llm-browser/backend/internal/browse/service_test.go

package browse_test

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"sync"
	"testing"

	"git.k6n.net/mats/fullstack-template/internal/browse"
)

// fakeFetcher is an in-memory browse.Fetcher: responses keyed by URL, an
// optional per-URL error, and a 404 default for anything unlisted.
type fakeFetcher struct {
	mu  sync.Mutex
	res map[string]*browse.Response
	err map[string]error
}

func newFakeFetcher(pages map[string]string) *fakeFetcher {
	f := &fakeFetcher{res: map[string]*browse.Response{}, err: map[string]error{}}
	for url, body := range pages {
		f.res[url] = &browse.Response{
			Status:      200,
			FinalURL:    url,
			ContentType: fakeContentType(url, body),
			Body:        []byte(body),
		}
	}
	return f
}

func (f *fakeFetcher) failOn(url string, err error) { f.err[url] = err }

func (f *fakeFetcher) Fetch(_ context.Context, url string) (*browse.Response, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	if err, ok := f.err[url]; ok {
		return nil, err
	}
	if r, ok := f.res[url]; ok {
		return r, nil
	}
	return &browse.Response{Status: 404, FinalURL: url, ContentType: "text/plain", Body: []byte("not found")}, nil
}

// fakeContentType guesses like a real server would: HTML pages as
// text/html, llms.txt as text/markdown, robots.txt as text/plain.
func fakeContentType(url, body string) string {
	b := strings.TrimSpace(body)
	switch {
	case strings.HasPrefix(b, "<!doctype") || strings.HasPrefix(b, "<html") || strings.Contains(b[:min(len(b), 200)], "<div"):
		return "text/html; charset=utf-8"
	case strings.HasSuffix(url, "llms.txt"):
		return "text/markdown; charset=utf-8"
	case strings.HasSuffix(url, "robots.txt"):
		return "text/plain"
	default:
		return "text/plain"
	}
}

// site is the smallest site that exercises every check path.
const site = `<!doctype html>
<html lang="en">
<head>
  <title>Test Site</title>
  <meta name="description" content="A site for tests.">
  <link rel="canonical" href="https://example.test/home">
</head>
<body>
  <h1>Welcome</h1>
  <p>Read the <a href="/docs">docs</a> or visit <a href="https://other.test/x">the hub</a>.</p>
  <a href="#" class="btn">dead</a>
  <script src="/app.js"></script>
</body>
</html>`

const llmsTxt = `# Test Site

> A site for tests.

## Docs

- [Guide](https://example.test/guide): how to use it
- [API](https://example.test/api "the api"): endpoints

## Community

- [Forum](https://forum.example.test): ask questions
`

func checkByID(t *testing.T, snap *browse.Snapshot, id string) browse.Check {
	t.Helper()
	for _, c := range snap.Checks {
		if c.ID == id {
			return c
		}
	}
	t.Fatalf("check %q not found in %v", id, snap.Checks)
	return browse.Check{}
}

func TestNormalizeURL(t *testing.T) {
	t.Parallel()
	cases := []struct {
		in, want string
		wantErr  bool
	}{
		{in: "example.com", want: "https://example.com"},
		{in: "  example.com/path  ", want: "https://example.com/path"},
		{in: "http://example.com", want: "http://example.com"},
		{in: "https://example.com/a?b=1#frag", want: "https://example.com/a?b=1"},
		{in: "user:pass@example.com", want: "https://user:pass@example.com"},
		{in: "ftp://example.com", wantErr: true},
		{in: "https://", wantErr: true},
		{in: "", wantErr: true},
		{in: "   ", wantErr: true},
	}
	for _, tc := range cases {
		got, err := browse.NormalizeURL(tc.in)
		if tc.wantErr {
			if err == nil {
				t.Errorf("NormalizeURL(%q) = %q, want error", tc.in, got)
			}
			var vErr *browse.ValidationError
			if err != nil && !errors.As(err, &vErr) {
				t.Errorf("NormalizeURL(%q) error type %T, want *ValidationError", tc.in, err)
			}
			continue
		}
		if err != nil {
			t.Errorf("NormalizeURL(%q) unexpected error: %v", tc.in, err)
			continue
		}
		// credentials survive url.Parse round-trip; the service drops them
		if u := got; strings.Contains(u, "user:pass") {
			t.Errorf("NormalizeURL(%q) = %q, want credentials dropped", tc.in, u)
		}
	}
}

func TestInspectConvertsHTML(t *testing.T) {
	t.Parallel()
	f := newFakeFetcher(map[string]string{"https://example.test": site})
	svc := browse.NewService(f)

	snap, err := svc.Inspect(context.Background(), "example.test")
	if err != nil {
		t.Fatalf("Inspect: %v", err)
	}
	if snap.RequestedURL != "https://example.test" {
		t.Errorf("requested URL = %q, want the normalized https origin", snap.RequestedURL)
	}
	if snap.Page == nil {
		t.Fatal("page metadata missing")
	}
	if snap.Page.Title != "Test Site" {
		t.Errorf("title = %q", snap.Page.Title)
	}
	if snap.Page.H1 != "Welcome" {
		t.Errorf("h1 = %q", snap.Page.H1)
	}
	if snap.Page.LinkCount != 2 || snap.Page.DeadLinks != 1 {
		t.Errorf("links = %d, dead = %d, want 2 and 1", snap.Page.LinkCount, snap.Page.DeadLinks)
	}
	for _, want := range []string{"# Welcome", "[docs](https://example.test/docs)", "[the hub](https://other.test/x)"} {
		if !strings.Contains(snap.Markdown, want) {
			t.Errorf("markdown missing %q:\n%s", want, snap.Markdown)
		}
	}
	if c := checkByID(t, snap, "page.status"); c.Severity != browse.SeverityOK {
		t.Errorf("page.status = %s (%s)", c.Severity, c.Detail)
	}
}

func TestInspectFetchError(t *testing.T) {
	t.Parallel()
	f := newFakeFetcher(nil)
	f.failOn("https://down.test", fmt.Errorf("dial tcp: refused"))
	svc := browse.NewService(f)

	_, err := svc.Inspect(context.Background(), "down.test")
	var fe *browse.FetchError
	if !errors.As(err, &fe) {
		t.Fatalf("err = %T (%v), want *browse.FetchError", err, err)
	}
	if fe.URL != "https://down.test" {
		t.Errorf("fetch error URL = %q", fe.URL)
	}
}

func TestInspectCompanions(t *testing.T) {
	t.Parallel()
	f := newFakeFetcher(map[string]string{
		"https://example.test":            site,
		"https://example.test/llms.txt":   llmsTxt,
		"https://example.test/robots.txt": "User-agent: *\nAllow: /\n\nSitemap: https://example.test/sitemap.xml",
	})
	svc := browse.NewService(f)

	snap, err := svc.Inspect(context.Background(), "https://example.test/")
	if err != nil {
		t.Fatalf("Inspect: %v", err)
	}

	llms := snap.Files.LlmsTxt
	if llms == nil || llms.Status >= 400 {
		t.Fatalf("llms.txt missing: %+v", snap.Files.LlmsTxt)
	}
	if llms.LlmsDoc == nil {
		t.Fatal("llms.txt was not parsed")
	}
	doc := llms.LlmsDoc
	if doc.Title != "Test Site" {
		t.Errorf("llms title = %q", doc.Title)
	}
	if doc.Description != "A site for tests." {
		t.Errorf("llms description = %q", doc.Description)
	}
	if len(doc.Sections) != 2 {
		t.Fatalf("llms sections = %d, want 2", len(doc.Sections))
	}
	if len(doc.Sections[0].Links) != 2 || doc.Sections[0].Links[1].Description != "endpoints" {
		t.Errorf("section links wrong: %+v", doc.Sections[0])
	}
	if doc.LinkCount != 3 {
		t.Errorf("llms link count = %d, want 3", doc.LinkCount)
	}

	if c := checkByID(t, snap, "llms.txt"); c.Severity != browse.SeverityOK {
		t.Errorf("llms.txt check = %s (%s)", c.Severity, c.Detail)
	}
	if c := checkByID(t, snap, "llms.txt.format"); c.Severity != browse.SeverityOK {
		t.Errorf("llms.txt.format = %s (%s)", c.Severity, c.Detail)
	}
	if c := checkByID(t, snap, "llms-full.txt"); c.Severity != browse.SeverityInfo {
		t.Errorf("llms-full.txt = %s (%s)", c.Severity, c.Detail)
	}
	if c := checkByID(t, snap, "robots.txt"); c.Severity != browse.SeverityOK {
		t.Errorf("robots.txt = %s (%s)", c.Severity, c.Detail)
	}
	if c := checkByID(t, snap, "robots.sitemap"); c.Detail == "" {
		t.Error("sitemap check detail empty")
	}
}

func TestInspectMissingLLMsTxtWarns(t *testing.T) {
	t.Parallel()
	f := newFakeFetcher(map[string]string{"https://bare.test": site})
	snap, err := browse.NewService(f).Inspect(context.Background(), "https://bare.test")
	if err != nil {
		t.Fatalf("Inspect: %v", err)
	}
	if c := checkByID(t, snap, "llms.txt"); c.Severity != browse.SeverityWarn {
		t.Errorf("llms.txt = %s (%s), want warn when the file is absent", c.Severity, c.Detail)
	}
}

func TestInspectLLMsTxtWithoutTitleIsAnError(t *testing.T) {
	t.Parallel()
	f := newFakeFetcher(map[string]string{
		"https://example.test/":         site,
		"https://example.test/llms.txt": "> just a quote, no H1\n",
	})
	snap, err := browse.NewService(f).Inspect(context.Background(), "https://example.test")
	if err != nil {
		t.Fatalf("Inspect: %v", err)
	}
	if c := checkByID(t, snap, "llms.txt.format"); c.Severity != browse.SeverityError {
		t.Errorf("llms.txt.format = %s (%s), want error for a missing H1", c.Severity, c.Detail)
	}
}

func TestInspectRobotsDisallowAllWarns(t *testing.T) {
	t.Parallel()
	f := newFakeFetcher(map[string]string{
		"https://closed.test":            site,
		"https://closed.test/robots.txt": "User-agent: *\nDisallow: /\n",
	})
	snap, err := browse.NewService(f).Inspect(context.Background(), "closed.test")
	if err != nil {
		t.Fatalf("Inspect: %v", err)
	}
	if c := checkByID(t, snap, "robots.txt"); c.Severity != browse.SeverityWarn {
		t.Errorf("robots.txt = %s (%s), want warn for a blanket disallow", c.Severity, c.Detail)
	}
}

func TestInspectPageProblems(t *testing.T) {
	t.Parallel()

	noindex := strings.Replace(site, "<title>Test Site</title>",
		`<title>Test Site</title><meta name="robots" content="noindex, nofollow">`, 1)

	jsShell := `<!doctype html><html><head><title>Shell</title></head><body>
		<div id="root"></div>
		<script src="/a.js"></script><script src="/b.js"></script><script src="/c.js"></script>
		</body></html>`

	f := newFakeFetcher(map[string]string{
		"https://noindex.test": noindex,
		"https://shell.test":   jsShell,
	})
	svc := browse.NewService(f)

	snap, err := svc.Inspect(context.Background(), "noindex.test")
	if err != nil {
		t.Fatalf("Inspect: %v", err)
	}
	if c := checkByID(t, snap, "page.robots-meta"); c.Severity != browse.SeverityError {
		t.Errorf("noindex = %s (%s), want error", c.Severity, c.Detail)
	}

	snap, err = svc.Inspect(context.Background(), "shell.test")
	if err != nil {
		t.Fatalf("Inspect: %v", err)
	}
	if c := checkByID(t, snap, "page.javascript"); c.Severity != browse.SeverityWarn {
		t.Errorf("js shell = %s (%s), want warn", c.Severity, c.Detail)
	}
}

func TestInspectRedirectAndMarkdownPage(t *testing.T) {
	t.Parallel()
	f := newFakeFetcher(map[string]string{
		"https://docs.test/guide": "# Guide\n\nAll there is.\n",
	})
	f.res["https://docs.test/guide"].ContentType = "text/markdown"

	snap, err := browse.NewService(f).Inspect(context.Background(), "https://docs.test/guide")
	if err != nil {
		t.Fatalf("Inspect: %v", err)
	}
	if snap.Page != nil {
		t.Errorf("markdown pages carry no HTML metadata, got %+v", snap.Page)
	}
	if !strings.HasPrefix(snap.Markdown, "# Guide") {
		t.Errorf("markdown = %q", snap.Markdown)
	}
	if c := checkByID(t, snap, "page.format"); c.Severity != browse.SeverityOK {
		t.Errorf("page.format = %s (%s)", c.Severity, c.Detail)
	}
}