package browse_test import ( "context" "errors" "fmt" "strings" "sync" "testing" "git.k6n.net/mats/fullstack-template/internal/browse" ) // fakeFetcher is an in-memory browse.Fetcher: responses keyed by URL, an // optional per-URL error, and a 404 default for anything unlisted. type fakeFetcher struct { mu sync.Mutex res map[string]*browse.Response err map[string]error } func newFakeFetcher(pages map[string]string) *fakeFetcher { f := &fakeFetcher{res: map[string]*browse.Response{}, err: map[string]error{}} for url, body := range pages { f.res[url] = &browse.Response{ Status: 200, FinalURL: url, ContentType: fakeContentType(url, body), Body: []byte(body), } } return f } func (f *fakeFetcher) failOn(url string, err error) { f.err[url] = err } func (f *fakeFetcher) Fetch(_ context.Context, url string) (*browse.Response, error) { f.mu.Lock() defer f.mu.Unlock() if err, ok := f.err[url]; ok { return nil, err } if r, ok := f.res[url]; ok { return r, nil } return &browse.Response{Status: 404, FinalURL: url, ContentType: "text/plain", Body: []byte("not found")}, nil } // fakeContentType guesses like a real server would: HTML pages as // text/html, llms.txt as text/markdown, robots.txt as text/plain. func fakeContentType(url, body string) string { b := strings.TrimSpace(body) switch { case strings.HasPrefix(b, " Test Site

Welcome

Read the docs or visit the hub.

dead ` const llmsTxt = `# Test Site > A site for tests. ## Docs - [Guide](https://example.test/guide): how to use it - [API](https://example.test/api "the api"): endpoints ## Community - [Forum](https://forum.example.test): ask questions ` func checkByID(t *testing.T, snap *browse.Snapshot, id string) browse.Check { t.Helper() for _, c := range snap.Checks { if c.ID == id { return c } } t.Fatalf("check %q not found in %v", id, snap.Checks) return browse.Check{} } func TestNormalizeURL(t *testing.T) { t.Parallel() cases := []struct { in, want string wantErr bool }{ {in: "example.com", want: "https://example.com"}, {in: " example.com/path ", want: "https://example.com/path"}, {in: "http://example.com", want: "http://example.com"}, {in: "https://example.com/a?b=1#frag", want: "https://example.com/a?b=1"}, {in: "user:pass@example.com", want: "https://user:pass@example.com"}, {in: "ftp://example.com", wantErr: true}, {in: "https://", wantErr: true}, {in: "", wantErr: true}, {in: " ", wantErr: true}, } for _, tc := range cases { got, err := browse.NormalizeURL(tc.in) if tc.wantErr { if err == nil { t.Errorf("NormalizeURL(%q) = %q, want error", tc.in, got) } var vErr *browse.ValidationError if err != nil && !errors.As(err, &vErr) { t.Errorf("NormalizeURL(%q) error type %T, want *ValidationError", tc.in, err) } continue } if err != nil { t.Errorf("NormalizeURL(%q) unexpected error: %v", tc.in, err) continue } // credentials survive url.Parse round-trip; the service drops them if u := got; strings.Contains(u, "user:pass") { t.Errorf("NormalizeURL(%q) = %q, want credentials dropped", tc.in, u) } } } func TestInspectConvertsHTML(t *testing.T) { t.Parallel() f := newFakeFetcher(map[string]string{"https://example.test": site}) svc := browse.NewService(f) snap, err := svc.Inspect(context.Background(), "example.test") if err != nil { t.Fatalf("Inspect: %v", err) } if snap.RequestedURL != "https://example.test" { t.Errorf("requested URL = %q, want the normalized https origin", snap.RequestedURL) } if snap.Page == nil { t.Fatal("page metadata missing") } if snap.Page.Title != "Test Site" { t.Errorf("title = %q", snap.Page.Title) } if snap.Page.H1 != "Welcome" { t.Errorf("h1 = %q", snap.Page.H1) } if snap.Page.LinkCount != 2 || snap.Page.DeadLinks != 1 { t.Errorf("links = %d, dead = %d, want 2 and 1", snap.Page.LinkCount, snap.Page.DeadLinks) } for _, want := range []string{"# Welcome", "[docs](https://example.test/docs)", "[the hub](https://other.test/x)"} { if !strings.Contains(snap.Markdown, want) { t.Errorf("markdown missing %q:\n%s", want, snap.Markdown) } } if c := checkByID(t, snap, "page.status"); c.Severity != browse.SeverityOK { t.Errorf("page.status = %s (%s)", c.Severity, c.Detail) } } func TestInspectFetchError(t *testing.T) { t.Parallel() f := newFakeFetcher(nil) f.failOn("https://down.test", fmt.Errorf("dial tcp: refused")) svc := browse.NewService(f) _, err := svc.Inspect(context.Background(), "down.test") var fe *browse.FetchError if !errors.As(err, &fe) { t.Fatalf("err = %T (%v), want *browse.FetchError", err, err) } if fe.URL != "https://down.test" { t.Errorf("fetch error URL = %q", fe.URL) } } func TestInspectCompanions(t *testing.T) { t.Parallel() f := newFakeFetcher(map[string]string{ "https://example.test": site, "https://example.test/llms.txt": llmsTxt, "https://example.test/robots.txt": "User-agent: *\nAllow: /\n\nSitemap: https://example.test/sitemap.xml", }) svc := browse.NewService(f) snap, err := svc.Inspect(context.Background(), "https://example.test/") if err != nil { t.Fatalf("Inspect: %v", err) } llms := snap.Files.LlmsTxt if llms == nil || llms.Status >= 400 { t.Fatalf("llms.txt missing: %+v", snap.Files.LlmsTxt) } if llms.LlmsDoc == nil { t.Fatal("llms.txt was not parsed") } doc := llms.LlmsDoc if doc.Title != "Test Site" { t.Errorf("llms title = %q", doc.Title) } if doc.Description != "A site for tests." { t.Errorf("llms description = %q", doc.Description) } if len(doc.Sections) != 2 { t.Fatalf("llms sections = %d, want 2", len(doc.Sections)) } if len(doc.Sections[0].Links) != 2 || doc.Sections[0].Links[1].Description != "endpoints" { t.Errorf("section links wrong: %+v", doc.Sections[0]) } if doc.LinkCount != 3 { t.Errorf("llms link count = %d, want 3", doc.LinkCount) } if c := checkByID(t, snap, "llms.txt"); c.Severity != browse.SeverityOK { t.Errorf("llms.txt check = %s (%s)", c.Severity, c.Detail) } if c := checkByID(t, snap, "llms.txt.format"); c.Severity != browse.SeverityOK { t.Errorf("llms.txt.format = %s (%s)", c.Severity, c.Detail) } if c := checkByID(t, snap, "llms-full.txt"); c.Severity != browse.SeverityInfo { t.Errorf("llms-full.txt = %s (%s)", c.Severity, c.Detail) } if c := checkByID(t, snap, "robots.txt"); c.Severity != browse.SeverityOK { t.Errorf("robots.txt = %s (%s)", c.Severity, c.Detail) } if c := checkByID(t, snap, "robots.sitemap"); c.Detail == "" { t.Error("sitemap check detail empty") } } func TestInspectMissingLLMsTxtWarns(t *testing.T) { t.Parallel() f := newFakeFetcher(map[string]string{"https://bare.test": site}) snap, err := browse.NewService(f).Inspect(context.Background(), "https://bare.test") if err != nil { t.Fatalf("Inspect: %v", err) } if c := checkByID(t, snap, "llms.txt"); c.Severity != browse.SeverityWarn { t.Errorf("llms.txt = %s (%s), want warn when the file is absent", c.Severity, c.Detail) } } func TestInspectLLMsTxtWithoutTitleIsAnError(t *testing.T) { t.Parallel() f := newFakeFetcher(map[string]string{ "https://example.test/": site, "https://example.test/llms.txt": "> just a quote, no H1\n", }) snap, err := browse.NewService(f).Inspect(context.Background(), "https://example.test") if err != nil { t.Fatalf("Inspect: %v", err) } if c := checkByID(t, snap, "llms.txt.format"); c.Severity != browse.SeverityError { t.Errorf("llms.txt.format = %s (%s), want error for a missing H1", c.Severity, c.Detail) } } func TestInspectRobotsDisallowAllWarns(t *testing.T) { t.Parallel() f := newFakeFetcher(map[string]string{ "https://closed.test": site, "https://closed.test/robots.txt": "User-agent: *\nDisallow: /\n", }) snap, err := browse.NewService(f).Inspect(context.Background(), "closed.test") if err != nil { t.Fatalf("Inspect: %v", err) } if c := checkByID(t, snap, "robots.txt"); c.Severity != browse.SeverityWarn { t.Errorf("robots.txt = %s (%s), want warn for a blanket disallow", c.Severity, c.Detail) } } func TestInspectPageProblems(t *testing.T) { t.Parallel() noindex := strings.Replace(site, "Test Site", `Test Site`, 1) jsShell := `Shell
` f := newFakeFetcher(map[string]string{ "https://noindex.test": noindex, "https://shell.test": jsShell, }) svc := browse.NewService(f) snap, err := svc.Inspect(context.Background(), "noindex.test") if err != nil { t.Fatalf("Inspect: %v", err) } if c := checkByID(t, snap, "page.robots-meta"); c.Severity != browse.SeverityError { t.Errorf("noindex = %s (%s), want error", c.Severity, c.Detail) } snap, err = svc.Inspect(context.Background(), "shell.test") if err != nil { t.Fatalf("Inspect: %v", err) } if c := checkByID(t, snap, "page.javascript"); c.Severity != browse.SeverityWarn { t.Errorf("js shell = %s (%s), want warn", c.Severity, c.Detail) } } func TestInspectRedirectAndMarkdownPage(t *testing.T) { t.Parallel() f := newFakeFetcher(map[string]string{ "https://docs.test/guide": "# Guide\n\nAll there is.\n", }) f.res["https://docs.test/guide"].ContentType = "text/markdown" snap, err := browse.NewService(f).Inspect(context.Background(), "https://docs.test/guide") if err != nil { t.Fatalf("Inspect: %v", err) } if snap.Page != nil { t.Errorf("markdown pages carry no HTML metadata, got %+v", snap.Page) } if !strings.HasPrefix(snap.Markdown, "# Guide") { t.Errorf("markdown = %q", snap.Markdown) } if c := checkByID(t, snap, "page.format"); c.Severity != browse.SeverityOK { t.Errorf("page.format = %s (%s)", c.Severity, c.Detail) } }