package browse import ( "context" "errors" "fmt" "net/url" "strings" "sync" "time" ) // Tunables for the checks. const ( // jsShellTextChars: with fewer server-rendered characters than this // next to real script traffic, the page probably needs JavaScript. jsShellTextChars = 400 // contextHeavyChars: markdown beyond this is a lot for one context. contextHeavyChars = 150_000 ) // Companion paths, always resolved against the site root of the page. const ( llmsTxtPath = "/llms.txt" llmsFullTxtPath = "/llms-full.txt" wellKnownTxtPath = "/.well-known/llms.txt" robotsTxtPath = "/robots.txt" ) // Service renders remote pages the way a text-only agent sees them. It is // stateless; the Fetcher is its only collaborator. type Service struct { fetch Fetcher } // NewService returns a Service over the given fetcher. func NewService(fetch Fetcher) *Service { return &Service{fetch: fetch} } // Inspect fetches target (a bare domain or a full URL), converts the page // to markdown, gathers the llms.txt companions and robots.txt in parallel, // and assembles the agent-readiness checks. func (s *Service) Inspect(ctx context.Context, raw string) (*Snapshot, error) { target, err := NormalizeURL(raw) if err != nil { return nil, err } started := time.Now() page, err := s.fetch.Fetch(ctx, target) if err != nil { var fe *FetchError if !errors.As(err, &fe) { fe = &FetchError{URL: target, Err: err} } return nil, fe } snap := &Snapshot{ RequestedURL: target, FinalURL: firstNonEmpty(page.FinalURL, target), Status: page.Status, ContentType: page.ContentType, FetchedAt: time.Now().UTC(), Checks: []Check{}, } var meta pageMeta switch { case isHTML(page.ContentType, string(page.Body)): md, m, convErr := htmlToMarkdown(page.Body, snap.FinalURL) if convErr != nil { return nil, fmt.Errorf("parse %s: %w", snap.FinalURL, convErr) } meta = m snap.Markdown, snap.MarkdownTruncated = truncateMarkdown(md, maxMarkdownChars) snap.Page = &PageView{ Title: meta.Title, Description: meta.Description, Canonical: meta.Canonical, RobotsMeta: meta.Robots, Lang: meta.Lang, H1: meta.H1, LinkCount: meta.LinkCount, DeadLinks: meta.DeadLinks, ScriptCount: meta.ScriptCount, ImageCount: meta.ImageCount, WordCount: meta.WordCount, TextLength: meta.TextLength, } case isText(page.ContentType): snap.Markdown, snap.MarkdownTruncated = truncateMarkdown(string(page.Body), maxMarkdownChars) default: snap.Markdown, snap.MarkdownTruncated = truncateMarkdown("```"+page.ContentType+"\n"+string(page.Body)+"\n```", maxMarkdownChars) } // Companions come from the site root the page ended up on. They are // fetched concurrently so one slow file cannot stretch the inspection. root := siteRoot(snap.FinalURL) var wg sync.WaitGroup wg.Add(4) go s.fetchFile(ctx, &wg, root+llmsTxtPath, true, &snap.Files.LlmsTxt) go s.fetchFile(ctx, &wg, root+llmsFullTxtPath, true, &snap.Files.LlmsFullTxt) go s.fetchFile(ctx, &wg, root+wellKnownTxtPath, true, &snap.Files.WellKnownTxt) go s.fetchFile(ctx, &wg, root+robotsTxtPath, false, &snap.Files.RobotsTxt) wg.Wait() snap.Checks = buildChecks(snap) snap.DurationMS = time.Since(started).Milliseconds() return snap, nil } // NormalizeURL turns user input into a fetchable URL: a bare domain gets // https://, credentials are dropped, fragments are dropped. func NormalizeURL(raw string) (string, error) { s := strings.TrimSpace(raw) if s == "" { return "", &ValidationError{Field: "url", Reason: "required"} } if !strings.Contains(s, "://") { s = "https://" + s } u, err := url.Parse(s) if err != nil || u.Host == "" { return "", &ValidationError{Field: "url", Reason: "must be a domain or URL agents can open"} } if u.Scheme != "http" && u.Scheme != "https" { return "", &ValidationError{Field: "url", Reason: "only http and https URLs are supported"} } u.User = nil u.Fragment = "" return u.String(), nil } // fetchFile fetches one companion into dst. A failed request is a result, // not an inspection failure — the checks describe what is missing. func (s *Service) fetchFile(ctx context.Context, wg *sync.WaitGroup, url string, parseLlms bool, dst **FileResult) { defer wg.Done() res, err := s.fetch.Fetch(ctx, url) if err != nil { var fe *FetchError if errors.As(err, &fe) { err = fe.Err } *dst = &FileResult{URL: url, Error: err.Error()} return } fr := &FileResult{ URL: firstNonEmpty(res.FinalURL, url), Status: res.Status, ContentType: res.ContentType, Truncated: res.Truncated, } if fr.found() { fr.Content = string(res.Body) if parseLlms && !isHTML(fr.ContentType, fr.Content) { fr.LlmsDoc = parseLlmsTxt(fr.Content) } } *dst = fr } // buildChecks turns a finished snapshot into the validation report. func buildChecks(snap *Snapshot) []Check { c := []Check{} add := func(id, label string, sev Severity, format string, args ...any) { c = append(c, Check{ID: id, Label: label, Severity: sev, Detail: fmt.Sprintf(format, args...)}) } // — the page itself — switch { case snap.Status >= 500: add("page.status", "Page reachable", SeverityError, "server answered %d", snap.Status) case snap.Status >= 400: add("page.status", "Page reachable", SeverityError, "server answered %d — agents see an error page", snap.Status) case snap.Status >= 300: add("page.status", "Page reachable", SeverityWarn, "server answered %d", snap.Status) default: add("page.status", "Page reachable", SeverityOK, "%d OK", snap.Status) } if u, err := url.Parse(snap.FinalURL); err == nil && u.Scheme == "http" { add("page.https", "Served over HTTPS", SeverityWarn, "the final URL is plain http; secure agents may refuse it") } else { add("page.https", "Served over HTTPS", SeverityOK, "https") } if snap.FinalURL != snap.RequestedURL { add("page.redirect", "Direct URL", SeverityInfo, "redirected to %s", snap.FinalURL) } if snap.Page != nil { p := snap.Page if p.Title == "" { add("page.title", "Title", SeverityError, "no — agents use it as the page name in their results") } else { add("page.title", "Title", SeverityOK, "%s", clip(p.Title, 80)) } if p.Description == "" { add("page.description", "Meta description", SeverityWarn, "missing — agents have to guess the page's purpose from body text") } else { add("page.description", "Meta description", SeverityOK, "%s", clip(p.Description, 140)) } robots := strings.ToLower(p.RobotsMeta) switch { case strings.Contains(robots, "noindex"): add("page.robots-meta", "Robots meta", SeverityError, "meta robots declares noindex — indexing agents are told to ignore this page") case strings.Contains(robots, "nofollow"): add("page.robots-meta", "Robots meta", SeverityWarn, "meta robots declares nofollow — agents are told not to follow this page's links") case p.RobotsMeta != "": add("page.robots-meta", "Robots meta", SeverityOK, "%s", p.RobotsMeta) default: add("page.robots-meta", "Robots meta", SeverityOK, "no restrictions declared") } if p.TextLength < jsShellTextChars && p.ScriptCount >= 3 { add("page.javascript", "Reads without JavaScript", SeverityWarn, "only %d characters of server-rendered text next to %d script tags — the page likely renders in the browser, which text agents never run", p.TextLength, p.ScriptCount) } else { add("page.javascript", "Reads without JavaScript", SeverityOK, "%d characters of server-rendered text", p.TextLength) } if p.H1 == "" { add("page.h1", "Primary heading", SeverityInfo, "no <h1>; the markdown starts without a top-level heading") } if p.DeadLinks > 0 { add("page.dead-links", "Followable links", SeverityWarn, "%d link(s) point at \"#\" or javascript: — agents cannot follow them", p.DeadLinks) } if len(snap.Markdown) > contextHeavyChars { add("markdown.size", "Markdown size", SeverityInfo, "%d characters — heavy for a single agent context window", len(snap.Markdown)) } } else if isText(snap.ContentType) { add("page.format", "Native format", SeverityOK, "served as markdown/plain text — agents read it directly") } // — the llms.txt family — f := snap.Files switch { case f.LlmsTxt.found(): add("llms.txt", "/llms.txt", SeverityOK, "found at %s", f.LlmsTxt.URL) if doc := f.LlmsTxt.LlmsDoc; doc != nil { if doc.Title == "" { add("llms.txt.format", "llms.txt format", SeverityError, "the spec requires the file to start with an `# H1 title`") } else if len(doc.Sections) == 0 { add("llms.txt.format", "llms.txt format", SeverityInfo, "title present but no `## sections` — fine, though sections organise links for agents") } else { add("llms.txt.format", "llms.txt format", SeverityOK, "%s — %d section(s)", clip(doc.Title, 60), len(doc.Sections)) } if doc.LinkCount == 0 { add("llms.txt.links", "llms.txt links", SeverityWarn, "no links — the file gives agents nothing to open") } else { add("llms.txt.links", "llms.txt links", SeverityOK, "%d curated link(s)", doc.LinkCount) } } if !strings.Contains(strings.ToLower(f.LlmsTxt.ContentType), "markdown") { add("llms.txt.content-type", "llms.txt content type", SeverityInfo, "served as %s; the spec recommends text/markdown", contentTypeOr(f.LlmsTxt.ContentType)) } case f.LlmsTxt == nil: add("llms.txt", "/llms.txt", SeverityWarn, "not reachable — agents are left to parse the HTML themselves; add one per llmstxt.org") default: add("llms.txt", "/llms.txt", SeverityWarn, "missing (HTTP %d) — agents are left to parse the HTML themselves; add one per llmstxt.org", f.LlmsTxt.Status) } switch { case f.LlmsFullTxt.found(): add("llms-full.txt", "/llms-full.txt", SeverityOK, "found — full-context variant for agents that want everything") case f.LlmsFullTxt != nil && f.LlmsFullTxt.Status > 0: add("llms-full.txt", "/llms-full.txt", SeverityInfo, "not published (HTTP %d) — optional", f.LlmsFullTxt.Status) default: add("llms-full.txt", "/llms-full.txt", SeverityInfo, "not published — optional") } switch { case f.WellKnownTxt.found(): add("llms.well-known", "/.well-known/llms.txt", SeverityOK, "found at %s", f.WellKnownTxt.URL) case f.WellKnownTxt != nil && f.WellKnownTxt.Status > 0: add("llms.well-known", "/.well-known/llms.txt", SeverityInfo, "not published (HTTP %d) — alternative location", f.WellKnownTxt.Status) default: add("llms.well-known", "/.well-known/llms.txt", SeverityInfo, "not published — alternative location") } // — robots.txt — switch { case f.RobotsTxt.found(): disallowAll, sitemap := robotsSummary(f.RobotsTxt.Content) if disallowAll { add("robots.txt", "robots.txt", SeverityWarn, "present but `User-agent: *` is disallowed for / — automated agents are told to keep out") } else { add("robots.txt", "robots.txt", SeverityOK, "no blanket disallow for `User-agent: *`") } if sitemap != "" { add("robots.sitemap", "Sitemap", SeverityInfo, "%s", sitemap) } default: add("robots.txt", "robots.txt", SeverityInfo, "not published — agents assume everything is allowed") } return c } // robotsSummary reports whether the wildcard agent is disallowed for / and // returns the first declared sitemap. func robotsSummary(content string) (disallowAll bool, sitemap string) { groupApplies := false for _, line := range strings.Split(content, "\n") { line = strings.TrimSpace(line) if line == "" || strings.HasPrefix(line, "#") { continue } key, value, _ := strings.Cut(line, ":") key = strings.ToLower(strings.TrimSpace(key)) value = strings.TrimSpace(value) switch key { case "user-agent": groupApplies = value == "*" case "disallow": if groupApplies && value == "/" { disallowAll = true } case "sitemap": if sitemap == "" { sitemap = value } } } return disallowAll, sitemap } // isHTML decides whether the body should go through the HTML converter. func isHTML(contentType, body string) bool { ct := strings.ToLower(contentType) if strings.Contains(ct, "html") { return true } if ct == "" || strings.HasPrefix(ct, "text/") { l := strings.ToLower(body[:min(len(body), 2048)]) return strings.Contains(l, "<!doctype html") || strings.Contains(l, "<html") } return false } // isText reports whether the content type is already markdown/plain text. func isText(contentType string) bool { ct := strings.ToLower(contentType) return strings.Contains(ct, "text/markdown") || strings.Contains(ct, "text/plain") || strings.Contains(ct, "text/x-markdown") } // siteRoot reduces a URL to scheme://host for companion lookups. func siteRoot(raw string) string { u, err := url.Parse(raw) if err != nil || u.Host == "" { return raw } return u.Scheme + "://" + u.Host } func contentTypeOr(ct string) string { if strings.TrimSpace(ct) == "" { return "an unknown type" } return ct } func clip(s string, max int) string { s = strings.TrimSpace(s) runes := []rune(s) if len(runes) <= max { return s } return string(runes[:max-1]) + "…" } func firstNonEmpty(vals ...string) string { for _, v := range vals { if v != "" { return v } } return "" }