399 lines
13 KiB
Go
399 lines
13 KiB
Go
package browse
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net/url"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// Tunables for the checks.
|
|
const (
|
|
// jsShellTextChars: with fewer server-rendered characters than this
|
|
// next to real script traffic, the page probably needs JavaScript.
|
|
jsShellTextChars = 400
|
|
// contextHeavyChars: markdown beyond this is a lot for one context.
|
|
contextHeavyChars = 150_000
|
|
)
|
|
|
|
// Companion paths, always resolved against the site root of the page.
|
|
const (
|
|
llmsTxtPath = "/llms.txt"
|
|
llmsFullTxtPath = "/llms-full.txt"
|
|
wellKnownTxtPath = "/.well-known/llms.txt"
|
|
robotsTxtPath = "/robots.txt"
|
|
)
|
|
|
|
// Service renders remote pages the way a text-only agent sees them. It is
|
|
// stateless; the Fetcher is its only collaborator.
|
|
type Service struct {
|
|
fetch Fetcher
|
|
}
|
|
|
|
// NewService returns a Service over the given fetcher.
|
|
func NewService(fetch Fetcher) *Service {
|
|
return &Service{fetch: fetch}
|
|
}
|
|
|
|
// Inspect fetches target (a bare domain or a full URL), converts the page
|
|
// to markdown, gathers the llms.txt companions and robots.txt in parallel,
|
|
// and assembles the agent-readiness checks.
|
|
func (s *Service) Inspect(ctx context.Context, raw string) (*Snapshot, error) {
|
|
target, err := NormalizeURL(raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
started := time.Now()
|
|
|
|
page, err := s.fetch.Fetch(ctx, target)
|
|
if err != nil {
|
|
var fe *FetchError
|
|
if !errors.As(err, &fe) {
|
|
fe = &FetchError{URL: target, Err: err}
|
|
}
|
|
return nil, fe
|
|
}
|
|
|
|
snap := &Snapshot{
|
|
RequestedURL: target,
|
|
FinalURL: firstNonEmpty(page.FinalURL, target),
|
|
Status: page.Status,
|
|
ContentType: page.ContentType,
|
|
FetchedAt: time.Now().UTC(),
|
|
Checks: []Check{},
|
|
}
|
|
|
|
var meta pageMeta
|
|
switch {
|
|
case isHTML(page.ContentType, string(page.Body)):
|
|
md, m, convErr := htmlToMarkdown(page.Body, snap.FinalURL)
|
|
if convErr != nil {
|
|
return nil, fmt.Errorf("parse %s: %w", snap.FinalURL, convErr)
|
|
}
|
|
meta = m
|
|
snap.Markdown, snap.MarkdownTruncated = truncateMarkdown(md, maxMarkdownChars)
|
|
snap.Page = &PageView{
|
|
Title: meta.Title,
|
|
Description: meta.Description,
|
|
Canonical: meta.Canonical,
|
|
RobotsMeta: meta.Robots,
|
|
Lang: meta.Lang,
|
|
H1: meta.H1,
|
|
LinkCount: meta.LinkCount,
|
|
DeadLinks: meta.DeadLinks,
|
|
ScriptCount: meta.ScriptCount,
|
|
ImageCount: meta.ImageCount,
|
|
WordCount: meta.WordCount,
|
|
TextLength: meta.TextLength,
|
|
}
|
|
case isText(page.ContentType):
|
|
snap.Markdown, snap.MarkdownTruncated = truncateMarkdown(string(page.Body), maxMarkdownChars)
|
|
default:
|
|
snap.Markdown, snap.MarkdownTruncated =
|
|
truncateMarkdown("```"+page.ContentType+"\n"+string(page.Body)+"\n```", maxMarkdownChars)
|
|
}
|
|
|
|
// Companions come from the site root the page ended up on. They are
|
|
// fetched concurrently so one slow file cannot stretch the inspection.
|
|
root := siteRoot(snap.FinalURL)
|
|
var wg sync.WaitGroup
|
|
wg.Add(4)
|
|
go s.fetchFile(ctx, &wg, root+llmsTxtPath, true, &snap.Files.LlmsTxt)
|
|
go s.fetchFile(ctx, &wg, root+llmsFullTxtPath, true, &snap.Files.LlmsFullTxt)
|
|
go s.fetchFile(ctx, &wg, root+wellKnownTxtPath, true, &snap.Files.WellKnownTxt)
|
|
go s.fetchFile(ctx, &wg, root+robotsTxtPath, false, &snap.Files.RobotsTxt)
|
|
wg.Wait()
|
|
|
|
snap.Checks = buildChecks(snap)
|
|
snap.DurationMS = time.Since(started).Milliseconds()
|
|
return snap, nil
|
|
}
|
|
|
|
// NormalizeURL turns user input into a fetchable URL: a bare domain gets
|
|
// https://, credentials are dropped, fragments are dropped.
|
|
func NormalizeURL(raw string) (string, error) {
|
|
s := strings.TrimSpace(raw)
|
|
if s == "" {
|
|
return "", &ValidationError{Field: "url", Reason: "required"}
|
|
}
|
|
if !strings.Contains(s, "://") {
|
|
s = "https://" + s
|
|
}
|
|
u, err := url.Parse(s)
|
|
if err != nil || u.Host == "" {
|
|
return "", &ValidationError{Field: "url", Reason: "must be a domain or URL agents can open"}
|
|
}
|
|
if u.Scheme != "http" && u.Scheme != "https" {
|
|
return "", &ValidationError{Field: "url", Reason: "only http and https URLs are supported"}
|
|
}
|
|
u.User = nil
|
|
u.Fragment = ""
|
|
return u.String(), nil
|
|
}
|
|
|
|
// fetchFile fetches one companion into dst. A failed request is a result,
|
|
// not an inspection failure — the checks describe what is missing.
|
|
func (s *Service) fetchFile(ctx context.Context, wg *sync.WaitGroup, url string, parseLlms bool, dst **FileResult) {
|
|
defer wg.Done()
|
|
|
|
res, err := s.fetch.Fetch(ctx, url)
|
|
if err != nil {
|
|
var fe *FetchError
|
|
if errors.As(err, &fe) {
|
|
err = fe.Err
|
|
}
|
|
*dst = &FileResult{URL: url, Error: err.Error()}
|
|
return
|
|
}
|
|
|
|
fr := &FileResult{
|
|
URL: firstNonEmpty(res.FinalURL, url),
|
|
Status: res.Status,
|
|
ContentType: res.ContentType,
|
|
Truncated: res.Truncated,
|
|
}
|
|
if fr.found() {
|
|
fr.Content = string(res.Body)
|
|
if parseLlms && !isHTML(fr.ContentType, fr.Content) {
|
|
fr.LlmsDoc = parseLlmsTxt(fr.Content)
|
|
}
|
|
}
|
|
*dst = fr
|
|
}
|
|
|
|
// buildChecks turns a finished snapshot into the validation report.
|
|
func buildChecks(snap *Snapshot) []Check {
|
|
c := []Check{}
|
|
add := func(id, label string, sev Severity, format string, args ...any) {
|
|
c = append(c, Check{ID: id, Label: label, Severity: sev, Detail: fmt.Sprintf(format, args...)})
|
|
}
|
|
|
|
// — the page itself —
|
|
switch {
|
|
case snap.Status >= 500:
|
|
add("page.status", "Page reachable", SeverityError, "server answered %d", snap.Status)
|
|
case snap.Status >= 400:
|
|
add("page.status", "Page reachable", SeverityError, "server answered %d — agents see an error page", snap.Status)
|
|
case snap.Status >= 300:
|
|
add("page.status", "Page reachable", SeverityWarn, "server answered %d", snap.Status)
|
|
default:
|
|
add("page.status", "Page reachable", SeverityOK, "%d OK", snap.Status)
|
|
}
|
|
|
|
if u, err := url.Parse(snap.FinalURL); err == nil && u.Scheme == "http" {
|
|
add("page.https", "Served over HTTPS", SeverityWarn, "the final URL is plain http; secure agents may refuse it")
|
|
} else {
|
|
add("page.https", "Served over HTTPS", SeverityOK, "https")
|
|
}
|
|
|
|
if snap.FinalURL != snap.RequestedURL {
|
|
add("page.redirect", "Direct URL", SeverityInfo, "redirected to %s", snap.FinalURL)
|
|
}
|
|
|
|
if snap.Page != nil {
|
|
p := snap.Page
|
|
if p.Title == "" {
|
|
add("page.title", "Title", SeverityError, "no <title> — agents use it as the page name in their results")
|
|
} else {
|
|
add("page.title", "Title", SeverityOK, "%s", clip(p.Title, 80))
|
|
}
|
|
|
|
if p.Description == "" {
|
|
add("page.description", "Meta description", SeverityWarn, "missing — agents have to guess the page's purpose from body text")
|
|
} else {
|
|
add("page.description", "Meta description", SeverityOK, "%s", clip(p.Description, 140))
|
|
}
|
|
|
|
robots := strings.ToLower(p.RobotsMeta)
|
|
switch {
|
|
case strings.Contains(robots, "noindex"):
|
|
add("page.robots-meta", "Robots meta", SeverityError, "meta robots declares noindex — indexing agents are told to ignore this page")
|
|
case strings.Contains(robots, "nofollow"):
|
|
add("page.robots-meta", "Robots meta", SeverityWarn, "meta robots declares nofollow — agents are told not to follow this page's links")
|
|
case p.RobotsMeta != "":
|
|
add("page.robots-meta", "Robots meta", SeverityOK, "%s", p.RobotsMeta)
|
|
default:
|
|
add("page.robots-meta", "Robots meta", SeverityOK, "no restrictions declared")
|
|
}
|
|
|
|
if p.TextLength < jsShellTextChars && p.ScriptCount >= 3 {
|
|
add("page.javascript", "Reads without JavaScript", SeverityWarn,
|
|
"only %d characters of server-rendered text next to %d script tags — the page likely renders in the browser, which text agents never run",
|
|
p.TextLength, p.ScriptCount)
|
|
} else {
|
|
add("page.javascript", "Reads without JavaScript", SeverityOK, "%d characters of server-rendered text", p.TextLength)
|
|
}
|
|
|
|
if p.H1 == "" {
|
|
add("page.h1", "Primary heading", SeverityInfo, "no <h1>; the markdown starts without a top-level heading")
|
|
}
|
|
|
|
if p.DeadLinks > 0 {
|
|
add("page.dead-links", "Followable links", SeverityWarn,
|
|
"%d link(s) point at \"#\" or javascript: — agents cannot follow them", p.DeadLinks)
|
|
}
|
|
|
|
if len(snap.Markdown) > contextHeavyChars {
|
|
add("markdown.size", "Markdown size", SeverityInfo,
|
|
"%d characters — heavy for a single agent context window", len(snap.Markdown))
|
|
}
|
|
} else if isText(snap.ContentType) {
|
|
add("page.format", "Native format", SeverityOK, "served as markdown/plain text — agents read it directly")
|
|
}
|
|
|
|
// — the llms.txt family —
|
|
f := snap.Files
|
|
switch {
|
|
case f.LlmsTxt.found():
|
|
add("llms.txt", "/llms.txt", SeverityOK, "found at %s", f.LlmsTxt.URL)
|
|
if doc := f.LlmsTxt.LlmsDoc; doc != nil {
|
|
if doc.Title == "" {
|
|
add("llms.txt.format", "llms.txt format", SeverityError,
|
|
"the spec requires the file to start with an `# H1 title`")
|
|
} else if len(doc.Sections) == 0 {
|
|
add("llms.txt.format", "llms.txt format", SeverityInfo,
|
|
"title present but no `## sections` — fine, though sections organise links for agents")
|
|
} else {
|
|
add("llms.txt.format", "llms.txt format", SeverityOK,
|
|
"%s — %d section(s)", clip(doc.Title, 60), len(doc.Sections))
|
|
}
|
|
if doc.LinkCount == 0 {
|
|
add("llms.txt.links", "llms.txt links", SeverityWarn, "no links — the file gives agents nothing to open")
|
|
} else {
|
|
add("llms.txt.links", "llms.txt links", SeverityOK, "%d curated link(s)", doc.LinkCount)
|
|
}
|
|
}
|
|
if !strings.Contains(strings.ToLower(f.LlmsTxt.ContentType), "markdown") {
|
|
add("llms.txt.content-type", "llms.txt content type", SeverityInfo,
|
|
"served as %s; the spec recommends text/markdown", contentTypeOr(f.LlmsTxt.ContentType))
|
|
}
|
|
case f.LlmsTxt == nil:
|
|
add("llms.txt", "/llms.txt", SeverityWarn, "not reachable — agents are left to parse the HTML themselves; add one per llmstxt.org")
|
|
default:
|
|
add("llms.txt", "/llms.txt", SeverityWarn,
|
|
"missing (HTTP %d) — agents are left to parse the HTML themselves; add one per llmstxt.org", f.LlmsTxt.Status)
|
|
}
|
|
|
|
switch {
|
|
case f.LlmsFullTxt.found():
|
|
add("llms-full.txt", "/llms-full.txt", SeverityOK, "found — full-context variant for agents that want everything")
|
|
case f.LlmsFullTxt != nil && f.LlmsFullTxt.Status > 0:
|
|
add("llms-full.txt", "/llms-full.txt", SeverityInfo, "not published (HTTP %d) — optional", f.LlmsFullTxt.Status)
|
|
default:
|
|
add("llms-full.txt", "/llms-full.txt", SeverityInfo, "not published — optional")
|
|
}
|
|
|
|
switch {
|
|
case f.WellKnownTxt.found():
|
|
add("llms.well-known", "/.well-known/llms.txt", SeverityOK, "found at %s", f.WellKnownTxt.URL)
|
|
case f.WellKnownTxt != nil && f.WellKnownTxt.Status > 0:
|
|
add("llms.well-known", "/.well-known/llms.txt", SeverityInfo, "not published (HTTP %d) — alternative location", f.WellKnownTxt.Status)
|
|
default:
|
|
add("llms.well-known", "/.well-known/llms.txt", SeverityInfo, "not published — alternative location")
|
|
}
|
|
|
|
// — robots.txt —
|
|
switch {
|
|
case f.RobotsTxt.found():
|
|
disallowAll, sitemap := robotsSummary(f.RobotsTxt.Content)
|
|
if disallowAll {
|
|
add("robots.txt", "robots.txt", SeverityWarn,
|
|
"present but `User-agent: *` is disallowed for / — automated agents are told to keep out")
|
|
} else {
|
|
add("robots.txt", "robots.txt", SeverityOK, "no blanket disallow for `User-agent: *`")
|
|
}
|
|
if sitemap != "" {
|
|
add("robots.sitemap", "Sitemap", SeverityInfo, "%s", sitemap)
|
|
}
|
|
default:
|
|
add("robots.txt", "robots.txt", SeverityInfo, "not published — agents assume everything is allowed")
|
|
}
|
|
|
|
return c
|
|
}
|
|
|
|
// robotsSummary reports whether the wildcard agent is disallowed for / and
|
|
// returns the first declared sitemap.
|
|
func robotsSummary(content string) (disallowAll bool, sitemap string) {
|
|
groupApplies := false
|
|
for _, line := range strings.Split(content, "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" || strings.HasPrefix(line, "#") {
|
|
continue
|
|
}
|
|
key, value, _ := strings.Cut(line, ":")
|
|
key = strings.ToLower(strings.TrimSpace(key))
|
|
value = strings.TrimSpace(value)
|
|
switch key {
|
|
case "user-agent":
|
|
groupApplies = value == "*"
|
|
case "disallow":
|
|
if groupApplies && value == "/" {
|
|
disallowAll = true
|
|
}
|
|
case "sitemap":
|
|
if sitemap == "" {
|
|
sitemap = value
|
|
}
|
|
}
|
|
}
|
|
return disallowAll, sitemap
|
|
}
|
|
|
|
// isHTML decides whether the body should go through the HTML converter.
|
|
func isHTML(contentType, body string) bool {
|
|
ct := strings.ToLower(contentType)
|
|
if strings.Contains(ct, "html") {
|
|
return true
|
|
}
|
|
if ct == "" || strings.HasPrefix(ct, "text/") {
|
|
l := strings.ToLower(body[:min(len(body), 2048)])
|
|
return strings.Contains(l, "<!doctype html") || strings.Contains(l, "<html")
|
|
}
|
|
return false
|
|
}
|
|
|
|
// isText reports whether the content type is already markdown/plain text.
|
|
func isText(contentType string) bool {
|
|
ct := strings.ToLower(contentType)
|
|
return strings.Contains(ct, "text/markdown") || strings.Contains(ct, "text/plain") ||
|
|
strings.Contains(ct, "text/x-markdown")
|
|
}
|
|
|
|
// siteRoot reduces a URL to scheme://host for companion lookups.
|
|
func siteRoot(raw string) string {
|
|
u, err := url.Parse(raw)
|
|
if err != nil || u.Host == "" {
|
|
return raw
|
|
}
|
|
return u.Scheme + "://" + u.Host
|
|
}
|
|
|
|
func contentTypeOr(ct string) string {
|
|
if strings.TrimSpace(ct) == "" {
|
|
return "an unknown type"
|
|
}
|
|
return ct
|
|
}
|
|
|
|
func clip(s string, max int) string {
|
|
s = strings.TrimSpace(s)
|
|
runes := []rune(s)
|
|
if len(runes) <= max {
|
|
return s
|
|
}
|
|
return string(runes[:max-1]) + "…"
|
|
}
|
|
|
|
func firstNonEmpty(vals ...string) string {
|
|
for _, v := range vals {
|
|
if v != "" {
|
|
return v
|
|
}
|
|
}
|
|
return ""
|
|
}
|