// Package browse renders a remote page the way a text-only agent sees it: // it fetches the URL, converts the HTML to markdown, gathers the llms.txt // companions and robots.txt, and reports agent-readiness checks. The // package is stateless — the Fetcher is its only collaborator. package browse import ( "context" "net/http" "time" ) // ValidationError reports an invalid input value. Handlers map it to 400. type ValidationError struct { Field string Reason string } func (e *ValidationError) Error() string { return e.Field + ": " + e.Reason } // FetchError reports a remote fetch that failed before an answer arrived // (DNS, TLS, connection refused, …). Handlers map it to 502 — the inspected // site is the problem, not this service. type FetchError struct { URL string Err error } func (e *FetchError) Error() string { return "fetch " + e.URL + ": " + e.Err.Error() } func (e *FetchError) Unwrap() error { return e.Err } // Severity classifies a Check. type Severity string const ( SeverityOK Severity = "ok" SeverityInfo Severity = "info" SeverityWarn Severity = "warn" SeverityError Severity = "error" ) // Check is one validated aspect of the inspected site, written so a human // can act on the detail without reading this package. type Check struct { ID string `json:"id"` Label string `json:"label"` Severity Severity `json:"severity"` Detail string `json:"detail"` } // Response is a capped remote fetch result. type Response struct { Status int FinalURL string // after redirects; empty means "same as requested" ContentType string Header http.Header Body []byte Truncated bool } // Fetcher retrieves remote documents. It plays the role the Repository // interface plays for database-backed resources: the service depends on the // interface, tests substitute an in-memory fake. type Fetcher interface { Fetch(ctx context.Context, url string) (*Response, error) } // PageView is what agents read out of the HTML head plus document shape // statistics that drive the checks. type PageView struct { Title string `json:"title"` Description string `json:"description"` Canonical string `json:"canonical"` RobotsMeta string `json:"robotsMeta"` Lang string `json:"lang"` H1 string `json:"h1"` LinkCount int `json:"linkCount"` DeadLinks int `json:"deadLinks"` ScriptCount int `json:"scriptCount"` ImageCount int `json:"imageCount"` WordCount int `json:"wordCount"` TextLength int `json:"textLength"` } // LlmsLink is one `- [title](url): description` entry of an llms.txt file. type LlmsLink struct { Title string `json:"title"` URL string `json:"url"` Description string `json:"description"` } // LlmsSection is one `## Heading` block of an llms.txt file. type LlmsSection struct { Heading string `json:"heading"` Description string `json:"description"` Links []LlmsLink `json:"links"` } // LlmsDoc is a parsed llms.txt document per llmstxt.org: an H1 title, an // optional blockquote summary, and optional H2 sections of curated links. type LlmsDoc struct { Title string `json:"title"` Description string `json:"description"` Sections []LlmsSection `json:"sections"` LinkCount int `json:"linkCount"` } // FileResult is the outcome of fetching one companion file. A Status of 0 // means the request itself failed (see Error); 4xx/5xx keep the status but // drop the content. type FileResult struct { URL string `json:"url"` Status int `json:"status"` ContentType string `json:"contentType"` Error string `json:"error,omitempty"` Content string `json:"content"` Truncated bool `json:"truncated"` LlmsDoc *LlmsDoc `json:"llmsDoc,omitempty"` } // found reports whether the file exists and answered with content. func (f *FileResult) found() bool { return f != nil && f.Status >= 200 && f.Status < 400 } // Files groups the companion documents fetched next to the page. type Files struct { LlmsTxt *FileResult `json:"llmsTxt"` LlmsFullTxt *FileResult `json:"llmsFullTxt"` WellKnownTxt *FileResult `json:"wellKnownLlmsTxt"` RobotsTxt *FileResult `json:"robotsTxt"` } // Snapshot is the full agent view of one URL: the rendered markdown, the // companion documents, and the validation checks. type Snapshot struct { RequestedURL string `json:"requestedUrl"` FinalURL string `json:"finalUrl"` Status int `json:"status"` ContentType string `json:"contentType"` DurationMS int64 `json:"durationMs"` FetchedAt time.Time `json:"fetchedAt"` Page *PageView `json:"page,omitempty"` Markdown string `json:"markdown"` MarkdownTruncated bool `json:"markdownTruncated"` Files Files `json:"files"` Checks []Check `json:"checks"` }