llm browser

backend/internal/browse/model.go

@@ -0,0 +1,146 @@
+// Package browse renders a remote page the way a text-only agent sees it:
+// it fetches the URL, converts the HTML to markdown, gathers the llms.txt
+// companions and robots.txt, and reports agent-readiness checks. The
+// package is stateless — the Fetcher is its only collaborator.
+package browse
+
+import (
+	"context"
+	"net/http"
+	"time"
+)
+
+// ValidationError reports an invalid input value. Handlers map it to 400.
+type ValidationError struct {
+	Field  string
+	Reason string
+}
+
+func (e *ValidationError) Error() string { return e.Field + ": " + e.Reason }
+
+// FetchError reports a remote fetch that failed before an answer arrived
+// (DNS, TLS, connection refused, …). Handlers map it to 502 — the inspected
+// site is the problem, not this service.
+type FetchError struct {
+	URL string
+	Err error
+}
+
+func (e *FetchError) Error() string { return "fetch " + e.URL + ": " + e.Err.Error() }
+func (e *FetchError) Unwrap() error { return e.Err }
+
+// Severity classifies a Check.
+type Severity string
+
+const (
+	SeverityOK    Severity = "ok"
+	SeverityInfo  Severity = "info"
+	SeverityWarn  Severity = "warn"
+	SeverityError Severity = "error"
+)
+
+// Check is one validated aspect of the inspected site, written so a human
+// can act on the detail without reading this package.
+type Check struct {
+	ID       string   `json:"id"`
+	Label    string   `json:"label"`
+	Severity Severity `json:"severity"`
+	Detail   string   `json:"detail"`
+}
+
+// Response is a capped remote fetch result.
+type Response struct {
+	Status      int
+	FinalURL    string // after redirects; empty means "same as requested"
+	ContentType string
+	Header      http.Header
+	Body        []byte
+	Truncated   bool
+}
+
+// Fetcher retrieves remote documents. It plays the role the Repository
+// interface plays for database-backed resources: the service depends on the
+// interface, tests substitute an in-memory fake.
+type Fetcher interface {
+	Fetch(ctx context.Context, url string) (*Response, error)
+}
+
+// PageView is what agents read out of the HTML head plus document shape
+// statistics that drive the checks.
+type PageView struct {
+	Title       string `json:"title"`
+	Description string `json:"description"`
+	Canonical   string `json:"canonical"`
+	RobotsMeta  string `json:"robotsMeta"`
+	Lang        string `json:"lang"`
+	H1          string `json:"h1"`
+	LinkCount   int    `json:"linkCount"`
+	DeadLinks   int    `json:"deadLinks"`
+	ScriptCount int    `json:"scriptCount"`
+	ImageCount  int    `json:"imageCount"`
+	WordCount   int    `json:"wordCount"`
+	TextLength  int    `json:"textLength"`
+}
+
+// LlmsLink is one `- [title](url): description` entry of an llms.txt file.
+type LlmsLink struct {
+	Title       string `json:"title"`
+	URL         string `json:"url"`
+	Description string `json:"description"`
+}
+
+// LlmsSection is one `## Heading` block of an llms.txt file.
+type LlmsSection struct {
+	Heading     string     `json:"heading"`
+	Description string     `json:"description"`
+	Links       []LlmsLink `json:"links"`
+}
+
+// LlmsDoc is a parsed llms.txt document per llmstxt.org: an H1 title, an
+// optional blockquote summary, and optional H2 sections of curated links.
+type LlmsDoc struct {
+	Title       string        `json:"title"`
+	Description string        `json:"description"`
+	Sections    []LlmsSection `json:"sections"`
+	LinkCount   int           `json:"linkCount"`
+}
+
+// FileResult is the outcome of fetching one companion file. A Status of 0
+// means the request itself failed (see Error); 4xx/5xx keep the status but
+// drop the content.
+type FileResult struct {
+	URL         string   `json:"url"`
+	Status      int      `json:"status"`
+	ContentType string   `json:"contentType"`
+	Error       string   `json:"error,omitempty"`
+	Content     string   `json:"content"`
+	Truncated   bool     `json:"truncated"`
+	LlmsDoc     *LlmsDoc `json:"llmsDoc,omitempty"`
+}
+
+// found reports whether the file exists and answered with content.
+func (f *FileResult) found() bool { return f != nil && f.Status >= 200 && f.Status < 400 }
+
+// Files groups the companion documents fetched next to the page.
+type Files struct {
+	LlmsTxt      *FileResult `json:"llmsTxt"`
+	LlmsFullTxt  *FileResult `json:"llmsFullTxt"`
+	WellKnownTxt *FileResult `json:"wellKnownLlmsTxt"`
+	RobotsTxt    *FileResult `json:"robotsTxt"`
+}
+
+// Snapshot is the full agent view of one URL: the rendered markdown, the
+// companion documents, and the validation checks.
+type Snapshot struct {
+	RequestedURL      string    `json:"requestedUrl"`
+	FinalURL          string    `json:"finalUrl"`
+	Status            int       `json:"status"`
+	ContentType       string    `json:"contentType"`
+	DurationMS        int64     `json:"durationMs"`
+	FetchedAt         time.Time `json:"fetchedAt"`
+	Page              *PageView `json:"page,omitempty"`
+	Markdown          string    `json:"markdown"`
+	MarkdownTruncated bool      `json:"markdownTruncated"`
+	Files             Files     `json:"files"`
+	Checks            []Check   `json:"checks"`
+}