llm-browser/backend/internal/browse/model.go

// Package browse renders a remote page the way a text-only agent sees it:
// it fetches the URL, converts the HTML to markdown, gathers the llms.txt
// companions and robots.txt, and reports agent-readiness checks. The
// package is stateless — the Fetcher is its only collaborator.
package browse

import (
	"context"
	"net/http"
	"time"
)

// ValidationError reports an invalid input value. Handlers map it to 400.
type ValidationError struct {
	Field  string
	Reason string
}

func (e *ValidationError) Error() string { return e.Field + ": " + e.Reason }

// FetchError reports a remote fetch that failed before an answer arrived
// (DNS, TLS, connection refused, …). Handlers map it to 502 — the inspected
// site is the problem, not this service.
type FetchError struct {
	URL string
	Err error
}

func (e *FetchError) Error() string { return "fetch " + e.URL + ": " + e.Err.Error() }
func (e *FetchError) Unwrap() error { return e.Err }

// Severity classifies a Check.
type Severity string

const (
	SeverityOK    Severity = "ok"
	SeverityInfo  Severity = "info"
	SeverityWarn  Severity = "warn"
	SeverityError Severity = "error"
)

// Check is one validated aspect of the inspected site, written so a human
// can act on the detail without reading this package.
type Check struct {
	ID       string   `json:"id"`
	Label    string   `json:"label"`
	Severity Severity `json:"severity"`
	Detail   string   `json:"detail"`
}

// Response is a capped remote fetch result.
type Response struct {
	Status      int
	FinalURL    string // after redirects; empty means "same as requested"
	ContentType string
	Header      http.Header
	Body        []byte
	Truncated   bool
}

// Fetcher retrieves remote documents. It plays the role the Repository
// interface plays for database-backed resources: the service depends on the
// interface, tests substitute an in-memory fake.
type Fetcher interface {
	Fetch(ctx context.Context, url string) (*Response, error)
}

// PageView is what agents read out of the HTML head plus document shape
// statistics that drive the checks.
type PageView struct {
	Title       string `json:"title"`
	Description string `json:"description"`
	Canonical   string `json:"canonical"`
	RobotsMeta  string `json:"robotsMeta"`
	Lang        string `json:"lang"`
	H1          string `json:"h1"`
	LinkCount   int    `json:"linkCount"`
	DeadLinks   int    `json:"deadLinks"`
	ScriptCount int    `json:"scriptCount"`
	ImageCount  int    `json:"imageCount"`
	WordCount   int    `json:"wordCount"`
	TextLength  int    `json:"textLength"`
}

// LlmsLink is one `- [title](url): description` entry of an llms.txt file.
type LlmsLink struct {
	Title       string `json:"title"`
	URL         string `json:"url"`
	Description string `json:"description"`
}

// LlmsSection is one `## Heading` block of an llms.txt file.
type LlmsSection struct {
	Heading     string     `json:"heading"`
	Description string     `json:"description"`
	Links       []LlmsLink `json:"links"`
}

// LlmsDoc is a parsed llms.txt document per llmstxt.org: an H1 title, an
// optional blockquote summary, and optional H2 sections of curated links.
type LlmsDoc struct {
	Title       string        `json:"title"`
	Description string        `json:"description"`
	Sections    []LlmsSection `json:"sections"`
	LinkCount   int           `json:"linkCount"`
}

// FileResult is the outcome of fetching one companion file. A Status of 0
// means the request itself failed (see Error); 4xx/5xx keep the status but
// drop the content.
type FileResult struct {
	URL         string   `json:"url"`
	Status      int      `json:"status"`
	ContentType string   `json:"contentType"`
	Error       string   `json:"error,omitempty"`
	Content     string   `json:"content"`
	Truncated   bool     `json:"truncated"`
	LlmsDoc     *LlmsDoc `json:"llmsDoc,omitempty"`
}

// found reports whether the file exists and answered with content.
func (f *FileResult) found() bool { return f != nil && f.Status >= 200 && f.Status < 400 }

// Files groups the companion documents fetched next to the page.
type Files struct {
	LlmsTxt      *FileResult `json:"llmsTxt"`
	LlmsFullTxt  *FileResult `json:"llmsFullTxt"`
	WellKnownTxt *FileResult `json:"wellKnownLlmsTxt"`
	RobotsTxt    *FileResult `json:"robotsTxt"`
}

// Snapshot is the full agent view of one URL: the rendered markdown, the
// companion documents, and the validation checks.
type Snapshot struct {
	RequestedURL      string    `json:"requestedUrl"`
	FinalURL          string    `json:"finalUrl"`
	Status            int       `json:"status"`
	ContentType       string    `json:"contentType"`
	DurationMS        int64     `json:"durationMs"`
	FetchedAt         time.Time `json:"fetchedAt"`
	Page              *PageView `json:"page,omitempty"`
	Markdown          string    `json:"markdown"`
	MarkdownTruncated bool      `json:"markdownTruncated"`
	Files             Files     `json:"files"`
	Checks            []Check   `json:"checks"`
}