llm browser
This commit is contained in:
developer committed
1 parent
afed023152
commit
a4dbb75195
39 files changed
+3709
-58
No files matched your search
@@ -0,0 +1,49 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.k6n.net/mats/fullstack-template/internal/browse"
|
||||
)
|
||||
|
||||
// browseHandler adapts HTTP requests to the browse service: read the url
|
||||
// query parameter, call Inspect, map errors to the wire. No fetching or
|
||||
// conversion logic lives here.
|
||||
type browseHandler struct {
|
||||
svc *browse.Service
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func (h *browseHandler) get(w http.ResponseWriter, r *http.Request) {
|
||||
raw := strings.TrimSpace(r.URL.Query().Get("url"))
|
||||
snap, err := h.svc.Inspect(r.Context(), raw)
|
||||
if err != nil {
|
||||
h.fail(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, snap)
|
||||
}
|
||||
|
||||
// fail is the translation point from browse errors to the wire:
|
||||
// *ValidationError → 400, *FetchError → 502 (the site is the problem),
|
||||
// context deadline → 504, anything else → 500 (logged).
|
||||
func (h *browseHandler) fail(w http.ResponseWriter, err error) {
|
||||
var vErr *browse.ValidationError
|
||||
var fErr *browse.FetchError
|
||||
switch {
|
||||
case errors.As(err, &vErr):
|
||||
writeError(w, http.StatusBadRequest, "validation_failed", vErr.Error())
|
||||
case errors.As(err, &fErr):
|
||||
h.logger.Info("browse fetch failed", "url", fErr.URL, "err", fErr.Err)
|
||||
writeError(w, http.StatusBadGateway, "unavailable", "could not fetch "+fErr.URL)
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
writeError(w, http.StatusGatewayTimeout, "timeout", "the site took too long to answer")
|
||||
default:
|
||||
h.logger.Error("browse failed", "err", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "an internal error occurred")
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.k6n.net/mats/fullstack-template/internal/browse"
|
||||
"git.k6n.net/mats/fullstack-template/internal/items"
|
||||
)
|
||||
|
||||
@@ -21,6 +22,7 @@ type Pinger interface {
|
||||
type Deps struct {
|
||||
Logger *slog.Logger
|
||||
Items *items.Service
|
||||
Browse *browse.Service
|
||||
DB Pinger
|
||||
AllowedOrigins []string
|
||||
RequestTimeout time.Duration // zero means the 30s default
|
||||
@@ -54,6 +56,7 @@ func New(deps Deps) http.Handler {
|
||||
logger: deps.Logger,
|
||||
db: deps.DB,
|
||||
items: &itemHandler{svc: deps.Items, logger: deps.Logger},
|
||||
browse: &browseHandler{svc: deps.Browse, logger: deps.Logger},
|
||||
auth: authH,
|
||||
}
|
||||
|
||||
@@ -80,6 +83,10 @@ func New(deps Deps) http.Handler {
|
||||
mux.HandleFunc("PATCH /api/v1/items/{id}", s.items.update)
|
||||
mux.HandleFunc("DELETE /api/v1/items/{id}", s.items.delete)
|
||||
|
||||
// Agent view: renders a remote page the way a text-only agent sees it.
|
||||
// The URL comes in via query parameter, so the response is a plain GET.
|
||||
mux.HandleFunc("GET /api/v1/browse", s.browse.get)
|
||||
|
||||
// The wrapper keeps the mux's automatic 405 + Allow responses for
|
||||
// wrong-method requests and renders all unmatched routes as JSON 404s
|
||||
// (the mux default is plain text).
|
||||
@@ -99,6 +106,7 @@ type server struct {
|
||||
logger *slog.Logger
|
||||
db Pinger
|
||||
items *itemHandler
|
||||
browse *browseHandler
|
||||
auth *authHandler
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.k6n.net/mats/fullstack-template/internal/api"
|
||||
"git.k6n.net/mats/fullstack-template/internal/browse"
|
||||
"git.k6n.net/mats/fullstack-template/internal/items"
|
||||
)
|
||||
|
||||
@@ -293,3 +294,100 @@ func TestCORSPreflight(t *testing.T) {
|
||||
t.Errorf("Allow-Origin for unknown origin = %q, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeFetcher is the same in-memory browse.Fetcher as
|
||||
// internal/browse/service_test.go, duplicated on purpose: test fakes stay
|
||||
// package-local. A nil response simulates a connection failure.
|
||||
type fakeFetcher struct {
|
||||
pages map[string]*browse.Response
|
||||
}
|
||||
|
||||
func (f fakeFetcher) Fetch(_ context.Context, url string) (*browse.Response, error) {
|
||||
r, ok := f.pages[url]
|
||||
if !ok {
|
||||
return &browse.Response{Status: 404, FinalURL: url, ContentType: "text/plain", Body: []byte("not found")}, nil
|
||||
}
|
||||
if r == nil {
|
||||
return nil, &browse.FetchError{URL: url, Err: errors.New("dial tcp: connection refused")}
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func newBrowseTestServer(t *testing.T) (http.Handler, fakeFetcher) {
|
||||
t.Helper()
|
||||
pages := map[string]*browse.Response{}
|
||||
add := func(url, ct, body string) {
|
||||
pages[url] = &browse.Response{Status: 200, FinalURL: url, ContentType: ct, Body: []byte(body)}
|
||||
}
|
||||
add("https://site.test/", "text/html; charset=utf-8",
|
||||
`<!doctype html><html><head><title>Site</title><meta name="description" content="d"></head>`+
|
||||
`<body><h1>Hello</h1><p><a href="/docs">docs</a></p></body></html>`)
|
||||
add("https://site.test/llms.txt", "text/markdown", "# Site\n\n> summary\n\n## Docs\n\n- [Docs](https://site.test/docs): the docs\n")
|
||||
add("https://site.test/robots.txt", "text/plain", "User-agent: *\nAllow: /\n")
|
||||
return api.New(api.Deps{
|
||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
Items: items.NewService(newFakeRepo()),
|
||||
Browse: browse.NewService(fakeFetcher{pages: pages}),
|
||||
DB: stubPinger{},
|
||||
AllowedOrigins: []string{"http://localhost:5173"},
|
||||
}), fakeFetcher{pages: pages}
|
||||
}
|
||||
|
||||
func TestBrowseEndpoint(t *testing.T) {
|
||||
h, _ := newBrowseTestServer(t)
|
||||
|
||||
rec := do(t, h, http.MethodGet, `/api/v1/browse?url=https%3A%2F%2Fsite.test%2F`, "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("browse = %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var snap browse.Snapshot
|
||||
decodeBody(t, rec, &snap)
|
||||
if snap.FinalURL != "https://site.test/" || snap.Status != 200 {
|
||||
t.Errorf("snapshot head = %d %s", snap.Status, snap.FinalURL)
|
||||
}
|
||||
if snap.Page == nil || snap.Page.Title != "Site" {
|
||||
t.Errorf("page = %+v", snap.Page)
|
||||
}
|
||||
if !strings.Contains(snap.Markdown, "# Hello") || !strings.Contains(snap.Markdown, "[docs](https://site.test/docs)") {
|
||||
t.Errorf("markdown = %q", snap.Markdown)
|
||||
}
|
||||
if snap.Files.LlmsTxt == nil || !strings.Contains(snap.Files.LlmsTxt.Content, "# Site") {
|
||||
t.Errorf("llms.txt = %+v", snap.Files.LlmsTxt)
|
||||
}
|
||||
if len(snap.Checks) == 0 {
|
||||
t.Error("checks empty")
|
||||
}
|
||||
// Bare-domain input normalizes to https on the wire.
|
||||
rec = do(t, h, http.MethodGet, `/api/v1/browse?url=site.test`, "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("bare domain = %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestBrowseEndpointErrors(t *testing.T) {
|
||||
h, fetcher := newBrowseTestServer(t)
|
||||
|
||||
// Missing and malformed input map to validation_failed.
|
||||
for _, target := range []string{"/api/v1/browse", "/api/v1/browse?url=", "/api/v1/browse?url=ftp://x.test"} {
|
||||
rec := do(t, h, http.MethodGet, target, "")
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Errorf("GET %s = %d, want 400: %s", target, rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A dead site maps to 502 unavailable.
|
||||
fetcher.pages["https://dead.test/"] = nil
|
||||
rec := do(t, h, http.MethodGet, `/api/v1/browse?url=https%3A%2F%2Fdead.test%2F`, "")
|
||||
if rec.Code != http.StatusBadGateway {
|
||||
t.Errorf("dead site = %d, want 502: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var errBody struct {
|
||||
Error struct {
|
||||
Code string `json:"code"`
|
||||
} `json:"error"`
|
||||
}
|
||||
decodeBody(t, rec, &errBody)
|
||||
if errBody.Error.Code != "unavailable" {
|
||||
t.Errorf("error code = %q, want unavailable", errBody.Error.Code)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package browse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Defaults applied when the FetchOptions field is left zero.
|
||||
const (
|
||||
// DefaultUserAgent announces the agent view; sites can see what is
|
||||
// browsing them and serve it like any other text agent.
|
||||
DefaultUserAgent = "LLM-Browser/0.1 (+https://git.k6n.net/mats/llm-browser; agent-view validator)"
|
||||
// DefaultFetchTimeout bounds one remote request, redirects included.
|
||||
DefaultFetchTimeout = 15 * time.Second
|
||||
// DefaultMaxBodySize caps how much of a response is read (2 MiB).
|
||||
DefaultMaxBodySize = 2 << 20
|
||||
)
|
||||
|
||||
// FetchOptions configures an HTTPFetcher. Zero fields fall back to the
|
||||
// matching Default* constant.
|
||||
type FetchOptions struct {
|
||||
UserAgent string
|
||||
Timeout time.Duration
|
||||
MaxBodySize int64
|
||||
}
|
||||
|
||||
// HTTPFetcher is the production Fetcher: one shared client, a fixed
|
||||
// user agent, and a hard cap on response bodies.
|
||||
type HTTPFetcher struct {
|
||||
client *http.Client
|
||||
opts FetchOptions
|
||||
}
|
||||
|
||||
// NewHTTPFetcher wires the fetcher. The client follows the standard
|
||||
// redirect policy (up to 10 hops); each request is bounded by opts.Timeout.
|
||||
func NewHTTPFetcher(opts FetchOptions) *HTTPFetcher {
|
||||
if opts.Timeout <= 0 {
|
||||
opts.Timeout = DefaultFetchTimeout
|
||||
}
|
||||
if opts.MaxBodySize <= 0 {
|
||||
opts.MaxBodySize = DefaultMaxBodySize
|
||||
}
|
||||
if opts.UserAgent == "" {
|
||||
opts.UserAgent = DefaultUserAgent
|
||||
}
|
||||
return &HTTPFetcher{
|
||||
client: &http.Client{Timeout: opts.Timeout},
|
||||
opts: opts,
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch performs a GET and reads at most MaxBodySize bytes. Transport
|
||||
// failures and read failures both come back as *FetchError.
|
||||
func (f *HTTPFetcher) Fetch(ctx context.Context, raw string) (*Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, raw, nil)
|
||||
if err != nil {
|
||||
return nil, &FetchError{URL: raw, Err: err}
|
||||
}
|
||||
req.Header.Set("User-Agent", f.opts.UserAgent)
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,text/markdown,text/plain;q=0.9,*/*;q=0.8")
|
||||
|
||||
resp, err := f.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, &FetchError{URL: raw, Err: err}
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, truncated, err := readCapped(resp.Body, f.opts.MaxBodySize)
|
||||
if err != nil {
|
||||
return nil, &FetchError{URL: raw, Err: err}
|
||||
}
|
||||
return &Response{
|
||||
Status: resp.StatusCode,
|
||||
FinalURL: resp.Request.URL.String(),
|
||||
ContentType: resp.Header.Get("Content-Type"),
|
||||
Header: resp.Header,
|
||||
Body: body,
|
||||
Truncated: truncated,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// readCapped reads r fully, stopping at max bytes; the second result
|
||||
// reports whether content beyond max was discarded.
|
||||
func readCapped(r io.Reader, max int64) ([]byte, bool, error) {
|
||||
body, err := io.ReadAll(io.LimitReader(r, max+1))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if int64(len(body)) > max {
|
||||
return body[:max], true, nil
|
||||
}
|
||||
return body, false, nil
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package browse
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// llmsLinkRe matches `- [title](url): description` (the optional quoted
|
||||
// title and the optional description are both per spec).
|
||||
var llmsLinkRe = regexp.MustCompile(`^\s*[-*]\s+\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)\s*(?::\s*(.*))?$`)
|
||||
|
||||
// parseLlmsTxt parses an llms.txt document per llmstxt.org:
|
||||
//
|
||||
// # Title (required, exactly once, first)
|
||||
// > Summary (optional blockquote)
|
||||
// ## Section (optional, starts a curated list)
|
||||
// > Section note (optional, directly after the heading)
|
||||
// - [Link](https://…): what it is
|
||||
//
|
||||
// Everything that is not one of those shapes is ignored — the parsed doc
|
||||
// describes what an agent can rely on, not everything the file contains.
|
||||
func parseLlmsTxt(content string) *LlmsDoc {
|
||||
doc := &LlmsDoc{Sections: []LlmsSection{}}
|
||||
var section *LlmsSection
|
||||
seenTitle := false
|
||||
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
t := strings.TrimSpace(line)
|
||||
switch {
|
||||
case strings.HasPrefix(t, "# ") && !seenTitle:
|
||||
doc.Title = strings.TrimSpace(t[2:])
|
||||
seenTitle = true
|
||||
case strings.HasPrefix(t, "## "):
|
||||
doc.Sections = append(doc.Sections, LlmsSection{
|
||||
Heading: strings.TrimSpace(t[3:]),
|
||||
Links: []LlmsLink{},
|
||||
})
|
||||
section = &doc.Sections[len(doc.Sections)-1]
|
||||
case strings.HasPrefix(t, ">"):
|
||||
note := strings.TrimSpace(strings.TrimLeft(t, "> "))
|
||||
if note == "" {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case !seenTitle:
|
||||
continue // blockquote before the title is not the summary
|
||||
case section != nil && section.Description == "":
|
||||
section.Description = note
|
||||
case section == nil:
|
||||
if doc.Description != "" {
|
||||
doc.Description += " "
|
||||
}
|
||||
doc.Description += note
|
||||
}
|
||||
default:
|
||||
m := llmsLinkRe.FindStringSubmatch(line)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
link := LlmsLink{Title: m[1], URL: m[2], Description: strings.TrimSpace(m[3])}
|
||||
if section == nil {
|
||||
doc.Sections = append(doc.Sections, LlmsSection{Links: []LlmsLink{}})
|
||||
section = &doc.Sections[len(doc.Sections)-1]
|
||||
}
|
||||
section.Links = append(section.Links, link)
|
||||
doc.LinkCount++
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
@@ -0,0 +1,565 @@
|
||||
package browse
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
"golang.org/x/net/html/atom"
|
||||
)
|
||||
|
||||
// pageMeta is what agents read from <head> plus document shape statistics
|
||||
// collected while walking the body.
|
||||
type pageMeta struct {
|
||||
Title string
|
||||
Description string
|
||||
Canonical string
|
||||
Robots string
|
||||
Lang string
|
||||
H1 string
|
||||
LinkCount int
|
||||
DeadLinks int
|
||||
ScriptCount int
|
||||
ImageCount int
|
||||
WordCount int
|
||||
TextLength int
|
||||
}
|
||||
|
||||
// maxMarkdownChars caps the converted markdown before it reaches the wire.
|
||||
const maxMarkdownChars = 200_000
|
||||
|
||||
var (
|
||||
hiddenStyleRe = regexp.MustCompile(`(?i)display\s*:\s*none|visibility\s*:\s*hidden`)
|
||||
wsRe = regexp.MustCompile(`\s+`)
|
||||
manyNlRe = regexp.MustCompile(`\n{3,}`)
|
||||
)
|
||||
|
||||
// htmlToMarkdown parses the document once and returns the body rendered as
|
||||
// markdown plus the head metadata. baseURL resolves relative links.
|
||||
func htmlToMarkdown(body []byte, baseURL string) (string, pageMeta, error) {
|
||||
var meta pageMeta
|
||||
doc, err := html.Parse(strings.NewReader(string(body)))
|
||||
if err != nil {
|
||||
return "", meta, err
|
||||
}
|
||||
|
||||
var htmlNode, bodyNode *html.Node
|
||||
var walk func(n *html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if c.Type != html.ElementNode {
|
||||
continue
|
||||
}
|
||||
switch c.DataAtom {
|
||||
case atom.Html:
|
||||
htmlNode = c
|
||||
walk(c)
|
||||
case atom.Head:
|
||||
collectHead(c, &meta)
|
||||
case atom.Body:
|
||||
bodyNode = c
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(doc)
|
||||
if htmlNode != nil {
|
||||
meta.Lang = attr(htmlNode, "lang")
|
||||
}
|
||||
if bodyNode != nil {
|
||||
inspectBody(bodyNode, &meta)
|
||||
}
|
||||
|
||||
r := &mdRenderer{base: parseBase(baseURL)}
|
||||
return r.renderBody(bodyNode), meta, nil
|
||||
}
|
||||
|
||||
// collectHead reads title, description, robots and canonical from <head>.
|
||||
func collectHead(head *html.Node, meta *pageMeta) {
|
||||
for c := head.FirstChild; c != nil; c = c.NextSibling {
|
||||
if c.Type != html.ElementNode {
|
||||
continue
|
||||
}
|
||||
switch c.DataAtom {
|
||||
case atom.Title:
|
||||
if meta.Title == "" {
|
||||
meta.Title = collapse(textContent(c))
|
||||
}
|
||||
case atom.Meta:
|
||||
content := collapse(attr(c, "content"))
|
||||
switch strings.ToLower(attr(c, "name")) {
|
||||
case "description":
|
||||
if meta.Description == "" {
|
||||
meta.Description = content
|
||||
}
|
||||
case "robots":
|
||||
if meta.Robots == "" {
|
||||
meta.Robots = content
|
||||
}
|
||||
}
|
||||
case atom.Link:
|
||||
if strings.EqualFold(attr(c, "rel"), "canonical") && meta.Canonical == "" {
|
||||
meta.Canonical = attr(c, "href")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// inspectBody counts the elements checks care about and measures the
|
||||
// visible text (script/style/svg content stays out of the count).
|
||||
func inspectBody(body *html.Node, meta *pageMeta) {
|
||||
var walk func(n *html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode {
|
||||
switch n.DataAtom {
|
||||
case atom.A:
|
||||
href := strings.TrimSpace(attr(n, "href"))
|
||||
if href == "" || strings.HasPrefix(href, "#") || strings.HasPrefix(strings.ToLower(href), "javascript:") {
|
||||
meta.DeadLinks++
|
||||
} else {
|
||||
meta.LinkCount++
|
||||
}
|
||||
case atom.Script:
|
||||
meta.ScriptCount++
|
||||
return
|
||||
case atom.Style, atom.Svg, atom.Noscript, atom.Template, atom.Iframe:
|
||||
return
|
||||
case atom.Img:
|
||||
meta.ImageCount++
|
||||
return
|
||||
case atom.H1:
|
||||
if meta.H1 == "" {
|
||||
meta.H1 = collapse(textContent(n))
|
||||
}
|
||||
}
|
||||
}
|
||||
if n.Type == html.TextNode {
|
||||
meta.TextLength += len(n.Data)
|
||||
meta.WordCount += len(strings.Fields(n.Data))
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(body)
|
||||
}
|
||||
|
||||
// mdRenderer renders a parsed document as markdown.
|
||||
type mdRenderer struct {
|
||||
base *url.URL
|
||||
}
|
||||
|
||||
func (r *mdRenderer) renderBody(body *html.Node) string {
|
||||
if body == nil {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
r.blocks(&b, body)
|
||||
out := strings.TrimSpace(b.String())
|
||||
return manyNlRe.ReplaceAllString(out, "\n\n")
|
||||
}
|
||||
|
||||
// skipped reports whether an element contributes nothing an agent could
|
||||
// act on: invisible content, script payloads, and embedded documents.
|
||||
func skipped(n *html.Node) bool {
|
||||
if n.Type != html.ElementNode {
|
||||
return false
|
||||
}
|
||||
switch n.DataAtom {
|
||||
case atom.Script, atom.Style, atom.Noscript, atom.Template, atom.Svg,
|
||||
atom.Head, atom.Link, atom.Meta, atom.Iframe, atom.Object, atom.Embed:
|
||||
return true
|
||||
}
|
||||
if hasAttr(n, "hidden") || strings.EqualFold(attr(n, "aria-hidden"), "true") {
|
||||
return true
|
||||
}
|
||||
return hiddenStyleRe.MatchString(attr(n, "style"))
|
||||
}
|
||||
|
||||
// blocks renders the children of n as a sequence of markdown blocks.
|
||||
func (r *mdRenderer) blocks(b *strings.Builder, n *html.Node) {
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if skipped(c) {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case c.Type == html.TextNode:
|
||||
if s := strings.TrimSpace(collapse(c.Data)); s != "" {
|
||||
b.WriteString(s + "\n\n")
|
||||
}
|
||||
case c.Type == html.ElementNode:
|
||||
r.element(b, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// element renders one block-level element.
|
||||
func (r *mdRenderer) element(b *strings.Builder, n *html.Node) {
|
||||
switch n.DataAtom {
|
||||
case atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6:
|
||||
level := int(n.DataAtom) - int(atom.H1) + 1
|
||||
b.WriteString(strings.Repeat("#", level) + " " + strings.TrimSpace(r.inline(n)) + "\n\n")
|
||||
case atom.P:
|
||||
if s := strings.TrimSpace(r.inline(n)); s != "" {
|
||||
b.WriteString(s + "\n\n")
|
||||
}
|
||||
case atom.Hr:
|
||||
b.WriteString("---\n\n")
|
||||
case atom.Blockquote:
|
||||
var inner strings.Builder
|
||||
r.blocks(&inner, n)
|
||||
b.WriteString(prefixLines(inner.String(), "> "))
|
||||
case atom.Pre:
|
||||
b.WriteString("```\n" + strings.Trim(textContent(n), "\n") + "\n```\n\n")
|
||||
case atom.Ul, atom.Ol:
|
||||
var lb strings.Builder
|
||||
r.list(&lb, n, "")
|
||||
b.WriteString(strings.TrimRight(lb.String(), "\n") + "\n\n")
|
||||
case atom.Table:
|
||||
r.table(b, n)
|
||||
case atom.Dl:
|
||||
r.defList(b, n)
|
||||
case atom.Button, atom.A, atom.Span, atom.Strong, atom.Em, atom.Code, atom.Small, atom.Label:
|
||||
// phrasing content at block position: keep it as its own paragraph
|
||||
if s := strings.TrimSpace(r.inline(n)); s != "" {
|
||||
b.WriteString(s + "\n\n")
|
||||
}
|
||||
default:
|
||||
r.blocks(b, n)
|
||||
}
|
||||
}
|
||||
|
||||
// list renders ul/ol items; nested lists indent under their item.
|
||||
func (r *mdRenderer) list(b *strings.Builder, n *html.Node, indent string) {
|
||||
ordered := n.DataAtom == atom.Ol
|
||||
index := 1
|
||||
if v := attr(n, "start"); v != "" {
|
||||
if i, err := strconv.Atoi(v); err == nil && i > 0 {
|
||||
index = i
|
||||
}
|
||||
}
|
||||
for li := n.FirstChild; li != nil; li = li.NextSibling {
|
||||
if li.Type != html.ElementNode || li.DataAtom != atom.Li {
|
||||
continue
|
||||
}
|
||||
marker := "- "
|
||||
if ordered {
|
||||
marker = strconv.Itoa(index) + ". "
|
||||
index++
|
||||
}
|
||||
var text strings.Builder
|
||||
var nested []*html.Node
|
||||
for c := li.FirstChild; c != nil; c = c.NextSibling {
|
||||
if skipped(c) {
|
||||
continue
|
||||
}
|
||||
if c.Type == html.ElementNode && (c.DataAtom == atom.Ul || c.DataAtom == atom.Ol) {
|
||||
nested = append(nested, c)
|
||||
continue
|
||||
}
|
||||
if c.Type == html.TextNode {
|
||||
text.WriteString(collapse(c.Data))
|
||||
} else {
|
||||
text.WriteString(r.inline(c))
|
||||
}
|
||||
}
|
||||
if item := strings.TrimSpace(text.String()); item != "" {
|
||||
b.WriteString(indent + marker + item + "\n")
|
||||
}
|
||||
for _, sub := range nested {
|
||||
var nb strings.Builder
|
||||
r.list(&nb, sub, indent+" ")
|
||||
b.WriteString(strings.TrimRight(nb.String(), "\n") + "\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// table renders a GFM pipe table; the first header row (th, else the first
|
||||
// tr) becomes the table head.
|
||||
func (r *mdRenderer) table(b *strings.Builder, n *html.Node) {
|
||||
var header []string
|
||||
var rows [][]string
|
||||
collectRows := func(scope *html.Node) {
|
||||
for tr := scope.FirstChild; tr != nil; tr = tr.NextSibling {
|
||||
if tr.Type != html.ElementNode || tr.DataAtom != atom.Tr {
|
||||
continue
|
||||
}
|
||||
var cells []string
|
||||
isHeader := false
|
||||
for cell := tr.FirstChild; cell != nil; cell = cell.NextSibling {
|
||||
if cell.Type != html.ElementNode {
|
||||
continue
|
||||
}
|
||||
switch cell.DataAtom {
|
||||
case atom.Th:
|
||||
isHeader = true
|
||||
cells = append(cells, r.cell(cell))
|
||||
case atom.Td:
|
||||
cells = append(cells, r.cell(cell))
|
||||
}
|
||||
}
|
||||
if len(cells) == 0 {
|
||||
continue
|
||||
}
|
||||
if isHeader && header == nil {
|
||||
header = cells
|
||||
} else {
|
||||
rows = append(rows, cells)
|
||||
}
|
||||
}
|
||||
}
|
||||
hasSections := false
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if c.Type != html.ElementNode {
|
||||
continue
|
||||
}
|
||||
switch c.DataAtom {
|
||||
case atom.Thead, atom.Tbody, atom.Tfoot:
|
||||
hasSections = true
|
||||
collectRows(c)
|
||||
}
|
||||
}
|
||||
if !hasSections {
|
||||
collectRows(n) // table with bare rows
|
||||
}
|
||||
if header == nil && len(rows) > 0 {
|
||||
header, rows = rows[0], rows[1:]
|
||||
}
|
||||
cols := len(header)
|
||||
for _, row := range rows {
|
||||
if len(row) > cols {
|
||||
cols = len(row)
|
||||
}
|
||||
}
|
||||
if cols == 0 {
|
||||
return
|
||||
}
|
||||
pad := func(row []string) []string {
|
||||
out := make([]string, cols)
|
||||
copy(out, row)
|
||||
return out
|
||||
}
|
||||
b.WriteString("| " + strings.Join(pad(header), " | ") + " |\n")
|
||||
b.WriteString("|" + strings.Repeat(" --- |", cols) + "\n")
|
||||
for _, row := range rows {
|
||||
b.WriteString("| " + strings.Join(pad(row), " | ") + " |\n")
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
// defList renders dt as a bold term and dd as its paragraph.
|
||||
func (r *mdRenderer) defList(b *strings.Builder, n *html.Node) {
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if c.Type != html.ElementNode {
|
||||
continue
|
||||
}
|
||||
switch c.DataAtom {
|
||||
case atom.Dt:
|
||||
if s := strings.TrimSpace(r.inline(c)); s != "" {
|
||||
b.WriteString("**" + s + "**\n\n")
|
||||
}
|
||||
case atom.Dd:
|
||||
if s := strings.TrimSpace(r.inline(c)); s != "" {
|
||||
b.WriteString(s + "\n\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cell flattens one table cell to a single line.
|
||||
func (r *mdRenderer) cell(n *html.Node) string {
|
||||
s := strings.TrimSpace(r.inline(n))
|
||||
return strings.ReplaceAll(s, "|", "\\|")
|
||||
}
|
||||
|
||||
// inline renders phrasing content to a markdown string.
|
||||
func (r *mdRenderer) inline(n *html.Node) string {
|
||||
var b strings.Builder
|
||||
r.inlineInto(&b, n)
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func (r *mdRenderer) inlineInto(b *strings.Builder, n *html.Node) {
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if skipped(c) {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case c.Type == html.TextNode:
|
||||
b.WriteString(collapse(c.Data))
|
||||
case c.Type == html.ElementNode:
|
||||
r.inlineElement(b, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *mdRenderer) inlineElement(b *strings.Builder, n *html.Node) {
|
||||
switch n.DataAtom {
|
||||
case atom.A:
|
||||
label := strings.TrimSpace(r.inline(n))
|
||||
if label == "" {
|
||||
label = attr(n, "href")
|
||||
}
|
||||
href := strings.TrimSpace(attr(n, "href"))
|
||||
lower := strings.ToLower(href)
|
||||
if href == "" || strings.HasPrefix(href, "#") || strings.HasPrefix(lower, "javascript:") {
|
||||
// an agent cannot follow these; keep the text only
|
||||
b.WriteString(label)
|
||||
return
|
||||
}
|
||||
b.WriteString("[" + label + "](" + r.resolve(href) + ")")
|
||||
case atom.Img:
|
||||
src := strings.TrimSpace(attr(n, "src"))
|
||||
if src == "" {
|
||||
return
|
||||
}
|
||||
alt := collapse(attr(n, "alt"))
|
||||
if alt == "" {
|
||||
alt = baseName(src)
|
||||
}
|
||||
b.WriteString("![" + alt + "](" + r.resolve(src) + ")")
|
||||
case atom.Br:
|
||||
b.WriteString("\n")
|
||||
case atom.Hr:
|
||||
b.WriteString("\n---\n")
|
||||
case atom.Strong, atom.B:
|
||||
b.WriteString("**" + strings.TrimSpace(r.inline(n)) + "**")
|
||||
case atom.Em, atom.I:
|
||||
b.WriteString("*" + strings.TrimSpace(r.inline(n)) + "*")
|
||||
case atom.Del, atom.S:
|
||||
b.WriteString("~~" + strings.TrimSpace(r.inline(n)) + "~~")
|
||||
case atom.Code:
|
||||
b.WriteString("`" + strings.TrimSpace(r.inline(n)) + "`")
|
||||
case atom.Summary:
|
||||
b.WriteString("**" + strings.TrimSpace(r.inline(n)) + "** ")
|
||||
case atom.Input, atom.Select, atom.Textarea, atom.Form:
|
||||
// form controls have no markdown meaning an agent can act on
|
||||
default:
|
||||
if blockish[n.DataAtom] {
|
||||
// block-level content inside inline context (nav menus, divs in
|
||||
// paragraphs): keep the words from running together
|
||||
padInline(b)
|
||||
r.inlineInto(b, n)
|
||||
padInline(b)
|
||||
return
|
||||
}
|
||||
r.inlineInto(b, n)
|
||||
}
|
||||
}
|
||||
|
||||
// blockish marks elements a browser starts on a fresh line; adjacent ones
|
||||
// need separating whitespace once flattened to inline text.
|
||||
var blockish = map[atom.Atom]bool{
|
||||
atom.Div: true, atom.Ul: true, atom.Ol: true, atom.Li: true, atom.Nav: true,
|
||||
atom.Section: true, atom.Article: true, atom.Header: true, atom.Footer: true,
|
||||
atom.Aside: true, atom.Main: true, atom.Form: true, atom.Fieldset: true,
|
||||
atom.Figure: true, atom.P: true, atom.Table: true, atom.Tr: true,
|
||||
}
|
||||
|
||||
func padInline(b *strings.Builder) {
|
||||
s := b.String()
|
||||
if s == "" {
|
||||
return
|
||||
}
|
||||
if last := s[len(s)-1]; last != ' ' && last != '\n' {
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
}
|
||||
|
||||
// resolve turns href into an absolute URL against the inspected page.
|
||||
func (r *mdRenderer) resolve(href string) string {
|
||||
if r.base == nil {
|
||||
return href
|
||||
}
|
||||
u, err := url.Parse(href)
|
||||
if err != nil {
|
||||
return href
|
||||
}
|
||||
return r.base.ResolveReference(u).String()
|
||||
}
|
||||
|
||||
func parseBase(raw string) *url.URL {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// collapse flattens whitespace runs to single spaces.
|
||||
func collapse(s string) string { return wsRe.ReplaceAllString(s, " ") }
|
||||
|
||||
// textContent returns the raw concatenated text under n.
|
||||
func textContent(n *html.Node) string {
|
||||
var b strings.Builder
|
||||
var walk func(n *html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.TextNode {
|
||||
b.WriteString(n.Data)
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(n)
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func prefixLines(s, prefix string) string {
|
||||
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
|
||||
for i, line := range lines {
|
||||
if line == "" {
|
||||
lines[i] = strings.TrimRight(prefix, " ")
|
||||
continue
|
||||
}
|
||||
lines[i] = prefix + line
|
||||
}
|
||||
return strings.Join(lines, "\n") + "\n\n"
|
||||
}
|
||||
|
||||
func baseName(src string) string {
|
||||
s := src
|
||||
if i := strings.IndexAny(s, "?#"); i >= 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
if i := strings.LastIndex(s, "/"); i >= 0 {
|
||||
s = s[i+1:]
|
||||
}
|
||||
if s == "" {
|
||||
return "image"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func attr(n *html.Node, key string) string {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == key {
|
||||
return a.Val
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func hasAttr(n *html.Node, key string) bool {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == key {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// truncateMarkdown caps markdown at max chars without splitting a rune.
|
||||
func truncateMarkdown(md string, max int) (string, bool) {
|
||||
if len(md) <= max {
|
||||
return md, false
|
||||
}
|
||||
// step back to a rune boundary so the cut stays valid UTF-8
|
||||
for i := max; i > max-4 && i > 0; i-- {
|
||||
if (md[i] & 0xC0) != 0x80 {
|
||||
return md[:i], true
|
||||
}
|
||||
}
|
||||
return md[:max], true
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package browse
|
||||
|
||||
// Internal test: it pins the unexported converter's exact output so
|
||||
// formatting drift is caught by the package itself.
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHTMLToMarkdown(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
name string
|
||||
html string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "headings and paragraphs",
|
||||
html: `<html><body><h1>Title</h1><p>First.</p><p>Second <strong>bold</strong> and <em>italic</em>.</p></body></html>`,
|
||||
want: "# Title\n\nFirst.\n\nSecond **bold** and *italic*.",
|
||||
},
|
||||
{
|
||||
name: "relative links resolve against the page",
|
||||
html: `<body><p><a href="/docs/guide">Guide</a> and <a href="https://abs.test/x">abs</a></p></body>`,
|
||||
want: "[Guide](https://page.test/docs/guide) and [abs](https://abs.test/x)",
|
||||
},
|
||||
{
|
||||
name: "dead links keep their text only",
|
||||
html: `<body><p><a href="#">top</a> <a href="javascript:go()">go</a></p></body>`,
|
||||
want: "top go",
|
||||
},
|
||||
{
|
||||
name: "lists nest and order",
|
||||
html: `<body><ul><li>one</li><li>two<ol start="3"><li>deep</li></ol></li></ul></body>`,
|
||||
want: "- one\n- two\n 3. deep",
|
||||
},
|
||||
{
|
||||
name: "blockquote prefixes every line",
|
||||
html: `<body><blockquote><p>line one</p><p>line two</p></blockquote></body>`,
|
||||
want: "> line one\n>\n> line two",
|
||||
},
|
||||
{
|
||||
name: "pre renders as a fenced block",
|
||||
html: `<body><pre>keep
|
||||
spacing</pre></body>`,
|
||||
want: "```\nkeep\n spacing\n```",
|
||||
},
|
||||
{
|
||||
name: "tables render GFM",
|
||||
html: `<body><table><thead><tr><th>Name</th><th>Size</th></tr></thead>` +
|
||||
`<tbody><tr><td>a|b</td><td>1</td></tr></tbody></table></body>`,
|
||||
want: "| Name | Size |\n| --- | --- |\n| a\\|b | 1 |",
|
||||
},
|
||||
{
|
||||
name: "script, style and hidden content vanish",
|
||||
html: `<body><style>p{}</style><script>var x=1;</script>` +
|
||||
`<p hidden>nope</p><p aria-hidden="true">nor this</p><p>visible</p></body>`,
|
||||
want: "visible",
|
||||
},
|
||||
{
|
||||
name: "images keep alt text",
|
||||
html: `<body><p><img src="/img/logo.png" alt="Logo"> and <img src="/img/x.png"></p></body>`,
|
||||
want: " and ",
|
||||
},
|
||||
{
|
||||
name: "consecutive blocks collapse to one blank line",
|
||||
html: `<body><div><p>a</p></div><div><p>b</p></div></body>`,
|
||||
want: "a\n\nb",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got, _, err := htmlToMarkdown([]byte(tc.html), "https://page.test/")
|
||||
if err != nil {
|
||||
t.Fatalf("htmlToMarkdown: %v", err)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Errorf("markdown =\n%q\nwant\n%q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTMLToMarkdownMetadata(t *testing.T) {
|
||||
t.Parallel()
|
||||
html := `<!doctype html><html lang="sv"><head>
|
||||
<title>Meta sidan</title>
|
||||
<meta name="description" content="Beskrivning.">
|
||||
<meta name="robots" content="noai">
|
||||
<link rel="canonical" href="https://page.test/canon">
|
||||
</head><body>
|
||||
<h1>Rubrik</h1>
|
||||
<p>lite text att räkna</p>
|
||||
<script src="/x.js"></script><img src="/y.png" alt="y">
|
||||
</body></html>`
|
||||
|
||||
_, meta, err := htmlToMarkdown([]byte(html), "https://page.test/")
|
||||
if err != nil {
|
||||
t.Fatalf("htmlToMarkdown: %v", err)
|
||||
}
|
||||
if meta.Title != "Meta sidan" || meta.Description != "Beskrivning." {
|
||||
t.Errorf("head metadata wrong: %+v", meta)
|
||||
}
|
||||
if meta.Lang != "sv" || meta.Canonical != "https://page.test/canon" {
|
||||
t.Errorf("lang/canonical wrong: %+v", meta)
|
||||
}
|
||||
if meta.H1 != "Rubrik" {
|
||||
t.Errorf("h1 = %q", meta.H1)
|
||||
}
|
||||
if meta.WordCount != 5 || meta.TextLength == 0 {
|
||||
t.Errorf("text stats wrong: words=%d chars=%d", meta.WordCount, meta.TextLength)
|
||||
}
|
||||
if meta.ScriptCount != 1 || meta.ImageCount != 1 {
|
||||
t.Errorf("element counts wrong: %+v", meta)
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(meta.Robots), "noai") {
|
||||
t.Errorf("robots = %q", meta.Robots)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
// Package browse renders a remote page the way a text-only agent sees it:
|
||||
// it fetches the URL, converts the HTML to markdown, gathers the llms.txt
|
||||
// companions and robots.txt, and reports agent-readiness checks. The
|
||||
// package is stateless — the Fetcher is its only collaborator.
|
||||
package browse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ValidationError reports an invalid input value. Handlers map it to 400.
|
||||
type ValidationError struct {
|
||||
Field string
|
||||
Reason string
|
||||
}
|
||||
|
||||
func (e *ValidationError) Error() string { return e.Field + ": " + e.Reason }
|
||||
|
||||
// FetchError reports a remote fetch that failed before an answer arrived
|
||||
// (DNS, TLS, connection refused, …). Handlers map it to 502 — the inspected
|
||||
// site is the problem, not this service.
|
||||
type FetchError struct {
|
||||
URL string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *FetchError) Error() string { return "fetch " + e.URL + ": " + e.Err.Error() }
|
||||
func (e *FetchError) Unwrap() error { return e.Err }
|
||||
|
||||
// Severity classifies a Check.
|
||||
type Severity string
|
||||
|
||||
const (
|
||||
SeverityOK Severity = "ok"
|
||||
SeverityInfo Severity = "info"
|
||||
SeverityWarn Severity = "warn"
|
||||
SeverityError Severity = "error"
|
||||
)
|
||||
|
||||
// Check is one validated aspect of the inspected site, written so a human
|
||||
// can act on the detail without reading this package.
|
||||
type Check struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
Severity Severity `json:"severity"`
|
||||
Detail string `json:"detail"`
|
||||
}
|
||||
|
||||
// Response is a capped remote fetch result.
|
||||
type Response struct {
|
||||
Status int
|
||||
FinalURL string // after redirects; empty means "same as requested"
|
||||
ContentType string
|
||||
Header http.Header
|
||||
Body []byte
|
||||
Truncated bool
|
||||
}
|
||||
|
||||
// Fetcher retrieves remote documents. It plays the role the Repository
|
||||
// interface plays for database-backed resources: the service depends on the
|
||||
// interface, tests substitute an in-memory fake.
|
||||
type Fetcher interface {
|
||||
Fetch(ctx context.Context, url string) (*Response, error)
|
||||
}
|
||||
|
||||
// PageView is what agents read out of the HTML head plus document shape
|
||||
// statistics that drive the checks.
|
||||
type PageView struct {
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Canonical string `json:"canonical"`
|
||||
RobotsMeta string `json:"robotsMeta"`
|
||||
Lang string `json:"lang"`
|
||||
H1 string `json:"h1"`
|
||||
LinkCount int `json:"linkCount"`
|
||||
DeadLinks int `json:"deadLinks"`
|
||||
ScriptCount int `json:"scriptCount"`
|
||||
ImageCount int `json:"imageCount"`
|
||||
WordCount int `json:"wordCount"`
|
||||
TextLength int `json:"textLength"`
|
||||
}
|
||||
|
||||
// LlmsLink is one `- [title](url): description` entry of an llms.txt file.
|
||||
type LlmsLink struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
// LlmsSection is one `## Heading` block of an llms.txt file.
|
||||
type LlmsSection struct {
|
||||
Heading string `json:"heading"`
|
||||
Description string `json:"description"`
|
||||
Links []LlmsLink `json:"links"`
|
||||
}
|
||||
|
||||
// LlmsDoc is a parsed llms.txt document per llmstxt.org: an H1 title, an
|
||||
// optional blockquote summary, and optional H2 sections of curated links.
|
||||
type LlmsDoc struct {
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Sections []LlmsSection `json:"sections"`
|
||||
LinkCount int `json:"linkCount"`
|
||||
}
|
||||
|
||||
// FileResult is the outcome of fetching one companion file. A Status of 0
|
||||
// means the request itself failed (see Error); 4xx/5xx keep the status but
|
||||
// drop the content.
|
||||
type FileResult struct {
|
||||
URL string `json:"url"`
|
||||
Status int `json:"status"`
|
||||
ContentType string `json:"contentType"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Content string `json:"content"`
|
||||
Truncated bool `json:"truncated"`
|
||||
LlmsDoc *LlmsDoc `json:"llmsDoc,omitempty"`
|
||||
}
|
||||
|
||||
// found reports whether the file exists and answered with content.
|
||||
func (f *FileResult) found() bool { return f != nil && f.Status >= 200 && f.Status < 400 }
|
||||
|
||||
// Files groups the companion documents fetched next to the page.
|
||||
type Files struct {
|
||||
LlmsTxt *FileResult `json:"llmsTxt"`
|
||||
LlmsFullTxt *FileResult `json:"llmsFullTxt"`
|
||||
WellKnownTxt *FileResult `json:"wellKnownLlmsTxt"`
|
||||
RobotsTxt *FileResult `json:"robotsTxt"`
|
||||
}
|
||||
|
||||
// Snapshot is the full agent view of one URL: the rendered markdown, the
|
||||
// companion documents, and the validation checks.
|
||||
type Snapshot struct {
|
||||
RequestedURL string `json:"requestedUrl"`
|
||||
FinalURL string `json:"finalUrl"`
|
||||
Status int `json:"status"`
|
||||
ContentType string `json:"contentType"`
|
||||
DurationMS int64 `json:"durationMs"`
|
||||
FetchedAt time.Time `json:"fetchedAt"`
|
||||
Page *PageView `json:"page,omitempty"`
|
||||
Markdown string `json:"markdown"`
|
||||
MarkdownTruncated bool `json:"markdownTruncated"`
|
||||
Files Files `json:"files"`
|
||||
Checks []Check `json:"checks"`
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
package browse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Tunables for the checks.
|
||||
const (
|
||||
// jsShellTextChars: with fewer server-rendered characters than this
|
||||
// next to real script traffic, the page probably needs JavaScript.
|
||||
jsShellTextChars = 400
|
||||
// contextHeavyChars: markdown beyond this is a lot for one context.
|
||||
contextHeavyChars = 150_000
|
||||
)
|
||||
|
||||
// Companion paths, always resolved against the site root of the page.
|
||||
const (
|
||||
llmsTxtPath = "/llms.txt"
|
||||
llmsFullTxtPath = "/llms-full.txt"
|
||||
wellKnownTxtPath = "/.well-known/llms.txt"
|
||||
robotsTxtPath = "/robots.txt"
|
||||
)
|
||||
|
||||
// Service renders remote pages the way a text-only agent sees them. It is
|
||||
// stateless; the Fetcher is its only collaborator.
|
||||
type Service struct {
|
||||
fetch Fetcher
|
||||
}
|
||||
|
||||
// NewService returns a Service over the given fetcher.
|
||||
func NewService(fetch Fetcher) *Service {
|
||||
return &Service{fetch: fetch}
|
||||
}
|
||||
|
||||
// Inspect fetches target (a bare domain or a full URL), converts the page
|
||||
// to markdown, gathers the llms.txt companions and robots.txt in parallel,
|
||||
// and assembles the agent-readiness checks.
|
||||
func (s *Service) Inspect(ctx context.Context, raw string) (*Snapshot, error) {
|
||||
target, err := NormalizeURL(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
started := time.Now()
|
||||
|
||||
page, err := s.fetch.Fetch(ctx, target)
|
||||
if err != nil {
|
||||
var fe *FetchError
|
||||
if !errors.As(err, &fe) {
|
||||
fe = &FetchError{URL: target, Err: err}
|
||||
}
|
||||
return nil, fe
|
||||
}
|
||||
|
||||
snap := &Snapshot{
|
||||
RequestedURL: target,
|
||||
FinalURL: firstNonEmpty(page.FinalURL, target),
|
||||
Status: page.Status,
|
||||
ContentType: page.ContentType,
|
||||
FetchedAt: time.Now().UTC(),
|
||||
Checks: []Check{},
|
||||
}
|
||||
|
||||
var meta pageMeta
|
||||
switch {
|
||||
case isHTML(page.ContentType, string(page.Body)):
|
||||
md, m, convErr := htmlToMarkdown(page.Body, snap.FinalURL)
|
||||
if convErr != nil {
|
||||
return nil, fmt.Errorf("parse %s: %w", snap.FinalURL, convErr)
|
||||
}
|
||||
meta = m
|
||||
snap.Markdown, snap.MarkdownTruncated = truncateMarkdown(md, maxMarkdownChars)
|
||||
snap.Page = &PageView{
|
||||
Title: meta.Title,
|
||||
Description: meta.Description,
|
||||
Canonical: meta.Canonical,
|
||||
RobotsMeta: meta.Robots,
|
||||
Lang: meta.Lang,
|
||||
H1: meta.H1,
|
||||
LinkCount: meta.LinkCount,
|
||||
DeadLinks: meta.DeadLinks,
|
||||
ScriptCount: meta.ScriptCount,
|
||||
ImageCount: meta.ImageCount,
|
||||
WordCount: meta.WordCount,
|
||||
TextLength: meta.TextLength,
|
||||
}
|
||||
case isText(page.ContentType):
|
||||
snap.Markdown, snap.MarkdownTruncated = truncateMarkdown(string(page.Body), maxMarkdownChars)
|
||||
default:
|
||||
snap.Markdown, snap.MarkdownTruncated =
|
||||
truncateMarkdown("```"+page.ContentType+"\n"+string(page.Body)+"\n```", maxMarkdownChars)
|
||||
}
|
||||
|
||||
// Companions come from the site root the page ended up on. They are
|
||||
// fetched concurrently so one slow file cannot stretch the inspection.
|
||||
root := siteRoot(snap.FinalURL)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(4)
|
||||
go s.fetchFile(ctx, &wg, root+llmsTxtPath, true, &snap.Files.LlmsTxt)
|
||||
go s.fetchFile(ctx, &wg, root+llmsFullTxtPath, true, &snap.Files.LlmsFullTxt)
|
||||
go s.fetchFile(ctx, &wg, root+wellKnownTxtPath, true, &snap.Files.WellKnownTxt)
|
||||
go s.fetchFile(ctx, &wg, root+robotsTxtPath, false, &snap.Files.RobotsTxt)
|
||||
wg.Wait()
|
||||
|
||||
snap.Checks = buildChecks(snap)
|
||||
snap.DurationMS = time.Since(started).Milliseconds()
|
||||
return snap, nil
|
||||
}
|
||||
|
||||
// NormalizeURL turns user input into a fetchable URL: a bare domain gets
|
||||
// https://, credentials are dropped, fragments are dropped.
|
||||
func NormalizeURL(raw string) (string, error) {
|
||||
s := strings.TrimSpace(raw)
|
||||
if s == "" {
|
||||
return "", &ValidationError{Field: "url", Reason: "required"}
|
||||
}
|
||||
if !strings.Contains(s, "://") {
|
||||
s = "https://" + s
|
||||
}
|
||||
u, err := url.Parse(s)
|
||||
if err != nil || u.Host == "" {
|
||||
return "", &ValidationError{Field: "url", Reason: "must be a domain or URL agents can open"}
|
||||
}
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return "", &ValidationError{Field: "url", Reason: "only http and https URLs are supported"}
|
||||
}
|
||||
u.User = nil
|
||||
u.Fragment = ""
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
// fetchFile fetches one companion into dst. A failed request is a result,
|
||||
// not an inspection failure — the checks describe what is missing.
|
||||
func (s *Service) fetchFile(ctx context.Context, wg *sync.WaitGroup, url string, parseLlms bool, dst **FileResult) {
|
||||
defer wg.Done()
|
||||
|
||||
res, err := s.fetch.Fetch(ctx, url)
|
||||
if err != nil {
|
||||
var fe *FetchError
|
||||
if errors.As(err, &fe) {
|
||||
err = fe.Err
|
||||
}
|
||||
*dst = &FileResult{URL: url, Error: err.Error()}
|
||||
return
|
||||
}
|
||||
|
||||
fr := &FileResult{
|
||||
URL: firstNonEmpty(res.FinalURL, url),
|
||||
Status: res.Status,
|
||||
ContentType: res.ContentType,
|
||||
Truncated: res.Truncated,
|
||||
}
|
||||
if fr.found() {
|
||||
fr.Content = string(res.Body)
|
||||
if parseLlms && !isHTML(fr.ContentType, fr.Content) {
|
||||
fr.LlmsDoc = parseLlmsTxt(fr.Content)
|
||||
}
|
||||
}
|
||||
*dst = fr
|
||||
}
|
||||
|
||||
// buildChecks turns a finished snapshot into the validation report.
|
||||
func buildChecks(snap *Snapshot) []Check {
|
||||
c := []Check{}
|
||||
add := func(id, label string, sev Severity, format string, args ...any) {
|
||||
c = append(c, Check{ID: id, Label: label, Severity: sev, Detail: fmt.Sprintf(format, args...)})
|
||||
}
|
||||
|
||||
// — the page itself —
|
||||
switch {
|
||||
case snap.Status >= 500:
|
||||
add("page.status", "Page reachable", SeverityError, "server answered %d", snap.Status)
|
||||
case snap.Status >= 400:
|
||||
add("page.status", "Page reachable", SeverityError, "server answered %d — agents see an error page", snap.Status)
|
||||
case snap.Status >= 300:
|
||||
add("page.status", "Page reachable", SeverityWarn, "server answered %d", snap.Status)
|
||||
default:
|
||||
add("page.status", "Page reachable", SeverityOK, "%d OK", snap.Status)
|
||||
}
|
||||
|
||||
if u, err := url.Parse(snap.FinalURL); err == nil && u.Scheme == "http" {
|
||||
add("page.https", "Served over HTTPS", SeverityWarn, "the final URL is plain http; secure agents may refuse it")
|
||||
} else {
|
||||
add("page.https", "Served over HTTPS", SeverityOK, "https")
|
||||
}
|
||||
|
||||
if snap.FinalURL != snap.RequestedURL {
|
||||
add("page.redirect", "Direct URL", SeverityInfo, "redirected to %s", snap.FinalURL)
|
||||
}
|
||||
|
||||
if snap.Page != nil {
|
||||
p := snap.Page
|
||||
if p.Title == "" {
|
||||
add("page.title", "Title", SeverityError, "no <title> — agents use it as the page name in their results")
|
||||
} else {
|
||||
add("page.title", "Title", SeverityOK, "%s", clip(p.Title, 80))
|
||||
}
|
||||
|
||||
if p.Description == "" {
|
||||
add("page.description", "Meta description", SeverityWarn, "missing — agents have to guess the page's purpose from body text")
|
||||
} else {
|
||||
add("page.description", "Meta description", SeverityOK, "%s", clip(p.Description, 140))
|
||||
}
|
||||
|
||||
robots := strings.ToLower(p.RobotsMeta)
|
||||
switch {
|
||||
case strings.Contains(robots, "noindex"):
|
||||
add("page.robots-meta", "Robots meta", SeverityError, "meta robots declares noindex — indexing agents are told to ignore this page")
|
||||
case strings.Contains(robots, "nofollow"):
|
||||
add("page.robots-meta", "Robots meta", SeverityWarn, "meta robots declares nofollow — agents are told not to follow this page's links")
|
||||
case p.RobotsMeta != "":
|
||||
add("page.robots-meta", "Robots meta", SeverityOK, "%s", p.RobotsMeta)
|
||||
default:
|
||||
add("page.robots-meta", "Robots meta", SeverityOK, "no restrictions declared")
|
||||
}
|
||||
|
||||
if p.TextLength < jsShellTextChars && p.ScriptCount >= 3 {
|
||||
add("page.javascript", "Reads without JavaScript", SeverityWarn,
|
||||
"only %d characters of server-rendered text next to %d script tags — the page likely renders in the browser, which text agents never run",
|
||||
p.TextLength, p.ScriptCount)
|
||||
} else {
|
||||
add("page.javascript", "Reads without JavaScript", SeverityOK, "%d characters of server-rendered text", p.TextLength)
|
||||
}
|
||||
|
||||
if p.H1 == "" {
|
||||
add("page.h1", "Primary heading", SeverityInfo, "no <h1>; the markdown starts without a top-level heading")
|
||||
}
|
||||
|
||||
if p.DeadLinks > 0 {
|
||||
add("page.dead-links", "Followable links", SeverityWarn,
|
||||
"%d link(s) point at \"#\" or javascript: — agents cannot follow them", p.DeadLinks)
|
||||
}
|
||||
|
||||
if len(snap.Markdown) > contextHeavyChars {
|
||||
add("markdown.size", "Markdown size", SeverityInfo,
|
||||
"%d characters — heavy for a single agent context window", len(snap.Markdown))
|
||||
}
|
||||
} else if isText(snap.ContentType) {
|
||||
add("page.format", "Native format", SeverityOK, "served as markdown/plain text — agents read it directly")
|
||||
}
|
||||
|
||||
// — the llms.txt family —
|
||||
f := snap.Files
|
||||
switch {
|
||||
case f.LlmsTxt.found():
|
||||
add("llms.txt", "/llms.txt", SeverityOK, "found at %s", f.LlmsTxt.URL)
|
||||
if doc := f.LlmsTxt.LlmsDoc; doc != nil {
|
||||
if doc.Title == "" {
|
||||
add("llms.txt.format", "llms.txt format", SeverityError,
|
||||
"the spec requires the file to start with an `# H1 title`")
|
||||
} else if len(doc.Sections) == 0 {
|
||||
add("llms.txt.format", "llms.txt format", SeverityInfo,
|
||||
"title present but no `## sections` — fine, though sections organise links for agents")
|
||||
} else {
|
||||
add("llms.txt.format", "llms.txt format", SeverityOK,
|
||||
"%s — %d section(s)", clip(doc.Title, 60), len(doc.Sections))
|
||||
}
|
||||
if doc.LinkCount == 0 {
|
||||
add("llms.txt.links", "llms.txt links", SeverityWarn, "no links — the file gives agents nothing to open")
|
||||
} else {
|
||||
add("llms.txt.links", "llms.txt links", SeverityOK, "%d curated link(s)", doc.LinkCount)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(f.LlmsTxt.ContentType), "markdown") {
|
||||
add("llms.txt.content-type", "llms.txt content type", SeverityInfo,
|
||||
"served as %s; the spec recommends text/markdown", contentTypeOr(f.LlmsTxt.ContentType))
|
||||
}
|
||||
case f.LlmsTxt == nil:
|
||||
add("llms.txt", "/llms.txt", SeverityWarn, "not reachable — agents are left to parse the HTML themselves; add one per llmstxt.org")
|
||||
default:
|
||||
add("llms.txt", "/llms.txt", SeverityWarn,
|
||||
"missing (HTTP %d) — agents are left to parse the HTML themselves; add one per llmstxt.org", f.LlmsTxt.Status)
|
||||
}
|
||||
|
||||
switch {
|
||||
case f.LlmsFullTxt.found():
|
||||
add("llms-full.txt", "/llms-full.txt", SeverityOK, "found — full-context variant for agents that want everything")
|
||||
case f.LlmsFullTxt != nil && f.LlmsFullTxt.Status > 0:
|
||||
add("llms-full.txt", "/llms-full.txt", SeverityInfo, "not published (HTTP %d) — optional", f.LlmsFullTxt.Status)
|
||||
default:
|
||||
add("llms-full.txt", "/llms-full.txt", SeverityInfo, "not published — optional")
|
||||
}
|
||||
|
||||
switch {
|
||||
case f.WellKnownTxt.found():
|
||||
add("llms.well-known", "/.well-known/llms.txt", SeverityOK, "found at %s", f.WellKnownTxt.URL)
|
||||
case f.WellKnownTxt != nil && f.WellKnownTxt.Status > 0:
|
||||
add("llms.well-known", "/.well-known/llms.txt", SeverityInfo, "not published (HTTP %d) — alternative location", f.WellKnownTxt.Status)
|
||||
default:
|
||||
add("llms.well-known", "/.well-known/llms.txt", SeverityInfo, "not published — alternative location")
|
||||
}
|
||||
|
||||
// — robots.txt —
|
||||
switch {
|
||||
case f.RobotsTxt.found():
|
||||
disallowAll, sitemap := robotsSummary(f.RobotsTxt.Content)
|
||||
if disallowAll {
|
||||
add("robots.txt", "robots.txt", SeverityWarn,
|
||||
"present but `User-agent: *` is disallowed for / — automated agents are told to keep out")
|
||||
} else {
|
||||
add("robots.txt", "robots.txt", SeverityOK, "no blanket disallow for `User-agent: *`")
|
||||
}
|
||||
if sitemap != "" {
|
||||
add("robots.sitemap", "Sitemap", SeverityInfo, "%s", sitemap)
|
||||
}
|
||||
default:
|
||||
add("robots.txt", "robots.txt", SeverityInfo, "not published — agents assume everything is allowed")
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// robotsSummary reports whether the wildcard agent is disallowed for / and
|
||||
// returns the first declared sitemap.
|
||||
func robotsSummary(content string) (disallowAll bool, sitemap string) {
|
||||
groupApplies := false
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
key, value, _ := strings.Cut(line, ":")
|
||||
key = strings.ToLower(strings.TrimSpace(key))
|
||||
value = strings.TrimSpace(value)
|
||||
switch key {
|
||||
case "user-agent":
|
||||
groupApplies = value == "*"
|
||||
case "disallow":
|
||||
if groupApplies && value == "/" {
|
||||
disallowAll = true
|
||||
}
|
||||
case "sitemap":
|
||||
if sitemap == "" {
|
||||
sitemap = value
|
||||
}
|
||||
}
|
||||
}
|
||||
return disallowAll, sitemap
|
||||
}
|
||||
|
||||
// isHTML decides whether the body should go through the HTML converter.
|
||||
func isHTML(contentType, body string) bool {
|
||||
ct := strings.ToLower(contentType)
|
||||
if strings.Contains(ct, "html") {
|
||||
return true
|
||||
}
|
||||
if ct == "" || strings.HasPrefix(ct, "text/") {
|
||||
l := strings.ToLower(body[:min(len(body), 2048)])
|
||||
return strings.Contains(l, "<!doctype html") || strings.Contains(l, "<html")
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isText reports whether the content type is already markdown/plain text.
|
||||
func isText(contentType string) bool {
|
||||
ct := strings.ToLower(contentType)
|
||||
return strings.Contains(ct, "text/markdown") || strings.Contains(ct, "text/plain") ||
|
||||
strings.Contains(ct, "text/x-markdown")
|
||||
}
|
||||
|
||||
// siteRoot reduces a URL to scheme://host for companion lookups.
|
||||
func siteRoot(raw string) string {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.Host == "" {
|
||||
return raw
|
||||
}
|
||||
return u.Scheme + "://" + u.Host
|
||||
}
|
||||
|
||||
func contentTypeOr(ct string) string {
|
||||
if strings.TrimSpace(ct) == "" {
|
||||
return "an unknown type"
|
||||
}
|
||||
return ct
|
||||
}
|
||||
|
||||
func clip(s string, max int) string {
|
||||
s = strings.TrimSpace(s)
|
||||
runes := []rune(s)
|
||||
if len(runes) <= max {
|
||||
return s
|
||||
}
|
||||
return string(runes[:max-1]) + "…"
|
||||
}
|
||||
|
||||
func firstNonEmpty(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,346 @@
|
||||
package browse_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.k6n.net/mats/fullstack-template/internal/browse"
|
||||
)
|
||||
|
||||
// fakeFetcher is an in-memory browse.Fetcher: responses keyed by URL, an
|
||||
// optional per-URL error, and a 404 default for anything unlisted.
|
||||
type fakeFetcher struct {
|
||||
mu sync.Mutex
|
||||
res map[string]*browse.Response
|
||||
err map[string]error
|
||||
}
|
||||
|
||||
func newFakeFetcher(pages map[string]string) *fakeFetcher {
|
||||
f := &fakeFetcher{res: map[string]*browse.Response{}, err: map[string]error{}}
|
||||
for url, body := range pages {
|
||||
f.res[url] = &browse.Response{
|
||||
Status: 200,
|
||||
FinalURL: url,
|
||||
ContentType: fakeContentType(url, body),
|
||||
Body: []byte(body),
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *fakeFetcher) failOn(url string, err error) { f.err[url] = err }
|
||||
|
||||
func (f *fakeFetcher) Fetch(_ context.Context, url string) (*browse.Response, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err, ok := f.err[url]; ok {
|
||||
return nil, err
|
||||
}
|
||||
if r, ok := f.res[url]; ok {
|
||||
return r, nil
|
||||
}
|
||||
return &browse.Response{Status: 404, FinalURL: url, ContentType: "text/plain", Body: []byte("not found")}, nil
|
||||
}
|
||||
|
||||
// fakeContentType guesses like a real server would: HTML pages as
|
||||
// text/html, llms.txt as text/markdown, robots.txt as text/plain.
|
||||
func fakeContentType(url, body string) string {
|
||||
b := strings.TrimSpace(body)
|
||||
switch {
|
||||
case strings.HasPrefix(b, "<!doctype") || strings.HasPrefix(b, "<html") || strings.Contains(b[:min(len(b), 200)], "<div"):
|
||||
return "text/html; charset=utf-8"
|
||||
case strings.HasSuffix(url, "llms.txt"):
|
||||
return "text/markdown; charset=utf-8"
|
||||
case strings.HasSuffix(url, "robots.txt"):
|
||||
return "text/plain"
|
||||
default:
|
||||
return "text/plain"
|
||||
}
|
||||
}
|
||||
|
||||
// site is the smallest site that exercises every check path.
|
||||
const site = `<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<title>Test Site</title>
|
||||
<meta name="description" content="A site for tests.">
|
||||
<link rel="canonical" href="https://example.test/home">
|
||||
</head>
|
||||
<body>
|
||||
<h1>Welcome</h1>
|
||||
<p>Read the <a href="/docs">docs</a> or visit <a href="https://other.test/x">the hub</a>.</p>
|
||||
<a href="#" class="btn">dead</a>
|
||||
<script src="/app.js"></script>
|
||||
</body>
|
||||
</html>`
|
||||
|
||||
const llmsTxt = `# Test Site
|
||||
|
||||
> A site for tests.
|
||||
|
||||
## Docs
|
||||
|
||||
- [Guide](https://example.test/guide): how to use it
|
||||
- [API](https://example.test/api "the api"): endpoints
|
||||
|
||||
## Community
|
||||
|
||||
- [Forum](https://forum.example.test): ask questions
|
||||
`
|
||||
|
||||
func checkByID(t *testing.T, snap *browse.Snapshot, id string) browse.Check {
|
||||
t.Helper()
|
||||
for _, c := range snap.Checks {
|
||||
if c.ID == id {
|
||||
return c
|
||||
}
|
||||
}
|
||||
t.Fatalf("check %q not found in %v", id, snap.Checks)
|
||||
return browse.Check{}
|
||||
}
|
||||
|
||||
func TestNormalizeURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
in, want string
|
||||
wantErr bool
|
||||
}{
|
||||
{in: "example.com", want: "https://example.com"},
|
||||
{in: " example.com/path ", want: "https://example.com/path"},
|
||||
{in: "http://example.com", want: "http://example.com"},
|
||||
{in: "https://example.com/a?b=1#frag", want: "https://example.com/a?b=1"},
|
||||
{in: "user:pass@example.com", want: "https://user:pass@example.com"},
|
||||
{in: "ftp://example.com", wantErr: true},
|
||||
{in: "https://", wantErr: true},
|
||||
{in: "", wantErr: true},
|
||||
{in: " ", wantErr: true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got, err := browse.NormalizeURL(tc.in)
|
||||
if tc.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("NormalizeURL(%q) = %q, want error", tc.in, got)
|
||||
}
|
||||
var vErr *browse.ValidationError
|
||||
if err != nil && !errors.As(err, &vErr) {
|
||||
t.Errorf("NormalizeURL(%q) error type %T, want *ValidationError", tc.in, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("NormalizeURL(%q) unexpected error: %v", tc.in, err)
|
||||
continue
|
||||
}
|
||||
// credentials survive url.Parse round-trip; the service drops them
|
||||
if u := got; strings.Contains(u, "user:pass") {
|
||||
t.Errorf("NormalizeURL(%q) = %q, want credentials dropped", tc.in, u)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectConvertsHTML(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newFakeFetcher(map[string]string{"https://example.test": site})
|
||||
svc := browse.NewService(f)
|
||||
|
||||
snap, err := svc.Inspect(context.Background(), "example.test")
|
||||
if err != nil {
|
||||
t.Fatalf("Inspect: %v", err)
|
||||
}
|
||||
if snap.RequestedURL != "https://example.test" {
|
||||
t.Errorf("requested URL = %q, want the normalized https origin", snap.RequestedURL)
|
||||
}
|
||||
if snap.Page == nil {
|
||||
t.Fatal("page metadata missing")
|
||||
}
|
||||
if snap.Page.Title != "Test Site" {
|
||||
t.Errorf("title = %q", snap.Page.Title)
|
||||
}
|
||||
if snap.Page.H1 != "Welcome" {
|
||||
t.Errorf("h1 = %q", snap.Page.H1)
|
||||
}
|
||||
if snap.Page.LinkCount != 2 || snap.Page.DeadLinks != 1 {
|
||||
t.Errorf("links = %d, dead = %d, want 2 and 1", snap.Page.LinkCount, snap.Page.DeadLinks)
|
||||
}
|
||||
for _, want := range []string{"# Welcome", "[docs](https://example.test/docs)", "[the hub](https://other.test/x)"} {
|
||||
if !strings.Contains(snap.Markdown, want) {
|
||||
t.Errorf("markdown missing %q:\n%s", want, snap.Markdown)
|
||||
}
|
||||
}
|
||||
if c := checkByID(t, snap, "page.status"); c.Severity != browse.SeverityOK {
|
||||
t.Errorf("page.status = %s (%s)", c.Severity, c.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectFetchError(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newFakeFetcher(nil)
|
||||
f.failOn("https://down.test", fmt.Errorf("dial tcp: refused"))
|
||||
svc := browse.NewService(f)
|
||||
|
||||
_, err := svc.Inspect(context.Background(), "down.test")
|
||||
var fe *browse.FetchError
|
||||
if !errors.As(err, &fe) {
|
||||
t.Fatalf("err = %T (%v), want *browse.FetchError", err, err)
|
||||
}
|
||||
if fe.URL != "https://down.test" {
|
||||
t.Errorf("fetch error URL = %q", fe.URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectCompanions(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newFakeFetcher(map[string]string{
|
||||
"https://example.test": site,
|
||||
"https://example.test/llms.txt": llmsTxt,
|
||||
"https://example.test/robots.txt": "User-agent: *\nAllow: /\n\nSitemap: https://example.test/sitemap.xml",
|
||||
})
|
||||
svc := browse.NewService(f)
|
||||
|
||||
snap, err := svc.Inspect(context.Background(), "https://example.test/")
|
||||
if err != nil {
|
||||
t.Fatalf("Inspect: %v", err)
|
||||
}
|
||||
|
||||
llms := snap.Files.LlmsTxt
|
||||
if llms == nil || llms.Status >= 400 {
|
||||
t.Fatalf("llms.txt missing: %+v", snap.Files.LlmsTxt)
|
||||
}
|
||||
if llms.LlmsDoc == nil {
|
||||
t.Fatal("llms.txt was not parsed")
|
||||
}
|
||||
doc := llms.LlmsDoc
|
||||
if doc.Title != "Test Site" {
|
||||
t.Errorf("llms title = %q", doc.Title)
|
||||
}
|
||||
if doc.Description != "A site for tests." {
|
||||
t.Errorf("llms description = %q", doc.Description)
|
||||
}
|
||||
if len(doc.Sections) != 2 {
|
||||
t.Fatalf("llms sections = %d, want 2", len(doc.Sections))
|
||||
}
|
||||
if len(doc.Sections[0].Links) != 2 || doc.Sections[0].Links[1].Description != "endpoints" {
|
||||
t.Errorf("section links wrong: %+v", doc.Sections[0])
|
||||
}
|
||||
if doc.LinkCount != 3 {
|
||||
t.Errorf("llms link count = %d, want 3", doc.LinkCount)
|
||||
}
|
||||
|
||||
if c := checkByID(t, snap, "llms.txt"); c.Severity != browse.SeverityOK {
|
||||
t.Errorf("llms.txt check = %s (%s)", c.Severity, c.Detail)
|
||||
}
|
||||
if c := checkByID(t, snap, "llms.txt.format"); c.Severity != browse.SeverityOK {
|
||||
t.Errorf("llms.txt.format = %s (%s)", c.Severity, c.Detail)
|
||||
}
|
||||
if c := checkByID(t, snap, "llms-full.txt"); c.Severity != browse.SeverityInfo {
|
||||
t.Errorf("llms-full.txt = %s (%s)", c.Severity, c.Detail)
|
||||
}
|
||||
if c := checkByID(t, snap, "robots.txt"); c.Severity != browse.SeverityOK {
|
||||
t.Errorf("robots.txt = %s (%s)", c.Severity, c.Detail)
|
||||
}
|
||||
if c := checkByID(t, snap, "robots.sitemap"); c.Detail == "" {
|
||||
t.Error("sitemap check detail empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectMissingLLMsTxtWarns(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newFakeFetcher(map[string]string{"https://bare.test": site})
|
||||
snap, err := browse.NewService(f).Inspect(context.Background(), "https://bare.test")
|
||||
if err != nil {
|
||||
t.Fatalf("Inspect: %v", err)
|
||||
}
|
||||
if c := checkByID(t, snap, "llms.txt"); c.Severity != browse.SeverityWarn {
|
||||
t.Errorf("llms.txt = %s (%s), want warn when the file is absent", c.Severity, c.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectLLMsTxtWithoutTitleIsAnError(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newFakeFetcher(map[string]string{
|
||||
"https://example.test/": site,
|
||||
"https://example.test/llms.txt": "> just a quote, no H1\n",
|
||||
})
|
||||
snap, err := browse.NewService(f).Inspect(context.Background(), "https://example.test")
|
||||
if err != nil {
|
||||
t.Fatalf("Inspect: %v", err)
|
||||
}
|
||||
if c := checkByID(t, snap, "llms.txt.format"); c.Severity != browse.SeverityError {
|
||||
t.Errorf("llms.txt.format = %s (%s), want error for a missing H1", c.Severity, c.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectRobotsDisallowAllWarns(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newFakeFetcher(map[string]string{
|
||||
"https://closed.test": site,
|
||||
"https://closed.test/robots.txt": "User-agent: *\nDisallow: /\n",
|
||||
})
|
||||
snap, err := browse.NewService(f).Inspect(context.Background(), "closed.test")
|
||||
if err != nil {
|
||||
t.Fatalf("Inspect: %v", err)
|
||||
}
|
||||
if c := checkByID(t, snap, "robots.txt"); c.Severity != browse.SeverityWarn {
|
||||
t.Errorf("robots.txt = %s (%s), want warn for a blanket disallow", c.Severity, c.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectPageProblems(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
noindex := strings.Replace(site, "<title>Test Site</title>",
|
||||
`<title>Test Site</title><meta name="robots" content="noindex, nofollow">`, 1)
|
||||
|
||||
jsShell := `<!doctype html><html><head><title>Shell</title></head><body>
|
||||
<div id="root"></div>
|
||||
<script src="/a.js"></script><script src="/b.js"></script><script src="/c.js"></script>
|
||||
</body></html>`
|
||||
|
||||
f := newFakeFetcher(map[string]string{
|
||||
"https://noindex.test": noindex,
|
||||
"https://shell.test": jsShell,
|
||||
})
|
||||
svc := browse.NewService(f)
|
||||
|
||||
snap, err := svc.Inspect(context.Background(), "noindex.test")
|
||||
if err != nil {
|
||||
t.Fatalf("Inspect: %v", err)
|
||||
}
|
||||
if c := checkByID(t, snap, "page.robots-meta"); c.Severity != browse.SeverityError {
|
||||
t.Errorf("noindex = %s (%s), want error", c.Severity, c.Detail)
|
||||
}
|
||||
|
||||
snap, err = svc.Inspect(context.Background(), "shell.test")
|
||||
if err != nil {
|
||||
t.Fatalf("Inspect: %v", err)
|
||||
}
|
||||
if c := checkByID(t, snap, "page.javascript"); c.Severity != browse.SeverityWarn {
|
||||
t.Errorf("js shell = %s (%s), want warn", c.Severity, c.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectRedirectAndMarkdownPage(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newFakeFetcher(map[string]string{
|
||||
"https://docs.test/guide": "# Guide\n\nAll there is.\n",
|
||||
})
|
||||
f.res["https://docs.test/guide"].ContentType = "text/markdown"
|
||||
|
||||
snap, err := browse.NewService(f).Inspect(context.Background(), "https://docs.test/guide")
|
||||
if err != nil {
|
||||
t.Fatalf("Inspect: %v", err)
|
||||
}
|
||||
if snap.Page != nil {
|
||||
t.Errorf("markdown pages carry no HTML metadata, got %+v", snap.Page)
|
||||
}
|
||||
if !strings.HasPrefix(snap.Markdown, "# Guide") {
|
||||
t.Errorf("markdown = %q", snap.Markdown)
|
||||
}
|
||||
if c := checkByID(t, snap, "page.format"); c.Severity != browse.SeverityOK {
|
||||
t.Errorf("page.format = %s (%s)", c.Severity, c.Detail)
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,8 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.k6n.net/mats/fullstack-template/internal/browse"
|
||||
)
|
||||
|
||||
// Config is the validated runtime configuration.
|
||||
@@ -34,6 +36,10 @@ type Config struct {
|
||||
LogFormat string // "json" or "text"
|
||||
// DebugQueries enables Bun query logging (bundebug).
|
||||
DebugQueries bool
|
||||
// FetchUserAgent identifies this server to the sites it inspects.
|
||||
FetchUserAgent string
|
||||
// FetchTimeout bounds each outbound fetch, redirects included.
|
||||
FetchTimeout time.Duration
|
||||
|
||||
// OIDCIssuer is the OpenID Connect issuer; endpoints are discovered,
|
||||
// never hardcoded. Auth is enabled when OIDCClientID is non-empty.
|
||||
@@ -60,6 +66,7 @@ func Load() (Config, error) {
|
||||
AllowedOrigins: splitCSV(env("CORS_ALLOWED_ORIGINS", "http://localhost:5173,http://127.0.0.1:5173")),
|
||||
LogFormat: strings.ToLower(env("LOG_FORMAT", "json")),
|
||||
DebugQueries: env("DB_DEBUG", "false") == "true",
|
||||
FetchUserAgent: env("FETCH_USER_AGENT", browse.DefaultUserAgent),
|
||||
|
||||
OIDCIssuer: strings.TrimRight(env("OIDC_ISSUER", "https://users.k6n.net"), "/"),
|
||||
OIDCClientID: env("OIDC_CLIENT_ID", ""),
|
||||
@@ -72,6 +79,8 @@ func Load() (Config, error) {
|
||||
cfg.AuthSessionTTL = sessionTTL
|
||||
}
|
||||
|
||||
fetchTimeout, fetchTimeoutErr := time.ParseDuration(env("FETCH_TIMEOUT", "15s"))
|
||||
|
||||
var errs []error
|
||||
|
||||
level := strings.ToLower(env("LOG_LEVEL", "info"))
|
||||
@@ -103,6 +112,13 @@ func Load() (Config, error) {
|
||||
} else if cfg.AuthSessionTTL <= 0 {
|
||||
errs = append(errs, errors.New("AUTH_SESSION_TTL must be positive"))
|
||||
}
|
||||
if fetchTimeoutErr != nil {
|
||||
errs = append(errs, fmt.Errorf("invalid FETCH_TIMEOUT: %v", fetchTimeoutErr))
|
||||
} else if fetchTimeout <= 0 {
|
||||
errs = append(errs, errors.New("FETCH_TIMEOUT must be positive"))
|
||||
} else {
|
||||
cfg.FetchTimeout = fetchTimeout
|
||||
}
|
||||
if _, err := url.Parse(cfg.OIDCIssuer); err != nil || !strings.HasPrefix(cfg.OIDCIssuer, "https://") && !strings.HasPrefix(cfg.OIDCIssuer, "http://") {
|
||||
errs = append(errs, fmt.Errorf("invalid OIDC_ISSUER %q (want an http(s) URL)", cfg.OIDCIssuer))
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user