llm browser
This commit is contained in:
developer committed
1 parent
afed023152
commit
a4dbb75195
39 files changed
+3709
-58
No files matched your search
@@ -0,0 +1,398 @@
|
||||
package browse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Tunables for the checks.
|
||||
const (
|
||||
// jsShellTextChars: with fewer server-rendered characters than this
|
||||
// next to real script traffic, the page probably needs JavaScript.
|
||||
jsShellTextChars = 400
|
||||
// contextHeavyChars: markdown beyond this is a lot for one context.
|
||||
contextHeavyChars = 150_000
|
||||
)
|
||||
|
||||
// Companion paths, always resolved against the site root of the page.
|
||||
const (
|
||||
llmsTxtPath = "/llms.txt"
|
||||
llmsFullTxtPath = "/llms-full.txt"
|
||||
wellKnownTxtPath = "/.well-known/llms.txt"
|
||||
robotsTxtPath = "/robots.txt"
|
||||
)
|
||||
|
||||
// Service renders remote pages the way a text-only agent sees them. It is
|
||||
// stateless; the Fetcher is its only collaborator.
|
||||
type Service struct {
|
||||
fetch Fetcher
|
||||
}
|
||||
|
||||
// NewService returns a Service over the given fetcher.
|
||||
func NewService(fetch Fetcher) *Service {
|
||||
return &Service{fetch: fetch}
|
||||
}
|
||||
|
||||
// Inspect fetches target (a bare domain or a full URL), converts the page
|
||||
// to markdown, gathers the llms.txt companions and robots.txt in parallel,
|
||||
// and assembles the agent-readiness checks.
|
||||
func (s *Service) Inspect(ctx context.Context, raw string) (*Snapshot, error) {
|
||||
target, err := NormalizeURL(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
started := time.Now()
|
||||
|
||||
page, err := s.fetch.Fetch(ctx, target)
|
||||
if err != nil {
|
||||
var fe *FetchError
|
||||
if !errors.As(err, &fe) {
|
||||
fe = &FetchError{URL: target, Err: err}
|
||||
}
|
||||
return nil, fe
|
||||
}
|
||||
|
||||
snap := &Snapshot{
|
||||
RequestedURL: target,
|
||||
FinalURL: firstNonEmpty(page.FinalURL, target),
|
||||
Status: page.Status,
|
||||
ContentType: page.ContentType,
|
||||
FetchedAt: time.Now().UTC(),
|
||||
Checks: []Check{},
|
||||
}
|
||||
|
||||
var meta pageMeta
|
||||
switch {
|
||||
case isHTML(page.ContentType, string(page.Body)):
|
||||
md, m, convErr := htmlToMarkdown(page.Body, snap.FinalURL)
|
||||
if convErr != nil {
|
||||
return nil, fmt.Errorf("parse %s: %w", snap.FinalURL, convErr)
|
||||
}
|
||||
meta = m
|
||||
snap.Markdown, snap.MarkdownTruncated = truncateMarkdown(md, maxMarkdownChars)
|
||||
snap.Page = &PageView{
|
||||
Title: meta.Title,
|
||||
Description: meta.Description,
|
||||
Canonical: meta.Canonical,
|
||||
RobotsMeta: meta.Robots,
|
||||
Lang: meta.Lang,
|
||||
H1: meta.H1,
|
||||
LinkCount: meta.LinkCount,
|
||||
DeadLinks: meta.DeadLinks,
|
||||
ScriptCount: meta.ScriptCount,
|
||||
ImageCount: meta.ImageCount,
|
||||
WordCount: meta.WordCount,
|
||||
TextLength: meta.TextLength,
|
||||
}
|
||||
case isText(page.ContentType):
|
||||
snap.Markdown, snap.MarkdownTruncated = truncateMarkdown(string(page.Body), maxMarkdownChars)
|
||||
default:
|
||||
snap.Markdown, snap.MarkdownTruncated =
|
||||
truncateMarkdown("```"+page.ContentType+"\n"+string(page.Body)+"\n```", maxMarkdownChars)
|
||||
}
|
||||
|
||||
// Companions come from the site root the page ended up on. They are
|
||||
// fetched concurrently so one slow file cannot stretch the inspection.
|
||||
root := siteRoot(snap.FinalURL)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(4)
|
||||
go s.fetchFile(ctx, &wg, root+llmsTxtPath, true, &snap.Files.LlmsTxt)
|
||||
go s.fetchFile(ctx, &wg, root+llmsFullTxtPath, true, &snap.Files.LlmsFullTxt)
|
||||
go s.fetchFile(ctx, &wg, root+wellKnownTxtPath, true, &snap.Files.WellKnownTxt)
|
||||
go s.fetchFile(ctx, &wg, root+robotsTxtPath, false, &snap.Files.RobotsTxt)
|
||||
wg.Wait()
|
||||
|
||||
snap.Checks = buildChecks(snap)
|
||||
snap.DurationMS = time.Since(started).Milliseconds()
|
||||
return snap, nil
|
||||
}
|
||||
|
||||
// NormalizeURL turns user input into a fetchable URL: a bare domain gets
|
||||
// https://, credentials are dropped, fragments are dropped.
|
||||
func NormalizeURL(raw string) (string, error) {
|
||||
s := strings.TrimSpace(raw)
|
||||
if s == "" {
|
||||
return "", &ValidationError{Field: "url", Reason: "required"}
|
||||
}
|
||||
if !strings.Contains(s, "://") {
|
||||
s = "https://" + s
|
||||
}
|
||||
u, err := url.Parse(s)
|
||||
if err != nil || u.Host == "" {
|
||||
return "", &ValidationError{Field: "url", Reason: "must be a domain or URL agents can open"}
|
||||
}
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return "", &ValidationError{Field: "url", Reason: "only http and https URLs are supported"}
|
||||
}
|
||||
u.User = nil
|
||||
u.Fragment = ""
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
// fetchFile fetches one companion into dst. A failed request is a result,
|
||||
// not an inspection failure — the checks describe what is missing.
|
||||
func (s *Service) fetchFile(ctx context.Context, wg *sync.WaitGroup, url string, parseLlms bool, dst **FileResult) {
|
||||
defer wg.Done()
|
||||
|
||||
res, err := s.fetch.Fetch(ctx, url)
|
||||
if err != nil {
|
||||
var fe *FetchError
|
||||
if errors.As(err, &fe) {
|
||||
err = fe.Err
|
||||
}
|
||||
*dst = &FileResult{URL: url, Error: err.Error()}
|
||||
return
|
||||
}
|
||||
|
||||
fr := &FileResult{
|
||||
URL: firstNonEmpty(res.FinalURL, url),
|
||||
Status: res.Status,
|
||||
ContentType: res.ContentType,
|
||||
Truncated: res.Truncated,
|
||||
}
|
||||
if fr.found() {
|
||||
fr.Content = string(res.Body)
|
||||
if parseLlms && !isHTML(fr.ContentType, fr.Content) {
|
||||
fr.LlmsDoc = parseLlmsTxt(fr.Content)
|
||||
}
|
||||
}
|
||||
*dst = fr
|
||||
}
|
||||
|
||||
// buildChecks turns a finished snapshot into the validation report.
|
||||
func buildChecks(snap *Snapshot) []Check {
|
||||
c := []Check{}
|
||||
add := func(id, label string, sev Severity, format string, args ...any) {
|
||||
c = append(c, Check{ID: id, Label: label, Severity: sev, Detail: fmt.Sprintf(format, args...)})
|
||||
}
|
||||
|
||||
// — the page itself —
|
||||
switch {
|
||||
case snap.Status >= 500:
|
||||
add("page.status", "Page reachable", SeverityError, "server answered %d", snap.Status)
|
||||
case snap.Status >= 400:
|
||||
add("page.status", "Page reachable", SeverityError, "server answered %d — agents see an error page", snap.Status)
|
||||
case snap.Status >= 300:
|
||||
add("page.status", "Page reachable", SeverityWarn, "server answered %d", snap.Status)
|
||||
default:
|
||||
add("page.status", "Page reachable", SeverityOK, "%d OK", snap.Status)
|
||||
}
|
||||
|
||||
if u, err := url.Parse(snap.FinalURL); err == nil && u.Scheme == "http" {
|
||||
add("page.https", "Served over HTTPS", SeverityWarn, "the final URL is plain http; secure agents may refuse it")
|
||||
} else {
|
||||
add("page.https", "Served over HTTPS", SeverityOK, "https")
|
||||
}
|
||||
|
||||
if snap.FinalURL != snap.RequestedURL {
|
||||
add("page.redirect", "Direct URL", SeverityInfo, "redirected to %s", snap.FinalURL)
|
||||
}
|
||||
|
||||
if snap.Page != nil {
|
||||
p := snap.Page
|
||||
if p.Title == "" {
|
||||
add("page.title", "Title", SeverityError, "no <title> — agents use it as the page name in their results")
|
||||
} else {
|
||||
add("page.title", "Title", SeverityOK, "%s", clip(p.Title, 80))
|
||||
}
|
||||
|
||||
if p.Description == "" {
|
||||
add("page.description", "Meta description", SeverityWarn, "missing — agents have to guess the page's purpose from body text")
|
||||
} else {
|
||||
add("page.description", "Meta description", SeverityOK, "%s", clip(p.Description, 140))
|
||||
}
|
||||
|
||||
robots := strings.ToLower(p.RobotsMeta)
|
||||
switch {
|
||||
case strings.Contains(robots, "noindex"):
|
||||
add("page.robots-meta", "Robots meta", SeverityError, "meta robots declares noindex — indexing agents are told to ignore this page")
|
||||
case strings.Contains(robots, "nofollow"):
|
||||
add("page.robots-meta", "Robots meta", SeverityWarn, "meta robots declares nofollow — agents are told not to follow this page's links")
|
||||
case p.RobotsMeta != "":
|
||||
add("page.robots-meta", "Robots meta", SeverityOK, "%s", p.RobotsMeta)
|
||||
default:
|
||||
add("page.robots-meta", "Robots meta", SeverityOK, "no restrictions declared")
|
||||
}
|
||||
|
||||
if p.TextLength < jsShellTextChars && p.ScriptCount >= 3 {
|
||||
add("page.javascript", "Reads without JavaScript", SeverityWarn,
|
||||
"only %d characters of server-rendered text next to %d script tags — the page likely renders in the browser, which text agents never run",
|
||||
p.TextLength, p.ScriptCount)
|
||||
} else {
|
||||
add("page.javascript", "Reads without JavaScript", SeverityOK, "%d characters of server-rendered text", p.TextLength)
|
||||
}
|
||||
|
||||
if p.H1 == "" {
|
||||
add("page.h1", "Primary heading", SeverityInfo, "no <h1>; the markdown starts without a top-level heading")
|
||||
}
|
||||
|
||||
if p.DeadLinks > 0 {
|
||||
add("page.dead-links", "Followable links", SeverityWarn,
|
||||
"%d link(s) point at \"#\" or javascript: — agents cannot follow them", p.DeadLinks)
|
||||
}
|
||||
|
||||
if len(snap.Markdown) > contextHeavyChars {
|
||||
add("markdown.size", "Markdown size", SeverityInfo,
|
||||
"%d characters — heavy for a single agent context window", len(snap.Markdown))
|
||||
}
|
||||
} else if isText(snap.ContentType) {
|
||||
add("page.format", "Native format", SeverityOK, "served as markdown/plain text — agents read it directly")
|
||||
}
|
||||
|
||||
// — the llms.txt family —
|
||||
f := snap.Files
|
||||
switch {
|
||||
case f.LlmsTxt.found():
|
||||
add("llms.txt", "/llms.txt", SeverityOK, "found at %s", f.LlmsTxt.URL)
|
||||
if doc := f.LlmsTxt.LlmsDoc; doc != nil {
|
||||
if doc.Title == "" {
|
||||
add("llms.txt.format", "llms.txt format", SeverityError,
|
||||
"the spec requires the file to start with an `# H1 title`")
|
||||
} else if len(doc.Sections) == 0 {
|
||||
add("llms.txt.format", "llms.txt format", SeverityInfo,
|
||||
"title present but no `## sections` — fine, though sections organise links for agents")
|
||||
} else {
|
||||
add("llms.txt.format", "llms.txt format", SeverityOK,
|
||||
"%s — %d section(s)", clip(doc.Title, 60), len(doc.Sections))
|
||||
}
|
||||
if doc.LinkCount == 0 {
|
||||
add("llms.txt.links", "llms.txt links", SeverityWarn, "no links — the file gives agents nothing to open")
|
||||
} else {
|
||||
add("llms.txt.links", "llms.txt links", SeverityOK, "%d curated link(s)", doc.LinkCount)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(f.LlmsTxt.ContentType), "markdown") {
|
||||
add("llms.txt.content-type", "llms.txt content type", SeverityInfo,
|
||||
"served as %s; the spec recommends text/markdown", contentTypeOr(f.LlmsTxt.ContentType))
|
||||
}
|
||||
case f.LlmsTxt == nil:
|
||||
add("llms.txt", "/llms.txt", SeverityWarn, "not reachable — agents are left to parse the HTML themselves; add one per llmstxt.org")
|
||||
default:
|
||||
add("llms.txt", "/llms.txt", SeverityWarn,
|
||||
"missing (HTTP %d) — agents are left to parse the HTML themselves; add one per llmstxt.org", f.LlmsTxt.Status)
|
||||
}
|
||||
|
||||
switch {
|
||||
case f.LlmsFullTxt.found():
|
||||
add("llms-full.txt", "/llms-full.txt", SeverityOK, "found — full-context variant for agents that want everything")
|
||||
case f.LlmsFullTxt != nil && f.LlmsFullTxt.Status > 0:
|
||||
add("llms-full.txt", "/llms-full.txt", SeverityInfo, "not published (HTTP %d) — optional", f.LlmsFullTxt.Status)
|
||||
default:
|
||||
add("llms-full.txt", "/llms-full.txt", SeverityInfo, "not published — optional")
|
||||
}
|
||||
|
||||
switch {
|
||||
case f.WellKnownTxt.found():
|
||||
add("llms.well-known", "/.well-known/llms.txt", SeverityOK, "found at %s", f.WellKnownTxt.URL)
|
||||
case f.WellKnownTxt != nil && f.WellKnownTxt.Status > 0:
|
||||
add("llms.well-known", "/.well-known/llms.txt", SeverityInfo, "not published (HTTP %d) — alternative location", f.WellKnownTxt.Status)
|
||||
default:
|
||||
add("llms.well-known", "/.well-known/llms.txt", SeverityInfo, "not published — alternative location")
|
||||
}
|
||||
|
||||
// — robots.txt —
|
||||
switch {
|
||||
case f.RobotsTxt.found():
|
||||
disallowAll, sitemap := robotsSummary(f.RobotsTxt.Content)
|
||||
if disallowAll {
|
||||
add("robots.txt", "robots.txt", SeverityWarn,
|
||||
"present but `User-agent: *` is disallowed for / — automated agents are told to keep out")
|
||||
} else {
|
||||
add("robots.txt", "robots.txt", SeverityOK, "no blanket disallow for `User-agent: *`")
|
||||
}
|
||||
if sitemap != "" {
|
||||
add("robots.sitemap", "Sitemap", SeverityInfo, "%s", sitemap)
|
||||
}
|
||||
default:
|
||||
add("robots.txt", "robots.txt", SeverityInfo, "not published — agents assume everything is allowed")
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// robotsSummary reports whether the wildcard agent is disallowed for / and
|
||||
// returns the first declared sitemap.
|
||||
func robotsSummary(content string) (disallowAll bool, sitemap string) {
|
||||
groupApplies := false
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
key, value, _ := strings.Cut(line, ":")
|
||||
key = strings.ToLower(strings.TrimSpace(key))
|
||||
value = strings.TrimSpace(value)
|
||||
switch key {
|
||||
case "user-agent":
|
||||
groupApplies = value == "*"
|
||||
case "disallow":
|
||||
if groupApplies && value == "/" {
|
||||
disallowAll = true
|
||||
}
|
||||
case "sitemap":
|
||||
if sitemap == "" {
|
||||
sitemap = value
|
||||
}
|
||||
}
|
||||
}
|
||||
return disallowAll, sitemap
|
||||
}
|
||||
|
||||
// isHTML decides whether the body should go through the HTML converter.
|
||||
func isHTML(contentType, body string) bool {
|
||||
ct := strings.ToLower(contentType)
|
||||
if strings.Contains(ct, "html") {
|
||||
return true
|
||||
}
|
||||
if ct == "" || strings.HasPrefix(ct, "text/") {
|
||||
l := strings.ToLower(body[:min(len(body), 2048)])
|
||||
return strings.Contains(l, "<!doctype html") || strings.Contains(l, "<html")
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isText reports whether the content type is already markdown/plain text.
|
||||
func isText(contentType string) bool {
|
||||
ct := strings.ToLower(contentType)
|
||||
return strings.Contains(ct, "text/markdown") || strings.Contains(ct, "text/plain") ||
|
||||
strings.Contains(ct, "text/x-markdown")
|
||||
}
|
||||
|
||||
// siteRoot reduces a URL to scheme://host for companion lookups.
|
||||
func siteRoot(raw string) string {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.Host == "" {
|
||||
return raw
|
||||
}
|
||||
return u.Scheme + "://" + u.Host
|
||||
}
|
||||
|
||||
func contentTypeOr(ct string) string {
|
||||
if strings.TrimSpace(ct) == "" {
|
||||
return "an unknown type"
|
||||
}
|
||||
return ct
|
||||
}
|
||||
|
||||
func clip(s string, max int) string {
|
||||
s = strings.TrimSpace(s)
|
||||
runes := []rune(s)
|
||||
if len(runes) <= max {
|
||||
return s
|
||||
}
|
||||
return string(runes[:max-1]) + "…"
|
||||
}
|
||||
|
||||
func firstNonEmpty(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Reference in new issue
Block a user