llm browser
This commit is contained in:
developer committed
1 parent
afed023152
commit
a4dbb75195
39 files changed
+3709
-58
No files matched your search
@@ -0,0 +1,565 @@
|
||||
package browse
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
"golang.org/x/net/html/atom"
|
||||
)
|
||||
|
||||
// pageMeta is what agents read from <head> plus document shape statistics
|
||||
// collected while walking the body.
|
||||
type pageMeta struct {
|
||||
Title string
|
||||
Description string
|
||||
Canonical string
|
||||
Robots string
|
||||
Lang string
|
||||
H1 string
|
||||
LinkCount int
|
||||
DeadLinks int
|
||||
ScriptCount int
|
||||
ImageCount int
|
||||
WordCount int
|
||||
TextLength int
|
||||
}
|
||||
|
||||
// maxMarkdownChars caps the converted markdown before it reaches the wire.
|
||||
const maxMarkdownChars = 200_000
|
||||
|
||||
var (
|
||||
hiddenStyleRe = regexp.MustCompile(`(?i)display\s*:\s*none|visibility\s*:\s*hidden`)
|
||||
wsRe = regexp.MustCompile(`\s+`)
|
||||
manyNlRe = regexp.MustCompile(`\n{3,}`)
|
||||
)
|
||||
|
||||
// htmlToMarkdown parses the document once and returns the body rendered as
|
||||
// markdown plus the head metadata. baseURL resolves relative links.
|
||||
func htmlToMarkdown(body []byte, baseURL string) (string, pageMeta, error) {
|
||||
var meta pageMeta
|
||||
doc, err := html.Parse(strings.NewReader(string(body)))
|
||||
if err != nil {
|
||||
return "", meta, err
|
||||
}
|
||||
|
||||
var htmlNode, bodyNode *html.Node
|
||||
var walk func(n *html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if c.Type != html.ElementNode {
|
||||
continue
|
||||
}
|
||||
switch c.DataAtom {
|
||||
case atom.Html:
|
||||
htmlNode = c
|
||||
walk(c)
|
||||
case atom.Head:
|
||||
collectHead(c, &meta)
|
||||
case atom.Body:
|
||||
bodyNode = c
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(doc)
|
||||
if htmlNode != nil {
|
||||
meta.Lang = attr(htmlNode, "lang")
|
||||
}
|
||||
if bodyNode != nil {
|
||||
inspectBody(bodyNode, &meta)
|
||||
}
|
||||
|
||||
r := &mdRenderer{base: parseBase(baseURL)}
|
||||
return r.renderBody(bodyNode), meta, nil
|
||||
}
|
||||
|
||||
// collectHead reads title, description, robots and canonical from <head>.
|
||||
func collectHead(head *html.Node, meta *pageMeta) {
|
||||
for c := head.FirstChild; c != nil; c = c.NextSibling {
|
||||
if c.Type != html.ElementNode {
|
||||
continue
|
||||
}
|
||||
switch c.DataAtom {
|
||||
case atom.Title:
|
||||
if meta.Title == "" {
|
||||
meta.Title = collapse(textContent(c))
|
||||
}
|
||||
case atom.Meta:
|
||||
content := collapse(attr(c, "content"))
|
||||
switch strings.ToLower(attr(c, "name")) {
|
||||
case "description":
|
||||
if meta.Description == "" {
|
||||
meta.Description = content
|
||||
}
|
||||
case "robots":
|
||||
if meta.Robots == "" {
|
||||
meta.Robots = content
|
||||
}
|
||||
}
|
||||
case atom.Link:
|
||||
if strings.EqualFold(attr(c, "rel"), "canonical") && meta.Canonical == "" {
|
||||
meta.Canonical = attr(c, "href")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// inspectBody counts the elements checks care about and measures the
|
||||
// visible text (script/style/svg content stays out of the count).
|
||||
func inspectBody(body *html.Node, meta *pageMeta) {
|
||||
var walk func(n *html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode {
|
||||
switch n.DataAtom {
|
||||
case atom.A:
|
||||
href := strings.TrimSpace(attr(n, "href"))
|
||||
if href == "" || strings.HasPrefix(href, "#") || strings.HasPrefix(strings.ToLower(href), "javascript:") {
|
||||
meta.DeadLinks++
|
||||
} else {
|
||||
meta.LinkCount++
|
||||
}
|
||||
case atom.Script:
|
||||
meta.ScriptCount++
|
||||
return
|
||||
case atom.Style, atom.Svg, atom.Noscript, atom.Template, atom.Iframe:
|
||||
return
|
||||
case atom.Img:
|
||||
meta.ImageCount++
|
||||
return
|
||||
case atom.H1:
|
||||
if meta.H1 == "" {
|
||||
meta.H1 = collapse(textContent(n))
|
||||
}
|
||||
}
|
||||
}
|
||||
if n.Type == html.TextNode {
|
||||
meta.TextLength += len(n.Data)
|
||||
meta.WordCount += len(strings.Fields(n.Data))
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(body)
|
||||
}
|
||||
|
||||
// mdRenderer renders a parsed document as markdown.
|
||||
type mdRenderer struct {
|
||||
base *url.URL
|
||||
}
|
||||
|
||||
func (r *mdRenderer) renderBody(body *html.Node) string {
|
||||
if body == nil {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
r.blocks(&b, body)
|
||||
out := strings.TrimSpace(b.String())
|
||||
return manyNlRe.ReplaceAllString(out, "\n\n")
|
||||
}
|
||||
|
||||
// skipped reports whether an element contributes nothing an agent could
|
||||
// act on: invisible content, script payloads, and embedded documents.
|
||||
func skipped(n *html.Node) bool {
|
||||
if n.Type != html.ElementNode {
|
||||
return false
|
||||
}
|
||||
switch n.DataAtom {
|
||||
case atom.Script, atom.Style, atom.Noscript, atom.Template, atom.Svg,
|
||||
atom.Head, atom.Link, atom.Meta, atom.Iframe, atom.Object, atom.Embed:
|
||||
return true
|
||||
}
|
||||
if hasAttr(n, "hidden") || strings.EqualFold(attr(n, "aria-hidden"), "true") {
|
||||
return true
|
||||
}
|
||||
return hiddenStyleRe.MatchString(attr(n, "style"))
|
||||
}
|
||||
|
||||
// blocks renders the children of n as a sequence of markdown blocks.
|
||||
func (r *mdRenderer) blocks(b *strings.Builder, n *html.Node) {
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if skipped(c) {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case c.Type == html.TextNode:
|
||||
if s := strings.TrimSpace(collapse(c.Data)); s != "" {
|
||||
b.WriteString(s + "\n\n")
|
||||
}
|
||||
case c.Type == html.ElementNode:
|
||||
r.element(b, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// element renders one block-level element.
|
||||
func (r *mdRenderer) element(b *strings.Builder, n *html.Node) {
|
||||
switch n.DataAtom {
|
||||
case atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6:
|
||||
level := int(n.DataAtom) - int(atom.H1) + 1
|
||||
b.WriteString(strings.Repeat("#", level) + " " + strings.TrimSpace(r.inline(n)) + "\n\n")
|
||||
case atom.P:
|
||||
if s := strings.TrimSpace(r.inline(n)); s != "" {
|
||||
b.WriteString(s + "\n\n")
|
||||
}
|
||||
case atom.Hr:
|
||||
b.WriteString("---\n\n")
|
||||
case atom.Blockquote:
|
||||
var inner strings.Builder
|
||||
r.blocks(&inner, n)
|
||||
b.WriteString(prefixLines(inner.String(), "> "))
|
||||
case atom.Pre:
|
||||
b.WriteString("```\n" + strings.Trim(textContent(n), "\n") + "\n```\n\n")
|
||||
case atom.Ul, atom.Ol:
|
||||
var lb strings.Builder
|
||||
r.list(&lb, n, "")
|
||||
b.WriteString(strings.TrimRight(lb.String(), "\n") + "\n\n")
|
||||
case atom.Table:
|
||||
r.table(b, n)
|
||||
case atom.Dl:
|
||||
r.defList(b, n)
|
||||
case atom.Button, atom.A, atom.Span, atom.Strong, atom.Em, atom.Code, atom.Small, atom.Label:
|
||||
// phrasing content at block position: keep it as its own paragraph
|
||||
if s := strings.TrimSpace(r.inline(n)); s != "" {
|
||||
b.WriteString(s + "\n\n")
|
||||
}
|
||||
default:
|
||||
r.blocks(b, n)
|
||||
}
|
||||
}
|
||||
|
||||
// list renders ul/ol items; nested lists indent under their item.
|
||||
func (r *mdRenderer) list(b *strings.Builder, n *html.Node, indent string) {
|
||||
ordered := n.DataAtom == atom.Ol
|
||||
index := 1
|
||||
if v := attr(n, "start"); v != "" {
|
||||
if i, err := strconv.Atoi(v); err == nil && i > 0 {
|
||||
index = i
|
||||
}
|
||||
}
|
||||
for li := n.FirstChild; li != nil; li = li.NextSibling {
|
||||
if li.Type != html.ElementNode || li.DataAtom != atom.Li {
|
||||
continue
|
||||
}
|
||||
marker := "- "
|
||||
if ordered {
|
||||
marker = strconv.Itoa(index) + ". "
|
||||
index++
|
||||
}
|
||||
var text strings.Builder
|
||||
var nested []*html.Node
|
||||
for c := li.FirstChild; c != nil; c = c.NextSibling {
|
||||
if skipped(c) {
|
||||
continue
|
||||
}
|
||||
if c.Type == html.ElementNode && (c.DataAtom == atom.Ul || c.DataAtom == atom.Ol) {
|
||||
nested = append(nested, c)
|
||||
continue
|
||||
}
|
||||
if c.Type == html.TextNode {
|
||||
text.WriteString(collapse(c.Data))
|
||||
} else {
|
||||
text.WriteString(r.inline(c))
|
||||
}
|
||||
}
|
||||
if item := strings.TrimSpace(text.String()); item != "" {
|
||||
b.WriteString(indent + marker + item + "\n")
|
||||
}
|
||||
for _, sub := range nested {
|
||||
var nb strings.Builder
|
||||
r.list(&nb, sub, indent+" ")
|
||||
b.WriteString(strings.TrimRight(nb.String(), "\n") + "\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// table renders a GFM pipe table; the first header row (th, else the first
|
||||
// tr) becomes the table head.
|
||||
func (r *mdRenderer) table(b *strings.Builder, n *html.Node) {
|
||||
var header []string
|
||||
var rows [][]string
|
||||
collectRows := func(scope *html.Node) {
|
||||
for tr := scope.FirstChild; tr != nil; tr = tr.NextSibling {
|
||||
if tr.Type != html.ElementNode || tr.DataAtom != atom.Tr {
|
||||
continue
|
||||
}
|
||||
var cells []string
|
||||
isHeader := false
|
||||
for cell := tr.FirstChild; cell != nil; cell = cell.NextSibling {
|
||||
if cell.Type != html.ElementNode {
|
||||
continue
|
||||
}
|
||||
switch cell.DataAtom {
|
||||
case atom.Th:
|
||||
isHeader = true
|
||||
cells = append(cells, r.cell(cell))
|
||||
case atom.Td:
|
||||
cells = append(cells, r.cell(cell))
|
||||
}
|
||||
}
|
||||
if len(cells) == 0 {
|
||||
continue
|
||||
}
|
||||
if isHeader && header == nil {
|
||||
header = cells
|
||||
} else {
|
||||
rows = append(rows, cells)
|
||||
}
|
||||
}
|
||||
}
|
||||
hasSections := false
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if c.Type != html.ElementNode {
|
||||
continue
|
||||
}
|
||||
switch c.DataAtom {
|
||||
case atom.Thead, atom.Tbody, atom.Tfoot:
|
||||
hasSections = true
|
||||
collectRows(c)
|
||||
}
|
||||
}
|
||||
if !hasSections {
|
||||
collectRows(n) // table with bare rows
|
||||
}
|
||||
if header == nil && len(rows) > 0 {
|
||||
header, rows = rows[0], rows[1:]
|
||||
}
|
||||
cols := len(header)
|
||||
for _, row := range rows {
|
||||
if len(row) > cols {
|
||||
cols = len(row)
|
||||
}
|
||||
}
|
||||
if cols == 0 {
|
||||
return
|
||||
}
|
||||
pad := func(row []string) []string {
|
||||
out := make([]string, cols)
|
||||
copy(out, row)
|
||||
return out
|
||||
}
|
||||
b.WriteString("| " + strings.Join(pad(header), " | ") + " |\n")
|
||||
b.WriteString("|" + strings.Repeat(" --- |", cols) + "\n")
|
||||
for _, row := range rows {
|
||||
b.WriteString("| " + strings.Join(pad(row), " | ") + " |\n")
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
// defList renders dt as a bold term and dd as its paragraph.
|
||||
func (r *mdRenderer) defList(b *strings.Builder, n *html.Node) {
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if c.Type != html.ElementNode {
|
||||
continue
|
||||
}
|
||||
switch c.DataAtom {
|
||||
case atom.Dt:
|
||||
if s := strings.TrimSpace(r.inline(c)); s != "" {
|
||||
b.WriteString("**" + s + "**\n\n")
|
||||
}
|
||||
case atom.Dd:
|
||||
if s := strings.TrimSpace(r.inline(c)); s != "" {
|
||||
b.WriteString(s + "\n\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cell flattens one table cell to a single line.
|
||||
func (r *mdRenderer) cell(n *html.Node) string {
|
||||
s := strings.TrimSpace(r.inline(n))
|
||||
return strings.ReplaceAll(s, "|", "\\|")
|
||||
}
|
||||
|
||||
// inline renders phrasing content to a markdown string.
|
||||
func (r *mdRenderer) inline(n *html.Node) string {
|
||||
var b strings.Builder
|
||||
r.inlineInto(&b, n)
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func (r *mdRenderer) inlineInto(b *strings.Builder, n *html.Node) {
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if skipped(c) {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case c.Type == html.TextNode:
|
||||
b.WriteString(collapse(c.Data))
|
||||
case c.Type == html.ElementNode:
|
||||
r.inlineElement(b, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *mdRenderer) inlineElement(b *strings.Builder, n *html.Node) {
|
||||
switch n.DataAtom {
|
||||
case atom.A:
|
||||
label := strings.TrimSpace(r.inline(n))
|
||||
if label == "" {
|
||||
label = attr(n, "href")
|
||||
}
|
||||
href := strings.TrimSpace(attr(n, "href"))
|
||||
lower := strings.ToLower(href)
|
||||
if href == "" || strings.HasPrefix(href, "#") || strings.HasPrefix(lower, "javascript:") {
|
||||
// an agent cannot follow these; keep the text only
|
||||
b.WriteString(label)
|
||||
return
|
||||
}
|
||||
b.WriteString("[" + label + "](" + r.resolve(href) + ")")
|
||||
case atom.Img:
|
||||
src := strings.TrimSpace(attr(n, "src"))
|
||||
if src == "" {
|
||||
return
|
||||
}
|
||||
alt := collapse(attr(n, "alt"))
|
||||
if alt == "" {
|
||||
alt = baseName(src)
|
||||
}
|
||||
b.WriteString("![" + alt + "](" + r.resolve(src) + ")")
|
||||
case atom.Br:
|
||||
b.WriteString("\n")
|
||||
case atom.Hr:
|
||||
b.WriteString("\n---\n")
|
||||
case atom.Strong, atom.B:
|
||||
b.WriteString("**" + strings.TrimSpace(r.inline(n)) + "**")
|
||||
case atom.Em, atom.I:
|
||||
b.WriteString("*" + strings.TrimSpace(r.inline(n)) + "*")
|
||||
case atom.Del, atom.S:
|
||||
b.WriteString("~~" + strings.TrimSpace(r.inline(n)) + "~~")
|
||||
case atom.Code:
|
||||
b.WriteString("`" + strings.TrimSpace(r.inline(n)) + "`")
|
||||
case atom.Summary:
|
||||
b.WriteString("**" + strings.TrimSpace(r.inline(n)) + "** ")
|
||||
case atom.Input, atom.Select, atom.Textarea, atom.Form:
|
||||
// form controls have no markdown meaning an agent can act on
|
||||
default:
|
||||
if blockish[n.DataAtom] {
|
||||
// block-level content inside inline context (nav menus, divs in
|
||||
// paragraphs): keep the words from running together
|
||||
padInline(b)
|
||||
r.inlineInto(b, n)
|
||||
padInline(b)
|
||||
return
|
||||
}
|
||||
r.inlineInto(b, n)
|
||||
}
|
||||
}
|
||||
|
||||
// blockish marks elements a browser starts on a fresh line; adjacent ones
|
||||
// need separating whitespace once flattened to inline text.
|
||||
var blockish = map[atom.Atom]bool{
|
||||
atom.Div: true, atom.Ul: true, atom.Ol: true, atom.Li: true, atom.Nav: true,
|
||||
atom.Section: true, atom.Article: true, atom.Header: true, atom.Footer: true,
|
||||
atom.Aside: true, atom.Main: true, atom.Form: true, atom.Fieldset: true,
|
||||
atom.Figure: true, atom.P: true, atom.Table: true, atom.Tr: true,
|
||||
}
|
||||
|
||||
func padInline(b *strings.Builder) {
|
||||
s := b.String()
|
||||
if s == "" {
|
||||
return
|
||||
}
|
||||
if last := s[len(s)-1]; last != ' ' && last != '\n' {
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
}
|
||||
|
||||
// resolve turns href into an absolute URL against the inspected page.
|
||||
func (r *mdRenderer) resolve(href string) string {
|
||||
if r.base == nil {
|
||||
return href
|
||||
}
|
||||
u, err := url.Parse(href)
|
||||
if err != nil {
|
||||
return href
|
||||
}
|
||||
return r.base.ResolveReference(u).String()
|
||||
}
|
||||
|
||||
func parseBase(raw string) *url.URL {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// collapse flattens whitespace runs to single spaces.
|
||||
func collapse(s string) string { return wsRe.ReplaceAllString(s, " ") }
|
||||
|
||||
// textContent returns the raw concatenated text under n.
|
||||
func textContent(n *html.Node) string {
|
||||
var b strings.Builder
|
||||
var walk func(n *html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.TextNode {
|
||||
b.WriteString(n.Data)
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(n)
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func prefixLines(s, prefix string) string {
|
||||
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
|
||||
for i, line := range lines {
|
||||
if line == "" {
|
||||
lines[i] = strings.TrimRight(prefix, " ")
|
||||
continue
|
||||
}
|
||||
lines[i] = prefix + line
|
||||
}
|
||||
return strings.Join(lines, "\n") + "\n\n"
|
||||
}
|
||||
|
||||
func baseName(src string) string {
|
||||
s := src
|
||||
if i := strings.IndexAny(s, "?#"); i >= 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
if i := strings.LastIndex(s, "/"); i >= 0 {
|
||||
s = s[i+1:]
|
||||
}
|
||||
if s == "" {
|
||||
return "image"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func attr(n *html.Node, key string) string {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == key {
|
||||
return a.Val
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func hasAttr(n *html.Node, key string) bool {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == key {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// truncateMarkdown caps markdown at max chars without splitting a rune.
|
||||
func truncateMarkdown(md string, max int) (string, bool) {
|
||||
if len(md) <= max {
|
||||
return md, false
|
||||
}
|
||||
// step back to a rune boundary so the cut stays valid UTF-8
|
||||
for i := max; i > max-4 && i > 0; i-- {
|
||||
if (md[i] & 0xC0) != 0x80 {
|
||||
return md[:i], true
|
||||
}
|
||||
}
|
||||
return md[:max], true
|
||||
}
|
||||
Reference in new issue
Block a user