llm-browser/backend/internal/browse/markdown.go

package browse

import (
	"net/url"
	"regexp"
	"strconv"
	"strings"

	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

// pageMeta is what agents read from <head> plus document shape statistics
// collected while walking the body.
type pageMeta struct {
	Title       string
	Description string
	Canonical   string
	Robots      string
	Lang        string
	H1          string
	LinkCount   int
	DeadLinks   int
	ScriptCount int
	ImageCount  int
	WordCount   int
	TextLength  int
}

// maxMarkdownChars caps the converted markdown before it reaches the wire.
const maxMarkdownChars = 200_000

var (
	hiddenStyleRe = regexp.MustCompile(`(?i)display\s*:\s*none|visibility\s*:\s*hidden`)
	wsRe          = regexp.MustCompile(`\s+`)
	manyNlRe      = regexp.MustCompile(`\n{3,}`)
)

// htmlToMarkdown parses the document once and returns the body rendered as
// markdown plus the head metadata. baseURL resolves relative links.
func htmlToMarkdown(body []byte, baseURL string) (string, pageMeta, error) {
	var meta pageMeta
	doc, err := html.Parse(strings.NewReader(string(body)))
	if err != nil {
		return "", meta, err
	}

	var htmlNode, bodyNode *html.Node
	var walk func(n *html.Node)
	walk = func(n *html.Node) {
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			if c.Type != html.ElementNode {
				continue
			}
			switch c.DataAtom {
			case atom.Html:
				htmlNode = c
				walk(c)
			case atom.Head:
				collectHead(c, &meta)
			case atom.Body:
				bodyNode = c
			}
		}
	}
	walk(doc)
	if htmlNode != nil {
		meta.Lang = attr(htmlNode, "lang")
	}
	if bodyNode != nil {
		inspectBody(bodyNode, &meta)
	}

	r := &mdRenderer{base: parseBase(baseURL)}
	return r.renderBody(bodyNode), meta, nil
}

// collectHead reads title, description, robots and canonical from <head>.
func collectHead(head *html.Node, meta *pageMeta) {
	for c := head.FirstChild; c != nil; c = c.NextSibling {
		if c.Type != html.ElementNode {
			continue
		}
		switch c.DataAtom {
		case atom.Title:
			if meta.Title == "" {
				meta.Title = collapse(textContent(c))
			}
		case atom.Meta:
			content := collapse(attr(c, "content"))
			switch strings.ToLower(attr(c, "name")) {
			case "description":
				if meta.Description == "" {
					meta.Description = content
				}
			case "robots":
				if meta.Robots == "" {
					meta.Robots = content
				}
			}
		case atom.Link:
			if strings.EqualFold(attr(c, "rel"), "canonical") && meta.Canonical == "" {
				meta.Canonical = attr(c, "href")
			}
		}
	}
}

// inspectBody counts the elements checks care about and measures the
// visible text (script/style/svg content stays out of the count).
func inspectBody(body *html.Node, meta *pageMeta) {
	var walk func(n *html.Node)
	walk = func(n *html.Node) {
		if n.Type == html.ElementNode {
			switch n.DataAtom {
			case atom.A:
				href := strings.TrimSpace(attr(n, "href"))
				if href == "" || strings.HasPrefix(href, "#") || strings.HasPrefix(strings.ToLower(href), "javascript:") {
					meta.DeadLinks++
				} else {
					meta.LinkCount++
				}
			case atom.Script:
				meta.ScriptCount++
				return
			case atom.Style, atom.Svg, atom.Noscript, atom.Template, atom.Iframe:
				return
			case atom.Img:
				meta.ImageCount++
				return
			case atom.H1:
				if meta.H1 == "" {
					meta.H1 = collapse(textContent(n))
				}
			}
		}
		if n.Type == html.TextNode {
			meta.TextLength += len(n.Data)
			meta.WordCount += len(strings.Fields(n.Data))
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(body)
}

// mdRenderer renders a parsed document as markdown.
type mdRenderer struct {
	base *url.URL
}

func (r *mdRenderer) renderBody(body *html.Node) string {
	if body == nil {
		return ""
	}
	var b strings.Builder
	r.blocks(&b, body)
	out := strings.TrimSpace(b.String())
	return manyNlRe.ReplaceAllString(out, "\n\n")
}

// skipped reports whether an element contributes nothing an agent could
// act on: invisible content, script payloads, and embedded documents.
func skipped(n *html.Node) bool {
	if n.Type != html.ElementNode {
		return false
	}
	switch n.DataAtom {
	case atom.Script, atom.Style, atom.Noscript, atom.Template, atom.Svg,
		atom.Head, atom.Link, atom.Meta, atom.Iframe, atom.Object, atom.Embed:
		return true
	}
	if hasAttr(n, "hidden") || strings.EqualFold(attr(n, "aria-hidden"), "true") {
		return true
	}
	return hiddenStyleRe.MatchString(attr(n, "style"))
}

// blocks renders the children of n as a sequence of markdown blocks.
func (r *mdRenderer) blocks(b *strings.Builder, n *html.Node) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if skipped(c) {
			continue
		}
		switch {
		case c.Type == html.TextNode:
			if s := strings.TrimSpace(collapse(c.Data)); s != "" {
				b.WriteString(s + "\n\n")
			}
		case c.Type == html.ElementNode:
			r.element(b, c)
		}
	}
}

// element renders one block-level element.
func (r *mdRenderer) element(b *strings.Builder, n *html.Node) {
	switch n.DataAtom {
	case atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6:
		level := int(n.DataAtom) - int(atom.H1) + 1
		b.WriteString(strings.Repeat("#", level) + " " + strings.TrimSpace(r.inline(n)) + "\n\n")
	case atom.P:
		if s := strings.TrimSpace(r.inline(n)); s != "" {
			b.WriteString(s + "\n\n")
		}
	case atom.Hr:
		b.WriteString("---\n\n")
	case atom.Blockquote:
		var inner strings.Builder
		r.blocks(&inner, n)
		b.WriteString(prefixLines(inner.String(), "> "))
	case atom.Pre:
		b.WriteString("```\n" + strings.Trim(textContent(n), "\n") + "\n```\n\n")
	case atom.Ul, atom.Ol:
		var lb strings.Builder
		r.list(&lb, n, "")
		b.WriteString(strings.TrimRight(lb.String(), "\n") + "\n\n")
	case atom.Table:
		r.table(b, n)
	case atom.Dl:
		r.defList(b, n)
	case atom.Button, atom.A, atom.Span, atom.Strong, atom.Em, atom.Code, atom.Small, atom.Label:
		// phrasing content at block position (nav links, card wrappers):
		// render with full inline semantics so anchors keep their href
		var ib strings.Builder
		r.inlineNode(&ib, n)
		if s := strings.TrimSpace(ib.String()); s != "" {
			b.WriteString(s + "\n\n")
		}
	default:
		r.blocks(b, n)
	}
}

// inlineNode renders one child node with inline semantics — the single
// dispatch point for list items and block-position phrasing elements, so
// anchors keep their href wherever they appear.
func (r *mdRenderer) inlineNode(b *strings.Builder, n *html.Node) {
	if n.Type == html.TextNode {
		b.WriteString(collapse(n.Data))
		return
	}
	if n.Type == html.ElementNode {
		r.inlineElement(b, n)
	}
}

// list renders ul/ol items; nested lists indent under their item.
func (r *mdRenderer) list(b *strings.Builder, n *html.Node, indent string) {
	ordered := n.DataAtom == atom.Ol
	index := 1
	if v := attr(n, "start"); v != "" {
		if i, err := strconv.Atoi(v); err == nil && i > 0 {
			index = i
		}
	}
	for li := n.FirstChild; li != nil; li = li.NextSibling {
		if li.Type != html.ElementNode || li.DataAtom != atom.Li {
			continue
		}
		marker := "- "
		if ordered {
			marker = strconv.Itoa(index) + ". "
			index++
		}
		var text strings.Builder
		var nested []*html.Node
		for c := li.FirstChild; c != nil; c = c.NextSibling {
			if skipped(c) {
				continue
			}
			if c.Type == html.ElementNode && (c.DataAtom == atom.Ul || c.DataAtom == atom.Ol) {
				nested = append(nested, c)
				continue
			}
			r.inlineNode(&text, c)
		}
		if item := strings.TrimSpace(text.String()); item != "" {
			b.WriteString(indent + marker + item + "\n")
		}
		for _, sub := range nested {
			var nb strings.Builder
			r.list(&nb, sub, indent+"  ")
			b.WriteString(strings.TrimRight(nb.String(), "\n") + "\n")
		}
	}
}

// table renders a GFM pipe table; the first header row (th, else the first
// tr) becomes the table head.
func (r *mdRenderer) table(b *strings.Builder, n *html.Node) {
	var header []string
	var rows [][]string
	collectRows := func(scope *html.Node) {
		for tr := scope.FirstChild; tr != nil; tr = tr.NextSibling {
			if tr.Type != html.ElementNode || tr.DataAtom != atom.Tr {
				continue
			}
			var cells []string
			isHeader := false
			for cell := tr.FirstChild; cell != nil; cell = cell.NextSibling {
				if cell.Type != html.ElementNode {
					continue
				}
				switch cell.DataAtom {
				case atom.Th:
					isHeader = true
					cells = append(cells, r.cell(cell))
				case atom.Td:
					cells = append(cells, r.cell(cell))
				}
			}
			if len(cells) == 0 {
				continue
			}
			if isHeader && header == nil {
				header = cells
			} else {
				rows = append(rows, cells)
			}
		}
	}
	hasSections := false
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if c.Type != html.ElementNode {
			continue
		}
		switch c.DataAtom {
		case atom.Thead, atom.Tbody, atom.Tfoot:
			hasSections = true
			collectRows(c)
		}
	}
	if !hasSections {
		collectRows(n) // table with bare rows
	}
	if header == nil && len(rows) > 0 {
		header, rows = rows[0], rows[1:]
	}
	cols := len(header)
	for _, row := range rows {
		if len(row) > cols {
			cols = len(row)
		}
	}
	if cols == 0 {
		return
	}
	pad := func(row []string) []string {
		out := make([]string, cols)
		copy(out, row)
		return out
	}
	b.WriteString("| " + strings.Join(pad(header), " | ") + " |\n")
	b.WriteString("|" + strings.Repeat(" --- |", cols) + "\n")
	for _, row := range rows {
		b.WriteString("| " + strings.Join(pad(row), " | ") + " |\n")
	}
	b.WriteString("\n")
}

// defList renders dt as a bold term and dd as its paragraph.
func (r *mdRenderer) defList(b *strings.Builder, n *html.Node) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if c.Type != html.ElementNode {
			continue
		}
		switch c.DataAtom {
		case atom.Dt:
			if s := strings.TrimSpace(r.inline(c)); s != "" {
				b.WriteString("**" + s + "**\n\n")
			}
		case atom.Dd:
			if s := strings.TrimSpace(r.inline(c)); s != "" {
				b.WriteString(s + "\n\n")
			}
		}
	}
}

// cell flattens one table cell to a single line.
func (r *mdRenderer) cell(n *html.Node) string {
	s := strings.TrimSpace(r.inline(n))
	return strings.ReplaceAll(s, "|", "\\|")
}

// inline renders phrasing content to a markdown string.
func (r *mdRenderer) inline(n *html.Node) string {
	var b strings.Builder
	r.inlineInto(&b, n)
	return b.String()
}

func (r *mdRenderer) inlineInto(b *strings.Builder, n *html.Node) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if skipped(c) {
			continue
		}
		switch {
		case c.Type == html.TextNode:
			b.WriteString(collapse(c.Data))
		case c.Type == html.ElementNode:
			r.inlineElement(b, c)
		}
	}
}

func (r *mdRenderer) inlineElement(b *strings.Builder, n *html.Node) {
	switch n.DataAtom {
	case atom.A:
		label := strings.TrimSpace(r.inline(n))
		if label == "" {
			label = attr(n, "href")
		}
		href := strings.TrimSpace(attr(n, "href"))
		lower := strings.ToLower(href)
		if href == "" || strings.HasPrefix(href, "#") || strings.HasPrefix(lower, "javascript:") {
			// an agent cannot follow these; keep the text only
			b.WriteString(label)
			return
		}
		b.WriteString("[" + label + "](" + r.resolve(href) + ")")
	case atom.Img:
		src := strings.TrimSpace(attr(n, "src"))
		if src == "" {
			return
		}
		alt := collapse(attr(n, "alt"))
		if alt == "" {
			alt = baseName(src)
		}
		b.WriteString("![" + alt + "](" + r.resolve(src) + ")")
	case atom.Br:
		b.WriteString("\n")
	case atom.Hr:
		b.WriteString("\n---\n")
	case atom.Strong, atom.B:
		b.WriteString("**" + strings.TrimSpace(r.inline(n)) + "**")
	case atom.Em, atom.I:
		b.WriteString("*" + strings.TrimSpace(r.inline(n)) + "*")
	case atom.Del, atom.S:
		b.WriteString("~~" + strings.TrimSpace(r.inline(n)) + "~~")
	case atom.Code:
		b.WriteString("`" + strings.TrimSpace(r.inline(n)) + "`")
	case atom.Summary:
		b.WriteString("**" + strings.TrimSpace(r.inline(n)) + "** ")
	case atom.Input, atom.Select, atom.Textarea, atom.Form:
		// form controls have no markdown meaning an agent can act on
	default:
		if blockish[n.DataAtom] {
			// block-level content inside inline context (nav menus, divs in
			// paragraphs): keep the words from running together
			padInline(b)
			r.inlineInto(b, n)
			padInline(b)
			return
		}
		r.inlineInto(b, n)
	}
}

// blockish marks elements a browser starts on a fresh line; adjacent ones
// need separating whitespace once flattened to inline text.
var blockish = map[atom.Atom]bool{
	atom.Div: true, atom.Ul: true, atom.Ol: true, atom.Li: true, atom.Nav: true,
	atom.Section: true, atom.Article: true, atom.Header: true, atom.Footer: true,
	atom.Aside: true, atom.Main: true, atom.Form: true, atom.Fieldset: true,
	atom.Figure: true, atom.P: true, atom.Table: true, atom.Tr: true,
}

func padInline(b *strings.Builder) {
	s := b.String()
	if s == "" {
		return
	}
	if last := s[len(s)-1]; last != ' ' && last != '\n' {
		b.WriteByte(' ')
	}
}

// resolve turns href into an absolute URL against the inspected page.
func (r *mdRenderer) resolve(href string) string {
	if r.base == nil {
		return href
	}
	u, err := url.Parse(href)
	if err != nil {
		return href
	}
	return r.base.ResolveReference(u).String()
}

func parseBase(raw string) *url.URL {
	u, err := url.Parse(raw)
	if err != nil {
		return nil
	}
	return u
}

// collapse flattens whitespace runs to single spaces.
func collapse(s string) string { return wsRe.ReplaceAllString(s, " ") }

// textContent returns the raw concatenated text under n.
func textContent(n *html.Node) string {
	var b strings.Builder
	var walk func(n *html.Node)
	walk = func(n *html.Node) {
		if n.Type == html.TextNode {
			b.WriteString(n.Data)
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(n)
	return b.String()
}

func prefixLines(s, prefix string) string {
	lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
	for i, line := range lines {
		if line == "" {
			lines[i] = strings.TrimRight(prefix, " ")
			continue
		}
		lines[i] = prefix + line
	}
	return strings.Join(lines, "\n") + "\n\n"
}

func baseName(src string) string {
	s := src
	if i := strings.IndexAny(s, "?#"); i >= 0 {
		s = s[:i]
	}
	if i := strings.LastIndex(s, "/"); i >= 0 {
		s = s[i+1:]
	}
	if s == "" {
		return "image"
	}
	return s
}

func attr(n *html.Node, key string) string {
	for _, a := range n.Attr {
		if a.Key == key {
			return a.Val
		}
	}
	return ""
}

func hasAttr(n *html.Node, key string) bool {
	for _, a := range n.Attr {
		if a.Key == key {
			return true
		}
	}
	return false
}

// truncateMarkdown caps markdown at max chars without splitting a rune.
func truncateMarkdown(md string, max int) (string, bool) {
	if len(md) <= max {
		return md, false
	}
	// step back to a rune boundary so the cut stays valid UTF-8
	for i := max; i > max-4 && i > 0; i-- {
		if (md[i] & 0xC0) != 0x80 {
			return md[:i], true
		}
	}
	return md[:max], true
}