578 lines
14 KiB
Go
578 lines
14 KiB
Go
package browse
|
|
|
|
import (
|
|
"net/url"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"golang.org/x/net/html"
|
|
"golang.org/x/net/html/atom"
|
|
)
|
|
|
|
// pageMeta is what agents read from <head> plus document shape statistics
|
|
// collected while walking the body.
|
|
type pageMeta struct {
|
|
Title string
|
|
Description string
|
|
Canonical string
|
|
Robots string
|
|
Lang string
|
|
H1 string
|
|
LinkCount int
|
|
DeadLinks int
|
|
ScriptCount int
|
|
ImageCount int
|
|
WordCount int
|
|
TextLength int
|
|
}
|
|
|
|
// maxMarkdownChars caps the converted markdown before it reaches the wire.
|
|
const maxMarkdownChars = 200_000
|
|
|
|
var (
|
|
hiddenStyleRe = regexp.MustCompile(`(?i)display\s*:\s*none|visibility\s*:\s*hidden`)
|
|
wsRe = regexp.MustCompile(`\s+`)
|
|
manyNlRe = regexp.MustCompile(`\n{3,}`)
|
|
)
|
|
|
|
// htmlToMarkdown parses the document once and returns the body rendered as
|
|
// markdown plus the head metadata. baseURL resolves relative links.
|
|
func htmlToMarkdown(body []byte, baseURL string) (string, pageMeta, error) {
|
|
var meta pageMeta
|
|
doc, err := html.Parse(strings.NewReader(string(body)))
|
|
if err != nil {
|
|
return "", meta, err
|
|
}
|
|
|
|
var htmlNode, bodyNode *html.Node
|
|
var walk func(n *html.Node)
|
|
walk = func(n *html.Node) {
|
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
|
if c.Type != html.ElementNode {
|
|
continue
|
|
}
|
|
switch c.DataAtom {
|
|
case atom.Html:
|
|
htmlNode = c
|
|
walk(c)
|
|
case atom.Head:
|
|
collectHead(c, &meta)
|
|
case atom.Body:
|
|
bodyNode = c
|
|
}
|
|
}
|
|
}
|
|
walk(doc)
|
|
if htmlNode != nil {
|
|
meta.Lang = attr(htmlNode, "lang")
|
|
}
|
|
if bodyNode != nil {
|
|
inspectBody(bodyNode, &meta)
|
|
}
|
|
|
|
r := &mdRenderer{base: parseBase(baseURL)}
|
|
return r.renderBody(bodyNode), meta, nil
|
|
}
|
|
|
|
// collectHead reads title, description, robots and canonical from <head>.
|
|
func collectHead(head *html.Node, meta *pageMeta) {
|
|
for c := head.FirstChild; c != nil; c = c.NextSibling {
|
|
if c.Type != html.ElementNode {
|
|
continue
|
|
}
|
|
switch c.DataAtom {
|
|
case atom.Title:
|
|
if meta.Title == "" {
|
|
meta.Title = collapse(textContent(c))
|
|
}
|
|
case atom.Meta:
|
|
content := collapse(attr(c, "content"))
|
|
switch strings.ToLower(attr(c, "name")) {
|
|
case "description":
|
|
if meta.Description == "" {
|
|
meta.Description = content
|
|
}
|
|
case "robots":
|
|
if meta.Robots == "" {
|
|
meta.Robots = content
|
|
}
|
|
}
|
|
case atom.Link:
|
|
if strings.EqualFold(attr(c, "rel"), "canonical") && meta.Canonical == "" {
|
|
meta.Canonical = attr(c, "href")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// inspectBody counts the elements checks care about and measures the
|
|
// visible text (script/style/svg content stays out of the count).
|
|
func inspectBody(body *html.Node, meta *pageMeta) {
|
|
var walk func(n *html.Node)
|
|
walk = func(n *html.Node) {
|
|
if n.Type == html.ElementNode {
|
|
switch n.DataAtom {
|
|
case atom.A:
|
|
href := strings.TrimSpace(attr(n, "href"))
|
|
if href == "" || strings.HasPrefix(href, "#") || strings.HasPrefix(strings.ToLower(href), "javascript:") {
|
|
meta.DeadLinks++
|
|
} else {
|
|
meta.LinkCount++
|
|
}
|
|
case atom.Script:
|
|
meta.ScriptCount++
|
|
return
|
|
case atom.Style, atom.Svg, atom.Noscript, atom.Template, atom.Iframe:
|
|
return
|
|
case atom.Img:
|
|
meta.ImageCount++
|
|
return
|
|
case atom.H1:
|
|
if meta.H1 == "" {
|
|
meta.H1 = collapse(textContent(n))
|
|
}
|
|
}
|
|
}
|
|
if n.Type == html.TextNode {
|
|
meta.TextLength += len(n.Data)
|
|
meta.WordCount += len(strings.Fields(n.Data))
|
|
}
|
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
|
walk(c)
|
|
}
|
|
}
|
|
walk(body)
|
|
}
|
|
|
|
// mdRenderer renders a parsed document as markdown.
|
|
type mdRenderer struct {
|
|
base *url.URL
|
|
}
|
|
|
|
func (r *mdRenderer) renderBody(body *html.Node) string {
|
|
if body == nil {
|
|
return ""
|
|
}
|
|
var b strings.Builder
|
|
r.blocks(&b, body)
|
|
out := strings.TrimSpace(b.String())
|
|
return manyNlRe.ReplaceAllString(out, "\n\n")
|
|
}
|
|
|
|
// skipped reports whether an element contributes nothing an agent could
|
|
// act on: invisible content, script payloads, and embedded documents.
|
|
func skipped(n *html.Node) bool {
|
|
if n.Type != html.ElementNode {
|
|
return false
|
|
}
|
|
switch n.DataAtom {
|
|
case atom.Script, atom.Style, atom.Noscript, atom.Template, atom.Svg,
|
|
atom.Head, atom.Link, atom.Meta, atom.Iframe, atom.Object, atom.Embed:
|
|
return true
|
|
}
|
|
if hasAttr(n, "hidden") || strings.EqualFold(attr(n, "aria-hidden"), "true") {
|
|
return true
|
|
}
|
|
return hiddenStyleRe.MatchString(attr(n, "style"))
|
|
}
|
|
|
|
// blocks renders the children of n as a sequence of markdown blocks.
|
|
func (r *mdRenderer) blocks(b *strings.Builder, n *html.Node) {
|
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
|
if skipped(c) {
|
|
continue
|
|
}
|
|
switch {
|
|
case c.Type == html.TextNode:
|
|
if s := strings.TrimSpace(collapse(c.Data)); s != "" {
|
|
b.WriteString(s + "\n\n")
|
|
}
|
|
case c.Type == html.ElementNode:
|
|
r.element(b, c)
|
|
}
|
|
}
|
|
}
|
|
|
|
// element renders one block-level element.
|
|
func (r *mdRenderer) element(b *strings.Builder, n *html.Node) {
|
|
switch n.DataAtom {
|
|
case atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6:
|
|
level := int(n.DataAtom) - int(atom.H1) + 1
|
|
b.WriteString(strings.Repeat("#", level) + " " + strings.TrimSpace(r.inline(n)) + "\n\n")
|
|
case atom.P:
|
|
if s := strings.TrimSpace(r.inline(n)); s != "" {
|
|
b.WriteString(s + "\n\n")
|
|
}
|
|
case atom.Hr:
|
|
b.WriteString("---\n\n")
|
|
case atom.Blockquote:
|
|
var inner strings.Builder
|
|
r.blocks(&inner, n)
|
|
b.WriteString(prefixLines(inner.String(), "> "))
|
|
case atom.Pre:
|
|
b.WriteString("```\n" + strings.Trim(textContent(n), "\n") + "\n```\n\n")
|
|
case atom.Ul, atom.Ol:
|
|
var lb strings.Builder
|
|
r.list(&lb, n, "")
|
|
b.WriteString(strings.TrimRight(lb.String(), "\n") + "\n\n")
|
|
case atom.Table:
|
|
r.table(b, n)
|
|
case atom.Dl:
|
|
r.defList(b, n)
|
|
case atom.Button, atom.A, atom.Span, atom.Strong, atom.Em, atom.Code, atom.Small, atom.Label:
|
|
// phrasing content at block position (nav links, card wrappers):
|
|
// render with full inline semantics so anchors keep their href
|
|
var ib strings.Builder
|
|
r.inlineNode(&ib, n)
|
|
if s := strings.TrimSpace(ib.String()); s != "" {
|
|
b.WriteString(s + "\n\n")
|
|
}
|
|
default:
|
|
r.blocks(b, n)
|
|
}
|
|
}
|
|
|
|
// inlineNode renders one child node with inline semantics — the single
|
|
// dispatch point for list items and block-position phrasing elements, so
|
|
// anchors keep their href wherever they appear.
|
|
func (r *mdRenderer) inlineNode(b *strings.Builder, n *html.Node) {
|
|
if n.Type == html.TextNode {
|
|
b.WriteString(collapse(n.Data))
|
|
return
|
|
}
|
|
if n.Type == html.ElementNode {
|
|
r.inlineElement(b, n)
|
|
}
|
|
}
|
|
|
|
// list renders ul/ol items; nested lists indent under their item.
|
|
func (r *mdRenderer) list(b *strings.Builder, n *html.Node, indent string) {
|
|
ordered := n.DataAtom == atom.Ol
|
|
index := 1
|
|
if v := attr(n, "start"); v != "" {
|
|
if i, err := strconv.Atoi(v); err == nil && i > 0 {
|
|
index = i
|
|
}
|
|
}
|
|
for li := n.FirstChild; li != nil; li = li.NextSibling {
|
|
if li.Type != html.ElementNode || li.DataAtom != atom.Li {
|
|
continue
|
|
}
|
|
marker := "- "
|
|
if ordered {
|
|
marker = strconv.Itoa(index) + ". "
|
|
index++
|
|
}
|
|
var text strings.Builder
|
|
var nested []*html.Node
|
|
for c := li.FirstChild; c != nil; c = c.NextSibling {
|
|
if skipped(c) {
|
|
continue
|
|
}
|
|
if c.Type == html.ElementNode && (c.DataAtom == atom.Ul || c.DataAtom == atom.Ol) {
|
|
nested = append(nested, c)
|
|
continue
|
|
}
|
|
r.inlineNode(&text, c)
|
|
}
|
|
if item := strings.TrimSpace(text.String()); item != "" {
|
|
b.WriteString(indent + marker + item + "\n")
|
|
}
|
|
for _, sub := range nested {
|
|
var nb strings.Builder
|
|
r.list(&nb, sub, indent+" ")
|
|
b.WriteString(strings.TrimRight(nb.String(), "\n") + "\n")
|
|
}
|
|
}
|
|
}
|
|
|
|
// table renders a GFM pipe table; the first header row (th, else the first
|
|
// tr) becomes the table head.
|
|
func (r *mdRenderer) table(b *strings.Builder, n *html.Node) {
|
|
var header []string
|
|
var rows [][]string
|
|
collectRows := func(scope *html.Node) {
|
|
for tr := scope.FirstChild; tr != nil; tr = tr.NextSibling {
|
|
if tr.Type != html.ElementNode || tr.DataAtom != atom.Tr {
|
|
continue
|
|
}
|
|
var cells []string
|
|
isHeader := false
|
|
for cell := tr.FirstChild; cell != nil; cell = cell.NextSibling {
|
|
if cell.Type != html.ElementNode {
|
|
continue
|
|
}
|
|
switch cell.DataAtom {
|
|
case atom.Th:
|
|
isHeader = true
|
|
cells = append(cells, r.cell(cell))
|
|
case atom.Td:
|
|
cells = append(cells, r.cell(cell))
|
|
}
|
|
}
|
|
if len(cells) == 0 {
|
|
continue
|
|
}
|
|
if isHeader && header == nil {
|
|
header = cells
|
|
} else {
|
|
rows = append(rows, cells)
|
|
}
|
|
}
|
|
}
|
|
hasSections := false
|
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
|
if c.Type != html.ElementNode {
|
|
continue
|
|
}
|
|
switch c.DataAtom {
|
|
case atom.Thead, atom.Tbody, atom.Tfoot:
|
|
hasSections = true
|
|
collectRows(c)
|
|
}
|
|
}
|
|
if !hasSections {
|
|
collectRows(n) // table with bare rows
|
|
}
|
|
if header == nil && len(rows) > 0 {
|
|
header, rows = rows[0], rows[1:]
|
|
}
|
|
cols := len(header)
|
|
for _, row := range rows {
|
|
if len(row) > cols {
|
|
cols = len(row)
|
|
}
|
|
}
|
|
if cols == 0 {
|
|
return
|
|
}
|
|
pad := func(row []string) []string {
|
|
out := make([]string, cols)
|
|
copy(out, row)
|
|
return out
|
|
}
|
|
b.WriteString("| " + strings.Join(pad(header), " | ") + " |\n")
|
|
b.WriteString("|" + strings.Repeat(" --- |", cols) + "\n")
|
|
for _, row := range rows {
|
|
b.WriteString("| " + strings.Join(pad(row), " | ") + " |\n")
|
|
}
|
|
b.WriteString("\n")
|
|
}
|
|
|
|
// defList renders dt as a bold term and dd as its paragraph.
|
|
func (r *mdRenderer) defList(b *strings.Builder, n *html.Node) {
|
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
|
if c.Type != html.ElementNode {
|
|
continue
|
|
}
|
|
switch c.DataAtom {
|
|
case atom.Dt:
|
|
if s := strings.TrimSpace(r.inline(c)); s != "" {
|
|
b.WriteString("**" + s + "**\n\n")
|
|
}
|
|
case atom.Dd:
|
|
if s := strings.TrimSpace(r.inline(c)); s != "" {
|
|
b.WriteString(s + "\n\n")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// cell flattens one table cell to a single line.
|
|
func (r *mdRenderer) cell(n *html.Node) string {
|
|
s := strings.TrimSpace(r.inline(n))
|
|
return strings.ReplaceAll(s, "|", "\\|")
|
|
}
|
|
|
|
// inline renders phrasing content to a markdown string.
|
|
func (r *mdRenderer) inline(n *html.Node) string {
|
|
var b strings.Builder
|
|
r.inlineInto(&b, n)
|
|
return b.String()
|
|
}
|
|
|
|
func (r *mdRenderer) inlineInto(b *strings.Builder, n *html.Node) {
|
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
|
if skipped(c) {
|
|
continue
|
|
}
|
|
switch {
|
|
case c.Type == html.TextNode:
|
|
b.WriteString(collapse(c.Data))
|
|
case c.Type == html.ElementNode:
|
|
r.inlineElement(b, c)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (r *mdRenderer) inlineElement(b *strings.Builder, n *html.Node) {
|
|
switch n.DataAtom {
|
|
case atom.A:
|
|
label := strings.TrimSpace(r.inline(n))
|
|
if label == "" {
|
|
label = attr(n, "href")
|
|
}
|
|
href := strings.TrimSpace(attr(n, "href"))
|
|
lower := strings.ToLower(href)
|
|
if href == "" || strings.HasPrefix(href, "#") || strings.HasPrefix(lower, "javascript:") {
|
|
// an agent cannot follow these; keep the text only
|
|
b.WriteString(label)
|
|
return
|
|
}
|
|
b.WriteString("[" + label + "](" + r.resolve(href) + ")")
|
|
case atom.Img:
|
|
src := strings.TrimSpace(attr(n, "src"))
|
|
if src == "" {
|
|
return
|
|
}
|
|
alt := collapse(attr(n, "alt"))
|
|
if alt == "" {
|
|
alt = baseName(src)
|
|
}
|
|
b.WriteString("![" + alt + "](" + r.resolve(src) + ")")
|
|
case atom.Br:
|
|
b.WriteString("\n")
|
|
case atom.Hr:
|
|
b.WriteString("\n---\n")
|
|
case atom.Strong, atom.B:
|
|
b.WriteString("**" + strings.TrimSpace(r.inline(n)) + "**")
|
|
case atom.Em, atom.I:
|
|
b.WriteString("*" + strings.TrimSpace(r.inline(n)) + "*")
|
|
case atom.Del, atom.S:
|
|
b.WriteString("~~" + strings.TrimSpace(r.inline(n)) + "~~")
|
|
case atom.Code:
|
|
b.WriteString("`" + strings.TrimSpace(r.inline(n)) + "`")
|
|
case atom.Summary:
|
|
b.WriteString("**" + strings.TrimSpace(r.inline(n)) + "** ")
|
|
case atom.Input, atom.Select, atom.Textarea, atom.Form:
|
|
// form controls have no markdown meaning an agent can act on
|
|
default:
|
|
if blockish[n.DataAtom] {
|
|
// block-level content inside inline context (nav menus, divs in
|
|
// paragraphs): keep the words from running together
|
|
padInline(b)
|
|
r.inlineInto(b, n)
|
|
padInline(b)
|
|
return
|
|
}
|
|
r.inlineInto(b, n)
|
|
}
|
|
}
|
|
|
|
// blockish marks elements a browser starts on a fresh line; adjacent ones
|
|
// need separating whitespace once flattened to inline text.
|
|
var blockish = map[atom.Atom]bool{
|
|
atom.Div: true, atom.Ul: true, atom.Ol: true, atom.Li: true, atom.Nav: true,
|
|
atom.Section: true, atom.Article: true, atom.Header: true, atom.Footer: true,
|
|
atom.Aside: true, atom.Main: true, atom.Form: true, atom.Fieldset: true,
|
|
atom.Figure: true, atom.P: true, atom.Table: true, atom.Tr: true,
|
|
}
|
|
|
|
func padInline(b *strings.Builder) {
|
|
s := b.String()
|
|
if s == "" {
|
|
return
|
|
}
|
|
if last := s[len(s)-1]; last != ' ' && last != '\n' {
|
|
b.WriteByte(' ')
|
|
}
|
|
}
|
|
|
|
// resolve turns href into an absolute URL against the inspected page.
|
|
func (r *mdRenderer) resolve(href string) string {
|
|
if r.base == nil {
|
|
return href
|
|
}
|
|
u, err := url.Parse(href)
|
|
if err != nil {
|
|
return href
|
|
}
|
|
return r.base.ResolveReference(u).String()
|
|
}
|
|
|
|
func parseBase(raw string) *url.URL {
|
|
u, err := url.Parse(raw)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
return u
|
|
}
|
|
|
|
// collapse flattens whitespace runs to single spaces.
|
|
func collapse(s string) string { return wsRe.ReplaceAllString(s, " ") }
|
|
|
|
// textContent returns the raw concatenated text under n.
|
|
func textContent(n *html.Node) string {
|
|
var b strings.Builder
|
|
var walk func(n *html.Node)
|
|
walk = func(n *html.Node) {
|
|
if n.Type == html.TextNode {
|
|
b.WriteString(n.Data)
|
|
}
|
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
|
walk(c)
|
|
}
|
|
}
|
|
walk(n)
|
|
return b.String()
|
|
}
|
|
|
|
func prefixLines(s, prefix string) string {
|
|
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
|
|
for i, line := range lines {
|
|
if line == "" {
|
|
lines[i] = strings.TrimRight(prefix, " ")
|
|
continue
|
|
}
|
|
lines[i] = prefix + line
|
|
}
|
|
return strings.Join(lines, "\n") + "\n\n"
|
|
}
|
|
|
|
func baseName(src string) string {
|
|
s := src
|
|
if i := strings.IndexAny(s, "?#"); i >= 0 {
|
|
s = s[:i]
|
|
}
|
|
if i := strings.LastIndex(s, "/"); i >= 0 {
|
|
s = s[i+1:]
|
|
}
|
|
if s == "" {
|
|
return "image"
|
|
}
|
|
return s
|
|
}
|
|
|
|
func attr(n *html.Node, key string) string {
|
|
for _, a := range n.Attr {
|
|
if a.Key == key {
|
|
return a.Val
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func hasAttr(n *html.Node, key string) bool {
|
|
for _, a := range n.Attr {
|
|
if a.Key == key {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// truncateMarkdown caps markdown at max chars without splitting a rune.
|
|
func truncateMarkdown(md string, max int) (string, bool) {
|
|
if len(md) <= max {
|
|
return md, false
|
|
}
|
|
// step back to a rune boundary so the cut stays valid UTF-8
|
|
for i := max; i > max-4 && i > 0; i-- {
|
|
if (md[i] & 0xC0) != 0x80 {
|
|
return md[:i], true
|
|
}
|
|
}
|
|
return md[:max], true
|
|
}
|