llm browser

backend/internal/browse/llmstxt.go

@@ -0,0 +1,70 @@
+package browse
+
+import (
+	"regexp"
+	"strings"
+)
+
+// llmsLinkRe matches `- [title](url): description` (the optional quoted
+// title and the optional description are both per spec).
+var llmsLinkRe = regexp.MustCompile(`^\s*[-*]\s+\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)\s*(?::\s*(.*))?$`)
+
+// parseLlmsTxt parses an llms.txt document per llmstxt.org:
+//
+//	# Title            (required, exactly once, first)
+//	> Summary          (optional blockquote)
+//	## Section         (optional, starts a curated list)
+//	> Section note     (optional, directly after the heading)
+//	- [Link](https://…): what it is
+//
+// Everything that is not one of those shapes is ignored — the parsed doc
+// describes what an agent can rely on, not everything the file contains.
+func parseLlmsTxt(content string) *LlmsDoc {
+	doc := &LlmsDoc{Sections: []LlmsSection{}}
+	var section *LlmsSection
+	seenTitle := false
+
+	for _, line := range strings.Split(content, "\n") {
+		t := strings.TrimSpace(line)
+		switch {
+		case strings.HasPrefix(t, "# ") && !seenTitle:
+			doc.Title = strings.TrimSpace(t[2:])
+			seenTitle = true
+		case strings.HasPrefix(t, "## "):
+			doc.Sections = append(doc.Sections, LlmsSection{
+				Heading: strings.TrimSpace(t[3:]),
+				Links:   []LlmsLink{},
+			})
+			section = &doc.Sections[len(doc.Sections)-1]
+		case strings.HasPrefix(t, ">"):
+			note := strings.TrimSpace(strings.TrimLeft(t, "> "))
+			if note == "" {
+				continue
+			}
+			switch {
+			case !seenTitle:
+				continue // blockquote before the title is not the summary
+			case section != nil && section.Description == "":
+				section.Description = note
+			case section == nil:
+				if doc.Description != "" {
+					doc.Description += " "
+				}
+				doc.Description += note
+			}
+		default:
+			m := llmsLinkRe.FindStringSubmatch(line)
+			if m == nil {
+				continue
+			}
+			link := LlmsLink{Title: m[1], URL: m[2], Description: strings.TrimSpace(m[3])}
+			if section == nil {
+				doc.Sections = append(doc.Sections, LlmsSection{Links: []LlmsLink{}})
+				section = &doc.Sections[len(doc.Sections)-1]
+			}
+			section.Links = append(section.Links, link)
+			doc.LinkCount++
+		}
+	}
+	return doc
+}