71 lines
2.1 KiB
Go
71 lines
2.1 KiB
Go
package browse
|
|
|
|
import (
|
|
"regexp"
|
|
"strings"
|
|
)
|
|
|
|
// llmsLinkRe matches `- [title](url): description` (the optional quoted
|
|
// title and the optional description are both per spec).
|
|
var llmsLinkRe = regexp.MustCompile(`^\s*[-*]\s+\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)\s*(?::\s*(.*))?$`)
|
|
|
|
// parseLlmsTxt parses an llms.txt document per llmstxt.org:
|
|
//
|
|
// # Title (required, exactly once, first)
|
|
// > Summary (optional blockquote)
|
|
// ## Section (optional, starts a curated list)
|
|
// > Section note (optional, directly after the heading)
|
|
// - [Link](https://…): what it is
|
|
//
|
|
// Everything that is not one of those shapes is ignored — the parsed doc
|
|
// describes what an agent can rely on, not everything the file contains.
|
|
func parseLlmsTxt(content string) *LlmsDoc {
|
|
doc := &LlmsDoc{Sections: []LlmsSection{}}
|
|
var section *LlmsSection
|
|
seenTitle := false
|
|
|
|
for _, line := range strings.Split(content, "\n") {
|
|
t := strings.TrimSpace(line)
|
|
switch {
|
|
case strings.HasPrefix(t, "# ") && !seenTitle:
|
|
doc.Title = strings.TrimSpace(t[2:])
|
|
seenTitle = true
|
|
case strings.HasPrefix(t, "## "):
|
|
doc.Sections = append(doc.Sections, LlmsSection{
|
|
Heading: strings.TrimSpace(t[3:]),
|
|
Links: []LlmsLink{},
|
|
})
|
|
section = &doc.Sections[len(doc.Sections)-1]
|
|
case strings.HasPrefix(t, ">"):
|
|
note := strings.TrimSpace(strings.TrimLeft(t, "> "))
|
|
if note == "" {
|
|
continue
|
|
}
|
|
switch {
|
|
case !seenTitle:
|
|
continue // blockquote before the title is not the summary
|
|
case section != nil && section.Description == "":
|
|
section.Description = note
|
|
case section == nil:
|
|
if doc.Description != "" {
|
|
doc.Description += " "
|
|
}
|
|
doc.Description += note
|
|
}
|
|
default:
|
|
m := llmsLinkRe.FindStringSubmatch(line)
|
|
if m == nil {
|
|
continue
|
|
}
|
|
link := LlmsLink{Title: m[1], URL: m[2], Description: strings.TrimSpace(m[3])}
|
|
if section == nil {
|
|
doc.Sections = append(doc.Sections, LlmsSection{Links: []LlmsLink{}})
|
|
section = &doc.Sections[len(doc.Sections)-1]
|
|
}
|
|
section.Links = append(section.Links, link)
|
|
doc.LinkCount++
|
|
}
|
|
}
|
|
return doc
|
|
}
|