fancy-dns/internal/blocklist/parse.go

package blocklist

import (
	"bufio"
	"bytes"
	"fmt"
	"net/netip"
	"strconv"
	"strings"

	"github.com/miekg/dns"

	"git.k8s-git-k6n.net/mats/fancy-dns/internal/zonefile"
)

// Format identifies a list's syntax.
type Format string

const (
	// FormatAuto sniffs the content.
	FormatAuto Format = "auto"
	// FormatHosts is "IP domain [domain...]" (also accepts "0 domain").
	FormatHosts Format = "hosts"
	// FormatAdblock is the Adblock Plus / EasyList syntax.
	FormatAdblock Format = "adblock"
	// FormatPlain is one domain per line.
	FormatPlain Format = "plain"
	// FormatRPZ is an RFC 1035 response policy zone.
	FormatRPZ Format = "rpz"
)

// ParseOptions tunes the parsers.
type ParseOptions struct {
	// Subdomains makes a bare domain also cover its subdomains. Defaults to
	// true; hosts files rely on it, and precise lists can opt out.
	Subdomains bool
	// DefaultTTL is applied to rules that carry none.
	DefaultTTL uint32
	// Categories are attached to every rule from this source.
	Categories []string
}

// ParseStats reports what a parser saw, for the UI and for tests.
type ParseStats struct {
	Rules      int      `json:"rules"`
	Comments   int      `json:"comments"`
	Skipped    int      `json:"skipped"`
	Malformed  int      `json:"malformed"`
	Duplicates int      `json:"duplicates"`
	Bytes      int64    `json:"bytes"`
	DurationMS float64  `json:"duration_ms"`
	Format     Format   `json:"format"`
	Sample     []string `json:"sample,omitempty"`
	Notices    []string `json:"notices,omitempty"`
}

// DetectFormat sniffs a list's syntax from its first meaningful lines.
func DetectFormat(data []byte) Format {
	sc := bufio.NewScanner(bytes.NewReader(stripBOM(data)))
	sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
	sawRecord := false
	for i := 0; sc.Scan() && i < 200; i++ {
		line := strings.TrimSpace(sc.Text())
		if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") {
			continue
		}
		if strings.HasPrefix(line, "[") {
			// "[Adblock Plus 2.0]" is a header, keep looking.
			continue
		}
		if strings.HasPrefix(line, "$TTL") || strings.HasPrefix(line, "$ORIGIN") ||
			strings.HasPrefix(line, "$INCLUDE") {
			return FormatRPZ
		}
		if sawRecord {
			return FormatAdblock
		}
		fields := strings.Fields(line)
		if len(fields) >= 2 && isAddr(fields[0]) {
			return FormatHosts
		}
		if strings.HasPrefix(line, "||") || strings.HasPrefix(line, "|") ||
			strings.HasPrefix(line, "@@") || strings.Contains(line, "##") ||
			strings.Contains(line, "#@#") || strings.Contains(line, "#?#") {
			return FormatAdblock
		}
		if len(fields) >= 3 && len(fields[1]) > 1 && allDigitsOrName(fields[1]) {
			// "name 60 IN A 1.2.3.4" style.
			return FormatRPZ
		}
		sawRecord = true
	}
	return FormatPlain
}

func allDigitsOrName(s string) bool {
	if _, err := strconv.Atoi(s); err == nil {
		return true
	}
	switch strings.ToUpper(s) {
	case "IN", "CH", "HS":
		return true
	}
	return false
}

// utf8BOM is the byte-order mark a UTF-8 file may start with. Editors on
// Windows add it, and it would otherwise glue itself to the first rule's
// owner name, silently dropping the first entry of the list.
var utf8BOM = []byte{0xef, 0xbb, 0xbf}

// stripBOM drops a leading UTF-8 byte-order mark. It is applied once, at the
// front door of the parse path, so every parser sees the same bytes.
func stripBOM(data []byte) []byte {
	return bytes.TrimPrefix(data, utf8BOM)
}

// Parse converts list data into rules using the given format (FormatAuto
// sniffs). It never returns partial garbage: on a hard error the rules
// accumulated so far are returned alongside it so the caller can decide
// whether to keep the last good set or adopt the partial one.
func Parse(format Format, data []byte, opt ParseOptions) ([]Rule, ParseStats, error) {
	data = stripBOM(data)
	if opt.DefaultTTL == 0 {
		opt.DefaultTTL = 60
	}
	if format == "" || format == FormatAuto {
		format = DetectFormat(data)
	}
	stats := ParseStats{Bytes: int64(len(data)), Format: format}
	var (
		rules []Rule
		err   error
	)
	switch format {
	case FormatHosts:
		rules, stats, err = parseHosts(data, opt)
	case FormatAdblock:
		rules, stats, err = parseAdblock(data, opt)
	case FormatPlain:
		rules, stats, err = parsePlain(data, opt)
	case FormatRPZ:
		rules, stats, err = parseRPZ(data, opt)
	default:
		return nil, stats, fmt.Errorf("blocklist: unknown format %q", format)
	}
	return rules, stats, err
}

func parseHosts(data []byte, opt ParseOptions) ([]Rule, ParseStats, error) {
	var rules []Rule
	stats := ParseStats{Bytes: int64(len(data)), Format: FormatHosts}
	sc := newLineScanner(data)
	for sc.Scan() {
		line := sc.Text()
		trimmed := strings.TrimSpace(line)
		if trimmed == "" {
			continue
		}
		if trimmed[0] == '#' || trimmed[0] == ';' {
			stats.Comments++
			continue
		}
		// Strip inline comments.
		if i := strings.IndexByte(trimmed, '#'); i >= 0 {
			trimmed = strings.TrimSpace(trimmed[:i])
			if trimmed == "" {
				stats.Comments++
				continue
			}
		}
		fields := strings.Fields(trimmed)
		if len(fields) < 2 {
			stats.Malformed++
			continue
		}
		addr := fields[0]
		if !isAddr(addr) && !strings.EqualFold(addr, "localhost") {
			// Not an address: this is not a hosts file after all, so treat
			// the whole line as a domain list. The notice has to travel with
			// the stats the plain parser returns, or the operator sees the
			// format switch with no explanation for it.
			const notice = "no address in first field, falling back to a plain domain list"
			rules, pstats, perr := parsePlain(data, opt)
			pstats.Notices = appendNotice(pstats.Notices, notice)
			return rules, pstats, perr
		}
		names := fields[1:]
		if isLoopbackName(names) {
			// "127.0.0.1 localhost" style bootstrap lines: never block these.
			stats.Skipped++
			continue
		}
		for _, n := range names {
			if !validDomain(n) {
				stats.Malformed++
				continue
			}
			rules = append(rules, hostsRule(n, opt))
		}
	}
	if err := sc.Err(); err != nil {
		return rules, stats, err
	}
	stats.Rules = len(rules)
	return rules, stats, nil
}

func isLoopbackName(names []string) bool {
	for _, n := range names {
		switch strings.ToLower(strings.TrimSuffix(n, ".")) {
		case "localhost", "localhost.localdomain", "broadcasthost", "ip6-localhost", "ip6-loopback":
			return true
		}
	}
	return false
}

// hostsRule builds a rule for a hosts-file or plain-list entry.
//
// A hosts file says "this name is bad" and everyone means the name *and*
// anything under it, so the pattern is "**." (apex included) rather than the
// strict-subdomain "*.". Getting this wrong is how a filter ends up blocking
// ads.example.com but happily resolving 1.ads.example.com.
func hostsRule(name string, opt ParseOptions) Rule {
	pattern := name
	if opt.Subdomains {
		pattern = "**." + name
	}
	return Rule{Name: pattern, Action: ActionBlock, TTL: opt.DefaultTTL}
}

func parseAdblock(data []byte, opt ParseOptions) ([]Rule, ParseStats, error) {
	var rules []Rule
	stats := ParseStats{Bytes: int64(len(data)), Format: FormatAdblock}
	sc := newLineScanner(data)
	for sc.Scan() {
		line := strings.TrimSpace(sc.Text())
		if line == "" {
			continue
		}
		if line[0] == '!' || line[0] == '#' {
			stats.Comments++
			continue
		}
		if line[0] == '[' {
			stats.Comments++
			continue
		}
		if strings.HasPrefix(line, "||") {
			host := hostOfAdblock(line[2:])
			if !validDomain(host) {
				stats.Skipped++
				continue
			}
			rules = append(rules, Rule{Name: "**." + host, Action: ActionBlock, TTL: opt.DefaultTTL})
			continue
		}
		if strings.HasPrefix(line, "@@") {
			rest := strings.TrimPrefix(line, "@@")
			host := hostOfAdblock(strings.TrimPrefix(rest, "||"))
			if !validDomain(host) {
				stats.Skipped++
				continue
			}
			rules = append(rules, Rule{Name: "**." + host, Action: ActionAllow, Priority: 10, TTL: opt.DefaultTTL})
			continue
		}
		if strings.HasPrefix(line, "|") {
			rest := strings.TrimPrefix(line, "|")
			if strings.HasPrefix(rest, "http://") || strings.HasPrefix(rest, "https://") {
				host := hostOfURL(rest)
				if !validDomain(host) {
					stats.Skipped++
					continue
				}
				// A path rule cannot be expressed in DNS; blocking the host
				// is the closest faithful approximation, so record it once.
				rules = append(rules, Rule{Name: "**." + host, Action: ActionBlock, TTL: opt.DefaultTTL})
				continue
			}
			stats.Skipped++
			continue
		}
		if line[0] == '/' && strings.HasSuffix(line, "/") {
			stats.Skipped++ // regular expressions are not DNS-expressible
			if len(stats.Notices) < 5 {
				stats.Notices = appendNotice(stats.Notices, "skipped regular-expression rules")
			}
			continue
		}
		// Element-hiding rules ("##", "#@#", "#?#", "##+js(") carry no DNS
		// meaning at all.
		if strings.Contains(line, "##") || strings.Contains(line, "#@#") || strings.Contains(line, "#?#") {
			stats.Skipped++
			continue
		}
		// A bare hostname with no adblock syntax at all.
		if validDomain(line) {
			rules = append(rules, hostsRule(line, opt))
			continue
		}
		stats.Skipped++
	}
	if err := sc.Err(); err != nil {
		return rules, stats, err
	}
	stats.Rules = len(rules)
	return rules, stats, nil
}

// hostOfAdblock extracts the domain from the part of an adblock rule between
// "||" (or a scheme) and the first separator (^, /, ?, |, $).
func hostOfAdblock(s string) string {
	if i := strings.IndexAny(s, "^/?|$"); i >= 0 {
		s = s[:i]
	}
	return strings.Trim(s, ".")
}

func hostOfURL(s string) string {
	if i := strings.Index(s, "://"); i >= 0 {
		s = s[i+3:]
	}
	if i := strings.IndexAny(s, "/?#"); i >= 0 {
		s = s[:i]
	}
	if i := strings.LastIndexByte(s, '@'); i >= 0 {
		s = s[i+1:]
	}
	// Strip credentials and port.
	if i := strings.LastIndexByte(s, ':'); i >= 0 {
		if _, err := strconv.Atoi(s[i+1:]); err == nil {
			s = s[:i]
		}
	}
	return strings.Trim(s, ".")
}

func parsePlain(data []byte, opt ParseOptions) ([]Rule, ParseStats, error) {
	var rules []Rule
	stats := ParseStats{Bytes: int64(len(data)), Format: FormatPlain}
	sc := newLineScanner(data)
	for sc.Scan() {
		line := strings.TrimSpace(sc.Text())
		if line == "" {
			continue
		}
		if line[0] == '#' || line[0] == ';' {
			stats.Comments++
			continue
		}
		// Accept "0.0.0.0 domain" and bare "domain" alike so operators do
		// not have to normalise their sources by hand.
		if fields := strings.Fields(line); len(fields) > 1 && isAddr(fields[0]) {
			for _, n := range fields[1:] {
				if validDomain(n) {
					rules = append(rules, hostsRule(n, opt))
				} else {
					stats.Malformed++
				}
			}
			continue
		}
		// Support inline allow markers: the documented "!allow example.com"
		// and the bare "!example.com", plus the adblock exception form
		// "@@||example.com^" operators paste into plain lists too.
		allow := false
		if strings.HasPrefix(line, "@@") {
			allow = true
			line = strings.TrimPrefix(line, "@@")
		} else if strings.HasPrefix(line, "!") {
			allow = true
			line = trimAllowWord(strings.TrimSpace(line[1:]))
		}
		// A plain list often carries a few adblock lines; drop the marker and
		// everything from the first adblock separator on, so they survive
		// instead of being counted as malformed.
		line = cutAdblockTail(strings.TrimPrefix(line, "||"))
		if !validDomain(line) {
			stats.Malformed++
			continue
		}
		pattern := line
		if opt.Subdomains && !strings.HasPrefix(line, "*") {
			pattern = "**." + line
		}
		action := ActionBlock
		if allow {
			action = ActionAllow
		}
		rules = append(rules, Rule{Name: pattern, Action: action, TTL: opt.DefaultTTL})
	}
	if err := sc.Err(); err != nil {
		return rules, stats, err
	}
	stats.Rules = len(rules)
	return rules, stats, nil
}

// cutAdblockTail drops everything from the first adblock separator onwards, so
// a plain list that someone pasted adblock lines into still yields the domain
// instead of a name with "^$|*//" decoration glued to it. A leading "*." is a
// wildcard prefix rather than a separator, and a lone "$" separator is only
// decoration when something follows it.
func cutAdblockTail(s string) string {
	for i := 0; i < len(s); i++ {
		switch s[i] {
		case '^', '|', '/':
			return s[:i]
		case '$':
			if i+1 < len(s) {
				return s[:i]
			}
		case '*':
			if i > 0 {
				return s[:i]
			}
		}
	}
	return s
}

// trimAllowWord drops a leading allow keyword that is separated from the
// domain by whitespace, so the documented "!allow example.com" and the bare
// "!example.com" mean the same thing, while a domain that merely starts with
// the word (allow.example.com) is left alone.
func trimAllowWord(s string) string {
	fields := strings.Fields(s)
	if len(fields) < 2 {
		return s
	}
	switch strings.ToLower(fields[0]) {
	case "allow", "allowlist", "whitelist":
		return strings.TrimSpace(s[len(fields[0]):])
	}
	return s
}

func parseRPZ(data []byte, opt ParseOptions) ([]Rule, ParseStats, error) {
	var rules []Rule
	stats := ParseStats{Bytes: int64(len(data)), Format: FormatRPZ}
	z, err := zonefile.Parse(bytes.NewReader(data), zonefile.Options{DefaultTTL: opt.DefaultTTL})
	if err != nil {
		return nil, stats, err
	}
	// A record line with no TTL column comes back from the zone reader
	// carrying its own placeholder, which would shadow the TTL the caller
	// asked for, so the zone's own default wins for those records.
	defaultTTL := z.DefaultTTL
	if defaultTTL == 0 {
		defaultTTL = opt.DefaultTTL
	}
	seen := make(map[string]bool, len(z.Records))
	for _, rec := range z.Records {
		ttl := rec.TTL
		if ttl == 0 || (ttl == implicitRRTTL && defaultTTL != implicitRRTTL) {
			ttl = defaultTTL
		}
		rule, ok := rpzRule(rec, ttl, opt)
		if !ok {
			stats.Skipped++
			continue
		}
		if seen[rule.Name] {
			stats.Duplicates++
			continue
		}
		seen[rule.Name] = true
		rules = append(rules, rule)
	}
	stats.Rules = len(rules)
	return rules, stats, nil
}

// implicitRRTTL is the TTL the zone reader stamps on a record whose line
// carries no TTL column. It is a placeholder rather than the file's intent,
// which is why parseRPZ replaces it with the zone's own default.
const implicitRRTTL = 3600

// rpzRule maps one policy-zone record onto a rule (RFC draft-ietf-dnsop-rpz).
// RPZ encodes the response in the rdata: a CNAME to "." means NXDOMAIN, to
// "rpz-drop." means drop, to "rpz-passthru." means resolve normally, and
// A/AAAA records return that address verbatim.
func rpzRule(rec zonefile.Record, ttl uint32, opt ParseOptions) (Rule, bool) {
	pattern := rec.Name
	target, action, resp := rpzTarget(rec.RR)
	if target == "" {
		return Rule{}, false
	}
	if !strings.HasPrefix(pattern, "*") && opt.Subdomains {
		pattern = "*." + pattern
	}
	return Rule{Name: pattern, Action: action, Response: resp, TTL: ttl}, true
}

// rpzTarget interprets a policy record's rdata. The returned target is only a
// "is this actionable" flag; action and resp carry the meaning.
func rpzTarget(rr dns.RR) (target string, action Action, resp string) {
	switch v := rr.(type) {
	case *dns.CNAME:
		t := v.Target
		switch {
		case t == ".":
			// The classic RPZ "this name does not exist" trigger.
			return ".", ActionNXDomain, ""
		case strings.EqualFold(t, "rpz-drop."):
			return t, ActionDrop, ""
		case strings.EqualFold(t, "rpz-passthru."), strings.EqualFold(t, "rpz-passthru."):
			return t, ActionPassthrough, ""
		case strings.EqualFold(t, "rpz-tcp-only."):
			// Force TCP upstream; as a DNS policy this is a no-op for us
			// since we already retry over TCP on truncation.
			return t, ActionPassthrough, ""
		default:
			return t, ActionCNAME, t
		}
	case *dns.A:
		if v.A == nil {
			return "", ActionBlock, ""
		}
		return v.A.String(), ActionRedirect, v.A.String()
	case *dns.AAAA:
		if v.AAAA == nil {
			return "", ActionBlock, ""
		}
		return v.AAAA.String(), ActionRedirect, v.AAAA.String()
	case *dns.TXT:
		// Some providers publish policy in TXT records; the payload is the
		// action name.
		t := strings.ToLower(strings.Join(v.Txt, ""))
		switch {
		case strings.Contains(t, "rpz-drop"):
			return t, ActionDrop, ""
		case strings.Contains(t, "rpz-passthru"):
			return t, ActionPassthrough, ""
		case strings.Contains(t, "allow"), strings.Contains(t, "whitelist"):
			return t, ActionAllow, ""
		default:
			return "", ActionBlock, ""
		}
	default:
		return "", ActionBlock, ""
	}
}

// isAddr reports whether s parses as an IP address literal.
func isAddr(s string) bool {
	_, err := netip.ParseAddr(s)
	return err == nil
}

// newLineScanner returns a line scanner sized for the very large list files
// real blocklists ship (multi-hundred-megabyte entries are not unheard of).
func newLineScanner(data []byte) *bufio.Scanner {
	sc := bufio.NewScanner(bytes.NewReader(data))
	sc.Buffer(make([]byte, 0, 128*1024), 4*1024*1024)
	return sc
}

func appendNotice(notices []string, n string) []string {
	for _, existing := range notices {
		if existing == n {
			return notices
		}
	}
	if len(notices) >= 8 {
		return notices
	}
	return append(notices, n)
}

// validDomain is a cheap sanity check: no scheme, no path, printable, and at
// least one dot or a known single-label TLD-ish name.
func validDomain(s string) bool {
	if s == "" || len(s) > 253 {
		return false
	}
	if strings.HasPrefix(s, ".") || strings.HasPrefix(s, "-") {
		return false
	}
	if strings.ContainsAny(s, "/ \t\\\"'<>|?*=") {
		return false
	}
	labels := strings.Split(strings.TrimSuffix(s, "."), ".")
	if len(labels) < 1 {
		return false
	}
	for _, l := range labels {
		if l == "" || len(l) > 63 {
			return false
		}
		for i := 0; i < len(l); i++ {
			c := l[i]
			ok := c == '-' || c == '_' || c == '*' ||
				(c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
				(c >= '0' && c <= '9')
			if !ok {
				return false
			}
		}
		if l[0] == '-' || l[len(l)-1] == '-' {
			return false
		}
	}
	// A bare label with no dot is only meaningful for the localhost-ish
	// names, which callers already filter; reject to avoid blocking a
	// single-label intranet name by accident.
	return strings.Contains(s, ".")
}