initial commit
This commit is contained in:
developer committed
1 parent
5b935301e5
commit
85453852d5
105 files changed
+24846
No files matched your search
@@ -0,0 +1,535 @@
|
||||
package blocklist
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
|
||||
"git.k8s-git-k6n.net/mats/fancy-dns/internal/zonefile"
|
||||
)
|
||||
|
||||
// Format identifies a list's syntax.
|
||||
type Format string
|
||||
|
||||
const (
|
||||
// FormatAuto sniffs the content.
|
||||
FormatAuto Format = "auto"
|
||||
// FormatHosts is "IP domain [domain...]" (also accepts "0 domain").
|
||||
FormatHosts Format = "hosts"
|
||||
// FormatAdblock is the Adblock Plus / EasyList syntax.
|
||||
FormatAdblock Format = "adblock"
|
||||
// FormatPlain is one domain per line.
|
||||
FormatPlain Format = "plain"
|
||||
// FormatRPZ is an RFC 1035 response policy zone.
|
||||
FormatRPZ Format = "rpz"
|
||||
)
|
||||
|
||||
// ParseOptions tunes the parsers.
|
||||
type ParseOptions struct {
|
||||
// Subdomains makes a bare domain also cover its subdomains. Defaults to
|
||||
// true; hosts files rely on it, and precise lists can opt out.
|
||||
Subdomains bool
|
||||
// DefaultTTL is applied to rules that carry none.
|
||||
DefaultTTL uint32
|
||||
// Categories are attached to every rule from this source.
|
||||
Categories []string
|
||||
}
|
||||
|
||||
// ParseStats reports what a parser saw, for the UI and for tests.
|
||||
type ParseStats struct {
|
||||
Rules int `json:"rules"`
|
||||
Comments int `json:"comments"`
|
||||
Skipped int `json:"skipped"`
|
||||
Malformed int `json:"malformed"`
|
||||
Duplicates int `json:"duplicates"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
DurationMS float64 `json:"duration_ms"`
|
||||
Format Format `json:"format"`
|
||||
Sample []string `json:"sample,omitempty"`
|
||||
Notices []string `json:"notices,omitempty"`
|
||||
}
|
||||
|
||||
// DetectFormat sniffs a list's syntax from its first meaningful lines.
|
||||
func DetectFormat(data []byte) Format {
|
||||
sc := bufio.NewScanner(bytes.NewReader(data))
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
sawRecord := false
|
||||
for i := 0; sc.Scan() && i < 200; i++ {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "[") {
|
||||
// "[Adblock Plus 2.0]" is a header, keep looking.
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "$TTL") || strings.HasPrefix(line, "$ORIGIN") ||
|
||||
strings.HasPrefix(line, "$INCLUDE") {
|
||||
return FormatRPZ
|
||||
}
|
||||
if sawRecord {
|
||||
return FormatAdblock
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) >= 2 && isAddr(fields[0]) {
|
||||
return FormatHosts
|
||||
}
|
||||
if strings.HasPrefix(line, "||") || strings.HasPrefix(line, "|") ||
|
||||
strings.HasPrefix(line, "@@") || strings.Contains(line, "##") ||
|
||||
strings.Contains(line, "#@#") || strings.Contains(line, "#?#") {
|
||||
return FormatAdblock
|
||||
}
|
||||
if len(fields) >= 3 && len(fields[1]) > 1 && allDigitsOrName(fields[1]) {
|
||||
// "name 60 IN A 1.2.3.4" style.
|
||||
return FormatRPZ
|
||||
}
|
||||
sawRecord = true
|
||||
}
|
||||
return FormatPlain
|
||||
}
|
||||
|
||||
func allDigitsOrName(s string) bool {
|
||||
if _, err := strconv.Atoi(s); err == nil {
|
||||
return true
|
||||
}
|
||||
switch strings.ToUpper(s) {
|
||||
case "IN", "CH", "HS":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Parse converts list data into rules using the given format (FormatAuto
|
||||
// sniffs). It never returns partial garbage: on a hard error the rules
|
||||
// accumulated so far are returned alongside it so the caller can decide
|
||||
// whether to keep the last good set or adopt the partial one.
|
||||
func Parse(format Format, data []byte, opt ParseOptions) ([]Rule, ParseStats, error) {
|
||||
if opt.DefaultTTL == 0 {
|
||||
opt.DefaultTTL = 60
|
||||
}
|
||||
if format == "" || format == FormatAuto {
|
||||
format = DetectFormat(data)
|
||||
}
|
||||
stats := ParseStats{Bytes: int64(len(data)), Format: format}
|
||||
var (
|
||||
rules []Rule
|
||||
err error
|
||||
)
|
||||
switch format {
|
||||
case FormatHosts:
|
||||
rules, stats, err = parseHosts(data, opt)
|
||||
case FormatAdblock:
|
||||
rules, stats, err = parseAdblock(data, opt)
|
||||
case FormatPlain:
|
||||
rules, stats, err = parsePlain(data, opt)
|
||||
case FormatRPZ:
|
||||
rules, stats, err = parseRPZ(data, opt)
|
||||
default:
|
||||
return nil, stats, fmt.Errorf("blocklist: unknown format %q", format)
|
||||
}
|
||||
return rules, stats, err
|
||||
}
|
||||
|
||||
func parseHosts(data []byte, opt ParseOptions) ([]Rule, ParseStats, error) {
|
||||
var rules []Rule
|
||||
stats := ParseStats{Bytes: int64(len(data)), Format: FormatHosts}
|
||||
sc := newLineScanner(data)
|
||||
for sc.Scan() {
|
||||
line := sc.Text()
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
if trimmed[0] == '#' || trimmed[0] == ';' {
|
||||
stats.Comments++
|
||||
continue
|
||||
}
|
||||
// Strip inline comments.
|
||||
if i := strings.IndexByte(trimmed, '#'); i >= 0 {
|
||||
trimmed = strings.TrimSpace(trimmed[:i])
|
||||
if trimmed == "" {
|
||||
stats.Comments++
|
||||
continue
|
||||
}
|
||||
}
|
||||
fields := strings.Fields(trimmed)
|
||||
if len(fields) < 2 {
|
||||
stats.Malformed++
|
||||
continue
|
||||
}
|
||||
addr := fields[0]
|
||||
if !isAddr(addr) && !strings.EqualFold(addr, "localhost") {
|
||||
// Not an address: this is not a hosts file after all, so treat
|
||||
// the whole line as a domain list.
|
||||
stats.Notices = appendNotice(stats.Notices, "no address in first field, falling back to a plain domain list")
|
||||
return parsePlain(data, opt)
|
||||
}
|
||||
names := fields[1:]
|
||||
if isLoopbackName(names) {
|
||||
// "127.0.0.1 localhost" style bootstrap lines: never block these.
|
||||
stats.Skipped++
|
||||
continue
|
||||
}
|
||||
for _, n := range names {
|
||||
if !validDomain(n) {
|
||||
stats.Malformed++
|
||||
continue
|
||||
}
|
||||
rules = append(rules, hostsRule(n, opt))
|
||||
}
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return rules, stats, err
|
||||
}
|
||||
stats.Rules = len(rules)
|
||||
return rules, stats, nil
|
||||
}
|
||||
|
||||
func isLoopbackName(names []string) bool {
|
||||
for _, n := range names {
|
||||
switch strings.ToLower(strings.TrimSuffix(n, ".")) {
|
||||
case "localhost", "localhost.localdomain", "broadcasthost", "ip6-localhost", "ip6-loopback":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// hostsRule builds a rule for a hosts-file or plain-list entry.
|
||||
//
|
||||
// A hosts file says "this name is bad" and everyone means the name *and*
|
||||
// anything under it, so the pattern is "**." (apex included) rather than the
|
||||
// strict-subdomain "*.". Getting this wrong is how a filter ends up blocking
|
||||
// ads.example.com but happily resolving 1.ads.example.com.
|
||||
func hostsRule(name string, opt ParseOptions) Rule {
|
||||
pattern := name
|
||||
if opt.Subdomains {
|
||||
pattern = "**." + name
|
||||
}
|
||||
return Rule{Name: pattern, Action: ActionBlock, TTL: opt.DefaultTTL}
|
||||
}
|
||||
|
||||
func parseAdblock(data []byte, opt ParseOptions) ([]Rule, ParseStats, error) {
|
||||
var rules []Rule
|
||||
stats := ParseStats{Bytes: int64(len(data)), Format: FormatAdblock}
|
||||
sc := newLineScanner(data)
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if line[0] == '!' || line[0] == '#' {
|
||||
stats.Comments++
|
||||
continue
|
||||
}
|
||||
if line[0] == '[' {
|
||||
stats.Comments++
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "||") {
|
||||
host := hostOfAdblock(line[2:])
|
||||
if !validDomain(host) {
|
||||
stats.Skipped++
|
||||
continue
|
||||
}
|
||||
rules = append(rules, Rule{Name: "**." + host, Action: ActionBlock, TTL: opt.DefaultTTL})
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "@@") {
|
||||
rest := strings.TrimPrefix(line, "@@")
|
||||
host := hostOfAdblock(strings.TrimPrefix(rest, "||"))
|
||||
if !validDomain(host) {
|
||||
stats.Skipped++
|
||||
continue
|
||||
}
|
||||
rules = append(rules, Rule{Name: "**." + host, Action: ActionAllow, Priority: 10, TTL: opt.DefaultTTL})
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "|") {
|
||||
rest := strings.TrimPrefix(line, "|")
|
||||
if strings.HasPrefix(rest, "http://") || strings.HasPrefix(rest, "https://") {
|
||||
host := hostOfURL(rest)
|
||||
if !validDomain(host) {
|
||||
stats.Skipped++
|
||||
continue
|
||||
}
|
||||
// A path rule cannot be expressed in DNS; blocking the host
|
||||
// is the closest faithful approximation, so record it once.
|
||||
rules = append(rules, Rule{Name: "**." + host, Action: ActionBlock, TTL: opt.DefaultTTL})
|
||||
continue
|
||||
}
|
||||
stats.Skipped++
|
||||
continue
|
||||
}
|
||||
if line[0] == '/' && strings.HasSuffix(line, "/") {
|
||||
stats.Skipped++ // regular expressions are not DNS-expressible
|
||||
if len(stats.Notices) < 5 {
|
||||
stats.Notices = appendNotice(stats.Notices, "skipped regular-expression rules")
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Element-hiding rules ("##", "#@#", "#?#", "##+js(") carry no DNS
|
||||
// meaning at all.
|
||||
if strings.Contains(line, "##") || strings.Contains(line, "#@#") || strings.Contains(line, "#?#") {
|
||||
stats.Skipped++
|
||||
continue
|
||||
}
|
||||
// A bare hostname with no adblock syntax at all.
|
||||
if validDomain(line) {
|
||||
rules = append(rules, hostsRule(line, opt))
|
||||
continue
|
||||
}
|
||||
stats.Skipped++
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return rules, stats, err
|
||||
}
|
||||
stats.Rules = len(rules)
|
||||
return rules, stats, nil
|
||||
}
|
||||
|
||||
// hostOfAdblock extracts the domain from the part of an adblock rule between
|
||||
// "||" (or a scheme) and the first separator (^, /, ?, |, $).
|
||||
func hostOfAdblock(s string) string {
|
||||
if i := strings.IndexAny(s, "^/?|$"); i >= 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
return strings.Trim(s, ".")
|
||||
}
|
||||
|
||||
func hostOfURL(s string) string {
|
||||
if i := strings.Index(s, "://"); i >= 0 {
|
||||
s = s[i+3:]
|
||||
}
|
||||
if i := strings.IndexAny(s, "/?#"); i >= 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
if i := strings.LastIndexByte(s, '@'); i >= 0 {
|
||||
s = s[i+1:]
|
||||
}
|
||||
// Strip credentials and port.
|
||||
if i := strings.LastIndexByte(s, ':'); i >= 0 {
|
||||
if _, err := strconv.Atoi(s[i+1:]); err == nil {
|
||||
s = s[:i]
|
||||
}
|
||||
}
|
||||
return strings.Trim(s, ".")
|
||||
}
|
||||
|
||||
func parsePlain(data []byte, opt ParseOptions) ([]Rule, ParseStats, error) {
|
||||
var rules []Rule
|
||||
stats := ParseStats{Bytes: int64(len(data)), Format: FormatPlain}
|
||||
sc := newLineScanner(data)
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if line[0] == '#' || line[0] == ';' {
|
||||
stats.Comments++
|
||||
continue
|
||||
}
|
||||
// Accept "0.0.0.0 domain" and bare "domain" alike so operators do
|
||||
// not have to normalise their sources by hand.
|
||||
if fields := strings.Fields(line); len(fields) > 1 && isAddr(fields[0]) {
|
||||
for _, n := range fields[1:] {
|
||||
if validDomain(n) {
|
||||
rules = append(rules, hostsRule(n, opt))
|
||||
} else {
|
||||
stats.Malformed++
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Support inline allow markers: "!allow example.com".
|
||||
allow := false
|
||||
if strings.HasPrefix(line, "!") {
|
||||
allow = true
|
||||
line = strings.TrimSpace(line[1:])
|
||||
}
|
||||
line = strings.TrimPrefix(line, "||")
|
||||
if !validDomain(line) {
|
||||
stats.Malformed++
|
||||
continue
|
||||
}
|
||||
pattern := line
|
||||
if opt.Subdomains && !strings.HasPrefix(line, "*") {
|
||||
pattern = "**." + line
|
||||
}
|
||||
action := ActionBlock
|
||||
if allow {
|
||||
action = ActionAllow
|
||||
}
|
||||
rules = append(rules, Rule{Name: pattern, Action: action, TTL: opt.DefaultTTL})
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return rules, stats, err
|
||||
}
|
||||
stats.Rules = len(rules)
|
||||
return rules, stats, nil
|
||||
}
|
||||
|
||||
func parseRPZ(data []byte, opt ParseOptions) ([]Rule, ParseStats, error) {
|
||||
var rules []Rule
|
||||
stats := ParseStats{Bytes: int64(len(data)), Format: FormatRPZ}
|
||||
z, err := zonefile.Parse(bytes.NewReader(data), zonefile.Options{DefaultTTL: opt.DefaultTTL})
|
||||
if err != nil {
|
||||
return nil, stats, err
|
||||
}
|
||||
seen := make(map[string]bool, len(z.Records))
|
||||
for _, rec := range z.Records {
|
||||
ttl := rec.TTL
|
||||
if ttl == 0 && opt.DefaultTTL == 0 {
|
||||
ttl = opt.DefaultTTL
|
||||
}
|
||||
rule, ok := rpzRule(rec, ttl, opt)
|
||||
if !ok {
|
||||
stats.Skipped++
|
||||
continue
|
||||
}
|
||||
if seen[rule.Name] {
|
||||
stats.Duplicates++
|
||||
continue
|
||||
}
|
||||
seen[rule.Name] = true
|
||||
rules = append(rules, rule)
|
||||
}
|
||||
stats.Rules = len(rules)
|
||||
return rules, stats, nil
|
||||
}
|
||||
|
||||
// rpzRule maps one policy-zone record onto a rule (RFC draft-ietf-dnsop-rpz).
|
||||
// RPZ encodes the response in the rdata: a CNAME to "." means NXDOMAIN, to
|
||||
// "rpz-drop." means drop, to "rpz-passthru." means resolve normally, and
|
||||
// A/AAAA records return that address verbatim.
|
||||
func rpzRule(rec zonefile.Record, ttl uint32, opt ParseOptions) (Rule, bool) {
|
||||
pattern := rec.Name
|
||||
if !strings.HasPrefix(pattern, "*") && opt.Subdomains {
|
||||
pattern = "*." + pattern
|
||||
}
|
||||
target, action, resp := rpzTarget(rec.RR)
|
||||
if target == "" {
|
||||
return Rule{}, false
|
||||
}
|
||||
return Rule{Name: pattern, Action: action, Response: resp, TTL: ttl}, true
|
||||
}
|
||||
|
||||
// rpzTarget interprets a policy record's rdata. The returned target is only a
|
||||
// "is this actionable" flag; action and resp carry the meaning.
|
||||
func rpzTarget(rr dns.RR) (target string, action Action, resp string) {
|
||||
switch v := rr.(type) {
|
||||
case *dns.CNAME:
|
||||
t := v.Target
|
||||
switch {
|
||||
case t == ".":
|
||||
// The classic RPZ "this name does not exist" trigger.
|
||||
return ".", ActionNXDomain, ""
|
||||
case strings.EqualFold(t, "rpz-drop."):
|
||||
return t, ActionDrop, ""
|
||||
case strings.EqualFold(t, "rpz-passthru."), strings.EqualFold(t, "rpz-passthru."):
|
||||
return t, ActionPassthrough, ""
|
||||
case strings.EqualFold(t, "rpz-tcp-only."):
|
||||
// Force TCP upstream; as a DNS policy this is a no-op for us
|
||||
// since we already retry over TCP on truncation.
|
||||
return t, ActionPassthrough, ""
|
||||
default:
|
||||
return t, ActionCNAME, t
|
||||
}
|
||||
case *dns.A:
|
||||
if v.A == nil {
|
||||
return "", ActionBlock, ""
|
||||
}
|
||||
return v.A.String(), ActionRedirect, v.A.String()
|
||||
case *dns.AAAA:
|
||||
if v.AAAA == nil {
|
||||
return "", ActionBlock, ""
|
||||
}
|
||||
return v.AAAA.String(), ActionRedirect, v.AAAA.String()
|
||||
case *dns.TXT:
|
||||
// Some providers publish policy in TXT records; the payload is the
|
||||
// action name.
|
||||
t := strings.ToLower(strings.Join(v.Txt, ""))
|
||||
switch {
|
||||
case strings.Contains(t, "rpz-drop"):
|
||||
return t, ActionDrop, ""
|
||||
case strings.Contains(t, "rpz-passthru"):
|
||||
return t, ActionPassthrough, ""
|
||||
case strings.Contains(t, "allow"), strings.Contains(t, "whitelist"):
|
||||
return t, ActionAllow, ""
|
||||
default:
|
||||
return "", ActionBlock, ""
|
||||
}
|
||||
default:
|
||||
return "", ActionBlock, ""
|
||||
}
|
||||
}
|
||||
|
||||
// isAddr reports whether s parses as an IP address literal.
|
||||
func isAddr(s string) bool {
|
||||
_, err := netip.ParseAddr(s)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// newLineScanner returns a line scanner sized for the very large list files
|
||||
// real blocklists ship (multi-hundred-megabyte entries are not unheard of).
|
||||
func newLineScanner(data []byte) *bufio.Scanner {
|
||||
sc := bufio.NewScanner(bytes.NewReader(data))
|
||||
sc.Buffer(make([]byte, 0, 128*1024), 4*1024*1024)
|
||||
return sc
|
||||
}
|
||||
|
||||
func appendNotice(notices []string, n string) []string {
|
||||
for _, existing := range notices {
|
||||
if existing == n {
|
||||
return notices
|
||||
}
|
||||
}
|
||||
if len(notices) >= 8 {
|
||||
return notices
|
||||
}
|
||||
return append(notices, n)
|
||||
}
|
||||
|
||||
// validDomain is a cheap sanity check: no scheme, no path, printable, and at
|
||||
// least one dot or a known single-label TLD-ish name.
|
||||
func validDomain(s string) bool {
|
||||
if s == "" || len(s) > 253 {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(s, ".") || strings.HasPrefix(s, "-") {
|
||||
return false
|
||||
}
|
||||
if strings.ContainsAny(s, "/ \t\\\"'<>|?*=") {
|
||||
return false
|
||||
}
|
||||
labels := strings.Split(strings.TrimSuffix(s, "."), ".")
|
||||
if len(labels) < 1 {
|
||||
return false
|
||||
}
|
||||
for _, l := range labels {
|
||||
if l == "" || len(l) > 63 {
|
||||
return false
|
||||
}
|
||||
for i := 0; i < len(l); i++ {
|
||||
c := l[i]
|
||||
ok := c == '-' || c == '_' || c == '*' ||
|
||||
(c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
|
||||
(c >= '0' && c <= '9')
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if l[0] == '-' || l[len(l)-1] == '-' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// A bare label with no dot is only meaningful for the localhost-ish
|
||||
// names, which callers already filter; reject to avoid blocking a
|
||||
// single-label intranet name by accident.
|
||||
return strings.Contains(s, ".")
|
||||
}
|
||||
Reference in new issue
Block a user