package blocklist import ( "strconv" "strings" "testing" ) // ruleStrings renders parsed rules compactly for table comparison. func ruleStrings(rules []Rule) []string { out := make([]string, 0, len(rules)) for _, r := range rules { s := r.Name + " " + r.Action.String() + " ttl=" + strconv.Itoa(int(r.TTL)) if r.Response != "" { s += " resp=" + r.Response } if r.Priority != 0 { s += " prio=" + strconv.Itoa(r.Priority) } out = append(out, s) } return out } // sameRules compares parsed rules against the expectation, treating "no rules" // as equal whether the parser returned a nil or an empty slice. func sameRules(got, want []string) bool { return strings.Join(got, "\n") == strings.Join(want, "\n") } func TestParseHosts(t *testing.T) { tests := []struct { name string data string opt ParseOptions want []string wantFormat Format wantComment int wantSkip int wantBad int wantNotice string }{ { name: "block entry", data: "0.0.0.0 ads.example.com\n", opt: ParseOptions{Subdomains: true}, want: []string{"**.ads.example.com block ttl=60"}, }, { name: "several names on one line", data: "0.0.0.0 a.example.com b.example.com\n", opt: ParseOptions{Subdomains: true}, want: []string{"**.a.example.com block ttl=60", "**.b.example.com block ttl=60"}, }, { name: "loopback bootstrap lines are never blocked", data: "127.0.0.1 localhost\n::1 ip6-localhost\n", opt: ParseOptions{Subdomains: true}, wantSkip: 2, }, { name: "comments, blank lines and inline comments", data: "# a comment\n; another\n\n \n0.0.0.0 ads.example.com # trailing note\n", opt: ParseOptions{Subdomains: true}, want: []string{"**.ads.example.com block ttl=60"}, wantComment: 2, }, { name: "custom TTL and no subdomain expansion", data: "0.0.0.0 ads.example.com\n", opt: ParseOptions{Subdomains: false, DefaultTTL: 300}, want: []string{"ads.example.com block ttl=300"}, }, { name: "a single field is not a hosts line", data: "ads.example.com\n", opt: ParseOptions{Subdomains: true}, wantBad: 1, }, { name: "a non-address first field falls back to a plain domain list", data: "0.0.0.0 ads.example.com\nfoo bar.example.com\n", opt: ParseOptions{Subdomains: true}, // The fallback re-parses the whole file as a plain list, so the // real hosts lines survive and the odd line is malformed. want: []string{"**.ads.example.com block ttl=60"}, wantFormat: FormatPlain, wantBad: 1, wantNotice: "no address in first field, falling back to a plain domain list", }, { name: "malformed lines", data: "0.0.0.0\n0.0.0.0 not_a_domain..x\n0.0.0.0 -bad.example.com\n", opt: ParseOptions{Subdomains: true}, wantBad: 3, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { rules, stats, err := Parse(FormatHosts, []byte(tc.data), tc.opt) if err != nil { t.Fatalf("Parse: %v", err) } if got := ruleStrings(rules); !sameRules(got, tc.want) { t.Errorf("rules = %v, want %v", got, tc.want) } wantFormat := tc.wantFormat if wantFormat == "" { wantFormat = FormatHosts } if stats.Format != wantFormat { t.Errorf("stats.Format = %q, want %q", stats.Format, wantFormat) } if stats.Comments != tc.wantComment { t.Errorf("Comments = %d, want %d", stats.Comments, tc.wantComment) } if stats.Skipped != tc.wantSkip { t.Errorf("Skipped = %d, want %d", stats.Skipped, tc.wantSkip) } if stats.Malformed != tc.wantBad { t.Errorf("Malformed = %d, want %d", stats.Malformed, tc.wantBad) } if stats.Rules != len(rules) { t.Errorf("Rules = %d, want %d", stats.Rules, len(rules)) } if stats.Bytes != int64(len(tc.data)) { t.Errorf("Bytes = %d, want %d", stats.Bytes, len(tc.data)) } if tc.wantNotice != "" { found := false for _, n := range stats.Notices { if n == tc.wantNotice { found = true } } if !found { t.Errorf("Notices = %v, want %q", stats.Notices, tc.wantNotice) } } }) } } func TestParseAdblock(t *testing.T) { data := strings.Join([]string{ "[Adblock Plus 2.0]", "! Title: test list", "! Version: 1", "||ads.example.com^", "||track.example.com^$third-party", "|http://banner.example.com/img.png|", "|https://user:pw@secure.example.com:8443/x?y#z|", "@@||ok.example.com^", "example.com##.ad", "banner.example.com#@#.sponsor", "/ad[0-9]+/", "/banner[0-9]+/", "||", "bare.example.com", }, "\n") rules, stats, err := Parse(FormatAdblock, []byte(data), ParseOptions{Subdomains: true, DefaultTTL: 60}) if err != nil { t.Fatalf("Parse: %v", err) } want := []string{ "**.ads.example.com block ttl=60", "**.track.example.com block ttl=60", "**.banner.example.com block ttl=60", "**.secure.example.com block ttl=60", "**.ok.example.com allow ttl=60 prio=10", "**.bare.example.com block ttl=60", } if got := ruleStrings(rules); !sameRules(got, want) { t.Errorf("rules = %v,\nwant %v", got, want) } if stats.Rules != len(want) { t.Errorf("Rules = %d, want %d", stats.Rules, len(want)) } if stats.Comments != 3 { // the header plus two "! " comments t.Errorf("Comments = %d, want 3", stats.Comments) } // 2 cosmetic rules, 2 regex rules, 1 empty "||" host. if stats.Skipped != 5 { t.Errorf("Skipped = %d, want 5", stats.Skipped) } foundNotice := false for _, n := range stats.Notices { if n == "skipped regular-expression rules" { foundNotice = true } } if !foundNotice { t.Errorf("Notices = %v, want the regex notice", stats.Notices) } if stats.Format != FormatAdblock { t.Errorf("Format = %q", stats.Format) } } func TestParsePlain(t *testing.T) { data := strings.Join([]string{ "# a comment", "; another comment", "", " spaced.example.com ", "!ok.example.com", "0.0.0.0 hoststyle.example.com", "||ads.example.com", "trailing.example.com.", "!not a domain line !!", "-bad.example.com", }, "\n") rules, stats, err := Parse(FormatPlain, []byte(data), ParseOptions{Subdomains: true}) if err != nil { t.Fatalf("Parse: %v", err) } want := []string{ "**.spaced.example.com block ttl=60", "**.ok.example.com allow ttl=60", "**.hoststyle.example.com block ttl=60", "**.ads.example.com block ttl=60", // A trailing dot survives parsing; Builder.Add strips it later. "**.trailing.example.com. block ttl=60", } if got := ruleStrings(rules); !sameRules(got, want) { t.Errorf("rules = %v,\nwant %v", got, want) } if stats.Comments != 2 { t.Errorf("Comments = %d, want 2", stats.Comments) } if stats.Malformed != 2 { t.Errorf("Malformed = %d, want 2", stats.Malformed) } // Without subdomain expansion the same list yields exact rules. exact, _, err := Parse(FormatPlain, []byte("spaced.example.com\n!ok.example.com\n"), ParseOptions{Subdomains: false}) if err != nil { t.Fatalf("Parse: %v", err) } wantExact := []string{"spaced.example.com block ttl=60", "ok.example.com allow ttl=60"} if got := ruleStrings(exact); !sameRules(got, wantExact) { t.Errorf("exact rules = %v, want %v", got, wantExact) } // The documented "!allow " spelling is accepted, so the keyword // does not end up glued to the name. allowForm, allowStats, err := Parse(FormatPlain, []byte("!allow ok.example.com\n"), ParseOptions{Subdomains: true}) if err != nil { t.Fatalf("Parse: %v", err) } if got := ruleStrings(allowForm); !sameRules(got, []string{"**.ok.example.com allow ttl=60"}) { t.Errorf("\"!allow ok.example.com\" rules = %v, malformed=%d", got, allowStats.Malformed) } // "!allow" with no name is still nothing at all. empty, emptyStats, err := Parse(FormatPlain, []byte("!allow\n"), ParseOptions{Subdomains: true}) if err != nil { t.Fatalf("Parse: %v", err) } if emptyStats.Rules != 0 || emptyStats.Malformed != 1 { t.Errorf("\"!allow\": rules=%d malformed=%d, want 0/1", emptyStats.Rules, emptyStats.Malformed) } if len(empty) != 0 { t.Errorf("\"!allow\" produced %v", ruleStrings(empty)) } } func TestParseRPZ(t *testing.T) { data := strings.Join([]string{ "$TTL 3600", "; an RPZ policy zone", "example.com CNAME .", "*.sub.example.org CNAME rpz-drop.", "**.both.example.net CNAME rpz-passthru.", "redirect.example.gov A 127.0.0.2", "redirect6.example.gov AAAA ::1", "example.com CNAME .", `skip.example.com TXT "not-a-known-action"`, }, "\n") rules, stats, err := Parse(FormatRPZ, []byte(data), ParseOptions{Subdomains: true, DefaultTTL: 60}) if err != nil { t.Fatalf("Parse: %v", err) } want := []string{ // A bare owner name becomes the STRICT-subdomain form "*." (RPZ // semantics: the policy covers what is under the name), unlike the // hosts parser which uses "**.". Zone owner names keep their // trailing dot; Builder.Add strips it at compile time. "*.example.com. nxdomain ttl=3600", "*.sub.example.org. drop ttl=3600", "**.both.example.net. passthrough ttl=3600", "*.redirect.example.gov. redirect ttl=3600 resp=127.0.0.2", "*.redirect6.example.gov. redirect ttl=3600 resp=::1", } if got := ruleStrings(rules); !sameRules(got, want) { t.Errorf("rules = %v,\nwant %v", got, want) } if stats.Duplicates != 1 { t.Errorf("Duplicates = %d, want 1 (the repeated example.com record)", stats.Duplicates) } if stats.Skipped != 1 { t.Errorf("Skipped = %d, want 1 (TXT with an unknown action)", stats.Skipped) } if stats.Format != FormatRPZ { t.Errorf("Format = %q, want rpz", stats.Format) } // Without subdomain expansion the wildcard prefix is left off entirely. plain, _, err := Parse(FormatRPZ, []byte("example.com CNAME .\n"), ParseOptions{Subdomains: false}) if err != nil { t.Fatalf("Parse: %v", err) } if got := ruleStrings(plain); !sameRules(got, []string{"example.com. nxdomain ttl=60"}) { t.Errorf("no-subdomain rules = %v", got) } // A CNAME to a real target is a rewrite rule and $TTL is honoured. cname, _, err := Parse(FormatRPZ, []byte("$TTL 60\nshop.example.com CNAME parked.example.net.\n"), ParseOptions{Subdomains: true}) if err != nil { t.Fatalf("Parse: %v", err) } if got := ruleStrings(cname); !sameRules(got, []string{"*.shop.example.com. cname ttl=60 resp=parked.example.net."}) { t.Errorf("cname rules = %v", got) } // The compiled forms behave as documented: "*." skips the apex while // "**." covers it. s := mustBuild(rules...) if e := s.Match("example.com"); e != nil { t.Errorf("RPZ \"example.com CNAME .\" matched the apex: %q, want nil (strict \"*.\")", e.Name) } if e := s.Match("www.example.com"); e == nil || e.Action != ActionNXDomain { t.Errorf("www.example.com: %+v, want nxdomain", e) } if e := s.Match("both.example.net"); e == nil || e.Action != ActionPassthrough { t.Errorf("both.example.net: %+v, want passthrough (\"**.\" covers the apex)", e) } if e := s.Match("deep.both.example.net"); e == nil || e.Action != ActionPassthrough { t.Errorf("deep.both.example.net: %+v, want passthrough", e) } } func TestDetectFormat(t *testing.T) { tests := []struct { name string data string want Format }{ {"hosts", "0.0.0.0 ads.example.com\n127.0.0.1 localhost\n", FormatHosts}, {"hosts after comments", "# comment\n\n 127.0.0.1 localhost\n", FormatHosts}, {"hosts with ::1", "::1 ip6-localhost\n", FormatHosts}, {"adblock double pipe", "||ads.example.com^\n", FormatAdblock}, {"adblock allow", "@@||ok.example.com^\n", FormatAdblock}, {"adblock cosmetic", "example.com##.ad\n", FormatAdblock}, {"adblock after header", "[Adblock Plus 2.0]\n! comment\n|http://a.example.com/x|\n", FormatAdblock}, {"rpz ttl", "$TTL 3600\nexample.com CNAME .\n", FormatRPZ}, {"rpz origin", "$ORIGIN example.com.\nexample.com CNAME .\n", FormatRPZ}, {"rpz record with ttl and class", "example.com 3600 IN CNAME .\n", FormatRPZ}, {"single plain domain", "ads.example.com\n", FormatPlain}, {"empty", "", FormatPlain}, {"only comments", "# nothing here\n; really\n", FormatPlain}, // A two-line plain list is sniffed as adblock: the second meaningful // line is taken to be an adblock rule. Both parsers turn a bare // hostname into a block rule, so the rules still come out right. {"two plain domains", "ads.example.com\ntrack.example.com\n", FormatAdblock}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { if got := DetectFormat([]byte(tc.data)); got != tc.want { t.Errorf("DetectFormat(%q) = %q, want %q", tc.data, got, tc.want) } }) } } func TestDetectFormatSuspectedBugs(t *testing.T) { // Real behaviour today, pinned so a future fix shows up as a failure. // allDigitsOrName() does not know RR type names, so a policy record // without an explicit TTL/class column is sniffed as a plain domain // list. Parsing the same content as RPZ works, which isolates the // detection step. if got := DetectFormat([]byte("example.com CNAME .\n")); got != FormatPlain { t.Errorf("DetectFormat(rpz record) = %q, want plain (current behaviour)", got) } rules, _, err := Parse(FormatRPZ, []byte("example.com CNAME .\n"), ParseOptions{Subdomains: true}) if err != nil || len(rules) != 1 { t.Errorf("explicit RPZ parse of the same record: rules=%v err=%v", ruleStrings(rules), err) } // The "0 address" form that FormatHosts documents as accepted is not an // IP literal for netip, so the hosts parser counts the line as malformed // instead of blocking the name. _, stats, err := Parse(FormatHosts, []byte("0 ads.example.com\n"), ParseOptions{Subdomains: true}) if err != nil { t.Fatalf("Parse: %v", err) } if stats.Rules != 0 || stats.Malformed != 1 { t.Errorf("\"0 ads.example.com\": rules=%d malformed=%d, want 0/1 (current behaviour)", stats.Rules, stats.Malformed) } } func TestParseAutoPicksFormat(t *testing.T) { rules, stats, err := Parse(FormatAuto, []byte("0.0.0.0 ads.example.com\n"), ParseOptions{Subdomains: true}) if err != nil { t.Fatalf("Parse: %v", err) } if stats.Format != FormatHosts { t.Errorf("Format = %q, want hosts", stats.Format) } if len(rules) != 1 || rules[0].Name != "**.ads.example.com" { t.Errorf("rules = %v", ruleStrings(rules)) } // An empty format string means "sniff" too. if _, stats, _ := Parse("", []byte("||ads.example.com^\n"), ParseOptions{Subdomains: true}); stats.Format != FormatAdblock { t.Errorf("Format = %q, want adblock", stats.Format) } if _, _, err := Parse("bogus", []byte("x"), ParseOptions{}); err == nil { t.Errorf("Parse with an unknown format returned no error") } } func TestParseRobustness(t *testing.T) { long := "long" + strings.Repeat("x", 100000) + ".example.com" bigDomain := strings.Repeat("d", 250) + ".example.com" // over the 253-char limit // Over the 4 MiB per-line cap the scanner enforces (see newLineScanner), // so the parser reports it instead of silently truncating. hugeDomain := strings.Repeat("d", 5*1024*1024) + ".example.com" listSamples := map[string]string{ "crlf": "0.0.0.0 ads.example.com\r\n0.0.0.0 b.example.com\r\n", "lone cr": "0.0.0.0 ads.example.com\r0.0.0.0 b.example.com\r", "leading/trailing space": "\t 0.0.0.0 ads.example.com \n", "blank and space-only": "\n\n \n\t\n", "very long line": "0.0.0.0 " + long + "\n", "over-long label": "0.0.0.0 " + bigDomain + "\n", "comments everywhere": "# c\n; c\n0.0.0.0 ads.example.com # tail\n\n", } for name, data := range listSamples { t.Run("list/"+name, func(t *testing.T) { for _, format := range []Format{FormatHosts, FormatPlain, FormatAdblock} { rules, stats, err := Parse(format, []byte(data), ParseOptions{Subdomains: true}) if err != nil { t.Errorf("%s: Parse: %v", format, err) continue } if stats.Rules != len(rules) { t.Errorf("%s: Rules = %d, want %d", format, stats.Rules, len(rules)) } for _, r := range rules { if strings.ContainsAny(r.Name, "\r\n\t ") { t.Errorf("%s: rule name kept whitespace: %q", format, r.Name) } } } }) } // The adblock and plain parsers must not be confused by CRLF either. for _, data := range []string{ "||ads.example.com^\r\n@@||ok.example.com^\r\n", "a.example.com\r\nb.example.com\r\n", } { for _, format := range []Format{FormatPlain, FormatAdblock} { rules, _, err := Parse(format, []byte(data), ParseOptions{Subdomains: true}) if err != nil { t.Errorf("%s: Parse(%q): %v", format, data, err) continue } if len(rules) == 0 { t.Errorf("%s: Parse(%q) produced no rules", format, data) } for _, r := range rules { if strings.ContainsAny(r.Name, "\r\n\t ") { t.Errorf("%s: rule name kept whitespace: %q", format, r.Name) } } } } // Specifically: CRLF does not leak a CR into the rule names. rules, _, err := Parse(FormatHosts, []byte("0.0.0.0 ads.example.com\r\n"), ParseOptions{Subdomains: true}) if err != nil { t.Fatalf("Parse: %v", err) } if len(rules) != 1 || rules[0].Name != "**.ads.example.com" { t.Errorf("CRLF rules = %v", ruleStrings(rules)) } s := mustBuild(rules...) if s.Match("ads.example.com") == nil || s.Match("x.ads.example.com") == nil { t.Errorf("CRLF-parsed rule does not match: %v", ruleStrings(rules)) } // The RPZ parser goes through the zone-file reader: CRLF, comments, // directives and blank lines are handled there too. for _, data := range []string{ "$TTL 60\r\nexample.com CNAME .\r\n*.sub.example.org CNAME rpz-drop.\r\n", "; comment\r\n\r\nexample.com 60 IN CNAME .\n", "$ORIGIN example.com.\n$TTL 60\nwww IN A 127.0.0.1\n", } { zrules, _, err := Parse(FormatRPZ, []byte(data), ParseOptions{Subdomains: true}) if err != nil { t.Errorf("Parse(rpz %q): %v", data, err) continue } for _, r := range zrules { if strings.ContainsAny(r.Name, "\r\n\t ") { t.Errorf("rule name kept whitespace: %q", r.Name) } } } // A line longer than the scanner's 4 MiB cap is reported, not silently // truncated. _, _, err = Parse(FormatPlain, []byte("a.example.com\n"+hugeDomain+"\n"), ParseOptions{Subdomains: true}) if err == nil { t.Errorf("an over-long line parsed without error") } else { t.Logf("over-long line error (expected): %v", err) } } func TestParseBOMBehaviour(t *testing.T) { // A UTF-8 BOM on the first line is stripped, so the first rule of the list // survives. An operator who saved the list from a Windows editor would // otherwise silently lose the first entry of every list. const bom = "\ufeff" _, stats, err := Parse(FormatAuto, []byte(bom+"0.0.0.0 ads.example.com\n"), ParseOptions{Subdomains: true}) if err != nil { t.Fatalf("Parse: %v", err) } if stats.Rules != 1 || stats.Malformed != 0 { t.Errorf("BOM sample: format=%s rules=%d malformed=%d, want 1/0", stats.Format, stats.Rules, stats.Malformed) } // Without the BOM the same content parses. _, ok, err := Parse(FormatAuto, []byte("0.0.0.0 ads.example.com\n"), ParseOptions{Subdomains: true}) if err != nil { t.Fatalf("Parse: %v", err) } if ok.Rules != 1 { t.Errorf("control: rules = %d, want 1", ok.Rules) } // A BOM on an adblock list behaves the same way. abRules, abStats, err := Parse(FormatAdblock, []byte(bom+"||ads.example.com^\n"), ParseOptions{Subdomains: true}) if err != nil { t.Fatalf("Parse: %v", err) } if got := ruleStrings(abRules); !sameRules(got, []string{"**.ads.example.com block ttl=60"}) { t.Errorf("BOM adblock rules = %v (skipped=%d)", got, abStats.Skipped) } // A BOM in the middle of a file is not a BOM: it stays put and the line // around it still parses, so a multi-line list is not damaged. _, mid, err := Parse(FormatPlain, []byte("a.example.com\n"+bom+"b.example.com\n"), ParseOptions{Subdomains: true}) if err != nil { t.Fatalf("Parse: %v", err) } if mid.Rules != 1 || mid.Malformed != 1 { t.Errorf("mid-file BOM: rules=%d malformed=%d, want 1/1", mid.Rules, mid.Malformed) } } func TestParsedRulesCompileAndMatch(t *testing.T) { // End to end: an adblock list with a broad root rule plus a narrow // allow rule compiles into a set where the allow wins for the name it // names and the root still covers everything else. data := "||example.com^\n@@||metrics.example.com^\n" rules, _, err := Parse(FormatAdblock, []byte(data), ParseOptions{Subdomains: true, DefaultTTL: 300}) if err != nil { t.Fatalf("Parse: %v", err) } b := NewBuilder() for _, r := range rules { b.Add(r, "list", []string{"ads"}) } s := b.Build() if e := s.Match("ads.example.com"); e == nil || e.Action != ActionBlock { t.Errorf("ads.example.com: %+v, want block", e) } if e := s.Match("example.com"); e == nil || e.Action != ActionBlock { t.Errorf("example.com: %+v, want block (root rule covers the apex)", e) } if e := s.Match("metrics.example.com"); e == nil || !e.IsAllow() { t.Errorf("metrics.example.com: %+v, want allow", e) } if e := s.Match("a.b.metrics.example.com"); e == nil || !e.IsAllow() { t.Errorf("a.b.metrics.example.com: %+v, want allow (the allow rule is a root rule)", e) } for _, r := range rules { if e := s.Lookup(r.Name); e != nil && e.TTL != 300 { t.Errorf("TTL = %d, want 300", e.TTL) } } }