fixes

internal/uaclass/uaclass.go

@@ -0,0 +1,267 @@
+// Package uaclass classifies a User-Agent string into a coarse client class
+// using a list of known crawlers, AI agents, monitors and scan tools.
+//
+// A User-Agent is self-declared and trivially spoofed, so the class is a
+// positive signal only: "ai_crawler" means the client says it is one, not that
+// it is. A missing or unrecognised UA is never evidence of a browser.
+package uaclass
+
+import "strings"
+
+// Class values are stable strings: they are stored in ClickHouse and used as
+// UI filter values.
+const (
+	None          = "none"           // no User-Agent sent
+	AICrawler     = "ai_crawler"     // model-training / AI search fetchers
+	AIAgent       = "ai_agent"       // agent harnesses and user-triggered AI fetchers
+	SearchCrawler = "search_crawler" // classic search engines and link previewers
+	SEOCrawler    = "seo_crawler"    // SEO / marketing crawlers
+	Monitor       = "monitor"        // uptime checks and cluster probes
+	ScanTool      = "scan_tool"      // known scanners and exploit tooling
+	OtherBot      = "other_bot"      // declares bot/crawler/spider but is not listed
+	Library       = "library"        // HTTP client libraries and CLIs
+	Browser       = "browser"        // looks like a real browser
+	Unknown       = "unknown"
+)
+
+// Classes lists every class in display order, for UI filters and docs.
+var Classes = []string{AICrawler, AIAgent, SearchCrawler, SEOCrawler, Monitor, ScanTool, OtherBot, Library, Browser, Unknown, None}
+
+// Result is the class plus the matched product token, when one matched.
+type Result struct {
+	Class string `json:"class"`
+	Name  string `json:"name,omitempty"`
+}
+
+type rule struct {
+	class string
+	name  string
+	token string // lower-case substring of the lower-cased UA
+}
+
+// Order matters: the first match wins, and specific products come before the
+// generic bot regexes and the "Mozilla/" browser fallback (most crawlers
+// prefix their UA with Mozilla/5.0).
+var rules = []rule{
+	// AI training and AI search crawlers.
+	{AICrawler, "GPTBot", "gptbot"},
+	{AICrawler, "OAI-SearchBot", "oai-searchbot"},
+	{AICrawler, "ClaudeBot", "claudebot"},
+	{AICrawler, "Claude-SearchBot", "claude-searchbot"},
+	{AICrawler, "anthropic-ai", "anthropic-ai"},
+	{AICrawler, "PerplexityBot", "perplexitybot"},
+	{AICrawler, "Google-Extended", "google-extended"},
+	{AICrawler, "GoogleOther", "googleother"},
+	{AICrawler, "Google-CloudVertexBot", "google-cloudvertexbot"},
+	{AICrawler, "CCBot", "ccbot"},
+	{AICrawler, "Bytespider", "bytespider"},
+	{AICrawler, "Amazonbot", "amazonbot"},
+	{AICrawler, "Applebot-Extended", "applebot-extended"},
+	{AICrawler, "meta-externalagent", "meta-externalagent"},
+	{AICrawler, "meta-externalfetcher", "meta-externalfetcher"},
+	{AICrawler, "FacebookBot", "facebookbot"},
+	{AICrawler, "cohere-ai", "cohere-ai"},
+	{AICrawler, "cohere-training-data-crawler", "cohere-training-data-crawler"},
+	{AICrawler, "Diffbot", "diffbot"},
+	{AICrawler, "YouBot", "youbot"},
+	{AICrawler, "DuckAssistBot", "duckassistbot"},
+	{AICrawler, "MistralAI-User", "mistralai-user"},
+	{AICrawler, "Timpibot", "timpibot"},
+	{AICrawler, "ImagesiftBot", "imagesiftbot"},
+	{AICrawler, "Omgilibot", "omgili"},
+	{AICrawler, "Kangaroo Bot", "kangaroo bot"},
+	{AICrawler, "AI2Bot", "ai2bot"},
+	{AICrawler, "Webzio", "webzio"},
+	{AICrawler, "PanguBot", "pangubot"},
+	{AICrawler, "SemrushBot-OCOB", "semrushbot-ocob"},
+
+	// User-triggered AI fetchers and agent / coding harnesses.
+	{AIAgent, "ChatGPT-User", "chatgpt-user"},
+	{AIAgent, "Claude-User", "claude-user"},
+	{AIAgent, "Perplexity-User", "perplexity-user"},
+	{AIAgent, "Claude Code", "claude-code"},
+	{AIAgent, "Claude CLI", "claude-cli"},
+	{AIAgent, "Claude Agent SDK", "claude-agent-sdk"},
+	{AIAgent, "Codex", "codex"},
+	{AIAgent, "Cursor", "cursor/"},
+	{AIAgent, "Windsurf", "windsurf"},
+	{AIAgent, "opencode", "opencode"},
+	{AIAgent, "aider", "aider/"},
+	{AIAgent, "Cline", "cline/"},
+	{AIAgent, "Continue", "continue/"},
+	{AIAgent, "Gemini CLI", "gemini-cli"},
+	{AIAgent, "Copilot", "copilot"},
+	{AIAgent, "browser-use", "browser-use"},
+	{AIAgent, "Playwright MCP", "playwright-mcp"},
+	{AIAgent, "Manus", "manus-user"},
+	{AIAgent, "LangChain", "langchain"},
+	{AIAgent, "LlamaIndex", "llamaindex"},
+	{AIAgent, "Anthropic SDK", "anthropic/"},
+	{AIAgent, "OpenAI SDK", "openai/"},
+
+	// Search engines and link-preview fetchers.
+	{SearchCrawler, "Googlebot", "googlebot"},
+	{SearchCrawler, "Google-InspectionTool", "google-inspectiontool"},
+	{SearchCrawler, "AdsBot-Google", "adsbot-google"},
+	{SearchCrawler, "Mediapartners-Google", "mediapartners-google"},
+	{SearchCrawler, "Storebot-Google", "storebot-google"},
+	{SearchCrawler, "bingbot", "bingbot"},
+	{SearchCrawler, "BingPreview", "bingpreview"},
+	{SearchCrawler, "DuckDuckBot", "duckduckbot"},
+	{SearchCrawler, "YandexBot", "yandex"},
+	{SearchCrawler, "Baiduspider", "baiduspider"},
+	{SearchCrawler, "Slurp", "slurp"},
+	{SearchCrawler, "Applebot", "applebot"},
+	{SearchCrawler, "PetalBot", "petalbot"},
+	{SearchCrawler, "Sogou", "sogou"},
+	{SearchCrawler, "Qwantify", "qwantify"},
+	{SearchCrawler, "SeznamBot", "seznambot"},
+	{SearchCrawler, "Naver Yeti", "yeti/"},
+	{SearchCrawler, "Exabot", "exabot"},
+	{SearchCrawler, "Ecosia", "ecosiabot"},
+	{SearchCrawler, "Kagi", "kagibot"},
+	{SearchCrawler, "Brave Search", "bravebot"},
+	{SearchCrawler, "facebookexternalhit", "facebookexternalhit"},
+	{SearchCrawler, "Twitterbot", "twitterbot"},
+	{SearchCrawler, "LinkedInBot", "linkedinbot"},
+	{SearchCrawler, "Slackbot", "slackbot"},
+	{SearchCrawler, "Discordbot", "discordbot"},
+	{SearchCrawler, "TelegramBot", "telegrambot"},
+	{SearchCrawler, "WhatsApp", "whatsapp/"},
+	{SearchCrawler, "Pinterestbot", "pinterestbot"},
+	{SearchCrawler, "redditbot", "redditbot"},
+	{SearchCrawler, "Embedly", "embedly"},
+
+	// SEO and marketing crawlers.
+	{SEOCrawler, "AhrefsBot", "ahrefsbot"},
+	{SEOCrawler, "AhrefsSiteAudit", "ahrefssiteaudit"},
+	{SEOCrawler, "SemrushBot", "semrushbot"},
+	{SEOCrawler, "MJ12bot", "mj12bot"},
+	{SEOCrawler, "DotBot", "dotbot"},
+	{SEOCrawler, "rogerbot", "rogerbot"},
+	{SEOCrawler, "BLEXBot", "blexbot"},
+	{SEOCrawler, "DataForSeoBot", "dataforseobot"},
+	{SEOCrawler, "serpstatbot", "serpstatbot"},
+	{SEOCrawler, "Screaming Frog", "screaming frog"},
+	{SEOCrawler, "MegaIndex", "megaindex"},
+	{SEOCrawler, "Barkrowler", "barkrowler"},
+	{SEOCrawler, "SISTRIX", "sistrix"},
+	{SEOCrawler, "Seekport", "seekport"},
+	{SEOCrawler, "BuiltWith", "builtwith"},
+	{SEOCrawler, "Netcraft", "netcraft"},
+	{SEOCrawler, "Expanse", "expanse, a palo alto networks"},
+
+	// Uptime checks and cluster probes.
+	{Monitor, "Uptime-Kuma", "uptime-kuma"},
+	{Monitor, "UptimeRobot", "uptimerobot"},
+	{Monitor, "Pingdom", "pingdom"},
+	{Monitor, "StatusCake", "statuscake"},
+	{Monitor, "Better Uptime", "betteruptime"},
+	{Monitor, "Site24x7", "site24x7"},
+	{Monitor, "Datadog", "datadog"},
+	{Monitor, "NewRelic", "newrelicpinger"},
+	{Monitor, "kube-probe", "kube-probe"},
+	{Monitor, "Prometheus", "prometheus/"},
+	{Monitor, "Blackbox Exporter", "blackbox exporter"},
+	{Monitor, "GoogleHC", "googlehc"},
+	{Monitor, "ELB-HealthChecker", "elb-healthchecker"},
+	{Monitor, "Trivy", "trivy/"},
+
+	// Scanners and exploit tooling.
+	{ScanTool, "CameraScanner", "camerascanner"},
+	{ScanTool, "zgrab", "zgrab"},
+	{ScanTool, "masscan", "masscan"},
+	{ScanTool, "Nmap", "nmap scripting engine"},
+	{ScanTool, "Nuclei", "nuclei"},
+	{ScanTool, "sqlmap", "sqlmap"},
+	{ScanTool, "Nikto", "nikto"},
+	{ScanTool, "WPScan", "wpscan"},
+	{ScanTool, "dirbuster", "dirbuster"},
+	{ScanTool, "gobuster", "gobuster"},
+	{ScanTool, "feroxbuster", "feroxbuster"},
+	{ScanTool, "ffuf", "fuzz faster u fool"},
+	{ScanTool, "Acunetix", "acunetix"},
+	{ScanTool, "Nessus", "nessus"},
+	{ScanTool, "OpenVAS", "openvas"},
+	{ScanTool, "Burp", "burp"},
+	{ScanTool, "Censys", "censys"},
+	{ScanTool, "Shodan", "shodan"},
+	{ScanTool, "Palo Alto Xpanse", "xpanse"},
+	{ScanTool, "Internet Measurement", "internet-measurement"},
+	{ScanTool, "ALittle Client", "alittle client"},
+	{ScanTool, "l9explore", "l9explore"},
+	{ScanTool, "LeakIX", "leakix"},
+	{ScanTool, "NetSystemsResearch", "netsystemsresearch"},
+	{ScanTool, "Hello, World", "hello, world"},
+
+	// Generic HTTP clients and CLIs.
+	{Library, "curl", "curl/"},
+	{Library, "Wget", "wget/"},
+	{Library, "python-requests", "python-requests"},
+	{Library, "python-httpx", "python-httpx"},
+	{Library, "aiohttp", "aiohttp"},
+	{Library, "urllib", "python-urllib"},
+	{Library, "Go-http-client", "go-http-client"},
+	{Library, "Scrapy", "scrapy"},
+	{Library, "Java", "java/"},
+	{Library, "Apache-HttpClient", "apache-httpclient"},
+	{Library, "okhttp", "okhttp"},
+	{Library, "axios", "axios/"},
+	{Library, "node-fetch", "node-fetch"},
+	{Library, "undici", "undici"},
+	{Library, "node", "node"},
+	{Library, "libwww-perl", "libwww-perl"},
+	{Library, "PostmanRuntime", "postmanruntime"},
+	{Library, "insomnia", "insomnia/"},
+	{Library, "HTTPie", "httpie"},
+	{Library, "Ruby", "ruby"},
+	{Library, "PHP", "php/"},
+	{Library, "Dart", "dart:"},
+	{Library, "Faraday", "faraday"},
+	{Library, "Deno", "deno/"},
+	{Library, "Bun", "bun/"},
+}
+
+// Classify returns the class of ua.
+func Classify(ua string) Result {
+	ua = strings.TrimSpace(ua)
+	if ua == "" || ua == "-" {
+		return Result{Class: None}
+	}
+	l := strings.ToLower(ua)
+	for _, r := range rules {
+		if r.name == "node" || r.name == "Ruby" {
+			// Short tokens only count when they are the whole product name,
+			// not a substring of a browser UA.
+			if l == r.token || strings.HasPrefix(l, r.token+"/") || strings.HasPrefix(l, r.token+" ") {
+				return Result{Class: r.class, Name: r.name}
+			}
+			continue
+		}
+		if strings.Contains(l, r.token) {
+			return Result{Class: r.class, Name: r.name}
+		}
+	}
+	for _, t := range []string{"bot", "crawler", "spider", "scraper", "fetcher", "headlesschrome", "phantomjs"} {
+		if strings.Contains(l, t) {
+			return Result{Class: OtherBot}
+		}
+	}
+	if strings.HasPrefix(l, "mozilla/") || strings.HasPrefix(l, "opera/") {
+		return Result{Class: Browser}
+	}
+	return Result{Class: Unknown}
+}
+
+// Of is Classify(ua).Class.
+func Of(ua string) string { return Classify(ua).Class }
+
+// IsBot reports whether the class is a self-declared non-human client
+// (everything except a browser, a missing UA or an unrecognised one).
+func IsBot(class string) bool {
+	switch class {
+	case AICrawler, AIAgent, SearchCrawler, SEOCrawler, Monitor, ScanTool, OtherBot, Library:
+		return true
+	}
+	return false
+}

internal/uaclass/uaclass_test.go

@@ -0,0 +1,37 @@
+package uaclass
+
+import "testing"
+
+func TestClassify(t *testing.T) {
+	cases := []struct{ ua, class, name string }{
+		{"", None, ""},
+		{"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)", AICrawler, "GPTBot"},
+		{"Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)", AICrawler, "ClaudeBot"},
+		{"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0", AIAgent, "ChatGPT-User"},
+		{"claude-code/2.1.0 (external, cli)", AIAgent, "Claude Code"},
+		{"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", SearchCrawler, "Googlebot"},
+		{"Mozilla/5.0 (compatible; AhrefsBot/7.0)", SEOCrawler, "AhrefsBot"},
+		{"Uptime-Kuma/1.23.11", Monitor, "Uptime-Kuma"},
+		{"kube-probe/1.31", Monitor, "kube-probe"},
+		{"CameraScannerV3/1.0", ScanTool, "CameraScanner"},
+		{"Go-http-client/2.0", Library, "Go-http-client"},
+		{"node", Library, "node"},
+		{"curl/8.5.0", Library, "curl"},
+		{"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36", Browser, ""},
+		{"Mozilla/5.0 (compatible; SomeNewFooBot/3.1)", OtherBot, ""},
+		{"Mozilla/5.0 (Macintosh) Nodeblock", Browser, ""}, // "node" must not match inside a browser UA
+		{"weird-thing", Unknown, ""},
+	}
+	for _, c := range cases {
+		got := Classify(c.ua)
+		if got.Class != c.class || got.Name != c.name {
+			t.Errorf("Classify(%q) = %+v, want %s/%s", c.ua, got, c.class, c.name)
+		}
+	}
+}
+
+func TestIsBot(t *testing.T) {
+	if !IsBot(AICrawler) || IsBot(Browser) || IsBot(None) || IsBot(Unknown) {
+		t.Fatal("IsBot mismatch")
+	}
+}