fixes
This commit is contained in:
1 parent
f30adf05af
commit
8824af8699
29 files changed
+1201
-115
No files matched your search
@@ -18,6 +18,7 @@ import (
|
||||
"git.tornberg.me/go-otel/internal/prometheus"
|
||||
"git.tornberg.me/go-otel/internal/sourcemaps"
|
||||
"git.tornberg.me/go-otel/internal/storage"
|
||||
"git.tornberg.me/go-otel/internal/uaclass"
|
||||
"git.tornberg.me/go-otel/internal/websocket"
|
||||
|
||||
"go.opentelemetry.io/collector/consumer"
|
||||
@@ -163,8 +164,9 @@ func (c *Collector) ListIssues(limit int) []issues.Issue {
|
||||
// ListIssuesFiltered returns a filtered page of deduplicated issues.
|
||||
func (c *Collector) ListIssuesFiltered(opts issues.ListOptions) issues.ListPage {
|
||||
page := c.issues.ListFiltered(opts)
|
||||
if c.sourceMaps != nil {
|
||||
for i := range page.Issues {
|
||||
for i := range page.Issues {
|
||||
classifyIssue(&page.Issues[i])
|
||||
if c.sourceMaps != nil {
|
||||
resolveIssue(&page.Issues[i], c.sourceMaps)
|
||||
}
|
||||
}
|
||||
@@ -174,8 +176,11 @@ func (c *Collector) ListIssuesFiltered(opts issues.ListOptions) issues.ListPage
|
||||
// GetIssue returns a single issue by id.
|
||||
func (c *Collector) GetIssue(id string) (issues.Issue, bool) {
|
||||
issue, ok := c.issues.Get(id)
|
||||
if ok && c.sourceMaps != nil {
|
||||
resolveIssue(&issue, c.sourceMaps)
|
||||
if ok {
|
||||
classifyIssue(&issue)
|
||||
if c.sourceMaps != nil {
|
||||
resolveIssue(&issue, c.sourceMaps)
|
||||
}
|
||||
}
|
||||
return issue, ok
|
||||
}
|
||||
@@ -194,6 +199,16 @@ func resolveIssue(issue *issues.Issue, resolver *sourcemaps.Resolver) {
|
||||
}
|
||||
}
|
||||
|
||||
// classifyIssue sets each sample's UAClass from its User-Agent. Samples with
|
||||
// no UA (backend logs, most spans) stay unclassified rather than "none".
|
||||
func classifyIssue(issue *issues.Issue) {
|
||||
for i := range issue.Samples {
|
||||
if ua := issue.Samples[i].UserAgent; ua != "" {
|
||||
issue.Samples[i].UAClass = uaclass.Of(ua)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SetIssueStatus updates the triage state of an issue.
|
||||
func (c *Collector) SetIssueStatus(id, status string) (issues.Issue, bool) {
|
||||
return c.issues.SetStatus(id, status)
|
||||
|
||||
@@ -175,6 +175,8 @@ func (c *Collector) HandleLogs(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
stampFrontendClient(logs, r)
|
||||
|
||||
if err := c.ConsumeLogs(context.Background(), logs); err != nil {
|
||||
log.Printf("failed to consume logs: %v", err)
|
||||
http.Error(w, "failed to consume logs", http.StatusInternalServerError)
|
||||
@@ -990,3 +992,31 @@ func (c *Collector) HandleNeedleClassify(w http.ResponseWriter, r *http.Request)
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
_, _ = io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
// stampFrontendClient fills in client.address and user_agent.original on
|
||||
// browser-originated log records (source=frontend) from the ingest request,
|
||||
// since the browser cannot know its own public IP. Values the SDK already
|
||||
// sent are kept, and backend services are left alone.
|
||||
func stampFrontendClient(logs plog.Logs, r *http.Request) {
|
||||
ip := strings.TrimSpace(clientIP(r))
|
||||
ua := r.Header.Get("User-Agent")
|
||||
rls := logs.ResourceLogs()
|
||||
for i := 0; i < rls.Len(); i++ {
|
||||
sls := rls.At(i).ScopeLogs()
|
||||
for j := 0; j < sls.Len(); j++ {
|
||||
recs := sls.At(j).LogRecords()
|
||||
for k := 0; k < recs.Len(); k++ {
|
||||
a := recs.At(k).Attributes()
|
||||
if v, ok := a.Get("source"); !ok || v.Str() != "frontend" {
|
||||
continue
|
||||
}
|
||||
if _, ok := a.Get("client.address"); !ok && ip != "" {
|
||||
a.PutStr("client.address", ip)
|
||||
}
|
||||
if _, ok := a.Get("browser.userAgent"); !ok && ua != "" {
|
||||
a.PutStr("user_agent.original", ua)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,12 @@ type SpikeConfig struct {
|
||||
Multiple float64 // a pattern must be novel or >= Multiple * its own baseline mean
|
||||
MinExcess int // and excess errors must be >= MinExcess
|
||||
TopPatterns int
|
||||
// DataSince is when data collection began (earliest row of any kind). When
|
||||
// set, no alert fires until WarmupWindows windows of baseline exist,
|
||||
// otherwise the first window after data begins reports everything as NEW.
|
||||
// Zero disables the guard.
|
||||
DataSince time.Time
|
||||
WarmupWindows int
|
||||
}
|
||||
|
||||
func DefaultSpikeConfig() SpikeConfig {
|
||||
@@ -64,6 +70,15 @@ func DetectSpikes(events []Event, at time.Time, cfg SpikeConfig) []Spike {
|
||||
at = at.UTC()
|
||||
start := at.Add(-cfg.Window)
|
||||
baseStart := start.Add(-time.Duration(cfg.BaselineWindows) * cfg.Window)
|
||||
if !cfg.DataSince.IsZero() {
|
||||
warm := cfg.WarmupWindows
|
||||
if warm <= 0 {
|
||||
warm = min(cfg.BaselineWindows, 6)
|
||||
}
|
||||
if start.Sub(cfg.DataSince.UTC()) < time.Duration(warm)*cfg.Window {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
type agg struct {
|
||||
cur int
|
||||
|
||||
@@ -77,3 +77,24 @@ func TestSuppressorCooldownAndEscalation(t *testing.T) {
|
||||
t.Fatal("alert after cooldown should pass")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectSpikesWarmupGuard(t *testing.T) {
|
||||
at := time.Date(2026, 9, 19, 12, 0, 0, 0, time.UTC)
|
||||
cfg := SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, WarmupWindows: 6}
|
||||
var events []Event
|
||||
for i := 0; i < 20; i++ {
|
||||
events = append(events, Event{Timestamp: at.Add(-time.Minute).Add(time.Duration(i) * time.Second), Service: "api", IsError: true, PatternSignature: "new"})
|
||||
}
|
||||
cfg.DataSince = at.Add(-10 * time.Minute) // data began 10 min ago: cold
|
||||
if got := DetectSpikes(events, at, cfg); len(got) != 0 {
|
||||
t.Fatalf("alerted during warm-up: %+v", got)
|
||||
}
|
||||
cfg.DataSince = at.Add(-2 * time.Hour) // warm
|
||||
if got := DetectSpikes(events, at, cfg); len(got) != 1 {
|
||||
t.Fatalf("expected alert once warm, got %+v", got)
|
||||
}
|
||||
cfg.DataSince = time.Time{} // guard disabled
|
||||
if got := DetectSpikes(events, at, cfg); len(got) != 1 {
|
||||
t.Fatalf("guard should be off when DataSince is zero")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
// Package geoip resolves IP addresses against local MaxMind (GeoLite2 / GeoIP2)
|
||||
// databases so the UI can show where an issue's client came from. Lookups are
|
||||
// purely local; no IP ever leaves the process.
|
||||
package geoip
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"strings"
|
||||
|
||||
geoip2 "github.com/oschwald/geoip2-golang/v2"
|
||||
)
|
||||
|
||||
// Lookup holds the opened databases. Either reader may be nil; a nil *Lookup
|
||||
// means no database is configured.
|
||||
type Lookup struct {
|
||||
city *geoip2.Reader
|
||||
asn *geoip2.Reader
|
||||
}
|
||||
|
||||
// Result is the JSON shape returned to the UI. Fields the databases don't
|
||||
// know are omitted.
|
||||
type Result struct {
|
||||
IP string `json:"ip"`
|
||||
Configured bool `json:"configured"`
|
||||
// Scope is set for addresses that can't be geolocated: "private",
|
||||
// "loopback", "link-local" or "unspecified".
|
||||
Scope string `json:"scope,omitempty"`
|
||||
|
||||
Country string `json:"country,omitempty"`
|
||||
CountryCode string `json:"countryCode,omitempty"`
|
||||
InEU bool `json:"inEu,omitempty"`
|
||||
Region string `json:"region,omitempty"`
|
||||
City string `json:"city,omitempty"`
|
||||
Postal string `json:"postal,omitempty"`
|
||||
Timezone string `json:"timezone,omitempty"`
|
||||
Latitude *float64 `json:"latitude,omitempty"`
|
||||
Longitude *float64 `json:"longitude,omitempty"`
|
||||
Network string `json:"network,omitempty"`
|
||||
ASN uint `json:"asn,omitempty"`
|
||||
ASOrg string `json:"asOrg,omitempty"`
|
||||
}
|
||||
|
||||
// Open opens the given database files; empty paths are skipped. It returns nil
|
||||
// (and no error) when both are empty.
|
||||
func Open(cityPath, asnPath string) (*Lookup, error) {
|
||||
if cityPath == "" && asnPath == "" {
|
||||
return nil, nil
|
||||
}
|
||||
l := &Lookup{}
|
||||
var err error
|
||||
if cityPath != "" {
|
||||
if l.city, err = geoip2.Open(cityPath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if asnPath != "" {
|
||||
if l.asn, err = geoip2.Open(asnPath); err != nil {
|
||||
l.Close()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// Close releases the memory-mapped databases.
|
||||
func (l *Lookup) Close() {
|
||||
if l == nil {
|
||||
return
|
||||
}
|
||||
if l.city != nil {
|
||||
l.city.Close()
|
||||
}
|
||||
if l.asn != nil {
|
||||
l.asn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// Lookup resolves ip. It never fails: an unparsable or unroutable address just
|
||||
// yields a Result without geo fields.
|
||||
func (l *Lookup) Lookup(ip netip.Addr) Result {
|
||||
ip = ip.Unmap()
|
||||
res := Result{IP: ip.String(), Configured: l != nil}
|
||||
switch {
|
||||
case ip.IsLoopback():
|
||||
res.Scope = "loopback"
|
||||
case ip.IsPrivate():
|
||||
res.Scope = "private"
|
||||
case ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast():
|
||||
res.Scope = "link-local"
|
||||
case ip.IsUnspecified():
|
||||
res.Scope = "unspecified"
|
||||
}
|
||||
if l == nil || res.Scope != "" {
|
||||
return res
|
||||
}
|
||||
if l.city != nil {
|
||||
if c, err := l.city.City(ip); err == nil && c.HasData() {
|
||||
res.Country = c.Country.Names.English
|
||||
res.CountryCode = c.Country.ISOCode
|
||||
res.InEU = c.Country.IsInEuropeanUnion
|
||||
res.City = c.City.Names.English
|
||||
res.Postal = c.Postal.Code
|
||||
res.Timezone = c.Location.TimeZone
|
||||
res.Latitude, res.Longitude = c.Location.Latitude, c.Location.Longitude
|
||||
if n := len(c.Subdivisions); n > 0 {
|
||||
res.Region = c.Subdivisions[0].Names.English
|
||||
}
|
||||
if c.Traits.Network.IsValid() {
|
||||
res.Network = c.Traits.Network.String()
|
||||
}
|
||||
}
|
||||
}
|
||||
if l.asn != nil {
|
||||
if a, err := l.asn.ASN(ip); err == nil && a.HasData() {
|
||||
res.ASN, res.ASOrg = a.AutonomousSystemNumber, a.AutonomousSystemOrganization
|
||||
if res.Network == "" && a.Network.IsValid() {
|
||||
res.Network = a.Network.String()
|
||||
}
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// Handler serves GET /api/ip/{ip}. Without a configured database it still
|
||||
// answers (Configured=false) so the UI can explain how to enable it.
|
||||
func (l *Lookup) Handler(w http.ResponseWriter, r *http.Request) {
|
||||
ip, err := netip.ParseAddr(strings.TrimSpace(r.PathValue("ip")))
|
||||
if err != nil {
|
||||
http.Error(w, "invalid ip address", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "private, max-age=3600")
|
||||
if err := json.NewEncoder(w).Encode(l.Lookup(ip)); err != nil {
|
||||
log.Printf("geoip: encode: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package geoip
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLookupWithoutDB(t *testing.T) {
|
||||
var l *Lookup
|
||||
if r := l.Lookup(netip.MustParseAddr("8.8.8.8")); r.Configured || r.Country != "" {
|
||||
t.Fatalf("unexpected result %+v", r)
|
||||
}
|
||||
if r := l.Lookup(netip.MustParseAddr("10.1.2.3")); r.Scope != "private" {
|
||||
t.Fatalf("want private scope, got %+v", r)
|
||||
}
|
||||
if r := l.Lookup(netip.MustParseAddr("::ffff:127.0.0.1")); r.Scope != "loopback" || r.IP != "127.0.0.1" {
|
||||
t.Fatalf("want unmapped loopback, got %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenEmpty(t *testing.T) {
|
||||
if l, err := Open("", ""); l != nil || err != nil {
|
||||
t.Fatalf("want nil,nil got %v,%v", l, err)
|
||||
}
|
||||
}
|
||||
@@ -51,7 +51,10 @@ type Sample struct {
|
||||
HTTPRoute string `json:"httpRoute,omitempty"`
|
||||
HTTPStatus int `json:"httpStatus,omitempty"`
|
||||
// Client/release context.
|
||||
UserAgent string `json:"userAgent,omitempty"`
|
||||
UserAgent string `json:"userAgent,omitempty"`
|
||||
// UAClass is derived from UserAgent when issues are served (see
|
||||
// internal/uaclass); it is not part of the stored sample.
|
||||
UAClass string `json:"uaClass,omitempty"`
|
||||
SourceIP string `json:"sourceIp,omitempty"`
|
||||
Release string `json:"release,omitempty"`
|
||||
Environment string `json:"environment,omitempty"`
|
||||
|
||||
@@ -12,9 +12,10 @@ import (
|
||||
|
||||
"git.tornberg.me/go-otel/internal/httpattr"
|
||||
"git.tornberg.me/go-otel/internal/issues"
|
||||
"git.tornberg.me/go-otel/internal/uaclass"
|
||||
)
|
||||
|
||||
const Version = "v1"
|
||||
const Version = "v2"
|
||||
|
||||
type Event struct {
|
||||
EventID string `json:"eventId"`
|
||||
@@ -29,6 +30,7 @@ type Event struct {
|
||||
Method string `json:"method,omitempty"`
|
||||
Host string `json:"host,omitempty"`
|
||||
UserAgent string `json:"userAgent,omitempty"`
|
||||
UAClass string `json:"uaClass,omitempty"`
|
||||
OriginHost string `json:"originHost,omitempty"`
|
||||
RefererHost string `json:"refererHost,omitempty"`
|
||||
FetchSite string `json:"fetchSite,omitempty"`
|
||||
@@ -53,6 +55,15 @@ type Event struct {
|
||||
MessageNormalized string `json:"messageNormalized,omitempty"`
|
||||
StackSignature string `json:"stackSignature,omitempty"`
|
||||
TraceID string `json:"traceId,omitempty"`
|
||||
SpanID string `json:"spanId,omitempty"`
|
||||
ParentSpanID string `json:"parentSpanId,omitempty"`
|
||||
SpanKind int `json:"spanKind,omitempty"`
|
||||
PeerIP string `json:"peerIp,omitempty"`
|
||||
PeerIPSource string `json:"peerIpSource,omitempty"`
|
||||
EdgePeerIP string `json:"edgePeerIp,omitempty"`
|
||||
EdgePeerSource string `json:"edgePeerSource,omitempty"`
|
||||
EdgeService string `json:"edgeService,omitempty"`
|
||||
EdgeSpanID string `json:"edgeSpanId,omitempty"`
|
||||
IssueFingerprint string `json:"issueFingerprint,omitempty"`
|
||||
IsRetry bool `json:"isRetry"`
|
||||
IsTimeout bool `json:"isTimeout"`
|
||||
@@ -96,35 +107,67 @@ func normalizeTraces(ingest int64, envelope map[string]json.RawMessage) []Event
|
||||
} `json:"scopeSpans"`
|
||||
}
|
||||
_ = json.Unmarshal(envelope["resourceSpans"], &resources)
|
||||
result := []Event{}
|
||||
type traceSpan struct {
|
||||
resource object
|
||||
item span
|
||||
attrs []attribute
|
||||
service string
|
||||
}
|
||||
var spans []traceSpan
|
||||
for _, resource := range resources {
|
||||
for _, scope := range resource.ScopeSpans {
|
||||
for _, item := range scope.Spans {
|
||||
attrs := mergeAttrs(resource.Resource.Attributes, item.Attributes)
|
||||
status := intValue(item.Status.Code)
|
||||
statusCode := intAttr(attrs, "http.response.status_code", "http.status_code")
|
||||
route := normalizeHTTPRoute(attrs)
|
||||
message := item.Status.Message
|
||||
exceptionType, exceptionMessage, stack := exception(item.Events)
|
||||
if exceptionMessage != "" {
|
||||
message = exceptionMessage
|
||||
}
|
||||
if message == "" {
|
||||
message = item.Name
|
||||
}
|
||||
if exceptionType != "" && message != "" {
|
||||
message = exceptionType + ": " + message
|
||||
}
|
||||
isError := status == 2 || isErrorStatus(statusCode) || intAttr(attrs, "rpc.grpc.status_code") > 0 || exceptionType != ""
|
||||
e := baseEvent(ingest, attrs, "span", item.TraceID)
|
||||
e.IsError, e.StatusCode, e.StatusClass = isError, statusCode, statusClass(statusCode)
|
||||
e.DurationMS = durationMS(item.Start, item.End)
|
||||
e.Route, e.SpanName, e.ExceptionType, e.MessageNormalized, e.StackSignature = route, item.Name, exceptionType, normalizeMessage(message), firstFrame(stack)
|
||||
e.IsRetry, e.IsTimeout, e.IsCancellation = operationFlags(attrs, statusCode, intAttr(attrs, "rpc.grpc.status_code"))
|
||||
result = append(result, finish(e))
|
||||
spans = append(spans, traceSpan{
|
||||
resource: resource.Resource,
|
||||
item: item,
|
||||
attrs: attrs,
|
||||
service: defaultString(stringAttr(attrs, "service.name"), "unknown"),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
contexts := make(map[string]spanContext, len(spans))
|
||||
for _, current := range spans {
|
||||
if current.item.SpanID == "" || current.item.TraceID == "" {
|
||||
continue
|
||||
}
|
||||
peer, peerSource := peerAddress(current.attrs)
|
||||
contexts[spanKey(current.item.TraceID, current.item.SpanID)] = spanContext{
|
||||
TraceID: current.item.TraceID, SpanID: current.item.SpanID,
|
||||
ParentSpanID: current.item.ParentSpanID, Service: current.service,
|
||||
PeerIP: peer, PeerSource: peerSource,
|
||||
Edge: isEdgeService(current.service),
|
||||
}
|
||||
}
|
||||
|
||||
result := make([]Event, 0, len(spans))
|
||||
for _, current := range spans {
|
||||
item, attrs := current.item, current.attrs
|
||||
status := intValue(item.Status.Code)
|
||||
statusCode := intAttr(attrs, "http.response.status_code", "http.status_code")
|
||||
route := normalizeHTTPRoute(attrs)
|
||||
message := item.Status.Message
|
||||
exceptionType, exceptionMessage, stack := exception(item.Events)
|
||||
if exceptionMessage != "" {
|
||||
message = exceptionMessage
|
||||
}
|
||||
if message == "" {
|
||||
message = item.Name
|
||||
}
|
||||
if exceptionType != "" && message != "" {
|
||||
message = exceptionType + ": " + message
|
||||
}
|
||||
isError := status == 2 || isErrorStatus(statusCode) || intAttr(attrs, "rpc.grpc.status_code") > 0 || exceptionType != ""
|
||||
e := baseEvent(ingest, attrs, "span", item.TraceID)
|
||||
e.SpanID, e.ParentSpanID, e.SpanKind = item.SpanID, item.ParentSpanID, item.Kind
|
||||
applyTraceContext(&e, contexts[spanKey(item.TraceID, item.SpanID)], contexts)
|
||||
e.IsError, e.StatusCode, e.StatusClass = isError, statusCode, statusClass(statusCode)
|
||||
e.DurationMS = durationMS(item.Start, item.End)
|
||||
e.Route, e.SpanName, e.ExceptionType, e.MessageNormalized, e.StackSignature = route, item.Name, exceptionType, normalizeMessage(message), firstFrame(stack)
|
||||
e.IsRetry, e.IsTimeout, e.IsCancellation = operationFlags(attrs, statusCode, intAttr(attrs, "rpc.grpc.status_code"))
|
||||
result = append(result, finish(e))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -198,17 +241,71 @@ type value struct {
|
||||
BoolValue bool `json:"boolValue"`
|
||||
}
|
||||
type span struct {
|
||||
TraceID string `json:"traceId"`
|
||||
Name string `json:"name"`
|
||||
Start string `json:"startTimeUnixNano"`
|
||||
End string `json:"endTimeUnixNano"`
|
||||
Attributes []attribute `json:"attributes"`
|
||||
Events []spanEvent `json:"events"`
|
||||
Status struct {
|
||||
TraceID string `json:"traceId"`
|
||||
SpanID string `json:"spanId"`
|
||||
ParentSpanID string `json:"parentSpanId"`
|
||||
Name string `json:"name"`
|
||||
Kind int `json:"kind"`
|
||||
Start string `json:"startTimeUnixNano"`
|
||||
End string `json:"endTimeUnixNano"`
|
||||
Attributes []attribute `json:"attributes"`
|
||||
Events []spanEvent `json:"events"`
|
||||
Status struct {
|
||||
Code json.Number `json:"code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"status"`
|
||||
}
|
||||
|
||||
type spanContext struct {
|
||||
TraceID, SpanID, ParentSpanID string
|
||||
Service string
|
||||
PeerIP, PeerSource string
|
||||
Edge bool
|
||||
}
|
||||
|
||||
func spanKey(traceID, spanID string) string { return traceID + "\x00" + spanID }
|
||||
|
||||
// applyTraceContext keeps the immediate peer separate from the peer observed
|
||||
// at an edge ancestor. In particular, a firewall/NAT address is never
|
||||
// relabeled as the original client address.
|
||||
func applyTraceContext(e *Event, current spanContext, contexts map[string]spanContext) {
|
||||
e.PeerIP, e.PeerIPSource = current.PeerIP, current.PeerSource
|
||||
edge := current
|
||||
found := current.Edge && current.PeerIP != ""
|
||||
for hops := 0; !found && edge.ParentSpanID != "" && hops < 32; hops++ {
|
||||
parent, ok := contexts[spanKey(edge.TraceID, edge.ParentSpanID)]
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
edge = parent
|
||||
if edge.Edge && edge.PeerIP != "" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if found {
|
||||
e.EdgePeerIP, e.EdgePeerSource = edge.PeerIP, edge.PeerSource
|
||||
e.EdgeService, e.EdgeSpanID = edge.Service, edge.SpanID
|
||||
}
|
||||
}
|
||||
|
||||
func peerAddress(attrs []attribute) (string, string) {
|
||||
for _, key := range []string{"net.peer.ip", "client.address", "net.sock.peer.addr"} {
|
||||
if value := stringAttr(attrs, key); value != "" {
|
||||
return value, key
|
||||
}
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
func isEdgeService(service string) bool {
|
||||
switch strings.ToLower(service) {
|
||||
case "nginx", "ingress-nginx", "envoy", "traefik", "haproxy", "gateway":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type spanEvent struct {
|
||||
Name string `json:"name"`
|
||||
Attributes []attribute `json:"attributes"`
|
||||
@@ -309,13 +406,18 @@ func baseEvent(timestamp int64, attrs []attribute, kind, traceID string) Event {
|
||||
source = "backend"
|
||||
}
|
||||
ep := httpattr.Extract(func(keys ...string) string { return stringAttr(attrs, keys...) })
|
||||
ua := stringAttr(attrs, "http.user_agent", "user_agent.original")
|
||||
uaClass := ""
|
||||
if ua != "" {
|
||||
uaClass = uaclass.Of(ua)
|
||||
}
|
||||
return Event{
|
||||
Timestamp: timestamp, NormalizerVersion: Version,
|
||||
Service: defaultString(stringAttr(attrs, "service.name"), "unknown"),
|
||||
Release: stringAttr(attrs, "service.release"),
|
||||
Environment: defaultString(stringAttr(attrs, "deployment.environment.name", "deployment.environment", "service.environment"), ""),
|
||||
Method: ep.Method, Host: ep.Host,
|
||||
UserAgent: stringAttr(attrs, "http.user_agent", "user_agent.original"),
|
||||
UserAgent: ua, UAClass: uaClass,
|
||||
OriginHost: attrHost(stringAttr(attrs, "http.request.header.origin", "http.origin", "origin")),
|
||||
RefererHost: attrHost(stringAttr(attrs, "http.request.header.referer", "http.referer")),
|
||||
FetchSite: stringAttr(attrs, "http.request.header.sec_fetch_site"),
|
||||
|
||||
@@ -46,6 +46,31 @@ func TestNormalizeTraceRedactsValuesAndCreatesStableSignatures(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeTraceCarriesPeerAndEdgeContext(t *testing.T) {
|
||||
data := []byte(`{"resourceSpans":[
|
||||
{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"nginx"}}]},"scopeSpans":[{"spans":[
|
||||
{"traceId":"trace-1","spanId":"edge-1","kind":2,"name":"GET /","startTimeUnixNano":"100","endTimeUnixNano":"200","attributes":[{"key":"net.peer.ip","value":{"stringValue":"10.10.0.1"}}]}
|
||||
]}]},
|
||||
{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"checkout"}}]},"scopeSpans":[{"spans":[
|
||||
{"traceId":"trace-1","spanId":"app-1","parentSpanId":"edge-1","kind":1,"name":"checkout","startTimeUnixNano":"120","endTimeUnixNano":"180","attributes":[{"key":"rpc.system","value":{"stringValue":"grpc"}}]}
|
||||
]}]}
|
||||
]}`)
|
||||
events, err := NormalizeEntry("traces", 99, data)
|
||||
if err != nil || len(events) != 2 {
|
||||
t.Fatalf("normalize failed: %v %+v", err, events)
|
||||
}
|
||||
if events[0].PeerIP != "10.10.0.1" || events[0].PeerIPSource != "net.peer.ip" {
|
||||
t.Fatalf("edge peer was not preserved: %+v", events[0])
|
||||
}
|
||||
child := events[1]
|
||||
if child.PeerIP != "" || child.EdgePeerIP != "10.10.0.1" || child.EdgeService != "nginx" || child.EdgeSpanID != "edge-1" {
|
||||
t.Fatalf("unexpected inherited edge context: %+v", child)
|
||||
}
|
||||
if child.SpanID != "app-1" || child.ParentSpanID != "edge-1" || child.SpanKind != 1 {
|
||||
t.Fatalf("missing span relationship fields: %+v", child)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeLogUsesEventTimestampAndFrontendSource(t *testing.T) {
|
||||
data := []byte(`{"resourceLogs":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"web"}}]},"scopeLogs":[{"logRecords":[{"timeUnixNano":"1230000000","severityNumber":17,"body":{"stringValue":"User 123 failed"},"attributes":[{"key":"source","value":{"stringValue":"frontend"}}]}]}]}]}`)
|
||||
events, err := NormalizeEntry("logs", 99, data)
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.tornberg.me/go-otel/internal/uaclass"
|
||||
)
|
||||
|
||||
// EndpointStats reads raw trace spans because the production entries table is
|
||||
@@ -85,6 +87,9 @@ func (s *ClickHouseStore) EndpointStats(ctx context.Context, host, service strin
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if stat.UserAgent != "" {
|
||||
stat.UAClass = uaclass.Of(stat.UserAgent)
|
||||
}
|
||||
if stat.Route == "/" && stat.Path != "/" {
|
||||
stat.Route = ""
|
||||
}
|
||||
|
||||
@@ -19,8 +19,9 @@ import (
|
||||
const normalizedColumns = `event_id, entry_type, event_index, ingest_timestamp, timestamp, service, release, environment,
|
||||
kind, source, is_error, status_class, status_code, duration_ms, route, span_name, metric_name, metric_type, metric_unit,
|
||||
metric_value, metric_count, metric_sum, metric_min, metric_max, exception_type, message_normalized, stack_signature,
|
||||
trace_id, issue_fingerprint, is_retry, is_timeout, is_cancellation, exact_signature, pattern_signature, version,
|
||||
method, host, user_agent, origin_host, referer_host, fetch_site, fetch_mode`
|
||||
trace_id, span_id, parent_span_id, span_kind, peer_ip, peer_ip_source, edge_peer_ip, edge_peer_source, edge_service, edge_span_id,
|
||||
issue_fingerprint, is_retry, is_timeout, is_cancellation, exact_signature, pattern_signature, version,
|
||||
method, host, user_agent, origin_host, referer_host, fetch_site, fetch_mode, ua_class`
|
||||
|
||||
// NormalizedInsertSQL is the INSERT prefix for a normalized_events batch.
|
||||
func NormalizedInsertSQL(db string) string {
|
||||
@@ -41,11 +42,15 @@ func EnsureNormalizedSchema(ctx context.Context, conn driver.Conn, db string) er
|
||||
status_class LowCardinality(String), status_code UInt16, duration_ms Float64,
|
||||
route String, span_name String, metric_name String, metric_type LowCardinality(String), metric_unit String,
|
||||
metric_value Float64, metric_count UInt64, metric_sum Float64, metric_min Float64, metric_max Float64,
|
||||
exception_type String, message_normalized String, stack_signature String, trace_id String, issue_fingerprint String,
|
||||
exception_type String, message_normalized String, stack_signature String, trace_id String,
|
||||
span_id String DEFAULT '', parent_span_id String DEFAULT '', span_kind UInt8 DEFAULT 0,
|
||||
peer_ip String DEFAULT '', peer_ip_source LowCardinality(String) DEFAULT '',
|
||||
edge_peer_ip String DEFAULT '', edge_peer_source LowCardinality(String) DEFAULT '',
|
||||
edge_service LowCardinality(String) DEFAULT '', edge_span_id String DEFAULT '', issue_fingerprint String,
|
||||
is_retry UInt8, is_timeout UInt8, is_cancellation UInt8,
|
||||
exact_signature String, pattern_signature String, version UInt8 DEFAULT 1,
|
||||
method String DEFAULT '', host String DEFAULT '', user_agent String DEFAULT '',
|
||||
origin_host String DEFAULT '', referer_host String DEFAULT '', fetch_site String DEFAULT '', fetch_mode String DEFAULT ''
|
||||
origin_host String DEFAULT '', referer_host String DEFAULT '', fetch_site String DEFAULT '', fetch_mode String DEFAULT '', ua_class LowCardinality(String) DEFAULT ''
|
||||
) ENGINE = ReplacingMergeTree(version)
|
||||
PARTITION BY toYYYYMM(timestamp)
|
||||
ORDER BY (service, kind, pattern_signature, timestamp, event_id)
|
||||
@@ -55,10 +60,14 @@ func EnsureNormalizedSchema(ctx context.Context, conn driver.Conn, db string) er
|
||||
}
|
||||
// Developer tables predate these columns; historical rows get safe defaults.
|
||||
for _, col := range []string{
|
||||
"span_id String DEFAULT ''", "parent_span_id String DEFAULT ''", "span_kind UInt8 DEFAULT 0",
|
||||
"peer_ip String DEFAULT ''", "peer_ip_source LowCardinality(String) DEFAULT ''",
|
||||
"edge_peer_ip String DEFAULT ''", "edge_peer_source LowCardinality(String) DEFAULT ''",
|
||||
"edge_service LowCardinality(String) DEFAULT ''", "edge_span_id String DEFAULT ''",
|
||||
"issue_fingerprint String DEFAULT ''", "is_retry UInt8 DEFAULT 0", "is_timeout UInt8 DEFAULT 0",
|
||||
"is_cancellation UInt8 DEFAULT 0", "method String DEFAULT ''", "host String DEFAULT ''",
|
||||
"user_agent String DEFAULT ''", "origin_host String DEFAULT ''", "referer_host String DEFAULT ''",
|
||||
"fetch_site String DEFAULT ''", "fetch_mode String DEFAULT ''",
|
||||
"fetch_site String DEFAULT ''", "fetch_mode String DEFAULT ''", "ua_class LowCardinality(String) DEFAULT ''",
|
||||
} {
|
||||
if err := conn.Exec(ctx, fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS %s", db, col)); err != nil {
|
||||
return err
|
||||
@@ -74,9 +83,10 @@ func AppendNormalizedEvent(batch driver.Batch, e normalization.Event) error {
|
||||
e.Service, e.Release, e.Environment, e.Kind, e.Source, boolToUint8(e.IsError),
|
||||
e.StatusClass, uint16(e.StatusCode), e.DurationMS, e.Route, e.SpanName, e.MetricName, e.MetricType, e.MetricUnit,
|
||||
e.MetricValue, e.MetricCount, e.MetricSum, e.MetricMin, e.MetricMax,
|
||||
e.ExceptionType, e.MessageNormalized, e.StackSignature, e.TraceID, e.IssueFingerprint,
|
||||
e.ExceptionType, e.MessageNormalized, e.StackSignature, e.TraceID, e.SpanID, e.ParentSpanID, uint8(e.SpanKind),
|
||||
e.PeerIP, e.PeerIPSource, e.EdgePeerIP, e.EdgePeerSource, e.EdgeService, e.EdgeSpanID, e.IssueFingerprint,
|
||||
boolToUint8(e.IsRetry), boolToUint8(e.IsTimeout), boolToUint8(e.IsCancellation), e.ExactSignature, e.PatternSignature,
|
||||
uint8(1), e.Method, e.Host, e.UserAgent, e.OriginHost, e.RefererHost, e.FetchSite, e.FetchMode,
|
||||
uint8(2), e.Method, e.Host, e.UserAgent, e.OriginHost, e.RefererHost, e.FetchSite, e.FetchMode, e.UAClass,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -56,6 +56,7 @@ type EndpointStat struct {
|
||||
Route string `json:"route,omitempty"`
|
||||
StatusCode uint16 `json:"statusCode"`
|
||||
UserAgent string `json:"userAgent,omitempty"`
|
||||
UAClass string `json:"uaClass,omitempty"`
|
||||
OriginHost string `json:"originHost,omitempty"`
|
||||
RefererHost string `json:"refererHost,omitempty"`
|
||||
FetchSite string `json:"fetchSite,omitempty"`
|
||||
|
||||
@@ -47,8 +47,9 @@ var (
|
||||
// control, CMS/admin paths, path traversal. Deliberately path-only.
|
||||
probeRoute = regexp.MustCompile(`(?i)(\.env\b|\.git\b|\.htpasswd|\.htaccess|\.npmrc|\.claude|\.docker|\.kube|\.aws|\.ssh|` +
|
||||
`wp-|/wp/|/wp$|wordpress|phpmyadmin|phpinfo|cgi-bin|xmlrpc|userfiles|@fs/|etc/passwd|proc/self|\.\./|%2e%2e|` +
|
||||
`\.php\b|credentials|secrets?\b|firebase|serviceaccount|/config(/|\.)|settings\.(js|json)$|bootstrap\.properties|` +
|
||||
`\.(ya?ml|toml|properties|sql|bak|old|swp|zip|tar|tgz|gz|ini|conf|cfg)$)`)
|
||||
`\.php\b|_debugbar|debug/pprof|server-info|actuator|__env|\.bash_|\.profile$|graphql/console|` +
|
||||
`credentials|secrets?\b|firebase|serviceaccount|/config(/|\.)|settings\.(js|json|py)$|bootstrap\.properties|` +
|
||||
`\.(ya?ml|toml|properties|sql|bak|old|swp|zip|tar|tgz|gz|ini|conf|cfg|key|pem)$)`)
|
||||
bugText = regexp.MustCompile(`(?i)panic|nil pointer|index out of range|typeerror|referenceerror|deadlock|unhandled|chunkloaderror|segfault`)
|
||||
capText = regexp.MustCompile(`(?i)timeout|deadline exceeded|\bpool\b|memory limit|\boom(killed)?\b|too many|queue full|exhaust`)
|
||||
depText = regexp.MustCompile(`(?i)dial|connection refused|\beof\b|no such host|upstream|unavailable|broken pipe|reset by peer`)
|
||||
@@ -87,7 +88,7 @@ func Classify(in Input) Result {
|
||||
return Result{StaleReference, "status-404"}
|
||||
case lifeText.MatchString(blob):
|
||||
return Result{ExpectedLifecycle, "lifecycle-text"}
|
||||
case in.Status == 499 || in.Status == 200 || in.Status == 202 || in.Status == 204 || (in.Status >= 300 && in.Status < 400):
|
||||
case in.Status == 499 || (in.Status >= 100 && in.Status < 400):
|
||||
return Result{ExpectedLifecycle, "benign-status"}
|
||||
case in.Status == 0 && (in.Kind == "span" || in.Kind == "log"):
|
||||
return Result{ExpectedLifecycle, "no-status-signal"}
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
// Package uaclass classifies a User-Agent string into a coarse client class
|
||||
// using a list of known crawlers, AI agents, monitors and scan tools.
|
||||
//
|
||||
// A User-Agent is self-declared and trivially spoofed, so the class is a
|
||||
// positive signal only: "ai_crawler" means the client says it is one, not that
|
||||
// it is. A missing or unrecognised UA is never evidence of a browser.
|
||||
package uaclass
|
||||
|
||||
import "strings"
|
||||
|
||||
// Class values are stable strings: they are stored in ClickHouse and used as
|
||||
// UI filter values.
|
||||
const (
|
||||
None = "none" // no User-Agent sent
|
||||
AICrawler = "ai_crawler" // model-training / AI search fetchers
|
||||
AIAgent = "ai_agent" // agent harnesses and user-triggered AI fetchers
|
||||
SearchCrawler = "search_crawler" // classic search engines and link previewers
|
||||
SEOCrawler = "seo_crawler" // SEO / marketing crawlers
|
||||
Monitor = "monitor" // uptime checks and cluster probes
|
||||
ScanTool = "scan_tool" // known scanners and exploit tooling
|
||||
OtherBot = "other_bot" // declares bot/crawler/spider but is not listed
|
||||
Library = "library" // HTTP client libraries and CLIs
|
||||
Browser = "browser" // looks like a real browser
|
||||
Unknown = "unknown"
|
||||
)
|
||||
|
||||
// Classes lists every class in display order, for UI filters and docs.
|
||||
var Classes = []string{AICrawler, AIAgent, SearchCrawler, SEOCrawler, Monitor, ScanTool, OtherBot, Library, Browser, Unknown, None}
|
||||
|
||||
// Result is the class plus the matched product token, when one matched.
|
||||
type Result struct {
|
||||
Class string `json:"class"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
type rule struct {
|
||||
class string
|
||||
name string
|
||||
token string // lower-case substring of the lower-cased UA
|
||||
}
|
||||
|
||||
// Order matters: the first match wins, and specific products come before the
|
||||
// generic bot regexes and the "Mozilla/" browser fallback (most crawlers
|
||||
// prefix their UA with Mozilla/5.0).
|
||||
var rules = []rule{
|
||||
// AI training and AI search crawlers.
|
||||
{AICrawler, "GPTBot", "gptbot"},
|
||||
{AICrawler, "OAI-SearchBot", "oai-searchbot"},
|
||||
{AICrawler, "ClaudeBot", "claudebot"},
|
||||
{AICrawler, "Claude-SearchBot", "claude-searchbot"},
|
||||
{AICrawler, "anthropic-ai", "anthropic-ai"},
|
||||
{AICrawler, "PerplexityBot", "perplexitybot"},
|
||||
{AICrawler, "Google-Extended", "google-extended"},
|
||||
{AICrawler, "GoogleOther", "googleother"},
|
||||
{AICrawler, "Google-CloudVertexBot", "google-cloudvertexbot"},
|
||||
{AICrawler, "CCBot", "ccbot"},
|
||||
{AICrawler, "Bytespider", "bytespider"},
|
||||
{AICrawler, "Amazonbot", "amazonbot"},
|
||||
{AICrawler, "Applebot-Extended", "applebot-extended"},
|
||||
{AICrawler, "meta-externalagent", "meta-externalagent"},
|
||||
{AICrawler, "meta-externalfetcher", "meta-externalfetcher"},
|
||||
{AICrawler, "FacebookBot", "facebookbot"},
|
||||
{AICrawler, "cohere-ai", "cohere-ai"},
|
||||
{AICrawler, "cohere-training-data-crawler", "cohere-training-data-crawler"},
|
||||
{AICrawler, "Diffbot", "diffbot"},
|
||||
{AICrawler, "YouBot", "youbot"},
|
||||
{AICrawler, "DuckAssistBot", "duckassistbot"},
|
||||
{AICrawler, "MistralAI-User", "mistralai-user"},
|
||||
{AICrawler, "Timpibot", "timpibot"},
|
||||
{AICrawler, "ImagesiftBot", "imagesiftbot"},
|
||||
{AICrawler, "Omgilibot", "omgili"},
|
||||
{AICrawler, "Kangaroo Bot", "kangaroo bot"},
|
||||
{AICrawler, "AI2Bot", "ai2bot"},
|
||||
{AICrawler, "Webzio", "webzio"},
|
||||
{AICrawler, "PanguBot", "pangubot"},
|
||||
{AICrawler, "SemrushBot-OCOB", "semrushbot-ocob"},
|
||||
|
||||
// User-triggered AI fetchers and agent / coding harnesses.
|
||||
{AIAgent, "ChatGPT-User", "chatgpt-user"},
|
||||
{AIAgent, "Claude-User", "claude-user"},
|
||||
{AIAgent, "Perplexity-User", "perplexity-user"},
|
||||
{AIAgent, "Claude Code", "claude-code"},
|
||||
{AIAgent, "Claude CLI", "claude-cli"},
|
||||
{AIAgent, "Claude Agent SDK", "claude-agent-sdk"},
|
||||
{AIAgent, "Codex", "codex"},
|
||||
{AIAgent, "Cursor", "cursor/"},
|
||||
{AIAgent, "Windsurf", "windsurf"},
|
||||
{AIAgent, "opencode", "opencode"},
|
||||
{AIAgent, "aider", "aider/"},
|
||||
{AIAgent, "Cline", "cline/"},
|
||||
{AIAgent, "Continue", "continue/"},
|
||||
{AIAgent, "Gemini CLI", "gemini-cli"},
|
||||
{AIAgent, "Copilot", "copilot"},
|
||||
{AIAgent, "browser-use", "browser-use"},
|
||||
{AIAgent, "Playwright MCP", "playwright-mcp"},
|
||||
{AIAgent, "Manus", "manus-user"},
|
||||
{AIAgent, "LangChain", "langchain"},
|
||||
{AIAgent, "LlamaIndex", "llamaindex"},
|
||||
{AIAgent, "Anthropic SDK", "anthropic/"},
|
||||
{AIAgent, "OpenAI SDK", "openai/"},
|
||||
|
||||
// Search engines and link-preview fetchers.
|
||||
{SearchCrawler, "Googlebot", "googlebot"},
|
||||
{SearchCrawler, "Google-InspectionTool", "google-inspectiontool"},
|
||||
{SearchCrawler, "AdsBot-Google", "adsbot-google"},
|
||||
{SearchCrawler, "Mediapartners-Google", "mediapartners-google"},
|
||||
{SearchCrawler, "Storebot-Google", "storebot-google"},
|
||||
{SearchCrawler, "bingbot", "bingbot"},
|
||||
{SearchCrawler, "BingPreview", "bingpreview"},
|
||||
{SearchCrawler, "DuckDuckBot", "duckduckbot"},
|
||||
{SearchCrawler, "YandexBot", "yandex"},
|
||||
{SearchCrawler, "Baiduspider", "baiduspider"},
|
||||
{SearchCrawler, "Slurp", "slurp"},
|
||||
{SearchCrawler, "Applebot", "applebot"},
|
||||
{SearchCrawler, "PetalBot", "petalbot"},
|
||||
{SearchCrawler, "Sogou", "sogou"},
|
||||
{SearchCrawler, "Qwantify", "qwantify"},
|
||||
{SearchCrawler, "SeznamBot", "seznambot"},
|
||||
{SearchCrawler, "Naver Yeti", "yeti/"},
|
||||
{SearchCrawler, "Exabot", "exabot"},
|
||||
{SearchCrawler, "Ecosia", "ecosiabot"},
|
||||
{SearchCrawler, "Kagi", "kagibot"},
|
||||
{SearchCrawler, "Brave Search", "bravebot"},
|
||||
{SearchCrawler, "facebookexternalhit", "facebookexternalhit"},
|
||||
{SearchCrawler, "Twitterbot", "twitterbot"},
|
||||
{SearchCrawler, "LinkedInBot", "linkedinbot"},
|
||||
{SearchCrawler, "Slackbot", "slackbot"},
|
||||
{SearchCrawler, "Discordbot", "discordbot"},
|
||||
{SearchCrawler, "TelegramBot", "telegrambot"},
|
||||
{SearchCrawler, "WhatsApp", "whatsapp/"},
|
||||
{SearchCrawler, "Pinterestbot", "pinterestbot"},
|
||||
{SearchCrawler, "redditbot", "redditbot"},
|
||||
{SearchCrawler, "Embedly", "embedly"},
|
||||
|
||||
// SEO and marketing crawlers.
|
||||
{SEOCrawler, "AhrefsBot", "ahrefsbot"},
|
||||
{SEOCrawler, "AhrefsSiteAudit", "ahrefssiteaudit"},
|
||||
{SEOCrawler, "SemrushBot", "semrushbot"},
|
||||
{SEOCrawler, "MJ12bot", "mj12bot"},
|
||||
{SEOCrawler, "DotBot", "dotbot"},
|
||||
{SEOCrawler, "rogerbot", "rogerbot"},
|
||||
{SEOCrawler, "BLEXBot", "blexbot"},
|
||||
{SEOCrawler, "DataForSeoBot", "dataforseobot"},
|
||||
{SEOCrawler, "serpstatbot", "serpstatbot"},
|
||||
{SEOCrawler, "Screaming Frog", "screaming frog"},
|
||||
{SEOCrawler, "MegaIndex", "megaindex"},
|
||||
{SEOCrawler, "Barkrowler", "barkrowler"},
|
||||
{SEOCrawler, "SISTRIX", "sistrix"},
|
||||
{SEOCrawler, "Seekport", "seekport"},
|
||||
{SEOCrawler, "BuiltWith", "builtwith"},
|
||||
{SEOCrawler, "Netcraft", "netcraft"},
|
||||
{SEOCrawler, "Expanse", "expanse, a palo alto networks"},
|
||||
|
||||
// Uptime checks and cluster probes.
|
||||
{Monitor, "Uptime-Kuma", "uptime-kuma"},
|
||||
{Monitor, "UptimeRobot", "uptimerobot"},
|
||||
{Monitor, "Pingdom", "pingdom"},
|
||||
{Monitor, "StatusCake", "statuscake"},
|
||||
{Monitor, "Better Uptime", "betteruptime"},
|
||||
{Monitor, "Site24x7", "site24x7"},
|
||||
{Monitor, "Datadog", "datadog"},
|
||||
{Monitor, "NewRelic", "newrelicpinger"},
|
||||
{Monitor, "kube-probe", "kube-probe"},
|
||||
{Monitor, "Prometheus", "prometheus/"},
|
||||
{Monitor, "Blackbox Exporter", "blackbox exporter"},
|
||||
{Monitor, "GoogleHC", "googlehc"},
|
||||
{Monitor, "ELB-HealthChecker", "elb-healthchecker"},
|
||||
{Monitor, "Trivy", "trivy/"},
|
||||
|
||||
// Scanners and exploit tooling.
|
||||
{ScanTool, "CameraScanner", "camerascanner"},
|
||||
{ScanTool, "zgrab", "zgrab"},
|
||||
{ScanTool, "masscan", "masscan"},
|
||||
{ScanTool, "Nmap", "nmap scripting engine"},
|
||||
{ScanTool, "Nuclei", "nuclei"},
|
||||
{ScanTool, "sqlmap", "sqlmap"},
|
||||
{ScanTool, "Nikto", "nikto"},
|
||||
{ScanTool, "WPScan", "wpscan"},
|
||||
{ScanTool, "dirbuster", "dirbuster"},
|
||||
{ScanTool, "gobuster", "gobuster"},
|
||||
{ScanTool, "feroxbuster", "feroxbuster"},
|
||||
{ScanTool, "ffuf", "fuzz faster u fool"},
|
||||
{ScanTool, "Acunetix", "acunetix"},
|
||||
{ScanTool, "Nessus", "nessus"},
|
||||
{ScanTool, "OpenVAS", "openvas"},
|
||||
{ScanTool, "Burp", "burp"},
|
||||
{ScanTool, "Censys", "censys"},
|
||||
{ScanTool, "Shodan", "shodan"},
|
||||
{ScanTool, "Palo Alto Xpanse", "xpanse"},
|
||||
{ScanTool, "Internet Measurement", "internet-measurement"},
|
||||
{ScanTool, "ALittle Client", "alittle client"},
|
||||
{ScanTool, "l9explore", "l9explore"},
|
||||
{ScanTool, "LeakIX", "leakix"},
|
||||
{ScanTool, "NetSystemsResearch", "netsystemsresearch"},
|
||||
{ScanTool, "Hello, World", "hello, world"},
|
||||
|
||||
// Generic HTTP clients and CLIs.
|
||||
{Library, "curl", "curl/"},
|
||||
{Library, "Wget", "wget/"},
|
||||
{Library, "python-requests", "python-requests"},
|
||||
{Library, "python-httpx", "python-httpx"},
|
||||
{Library, "aiohttp", "aiohttp"},
|
||||
{Library, "urllib", "python-urllib"},
|
||||
{Library, "Go-http-client", "go-http-client"},
|
||||
{Library, "Scrapy", "scrapy"},
|
||||
{Library, "Java", "java/"},
|
||||
{Library, "Apache-HttpClient", "apache-httpclient"},
|
||||
{Library, "okhttp", "okhttp"},
|
||||
{Library, "axios", "axios/"},
|
||||
{Library, "node-fetch", "node-fetch"},
|
||||
{Library, "undici", "undici"},
|
||||
{Library, "node", "node"},
|
||||
{Library, "libwww-perl", "libwww-perl"},
|
||||
{Library, "PostmanRuntime", "postmanruntime"},
|
||||
{Library, "insomnia", "insomnia/"},
|
||||
{Library, "HTTPie", "httpie"},
|
||||
{Library, "Ruby", "ruby"},
|
||||
{Library, "PHP", "php/"},
|
||||
{Library, "Dart", "dart:"},
|
||||
{Library, "Faraday", "faraday"},
|
||||
{Library, "Deno", "deno/"},
|
||||
{Library, "Bun", "bun/"},
|
||||
}
|
||||
|
||||
// Classify returns the class of ua.
|
||||
func Classify(ua string) Result {
|
||||
ua = strings.TrimSpace(ua)
|
||||
if ua == "" || ua == "-" {
|
||||
return Result{Class: None}
|
||||
}
|
||||
l := strings.ToLower(ua)
|
||||
for _, r := range rules {
|
||||
if r.name == "node" || r.name == "Ruby" {
|
||||
// Short tokens only count when they are the whole product name,
|
||||
// not a substring of a browser UA.
|
||||
if l == r.token || strings.HasPrefix(l, r.token+"/") || strings.HasPrefix(l, r.token+" ") {
|
||||
return Result{Class: r.class, Name: r.name}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if strings.Contains(l, r.token) {
|
||||
return Result{Class: r.class, Name: r.name}
|
||||
}
|
||||
}
|
||||
for _, t := range []string{"bot", "crawler", "spider", "scraper", "fetcher", "headlesschrome", "phantomjs"} {
|
||||
if strings.Contains(l, t) {
|
||||
return Result{Class: OtherBot}
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(l, "mozilla/") || strings.HasPrefix(l, "opera/") {
|
||||
return Result{Class: Browser}
|
||||
}
|
||||
return Result{Class: Unknown}
|
||||
}
|
||||
|
||||
// Of is Classify(ua).Class.
|
||||
func Of(ua string) string { return Classify(ua).Class }
|
||||
|
||||
// IsBot reports whether the class is a self-declared non-human client
|
||||
// (everything except a browser, a missing UA or an unrecognised one).
|
||||
func IsBot(class string) bool {
|
||||
switch class {
|
||||
case AICrawler, AIAgent, SearchCrawler, SEOCrawler, Monitor, ScanTool, OtherBot, Library:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package uaclass
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestClassify(t *testing.T) {
|
||||
cases := []struct{ ua, class, name string }{
|
||||
{"", None, ""},
|
||||
{"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)", AICrawler, "GPTBot"},
|
||||
{"Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)", AICrawler, "ClaudeBot"},
|
||||
{"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0", AIAgent, "ChatGPT-User"},
|
||||
{"claude-code/2.1.0 (external, cli)", AIAgent, "Claude Code"},
|
||||
{"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", SearchCrawler, "Googlebot"},
|
||||
{"Mozilla/5.0 (compatible; AhrefsBot/7.0)", SEOCrawler, "AhrefsBot"},
|
||||
{"Uptime-Kuma/1.23.11", Monitor, "Uptime-Kuma"},
|
||||
{"kube-probe/1.31", Monitor, "kube-probe"},
|
||||
{"CameraScannerV3/1.0", ScanTool, "CameraScanner"},
|
||||
{"Go-http-client/2.0", Library, "Go-http-client"},
|
||||
{"node", Library, "node"},
|
||||
{"curl/8.5.0", Library, "curl"},
|
||||
{"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36", Browser, ""},
|
||||
{"Mozilla/5.0 (compatible; SomeNewFooBot/3.1)", OtherBot, ""},
|
||||
{"Mozilla/5.0 (Macintosh) Nodeblock", Browser, ""}, // "node" must not match inside a browser UA
|
||||
{"weird-thing", Unknown, ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := Classify(c.ua)
|
||||
if got.Class != c.class || got.Name != c.name {
|
||||
t.Errorf("Classify(%q) = %+v, want %s/%s", c.ua, got, c.class, c.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsBot(t *testing.T) {
|
||||
if !IsBot(AICrawler) || IsBot(Browser) || IsBot(None) || IsBot(Unknown) {
|
||||
t.Fatal("IsBot mismatch")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user