72 lines
2.4 KiB
Go
72 lines
2.4 KiB
Go
package features
|
|
|
|
import (
|
|
"git.tornberg.me/go-otel/internal/triage"
|
|
"git.tornberg.me/go-otel/internal/uaclass"
|
|
)
|
|
|
|
// Reasons an event is attributed to a bad actor. The empty string means it is
|
|
// not. These are noise for alerting and for the model's label, not verdicts:
|
|
// the rows are kept, so crawler traffic stays available as training data.
|
|
const (
|
|
BadActorProbe = "probe-scanner" // client hit several probe paths
|
|
BadActorEnumerator = "enumerator" // client walked many routes that do not exist
|
|
BadActorBot = "bot-4xx" // self-declared crawler/agent/scanner getting a client error
|
|
)
|
|
|
|
// BadActorConfig sets the per-client thresholds, evaluated over the events
|
|
// passed to MarkBadActors (a detector range or an export).
|
|
type BadActorConfig struct {
|
|
ProbePaths int // distinct probe paths that make a client a scanner
|
|
EnumPaths int // distinct not-found routes that make a client an enumerator
|
|
}
|
|
|
|
func DefaultBadActorConfig() BadActorConfig { return BadActorConfig{ProbePaths: 3, EnumPaths: 15} }
|
|
|
|
// MarkBadActors sets Event.BadActor in place.
|
|
//
|
|
// Client rules need a resolved ClientIP (see internal/tracectx) and apply to
|
|
// every event of the client, so a scanner's innocent-looking 503 on "/" is
|
|
// covered too. The bot rule needs only the User-Agent class, and covers 4xx
|
|
// only: a crawler that provokes a 5xx has found a real defect.
|
|
func MarkBadActors(events []Event, cfg BadActorConfig) {
|
|
d := DefaultBadActorConfig()
|
|
if cfg.ProbePaths <= 0 {
|
|
cfg.ProbePaths = d.ProbePaths
|
|
}
|
|
if cfg.EnumPaths <= 0 {
|
|
cfg.EnumPaths = d.EnumPaths
|
|
}
|
|
probe := map[string]map[string]struct{}{}
|
|
missing := map[string]map[string]struct{}{}
|
|
add := func(m map[string]map[string]struct{}, ip, route string) {
|
|
if m[ip] == nil {
|
|
m[ip] = map[string]struct{}{}
|
|
}
|
|
m[ip][route] = struct{}{}
|
|
}
|
|
for _, e := range events {
|
|
if e.ClientIP == "" || e.IsDuplicate {
|
|
continue
|
|
}
|
|
if triage.IsProbePath(e.Route) {
|
|
add(probe, e.ClientIP, e.Route)
|
|
}
|
|
switch e.StatusCode {
|
|
case 400, 404, 405, 410:
|
|
add(missing, e.ClientIP, e.Route)
|
|
}
|
|
}
|
|
for i := range events {
|
|
e := &events[i]
|
|
switch {
|
|
case e.ClientIP != "" && len(probe[e.ClientIP]) >= cfg.ProbePaths:
|
|
e.BadActor = BadActorProbe
|
|
case e.ClientIP != "" && len(missing[e.ClientIP]) >= cfg.EnumPaths:
|
|
e.BadActor = BadActorEnumerator
|
|
case uaclass.IsCrawler(e.UAClass) && e.StatusCode >= 400 && e.StatusCode < 500:
|
|
e.BadActor = BadActorBot
|
|
}
|
|
}
|
|
}
|