CSP violation reports: ingest endpoint, ClickHouse storage, UI view

- POST /v1/csp collects browser CSP reports in both wire formats (legacy application/csp-report and Reporting API application/reports+json), with CORS limited to the operator's domains (any subdomain of k6n.net, tornberg.me, knatofs.se) and preflight handling. - Reports are stored in a dedicated typed ClickHouse table (csp_reports, 90-day TTL) with a JSONL file fallback (OTEL_CSP_PATH). - New authenticated view endpoints /api/csp/reports and /api/csp/summary plus a 'CSP reports' screen in the UI (tab 12) with time window, site and directive filters and report drill-down. - Documented in instrumentation.md; unit tests for parsing/handlers/file store and an env-gated ClickHouse integration test.
2026-09-23 16:50:43 +00:00
1 parent 9685de64da
commit 36f00d49b9
@@ -0,0 +1,368 @@
+package storage
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+)
+
+// CSPReport is one Content-Security-Policy violation report, normalized from
+// either the legacy report-uri format ({"csp-report": {...}}) or the newer
+// Reporting API (application/reports+json).
+type CSPReport struct {
+	// ID is a stable content hash, computed at ingest, usable as a UI key.
+	ID                 string    `json:"id"`
+	Timestamp          time.Time `json:"timestamp"`
+	Site               string    `json:"site"` // host of document_url, e.g. app.k6n.net
+	DocumentURL        string    `json:"documentUrl"`
+	Referrer           string    `json:"referrer,omitempty"`
+	BlockedURL         string    `json:"blockedUrl,omitempty"`
+	ViolatedDirective  string    `json:"violatedDirective,omitempty"`
+	EffectiveDirective string    `json:"effectiveDirective,omitempty"`
+	OriginalPolicy     string    `json:"originalPolicy,omitempty"`
+	Disposition        string    `json:"disposition,omitempty"` // enforce | report
+	StatusCode         int       `json:"statusCode,omitempty"`
+	Sample             string    `json:"sample,omitempty"`
+	// ReportType is the Reporting API type ("csp-violation"); empty for
+	// legacy report-uri payloads.
+	ReportType string `json:"reportType,omitempty"`
+	UserAgent  string `json:"userAgent,omitempty"`
+	SourceIP   string `json:"sourceIP,omitempty"`
+}
+
+// CSPReportsQuery filters the stored reports for the UI list.
+type CSPReportsQuery struct {
+	// Site filters on the document host ("app.k6n.net"); empty = all.
+	Site string
+	// Directive filters on the effective directive ("script-src"); empty = all.
+	Directive string
+	// Search is a substring match across urls, referrer and sample.
+	Search string
+	// Since (zero = all time) drops reports older than the given instant.
+	Since time.Time
+	Limit int
+}
+
+// CSPCount is one named bucket of the summary (directive, host, site).
+type CSPCount struct {
+	Name  string `json:"name"`
+	Count uint64 `json:"count"`
+}
+
+// CSPSummary aggregates reports over one window for the view's overview cards.
+type CSPSummary struct {
+	Total        uint64     `json:"total"`
+	Sites        uint64     `json:"sites"`
+	ByDirective  []CSPCount `json:"byDirective"`
+	ByBlockedURL []CSPCount `json:"byBlockedHost"`
+	BySite       []CSPCount `json:"bySite"`
+}
+
+// CSPStore persists CSP violation reports and answers the view's queries.
+// ClickHouseStore implements it; CSPFileStore is the fallback when no
+// ClickHouse is configured.
+type CSPStore interface {
+	AppendCSPReport(ctx context.Context, r CSPReport) error
+	CSPReports(ctx context.Context, q CSPReportsQuery) ([]CSPReport, error)
+	CSPSummary(ctx context.Context, hours int) (CSPSummary, error)
+}
+
+// EnsureCSPSchema creates the dedicated csp_reports table: one row per
+// browser report, typed columns instead of a JSON blob so the view can
+// aggregate server-side. Idempotent.
+func EnsureCSPSchema(ctx context.Context, conn driver.Conn, db string) error {
+	return conn.Exec(ctx, fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.csp_reports (
+		id                  String,
+		timestamp           DateTime64(9, 'UTC'),
+		site                LowCardinality(String),
+		document_url        String,
+		referrer            String,
+		blocked_url         String,
+		violated_directive  LowCardinality(String),
+		effective_directive LowCardinality(String),
+		original_policy     String,
+		disposition         LowCardinality(String),
+		status_code         UInt16,
+		sample              String,
+		report_type         LowCardinality(String),
+		user_agent          String,
+		source_ip           String
+	) ENGINE = MergeTree
+	PARTITION BY toYYYYMM(timestamp)
+	ORDER BY (site, effective_directive, timestamp)
+	TTL toDateTime(timestamp) + INTERVAL 90 DAY`, db))
+}
+
+func (s *ClickHouseStore) AppendCSPReport(ctx context.Context, r CSPReport) error {
+	return s.conn.Exec(ctx, fmt.Sprintf(`INSERT INTO %s.csp_reports
+		(id, timestamp, site, document_url, referrer, blocked_url, violated_directive,
+		 effective_directive, original_policy, disposition, status_code, sample,
+		 report_type, user_agent, source_ip) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, s.db),
+		r.ID, r.Timestamp.UTC(), r.Site, r.DocumentURL, r.Referrer, r.BlockedURL,
+		r.ViolatedDirective, r.EffectiveDirective, r.OriginalPolicy, r.Disposition,
+		r.StatusCode, r.Sample, r.ReportType, r.UserAgent, r.SourceIP)
+}
+
+func (s *ClickHouseStore) CSPReports(ctx context.Context, q CSPReportsQuery) ([]CSPReport, error) {
+	if q.Limit <= 0 || q.Limit > 1000 {
+		q.Limit = 200
+	}
+	where, args := "1", []any{}
+	if q.Site != "" {
+		where += " AND site = ?"
+		args = append(args, q.Site)
+	}
+	if q.Directive != "" {
+		where += " AND effective_directive = ?"
+		args = append(args, q.Directive)
+	}
+	if q.Search != "" {
+		where += " AND (position(document_url, ?) > 0 OR position(blocked_url, ?) > 0 OR position(referrer, ?) > 0 OR position(sample, ?) > 0)"
+		for i := 0; i < 4; i++ {
+			args = append(args, q.Search)
+		}
+	}
+	if !q.Since.IsZero() {
+		where += " AND timestamp >= ?"
+		args = append(args, q.Since.UTC())
+	}
+	args = append(args, q.Limit)
+	rows, err := s.conn.Query(ctx, fmt.Sprintf(`SELECT id, timestamp, site, document_url, referrer, blocked_url,
+		violated_directive, effective_directive, original_policy, disposition, status_code,
+		sample, report_type, user_agent, source_ip
+		FROM %s.csp_reports WHERE %s ORDER BY timestamp DESC LIMIT ?`, s.db, where), args...)
+	if err != nil {
+		return nil, fmt.Errorf("clickhouse csp reports: %w", err)
+	}
+	defer rows.Close()
+	out := []CSPReport{}
+	for rows.Next() {
+		var r CSPReport
+		var code uint16
+		if err := rows.Scan(&r.ID, &r.Timestamp, &r.Site, &r.DocumentURL, &r.Referrer,
+			&r.BlockedURL, &r.ViolatedDirective, &r.EffectiveDirective, &r.OriginalPolicy,
+			&r.Disposition, &code, &r.Sample, &r.ReportType, &r.UserAgent, &r.SourceIP); err != nil {
+			return nil, err
+		}
+		r.StatusCode = int(code)
+		out = append(out, r)
+	}
+	return out, rows.Err()
+}
+
+func (s *ClickHouseStore) CSPSummary(ctx context.Context, hours int) (CSPSummary, error) {
+	if hours <= 0 {
+		hours = 24
+	}
+	since := time.Now().UTC().Add(-time.Duration(hours) * time.Hour)
+	var sum CSPSummary
+	if err := s.conn.QueryRow(ctx, fmt.Sprintf(
+		`SELECT count(), uniqExact(site) FROM %s.csp_reports WHERE timestamp >= ?`, s.db),
+		since).Scan(&sum.Total, &sum.Sites); err != nil {
+		return sum, fmt.Errorf("clickhouse csp summary: %w", err)
+	}
+	counts := func(expr string) ([]CSPCount, error) {
+		rows, err := s.conn.Query(ctx, fmt.Sprintf(
+			`SELECT %s AS name, count() AS c FROM %s.csp_reports
+			 WHERE timestamp >= ? GROUP BY name ORDER BY c DESC LIMIT 20`, expr, s.db), since)
+		if err != nil {
+			return nil, err
+		}
+		defer rows.Close()
+		var out []CSPCount
+		for rows.Next() {
+			var v CSPCount
+			if err := rows.Scan(&v.Name, &v.Count); err != nil {
+				return nil, err
+			}
+			out = append(out, v)
+		}
+		return out, rows.Err()
+	}
+	var err error
+	if sum.ByDirective, err = counts("effective_directive"); err != nil {
+		return sum, err
+	}
+	if sum.ByBlockedURL, err = counts("domain(blocked_url)"); err != nil {
+		return sum, err
+	}
+	if sum.BySite, err = counts("site"); err != nil {
+		return sum, err
+	}
+	return sum, nil
+}
+
+// URLLowerHost extracts the lowercase host (port stripped) from a URL-ish
+// string without failing on scheme-relative or malformed input; shared by
+// ingest normalization and the file-store summary.
+func URLLowerHost(raw string) string {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return ""
+	}
+	if i := strings.IndexAny(raw, "?#"); i != -1 {
+		raw = raw[:i]
+	}
+	raw = strings.TrimPrefix(strings.TrimPrefix(raw, "https://"), "http://")
+	raw = strings.TrimPrefix(raw, "//")
+	if i := strings.IndexByte(raw, '/'); i != -1 {
+		raw = raw[:i]
+	}
+	if i := strings.LastIndexByte(raw, '@'); i != -1 {
+		raw = raw[i+1:]
+	}
+	if i := strings.LastIndexByte(raw, ':'); i != -1 && strings.LastIndexByte(raw, ']') < i {
+		raw = raw[:i]
+	}
+	return strings.ToLower(raw)
+}
+
+// CSPFileStore is the no-ClickHouse fallback: newline-delimited JSON next to
+// the telemetry log, read back with in-memory filtering.
+type CSPFileStore struct {
+	filename string
+	mu       sync.Mutex
+}
+
+func NewCSPFileStore(filename string) (*CSPFileStore, error) {
+	if dir := filepath.Dir(filename); dir != "" && dir != "." {
+		if err := os.MkdirAll(dir, 0o755); err != nil {
+			return nil, err
+		}
+	}
+	return &CSPFileStore{filename: filename}, nil
+}
+
+func (fs *CSPFileStore) AppendCSPReport(_ context.Context, r CSPReport) error {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+	f, err := os.OpenFile(fs.filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	return json.NewEncoder(f).Encode(r)
+}
+
+// readAll loads every stored report, oldest first.
+func (fs *CSPFileStore) readAll() ([]CSPReport, error) {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+	f, err := os.Open(fs.filename)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	defer f.Close()
+	var out []CSPReport
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		line := strings.TrimSpace(sc.Text())
+		if line == "" {
+			continue
+		}
+		var r CSPReport
+		if err := json.Unmarshal([]byte(line), &r); err == nil {
+			out = append(out, r)
+		}
+	}
+	return out, sc.Err()
+}
+
+func (fs *CSPFileStore) CSPReports(_ context.Context, q CSPReportsQuery) ([]CSPReport, error) {
+	if q.Limit <= 0 || q.Limit > 1000 {
+		q.Limit = 200
+	}
+	all, err := fs.readAll()
+	if err != nil {
+		return nil, err
+	}
+	match := func(hay, needle string) bool { return needle == "" || strings.Contains(hay, needle) }
+	out := []CSPReport{}
+	for _, r := range all {
+		if q.Site != "" && r.Site != q.Site {
+			continue
+		}
+		if q.Directive != "" && r.EffectiveDirective != q.Directive {
+			continue
+		}
+		if !q.Since.IsZero() && r.Timestamp.Before(q.Since) {
+			continue
+		}
+		if q.Search != "" &&
+			!match(r.DocumentURL, q.Search) && !match(r.BlockedURL, q.Search) &&
+			!match(r.Referrer, q.Search) && !match(r.Sample, q.Search) {
+			continue
+		}
+		out = append(out, r)
+	}
+	// Newest first, bounded.
+	sort.Slice(out, func(i, j int) bool { return out[i].Timestamp.After(out[j].Timestamp) })
+	if len(out) > q.Limit {
+		out = out[:q.Limit]
+	}
+	return out, nil
+}
+
+func (fs *CSPFileStore) CSPSummary(_ context.Context, hours int) (CSPSummary, error) {
+	if hours <= 0 {
+		hours = 24
+	}
+	since := time.Now().Add(-time.Duration(hours) * time.Hour)
+	all, err := fs.readAll()
+	if err != nil {
+		return CSPSummary{}, err
+	}
+	var sum CSPSummary
+	directive, blocked, site := map[string]uint64{}, map[string]uint64{}, map[string]uint64{}
+	sites := map[string]bool{}
+	add := func(m map[string]uint64, name string) {
+		if name != "" {
+			m[name]++
+		}
+	}
+	for _, r := range all {
+		if r.Timestamp.Before(since) {
+			continue
+		}
+		sum.Total++
+		sites[r.Site] = true
+		add(directive, r.EffectiveDirective)
+		add(blocked, URLLowerHost(r.BlockedURL))
+		add(site, r.Site)
+	}
+	sum.Sites = uint64(len(sites))
+	sum.ByDirective = topCounts(directive)
+	sum.ByBlockedURL = topCounts(blocked)
+	sum.BySite = topCounts(site)
+	return sum, nil
+}
+
+// topCounts sorts a counter map, heaviest first, capped like the ClickHouse
+// summary so both stores return comparable payloads.
+func topCounts(m map[string]uint64) []CSPCount {
+	out := make([]CSPCount, 0, len(m))
+	for name, count := range m {
+		out = append(out, CSPCount{Name: name, Count: count})
+	}
+	for i := 1; i < len(out); i++ {
+		for j := i; j > 0 && out[j].Count > out[j-1].Count; j-- {
+			out[j], out[j-1] = out[j-1], out[j]
+		}
+	}
+	if len(out) > 20 {
+		out = out[:20]
+	}
+	return out
+}