CSP violation reports: ingest endpoint, ClickHouse storage, UI view

- POST /v1/csp collects browser CSP reports in both wire formats (legacy
  application/csp-report and Reporting API application/reports+json), with
  CORS limited to the operator's domains (any subdomain of k6n.net,
  tornberg.me, knatofs.se) and preflight handling.
- Reports are stored in a dedicated typed ClickHouse table (csp_reports,
  90-day TTL) with a JSONL file fallback (OTEL_CSP_PATH).
- New authenticated view endpoints /api/csp/reports and /api/csp/summary
  plus a 'CSP reports' screen in the UI (tab 12) with time window, site and
  directive filters and report drill-down.
- Documented in instrumentation.md; unit tests for parsing/handlers/file
  store and an env-gated ClickHouse integration test.

internal/collector/collector.go

@@ -47,6 +47,7 @@ type Collector struct {
 	ingestKey     string
 	todoStore     todos.Store
 	sourceMapKey  string
+	cspStore      storage.CSPStore
 
 	notifyIssuesFn func()
 

internal/collector/csp.go

@@ -0,0 +1,257 @@
+package collector
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/storage"
+)
+
+const (
+	// CSP reports are small; URLs and the echoed policy are the long parts.
+	maxCSPBody       = 16 << 10
+	maxCSPURL        = 512
+	maxCSPPolicy     = 2048
+	maxCSPSample     = 300
+	maxCSPBatch      = 50
+	defaultCSPWindow = 24
+)
+
+// SetCSPStore enables CSP violation report collection. Both the ClickHouse
+// store and the file fallback satisfy the interface; nil disables the
+// endpoints.
+func (c *Collector) SetCSPStore(s storage.CSPStore) { c.cspStore = s }
+
+// cspReportBody mirrors the legacy report-uri payload
+// (Content-Type: application/csp-report).
+type cspReportBody struct {
+	CSPReport struct {
+		DocumentURI        string `json:"document-uri"`
+		Referrer           string `json:"referrer"`
+		BlockedURI         string `json:"blocked-uri"`
+		ViolatedDirective  string `json:"violated-directive"`
+		EffectiveDirective string `json:"effective-directive"`
+		OriginalPolicy     string `json:"original-policy"`
+		Disposition        string `json:"disposition"`
+		StatusCode         int    `json:"status-code"`
+		ScriptSample       string `json:"script-sample"`
+	} `json:"csp-report"`
+}
+
+// cspAPIReport mirrors one entry of the Reporting API payload
+// (Content-Type: application/reports+json); body repeats the csp-report
+// fields in camelCase.
+type cspAPIReport struct {
+	Type      string `json:"type"`
+	UserAgent string `json:"user_agent"`
+	Body      struct {
+		DocumentURL        string `json:"documentURL"`
+		Referrer           string `json:"referrer"`
+		BlockedURL         string `json:"blockedURL"`
+		ViolatedDirective  string `json:"violatedDirective"`
+		EffectiveDirective string `json:"effectiveDirective"`
+		OriginalPolicy     string `json:"originalPolicy"`
+		Disposition        string `json:"disposition"`
+		StatusCode         int    `json:"statusCode"`
+		Sample             string `json:"sample"`
+	} `json:"body"`
+}
+
+func clipCSP(s string, max int) string {
+	if len(s) > max {
+		return s[:max]
+	}
+	return s
+}
+
+// parseCSPReports decodes either wire format into normalized reports; an
+// unrecognized shape returns nil (the handler rejects those).
+func parseCSPReports(body []byte) []storage.CSPReport {
+	var reports []storage.CSPReport
+	if len(body) > 0 && body[0] == '[' {
+		var api []cspAPIReport
+		if err := json.Unmarshal(body, &api); err != nil || len(api) == 0 {
+			return nil
+		}
+		if len(api) > maxCSPBatch {
+			api = api[:maxCSPBatch]
+		}
+		for _, r := range api {
+			// The endpoint only serves CSP; skip other report types instead
+			// of guessing at their bodies, and drop empty entries.
+			if r.Type != "" && r.Type != "csp-violation" {
+				continue
+			}
+			if r.Body.DocumentURL == "" && r.Body.BlockedURL == "" && r.Body.EffectiveDirective == "" &&
+				r.Body.ViolatedDirective == "" && r.Body.Sample == "" {
+				continue
+			}
+			reports = append(reports, storage.CSPReport{
+				DocumentURL:        clipCSP(r.Body.DocumentURL, maxCSPURL),
+				Referrer:           clipCSP(r.Body.Referrer, maxCSPURL),
+				BlockedURL:         clipCSP(r.Body.BlockedURL, maxCSPURL),
+				ViolatedDirective:  clipCSP(r.Body.ViolatedDirective, maxCSPSample),
+				EffectiveDirective: clipCSP(r.Body.EffectiveDirective, maxCSPSample),
+				OriginalPolicy:     clipCSP(r.Body.OriginalPolicy, maxCSPPolicy),
+				Disposition:        clipCSP(r.Body.Disposition, 32),
+				StatusCode:         r.Body.StatusCode,
+				Sample:             clipCSP(r.Body.Sample, maxCSPSample),
+				ReportType:         clipCSP(r.Type, 64),
+				UserAgent:          clipCSP(r.UserAgent, maxCSPSample),
+			})
+		}
+		return reports
+	}
+
+	var legacy cspReportBody
+	if err := json.Unmarshal(body, &legacy); err != nil {
+		return nil
+	}
+	v := legacy.CSPReport
+	if v.DocumentURI == "" && v.BlockedURI == "" && v.ViolatedDirective == "" {
+		return nil
+	}
+	return []storage.CSPReport{{
+		DocumentURL:        clipCSP(v.DocumentURI, maxCSPURL),
+		Referrer:           clipCSP(v.Referrer, maxCSPURL),
+		BlockedURL:         clipCSP(v.BlockedURI, maxCSPURL),
+		ViolatedDirective:  clipCSP(v.ViolatedDirective, maxCSPSample),
+		EffectiveDirective: clipCSP(v.EffectiveDirective, maxCSPSample),
+		OriginalPolicy:     clipCSP(v.OriginalPolicy, maxCSPPolicy),
+		Disposition:        clipCSP(v.Disposition, 32),
+		StatusCode:         v.StatusCode,
+		Sample:             clipCSP(v.ScriptSample, maxCSPSample),
+	}}
+}
+
+// finishCSPReport derives site/blocked host and a stable content id, and
+// stamps server-side context (time, peer IP, request UA when the report
+// carries none).
+func finishCSPReport(r storage.CSPReport, now time.Time, peerIP, requestUA string) storage.CSPReport {
+	if r.Site = storage.URLLowerHost(r.DocumentURL); r.Site == "" {
+		r.Site = "unknown"
+	}
+	if r.ReportType == "" {
+		r.ReportType = "csp-report"
+	}
+	if r.UserAgent == "" {
+		r.UserAgent = clipCSP(requestUA, maxCSPSample)
+	}
+	r.SourceIP = peerIP
+	r.Timestamp = now.UTC()
+	// Deterministic id over the report content: a stable UI key that also
+	// makes duplicate re-deliveries of one violation identifiable.
+	hash := sha256.Sum256([]byte(fmt.Sprintf("%s|%s|%s|%s|%s|%s",
+		r.Site, r.DocumentURL, r.BlockedURL, r.EffectiveDirective, r.Sample, r.Timestamp.Format(time.RFC3339Nano))))
+	r.ID = hex.EncodeToString(hash[:8])
+	return r
+}
+
+// HandleCSPIngest collects Content-Security-Policy violation reports posted
+// by browsers (report-uri / report-to). Both the legacy application/csp-report
+// object and the Reporting API's application/reports+json array are accepted.
+// Browsers cannot attach auth headers, so like /v1/vitals the optional ingest
+// key is accepted as ?key= on the configured reporting URL; CORS is limited
+// to the operator's own domains (withReportingCORS in cmd/server).
+func (c *Collector) HandleCSPIngest(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.checkIngest(w, r) {
+		return
+	}
+	if c.cspStore == nil {
+		http.Error(w, "csp report storage not configured", http.StatusServiceUnavailable)
+		return
+	}
+	// Browsers post application/csp-report or application/reports+json; some
+	// proxies rewrite the type, so decode regardless of content type.
+	raw, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxCSPBody))
+	if err != nil {
+		http.Error(w, "invalid csp report payload", http.StatusBadRequest)
+		return
+	}
+	reports := parseCSPReports(raw)
+	if len(reports) == 0 {
+		http.Error(w, "no csp reports in payload", http.StatusBadRequest)
+		return
+	}
+	now := time.Now()
+	peerIP := clientIP(r)
+	requestUA := r.UserAgent()
+	for i := range reports {
+		reports[i] = finishCSPReport(reports[i], now, peerIP, requestUA)
+	}
+	ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
+	defer cancel()
+	for _, report := range reports {
+		if err := c.cspStore.AppendCSPReport(ctx, report); err != nil {
+			log.Printf("csp: failed to store report: %v", err)
+			http.Error(w, "failed to store report", http.StatusInternalServerError)
+			return
+		}
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// HandleCSPReports serves the view's list.
+// Query: site, directive, q (substring), hours, limit.
+func (c *Collector) HandleCSPReports(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if c.cspStore == nil {
+		http.Error(w, "csp report storage not configured", http.StatusNotImplemented)
+		return
+	}
+	q := storage.CSPReportsQuery{
+		Site:      r.URL.Query().Get("site"),
+		Directive: r.URL.Query().Get("directive"),
+		Search:    r.URL.Query().Get("q"),
+		Limit:     parseLimit(r, 200),
+	}
+	if _, hours := parseStatsCommon(r); r.URL.Query().Get("hours") != "" {
+		q.Since = time.Now().Add(-time.Duration(hours) * time.Hour)
+	}
+	reports, err := c.cspStore.CSPReports(r.Context(), q)
+	if err != nil {
+		log.Printf("csp: list failed: %v", err)
+		http.Error(w, "failed to list csp reports", http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]any{"reports": reports})
+}
+
+// HandleCSPSummary serves the view's aggregates. Query: hours (default 24).
+func (c *Collector) HandleCSPSummary(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if c.cspStore == nil {
+		http.Error(w, "csp report storage not configured", http.StatusNotImplemented)
+		return
+	}
+	_, hours := parseStatsCommon(r)
+	if r.URL.Query().Get("hours") == "" {
+		hours = defaultCSPWindow
+	}
+	summary, err := c.cspStore.CSPSummary(r.Context(), hours)
+	if err != nil {
+		log.Printf("csp: summary failed: %v", err)
+		http.Error(w, "failed to summarize csp reports", http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]any{"summary": summary})
+}

internal/collector/csp_test.go

@@ -0,0 +1,187 @@
+package collector
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/storage"
+)
+
+// cspMemStore records ingested reports and serves canned query results.
+type cspMemStore struct {
+	reports []storage.CSPReport
+}
+
+func (s *cspMemStore) AppendCSPReport(_ context.Context, r storage.CSPReport) error {
+	s.reports = append(s.reports, r)
+	return nil
+}
+
+func (s *cspMemStore) CSPReports(_ context.Context, q storage.CSPReportsQuery) ([]storage.CSPReport, error) {
+	return s.reports, nil
+}
+
+func (s *cspMemStore) CSPSummary(_ context.Context, _ int) (storage.CSPSummary, error) {
+	return storage.CSPSummary{Total: uint64(len(s.reports))}, nil
+}
+
+func TestParseCSPReportsLegacy(t *testing.T) {
+	body := []byte(`{"csp-report":{
+		"document-uri":"https://app.k6n.net/page?x=1",
+		"referrer":"https://k6n.net/",
+		"blocked-uri":"https://evil.example.com/tracker.js",
+		"violated-directive":"script-src-elem",
+		"effective-directive":"script-src-elem",
+		"original-policy":"script-src 'self'; default-src 'self'",
+		"disposition":"enforce",
+		"status-code":200,
+		"script-sample":"evil()"}}`)
+	reports := parseCSPReports(body)
+	if len(reports) != 1 {
+		t.Fatalf("want 1 report, got %d", len(reports))
+	}
+	r := reports[0]
+	if r.DocumentURL != "https://app.k6n.net/page?x=1" || r.BlockedURL != "https://evil.example.com/tracker.js" {
+		t.Errorf("urls not parsed: %+v", r)
+	}
+	if r.EffectiveDirective != "script-src-elem" || r.Disposition != "enforce" || r.StatusCode != 200 {
+		t.Errorf("directives/disposition/status not parsed: %+v", r)
+	}
+	if r.OriginalPolicy != "script-src 'self'; default-src 'self'" || r.Sample != "evil()" {
+		t.Errorf("policy/sample not parsed: %+v", r)
+	}
+}
+
+func TestParseCSPReportsReportingAPI(t *testing.T) {
+	body := []byte(`[
+		{"age":1,"type":"csp-violation","url":"https://docs.tornberg.me/a","user_agent":"Mozilla/5.0",
+		 "body":{"documentURL":"https://docs.tornberg.me/a","blockedURL":"inline","effectiveDirective":"style-src",
+		          "disposition":"report","statusCode":200,"sample":"color:red","sourceFile":"x.js"}},
+		{"age":2,"type":"deprecation","body":{}}]`)
+	reports := parseCSPReports(body)
+	if len(reports) != 1 {
+		t.Fatalf("want only the csp-violation report, got %d", len(reports))
+	}
+	r := reports[0]
+	if r.EffectiveDirective != "style-src" || r.UserAgent != "Mozilla/5.0" || r.ReportType != "csp-violation" {
+		t.Errorf("reporting-api fields not parsed: %+v", r)
+	}
+}
+
+func TestParseCSPReportsRejectsGarbage(t *testing.T) {
+	for _, body := range []string{"", "not json", `{"nope":1}`, `[]`, `[{"body":{}}]`} {
+		if got := parseCSPReports([]byte(body)); got != nil {
+			t.Errorf("parseCSPReports(%q) = %+v, want nil", body, got)
+		}
+	}
+}
+
+func TestHandleCSPIngest(t *testing.T) {
+	store := &cspMemStore{}
+	c := New(nil, 10, nil)
+	c.SetCSPStore(store)
+
+	// Legacy payload via sendBeacon (text/plain, no UA in body).
+	req := httptest.NewRequest(http.MethodPost, "/v1/csp", strings.NewReader(
+		`{"csp-report":{"document-uri":"https://app.k6n.net/p","blocked-uri":"https://ads.knatofs.se/x.js",
+		  "effective-directive":"script-src","disposition":"enforce"}}`))
+	req.Header.Set("User-Agent", "TestUA/1.0")
+	req.RemoteAddr = "203.0.113.7:1234"
+	rec := httptest.NewRecorder()
+	c.HandleCSPIngest(rec, req)
+	if rec.Code != http.StatusNoContent {
+		t.Fatalf("legacy ingest: code=%d body=%s", rec.Code, rec.Body.String())
+	}
+	if len(store.reports) != 1 {
+		t.Fatalf("legacy ingest: stored %d reports", len(store.reports))
+	}
+	got := store.reports[0]
+	if got.Site != "app.k6n.net" || got.SourceIP != "203.0.113.7" {
+		t.Errorf("site/source ip not derived: %+v", got)
+	}
+	if got.UserAgent != "TestUA/1.0" || got.ReportType != "csp-report" || got.ID == "" {
+		t.Errorf("ua/type/id not finished: %+v", got)
+	}
+
+	// Reporting API batch.
+	req = httptest.NewRequest(http.MethodPost, "/v1/csp", strings.NewReader(
+		`[{"type":"csp-violation","body":{"documentURL":"https://www.tornberg.me/","blockedURL":"https://b.example/"}}]`))
+	req.Header.Set("X-Forwarded-For", "198.51.100.9, 10.0.0.1")
+	rec = httptest.NewRecorder()
+	c.HandleCSPIngest(rec, req)
+	if rec.Code != http.StatusNoContent || len(store.reports) != 2 {
+		t.Fatalf("api ingest: code=%d stored=%d", rec.Code, len(store.reports))
+	}
+	if got := store.reports[1]; got.Site != "www.tornberg.me" || got.SourceIP != "198.51.100.9" {
+		t.Errorf("api ingest normalization: %+v", got)
+	}
+
+	// Garbage is rejected; reads stay 405.
+	if rec := post(c, "/v1/csp", "garbage"); rec.Code != http.StatusBadRequest {
+		t.Errorf("garbage: code=%d", rec.Code)
+	}
+	getRec := httptest.NewRecorder()
+	c.HandleCSPIngest(getRec, httptest.NewRequest(http.MethodGet, "/v1/csp", nil))
+	if getRec.Code != http.StatusMethodNotAllowed {
+		t.Errorf("GET: code=%d", getRec.Code)
+	}
+}
+
+func post(c *Collector, path, body string) *httptest.ResponseRecorder {
+	rec := httptest.NewRecorder()
+	c.HandleCSPIngest(rec, httptest.NewRequest(http.MethodPost, path, strings.NewReader(body)))
+	return rec
+}
+
+func TestHandleCSPViews(t *testing.T) {
+	store := &cspMemStore{reports: []storage.CSPReport{{
+		ID: "abc", Timestamp: time.Now(), Site: "app.k6n.net", EffectiveDirective: "script-src",
+	}}}
+	c := New(nil, 10, nil)
+	c.SetCSPStore(store)
+
+	rec := httptest.NewRecorder()
+	c.HandleCSPReports(rec, httptest.NewRequest(http.MethodGet, "/api/csp/reports?site=app.k6n.net&hours=24", nil))
+	if rec.Code != http.StatusOK {
+		t.Fatalf("reports: code=%d", rec.Code)
+	}
+	var list struct {
+		Reports []storage.CSPReport `json:"reports"`
+	}
+	if err := json.Unmarshal(rec.Body.Bytes(), &list); err != nil || len(list.Reports) != 1 {
+		t.Fatalf("reports: err=%v body=%s", err, rec.Body.String())
+	}
+
+	rec = httptest.NewRecorder()
+	c.HandleCSPSummary(rec, httptest.NewRequest(http.MethodGet, "/api/csp/summary?hours=168", nil))
+	if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"total":1`) {
+		t.Fatalf("summary: code=%d body=%s", rec.Code, rec.Body.String())
+	}
+
+	// Without a store both views answer 501 politely.
+	c2 := New(nil, 10, nil)
+	rec = httptest.NewRecorder()
+	c2.HandleCSPReports(rec, httptest.NewRequest(http.MethodGet, "/api/csp/reports", nil))
+	if rec.Code != http.StatusNotImplemented {
+		t.Fatalf("no store: code=%d", rec.Code)
+	}
+}
+
+func TestFinishCSPReportIDStable(t *testing.T) {
+	now := time.Date(2025, 1, 2, 3, 4, 5, 0, time.UTC)
+	a := finishCSPReport(storage.CSPReport{DocumentURL: "https://a.k6n.net/", BlockedURL: "https://b/"}, now, "1.2.3.4", "UA")
+	b := finishCSPReport(storage.CSPReport{DocumentURL: "https://a.k6n.net/", BlockedURL: "https://b/"}, now, "5.6.7.8", "Other")
+	if a.ID != b.ID {
+		// The id intentionally covers report content only, not delivery
+		// context, so duplicate re-deliveries share it.
+		t.Errorf("ids differ across delivery context: %s vs %s", a.ID, b.ID)
+	}
+	if a.Site != "a.k6n.net" || a.Timestamp != now.UTC() {
+		t.Errorf("finish: %+v", a)
+	}
+}