all the changes

cmd/normalized-backfill/main.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"git.tornberg.me/go-otel/internal/normalization"
+	"git.tornberg.me/go-otel/internal/storage"
 	"github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
 )
@@ -50,7 +51,7 @@ func main() {
 		log.Fatal(err)
 	}
 	if *write {
-		if err := createSchema(ctx, conn, *database); err != nil {
+		if err := storage.EnsureNormalizedSchema(ctx, conn, *database); err != nil {
 			log.Fatal(err)
 		}
 	}
@@ -82,7 +83,7 @@ func main() {
 	rawCount, eventCount := 0, 0
 	var batch driver.Batch
 	if *write {
-		batch, err = conn.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.normalized_events", *database))
+		batch, err = conn.PrepareBatch(ctx, storage.NormalizedInsertSQL(*database))
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -115,12 +116,12 @@ func main() {
 			}
 			counts[key].Count++
 			if *write {
-				if err := appendEvent(batch, event); err != nil {
+				if err := storage.AppendNormalizedEvent(batch, event); err != nil {
 					log.Fatal(err)
 				}
 				if eventCount%*batchSize == 0 {
 					flush()
-					batch, err = conn.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.normalized_events", *database))
+					batch, err = conn.PrepareBatch(ctx, storage.NormalizedInsertSQL(*database))
 					if err != nil {
 						log.Fatal(err)
 					}
@@ -146,70 +147,6 @@ func main() {
 	}
 }
 
-func createSchema(ctx context.Context, conn clickhouse.Conn, database string) error {
-	query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.normalized_events (
-		event_id String,
-		entry_type LowCardinality(String),
-		event_index UInt32,
-		ingest_timestamp DateTime64(9, 'UTC'),
-		timestamp DateTime64(9, 'UTC'),
-		service LowCardinality(String), release LowCardinality(String), environment LowCardinality(String),
-		kind LowCardinality(String), source LowCardinality(String), is_error UInt8,
-		status_class LowCardinality(String), status_code UInt16, duration_ms Float64,
-		route String, span_name String, metric_name String, metric_type LowCardinality(String), metric_unit String,
-		metric_value Float64, metric_count UInt64, metric_sum Float64, metric_min Float64, metric_max Float64,
-		exception_type String, message_normalized String, stack_signature String, trace_id String, issue_fingerprint String,
-		is_retry UInt8, is_timeout UInt8, is_cancellation UInt8,
-		exact_signature String, pattern_signature String, version UInt8 DEFAULT 1,
-		method String DEFAULT '', host String DEFAULT '', user_agent String DEFAULT '',
-		origin_host String DEFAULT '', referer_host String DEFAULT '', fetch_site String DEFAULT '', fetch_mode String DEFAULT ''
-	) ENGINE = ReplacingMergeTree(version)
-	PARTITION BY toYYYYMM(timestamp)
-	ORDER BY (service, kind, pattern_signature, timestamp, event_id)
-	TTL toDateTime(timestamp) + INTERVAL 180 DAY`, database)
-	if err := conn.Exec(ctx, query); err != nil {
-		return err
-	}
-	// Existing developer tables predate the explicit model flags. These
-	// additive migrations preserve historical rows with safe zero defaults.
-	for _, statement := range []string{
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS issue_fingerprint String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_retry UInt8 DEFAULT 0", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_timeout UInt8 DEFAULT 0", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_cancellation UInt8 DEFAULT 0", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS method String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS host String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS user_agent String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS origin_host String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS referer_host String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS fetch_site String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS fetch_mode String DEFAULT ''", database),
-	} {
-		if err := conn.Exec(ctx, statement); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func appendEvent(batch driver.Batch, event normalization.Event) error {
-	return batch.Append(
-		event.EventID, event.EntryType, event.EventIndex, time.Unix(0, event.IngestTimestamp), time.Unix(0, event.Timestamp),
-		event.Service, event.Release, event.Environment, event.Kind, event.Source, boolUint8(event.IsError),
-		event.StatusClass, event.StatusCode, event.DurationMS, event.Route, event.SpanName, event.MetricName, event.MetricType, event.MetricUnit,
-		event.MetricValue, event.MetricCount, event.MetricSum, event.MetricMin, event.MetricMax,
-		event.ExceptionType, event.MessageNormalized, event.StackSignature, event.TraceID, event.IssueFingerprint,
-		boolUint8(event.IsRetry), boolUint8(event.IsTimeout), boolUint8(event.IsCancellation), event.ExactSignature, event.PatternSignature,
-		uint8(1), event.Method, event.Host, event.UserAgent, event.OriginHost, event.RefererHost, event.FetchSite, event.FetchMode,
-	)
-}
-
-func boolUint8(value bool) uint8 {
-	if value {
-		return 1
-	}
-	return 0
-}
 func envOr(key, fallback string) string {
 	if value := os.Getenv(key); value != "" {
 		return value