ml
This commit is contained in:
mats committed
1 parent
21f04d696b
commit
8f5c267444
26 files changed
+3039
-10
No files matched your search
@@ -0,0 +1,130 @@
|
||||
// training-sample exports a bounded, time-windowed JSONL dataset from
|
||||
// normalized_events. It never writes to ClickHouse.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
)
|
||||
|
||||
type sample struct {
|
||||
EventID string `json:"eventId"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Service string `json:"service"`
|
||||
Release string `json:"release,omitempty"`
|
||||
Kind string `json:"kind"`
|
||||
Source string `json:"source"`
|
||||
IsError uint8 `json:"isError"`
|
||||
StatusClass string `json:"statusClass,omitempty"`
|
||||
StatusCode uint16 `json:"statusCode,omitempty"`
|
||||
DurationMS float64 `json:"durationMs,omitempty"`
|
||||
Route string `json:"route,omitempty"`
|
||||
SpanName string `json:"spanName,omitempty"`
|
||||
MetricName string `json:"metricName,omitempty"`
|
||||
MetricType string `json:"metricType,omitempty"`
|
||||
MetricValue float64 `json:"metricValue,omitempty"`
|
||||
MetricCount uint64 `json:"metricCount,omitempty"`
|
||||
MetricSum float64 `json:"metricSum,omitempty"`
|
||||
MetricMin float64 `json:"metricMin,omitempty"`
|
||||
MetricMax float64 `json:"metricMax,omitempty"`
|
||||
ExceptionType string `json:"exceptionType,omitempty"`
|
||||
Message string `json:"messageNormalized,omitempty"`
|
||||
StackSignature string `json:"stackSignature,omitempty"`
|
||||
TraceID string `json:"traceId,omitempty"`
|
||||
IssueFingerprint string `json:"issueFingerprint,omitempty"`
|
||||
IsRetry uint8 `json:"isRetry"`
|
||||
IsTimeout uint8 `json:"isTimeout"`
|
||||
IsCancellation uint8 `json:"isCancellation"`
|
||||
ExactSignature string `json:"exactSignature"`
|
||||
PatternSignature string `json:"patternSignature"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
addr := flag.String("addr", envOr("OTEL_CLICKHOUSE_ADDR", "127.0.0.1:9000"), "ClickHouse host:port")
|
||||
database := flag.String("database", envOr("OTEL_CLICKHOUSE_DB", "otel"), "ClickHouse database")
|
||||
user := flag.String("user", envOr("OTEL_CLICKHOUSE_USER", "default"), "ClickHouse user")
|
||||
password := flag.String("password", os.Getenv("OTEL_CLICKHOUSE_PASSWORD"), "ClickHouse password")
|
||||
since := flag.String("since", "", "inclusive UTC RFC3339 timestamp")
|
||||
until := flag.String("until", "", "exclusive UTC RFC3339 timestamp")
|
||||
maxPerPattern := flag.Int("max-per-pattern", 25, "maximum examples per pattern")
|
||||
maxEvents := flag.Int("max-events", 10000, "maximum exported events")
|
||||
output := flag.String("output", "", "write JSONL to this path instead of stdout")
|
||||
flag.Parse()
|
||||
if *since == "" || *until == "" || *maxPerPattern < 1 || *maxEvents < 1 {
|
||||
log.Fatal("-since, -until, -max-per-pattern, and -max-events are required and must be valid")
|
||||
}
|
||||
start, err := time.Parse(time.RFC3339Nano, *since)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
end, err := time.Parse(time.RFC3339Nano, *until)
|
||||
if err != nil || !start.Before(end) {
|
||||
log.Fatal("invalid training window")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{*addr}, Auth: clickhouse.Auth{Database: *database, Username: *user, Password: *password}})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if err := conn.Ping(ctx); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
writer := os.Stdout
|
||||
if *output != "" {
|
||||
file, err := os.Create(*output)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
writer = file
|
||||
}
|
||||
query := fmt.Sprintf(`SELECT event_id, timestamp, service, release, kind, source, is_error, status_class, status_code, duration_ms,
|
||||
route, span_name, metric_name, metric_type, metric_value, metric_count, metric_sum, metric_min, metric_max,
|
||||
exception_type, message_normalized, stack_signature, trace_id, issue_fingerprint, is_retry, is_timeout, is_cancellation, exact_signature, pattern_signature
|
||||
FROM %s.normalized_events FINAL
|
||||
WHERE timestamp >= toDateTime64('%s', 9, 'UTC') AND timestamp < toDateTime64('%s', 9, 'UTC')
|
||||
ORDER BY pattern_signature, timestamp
|
||||
LIMIT %d BY pattern_signature
|
||||
LIMIT %d`, *database, clickhouseTime(start), clickhouseTime(end), *maxPerPattern, *maxEvents)
|
||||
rows, err := conn.Query(ctx, query)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
encoder := json.NewEncoder(writer)
|
||||
count := 0
|
||||
for rows.Next() {
|
||||
var item sample
|
||||
if err := rows.Scan(&item.EventID, &item.Timestamp, &item.Service, &item.Release, &item.Kind, &item.Source, &item.IsError, &item.StatusClass, &item.StatusCode, &item.DurationMS, &item.Route, &item.SpanName, &item.MetricName, &item.MetricType, &item.MetricValue, &item.MetricCount, &item.MetricSum, &item.MetricMin, &item.MetricMax, &item.ExceptionType, &item.Message, &item.StackSignature, &item.TraceID, &item.IssueFingerprint, &item.IsRetry, &item.IsTimeout, &item.IsCancellation, &item.ExactSignature, &item.PatternSignature); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := encoder.Encode(item); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
count++
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Printf("exported %d training events", count)
|
||||
}
|
||||
|
||||
func clickhouseTime(value time.Time) string {
|
||||
return value.UTC().Format("2006-01-02 15:04:05.000000000")
|
||||
}
|
||||
func envOr(key, fallback string) string {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
Reference in new issue
Block a user