more

otel-ui/src/lib/ingest.ts

@@ -5,6 +5,37 @@
 import type { LogEntry, MetricData, OtelMessage, TraceData } from "./types"
 import { normalizeLogEntry, normalizeMetricData, normalizeTrace } from "./normalize"
 
+/**
+ * Live telemetry is a rolling window. Without a bound, a long-lived browser
+ * session eventually retains every span/log/metric it has ever received.
+ */
+export const STREAM_LIMITS = {
+  traces: 500,
+  logs: 1_000,
+  metrics: 250,
+  bufferedMessages: 2_000,
+  knownTraceIds: 5_000,
+  logsPerTrace: 100,
+  logLinks: 10_000,
+} as const
+
+/** Append items while retaining only the newest `limit` entries. */
+export function appendFifo<T>(previous: T[], additions: T[], limit: number): T[] {
+  if (additions.length === 0) return previous
+  const combined = previous.length + additions.length <= limit
+    ? [...previous, ...additions]
+    : [...previous, ...additions].slice(-limit)
+  return combined
+}
+
+/** Add ids in arrival order while retaining only the newest ids. */
+export function addFifoSet<T>(previous: Set<T>, additions: Iterable<T>, limit: number): Set<T> {
+  const next = new Set(previous)
+  for (const value of additions) next.add(value)
+  while (next.size > limit) next.delete(next.values().next().value as T)
+  return next
+}
+
 export interface PartitionedMessages {
   traces: TraceData[]
   metrics: MetricData[]
@@ -75,11 +106,27 @@ export function partitionMessages(messages: OtelMessage[]): PartitionedMessages
 export function mergeLogsByTraceId(
   prev: Record<string, LogEntry[]>,
   additions: Record<string, LogEntry[]>,
+  limit = STREAM_LIMITS.logsPerTrace,
+  maxTraceIds = STREAM_LIMITS.knownTraceIds,
+  maxTotalEntries = STREAM_LIMITS.logLinks,
 ): Record<string, LogEntry[]> {
   if (Object.keys(additions).length === 0) return prev
   const next = { ...prev }
   Object.entries(additions).forEach(([traceId, entries]) => {
-    next[traceId] = [...(next[traceId] ?? []), ...entries]
+    next[traceId] = appendFifo(next[traceId] ?? [], entries, limit)
   })
+  const traceIds = Object.keys(next)
+  while (traceIds.length > maxTraceIds) {
+    delete next[traceIds.shift()!]
+  }
+  let totalEntries = Object.values(next).reduce((sum, entries) => sum + entries.length, 0)
+  while (totalEntries > maxTotalEntries) {
+    const oldestTraceId = Object.keys(next)[0]
+    if (!oldestTraceId) break
+    const entries = next[oldestTraceId]
+    entries.shift()
+    totalEntries--
+    if (entries.length === 0) delete next[oldestTraceId]
+  }
   return next
 }