platform/catalog: generic ProjectionStore[T] consumer + race-free holder; finish C1

- Generic SnapshotWriter[T]/SnapshotReader[T] over a value-agnostic raw layer;
  SnapshotHolder[T] now RWMutex-guards munmap vs in-flight Get (race-free swap,
  TestSnapshotHolder_SwapRace) — the drain grace is now just close-batching.
- ProjectionStore[T]: snapshot begin|chunk|end assembly → atomic swap, epoch-gated
  delta overlay, tombstones, atomic epoch sidecar (ordered before the swap),
  cold-start reopen, mandatory ProjectionTransform[T]. Future-epoch deltas are
  dropped by contract (recover via next snapshot), not buffered.
- Shared bus-framing constants so producer + consumer agree.
- Bump slask-finder / go-cart-actor / backoffice pointers; refresh status.md +
  catalog-projection.md to the finished state.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
matsandClaude Opus 4.8 committed 2026-06-28 21:07:39 +02:00
1 parent 5a3ffcb6bc
commit 8b454bccac
8 files changed
+959 -170

No files matched your search

+323
View File
@@ -0,0 +1,323 @@
package catalog
// ProjectionStore — the consumer half of the projection seam (see
// docs/catalog-projection.md). A commerce service (cart, checkout, order, UCP)
// feeds it the catalog.projection_published frames it receives off the bus and
// reads catalog facts back by SKU. The store owns a LOCAL read-only projection:
//
// - A full snapshot stream (begin → chunk* → end, one MetaEpoch) is assembled
// into the store's OWN .bin (SnapshotWriter), atomically swapped in as the
// mmap base (SnapshotHolder). The base is the cold-start substrate — it
// survives a restart (reopened from disk), so the consumer serves reads
// immediately without waiting for a bus replay.
// - Per-batch deltas of the SAME epoch are applied to an in-memory overlay on
// top of the base (overlay wins; a delete leaves a tombstone that shadows the
// base and the fallback). The next snapshot folds the overlay into a new base.
//
// Read order on Get: overlay → mmap base → optional caller fallback (e.g. an
// HTTP fetch from the reader) — last-resort only. The access pattern is point
// lookup by SKU; nothing enumerates the catalog.
//
// No shared volume: each consumer builds its own base from the bus. Concurrency:
// HandleFrame is single-writer (drive it from one delivery goroutine); Get is
// safe from many readers.
import (
"fmt"
"log"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
)
// ProjectionFallback resolves an SKU the local store doesn't have yet (cold
// start before the first snapshot, or an item added since the last base). It
// returns a raw Projection — the store applies the SAME Transform to it, so a
// fallback hit is shaped identically to bus-fed data. Return (_, false) to
// report a genuine miss.
type ProjectionFallback func(sku string) (Projection, bool)
// ProjectionTransform projects the wire Projection down to the consumer's own
// stored type T. It is MANDATORY: it is the one place a consumer declares which
// catalog facts it actually needs (and any derivation, e.g. tax_class → rate),
// so its local .bin holds only that — smaller, and read-ready with no per-Get
// conversion. Every value entering the store — snapshot, delta, and fallback —
// passes through it exactly once.
type ProjectionTransform[T any] func(Projection) T
// ProjectionStore is a consumer's local, cold-start-ready projection cache of T.
type ProjectionStore[T any] struct {
dir, name string
transform ProjectionTransform[T]
fallback ProjectionFallback
holder *SnapshotHolder[T] // mmap base; atomic-swap + drain-grace
mu sync.RWMutex
epoch int64 // epoch of the live base
overlay map[string]overlayEntry[T] // same-epoch deltas layered on the base
// in-progress snapshot assembly (one epoch at a time)
building *SnapshotWriter[T]
buildingEpoch int64
buildingSeen int
}
type overlayEntry[T any] struct {
val T
deleted bool
}
// NewProjectionStore opens a store of T backed by dir/name (its own .bin).
// transform is REQUIRED — it defines the consumer's projection shape (see
// ProjectionTransform). If a base from a previous run is present it is reopened
// immediately (cold-start ready); its epoch is read from the sidecar so
// same-epoch deltas can resume. fallback may be nil. Close() when done.
func NewProjectionStore[T any](dir, name string, transform ProjectionTransform[T], fallback ProjectionFallback) (*ProjectionStore[T], error) {
if dir == "" || name == "" {
return nil, fmt.Errorf("catalog: ProjectionStore needs dir and name")
}
if transform == nil {
return nil, fmt.Errorf("catalog: ProjectionStore needs a transform (declare which catalog facts this consumer stores)")
}
s := &ProjectionStore[T]{
dir: dir,
name: name,
transform: transform,
fallback: fallback,
holder: NewSnapshotHolder[T](0),
overlay: make(map[string]overlayEntry[T]),
}
path := filepath.Join(dir, name)
if err := s.holder.Open(path); err == nil {
s.epoch = readEpochSidecar(path)
log.Printf("catalog: ProjectionStore reopened base %s (epoch=%d, %d entries)", path, s.epoch, s.holder.Len())
} else if !os.IsNotExist(err) {
// A corrupt/partial base on disk shouldn't crash boot — start empty and
// wait for the next snapshot; reads fall back meanwhile.
log.Printf("catalog: ProjectionStore ignoring unreadable base %s: %v", path, err)
}
return s, nil
}
// HandleFrame applies one catalog.projection_published event (snapshot frame or
// delta), routed by its Meta. Drive this from a single delivery goroutine. A
// returned error means the payload was undecodable (the caller may NACK); a
// frame that simply doesn't apply (stale epoch, wrong assembly state) returns
// nil so the delivery is acked.
func (s *ProjectionStore[T]) HandleFrame(meta map[string]string, payload []byte) error {
epoch, _ := strconv.ParseInt(meta[MetaEpoch], 10, 64)
switch meta[MetaSnapshotPhase] {
case SnapshotBegin:
return s.beginSnapshot(epoch, meta[MetaTenant])
case SnapshotChunk:
return s.chunkSnapshot(epoch, payload)
case SnapshotEnd:
return s.endSnapshot(epoch, meta)
case "":
return s.applyDelta(epoch, payload)
default:
return nil // unknown phase — ignore forward-compatibly
}
}
func (s *ProjectionStore[T]) beginSnapshot(epoch int64, tenant string) error {
s.mu.Lock()
defer s.mu.Unlock()
if epoch <= s.epoch {
return nil // we already hold this base or newer
}
// Start (or restart) assembly for this epoch. A new begin supersedes any
// half-built snapshot (e.g. a producer that aborted mid-stream).
s.building = NewSnapshotWriter[T]().WithTenant(tenant)
s.buildingEpoch = epoch
s.buildingSeen = 0
return nil
}
func (s *ProjectionStore[T]) chunkSnapshot(epoch int64, payload []byte) error {
ups, err := DecodeUpdates(payload)
if err != nil {
return fmt.Errorf("catalog: decode snapshot chunk: %w", err)
}
s.mu.Lock()
defer s.mu.Unlock()
if s.building == nil || epoch != s.buildingEpoch {
return nil // not assembling this epoch
}
for _, u := range ups {
if u.Deleted || u.SKU == "" {
continue // a snapshot carries live items only
}
// Transform applied here — the base stores the consumer's shape, not the
// full Projection.
if err := s.building.Append(u.SKU, s.transform(u.Projection)); err != nil {
log.Printf("catalog: ProjectionStore skip %q in snapshot: %v", u.SKU, err)
continue
}
s.buildingSeen++
}
return nil
}
func (s *ProjectionStore[T]) endSnapshot(epoch int64, meta map[string]string) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.building == nil || epoch != s.buildingEpoch {
return nil
}
w := s.building
s.building = nil
// Completeness guard: if begin/end advertised a count and we appended fewer
// (a lost chunk), DON'T swap — keep the old base and wait for the next
// snapshot. Serving an incomplete catalog silently is worse than stale.
if cs := meta[MetaCount]; cs != "" {
if want, perr := strconv.Atoi(cs); perr == nil && want != s.buildingSeen {
log.Printf("catalog: ProjectionStore snapshot epoch=%d incomplete (%d/%d) — discarding, keeping base epoch=%d",
epoch, s.buildingSeen, want, s.epoch)
return nil
}
}
path, err := w.Write(s.dir, s.name)
if err != nil {
return fmt.Errorf("catalog: write snapshot base: %w", err)
}
// Persist the epoch sidecar (atomic) BEFORE swapping the in-memory reader, so
// a reopened store's epoch always matches the .bin already on disk (w.Write
// renamed the new file into place above). Writing it after Reload would leave
// a crash window where the new .bin carries a stale/absent epoch — the store
// would then mislabel its base and drop the current epoch's deltas on restart.
writeEpochSidecar(path, epoch)
if err := s.holder.Reload(path); err != nil {
return fmt.Errorf("catalog: reload snapshot base: %w", err)
}
s.epoch = epoch
s.overlay = make(map[string]overlayEntry[T]) // base now includes everything ≤ epoch
log.Printf("catalog: ProjectionStore base swapped epoch=%d (%d entries)", epoch, s.buildingSeen)
return nil
}
func (s *ProjectionStore[T]) applyDelta(epoch int64, payload []byte) error {
ups, err := DecodeUpdates(payload)
if err != nil {
return fmt.Errorf("catalog: decode delta: %w", err)
}
s.mu.Lock()
defer s.mu.Unlock()
switch {
case s.building != nil:
// Under FIFO delivery a delta won't interleave a snapshot stream; if it
// does, the snapshot is authoritative for its epoch, so drop the delta.
return nil
case epoch < s.epoch:
return nil // stale — predates our base
case epoch > s.epoch:
// CONTRACT: a future-epoch delta is DROPPED, not buffered. Under FIFO
// per-queue delivery begin(N) always precedes delta(N), so this only
// happens after a reconnect that missed the whole snapshot — which a
// buffer couldn't reassemble anyway. Recovery is the next snapshot of
// this epoch; the consumer's fallback covers reads until then.
return nil
}
for _, u := range ups {
if u.SKU == "" {
continue
}
if u.Deleted {
s.overlay[u.SKU] = overlayEntry[T]{deleted: true}
continue
}
// Transform applied here too — overlay holds the consumer's shape.
s.overlay[u.SKU] = overlayEntry[T]{val: s.transform(u.Projection)}
}
return nil
}
// Get resolves a value by SKU: overlay (incl. tombstone) → mmap base →
// fallback. A tombstoned SKU returns (zero, false) and never reaches the
// fallback — the catalog explicitly removed it. A fallback hit is run through
// the same Transform, so it's shaped like everything else.
func (s *ProjectionStore[T]) Get(sku string) (T, bool) {
var zero T
s.mu.RLock()
e, ok := s.overlay[sku]
s.mu.RUnlock()
if ok {
if e.deleted {
return zero, false
}
return e.val, true
}
if v, ok := s.holder.Get(sku); ok {
return v, true
}
if s.fallback != nil {
if p, ok := s.fallback(sku); ok {
return s.transform(p), true
}
}
return zero, false
}
// IsDeleted reports whether the overlay holds a tombstone for sku — a delete
// seen on the bus since the current base. Lets a caller skip a downstream fetch
// for an SKU the catalog just removed. After a snapshot folds the delete into a
// new base the SKU is simply absent (IsDeleted → false); "absent" is then the
// caller's fetch/miss path, which a deleted item resolves as a downstream 404.
func (s *ProjectionStore[T]) IsDeleted(sku string) bool {
s.mu.RLock()
defer s.mu.RUnlock()
e, ok := s.overlay[sku]
return ok && e.deleted
}
// Epoch returns the epoch of the current base (0 if none yet).
func (s *ProjectionStore[T]) Epoch() int64 {
s.mu.RLock()
defer s.mu.RUnlock()
return s.epoch
}
// Stats returns observability counters (base size, overlay size, epoch).
func (s *ProjectionStore[T]) Stats() (epoch int64, baseCount, overlayCount int) {
s.mu.RLock()
defer s.mu.RUnlock()
return s.epoch, s.holder.Len(), len(s.overlay)
}
// Close releases the mmap base.
func (s *ProjectionStore[T]) Close() error { return s.holder.Close() }
// ── epoch sidecar ────────────────────────────────────────────────────────────
// The .bin header has no room for the epoch, so it's persisted next to it as
// "<name>.epoch" — read on boot so a reopened base can resume same-epoch deltas.
func epochSidecarPath(binPath string) string { return binPath + ".epoch" }
// writeEpochSidecar writes the epoch atomically (temp+rename) so a crash mid-write
// can't leave a torn file that parses as a bogus epoch — a reader either sees the
// old epoch or the new one, never garbage.
func writeEpochSidecar(binPath string, epoch int64) {
p := epochSidecarPath(binPath)
tmp := p + ".tmp"
if err := os.WriteFile(tmp, []byte(strconv.FormatInt(epoch, 10)), 0o644); err != nil {
log.Printf("catalog: ProjectionStore epoch sidecar write: %v", err)
return
}
if err := os.Rename(tmp, p); err != nil {
log.Printf("catalog: ProjectionStore epoch sidecar rename: %v", err)
_ = os.Remove(tmp)
}
}
func readEpochSidecar(binPath string) int64 {
b, err := os.ReadFile(epochSidecarPath(binPath))
if err != nil {
return 0
}
n, _ := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64)
return n
}
+225
View File
@@ -0,0 +1,225 @@
package catalog
import (
"strconv"
"testing"
"git.k6n.net/mats/platform/money"
)
func snapMeta(phase string, epoch int64, seq, count int) map[string]string {
m := map[string]string{MetaEpoch: strconv.FormatInt(epoch, 10), MetaTenant: "SE"}
if phase != "" {
m[MetaSnapshotPhase] = phase
m[MetaSeq] = strconv.Itoa(seq)
}
if phase == SnapshotBegin || phase == SnapshotEnd {
m[MetaCount] = strconv.Itoa(count)
}
return m
}
func mustEnc(t *testing.T, ups []ProjectionUpdate) []byte {
t.Helper()
b, err := EncodeUpdates(ups)
if err != nil {
t.Fatalf("encode: %v", err)
}
return b
}
// identity is the transform used by tests: store the full Projection (T = Projection).
func identity(p Projection) Projection { return p }
// emitSnapshot drives a full begin→chunk→end stream of projs at epoch.
func emitSnapshot(t *testing.T, s *ProjectionStore[Projection], epoch int64, projs []Projection) {
t.Helper()
if err := s.HandleFrame(snapMeta(SnapshotBegin, epoch, 0, len(projs)), mustEnc(t, nil)); err != nil {
t.Fatalf("begin: %v", err)
}
ups := make([]ProjectionUpdate, len(projs))
for i, p := range projs {
ups[i] = ProjectionUpdate{Projection: p}
}
if err := s.HandleFrame(snapMeta(SnapshotChunk, epoch, 0, 0), mustEnc(t, ups)); err != nil {
t.Fatalf("chunk: %v", err)
}
if err := s.HandleFrame(snapMeta(SnapshotEnd, epoch, 1, len(projs)), mustEnc(t, nil)); err != nil {
t.Fatalf("end: %v", err)
}
}
func proj(sku string, price money.Cents) Projection {
return Projection{ID: "sku:" + sku, SKU: sku, PriceIncVat: price}
}
func newStore(t *testing.T, fb ProjectionFallback) *ProjectionStore[Projection] {
t.Helper()
s, err := NewProjectionStore(t.TempDir(), "consumer.bin", identity, fb)
if err != nil {
t.Fatalf("new store: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
return s
}
func TestProjectionStore_SnapshotThenGet(t *testing.T) {
s := newStore(t, nil)
emitSnapshot(t, s, 100, []Projection{proj("A", 100_00), proj("B", 150_00)})
if got, ok := s.Get("A"); !ok || got.PriceIncVat != 100_00 {
t.Fatalf("A = %+v ok=%v, want 10000", got, ok)
}
if e, base, ov := s.Stats(); e != 100 || base != 2 || ov != 0 {
t.Fatalf("stats = epoch %d base %d overlay %d, want 100/2/0", e, base, ov)
}
}
func TestProjectionStore_DeltaOverlayWinsOverBase(t *testing.T) {
s := newStore(t, nil)
emitSnapshot(t, s, 100, []Projection{proj("A", 100_00)})
// Same-epoch delta with a fresher price overrides the base.
if err := s.HandleFrame(snapMeta("", 100, 0, 0), mustEnc(t, []ProjectionUpdate{{Projection: proj("A", 80_00)}})); err != nil {
t.Fatalf("delta: %v", err)
}
if got, _ := s.Get("A"); got.PriceIncVat != 80_00 {
t.Fatalf("A after delta = %d, want 8000", got.PriceIncVat)
}
}
func TestProjectionStore_DeltaTombstoneShadowsBaseAndFallback(t *testing.T) {
fbHits := 0
s := newStore(t, func(string) (Projection, bool) { fbHits++; return proj("A", 1), true })
emitSnapshot(t, s, 100, []Projection{proj("A", 100_00)})
if err := s.HandleFrame(snapMeta("", 100, 0, 0), mustEnc(t, []ProjectionUpdate{{Projection: Projection{SKU: "A"}, Deleted: true}})); err != nil {
t.Fatalf("delete: %v", err)
}
if got, ok := s.Get("A"); ok {
t.Fatalf("A after tombstone = %+v, want miss (no resurrect, no fallback)", got)
}
if fbHits != 0 {
t.Fatalf("fallback hit %d times for a tombstoned SKU, want 0", fbHits)
}
}
func TestProjectionStore_FallbackOnMiss(t *testing.T) {
s := newStore(t, func(sku string) (Projection, bool) {
if sku == "Z" {
return proj("Z", 42), true
}
return Projection{}, false
})
emitSnapshot(t, s, 100, []Projection{proj("A", 100_00)})
if got, ok := s.Get("Z"); !ok || got.PriceIncVat != 42 {
t.Fatalf("Z = %+v ok=%v, want fallback 42", got, ok)
}
if _, ok := s.Get("nope"); ok {
t.Fatalf("unknown SKU should miss")
}
}
func TestProjectionStore_StaleEpochIgnored(t *testing.T) {
s := newStore(t, nil)
emitSnapshot(t, s, 200, []Projection{proj("A", 100_00)})
// An older snapshot must not replace the newer base.
emitSnapshot(t, s, 100, []Projection{proj("A", 999_00), proj("OLD", 1)})
if got, _ := s.Get("A"); got.PriceIncVat != 100_00 {
t.Fatalf("A = %d, want base epoch 200 price 10000 (stale snapshot ignored)", got.PriceIncVat)
}
if _, ok := s.Get("OLD"); ok {
t.Fatalf("OLD from a stale snapshot should not be present")
}
// An older-epoch delta is ignored too.
_ = s.HandleFrame(snapMeta("", 100, 0, 0), mustEnc(t, []ProjectionUpdate{{Projection: proj("A", 1)}}))
if got, _ := s.Get("A"); got.PriceIncVat != 100_00 {
t.Fatalf("A after stale delta = %d, want 10000", got.PriceIncVat)
}
}
// A future-epoch delta is DROPPED, by contract. Under FIFO per-queue delivery a
// delta never legitimately precedes its snapshot (the producer emits begin(N)
// before any delta(N)); the only way to see one is a reconnect that missed the
// whole snapshot, which a buffer couldn't reassemble anyway. Recovery is the next
// snapshot of that epoch (and the consumer's fallback in the meantime). See
// docs/catalog-projection.md.
func TestProjectionStore_FutureEpochDeltaDropped(t *testing.T) {
s := newStore(t, nil)
emitSnapshot(t, s, 100, []Projection{proj("A", 100_00)})
// A delta ahead of our base (gap) is dropped; only a matching snapshot applies it.
_ = s.HandleFrame(snapMeta("", 200, 0, 0), mustEnc(t, []ProjectionUpdate{{Projection: proj("A", 7)}}))
if got, _ := s.Get("A"); got.PriceIncVat != 100_00 {
t.Fatalf("A after future delta = %d, want 10000 (gap delta dropped)", got.PriceIncVat)
}
}
func TestProjectionStore_IncompleteSnapshotDiscarded(t *testing.T) {
s := newStore(t, nil)
emitSnapshot(t, s, 100, []Projection{proj("A", 100_00)})
// begin says 3, but only 1 chunked → end must discard and keep the old base.
if err := s.HandleFrame(snapMeta(SnapshotBegin, 200, 0, 3), mustEnc(t, nil)); err != nil {
t.Fatal(err)
}
if err := s.HandleFrame(snapMeta(SnapshotChunk, 200, 0, 0), mustEnc(t, []ProjectionUpdate{{Projection: proj("B", 5)}})); err != nil {
t.Fatal(err)
}
if err := s.HandleFrame(snapMeta(SnapshotEnd, 200, 1, 3), mustEnc(t, nil)); err != nil {
t.Fatal(err)
}
if e := s.Epoch(); e != 100 {
t.Fatalf("epoch = %d, want 100 (incomplete snapshot discarded)", e)
}
if _, ok := s.Get("B"); ok {
t.Fatalf("B from an incomplete snapshot must not be visible")
}
if got, _ := s.Get("A"); got.PriceIncVat != 100_00 {
t.Fatalf("A = %d, want old base intact", got.PriceIncVat)
}
}
func TestProjectionStore_ReopenFromDiskColdStart(t *testing.T) {
dir := t.TempDir()
s1, err := NewProjectionStore(dir, "consumer.bin", identity, nil)
if err != nil {
t.Fatal(err)
}
emitSnapshot(t, s1, 500, []Projection{proj("A", 100_00), proj("B", 150_00)})
_ = s1.Close()
// A fresh store on the same dir must serve immediately from the persisted base
// and resume same-epoch deltas (epoch read from the sidecar).
s2, err := NewProjectionStore(dir, "consumer.bin", identity, nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = s2.Close() })
if e := s2.Epoch(); e != 500 {
t.Fatalf("reopened epoch = %d, want 500 (from sidecar)", e)
}
if got, ok := s2.Get("A"); !ok || got.PriceIncVat != 100_00 {
t.Fatalf("A after reopen = %+v ok=%v, want 10000", got, ok)
}
if err := s2.HandleFrame(snapMeta("", 500, 0, 0), mustEnc(t, []ProjectionUpdate{{Projection: proj("A", 90_00)}})); err != nil {
t.Fatalf("post-reopen delta: %v", err)
}
if got, _ := s2.Get("A"); got.PriceIncVat != 90_00 {
t.Fatalf("A after post-reopen delta = %d, want 9000", got.PriceIncVat)
}
}
func TestProjectionStore_BadPayloadErrors(t *testing.T) {
s := newStore(t, nil)
if err := s.HandleFrame(snapMeta(SnapshotChunk, 1, 0, 0), []byte("{not json")); err == nil {
t.Fatal("chunk with bad payload should error (caller may NACK)")
}
if err := s.HandleFrame(snapMeta("", 1, 0, 0), []byte("{not json")); err == nil {
t.Fatal("delta with bad payload should error")
}
}
+24
View File
@@ -148,3 +148,27 @@ func Decode(raw []byte) (Projection, error) {
err := json.Unmarshal(raw, &p) err := json.Unmarshal(raw, &p)
return p, err return p, err
} }
// ── Bus framing contract (shared by producer + consumer) ─────────────────────
//
// catalog.projection_published carries two channels distinguished only by
// event.Event.Meta. The producer (slask-finder writeradmin) sets these keys and
// the consumer (ProjectionStore, consumer.go) reads them — one definition so the
// two never drift. See docs/catalog-projection.md.
//
// - A FULL SNAPSHOT is a framed begin → chunk* → end stream, all tagged with one
// MetaEpoch. begin/end carry MetaCount (total projections) for completeness
// validation; chunks carry the projections.
// - A DELTA has NO MetaSnapshotPhase key, and MetaEpoch names the snapshot base
// it builds on (consumers ignore a delta older than the base they hold).
const (
MetaSnapshotPhase = "snapshot" // begin|chunk|end; absent ⇒ delta
MetaEpoch = "epoch" // int64 (Unix-ms), monotonic across rebuilds
MetaSeq = "seq" // chunk sequence within a snapshot
MetaCount = "count" // total projections (begin/end only)
MetaTenant = "tenant" // namespace tag
SnapshotBegin = "begin"
SnapshotChunk = "chunk"
SnapshotEnd = "end"
)
+140 -75
View File
@@ -68,13 +68,21 @@ var (
ErrTenantTooLong = fmt.Errorf("catalog: tenant name longer than %d bytes", maxTenantLen) ErrTenantTooLong = fmt.Errorf("catalog: tenant name longer than %d bytes", maxTenantLen)
) )
// SnapshotWriter is single-shot: Append() all projections, then Write() once // The snapshot is generic over the stored value T (see SnapshotWriter[T] /
// (atomic rename-into-place so readers never see a partial file). // SnapshotReader[T] below). The binary format, sorting, mmap, and bounds-check
type SnapshotWriter struct { // logic is value-agnostic and lives in the unexported raw* layer; the generic
Tenant string // optional namespace tag (writes into header) // wrappers only marshal T → blob (write) and blob → T (read). The producer
blobs []byte // stores catalog.Projection; each consumer stores its OWN projected type via a
// transform (see ProjectionStore[T] in consumer.go) — that's the per-consumer
// read-only projection from docs/catalog-projection.md.
// rawSnapshotWriter is the value-agnostic writer: it indexes (sku → blob bytes).
// Single-shot: append() all, then write() once (atomic rename).
type rawSnapshotWriter struct {
tenant string
blobs []byte
records []snapshotRecord records []snapshotRecord
dirty bool dirty bool
} }
// snapshotRecord is the on-disk layout: SKU zero-padded to 32 bytes, then // snapshotRecord is the on-disk layout: SKU zero-padded to 32 bytes, then
@@ -85,56 +93,78 @@ type snapshotRecord struct {
BlobLength uint32 BlobLength uint32
} }
// NewSnapshotWriter returns an empty writer. Caller Appends every projection // append adds a pre-encoded blob under sku. ErrSKUTooLong if sku exceeds the
// and calls Write once. // on-disk width.
func NewSnapshotWriter() *SnapshotWriter { func (w *rawSnapshotWriter) append(sku string, blob []byte) error {
return &SnapshotWriter{} if len(sku) > maxSKULen {
}
// WithTenant tags this writer with a tenant namespace that lands in the
// snapshot header. Optional — empty tenant gets zero-padded; readers see an
// empty string when the writer was untagged.
func (w *SnapshotWriter) WithTenant(tenant string) *SnapshotWriter {
w.Tenant = tenant
return w
}
// Append encodes and adds a projection. Returns ErrSKUTooLong if SKU exceeds
// the on-disk width; returns marshal errors as wrapped fmt.Errorf.
func (w *SnapshotWriter) Append(p Projection) error {
if len(p.SKU) > maxSKULen {
return ErrSKUTooLong return ErrSKUTooLong
} }
jb, err := json.Marshal(p)
if err != nil {
return fmt.Errorf("catalog: marshal projection %q: %w", p.SKU, err)
}
var rec snapshotRecord var rec snapshotRecord
copy(rec.SKU[:], p.SKU) copy(rec.SKU[:], sku)
rec.BlobOffset = uint64(headerSize + len(w.blobs)) rec.BlobOffset = uint64(headerSize + len(w.blobs))
rec.BlobLength = uint32(len(jb)) rec.BlobLength = uint32(len(blob))
w.blobs = append(w.blobs, jb...) w.blobs = append(w.blobs, blob...)
w.records = append(w.records, rec) w.records = append(w.records, rec)
w.dirty = true w.dirty = true
return nil return nil
} }
// Len returns how many projections have been Append'd. // SnapshotWriter[T] is single-shot: Append() all values, then Write() once
func (w *SnapshotWriter) Len() int { return len(w.records) } // (atomic rename-into-place so readers never see a partial file). T is the
// stored value type (Projection for the producer; a consumer-specific type for
// a ProjectionStore[T]).
type SnapshotWriter[T any] struct {
raw rawSnapshotWriter
}
// Write serialises to a temp file in dir, fsync's, then renames into the // NewSnapshotWriter returns an empty writer for value type T. Caller Appends
// every value and calls Write once.
func NewSnapshotWriter[T any]() *SnapshotWriter[T] {
return &SnapshotWriter[T]{}
}
// WithTenant tags this writer with a tenant namespace that lands in the
// snapshot header. Optional — empty tenant gets zero-padded; readers see an
// empty string when the writer was untagged.
func (w *SnapshotWriter[T]) WithTenant(tenant string) *SnapshotWriter[T] {
w.raw.tenant = tenant
return w
}
// Append encodes v and indexes it under sku. Returns ErrSKUTooLong if sku
// exceeds the on-disk width; returns marshal errors as wrapped fmt.Errorf.
// SKU is explicit because T is opaque — the caller knows the key.
func (w *SnapshotWriter[T]) Append(sku string, v T) error {
if len(sku) > maxSKULen {
return ErrSKUTooLong
}
jb, err := json.Marshal(v)
if err != nil {
return fmt.Errorf("catalog: marshal snapshot value %q: %w", sku, err)
}
return w.raw.append(sku, jb)
}
// Len returns how many values have been Append'd.
func (w *SnapshotWriter[T]) Len() int { return len(w.raw.records) }
// Write serialises to dir/name (atomic temp+rename). See rawSnapshotWriter.write.
func (w *SnapshotWriter[T]) Write(dir, name string) (finalPath string, err error) {
return w.raw.write(dir, name)
}
// write serialises to a temp file in dir, fsync's, then renames into the
// final path. Records are sorted by SKU asc before serialisation so the // final path. Records are sorted by SKU asc before serialisation so the
// reader's binary search works. // reader's binary search works.
func (w *SnapshotWriter) Write(dir, name string) (finalPath string, err error) { func (w *rawSnapshotWriter) write(dir, name string) (finalPath string, err error) {
if w.dirty { if w.dirty {
sort.Slice(w.records, func(i, j int) bool { sort.Slice(w.records, func(i, j int) bool {
return compareSKU(w.records[i].SKU[:], w.records[j].SKU[:]) < 0 return compareSKU(w.records[i].SKU[:], w.records[j].SKU[:]) < 0
}) })
w.dirty = false w.dirty = false
} }
if len(w.Tenant) > maxTenantLen { if len(w.tenant) > maxTenantLen {
err = ErrTenantTooLong err = ErrTenantTooLong
return "", err return "", err
} }
@@ -166,7 +196,7 @@ func (w *SnapshotWriter) Write(dir, name string) (finalPath string, err error) {
binary.LittleEndian.PutUint64(hdr[16:24], uint64(len(w.records))) binary.LittleEndian.PutUint64(hdr[16:24], uint64(len(w.records)))
binary.LittleEndian.PutUint64(hdr[24:32], blobStart) binary.LittleEndian.PutUint64(hdr[24:32], blobStart)
binary.LittleEndian.PutUint64(hdr[32:40], recordStart) binary.LittleEndian.PutUint64(hdr[32:40], recordStart)
copy(hdr[40:40+len(w.Tenant)], w.Tenant) // rest is zero-pad copy(hdr[40:40+len(w.tenant)], w.tenant) // rest is zero-pad
if _, werr := f.Write(hdr[:]); werr != nil { if _, werr := f.Write(hdr[:]); werr != nil {
err = werr err = werr
@@ -251,14 +281,10 @@ func compareRecordLookup(padded, want []byte) int {
return compareSKU(padded, want) return compareSKU(padded, want)
} }
// SnapshotReader is read-only and concurrency-safe — parallel Get calls are // rawSnapshotReader is the value-agnostic reader: binary search by SKU, returns
// OK because no fields are mutated after construction. Held open via an // the raw blob bytes. Concurrency-safe for parallel getRaw; NOT safe racing
// embedded file handle so the underlying mmap stays pinned. Close uses // Close with getRaw (see package doc — use SnapshotHolder for reload).
// sync.Once so concurrent shutdown paths can't double-munmap. type rawSnapshotReader struct {
//
// IMPORTANT — see package doc: SnapshotReader is NOT safe racing Close
// concurrently with Get. Use SnapshotHolder for any reload-bearing workflow.
type SnapshotReader struct {
data []byte data []byte
count uint64 count uint64
blobStart uint64 blobStart uint64
@@ -269,14 +295,64 @@ type SnapshotReader struct {
closer func() error // munmap + close on Unix; no-op on ReadFile fallback closer func() error // munmap + close on Unix; no-op on ReadFile fallback
} }
// ReadSnapshotFile opens + parses a snapshot file. On Unix uses mmap; on // SnapshotReader[T] is the typed view over a raw snapshot: Get decodes the blob
// into T. Read-only and concurrency-safe — parallel Get calls are OK. NOT safe
// racing Close concurrently with Get; use SnapshotHolder[T] for reload.
type SnapshotReader[T any] struct {
raw *rawSnapshotReader
}
// ReadSnapshotFile opens + parses a snapshot file as a reader of T. On Unix uses
// mmap; on other platforms reads into memory (functionally identical, higher RSS).
func ReadSnapshotFile[T any](path string) (*SnapshotReader[T], error) {
rr, err := readRawSnapshotFile(path)
if err != nil {
return nil, err
}
return &SnapshotReader[T]{raw: rr}, nil
}
// NewSnapshotReader wraps an already-loaded []byte as a reader of T (use in
// tests; production paths go through ReadSnapshotFile).
func NewSnapshotReader[T any](data []byte) (*SnapshotReader[T], error) {
rr, err := newRawSnapshotReader(data)
if err != nil {
return nil, err
}
return &SnapshotReader[T]{raw: rr}, nil
}
// Get returns the value for sku, or (zero, false) if absent / undecodable.
func (r *SnapshotReader[T]) Get(sku string) (T, bool) {
var zero T
blob, ok := r.raw.getRaw(sku)
if !ok {
return zero, false
}
var v T
if err := json.Unmarshal(blob, &v); err != nil {
return zero, false
}
return v, true
}
// Len returns the number of values in this snapshot.
func (r *SnapshotReader[T]) Len() int { return r.raw.Len() }
// Tenant returns the snapshot's namespace tag ("" if untagged).
func (r *SnapshotReader[T]) Tenant() string { return r.raw.Tenant }
// Close releases the file handle and mmap region. Idempotent.
func (r *SnapshotReader[T]) Close() error { return r.raw.Close() }
// readRawSnapshotFile opens + parses a snapshot file. On Unix uses mmap; on
// other platforms reads into memory (functionally identical, higher RSS). // other platforms reads into memory (functionally identical, higher RSS).
func ReadSnapshotFile(path string) (*SnapshotReader, error) { func readRawSnapshotFile(path string) (*rawSnapshotReader, error) {
data, closer, err := loadSnapshotBytes(path) data, closer, err := loadSnapshotBytes(path)
if err != nil { if err != nil {
return nil, err return nil, err
} }
r, err := newSnapshotReader(data) r, err := newRawSnapshotReader(data)
if err != nil { if err != nil {
if closer != nil { if closer != nil {
_ = closer() _ = closer()
@@ -287,13 +363,7 @@ func ReadSnapshotFile(path string) (*SnapshotReader, error) {
return r, nil return r, nil
} }
// NewSnapshotReader wraps an already-loaded []byte (use this in tests; for func newRawSnapshotReader(data []byte) (*rawSnapshotReader, error) {
// production paths go through ReadSnapshotFile).
func NewSnapshotReader(data []byte) (*SnapshotReader, error) {
return newSnapshotReader(data)
}
func newSnapshotReader(data []byte) (*SnapshotReader, error) {
if len(data) < headerSize { if len(data) < headerSize {
return nil, ErrSnapshotTruncated return nil, ErrSnapshotTruncated
} }
@@ -355,7 +425,7 @@ func newSnapshotReader(data []byte) (*SnapshotReader, error) {
} }
tenant := string(tenantBytes[:tenantEnd]) tenant := string(tenantBytes[:tenantEnd])
return &SnapshotReader{ return &rawSnapshotReader{
data: data, data: data,
count: count, count: count,
blobStart: blobStart, blobStart: blobStart,
@@ -364,15 +434,15 @@ func newSnapshotReader(data []byte) (*SnapshotReader, error) {
}, nil }, nil
} }
// Len returns the number of Projections in this snapshot. // Len returns the number of records in this snapshot.
func (r *SnapshotReader) Len() int { return int(r.count) } func (r *rawSnapshotReader) Len() int { return int(r.count) }
// Close releases the file handle and mmap region (if any). Idempotent — safe // Close releases the file handle and mmap region (if any). Idempotent — safe
// to call from multiple goroutines (uses sync.Once under the hood). // to call from multiple goroutines (uses sync.Once under the hood).
// //
// SAFETY: this is NOT safe against concurrent Get() — see package doc. Use // SAFETY: this is NOT safe against concurrent getRaw() — see package doc. Use
// SnapshotHolder for any reload-bearing workflow. // SnapshotHolder for any reload-bearing workflow.
func (r *SnapshotReader) Close() error { func (r *rawSnapshotReader) Close() error {
var err error var err error
r.closeOnce.Do(func() { r.closeOnce.Do(func() {
if r.closer != nil { if r.closer != nil {
@@ -382,16 +452,16 @@ func (r *SnapshotReader) Close() error {
return err return err
} }
// Get returns the projection for sku, or (zero, false) if not found. // getRaw returns the blob bytes for sku, or (nil, false) if not found. The
// O(log N) — binary search on the sorted record section; on hit, reads the // returned slice aliases the mmap region — valid until Close. O(log N) binary
// JSON blob and decodes it (one allocation). // search on the sorted record section.
func (r *SnapshotReader) Get(sku string) (Projection, bool) { func (r *rawSnapshotReader) getRaw(sku string) ([]byte, bool) {
if r.count == 0 || sku == "" { if r.count == 0 || sku == "" {
return Projection{}, false return nil, false
} }
skuBytes := []byte(sku) skuBytes := []byte(sku)
if len(skuBytes) > maxSKULen { if len(skuBytes) > maxSKULen {
return Projection{}, false return nil, false
} }
lo, hi := uint64(0), r.count lo, hi := uint64(0), r.count
@@ -408,16 +478,11 @@ func (r *SnapshotReader) Get(sku string) (Projection, bool) {
default: default:
blobLen := binary.LittleEndian.Uint32(r.data[recOff+40 : recOff+44]) blobLen := binary.LittleEndian.Uint32(r.data[recOff+40 : recOff+44])
blobOff := binary.LittleEndian.Uint64(r.data[recOff+32 : recOff+40]) blobOff := binary.LittleEndian.Uint64(r.data[recOff+32 : recOff+40])
// Per-record bounds check at Open guarantees this slice is in-range // Per-record bounds check at Open guarantees this slice is in-range.
rdata := r.data[blobOff : blobOff+uint64(blobLen)] return r.data[blobOff : blobOff+uint64(blobLen)], true
var p Projection
if err := json.Unmarshal(rdata, &p); err != nil {
return Projection{}, false
}
return p, true
} }
} }
return Projection{}, false return nil, false
} }
+6
View File
@@ -13,6 +13,12 @@ func os2Stat(path string) (os.FileInfo, error) {
return os.Stat(path) return os.Stat(path)
} }
// appendP appends a Projection keyed by its own SKU — the producer/test pattern
// over the now-generic SnapshotWriter[Projection] (Append takes an explicit key).
func appendP(w *SnapshotWriter[Projection], p Projection) error {
return w.Append(p.SKU, p)
}
func repeat32(s string, n int) string { func repeat32(s string, n int) string {
out := make([]byte, 0, n*len(s)) out := make([]byte, 0, n*len(s))
for i := 0; i < n; i++ { for i := 0; i < n; i++ {
+67 -34
View File
@@ -1,12 +1,18 @@
// SnapshotHolder provides atomic hot-reload for SnapshotReader with drain // SnapshotHolder provides atomic hot-reload for SnapshotReader and solves the
// grace — solves the classic "Get concurrent with Close/munmap" race that // classic "Get concurrent with Close/munmap" race that would otherwise produce
// would otherwise produce uncatchable SIGBUS / SIGSEGV under reload. // uncatchable SIGBUS / SIGSEGV under reload.
// //
// Pattern source: slask-finder's `RawItemsStore` and similar mmap+reload // Safety is an RWMutex (readMu): Get holds RLock across the load-and-read of a
// stores, which delay unmap behind a configurable drain-grace timer so any // reader, and a reader is only Close()d (munmap) under Lock, so a reader can
// in-flight Get on the previously-current reader completes before the // never be unmapped while a Get is reading it — regardless of how long that
// underlying pages become unmapped. See docs/inventory-shape.md & the // Get's goroutine was descheduled. (A drain-grace timer alone, the original
// finder WARP doc on mmap reload for the prior art. // approach, is only best-effort: a goroutine paused longer than the grace can
// still read freed pages. TestSnapshotHolder_SwapRace reproduces that crash.)
//
// The drain grace is retained as a close-BATCHING optimization: it defers and
// coalesces reader closes onto the reaper tick instead of taking the exclusive
// Lock on every swap. Pattern source: slask-finder's mmap+reload stores.
// See docs/inventory-shape.md for the prior art.
// //
// Use: // Use:
// //
@@ -36,30 +42,39 @@ const DefaultDrainGrace = 30 * time.Second
// out from under in-flight Get()s. Set retains the previous reader behind a // out from under in-flight Get()s. Set retains the previous reader behind a
// drain timer; after the timer elapses the reaper goroutine closes the old // drain timer; after the timer elapses the reaper goroutine closes the old
// reader (and runs its munmap). // reader (and runs its munmap).
type SnapshotHolder struct { type SnapshotHolder[T any] struct {
current atomic.Pointer[SnapshotReader] current atomic.Pointer[SnapshotReader[T]]
drainsMu sync.Mutex // readMu makes reclamation race-free: Get holds RLock across the
drains []*pendingClose // Load-and-read of a reader, and a reader is only Close()d (munmap) under
// Lock — so a reader can never be unmapped while a Get is reading it, even
// if that Get's goroutine was descheduled past the drain grace. The grace
// (below) is now only close-BATCHING — defer + coalesce closes off the swap
// path — NOT the safety mechanism it was before (which was best-effort and
// could SIGSEGV under a long enough scheduler pause).
readMu sync.RWMutex
drainsMu sync.Mutex
drains []*pendingClose[T]
drainGrace time.Duration drainGrace time.Duration
stopCh chan struct{} stopCh chan struct{}
once sync.Once once sync.Once
} }
type pendingClose struct { type pendingClose[T any] struct {
r *SnapshotReader r *SnapshotReader[T]
expireAt time.Time expireAt time.Time
} }
// NewSnapshotHolder returns a holder with the given drain grace (the time // NewSnapshotHolder returns a holder of T with the given drain grace (the time
// the previous reader lives behind Set before being closed). Zero or // the previous reader lives behind Set before being closed). Zero or
// negative grace falls back to DefaultDrainGrace. // negative grace falls back to DefaultDrainGrace.
func NewSnapshotHolder(grace time.Duration) *SnapshotHolder { func NewSnapshotHolder[T any](grace time.Duration) *SnapshotHolder[T] {
if grace <= 0 { if grace <= 0 {
grace = DefaultDrainGrace grace = DefaultDrainGrace
} }
h := &SnapshotHolder{ h := &SnapshotHolder[T]{
drainGrace: grace, drainGrace: grace,
stopCh: make(chan struct{}), stopCh: make(chan struct{}),
} }
@@ -69,8 +84,8 @@ func NewSnapshotHolder(grace time.Duration) *SnapshotHolder {
// Open is shorthand for Set(ReadSnapshotFile(path)). Errors leave the // Open is shorthand for Set(ReadSnapshotFile(path)). Errors leave the
// holder's current reader unchanged. // holder's current reader unchanged.
func (h *SnapshotHolder) Open(path string) error { func (h *SnapshotHolder[T]) Open(path string) error {
r, err := ReadSnapshotFile(path) r, err := ReadSnapshotFile[T](path)
if err != nil { if err != nil {
return err return err
} }
@@ -81,8 +96,8 @@ func (h *SnapshotHolder) Open(path string) error {
// Reload atomically swaps the current reader for one freshly opened from // Reload atomically swaps the current reader for one freshly opened from
// path. The old reader (if any) is queued for close after the drain grace // path. The old reader (if any) is queued for close after the drain grace
// elapses; in-flight Get(s) on the old reader continue to work. // elapses; in-flight Get(s) on the old reader continue to work.
func (h *SnapshotHolder) Reload(path string) error { func (h *SnapshotHolder[T]) Reload(path string) error {
r, err := ReadSnapshotFile(path) r, err := ReadSnapshotFile[T](path)
if err != nil { if err != nil {
return err return err
} }
@@ -92,7 +107,7 @@ func (h *SnapshotHolder) Reload(path string) error {
// Set replaces the current reader with r. The previous reader is queued // Set replaces the current reader with r. The previous reader is queued
// for close after the drain grace elapses. r may be nil to clear. // for close after the drain grace elapses. r may be nil to clear.
func (h *SnapshotHolder) Set(r *SnapshotReader) { func (h *SnapshotHolder[T]) Set(r *SnapshotReader[T]) {
if r == nil { if r == nil {
h.current.Store(nil) h.current.Store(nil)
return return
@@ -102,7 +117,7 @@ func (h *SnapshotHolder) Set(r *SnapshotReader) {
return return
} }
h.drainsMu.Lock() h.drainsMu.Lock()
h.drains = append(h.drains, &pendingClose{ h.drains = append(h.drains, &pendingClose[T]{
r: old, r: old,
expireAt: time.Now().Add(h.drainGrace), expireAt: time.Now().Add(h.drainGrace),
}) })
@@ -110,17 +125,23 @@ func (h *SnapshotHolder) Set(r *SnapshotReader) {
} }
// Get is the concurrent-safe shim over the current reader's Get. Returns // Get is the concurrent-safe shim over the current reader's Get. Returns
// (zero, false) when the holder is empty. // (zero, false) when the holder is empty. The RLock is held across the Load AND
func (h *SnapshotHolder) Get(sku string) (Projection, bool) { // the read so a concurrent reaper can't munmap this reader mid-read.
func (h *SnapshotHolder[T]) Get(sku string) (T, bool) {
h.readMu.RLock()
defer h.readMu.RUnlock()
r := h.current.Load() r := h.current.Load()
if r == nil { if r == nil {
return Projection{}, false var zero T
return zero, false
} }
return r.Get(sku) return r.Get(sku)
} }
// Len returns the count of the current reader (0 if empty). // Len returns the count of the current reader (0 if empty).
func (h *SnapshotHolder) Len() int { func (h *SnapshotHolder[T]) Len() int {
h.readMu.RLock()
defer h.readMu.RUnlock()
r := h.current.Load() r := h.current.Load()
if r == nil { if r == nil {
return 0 return 0
@@ -131,22 +152,29 @@ func (h *SnapshotHolder) Len() int {
// Close stops the reaper goroutine and closes the current reader + any // Close stops the reaper goroutine and closes the current reader + any
// pending drains without waiting for the grace period. Safe to call // pending drains without waiting for the grace period. Safe to call
// multiple times. // multiple times.
func (h *SnapshotHolder) Close() error { func (h *SnapshotHolder[T]) Close() error {
var firstErr error var firstErr error
h.once.Do(func() { h.once.Do(func() {
close(h.stopCh) close(h.stopCh)
// Close under Lock so no in-flight Get is reading these readers' mmap
// when we munmap them.
h.readMu.Lock()
r := h.current.Swap(nil) r := h.current.Swap(nil)
if r != nil { if r != nil {
if err := r.Close(); err != nil { if err := r.Close(); err != nil {
firstErr = err firstErr = err
} }
} }
h.readMu.Unlock()
h.drainsMu.Lock() h.drainsMu.Lock()
drained := h.drains drained := h.drains
h.drains = nil h.drains = nil
h.drainsMu.Unlock() h.drainsMu.Unlock()
for _, d := range drained { for _, d := range drained {
if err := d.r.Close(); err != nil && firstErr == nil { h.readMu.Lock()
err := d.r.Close()
h.readMu.Unlock()
if err != nil && firstErr == nil {
firstErr = err firstErr = err
} }
} }
@@ -156,7 +184,7 @@ func (h *SnapshotHolder) Close() error {
// DrainCount returns the number of readers currently in the drain queue. // DrainCount returns the number of readers currently in the drain queue.
// Useful for tests and operational metrics. // Useful for tests and operational metrics.
func (h *SnapshotHolder) DrainCount() int { func (h *SnapshotHolder[T]) DrainCount() int {
h.drainsMu.Lock() h.drainsMu.Lock()
defer h.drainsMu.Unlock() defer h.drainsMu.Unlock()
return len(h.drains) return len(h.drains)
@@ -164,10 +192,10 @@ func (h *SnapshotHolder) DrainCount() int {
// reapNow scans the drain queue and Closes any readers whose grace has // reapNow scans the drain queue and Closes any readers whose grace has
// elapsed. Called by the sweeper goroutine on a tick. // elapsed. Called by the sweeper goroutine on a tick.
func (h *SnapshotHolder) reapNow() { func (h *SnapshotHolder[T]) reapNow() {
now := time.Now() now := time.Now()
var alive []*pendingClose var alive []*pendingClose[T]
var reaped []*pendingClose var reaped []*pendingClose[T]
h.drainsMu.Lock() h.drainsMu.Lock()
for _, d := range h.drains { for _, d := range h.drains {
if !d.expireAt.After(now) { if !d.expireAt.After(now) {
@@ -179,11 +207,16 @@ func (h *SnapshotHolder) reapNow() {
h.drains = alive h.drains = alive
h.drainsMu.Unlock() h.drainsMu.Unlock()
for _, d := range reaped { for _, d := range reaped {
// Lock briefly so the munmap can't race an in-flight Get on this reader.
// Gets that loaded this reader hold RLock and finish before the Lock is
// granted; Gets starting after see the swapped-in reader, never this one.
h.readMu.Lock()
_ = d.r.Close() _ = d.r.Close()
h.readMu.Unlock()
} }
} }
func (h *SnapshotHolder) sweepLoop() { func (h *SnapshotHolder[T]) sweepLoop() {
interval := h.drainGrace / 4 interval := h.drainGrace / 4
if interval < 10*time.Millisecond { if interval < 10*time.Millisecond {
interval = 10 * time.Millisecond interval = 10 * time.Millisecond
+131 -18
View File
@@ -1,11 +1,124 @@
package catalog package catalog
import ( import (
"fmt"
"path/filepath" "path/filepath"
"sync"
"sync/atomic"
"testing" "testing"
"time" "time"
"git.k6n.net/mats/platform/money"
) )
// TestSnapshotHolder_SwapRace storms the holder: 100 readers Get in tight loops
// while a rotator Reloads a fresh snapshot every 50ms. It asserts the holder's
// safety + liveness invariants under concurrency (run with -race):
//
// - No panic / SIGBUS — completion is the proof the drain grace never unmaps a
// reader while an in-flight Get is reading it.
// - No torn reads — every hit decodes to a valid Projection (SKU matches,
// price > 0). A read of a munmapped page would crash or fail to decode.
// - No gap — since every snapshot version contains every SKU, a Get must never
// miss; misses would mean the holder briefly exposed an empty/nil reader
// mid-swap. (Exact "stale beyond drain grace" timing is intentionally not
// asserted — it's timing-flaky, and the no-torn-read + no-crash checks
// already cover the drain-safety property it stands in for.)
func TestSnapshotHolder_SwapRace(t *testing.T) {
dir := t.TempDir()
const versions = 4
const skuCount = 200
paths := make([]string, versions)
for v := 0; v < versions; v++ {
w := NewSnapshotWriter[Projection]()
for i := 0; i < skuCount; i++ {
sku := fmt.Sprintf("SKU-%03d", i)
// price encodes the version (always > 0) so a torn read is detectable.
if err := w.Append(sku, Projection{ID: "sku:" + sku, SKU: sku, PriceIncVat: money.Cents(int64((v + 1) * 100))}); err != nil {
t.Fatal(err)
}
}
p, err := w.Write(dir, fmt.Sprintf("snap-%d.bin", v))
if err != nil {
t.Fatal(err)
}
paths[v] = p
}
h := NewSnapshotHolder[Projection](100 * time.Millisecond)
if err := h.Open(paths[0]); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = h.Close() })
stop := make(chan struct{})
var reads, misses int64
var torn int32
var wg sync.WaitGroup
for r := 0; r < 100; r++ {
wg.Add(1)
go func(seed int) {
defer wg.Done()
i := seed
for {
select {
case <-stop:
return
default:
}
sku := fmt.Sprintf("SKU-%03d", i%skuCount)
if p, ok := h.Get(sku); ok {
if p.SKU != sku || p.PriceIncVat <= 0 {
atomic.StoreInt32(&torn, 1)
return
}
atomic.AddInt64(&reads, 1)
} else {
atomic.AddInt64(&misses, 1)
}
i++
}
}(r)
}
wg.Add(1)
go func() {
defer wg.Done()
tick := time.NewTicker(50 * time.Millisecond)
defer tick.Stop()
v := 0
for {
select {
case <-stop:
return
case <-tick.C:
v = (v + 1) % versions
if err := h.Reload(paths[v]); err != nil {
atomic.StoreInt32(&torn, 1)
return
}
}
}
}()
time.Sleep(1500 * time.Millisecond)
close(stop)
wg.Wait()
if atomic.LoadInt32(&torn) != 0 {
t.Fatal("torn/garbage read or reload error during swap storm")
}
if m := atomic.LoadInt64(&misses); m != 0 {
t.Fatalf("got %d misses — holder exposed an empty reader mid-swap (every version has every SKU)", m)
}
if atomic.LoadInt64(&reads) == 0 {
t.Fatal("no successful reads — test did not exercise the holder")
}
t.Logf("SwapRace OK: %d reads across %d rotations, 0 torn, 0 miss", atomic.LoadInt64(&reads), versions)
}
// fakeCloser counts how many times it's been called. Used to assert the // fakeCloser counts how many times it's been called. Used to assert the
// drain-grace semantics of SnapshotHolder. // drain-grace semantics of SnapshotHolder.
type fakeCloser struct{ hits int } type fakeCloser struct{ hits int }
@@ -15,14 +128,14 @@ func (f *fakeCloser) Close() error { f.hits++; return nil }
// withFakeCloser swaps SnapshotReader's closer for our counter. Returns the // withFakeCloser swaps SnapshotReader's closer for our counter. Returns the
// reader and the counter; tests then call holder.Set(reloaded) and assert // reader and the counter; tests then call holder.Set(reloaded) and assert
// that close is invoked after the drain grace. // that close is invoked after the drain grace.
func withFakeCloser(t *testing.T, path string, fc *fakeCloser) *SnapshotReader { func withFakeCloser(t *testing.T, path string, fc *fakeCloser) *SnapshotReader[Projection] {
t.Helper() t.Helper()
r, err := ReadSnapshotFile(path) r, err := ReadSnapshotFile[Projection](path)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
// replace real closer with our counter; the test asserts only via fc // replace real closer with our counter; the test asserts only via fc
r.closer = fc.Close r.raw.closer = fc.Close
return r return r
} }
@@ -35,11 +148,11 @@ func TestSnapshotHolderReloadDrainsOld(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
// 1) Build a valid first snapshot (skus A-1 / A-2). // 1) Build a valid first snapshot (skus A-1 / A-2).
w := NewSnapshotWriter() w := NewSnapshotWriter[Projection]()
if err := w.Append(makeProjection("A-1", 100)); err != nil { if err := appendP(w, makeProjection("A-1", 100)); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := w.Append(makeProjection("A-2", 200)); err != nil { if err := appendP(w, makeProjection("A-2", 200)); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err := w.Write(dir, "first.bin"); err != nil { if _, err := w.Write(dir, "first.bin"); err != nil {
@@ -50,8 +163,8 @@ func TestSnapshotHolderReloadDrainsOld(t *testing.T) {
defer first.Close() defer first.Close()
// 2) Build a second snapshot (skus B-1). // 2) Build a second snapshot (skus B-1).
w2 := NewSnapshotWriter() w2 := NewSnapshotWriter[Projection]()
if err := w2.Append(makeProjection("B-1", 300)); err != nil { if err := appendP(w2, makeProjection("B-1", 300)); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err := w2.Write(dir, "second.bin"); err != nil { if _, err := w2.Write(dir, "second.bin"); err != nil {
@@ -61,7 +174,7 @@ func TestSnapshotHolderReloadDrainsOld(t *testing.T) {
second := withFakeCloser(t, filepath.Join(dir, "second.bin"), secondCloser) second := withFakeCloser(t, filepath.Join(dir, "second.bin"), secondCloser)
defer second.Close() defer second.Close()
h := NewSnapshotHolder(50 * time.Millisecond) h := NewSnapshotHolder[Projection](50 * time.Millisecond)
defer h.Close() defer h.Close()
h.Set(first) h.Set(first)
if h.Len() != 2 { if h.Len() != 2 {
@@ -96,7 +209,7 @@ func TestSnapshotHolderReloadDrainsOld(t *testing.T) {
// TestSnapshotHolderEmptyGet — holder with no current serves empty. // TestSnapshotHolderEmptyGet — holder with no current serves empty.
func TestSnapshotHolderEmptyGet(t *testing.T) { func TestSnapshotHolderEmptyGet(t *testing.T) {
h := NewSnapshotHolder(100 * time.Millisecond) h := NewSnapshotHolder[Projection](100 * time.Millisecond)
defer h.Close() defer h.Close()
if _, ok := h.Get("anything"); ok { if _, ok := h.Get("anything"); ok {
t.Error("empty holder should return false") t.Error("empty holder should return false")
@@ -112,8 +225,8 @@ func TestSnapshotHolderEmptyGet(t *testing.T) {
func TestSnapshotHolderCloseImmediatelyReaps(t *testing.T) { func TestSnapshotHolderCloseImmediatelyReaps(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
w := NewSnapshotWriter() w := NewSnapshotWriter[Projection]()
if err := w.Append(makeProjection("X-1", 1)); err != nil { if err := appendP(w, makeProjection("X-1", 1)); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err := w.Write(dir, "x.bin"); err != nil { if _, err := w.Write(dir, "x.bin"); err != nil {
@@ -122,8 +235,8 @@ func TestSnapshotHolderCloseImmediatelyReaps(t *testing.T) {
fc1 := &fakeCloser{} fc1 := &fakeCloser{}
r1 := withFakeCloser(t, filepath.Join(dir, "x.bin"), fc1) r1 := withFakeCloser(t, filepath.Join(dir, "x.bin"), fc1)
w2 := NewSnapshotWriter() w2 := NewSnapshotWriter[Projection]()
if err := w2.Append(makeProjection("Y-1", 2)); err != nil { if err := appendP(w2, makeProjection("Y-1", 2)); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err := w2.Write(dir, "y.bin"); err != nil { if _, err := w2.Write(dir, "y.bin"); err != nil {
@@ -132,7 +245,7 @@ func TestSnapshotHolderCloseImmediatelyReaps(t *testing.T) {
fc2 := &fakeCloser{} fc2 := &fakeCloser{}
r2 := withFakeCloser(t, filepath.Join(dir, "y.bin"), fc2) r2 := withFakeCloser(t, filepath.Join(dir, "y.bin"), fc2)
h := NewSnapshotHolder(10 * time.Second) // long grace so reap wouldn't happen on its own h := NewSnapshotHolder[Projection](10 * time.Second) // long grace so reap wouldn't happen on its own
h.Set(r1) h.Set(r1)
h.Set(r2) h.Set(r2)
@@ -152,15 +265,15 @@ func TestSnapshotHolderCloseImmediatelyReaps(t *testing.T) {
// until grace elapses. // until grace elapses.
func TestSnapshotHolderIdentityReload(t *testing.T) { func TestSnapshotHolderIdentityReload(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
w := NewSnapshotWriter() w := NewSnapshotWriter[Projection]()
if err := w.Append(makeProjection("ID-1", 1)); err != nil { if err := appendP(w, makeProjection("ID-1", 1)); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err := w.Write(dir, "id.bin"); err != nil { if _, err := w.Write(dir, "id.bin"); err != nil {
t.Fatal(err) t.Fatal(err)
} }
h := NewSnapshotHolder(10 * time.Second) h := NewSnapshotHolder[Projection](10 * time.Second)
defer h.Close() defer h.Close()
if err := h.Reload(filepath.Join(dir, "id.bin")); err != nil { if err := h.Reload(filepath.Join(dir, "id.bin")); err != nil {
t.Fatal(err) t.Fatal(err)
+43 -43
View File
@@ -20,12 +20,12 @@ func TestSnapshotRoundTrip(t *testing.T) {
path := filepath.Join(dir, "snap.bin") path := filepath.Join(dir, "snap.bin")
const n = 1000 const n = 1000
w := NewSnapshotWriter().WithTenant("default") w := NewSnapshotWriter[Projection]().WithTenant("default")
// intentionally shuffled input — sort is what reorders at Write. // intentionally shuffled input — sort is what reorders at Write.
perm := rand.Perm(n) perm := rand.Perm(n)
for _, i := range perm { for _, i := range perm {
sku := fmt.Sprintf("SKU-%05d", i) sku := fmt.Sprintf("SKU-%05d", i)
if err := w.Append(makeProjection(sku, int64(100+(i%97)*100))); err != nil { if err := appendP(w, makeProjection(sku, int64(100+(i%97)*100))); err != nil {
t.Fatalf("Append %s: %v", sku, err) t.Fatalf("Append %s: %v", sku, err)
} }
} }
@@ -40,7 +40,7 @@ func TestSnapshotRoundTrip(t *testing.T) {
t.Errorf("Write returned %q expected %q", final, path) t.Errorf("Write returned %q expected %q", final, path)
} }
r, err := ReadSnapshotFile(path) r, err := ReadSnapshotFile[Projection](path)
if err != nil { if err != nil {
t.Fatalf("ReadSnapshotFile: %v", err) t.Fatalf("ReadSnapshotFile: %v", err)
} }
@@ -49,8 +49,8 @@ func TestSnapshotRoundTrip(t *testing.T) {
if r.Len() != n { if r.Len() != n {
t.Errorf("Len: got %d want %d", r.Len(), n) t.Errorf("Len: got %d want %d", r.Len(), n)
} }
if r.Tenant != "default" { if r.Tenant() != "default" {
t.Errorf("Tenant: got %q want %q", r.Tenant, "default") t.Errorf("Tenant: got %q want %q", r.Tenant(), "default")
} }
// every SKU present and correct // every SKU present and correct
@@ -79,19 +79,19 @@ func TestSnapshotRoundTrip(t *testing.T) {
// — this test pins that invariant. // — this test pins that invariant.
func TestSnapshotCompareSKUInvariant(t *testing.T) { func TestSnapshotCompareSKUInvariant(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
w := NewSnapshotWriter() w := NewSnapshotWriter[Projection]()
// deliberately inverted order after Append (so sort has work to do) // deliberately inverted order after Append (so sort has work to do)
perm := rand.Perm(500) perm := rand.Perm(500)
for _, i := range perm { for _, i := range perm {
sku := fmt.Sprintf("XX-%05d", i) sku := fmt.Sprintf("XX-%05d", i)
if err := w.Append(makeProjection(sku, int64(i))); err != nil { if err := appendP(w, makeProjection(sku, int64(i))); err != nil {
t.Fatal(err) t.Fatal(err)
} }
} }
if _, err := w.Write(dir, "snap.bin"); err != nil { if _, err := w.Write(dir, "snap.bin"); err != nil {
t.Fatal(err) t.Fatal(err)
} }
r, err := ReadSnapshotFile(filepath.Join(dir, "snap.bin")) r, err := ReadSnapshotFile[Projection](filepath.Join(dir, "snap.bin"))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -101,8 +101,8 @@ func TestSnapshotCompareSKUInvariant(t *testing.T) {
// form, and assert non-decreasing order under compareSKU. // form, and assert non-decreasing order under compareSKU.
prevSKU := "" prevSKU := ""
for i := 0; i < r.Len(); i++ { for i := 0; i < r.Len(); i++ {
recOff := int(r.recordStart) + i*recordSize recOff := int(r.raw.recordStart) + i*recordSize
raw := r.data[recOff : recOff+32] raw := r.raw.data[recOff : recOff+32]
// trim trailing zeros // trim trailing zeros
end := len(raw) end := len(raw)
for end > 0 && raw[end-1] == 0 { for end > 0 && raw[end-1] == 0 {
@@ -131,45 +131,45 @@ func runesToPadded32(s string) []byte {
// with no on-disk evidence. // with no on-disk evidence.
func TestSnapshotTenantRoundTrip(t *testing.T) { func TestSnapshotTenantRoundTrip(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
w := NewSnapshotWriter().WithTenant("bygglagret") w := NewSnapshotWriter[Projection]().WithTenant("bygglagret")
if err := w.Append(makeProjection("ABC-1", 100)); err != nil { if err := appendP(w, makeProjection("ABC-1", 100)); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err := w.Write(dir, "snap.bin"); err != nil { if _, err := w.Write(dir, "snap.bin"); err != nil {
t.Fatal(err) t.Fatal(err)
} }
r, err := ReadSnapshotFile(filepath.Join(dir, "snap.bin")) r, err := ReadSnapshotFile[Projection](filepath.Join(dir, "snap.bin"))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
defer r.Close() defer r.Close()
if r.Tenant != "bygglagret" { if r.Tenant() != "bygglagret" {
t.Errorf("Tenant: got %q want %q", r.Tenant, "bygglagret") t.Errorf("Tenant: got %q want %q", r.Tenant(), "bygglagret")
} }
// Empty tenant writes zero-padded. // Empty tenant writes zero-padded.
w2 := NewSnapshotWriter() w2 := NewSnapshotWriter[Projection]()
if err := w2.Append(makeProjection("ABC-2", 200)); err != nil { if err := appendP(w2, makeProjection("ABC-2", 200)); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err := w2.Write(dir, "snap2.bin"); err != nil { if _, err := w2.Write(dir, "snap2.bin"); err != nil {
t.Fatal(err) t.Fatal(err)
} }
r2, err := ReadSnapshotFile(filepath.Join(dir, "snap2.bin")) r2, err := ReadSnapshotFile[Projection](filepath.Join(dir, "snap2.bin"))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
defer r2.Close() defer r2.Close()
if r2.Tenant != "" { if r2.Tenant() != "" {
t.Errorf("Tenant (no tag): got %q want empty", r2.Tenant) t.Errorf("Tenant (no tag): got %q want empty", r2.Tenant())
} }
} }
// TestSnapshotTenantTooLong — over-long tenant name rejected at Write. // TestSnapshotTenantTooLong — over-long tenant name rejected at Write.
func TestSnapshotTenantTooLong(t *testing.T) { func TestSnapshotTenantTooLong(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
w := NewSnapshotWriter().WithTenant(string(make([]byte, 33))) w := NewSnapshotWriter[Projection]().WithTenant(string(make([]byte, 33)))
if err := w.Append(makeProjection("ABC", 1)); err != nil { if err := appendP(w, makeProjection("ABC", 1)); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err := w.Write(dir, "snap.bin"); err == nil { if _, err := w.Write(dir, "snap.bin"); err == nil {
@@ -196,7 +196,7 @@ func TestSnapshotCountOverflowRejected(t *testing.T) {
if err := os2WriteFile(path, bs, 0o644); err != nil { if err := os2WriteFile(path, bs, 0o644); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err := ReadSnapshotFile(path); !errors.Is(err, ErrSnapshotOverflow) { if _, err := ReadSnapshotFile[Projection](path); !errors.Is(err, ErrSnapshotOverflow) {
t.Errorf("overflow count rejected with %v, want ErrSnapshotOverflow", err) t.Errorf("overflow count rejected with %v, want ErrSnapshotOverflow", err)
} }
} }
@@ -205,10 +205,10 @@ func TestSnapshotCountOverflowRejected(t *testing.T) {
// leaves a half-written snap.bin visible to readers. // leaves a half-written snap.bin visible to readers.
func TestSnapshotAtomicSwap(t *testing.T) { func TestSnapshotAtomicSwap(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
w := NewSnapshotWriter() w := NewSnapshotWriter[Projection]()
mustAppend := func(w *SnapshotWriter, sku string, price int64) { mustAppend := func(w *SnapshotWriter[Projection], sku string, price int64) {
t.Helper() t.Helper()
if err := w.Append(makeProjection(sku, price)); err != nil { if err := appendP(w, makeProjection(sku, price)); err != nil {
t.Fatalf("Append %s: %v", sku, err) t.Fatalf("Append %s: %v", sku, err)
} }
} }
@@ -223,13 +223,13 @@ func TestSnapshotAtomicSwap(t *testing.T) {
} }
// replace // replace
w2 := NewSnapshotWriter() w2 := NewSnapshotWriter[Projection]()
mustAppend(w2, "ZZZ", 200) mustAppend(w2, "ZZZ", 200)
if _, err := w2.Write(dir, "snap.bin"); err != nil { if _, err := w2.Write(dir, "snap.bin"); err != nil {
t.Fatal(err) t.Fatal(err)
} }
r, err := ReadSnapshotFile(final) r, err := ReadSnapshotFile[Projection](final)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -251,7 +251,7 @@ func TestSnapshotRejectBadInput(t *testing.T) {
if err := os2WriteFile(path, []byte{}, 0o644); err != nil { if err := os2WriteFile(path, []byte{}, 0o644); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err := ReadSnapshotFile(path); !errors.Is(err, ErrSnapshotTruncated) { if _, err := ReadSnapshotFile[Projection](path); !errors.Is(err, ErrSnapshotTruncated) {
t.Errorf("empty file rejected with %v, want ErrSnapshotTruncated", err) t.Errorf("empty file rejected with %v, want ErrSnapshotTruncated", err)
} }
@@ -265,7 +265,7 @@ func TestSnapshotRejectBadInput(t *testing.T) {
if err := os2WriteFile(path, wrongMagic, 0o644); err != nil { if err := os2WriteFile(path, wrongMagic, 0o644); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err := ReadSnapshotFile(path); !errors.Is(err, ErrSnapshotMagic) { if _, err := ReadSnapshotFile[Projection](path); !errors.Is(err, ErrSnapshotMagic) {
t.Errorf("wrong magic rejected with %v, want ErrSnapshotMagic", err) t.Errorf("wrong magic rejected with %v, want ErrSnapshotMagic", err)
} }
@@ -274,7 +274,7 @@ func TestSnapshotRejectBadInput(t *testing.T) {
if err := os2WriteFile(path, buildValidHeader(5), 0o644); err != nil { if err := os2WriteFile(path, buildValidHeader(5), 0o644); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err := ReadSnapshotFile(path); !errors.Is(err, ErrSnapshotTruncated) { if _, err := ReadSnapshotFile[Projection](path); !errors.Is(err, ErrSnapshotTruncated) {
t.Errorf("truncated snapshot rejected with %v, want ErrSnapshotTruncated", err) t.Errorf("truncated snapshot rejected with %v, want ErrSnapshotTruncated", err)
} }
@@ -284,7 +284,7 @@ func TestSnapshotRejectBadInput(t *testing.T) {
if err := os2WriteFile(path, tampered, 0o644); err != nil { if err := os2WriteFile(path, tampered, 0o644); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err := ReadSnapshotFile(path); !errors.Is(err, ErrSnapshotBadBlob) { if _, err := ReadSnapshotFile[Projection](path); !errors.Is(err, ErrSnapshotBadBlob) {
t.Errorf("tampered-blob rejected with %v, want ErrSnapshotBadBlob", err) t.Errorf("tampered-blob rejected with %v, want ErrSnapshotBadBlob", err)
} }
} }
@@ -292,11 +292,11 @@ func TestSnapshotRejectBadInput(t *testing.T) {
// TestSnapshotEmpty — empty (count=0) snapshot is valid; Get returns false. // TestSnapshotEmpty — empty (count=0) snapshot is valid; Get returns false.
func TestSnapshotEmpty(t *testing.T) { func TestSnapshotEmpty(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
w := NewSnapshotWriter() w := NewSnapshotWriter[Projection]()
if _, err := w.Write(dir, "empty.bin"); err != nil { if _, err := w.Write(dir, "empty.bin"); err != nil {
t.Fatal(err) t.Fatal(err)
} }
r, err := ReadSnapshotFile(filepath.Join(dir, "empty.bin")) r, err := ReadSnapshotFile[Projection](filepath.Join(dir, "empty.bin"))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -316,14 +316,14 @@ func TestSnapshotSKU32(t *testing.T) {
if len(sku) != 32 { if len(sku) != 32 {
t.Fatalf("test bug: sku length %d", len(sku)) t.Fatalf("test bug: sku length %d", len(sku))
} }
w := NewSnapshotWriter() w := NewSnapshotWriter[Projection]()
if err := w.Append(makeProjection(sku, 1)); err != nil { if err := appendP(w, makeProjection(sku, 1)); err != nil {
t.Fatalf("Append 32-char SKU: %v", err) t.Fatalf("Append 32-char SKU: %v", err)
} }
if _, err := w.Write(dir, "snap.bin"); err != nil { if _, err := w.Write(dir, "snap.bin"); err != nil {
t.Fatal(err) t.Fatal(err)
} }
r, err := ReadSnapshotFile(filepath.Join(dir, "snap.bin")) r, err := ReadSnapshotFile[Projection](filepath.Join(dir, "snap.bin"))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -335,9 +335,9 @@ func TestSnapshotSKU32(t *testing.T) {
// TestSnapshotSKU33Rejected — one char over should error at Append. // TestSnapshotSKU33Rejected — one char over should error at Append.
func TestSnapshotSKU33Rejected(t *testing.T) { func TestSnapshotSKU33Rejected(t *testing.T) {
w := NewSnapshotWriter() w := NewSnapshotWriter[Projection]()
sku := repeat32("X", 33) sku := repeat32("X", 33)
err := w.Append(makeProjection(sku, 1)) err := appendP(w, makeProjection(sku, 1))
if !errors.Is(err, ErrSKUTooLong) { if !errors.Is(err, ErrSKUTooLong) {
t.Errorf("33-char SKU Append: got %v want ErrSKUTooLong", err) t.Errorf("33-char SKU Append: got %v want ErrSKUTooLong", err)
} }
@@ -351,18 +351,18 @@ func TestSnapshotGetSpeed(t *testing.T) {
} }
dir := t.TempDir() dir := t.TempDir()
const n = 5000 const n = 5000
w := NewSnapshotWriter() w := NewSnapshotWriter[Projection]()
skus := make([]string, n) skus := make([]string, n)
for i := 0; i < n; i++ { for i := 0; i < n; i++ {
skus[i] = fmt.Sprintf("Z%06d", i) skus[i] = fmt.Sprintf("Z%06d", i)
if err := w.Append(makeProjection(skus[i], int64(i))); err != nil { if err := appendP(w, makeProjection(skus[i], int64(i))); err != nil {
t.Fatalf("Append %s: %v", skus[i], err) t.Fatalf("Append %s: %v", skus[i], err)
} }
} }
if _, err := w.Write(dir, "snap.bin"); err != nil { if _, err := w.Write(dir, "snap.bin"); err != nil {
t.Fatal(err) t.Fatal(err)
} }
r, err := ReadSnapshotFile(filepath.Join(dir, "snap.bin")) r, err := ReadSnapshotFile[Projection](filepath.Join(dir, "snap.bin"))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -429,7 +429,7 @@ func TestSnapshotRejectBadVersion(t *testing.T) {
if err := os2WriteFile(path, bs, 0o644); err != nil { if err := os2WriteFile(path, bs, 0o644); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err := ReadSnapshotFile(path); !errors.Is(err, ErrSnapshotVersion) { if _, err := ReadSnapshotFile[Projection](path); !errors.Is(err, ErrSnapshotVersion) {
t.Errorf("v2-not-supported header rejected with %v, want ErrSnapshotVersion", err) t.Errorf("v2-not-supported header rejected with %v, want ErrSnapshotVersion", err)
} }
} }