mats/platform
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

platform/catalog: generic ProjectionStore[T] consumer + race-free holder; finish C1

Browse Source 
- Generic SnapshotWriter[T]/SnapshotReader[T] over a value-agnostic raw layer;
  SnapshotHolder[T] now RWMutex-guards munmap vs in-flight Get (race-free swap,
  TestSnapshotHolder_SwapRace) — the drain grace is now just close-batching.
- ProjectionStore[T]: snapshot begin|chunk|end assembly → atomic swap, epoch-gated
  delta overlay, tombstones, atomic epoch sidecar (ordered before the swap),
  cold-start reopen, mandatory ProjectionTransform[T]. Future-epoch deltas are
  dropped by contract (recover via next snapshot), not buffered.
- Shared bus-framing constants so producer + consumer agree.
- Bump slask-finder / go-cart-actor / backoffice pointers; refresh status.md +
  catalog-projection.md to the finished state.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
matsandClaude Opus 4.8 committed 2026-06-28 21:07:39 +02:00
1 parent 5a3ffcb6bc
commit 8b454bccac
8 files changed
+959 -170
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

catalog/snapshot.go
+140 -75
View File
@@ -68,13 +68,21 @@ var (
ErrTenantTooLong = fmt.Errorf("catalog: tenant name longer than %d bytes", maxTenantLen)
)
// SnapshotWriter is single-shot: Append() all projections, then Write() once
// (atomic rename-into-place so readers never see a partial file).
type SnapshotWriter struct {
Tenant string // optional namespace tag (writes into header)
blobs []byte
// The snapshot is generic over the stored value T (see SnapshotWriter[T] /
// SnapshotReader[T] below). The binary format, sorting, mmap, and bounds-check
// logic is value-agnostic and lives in the unexported raw* layer; the generic
// wrappers only marshal T → blob (write) and blob → T (read). The producer
// stores catalog.Projection; each consumer stores its OWN projected type via a
// transform (see ProjectionStore[T] in consumer.go) — that's the per-consumer
// read-only projection from docs/catalog-projection.md.
// rawSnapshotWriter is the value-agnostic writer: it indexes (sku → blob bytes).
// Single-shot: append() all, then write() once (atomic rename).
type rawSnapshotWriter struct {
tenant string
blobs []byte
records []snapshotRecord
dirty bool
dirty bool
}
// snapshotRecord is the on-disk layout: SKU zero-padded to 32 bytes, then
@@ -85,56 +93,78 @@ type snapshotRecord struct {
BlobLength uint32
}
// NewSnapshotWriter returns an empty writer. Caller Appends every projection
// and calls Write once.
func NewSnapshotWriter() *SnapshotWriter {
return &SnapshotWriter{}
}
// WithTenant tags this writer with a tenant namespace that lands in the
// snapshot header. Optional — empty tenant gets zero-padded; readers see an
// empty string when the writer was untagged.
func (w *SnapshotWriter) WithTenant(tenant string) *SnapshotWriter {
w.Tenant = tenant
return w
}
// Append encodes and adds a projection. Returns ErrSKUTooLong if SKU exceeds
// the on-disk width; returns marshal errors as wrapped fmt.Errorf.
func (w *SnapshotWriter) Append(p Projection) error {
if len(p.SKU) > maxSKULen {
// append adds a pre-encoded blob under sku. ErrSKUTooLong if sku exceeds the
// on-disk width.
func (w *rawSnapshotWriter) append(sku string, blob []byte) error {
if len(sku) > maxSKULen {
return ErrSKUTooLong
}
jb, err := json.Marshal(p)
if err != nil {
return fmt.Errorf("catalog: marshal projection %q: %w", p.SKU, err)
}
var rec snapshotRecord
copy(rec.SKU[:], p.SKU)
copy(rec.SKU[:], sku)
rec.BlobOffset = uint64(headerSize + len(w.blobs))
rec.BlobLength = uint32(len(jb))
rec.BlobLength = uint32(len(blob))
w.blobs = append(w.blobs, jb...)
w.blobs = append(w.blobs, blob...)
w.records = append(w.records, rec)
w.dirty = true
return nil
}
// Len returns how many projections have been Append'd.
func (w *SnapshotWriter) Len() int { return len(w.records) }
// SnapshotWriter[T] is single-shot: Append() all values, then Write() once
// (atomic rename-into-place so readers never see a partial file). T is the
// stored value type (Projection for the producer; a consumer-specific type for
// a ProjectionStore[T]).
type SnapshotWriter[T any] struct {
raw rawSnapshotWriter
}
// Write serialises to a temp file in dir, fsync's, then renames into the
// NewSnapshotWriter returns an empty writer for value type T. Caller Appends
// every value and calls Write once.
func NewSnapshotWriter[T any]() *SnapshotWriter[T] {
return &SnapshotWriter[T]{}
}
// WithTenant tags this writer with a tenant namespace that lands in the
// snapshot header. Optional — empty tenant gets zero-padded; readers see an
// empty string when the writer was untagged.
func (w *SnapshotWriter[T]) WithTenant(tenant string) *SnapshotWriter[T] {
w.raw.tenant = tenant
return w
}
// Append encodes v and indexes it under sku. Returns ErrSKUTooLong if sku
// exceeds the on-disk width; returns marshal errors as wrapped fmt.Errorf.
// SKU is explicit because T is opaque — the caller knows the key.
func (w *SnapshotWriter[T]) Append(sku string, v T) error {
if len(sku) > maxSKULen {
return ErrSKUTooLong
}
jb, err := json.Marshal(v)
if err != nil {
return fmt.Errorf("catalog: marshal snapshot value %q: %w", sku, err)
}
return w.raw.append(sku, jb)
}
// Len returns how many values have been Append'd.
func (w *SnapshotWriter[T]) Len() int { return len(w.raw.records) }
// Write serialises to dir/name (atomic temp+rename). See rawSnapshotWriter.write.
func (w *SnapshotWriter[T]) Write(dir, name string) (finalPath string, err error) {
return w.raw.write(dir, name)
}
// write serialises to a temp file in dir, fsync's, then renames into the
// final path. Records are sorted by SKU asc before serialisation so the
// reader's binary search works.
func (w *SnapshotWriter) Write(dir, name string) (finalPath string, err error) {
func (w *rawSnapshotWriter) write(dir, name string) (finalPath string, err error) {
if w.dirty {
sort.Slice(w.records, func(i, j int) bool {
return compareSKU(w.records[i].SKU[:], w.records[j].SKU[:]) < 0
})
w.dirty = false
}
if len(w.Tenant) > maxTenantLen {
if len(w.tenant) > maxTenantLen {
err = ErrTenantTooLong
return "", err
}
@@ -166,7 +196,7 @@ func (w *SnapshotWriter) Write(dir, name string) (finalPath string, err error) {
binary.LittleEndian.PutUint64(hdr[16:24], uint64(len(w.records)))
binary.LittleEndian.PutUint64(hdr[24:32], blobStart)
binary.LittleEndian.PutUint64(hdr[32:40], recordStart)
copy(hdr[40:40+len(w.Tenant)], w.Tenant) // rest is zero-pad
copy(hdr[40:40+len(w.tenant)], w.tenant) // rest is zero-pad
if _, werr := f.Write(hdr[:]); werr != nil {
err = werr
@@ -251,14 +281,10 @@ func compareRecordLookup(padded, want []byte) int {
return compareSKU(padded, want)
}
// SnapshotReader is read-only and concurrency-safe — parallel Get calls are
// OK because no fields are mutated after construction. Held open via an
// embedded file handle so the underlying mmap stays pinned. Close uses
// sync.Once so concurrent shutdown paths can't double-munmap.
//
// IMPORTANT — see package doc: SnapshotReader is NOT safe racing Close
// concurrently with Get. Use SnapshotHolder for any reload-bearing workflow.
type SnapshotReader struct {
// rawSnapshotReader is the value-agnostic reader: binary search by SKU, returns
// the raw blob bytes. Concurrency-safe for parallel getRaw; NOT safe racing
// Close with getRaw (see package doc — use SnapshotHolder for reload).
type rawSnapshotReader struct {
data []byte
count uint64
blobStart uint64
@@ -269,14 +295,64 @@ type SnapshotReader struct {
closer func() error // munmap + close on Unix; no-op on ReadFile fallback
}
// ReadSnapshotFile opens + parses a snapshot file. On Unix uses mmap; on
// SnapshotReader[T] is the typed view over a raw snapshot: Get decodes the blob
// into T. Read-only and concurrency-safe — parallel Get calls are OK. NOT safe
// racing Close concurrently with Get; use SnapshotHolder[T] for reload.
type SnapshotReader[T any] struct {
raw *rawSnapshotReader
}
// ReadSnapshotFile opens + parses a snapshot file as a reader of T. On Unix uses
// mmap; on other platforms reads into memory (functionally identical, higher RSS).
func ReadSnapshotFile[T any](path string) (*SnapshotReader[T], error) {
rr, err := readRawSnapshotFile(path)
if err != nil {
return nil, err
}
return &SnapshotReader[T]{raw: rr}, nil
}
// NewSnapshotReader wraps an already-loaded []byte as a reader of T (use in
// tests; production paths go through ReadSnapshotFile).
func NewSnapshotReader[T any](data []byte) (*SnapshotReader[T], error) {
rr, err := newRawSnapshotReader(data)
if err != nil {
return nil, err
}
return &SnapshotReader[T]{raw: rr}, nil
}
// Get returns the value for sku, or (zero, false) if absent / undecodable.
func (r *SnapshotReader[T]) Get(sku string) (T, bool) {
var zero T
blob, ok := r.raw.getRaw(sku)
if !ok {
return zero, false
}
var v T
if err := json.Unmarshal(blob, &v); err != nil {
return zero, false
}
return v, true
}
// Len returns the number of values in this snapshot.
func (r *SnapshotReader[T]) Len() int { return r.raw.Len() }
// Tenant returns the snapshot's namespace tag ("" if untagged).
func (r *SnapshotReader[T]) Tenant() string { return r.raw.Tenant }
// Close releases the file handle and mmap region. Idempotent.
func (r *SnapshotReader[T]) Close() error { return r.raw.Close() }
// readRawSnapshotFile opens + parses a snapshot file. On Unix uses mmap; on
// other platforms reads into memory (functionally identical, higher RSS).
func ReadSnapshotFile(path string) (*SnapshotReader, error) {
func readRawSnapshotFile(path string) (*rawSnapshotReader, error) {
data, closer, err := loadSnapshotBytes(path)
if err != nil {
return nil, err
}
r, err := newSnapshotReader(data)
r, err := newRawSnapshotReader(data)
if err != nil {
if closer != nil {
_ = closer()
@@ -287,13 +363,7 @@ func ReadSnapshotFile(path string) (*SnapshotReader, error) {
return r, nil
}
// NewSnapshotReader wraps an already-loaded []byte (use this in tests; for
// production paths go through ReadSnapshotFile).
func NewSnapshotReader(data []byte) (*SnapshotReader, error) {
return newSnapshotReader(data)
}
func newSnapshotReader(data []byte) (*SnapshotReader, error) {
func newRawSnapshotReader(data []byte) (*rawSnapshotReader, error) {
if len(data) < headerSize {
return nil, ErrSnapshotTruncated
}
@@ -355,7 +425,7 @@ func newSnapshotReader(data []byte) (*SnapshotReader, error) {
}
tenant := string(tenantBytes[:tenantEnd])
return &SnapshotReader{
return &rawSnapshotReader{
data: data,
count: count,
blobStart: blobStart,
@@ -364,15 +434,15 @@ func newSnapshotReader(data []byte) (*SnapshotReader, error) {
}, nil
}
// Len returns the number of Projections in this snapshot.
func (r *SnapshotReader) Len() int { return int(r.count) }
// Len returns the number of records in this snapshot.
func (r *rawSnapshotReader) Len() int { return int(r.count) }
// Close releases the file handle and mmap region (if any). Idempotent — safe
// to call from multiple goroutines (uses sync.Once under the hood).
//
// SAFETY: this is NOT safe against concurrent Get() — see package doc. Use
// SAFETY: this is NOT safe against concurrent getRaw() — see package doc. Use
// SnapshotHolder for any reload-bearing workflow.
func (r *SnapshotReader) Close() error {
func (r *rawSnapshotReader) Close() error {
var err error
r.closeOnce.Do(func() {
if r.closer != nil {
@@ -382,16 +452,16 @@ func (r *SnapshotReader) Close() error {
return err
}
// Get returns the projection for sku, or (zero, false) if not found.
// O(log N) — binary search on the sorted record section; on hit, reads the
// JSON blob and decodes it (one allocation).
func (r *SnapshotReader) Get(sku string) (Projection, bool) {
// getRaw returns the blob bytes for sku, or (nil, false) if not found. The
// returned slice aliases the mmap region — valid until Close. O(log N) binary
// search on the sorted record section.
func (r *rawSnapshotReader) getRaw(sku string) ([]byte, bool) {
if r.count == 0 || sku == "" {
return Projection{}, false
return nil, false
}
skuBytes := []byte(sku)
if len(skuBytes) > maxSKULen {
return Projection{}, false
return nil, false
}
lo, hi := uint64(0), r.count
@@ -408,16 +478,11 @@ func (r *SnapshotReader) Get(sku string) (Projection, bool) {
default:
blobLen := binary.LittleEndian.Uint32(r.data[recOff+40 : recOff+44])
blobOff := binary.LittleEndian.Uint64(r.data[recOff+32 : recOff+40])
// Per-record bounds check at Open guarantees this slice is in-range
rdata := r.data[blobOff : blobOff+uint64(blobLen)]
var p Projection
if err := json.Unmarshal(rdata, &p); err != nil {
return Projection{}, false
}
return p, true
// Per-record bounds check at Open guarantees this slice is in-range.
return r.data[blobOff : blobOff+uint64(blobLen)], true
}
}
return Projection{}, false
return nil, false
}