platform/catalog/snapshot.go

// Snapshot reader/writer for the published catalog Projection.
//
// Why a snapshot file at all? The bus is the live channel, but it doesn't
// carry the historical population on cold start. A snapshot file gives
// consumers (cart / checkout / order / UCP add-to-cart) a fast mmap-loaded
// cold-start substrate, so the first add-line request doesn't have to wait for
// the bus to replay. The bus then keeps it fresh.
//
// Format v2 (little-endian, fixed-width, mmap-friendly):
//
//	header   [72 bytes]   magic(8) version(4) reserved(4) count(8)
//	                       blobStart(8) recordStart(8) tenant(32, zero-padded)
//	blobs    variable     count sequential JSON-encoded Projection bodies, no padding
//	records  count*44     each: sku(32, zero-padded) blobOffset(8) blobLength(4);
//	                       sorted by SKU via the single compareSKU total order
//
// Tenant field at offsets 40–71 (zero-padded) names the namespace this
// snapshot covers. Independent SKU schemas can produce Projections into
// different tenant files without conflict. Hot-reload across tenants stays
// safe under SnapshotHolder (snapshot_manager.go).
//
// Sorting + binary search gives O(log N) per-SKU lookup with no on-disk hash
// table and no collisions. SKUs longer than 32 bytes are rejected at write
// time (Cart SKUs in this codebase are well under that).
//
// Memory model: the Reader holds one []byte for the entire file (mmap on
// Unix, ReadFile elsewhere). Per-SKU Get reads at most log2(N) record slots
// (each 44 B) and one JSON blob (~200 B). For 220 k projections the
// resident set is "pages touched this minute", not the whole file.
//
// SAFETY: SnapshotReader alone is NOT safe across hot-reload. During reload,
// in-flight Get() calls on the OLD reader continue to work until disposal
// of the old reader is finalised; calling SnapshotReader.Close() concurrently
// with Get() on the same reader risks unmapping pages-under-read (SIGBUS).
// Use SnapshotHolder (snapshot_manager.go) for any reload-bearing setup —
// it gives you an atomic.Pointer swap with a drain-grace (default 30s) so
// concurrent Gets on the OLD reader finish naturally before munmap fires.
package catalog

import (
	"encoding/binary"
	"encoding/json"
	"errors"
	"fmt"
	"math"
	"os"
	"path/filepath"
	"sort"
	"sync"
)

const (
	snapshotMagic   uint64 = 0x3230564a53435050 // "PCSJ002" v2 (was 0x3130564a53435050 = v1)
	snapshotVersion uint32 = 2
	maxSKULen             = 32
	maxTenantLen          = 32
	headerSize            = 8 + 4 + 4 + 8 + 8 + 8 + 32 // 72 bytes
	recordSize            = 32 + 8 + 4                 // 44 bytes per record
)

var (
	ErrSnapshotMagic     = errors.New("catalog: not a snapshot file (bad magic)")
	ErrSnapshotVersion   = errors.New("catalog: unsupported snapshot version")
	ErrSnapshotTruncated = errors.New("catalog: snapshot truncated")
	ErrSnapshotBadBlob   = errors.New("catalog: snapshot record points past file")
	ErrSnapshotOverflow  = errors.New("catalog: snapshot record-section size overflows uint64")
	ErrSKUTooLong        = fmt.Errorf("catalog: SKU longer than %d bytes", maxSKULen)
	ErrTenantTooLong     = fmt.Errorf("catalog: tenant name longer than %d bytes", maxTenantLen)
)

// The snapshot is generic over the stored value T (see SnapshotWriter[T] /
// SnapshotReader[T] below). The binary format, sorting, mmap, and bounds-check
// logic is value-agnostic and lives in the unexported raw* layer; the generic
// wrappers only marshal T → blob (write) and blob → T (read). The producer
// stores catalog.Projection; each consumer stores its OWN projected type via a
// transform (see ProjectionStore[T] in consumer.go) — that's the per-consumer
// read-only projection from docs/catalog-projection.md.

// rawSnapshotWriter is the value-agnostic writer: it indexes (sku → blob bytes).
// Single-shot: append() all, then write() once (atomic rename).
type rawSnapshotWriter struct {
	tenant  string
	blobs   []byte
	records []snapshotRecord
	dirty   bool
}

// snapshotRecord is the on-disk layout: SKU zero-padded to 32 bytes, then
// an 8-byte offset (from file start) plus a 4-byte length for the JSON blob.
type snapshotRecord struct {
	SKU        [32]byte
	BlobOffset uint64
	BlobLength uint32
}

// append adds a pre-encoded blob under sku. ErrSKUTooLong if sku exceeds the
// on-disk width.
func (w *rawSnapshotWriter) append(sku string, blob []byte) error {
	if len(sku) > maxSKULen {
		return ErrSKUTooLong
	}
	var rec snapshotRecord
	copy(rec.SKU[:], sku)
	rec.BlobOffset = uint64(headerSize + len(w.blobs))
	rec.BlobLength = uint32(len(blob))

	w.blobs = append(w.blobs, blob...)
	w.records = append(w.records, rec)
	w.dirty = true
	return nil
}

// SnapshotWriter[T] is single-shot: Append() all values, then Write() once
// (atomic rename-into-place so readers never see a partial file). T is the
// stored value type (Projection for the producer; a consumer-specific type for
// a ProjectionStore[T]).
type SnapshotWriter[T any] struct {
	raw rawSnapshotWriter
}

// NewSnapshotWriter returns an empty writer for value type T. Caller Appends
// every value and calls Write once.
func NewSnapshotWriter[T any]() *SnapshotWriter[T] {
	return &SnapshotWriter[T]{}
}

// WithTenant tags this writer with a tenant namespace that lands in the
// snapshot header. Optional — empty tenant gets zero-padded; readers see an
// empty string when the writer was untagged.
func (w *SnapshotWriter[T]) WithTenant(tenant string) *SnapshotWriter[T] {
	w.raw.tenant = tenant
	return w
}

// Append encodes v and indexes it under sku. Returns ErrSKUTooLong if sku
// exceeds the on-disk width; returns marshal errors as wrapped fmt.Errorf.
// SKU is explicit because T is opaque — the caller knows the key.
func (w *SnapshotWriter[T]) Append(sku string, v T) error {
	if len(sku) > maxSKULen {
		return ErrSKUTooLong
	}
	jb, err := json.Marshal(v)
	if err != nil {
		return fmt.Errorf("catalog: marshal snapshot value %q: %w", sku, err)
	}
	return w.raw.append(sku, jb)
}

// Len returns how many values have been Append'd.
func (w *SnapshotWriter[T]) Len() int { return len(w.raw.records) }

// Write serialises to dir/name (atomic temp+rename). See rawSnapshotWriter.write.
func (w *SnapshotWriter[T]) Write(dir, name string) (finalPath string, err error) {
	return w.raw.write(dir, name)
}

// write serialises to a temp file in dir, fsync's, then renames into the
// final path. Records are sorted by SKU asc before serialisation so the
// reader's binary search works.
func (w *rawSnapshotWriter) write(dir, name string) (finalPath string, err error) {
	if w.dirty {
		sort.Slice(w.records, func(i, j int) bool {
			return compareSKU(w.records[i].SKU[:], w.records[j].SKU[:]) < 0
		})
		w.dirty = false
	}
	if len(w.tenant) > maxTenantLen {
		err = ErrTenantTooLong
		return "", err
	}

	if err = os.MkdirAll(dir, 0o755); err != nil {
		return "", err
	}
	finalPath = filepath.Join(dir, name)
	tmp := finalPath + ".tmp"

	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
	if err != nil {
		return "", err
	}
	defer func() {
		if err != nil {
			_ = os.Remove(tmp)
			_ = f.Close()
		}
	}()

	// Header: fixed 72 bytes (8 + 4 + 4 + 8 + 8 + 8 + 32).
	blobStart := uint64(headerSize)
	recordStart := blobStart + uint64(len(w.blobs))
	var hdr [headerSize]byte
	binary.LittleEndian.PutUint64(hdr[0:8], snapshotMagic)
	binary.LittleEndian.PutUint32(hdr[8:12], snapshotVersion)
	binary.LittleEndian.PutUint32(hdr[12:16], 0)
	binary.LittleEndian.PutUint64(hdr[16:24], uint64(len(w.records)))
	binary.LittleEndian.PutUint64(hdr[24:32], blobStart)
	binary.LittleEndian.PutUint64(hdr[32:40], recordStart)
	copy(hdr[40:40+len(w.tenant)], w.tenant) // rest is zero-pad

	if _, werr := f.Write(hdr[:]); werr != nil {
		err = werr
		return "", err
	}
	if _, werr := f.Write(w.blobs); werr != nil {
		err = werr
		return "", err
	}
	recBytes := make([]byte, recordSize*len(w.records))
	for i, r := range w.records {
		off := i * recordSize
		copy(recBytes[off:off+32], r.SKU[:])
		binary.LittleEndian.PutUint64(recBytes[off+32:off+40], r.BlobOffset)
		binary.LittleEndian.PutUint32(recBytes[off+40:off+44], r.BlobLength)
	}
	if _, werr := f.Write(recBytes); werr != nil {
		err = werr
		return "", err
	}

	if werr := f.Sync(); werr != nil {
		err = werr
		return "", err
	}
	if werr := f.Close(); werr != nil {
		err = werr
		return "", err
	}
	if werr := os.Rename(tmp, finalPath); werr != nil {
		err = werr
		return "", err
	}
	return finalPath, nil
}

// compareSKU defines the SINGLE TOTAL ORDER for SKUs across both Writer
// (uses it for sort.Slice) and Reader (uses it for binary search). One
// definition; invariant test verifies the on-disk result is sorted under
// it. Source SKUs SHOULD NOT contain embedded null bytes — the padded
// comparator treats them as terminators, which silently truncates an SKU.
// Realistically no SKU has NUL.
func compareSKU(padded, other []byte) int {
	// padded is a fixed-size zero-padded slot; trim trailing zeros.
	end := len(padded)
	for end > 0 && padded[end-1] == 0 {
		end--
	}
	a := padded[:end]
	b := other

	n := len(a)
	if len(b) < n {
		n = len(b)
	}
	for i := 0; i < n; i++ {
		if a[i] != b[i] {
			if a[i] < b[i] {
				return -1
			}
			return 1
		}
	}
	switch {
	case len(a) < len(b):
		return -1
	case len(a) > len(b):
		return 1
	default:
		return 0
	}
}

// compareRecordLookup is compareSKU specialised for Reader lookups so the
// argument order matches the lex order: "padded" (fixed-slot) and "want"
// (query).
//
// compareSKU is symmetric so this is just an alias used by Reader code
// paths to make intent obvious. Any change to the total order here MUST be
// reflected in Writer.sort.Slice and verified by the invariant test.
func compareRecordLookup(padded, want []byte) int {
	return compareSKU(padded, want)
}

// rawSnapshotReader is the value-agnostic reader: binary search by SKU, returns
// the raw blob bytes. Concurrency-safe for parallel getRaw; NOT safe racing
// Close with getRaw (see package doc — use SnapshotHolder for reload).
type rawSnapshotReader struct {
	data        []byte
	count       uint64
	blobStart   uint64
	recordStart uint64
	Tenant      string // populated from header at Open time; "" if not set

	closeOnce sync.Once
	closer    func() error // munmap + close on Unix; no-op on ReadFile fallback
}

// SnapshotReader[T] is the typed view over a raw snapshot: Get decodes the blob
// into T. Read-only and concurrency-safe — parallel Get calls are OK. NOT safe
// racing Close concurrently with Get; use SnapshotHolder[T] for reload.
type SnapshotReader[T any] struct {
	raw *rawSnapshotReader
}

// ReadSnapshotFile opens + parses a snapshot file as a reader of T. On Unix uses
// mmap; on other platforms reads into memory (functionally identical, higher RSS).
func ReadSnapshotFile[T any](path string) (*SnapshotReader[T], error) {
	rr, err := readRawSnapshotFile(path)
	if err != nil {
		return nil, err
	}
	return &SnapshotReader[T]{raw: rr}, nil
}

// NewSnapshotReader wraps an already-loaded []byte as a reader of T (use in
// tests; production paths go through ReadSnapshotFile).
func NewSnapshotReader[T any](data []byte) (*SnapshotReader[T], error) {
	rr, err := newRawSnapshotReader(data)
	if err != nil {
		return nil, err
	}
	return &SnapshotReader[T]{raw: rr}, nil
}

// Get returns the value for sku, or (zero, false) if absent / undecodable.
func (r *SnapshotReader[T]) Get(sku string) (T, bool) {
	var zero T
	blob, ok := r.raw.getRaw(sku)
	if !ok {
		return zero, false
	}
	var v T
	if err := json.Unmarshal(blob, &v); err != nil {
		return zero, false
	}
	return v, true
}

// Len returns the number of values in this snapshot.
func (r *SnapshotReader[T]) Len() int { return r.raw.Len() }

// Tenant returns the snapshot's namespace tag ("" if untagged).
func (r *SnapshotReader[T]) Tenant() string { return r.raw.Tenant }

// Close releases the file handle and mmap region. Idempotent.
func (r *SnapshotReader[T]) Close() error { return r.raw.Close() }

// readRawSnapshotFile opens + parses a snapshot file. On Unix uses mmap; on
// other platforms reads into memory (functionally identical, higher RSS).
func readRawSnapshotFile(path string) (*rawSnapshotReader, error) {
	data, closer, err := loadSnapshotBytes(path)
	if err != nil {
		return nil, err
	}
	r, err := newRawSnapshotReader(data)
	if err != nil {
		if closer != nil {
			_ = closer()
		}
		return nil, err
	}
	r.closer = closer
	return r, nil
}

func newRawSnapshotReader(data []byte) (*rawSnapshotReader, error) {
	if len(data) < headerSize {
		return nil, ErrSnapshotTruncated
	}
	magic := binary.LittleEndian.Uint64(data[0:8])
	if magic != snapshotMagic {
		return nil, ErrSnapshotMagic
	}
	version := binary.LittleEndian.Uint32(data[8:12])
	if version != snapshotVersion {
		return nil, ErrSnapshotVersion
	}
	count := binary.LittleEndian.Uint64(data[16:24])
	blobStart := binary.LittleEndian.Uint64(data[24:32])
	recordStart := binary.LittleEndian.Uint64(data[32:40])
	if blobStart != uint64(headerSize) {
		return nil, fmt.Errorf("catalog: blobStart=%d expected %d", blobStart, headerSize)
	}

	// Multiplication overflow guard: for a malformed/tampered count, the
	// naive int(count)*recordSize in Go silently wraps on overflow,
	// making expectedSize negative and bypassing the truncation guard.
	// Compute as uint64 with an explicit overflow check.
	recordBytes := uint64(recordSize) * count
	if count > 0 && recordBytes/uint64(count) != uint64(recordSize) {
		return nil, fmt.Errorf("%w: count*recordSize overflows (count=%d)", ErrSnapshotOverflow, count)
	}
	if recordStart > math.MaxUint64-recordBytes {
		return nil, fmt.Errorf("%w: recordStart+records exceeds uint64 (recordStart=%d count=%d)",
			ErrSnapshotOverflow, recordStart, count)
	}
	expectedSize := recordStart + recordBytes
	if uint64(len(data)) < expectedSize {
		return nil, ErrSnapshotTruncated
	}

	// Per-record blob-offset bounds check: defends against malformed /
	// tampered files where the record section is intact but the blob
	// pointer lies past EOF. A linear pass; cheap (44 B × count at open).
	//
	// Avoid the naive blobOff+blobLen > dataLen check — blobOff can be
	// MaxUint64 in tampered input and uint64 addition wraps, falsely
	// passing the test. Split the check into overflow-safe parts.
	dataLen := uint64(len(data))
	for i := uint64(0); i < count; i++ {
		recOff := recordStart + i*uint64(recordSize)
		blobLen := binary.LittleEndian.Uint32(data[recOff+40 : recOff+44])
		blobOff := binary.LittleEndian.Uint64(data[recOff+32 : recOff+40])
		if blobOff > dataLen || uint64(blobLen) > dataLen-blobOff {
			return nil, fmt.Errorf("%w: record %d blobOff=%d len=%d exceeds file size %d",
				ErrSnapshotBadBlob, i, blobOff, blobLen, dataLen)
		}
	}

	// Tenant: read zero-padded bytes at offset 40, trim trailing zeros.
	tenantBytes := data[40 : 40+maxTenantLen]
	tenantEnd := len(tenantBytes)
	for tenantEnd > 0 && tenantBytes[tenantEnd-1] == 0 {
		tenantEnd--
	}
	tenant := string(tenantBytes[:tenantEnd])

	return &rawSnapshotReader{
		data:        data,
		count:       count,
		blobStart:   blobStart,
		recordStart: recordStart,
		Tenant:      tenant,
	}, nil
}

// Len returns the number of records in this snapshot.
func (r *rawSnapshotReader) Len() int { return int(r.count) }

// Close releases the file handle and mmap region (if any). Idempotent — safe
// to call from multiple goroutines (uses sync.Once under the hood).
//
// SAFETY: this is NOT safe against concurrent getRaw() — see package doc. Use
// SnapshotHolder for any reload-bearing workflow.
func (r *rawSnapshotReader) Close() error {
	var err error
	r.closeOnce.Do(func() {
		if r.closer != nil {
			err = r.closer()
		}
	})
	return err
}

// getRaw returns the blob bytes for sku, or (nil, false) if not found. The
// returned slice aliases the mmap region — valid until Close. O(log N) binary
// search on the sorted record section.
func (r *rawSnapshotReader) getRaw(sku string) ([]byte, bool) {
	if r.count == 0 || sku == "" {
		return nil, false
	}
	skuBytes := []byte(sku)
	if len(skuBytes) > maxSKULen {
		return nil, false
	}

	lo, hi := uint64(0), r.count
	for lo < hi {
		mid := (lo + hi) / 2
		recOff := int(r.recordStart) + int(mid)*recordSize
		recSKU := r.data[recOff : recOff+32]
		cmp := compareRecordLookup(recSKU, skuBytes)
		switch {
		case cmp < 0:
			lo = mid + 1
		case cmp > 0:
			hi = mid
		default:
			blobLen := binary.LittleEndian.Uint32(r.data[recOff+40 : recOff+44])
			blobOff := binary.LittleEndian.Uint64(r.data[recOff+32 : recOff+40])
			// Per-record bounds check at Open guarantees this slice is in-range.
			return r.data[blobOff : blobOff+uint64(blobLen)], true
		}
	}
	return nil, false
}