platform/projection/snapshot.go

// Snapshot reader/writer — a value-agnostic, mmap-friendly key→blob store.
//
// This is the domain-neutral substrate extracted from platform/catalog: the
// bus is the live channel, but it doesn't carry the historical population on
// cold start. A snapshot file gives a consumer a fast mmap-loaded cold-start
// substrate, so the first read doesn't have to wait for the bus to replay. The
// bus then keeps it fresh. catalog re-exports these types for its Projection;
// other domains (tracking popularity/suggestions) instantiate them over their
// own value type.
//
// Format v2 (little-endian, fixed-width, mmap-friendly):
//
//	header   [72 bytes]   magic(8) version(4) reserved(4) count(8)
//	                       blobStart(8) recordStart(8) tenant(32, zero-padded)
//	blobs    variable     count sequential JSON-encoded value bodies, no padding
//	records  count*44     each: key(32, zero-padded) blobOffset(8) blobLength(4);
//	                       sorted by key via the single compareSKU total order
//
// Tenant field at offsets 40–71 (zero-padded) names the namespace this
// snapshot covers. Independent key schemas can produce values into different
// tenant files without conflict. Hot-reload across tenants stays safe under
// SnapshotHolder (manager.go).
//
// Sorting + binary search gives O(log N) per-key lookup with no on-disk hash
// table and no collisions. Keys longer than 32 bytes are rejected at write
// time. (Domains with longer natural keys — page paths, query strings — hash
// the key to a fixed-width digest and carry the original inside the value blob.)
//
// Memory model: the Reader holds one []byte for the entire file (mmap on
// Unix, ReadFile elsewhere). Per-key Get reads at most log2(N) record slots
// (each 44 B) and one JSON blob. Scan walks the sorted record section once.
//
// SAFETY: SnapshotReader alone is NOT safe across hot-reload. During reload,
// in-flight Get()/Scan() calls on the OLD reader continue to work until disposal
// of the old reader is finalised; calling SnapshotReader.Close() concurrently
// with Get() on the same reader risks unmapping pages-under-read (SIGBUS).
// Use SnapshotHolder (manager.go) for any reload-bearing setup — it gives you
// an atomic.Pointer swap with a drain-grace (default 30s) so concurrent Gets on
// the OLD reader finish naturally before munmap fires.
package projection

import (
	"crypto/sha256"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"math"
	"os"
	"path/filepath"
	"sort"
	"sync"
)

const (
	snapshotMagic   uint64 = 0x3230564a53435050 // "PCSJ002" v2 (was 0x3130564a53435050 = v1)
	snapshotVersion uint32 = 2
	maxSKULen              = 32
	maxTenantLen           = 32
	headerSize             = 8 + 4 + 4 + 8 + 8 + 8 + 32 // 72 bytes
	recordSize             = 32 + 8 + 4                 // 44 bytes per record
)

// Key encoding (stored in the formerly-reserved uint32 at header offset 12).
// 0 = raw, which is byte-identical to a pre-hashing v2 file — so catalog's
// existing .bin files keep parsing unchanged. 1 = hashed: keys are stored as
// hex(sha256(key)[:16]) — 32 ASCII chars, so an arbitrarily long key (page
// path, query string) fits the fixed 32-byte slot. The original key is NOT
// recoverable from the index in hashed mode — the consumer carries it inside
// the value blob (the id/sku stays the link). A whole file is one mode: Get
// applies the same transform the Writer used, read from the header.
const (
	keyEncRaw    uint32 = 0
	keyEncHashed uint32 = 1
)

var (
	ErrSnapshotMagic     = errors.New("projection: not a snapshot file (bad magic)")
	ErrSnapshotVersion   = errors.New("projection: unsupported snapshot version")
	ErrSnapshotTruncated = errors.New("projection: snapshot truncated")
	ErrSnapshotBadBlob   = errors.New("projection: snapshot record points past file")
	ErrSnapshotOverflow  = errors.New("projection: snapshot record-section size overflows uint64")
	ErrSnapshotKeyEnc    = errors.New("projection: unknown key encoding in snapshot header")
	ErrSKUTooLong        = fmt.Errorf("projection: key longer than %d bytes (enable key hashing for unbounded keys)", maxSKULen)
	ErrTenantTooLong     = fmt.Errorf("projection: tenant name longer than %d bytes", maxTenantLen)
)

// hashKey maps an arbitrary-length key to a fixed 32-char hex digest for the
// hashed key encoding. 128 bits (truncated SHA-256) — collision-negligible at
// the cardinalities this store targets (tens of thousands of keys). Hex output
// is pure ASCII, so it never contains the NUL bytes the padded comparator
// treats as terminators.
func hashKey(key string) string {
	sum := sha256.Sum256([]byte(key))
	return hex.EncodeToString(sum[:16])
}

// The snapshot is generic over the stored value T (see SnapshotWriter[T] /
// SnapshotReader[T] below). The binary format, sorting, mmap, and bounds-check
// logic is value-agnostic and lives in the unexported raw* layer; the generic
// wrappers only marshal T → blob (write) and blob → T (read).

// rawSnapshotWriter is the value-agnostic writer: it indexes (key → blob bytes).
// Single-shot: append() all, then write() once (atomic rename).
type rawSnapshotWriter struct {
	tenant  string
	keyEnc  uint32
	blobs   []byte
	records []snapshotRecord
	dirty   bool
}

// snapshotRecord is the on-disk layout: key zero-padded to 32 bytes, then
// an 8-byte offset (from file start) plus a 4-byte length for the JSON blob.
type snapshotRecord struct {
	SKU        [32]byte
	BlobOffset uint64
	BlobLength uint32
}

// append adds a pre-encoded blob under key. In raw mode, ErrSKUTooLong if key
// exceeds the on-disk width; in hashed mode the key is digested to a fixed
// 32-char slot first, so any length is accepted.
func (w *rawSnapshotWriter) append(sku string, blob []byte) error {
	key := sku
	if w.keyEnc == keyEncHashed {
		key = hashKey(sku)
	} else if len(sku) > maxSKULen {
		return ErrSKUTooLong
	}
	var rec snapshotRecord
	copy(rec.SKU[:], key)
	rec.BlobOffset = uint64(headerSize + len(w.blobs))
	rec.BlobLength = uint32(len(blob))

	w.blobs = append(w.blobs, blob...)
	w.records = append(w.records, rec)
	w.dirty = true
	return nil
}

// SnapshotWriter[T] is single-shot: Append() all values, then Write() once
// (atomic rename-into-place so readers never see a partial file). T is the
// stored value type.
type SnapshotWriter[T any] struct {
	raw rawSnapshotWriter
}

// NewSnapshotWriter returns an empty writer for value type T. Caller Appends
// every value and calls Write once.
func NewSnapshotWriter[T any]() *SnapshotWriter[T] {
	return &SnapshotWriter[T]{}
}

// WithTenant tags this writer with a tenant namespace that lands in the
// snapshot header. Optional — empty tenant gets zero-padded; readers see an
// empty string when the writer was untagged.
func (w *SnapshotWriter[T]) WithTenant(tenant string) *SnapshotWriter[T] {
	w.raw.tenant = tenant
	return w
}

// WithKeyHashing switches this writer to the hashed key encoding: keys are
// stored as a fixed 32-char digest, so arbitrarily long keys (page paths, query
// strings) are accepted. The original key is NOT recoverable from the index —
// carry it inside the value if you need it back (see Scan). A reader opening the
// file applies the same hashing on Get automatically (the mode is in the
// header). Set this before any Append.
func (w *SnapshotWriter[T]) WithKeyHashing() *SnapshotWriter[T] {
	w.raw.keyEnc = keyEncHashed
	return w
}

// Append encodes v and indexes it under key. In raw mode returns ErrSKUTooLong
// if key exceeds the on-disk width; in hashed mode any length is accepted.
// Returns marshal errors as wrapped fmt.Errorf. key is explicit because T is
// opaque — the caller knows the key.
func (w *SnapshotWriter[T]) Append(sku string, v T) error {
	if w.raw.keyEnc == keyEncRaw && len(sku) > maxSKULen {
		return ErrSKUTooLong
	}
	jb, err := json.Marshal(v)
	if err != nil {
		return fmt.Errorf("projection: marshal snapshot value %q: %w", sku, err)
	}
	return w.raw.append(sku, jb)
}

// Len returns how many values have been Append'd.
func (w *SnapshotWriter[T]) Len() int { return len(w.raw.records) }

// Write serialises to dir/name (atomic temp+rename). See rawSnapshotWriter.write.
func (w *SnapshotWriter[T]) Write(dir, name string) (finalPath string, err error) {
	return w.raw.write(dir, name)
}

// write serialises to a temp file in dir, fsync's, then renames into the
// final path. Records are sorted by key asc before serialisation so the
// reader's binary search works.
func (w *rawSnapshotWriter) write(dir, name string) (finalPath string, err error) {
	if w.dirty {
		sort.Slice(w.records, func(i, j int) bool {
			return compareSKU(w.records[i].SKU[:], w.records[j].SKU[:]) < 0
		})
		w.dirty = false
	}
	if len(w.tenant) > maxTenantLen {
		err = ErrTenantTooLong
		return "", err
	}

	if err = os.MkdirAll(dir, 0o755); err != nil {
		return "", err
	}
	finalPath = filepath.Join(dir, name)
	tmp := finalPath + ".tmp"

	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
	if err != nil {
		return "", err
	}
	defer func() {
		if err != nil {
			_ = os.Remove(tmp)
			_ = f.Close()
		}
	}()

	// Header: fixed 72 bytes (8 + 4 + 4 + 8 + 8 + 8 + 32).
	blobStart := uint64(headerSize)
	recordStart := blobStart + uint64(len(w.blobs))
	var hdr [headerSize]byte
	binary.LittleEndian.PutUint64(hdr[0:8], snapshotMagic)
	binary.LittleEndian.PutUint32(hdr[8:12], snapshotVersion)
	binary.LittleEndian.PutUint32(hdr[12:16], w.keyEnc) // 0 = raw (byte-identical to pre-hashing v2)
	binary.LittleEndian.PutUint64(hdr[16:24], uint64(len(w.records)))
	binary.LittleEndian.PutUint64(hdr[24:32], blobStart)
	binary.LittleEndian.PutUint64(hdr[32:40], recordStart)
	copy(hdr[40:40+len(w.tenant)], w.tenant) // rest is zero-pad

	if _, werr := f.Write(hdr[:]); werr != nil {
		err = werr
		return "", err
	}
	if _, werr := f.Write(w.blobs); werr != nil {
		err = werr
		return "", err
	}
	recBytes := make([]byte, recordSize*len(w.records))
	for i, r := range w.records {
		off := i * recordSize
		copy(recBytes[off:off+32], r.SKU[:])
		binary.LittleEndian.PutUint64(recBytes[off+32:off+40], r.BlobOffset)
		binary.LittleEndian.PutUint32(recBytes[off+40:off+44], r.BlobLength)
	}
	if _, werr := f.Write(recBytes); werr != nil {
		err = werr
		return "", err
	}

	if werr := f.Sync(); werr != nil {
		err = werr
		return "", err
	}
	if werr := f.Close(); werr != nil {
		err = werr
		return "", err
	}
	if werr := os.Rename(tmp, finalPath); werr != nil {
		err = werr
		return "", err
	}
	return finalPath, nil
}

// compareSKU defines the SINGLE TOTAL ORDER for keys across both Writer
// (uses it for sort.Slice) and Reader (uses it for binary search). One
// definition; invariant test verifies the on-disk result is sorted under
// it. Source keys SHOULD NOT contain embedded null bytes — the padded
// comparator treats them as terminators, which silently truncates a key.
func compareSKU(padded, other []byte) int {
	// padded is a fixed-size zero-padded slot; trim trailing zeros.
	end := len(padded)
	for end > 0 && padded[end-1] == 0 {
		end--
	}
	a := padded[:end]
	b := other

	n := len(a)
	if len(b) < n {
		n = len(b)
	}
	for i := 0; i < n; i++ {
		if a[i] != b[i] {
			if a[i] < b[i] {
				return -1
			}
			return 1
		}
	}
	switch {
	case len(a) < len(b):
		return -1
	case len(a) > len(b):
		return 1
	default:
		return 0
	}
}

// compareRecordLookup is compareSKU specialised for Reader lookups so the
// argument order matches the lex order: "padded" (fixed-slot) and "want"
// (query). compareSKU is symmetric so this is just an alias.
func compareRecordLookup(padded, want []byte) int {
	return compareSKU(padded, want)
}

// rawSnapshotReader is the value-agnostic reader: binary search by key, returns
// the raw blob bytes. Concurrency-safe for parallel getRaw; NOT safe racing
// Close with getRaw (see package doc — use SnapshotHolder for reload).
type rawSnapshotReader struct {
	data        []byte
	count       uint64
	blobStart   uint64
	recordStart uint64
	keyEnc      uint32 // raw | hashed; applied to the query key in getRaw
	Tenant      string // populated from header at Open time; "" if not set

	closeOnce sync.Once
	closer    func() error // munmap + close on Unix; no-op on ReadFile fallback
}

// SnapshotReader[T] is the typed view over a raw snapshot: Get decodes the blob
// into T. Read-only and concurrency-safe — parallel Get calls are OK. NOT safe
// racing Close concurrently with Get; use SnapshotHolder[T] for reload.
type SnapshotReader[T any] struct {
	raw *rawSnapshotReader
}

// ReadSnapshotFile opens + parses a snapshot file as a reader of T. On Unix uses
// mmap; on other platforms reads into memory (functionally identical, higher RSS).
func ReadSnapshotFile[T any](path string) (*SnapshotReader[T], error) {
	rr, err := readRawSnapshotFile(path)
	if err != nil {
		return nil, err
	}
	return &SnapshotReader[T]{raw: rr}, nil
}

// NewSnapshotReader wraps an already-loaded []byte as a reader of T (use in
// tests; production paths go through ReadSnapshotFile).
func NewSnapshotReader[T any](data []byte) (*SnapshotReader[T], error) {
	rr, err := newRawSnapshotReader(data)
	if err != nil {
		return nil, err
	}
	return &SnapshotReader[T]{raw: rr}, nil
}

// Get returns the value for key, or (zero, false) if absent / undecodable.
func (r *SnapshotReader[T]) Get(sku string) (T, bool) {
	var zero T
	blob, ok := r.raw.getRaw(sku)
	if !ok {
		return zero, false
	}
	var v T
	if err := json.Unmarshal(blob, &v); err != nil {
		return zero, false
	}
	return v, true
}

// Scan walks every (key, value) in stored-key-sorted order, calling fn for
// each. Return false from fn to stop early. A value that fails to decode is
// skipped (fn is not called for it). This is the enumeration / ranked-scan
// substrate: callers that need top-N-by-value collect into a slice and sort in
// memory — the on-disk order is by key, not value. NOT safe racing Close; drive
// it through SnapshotHolder.Scan under reload.
//
// In hashed key mode the `key` passed to fn is the DIGEST, not the original —
// recover the original from the value (that's why the consumer carries the
// id/sku in the value blob).
func (r *SnapshotReader[T]) Scan(fn func(key string, v T) bool) {
	r.raw.scan(func(key string, blob []byte) bool {
		var v T
		if err := json.Unmarshal(blob, &v); err != nil {
			return true // skip undecodable, keep going
		}
		return fn(key, v)
	})
}

// Len returns the number of values in this snapshot.
func (r *SnapshotReader[T]) Len() int { return r.raw.Len() }

// Tenant returns the snapshot's namespace tag ("" if untagged).
func (r *SnapshotReader[T]) Tenant() string { return r.raw.Tenant }

// Close releases the file handle and mmap region. Idempotent.
func (r *SnapshotReader[T]) Close() error { return r.raw.Close() }

// readRawSnapshotFile opens + parses a snapshot file. On Unix uses mmap; on
// other platforms reads into memory (functionally identical, higher RSS).
func readRawSnapshotFile(path string) (*rawSnapshotReader, error) {
	data, closer, err := loadSnapshotBytes(path)
	if err != nil {
		return nil, err
	}
	r, err := newRawSnapshotReader(data)
	if err != nil {
		if closer != nil {
			_ = closer()
		}
		return nil, err
	}
	r.closer = closer
	return r, nil
}

func newRawSnapshotReader(data []byte) (*rawSnapshotReader, error) {
	if len(data) < headerSize {
		return nil, ErrSnapshotTruncated
	}
	magic := binary.LittleEndian.Uint64(data[0:8])
	if magic != snapshotMagic {
		return nil, ErrSnapshotMagic
	}
	version := binary.LittleEndian.Uint32(data[8:12])
	if version != snapshotVersion {
		return nil, ErrSnapshotVersion
	}
	keyEnc := binary.LittleEndian.Uint32(data[12:16])
	if keyEnc != keyEncRaw && keyEnc != keyEncHashed {
		// Reject rather than mis-look-up: an unknown encoding means Get can't
		// reproduce the key transform the writer used.
		return nil, fmt.Errorf("%w: %d", ErrSnapshotKeyEnc, keyEnc)
	}
	count := binary.LittleEndian.Uint64(data[16:24])
	blobStart := binary.LittleEndian.Uint64(data[24:32])
	recordStart := binary.LittleEndian.Uint64(data[32:40])
	if blobStart != uint64(headerSize) {
		return nil, fmt.Errorf("projection: blobStart=%d expected %d", blobStart, headerSize)
	}

	// Multiplication overflow guard: for a malformed/tampered count, the
	// naive int(count)*recordSize in Go silently wraps on overflow,
	// making expectedSize negative and bypassing the truncation guard.
	// Compute as uint64 with an explicit overflow check.
	recordBytes := uint64(recordSize) * count
	if count > 0 && recordBytes/uint64(count) != uint64(recordSize) {
		return nil, fmt.Errorf("%w: count*recordSize overflows (count=%d)", ErrSnapshotOverflow, count)
	}
	if recordStart > math.MaxUint64-recordBytes {
		return nil, fmt.Errorf("%w: recordStart+records exceeds uint64 (recordStart=%d count=%d)",
			ErrSnapshotOverflow, recordStart, count)
	}
	expectedSize := recordStart + recordBytes
	if uint64(len(data)) < expectedSize {
		return nil, ErrSnapshotTruncated
	}

	// Per-record blob-offset bounds check: defends against malformed /
	// tampered files where the record section is intact but the blob
	// pointer lies past EOF. A linear pass; cheap (44 B × count at open).
	dataLen := uint64(len(data))
	for i := uint64(0); i < count; i++ {
		recOff := recordStart + i*uint64(recordSize)
		blobLen := binary.LittleEndian.Uint32(data[recOff+40 : recOff+44])
		blobOff := binary.LittleEndian.Uint64(data[recOff+32 : recOff+40])
		if blobOff > dataLen || uint64(blobLen) > dataLen-blobOff {
			return nil, fmt.Errorf("%w: record %d blobOff=%d len=%d exceeds file size %d",
				ErrSnapshotBadBlob, i, blobOff, blobLen, dataLen)
		}
	}

	// Tenant: read zero-padded bytes at offset 40, trim trailing zeros.
	tenantBytes := data[40 : 40+maxTenantLen]
	tenantEnd := len(tenantBytes)
	for tenantEnd > 0 && tenantBytes[tenantEnd-1] == 0 {
		tenantEnd--
	}
	tenant := string(tenantBytes[:tenantEnd])

	return &rawSnapshotReader{
		data:        data,
		count:       count,
		blobStart:   blobStart,
		recordStart: recordStart,
		keyEnc:      keyEnc,
		Tenant:      tenant,
	}, nil
}

// Len returns the number of records in this snapshot.
func (r *rawSnapshotReader) Len() int { return int(r.count) }

// Close releases the file handle and mmap region (if any). Idempotent — safe
// to call from multiple goroutines (uses sync.Once under the hood).
func (r *rawSnapshotReader) Close() error {
	var err error
	r.closeOnce.Do(func() {
		if r.closer != nil {
			err = r.closer()
		}
	})
	return err
}

// getRaw returns the blob bytes for key, or (nil, false) if not found. The
// returned slice aliases the mmap region — valid until Close. O(log N) binary
// search on the sorted record section.
func (r *rawSnapshotReader) getRaw(sku string) ([]byte, bool) {
	if r.count == 0 || sku == "" {
		return nil, false
	}
	if r.keyEnc == keyEncHashed {
		sku = hashKey(sku) // same transform the writer applied
	}
	skuBytes := []byte(sku)
	if len(skuBytes) > maxSKULen {
		return nil, false
	}

	lo, hi := uint64(0), r.count
	for lo < hi {
		mid := (lo + hi) / 2
		recOff := int(r.recordStart) + int(mid)*recordSize
		recSKU := r.data[recOff : recOff+32]
		cmp := compareRecordLookup(recSKU, skuBytes)
		switch {
		case cmp < 0:
			lo = mid + 1
		case cmp > 0:
			hi = mid
		default:
			blobLen := binary.LittleEndian.Uint32(r.data[recOff+40 : recOff+44])
			blobOff := binary.LittleEndian.Uint64(r.data[recOff+32 : recOff+40])
			// Per-record bounds check at Open guarantees this slice is in-range.
			return r.data[blobOff : blobOff+uint64(blobLen)], true
		}
	}
	return nil, false
}

// scan walks every record in key-sorted order, yielding (key, blob). Return
// false from fn to stop. The blob aliases the mmap region — valid until Close.
func (r *rawSnapshotReader) scan(fn func(key string, blob []byte) bool) {
	for i := uint64(0); i < r.count; i++ {
		recOff := int(r.recordStart) + int(i)*recordSize
		recSKU := r.data[recOff : recOff+32]
		end := len(recSKU)
		for end > 0 && recSKU[end-1] == 0 {
			end--
		}
		key := string(recSKU[:end])
		blobLen := binary.LittleEndian.Uint32(r.data[recOff+40 : recOff+44])
		blobOff := binary.LittleEndian.Uint64(r.data[recOff+32 : recOff+40])
		if !fn(key, r.data[blobOff:blobOff+uint64(blobLen)]) {
			return
		}
	}
}