more platform
This commit is contained in:
mats committed
1 parent
8a71e32149
commit
2a66754f9e
27 files changed
+2435
-540
No files matched your search
@@ -0,0 +1,507 @@
|
||||
// Snapshot reader/writer — a value-agnostic, mmap-friendly key→blob store.
|
||||
//
|
||||
// This is the domain-neutral substrate extracted from platform/catalog: the
|
||||
// bus is the live channel, but it doesn't carry the historical population on
|
||||
// cold start. A snapshot file gives a consumer a fast mmap-loaded cold-start
|
||||
// substrate, so the first read doesn't have to wait for the bus to replay. The
|
||||
// bus then keeps it fresh. catalog re-exports these types for its Projection;
|
||||
// other domains (tracking popularity/suggestions) instantiate them over their
|
||||
// own value type.
|
||||
//
|
||||
// Format v2 (little-endian, fixed-width, mmap-friendly):
|
||||
//
|
||||
// header [72 bytes] magic(8) version(4) reserved(4) count(8)
|
||||
// blobStart(8) recordStart(8) tenant(32, zero-padded)
|
||||
// blobs variable count sequential JSON-encoded value bodies, no padding
|
||||
// records count*44 each: key(32, zero-padded) blobOffset(8) blobLength(4);
|
||||
// sorted by key via the single compareSKU total order
|
||||
//
|
||||
// Tenant field at offsets 40–71 (zero-padded) names the namespace this
|
||||
// snapshot covers. Independent key schemas can produce values into different
|
||||
// tenant files without conflict. Hot-reload across tenants stays safe under
|
||||
// SnapshotHolder (manager.go).
|
||||
//
|
||||
// Sorting + binary search gives O(log N) per-key lookup with no on-disk hash
|
||||
// table and no collisions. Keys longer than 32 bytes are rejected at write
|
||||
// time. (Domains with longer natural keys — page paths, query strings — hash
|
||||
// the key to a fixed-width digest and carry the original inside the value blob.)
|
||||
//
|
||||
// Memory model: the Reader holds one []byte for the entire file (mmap on
|
||||
// Unix, ReadFile elsewhere). Per-key Get reads at most log2(N) record slots
|
||||
// (each 44 B) and one JSON blob. Scan walks the sorted record section once.
|
||||
//
|
||||
// SAFETY: SnapshotReader alone is NOT safe across hot-reload. During reload,
|
||||
// in-flight Get()/Scan() calls on the OLD reader continue to work until disposal
|
||||
// of the old reader is finalised; calling SnapshotReader.Close() concurrently
|
||||
// with Get() on the same reader risks unmapping pages-under-read (SIGBUS).
|
||||
// Use SnapshotHolder (manager.go) for any reload-bearing setup — it gives you
|
||||
// an atomic.Pointer swap with a drain-grace (default 30s) so concurrent Gets on
|
||||
// the OLD reader finish naturally before munmap fires.
|
||||
package projection
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
snapshotMagic uint64 = 0x3230564a53435050 // "PCSJ002" v2 (was 0x3130564a53435050 = v1)
|
||||
snapshotVersion uint32 = 2
|
||||
maxSKULen = 32
|
||||
maxTenantLen = 32
|
||||
headerSize = 8 + 4 + 4 + 8 + 8 + 8 + 32 // 72 bytes
|
||||
recordSize = 32 + 8 + 4 // 44 bytes per record
|
||||
)
|
||||
|
||||
var (
|
||||
ErrSnapshotMagic = errors.New("projection: not a snapshot file (bad magic)")
|
||||
ErrSnapshotVersion = errors.New("projection: unsupported snapshot version")
|
||||
ErrSnapshotTruncated = errors.New("projection: snapshot truncated")
|
||||
ErrSnapshotBadBlob = errors.New("projection: snapshot record points past file")
|
||||
ErrSnapshotOverflow = errors.New("projection: snapshot record-section size overflows uint64")
|
||||
ErrSKUTooLong = fmt.Errorf("projection: key longer than %d bytes", maxSKULen)
|
||||
ErrTenantTooLong = fmt.Errorf("projection: tenant name longer than %d bytes", maxTenantLen)
|
||||
)
|
||||
|
||||
// The snapshot is generic over the stored value T (see SnapshotWriter[T] /
|
||||
// SnapshotReader[T] below). The binary format, sorting, mmap, and bounds-check
|
||||
// logic is value-agnostic and lives in the unexported raw* layer; the generic
|
||||
// wrappers only marshal T → blob (write) and blob → T (read).
|
||||
|
||||
// rawSnapshotWriter is the value-agnostic writer: it indexes (key → blob bytes).
|
||||
// Single-shot: append() all, then write() once (atomic rename).
|
||||
type rawSnapshotWriter struct {
|
||||
tenant string
|
||||
blobs []byte
|
||||
records []snapshotRecord
|
||||
dirty bool
|
||||
}
|
||||
|
||||
// snapshotRecord is the on-disk layout: key zero-padded to 32 bytes, then
|
||||
// an 8-byte offset (from file start) plus a 4-byte length for the JSON blob.
|
||||
type snapshotRecord struct {
|
||||
SKU [32]byte
|
||||
BlobOffset uint64
|
||||
BlobLength uint32
|
||||
}
|
||||
|
||||
// append adds a pre-encoded blob under key. ErrSKUTooLong if key exceeds the
|
||||
// on-disk width.
|
||||
func (w *rawSnapshotWriter) append(sku string, blob []byte) error {
|
||||
if len(sku) > maxSKULen {
|
||||
return ErrSKUTooLong
|
||||
}
|
||||
var rec snapshotRecord
|
||||
copy(rec.SKU[:], sku)
|
||||
rec.BlobOffset = uint64(headerSize + len(w.blobs))
|
||||
rec.BlobLength = uint32(len(blob))
|
||||
|
||||
w.blobs = append(w.blobs, blob...)
|
||||
w.records = append(w.records, rec)
|
||||
w.dirty = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// SnapshotWriter[T] is single-shot: Append() all values, then Write() once
|
||||
// (atomic rename-into-place so readers never see a partial file). T is the
|
||||
// stored value type.
|
||||
type SnapshotWriter[T any] struct {
|
||||
raw rawSnapshotWriter
|
||||
}
|
||||
|
||||
// NewSnapshotWriter returns an empty writer for value type T. Caller Appends
|
||||
// every value and calls Write once.
|
||||
func NewSnapshotWriter[T any]() *SnapshotWriter[T] {
|
||||
return &SnapshotWriter[T]{}
|
||||
}
|
||||
|
||||
// WithTenant tags this writer with a tenant namespace that lands in the
|
||||
// snapshot header. Optional — empty tenant gets zero-padded; readers see an
|
||||
// empty string when the writer was untagged.
|
||||
func (w *SnapshotWriter[T]) WithTenant(tenant string) *SnapshotWriter[T] {
|
||||
w.raw.tenant = tenant
|
||||
return w
|
||||
}
|
||||
|
||||
// Append encodes v and indexes it under key. Returns ErrSKUTooLong if key
|
||||
// exceeds the on-disk width; returns marshal errors as wrapped fmt.Errorf.
|
||||
// key is explicit because T is opaque — the caller knows the key.
|
||||
func (w *SnapshotWriter[T]) Append(sku string, v T) error {
|
||||
if len(sku) > maxSKULen {
|
||||
return ErrSKUTooLong
|
||||
}
|
||||
jb, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return fmt.Errorf("projection: marshal snapshot value %q: %w", sku, err)
|
||||
}
|
||||
return w.raw.append(sku, jb)
|
||||
}
|
||||
|
||||
// Len returns how many values have been Append'd.
|
||||
func (w *SnapshotWriter[T]) Len() int { return len(w.raw.records) }
|
||||
|
||||
// Write serialises to dir/name (atomic temp+rename). See rawSnapshotWriter.write.
|
||||
func (w *SnapshotWriter[T]) Write(dir, name string) (finalPath string, err error) {
|
||||
return w.raw.write(dir, name)
|
||||
}
|
||||
|
||||
// write serialises to a temp file in dir, fsync's, then renames into the
|
||||
// final path. Records are sorted by key asc before serialisation so the
|
||||
// reader's binary search works.
|
||||
func (w *rawSnapshotWriter) write(dir, name string) (finalPath string, err error) {
|
||||
if w.dirty {
|
||||
sort.Slice(w.records, func(i, j int) bool {
|
||||
return compareSKU(w.records[i].SKU[:], w.records[j].SKU[:]) < 0
|
||||
})
|
||||
w.dirty = false
|
||||
}
|
||||
if len(w.tenant) > maxTenantLen {
|
||||
err = ErrTenantTooLong
|
||||
return "", err
|
||||
}
|
||||
|
||||
if err = os.MkdirAll(dir, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
finalPath = filepath.Join(dir, name)
|
||||
tmp := finalPath + ".tmp"
|
||||
|
||||
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
_ = f.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
// Header: fixed 72 bytes (8 + 4 + 4 + 8 + 8 + 8 + 32).
|
||||
blobStart := uint64(headerSize)
|
||||
recordStart := blobStart + uint64(len(w.blobs))
|
||||
var hdr [headerSize]byte
|
||||
binary.LittleEndian.PutUint64(hdr[0:8], snapshotMagic)
|
||||
binary.LittleEndian.PutUint32(hdr[8:12], snapshotVersion)
|
||||
binary.LittleEndian.PutUint32(hdr[12:16], 0)
|
||||
binary.LittleEndian.PutUint64(hdr[16:24], uint64(len(w.records)))
|
||||
binary.LittleEndian.PutUint64(hdr[24:32], blobStart)
|
||||
binary.LittleEndian.PutUint64(hdr[32:40], recordStart)
|
||||
copy(hdr[40:40+len(w.tenant)], w.tenant) // rest is zero-pad
|
||||
|
||||
if _, werr := f.Write(hdr[:]); werr != nil {
|
||||
err = werr
|
||||
return "", err
|
||||
}
|
||||
if _, werr := f.Write(w.blobs); werr != nil {
|
||||
err = werr
|
||||
return "", err
|
||||
}
|
||||
recBytes := make([]byte, recordSize*len(w.records))
|
||||
for i, r := range w.records {
|
||||
off := i * recordSize
|
||||
copy(recBytes[off:off+32], r.SKU[:])
|
||||
binary.LittleEndian.PutUint64(recBytes[off+32:off+40], r.BlobOffset)
|
||||
binary.LittleEndian.PutUint32(recBytes[off+40:off+44], r.BlobLength)
|
||||
}
|
||||
if _, werr := f.Write(recBytes); werr != nil {
|
||||
err = werr
|
||||
return "", err
|
||||
}
|
||||
|
||||
if werr := f.Sync(); werr != nil {
|
||||
err = werr
|
||||
return "", err
|
||||
}
|
||||
if werr := f.Close(); werr != nil {
|
||||
err = werr
|
||||
return "", err
|
||||
}
|
||||
if werr := os.Rename(tmp, finalPath); werr != nil {
|
||||
err = werr
|
||||
return "", err
|
||||
}
|
||||
return finalPath, nil
|
||||
}
|
||||
|
||||
// compareSKU defines the SINGLE TOTAL ORDER for keys across both Writer
|
||||
// (uses it for sort.Slice) and Reader (uses it for binary search). One
|
||||
// definition; invariant test verifies the on-disk result is sorted under
|
||||
// it. Source keys SHOULD NOT contain embedded null bytes — the padded
|
||||
// comparator treats them as terminators, which silently truncates a key.
|
||||
func compareSKU(padded, other []byte) int {
|
||||
// padded is a fixed-size zero-padded slot; trim trailing zeros.
|
||||
end := len(padded)
|
||||
for end > 0 && padded[end-1] == 0 {
|
||||
end--
|
||||
}
|
||||
a := padded[:end]
|
||||
b := other
|
||||
|
||||
n := len(a)
|
||||
if len(b) < n {
|
||||
n = len(b)
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
if a[i] != b[i] {
|
||||
if a[i] < b[i] {
|
||||
return -1
|
||||
}
|
||||
return 1
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(a) < len(b):
|
||||
return -1
|
||||
case len(a) > len(b):
|
||||
return 1
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// compareRecordLookup is compareSKU specialised for Reader lookups so the
|
||||
// argument order matches the lex order: "padded" (fixed-slot) and "want"
|
||||
// (query). compareSKU is symmetric so this is just an alias.
|
||||
func compareRecordLookup(padded, want []byte) int {
|
||||
return compareSKU(padded, want)
|
||||
}
|
||||
|
||||
// rawSnapshotReader is the value-agnostic reader: binary search by key, returns
|
||||
// the raw blob bytes. Concurrency-safe for parallel getRaw; NOT safe racing
|
||||
// Close with getRaw (see package doc — use SnapshotHolder for reload).
|
||||
type rawSnapshotReader struct {
|
||||
data []byte
|
||||
count uint64
|
||||
blobStart uint64
|
||||
recordStart uint64
|
||||
Tenant string // populated from header at Open time; "" if not set
|
||||
|
||||
closeOnce sync.Once
|
||||
closer func() error // munmap + close on Unix; no-op on ReadFile fallback
|
||||
}
|
||||
|
||||
// SnapshotReader[T] is the typed view over a raw snapshot: Get decodes the blob
|
||||
// into T. Read-only and concurrency-safe — parallel Get calls are OK. NOT safe
|
||||
// racing Close concurrently with Get; use SnapshotHolder[T] for reload.
|
||||
type SnapshotReader[T any] struct {
|
||||
raw *rawSnapshotReader
|
||||
}
|
||||
|
||||
// ReadSnapshotFile opens + parses a snapshot file as a reader of T. On Unix uses
|
||||
// mmap; on other platforms reads into memory (functionally identical, higher RSS).
|
||||
func ReadSnapshotFile[T any](path string) (*SnapshotReader[T], error) {
|
||||
rr, err := readRawSnapshotFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &SnapshotReader[T]{raw: rr}, nil
|
||||
}
|
||||
|
||||
// NewSnapshotReader wraps an already-loaded []byte as a reader of T (use in
|
||||
// tests; production paths go through ReadSnapshotFile).
|
||||
func NewSnapshotReader[T any](data []byte) (*SnapshotReader[T], error) {
|
||||
rr, err := newRawSnapshotReader(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &SnapshotReader[T]{raw: rr}, nil
|
||||
}
|
||||
|
||||
// Get returns the value for key, or (zero, false) if absent / undecodable.
|
||||
func (r *SnapshotReader[T]) Get(sku string) (T, bool) {
|
||||
var zero T
|
||||
blob, ok := r.raw.getRaw(sku)
|
||||
if !ok {
|
||||
return zero, false
|
||||
}
|
||||
var v T
|
||||
if err := json.Unmarshal(blob, &v); err != nil {
|
||||
return zero, false
|
||||
}
|
||||
return v, true
|
||||
}
|
||||
|
||||
// Scan walks every (key, value) in key-sorted order, calling fn for each.
|
||||
// Return false from fn to stop early. A value that fails to decode is skipped
|
||||
// (fn is not called for it). This is the enumeration / ranked-scan substrate:
|
||||
// callers that need top-N-by-value collect into a slice and sort in memory —
|
||||
// the on-disk order is by key, not value. NOT safe racing Close; drive it
|
||||
// through SnapshotHolder.Scan under reload.
|
||||
func (r *SnapshotReader[T]) Scan(fn func(key string, v T) bool) {
|
||||
r.raw.scan(func(key string, blob []byte) bool {
|
||||
var v T
|
||||
if err := json.Unmarshal(blob, &v); err != nil {
|
||||
return true // skip undecodable, keep going
|
||||
}
|
||||
return fn(key, v)
|
||||
})
|
||||
}
|
||||
|
||||
// Len returns the number of values in this snapshot.
|
||||
func (r *SnapshotReader[T]) Len() int { return r.raw.Len() }
|
||||
|
||||
// Tenant returns the snapshot's namespace tag ("" if untagged).
|
||||
func (r *SnapshotReader[T]) Tenant() string { return r.raw.Tenant }
|
||||
|
||||
// Close releases the file handle and mmap region. Idempotent.
|
||||
func (r *SnapshotReader[T]) Close() error { return r.raw.Close() }
|
||||
|
||||
// readRawSnapshotFile opens + parses a snapshot file. On Unix uses mmap; on
|
||||
// other platforms reads into memory (functionally identical, higher RSS).
|
||||
func readRawSnapshotFile(path string) (*rawSnapshotReader, error) {
|
||||
data, closer, err := loadSnapshotBytes(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, err := newRawSnapshotReader(data)
|
||||
if err != nil {
|
||||
if closer != nil {
|
||||
_ = closer()
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
r.closer = closer
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func newRawSnapshotReader(data []byte) (*rawSnapshotReader, error) {
|
||||
if len(data) < headerSize {
|
||||
return nil, ErrSnapshotTruncated
|
||||
}
|
||||
magic := binary.LittleEndian.Uint64(data[0:8])
|
||||
if magic != snapshotMagic {
|
||||
return nil, ErrSnapshotMagic
|
||||
}
|
||||
version := binary.LittleEndian.Uint32(data[8:12])
|
||||
if version != snapshotVersion {
|
||||
return nil, ErrSnapshotVersion
|
||||
}
|
||||
count := binary.LittleEndian.Uint64(data[16:24])
|
||||
blobStart := binary.LittleEndian.Uint64(data[24:32])
|
||||
recordStart := binary.LittleEndian.Uint64(data[32:40])
|
||||
if blobStart != uint64(headerSize) {
|
||||
return nil, fmt.Errorf("projection: blobStart=%d expected %d", blobStart, headerSize)
|
||||
}
|
||||
|
||||
// Multiplication overflow guard: for a malformed/tampered count, the
|
||||
// naive int(count)*recordSize in Go silently wraps on overflow,
|
||||
// making expectedSize negative and bypassing the truncation guard.
|
||||
// Compute as uint64 with an explicit overflow check.
|
||||
recordBytes := uint64(recordSize) * count
|
||||
if count > 0 && recordBytes/uint64(count) != uint64(recordSize) {
|
||||
return nil, fmt.Errorf("%w: count*recordSize overflows (count=%d)", ErrSnapshotOverflow, count)
|
||||
}
|
||||
if recordStart > math.MaxUint64-recordBytes {
|
||||
return nil, fmt.Errorf("%w: recordStart+records exceeds uint64 (recordStart=%d count=%d)",
|
||||
ErrSnapshotOverflow, recordStart, count)
|
||||
}
|
||||
expectedSize := recordStart + recordBytes
|
||||
if uint64(len(data)) < expectedSize {
|
||||
return nil, ErrSnapshotTruncated
|
||||
}
|
||||
|
||||
// Per-record blob-offset bounds check: defends against malformed /
|
||||
// tampered files where the record section is intact but the blob
|
||||
// pointer lies past EOF. A linear pass; cheap (44 B × count at open).
|
||||
dataLen := uint64(len(data))
|
||||
for i := uint64(0); i < count; i++ {
|
||||
recOff := recordStart + i*uint64(recordSize)
|
||||
blobLen := binary.LittleEndian.Uint32(data[recOff+40 : recOff+44])
|
||||
blobOff := binary.LittleEndian.Uint64(data[recOff+32 : recOff+40])
|
||||
if blobOff > dataLen || uint64(blobLen) > dataLen-blobOff {
|
||||
return nil, fmt.Errorf("%w: record %d blobOff=%d len=%d exceeds file size %d",
|
||||
ErrSnapshotBadBlob, i, blobOff, blobLen, dataLen)
|
||||
}
|
||||
}
|
||||
|
||||
// Tenant: read zero-padded bytes at offset 40, trim trailing zeros.
|
||||
tenantBytes := data[40 : 40+maxTenantLen]
|
||||
tenantEnd := len(tenantBytes)
|
||||
for tenantEnd > 0 && tenantBytes[tenantEnd-1] == 0 {
|
||||
tenantEnd--
|
||||
}
|
||||
tenant := string(tenantBytes[:tenantEnd])
|
||||
|
||||
return &rawSnapshotReader{
|
||||
data: data,
|
||||
count: count,
|
||||
blobStart: blobStart,
|
||||
recordStart: recordStart,
|
||||
Tenant: tenant,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Len returns the number of records in this snapshot.
|
||||
func (r *rawSnapshotReader) Len() int { return int(r.count) }
|
||||
|
||||
// Close releases the file handle and mmap region (if any). Idempotent — safe
|
||||
// to call from multiple goroutines (uses sync.Once under the hood).
|
||||
func (r *rawSnapshotReader) Close() error {
|
||||
var err error
|
||||
r.closeOnce.Do(func() {
|
||||
if r.closer != nil {
|
||||
err = r.closer()
|
||||
}
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// getRaw returns the blob bytes for key, or (nil, false) if not found. The
|
||||
// returned slice aliases the mmap region — valid until Close. O(log N) binary
|
||||
// search on the sorted record section.
|
||||
func (r *rawSnapshotReader) getRaw(sku string) ([]byte, bool) {
|
||||
if r.count == 0 || sku == "" {
|
||||
return nil, false
|
||||
}
|
||||
skuBytes := []byte(sku)
|
||||
if len(skuBytes) > maxSKULen {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
lo, hi := uint64(0), r.count
|
||||
for lo < hi {
|
||||
mid := (lo + hi) / 2
|
||||
recOff := int(r.recordStart) + int(mid)*recordSize
|
||||
recSKU := r.data[recOff : recOff+32]
|
||||
cmp := compareRecordLookup(recSKU, skuBytes)
|
||||
switch {
|
||||
case cmp < 0:
|
||||
lo = mid + 1
|
||||
case cmp > 0:
|
||||
hi = mid
|
||||
default:
|
||||
blobLen := binary.LittleEndian.Uint32(r.data[recOff+40 : recOff+44])
|
||||
blobOff := binary.LittleEndian.Uint64(r.data[recOff+32 : recOff+40])
|
||||
// Per-record bounds check at Open guarantees this slice is in-range.
|
||||
return r.data[blobOff : blobOff+uint64(blobLen)], true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// scan walks every record in key-sorted order, yielding (key, blob). Return
|
||||
// false from fn to stop. The blob aliases the mmap region — valid until Close.
|
||||
func (r *rawSnapshotReader) scan(fn func(key string, blob []byte) bool) {
|
||||
for i := uint64(0); i < r.count; i++ {
|
||||
recOff := int(r.recordStart) + int(i)*recordSize
|
||||
recSKU := r.data[recOff : recOff+32]
|
||||
end := len(recSKU)
|
||||
for end > 0 && recSKU[end-1] == 0 {
|
||||
end--
|
||||
}
|
||||
key := string(recSKU[:end])
|
||||
blobLen := binary.LittleEndian.Uint32(r.data[recOff+40 : recOff+44])
|
||||
blobOff := binary.LittleEndian.Uint64(r.data[recOff+32 : recOff+40])
|
||||
if !fn(key, r.data[blobOff:blobOff+uint64(blobLen)]) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user