better convert

projection/snapshot.go

@@ -40,7 +40,9 @@
 package projection
 
 import (
+	"crypto/sha256"
 	"encoding/binary"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -60,16 +62,40 @@ const (
 	recordSize             = 32 + 8 + 4                 // 44 bytes per record
 )
 
+// Key encoding (stored in the formerly-reserved uint32 at header offset 12).
+// 0 = raw, which is byte-identical to a pre-hashing v2 file — so catalog's
+// existing .bin files keep parsing unchanged. 1 = hashed: keys are stored as
+// hex(sha256(key)[:16]) — 32 ASCII chars, so an arbitrarily long key (page
+// path, query string) fits the fixed 32-byte slot. The original key is NOT
+// recoverable from the index in hashed mode — the consumer carries it inside
+// the value blob (the id/sku stays the link). A whole file is one mode: Get
+// applies the same transform the Writer used, read from the header.
+const (
+	keyEncRaw    uint32 = 0
+	keyEncHashed uint32 = 1
+)
+
 var (
 	ErrSnapshotMagic     = errors.New("projection: not a snapshot file (bad magic)")
 	ErrSnapshotVersion   = errors.New("projection: unsupported snapshot version")
 	ErrSnapshotTruncated = errors.New("projection: snapshot truncated")
 	ErrSnapshotBadBlob   = errors.New("projection: snapshot record points past file")
 	ErrSnapshotOverflow  = errors.New("projection: snapshot record-section size overflows uint64")
-	ErrSKUTooLong        = fmt.Errorf("projection: key longer than %d bytes", maxSKULen)
+	ErrSnapshotKeyEnc    = errors.New("projection: unknown key encoding in snapshot header")
+	ErrSKUTooLong        = fmt.Errorf("projection: key longer than %d bytes (enable key hashing for unbounded keys)", maxSKULen)
 	ErrTenantTooLong     = fmt.Errorf("projection: tenant name longer than %d bytes", maxTenantLen)
 )
 
+// hashKey maps an arbitrary-length key to a fixed 32-char hex digest for the
+// hashed key encoding. 128 bits (truncated SHA-256) — collision-negligible at
+// the cardinalities this store targets (tens of thousands of keys). Hex output
+// is pure ASCII, so it never contains the NUL bytes the padded comparator
+// treats as terminators.
+func hashKey(key string) string {
+	sum := sha256.Sum256([]byte(key))
+	return hex.EncodeToString(sum[:16])
+}
+
 // The snapshot is generic over the stored value T (see SnapshotWriter[T] /
 // SnapshotReader[T] below). The binary format, sorting, mmap, and bounds-check
 // logic is value-agnostic and lives in the unexported raw* layer; the generic
@@ -79,6 +105,7 @@ var (
 // Single-shot: append() all, then write() once (atomic rename).
 type rawSnapshotWriter struct {
 	tenant  string
+	keyEnc  uint32
 	blobs   []byte
 	records []snapshotRecord
 	dirty   bool
@@ -92,14 +119,18 @@ type snapshotRecord struct {
 	BlobLength uint32
 }
 
-// append adds a pre-encoded blob under key. ErrSKUTooLong if key exceeds the
-// on-disk width.
+// append adds a pre-encoded blob under key. In raw mode, ErrSKUTooLong if key
+// exceeds the on-disk width; in hashed mode the key is digested to a fixed
+// 32-char slot first, so any length is accepted.
 func (w *rawSnapshotWriter) append(sku string, blob []byte) error {
-	if len(sku) > maxSKULen {
+	key := sku
+	if w.keyEnc == keyEncHashed {
+		key = hashKey(sku)
+	} else if len(sku) > maxSKULen {
 		return ErrSKUTooLong
 	}
 	var rec snapshotRecord
-	copy(rec.SKU[:], sku)
+	copy(rec.SKU[:], key)
 	rec.BlobOffset = uint64(headerSize + len(w.blobs))
 	rec.BlobLength = uint32(len(blob))
 
@@ -130,11 +161,23 @@ func (w *SnapshotWriter[T]) WithTenant(tenant string) *SnapshotWriter[T] {
 	return w
 }
 
-// Append encodes v and indexes it under key. Returns ErrSKUTooLong if key
-// exceeds the on-disk width; returns marshal errors as wrapped fmt.Errorf.
-// key is explicit because T is opaque — the caller knows the key.
+// WithKeyHashing switches this writer to the hashed key encoding: keys are
+// stored as a fixed 32-char digest, so arbitrarily long keys (page paths, query
+// strings) are accepted. The original key is NOT recoverable from the index —
+// carry it inside the value if you need it back (see Scan). A reader opening the
+// file applies the same hashing on Get automatically (the mode is in the
+// header). Set this before any Append.
+func (w *SnapshotWriter[T]) WithKeyHashing() *SnapshotWriter[T] {
+	w.raw.keyEnc = keyEncHashed
+	return w
+}
+
+// Append encodes v and indexes it under key. In raw mode returns ErrSKUTooLong
+// if key exceeds the on-disk width; in hashed mode any length is accepted.
+// Returns marshal errors as wrapped fmt.Errorf. key is explicit because T is
+// opaque — the caller knows the key.
 func (w *SnapshotWriter[T]) Append(sku string, v T) error {
-	if len(sku) > maxSKULen {
+	if w.raw.keyEnc == keyEncRaw && len(sku) > maxSKULen {
 		return ErrSKUTooLong
 	}
 	jb, err := json.Marshal(v)
@@ -190,7 +233,7 @@ func (w *rawSnapshotWriter) write(dir, name string) (finalPath string, err error
 	var hdr [headerSize]byte
 	binary.LittleEndian.PutUint64(hdr[0:8], snapshotMagic)
 	binary.LittleEndian.PutUint32(hdr[8:12], snapshotVersion)
-	binary.LittleEndian.PutUint32(hdr[12:16], 0)
+	binary.LittleEndian.PutUint32(hdr[12:16], w.keyEnc) // 0 = raw (byte-identical to pre-hashing v2)
 	binary.LittleEndian.PutUint64(hdr[16:24], uint64(len(w.records)))
 	binary.LittleEndian.PutUint64(hdr[24:32], blobStart)
 	binary.LittleEndian.PutUint64(hdr[32:40], recordStart)
@@ -282,6 +325,7 @@ type rawSnapshotReader struct {
 	count       uint64
 	blobStart   uint64
 	recordStart uint64
+	keyEnc      uint32 // raw | hashed; applied to the query key in getRaw
 	Tenant      string // populated from header at Open time; "" if not set
 
 	closeOnce sync.Once
@@ -329,12 +373,16 @@ func (r *SnapshotReader[T]) Get(sku string) (T, bool) {
 	return v, true
 }
 
-// Scan walks every (key, value) in key-sorted order, calling fn for each.
-// Return false from fn to stop early. A value that fails to decode is skipped
-// (fn is not called for it). This is the enumeration / ranked-scan substrate:
-// callers that need top-N-by-value collect into a slice and sort in memory —
-// the on-disk order is by key, not value. NOT safe racing Close; drive it
-// through SnapshotHolder.Scan under reload.
+// Scan walks every (key, value) in stored-key-sorted order, calling fn for
+// each. Return false from fn to stop early. A value that fails to decode is
+// skipped (fn is not called for it). This is the enumeration / ranked-scan
+// substrate: callers that need top-N-by-value collect into a slice and sort in
+// memory — the on-disk order is by key, not value. NOT safe racing Close; drive
+// it through SnapshotHolder.Scan under reload.
+//
+// In hashed key mode the `key` passed to fn is the DIGEST, not the original —
+// recover the original from the value (that's why the consumer carries the
+// id/sku in the value blob).
 func (r *SnapshotReader[T]) Scan(fn func(key string, v T) bool) {
 	r.raw.scan(func(key string, blob []byte) bool {
 		var v T
@@ -384,6 +432,12 @@ func newRawSnapshotReader(data []byte) (*rawSnapshotReader, error) {
 	if version != snapshotVersion {
 		return nil, ErrSnapshotVersion
 	}
+	keyEnc := binary.LittleEndian.Uint32(data[12:16])
+	if keyEnc != keyEncRaw && keyEnc != keyEncHashed {
+		// Reject rather than mis-look-up: an unknown encoding means Get can't
+		// reproduce the key transform the writer used.
+		return nil, fmt.Errorf("%w: %d", ErrSnapshotKeyEnc, keyEnc)
+	}
 	count := binary.LittleEndian.Uint64(data[16:24])
 	blobStart := binary.LittleEndian.Uint64(data[24:32])
 	recordStart := binary.LittleEndian.Uint64(data[32:40])
@@ -435,6 +489,7 @@ func newRawSnapshotReader(data []byte) (*rawSnapshotReader, error) {
 		count:       count,
 		blobStart:   blobStart,
 		recordStart: recordStart,
+		keyEnc:      keyEnc,
 		Tenant:      tenant,
 	}, nil
 }
@@ -461,6 +516,9 @@ func (r *rawSnapshotReader) getRaw(sku string) ([]byte, bool) {
 	if r.count == 0 || sku == "" {
 		return nil, false
 	}
+	if r.keyEnc == keyEncHashed {
+		sku = hashKey(sku) // same transform the writer applied
+	}
 	skuBytes := []byte(sku)
 	if len(skuBytes) > maxSKULen {
 		return nil, false