better convert

projection/store.go

@@ -45,9 +45,11 @@ type Store[W, T any] struct {
 	fallback  Fallback[W]
 	holder    *SnapshotHolder[T] // mmap base; atomic-swap + drain-grace
 
+	hashKeys bool // build the base + dedup Scan under the hashed key encoding
+
 	mu      sync.RWMutex
 	epoch   int64                      // epoch of the live base
-	overlay map[string]overlayEntry[T] // same-epoch deltas layered on the base
+	overlay map[string]overlayEntry[T] // same-epoch deltas layered on the base (keyed by ORIGINAL key)
 
 	// in-progress snapshot assembly (one epoch at a time)
 	building      *SnapshotWriter[T]
@@ -60,13 +62,24 @@ type overlayEntry[T any] struct {
 	deleted bool
 }
 
+// StoreOption configures a Store at construction.
+type StoreOption func(*storeConfig)
+
+type storeConfig struct{ hashKeys bool }
+
+// WithKeyHashing makes the store build its base under the hashed key encoding,
+// so unbounded keys (page paths, query strings) fit the fixed-width index. Get
+// still takes the original key (it is hashed internally); Scan yields digests
+// (recover the original from the value). Match this to how the producer emits.
+func WithKeyHashing() StoreOption { return func(c *storeConfig) { c.hashKeys = true } }
+
 // NewStore opens a store backed by dir/name (its own .bin). decode and
 // transform are REQUIRED: decode parses the domain wire schema from a frame
 // payload; transform projects each wire value to the stored shape. If a base
 // from a previous run is present it is reopened immediately (cold-start ready);
 // its epoch is read from the sidecar so same-epoch deltas can resume. fallback
 // may be nil. Close() when done.
-func NewStore[W, T any](dir, name string, decode Decoder[W], transform Transform[W, T], fallback Fallback[W]) (*Store[W, T], error) {
+func NewStore[W, T any](dir, name string, decode Decoder[W], transform Transform[W, T], fallback Fallback[W], opts ...StoreOption) (*Store[W, T], error) {
 	if dir == "" || name == "" {
 		return nil, fmt.Errorf("projection: Store needs dir and name")
 	}
@@ -76,12 +89,17 @@ func NewStore[W, T any](dir, name string, decode Decoder[W], transform Transform
 	if transform == nil {
 		return nil, fmt.Errorf("projection: Store needs a transform (declare which facts this consumer stores)")
 	}
+	var cfg storeConfig
+	for _, o := range opts {
+		o(&cfg)
+	}
 	s := &Store[W, T]{
 		dir:       dir,
 		name:      name,
 		decode:    decode,
 		transform: transform,
 		fallback:  fallback,
+		hashKeys:  cfg.hashKeys,
 		holder:    NewSnapshotHolder[T](0),
 		overlay:   make(map[string]overlayEntry[T]),
 	}
@@ -127,6 +145,9 @@ func (s *Store[W, T]) beginSnapshot(epoch int64, tenant string) error {
 	// Start (or restart) assembly for this epoch. A new begin supersedes any
 	// half-built snapshot (e.g. a producer that aborted mid-stream).
 	s.building = NewSnapshotWriter[T]().WithTenant(tenant)
+	if s.hashKeys {
+		s.building.WithKeyHashing()
+	}
 	s.buildingEpoch = epoch
 	s.buildingSeen = 0
 	return nil
@@ -271,13 +292,23 @@ func (s *Store[W, T]) Get(sku string) (T, bool) {
 func (s *Store[W, T]) Scan(fn func(key string, v T) bool) {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
+	// The base (holder) is keyed by stored key — a digest in hashed mode. To
+	// dedup overlay against it and emit one consistent key space, project the
+	// overlay's original keys through the same encoding.
+	storeKey := func(k string) string {
+		if s.hashKeys {
+			return hashKey(k)
+		}
+		return k
+	}
 	seen := make(map[string]struct{}, len(s.overlay))
 	for k, e := range s.overlay {
-		seen[k] = struct{}{}
+		sk := storeKey(k)
+		seen[sk] = struct{}{}
 		if e.deleted {
 			continue
 		}
-		if !fn(k, e.val) {
+		if !fn(sk, e.val) {
 			return
 		}
 	}