platform
This commit is contained in:
1 parent
e706f7d907
commit
21bcb63440
42 files changed
+5903
No files matched your search
@@ -0,0 +1,386 @@
|
||||
// Builder applies a Schema to a generic source map[string]any and returns
|
||||
// the platform-projected Projection. Pure function — no I/O, no mutation,
|
||||
// safe to call concurrently. Coercions are best-effort:
|
||||
//
|
||||
// - missing source field ⇒ Schema default (FixedCurrency/DefaultTaxClass
|
||||
// when present, otherwise empty/zero)
|
||||
// - wrong-typed source value ⇒ the Builder either coerces (bools from
|
||||
// truthy strings, ints from numeric strings, json.Number, etc.) or
|
||||
// returns ErrProjectionBuild so the caller can decide whether to
|
||||
// log+skip or fail.
|
||||
//
|
||||
// Builder never panics on bad input — a malformed source line is a log+skip,
|
||||
// not a process crash.
|
||||
//
|
||||
// Three coercers cover realistic producer output shapes:
|
||||
//
|
||||
// - stock json.Unmarshal gives numeric fields as float64 unless
|
||||
// `dec.UseNumber()` is on, in which case they come as json.Number.
|
||||
// - json.Number preserves up to ~17 digits of precision; float64
|
||||
// silently loses precision past 2^53. A producer that wants exact
|
||||
// prices sets UseNumber(); this Builder must accept the result.
|
||||
// - some producers serialise via reflection and emit `int` / `int64`
|
||||
// directly; this Builder must accept those too.
|
||||
//
|
||||
// All amounts leave the Builder as money.Cents (integer minor units). The
|
||||
// only float-side conversion lives in platform/money (FromFloatMajor) and
|
||||
// is the explicit lossy-edge; once in Cents, no float is ever summed,
|
||||
// multiplied, or compared downstream.
|
||||
//
|
||||
// SCALE SEMANTICS used by coercePrice:
|
||||
//
|
||||
// - PriceScale == 1: the source value is ALREADY in minor units.
|
||||
// * For string / json.Number: ParseInt directly (integer form only —
|
||||
// the fractional-vs-low-scale guard above rejects "."-containing
|
||||
// inputs).
|
||||
// * For int / int64: identity (multiply by 1).
|
||||
// * For float64 / float32: the float must be integral (else rejected);
|
||||
// cast to int gives minor units directly.
|
||||
//
|
||||
// - PriceScale > 1 (e.g. 100): the source value is in MAJOR units.
|
||||
// * For string / json.Number: ParseCents treats the decimal form as
|
||||
// MAJOR and returns MINOR cents exactly — no further multiply.
|
||||
// * For int / int64: integer multiply (lossless).
|
||||
// * For float64 / float32: float multiply + round-half-away (lossy).
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.k6n.net/mats/platform/money"
|
||||
)
|
||||
|
||||
// ErrProjectionBuild is returned when a source value cannot be coerced
|
||||
// into the Projection shape. Callers typically log + skip so a single bad
|
||||
// item doesn't poison a whole batch.
|
||||
var ErrProjectionBuild = errors.New("catalog: projection build failed")
|
||||
|
||||
// Apply is the pure helper signature: schema, source map → Projection,
|
||||
// error. Use the package-level BuildFromMap helper for the most common
|
||||
// shape (schema + map → Projection); use Apply when you want fine control.
|
||||
func Apply(s Schema, source map[string]any) (Projection, error) {
|
||||
if s.SKUField == "" || s.PriceField == "" {
|
||||
return Projection{}, fmt.Errorf("%w: schema missing sku_field or price_field", ErrProjectionBuild)
|
||||
}
|
||||
if source == nil {
|
||||
return Projection{}, fmt.Errorf("%w: nil source map", ErrProjectionBuild)
|
||||
}
|
||||
sku := stringOf(source, s.SKUField)
|
||||
if sku == "" {
|
||||
return Projection{}, fmt.Errorf("%w: missing SKU at %q", ErrProjectionBuild, s.SKUField)
|
||||
}
|
||||
|
||||
priceRaw, ok := source[s.PriceField]
|
||||
if !ok {
|
||||
return Projection{}, fmt.Errorf("%w: missing price at %q", ErrProjectionBuild, s.PriceField)
|
||||
}
|
||||
price, err := coercePrice(priceRaw, s.PriceScale)
|
||||
if err != nil {
|
||||
return Projection{}, fmt.Errorf("%w: price at %q: %w", ErrProjectionBuild, s.PriceField, err)
|
||||
}
|
||||
|
||||
cur := ""
|
||||
if s.CurrencyField != "" {
|
||||
cur = stringOf(source, s.CurrencyField)
|
||||
}
|
||||
if cur == "" {
|
||||
cur = s.FixedCurrency
|
||||
}
|
||||
|
||||
tax := ""
|
||||
if s.TaxClassField != "" {
|
||||
tax = stringOf(source, s.TaxClassField)
|
||||
}
|
||||
if tax == "" {
|
||||
tax = s.DefaultTaxClass
|
||||
}
|
||||
|
||||
id := sku
|
||||
if s.IDPrefix != "" {
|
||||
id = s.IDPrefix + sku
|
||||
}
|
||||
|
||||
upd, err := coerceUpdatedAt(s, source)
|
||||
if err != nil {
|
||||
return Projection{}, fmt.Errorf("%w: updated_at: %w", ErrProjectionBuild, err)
|
||||
}
|
||||
|
||||
truthy := s.TrueValues
|
||||
if truthy == nil {
|
||||
truthy = DefaultTrueValues()
|
||||
}
|
||||
|
||||
p := Projection{
|
||||
ID: id,
|
||||
SKU: sku,
|
||||
Title: stringOf(source, s.TitleField),
|
||||
PriceIncVat: price,
|
||||
Currency: cur,
|
||||
TaxClass: tax,
|
||||
InventoryTracked: boolOf(source, s.InventoryTrackedF, truthy),
|
||||
DropShip: boolOf(source, s.DropShipF, truthy),
|
||||
Category: stringOf(source, s.CategoryField),
|
||||
Brand: stringOf(source, s.BrandField),
|
||||
Image: stringOf(source, s.ImageField),
|
||||
UpdatedAt: upd,
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// BuildFromMap is the convenience: schema + map → Projection with error.
|
||||
// Use this from producer adapters where the source is a parsed map[string]any.
|
||||
func BuildFromMap(s Schema, source map[string]any) (Projection, error) {
|
||||
return Apply(s, source)
|
||||
}
|
||||
|
||||
// stringOf returns the value at key as a string. JSON unmarshalling gives
|
||||
// strings as string; we accept the common numeric source shapes too so a
|
||||
// map[string]any that came from a different upstream parser (e.g. typed
|
||||
// structs serialised with one round of reflection) still works.
|
||||
//
|
||||
// json.Number is included because precision-preserving producers turn it on
|
||||
// via json.Decoder.UseNumber(); without this case the Builder would reject
|
||||
// every projection in the catalogue.
|
||||
func stringOf(m map[string]any, key string) string {
|
||||
if key == "" {
|
||||
return ""
|
||||
}
|
||||
v, ok := m[key]
|
||||
if !ok || v == nil {
|
||||
return ""
|
||||
}
|
||||
switch x := v.(type) {
|
||||
case string:
|
||||
return x
|
||||
case json.Number:
|
||||
// Use Number preserves up to ~17 digits as a string-token. For SKUs
|
||||
// digit-strings >2^53 (rare but possible), UseNumber is the only
|
||||
// way to round-trip safely. String() gives back the raw digits.
|
||||
return x.String()
|
||||
case bool:
|
||||
if x {
|
||||
return "true"
|
||||
}
|
||||
return "false"
|
||||
case float64:
|
||||
if x == float64(int64(x)) {
|
||||
return strconv.FormatInt(int64(x), 10)
|
||||
}
|
||||
return strconv.FormatFloat(x, 'f', -1, 64)
|
||||
case int:
|
||||
return strconv.Itoa(x)
|
||||
case int64:
|
||||
return strconv.FormatInt(x, 10)
|
||||
}
|
||||
return fmt.Sprintf("%v", v)
|
||||
}
|
||||
|
||||
// boolOf returns the value at key coerced through truthy. nil/missing keys
|
||||
// are false; strings lower-cased and matched against truthy; bools passed
|
||||
// through; numerics "!= 0"; json.Number goes through Float64.
|
||||
func boolOf(m map[string]any, key string, truthy []string) bool {
|
||||
if key == "" {
|
||||
return false
|
||||
}
|
||||
v, ok := m[key]
|
||||
if !ok || v == nil {
|
||||
return false
|
||||
}
|
||||
switch x := v.(type) {
|
||||
case bool:
|
||||
return x
|
||||
case string:
|
||||
low := strings.ToLower(strings.TrimSpace(x))
|
||||
for _, t := range truthy {
|
||||
if low == t {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
case float64:
|
||||
return x != 0
|
||||
case int:
|
||||
return x != 0
|
||||
case int64:
|
||||
return x != 0
|
||||
case json.Number:
|
||||
if f, err := x.Float64(); err == nil {
|
||||
return f != 0
|
||||
}
|
||||
return false
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// coercePrice normalises a price source value to money.Cents (integer minor
|
||||
// units). The schema's PriceScale determines whether the source is already
|
||||
// in minor units (scale==1) or in MAJOR units that need interpretation
|
||||
// (scale > 1, typically 100). See the package doc for the full table.
|
||||
//
|
||||
// The fractional-vs-low-scale guard REJECTS a fractional value when scale ≤ 1
|
||||
// at any of the float / string / json.Number arms — that's almost certainly a
|
||||
// tenant who forgot PriceScale=100 and would have produced a silent 100×-off
|
||||
// price. Loud-fail beats silent corruption.
|
||||
//
|
||||
// At scale > 1, decimals come through money.ParseCents (exact, integer-only)
|
||||
// rather than ParseFloat (lossy past 2^53). Float arms remain the explicit
|
||||
// lossy bits, intentionally isolated.
|
||||
func coercePrice(v any, scale int) (money.Cents, error) {
|
||||
if scale < 0 {
|
||||
return 0, fmt.Errorf("negative price_scale: %d", scale)
|
||||
}
|
||||
effective := int64(scale)
|
||||
if effective < 1 {
|
||||
effective = 1
|
||||
}
|
||||
// Fractional-vs-low-scale guard is inlined at each arm so the error
|
||||
// message keeps the actual value (the float stringification is noisier
|
||||
// than the JSON-token ".eE" check, so a single helper would lose
|
||||
// fidelity).
|
||||
switch x := v.(type) {
|
||||
case float64:
|
||||
if x != float64(int64(x)) && effective == 1 {
|
||||
return 0, fmt.Errorf("fractional price %v with PriceScale=1 — did you forget PriceScale=100?", x)
|
||||
}
|
||||
return money.Cents(roundHalfAwayFromZero(x * float64(effective))), nil
|
||||
case float32:
|
||||
if x != float32(int64(x)) && effective == 1 {
|
||||
return 0, fmt.Errorf("fractional price %v with PriceScale=1 — did you forget PriceScale=100?", x)
|
||||
}
|
||||
return money.Cents(roundHalfAwayFromZero(float64(x) * float64(effective))), nil
|
||||
case int:
|
||||
return money.Cents(int64(x) * effective), nil
|
||||
case int64:
|
||||
return money.Cents(x * effective), nil
|
||||
case string:
|
||||
s := strings.TrimSpace(x)
|
||||
if s == "" {
|
||||
return 0, fmt.Errorf("empty price string")
|
||||
}
|
||||
// Scale 1 with a fractional character set ⇒ foot-gun 4.
|
||||
if effective == 1 && strings.ContainsAny(s, ".eE") {
|
||||
return 0, fmt.Errorf("fractional price %q with PriceScale=1 — did you forget PriceScale=100?", s)
|
||||
}
|
||||
if effective == 1 {
|
||||
// Scale 1 means source IS minor units. Remaining forms are
|
||||
// integer decimal strings — ParseInt directly. ParseCents would
|
||||
// misread "14995" as 14995.00 SEK = 1499500.
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("parse price %q at scale=1: %w", x, err)
|
||||
}
|
||||
return money.Cents(n), nil
|
||||
}
|
||||
// Scale > 1: source is MAJOR units. ParseCents interprets the
|
||||
// decimal form as MAJOR and returns MINOR (cents) EXACTLY — no
|
||||
// further multiply.
|
||||
if c, err := money.ParseCents(s); err == nil {
|
||||
return c, nil
|
||||
}
|
||||
f, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("parse price %q: %w", x, err)
|
||||
}
|
||||
return money.Cents(roundHalfAwayFromZero(f * float64(effective))), nil
|
||||
case json.Number:
|
||||
s := x.String()
|
||||
if effective == 1 && strings.ContainsAny(s, ".eE") {
|
||||
return 0, fmt.Errorf("fractional price %q with PriceScale=1 — did you forget PriceScale=100?", s)
|
||||
}
|
||||
if effective == 1 {
|
||||
// x is the original json.Number — Int64() works.
|
||||
if i, err := x.Int64(); err == nil {
|
||||
return money.Cents(i), nil
|
||||
}
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("parse price json.Number %q at scale=1: %w", s, err)
|
||||
}
|
||||
return money.Cents(n), nil
|
||||
}
|
||||
// Scale > 1: ParseCents token-aware path. ParseCents interprets
|
||||
// the token as MAJOR-units decimal and returns MINOR cents exactly.
|
||||
if c, err := money.ParseCents(s); err == nil {
|
||||
return c, nil
|
||||
}
|
||||
f, err := x.Float64()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("parse price json.Number %q: %w", s, err)
|
||||
}
|
||||
return money.Cents(roundHalfAwayFromZero(f * float64(effective))), nil
|
||||
}
|
||||
return 0, fmt.Errorf("unsupported price type %T", v)
|
||||
}
|
||||
|
||||
// roundHalfAwayFromZero rounds half-away-from-zero instead of banker's
|
||||
// rounding so e.g. 99.995 → 100 (with scale=1) not 99, and -99.995 → -100
|
||||
// not -99. We accept the very small skew in favour of deterministic
|
||||
// conventional rounding for signed amounts (refunds, discounts).
|
||||
func roundHalfAwayFromZero(f float64) int64 {
|
||||
if f >= 0 {
|
||||
return int64(f + 0.5)
|
||||
}
|
||||
return int64(f - 0.5)
|
||||
}
|
||||
|
||||
// coerceUpdatedAt returns Unix-ms for the Projection.UpdatedAt stamp based
|
||||
// on Schema.UpdatedAtSource.
|
||||
//
|
||||
// IMPORTANT contract (foot-gun 3, resolved):
|
||||
//
|
||||
// - StampAtPublish is the RESERVED deliberate-fallback. Builder sets
|
||||
// time.Now().UnixMilli(). Every rebuild re-stamps every projection,
|
||||
// invalidating every consumer's whole cache even when nothing changed.
|
||||
// Pick this only when the producer genuinely has no version stamp.
|
||||
//
|
||||
// - UseSourceField (DefaultSchema, BygglagretSchema) reads the value
|
||||
// from source[Schema.UpdatedAtField]. If the field is MISSING or the
|
||||
// value can't be parsed, Apply returns ErrProjectionBuild (wrapped)
|
||||
// rather than stamp-now'ing. That keeps UpdatedAt a true content
|
||||
// version across rebuilds — same source content ⇒ same UpdatedAt ⇒
|
||||
// consumer caches stay valid.
|
||||
//
|
||||
// Consumers compare UpdatedAt with "strictly greater wins" semantics; a
|
||||
// silent stamp-now fallback would defeat this contract.
|
||||
func coerceUpdatedAt(s Schema, source map[string]any) (int64, error) {
|
||||
if s.UpdatedAtSource == StampAtPublish || s.UpdatedAtField == "" {
|
||||
return time.Now().UnixMilli(), nil
|
||||
}
|
||||
v, ok := source[s.UpdatedAtField]
|
||||
if !ok || v == nil {
|
||||
return 0, fmt.Errorf("missing updated_at at %q (UseSourceField requires the source to supply it)", s.UpdatedAtField)
|
||||
}
|
||||
switch x := v.(type) {
|
||||
case int64:
|
||||
return x, nil
|
||||
case int:
|
||||
return int64(x), nil
|
||||
case float64:
|
||||
return int64(x), nil
|
||||
case json.Number:
|
||||
if i, err := x.Int64(); err == nil {
|
||||
return i, nil
|
||||
}
|
||||
f, err := x.Float64()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("updated_at json.Number %q: %w", x.String(), err)
|
||||
}
|
||||
return int64(f), nil
|
||||
case string:
|
||||
t := strings.TrimSpace(x)
|
||||
if t == "" {
|
||||
return 0, fmt.Errorf("empty updated_at string at %q (UseSourceField requires a value)", s.UpdatedAtField)
|
||||
}
|
||||
n, err := strconv.ParseInt(t, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("parse updated_at %q: %w", x, err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
return 0, fmt.Errorf("unsupported updated_at type %T at %q", v, s.UpdatedAtField)
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// json.Number from a json.Decoder running with UseNumber() is the
|
||||
// canonical path for precision-preserving producer parsing. Without this
|
||||
// case, BuildFromMap would log+skip every projection that came through
|
||||
// UseNumber — silently losing items.
|
||||
func TestBuilderJSONNumber(t *testing.T) {
|
||||
// json.Number for SKU
|
||||
src := map[string]any{
|
||||
"sku": json.Number("123456789012345"), // float64-unsafe
|
||||
"title": "BigNumber SKU",
|
||||
"price_inc_vat": json.Number("14995"), // already øre
|
||||
"currency": "SEK",
|
||||
"tax_class": "standard",
|
||||
"updated_at": json.Number("1700000000000"),
|
||||
}
|
||||
p, err := Apply(DefaultSchema(), src)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Number source rejected: %v", err)
|
||||
}
|
||||
if p.SKU != "123456789012345" {
|
||||
t.Errorf("SKU: got %q want %q", p.SKU, "123456789012345")
|
||||
}
|
||||
if p.PriceIncVat != 14995 {
|
||||
t.Errorf("PriceIncVat: got %d want %d", p.PriceIncVat, 14995)
|
||||
}
|
||||
if p.UpdatedAt != 1700000000000 {
|
||||
t.Errorf("UpdatedAt: got %d want %d", p.UpdatedAt, int64(1700000000000))
|
||||
}
|
||||
}
|
||||
|
||||
// Updated=UseSourceField: missing source updated_at MUST be a loud error,
|
||||
// not a stamp-now fallback. (Foot-gun 3 resolution: silence here invalidates
|
||||
// every consumer's cache on rebuild.)
|
||||
func TestBuilderRequiresUpdatedAt(t *testing.T) {
|
||||
src := map[string]any{
|
||||
"sku": "X-1",
|
||||
"price_inc_vat": int64(100),
|
||||
// no "updated_at" key intentionally
|
||||
}
|
||||
_, err := Apply(DefaultSchema(), src)
|
||||
if err == nil {
|
||||
t.Fatal("UseSourceField + missing updated_at should error, not stamp-now")
|
||||
}
|
||||
if !errors.Is(err, ErrProjectionBuild) {
|
||||
t.Errorf("got %v want ErrProjectionBuild", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "updated_at") {
|
||||
t.Errorf("error should mention updated_at: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Same projection under StampAtPublish succeeds even without updated_at —
|
||||
// that's the deliberate-fallback path. Used by sources with no other stamp.
|
||||
// Bind the stamp to real wall-clock time (before ≤ actual ≤ after+slack)
|
||||
// so the test distinguishes "stamp-now was called" from any other non-zero
|
||||
// path.
|
||||
func TestBuilderStampAtPublishFallback(t *testing.T) {
|
||||
s := DefaultSchema()
|
||||
s.UpdatedAtSource = StampAtPublish
|
||||
s.UpdatedAtField = "" // ignored when source is StampAtPublish
|
||||
before := time.Now().UnixMilli()
|
||||
src := map[string]any{
|
||||
"sku": "X-2",
|
||||
"price_inc_vat": int64(100),
|
||||
}
|
||||
p, err := Apply(s, src)
|
||||
if err != nil {
|
||||
t.Fatalf("StampAtPublish should accept missing updated_at: %v", err)
|
||||
}
|
||||
after := time.Now().UnixMilli() + 1000 // generous slack for slow CI
|
||||
if p.UpdatedAt < before || p.UpdatedAt > after {
|
||||
t.Errorf("UpdatedAt should be stamp-now Unix-ms in [%d,%d], got %d", before, after, p.UpdatedAt)
|
||||
}
|
||||
}
|
||||
|
||||
// Fractional float price with scale=1 must be loud-fail — the float arm of
|
||||
// coercePrice has the same foot-gun 4 risk as the string arm (silent 100x
|
||||
// bias if a tenant forgets PriceScale=100 and ships a float literal).
|
||||
func TestBuilderFractionalFloatLowScaleRejected(t *testing.T) {
|
||||
src := map[string]any{
|
||||
"sku": "F-1",
|
||||
"price_inc_vat": 99.95, // float major-units at scale=1
|
||||
"updated_at": int64(1700000000000),
|
||||
}
|
||||
_, err := Apply(DefaultSchema(), src)
|
||||
if err == nil {
|
||||
t.Fatal("fractional float price at scale=1 should error, not silently produce 100 øre")
|
||||
}
|
||||
if !errors.Is(err, ErrProjectionBuild) {
|
||||
t.Errorf("got %v want ErrProjectionBuild", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "PriceScale=100") {
|
||||
t.Errorf("error message should mention PriceScale=100 hint: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// json.Number("149.95") at scale=1 must reach the fractional guard via
|
||||
// the json.Number branch — the precision-safe producer route.
|
||||
func TestBuilderFractionalJSONNumberLowScaleRejected(t *testing.T) {
|
||||
src := map[string]any{
|
||||
"sku": "JN-1",
|
||||
"price_inc_vat": json.Number("149.95"),
|
||||
"updated_at": int64(1700000000000),
|
||||
}
|
||||
_, err := Apply(DefaultSchema(), src)
|
||||
if err == nil {
|
||||
t.Fatal("fractional json.Number at scale=1 should error, not silently produce 150 øre")
|
||||
}
|
||||
if !errors.Is(err, ErrProjectionBuild) {
|
||||
t.Errorf("got %v want ErrProjectionBuild", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "PriceScale=100") {
|
||||
t.Errorf("error message should mention PriceScale=100 hint: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Fractional string price with scale=1 must be loud-fail (a misconfigured
|
||||
// tenant shipping major units without PriceScale=100 — silent 100x bias).
|
||||
func TestBuilderFractionalStringPriceLowScaleRejected(t *testing.T) {
|
||||
src := map[string]any{
|
||||
"sku": "X-1",
|
||||
"price_inc_vat": "149.95", // major units FRACTION at scale=1
|
||||
"updated_at": int64(1700000000000),
|
||||
}
|
||||
_, err := Apply(DefaultSchema(), src)
|
||||
if err == nil {
|
||||
t.Fatal("fractional string at scale=1 should error, not silently produce 150 øre")
|
||||
}
|
||||
if !errors.Is(err, ErrProjectionBuild) {
|
||||
t.Errorf("got %v want ErrProjectionBuild", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "PriceScale=100") {
|
||||
t.Errorf("error message should mention PriceScale=100 hint: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Same string with scale=100 (correct major-unit config) succeeds.
|
||||
func TestBuilderFractionalStringPriceCorrectScale(t *testing.T) {
|
||||
s := DefaultSchema()
|
||||
s.PriceScale = 100
|
||||
src := map[string]any{
|
||||
"sku": "X-2",
|
||||
"price_inc_vat": "149.95",
|
||||
"updated_at": int64(1700000000000),
|
||||
}
|
||||
p, err := Apply(s, src)
|
||||
if err != nil {
|
||||
t.Fatalf("scale=100 string price rejected: %v", err)
|
||||
}
|
||||
if p.PriceIncVat != 14995 {
|
||||
t.Errorf("PriceIncVat: got %d want %d", p.PriceIncVat, 14995)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
// Package catalog defines the published, versioned catalog projection that
|
||||
// commerce modules (cart, checkout, order) consume — and the configurable
|
||||
// per-tenant Schema that maps a producer's private item JSON into that
|
||||
// projection.
|
||||
//
|
||||
// Why configurable: different tenants ship with different catalog shapes
|
||||
// (one has ArticleNumber, another has sku; one's price is inc-VAT minor units,
|
||||
// another's is a float with a separate currency; some include a Brand field,
|
||||
// others don't). A one-size-fits-all Projection field set would force every
|
||||
// tenant to field the union of all constraints; the configurable Schema keeps
|
||||
// each tenant's producer adapter small while still emitting the same
|
||||
// downstream wire shape, so commerce consumers do not change.
|
||||
//
|
||||
// What's here:
|
||||
//
|
||||
// - Projection: the published, versioned entry (struct + Encode helpers).
|
||||
// - Schema: declarative field-map for one tenant's source JSON.
|
||||
// - DefaultSchema: the consensus shape that works out of the box when no
|
||||
// tenant-specific override is supplied.
|
||||
// - Registry: per-tenant lookup with DefaultSchema fallback.
|
||||
// - Builder: applies a Schema to a generic source map; pure function.
|
||||
//
|
||||
// What is NOT here (kept out by design):
|
||||
//
|
||||
// - Numerics live in minor units (öre) and stay int64 — never float. The
|
||||
// platform-wide rate-scale convention (basis-points-of-a-percent) is
|
||||
// documented in platform/tax; TaxClass is a STRING identifier on the
|
||||
// Projection side and is resolved to a numeric rate by platform/tax, never
|
||||
// on the wire.
|
||||
// - No transport, no AMQP consumer/producer, no event-subscriber wiring —
|
||||
// callers publish / consume `Projection` JSON via the existing
|
||||
// `platform/event` + `platform/rabbit` package plumbing.
|
||||
// - No HTTP lookup. Finder remains the federation source; no commerce module
|
||||
// issues a "lookup this SKU" RPC today and resolving it here would be
|
||||
// premature.
|
||||
//
|
||||
// See docs/published-document-vocabulary-pass.md §11 for the wider vocabulary
|
||||
// context (this package implements the cart side of that contract; the full
|
||||
// vocabulary is the next pass and lives in `platform/published` once proven
|
||||
// out).
|
||||
package catalog
|
||||
@@ -0,0 +1,134 @@
|
||||
// Package catalog — see doc.go for the package summary.
|
||||
//
|
||||
// This file holds the published Projection struct: the wire shape that
|
||||
// commerce modules receive. Every producer adapter (writeradmin, future CMS
|
||||
// catalog adapter, ...) emits this exact shape; every consumer (cart,
|
||||
// checkout, order, UCP add-to-cart) reads from this exact shape. Per-tenant
|
||||
// field remapping happens in Schema + Builder (schema.go, builder.go).
|
||||
//
|
||||
// Mapping source JSON → Projection is per-tenant and lives in Schema.
|
||||
// The consensus mapping for DefaultSchema() is:
|
||||
//
|
||||
// id ← IDPrefix + SKU (e.g. "sku:<SKU>")
|
||||
// sku ← source[SKUField]
|
||||
// title ← source[TitleField] (omitempty, optional)
|
||||
// price_inc_vat ← coercive parse of source[PriceField] × PriceScale
|
||||
// currency ← source[CurrencyField], or FixedCurrency
|
||||
// tax_class ← source[TaxClassField] (string identifier)
|
||||
// inventory_tracked ← bool coercion of source[InventoryTrackedField]
|
||||
// drop_ship ← bool coercion of source[DropShipField]
|
||||
// category ← source[CategoryField] (omitempty)
|
||||
// brand ← source[BrandField] (omitempty)
|
||||
// image ← source[ImageField] (omitempty)
|
||||
// updated_at ← Unix-ms from source[UpdatedAtField] OR stamped at
|
||||
// publish time when UpdatedAtSource == StampAtPublish
|
||||
//
|
||||
// SCALE (rate-bearing fields): any field that carries a tax rate uses the
|
||||
// platform-wide basis-points-of-a-percent convention (rate × 100; 25% = 2500,
|
||||
// 12.5% = 1250, 8.1% = 810). Never float. Producer-side fields stay in
|
||||
// minor currency units (öre / cents) for amounts — already int64. TaxClass
|
||||
// on the wire is a STRING identifier; numeric rate resolution is a platform/tax
|
||||
// concern (out of package scope by design).
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"git.k6n.net/mats/platform/money"
|
||||
)
|
||||
|
||||
// Projection is the published, versioned catalog entry that commerce modules
|
||||
// consume. Finder owns updating it; cart / checkout / order read it.
|
||||
//
|
||||
// Versioning: UpdatedAt is a monotonic Unix-millisecond timestamp stamped by
|
||||
// the producer at the time the AMQP event is emitted (or read from a
|
||||
// designated source field, see Schema.UpdatedAtSource). Consumers, when
|
||||
// revalidating, treat a strictly greater UpdatedAt as the authoritative
|
||||
// replacement and refresh the local copy.
|
||||
//
|
||||
// Field rationale:
|
||||
//
|
||||
// ID — opaque published identity; SKU alone is not a stable
|
||||
// cross-module id (different scopes reuse it).
|
||||
// SKU — domain identifier; the join key cart already uses.
|
||||
// PriceIncVat — display + checkout projection; NOT transactional truth.
|
||||
// System of record for the total is settled via the
|
||||
// checkout/order payload — see platform-uniforming-plan.md.
|
||||
// Currency — optional but recommended; absent ⇒ caller supplies.
|
||||
// TaxClass — class identifier (string). Numeric rate resolution is
|
||||
// a platform/tax concern (see platform/tax.Provider). This
|
||||
// struct never carries a numeric rate; that keeps the
|
||||
// rate-scale convention in one place and keeps the wire
|
||||
// free of consumer-policy values.
|
||||
// InventoryTracked / DropShip — fulfillment-routing flags. The C1 contract
|
||||
// asks for these so the cart can pre-mark drop-ship
|
||||
// items before checkout and inventory can opt-out of
|
||||
// tracking for non-stock items.
|
||||
// Category / Brand / Image — display projection only.
|
||||
// UpdatedAt — version stamp for snapshot + revalidate.
|
||||
type Projection struct {
|
||||
ID string `json:"id"`
|
||||
SKU string `json:"sku"`
|
||||
Title string `json:"title,omitempty"`
|
||||
PriceIncVat money.Cents `json:"price_inc_vat"`
|
||||
Currency string `json:"currency,omitempty"`
|
||||
// TaxClass uses a string identifier (see platform/tax class constants).
|
||||
// Empty string means "unclassified"; the consumer's tax.Provider is what
|
||||
// resolves the identifier to a numeric rate.
|
||||
TaxClass string `json:"tax_class"`
|
||||
InventoryTracked bool `json:"inventory_tracked"`
|
||||
DropShip bool `json:"drop_ship"`
|
||||
Category string `json:"category,omitempty"`
|
||||
Brand string `json:"brand,omitempty"`
|
||||
Image string `json:"image,omitempty"`
|
||||
UpdatedAt int64 `json:"updated_at"`
|
||||
}
|
||||
|
||||
// PayloadKey is the envelope key producers use when publishing a slice of
|
||||
// projections over the bus as an `event.Event.Payload` body. Consumers parse
|
||||
// this exact key — never a per-producer ad-hoc name — so the wire shape is
|
||||
// stable.
|
||||
const PayloadKey = "catalog_projection"
|
||||
|
||||
// Encode serialises a Projection to JSON suitable for the event.Payload.
|
||||
// Kept as a package-level helper so producers and tests share the exact wire
|
||||
// shape; any future wire renaming lands here, not in every caller.
|
||||
func (p Projection) Encode() ([]byte, error) {
|
||||
return json.Marshal(p)
|
||||
}
|
||||
|
||||
// EncodeBatch serialises a slice to a JSON array.
|
||||
func EncodeBatch(batch []Projection) ([]byte, error) {
|
||||
return json.Marshal(batch)
|
||||
}
|
||||
|
||||
// ProjectionUpdate is one entry in a CatalogProjectionPublished batch. An
|
||||
// upsert carries a fully-built Projection. A delete sets Deleted=true; only SKU
|
||||
// (and ID) are then meaningful and the consumer drops that SKU from its
|
||||
// snapshot / cache. Modelling delete as a flag (not a nil payload) keeps a
|
||||
// change-set's upserts and deletes together in one ordered event.
|
||||
type ProjectionUpdate struct {
|
||||
Projection
|
||||
Deleted bool `json:"deleted,omitempty"`
|
||||
}
|
||||
|
||||
// EncodeUpdates serialises a batch for the event.Event.Payload body.
|
||||
func EncodeUpdates(batch []ProjectionUpdate) ([]byte, error) {
|
||||
return json.Marshal(batch)
|
||||
}
|
||||
|
||||
// DecodeUpdates parses a CatalogProjectionPublished payload — the consumer side
|
||||
// of EncodeUpdates.
|
||||
func DecodeUpdates(raw []byte) ([]ProjectionUpdate, error) {
|
||||
var out []ProjectionUpdate
|
||||
err := json.Unmarshal(raw, &out)
|
||||
return out, err
|
||||
}
|
||||
|
||||
// Decode parses a Projection from JSON. Symmetric with Encode so consumer
|
||||
// tests can round-trip a fixture without depending on the Builder.
|
||||
func Decode(raw []byte) (Projection, error) {
|
||||
var p Projection
|
||||
err := json.Unmarshal(raw, &p)
|
||||
return p, err
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
// Registry is the per-tenant Schema lookup. A producer processes tenant X:
|
||||
// it builds an event.Event with Source="catalog:<tenant>" and asks the
|
||||
// Registry for X's Schema; the Builder applies that Schema to the source
|
||||
// JSON. Unknown tenants get DefaultSchema — that makes the platform
|
||||
// "just work" for new tenants while the operator rolls out the proper
|
||||
// override file.
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Registry is concurrency-safe: Register / SchemaFor can be called from any
|
||||
// goroutine. SchemaFor is the hot path; it takes a read lock.
|
||||
type Registry struct {
|
||||
mu sync.RWMutex
|
||||
schemas map[string]Schema
|
||||
def Schema // the "default" / DefaultSchema() fallback
|
||||
}
|
||||
|
||||
// NewRegistry returns an empty Registry whose DefaultSchema fallback is
|
||||
// DefaultSchema(). Use Register to add tenant-specific overrides.
|
||||
func NewRegistry() *Registry {
|
||||
return &Registry{
|
||||
schemas: map[string]Schema{},
|
||||
def: DefaultSchema(),
|
||||
}
|
||||
}
|
||||
|
||||
// WithDefault swaps the DefaultSchema fallback (used when SchemaFor is
|
||||
// called with an unknown tenant name, or with the literal "default").
|
||||
// Returns the receiver so it composes with NewRegistry().
|
||||
func (r *Registry) WithDefault(s Schema) *Registry {
|
||||
if s.Tenant == "" {
|
||||
s.Tenant = "default"
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.def = s
|
||||
return r
|
||||
}
|
||||
|
||||
// Register stores s under s.Tenant. A Schema with empty Tenant is stored
|
||||
// under "default" and ALSO replaces the DefaultSchema fallback.
|
||||
func (r *Registry) Register(s Schema) error {
|
||||
if s.SKUField == "" || s.PriceField == "" {
|
||||
return fmt.Errorf("catalog: schema for %q must declare sku_field and price_field", s.Tenant)
|
||||
}
|
||||
if s.PriceScale < 0 {
|
||||
return fmt.Errorf("catalog: schema %q has negative price_scale (%d)", s.Tenant, s.PriceScale)
|
||||
}
|
||||
// Foot-gun 3, registry-side guard: UseSourceField without a field is
|
||||
// the silent stamp-now path. Refuse at load time so a JSON typo
|
||||
// surfaces immediately rather than invalidating caches per-rebuild.
|
||||
if s.UpdatedAtSource == UseSourceField && s.UpdatedAtField == "" {
|
||||
return fmt.Errorf("catalog: schema %q has updated_at_source=UseSourceField but no updated_at_field (would silently stamp-now)", s.Tenant)
|
||||
}
|
||||
if s.TrueValues == nil {
|
||||
s.TrueValues = DefaultTrueValues()
|
||||
}
|
||||
key := s.Tenant
|
||||
if key == "" {
|
||||
key = "default"
|
||||
s.Tenant = key
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.def = s
|
||||
r.schemas[key] = s
|
||||
return nil
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.schemas[key] = s
|
||||
return nil
|
||||
}
|
||||
|
||||
// SchemaFor returns the Schema registered for tenant, or the DefaultSchema
|
||||
// fallback when tenant is "" or not registered.
|
||||
func (r *Registry) SchemaFor(tenant string) Schema {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
if tenant == "" {
|
||||
return r.def
|
||||
}
|
||||
if s, ok := r.schemas[tenant]; ok {
|
||||
return s
|
||||
}
|
||||
return r.def
|
||||
}
|
||||
|
||||
// Tenants returns the sorted list of registered tenant names (excluding the
|
||||
// default fallback). Useful for ops dashboards and "what schemas are loaded".
|
||||
func (r *Registry) Tenants() []string {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
out := make([]string, 0, len(r.schemas))
|
||||
for k := range r.schemas {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// LoadFile reads a JSON file of { "tenant": Schema, ... } and registers each
|
||||
// entry into r. The JSON shape mirrors the Schema struct exactly (json tags
|
||||
// in schema.go). Missing file is OK (file is optional); unreadable file
|
||||
// returns the error so a misconfigured deployment fails loudly on startup.
|
||||
func (r *Registry) LoadFile(path string) error {
|
||||
if path == "" {
|
||||
return nil
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("catalog: read schema file %q: %w", path, err)
|
||||
}
|
||||
var raw map[string]Schema
|
||||
if err := json.Unmarshal(data, &raw); err != nil {
|
||||
return fmt.Errorf("catalog: parse schema file %q: %w", path, err)
|
||||
}
|
||||
// Walk in sorted order so operator-visible loading is deterministic
|
||||
// (helps when something fails partway through).
|
||||
keys := make([]string, 0, len(raw))
|
||||
for k := range raw {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
s := raw[k]
|
||||
if s.Tenant == "" {
|
||||
s.Tenant = k
|
||||
}
|
||||
if err := r.Register(s); err != nil {
|
||||
return fmt.Errorf("catalog: schema %q in %q: %w", k, path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestRegistryLookupAndDefault — registered tenant wins, unknown falls back.
|
||||
func TestRegistryLookupAndDefault(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
if got := r.SchemaFor("missing"); !reflect.DeepEqual(got, DefaultSchema()) {
|
||||
t.Errorf("unknown tenant did not fall back to DefaultSchema")
|
||||
}
|
||||
bygg := BygglagretSchema()
|
||||
if err := r.Register(bygg); err != nil {
|
||||
t.Fatalf("Register: %v", err)
|
||||
}
|
||||
got := r.SchemaFor("bygglagret")
|
||||
if got.SKUField != "ArticleNumber" {
|
||||
t.Errorf("Register did not store under Tenant: %+v", got)
|
||||
}
|
||||
if got.SKUField == DefaultSchema().SKUField {
|
||||
t.Errorf("Expected bygglagret to differ from default")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRegisterRejectsInvalidSchema — guard against typos in operator JSON.
|
||||
func TestRegisterRejectsInvalidSchema(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
err := r.Register(Schema{Tenant: "broken", PriceField: "x"}) // missing SKUField
|
||||
if err == nil {
|
||||
t.Error("expected error for schema without SKUField")
|
||||
}
|
||||
err = r.Register(Schema{Tenant: "negscale", SKUField: "s", PriceField: "p", PriceScale: -1})
|
||||
if err == nil {
|
||||
t.Error("expected error for negative PriceScale")
|
||||
}
|
||||
}
|
||||
|
||||
// TestTenantsSorted — dashboard-friendly ordering.
|
||||
func TestTenantsSorted(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
for _, k := range []string{"zeta", "alpha", "mu"} {
|
||||
s := Schema{Tenant: k, SKUField: "s", PriceField: "p"}
|
||||
if err := r.Register(s); err != nil {
|
||||
t.Fatalf("Register %s: %v", k, err)
|
||||
}
|
||||
}
|
||||
got := r.Tenants()
|
||||
want := []string{"alpha", "mu", "zeta"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("Tenants() got %v want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadFileMissingIsOK — file is optional.
|
||||
func TestLoadFileMissingIsOK(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
if err := r.LoadFile(filepath.Join(t.TempDir(), "nope.json")); err != nil {
|
||||
t.Errorf("missing file should be a no-op, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadFileHappy — load tenant schemas from disk and look one up.
|
||||
func TestLoadFileHappy(t *testing.T) {
|
||||
// JSON shape mirrors schema.go json tags; tenant keys ("bygglagret") are
|
||||
// the registry names.
|
||||
jsonBody := `{
|
||||
"bygglagret": {
|
||||
"sku_field": "ArticleNumber",
|
||||
"price_field": "PriceInclVAT",
|
||||
"price_scale": 100,
|
||||
"id_prefix": "sku:",
|
||||
"updated_at_source": 1,
|
||||
"updated_at_field": "LastModifiedMs",
|
||||
"fixed_currency": "SEK"
|
||||
},
|
||||
"north": {
|
||||
"sku_field": "sku",
|
||||
"price_field": "price_minor",
|
||||
"price_scale": 1
|
||||
}
|
||||
}`
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "schemas.json")
|
||||
writeFile(t, path, jsonBody)
|
||||
r := NewRegistry()
|
||||
if err := r.LoadFile(path); err != nil {
|
||||
t.Fatalf("LoadFile: %v", err)
|
||||
}
|
||||
if got := r.SchemaFor("bygglagret").PriceField; got != "PriceInclVAT" {
|
||||
t.Errorf("bygglagret not loaded: %q", got)
|
||||
}
|
||||
if got := r.SchemaFor("north").PriceScale; got != 1 {
|
||||
t.Errorf("north not loaded: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// writeFile is a local helper; writes the JSON tenant-schema fixture to path.
|
||||
func writeFile(t *testing.T, path, body string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
// Schema declarations and the DefaultSchema consensus shape.
|
||||
//
|
||||
// A Schema maps a producer's private source JSON to the platform-wide
|
||||
// Projection. Field names point at keys in the source map; Value coercions
|
||||
// happen in the Builder. A Schema is the unit of configuration per tenant;
|
||||
// the Registry (registry.go) carries the per-tenant map.
|
||||
package catalog
|
||||
|
||||
// UpdatedAtSource controls where the Projection's UpdatedAt comes from.
|
||||
//
|
||||
// - StampAtPublish — Builder sets UpdatedAt to time.Now().UnixMilli() at the
|
||||
// moment Build() runs. RESERVED as a deliberate-fallback for sources
|
||||
// that genuinely have no other version stamp. NOT a content-version
|
||||
// address: a full snapshot rebuild re-stamps every projection and
|
||||
// invalidates every consumer cache even when nothing changed. Use only
|
||||
// when the producer side has NO source mtime / hash to set.
|
||||
//
|
||||
// - UseSourceField — the production-correct choice. Builder parses
|
||||
// source[Schema.UpdatedAtField] as a Unix-millisecond integer (or
|
||||
// decimal-fraction-of-Unix-second if the source uses seconds — Apply
|
||||
// accepts that and converts). This is the content-version value;
|
||||
// rebuilds produce identical UpdatedAt when content is identical.
|
||||
//
|
||||
// Critical contract: UpdatedAt is the CONTENT VERSION, not the publish
|
||||
// timestamp. Consumers use it for snapshot + revalidate. If your schema
|
||||
// sets UpdatedAtSource=StampAtPublish, every snapshot rebuild churns the
|
||||
// version. Choose UseSourceField for any consumer-cache scenario.
|
||||
type UpdatedAtSource int
|
||||
|
||||
const (
|
||||
// StampAtPublish sets UpdatedAt at Build() time. DefaultSchema
|
||||
// deliberately does NOT use this; it's reserved for callers who know
|
||||
// they want publish-time semantics.
|
||||
StampAtPublish UpdatedAtSource = iota
|
||||
// UseSourceField reads the version from Schema.UpdatedAtField. This is
|
||||
// the contract for content-versioned snapshots.
|
||||
UseSourceField
|
||||
)
|
||||
|
||||
// Schema is the declarative per-tenant mapping from a producer's source JSON
|
||||
// to the published Projection. Zero value is NOT valid — always start from
|
||||
// DefaultSchema() and override what differs, so unset-but-required fields
|
||||
// surface immediately rather than silently emitting empty Projections.
|
||||
//
|
||||
// All "Field" strings refer to keys in the source map[string]any that the
|
||||
// producer hands the Builder. Empty Field means "this Projection carrier is
|
||||
// supplied by Fixed* / Default* / Build-time, not read from source".
|
||||
type Schema struct {
|
||||
// Tenant is the registry key this schema is registered under. Empty
|
||||
// means "default" — see Registry.
|
||||
Tenant string `json:"tenant,omitempty"`
|
||||
|
||||
// --- source field references (per carrier) ---
|
||||
SKUField string `json:"sku_field"`
|
||||
TitleField string `json:"title_field,omitempty"`
|
||||
PriceField string `json:"price_field"`
|
||||
CurrencyField string `json:"currency_field,omitempty"`
|
||||
TaxClassField string `json:"tax_class_field,omitempty"`
|
||||
InventoryTrackedF string `json:"inventory_tracked_field,omitempty"`
|
||||
DropShipF string `json:"drop_ship_field,omitempty"`
|
||||
CategoryField string `json:"category_field,omitempty"`
|
||||
BrandField string `json:"brand_field,omitempty"`
|
||||
ImageField string `json:"image_field,omitempty"`
|
||||
|
||||
// --- fixed / default values used when a source field is absent ---.
|
||||
FixedCurrency string `json:"fixed_currency,omitempty"`
|
||||
// DefaultTaxClass is what the Builder writes when the source has no
|
||||
// TaxClass field (e.g. producer doesn't classify). Empty means "leave
|
||||
// empty" — consumer's tax.Provider gets called with empty string.
|
||||
DefaultTaxClass string `json:"default_tax_class,omitempty"`
|
||||
|
||||
// --- derived-value controls ---
|
||||
// PriceScale: source value interpreted as MINOR units if ≤ 1 (pass
|
||||
// through; floats round half-away-from-zero, ints pass through exact),
|
||||
// MAJOR units if 100 (multiply by 100 to get øre), etc. Negative =
|
||||
// rejected. The Builder rounds half-away-from-zero on fractional
|
||||
// results so the integer Projection never silently truncates.
|
||||
PriceScale int `json:"price_scale"`
|
||||
// IDPrefix is prepended to the normalised ID. Use "sku:" for the
|
||||
// consensus convention; an empty prefix means raw SKU.
|
||||
IDPrefix string `json:"id_prefix,omitempty"`
|
||||
// UpdatedAtSource — see type doc. DefaultSchema uses UseSourceField and
|
||||
// pins UpdatedAtField="updated_at". Producers MUST supply an updated_at
|
||||
// value (Unix-ms int); Apply returns ErrProjectionBuild (wrapped) when
|
||||
// it's missing rather than silently stamp-now'ing and churning every
|
||||
// rebuild. StampAtPublish is RESERVED for sources with no other stamp.
|
||||
UpdatedAtSource UpdatedAtSource `json:"updated_at_source"`
|
||||
// UpdatedAtField is the SOURCE-side key (when UpdatedAtSource ==
|
||||
// UseSourceField) the Builder reads the version from.
|
||||
UpdatedAtField string `json:"updated_at_field,omitempty"`
|
||||
// TrueValues is the set of source values that map to InventoryTracked
|
||||
// / DropShip = true. Comparison is lower-cased. Defaults to
|
||||
// DefaultTrueValues below if left nil — never add "false" / "0" etc,
|
||||
// the Builder treats anything not in TrueValues as false.
|
||||
TrueValues []string `json:"true_values,omitempty"`
|
||||
}
|
||||
|
||||
// DefaultSchema is the consensus schema: SKU under "sku", price under
|
||||
// "price_inc_vat" already in øre, currency from source "currency" or empty,
|
||||
// tax_class from "tax_class", inventory_tracked / drop_ship from
|
||||
// "inventory_tracked" / "drop_ship" bool fields. updated_at MUST come from
|
||||
// the source under the field name "updated_at" (Unix-ms) — Apply rejects
|
||||
// missing values loudly rather than silent stamp-now churn.
|
||||
//
|
||||
// Use this when a tenant has no override file; it is the safe default for
|
||||
// producers that already emit platform-shaped JSON.
|
||||
func DefaultSchema() Schema {
|
||||
return Schema{
|
||||
Tenant: "default",
|
||||
// source-side field names
|
||||
SKUField: "sku",
|
||||
TitleField: "title",
|
||||
PriceField: "price_inc_vat",
|
||||
CurrencyField: "currency",
|
||||
TaxClassField: "tax_class",
|
||||
InventoryTrackedF: "inventory_tracked",
|
||||
DropShipF: "drop_ship",
|
||||
CategoryField: "category",
|
||||
BrandField: "brand",
|
||||
ImageField: "image",
|
||||
UpdatedAtField: "updated_at",
|
||||
|
||||
FixedCurrency: "", // read from source if present
|
||||
DefaultTaxClass: "standard",
|
||||
|
||||
PriceScale: 1, // price_inc_vat already in minor units
|
||||
IDPrefix: "sku:",
|
||||
UpdatedAtSource: UseSourceField, // change to StampAtPublish only with intent
|
||||
TrueValues: DefaultTrueValues(),
|
||||
}
|
||||
}
|
||||
|
||||
// BygglagretSchema is an explicit preset for the Swedish building-supplies
|
||||
// tenant that ships inc-VAT float prices in a "PriceInclVAT" string field.
|
||||
//
|
||||
// CRITICAL: TaxClassField (VATClass) is a STRING CLASS IDENTIFIER, not a raw
|
||||
// percent. The producer MUST pre-classify numeric VAT codes ("25" →
|
||||
// "standard", "12" → "reduced-12", "6" → "reduced-6") before publishing.
|
||||
// The wire contract is class identifiers; platform/tax resolves them to
|
||||
// numeric rates out of band. Shipping "25" as the class would break that
|
||||
// resolution path on the consumer side.
|
||||
func BygglagretSchema() Schema {
|
||||
return Schema{
|
||||
Tenant: "bygglagret",
|
||||
SKUField: "ArticleNumber",
|
||||
TitleField: "Name",
|
||||
PriceField: "PriceInclVAT",
|
||||
CurrencyField: "", // fixed below
|
||||
TaxClassField: "VATClass",
|
||||
InventoryTrackedF: "StockTracked",
|
||||
DropShipF: "DropshipFlag",
|
||||
CategoryField: "CategoryPath",
|
||||
BrandField: "Brand",
|
||||
ImageField: "PrimaryImage",
|
||||
UpdatedAtField: "LastModifiedMs",
|
||||
|
||||
FixedCurrency: "SEK",
|
||||
DefaultTaxClass: "standard",
|
||||
|
||||
PriceScale: 100, // PriceInclVAT is in SEK; multiply for øre
|
||||
IDPrefix: "sku:",
|
||||
UpdatedAtSource: UseSourceField,
|
||||
TrueValues: DefaultTrueValues(),
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultTrueValues is the consensus truthy-string set for bool-typed
|
||||
// source fields. Lower-cased comparison in Builder.
|
||||
func DefaultTrueValues() []string {
|
||||
return []string{"true", "yes", "1", "on"}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestSchemaRoundTrip verifies a Schema can be JSON-encoded and decoded
|
||||
// without losing operator-tunable fields. Round-trip is the contract
|
||||
// LoadFile relies on.
|
||||
func TestSchemaRoundTrip(t *testing.T) {
|
||||
original := BygglagretSchema()
|
||||
original.TrueValues = []string{"true", "ja"} // tenant-specific
|
||||
|
||||
raw, err := json.Marshal(original)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
var decoded Schema
|
||||
if err := json.Unmarshal(raw, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if decoded.SKUField != original.SKUField {
|
||||
t.Errorf("SKUField: got %q want %q", decoded.SKUField, original.SKUField)
|
||||
}
|
||||
if decoded.PriceScale != original.PriceScale {
|
||||
t.Errorf("PriceScale: got %d want %d", decoded.PriceScale, original.PriceScale)
|
||||
}
|
||||
if decoded.UpdatedAtSource != original.UpdatedAtSource {
|
||||
t.Errorf("UpdatedAtSource: got %v want %v", decoded.UpdatedAtSource, original.UpdatedAtSource)
|
||||
}
|
||||
if decoded.UpdatedAtField != original.UpdatedAtField {
|
||||
t.Errorf("UpdatedAtField: got %q want %q", decoded.UpdatedAtField, original.UpdatedAtField)
|
||||
}
|
||||
if len(decoded.TrueValues) != 2 || decoded.TrueValues[1] != "ja" {
|
||||
t.Errorf("TrueValues lost on round-trip: %v", decoded.TrueValues)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefaultSchemaConsensusShape pins the field-name contract for the
|
||||
// zero-config tenant (the operator-facing shape).
|
||||
func TestDefaultSchemaConsensusShape(t *testing.T) {
|
||||
s := DefaultSchema()
|
||||
if s.Tenant != "default" {
|
||||
t.Errorf("DefaultSchema().Tenant = %q, want %q", s.Tenant, "default")
|
||||
}
|
||||
if s.SKUField != "sku" || s.PriceField != "price_inc_vat" {
|
||||
t.Errorf("consensus shape rotated: SKUField=%q PriceField=%q", s.SKUField, s.PriceField)
|
||||
}
|
||||
if s.PriceScale != 1 || s.IDPrefix != "sku:" {
|
||||
t.Errorf("consensus defaults wrong: PriceScale=%d IDPrefix=%q", s.PriceScale, s.IDPrefix)
|
||||
}
|
||||
if s.UpdatedAtSource != UseSourceField {
|
||||
t.Errorf("consensus updated_at_source = %v, want UseSourceField "+
|
||||
"(foot-gun 3: stamp-now'ing on rebuild invalidates every consumer cache)", s.UpdatedAtSource)
|
||||
}
|
||||
if len(s.TrueValues) == 0 {
|
||||
t.Errorf("Truth set should not be empty in consensus")
|
||||
}
|
||||
}
|
||||
|
||||
// TestTrueValueCaseInsensitive pins the lower-cased match used by boolOf.
|
||||
func TestTrueValueCaseInsensitive(t *testing.T) {
|
||||
truthy := DefaultTrueValues()
|
||||
for _, s := range []string{"TRUE", "Yes", "ON", "1"} {
|
||||
// Re-implement the matching here to avoid coupling to builder.go's
|
||||
// internals, but flag if it ever drifts.
|
||||
low := ""
|
||||
for _, c := range s {
|
||||
if c >= 'A' && c <= 'Z' {
|
||||
low += string(c + 32)
|
||||
} else {
|
||||
low += string(c)
|
||||
}
|
||||
}
|
||||
found := false
|
||||
for _, t := range truthy {
|
||||
if low == t {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("truthy set is missing lower-cased %q", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSchemaZeroValueInvalid documents that the zero value of Schema is
|
||||
// explicitly NOT a usable schema — Apply validates and refuses it.
|
||||
func TestSchemaZeroValueInvalid(t *testing.T) {
|
||||
_, err := Apply(Schema{}, map[string]any{"sku": "ABC", "price_inc_vat": 1})
|
||||
if err == nil {
|
||||
t.Errorf("zero Schema should be rejected by Apply")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,423 @@
|
||||
// Snapshot reader/writer for the published catalog Projection.
|
||||
//
|
||||
// Why a snapshot file at all? The bus is the live channel, but it doesn't
|
||||
// carry the historical population on cold start. A snapshot file gives
|
||||
// consumers (cart / checkout / order / UCP add-to-cart) a fast mmap-loaded
|
||||
// cold-start substrate, so the first add-line request doesn't have to wait for
|
||||
// the bus to replay. The bus then keeps it fresh.
|
||||
//
|
||||
// Format v2 (little-endian, fixed-width, mmap-friendly):
|
||||
//
|
||||
// header [72 bytes] magic(8) version(4) reserved(4) count(8)
|
||||
// blobStart(8) recordStart(8) tenant(32, zero-padded)
|
||||
// blobs variable count sequential JSON-encoded Projection bodies, no padding
|
||||
// records count*44 each: sku(32, zero-padded) blobOffset(8) blobLength(4);
|
||||
// sorted by SKU via the single compareSKU total order
|
||||
//
|
||||
// Tenant field at offsets 40–71 (zero-padded) names the namespace this
|
||||
// snapshot covers. Independent SKU schemas can produce Projections into
|
||||
// different tenant files without conflict. Hot-reload across tenants stays
|
||||
// safe under SnapshotHolder (snapshot_manager.go).
|
||||
//
|
||||
// Sorting + binary search gives O(log N) per-SKU lookup with no on-disk hash
|
||||
// table and no collisions. SKUs longer than 32 bytes are rejected at write
|
||||
// time (Cart SKUs in this codebase are well under that).
|
||||
//
|
||||
// Memory model: the Reader holds one []byte for the entire file (mmap on
|
||||
// Unix, ReadFile elsewhere). Per-SKU Get reads at most log2(N) record slots
|
||||
// (each 44 B) and one JSON blob (~200 B). For 220 k projections the
|
||||
// resident set is "pages touched this minute", not the whole file.
|
||||
//
|
||||
// SAFETY: SnapshotReader alone is NOT safe across hot-reload. During reload,
|
||||
// in-flight Get() calls on the OLD reader continue to work until disposal
|
||||
// of the old reader is finalised; calling SnapshotReader.Close() concurrently
|
||||
// with Get() on the same reader risks unmapping pages-under-read (SIGBUS).
|
||||
// Use SnapshotHolder (snapshot_manager.go) for any reload-bearing setup —
|
||||
// it gives you an atomic.Pointer swap with a drain-grace (default 30s) so
|
||||
// concurrent Gets on the OLD reader finish naturally before munmap fires.
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
snapshotMagic uint64 = 0x3230564a53435050 // "PCSJ002" v2 (was 0x3130564a53435050 = v1)
|
||||
snapshotVersion uint32 = 2
|
||||
maxSKULen = 32
|
||||
maxTenantLen = 32
|
||||
headerSize = 8 + 4 + 4 + 8 + 8 + 8 + 32 // 72 bytes
|
||||
recordSize = 32 + 8 + 4 // 44 bytes per record
|
||||
)
|
||||
|
||||
var (
|
||||
ErrSnapshotMagic = errors.New("catalog: not a snapshot file (bad magic)")
|
||||
ErrSnapshotVersion = errors.New("catalog: unsupported snapshot version")
|
||||
ErrSnapshotTruncated = errors.New("catalog: snapshot truncated")
|
||||
ErrSnapshotBadBlob = errors.New("catalog: snapshot record points past file")
|
||||
ErrSnapshotOverflow = errors.New("catalog: snapshot record-section size overflows uint64")
|
||||
ErrSKUTooLong = fmt.Errorf("catalog: SKU longer than %d bytes", maxSKULen)
|
||||
ErrTenantTooLong = fmt.Errorf("catalog: tenant name longer than %d bytes", maxTenantLen)
|
||||
)
|
||||
|
||||
// SnapshotWriter is single-shot: Append() all projections, then Write() once
|
||||
// (atomic rename-into-place so readers never see a partial file).
|
||||
type SnapshotWriter struct {
|
||||
Tenant string // optional namespace tag (writes into header)
|
||||
blobs []byte
|
||||
records []snapshotRecord
|
||||
dirty bool
|
||||
}
|
||||
|
||||
// snapshotRecord is the on-disk layout: SKU zero-padded to 32 bytes, then
|
||||
// an 8-byte offset (from file start) plus a 4-byte length for the JSON blob.
|
||||
type snapshotRecord struct {
|
||||
SKU [32]byte
|
||||
BlobOffset uint64
|
||||
BlobLength uint32
|
||||
}
|
||||
|
||||
// NewSnapshotWriter returns an empty writer. Caller Appends every projection
|
||||
// and calls Write once.
|
||||
func NewSnapshotWriter() *SnapshotWriter {
|
||||
return &SnapshotWriter{}
|
||||
}
|
||||
|
||||
// WithTenant tags this writer with a tenant namespace that lands in the
|
||||
// snapshot header. Optional — empty tenant gets zero-padded; readers see an
|
||||
// empty string when the writer was untagged.
|
||||
func (w *SnapshotWriter) WithTenant(tenant string) *SnapshotWriter {
|
||||
w.Tenant = tenant
|
||||
return w
|
||||
}
|
||||
|
||||
// Append encodes and adds a projection. Returns ErrSKUTooLong if SKU exceeds
|
||||
// the on-disk width; returns marshal errors as wrapped fmt.Errorf.
|
||||
func (w *SnapshotWriter) Append(p Projection) error {
|
||||
if len(p.SKU) > maxSKULen {
|
||||
return ErrSKUTooLong
|
||||
}
|
||||
jb, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return fmt.Errorf("catalog: marshal projection %q: %w", p.SKU, err)
|
||||
}
|
||||
|
||||
var rec snapshotRecord
|
||||
copy(rec.SKU[:], p.SKU)
|
||||
rec.BlobOffset = uint64(headerSize + len(w.blobs))
|
||||
rec.BlobLength = uint32(len(jb))
|
||||
|
||||
w.blobs = append(w.blobs, jb...)
|
||||
w.records = append(w.records, rec)
|
||||
w.dirty = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Len returns how many projections have been Append'd.
|
||||
func (w *SnapshotWriter) Len() int { return len(w.records) }
|
||||
|
||||
// Write serialises to a temp file in dir, fsync's, then renames into the
|
||||
// final path. Records are sorted by SKU asc before serialisation so the
|
||||
// reader's binary search works.
|
||||
func (w *SnapshotWriter) Write(dir, name string) (finalPath string, err error) {
|
||||
if w.dirty {
|
||||
sort.Slice(w.records, func(i, j int) bool {
|
||||
return compareSKU(w.records[i].SKU[:], w.records[j].SKU[:]) < 0
|
||||
})
|
||||
w.dirty = false
|
||||
}
|
||||
if len(w.Tenant) > maxTenantLen {
|
||||
err = ErrTenantTooLong
|
||||
return "", err
|
||||
}
|
||||
|
||||
if err = os.MkdirAll(dir, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
finalPath = filepath.Join(dir, name)
|
||||
tmp := finalPath + ".tmp"
|
||||
|
||||
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
_ = f.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
// Header: fixed 72 bytes (8 + 4 + 4 + 8 + 8 + 8 + 32).
|
||||
blobStart := uint64(headerSize)
|
||||
recordStart := blobStart + uint64(len(w.blobs))
|
||||
var hdr [headerSize]byte
|
||||
binary.LittleEndian.PutUint64(hdr[0:8], snapshotMagic)
|
||||
binary.LittleEndian.PutUint32(hdr[8:12], snapshotVersion)
|
||||
binary.LittleEndian.PutUint32(hdr[12:16], 0)
|
||||
binary.LittleEndian.PutUint64(hdr[16:24], uint64(len(w.records)))
|
||||
binary.LittleEndian.PutUint64(hdr[24:32], blobStart)
|
||||
binary.LittleEndian.PutUint64(hdr[32:40], recordStart)
|
||||
copy(hdr[40:40+len(w.Tenant)], w.Tenant) // rest is zero-pad
|
||||
|
||||
if _, werr := f.Write(hdr[:]); werr != nil {
|
||||
err = werr
|
||||
return "", err
|
||||
}
|
||||
if _, werr := f.Write(w.blobs); werr != nil {
|
||||
err = werr
|
||||
return "", err
|
||||
}
|
||||
recBytes := make([]byte, recordSize*len(w.records))
|
||||
for i, r := range w.records {
|
||||
off := i * recordSize
|
||||
copy(recBytes[off:off+32], r.SKU[:])
|
||||
binary.LittleEndian.PutUint64(recBytes[off+32:off+40], r.BlobOffset)
|
||||
binary.LittleEndian.PutUint32(recBytes[off+40:off+44], r.BlobLength)
|
||||
}
|
||||
if _, werr := f.Write(recBytes); werr != nil {
|
||||
err = werr
|
||||
return "", err
|
||||
}
|
||||
|
||||
if werr := f.Sync(); werr != nil {
|
||||
err = werr
|
||||
return "", err
|
||||
}
|
||||
if werr := f.Close(); werr != nil {
|
||||
err = werr
|
||||
return "", err
|
||||
}
|
||||
if werr := os.Rename(tmp, finalPath); werr != nil {
|
||||
err = werr
|
||||
return "", err
|
||||
}
|
||||
return finalPath, nil
|
||||
}
|
||||
|
||||
// compareSKU defines the SINGLE TOTAL ORDER for SKUs across both Writer
|
||||
// (uses it for sort.Slice) and Reader (uses it for binary search). One
|
||||
// definition; invariant test verifies the on-disk result is sorted under
|
||||
// it. Source SKUs SHOULD NOT contain embedded null bytes — the padded
|
||||
// comparator treats them as terminators, which silently truncates an SKU.
|
||||
// Realistically no SKU has NUL.
|
||||
func compareSKU(padded, other []byte) int {
|
||||
// padded is a fixed-size zero-padded slot; trim trailing zeros.
|
||||
end := len(padded)
|
||||
for end > 0 && padded[end-1] == 0 {
|
||||
end--
|
||||
}
|
||||
a := padded[:end]
|
||||
b := other
|
||||
|
||||
n := len(a)
|
||||
if len(b) < n {
|
||||
n = len(b)
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
if a[i] != b[i] {
|
||||
if a[i] < b[i] {
|
||||
return -1
|
||||
}
|
||||
return 1
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(a) < len(b):
|
||||
return -1
|
||||
case len(a) > len(b):
|
||||
return 1
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// compareRecordLookup is compareSKU specialised for Reader lookups so the
|
||||
// argument order matches the lex order: "padded" (fixed-slot) and "want"
|
||||
// (query).
|
||||
//
|
||||
// compareSKU is symmetric so this is just an alias used by Reader code
|
||||
// paths to make intent obvious. Any change to the total order here MUST be
|
||||
// reflected in Writer.sort.Slice and verified by the invariant test.
|
||||
func compareRecordLookup(padded, want []byte) int {
|
||||
return compareSKU(padded, want)
|
||||
}
|
||||
|
||||
// SnapshotReader is read-only and concurrency-safe — parallel Get calls are
|
||||
// OK because no fields are mutated after construction. Held open via an
|
||||
// embedded file handle so the underlying mmap stays pinned. Close uses
|
||||
// sync.Once so concurrent shutdown paths can't double-munmap.
|
||||
//
|
||||
// IMPORTANT — see package doc: SnapshotReader is NOT safe racing Close
|
||||
// concurrently with Get. Use SnapshotHolder for any reload-bearing workflow.
|
||||
type SnapshotReader struct {
|
||||
data []byte
|
||||
count uint64
|
||||
blobStart uint64
|
||||
recordStart uint64
|
||||
Tenant string // populated from header at Open time; "" if not set
|
||||
|
||||
closeOnce sync.Once
|
||||
closer func() error // munmap + close on Unix; no-op on ReadFile fallback
|
||||
}
|
||||
|
||||
// ReadSnapshotFile opens + parses a snapshot file. On Unix uses mmap; on
|
||||
// other platforms reads into memory (functionally identical, higher RSS).
|
||||
func ReadSnapshotFile(path string) (*SnapshotReader, error) {
|
||||
data, closer, err := loadSnapshotBytes(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, err := newSnapshotReader(data)
|
||||
if err != nil {
|
||||
if closer != nil {
|
||||
_ = closer()
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
r.closer = closer
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// NewSnapshotReader wraps an already-loaded []byte (use this in tests; for
|
||||
// production paths go through ReadSnapshotFile).
|
||||
func NewSnapshotReader(data []byte) (*SnapshotReader, error) {
|
||||
return newSnapshotReader(data)
|
||||
}
|
||||
|
||||
func newSnapshotReader(data []byte) (*SnapshotReader, error) {
|
||||
if len(data) < headerSize {
|
||||
return nil, ErrSnapshotTruncated
|
||||
}
|
||||
magic := binary.LittleEndian.Uint64(data[0:8])
|
||||
if magic != snapshotMagic {
|
||||
return nil, ErrSnapshotMagic
|
||||
}
|
||||
version := binary.LittleEndian.Uint32(data[8:12])
|
||||
if version != snapshotVersion {
|
||||
return nil, ErrSnapshotVersion
|
||||
}
|
||||
count := binary.LittleEndian.Uint64(data[16:24])
|
||||
blobStart := binary.LittleEndian.Uint64(data[24:32])
|
||||
recordStart := binary.LittleEndian.Uint64(data[32:40])
|
||||
if blobStart != uint64(headerSize) {
|
||||
return nil, fmt.Errorf("catalog: blobStart=%d expected %d", blobStart, headerSize)
|
||||
}
|
||||
|
||||
// Multiplication overflow guard: for a malformed/tampered count, the
|
||||
// naive int(count)*recordSize in Go silently wraps on overflow,
|
||||
// making expectedSize negative and bypassing the truncation guard.
|
||||
// Compute as uint64 with an explicit overflow check.
|
||||
recordBytes := uint64(recordSize) * count
|
||||
if count > 0 && recordBytes/uint64(count) != uint64(recordSize) {
|
||||
return nil, fmt.Errorf("%w: count*recordSize overflows (count=%d)", ErrSnapshotOverflow, count)
|
||||
}
|
||||
if recordStart > math.MaxUint64-recordBytes {
|
||||
return nil, fmt.Errorf("%w: recordStart+records exceeds uint64 (recordStart=%d count=%d)",
|
||||
ErrSnapshotOverflow, recordStart, count)
|
||||
}
|
||||
expectedSize := recordStart + recordBytes
|
||||
if uint64(len(data)) < expectedSize {
|
||||
return nil, ErrSnapshotTruncated
|
||||
}
|
||||
|
||||
// Per-record blob-offset bounds check: defends against malformed /
|
||||
// tampered files where the record section is intact but the blob
|
||||
// pointer lies past EOF. A linear pass; cheap (44 B × count at open).
|
||||
//
|
||||
// Avoid the naive blobOff+blobLen > dataLen check — blobOff can be
|
||||
// MaxUint64 in tampered input and uint64 addition wraps, falsely
|
||||
// passing the test. Split the check into overflow-safe parts.
|
||||
dataLen := uint64(len(data))
|
||||
for i := uint64(0); i < count; i++ {
|
||||
recOff := recordStart + i*uint64(recordSize)
|
||||
blobLen := binary.LittleEndian.Uint32(data[recOff+40 : recOff+44])
|
||||
blobOff := binary.LittleEndian.Uint64(data[recOff+32 : recOff+40])
|
||||
if blobOff > dataLen || uint64(blobLen) > dataLen-blobOff {
|
||||
return nil, fmt.Errorf("%w: record %d blobOff=%d len=%d exceeds file size %d",
|
||||
ErrSnapshotBadBlob, i, blobOff, blobLen, dataLen)
|
||||
}
|
||||
}
|
||||
|
||||
// Tenant: read zero-padded bytes at offset 40, trim trailing zeros.
|
||||
tenantBytes := data[40 : 40+maxTenantLen]
|
||||
tenantEnd := len(tenantBytes)
|
||||
for tenantEnd > 0 && tenantBytes[tenantEnd-1] == 0 {
|
||||
tenantEnd--
|
||||
}
|
||||
tenant := string(tenantBytes[:tenantEnd])
|
||||
|
||||
return &SnapshotReader{
|
||||
data: data,
|
||||
count: count,
|
||||
blobStart: blobStart,
|
||||
recordStart: recordStart,
|
||||
Tenant: tenant,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Len returns the number of Projections in this snapshot.
|
||||
func (r *SnapshotReader) Len() int { return int(r.count) }
|
||||
|
||||
// Close releases the file handle and mmap region (if any). Idempotent — safe
|
||||
// to call from multiple goroutines (uses sync.Once under the hood).
|
||||
//
|
||||
// SAFETY: this is NOT safe against concurrent Get() — see package doc. Use
|
||||
// SnapshotHolder for any reload-bearing workflow.
|
||||
func (r *SnapshotReader) Close() error {
|
||||
var err error
|
||||
r.closeOnce.Do(func() {
|
||||
if r.closer != nil {
|
||||
err = r.closer()
|
||||
}
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// Get returns the projection for sku, or (zero, false) if not found.
|
||||
// O(log N) — binary search on the sorted record section; on hit, reads the
|
||||
// JSON blob and decodes it (one allocation).
|
||||
func (r *SnapshotReader) Get(sku string) (Projection, bool) {
|
||||
if r.count == 0 || sku == "" {
|
||||
return Projection{}, false
|
||||
}
|
||||
skuBytes := []byte(sku)
|
||||
if len(skuBytes) > maxSKULen {
|
||||
return Projection{}, false
|
||||
}
|
||||
|
||||
lo, hi := uint64(0), r.count
|
||||
for lo < hi {
|
||||
mid := (lo + hi) / 2
|
||||
recOff := int(r.recordStart) + int(mid)*recordSize
|
||||
recSKU := r.data[recOff : recOff+32]
|
||||
cmp := compareRecordLookup(recSKU, skuBytes)
|
||||
switch {
|
||||
case cmp < 0:
|
||||
lo = mid + 1
|
||||
case cmp > 0:
|
||||
hi = mid
|
||||
default:
|
||||
blobLen := binary.LittleEndian.Uint32(r.data[recOff+40 : recOff+44])
|
||||
blobOff := binary.LittleEndian.Uint64(r.data[recOff+32 : recOff+40])
|
||||
// Per-record bounds check at Open guarantees this slice is in-range
|
||||
rdata := r.data[blobOff : blobOff+uint64(blobLen)]
|
||||
var p Projection
|
||||
if err := json.Unmarshal(rdata, &p); err != nil {
|
||||
return Projection{}, false
|
||||
}
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
return Projection{}, false
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
// Local helpers for snapshot_test.go — production code does not see them.
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
func os2WriteFile(path string, data []byte, perm os.FileMode) error {
|
||||
return os.WriteFile(path, data, perm)
|
||||
}
|
||||
|
||||
func os2Stat(path string) (os.FileInfo, error) {
|
||||
return os.Stat(path)
|
||||
}
|
||||
|
||||
func repeat32(s string, n int) string {
|
||||
out := make([]byte, 0, n*len(s))
|
||||
for i := 0; i < n; i++ {
|
||||
out = append(out, s...)
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
// SnapshotHolder provides atomic hot-reload for SnapshotReader with drain
|
||||
// grace — solves the classic "Get concurrent with Close/munmap" race that
|
||||
// would otherwise produce uncatchable SIGBUS / SIGSEGV under reload.
|
||||
//
|
||||
// Pattern source: slask-finder's `RawItemsStore` and similar mmap+reload
|
||||
// stores, which delay unmap behind a configurable drain-grace timer so any
|
||||
// in-flight Get on the previously-current reader completes before the
|
||||
// underlying pages become unmapped. See docs/inventory-shape.md & the
|
||||
// finder WARP doc on mmap reload for the prior art.
|
||||
//
|
||||
// Use:
|
||||
//
|
||||
// holder := catalog.NewSnapshotHolder(30 * time.Second) // drain grace
|
||||
// if err := holder.Open("/var/lib/catalog/bygglagret.bin"); err != nil { ... }
|
||||
// // later, when a fresh snapshot lands:
|
||||
// if err := holder.Reload("/var/lib/catalog/bygglagret.bin"); err != nil { ... }
|
||||
// // pulls the projection; always hits the current reader:
|
||||
// p, ok := holder.Get(sku)
|
||||
//
|
||||
// holder.Get / Len are safe for parallel use. Open and Reload are also
|
||||
// safe for parallel use. Close stops the reaper goroutine and shuts the
|
||||
// current reader + drains down.
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultDrainGrace is the default drain period for SnapshotHolder. Finder's
|
||||
// analogous reload path uses ~30s; we mirror that value.
|
||||
const DefaultDrainGrace = 30 * time.Second
|
||||
|
||||
// SnapshotHolder swaps a SnapshotReader under load without unmapping pages
|
||||
// out from under in-flight Get()s. Set retains the previous reader behind a
|
||||
// drain timer; after the timer elapses the reaper goroutine closes the old
|
||||
// reader (and runs its munmap).
|
||||
type SnapshotHolder struct {
|
||||
current atomic.Pointer[SnapshotReader]
|
||||
|
||||
drainsMu sync.Mutex
|
||||
drains []*pendingClose
|
||||
drainGrace time.Duration
|
||||
|
||||
stopCh chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
type pendingClose struct {
|
||||
r *SnapshotReader
|
||||
expireAt time.Time
|
||||
}
|
||||
|
||||
// NewSnapshotHolder returns a holder with the given drain grace (the time
|
||||
// the previous reader lives behind Set before being closed). Zero or
|
||||
// negative grace falls back to DefaultDrainGrace.
|
||||
func NewSnapshotHolder(grace time.Duration) *SnapshotHolder {
|
||||
if grace <= 0 {
|
||||
grace = DefaultDrainGrace
|
||||
}
|
||||
h := &SnapshotHolder{
|
||||
drainGrace: grace,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
go h.sweepLoop()
|
||||
return h
|
||||
}
|
||||
|
||||
// Open is shorthand for Set(ReadSnapshotFile(path)). Errors leave the
|
||||
// holder's current reader unchanged.
|
||||
func (h *SnapshotHolder) Open(path string) error {
|
||||
r, err := ReadSnapshotFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h.Set(r)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reload atomically swaps the current reader for one freshly opened from
|
||||
// path. The old reader (if any) is queued for close after the drain grace
|
||||
// elapses; in-flight Get(s) on the old reader continue to work.
|
||||
func (h *SnapshotHolder) Reload(path string) error {
|
||||
r, err := ReadSnapshotFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h.Set(r)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Set replaces the current reader with r. The previous reader is queued
|
||||
// for close after the drain grace elapses. r may be nil to clear.
|
||||
func (h *SnapshotHolder) Set(r *SnapshotReader) {
|
||||
if r == nil {
|
||||
h.current.Store(nil)
|
||||
return
|
||||
}
|
||||
old := h.current.Swap(r)
|
||||
if old == nil {
|
||||
return
|
||||
}
|
||||
h.drainsMu.Lock()
|
||||
h.drains = append(h.drains, &pendingClose{
|
||||
r: old,
|
||||
expireAt: time.Now().Add(h.drainGrace),
|
||||
})
|
||||
h.drainsMu.Unlock()
|
||||
}
|
||||
|
||||
// Get is the concurrent-safe shim over the current reader's Get. Returns
|
||||
// (zero, false) when the holder is empty.
|
||||
func (h *SnapshotHolder) Get(sku string) (Projection, bool) {
|
||||
r := h.current.Load()
|
||||
if r == nil {
|
||||
return Projection{}, false
|
||||
}
|
||||
return r.Get(sku)
|
||||
}
|
||||
|
||||
// Len returns the count of the current reader (0 if empty).
|
||||
func (h *SnapshotHolder) Len() int {
|
||||
r := h.current.Load()
|
||||
if r == nil {
|
||||
return 0
|
||||
}
|
||||
return r.Len()
|
||||
}
|
||||
|
||||
// Close stops the reaper goroutine and closes the current reader + any
|
||||
// pending drains without waiting for the grace period. Safe to call
|
||||
// multiple times.
|
||||
func (h *SnapshotHolder) Close() error {
|
||||
var firstErr error
|
||||
h.once.Do(func() {
|
||||
close(h.stopCh)
|
||||
r := h.current.Swap(nil)
|
||||
if r != nil {
|
||||
if err := r.Close(); err != nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
h.drainsMu.Lock()
|
||||
drained := h.drains
|
||||
h.drains = nil
|
||||
h.drainsMu.Unlock()
|
||||
for _, d := range drained {
|
||||
if err := d.r.Close(); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
})
|
||||
return firstErr
|
||||
}
|
||||
|
||||
// DrainCount returns the number of readers currently in the drain queue.
|
||||
// Useful for tests and operational metrics.
|
||||
func (h *SnapshotHolder) DrainCount() int {
|
||||
h.drainsMu.Lock()
|
||||
defer h.drainsMu.Unlock()
|
||||
return len(h.drains)
|
||||
}
|
||||
|
||||
// reapNow scans the drain queue and Closes any readers whose grace has
|
||||
// elapsed. Called by the sweeper goroutine on a tick.
|
||||
func (h *SnapshotHolder) reapNow() {
|
||||
now := time.Now()
|
||||
var alive []*pendingClose
|
||||
var reaped []*pendingClose
|
||||
h.drainsMu.Lock()
|
||||
for _, d := range h.drains {
|
||||
if !d.expireAt.After(now) {
|
||||
reaped = append(reaped, d)
|
||||
} else {
|
||||
alive = append(alive, d)
|
||||
}
|
||||
}
|
||||
h.drains = alive
|
||||
h.drainsMu.Unlock()
|
||||
for _, d := range reaped {
|
||||
_ = d.r.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (h *SnapshotHolder) sweepLoop() {
|
||||
interval := h.drainGrace / 4
|
||||
if interval < 10*time.Millisecond {
|
||||
interval = 10 * time.Millisecond
|
||||
}
|
||||
t := time.NewTicker(interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-h.stopCh:
|
||||
return
|
||||
case <-t.C:
|
||||
h.reapNow()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeCloser counts how many times it's been called. Used to assert the
|
||||
// drain-grace semantics of SnapshotHolder.
|
||||
type fakeCloser struct{ hits int }
|
||||
|
||||
func (f *fakeCloser) Close() error { f.hits++; return nil }
|
||||
|
||||
// withFakeCloser swaps SnapshotReader's closer for our counter. Returns the
|
||||
// reader and the counter; tests then call holder.Set(reloaded) and assert
|
||||
// that close is invoked after the drain grace.
|
||||
func withFakeCloser(t *testing.T, path string, fc *fakeCloser) *SnapshotReader {
|
||||
t.Helper()
|
||||
r, err := ReadSnapshotFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// replace real closer with our counter; the test asserts only via fc
|
||||
r.closer = fc.Close
|
||||
return r
|
||||
}
|
||||
|
||||
// TestSnapshotHolderReloadDrainsOld — Set swaps the current reader; the
|
||||
// previous reader is queued. Before grace: still Get-able. After grace:
|
||||
// reaped and Close fires.
|
||||
//
|
||||
// We use a very short grace so the test doesn't sleep long.
|
||||
func TestSnapshotHolderReloadDrainsOld(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
// 1) Build a valid first snapshot (skus A-1 / A-2).
|
||||
w := NewSnapshotWriter()
|
||||
if err := w.Append(makeProjection("A-1", 100)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Append(makeProjection("A-2", 200)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.Write(dir, "first.bin"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
firstCloser := &fakeCloser{}
|
||||
first := withFakeCloser(t, filepath.Join(dir, "first.bin"), firstCloser)
|
||||
defer first.Close()
|
||||
|
||||
// 2) Build a second snapshot (skus B-1).
|
||||
w2 := NewSnapshotWriter()
|
||||
if err := w2.Append(makeProjection("B-1", 300)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w2.Write(dir, "second.bin"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondCloser := &fakeCloser{}
|
||||
second := withFakeCloser(t, filepath.Join(dir, "second.bin"), secondCloser)
|
||||
defer second.Close()
|
||||
|
||||
h := NewSnapshotHolder(50 * time.Millisecond)
|
||||
defer h.Close()
|
||||
h.Set(first)
|
||||
if h.Len() != 2 {
|
||||
t.Errorf("Len after first Set: got %d want 2", h.Len())
|
||||
}
|
||||
if _, ok := h.Get("A-1"); !ok {
|
||||
t.Error("A-1 not found in initial state")
|
||||
}
|
||||
|
||||
// 3) Swap to second — first is now queued.
|
||||
h.Set(second)
|
||||
if h.Len() != 1 {
|
||||
t.Errorf("Len after second Set: got %d want 1", h.Len())
|
||||
}
|
||||
if firstCloser.hits != 0 {
|
||||
t.Errorf("firstCloser hits: got %d want 0 (still in drain grace)", firstCloser.hits)
|
||||
}
|
||||
if h.DrainCount() != 1 {
|
||||
t.Errorf("DrainCount: got %d want 1", h.DrainCount())
|
||||
}
|
||||
|
||||
// 4) Wait past grace, force a reap tick.
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
h.reapNow()
|
||||
if firstCloser.hits != 1 {
|
||||
t.Errorf("firstCloser hits: got %d want 1 (should have reaped)", firstCloser.hits)
|
||||
}
|
||||
if h.DrainCount() != 0 {
|
||||
t.Errorf("DrainCount after reap: got %d want 0", h.DrainCount())
|
||||
}
|
||||
}
|
||||
|
||||
// TestSnapshotHolderEmptyGet — holder with no current serves empty.
|
||||
func TestSnapshotHolderEmptyGet(t *testing.T) {
|
||||
h := NewSnapshotHolder(100 * time.Millisecond)
|
||||
defer h.Close()
|
||||
if _, ok := h.Get("anything"); ok {
|
||||
t.Error("empty holder should return false")
|
||||
}
|
||||
if h.Len() != 0 {
|
||||
t.Errorf("Len: got %d want 0", h.Len())
|
||||
}
|
||||
}
|
||||
|
||||
// TestSnapshotHolderCloseImmediatelyReaps — Close with pending drains
|
||||
// should close them now (no grace wait). Confirms the drain queue is
|
||||
// drained on Close.
|
||||
func TestSnapshotHolderCloseImmediatelyReaps(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
w := NewSnapshotWriter()
|
||||
if err := w.Append(makeProjection("X-1", 1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.Write(dir, "x.bin"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fc1 := &fakeCloser{}
|
||||
r1 := withFakeCloser(t, filepath.Join(dir, "x.bin"), fc1)
|
||||
|
||||
w2 := NewSnapshotWriter()
|
||||
if err := w2.Append(makeProjection("Y-1", 2)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w2.Write(dir, "y.bin"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fc2 := &fakeCloser{}
|
||||
r2 := withFakeCloser(t, filepath.Join(dir, "y.bin"), fc2)
|
||||
|
||||
h := NewSnapshotHolder(10 * time.Second) // long grace so reap wouldn't happen on its own
|
||||
h.Set(r1)
|
||||
h.Set(r2)
|
||||
|
||||
if fc1.hits != 0 || fc2.hits != 0 {
|
||||
t.Errorf("premature Close hits: fc1=%d fc2=%d", fc1.hits, fc2.hits)
|
||||
}
|
||||
if err := h.Close(); err != nil {
|
||||
t.Errorf("Close: %v", err)
|
||||
}
|
||||
if fc1.hits != 1 || fc2.hits != 1 {
|
||||
t.Errorf("after Close: fc1=%d fc2=%d want 1 each", fc1.hits, fc2.hits)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSnapshotHolderIdentityReload — Open the same path twice; current reader
|
||||
// is replaced; DrainCount returns 1 (old queued). Old's data still served
|
||||
// until grace elapses.
|
||||
func TestSnapshotHolderIdentityReload(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
w := NewSnapshotWriter()
|
||||
if err := w.Append(makeProjection("ID-1", 1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.Write(dir, "id.bin"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
h := NewSnapshotHolder(10 * time.Second)
|
||||
defer h.Close()
|
||||
if err := h.Reload(filepath.Join(dir, "id.bin")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := h.Reload(filepath.Join(dir, "id.bin")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h.DrainCount() != 1 {
|
||||
t.Errorf("DrainCount after two reloads of same path: got %d want 1", h.DrainCount())
|
||||
}
|
||||
if _, ok := h.Get("ID-1"); !ok {
|
||||
t.Error("current reader did not surface ID-1")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
//go:build !unix
|
||||
|
||||
// Non-unix fallback for SnapshotReader: ReadFile into a []byte. Functionally
|
||||
// identical to the unix mmap path — same reader code, same lookups — just
|
||||
// pays the full resident-set cost at Open. Production mmap-on-Linux/Darwin;
|
||||
// this exists for windows / plan9 / wasm builds.
|
||||
package catalog
|
||||
|
||||
import "os"
|
||||
|
||||
func loadSnapshotBytes(path string) (data []byte, closer func() error, err error) {
|
||||
data, err = os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return data, func() error { return nil }, nil
|
||||
}
|
||||
@@ -0,0 +1,448 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"math/rand"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.k6n.net/mats/platform/money"
|
||||
)
|
||||
|
||||
// TestSnapshotRoundTrip — write 1000 projections in shuffled order, read back
|
||||
// and verify all 1000 are found at the right SKU.
|
||||
func TestSnapshotRoundTrip(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "snap.bin")
|
||||
|
||||
const n = 1000
|
||||
w := NewSnapshotWriter().WithTenant("default")
|
||||
// intentionally shuffled input — sort is what reorders at Write.
|
||||
perm := rand.Perm(n)
|
||||
for _, i := range perm {
|
||||
sku := fmt.Sprintf("SKU-%05d", i)
|
||||
if err := w.Append(makeProjection(sku, int64(100+(i%97)*100))); err != nil {
|
||||
t.Fatalf("Append %s: %v", sku, err)
|
||||
}
|
||||
}
|
||||
if w.Len() != n {
|
||||
t.Fatalf("Len: got %d want %d", w.Len(), n)
|
||||
}
|
||||
final, err := w.Write(dir, "snap.bin")
|
||||
if err != nil {
|
||||
t.Fatalf("Write: %v", err)
|
||||
}
|
||||
if final != path {
|
||||
t.Errorf("Write returned %q expected %q", final, path)
|
||||
}
|
||||
|
||||
r, err := ReadSnapshotFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadSnapshotFile: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
if r.Len() != n {
|
||||
t.Errorf("Len: got %d want %d", r.Len(), n)
|
||||
}
|
||||
if r.Tenant != "default" {
|
||||
t.Errorf("Tenant: got %q want %q", r.Tenant, "default")
|
||||
}
|
||||
|
||||
// every SKU present and correct
|
||||
for _, i := range rand.Perm(n)[:50] { // probe 50 random lookups
|
||||
sku := fmt.Sprintf("SKU-%05d", i)
|
||||
got, ok := r.Get(sku)
|
||||
if !ok {
|
||||
t.Errorf("Get(%q) not found", sku)
|
||||
continue
|
||||
}
|
||||
want := money.Cents(100 + (i%97)*100)
|
||||
if got.PriceIncVat != want {
|
||||
t.Errorf("Get(%q) PriceIncVat: got %d want %d", sku, got.PriceIncVat, want)
|
||||
}
|
||||
}
|
||||
|
||||
// missing SKU
|
||||
if _, ok := r.Get("NOPE-99999"); ok {
|
||||
t.Error("Get on missing SKU should return ok=false")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSnapshotCompareSKUInvariant — verify the on-disk record section is
|
||||
// sorted under the same compareSKU total order that BinarySearch uses.
|
||||
// If sort.Slice and Reader.Get ever drift apart, lookups silently miss
|
||||
// — this test pins that invariant.
|
||||
func TestSnapshotCompareSKUInvariant(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
w := NewSnapshotWriter()
|
||||
// deliberately inverted order after Append (so sort has work to do)
|
||||
perm := rand.Perm(500)
|
||||
for _, i := range perm {
|
||||
sku := fmt.Sprintf("XX-%05d", i)
|
||||
if err := w.Append(makeProjection(sku, int64(i))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := w.Write(dir, "snap.bin"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := ReadSnapshotFile(filepath.Join(dir, "snap.bin"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
// Walk every record slot, harvest SKU bytes from the pad-then-trim
|
||||
// form, and assert non-decreasing order under compareSKU.
|
||||
prevSKU := ""
|
||||
for i := 0; i < r.Len(); i++ {
|
||||
recOff := int(r.recordStart) + i*recordSize
|
||||
raw := r.data[recOff : recOff+32]
|
||||
// trim trailing zeros
|
||||
end := len(raw)
|
||||
for end > 0 && raw[end-1] == 0 {
|
||||
end--
|
||||
}
|
||||
cur := string(raw[:end])
|
||||
if i > 0 {
|
||||
if compareSKU(runesToPadded32(prevSKU), []byte(cur)) > 0 {
|
||||
t.Errorf("record section out of order at i=%d: prev=%q cur=%q", i, prevSKU, cur)
|
||||
}
|
||||
}
|
||||
prevSKU = cur
|
||||
}
|
||||
}
|
||||
|
||||
// runesToPadded32 zero-pads s into a 32-byte slot for compareSKU in the
|
||||
// invariant test.
|
||||
func runesToPadded32(s string) []byte {
|
||||
out := make([]byte, 32)
|
||||
copy(out, s)
|
||||
return out
|
||||
}
|
||||
|
||||
// TestSnapshotTenantRoundTrip — Writer.WithTenant lands in the header and
|
||||
// Reader surfaces it. Without this, one file per tenant is just a convention
|
||||
// with no on-disk evidence.
|
||||
func TestSnapshotTenantRoundTrip(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
w := NewSnapshotWriter().WithTenant("bygglagret")
|
||||
if err := w.Append(makeProjection("ABC-1", 100)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.Write(dir, "snap.bin"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := ReadSnapshotFile(filepath.Join(dir, "snap.bin"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
if r.Tenant != "bygglagret" {
|
||||
t.Errorf("Tenant: got %q want %q", r.Tenant, "bygglagret")
|
||||
}
|
||||
|
||||
// Empty tenant writes zero-padded.
|
||||
w2 := NewSnapshotWriter()
|
||||
if err := w2.Append(makeProjection("ABC-2", 200)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w2.Write(dir, "snap2.bin"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r2, err := ReadSnapshotFile(filepath.Join(dir, "snap2.bin"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r2.Close()
|
||||
if r2.Tenant != "" {
|
||||
t.Errorf("Tenant (no tag): got %q want empty", r2.Tenant)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSnapshotTenantTooLong — over-long tenant name rejected at Write.
|
||||
func TestSnapshotTenantTooLong(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
w := NewSnapshotWriter().WithTenant(string(make([]byte, 33)))
|
||||
if err := w.Append(makeProjection("ABC", 1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.Write(dir, "snap.bin"); err == nil {
|
||||
t.Error("expected error for over-long tenant")
|
||||
} else if !errors.Is(err, ErrTenantTooLong) {
|
||||
t.Errorf("got %v want ErrTenantTooLong", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSnapshotCountOverflowRejected — tampered count that would cause
|
||||
// recordStart + records*recordSize to overflow uint64 must be rejected
|
||||
// with ErrSnapshotOverflow (formerly a silent wraparound vulnerability).
|
||||
func TestSnapshotCountOverflowRejected(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "over.bin")
|
||||
bs := make([]byte, headerSize)
|
||||
binary.LittleEndian.PutUint64(bs[0:8], snapshotMagic)
|
||||
binary.LittleEndian.PutUint32(bs[8:12], snapshotVersion)
|
||||
binary.LittleEndian.PutUint32(bs[12:16], 0)
|
||||
// count that overflows when multiplied by recordSize=44
|
||||
binary.LittleEndian.PutUint64(bs[16:24], math.MaxUint64/uint64(recordSize)+1)
|
||||
binary.LittleEndian.PutUint64(bs[24:32], uint64(headerSize))
|
||||
binary.LittleEndian.PutUint64(bs[32:40], uint64(headerSize))
|
||||
if err := os2WriteFile(path, bs, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ReadSnapshotFile(path); !errors.Is(err, ErrSnapshotOverflow) {
|
||||
t.Errorf("overflow count rejected with %v, want ErrSnapshotOverflow", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSnapshotAtomicSwap — verify the temporary-then-rename discipline never
|
||||
// leaves a half-written snap.bin visible to readers.
|
||||
func TestSnapshotAtomicSwap(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
w := NewSnapshotWriter()
|
||||
mustAppend := func(w *SnapshotWriter, sku string, price int64) {
|
||||
t.Helper()
|
||||
if err := w.Append(makeProjection(sku, price)); err != nil {
|
||||
t.Fatalf("Append %s: %v", sku, err)
|
||||
}
|
||||
}
|
||||
mustAppend(w, "AAA", 100)
|
||||
final, err := w.Write(dir, "snap.bin")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// tmp must not exist anymore
|
||||
if _, err := os2Stat(final + ".tmp"); err == nil {
|
||||
t.Errorf(".tmp still present after Write")
|
||||
}
|
||||
|
||||
// replace
|
||||
w2 := NewSnapshotWriter()
|
||||
mustAppend(w2, "ZZZ", 200)
|
||||
if _, err := w2.Write(dir, "snap.bin"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
r, err := ReadSnapshotFile(final)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
if _, ok := r.Get("ZZZ"); !ok {
|
||||
t.Error("second-snapshot Get(ZZZ) missing")
|
||||
}
|
||||
if _, ok := r.Get("AAA"); ok {
|
||||
t.Error("first-snapshot residue Get(AAA) found — atomic swap bug")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSnapshotRejectBadInput — magic, version, truncation, blob-offset OOB.
|
||||
func TestSnapshotRejectBadInput(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "bad.bin")
|
||||
|
||||
// Empty file: truncated.
|
||||
if err := os2WriteFile(path, []byte{}, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ReadSnapshotFile(path); !errors.Is(err, ErrSnapshotTruncated) {
|
||||
t.Errorf("empty file rejected with %v, want ErrSnapshotTruncated", err)
|
||||
}
|
||||
|
||||
// Wrong magic. The fixture must be at least headerSize bytes long
|
||||
// (v2 header is 72 B), otherwise we'd hit ErrSnapshotTruncated
|
||||
// before the magic check fires.
|
||||
wrongMagic := make([]byte, headerSize+1)
|
||||
for i := range wrongMagic {
|
||||
wrongMagic[i] = 0xff
|
||||
}
|
||||
if err := os2WriteFile(path, wrongMagic, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ReadSnapshotFile(path); !errors.Is(err, ErrSnapshotMagic) {
|
||||
t.Errorf("wrong magic rejected with %v, want ErrSnapshotMagic", err)
|
||||
}
|
||||
|
||||
// Truncated mid-record. A valid header claiming 5 records but with no
|
||||
// record section.
|
||||
if err := os2WriteFile(path, buildValidHeader(5), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ReadSnapshotFile(path); !errors.Is(err, ErrSnapshotTruncated) {
|
||||
t.Errorf("truncated snapshot rejected with %v, want ErrSnapshotTruncated", err)
|
||||
}
|
||||
|
||||
// Tampered blob offsets: valid header, valid record section with a
|
||||
// blob pointer that lies past EOF. Should reject with ErrSnapshotBadBlob.
|
||||
tampered := buildTamperedHeader()
|
||||
if err := os2WriteFile(path, tampered, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ReadSnapshotFile(path); !errors.Is(err, ErrSnapshotBadBlob) {
|
||||
t.Errorf("tampered-blob rejected with %v, want ErrSnapshotBadBlob", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSnapshotEmpty — empty (count=0) snapshot is valid; Get returns false.
|
||||
func TestSnapshotEmpty(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
w := NewSnapshotWriter()
|
||||
if _, err := w.Write(dir, "empty.bin"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := ReadSnapshotFile(filepath.Join(dir, "empty.bin"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
if r.Len() != 0 {
|
||||
t.Errorf("Len: got %d want 0", r.Len())
|
||||
}
|
||||
if _, ok := r.Get("anything"); ok {
|
||||
t.Error("empty snapshot should never return ok=true")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSnapshotSKU32 — SKU exactly 32 chars (the boundary case) must succeed.
|
||||
func TestSnapshotSKU32(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
sku := "SKU-" + repeat32("X", 32-4) // 4 + 28 = 32 chars
|
||||
if len(sku) != 32 {
|
||||
t.Fatalf("test bug: sku length %d", len(sku))
|
||||
}
|
||||
w := NewSnapshotWriter()
|
||||
if err := w.Append(makeProjection(sku, 1)); err != nil {
|
||||
t.Fatalf("Append 32-char SKU: %v", err)
|
||||
}
|
||||
if _, err := w.Write(dir, "snap.bin"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := ReadSnapshotFile(filepath.Join(dir, "snap.bin"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
if _, ok := r.Get(sku); !ok {
|
||||
t.Error("32-char SKU Get returned false")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSnapshotSKU33Rejected — one char over should error at Append.
|
||||
func TestSnapshotSKU33Rejected(t *testing.T) {
|
||||
w := NewSnapshotWriter()
|
||||
sku := repeat32("X", 33)
|
||||
err := w.Append(makeProjection(sku, 1))
|
||||
if !errors.Is(err, ErrSKUTooLong) {
|
||||
t.Errorf("33-char SKU Append: got %v want ErrSKUTooLong", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSnapshotGetSpeed — probe thousands of random SKU lookups to give a
|
||||
// sense of the binary-search cost (not a benchmark per se).
|
||||
func TestSnapshotGetSpeed(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("short mode")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
const n = 5000
|
||||
w := NewSnapshotWriter()
|
||||
skus := make([]string, n)
|
||||
for i := 0; i < n; i++ {
|
||||
skus[i] = fmt.Sprintf("Z%06d", i)
|
||||
if err := w.Append(makeProjection(skus[i], int64(i))); err != nil {
|
||||
t.Fatalf("Append %s: %v", skus[i], err)
|
||||
}
|
||||
}
|
||||
if _, err := w.Write(dir, "snap.bin"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := ReadSnapshotFile(filepath.Join(dir, "snap.bin"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
start := time.Now()
|
||||
hits, misses := 0, 0
|
||||
for _, i := range rand.Perm(n) {
|
||||
if _, ok := r.Get(skus[i]); ok {
|
||||
hits++
|
||||
} else {
|
||||
misses++
|
||||
}
|
||||
}
|
||||
elapsed := time.Since(start)
|
||||
t.Logf("100%% hit Get on %d records: %d hits, %d misses, %v (%v/lookup)",
|
||||
n, hits, misses, elapsed, elapsed/time.Duration(n))
|
||||
if misses != 0 {
|
||||
t.Errorf("expected zero misses on full-key set, got %d", misses)
|
||||
}
|
||||
}
|
||||
|
||||
// buildValidHeader synthesises a header byte slice with the given Count
|
||||
// and zero-blob / zero-record section, used to exercise the truncation
|
||||
// branch in newSnapshotReader.
|
||||
func buildValidHeader(count uint64) []byte {
|
||||
bs := make([]byte, headerSize)
|
||||
binary.LittleEndian.PutUint64(bs[0:8], snapshotMagic)
|
||||
binary.LittleEndian.PutUint32(bs[8:12], snapshotVersion)
|
||||
binary.LittleEndian.PutUint32(bs[12:16], 0)
|
||||
binary.LittleEndian.PutUint64(bs[16:24], count)
|
||||
binary.LittleEndian.PutUint64(bs[24:32], uint64(headerSize)) // blobStart
|
||||
binary.LittleEndian.PutUint64(bs[32:40], uint64(headerSize)) // recordStart (truncated)
|
||||
return bs
|
||||
}
|
||||
|
||||
// buildTamperedHeader returns a buffer with a valid header claiming 1 record
|
||||
// AND its 44-byte record section, but the record's blob pointer lies past
|
||||
// EOF. Used to exercise ErrSnapshotBadBlob.
|
||||
func buildTamperedHeader() []byte {
|
||||
bs := make([]byte, headerSize+recordSize)
|
||||
binary.LittleEndian.PutUint64(bs[0:8], snapshotMagic)
|
||||
binary.LittleEndian.PutUint32(bs[8:12], snapshotVersion)
|
||||
binary.LittleEndian.PutUint32(bs[12:16], 0)
|
||||
binary.LittleEndian.PutUint64(bs[16:24], 1)
|
||||
binary.LittleEndian.PutUint64(bs[24:32], uint64(headerSize))
|
||||
binary.LittleEndian.PutUint64(bs[32:40], uint64(headerSize))
|
||||
binary.LittleEndian.PutUint64(bs[headerSize+32:headerSize+40], ^uint64(0))
|
||||
binary.LittleEndian.PutUint32(bs[headerSize+40:headerSize+44], ^uint32(0))
|
||||
return bs
|
||||
}
|
||||
|
||||
// TestSnapshotRejectBadVersion — pin the ErrSnapshotVersion rejection path.
|
||||
func TestSnapshotRejectBadVersion(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "badver.bin")
|
||||
bs := make([]byte, headerSize)
|
||||
binary.LittleEndian.PutUint64(bs[0:8], snapshotMagic)
|
||||
binary.LittleEndian.PutUint32(bs[8:12], snapshotVersion+1)
|
||||
binary.LittleEndian.PutUint32(bs[12:16], 0)
|
||||
binary.LittleEndian.PutUint64(bs[16:24], 0)
|
||||
binary.LittleEndian.PutUint64(bs[24:32], uint64(headerSize))
|
||||
binary.LittleEndian.PutUint64(bs[32:40], uint64(headerSize))
|
||||
if err := os2WriteFile(path, bs, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ReadSnapshotFile(path); !errors.Is(err, ErrSnapshotVersion) {
|
||||
t.Errorf("v2-not-supported header rejected with %v, want ErrSnapshotVersion", err)
|
||||
}
|
||||
}
|
||||
|
||||
// makeProjection helper repeated here for the new tests.
|
||||
func makeProjection(sku string, price int64) Projection {
|
||||
return Projection{
|
||||
ID: "sku:" + sku,
|
||||
SKU: sku,
|
||||
Title: "Item " + sku,
|
||||
PriceIncVat: money.Cents(price),
|
||||
Currency: "SEK",
|
||||
TaxClass: "standard",
|
||||
UpdatedAt: time.Now().UnixMilli(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
//go:build unix
|
||||
|
||||
// mmap-on-unix loader for SnapshotReader. The file handle stays alive for
|
||||
// the lifetime of the mmap region (the kernel needs it pinned). Closer
|
||||
// munmaps AND closes the fd. Uses MADV_RANDOM because the dominant access
|
||||
// pattern is binary-search-by-SKU — touching random record slots, not
|
||||
// streaming the file.
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// loadSnapshotBytes returns the file contents (mmap'd) + a closer that munmaps
|
||||
// and closes the fd.
|
||||
func loadSnapshotBytes(path string) (data []byte, closer func() error, err error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
size := info.Size()
|
||||
if size == 0 {
|
||||
_ = f.Close()
|
||||
return []byte{}, func() error { return nil }, nil
|
||||
}
|
||||
if size < int64(headerSize) {
|
||||
_ = f.Close()
|
||||
return nil, nil, ErrSnapshotTruncated
|
||||
}
|
||||
|
||||
data, mmapErr := unix.Mmap(int(f.Fd()), 0, int(size), unix.PROT_READ, unix.MAP_SHARED)
|
||||
if mmapErr != nil {
|
||||
_ = f.Close()
|
||||
return nil, nil, fmt.Errorf("catalog: mmap %s: %w", path, mmapErr)
|
||||
}
|
||||
|
||||
// Binary-search by SKU = random page access; MADV_RANDOM tells the
|
||||
// kernel NOT to do read-ahead — saves I/O on cold layers.
|
||||
if maErr := unix.Madvise(data, unix.MADV_RANDOM); maErr != nil {
|
||||
// non-fatal; degrade gracefully
|
||||
_ = maErr
|
||||
}
|
||||
|
||||
closer = func() error {
|
||||
mErr := unix.Munmap(data)
|
||||
cErr := f.Close()
|
||||
if mErr != nil {
|
||||
return mErr
|
||||
}
|
||||
return cErr
|
||||
}
|
||||
return data, closer, nil
|
||||
}
|
||||
Reference in new issue
Block a user