more

cmd/order/amqp.go

@@ -60,7 +60,12 @@ func (p *rabbitPublisher) Publish(exchange, routingKey string, body []byte) erro
 			return err
 		}
 	}
-	err := p.ch.PublishWithContext(context.Background(), exchange, routingKey, false, false,
+	// Bound the publish so a half-open/black-holed connection can't wedge the
+	// outbox relay indefinitely; on timeout the error path resets and the relay
+	// retries on its next tick.
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	err := p.ch.PublishWithContext(ctx, exchange, routingKey, false, false,
 		amqp.Publishing{ContentType: "application/json", Body: body})
 	if err != nil {
 		p.reset() // force a reconnect on the next attempt

cmd/order/handlers_checkout.go

@@ -134,11 +134,12 @@ func (s *server) handleFromCheckout(w http.ResponseWriter, r *http.Request) {
 		s.logger.Warn("from-checkout flow failed", "orderId", id.String(), "err", runErr)
 	}
 
-	// Record the idempotency key durably so retries (incl. after a restart) are
+	// Record the idempotency key durably only on success, so retries (incl. after
-	// safe. We record even on a failed flow: the failed order grain exists, and a
+	// a restart) return the captured order. A failed flow (402) is deliberately
-	// blind retry with the same key should return it rather than place a second
+	// NOT recorded: the failed grain is terminal, and a retry with the same key
-	// order — the caller is told to retry with a fresh key (402 contract).
+	// should re-attempt rather than be handed back a dead order as a 409 hit
-	if req.IdempotencyKey != "" {
+	// (which the checkout client treats as a successful, existing order).
+	if req.IdempotencyKey != "" && runErr == nil {
 		if err := s.idem.Put(req.IdempotencyKey, ordID); err != nil {
 			s.logger.Error("idempotency record failed", "key", req.IdempotencyKey, "orderId", id.String(), "err", err)
 		}

cmd/order/main.go

@@ -102,12 +102,13 @@ func main() {
 		// event survives a broker outage or a crash after the state change.
 		box, err := outbox.Open(filepath.Join(dataDir, "outbox.log"))
 		if err != nil {
-			logger.Warn("outbox disabled: open failed", "err", err)
+			// Durability-critical (same as the idempotency store): fail fast rather
-		} else {
+			// than silently publish events nowhere for the whole process lifetime.
+			log.Fatalf("open outbox: %v", err)
+		}
 		go box.Run(context.Background(), newRabbitPublisher(amqpURL), 5*time.Second, logger)
 		emitPub = box
 	}
-	}
 	freg := flow.NewRegistry()
 	flow.RegisterBuiltinHooks(freg)
 	order.RegisterFlowActions(freg, applier, provider)

pkg/idempotency/store.go

@@ -14,19 +14,23 @@ import (
 	"bufio"
 	"encoding/json"
 	"errors"
+	"hash/fnv"
 	"os"
 	"sync"
 )
 
+// lockShards bounds the per-key lock memory: keys hash into a fixed set of
+// mutexes rather than a map that grows with every distinct key ever seen. Two
+// unrelated keys that collide on a shard serialise occasionally — harmless.
+const lockShards = 256
+
 // Store maps an idempotency key to a result id (e.g. an order id), cached in
 // memory and persisted to an append-only file.
 type Store struct {
 	mu    sync.Mutex
 	m     map[string]uint64
 	f     *os.File
-
+	locks [lockShards]sync.Mutex
-	klMu     sync.Mutex
-	keyLocks map[string]*sync.Mutex
 }
 
 type record struct {
@@ -62,7 +66,7 @@ func Open(path string) (*Store, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &Store{m: m, f: f, keyLocks: map[string]*sync.Mutex{}}, nil
+	return &Store{m: m, f: f}, nil
 }
 
 // Get returns the recorded id for key and whether it was present.
@@ -103,13 +107,9 @@ func (s *Store) Lock(key string) func() {
 	if key == "" {
 		return func() {}
 	}
-	s.klMu.Lock()
+	h := fnv.New32a()
-	mu, ok := s.keyLocks[key]
+	_, _ = h.Write([]byte(key))
-	if !ok {
+	mu := &s.locks[h.Sum32()%lockShards]
-		mu = &sync.Mutex{}
-		s.keyLocks[key] = mu
-	}
-	s.klMu.Unlock()
 	mu.Lock()
 	return mu.Unlock
 }

pkg/outbox/outbox.go

@@ -23,6 +23,7 @@ import (
 	"errors"
 	"log/slog"
 	"os"
+	"path/filepath"
 	"sort"
 	"sync"
 	"time"
@@ -144,6 +145,12 @@ func (o *Outbox) compact() error {
 	if err := os.Rename(tmp, o.path); err != nil {
 		return err
 	}
+	// fsync the directory so the rename (the file swap) is itself durable — a
+	// crash right after must not leave the pending log lost or a stale .tmp.
+	if d, derr := os.Open(filepath.Dir(o.path)); derr == nil {
+		_ = d.Sync()
+		_ = d.Close()
+	}
 	o.f, err = os.OpenFile(o.path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
 	return err
 }