update

cmd/profile/clustered_applier.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"context"
+
+	"google.golang.org/protobuf/proto"
+
+	"git.k6n.net/mats/go-cart-actor/pkg/actor"
+	"git.k6n.net/mats/go-cart-actor/pkg/profile"
+)
+
+// clusterAwareApplier is a ProfileApplier that chooses between an
+// in-process pool call and a remote forward based on which replica
+// currently owns the grain. It is the single seam at which the auth
+// server (`customerauth.Server`) and the UCP customer handler
+// (`ucp.CustomerServer`) ever speak to the grain pool — keeping the
+// decision of "go to the authoritative owner" vs "spawn locally" in
+// one place avoids the split-brain hazard introduced when multiple
+// code paths (HTTP middleware + the handlers' own pool.Get) could each
+// independently decide to spawn a stale or empty grain and broadcast
+// conflicting ownership for it.
+//
+// The decision rule:
+//
+//   - pool.OwnerHost(id) returns a remote host  →  forward Get/Apply
+//     to that host. The host holds the authoritative in-memory state.
+//   - pool.OwnerHost(id) returns no host        →  delegate to
+//     pool.Get / pool.Apply. On a local cache miss pool.Get spawns
+//     the grain from disk on this pod, broadcasts the new ownership,
+//     and returns the grain; this is the only code path that ever
+//     claims a grain on this pod.
+//
+// pool.Get's spawn path is unchanged on purpose: the disk-backed event
+// log is the source of truth, and the local cache is just a projection
+// rebuilt from it. The risk surface that the applicr layer removes is
+// the HTTP-middleware fall-through that re-entered the same pool.Get
+// from a different code path and duplicated the decision.
+type clusterAwareApplier struct {
+	pool *actor.SimpleGrainPool[profile.ProfileGrain]
+}
+
+// Get returns the current state of grain id, preferring the
+// authoritative remote owner when one is registered. With no remote
+// owner the pool spawns the grain locally (from the disk-backed event
+// log), takes ownership, and returns the grain.
+func (a *clusterAwareApplier) Get(ctx context.Context, id uint64) (*profile.ProfileGrain, error) {
+	if owner, ok := a.pool.OwnerHost(id); ok {
+		return owner.Get(ctx, id)
+	}
+	return a.pool.Get(ctx, id)
+}
+
+// Apply mutates grain id with messages, forwarding to the authoritative
+// remote owner when one is registered. With no remote owner the pool
+// spawns the grain locally, applies the mutation, takes ownership, and
+// returns the mutated state — its listeners (including the AMQP
+// mutation feed) fire from this pod.
+func (a *clusterAwareApplier) Apply(ctx context.Context, id uint64, msgs ...proto.Message) (*actor.MutationResult[profile.ProfileGrain], error) {
+	if owner, ok := a.pool.OwnerHost(id); ok {
+		return owner.Apply(ctx, id, msgs...)
+	}
+	return a.pool.Apply(ctx, id, msgs...)
+}

cmd/profile/main.go

@@ -243,12 +243,28 @@ func main() {
 
 	// UCP Customer REST adapter
 	auditLogPath := filepath.Join(profileDir, "audit.log")
-	customerUCP := ucp.CustomerHandler(pool, auditLogPath,
+	// Session signer is shared by the auth server (HMAC verifier for the
+	// "sid" cookie); NewSigner is stateless once built so a single
+	// instance can be reused across the auth server's NewServer call.
+	sessionSigner := customerauth.NewSigner(authSecret())
+
+	// Cluster-aware ProfileApplier: routes Get/Apply to the replica that
+	// currently owns the grain. With a remote owner we forward there so
+	// the request reads the authoritative in-memory state; with no owner
+	// we let pool.Get / pool.Apply spawn the grain locally from the
+	// disk-backed event log and broadcast ownership. Centralising this
+	// choice in the applicr (rather than HTTP middleware) prevents the
+	// split-brain hazard where a non-owner pod reads a stale or empty
+	// grain from disk and caches it under its own (conflicting)
+	// ownership.
+	applier := &clusterAwareApplier{pool: pool}
+
+	customerUCP := ucp.CustomerHandler(applier, auditLogPath,
 		ucp.WithEmailIndex(emailIndex),
 		ucp.WithCredentialDeleter(credStore),
 	)
-	if signer := loadUCPProfileSigner(); signer != nil {
-		customerUCP = ucp.WithSigning(customerUCP, signer)
+	if ucpSigner := loadUCPProfileSigner(); ucpSigner != nil {
+		customerUCP = ucp.WithSigning(customerUCP, ucpSigner)
 		log.Print("ucp customer signing enabled")
 	}
 	// StripPrefix is required because the sub-mux (CustomerHandler) registers
@@ -258,7 +274,7 @@ func main() {
 
 	// Customer auth: password signup/login + session cookies + identity linking.
 	authNotifier := selectAuthNotifier()
-	authHandler := customerauth.NewServer(credStore, pool, customerauth.NewSigner(authSecret()), 0, customerauth.Options{
+	authHandler := customerauth.NewServer(credStore, applier, sessionSigner, 0, customerauth.Options{
 		Notifier:             authNotifier,
 		Limiter:              limiter,
 		BaseURL:              os.Getenv("CUSTOMER_AUTH_BASE_URL"),