go-cart-actor/pkg/outbox/outbox.go

// Package outbox implements a durable transactional-outbox relay for AMQP
// publishing. Today the order service's amqp_emit hook publishes best-effort: a
// broker outage or a crash after a state change but before the publish loses
// the event. The outbox closes that gap.
//
// Producers call Publish, which appends the message to an append-only JSONL file
// and fsyncs before returning — so the intent to emit is durable the moment
// Publish returns, independent of broker availability. A single relay goroutine
// (Run) drains undelivered entries to the broker and records an ack line on
// success, retrying on failure. Delivery is therefore at-least-once: a crash
// between the broker publish and the ack line replays that message on restart,
// which downstream consumers must tolerate (the commerce events are keyed and
// already designed to be idempotent).
//
// This targets the single-node order service. The file is compacted on Open
// (acked entries are dropped), which bounds growth across restarts.
package outbox

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"log/slog"
	"os"
	"sort"
	"sync"
	"time"
)

// Publisher delivers a message to the broker. The relay retries on error, so an
// implementation should surface a transient failure (broker down) as an error
// rather than dropping the message.
type Publisher interface {
	Publish(exchange, routingKey string, body []byte) error
}

// line is one record in the log: either a pending message (Seq>0 with payload)
// or an ack marker (Ack>0). Body round-trips as base64 via encoding/json.
type line struct {
	Seq        uint64 `json:"seq,omitempty"`
	Exchange   string `json:"exchange,omitempty"`
	RoutingKey string `json:"rk,omitempty"`
	Body       []byte `json:"body,omitempty"`
	Ack        uint64 `json:"ack,omitempty"`
}

type entry struct {
	seq        uint64
	exchange   string
	routingKey string
	body       []byte
}

// Outbox is a durable, file-backed message buffer with an at-least-once relay.
type Outbox struct {
	mu      sync.Mutex
	f       *os.File
	path    string
	seq     uint64
	pending map[uint64]entry
	wake    chan struct{}
}

// Open loads the outbox at path (creating it if absent), drops already-acked
// entries, and rewrites the file with only the still-pending ones (compaction).
// Returns an Outbox whose pending set is ready for the relay to drain.
func Open(path string) (*Outbox, error) {
	pending := map[uint64]entry{}
	acked := map[uint64]struct{}{}
	var maxSeq uint64

	if f, err := os.Open(path); err == nil {
		sc := bufio.NewScanner(f)
		sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
		for sc.Scan() {
			b := sc.Bytes()
			if len(b) == 0 {
				continue
			}
			var l line
			if err := json.Unmarshal(b, &l); err != nil {
				continue // skip a torn/corrupt line
			}
			switch {
			case l.Ack > 0:
				acked[l.Ack] = struct{}{}
			case l.Seq > 0:
				pending[l.Seq] = entry{seq: l.Seq, exchange: l.Exchange, routingKey: l.RoutingKey, body: l.Body}
				if l.Seq > maxSeq {
					maxSeq = l.Seq
				}
			}
		}
		_ = f.Close()
	} else if !errors.Is(err, os.ErrNotExist) {
		return nil, err
	}
	for seq := range acked {
		delete(pending, seq)
	}

	ob := &Outbox{path: path, seq: maxSeq, pending: pending, wake: make(chan struct{}, 1)}
	if err := ob.compact(); err != nil {
		return nil, err
	}
	return ob, nil
}

// compact rewrites the file with only the current pending entries, then reopens
// it for appends. Called on Open so the log can't grow without bound across
// restarts. Must be called with no concurrent writers (Open, before Run).
func (o *Outbox) compact() error {
	tmp := o.path + ".tmp"
	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
	if err != nil {
		return err
	}
	seqs := make([]uint64, 0, len(o.pending))
	for seq := range o.pending {
		seqs = append(seqs, seq)
	}
	sort.Slice(seqs, func(i, j int) bool { return seqs[i] < seqs[j] })
	w := bufio.NewWriter(f)
	for _, seq := range seqs {
		e := o.pending[seq]
		b, _ := json.Marshal(line{Seq: e.seq, Exchange: e.exchange, RoutingKey: e.routingKey, Body: e.body})
		if _, err := w.Write(append(b, '\n')); err != nil {
			_ = f.Close()
			return err
		}
	}
	if err := w.Flush(); err != nil {
		_ = f.Close()
		return err
	}
	if err := f.Sync(); err != nil {
		_ = f.Close()
		return err
	}
	if err := f.Close(); err != nil {
		return err
	}
	if err := os.Rename(tmp, o.path); err != nil {
		return err
	}
	o.f, err = os.OpenFile(o.path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
	return err
}

// Publish durably records a message and returns once it is fsync'd. It satisfies
// the order.Publisher seam, so the amqp_emit hook records to the outbox instead
// of publishing to the broker directly. The relay delivers it asynchronously.
func (o *Outbox) Publish(exchange, routingKey string, body []byte) error {
	o.mu.Lock()
	o.seq++
	seq := o.seq
	b, err := json.Marshal(line{Seq: seq, Exchange: exchange, RoutingKey: routingKey, Body: body})
	if err != nil {
		o.mu.Unlock()
		return err
	}
	if _, err := o.f.Write(append(b, '\n')); err != nil {
		o.mu.Unlock()
		return err
	}
	if err := o.f.Sync(); err != nil {
		o.mu.Unlock()
		return err
	}
	o.pending[seq] = entry{seq: seq, exchange: exchange, routingKey: routingKey, body: body}
	o.mu.Unlock()

	select {
	case o.wake <- struct{}{}: // nudge the relay
	default:
	}
	return nil
}

// PendingCount reports how many messages are awaiting delivery (for tests/metrics).
func (o *Outbox) PendingCount() int {
	o.mu.Lock()
	defer o.mu.Unlock()
	return len(o.pending)
}

// Run drains the outbox to pub, retrying failed rounds every retry interval and
// waking immediately when a new message arrives. It blocks until ctx is done.
func (o *Outbox) Run(ctx context.Context, pub Publisher, retry time.Duration, logger *slog.Logger) {
	t := time.NewTicker(retry)
	defer t.Stop()
	for {
		o.drain(pub, logger)
		select {
		case <-ctx.Done():
			return
		case <-t.C:
		case <-o.wake:
		}
	}
}

// drain attempts to deliver every pending message in seq order. It stops the
// round at the first delivery error (likely the broker is down) so messages keep
// their order and the next tick retries.
func (o *Outbox) drain(pub Publisher, logger *slog.Logger) {
	o.mu.Lock()
	seqs := make([]uint64, 0, len(o.pending))
	for seq := range o.pending {
		seqs = append(seqs, seq)
	}
	sort.Slice(seqs, func(i, j int) bool { return seqs[i] < seqs[j] })
	batch := make([]entry, 0, len(seqs))
	for _, seq := range seqs {
		batch = append(batch, o.pending[seq])
	}
	o.mu.Unlock()

	for _, e := range batch {
		if err := pub.Publish(e.exchange, e.routingKey, e.body); err != nil {
			if logger != nil {
				logger.Warn("outbox: delivery failed, will retry", "seq", e.seq, "err", err)
			}
			return // broker likely down; retry the whole round next tick
		}
		if err := o.ack(e.seq); err != nil && logger != nil {
			// The message was delivered; failing to persist the ack only risks a
			// redelivery on restart (at-least-once), so log and continue.
			logger.Warn("outbox: ack write failed", "seq", e.seq, "err", err)
		}
	}
}

// ack records that seq was delivered and drops it from the pending set.
func (o *Outbox) ack(seq uint64) error {
	o.mu.Lock()
	defer o.mu.Unlock()
	b, _ := json.Marshal(line{Ack: seq})
	if _, err := o.f.Write(append(b, '\n')); err != nil {
		return err
	}
	if err := o.f.Sync(); err != nil {
		return err
	}
	delete(o.pending, seq)
	return nil
}

// Close closes the underlying file.
func (o *Outbox) Close() error {
	o.mu.Lock()
	defer o.mu.Unlock()
	return o.f.Close()
}