editor: experiments QA panel (P0 of experiments plan)

New Experiments section in the rail: lists tracking experiment rules
(status, traffic, arms, targeting), shows this session's assignment via
evaluate-experiments, per-arm analytics with winner/significance, a QA
arm override (sets the storefront variant cookie), and a two-step
pause/resume kill-switch. Admin reads degrade when the tracking gate
refuses; eligibility always works. Mock tracking endpoints + harness
tests.

🤖 Generated with Codebuff
Co-Authored-By: Codebuff <noreply@codebuff.com>

mock/server.ts

@@ -15,6 +15,11 @@ export interface MockOptions {
 export function createMockCms(opts: MockOptions = {}) {
   let graph = seedGraph()
   const published: string[] = []
+  // --- mock slask-tracking experiments (P0 QA panel) ---
+  const expRules: { experiment_id: string; targeting?: { field: string; op: string; value?: string }[]; traffic_fraction?: number; variant_weights?: Record<string, number>; disabled?: boolean }[] = [
+    { experiment_id: 'exp_hero', targeting: [{ field: 'page_path', op: 'prefix', value: '/' }], traffic_fraction: 1, variant_weights: { a: 1, b: 1 } },
+    { experiment_id: 'exp_checkout', targeting: [{ field: 'country', op: 'eq', value: 'SE' }], traffic_fraction: 0.5, variant_weights: { control: 9, onepage: 1 }, disabled: true },
+  ]
   const requests: string[] = []
   let previewApi = true
   // --- mock go-auth (RFC 8628 device grant) ---
@@ -102,6 +107,44 @@ export function createMockCms(opts: MockOptions = {}) {
         }
         return json(200, walk(r[1]))
       }
+      if (path === '/api/tracking/config' && m === 'GET') return json(200, { experiment_rules: expRules })
+      if (path === '/api/tracking/evaluate-experiments' && m === 'POST') {
+        const body = (await req.json()) as { experiments?: { id: string; cookieName?: string }[] }
+        const cookies = Object.fromEntries((req.headers.get('cookie') ?? '').split(';').map((p) => p.trim().split('=').map((s) => s.trim())).filter((kv) => kv.length === 2)) as Record<string, string>
+        const assignments = (body.experiments ?? []).map((e) => {
+          const rule = expRules.find((r) => r.experiment_id === e.id)
+          const cookieName = e.cookieName || `exp_${e.id}`
+          if (!rule || rule.disabled) return { experimentId: e.id, cookieName, variant: '', eligible: false }
+          const arms = Object.keys(rule.variant_weights ?? { a: 1, b: 1 })
+          const forced = cookies[cookieName]
+          return { experimentId: e.id, cookieName, variant: forced && arms.includes(forced) ? forced : arms[0], eligible: true }
+        })
+        return json(200, { assignments })
+      }
+      if (path === '/api/tracking/experiments/analytics' && m === 'GET')
+        return json(200, [
+          {
+            experiment_id: 'exp_hero',
+            variants: [
+              { variant: 'a', sessions: 120, conversions: 12, conversion_rate: 0.1 },
+              { variant: 'b', sessions: 130, conversions: 22, conversion_rate: 0.169 },
+            ],
+            winner: 'b',
+            winner_probability: 0.87,
+            significant: true,
+            conclusion: 'b beats a',
+            total_sessions: 250,
+          },
+        ])
+      const expToggle = /^\/api\/tracking\/config\/experiments\/([^/]+)\/disabled$/.exec(path)
+      if (expToggle && m === 'PATCH') {
+        const rule = expRules.find((x) => x.experiment_id === decodeURIComponent(expToggle[1]))
+        if (!rule) return json(404, { error: 'experiment rule not found' })
+        const body = (await req.json()) as { disabled?: boolean }
+        if (typeof body.disabled !== 'boolean') return json(422, { error: 'missing disabled' })
+        rule.disabled = body.disabled
+        return json(200, { disabled: rule.disabled })
+      }
       if (path === '/api/edit' && m === 'POST') {
         const body = (await req.json()) as { ops: any[] }
         const res = graph.edit(body.ops)

src/api.ts

@@ -35,6 +35,48 @@ export interface Blueprint {
 }
 export type JSONSchema = Record<string, any>
 
+export interface ExperimentTargetingRule {
+  field: string
+  op: string
+  value?: string
+  values?: string[]
+}
+export interface ExperimentRule {
+  experiment_id: string
+  targeting?: ExperimentTargetingRule[]
+  traffic_fraction?: number
+  variant_weights?: Record<string, number>
+  disabled?: boolean
+}
+export interface EvaluateContext {
+  pagePath?: string
+  referrer?: string
+  language?: string
+  personaGroups?: string[]
+  [k: string]: unknown
+}
+export interface ExperimentAssignment {
+  experimentId: string
+  cookieName: string
+  variant: string
+  eligible: boolean
+}
+export interface VariantAnalytics {
+  variant: string
+  sessions: number
+  conversions: number
+  conversion_rate: number
+}
+export interface ExperimentAnalytics {
+  experiment_id: string
+  variants: VariantAnalytics[]
+  winner: string
+  winner_probability: number
+  significant: boolean
+  conclusion: string
+  total_sessions: number
+}
+
 export interface RequestLog {
   n: number
   method: string
@@ -164,6 +206,19 @@ export class Api {
   types(): Promise<TypeInfo[]> {
     return (this.typesP ??= this.get<TypeInfo[] | { types: TypeInfo[] }>('/api/types').then((r) => (Array.isArray(r) ? r : r.types ?? [])))
   }
+
+  // --- experiments & personalisation (slask-tracking; same-origin /api/tracking/*) ------------------
+  // evaluate/assign are public (cookie-scoped); config/analytics are admin-gated — the editor's bearer
+  // satisfies the gate in prod, and the gate is open in dev. Callers degrade when admin reads refuse.
+  trackingConfig = () => this.get<{ experiment_rules?: ExperimentRule[] }>('/api/tracking/config')
+  evaluateExperiments = (ids: string[], context: EvaluateContext) =>
+    this.post<{ assignments: ExperimentAssignment[] }>('/api/tracking/evaluate-experiments', {
+      experiments: ids.map((id) => ({ id })),
+      context,
+    })
+  experimentAnalytics = () => this.get<ExperimentAnalytics[]>('/api/tracking/experiments/analytics')
+  setExperimentDisabled = (id: string, disabled: boolean) =>
+    this.request<{ disabled: boolean }>('PATCH', `/api/tracking/config/experiments/${encodeURIComponent(id)}/disabled`, { disabled })
   blueprints(): Promise<Blueprint[]> {
     return (this.blueprintsP ??= this.get<Blueprint[]>('/api/blueprints').catch(() => []))
   }

src/ui/Icon.tsx

@@ -35,7 +35,9 @@ import {
   mdiPageLayoutFooter,
   mdiPageLayoutHeader,
   mdiPaletteOutline,
+  mdiPause,
   mdiPencilOutline,
+  mdiPlay,
   mdiPlus,
   mdiPlusBoxOutline,
   mdiPower,
@@ -86,6 +88,8 @@ export const ICONS = {
   publish: mdiCloudUploadOutline,
   link: mdiLinkVariant,
   check: mdiCheck,
+  pause: mdiPause,
+  play: mdiPlay,
   refresh: mdiRefresh,
   info: mdiInformationOutline,
   warning: mdiAlertCircleOutline,

src/ui/Rail.tsx

@@ -1,6 +1,6 @@
 import { createElement, useEffect, useMemo, useRef, useState } from 'react'
 import { InsertPicker } from '../editors/builtin'
-import type { CmsNode, TypeInfo } from '../api'
+import type { CmsNode, EvaluateContext, ExperimentAnalytics, ExperimentAssignment, ExperimentRule, TypeInfo } from '../api'
 import { allowedChildren, connectionsOf, labelKey, sortGroups, typeName } from '../schema'
 import type { DropTarget } from '../page/dnd'
 import { Icon, TypeIcon } from './Icon'
@@ -47,6 +47,7 @@ export function Rail() {
         <OutlineTree />
         <Palette />
         <NodeSearch />
+        <ExperimentsPanel />
       </div>
     </aside>
   )
@@ -562,3 +563,144 @@ function NodeSearch() {
     </section>
   )
 }
+
+// --- experiments: QA + eligibility on the page (plan-onpage-experiments-personalisation.md P0) ---------
+
+const expCookie = (name: string): string | null => {
+  const m = document.cookie.match(new RegExp(`(?:^|; )${name}=([^;]*)`))
+  return m ? decodeURIComponent(m[1]) : null
+}
+const setExpCookie = (name: string, value: string | null) => {
+  document.cookie = value === null ? `${name}=; expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/` : `${name}=${encodeURIComponent(value)}; path=/; SameSite=Lax`
+}
+const targetingSummary = (rules?: ExperimentRule['targeting']): string =>
+  !rules?.length ? 'everyone' : rules.map((r) => `${r.field} ${r.op} ${r.value ?? (r.values ?? []).join('|')}`).join(' · ')
+const armNames = (rule: ExperimentRule): string[] => Object.keys(rule.variant_weights ?? {})
+
+function ExperimentsPanel() {
+  const ctrl = useCtrl()
+  const [rules, setRules] = useState<ExperimentRule[] | null>(null)
+  const [adminDenied, setAdminDenied] = useState(false)
+  const [assign, setAssign] = useState<Record<string, ExperimentAssignment>>({})
+  const [stats, setStats] = useState<Record<string, ExperimentAnalytics>>({})
+  const [confirmOff, setConfirmOff] = useState<string | null>(null)
+  const [tick, setTick] = useState(0) // re-read override cookies after set/clear
+  const [open, setOpen] = useState(false)
+
+  useEffect(() => {
+    if (!open) return
+    let live = true
+    const ctx: EvaluateContext = { pagePath: location.pathname, referrer: document.referrer || undefined, language: navigator.language || undefined }
+    void ctrl.api
+      .trackingConfig()
+      .then((cfg) => {
+        if (!live) return
+        const rs = cfg.experiment_rules ?? []
+        setRules(rs)
+        if (!rs.length) return
+        void ctrl.api.evaluateExperiments(rs.map((r) => r.experiment_id), ctx).then(
+          (res) => live && setAssign(Object.fromEntries((res.assignments ?? []).map((a) => [a.experimentId, a]))),
+          () => {}, // evaluate is public; a failure just leaves assignments blank
+        )
+        void ctrl.api.experimentAnalytics().then(
+          (rows) => live && setStats(Object.fromEntries((rows ?? []).map((s) => [s.experiment_id, s]))),
+          () => live && setAdminDenied(true), // admin-gated: eligibility still works, stats hide
+        )
+      })
+      .catch(() => live && (setRules([]), setAdminDenied(true)))
+    return () => void (live = false)
+  }, [open, ctrl])
+
+  const toggleDisabled = async (rule: ExperimentRule) => {
+    if (confirmOff !== rule.experiment_id) return void setConfirmOff(rule.experiment_id)
+    setConfirmOff(null)
+    try {
+      const res = await ctrl.api.setExperimentDisabled(rule.experiment_id, !rule.disabled)
+      setRules((rs) => rs?.map((r) => (r.experiment_id === rule.experiment_id ? { ...r, disabled: res.disabled } : r)) ?? rs)
+      ctrl.toast('info', `${rule.experiment_id} ${res.disabled ? 'paused' : 'resumed'}`)
+    } catch (e) {
+      ctrl.toast('error', (e as Error).message)
+    }
+  }
+
+  return (
+    <section aria-label="Experiments panel">
+      <h4 className="cms-rail__sec">
+        <button className="cms-sectoggle" aria-expanded={open} aria-label="Experiments" onClick={() => setOpen((o) => !o)}>
+          Experiments{rules ? ` (${rules.length})` : ''}
+        </button>
+      </h4>
+      {open && (
+        <div className="cms-pad">
+          {rules === null && <p className="cms-muted">Loading experiments…</p>}
+          {rules !== null && !rules.length && <p className="cms-muted">No experiments configured in tracking.</p>}
+          {adminDenied && !!rules?.length && <p className="cms-muted cms-hint">Tracking admin refused — eligibility still works; stats and pause need an admin credential.</p>}
+          {!!rules?.length && (
+            <ul className="cms-explist">
+              {rules.map((rule) => {
+                const a = assign[rule.experiment_id]
+                const s = stats[rule.experiment_id]
+                const arms = armNames(rule)
+                const cookie = expCookie(a?.cookieName ?? `exp_${rule.experiment_id}`)
+                void tick
+                return (
+                  <li key={rule.experiment_id} className="cms-exp">
+                    <div className="cms-exp__head">
+                      <strong>{rule.experiment_id}</strong>
+                      <span className={`cms-pill${rule.disabled ? ' is-off' : ''}`}>{rule.disabled ? 'paused' : 'live'}</span>
+                    </div>
+                    <div className="cms-muted cms-exp__meta">
+                      traffic {Math.round((rule.traffic_fraction ?? 1) * 100)}% · arms {arms.length ? arms.join(' / ') : 'a / b'} · {targetingSummary(rule.targeting)}
+                    </div>
+                    <div className="cms-exp__assign">
+                      {a ? (a.eligible ? <>you: <b>{a.variant || '—'}</b></> : 'you: not eligible') : 'you: …'}
+                      {cookie && cookie !== a?.variant && <span className="cms-muted"> (override: {cookie})</span>}
+                    </div>
+                    {s && (
+                      <div className="cms-muted cms-exp__meta">
+                        {s.total_sessions} sessions · winner <b>{s.winner || '—'}</b>
+                        {s.winner ? ` (${Math.round((s.winner_probability ?? 0) * 100)}%)` : ''}
+                        {s.significant ? ' · significant' : ''} — {s.conclusion || 'collecting data'}
+                      </div>
+                    )}
+                    <div className="cms-exp__actions">
+                      <label className="cms-muted">
+                        QA arm{' '}
+                        <select
+                          className="cms-input"
+                          aria-label={`Force variant for ${rule.experiment_id}`}
+                          value={cookie ?? ''}
+                          onChange={(e) => {
+                            setExpCookie(a?.cookieName ?? `exp_${rule.experiment_id}`, e.target.value || null)
+                            setTick((t) => t + 1)
+                          }}
+                        >
+                          <option value="">session default</option>
+                          {arms.map((v) => (
+                            <option key={v} value={v}>
+                              {v}
+                            </option>
+                          ))}
+                        </select>
+                      </label>
+                      <button
+                        className={`cms-ico${confirmOff === rule.experiment_id ? ' is-on' : ''}`}
+                        aria-label={rule.disabled ? `Resume ${rule.experiment_id}` : `Pause ${rule.experiment_id}`}
+                        title={rule.disabled ? 'Resume (re-buckets new sessions)' : 'Pause (kill-switch: new sessions see control)'}
+                        onClick={() => void toggleDisabled(rule)}
+                      >
+                        <Icon name={confirmOff === rule.experiment_id ? 'check' : rule.disabled ? 'play' : 'pause'} size={15} />
+                        {confirmOff === rule.experiment_id && <span>confirm</span>}
+                      </button>
+                    </div>
+                  </li>
+                )
+              })}
+            </ul>
+          )}
+          <p className="cms-muted cms-hint">Overrides set the storefront variant cookie — reload the page to preview the arm.</p>
+        </div>
+      )}
+    </section>
+  )
+}

src/ui/styles.css

@@ -255,6 +255,20 @@ input.cms-map__key { padding-top: 6px; }
 .cms-noderow:active { cursor: grabbing; }
 .cms-noderow__main { flex: 1; min-width: 0; display: flex; flex-direction: column; overflow: hidden; white-space: nowrap; }
 .cms-pad { padding: 10px 12px 4px; }
+.cms-sectoggle { background: none; border: 0; padding: 0; font: inherit; letter-spacing: inherit; text-transform: inherit; color: inherit; cursor: pointer; display: inline-flex; align-items: center; gap: 6px; }
+.cms-sectoggle:hover { color: var(--cms-accent); }
+
+/* experiments QA panel */
+.cms-explist { list-style: none; margin: 8px 0; padding: 0; display: flex; flex-direction: column; gap: 10px; }
+.cms-exp { border: 1px solid var(--cms-line); border-radius: 10px; padding: 8px 10px; display: flex; flex-direction: column; gap: 4px; }
+.cms-exp__head { display: flex; align-items: center; gap: 8px; justify-content: space-between; }
+.cms-exp__head strong { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+.cms-pill.is-off { border-color: #a33a4a; color: #ff8a8a; }
+.cms-exp__meta { font-size: 12px; }
+.cms-exp__assign { font-size: 12px; }
+.cms-exp__actions { display: flex; align-items: center; gap: 8px; justify-content: space-between; margin-top: 2px; }
+.cms-exp__actions .cms-input { max-width: 150px; }
+.cms-exp__actions .cms-ico.is-on { border-color: #a33a4a; color: #ff8a8a; }
 .cms-hint { font-size: 11.5px; margin-top: 6px; }
 .cms-outline { list-style: none; margin: 0; padding: 4px; overflow: auto; }
 .cms-outline li button { display: flex; align-items: center; gap: 8px; width: 100%; background: none; border: 0; border-radius: 6px; padding: 4px 8px; text-align: left; overflow: hidden; white-space: nowrap; }

test/experiments.test.tsx

@@ -0,0 +1,66 @@
+import { afterEach, describe, expect, it } from 'vitest'
+import { act } from 'react'
+import { setup } from './harness'
+import type { Harness } from './harness'
+
+let h: Harness
+afterEach(() => h?.cleanup())
+
+const q = (sel: string) => h.shadow.querySelector<HTMLElement>(sel)
+const qa = (sel: string) => [...h.shadow.querySelectorAll<HTMLElement>(sel)]
+
+describe('experiments QA panel', () => {
+  it('lists experiments with status, eligibility and analytics', async () => {
+    h = await setup()
+    await h.settle()
+    await h.press('Panel')
+    await h.press('Experiments')
+    await h.settle()
+    const rows = qa('.cms-exp')
+    expect(rows.map((r) => r.querySelector('strong')?.textContent)).toEqual(['exp_hero', 'exp_checkout'])
+    expect(rows[0].textContent).toContain('live')
+    expect(rows[1].textContent).toContain('paused')
+    expect(rows[0].textContent).toContain('traffic 100%')
+    expect(rows[0].textContent).toContain('arms a / b')
+    expect(rows[0].textContent).toContain('page_path prefix /')
+    // Public evaluate: hero eligible (arm a), disabled checkout not.
+    expect(rows[0].textContent).toContain('you: a')
+    expect(rows[1].textContent).toContain('not eligible')
+    // Admin analytics for the live rule.
+    expect(rows[0].textContent).toContain('250 sessions')
+    expect(rows[0].textContent).toContain('b')
+    expect(rows[0].textContent).toContain('significant')
+  })
+
+  it('forcing a QA arm sets the storefront variant cookie', async () => {
+    h = await setup()
+    await h.settle()
+    await h.press('Panel')
+    await h.press('Experiments')
+    await h.settle()
+    const sel = q('select[aria-label=\"Force variant for exp_hero\"]') as HTMLSelectElement
+    const set = Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype, 'value')!.set!
+    await act(async () => {
+      set.call(sel, 'b')
+      sel.dispatchEvent(new Event('change', { bubbles: true }))
+    })
+    expect(document.cookie).toContain('exp_exp_hero=b')
+  })
+
+  it('pausing asks to confirm, then flips the rule via PATCH', async () => {
+    h = await setup()
+    await h.settle()
+    await h.press('Panel')
+    await h.press('Experiments')
+    await h.settle()
+    await act(async () => void q('[aria-label=\"Pause exp_hero\"]')!.click())
+    await h.settle()
+    // First click only arms the confirm …
+    expect(q('[aria-label=\"Pause exp_hero\"]')?.textContent).toContain('confirm')
+    await act(async () => void q('[aria-label=\"Pause exp_hero\"]')!.click())
+    await h.settle()
+    const rows = qa('.cms-exp')
+    expect(rows[0].textContent).toContain('paused')
+    expect(h.cms.requests.some((r) => r === 'PATCH /api/tracking/config/experiments/exp_hero/disabled')).toBe(true)
+  })
+})