editor: experiments QA panel (P0 of experiments plan)
New Experiments section in the rail: lists tracking experiment rules (status, traffic, arms, targeting), shows this session's assignment via evaluate-experiments, per-arm analytics with winner/significance, a QA arm override (sets the storefront variant cookie), and a two-step pause/resume kill-switch. Admin reads degrade when the tracking gate refuses; eligibility always works. Mock tracking endpoints + harness tests. 🤖 Generated with Codebuff Co-Authored-By: Codebuff <noreply@codebuff.com>
This commit is contained in:
mats committed
1 parent
40b33f89a4
commit
922562e1c2
6 files changed
+325
-1
No files matched your search
+143
-1
@@ -1,6 +1,6 @@
|
||||
import { createElement, useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { InsertPicker } from '../editors/builtin'
|
||||
import type { CmsNode, TypeInfo } from '../api'
|
||||
import type { CmsNode, EvaluateContext, ExperimentAnalytics, ExperimentAssignment, ExperimentRule, TypeInfo } from '../api'
|
||||
import { allowedChildren, connectionsOf, labelKey, sortGroups, typeName } from '../schema'
|
||||
import type { DropTarget } from '../page/dnd'
|
||||
import { Icon, TypeIcon } from './Icon'
|
||||
@@ -47,6 +47,7 @@ export function Rail() {
|
||||
<OutlineTree />
|
||||
<Palette />
|
||||
<NodeSearch />
|
||||
<ExperimentsPanel />
|
||||
</div>
|
||||
</aside>
|
||||
)
|
||||
@@ -562,3 +563,144 @@ function NodeSearch() {
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
// --- experiments: QA + eligibility on the page (plan-onpage-experiments-personalisation.md P0) ---------
|
||||
|
||||
const expCookie = (name: string): string | null => {
|
||||
const m = document.cookie.match(new RegExp(`(?:^|; )${name}=([^;]*)`))
|
||||
return m ? decodeURIComponent(m[1]) : null
|
||||
}
|
||||
const setExpCookie = (name: string, value: string | null) => {
|
||||
document.cookie = value === null ? `${name}=; expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/` : `${name}=${encodeURIComponent(value)}; path=/; SameSite=Lax`
|
||||
}
|
||||
const targetingSummary = (rules?: ExperimentRule['targeting']): string =>
|
||||
!rules?.length ? 'everyone' : rules.map((r) => `${r.field} ${r.op} ${r.value ?? (r.values ?? []).join('|')}`).join(' · ')
|
||||
const armNames = (rule: ExperimentRule): string[] => Object.keys(rule.variant_weights ?? {})
|
||||
|
||||
function ExperimentsPanel() {
|
||||
const ctrl = useCtrl()
|
||||
const [rules, setRules] = useState<ExperimentRule[] | null>(null)
|
||||
const [adminDenied, setAdminDenied] = useState(false)
|
||||
const [assign, setAssign] = useState<Record<string, ExperimentAssignment>>({})
|
||||
const [stats, setStats] = useState<Record<string, ExperimentAnalytics>>({})
|
||||
const [confirmOff, setConfirmOff] = useState<string | null>(null)
|
||||
const [tick, setTick] = useState(0) // re-read override cookies after set/clear
|
||||
const [open, setOpen] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) return
|
||||
let live = true
|
||||
const ctx: EvaluateContext = { pagePath: location.pathname, referrer: document.referrer || undefined, language: navigator.language || undefined }
|
||||
void ctrl.api
|
||||
.trackingConfig()
|
||||
.then((cfg) => {
|
||||
if (!live) return
|
||||
const rs = cfg.experiment_rules ?? []
|
||||
setRules(rs)
|
||||
if (!rs.length) return
|
||||
void ctrl.api.evaluateExperiments(rs.map((r) => r.experiment_id), ctx).then(
|
||||
(res) => live && setAssign(Object.fromEntries((res.assignments ?? []).map((a) => [a.experimentId, a]))),
|
||||
() => {}, // evaluate is public; a failure just leaves assignments blank
|
||||
)
|
||||
void ctrl.api.experimentAnalytics().then(
|
||||
(rows) => live && setStats(Object.fromEntries((rows ?? []).map((s) => [s.experiment_id, s]))),
|
||||
() => live && setAdminDenied(true), // admin-gated: eligibility still works, stats hide
|
||||
)
|
||||
})
|
||||
.catch(() => live && (setRules([]), setAdminDenied(true)))
|
||||
return () => void (live = false)
|
||||
}, [open, ctrl])
|
||||
|
||||
const toggleDisabled = async (rule: ExperimentRule) => {
|
||||
if (confirmOff !== rule.experiment_id) return void setConfirmOff(rule.experiment_id)
|
||||
setConfirmOff(null)
|
||||
try {
|
||||
const res = await ctrl.api.setExperimentDisabled(rule.experiment_id, !rule.disabled)
|
||||
setRules((rs) => rs?.map((r) => (r.experiment_id === rule.experiment_id ? { ...r, disabled: res.disabled } : r)) ?? rs)
|
||||
ctrl.toast('info', `${rule.experiment_id} ${res.disabled ? 'paused' : 'resumed'}`)
|
||||
} catch (e) {
|
||||
ctrl.toast('error', (e as Error).message)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<section aria-label="Experiments panel">
|
||||
<h4 className="cms-rail__sec">
|
||||
<button className="cms-sectoggle" aria-expanded={open} aria-label="Experiments" onClick={() => setOpen((o) => !o)}>
|
||||
Experiments{rules ? ` (${rules.length})` : ''}
|
||||
</button>
|
||||
</h4>
|
||||
{open && (
|
||||
<div className="cms-pad">
|
||||
{rules === null && <p className="cms-muted">Loading experiments…</p>}
|
||||
{rules !== null && !rules.length && <p className="cms-muted">No experiments configured in tracking.</p>}
|
||||
{adminDenied && !!rules?.length && <p className="cms-muted cms-hint">Tracking admin refused — eligibility still works; stats and pause need an admin credential.</p>}
|
||||
{!!rules?.length && (
|
||||
<ul className="cms-explist">
|
||||
{rules.map((rule) => {
|
||||
const a = assign[rule.experiment_id]
|
||||
const s = stats[rule.experiment_id]
|
||||
const arms = armNames(rule)
|
||||
const cookie = expCookie(a?.cookieName ?? `exp_${rule.experiment_id}`)
|
||||
void tick
|
||||
return (
|
||||
<li key={rule.experiment_id} className="cms-exp">
|
||||
<div className="cms-exp__head">
|
||||
<strong>{rule.experiment_id}</strong>
|
||||
<span className={`cms-pill${rule.disabled ? ' is-off' : ''}`}>{rule.disabled ? 'paused' : 'live'}</span>
|
||||
</div>
|
||||
<div className="cms-muted cms-exp__meta">
|
||||
traffic {Math.round((rule.traffic_fraction ?? 1) * 100)}% · arms {arms.length ? arms.join(' / ') : 'a / b'} · {targetingSummary(rule.targeting)}
|
||||
</div>
|
||||
<div className="cms-exp__assign">
|
||||
{a ? (a.eligible ? <>you: <b>{a.variant || '—'}</b></> : 'you: not eligible') : 'you: …'}
|
||||
{cookie && cookie !== a?.variant && <span className="cms-muted"> (override: {cookie})</span>}
|
||||
</div>
|
||||
{s && (
|
||||
<div className="cms-muted cms-exp__meta">
|
||||
{s.total_sessions} sessions · winner <b>{s.winner || '—'}</b>
|
||||
{s.winner ? ` (${Math.round((s.winner_probability ?? 0) * 100)}%)` : ''}
|
||||
{s.significant ? ' · significant' : ''} — {s.conclusion || 'collecting data'}
|
||||
</div>
|
||||
)}
|
||||
<div className="cms-exp__actions">
|
||||
<label className="cms-muted">
|
||||
QA arm{' '}
|
||||
<select
|
||||
className="cms-input"
|
||||
aria-label={`Force variant for ${rule.experiment_id}`}
|
||||
value={cookie ?? ''}
|
||||
onChange={(e) => {
|
||||
setExpCookie(a?.cookieName ?? `exp_${rule.experiment_id}`, e.target.value || null)
|
||||
setTick((t) => t + 1)
|
||||
}}
|
||||
>
|
||||
<option value="">session default</option>
|
||||
{arms.map((v) => (
|
||||
<option key={v} value={v}>
|
||||
{v}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<button
|
||||
className={`cms-ico${confirmOff === rule.experiment_id ? ' is-on' : ''}`}
|
||||
aria-label={rule.disabled ? `Resume ${rule.experiment_id}` : `Pause ${rule.experiment_id}`}
|
||||
title={rule.disabled ? 'Resume (re-buckets new sessions)' : 'Pause (kill-switch: new sessions see control)'}
|
||||
onClick={() => void toggleDisabled(rule)}
|
||||
>
|
||||
<Icon name={confirmOff === rule.experiment_id ? 'check' : rule.disabled ? 'play' : 'pause'} size={15} />
|
||||
{confirmOff === rule.experiment_id && <span>confirm</span>}
|
||||
</button>
|
||||
</div>
|
||||
</li>
|
||||
)
|
||||
})}
|
||||
</ul>
|
||||
)}
|
||||
<p className="cms-muted cms-hint">Overrides set the storefront variant cookie — reload the page to preview the arm.</p>
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
)
|
||||
}
|
||||
Reference in new issue
Block a user