diff --git a/mock/server.ts b/mock/server.ts index 44be178..896ca5f 100644 --- a/mock/server.ts +++ b/mock/server.ts @@ -15,6 +15,11 @@ export interface MockOptions { export function createMockCms(opts: MockOptions = {}) { let graph = seedGraph() const published: string[] = [] + // --- mock slask-tracking experiments (P0 QA panel) --- + const expRules: { experiment_id: string; targeting?: { field: string; op: string; value?: string }[]; traffic_fraction?: number; variant_weights?: Record; disabled?: boolean }[] = [ + { experiment_id: 'exp_hero', targeting: [{ field: 'page_path', op: 'prefix', value: '/' }], traffic_fraction: 1, variant_weights: { a: 1, b: 1 } }, + { experiment_id: 'exp_checkout', targeting: [{ field: 'country', op: 'eq', value: 'SE' }], traffic_fraction: 0.5, variant_weights: { control: 9, onepage: 1 }, disabled: true }, + ] const requests: string[] = [] let previewApi = true // --- mock go-auth (RFC 8628 device grant) --- @@ -102,6 +107,44 @@ export function createMockCms(opts: MockOptions = {}) { } return json(200, walk(r[1])) } + if (path === '/api/tracking/config' && m === 'GET') return json(200, { experiment_rules: expRules }) + if (path === '/api/tracking/evaluate-experiments' && m === 'POST') { + const body = (await req.json()) as { experiments?: { id: string; cookieName?: string }[] } + const cookies = Object.fromEntries((req.headers.get('cookie') ?? '').split(';').map((p) => p.trim().split('=').map((s) => s.trim())).filter((kv) => kv.length === 2)) as Record + const assignments = (body.experiments ?? []).map((e) => { + const rule = expRules.find((r) => r.experiment_id === e.id) + const cookieName = e.cookieName || `exp_${e.id}` + if (!rule || rule.disabled) return { experimentId: e.id, cookieName, variant: '', eligible: false } + const arms = Object.keys(rule.variant_weights ?? { a: 1, b: 1 }) + const forced = cookies[cookieName] + return { experimentId: e.id, cookieName, variant: forced && arms.includes(forced) ? forced : arms[0], eligible: true } + }) + return json(200, { assignments }) + } + if (path === '/api/tracking/experiments/analytics' && m === 'GET') + return json(200, [ + { + experiment_id: 'exp_hero', + variants: [ + { variant: 'a', sessions: 120, conversions: 12, conversion_rate: 0.1 }, + { variant: 'b', sessions: 130, conversions: 22, conversion_rate: 0.169 }, + ], + winner: 'b', + winner_probability: 0.87, + significant: true, + conclusion: 'b beats a', + total_sessions: 250, + }, + ]) + const expToggle = /^\/api\/tracking\/config\/experiments\/([^/]+)\/disabled$/.exec(path) + if (expToggle && m === 'PATCH') { + const rule = expRules.find((x) => x.experiment_id === decodeURIComponent(expToggle[1])) + if (!rule) return json(404, { error: 'experiment rule not found' }) + const body = (await req.json()) as { disabled?: boolean } + if (typeof body.disabled !== 'boolean') return json(422, { error: 'missing disabled' }) + rule.disabled = body.disabled + return json(200, { disabled: rule.disabled }) + } if (path === '/api/edit' && m === 'POST') { const body = (await req.json()) as { ops: any[] } const res = graph.edit(body.ops) diff --git a/src/api.ts b/src/api.ts index 3feeb75..5e25cf0 100644 --- a/src/api.ts +++ b/src/api.ts @@ -35,6 +35,48 @@ export interface Blueprint { } export type JSONSchema = Record +export interface ExperimentTargetingRule { + field: string + op: string + value?: string + values?: string[] +} +export interface ExperimentRule { + experiment_id: string + targeting?: ExperimentTargetingRule[] + traffic_fraction?: number + variant_weights?: Record + disabled?: boolean +} +export interface EvaluateContext { + pagePath?: string + referrer?: string + language?: string + personaGroups?: string[] + [k: string]: unknown +} +export interface ExperimentAssignment { + experimentId: string + cookieName: string + variant: string + eligible: boolean +} +export interface VariantAnalytics { + variant: string + sessions: number + conversions: number + conversion_rate: number +} +export interface ExperimentAnalytics { + experiment_id: string + variants: VariantAnalytics[] + winner: string + winner_probability: number + significant: boolean + conclusion: string + total_sessions: number +} + export interface RequestLog { n: number method: string @@ -164,6 +206,19 @@ export class Api { types(): Promise { return (this.typesP ??= this.get('/api/types').then((r) => (Array.isArray(r) ? r : r.types ?? []))) } + + // --- experiments & personalisation (slask-tracking; same-origin /api/tracking/*) ------------------ + // evaluate/assign are public (cookie-scoped); config/analytics are admin-gated — the editor's bearer + // satisfies the gate in prod, and the gate is open in dev. Callers degrade when admin reads refuse. + trackingConfig = () => this.get<{ experiment_rules?: ExperimentRule[] }>('/api/tracking/config') + evaluateExperiments = (ids: string[], context: EvaluateContext) => + this.post<{ assignments: ExperimentAssignment[] }>('/api/tracking/evaluate-experiments', { + experiments: ids.map((id) => ({ id })), + context, + }) + experimentAnalytics = () => this.get('/api/tracking/experiments/analytics') + setExperimentDisabled = (id: string, disabled: boolean) => + this.request<{ disabled: boolean }>('PATCH', `/api/tracking/config/experiments/${encodeURIComponent(id)}/disabled`, { disabled }) blueprints(): Promise { return (this.blueprintsP ??= this.get('/api/blueprints').catch(() => [])) } diff --git a/src/ui/Icon.tsx b/src/ui/Icon.tsx index 571f40b..c3e2957 100644 --- a/src/ui/Icon.tsx +++ b/src/ui/Icon.tsx @@ -35,7 +35,9 @@ import { mdiPageLayoutFooter, mdiPageLayoutHeader, mdiPaletteOutline, + mdiPause, mdiPencilOutline, + mdiPlay, mdiPlus, mdiPlusBoxOutline, mdiPower, @@ -86,6 +88,8 @@ export const ICONS = { publish: mdiCloudUploadOutline, link: mdiLinkVariant, check: mdiCheck, + pause: mdiPause, + play: mdiPlay, refresh: mdiRefresh, info: mdiInformationOutline, warning: mdiAlertCircleOutline, diff --git a/src/ui/Rail.tsx b/src/ui/Rail.tsx index 2618c17..cae46da 100644 --- a/src/ui/Rail.tsx +++ b/src/ui/Rail.tsx @@ -1,6 +1,6 @@ import { createElement, useEffect, useMemo, useRef, useState } from 'react' import { InsertPicker } from '../editors/builtin' -import type { CmsNode, TypeInfo } from '../api' +import type { CmsNode, EvaluateContext, ExperimentAnalytics, ExperimentAssignment, ExperimentRule, TypeInfo } from '../api' import { allowedChildren, connectionsOf, labelKey, sortGroups, typeName } from '../schema' import type { DropTarget } from '../page/dnd' import { Icon, TypeIcon } from './Icon' @@ -47,6 +47,7 @@ export function Rail() { + ) @@ -562,3 +563,144 @@ function NodeSearch() { ) } + +// --- experiments: QA + eligibility on the page (plan-onpage-experiments-personalisation.md P0) --------- + +const expCookie = (name: string): string | null => { + const m = document.cookie.match(new RegExp(`(?:^|; )${name}=([^;]*)`)) + return m ? decodeURIComponent(m[1]) : null +} +const setExpCookie = (name: string, value: string | null) => { + document.cookie = value === null ? `${name}=; expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/` : `${name}=${encodeURIComponent(value)}; path=/; SameSite=Lax` +} +const targetingSummary = (rules?: ExperimentRule['targeting']): string => + !rules?.length ? 'everyone' : rules.map((r) => `${r.field} ${r.op} ${r.value ?? (r.values ?? []).join('|')}`).join(' · ') +const armNames = (rule: ExperimentRule): string[] => Object.keys(rule.variant_weights ?? {}) + +function ExperimentsPanel() { + const ctrl = useCtrl() + const [rules, setRules] = useState(null) + const [adminDenied, setAdminDenied] = useState(false) + const [assign, setAssign] = useState>({}) + const [stats, setStats] = useState>({}) + const [confirmOff, setConfirmOff] = useState(null) + const [tick, setTick] = useState(0) // re-read override cookies after set/clear + const [open, setOpen] = useState(false) + + useEffect(() => { + if (!open) return + let live = true + const ctx: EvaluateContext = { pagePath: location.pathname, referrer: document.referrer || undefined, language: navigator.language || undefined } + void ctrl.api + .trackingConfig() + .then((cfg) => { + if (!live) return + const rs = cfg.experiment_rules ?? [] + setRules(rs) + if (!rs.length) return + void ctrl.api.evaluateExperiments(rs.map((r) => r.experiment_id), ctx).then( + (res) => live && setAssign(Object.fromEntries((res.assignments ?? []).map((a) => [a.experimentId, a]))), + () => {}, // evaluate is public; a failure just leaves assignments blank + ) + void ctrl.api.experimentAnalytics().then( + (rows) => live && setStats(Object.fromEntries((rows ?? []).map((s) => [s.experiment_id, s]))), + () => live && setAdminDenied(true), // admin-gated: eligibility still works, stats hide + ) + }) + .catch(() => live && (setRules([]), setAdminDenied(true))) + return () => void (live = false) + }, [open, ctrl]) + + const toggleDisabled = async (rule: ExperimentRule) => { + if (confirmOff !== rule.experiment_id) return void setConfirmOff(rule.experiment_id) + setConfirmOff(null) + try { + const res = await ctrl.api.setExperimentDisabled(rule.experiment_id, !rule.disabled) + setRules((rs) => rs?.map((r) => (r.experiment_id === rule.experiment_id ? { ...r, disabled: res.disabled } : r)) ?? rs) + ctrl.toast('info', `${rule.experiment_id} ${res.disabled ? 'paused' : 'resumed'}`) + } catch (e) { + ctrl.toast('error', (e as Error).message) + } + } + + return ( +
+

+ +

+ {open && ( +
+ {rules === null &&

Loading experiments…

} + {rules !== null && !rules.length &&

No experiments configured in tracking.

} + {adminDenied && !!rules?.length &&

Tracking admin refused — eligibility still works; stats and pause need an admin credential.

} + {!!rules?.length && ( +
    + {rules.map((rule) => { + const a = assign[rule.experiment_id] + const s = stats[rule.experiment_id] + const arms = armNames(rule) + const cookie = expCookie(a?.cookieName ?? `exp_${rule.experiment_id}`) + void tick + return ( +
  • +
    + {rule.experiment_id} + {rule.disabled ? 'paused' : 'live'} +
    +
    + traffic {Math.round((rule.traffic_fraction ?? 1) * 100)}% · arms {arms.length ? arms.join(' / ') : 'a / b'} · {targetingSummary(rule.targeting)} +
    +
    + {a ? (a.eligible ? <>you: {a.variant || '—'} : 'you: not eligible') : 'you: …'} + {cookie && cookie !== a?.variant && (override: {cookie})} +
    + {s && ( +
    + {s.total_sessions} sessions · winner {s.winner || '—'} + {s.winner ? ` (${Math.round((s.winner_probability ?? 0) * 100)}%)` : ''} + {s.significant ? ' · significant' : ''} — {s.conclusion || 'collecting data'} +
    + )} +
    + + +
    +
  • + ) + })} +
+ )} +

Overrides set the storefront variant cookie — reload the page to preview the arm.

+
+ )} +
+ ) +} diff --git a/src/ui/styles.css b/src/ui/styles.css index f8bfab7..40a77ce 100644 --- a/src/ui/styles.css +++ b/src/ui/styles.css @@ -255,6 +255,20 @@ input.cms-map__key { padding-top: 6px; } .cms-noderow:active { cursor: grabbing; } .cms-noderow__main { flex: 1; min-width: 0; display: flex; flex-direction: column; overflow: hidden; white-space: nowrap; } .cms-pad { padding: 10px 12px 4px; } +.cms-sectoggle { background: none; border: 0; padding: 0; font: inherit; letter-spacing: inherit; text-transform: inherit; color: inherit; cursor: pointer; display: inline-flex; align-items: center; gap: 6px; } +.cms-sectoggle:hover { color: var(--cms-accent); } + +/* experiments QA panel */ +.cms-explist { list-style: none; margin: 8px 0; padding: 0; display: flex; flex-direction: column; gap: 10px; } +.cms-exp { border: 1px solid var(--cms-line); border-radius: 10px; padding: 8px 10px; display: flex; flex-direction: column; gap: 4px; } +.cms-exp__head { display: flex; align-items: center; gap: 8px; justify-content: space-between; } +.cms-exp__head strong { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.cms-pill.is-off { border-color: #a33a4a; color: #ff8a8a; } +.cms-exp__meta { font-size: 12px; } +.cms-exp__assign { font-size: 12px; } +.cms-exp__actions { display: flex; align-items: center; gap: 8px; justify-content: space-between; margin-top: 2px; } +.cms-exp__actions .cms-input { max-width: 150px; } +.cms-exp__actions .cms-ico.is-on { border-color: #a33a4a; color: #ff8a8a; } .cms-hint { font-size: 11.5px; margin-top: 6px; } .cms-outline { list-style: none; margin: 0; padding: 4px; overflow: auto; } .cms-outline li button { display: flex; align-items: center; gap: 8px; width: 100%; background: none; border: 0; border-radius: 6px; padding: 4px 8px; text-align: left; overflow: hidden; white-space: nowrap; } diff --git a/test/experiments.test.tsx b/test/experiments.test.tsx new file mode 100644 index 0000000..f11306e --- /dev/null +++ b/test/experiments.test.tsx @@ -0,0 +1,66 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { act } from 'react' +import { setup } from './harness' +import type { Harness } from './harness' + +let h: Harness +afterEach(() => h?.cleanup()) + +const q = (sel: string) => h.shadow.querySelector(sel) +const qa = (sel: string) => [...h.shadow.querySelectorAll(sel)] + +describe('experiments QA panel', () => { + it('lists experiments with status, eligibility and analytics', async () => { + h = await setup() + await h.settle() + await h.press('Panel') + await h.press('Experiments') + await h.settle() + const rows = qa('.cms-exp') + expect(rows.map((r) => r.querySelector('strong')?.textContent)).toEqual(['exp_hero', 'exp_checkout']) + expect(rows[0].textContent).toContain('live') + expect(rows[1].textContent).toContain('paused') + expect(rows[0].textContent).toContain('traffic 100%') + expect(rows[0].textContent).toContain('arms a / b') + expect(rows[0].textContent).toContain('page_path prefix /') + // Public evaluate: hero eligible (arm a), disabled checkout not. + expect(rows[0].textContent).toContain('you: a') + expect(rows[1].textContent).toContain('not eligible') + // Admin analytics for the live rule. + expect(rows[0].textContent).toContain('250 sessions') + expect(rows[0].textContent).toContain('b') + expect(rows[0].textContent).toContain('significant') + }) + + it('forcing a QA arm sets the storefront variant cookie', async () => { + h = await setup() + await h.settle() + await h.press('Panel') + await h.press('Experiments') + await h.settle() + const sel = q('select[aria-label=\"Force variant for exp_hero\"]') as HTMLSelectElement + const set = Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype, 'value')!.set! + await act(async () => { + set.call(sel, 'b') + sel.dispatchEvent(new Event('change', { bubbles: true })) + }) + expect(document.cookie).toContain('exp_exp_hero=b') + }) + + it('pausing asks to confirm, then flips the rule via PATCH', async () => { + h = await setup() + await h.settle() + await h.press('Panel') + await h.press('Experiments') + await h.settle() + await act(async () => void q('[aria-label=\"Pause exp_hero\"]')!.click()) + await h.settle() + // First click only arms the confirm … + expect(q('[aria-label=\"Pause exp_hero\"]')?.textContent).toContain('confirm') + await act(async () => void q('[aria-label=\"Pause exp_hero\"]')!.click()) + await h.settle() + const rows = qa('.cms-exp') + expect(rows[0].textContent).toContain('paused') + expect(h.cms.requests.some((r) => r === 'PATCH /api/tracking/config/experiments/exp_hero/disabled')).toBe(true) + }) +})