1/**
2 * End-to-end live-mode tests — full click-to-accept cycle.
3 *
4 * For every framework fixture with a `runtime` block in fixture.json, this
5 * runner exercises the entire user-visible chain:
6 *
7 * 1. Stage → install → start live-server + dev server → inject script tag
8 * 2. Open Playwright Chromium, assert the live handshake fires
9 * 3. Spawn a deterministic fake-agent polling loop in this same process
10 * 4. Drive the bar UI: pick element → Go → wait CYCLING → cycle → Accept
11 * 5. Assert source rewrite (variants block, then accepted-only after accept)
12 * 6. Assert DOM reflects the accepted variant via getComputedStyle
13 * 7. Tear down (browser, dev server, agent loop, live-server, tmp)
14 *
15 * The fake agent is pluggable — see tests/live-e2e/agent.mjs. A future
16 * LLM-backed agent slots in by implementing the same VariantAgent interface.
17 *
18 * Run with: bun run test:live-e2e
19 */
20
21import { describe, it, before, after } from 'node:test';
22import assert from 'node:assert/strict';
23import { existsSync, readdirSync, readFileSync } from 'node:fs';
24import { dirname, join } from 'node:path';
25import { fileURLToPath } from 'node:url';
26
27import { createFakeAgent } from './live-e2e/agent.mjs';
28import { createLlmAgent } from './live-e2e/agents/llm-agent.mjs';
29import { bootFixtureSession, FIXTURES_DIR } from './live-e2e/session.mjs';
30import {
31 clickAccept,
32 clickGo,
33 clickNext,
34 getVisibleVariant,
35 pickElement,
36 waitForCycling,
37 waitForHandshake,
38} from './live-e2e/ui.mjs';
39
40const __dirname = dirname(fileURLToPath(import.meta.url));
41
42// Discover fixtures that opt into the runtime E2E pass.
43function listRuntimeFixtures() {
44 const names = readdirSync(FIXTURES_DIR, { withFileTypes: true })
45 .filter((e) => e.isDirectory())
46 .map((e) => e.name);
47
48 const out = [];
49 for (const name of names) {
50 const fixturePath = join(FIXTURES_DIR, name, 'fixture.json');
51 if (!existsSync(fixturePath)) continue;
52 const fixture = JSON.parse(readFileSync(fixturePath, 'utf-8'));
53 if (fixture.runtime) out.push({ name, fixture });
54 }
55 return out;
56}
57
58const allFixtures = listRuntimeFixtures();
59
60// During development of the full-cycle test, a single fixture is much faster
61// to iterate on. Set IMPECCABLE_E2E_ONLY=<name> to scope the run.
62const onlyName = process.env.IMPECCABLE_E2E_ONLY;
63const fixtures = onlyName
64 ? allFixtures.filter((f) => f.name === onlyName)
65 : allFixtures;
66
67if (fixtures.length === 0) {
68 describe('live-e2e (no runtime fixtures registered)', () => {
69 it('is a no-op', () => assert.ok(true));
70 });
71}
72
73let playwright;
74let browser;
75
76before(async () => {
77 if (fixtures.length === 0) return;
78 try {
79 playwright = await import('playwright');
80 } catch (err) {
81 throw new Error(
82 `Playwright is required for live-e2e tests (${err.message}). Run: npx playwright install chromium`,
83 );
84 }
85 try {
86 browser = await playwright.chromium.launch({ headless: true });
87 } catch (err) {
88 throw new Error(`Failed to launch Chromium (${err.message}). Run: npx playwright install chromium`);
89 }
90});
91
92after(async () => {
93 if (browser) await browser.close();
94});
95
96for (const { name, fixture } of fixtures) {
97 describe(`live-e2e · ${name} (${fixture.runtime.styling || 'unknown-styling'})`, () => {
98 it('drives the full click → Go → cycle → accept cycle', async (t) => {
99 // Fixtures may declare `runtime.knownLimitation` to flag a scenario
100 // that exposes a genuine live-mode gap rather than a test bug. The
101 // test still attempts the full chain but does not fail the suite when
102 // the documented failure mode appears — it surfaces the diagnostic so
103 // the limitation is visible in the run output.
104 const knownLimitation = fixture.runtime.knownLimitation;
105
106 // Pick the agent. `IMPECCABLE_E2E_AGENT=llm` opts into the real Claude
107 // API; everything else uses the deterministic fake. Skip rather than
108 // fail when LLM is requested but no API key is set so default suite
109 // runs in unauthenticated environments still pass.
110 const agentMode = process.env.IMPECCABLE_E2E_AGENT || 'fake';
111 let agent;
112 if (agentMode === 'llm') {
113 agent = await createLlmAgent({
114 model: process.env.IMPECCABLE_E2E_LLM_MODEL,
115 log: (m) => t.diagnostic('[llm] ' + m),
116 });
117 if (!agent) {
118 t.skip('IMPECCABLE_E2E_AGENT=llm requires ANTHROPIC_API_KEY');
119 return;
120 }
121 t.diagnostic(`Using LLM agent (model=${process.env.IMPECCABLE_E2E_LLM_MODEL || 'claude-haiku-4-5'})`);
122 } else {
123 agent = createFakeAgent();
124 }
125
126 t.diagnostic(`Booting fixture ${name}`);
127 const session = await bootFixtureSession({
128 name,
129 fixture,
130 browser,
131 agent,
132 log: (m) => t.diagnostic(m),
133 });
134
135 const { page, tmp, consoleErrors, teardown } = session;
136 const expectedCount = 3;
137 const pickSelector = fixture.runtime.pickSelector || 'h1.hero-title';
138
139 try {
140 // 1. Handshake
141 t.diagnostic('Waiting for live handshake');
142 await waitForHandshake(page);
143
144 // 2. preActions — fixtures with hidden/conditional content (modals,
145 // tabs, routes) drive the page into the right state before pick.
146 if (fixture.runtime.preActions) {
147 t.diagnostic(`Running ${fixture.runtime.preActions.length} preAction(s)`);
148 await runPreActions(page, fixture.runtime.preActions);
149 }
150
151 // 3. Pick the target element
152 t.diagnostic(`Picking ${pickSelector}`);
153 await pickElement(page, pickSelector);
154
155 if (process.env.IMPECCABLE_E2E_DEBUG) {
156 const barText = await page.evaluate(() => {
157 const bar = document.querySelector('#impeccable-live-bar');
158 return bar ? { display: bar.style.display, text: bar.textContent || '', html: bar.innerHTML.slice(0, 500) } : null;
159 });
160 t.diagnostic(`Bar after pick: ${JSON.stringify(barText)}`);
161 }
162
163 // 3. Click Go (default action 'impeccable', default count 3 — fixture-stable)
164 t.diagnostic('Clicking Go');
165 await clickGo(page);
166
167 // 4. Wait for the agent's variants to land (HMR + MutationObserver).
168 // For fixtures whose picked element lives inside a conditional
169 // render (modal, tab, route), HMR can remount the parent and lose
170 // the open/active state — the wrapper exists in source but isn't
171 // in the DOM, so MutationObserver never sees it. Live mode now
172 // surfaces a toast asking the user to retrace the path; we mirror
173 // that here by re-running preActions on the first short timeout.
174 //
175 // The first-pass timeout has to be long enough to cover the agent's
176 // generate latency before declaring "state was lost, retrace." A
177 // fake agent finishes in <100ms. The real LLM path usually lands
178 // quickly too, but full-matrix runs can see minute-scale API or
179 // install pressure, so keep this gate patient enough that we do
180 // not retrace while the agent is still writing the variants.
181 t.diagnostic(`Waiting for CYCLING state with ${expectedCount} variants`);
182 const firstPassTimeoutMs = agentMode === 'llm' ? 90_000 : 5_000;
183 let cyclingReached = false;
184 if (fixture.runtime.preActions) {
185 try {
186 await waitForCycling(page, expectedCount, { timeout: firstPassTimeoutMs });
187 cyclingReached = true;
188 } catch {
189 t.diagnostic(`Cycling not reached in ${firstPassTimeoutMs}ms — retracing preActions`);
190 await runPreActions(page, fixture.runtime.preActions);
191 }
192 }
193 try {
194 if (!cyclingReached) {
195 // Default 30s; LLM mode bumps to 90s to absorb API latency on
196 // top of HMR settle time.
197 const finalTimeoutMs = agentMode === 'llm' ? 90_000 : 30_000;
198 await waitForCycling(page, expectedCount, { timeout: finalTimeoutMs });
199 }
200 } catch (err) {
201 if (process.env.IMPECCABLE_E2E_DEBUG) {
202 const variantCount = await page.evaluate(() =>
203 document.querySelectorAll('[data-impeccable-variant]').length,
204 );
205 const barInfo = await page.evaluate(() => {
206 const bars = document.querySelectorAll('#impeccable-live-bar');
207 return {
208 count: bars.length,
209 bars: [...bars].map((bar) => ({
210 display: bar.style.display,
211 opacity: bar.style.opacity,
212 text: bar.textContent || '',
213 innerHtml: bar.innerHTML.slice(0, 600),
214 })),
215 __init: window.__IMPECCABLE_LIVE_INIT__,
216 };
217 });
218 t.diagnostic(`waitForCycling failed; variants in DOM: ${variantCount}`);
219 t.diagnostic(`Bar state: ${JSON.stringify(barInfo)}`);
220 t.diagnostic(`--- dev server tail ---\n${session.dev.log()}`);
221 }
222 throw err;
223 }
224
225 // 5. Source-side check: wrapper + style + variants are present
226 const sourceFile = await locateSessionFile(tmp);
227 const after = readFileSync(sourceFile, 'utf-8');
228 assert.match(after, /data-impeccable-variants="/, 'wrapper inserted');
229 if (sourceFile.endsWith('.astro')) {
230 assert.match(after, /<style is:inline data-impeccable-css="/, 'Astro live CSS uses an inline compiler-bypassing style block');
231 assert.match(
232 after,
233 /\[data-impeccable-variant="1"\]\s*>\s*(?:h1|\.[\w-]+)/,
234 'event=live_e2e.astro_css_prefix actor=agent operation=write_variants risk=astro_scopes_preview_css_away expected=variant-prefixed global selector actual=missing suggestion=inspect fake agent styleMode handling',
235 );
236 assert.doesNotMatch(after, /@scope \(\[data-impeccable-variant="1"\]\)/, 'Astro live CSS does not use raw @scope');
237 } else {
238 assert.match(after, /<style data-impeccable-css="/, 'colocated <style> block present');
239 assert.match(after, /@scope \(\[data-impeccable-variant="1"\]\)/, 'scoped CSS for variant 1');
240 assert.match(after, /@scope \(\[data-impeccable-variant="2"\]\)/, 'scoped CSS for variant 2');
241 assert.match(after, /@scope \(\[data-impeccable-variant="3"\]\)/, 'scoped CSS for variant 3');
242 }
243 // Param manifest assertions are scoped to fake-agent mode. The fake
244 // agent deterministically emits one param per variant covering all
245 // three kinds; the LLM agent is non-deterministic and may legitimately
246 // emit no params per the live.md spec ("variants are fixed points").
247 if (agentMode === 'fake') {
248 assert.match(after, /data-impeccable-params=/, 'data-impeccable-params manifest emitted');
249 for (const kind of ['range', 'steps', 'toggle']) {
250 assert.match(after, new RegExp(`"kind"\\s*:\\s*"${kind}"`), `param kind ${kind} present`);
251 }
252 }
253
254 // 6. Cycle to variant 2 (the bold one in the fake agent)
255 t.diagnostic('Cycling to variant 2');
256 await clickNext(page);
257 const visible = await getVisibleVariant(page);
258 assert.equal(visible, 2, 'variant 2 visible after one Next');
259 if (agentMode === 'fake') {
260 await page.waitForFunction(() => {
261 const h1 = document.querySelector('[data-impeccable-variant="2"] > h1');
262 return h1 && getComputedStyle(h1).fontWeight === '900';
263 }, null, { timeout: 5_000 }).catch(() => {});
264 const variantWeight = await page.evaluate(() => {
265 const h1 = document.querySelector('[data-impeccable-variant="2"] > h1');
266 return h1 ? getComputedStyle(h1).fontWeight : null;
267 });
268 assert.equal(
269 variantWeight,
270 '900',
271 'event=live_e2e.variant_css_applied actor=browser operation=render_visible_variant risk=unstyled_live_preview expected=font-weight 900 actual=' + variantWeight + ' suggestion=inspect live CSS style mode and selector shape',
272 );
273 }
274
275 // 7. Accept variant 2
276 t.diagnostic('Accepting variant 2');
277 await clickAccept(page, { expectedVariant: 2 });
278
279 // 8. Wait for live-accept + the agent's carbonize cleanup to land.
280 // File-side: wrapper, all variants, and carbonize markers gone;
281 // only the accepted inner element survives.
282 t.diagnostic('Waiting for accept + carbonize cleanup to land');
283 const final = await waitForSourceClean(sourceFile, 20_000);
284 assert.doesNotMatch(final, /data-impeccable-variants="/, 'variants wrapper removed');
285 assert.doesNotMatch(final, /impeccable-variants-start/, 'variants-start marker removed');
286 assert.doesNotMatch(final, /impeccable-carbonize-start/, 'carbonize-start marker removed');
287 assert.doesNotMatch(final, /impeccable-carbonize-end/, 'carbonize-end marker removed');
288 assert.doesNotMatch(final, /data-impeccable-variant="/, 'no leftover variant scaffolding');
289 // Accept the original class as a substring of the className value so
290 // an LLM agent that adds classes around the original (e.g.
291 // class="hero-title bold red") still passes — only the literal
292 // class="hero-title" form would otherwise match.
293 assert.match(
294 final,
295 /<h1[^>]*(class|className)="[^"]*\bhero-title\b[^"]*"/,
296 'accepted h1 survives with hero-title class',
297 );
298
299 // Optional fixture hook: assert that arbitrary strings survive the
300 // wrap → accept → carbonize cycle. Used by repeated-branch fixtures
301 // to prove wrap disambiguated correctly — sibling branches the test
302 // didn't pick should be untouched.
303 if (Array.isArray(fixture.runtime.assertSourceContains)) {
304 for (const needle of fixture.runtime.assertSourceContains) {
305 assert.ok(
306 final.includes(needle),
307 `source still contains ${JSON.stringify(needle)} after accept (sibling branch must not be rewritten)`,
308 );
309 }
310 }
311
312 // 9. DOM-side: at least one matching element, none inside any wrapper.
313 await page.waitForFunction(
314 (sel) => {
315 const all = document.querySelectorAll(sel);
316 if (all.length < 1) return false;
317 for (const el of all) {
318 if (el.closest('[data-impeccable-variants],[data-impeccable-variant]')) return false;
319 }
320 return true;
321 },
322 pickSelector,
323 { timeout: 20_000 },
324 );
325
326 // 9b. reloadProbe — fixtures with conditional render assert that the
327 // accepted variant survives a full page reload. The picked element
328 // may be hidden by default (closed modal, non-default tab); the
329 // probe re-runs preActions to bring it back into the DOM.
330 if (fixture.runtime.reloadProbe) {
331 t.diagnostic('Running reloadProbe (reload + reach + assert)');
332 await page.reload({ waitUntil: 'domcontentloaded' });
333 if (fixture.runtime.reloadProbe.preActions) {
334 await runPreActions(page, fixture.runtime.reloadProbe.preActions);
335 }
336 const expectSelector = fixture.runtime.reloadProbe.expectSelector || pickSelector;
337 await page.waitForSelector(expectSelector, { timeout: 10_000 });
338 }
339
340 // 10. Console hygiene — no errors during the whole flow.
341 if (fixture.runtime.probe?.expectConsoleClean) {
342 const realErrors = consoleErrors.filter((e) =>
343 !/(Download the React DevTools|StrictMode|Failed to load resource: the server responded with a status of 404)/i.test(e),
344 );
345 if (realErrors.length > 0) {
346 t.diagnostic('--- console errors ---');
347 for (const e of realErrors) t.diagnostic(e);
348 t.diagnostic('--- final source ---');
349 t.diagnostic(readFileSync(sourceFile, 'utf-8'));
350 }
351 assert.equal(
352 realErrors.length,
353 0,
354 `expected clean console, got:\n${realErrors.join('\n')}`,
355 );
356 }
357 } catch (err) {
358 if (knownLimitation) {
359 t.diagnostic(`KNOWN LIMITATION: ${knownLimitation}`);
360 t.diagnostic(`Failure: ${err.message?.split('\n')[0] || err}`);
361 t.skip(`known limitation: ${knownLimitation}`);
362 return;
363 }
364 throw err;
365 } finally {
366 await teardown();
367 }
368 });
369 });
370}
371
372// ---------------------------------------------------------------------------
373// Helpers
374// ---------------------------------------------------------------------------
375
376/**
377 * Drive a list of pre-pick / reload-probe actions. Used to set up tricky
378 * scenarios: open a modal, switch tabs, navigate routes.
379 *
380 * Live mode's element picker intercepts every page click in capture phase
381 * while `pickActive === true`, so any action that depends on the page's own
382 * click handler (open a modal, switch a tab) gets swallowed. We bracket the
383 * action sequence with two clicks of the global bar's pick toggle and leave
384 * the picker in its original state once preActions complete.
385 *
386 * Supported action shapes:
387 * { "type": "click", "selector": "..." }
388 * { "type": "goto", "path": "/about" }
389 * { "type": "wait", "selector": "..." }
390 */
391async function runPreActions(page, actions) {
392 const PICK_TOGGLE = '#impeccable-live-pick-toggle';
393 const pickerToggle = await page.$(PICK_TOGGLE);
394 const wasActive = pickerToggle
395 ? await pickerToggle.evaluate((el) => el.dataset.active === 'true')
396 : false;
397 if (wasActive) await clickPickToggle(page, PICK_TOGGLE);
398
399 try {
400 for (let i = 0; i < actions.length; i++) {
401 const a = actions[i];
402 if (a.type === 'click') {
403 const next = actions[i + 1];
404 if (next?.type === 'wait') {
405 const alreadyVisible = await page.locator(next.selector).first().isVisible().catch(() => false);
406 if (alreadyVisible) continue;
407 }
408 const loc = page.locator(a.selector);
409 await loc.first().waitFor({ state: 'visible', timeout: 5_000 });
410 await loc.first().click();
411 continue;
412 }
413 if (a.type === 'goto') {
414 const target = new URL(a.path, page.url()).href;
415 await page.goto(target, { waitUntil: 'domcontentloaded', timeout: 10_000 });
416 continue;
417 }
418 if (a.type === 'wait') {
419 await page.waitForSelector(a.selector, { timeout: 5_000 });
420 continue;
421 }
422 throw new Error(`unknown preAction type: ${a.type}`);
423 }
424 } finally {
425 if (wasActive) {
426 // Re-arm the picker. If the page navigated mid-action the toggle may
427 // belong to a freshly mounted bar — best-effort, no throw.
428 const after = await page.$(PICK_TOGGLE);
429 if (after) {
430 const isActive = await after.evaluate((el) => el.dataset.active === 'true');
431 if (!isActive) await clickPickToggle(page, PICK_TOGGLE);
432 }
433 }
434 }
435}
436
437async function clickPickToggle(page, selector) {
438 try {
439 await page.locator(selector).click({ timeout: 5_000 });
440 return;
441 } catch (err) {
442 const clicked = await page.evaluate((sel) => {
443 const btn = document.querySelector(sel);
444 if (!btn) return false;
445 btn.click();
446 return true;
447 }, selector);
448 if (!clicked) throw err;
449 }
450}
451
452/**
453 * Poll the file until carbonize cleanup has landed: no variants wrapper, no
454 * carbonize markers, no leftover variant divs. Returns the final contents.
455 */
456async function waitForSourceClean(filePath, timeoutMs) {
457 const start = Date.now();
458 let last = '';
459 while (Date.now() - start < timeoutMs) {
460 last = readFileSync(filePath, 'utf-8');
461 const dirty =
462 last.includes('data-impeccable-variants=') ||
463 last.includes('impeccable-variants-start') ||
464 last.includes('impeccable-carbonize-start') ||
465 last.includes('data-impeccable-variant=');
466 if (!dirty) return last;
467 await new Promise((r) => setTimeout(r, 100));
468 }
469 throw new Error(`source not clean after ${timeoutMs}ms — last contents:\n${last}`);
470}
471
472/**
473 * Find the source file that received the wrapper. We look for any tracked
474 * file containing the variants marker — the agent always writes to exactly
475 * one file per session.
476 */
477async function locateSessionFile(tmp) {
478 const candidates = walkSources(tmp);
479 for (const f of candidates) {
480 const body = readFileSync(f, 'utf-8');
481 if (
482 body.includes('data-impeccable-variants=') ||
483 body.includes('impeccable-carbonize-start') ||
484 body.includes('impeccable-variants-start')
485 ) {
486 return f;
487 }
488 }
489 throw new Error('Could not locate session source file under ' + tmp);
490}
491
492function walkSources(root) {
493 const results = [];
494 const stack = [root];
495 const SKIP = new Set(['node_modules', '.git', '.svelte-kit', 'dist', '.vite', 'build', '.next']);
496 const EXTS = ['.html', '.jsx', '.tsx', '.svelte', '.astro', '.vue'];
497 while (stack.length) {
498 const dir = stack.pop();
499 let entries;
500 try { entries = readdirSync(dir, { withFileTypes: true }); } catch { continue; }
501 for (const e of entries) {
502 const full = join(dir, e.name);
503 if (e.isDirectory()) {
504 if (!SKIP.has(e.name)) stack.push(full);
505 continue;
506 }
507 if (EXTS.some((x) => e.name.endsWith(x))) results.push(full);
508 }
509 }
510 return results;
511}