/** * `vgai probe` — Task 3.4/3.5 of docs/ACCEPTANCE-DRIVER-BUILD-PLAN.md, run * sequence per docs/ACCEPTANCE-DRIVER-SPEC.md §2.2. Doctor-shaped (same * harness idioms as `run-doctor.ts`: deadline-clamped waits, CLI-owned * headless Chromium + SwiftShader flags, `stopProcess` for teardown), but * answers a different question: doctor asks "does this project mount in the * editor?"; probe asks "do this game's acceptance specs * (`tests/acceptance/*.spec.ts`) pass on the standalone game page?" * * Run sequence (each step's own exit path; exit codes below): * 1. zero-specs usage check -> `checkZeroSpecs` * 2. `tsc --noEmit -p tests/tsconfig.json` -> `runAcceptanceTypecheck` * 3. pre-boot lint (4 bans + D16) -> `lintAcceptanceSpecs` * 4. boot/reuse the standalone game server * (+ the project's Colyseus room, if declared) -> `resolveServers` * 5. generate the Playwright config -> `writeProbeConfig` * 6. run the suite, watch for a harness wedge -> `runPlaywrightSuite` * 7. read back `last-run.json`, print, propagate the exit code * * Exit codes: 0 all pass · 1 spec failure (including step 2's type errors) · * 2 usage (zero specs, pre-boot lint, bad args) · 3 missing capability (the * page loaded but `window.__vgai` never appeared) · 5 harness wedge (the * spawned `playwright test` child stopped making progress). * * Invoked via `npx tsx run-probe.ts [file] [-g ] * [--init] [--last-failure] [--junit] [--fresh] [--json] [--headed] [--headless] * [--headed-auto] [--live-session-hint]` from the repo root * (`packages/vgai-cli/src/index.ts` spawns this exact script, mirroring * `runDoctor`/`runConformance`/`runBundle`) — `` is always * supplied by the CLI wrapper: the user's optional `[folder]` positional. * `--headed-auto`/`--live-session-hint` are CLI-INTERNAL plumbing (#125 item * 1, `ProbeCliArgs`'s own doc comments) — never part of the public `vgai * probe` flag surface a person types. * resolved against their OWN cwd, defaulting to the cwd itself when absent * (the `npm test` convention), so argv[0] here is always the project folder * and any later positional is a Playwright passthrough. * * KNOWN GAPS (disclosed, not silently narrowed): * - The renderer-wedge/hidden-tab watchdog hollowstone's `bot-probe.mjs` * implements assumes a LIVE Page/Browser handle (it launches Chromium * itself). This runner instead spawns `npx playwright test` as a child * process and never holds a page handle of its own — the fixture that * does (`@vgai/probe`'s `fixture.ts`) is another package's territory. * `runPlaywrightSuite` below therefore adapts the SEMANTICS (2 silent * 90s windows -> kill -> exit 5) onto the one signal available at this * layer: the child's stdout/stderr liveness. True page-level recovery * (bringToFront, an eval-race health check) is not implemented here. * - Full per-test fps/frame-time-percentile/sim-speed perf capture needs * page-side instrumentation in `@vgai/probe`'s fixture teardown (also not * this package's territory) — `probe-reporter.ts` honestly records only * wall-clock duration per test, never a fabricated number. * - `packages/editor/scripts/*.ts` is not covered by any `tsc` project * today (verified: neither `tsconfig.json` nor `tsconfig.test.json` * `include`s `scripts/`, matching the pre-existing `run-doctor.ts`/ * `run-conformance.ts` — this is a pre-existing gap, not one this file * introduces). */ import { type ChildProcess, spawn } from 'node:child_process'; import { createHash } from 'node:crypto'; import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, statSync, writeFileSync, } from 'node:fs'; import { createRequire } from 'node:module'; import net from 'node:net'; import { homedir } from 'node:os'; import { join, relative, resolve, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; import type { ResolvedGameManifest } from '@vgai/engine/manifest/load'; import { loadGameManifestFile } from '@vgai/engine/manifest/load-file'; import { stopProcess } from '../server/process-shutdown'; import { type EditorSession, liveSessions, registerSession, unregisterSession, } from '../server/session-registry'; import { allFailuresAreMissingCapability, buildWedgeEnvelope, type LastRunEnvelope, type ProbeFailureRecord, } from './probe-reporter'; // --------------------------------------------------------------------------- // Exit codes (docs/ACCEPTANCE-DRIVER-SPEC.md §2.2) // --------------------------------------------------------------------------- export const PROBE_EXIT_CODES = { PASS: 0, SPEC_FAILURE: 1, USAGE: 2, MISSING_CAPABILITY: 3, HARNESS_WEDGE: 5, } as const; /** Human-readable meaning for each code above — the SAME wording as the * `vgai probe --help` / global `--help` exit-code table (kept in sync by * hand; both are short and rarely change). Fed to `describeExitCode`, * which `main()` uses to print a final `vgai probe: exit ()` * line on any non-zero exit (Wave-6.1 dry-run friction: a bare non-zero * code with no reminder of what it means sent people back to the docs). */ const EXIT_CODE_MEANINGS: Record = { [PROBE_EXIT_CODES.PASS]: 'all specs passed', [PROBE_EXIT_CODES.SPEC_FAILURE]: 'a spec failed, including a tests/ type error', [PROBE_EXIT_CODES.USAGE]: 'usage error, including zero specs', [PROBE_EXIT_CODES.MISSING_CAPABILITY]: 'the page loaded but no debug bridge appeared', [PROBE_EXIT_CODES.HARNESS_WEDGE]: 'the harness stopped making progress (renderer/test-runner wedge)', }; /** Pure — maps an exit code to its documented meaning; an unrecognized code * (should never happen — defensive only) still names itself rather than * throwing. */ export function describeExitCode(code: number): string { return EXIT_CODE_MEANINGS[code] ?? `unrecognized exit code ${code}`; } /** Sets `process.exitCode` and — on any NON-zero code — prints a final * `vgai probe: exit ()` line so a failure is never left to a * bare number (Wave-6.1 dry-run friction #3). Centralized here so every * exit path in `main()` gets the line for free. */ function setExit(code: number): void { process.exitCode = code; if (code !== 0) { console.error(`vgai probe: exit ${code} (${describeExitCode(code)})`); } } export class ProbeUsageError extends Error {} export const PROBE_USAGE_TEXT = 'Usage: run-probe.ts [file] [-g ] [--init] [--last-failure] ' + '[--run-file ] [--junit] [--fresh] [--json] [--headed] [--headless]'; // --------------------------------------------------------------------------- // Arg parsing // --------------------------------------------------------------------------- export interface ProbeCliArgs { projectDir: string; init: boolean; lastFailure: boolean; /** * `runs/` archive (Wave-6 findings ledger) — an EXPLICIT, always-value * `--run-file ` flag naming one of the archived `.vgai/probe/runs/ * -.json` envelopes to render instead of the * single-slot `last-run.json`. Deliberately a SEPARATE flag from * `--last-failure` rather than an optional positional after it — an * earlier design let `--last-failure` optionally swallow the next bare * token as a run-file path, which silently ate the `[projectDir]` * positional on the common `vgai probe --last-failure ` shape * (flag-before-project). A `-g`-style "peek at the next token" trick only * works when the flag's value is UNAMBIGUOUSLY distinguishable from every * other positional; a run-file path is not. `--run-file` always consumes * its next argv token, full stop — same discipline as `-g ` * below, but with no ambiguity to resolve. Resolved against `projectDir` * when relative (mirrors how `projectDir` itself is always resolved here). */ runFile?: string; junit: boolean; fresh: boolean; json: boolean; /** Launch a VISIBLE (non-headless) browser so a human can watch the * synthetic player play the game live — same runner, same suite, just * `headless: false` in the generated config (see `buildProbeConfigSource`) * and no headless-shell executablePath resolution (headed needs full * chromium, resolved via Playwright's own default). */ headed: boolean; /** * #125 item 1 — CLI-INTERNAL: true when `headed` was chosen by the CLI's * default gate (`resolveProbeHeadedMode`, `@vgai/cli`'s * `probe-headed-gate.ts`) rather than an explicit `--headed` typed by a * person. Not part of the public `vgai probe` flag surface — only * `packages/vgai-cli/src/index.ts` ever sets this when it spawns this * script. Governs whether a "no display" launch failure hard-fails (an * explicit request — unchanged pre-#125 behavior) or silently retries * headless (a default nobody typed must never fail the run for display * reasons — see `main()`'s headed-fallback handling below). */ headedAuto: boolean; /** Explicit `--headless` opt-out (#125): forces headless even if `headed`/ * `headedAuto` were also (mistakenly) set, and suppresses the "you could * have watched this" tip below. Also settable by a human directly. */ headless: boolean; /** * #125 item 1 — CLI-INTERNAL: true when the CLI found a live editor * session for this project when it decided the launch mode. Used only to * decide whether to print the "re-run with --headed to watch" tip when the * run ends up headless. Not part of the public flag surface. */ liveSessionHint: boolean; /** * #140 — play the spec INSIDE the already-live editor session's tab * (`vgai probe --in-editor`) instead of booting a standalone vite+browser: * skips step 4 (`resolveServers`) entirely and drives the suite over * `@vgai/probe`'s `RelayTransport` against `sessionPort` below. Mutually * exclusive with `headed`/`headless` in practice (there is no browser this * runner itself launches to make headed/headless), enforced by the CLI * wrapper, not re-validated here. */ inEditor: boolean; /** * #140 — CLI-INTERNAL: the live editor session's dev-server port, set by * `packages/vgai-cli/src/index.ts` whenever it passes `inEditor`. Not part * of the public flag surface a person types (mirrors `headedAuto`/ * `liveSessionHint`'s own "CLI-internal" convention above). */ sessionPort?: number; /** `[file]` / `-g ` — forwarded to `npx playwright test` untouched. * #146 `--bail` is the one mapped flag: it becomes a leading * `--max-failures=1` here at parse time (single point of truth — every * `runPlaywrightSuite` call site forwards `playwrightArgs` as-is, so the * mapping rides into the headed-fallback retry and the in-editor run * without each call site knowing about it). */ playwrightArgs: string[]; /** * D15/T-D15.6 (objection-4 fix) — `--seed `: bakes a deterministic seed * into every spec this run drives, via `VGAI_PROBE_SEED` (same * env-var-into-the-suite pattern as `VGAI_PROBE_URL`/`VGAI_PROBE_WARM` * below) — `@vgai/probe`'s `fixture.ts` appends `&vgai-seed=` to the * game URL alongside `?vgai-debug=1`, reaching the SAME * `resolveDeterminismSeed` precedence a standalone/editor boot already * honors (requires the project to declare `determinism.seededRandom`). * `undefined` (the default) leaves the manifest/no-seed precedence * untouched. */ seed: number | undefined; } /** Pure — `argv` is the script's own `process.argv.slice(2)` (or an * equivalent test fixture): `argv[0]` is always the project folder (the CLI * wrapper resolves it from the user's cwd before spawning this script). */ export function parseProbeArgs(argv: string[]): ProbeCliArgs { const folder = argv[0]; if (!folder) { throw new ProbeUsageError('run-probe.ts: missing argument'); } let init = false; let lastFailure = false; let runFile: string | undefined; let junit = false; let fresh = false; let json = false; let headed = false; let headedAuto = false; let headless = false; let liveSessionHint = false; let inEditor = false; let sessionPort: number | undefined; let seed: number | undefined; let bail = false; const playwrightArgs: string[] = []; for (let i = 1; i < argv.length; i++) { const a = argv[i]!; if (a === '--init') init = true; else if (a === '--last-failure') lastFailure = true; else if (a === '--run-file') { const value = argv[++i]; if (value === undefined) throw new ProbeUsageError('--run-file requires a path argument'); runFile = value; } else if (a === '--junit') junit = true; else if (a === '--fresh') fresh = true; else if (a === '--json') json = true; else if (a === '--headed') headed = true; else if (a === '--headed-auto') headedAuto = true; else if (a === '--headless') headless = true; else if (a === '--live-session-hint') liveSessionHint = true; else if (a === '--in-editor') inEditor = true; else if (a === '--bail') bail = true; else if (a === '--session-port') { const raw = argv[++i]; if (raw === undefined) throw new ProbeUsageError('--session-port requires a value'); sessionPort = Number(raw); if (!Number.isFinite(sessionPort)) { throw new ProbeUsageError(`--session-port expects a number, got: ${raw}`); } } else if (a === '--seed') { const raw = argv[++i]; const n = raw === undefined ? Number.NaN : Number(raw); if (!Number.isFinite(n)) { throw new ProbeUsageError(`--seed expects a number, got: ${raw ?? '(nothing)'}`); } seed = n; } else if (a === '-g') { const pattern = argv[++i]; if (pattern === undefined) throw new ProbeUsageError('-g requires a pattern argument'); playwrightArgs.push('-g', pattern); } else { playwrightArgs.push(a); } } return { projectDir: resolve(folder), init, lastFailure, ...(runFile !== undefined ? { runFile } : {}), junit, fresh, json, headed, headedAuto, headless, liveSessionHint, inEditor, ...(sessionPort !== undefined ? { sessionPort } : {}), // #146 `--bail`: stop at the first failing spec. Mapped to Playwright's // own `--max-failures=1` HERE (see the `playwrightArgs` doc comment). playwrightArgs: bail ? ['--max-failures=1', ...playwrightArgs] : playwrightArgs, seed, }; } // --------------------------------------------------------------------------- // Step 1 — zero-specs usage check // --------------------------------------------------------------------------- export const ZERO_SPECS_MESSAGE = 'acceptance: 0 specs in tests/acceptance/ — this game has no bot-proven features. ' + "Scaffold the worked example with 'vgai probe --init'."; /** Recursive walk of `/tests/acceptance` for `*.spec.ts` files — * `[]` when the directory is absent (never throws on a missing dir). */ export function findAcceptanceSpecFiles(projectDir: string): string[] { const dir = join(projectDir, 'tests', 'acceptance'); if (!existsSync(dir) || !statSync(dir).isDirectory()) return []; const out: string[] = []; const walk = (d: string): void => { for (const entry of readdirSync(d, { withFileTypes: true })) { const full = join(d, entry.name); if (entry.isDirectory()) walk(full); else if (entry.isFile() && entry.name.endsWith('.spec.ts')) out.push(full); } }; walk(dir); return out.sort(); } export interface ZeroSpecsCheck { zero: boolean; specFiles: string[]; } export function checkZeroSpecs(projectDir: string): ZeroSpecsCheck { const specFiles = findAcceptanceSpecFiles(projectDir); return { zero: specFiles.length === 0, specFiles }; } // --------------------------------------------------------------------------- // Step 2 — `tsc --noEmit -p tests/tsconfig.json` // --------------------------------------------------------------------------- export interface TypecheckResult { ok: boolean; output: string; } /** Spawns the project's OWN installed `typescript` (via `npx tsc`, `cwd: * projectDir`) against `tests/tsconfig.json` — AC-B1.5: a type-broken spec * fails in seconds, before any browser boots. */ export async function runAcceptanceTypecheck(projectDir: string): Promise { const tsconfigPath = join(projectDir, 'tests', 'tsconfig.json'); return new Promise((resolvePromise) => { const child = spawn('npx', ['tsc', '--noEmit', '-p', tsconfigPath], { cwd: projectDir, stdio: ['ignore', 'pipe', 'pipe'], shell: process.platform === 'win32', }); let output = ''; child.stdout?.on('data', (d: Buffer) => { output += d.toString(); }); child.stderr?.on('data', (d: Buffer) => { output += d.toString(); }); child.on('close', (code) => resolvePromise({ ok: code === 0, output })); child.on('error', (err) => resolvePromise({ ok: false, output: `${output}\n${err.message}` })); }); } // --------------------------------------------------------------------------- // Step 3 — pre-boot lint: the four bans + D16 // --------------------------------------------------------------------------- export type LintRule = | 'wait-for-timeout' | 'networkidle' | 'raw-goto' | 'command-only' | 'wall-clock-budget'; export interface LintViolation { file: string; line: number; rule: LintRule; message: string; } const WAIT_FOR_TIMEOUT_RE = /\.waitForTimeout\s*\(/; const NETWORKIDLE_RE = /networkidle/; const RAW_GOTO_RE = /\bpage\.goto\s*\(/; // A blind validation run re-introduced `test.setTimeout(120_000)` in its own // specs under 0.2x sim speed (SwiftShader has no GPU) — the runner's own // generated config sets `timeout: 0` for exactly this reason (D1, above), but // Playwright's PER-TEST `test.setTimeout`/`test.slow` override that from // inside the spec file itself, where this pre-boot lint hadn't been looking. const TEST_SET_TIMEOUT_RE = /\btest\.setTimeout\s*\(/; const TEST_SLOW_RE = /\btest\.slow\s*\(/; const WALL_CLOCK_BUDGET_MESSAGE = 'banned: wall-clock test budgets fight the sim clock (the run may be at 0.2x under ' + 'SwiftShader); budgets belong in { simSeconds } — the runner already disables ' + "Playwright's wall timeout"; const GAME_INPUT_RE = /\bgame\.input\b/; const GAME_WAIT_FOR_RE = /\bgame\.waitFor\s*\(/; const GAME_EVENTS_EXPECT_RE = /\bgame\.events\.expect\s*\(/; const GAME_PAGE_RE = /\bgame\.page\s*\(/; const GAME_COMMAND_OR_SCREENSHOT_RE = /\bgame\.(?:command|screenshot)\s*\(/; /** Pure — lints one spec file's already-read source text. Per-line bans get * their own violation per matching line; D16 ("commands are fixtures, never * proofs") is a file-level check per the build plan's own wording ("a spec * file whose only interactions are commands"). */ export function lintSpecFile(filePath: string, contents: string): LintViolation[] { const violations: LintViolation[] = []; const lines = contents.split('\n'); lines.forEach((lineText, idx) => { const line = idx + 1; if (WAIT_FOR_TIMEOUT_RE.test(lineText)) { violations.push({ file: filePath, line, rule: 'wait-for-timeout', message: 'banned: sleeps wall-clock while the game runs sim-time; use game.waitFor(pred, { simSeconds })', }); } if (NETWORKIDLE_RE.test(lineText)) { violations.push({ file: filePath, line, rule: 'networkidle', message: 'banned: hangs forever against Vite dev servers; the game fixture owns page.goto', }); } if (RAW_GOTO_RE.test(lineText)) { violations.push({ file: filePath, line, rule: 'raw-goto', message: 'banned: the game fixture owns navigation', }); } if (TEST_SET_TIMEOUT_RE.test(lineText) || TEST_SLOW_RE.test(lineText)) { violations.push({ file: filePath, line, rule: 'wall-clock-budget', message: WALL_CLOCK_BUDGET_MESSAGE, }); } }); const hasProofCall = GAME_INPUT_RE.test(contents) || GAME_PAGE_RE.test(contents) || GAME_WAIT_FOR_RE.test(contents) || GAME_EVENTS_EXPECT_RE.test(contents); const hasCommandOrScreenshot = GAME_COMMAND_OR_SCREENSHOT_RE.test(contents); if (hasCommandOrScreenshot && !hasProofCall) { const firstMatchLine = lines.findIndex((l) => GAME_COMMAND_OR_SCREENSHOT_RE.test(l)); violations.push({ file: filePath, line: firstMatchLine >= 0 ? firstMatchLine + 1 : 1, rule: 'command-only', message: 'this spec only issues debug commands — commands are fixtures, never proofs (D16); ' + 'drive input and assert observable state', }); } return violations; } export function lintAcceptanceSpecs(specFiles: string[]): LintViolation[] { const violations: LintViolation[] = []; for (const file of specFiles) { violations.push(...lintSpecFile(file, readFileSync(file, 'utf-8'))); } return violations; } export function formatLintViolation(v: LintViolation, projectDir: string): string { return `${relative(projectDir, v.file)}:${v.line}: ${v.message}`; } // --------------------------------------------------------------------------- // Step 5 — the generated Playwright config // --------------------------------------------------------------------------- export interface ProbeConfigOptions { projectDir: string; testDir: string; reporterModulePath: string; lastRunJsonPath: string; artifactsDir: string; junitOutputPath?: string; chromiumExecutablePath?: string; /** `--headed`: emits `use.headless: false` so a human can watch the run — * video/artifacts/SwiftShader launch args stay identical either way. */ headed?: boolean; } /** Pure — returns the generated config's TS SOURCE TEXT (never a live * object): `workers: 1` enforced, reporters `[['list'], [], * ...(junit)]` — `'html'` never appears (it blocks the terminal — a wedged * Bash call to an agent, per the spec's own reasoning). */ export function buildProbeConfigSource(opts: ProbeConfigOptions): string { const reporterOptions = { outputFile: opts.lastRunJsonPath, projectDir: opts.projectDir, outputDir: opts.artifactsDir, }; const reporterEntries = [ "['list']", `[${JSON.stringify(opts.reporterModulePath)}, ${JSON.stringify(reporterOptions)}]`, ]; if (opts.junitOutputPath) { reporterEntries.push(`['junit', ${JSON.stringify({ outputFile: opts.junitOutputPath })}]`); } const launchOptions: { args: string[]; executablePath?: string } = { // The three SwiftShader flags this checkout's `run-doctor.ts` is missing // the third of — a newer headless-shell needs it (build plan Task 3.4 anchors). args: ['--use-gl=angle', '--use-angle=swiftshader', '--enable-unsafe-swiftshader'], }; if (opts.chromiumExecutablePath) launchOptions.executablePath = opts.chromiumExecutablePath; return `// GENERATED by \`vgai probe\` — do not edit by hand; regenerated every run. import { defineConfig } from '@playwright/test'; export default defineConfig({ testDir: ${JSON.stringify(opts.testDir)}, fullyParallel: false, workers: 1, // D1: Playwright's own per-test wall-clock timeout defaults to 30s, which // silently caps every sim-time \`waitFor\`/\`hold\` budget — a test with a // genuinely generous sim-time budget (SwiftShader can run well under 1x) // would die as a raw 'Test timeout of 30000ms exceeded' with none of this // package's own failure-block rendering. There is no wall-clock budget to // protect here BY DESIGN (see wait-for.ts's WAIT_FOR_TIMEOUT_OPTION_MESSAGE) // — the backstops are this runner's own renderer-wedge watchdog (2x90s, // above) and wait-for.ts's internal frozen-clock stall guard (D2). timeout: 0, reporter: [ ${reporterEntries.map((e) => ` ${e},`).join('\n')} ], outputDir: ${JSON.stringify(opts.artifactsDir)}, use: { video: 'retain-on-failure', launchOptions: ${JSON.stringify(launchOptions)},${ opts.headed ? ` // --headed: a human is watching this run live — SwiftShader launch args // above are harmless headed (they only pick the GL backend) and stay on. headless: false,` : '' } }, }); `; } export function computeConfigHash(opts: ProbeConfigOptions): string { return createHash('sha1').update(JSON.stringify(opts)).digest('hex').slice(0, 8); } /** Writes the generated config to `/.vgai/probe/config..ts` * and returns its absolute path. */ export function writeProbeConfig(opts: ProbeConfigOptions): string { const dir = join(opts.projectDir, '.vgai', 'probe'); mkdirSync(dir, { recursive: true }); const configPath = join(dir, `config.${computeConfigHash(opts)}.ts`); writeFileSync(configPath, buildProbeConfigSource(opts)); return configPath; } /** * Playwright preflights custom ESM reporters by importing a virtual * `.esm.preflight` sibling. Node rejects that import before * Playwright's TypeScript loader can see it when the reporter lives under * `node_modules` (`ERR_UNSUPPORTED_NODE_MODULES_TYPE_STRIPPING`). A packaged * editor necessarily ships this source under `node_modules`, so stage the * small reporter beside the generated config in the project's own probe * workspace. In-repo and packaged runs then exercise the same path. */ export function stageProbeReporter(projectDir: string, sourcePath: string): string { const dir = join(projectDir, '.vgai', 'probe'); mkdirSync(dir, { recursive: true }); const stagedPath = join(dir, 'probe-reporter.ts'); writeFileSync(stagedPath, readFileSync(sourcePath)); return stagedPath; } // --------------------------------------------------------------------------- // resolveHeadlessShell — ported from hollowstone's bot-probe.mjs semantics. // The actual hollowstone checkout was not reachable from this worktree/agent // sandbox (verified: no ../hollowstone dir, no bot-probe.mjs anywhere on // disk here) — this is a faithful REIMPLEMENTATION from the build plan's // description of that file's `resolveHeadlessShell` (env override renamed // `VGAI_CHROME`, linux64 cache path, `undefined` fallback so CI's own // `playwright install chromium` still resolves via Playwright's default // lookup), not a byte-for-byte port. Flagged as a deviation in the task report. // --------------------------------------------------------------------------- export function resolveHeadlessShell(env: NodeJS.ProcessEnv = process.env): string | undefined { const override = env['VGAI_CHROME']; if (override) return override; if (process.platform !== 'linux') return undefined; const cacheDir = join(env['HOME'] ?? homedir(), '.cache', 'ms-playwright'); try { if (!existsSync(cacheDir)) return undefined; const entries = readdirSync(cacheDir).filter( (n) => n.startsWith('chromium_headless_shell-') || n.startsWith('chromium-'), ); if (entries.length === 0) return undefined; const headlessOnly = entries.filter((n) => n.startsWith('chromium_headless_shell-')).sort(); const chosen = headlessOnly[headlessOnly.length - 1] ?? entries.sort()[entries.length - 1]; if (!chosen) return undefined; const shellPath = join(cacheDir, chosen, 'chrome-linux', 'headless_shell'); if (existsSync(shellPath)) return shellPath; const chromePath = join(cacheDir, chosen, 'chrome-linux', 'chrome'); return existsSync(chromePath) ? chromePath : undefined; } catch { return undefined; } } // --------------------------------------------------------------------------- // --headed on a display-less host — Playwright's own launch failure, mapped // to a teaching hint. This layer never holds a live Page/Browser handle (see // this file's header "KNOWN GAPS" note), so the spawned child's stdout/stderr // text is the only signal available; `classifyHeadedLaunchFailure` is pure so // it can be exercised directly against a captured/fake Playwright error // string without ever actually launching a headed browser in CI. // --------------------------------------------------------------------------- export const HEADED_NO_DISPLAY_HINT = 'headed mode needs a display; on WSL use an X server or run from a desktop session'; // Matches both Playwright's own launch-time message ("Looks like you // launched a headed browser without having a XServer running...") and the // lower-level X11/Chromium errors it can surface instead ("Missing X server // or $DISPLAY", "cannot open display", "no protocol specified"). const HEADED_NO_DISPLAY_RE = /(headed browser without having a x ?server|missing x server or \$display|cannot open display|no protocol specified)/i; /** Pure — inspects a spawned `playwright test` child's combined stdout+stderr * text for its own "no display" launch failure and maps it to * `HEADED_NO_DISPLAY_HINT`. Returns `undefined` when the output doesn't * match (an ordinary failure, or headed wasn't requested at all). */ export function classifyHeadedLaunchFailure(output: string): string | undefined { return HEADED_NO_DISPLAY_RE.test(output) ? HEADED_NO_DISPLAY_HINT : undefined; } // --------------------------------------------------------------------------- // Exit-code mapping table // --------------------------------------------------------------------------- export interface ExitCodeInputs { zeroSpecs?: boolean; lintViolationCount?: number; typecheckFailed?: boolean; wedged?: boolean; missingCapability?: boolean; anyTestFailed?: boolean; } /** Pure decision table mirroring docs/ACCEPTANCE-DRIVER-SPEC.md §2.2's exit * code list. Order matters: usage-class failures (steps 1/3) are checked * before the type-check's spec-failure class (step 2), before the harness * wedge, before the missing-capability/ordinary-failure split from an * actual suite run. */ export function computeExitCode(input: ExitCodeInputs): number { if (input.zeroSpecs) return PROBE_EXIT_CODES.USAGE; if ((input.lintViolationCount ?? 0) > 0) return PROBE_EXIT_CODES.USAGE; if (input.typecheckFailed) return PROBE_EXIT_CODES.SPEC_FAILURE; if (input.wedged) return PROBE_EXIT_CODES.HARNESS_WEDGE; if (input.missingCapability) return PROBE_EXIT_CODES.MISSING_CAPABILITY; if (input.anyTestFailed) return PROBE_EXIT_CODES.SPEC_FAILURE; return PROBE_EXIT_CODES.PASS; } // --------------------------------------------------------------------------- // D4 — stale/incomplete envelope must never read as a false green // --------------------------------------------------------------------------- /** D4(a): deletes any `last-run.json` left over from a PREVIOUS run before * this run's suite spawns. Without this, a run whose child process crashes * before the reporter's `onEnd` ever fires (its own config failed to load, * the Playwright CLI's "No tests found" short-circuit, ...) would fall * through to step 7's `readFileSync` and silently read the PRIOR run's * envelope as if it were this run's result — including a stale `passed`. * Best-effort: a missing file (the common case, nothing stale) is a no-op. */ export function clearStaleLastRun(lastRunJsonPath: string): void { try { rmSync(lastRunJsonPath, { force: true }); } catch { // best-effort — see doc comment above } } /** * Run-2 frictions fix #4: `.vgai/probe/last-run/` (screenshots, videos, * console logs — Playwright's own `outputDir`) gets WIPED at the start of * every run — that's Playwright's own default behavior for `outputDir`, not * something this file does explicitly — so a green run's PNG evidence * vanished the moment the next run started, often before anyone got to * review it. Before this run's own `outputDir` is (re)created, archive * whatever the PREVIOUS run left in `last-run/` to `prev-run/` — a plain * rename, not a copy, so it costs nothing extra on disk. Exactly ONE * archived generation is kept (an existing `prev-run/` is deleted first), * so this stays disk-bounded no matter how many runs happen back to back — * unlike the `runs/` JSON archive above, which intentionally keeps * `RUNS_ARCHIVE_KEEP_COUNT` envelopes; screenshots/videos are too large for * the same treatment. * * Must run BEFORE `artifactsDir` is (re)created for this run — call this * first, then `mkdirSync(artifactsDir, { recursive: true })`. * * Best-effort and silent on any failure (permissions, a locked file on * Windows, no `last-run/` yet on a project's very first run): losing the * archived generation is far cheaper than failing the run over it. */ export function archiveLastRunArtifacts(probeDir: string): void { const lastRunDir = join(probeDir, 'last-run'); const prevRunDir = join(probeDir, 'prev-run'); if (!existsSync(lastRunDir)) return; try { rmSync(prevRunDir, { recursive: true, force: true }); } catch { // best-effort — see doc comment above } try { renameSync(lastRunDir, prevRunDir); } catch { // best-effort — e.g. a cross-device rename, or a file locked by an // antivirus/indexer on Windows; the new run simply proceeds without an // archived prior generation, same as a project's very first run. } } // --------------------------------------------------------------------------- // `runs/` archive (Wave-6 findings ledger: "last-run.json single-slot erases // run history"). `last-run.json` stays EXACTLY as-is — every existing reader // (`--last-failure`'s default path, a panel, a human `cat`-ing it) keeps its // stable single-slot contract untouched. This is purely additive: every // finished run's envelope is ALSO copied into `.vgai/probe/runs/-.json` (colons/dots swapped for `-` — Windows-safe filenames, // still lexicographically sortable since every timestamp is the same fixed // ISO width), pruned to the newest `RUNS_ARCHIVE_KEEP_COUNT` afterward. // --------------------------------------------------------------------------- export const RUNS_ARCHIVE_DIR_NAME = 'runs'; export const RUNS_ARCHIVE_KEEP_COUNT = 20; /** Windows-safe, still-sortable filename stem from an ISO timestamp — every * `:`/`.` (both illegal in a Windows filename) becomes `-`; the result * stays lexicographically ordered because `toISOString()` is fixed-width. */ function sanitizeTimestampForFilename(iso: string): string { return iso.replace(/[:.]/g, '-'); } /** Lowercase, filesystem-safe label for the archive filename — `'wedged'` * when the envelope carries `wedged: true` (AC-B1.7's synthetic envelope, * or a real one annotated in place — see `handleWedgeOutcome`), else the * envelope's own Playwright `status` verbatim. */ function resultLabelForFilename(envelope: LastRunEnvelope): string { const raw = envelope.wedged ? 'wedged' : envelope.status; return String(raw) .toLowerCase() .replace(/[^a-z0-9]+/g, '-'); } /** Deletes the OLDEST archived run files past `keep` — plain lexicographic * sort works because every filename is prefixed with a fixed-width, * sanitized ISO timestamp (see `sanitizeTimestampForFilename`), which sorts * identically to chronological order. Best-effort: a missing/unreadable * directory is a no-op (mirrors `clearStaleLastRun`'s own discipline). */ export function pruneRunsArchive(runsDir: string, keep: number = RUNS_ARCHIVE_KEEP_COUNT): void { let entries: string[]; try { entries = readdirSync(runsDir).filter((f) => f.endsWith('.json')); } catch { return; } entries.sort(); const excessCount = entries.length - keep; for (let i = 0; i < excessCount; i++) { try { rmSync(join(runsDir, entries[i]!), { force: true }); } catch { // best-effort — see doc comment above } } } /** * Copies the just-finalized `last-run.json` (already written by either * `ProbeJsonReporter.onEnd` or `handleWedgeOutcome`, by the time `main()` * calls this) into the `runs/` archive, then prunes to the newest * `RUNS_ARCHIVE_KEEP_COUNT`. Best-effort and silent on any failure — an * archive-write problem must never fail the run itself or shadow the real * exit code; `last-run.json` (this function's INPUT, never touched here) is * the one contract callers already depend on. */ export function archiveRun(input: { lastRunPath: string; runsDir: string }): void { let envelope: LastRunEnvelope; try { envelope = JSON.parse(readFileSync(input.lastRunPath, 'utf-8')) as LastRunEnvelope; } catch { return; } try { mkdirSync(input.runsDir, { recursive: true }); const fileName = `${sanitizeTimestampForFilename(envelope.startedAt)}-${resultLabelForFilename(envelope)}.json`; writeFileSync(join(input.runsDir, fileName), `${JSON.stringify(envelope, null, 2)}\n`); } catch { return; } pruneRunsArchive(input.runsDir); } /** D4(b): the final exit code must fold in the CHILD's own exit code and the * envelope's own `status`, not just `failures.length` — otherwise a run * that recorded a freshly-written envelope with zero failures (e.g. * Playwright's "No tests found" path, which can still invoke `onEnd` with * an empty suite) would compute `anyTestFailed: false` and exit 0 even * though the child itself exited non-zero / never actually passed. * `envelope === undefined` (no envelope at all — step 7's `readFileSync` * failed, likely because `clearStaleLastRun` above removed a stale one and * nothing replaced it) falls back to the child's own exit code, exactly as * before. * * D4-follow-up (Wave-6.1 dry-run friction #2): the fold above still leaves * one dishonest-green seam — an envelope that reports `status: 'passed'`, * `playwrightExit: 0`, zero `failures`, but ALSO zero `verdicts` (e.g. a * `-g ` that matches nothing inside acceptance spec files that DO * exist on disk; some Playwright versions treat that as a vacuous pass * rather than an error). `specFilesFound` — step 1's own * `checkZeroSpecs(...).specFiles.length`, always > 0 by the time this is * called since step 1 already returns exit 2 otherwise — makes that case * count as a real failure: zero tests actually running is never a real * green when specs are known to exist. Optional/defaulted so every * existing caller/fixture that doesn't pass `verdicts`/`specFilesFound` * keeps its prior behavior untouched. */ export function computeSuiteExitCode(input: { envelope: | (Pick & { verdicts?: LastRunEnvelope['verdicts']; }) | undefined; playwrightExit: number; specFilesFound?: number; }): number { if (!input.envelope) { return input.playwrightExit === 0 ? PROBE_EXIT_CODES.PASS : PROBE_EXIT_CODES.SPEC_FAILURE; } const zeroVerdictsMismatch = (input.specFilesFound ?? 0) > 0 && (input.envelope.verdicts?.length ?? 0) === 0; const anyTestFailed = input.playwrightExit !== 0 || input.envelope.status !== 'passed' || input.envelope.failures.length > 0 || zeroVerdictsMismatch; return computeExitCode({ anyTestFailed, missingCapability: !zeroVerdictsMismatch && allFailuresAreMissingCapability(input.envelope.failures), }); } /** Companion to `computeSuiteExitCode`'s zero-verdicts fold above — `main()` * calls this separately (same inputs) purely to print a line NAMING the * mismatch when it's what forced the failure, so `vgai probe: exit 1 (...)` * is never left to speak for itself when the actual cause is "0 tests ran * despite N spec files on disk". `undefined` when the condition doesn't * hold (an ordinary failure already explains itself via its own output). */ export function describeZeroVerdictsMismatch(input: { envelope: Pick | undefined; specFilesFound: number; }): string | undefined { if (!input.envelope) return undefined; if (input.specFilesFound === 0 || input.envelope.verdicts.length > 0) return undefined; return ( `vgai probe: 0 verdicts recorded despite ${input.specFilesFound} acceptance spec file(s) ` + 'in tests/acceptance/ — a vacuous pass is not a real green; treating as a failure.' ); } // --------------------------------------------------------------------------- // --last-failure // --------------------------------------------------------------------------- // #105b — `@vgai/probe`'s frozen attachment name for the AC-B1.8 // console/page-error + hidden-recovery + (since #105b) last-state/last-events // artifact (`fixture.ts`'s `VGAI_PROBE_CONSOLE_LOG_ATTACHMENT`). Matched by // literal string, not an import — this package deliberately never depends on // `@vgai/probe` (see this file's module doc: the fixture is "another // package's territory"; `probe-reporter.ts`'s `KNOWN_MESSAGE_CODE_PREFIXES` // is the same precedent, matching a frozen token by value rather than by // cross-package import). const CONSOLE_LOG_ATTACHMENT_NAME = 'vgai-probe-console-log'; /** Reads a failure's `vgai-probe-console-log` attachment off disk, if one was * recorded and its file still exists (a wedge/interrupted run, or a run * whose `.vgai/probe/last-run/` output dir was since cleaned, has neither). * `undefined` for a failure with no such attachment — never thrown. */ function readConsoleLogAttachment(f: ProbeFailureRecord): string | undefined { const attachment = f.attachments.find((a) => a.name === CONSOLE_LOG_ATTACHMENT_NAME); if (!attachment?.path || !existsSync(attachment.path)) return undefined; try { return readFileSync(attachment.path, 'utf-8').trimEnd(); } catch { return undefined; } } export function renderLastFailureReport(envelope: LastRunEnvelope): string { if (envelope.failures.length === 0) { return ( `vgai probe --last-failure: last run (${envelope.finishedAt}) had no failures ` + `(status: ${envelope.status}).` ); } const blocks = envelope.failures.map((f: ProbeFailureRecord) => { const tag = f.code ? ` [${f.code}]` : ''; // #105b: a `game.waitFor`/`game.events.expect` failure's `message` is // already the full eight-member failure block (headline through // console/page errors — see `failure-block.ts`), but a PLAIN `expect()` // assertion failure written directly in a spec (never routed through // that assembler) has only the bare assertion text here. Either way, // append the console-log artifact when one was captured (AC-B1.8, fixture // teardown, ALWAYS attempted on a failed test) — it carries capped // last-observed state + recent events the bare message never had, so // `--last-failure` is actionable without a separate `cat // .vgai/probe/last-run/.../console-log.txt`. const consoleLog = readConsoleLogAttachment(f); const body = consoleLog ? `${f.message}\n\n${consoleLog}` : f.message; return `--- ${f.title} (${f.file}:${f.line})${tag} ---\n${body}`; }); return [ `vgai probe --last-failure: ${envelope.failures.length} failure(s) from the last run ` + `(${envelope.finishedAt}):`, '', ...blocks, ].join('\n\n'); } // --------------------------------------------------------------------------- // --init (Task 3.5) // --------------------------------------------------------------------------- // M11: this content must be SELF-CONTAINED against @vgai/probe's built-in // providers ONLY (`time`, `input.actions` — Layer A, no game code required) // — a prior version referenced a fictional `give`/`axe`/`wood`/`attack` // game (never even matching the template's own real example component) and // failed out of the box on any real project the moment `--init` promised a // "worked example". The predicate callback reads `unknown` off `s(name)` // (DebugSnapshot's state reader) — the one cast below is exactly what a // real spec must do too. export const EXAMPLE_SPEC_CONTENT = `import { expect, test } from '@vgai/probe'; // Scaffolded by \`vgai probe --init\`. Every acceptance spec goes through the // same @vgai/probe fixture (see CLAUDE.md's "Acceptance tests" section). // This spec is deliberately generic -- it only touches providers every vgai // project gets for free (Layer A), so it passes unmodified on a brand-new // project with no gameplay code yet. Swap in YOUR game's own // providers/commands/actions (ctx.debug.registerStateProvider / // registerCommand -- server-locus commands need { locus: 'server' } -- and // your inputmap's declared actions) as soon as you have some. For the full // game-specific pattern -- a fixture command (D16: commands are never // proofs), the honest-input path via game.input.hold, game.events.expect, // and a boot-race-safe waitFor before a one-shot read -- see the vgai-engine // repo's own worked example: // packages/editor/template/tests/acceptance/example.spec.ts // The built-in providers every project gets for free, no game code required. const BUILT_IN_PROVIDERS = ['time', 'input.actions']; test('the debug bridge is alive: sim ticks forward from a cold boot', async ({ game }) => { // a one-shot read would race cold boot (the sim may not have ticked yet // under load), so this is a waitFor like every other time-dependent read. await game.waitFor((s) => (s('time') as { tick: number }).tick > 0, { simSeconds: 10 }); }); // Parametrization over content, not copy-pasted specs per case. for (const name of BUILT_IN_PROVIDERS) { test(\`built-in provider "\${name}" is enumerated\`, async ({ game }) => { const providers = await game.providers(); expect(providers.map((p) => p.name)).toContain(name); }); } test('a mid-run screenshot can be captured', async ({ game }) => { await game.waitFor((s) => (s('time') as { tick: number }).tick > 0, { simSeconds: 10 }); await game.screenshot('boot'); }); `; export const TESTS_TSCONFIG_CONTENT = `{ "compilerOptions": { "target": "ES2022", "module": "ESNext", "moduleResolution": "bundler", "lib": ["ES2022", "DOM"], "strict": true, "skipLibCheck": true, "esModuleInterop": true, "resolveJsonModule": true, "isolatedModules": true, "noEmit": true }, "include": ["acceptance/**/*.ts"] } `; export interface InitResult { written: string[]; skipped: string[]; } /** Idempotent — never overwrites an existing file. */ export function runInit(projectDir: string): InitResult { const written: string[] = []; const skipped: string[] = []; const specDir = join(projectDir, 'tests', 'acceptance'); const specPath = join(specDir, 'example.spec.ts'); const tsconfigPath = join(projectDir, 'tests', 'tsconfig.json'); mkdirSync(specDir, { recursive: true }); if (existsSync(specPath)) { skipped.push(specPath); } else { writeFileSync(specPath, EXAMPLE_SPEC_CONTENT); written.push(specPath); } if (existsSync(tsconfigPath)) { skipped.push(tsconfigPath); } else { writeFileSync(tsconfigPath, TESTS_TSCONFIG_CONTENT); written.push(tsconfigPath); } return { written, skipped }; } // --------------------------------------------------------------------------- // Small process/network helpers (deliberately NOT imported from // e2e/helpers/server.ts beyond `stopProcess` — see that file's own // `spawnDevServer` vs `startEditorServer` split, which `run-doctor.ts`'s // header comment explains: a caller that needs the child handle/pid // immediately writes its own spawn function; `stopProcess` alone is reused). // --------------------------------------------------------------------------- function getFreePort(): Promise { return new Promise((res, rej) => { const srv = net.createServer(); srv.on('error', rej); srv.listen(0, '127.0.0.1', () => { const addr = srv.address(); if (addr && typeof addr === 'object') { const port = addr.port; srv.close(() => res(port)); } else { srv.close(() => rej(new Error('run-probe: could not determine a free port.'))); } }); }); } function isPortOpen(port: number, host = '127.0.0.1'): Promise { return new Promise((resolveProbe) => { const sock = net.connect({ port, host }, () => { sock.end(); resolveProbe(true); }); sock.on('error', () => { sock.destroy(); resolveProbe(false); }); }); } async function waitForPortFree( port: number, host = '127.0.0.1', timeoutMs = 15_000, ): Promise { const start = Date.now(); while (Date.now() - start < timeoutMs) { if (!(await isPortOpen(port, host))) return; await new Promise((r) => setTimeout(r, 250)); } throw new Error( `run-probe: refusing to start on ${host}:${port} — something is still bound there after ${timeoutMs}ms.`, ); } async function waitForPortOpenUntil(port: number, host: string, deadline: number): Promise { for (;;) { if (await isPortOpen(port, host)) return; if (Date.now() >= deadline) { throw new Error(`run-probe: timed out waiting for ${host}:${port} to open`); } await new Promise((r) => setTimeout(r, 250)); } } async function waitForUrlUntil(url: string, deadline: number): Promise { for (;;) { const left = deadline - Date.now(); if (left <= 0) throw new Error(`run-probe: timed out waiting for ${url}`); try { const res = await fetch(url, { signal: AbortSignal.timeout(Math.min(left, 5000)) }); if (res.ok) return; } catch { // not ready yet, or this probe's own short timeout fired — keep polling } await new Promise((r) => setTimeout(r, Math.min(250, Math.max(0, deadline - Date.now())))); } } function canonicalPath(p: string): string { try { return realpathSync(p); } catch { return p; } } // --------------------------------------------------------------------------- // Runner hardening preflights (docs/PROBE-EXTERNAL-INSTALL-DESIGN.md T3): a // project scaffolded outside the checkout resolves `@vgai/probe`'s // `@playwright/test` peerDependency via WHATEVER npm materialized // `node_modules/@vgai/probe` as — a COPY under the template's own // `install-links=true` .npmrc pin, or a SYMLINK back into the checkout if // that pin is ever missing/overridden. These preflights turn the two // concrete failure modes the design doc's Problem section found (a stale // raw-TS entry; two live `@playwright/test` instances in one process) into // a named, actionable exit 2 — never Playwright's own opaque "Requiring // @playwright/test second time" crash, and never a `.ts` file handed to // Node's loader. // --------------------------------------------------------------------------- const PROBE_REINSTALL_HINT = 'reinstall the project (rm -rf node_modules && npm install)'; /** Resolves `@playwright/test`'s package.json path as seen from a * `createRequire` rooted at `fromDir` — `undefined` when unresolvable (no * such package reachable from there at all). */ function resolvePlaywrightTestPkgFrom(fromDir: string): string | undefined { try { const req = createRequire(join(fromDir, 'package.json')); return req.resolve('@playwright/test/package.json'); } catch { return undefined; } } /** * Raw-TS preflight (test plan item 2): a pre-T1 scaffold (or a project whose * `@vgai/probe` install predates the shipped-dist change) resolves * `@vgai/probe`'s main entry to a `.ts` file under `node_modules` — Node's * loader has no transform hook for that and throws * `ERR_UNKNOWN_FILE_EXTENSION`. Caught here first so the message names the * actual cause instead of that generic Node error. */ export function checkProbeEntryIsBuilt(projectDir: string): string | undefined { let entry: string; try { entry = createRequire(join(projectDir, 'package.json')).resolve('@vgai/probe'); } catch { return undefined; // unresolvable entirely — a different failure surfaces downstream } if (!entry.endsWith('.ts')) return undefined; return ( `vgai probe: @vgai/probe resolved to a raw TypeScript entry (${entry}) instead of compiled ` + `output — ${PROBE_REINSTALL_HINT} to pick up the current @vgai/probe, which ships dist/.` ); } /** * Single-@playwright/test-instance preflight (test plan items 2/3): resolves * `@playwright/test` from two vantage points — this project's own root, and * `@vgai/probe`'s own REAL (symlink-resolved) location — and fails the * moment they are two DIFFERENT physical files. Passing (one physical tree) * is the invariant, NOT version equality: a project whose own * `@playwright/test` is merely version-compatible with the one `@vgai/probe` * would reach is still a dual-instance hazard if they're different installs. * `undefined` (no failure) when either side can't be resolved at all — * that's a plain "not installed" case the ordinary run surfaces on its own. */ export function checkSinglePlaywrightInstance(projectDir: string): string | undefined { const probeDir = join(projectDir, 'node_modules', '@vgai', 'probe'); if (!existsSync(probeDir)) return undefined; // no npm install yet — nothing to compare const probeRealDir = canonicalPath(probeDir); const fromProject = resolvePlaywrightTestPkgFrom(projectDir); const fromProbe = resolvePlaywrightTestPkgFrom(probeRealDir); if (!fromProject || !fromProbe) return undefined; if (canonicalPath(fromProject) === canonicalPath(fromProbe)) return undefined; return ( 'vgai probe: two different @playwright/test installations are reachable — one from this ' + `project (${fromProject}), one from @vgai/probe's own resolution (${fromProbe}). ` + 'Playwright fatals the moment both load in one process ("Requiring @playwright/test ' + 'second time"). This project\'s .npmrc should pin `install-links=true` (copy semantics) so ' + `@vgai/probe resolves this project's OWN @playwright/test — ${PROBE_REINSTALL_HINT} after ` + 'confirming .npmrc is present and un-edited.' ); } /** Resolves the project-local `@playwright/test` CLI entry (`playwright * test`'s actual implementation) via `createRequire` rooted at the * project's own `package.json` — replaces the former bare `npx playwright * test` spawn, which could silently fall through to npm's registry-fetch * fallback instead of the project's own installed copy. In-checkout * projects still resolve up to the root workspace install, unchanged. * Throws when unresolvable — callers map that to exit 2 naming `npm * install`. */ export function resolvePlaywrightCliPath(projectDir: string): string { const req = createRequire(join(projectDir, 'package.json')); return req.resolve('@playwright/test/cli'); } async function probeGetRoot(port: number, timeoutMs = 1500): Promise { try { const res = await fetch(`http://127.0.0.1:${port}/`, { signal: AbortSignal.timeout(timeoutMs), }); return res.ok; } catch { return false; } } // --------------------------------------------------------------------------- // Step 4 — boot/reuse the standalone game server (+ Colyseus, if declared) // --------------------------------------------------------------------------- interface BootedServer { origin: string; port: number; pid: number; stop(): Promise; } /** Resolves `vite`'s CLI entry via the monorepo's own installation (same * resolution shape as `e2e/helpers/server.ts`'s `startStandaloneServer`), * duplicated here (rather than imported) because that helper's * `ServerHandle` never exposes the child's pid, which the session registry * needs to register/verify/kill a warm probe session. */ function resolveViteBinJs(): string { const req = createRequire(import.meta.url); const viteMain = req.resolve('vite'); const viteDir = viteMain.slice(0, viteMain.lastIndexOf(`${sep}vite${sep}`) + `${sep}vite`.length); return join(viteDir, 'bin', 'vite.js'); } async function bootGameServer(projectDir: string, port: number): Promise { await waitForPortFree(port); const env: Record = { ...(process.env as Record) }; // Watch disabled (the hollowstone reload-kills-the-run lesson) — harmless // and forward-compatible today: the template vite config learns this flag // in a concurrent Wave-4 task (Task 4.1), not yet landed. env['VGAI_NO_WATCH'] = '1'; env['NO_COLOR'] = '1'; env['FORCE_COLOR'] = '0'; const viteBinJs = resolveViteBinJs(); const proc = spawn( process.execPath, [viteBinJs, '--host', '127.0.0.1', '--strictPort', '--port', String(port)], { cwd: projectDir, env, stdio: 'ignore', // Own process group + unref'd: this server is meant to OUTLIVE this // `vgai probe` invocation (warm reuse — "reruns pay page navigation, // not server boot"). Only `--fresh` or `vgai close` tear it down. detached: process.platform !== 'win32', }, ); proc.unref(); if (proc.pid === undefined) { throw new Error('run-probe: game server spawn produced no pid'); } await waitForUrlUntil(`http://127.0.0.1:${port}/`, Date.now() + 30_000); const pid = proc.pid; return { origin: `http://127.0.0.1:${port}`, port, pid, stop: () => stopProcess(proc), }; } const COLYSEUS_DEFAULT_PORT = 2567; async function bootColyseusServer(projectDir: string): Promise { const port = COLYSEUS_DEFAULT_PORT; await waitForPortFree(port); const env: Record = { ...(process.env as Record) }; // Frozen decision (build plan Task 3.4 anchors): the room-side // `__vgai:debugCommand` handler gates on this (landing in Wave 4). env['VGAI_ALLOW_DEBUG_COMMANDS'] = '1'; const tsconfigPath = join(projectDir, 'server', 'tsconfig.json'); const mainPath = join(projectDir, 'server', 'main.ts'); const proc = spawn('npx', ['tsx', '--tsconfig', tsconfigPath, mainPath], { cwd: projectDir, env, stdio: 'ignore', shell: process.platform === 'win32', detached: process.platform !== 'win32', }); proc.unref(); if (proc.pid === undefined) { throw new Error('run-probe: colyseus server spawn produced no pid'); } // Colyseus's HTTP upgrade endpoint doesn't answer a plain GET the way a // Vite server's index page does — a TCP-open probe is the honest readiness // signal here (matches this file's own `waitForPortFree`'s probe style). await waitForPortOpenUntil(port, '127.0.0.1', Date.now() + 30_000); const pid = proc.pid; return { origin: `ws://127.0.0.1:${port}`, port, pid, stop: () => stopProcess(proc), }; } export interface ResolvedServers { gameServer: { origin: string; port: number }; colyseus: { origin: string; port: number } | null; reused: boolean; } /** * A warm probe server has file watching disabled so an in-flight acceptance * run cannot be destroyed by HMR. Consequently it must be restarted before * reuse when game inputs changed after the session was registered. */ export function projectChangedSinceProbeStarted(projectDir: string, startedAt: string): boolean { const startedMs = Date.parse(startedAt); if (!Number.isFinite(startedMs)) return true; const roots = [ 'src', 'public', 'server', 'vgai.game.json', 'vite.config.ts', 'index.html', 'package.json', ]; const newer = (path: string): boolean => { if (!existsSync(path)) return false; const stat = statSync(path); if (stat.mtimeMs > startedMs) return true; if (!stat.isDirectory()) return false; return readdirSync(path, { withFileTypes: true }).some((entry) => newer(join(path, entry.name)), ); }; return roots.some((path) => newer(join(projectDir, path))); } /** Warm reuse (AC-B1.6): a live `kind: 'probe'` session for this project that * answers `GET /` skips the boot entirely. `--fresh` kills it by pid first. * Multiplayer games get a second `kind: 'probe'` session entry for the * Colyseus leg (same `project`, so `vgai close ` tears down both * with no changes needed there — per the build plan's frozen decision). */ async function resolveServers( projectDir: string, needsColyseus: boolean, fresh: boolean, ): Promise { const canon = canonicalPath(projectDir); const killMatchingSessions = (): void => { for (const s of liveSessions()) { if (s.kind === 'probe' && s.project !== null && canonicalPath(s.project) === canon) { try { process.kill(s.pid, 'SIGTERM'); } catch { // already gone } unregisterSession(s.pid); } } }; if (fresh) { killMatchingSessions(); } else { const matching = liveSessions().filter( (s: EditorSession) => s.kind === 'probe' && s.project !== null && canonicalPath(s.project) === canon, ); const gamePageSession = matching.find((s) => s.port !== COLYSEUS_DEFAULT_PORT); const projectChanged = gamePageSession ? projectChangedSinceProbeStarted(projectDir, gamePageSession.startedAt) : false; if (gamePageSession && projectChanged) { killMatchingSessions(); } if (gamePageSession && !projectChanged && (await probeGetRoot(gamePageSession.port))) { let colyseus: { origin: string; port: number } | null = null; if (needsColyseus) { if (await isPortOpen(COLYSEUS_DEFAULT_PORT)) { colyseus = { origin: `ws://127.0.0.1:${COLYSEUS_DEFAULT_PORT}`, port: COLYSEUS_DEFAULT_PORT, }; } else { const booted = await bootColyseusServer(projectDir); registerSession({ project: canon, port: booted.port, pid: booted.pid, startedAt: new Date().toISOString(), kind: 'probe', }); colyseus = { origin: booted.origin, port: booted.port }; } } return { gameServer: { origin: `http://127.0.0.1:${gamePageSession.port}`, port: gamePageSession.port, }, colyseus, reused: true, }; } } const port = await getFreePort(); const booted = await bootGameServer(projectDir, port); registerSession({ project: canon, port: booted.port, pid: booted.pid, startedAt: new Date().toISOString(), kind: 'probe', }); let colyseus: { origin: string; port: number } | null = null; if (needsColyseus) { const bootedColyseus = await bootColyseusServer(projectDir); registerSession({ project: canon, port: bootedColyseus.port, pid: bootedColyseus.pid, startedAt: new Date().toISOString(), kind: 'probe', }); colyseus = { origin: bootedColyseus.origin, port: bootedColyseus.port }; } return { gameServer: { origin: booted.origin, port: booted.port }, colyseus, reused: false, }; } // --------------------------------------------------------------------------- // Step 6 — run the suite; watch for a harness wedge // --------------------------------------------------------------------------- export interface PlaywrightRunResult { exitCode: number; wedged: boolean; /** Set only when `opts.headed` and the child's output matched * `classifyHeadedLaunchFailure` (no display available) — the caller * reports this teaching hint and treats it as a usage-class failure * rather than an ordinary spec failure. */ headedNoDisplayHint?: string; /** Set only when `wedged` — the actual silent-window budget (ms) that * triggered the kill (`strikeIntervalMs * strikesToWedge`, whichever * values were in effect for this run), fed to `formatHarnessWedgeMessage` * so the printed message never hardcodes a number the watchdog wasn't * actually using. */ wedgeSilenceMs?: number; } /** Default silent-window budget: two 90s strikes (3 minutes total) — see * `runPlaywrightSuite`'s doc comment for why. Named here (not just inlined * as `?? 90_000`) so `main()` can reconstruct the same default for the * wedge message when a caller never overrides it. */ const DEFAULT_STRIKE_INTERVAL_MS = 90_000; const DEFAULT_STRIKES_TO_WEDGE = 2; /** Pure — renders the exit-5 wedge message. Without a live Page/Browser * handle (see this file's header "KNOWN GAPS" note) the runner genuinely * cannot distinguish "the renderer/test-runner truly hung" from "a single * test is legitimately running long and silent under low sim speed" — a * blind validation run's own finding was that the bare "stopped making * progress" wording read as an unconditional hang diagnosis. This names * BOTH possibilities and the remedy for the benign one. */ export function formatHarnessWedgeMessage(silenceMs: number): string { const silenceSeconds = Math.round(silenceMs / 1000); return ( `vgai probe: the harness produced no output for ${silenceSeconds}s — either a true hang, ` + 'or a single test legitimately running long and silent under low sim speed; if the perf ' + 'lines above show simSpeed well below 1x, re-run with -g on the long test or reduce ' + 'staged work with fixtures.' ); } /** * AC-B1.7 — a wedge kills the child before Playwright's OWN `--junit` * reporter can ever run its `onEnd` (same lifecycle as our own * `ProbeJsonReporter` — see `buildWedgeEnvelope`'s doc comment), so a * `--junit` run that wedges either writes NO XML at all, or leaves a STALE * file from some earlier run sitting there untouched (nothing clears * `junit.xml` the way `clearStaleLastRun` clears `last-run.json`). Silence * here would let a caller who only checks "does the file exist" believe a * stale leftover is this run's result. Pure — `xmlExists` is the caller's own * `existsSync` read, not performed here, so this is trivially unit-testable * without touching the filesystem. `undefined` when `--junit` was never * requested at all (nothing to say). * * `suiteAlreadyFinished` covers the OTHER wedge shape (see * `handleWedgeOutcome`): the child actually reached Playwright's `onEnd` * (both reporters ran — `last-run.json` already existed at wedge time) and * only hung AFTERWARD, e.g. on trailing teardown that never quiesces the * watchdog's stdout-liveness check. There, an existing `junit.xml` is this * run's real, freshly-written report — calling it "STALE" would be false. */ export function describeWedgeJunitOutcome(opts: { junitRequested: boolean; junitOutputPath: string | undefined; xmlExists: boolean; suiteAlreadyFinished?: boolean; }): string | undefined { if (!opts.junitRequested) return undefined; if (opts.xmlExists) { if (opts.suiteAlreadyFinished) { return ( `vgai probe: --junit was requested (${opts.junitOutputPath}) and was written — this run ` + "completed (Playwright's reporters already ran onEnd) before the harness wedge fired on " + "trailing silence; the XML is this run's real result, not a stale leftover." ); } return ( `vgai probe: --junit was requested (${opts.junitOutputPath}) but this run wedged (exit 5) ` + "before Playwright's own junit reporter could run — the file that exists is a STALE " + 'leftover from an earlier run, not this one.' ); } return ( `vgai probe: --junit was requested (${opts.junitOutputPath}) but no JUnit XML was written — ` + "this run wedged (exit 5) before Playwright's own reporters ever ran onEnd." ); } export interface WedgeOutcomeInput { /** The actual silent-window budget that triggered the kill, straight off * `runPlaywrightSuite`'s result — `undefined` only defensively (should * always be set alongside `wedged: true`), falls back to the real-world * default. */ wedgeSilenceMs: number | undefined; lastRunPath: string; projectDir: string; artifactsDir: string; /** ISO timestamp captured right before step 6 spawned the suite — NOT * computed inside this function, so a caller can pass a fixed value in a * test rather than racing `Date.now()`. */ suiteStartedAt: string; junitRequested: boolean; junitOutputPath: string | undefined; } /** * AC-B1.7 — everything `main()` does once `runPlaywrightSuite` reports * `wedged: true`: print the wedge message, record the wedge on * `last-run.json`, and print an honest note about the `--junit` XML via * {@link describeWedgeJunitOutcome}. Extracted out of `main()` as its own * function — real fs side effects, real paths, no fake filesystem — purely * so this glue is directly unit-testable without spawning a real child and * waiting out the real-world 90s x2 wedge budget (see `probe-runner.test.ts`'s * wedge-outcome tests, which pass a synthetic `wedgeSilenceMs` the way a * genuinely wedged run would). Returns the exit code `main()` should set * (always `PROBE_EXIT_CODES.HARNESS_WEDGE`) so `setExit` stays the one call * site that touches `process.exitCode`. * * Two distinct shapes hide behind one `wedged: true` result from * `runPlaywrightSuite`, distinguished here by whether `last-run.json` already * exists: * * - **Genuine mid-suite kill** (the common case): `clearStaleLastRun` already * deleted any PRIOR `last-run.json` before the suite spawned, and * `ProbeJsonReporter.onEnd` never got to write a fresh one — the child was * killed before any test reached `onTestEnd`. Without a write here, * `last-run.json` would simply not exist, indistinguishable from "no run * has ever happened" to `--last-failure` or any other reader — so this * writes the synthetic {@link buildWedgeEnvelope}. * - **Finished, then hung** (e.g. dangling teardown handle keeping the * process alive after the suite is done): both reporters' `onEnd` already * ran — `last-run.json` on disk is the REAL envelope, with real * verdicts/failures/status, before the stdout-liveness watchdog ever fired. * Overwriting it with the empty synthetic envelope would silently destroy * a real, possibly all-green result and misreport it as `interrupted`. This * branch instead reads the existing envelope back and merges `wedged: true` * onto it in place — preserving every real field — so a consumer still * learns the harness had to intervene, without losing the actual outcome. */ export function handleWedgeOutcome(input: WedgeOutcomeInput): number { const resolvedWedgeSilenceMs = input.wedgeSilenceMs ?? DEFAULT_STRIKE_INTERVAL_MS * DEFAULT_STRIKES_TO_WEDGE; console.error(formatHarnessWedgeMessage(resolvedWedgeSilenceMs)); const suiteAlreadyFinished = existsSync(input.lastRunPath); if (suiteAlreadyFinished) { // `onEnd` already ran and wrote the real envelope — annotate, don't clobber. const existingEnvelope = JSON.parse( readFileSync(input.lastRunPath, 'utf-8'), ) as LastRunEnvelope; writeFileSync( input.lastRunPath, `${JSON.stringify({ ...existingEnvelope, wedged: true }, null, 2)}\n`, ); } else { writeFileSync( input.lastRunPath, `${JSON.stringify( buildWedgeEnvelope({ project: input.projectDir, startedAt: input.suiteStartedAt, wedgeSilenceMs: resolvedWedgeSilenceMs, outputDir: input.artifactsDir, }), null, 2, )}\n`, ); } const junitNote = describeWedgeJunitOutcome({ junitRequested: input.junitRequested, junitOutputPath: input.junitOutputPath, xmlExists: input.junitOutputPath !== undefined && existsSync(input.junitOutputPath), suiteAlreadyFinished, }); if (junitNote) console.error(junitNote); return PROBE_EXIT_CODES.HARNESS_WEDGE; } // Bound on how much combined stdout/stderr text `runPlaywrightSuite` retains // purely for the headed-no-display check below — the full output is already // mirrored to this process's own stdout/stderr as it streams; this is a // rolling tail, not a second full copy, so a long-running suite never grows // this buffer unbounded. const HEADED_ERROR_SCAN_TAIL_BYTES = 16_384; /** Spawns the project-resolved `@playwright/test` CLI (`node test * --config ` — see `resolvePlaywrightCliPath`'s doc comment for * why this replaced a bare `npx playwright test` spawn: `npx` can silently * fall through to a registry-fetch fallback instead of the project's own * installed copy, which is exactly the dual-@playwright/test hazard T3's * preflights exist to prevent) and mirrors its stdout/stderr. See this * file's header "KNOWN GAPS" note for why the wedge watchdog here is * stdout-liveness-based (2 silent 90s windows -> kill the child process * TREE -> exit 5) rather than hollowstone's page-level eval-race/ * bringToFront recovery — this layer never holds a live Page handle. * * Why 90s x2 (3 minutes of total silence) and not something tighter: the * Playwright list reporter only prints a per-test line when a test ENDS, * not while it runs — so any single test that is legitimately quiet for a * while (the template's ~22s spinner spec under load, any sim-time * `waitFor` with a generous `simSeconds` budget) produces long stretches * with zero stdout even though the suite is making fine progress. The * budget here must clear the longest legitimately-silent test by a wide * margin — it exists to catch a genuinely hung child (renderer wedge, * crashed browser, deadlocked test), not to police slow tests. 3 minutes * is still far below any CI job timeout. * `strikeIntervalMs`/`strikesToWedge` are injectable (tests only) so the * watchdog's real-world defaults never have to be temporarily shrunk to * exercise this logic under fake timers. */ export async function runPlaywrightSuite(opts: { cwd: string; configPath: string; env: NodeJS.ProcessEnv; passthroughArgs: string[]; /** The resolved `@playwright/test/cli` entry (`resolvePlaywrightCliPath`) — * spawned via `process.execPath`, never `npx`. */ cliPath: string; /** `--headed`: on a non-zero exit, the tail of the child's own output is * scanned for its "no display" launch failure (see * `classifyHeadedLaunchFailure`) and surfaced as `headedNoDisplayHint`. */ headed?: boolean; /** Test-only override; defaults to 90_000. */ strikeIntervalMs?: number; /** Test-only override; defaults to 2. */ strikesToWedge?: number; }): Promise { return new Promise((resolvePromise) => { const child: ChildProcess = spawn( process.execPath, [opts.cliPath, 'test', '--config', opts.configPath, ...opts.passthroughArgs], { cwd: opts.cwd, env: opts.env, stdio: ['ignore', 'pipe', 'pipe'], detached: process.platform !== 'win32', shell: process.platform === 'win32', }, ); let settled = false; let lastOutputAt = Date.now(); let strikes = 0; let outputTail = ''; const STRIKE_INTERVAL_MS = opts.strikeIntervalMs ?? DEFAULT_STRIKE_INTERVAL_MS; const STRIKES_TO_WEDGE = opts.strikesToWedge ?? DEFAULT_STRIKES_TO_WEDGE; // Live-proven race (found while writing this file's own wedge test, // AC-B1.7): on a REAL spawned child, sending SIGTERM/SIGKILL // (`stopProcess`, below) makes the OS kill the process directly — its // 'close' event (registered at the bottom of this function) then fires // almost immediately, well before `stopProcess`'s OWN promise resolves // (that promise waits on its own 'exit' listener, attached only once // `stopProcess` is called, or its 5s internal fallback timer). The old // code decided "wedged" ONLY in the `.then()` continuation below — so the // 'close' handler's ordinary `finishWithExit` almost always won that // race in practice and reported a plain exit code (e.g. `1`, from a // signal-killed child's `code === null` folding to `?? 1`), never // `wedged: true`/exit 5. A `FakeChild` test double never emits a real // 'close' event during this scenario (nothing calls `.emit('close', …)` // in the wedge tests), which is why this never surfaced before. Fixed by // deciding "this run is a wedge" the INSTANT the strike budget is // crossed — `wedgeTriggered` below — so whichever completion path fires // first (the real child's 'close', or `stopProcess`'s own fallback // resolution) reports the SAME correct wedged result. let wedgeTriggered = false; let triggeredWedgeSilenceMs = 0; const wedgeResult = (): PlaywrightRunResult => ({ exitCode: PROBE_EXIT_CODES.HARNESS_WEDGE, wedged: true, wedgeSilenceMs: triggeredWedgeSilenceMs, }); const trackTail = (d: Buffer): void => { if (!opts.headed) return; outputTail = (outputTail + d.toString()).slice(-HEADED_ERROR_SCAN_TAIL_BYTES); }; child.stdout?.on('data', (d: Buffer) => { lastOutputAt = Date.now(); trackTail(d); process.stdout.write(d); }); child.stderr?.on('data', (d: Buffer) => { lastOutputAt = Date.now(); trackTail(d); process.stderr.write(d); }); const finish = (result: PlaywrightRunResult): void => { if (settled) return; settled = true; clearInterval(watchdog); resolvePromise(result); }; const watchdog = setInterval(() => { if (settled || wedgeTriggered) return; if (Date.now() - lastOutputAt >= STRIKE_INTERVAL_MS) { strikes += 1; if (strikes >= STRIKES_TO_WEDGE) { wedgeTriggered = true; triggeredWedgeSilenceMs = STRIKE_INTERVAL_MS * STRIKES_TO_WEDGE; // Fire-and-forget: `finishWithExit` below now reports the wedge // the moment the real child's 'close' fires (the common case — // see the doc comment above `wedgeTriggered`'s declaration); this // `.then()` is only the BACKSTOP for a double that never emits // 'close' at all (a test double, or the pathological case of a // real child whose stdio streams somehow never close). void stopProcess(child).then(() => finish(wedgeResult())); } } else { strikes = 0; } }, STRIKE_INTERVAL_MS); const finishWithExit = (code: number | null): void => { if (wedgeTriggered) { finish(wedgeResult()); return; } const exitCode = code ?? 1; const headedNoDisplayHint = opts.headed && exitCode !== 0 ? classifyHeadedLaunchFailure(outputTail) : undefined; finish({ exitCode, wedged: false, ...(headedNoDisplayHint ? { headedNoDisplayHint } : {}), }); }; child.on('close', (code) => finishWithExit(code)); child.on('error', () => finishWithExit(1)); }); } // --------------------------------------------------------------------------- // Main // --------------------------------------------------------------------------- async function main(): Promise { const argv = process.argv.slice(2); let args: ProbeCliArgs; try { args = parseProbeArgs(argv); } catch (err) { console.error(err instanceof Error ? err.message : String(err)); console.error(PROBE_USAGE_TEXT); setExit(PROBE_EXIT_CODES.USAGE); return; } const { projectDir } = args; const jsonMode = args.json; const log = (...a: unknown[]): void => { if (jsonMode) console.error(...a); else console.log(...a); }; // `--run-file` only means anything alongside `--last-failure` (it names // which archived run to re-print). Without it, it's silently a no-op — // warn once rather than let a typo'd/misordered flag combination look like // it did something. Behavior is unchanged either way. if (args.runFile !== undefined && !args.lastFailure) { console.error('--run-file has no effect without --last-failure'); } if (!existsSync(projectDir) || !statSync(projectDir).isDirectory()) { console.error(`vgai probe: folder not found: ${projectDir}`); setExit(PROBE_EXIT_CODES.USAGE); return; } if (args.init) { const result = runInit(projectDir); for (const f of result.written) console.log(`vgai probe --init: wrote ${relative(projectDir, f)}`); for (const f of result.skipped) { console.log(`vgai probe --init: ${relative(projectDir, f)} already exists — left untouched`); } setExit(PROBE_EXIT_CODES.PASS); return; } const lastRunPath = join(projectDir, '.vgai', 'probe', 'last-run.json'); const runsDir = join(projectDir, '.vgai', 'probe', RUNS_ARCHIVE_DIR_NAME); if (args.lastFailure) { // `runs/` archive: `--run-file ` reads one of the archived // envelopes instead of the single-slot `last-run.json` — relative paths // resolve against `projectDir` (same convention as `` itself). const targetPath = args.runFile ? resolve(projectDir, args.runFile) : lastRunPath; if (!existsSync(targetPath)) { if (args.runFile) { // An EXPLICIT --run-file path that doesn't exist is almost always a // typo/bad path, not a "never probed this project" situation — don't // send the user off to re-run the whole suite when the fix is to // check the path they typed. console.error( `vgai probe --run-file: ${relative(projectDir, targetPath)} not found — check the path.`, ); } else { console.error( `vgai probe --last-failure: no ${relative(projectDir, targetPath)} found — run \`vgai probe\` first.`, ); } setExit(PROBE_EXIT_CODES.USAGE); return; } const envelope = JSON.parse(readFileSync(targetPath, 'utf-8')) as LastRunEnvelope; if (jsonMode) console.log(JSON.stringify(envelope, null, 2)); else console.log(renderLastFailureReport(envelope)); setExit( computeExitCode({ anyTestFailed: envelope.failures.length > 0, missingCapability: allFailuresAreMissingCapability(envelope.failures), }), ); return; } // Step 1 const { zero, specFiles } = checkZeroSpecs(projectDir); if (zero) { console.error(ZERO_SPECS_MESSAGE); setExit(PROBE_EXIT_CODES.USAGE); return; } // Step 2 log('vgai probe: type-checking tests/acceptance (tests/tsconfig.json)...'); const typecheck = await runAcceptanceTypecheck(projectDir); if (!typecheck.ok) { console.error(typecheck.output); console.error('probe: type errors in tests/acceptance — fix before the browser boots'); setExit(PROBE_EXIT_CODES.SPEC_FAILURE); return; } // Step 3 const violations = lintAcceptanceSpecs(specFiles); if (violations.length > 0) { for (const v of violations) console.error(formatLintViolation(v, projectDir)); setExit(PROBE_EXIT_CODES.USAGE); return; } let manifest: ResolvedGameManifest; try { manifest = loadGameManifestFile(join(projectDir, 'vgai.game.json')); } catch (err) { console.error( `vgai probe: could not load vgai.game.json — ${err instanceof Error ? err.message : err}`, ); setExit(PROBE_EXIT_CODES.USAGE); return; } const needsColyseus = !!manifest.server?.room; for (const name of ['playwright.config.ts', 'playwright.config.js', 'playwright.config.mjs']) { if (existsSync(join(projectDir, name))) { console.error(`probe: ignoring ${name} — probe owns the runner config`); } } // Step 3.5 — external-install preflights (docs/PROBE-EXTERNAL-INSTALL-DESIGN.md // T3): resolve the project's own @playwright/test CLI, and fail loudly on // either of the two hazards a project scaffolded outside the checkout can // hit (a raw-TS @vgai/probe entry; two live @playwright/test instances) — // before booting any server, so a doomed run fails fast. const rawTsFailure = checkProbeEntryIsBuilt(projectDir); if (rawTsFailure) { console.error(rawTsFailure); setExit(PROBE_EXIT_CODES.USAGE); return; } const dualPlaywrightFailure = checkSinglePlaywrightInstance(projectDir); if (dualPlaywrightFailure) { console.error(dualPlaywrightFailure); setExit(PROBE_EXIT_CODES.USAGE); return; } let playwrightCliPath: string; try { playwrightCliPath = resolvePlaywrightCliPath(projectDir); } catch { console.error( 'vgai probe: could not resolve @playwright/test from this project — run `npm install`.', ); setExit(PROBE_EXIT_CODES.USAGE); return; } // Step 4 — #140: `--in-editor` skips this ENTIRELY (no vite boot, no // Colyseus boot, no new session-registry entry) — the live editor session // named by `args.sessionPort` IS the server this run drives, over // `@vgai/probe`'s `RelayTransport` instead of a page navigated to a // standalone origin. Multiplayer projects (`needsColyseus`) are a known // gap in this mode for now (see the PR body) — their room is whatever the // live session itself has running (or not), never booted here. let servers: ResolvedServers | undefined; if (args.inEditor) { if (args.sessionPort === undefined) { console.error( 'vgai probe --in-editor: no --session-port was supplied — this is CLI-internal ' + 'plumbing (packages/vgai-cli), not something to pass by hand.', ); setExit(PROBE_EXIT_CODES.USAGE); return; } log(`vgai probe --in-editor: driving the live editor session on :${args.sessionPort}...`); } else { log('vgai probe: resolving the standalone game server...'); try { servers = await resolveServers(projectDir, needsColyseus, args.fresh); } catch (err) { console.error( `vgai probe: failed to boot the standalone game server — ${err instanceof Error ? err.message : err}`, ); setExit(PROBE_EXIT_CODES.SPEC_FAILURE); return; } log( servers.reused ? `vgai probe: reusing the warm session at ${servers.gameServer.origin}` : `vgai probe: game server ready at ${servers.gameServer.origin}`, ); } // Step 5 const probeDir = join(projectDir, '.vgai', 'probe'); // Run-2 frictions fix #4: archive the PREVIOUS run's screenshots/videos to // prev-run/ before this run's own outputDir gets (re)created — must run // before Playwright's own outputDir-clearing default gets a chance to wipe // last-run/ out from under the prior run's evidence. archiveLastRunArtifacts(probeDir); const artifactsDir = join(probeDir, 'last-run'); mkdirSync(artifactsDir, { recursive: true }); const reporterModulePath = stageProbeReporter( projectDir, fileURLToPath(new URL('./probe-reporter.ts', import.meta.url)), ); const junitOutputPath = args.junit ? join(projectDir, '.vgai', 'probe', 'junit.xml') : undefined; // #125: `--headless` is defensive belt-and-suspenders — it forces headless // even if `--headed`/`--headed-auto` were (mistakenly) also passed down by // the CLI wrapper. `attemptHeaded` is what actually drives this run. // #140: `--in-editor` never launches a browser of its own at all — headed/ // headless are meaningless there, forced off regardless of what was passed. const attemptHeaded = !args.inEditor && args.headed && !args.headless; // --headed needs a full chromium (not the headless-shell build) — skip the // executablePath resolution entirely and let Playwright resolve its own // default. The SwiftShader launch args stay on regardless (harmless headed). const chromiumExecutablePath = args.inEditor ? undefined : attemptHeaded ? undefined : resolveHeadlessShell(); const configOpts: ProbeConfigOptions = { projectDir, testDir: join(projectDir, 'tests', 'acceptance'), reporterModulePath, lastRunJsonPath: lastRunPath, artifactsDir, ...(junitOutputPath !== undefined ? { junitOutputPath } : {}), ...(chromiumExecutablePath !== undefined ? { chromiumExecutablePath } : {}), ...(attemptHeaded ? { headed: true } : {}), }; const configPath = writeProbeConfig(configOpts); // D4(a): drop any last-run.json a PREVIOUS invocation left behind before // this run's suite spawns — otherwise a run whose child never reaches the // reporter's onEnd (crashed config, "No tests found", ...) would silently // fall through to step 7 reading the prior run's envelope as this run's // result. clearStaleLastRun(lastRunPath); // Step 6 const suiteStartedAt = new Date().toISOString(); log('vgai probe: running the acceptance suite...'); if (attemptHeaded) { log('headed mode: watch the run; timings will differ from headless CI'); } const env: Record = { ...(process.env as Record) }; if (args.inEditor) { // #140: selects `@vgai/probe`'s relay-backed fixture set // (`relay-fixture.ts`) over the page-backed one — see `index.ts`'s // module doc for the selection mechanics. `VGAI_PROBE_URL` is // deliberately left UNSET here (no standalone page ever gets navigated). env['VGAI_PROBE_RELAY_PORT'] = String(args.sessionPort); } else { env['VGAI_PROBE_URL'] = servers!.gameServer.origin; } env['VGAI_PROBE_ARTIFACTS'] = artifactsDir; // Blind session B's finding: a DEBUG_COMMAND_NOT_REGISTERED failure against // a REUSED warm session is ambiguous from inside the spec alone — the // command may never have existed, OR it may have been added to the game // since the warm server booted. This runner already knows which case it // is (`servers.reused`, above); threading it through as an env var lets // `@vgai/probe`'s fixture/client append an honest hint ONLY when it's true. if (servers?.reused) env['VGAI_PROBE_WARM'] = '1'; // D15/T-D15.6 (objection-4 fix) — `--seed ` reaches `@vgai/probe`'s // fixture the same env-var way `VGAI_PROBE_URL`/`VGAI_PROBE_WARM` do. // (In `--in-editor` mode this env var is set but unread — the relay // fixture navigates no URL; the CLI wrapper threads --seed into its own // `play` relay command instead. See `runProbe` in packages/vgai-cli.) if (args.seed !== undefined) env['VGAI_PROBE_SEED'] = String(args.seed); let { exitCode: playwrightExit, wedged, wedgeSilenceMs, headedNoDisplayHint, } = await runPlaywrightSuite({ cwd: projectDir, configPath, env, passthroughArgs: args.playwrightArgs, cliPath: playwrightCliPath, headed: attemptHeaded, }); if (wedged) { const exitCode = handleWedgeOutcome({ wedgeSilenceMs, lastRunPath, projectDir, artifactsDir, suiteStartedAt, junitRequested: args.junit, junitOutputPath, }); // `runs/` archive: `handleWedgeOutcome` above always leaves SOME // envelope at `lastRunPath` (synthetic wedge, or the real one annotated // in place) — archive it the same as an ordinary finish, below. archiveRun({ lastRunPath, runsDir }); setExit(exitCode); return; } // Ended up headed for real (no fallback needed) unless the block below flips it. let endedHeaded = attemptHeaded; if (headedNoDisplayHint) { if (!args.headedAuto) { // Explicit `--headed` on a display-less host: unchanged pre-#125 // behavior — a person asked for headed specifically, so a launch // failure is reported and the run fails (exit 2). console.error(`vgai probe: ${headedNoDisplayHint}`); setExit(PROBE_EXIT_CODES.USAGE); return; } // #125: `headed` here came from the CLI's DEFAULT gate — nobody typed // `--headed` — so a display-less host must never fail the run. Fall back // to a fresh headless re-run instead, transparently, with no change to // the eventual exit code for the same spec results. console.error( 'vgai probe: no display available — running headless ' + '(re-run with --headed after setting up a display to watch)', ); endedHeaded = false; const fallbackChromiumPath = resolveHeadlessShell(); const fallbackConfigOpts: ProbeConfigOptions = { projectDir, testDir: join(projectDir, 'tests', 'acceptance'), reporterModulePath, lastRunJsonPath: lastRunPath, artifactsDir, ...(junitOutputPath !== undefined ? { junitOutputPath } : {}), ...(fallbackChromiumPath !== undefined ? { chromiumExecutablePath: fallbackChromiumPath } : {}), }; const fallbackConfigPath = writeProbeConfig(fallbackConfigOpts); // Same reasoning as the D4(a) clear above: the failed headed attempt // never reached a real spec (Playwright's own browser launch failed // before any test ran), so there is nothing worth preserving from it. clearStaleLastRun(lastRunPath); const retry = await runPlaywrightSuite({ cwd: projectDir, configPath: fallbackConfigPath, env, passthroughArgs: args.playwrightArgs, cliPath: playwrightCliPath, headed: false, }); if (retry.wedged) { const exitCode = handleWedgeOutcome({ wedgeSilenceMs: retry.wedgeSilenceMs, lastRunPath, projectDir, artifactsDir, suiteStartedAt, junitRequested: args.junit, junitOutputPath, }); archiveRun({ lastRunPath, runsDir }); setExit(exitCode); return; } playwrightExit = retry.exitCode; } // #125: someone might have wanted to watch this — nudge them toward // `--headed` next time. Skipped in --json mode (keep stdout parseable) and // whenever `--headless` was explicit (they already told us not to watch). if (!endedHeaded && args.liveSessionHint && !args.headless && !jsonMode) { console.error( 'tip: this project has a live editor session — the synthetic player ran headless; ' + 're-run with --headed to watch it play', ); } // Step 7 let envelope: LastRunEnvelope | undefined; try { envelope = JSON.parse(readFileSync(lastRunPath, 'utf-8')) as LastRunEnvelope; } catch { // The reporter should always have written this — a missing file means // Playwright itself never got far enough to run onEnd (its OWN config // failed to load, say). Fall back to the raw child exit code below. } if (envelope) { for (const s of envelope.perf.samples) { log(` ${s.title}: ${s.wallMs}ms wall`); } } // `runs/` archive: best-effort, silent — see `archiveRun`'s own doc // comment for why an archive-write problem never touches the exit code. // No-ops on its own when `lastRunPath` doesn't exist (the `catch` inside // `envelope` above already covers that case; `archiveRun` re-checks). archiveRun({ lastRunPath, runsDir }); console.log(lastRunPath); // D4(b): folds in the child's own exit code and the envelope's own status // (not just failures.length), AND the zero-verdicts-despite-specs-on-disk // mismatch (D4-follow-up) — see computeSuiteExitCode's doc comment. const exitCode = computeSuiteExitCode({ envelope, playwrightExit, specFilesFound: specFiles.length, }); const zeroVerdictsMessage = describeZeroVerdictsMismatch({ envelope, specFilesFound: specFiles.length, }); if (zeroVerdictsMessage) console.error(zeroVerdictsMessage); if (jsonMode) { console.log(JSON.stringify({ lastRunPath, exitCode, envelope }, null, 2)); } setExit(exitCode); } // Only run when this file is the actual entry point (`npx tsx run-probe.ts // ...`) — importing it (as `probe-runner.test.ts` does, for the pure // functions above) must never trigger a real run. function isEntryPoint(): boolean { try { const thisFile = fileURLToPath(import.meta.url); const argvPath = process.argv[1]; if (!argvPath) return false; return realpathSync(thisFile) === realpathSync(resolve(argvPath)); } catch { return false; } } if (isEntryPoint()) { main().catch((err: unknown) => { console.error( `vgai probe: unexpected error — ${err instanceof Error ? (err.stack ?? err.message) : String(err)}`, ); setExit(PROBE_EXIT_CODES.SPEC_FAILURE); }); }