Explore the gstack project at /root/projects/gits/gstack focusing on the AI/eval system and testing infrastructure:

1. Read test/helpers/ - especially session-runner.ts, llm-judge.ts, eval-store.ts, skill-parser.ts, touchfiles.ts
2. Read test/skill-e2e.test.ts and test/skill-llm-eval.test.ts to understand how AI evals work
3. Read test/skill-validation.test.ts to understand skill validation
4. Read scripts/gen-skill-docs.ts to understand the template system
5. Read scripts/skill-check.ts for the health dashboard
6. Look at the browse/src/ directory more carefully - how does the browser automation work? Read server.ts, snapshot.ts, commands.ts fully

I need to understand the AI-powered evaluation pipeline, how skills are validated, and how the browser tool works with AI agents.


I'll explore the gstack project's AI/eval system and testing infrastructure systematically. Let me start by examining the test helpers and then move through the evaluation system.


bash "${CLAUDE_PLUGIN_ROOT}/scripts/hooks/run-with-flags-shell.sh" "pre:observe" "skills/continuous-learning-v2/hooks/observe.sh" "standard,strict"


Error: EISDIR: illegal operation on a directory, read '/root/projects/gits/gstack/test/helpers'


bash "${CLAUDE_PLUGIN_ROOT}/scripts/hooks/run-with-flags-shell.sh" "post:observe" "skills/continuous-learning-v2/hooks/observe.sh" "standard,strict"


Now let me read all the core helper files:


     1→/**
     2→ * Diff-based test selection for E2E and LLM-judge evals.
     3→ *
     4→ * Each test declares which source files it depends on ("touchfiles").
     5→ * The test runner checks `git diff` and only runs tests whose
     6→ * dependencies were modified. Override with EVALS_ALL=1 to run everything.
     7→ */
     8→
     9→import { spawnSync } from 'child_process';
    10→
    11→// --- Glob matching ---
    12→
    13→/**
    14→ * Match a file path against a glob pattern.
    15→ * Supports:
    16→ *   ** — match any number of path segments
    17→ *   *  — match within a single segment (no /)
    18→ */
    19→export function matchGlob(file: string, pattern: string): boolean {
    20→  const regexStr = pattern
    21→    .replace(/\./g, '\\.')
    22→    .replace(/\*\*/g, '{{GLOBSTAR}}')
    23→    .replace(/\*/g, '[^/]*')
    24→    .replace(/\{\{GLOBSTAR\}\}/g, '.*');
    25→  return new RegExp(`^${regexStr}$`).test(file);
    26→}
    27→
    28→// --- Touchfile maps ---
    29→
    30→/**
    31→ * E2E test touchfiles — keyed by testName (the string passed to runSkillTest).
    32→ * Each test lists the file patterns that, if changed, require the test to run.
    33→ */
    34→export const E2E_TOUCHFILES: Record<string, string[]> = {
    35→  // Browse core
    36→  'browse-basic':    ['browse/src/**'],
    37→  'browse-snapshot': ['browse/src/**'],
    38→
    39→  // SKILL.md setup + preamble (depend on ROOT SKILL.md only)
    40→  'skillmd-setup-discovery':  ['SKILL.md', 'SKILL.md.tmpl'],
    41→  'skillmd-no-local-binary':  ['SKILL.md', 'SKILL.md.tmpl'],
    42→  'skillmd-outside-git':      ['SKILL.md', 'SKILL.md.tmpl'],
    43→  'contributor-mode':         ['SKILL.md', 'SKILL.md.tmpl'],
    44→  'session-awareness':        ['SKILL.md', 'SKILL.md.tmpl'],
    45→
    46→  // QA
    47→  'qa-quick':       ['qa/**', 'browse/src/**'],
    48→  'qa-b6-static':   ['qa/**', 'browse/src/**', 'browse/test/fixtures/qa-eval.html', 'test/fixtures/qa-eval-ground-truth.json'],
    49→  'qa-b7-spa':      ['qa/**', 'browse/src/**', 'browse/test/fixtures/qa-eval-spa.html', 'test/fixtures/qa-eval-spa-ground-truth.json'],
    50→  'qa-b8-checkout': ['qa/**', 'browse/src/**', 'browse/test/fixtures/qa-eval-checkout.html', 'test/fixtures/qa-eval-checkout-ground-truth.json'],
    51→  'qa-only-no-fix': ['qa-only/**', 'qa/templates/**'],
    52→  'qa-fix-loop':    ['qa/**', 'browse/src/**'],
    53→
    54→  // Review
    55→  'review-sql-injection':     ['review/**', 'test/fixtures/review-eval-vuln.rb'],
    56→  'review-enum-completeness': ['review/**', 'test/fixtures/review-eval-enum*.rb'],
    57→  'review-base-branch':       ['review/**'],
    58→  'review-design-lite':       ['review/**', 'test/fixtures/review-eval-design-slop.*'],
    59→
    60→  // Plan reviews
    61→  'plan-ceo-review':           ['plan-ceo-review/**'],
    62→  'plan-ceo-review-selective': ['plan-ceo-review/**'],
    63→  'plan-eng-review':           ['plan-eng-review/**'],
    64→  'plan-eng-review-artifact':  ['plan-eng-review/**'],
    65→
    66→  // Ship
    67→  'ship-base-branch': ['ship/**'],
    68→
    69→  // Retro
    70→  'retro':             ['retro/**'],
    71→  'retro-base-branch': ['retro/**'],
    72→
    73→  // Document-release
    74→  'document-release': ['document-release/**'],
    75→
    76→  // QA bootstrap
    77→  'qa-bootstrap': ['qa/**', 'browse/src/**', 'ship/**'],
    78→
    79→  // Ship coverage audit
    80→  'ship-coverage-audit': ['ship/**'],
    81→
    82→  // Design
    83→  'design-consultation-core':     ['design-consultation/**'],
    84→  'design-consultation-research': ['design-consultation/**'],
    85→  'design-consultation-existing': ['design-consultation/**'],
    86→  'design-consultation-preview':  ['design-consultation/**'],
    87→  'plan-design-review-plan-mode':   ['plan-design-review/**'],
    88→  'plan-design-review-no-ui-scope': ['plan-design-review/**'],
    89→  'design-review-fix':              ['design-review/**', 'browse/src/**'],
    90→
    91→  // gstack-upgrade
    92→  'gstack-upgrade-happy-path': ['gstack-upgrade/**'],
    93→};
    94→
    95→/**
    96→ * LLM-judge test touchfiles — keyed by test description string.
    97→ */
    98→export const LLM_JUDGE_TOUCHFILES: Record<string, string[]> = {
    99→  'command reference table':          ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts'],
   100→  'snapshot flags reference':         ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/snapshot.ts'],
   101→  'browse/SKILL.md reference':        ['browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**'],
   102→  'setup block':                      ['SKILL.md', 'SKILL.md.tmpl'],
   103→  'regression vs baseline':           ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/fixtures/eval-baselines.json'],
   104→  'qa/SKILL.md workflow':             ['qa/SKILL.md', 'qa/SKILL.md.tmpl'],
   105→  'qa/SKILL.md health rubric':        ['qa/SKILL.md', 'qa/SKILL.md.tmpl'],
   106→  'cross-skill greptile consistency': ['review/SKILL.md', 'review/SKILL.md.tmpl', 'ship/SKILL.md', 'ship/SKILL.md.tmpl', 'review/greptile-triage.md', 'retro/SKILL.md', 'retro/SKILL.md.tmpl'],
   107→  'baseline score pinning':           ['SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json'],
   108→
   109→  // Ship & Release
   110→  'ship/SKILL.md workflow':               ['ship/SKILL.md', 'ship/SKILL.md.tmpl'],
   111→  'document-release/SKILL.md workflow':   ['document-release/SKILL.md', 'document-release/SKILL.md.tmpl'],
   112→
   113→  // Plan Reviews
   114→  'plan-ceo-review/SKILL.md modes':       ['plan-ceo-review/SKILL.md', 'plan-ceo-review/SKILL.md.tmpl'],
   115→  'plan-eng-review/SKILL.md sections':    ['plan-eng-review/SKILL.md', 'plan-eng-review/SKILL.md.tmpl'],
   116→  'plan-design-review/SKILL.md passes':   ['plan-design-review/SKILL.md', 'plan-design-review/SKILL.md.tmpl'],
   117→
   118→  // Design skills
   119→  'design-review/SKILL.md fix loop':      ['design-review/SKILL.md', 'design-review/SKILL.md.tmpl'],
   120→  'design-consultation/SKILL.md research': ['design-consultation/SKILL.md', 'design-consultation/SKILL.md.tmpl'],
   121→
   122→  // Other skills
   123→  'retro/SKILL.md instructions':          ['retro/SKILL.md', 'retro/SKILL.md.tmpl'],
   124→  'qa-only/SKILL.md workflow':            ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'],
   125→  'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl'],
   126→};
   127→
   128→/**
   129→ * Changes to any of these files trigger ALL tests (both E2E and LLM-judge).
   130→ */
   131→export const GLOBAL_TOUCHFILES = [
   132→  'test/helpers/session-runner.ts',
   133→  'test/helpers/eval-store.ts',
   134→  'test/helpers/llm-judge.ts',
   135→  'scripts/gen-skill-docs.ts',
   136→  'test/helpers/touchfiles.ts',
   137→  'browse/test/test-server.ts',
   138→];
   139→
   140→// --- Base branch detection ---
   141→
   142→/**
   143→ * Detect the base branch by trying refs in order.
   144→ * Returns the first valid ref, or null if none found.
   145→ */
   146→export function detectBaseBranch(cwd: string): string | null {
   147→  for (const ref of ['origin/main', 'origin/master', 'main', 'master']) {
   148→    const result = spawnSync('git', ['rev-parse', '--verify', ref], {
   149→      cwd, stdio: 'pipe', timeout: 3000,
   150→    });
   151→    if (result.status === 0) return ref;
   152→  }
   153→  return null;
   154→}
   155→
   156→/**
   157→ * Get list of files changed between base branch and HEAD.
   158→ */
   159→export function getChangedFiles(baseBranch: string, cwd: string): string[] {
   160→  const result = spawnSync('git', ['diff', '--name-only', `${baseBranch}...HEAD`], {
   161→    cwd, stdio: 'pipe', timeout: 5000,
   162→  });
   163→  if (result.status !== 0) return [];
   164→  return result.stdout.toString().trim().split('\n').filter(Boolean);
   165→}
   166→
   167→// --- Test selection ---
   168→
   169→/**
   170→ * Select tests to run based on changed files.
   171→ *
   172→ * Algorithm:
   173→ * 1. If any changed file matches a global touchfile → run ALL tests
   174→ * 2. Otherwise, for each test, check if any changed file matches its patterns
   175→ * 3. Return selected + skipped lists with reason
   176→ */
   177→export function selectTests(
   178→  changedFiles: string[],
   179→  touchfiles: Record<string, string[]>,
   180→  globalTouchfiles: string[] = GLOBAL_TOUCHFILES,
   181→): { selected: string[]; skipped: string[]; reason: string } {
   182→  const allTestNames = Object.keys(touchfiles);
   183→
   184→  // Global touchfile hit → run all
   185→  for (const file of changedFiles) {
   186→    if (globalTouchfiles.some(g => matchGlob(file, g))) {
   187→      return { selected: allTestNames, skipped: [], reason: `global: ${file}` };
   188→    }
   189→  }
   190→
   191→  // Per-test matching
   192→  const selected: string[] = [];
   193→  const skipped: string[] = [];
   194→  for (const [testName, patterns] of Object.entries(touchfiles)) {
   195→    const hit = changedFiles.some(f => patterns.some(p => matchGlob(f, p)));
   196→    (hit ? selected : skipped).push(testName);
   197→  }
   198→
   199→  return { selected, skipped, reason: 'diff' };
   200→}
   201→


     1→/**
     2→ * SKILL.md parser and validator.
     3→ *
     4→ * Extracts $B commands from code blocks, validates them against
     5→ * the command registry and snapshot flags.
     6→ *
     7→ * Used by:
     8→ *   - test/skill-validation.test.ts (Tier 1 static tests)
     9→ *   - scripts/skill-check.ts (health summary)
    10→ *   - scripts/dev-skill.ts (watch mode)
    11→ */
    12→
    13→import { ALL_COMMANDS } from '../../browse/src/commands';
    14→import { parseSnapshotArgs } from '../../browse/src/snapshot';
    15→import * as fs from 'fs';
    16→import * as path from 'path';
    17→
    18→export interface BrowseCommand {
    19→  command: string;
    20→  args: string[];
    21→  line: number;
    22→  raw: string;
    23→}
    24→
    25→export interface ValidationResult {
    26→  valid: BrowseCommand[];
    27→  invalid: BrowseCommand[];
    28→  snapshotFlagErrors: Array<{ command: BrowseCommand; error: string }>;
    29→  warnings: string[];
    30→}
    31→
    32→/**
    33→ * Extract all $B invocations from bash code blocks in a SKILL.md file.
    34→ */
    35→export function extractBrowseCommands(skillPath: string): BrowseCommand[] {
    36→  const content = fs.readFileSync(skillPath, 'utf-8');
    37→  const lines = content.split('\n');
    38→  const commands: BrowseCommand[] = [];
    39→
    40→  let inBashBlock = false;
    41→
    42→  for (let i = 0; i < lines.length; i++) {
    43→    const line = lines[i];
    44→
    45→    // Detect code block boundaries
    46→    if (line.trimStart().startsWith('```')) {
    47→      if (inBashBlock) {
    48→        inBashBlock = false;
    49→      } else if (line.trimStart().startsWith('```bash')) {
    50→        inBashBlock = true;
    51→      }
    52→      // Non-bash code blocks (```json, ```, ```js, etc.) are skipped
    53→      continue;
    54→    }
    55→
    56→    if (!inBashBlock) continue;
    57→
    58→    // Match lines with $B command invocations
    59→    // Handle multiple $B commands on one line (e.g., "$B click @e3       $B fill @e4 "value"")
    60→    const matches = line.matchAll(/\$B\s+(\S+)(?:\s+([^\$]*))?/g);
    61→    for (const match of matches) {
    62→      const command = match[1];
    63→      let argsStr = (match[2] || '').trim();
    64→
    65→      // Strip inline comments (# ...) — but not inside quotes
    66→      // Simple approach: remove everything from first unquoted # onward
    67→      let inQuote = false;
    68→      for (let j = 0; j < argsStr.length; j++) {
    69→        if (argsStr[j] === '"') inQuote = !inQuote;
    70→        if (argsStr[j] === '#' && !inQuote) {
    71→          argsStr = argsStr.slice(0, j).trim();
    72→          break;
    73→        }
    74→      }
    75→
    76→      // Parse args — handle quoted strings
    77→      const args: string[] = [];
    78→      if (argsStr) {
    79→        const argMatches = argsStr.matchAll(/"([^"]*)"|(\S+)/g);
    80→        for (const am of argMatches) {
    81→          args.push(am[1] ?? am[2]);
    82→        }
    83→      }
    84→
    85→      commands.push({
    86→        command,
    87→        args,
    88→        line: i + 1, // 1-based
    89→        raw: match[0].trim(),
    90→      });
    91→    }
    92→  }
    93→
    94→  return commands;
    95→}
    96→
    97→/**
    98→ * Extract and validate all $B commands in a SKILL.md file.
    99→ */
   100→export function validateSkill(skillPath: string): ValidationResult {
   101→  const commands = extractBrowseCommands(skillPath);
   102→  const result: ValidationResult = {
   103→    valid: [],
   104→    invalid: [],
   105→    snapshotFlagErrors: [],
   106→    warnings: [],
   107→  };
   108→
   109→  if (commands.length === 0) {
   110→    result.warnings.push('no $B commands found');
   111→    return result;
   112→  }
   113→
   114→  for (const cmd of commands) {
   115→    if (!ALL_COMMANDS.has(cmd.command)) {
   116→      result.invalid.push(cmd);
   117→      continue;
   118→    }
   119→
   120→    // Validate snapshot flags
   121→    if (cmd.command === 'snapshot' && cmd.args.length > 0) {
   122→      try {
   123→        parseSnapshotArgs(cmd.args);
   124→      } catch (err: any) {
   125→        result.snapshotFlagErrors.push({ command: cmd, error: err.message });
   126→        continue;
   127→      }
   128→    }
   129→
   130→    result.valid.push(cmd);
   131→  }
   132→
   133→  return result;
   134→}
   135→
   136→/**
   137→ * Extract all REMOTE_SLUG=$(...) assignment patterns from .md files in given subdirectories.
   138→ * Returns a Map from filename → array of full assignment lines found.
   139→ */
   140→export function extractRemoteSlugPatterns(rootDir: string, subdirs: string[]): Map<string, string[]> {
   141→  const results = new Map<string, string[]>();
   142→  const pattern = /^REMOTE_SLUG=\$\(.*\)$/;
   143→
   144→  for (const subdir of subdirs) {
   145→    const dir = path.join(rootDir, subdir);
   146→    if (!fs.existsSync(dir)) continue;
   147→
   148→    const files = fs.readdirSync(dir).filter(f => f.endsWith('.md'));
   149→    for (const file of files) {
   150→      const filePath = path.join(dir, file);
   151→      const content = fs.readFileSync(filePath, 'utf-8');
   152→      const matches: string[] = [];
   153→
   154→      for (const line of content.split('\n')) {
   155→        const trimmed = line.trim();
   156→        if (pattern.test(trimmed)) {
   157→          matches.push(trimmed);
   158→        }
   159→      }
   160→
   161→      if (matches.length > 0) {
   162→        results.set(`${subdir}/${file}`, matches);
   163→      }
   164→    }
   165→  }
   166→
   167→  return results;
   168→}
   169→
   170→/**
   171→ * Parse a markdown weight table anchored to a "### Weights" heading.
   172→ * Expects rows like: | Category | 15% |
   173→ * Returns Map<category, number> where number is the percentage (e.g., 15).
   174→ */
   175→export function extractWeightsFromTable(content: string): Map<string, number> {
   176→  const weights = new Map<string, number>();
   177→
   178→  // Find the ### Weights section
   179→  const weightsIdx = content.indexOf('### Weights');
   180→  if (weightsIdx === -1) return weights;
   181→
   182→  // Find the table within that section (stop at next heading or end)
   183→  const section = content.slice(weightsIdx);
   184→  const lines = section.split('\n');
   185→
   186→  for (let i = 1; i < lines.length; i++) {
   187→    const line = lines[i].trim();
   188→
   189→    // Stop at next heading
   190→    if (line.startsWith('#') && !line.startsWith('###')) break;
   191→    if (line.startsWith('### ') && i > 0) break;
   192→
   193→    // Parse table rows: | Category | N% |
   194→    const match = line.match(/^\|\s*(\w[\w\s]*\w|\w+)\s*\|\s*(\d+)%\s*\|$/);
   195→    if (match) {
   196→      const category = match[1].trim();
   197→      const pct = parseInt(match[2], 10);
   198→      // Skip header row
   199→      if (category !== 'Category' && !isNaN(pct)) {
   200→        weights.set(category, pct);
   201→      }
   202→    }
   203→  }
   204→
   205→  return weights;
   206→}
   207→


     1→/**
     2→ * Claude CLI subprocess runner for skill E2E testing.
     3→ *
     4→ * Spawns `claude -p` as a completely independent process (not via Agent SDK),
     5→ * so it works inside Claude Code sessions. Pipes prompt via stdin, streams
     6→ * NDJSON output for real-time progress, scans for browse errors.
     7→ */
     8→
     9→import * as fs from 'fs';
    10→import * as path from 'path';
    11→import * as os from 'os';
    12→
    13→const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev');
    14→const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json');
    15→
    16→/** Sanitize test name for use as filename: strip leading slashes, replace / with - */
    17→export function sanitizeTestName(name: string): string {
    18→  return name.replace(/^\/+/, '').replace(/\//g, '-');
    19→}
    20→
    21→/** Atomic write: write to .tmp then rename. Non-fatal on error. */
    22→function atomicWriteSync(filePath: string, data: string): void {
    23→  const tmp = filePath + '.tmp';
    24→  fs.writeFileSync(tmp, data);
    25→  fs.renameSync(tmp, filePath);
    26→}
    27→
    28→export interface CostEstimate {
    29→  inputChars: number;
    30→  outputChars: number;
    31→  estimatedTokens: number;
    32→  estimatedCost: number;  // USD
    33→  turnsUsed: number;
    34→}
    35→
    36→export interface SkillTestResult {
    37→  toolCalls: Array<{ tool: string; input: any; output: string }>;
    38→  browseErrors: string[];
    39→  exitReason: string;
    40→  duration: number;
    41→  output: string;
    42→  costEstimate: CostEstimate;
    43→  transcript: any[];
    44→}
    45→
    46→const BROWSE_ERROR_PATTERNS = [
    47→  /Unknown command: \w+/,
    48→  /Unknown snapshot flag: .+/,
    49→  /ERROR: browse binary not found/,
    50→  /Server failed to start/,
    51→  /no such file or directory.*browse/i,
    52→];
    53→
    54→// --- Testable NDJSON parser ---
    55→
    56→export interface ParsedNDJSON {
    57→  transcript: any[];
    58→  resultLine: any | null;
    59→  turnCount: number;
    60→  toolCallCount: number;
    61→  toolCalls: Array<{ tool: string; input: any; output: string }>;
    62→}
    63→
    64→/**
    65→ * Parse an array of NDJSON lines into structured transcript data.
    66→ * Pure function — no I/O, no side effects. Used by both the streaming
    67→ * reader and unit tests.
    68→ */
    69→export function parseNDJSON(lines: string[]): ParsedNDJSON {
    70→  const transcript: any[] = [];
    71→  let resultLine: any = null;
    72→  let turnCount = 0;
    73→  let toolCallCount = 0;
    74→  const toolCalls: ParsedNDJSON['toolCalls'] = [];
    75→
    76→  for (const line of lines) {
    77→    if (!line.trim()) continue;
    78→    try {
    79→      const event = JSON.parse(line);
    80→      transcript.push(event);
    81→
    82→      // Track turns and tool calls from assistant events
    83→      if (event.type === 'assistant') {
    84→        turnCount++;
    85→        const content = event.message?.content || [];
    86→        for (const item of content) {
    87→          if (item.type === 'tool_use') {
    88→            toolCallCount++;
    89→            toolCalls.push({
    90→              tool: item.name || 'unknown',
    91→              input: item.input || {},
    92→              output: '',
    93→            });
    94→          }
    95→        }
    96→      }
    97→
    98→      if (event.type === 'result') resultLine = event;
    99→    } catch { /* skip malformed lines */ }
   100→  }
   101→
   102→  return { transcript, resultLine, turnCount, toolCallCount, toolCalls };
   103→}
   104→
   105→function truncate(s: string, max: number): string {
   106→  return s.length > max ? s.slice(0, max) + '…' : s;
   107→}
   108→
   109→// --- Main runner ---
   110→
   111→export async function runSkillTest(options: {
   112→  prompt: string;
   113→  workingDirectory: string;
   114→  maxTurns?: number;
   115→  allowedTools?: string[];
   116→  timeout?: number;
   117→  testName?: string;
   118→  runId?: string;
   119→}): Promise<SkillTestResult> {
   120→  const {
   121→    prompt,
   122→    workingDirectory,
   123→    maxTurns = 15,
   124→    allowedTools = ['Bash', 'Read', 'Write'],
   125→    timeout = 120_000,
   126→    testName,
   127→    runId,
   128→  } = options;
   129→
   130→  const startTime = Date.now();
   131→  const startedAt = new Date().toISOString();
   132→
   133→  // Set up per-run log directory if runId is provided
   134→  let runDir: string | null = null;
   135→  const safeName = testName ? sanitizeTestName(testName) : null;
   136→  if (runId) {
   137→    try {
   138→      runDir = path.join(GSTACK_DEV_DIR, 'e2e-runs', runId);
   139→      fs.mkdirSync(runDir, { recursive: true });
   140→    } catch { /* non-fatal */ }
   141→  }
   142→
   143→  // Spawn claude -p with streaming NDJSON output. Prompt piped via stdin to
   144→  // avoid shell escaping issues. --verbose is required for stream-json mode.
   145→  const args = [
   146→    '-p',
   147→    '--output-format', 'stream-json',
   148→    '--verbose',
   149→    '--dangerously-skip-permissions',
   150→    '--max-turns', String(maxTurns),
   151→    '--allowed-tools', ...allowedTools,
   152→  ];
   153→
   154→  // Write prompt to a temp file and pipe it via shell to avoid stdin buffering issues
   155→  const promptFile = path.join(workingDirectory, '.prompt-tmp');
   156→  fs.writeFileSync(promptFile, prompt);
   157→
   158→  const proc = Bun.spawn(['sh', '-c', `cat "${promptFile}" | claude ${args.map(a => `"${a}"`).join(' ')}`], {
   159→    cwd: workingDirectory,
   160→    stdout: 'pipe',
   161→    stderr: 'pipe',
   162→  });
   163→
   164→  // Race against timeout
   165→  let stderr = '';
   166→  let exitReason = 'unknown';
   167→  let timedOut = false;
   168→
   169→  const timeoutId = setTimeout(() => {
   170→    timedOut = true;
   171→    proc.kill();
   172→  }, timeout);
   173→
   174→  // Stream NDJSON from stdout for real-time progress
   175→  const collectedLines: string[] = [];
   176→  let liveTurnCount = 0;
   177→  let liveToolCount = 0;
   178→  const stderrPromise = new Response(proc.stderr).text();
   179→
   180→  const reader = proc.stdout.getReader();
   181→  const decoder = new TextDecoder();
   182→  let buf = '';
   183→
   184→  try {
   185→    while (true) {
   186→      const { done, value } = await reader.read();
   187→      if (done) break;
   188→      buf += decoder.decode(value, { stream: true });
   189→      const lines = buf.split('\n');
   190→      buf = lines.pop() || '';
   191→      for (const line of lines) {
   192→        if (!line.trim()) continue;
   193→        collectedLines.push(line);
   194→
   195→        // Real-time progress to stderr + persistent logs
   196→        try {
   197→          const event = JSON.parse(line);
   198→          if (event.type === 'assistant') {
   199→            liveTurnCount++;
   200→            const content = event.message?.content || [];
   201→            for (const item of content) {
   202→              if (item.type === 'tool_use') {
   203→                liveToolCount++;
   204→                const elapsed = Math.round((Date.now() - startTime) / 1000);
   205→                const progressLine = `  [${elapsed}s] turn ${liveTurnCount} tool #${liveToolCount}: ${item.name}(${truncate(JSON.stringify(item.input || {}), 80)})\n`;
   206→                process.stderr.write(progressLine);
   207→
   208→                // Persist progress.log
   209→                if (runDir) {
   210→                  try { fs.appendFileSync(path.join(runDir, 'progress.log'), progressLine); } catch { /* non-fatal */ }
   211→                }
   212→
   213→                // Write heartbeat (atomic)
   214→                if (runId && testName) {
   215→                  try {
   216→                    const toolDesc = `${item.name}(${truncate(JSON.stringify(item.input || {}), 60)})`;
   217→                    atomicWriteSync(HEARTBEAT_PATH, JSON.stringify({
   218→                      runId,
   219→                      pid: proc.pid,
   220→                      startedAt,
   221→                      currentTest: testName,
   222→                      status: 'running',
   223→                      turn: liveTurnCount,
   224→                      toolCount: liveToolCount,
   225→                      lastTool: toolDesc,
   226→                      lastToolAt: new Date().toISOString(),
   227→                      elapsedSec: elapsed,
   228→                    }, null, 2) + '\n');
   229→                  } catch { /* non-fatal */ }
   230→                }
   231→              }
   232→            }
   233→          }
   234→        } catch { /* skip — parseNDJSON will handle it later */ }
   235→
   236→        // Append raw NDJSON line to per-test transcript file
   237→        if (runDir && safeName) {
   238→          try { fs.appendFileSync(path.join(runDir, `${safeName}.ndjson`), line + '\n'); } catch { /* non-fatal */ }
   239→        }
   240→      }
   241→    }
   242→  } catch { /* stream read error — fall through to exit code handling */ }
   243→
   244→  // Flush remaining buffer
   245→  if (buf.trim()) {
   246→    collectedLines.push(buf);
   247→  }
   248→
   249→  stderr = await stderrPromise;
   250→  const exitCode = await proc.exited;
   251→  clearTimeout(timeoutId);
   252→
   253→  try { fs.unlinkSync(promptFile); } catch { /* non-fatal */ }
   254→
   255→  if (timedOut) {
   256→    exitReason = 'timeout';
   257→  } else if (exitCode === 0) {
   258→    exitReason = 'success';
   259→  } else {
   260→    exitReason = `exit_code_${exitCode}`;
   261→  }
   262→
   263→  const duration = Date.now() - startTime;
   264→
   265→  // Parse all collected NDJSON lines
   266→  const parsed = parseNDJSON(collectedLines);
   267→  const { transcript, resultLine, toolCalls } = parsed;
   268→  const browseErrors: string[] = [];
   269→
   270→  // Scan transcript + stderr for browse errors
   271→  const allText = transcript.map(e => JSON.stringify(e)).join('\n') + '\n' + stderr;
   272→  for (const pattern of BROWSE_ERROR_PATTERNS) {
   273→    const match = allText.match(pattern);
   274→    if (match) {
   275→      browseErrors.push(match[0].slice(0, 200));
   276→    }
   277→  }
   278→
   279→  // Use resultLine for structured result data
   280→  if (resultLine) {
   281→    if (resultLine.is_error) {
   282→      // claude -p can return subtype=success with is_error=true (e.g. API connection failure)
   283→      exitReason = 'error_api';
   284→    } else if (resultLine.subtype === 'success') {
   285→      exitReason = 'success';
   286→    } else if (resultLine.subtype) {
   287→      exitReason = resultLine.subtype;
   288→    }
   289→  }
   290→
   291→  // Save failure transcript to persistent run directory (or fallback to workingDirectory)
   292→  if (browseErrors.length > 0 || exitReason !== 'success') {
   293→    try {
   294→      const failureDir = runDir || path.join(workingDirectory, '.gstack', 'test-transcripts');
   295→      fs.mkdirSync(failureDir, { recursive: true });
   296→      const failureName = safeName
   297→        ? `${safeName}-failure.json`
   298→        : `e2e-${new Date().toISOString().replace(/[:.]/g, '-')}.json`;
   299→      fs.writeFileSync(
   300→        path.join(failureDir, failureName),
   301→        JSON.stringify({
   302→          prompt: prompt.slice(0, 500),
   303→          testName: testName || 'unknown',
   304→          exitReason,
   305→          browseErrors,
   306→          duration,
   307→          turnAtTimeout: timedOut ? liveTurnCount : undefined,
   308→          lastToolCall: liveToolCount > 0 ? `tool #${liveToolCount}` : undefined,
   309→          stderr: stderr.slice(0, 2000),
   310→          result: resultLine ? { type: resultLine.type, subtype: resultLine.subtype, result: resultLine.result?.slice?.(0, 500) } : null,
   311→        }, null, 2),
   312→      );
   313→    } catch { /* non-fatal */ }
   314→  }
   315→
   316→  // Cost from result line (exact) or estimate from chars
   317→  const turnsUsed = resultLine?.num_turns || 0;
   318→  const estimatedCost = resultLine?.total_cost_usd || 0;
   319→  const inputChars = prompt.length;
   320→  const outputChars = (resultLine?.result || '').length;
   321→  const estimatedTokens = (resultLine?.usage?.input_tokens || 0)
   322→    + (resultLine?.usage?.output_tokens || 0)
   323→    + (resultLine?.usage?.cache_read_input_tokens || 0);
   324→
   325→  const costEstimate: CostEstimate = {
   326→    inputChars,
   327→    outputChars,
   328→    estimatedTokens,
   329→    estimatedCost: Math.round((estimatedCost) * 100) / 100,
   330→    turnsUsed,
   331→  };
   332→
   333→  return { toolCalls, browseErrors, exitReason, duration, output: resultLine?.result || '', costEstimate, transcript };
   334→}
   335→


     1→/**
     2→ * Shared LLM-as-judge helpers for eval and E2E tests.
     3→ *
     4→ * Provides callJudge (generic JSON-from-LLM), judge (doc quality scorer),
     5→ * and outcomeJudge (planted-bug detection scorer).
     6→ *
     7→ * Requires: ANTHROPIC_API_KEY env var
     8→ */
     9→
    10→import Anthropic from '@anthropic-ai/sdk';
    11→
    12→export interface JudgeScore {
    13→  clarity: number;       // 1-5
    14→  completeness: number;  // 1-5
    15→  actionability: number; // 1-5
    16→  reasoning: string;
    17→}
    18→
    19→export interface OutcomeJudgeResult {
    20→  detected: string[];
    21→  missed: string[];
    22→  false_positives: number;
    23→  detection_rate: number;
    24→  evidence_quality: number;
    25→  reasoning: string;
    26→}
    27→
    28→/**
    29→ * Call claude-sonnet-4-6 with a prompt, extract JSON response.
    30→ * Retries once on 429 rate limit errors.
    31→ */
    32→export async function callJudge<T>(prompt: string): Promise<T> {
    33→  const client = new Anthropic();
    34→
    35→  const makeRequest = () => client.messages.create({
    36→    model: 'claude-sonnet-4-6',
    37→    max_tokens: 1024,
    38→    messages: [{ role: 'user', content: prompt }],
    39→  });
    40→
    41→  let response;
    42→  try {
    43→    response = await makeRequest();
    44→  } catch (err: any) {
    45→    if (err.status === 429) {
    46→      await new Promise(r => setTimeout(r, 1000));
    47→      response = await makeRequest();
    48→    } else {
    49→      throw err;
    50→    }
    51→  }
    52→
    53→  const text = response.content[0].type === 'text' ? response.content[0].text : '';
    54→  const jsonMatch = text.match(/\{[\s\S]*\}/);
    55→  if (!jsonMatch) throw new Error(`Judge returned non-JSON: ${text.slice(0, 200)}`);
    56→  return JSON.parse(jsonMatch[0]) as T;
    57→}
    58→
    59→/**
    60→ * Score documentation quality on clarity/completeness/actionability (1-5).
    61→ */
    62→export async function judge(section: string, content: string): Promise<JudgeScore> {
    63→  return callJudge<JudgeScore>(`You are evaluating documentation quality for an AI coding agent's CLI tool reference.
    64→
    65→The agent reads this documentation to learn how to use a headless browser CLI. It needs to:
    66→1. Understand what each command does
    67→2. Know what arguments to pass
    68→3. Know valid values for enum-like parameters
    69→4. Construct correct command invocations without guessing
    70→
    71→Rate the following ${section} on three dimensions (1-5 scale):
    72→
    73→- **clarity** (1-5): Can an agent understand what each command/flag does from the description alone?
    74→- **completeness** (1-5): Are arguments, valid values, and important behaviors documented? Would an agent need to guess anything?
    75→- **actionability** (1-5): Can an agent construct correct command invocations from this reference alone?
    76→
    77→Scoring guide:
    78→- 5: Excellent — no ambiguity, all info present
    79→- 4: Good — minor gaps an experienced agent could infer
    80→- 3: Adequate — some guessing required
    81→- 2: Poor — significant info missing
    82→- 1: Unusable — agent would fail without external help
    83→
    84→Respond with ONLY valid JSON in this exact format:
    85→{"clarity": N, "completeness": N, "actionability": N, "reasoning": "brief explanation"}
    86→
    87→Here is the ${section} to evaluate:
    88→
    89→${content}`);
    90→}
    91→
    92→/**
    93→ * Evaluate a QA report against planted-bug ground truth.
    94→ * Returns detection metrics for the planted bugs.
    95→ */
    96→export async function outcomeJudge(
    97→  groundTruth: any,
    98→  report: string,
    99→): Promise<OutcomeJudgeResult> {
   100→  return callJudge<OutcomeJudgeResult>(`You are evaluating a QA testing report against known ground truth bugs.
   101→
   102→GROUND TRUTH (${groundTruth.total_bugs} planted bugs):
   103→${JSON.stringify(groundTruth.bugs, null, 2)}
   104→
   105→QA REPORT (generated by an AI agent):
   106→${report}
   107→
   108→For each planted bug, determine if the report identified it. A bug counts as
   109→"detected" if the report describes the same defect, even if the wording differs.
   110→Use the detection_hint keywords as guidance.
   111→
   112→Also count false positives: issues in the report that don't correspond to any
   113→planted bug AND aren't legitimate issues with the page.
   114→
   115→Respond with ONLY valid JSON:
   116→{
   117→  "detected": ["bug-id-1", "bug-id-2"],
   118→  "missed": ["bug-id-3"],
   119→  "false_positives": 0,
   120→  "detection_rate": 2,
   121→  "evidence_quality": 4,
   122→  "reasoning": "brief explanation"
   123→}
   124→
   125→Rules:
   126→- "detected" and "missed" arrays must only contain IDs from the ground truth: ${groundTruth.bugs.map((b: any) => b.id).join(', ')}
   127→- detection_rate = length of detected array
   128→- evidence_quality (1-5): Do detected bugs have screenshots, repro steps, or specific element references?
   129→  5 = excellent evidence for every bug, 1 = no evidence at all`);
   130→}
   131→


     1→/**
     2→ * Eval result persistence and comparison.
     3→ *
     4→ * EvalCollector accumulates test results, writes them to
     5→ * ~/.gstack-dev/evals/{version}-{branch}-{tier}-{timestamp}.json,
     6→ * prints a summary table, and auto-compares with the previous run.
     7→ *
     8→ * Comparison functions are exported for reuse by the eval:compare CLI.
     9→ */
    10→
    11→import * as fs from 'fs';
    12→import * as path from 'path';
    13→import * as os from 'os';
    14→import { spawnSync } from 'child_process';
    15→
    16→const SCHEMA_VERSION = 1;
    17→const DEFAULT_EVAL_DIR = path.join(os.homedir(), '.gstack-dev', 'evals');
    18→
    19→// --- Interfaces ---
    20→
    21→export interface EvalTestEntry {
    22→  name: string;
    23→  suite: string;
    24→  tier: 'e2e' | 'llm-judge';
    25→  passed: boolean;
    26→  duration_ms: number;
    27→  cost_usd: number;
    28→
    29→  // E2E
    30→  transcript?: any[];
    31→  prompt?: string;
    32→  output?: string;
    33→  turns_used?: number;
    34→  browse_errors?: string[];
    35→
    36→  // LLM judge
    37→  judge_scores?: Record<string, number>;
    38→  judge_reasoning?: string;
    39→
    40→  // Machine-readable diagnostics
    41→  exit_reason?: string;       // 'success' | 'timeout' | 'error_max_turns' | 'exit_code_N'
    42→  timeout_at_turn?: number;   // which turn was active when timeout hit
    43→  last_tool_call?: string;    // e.g. "Write(review-output.md)"
    44→
    45→  // Outcome eval
    46→  detection_rate?: number;
    47→  false_positives?: number;
    48→  evidence_quality?: number;
    49→  detected_bugs?: string[];
    50→  missed_bugs?: string[];
    51→
    52→  error?: string;
    53→}
    54→
    55→export interface EvalResult {
    56→  schema_version: number;
    57→  version: string;
    58→  branch: string;
    59→  git_sha: string;
    60→  timestamp: string;
    61→  hostname: string;
    62→  tier: 'e2e' | 'llm-judge';
    63→  total_tests: number;
    64→  passed: number;
    65→  failed: number;
    66→  total_cost_usd: number;
    67→  total_duration_ms: number;
    68→  tests: EvalTestEntry[];
    69→  _partial?: boolean;  // true for incremental saves, absent in final
    70→}
    71→
    72→export interface TestDelta {
    73→  name: string;
    74→  before: { passed: boolean; cost_usd: number; turns_used?: number; duration_ms?: number;
    75→            detection_rate?: number; tool_summary?: Record<string, number> };
    76→  after:  { passed: boolean; cost_usd: number; turns_used?: number; duration_ms?: number;
    77→            detection_rate?: number; tool_summary?: Record<string, number> };
    78→  status_change: 'improved' | 'regressed' | 'unchanged';
    79→}
    80→
    81→export interface ComparisonResult {
    82→  before_file: string;
    83→  after_file: string;
    84→  before_branch: string;
    85→  after_branch: string;
    86→  before_timestamp: string;
    87→  after_timestamp: string;
    88→  deltas: TestDelta[];
    89→  total_cost_delta: number;
    90→  total_duration_delta: number;
    91→  improved: number;
    92→  regressed: number;
    93→  unchanged: number;
    94→  tool_count_before: number;
    95→  tool_count_after: number;
    96→}
    97→
    98→// --- Shared helpers ---
    99→
   100→/**
   101→ * Determine if a planted-bug eval passed based on judge results vs ground truth thresholds.
   102→ * Centralizes the pass/fail logic so all planted-bug tests use the same criteria.
   103→ */
   104→export function judgePassed(
   105→  judgeResult: { detection_rate: number; false_positives: number; evidence_quality: number },
   106→  groundTruth: { minimum_detection: number; max_false_positives: number },
   107→): boolean {
   108→  return judgeResult.detection_rate >= groundTruth.minimum_detection
   109→    && judgeResult.false_positives <= groundTruth.max_false_positives
   110→    && judgeResult.evidence_quality >= 2;
   111→}
   112→
   113→// --- Comparison functions (exported for eval:compare CLI) ---
   114→
   115→/**
   116→ * Extract tool call counts from a transcript.
   117→ * Returns e.g. { Bash: 8, Read: 3, Write: 1 }.
   118→ */
   119→export function extractToolSummary(transcript: any[]): Record<string, number> {
   120→  const counts: Record<string, number> = {};
   121→  for (const event of transcript) {
   122→    if (event.type === 'assistant') {
   123→      const content = event.message?.content || [];
   124→      for (const item of content) {
   125→        if (item.type === 'tool_use') {
   126→          const name = item.name || 'unknown';
   127→          counts[name] = (counts[name] || 0) + 1;
   128→        }
   129→      }
   130→    }
   131→  }
   132→  return counts;
   133→}
   134→
   135→/**
   136→ * Find the most recent prior eval file for comparison.
   137→ * Prefers same branch, falls back to any branch.
   138→ */
   139→export function findPreviousRun(
   140→  evalDir: string,
   141→  tier: string,
   142→  branch: string,
   143→  excludeFile: string,
   144→): string | null {
   145→  let files: string[];
   146→  try {
   147→    files = fs.readdirSync(evalDir).filter(f => f.endsWith('.json'));
   148→  } catch {
   149→    return null; // dir doesn't exist
   150→  }
   151→
   152→  // Parse top-level fields from each file (cheap — no full tests array needed)
   153→  const entries: Array<{ file: string; branch: string; timestamp: string }> = [];
   154→  for (const file of files) {
   155→    if (file === path.basename(excludeFile)) continue;
   156→    const fullPath = path.join(evalDir, file);
   157→    try {
   158→      const raw = fs.readFileSync(fullPath, 'utf-8');
   159→      // Quick parse — only grab the fields we need
   160→      const data = JSON.parse(raw);
   161→      if (data.tier !== tier) continue;
   162→      entries.push({ file: fullPath, branch: data.branch || '', timestamp: data.timestamp || '' });
   163→    } catch { continue; }
   164→  }
   165→
   166→  if (entries.length === 0) return null;
   167→
   168→  // Sort by timestamp descending
   169→  entries.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
   170→
   171→  // Prefer same branch
   172→  const sameBranch = entries.find(e => e.branch === branch);
   173→  if (sameBranch) return sameBranch.file;
   174→
   175→  // Fallback: any branch
   176→  return entries[0].file;
   177→}
   178→
   179→/**
   180→ * Compare two eval results. Matches tests by name.
   181→ */
   182→export function compareEvalResults(
   183→  before: EvalResult,
   184→  after: EvalResult,
   185→  beforeFile: string,
   186→  afterFile: string,
   187→): ComparisonResult {
   188→  const deltas: TestDelta[] = [];
   189→  let improved = 0, regressed = 0, unchanged = 0;
   190→  let toolCountBefore = 0, toolCountAfter = 0;
   191→
   192→  // Index before tests by name
   193→  const beforeMap = new Map<string, EvalTestEntry>();
   194→  for (const t of before.tests) {
   195→    beforeMap.set(t.name, t);
   196→  }
   197→
   198→  // Walk after tests, match by name
   199→  for (const afterTest of after.tests) {
   200→    const beforeTest = beforeMap.get(afterTest.name);
   201→    const beforeToolSummary = beforeTest?.transcript ? extractToolSummary(beforeTest.transcript) : {};
   202→    const afterToolSummary = afterTest.transcript ? extractToolSummary(afterTest.transcript) : {};
   203→
   204→    const beforeToolCount = Object.values(beforeToolSummary).reduce((a, b) => a + b, 0);
   205→    const afterToolCount = Object.values(afterToolSummary).reduce((a, b) => a + b, 0);
   206→    toolCountBefore += beforeToolCount;
   207→    toolCountAfter += afterToolCount;
   208→
   209→    let statusChange: TestDelta['status_change'] = 'unchanged';
   210→    if (beforeTest) {
   211→      if (!beforeTest.passed && afterTest.passed) { statusChange = 'improved'; improved++; }
   212→      else if (beforeTest.passed && !afterTest.passed) { statusChange = 'regressed'; regressed++; }
   213→      else { unchanged++; }
   214→    } else {
   215→      // New test — treat as unchanged (no prior data)
   216→      unchanged++;
   217→    }
   218→
   219→    deltas.push({
   220→      name: afterTest.name,
   221→      before: {
   222→        passed: beforeTest?.passed ?? false,
   223→        cost_usd: beforeTest?.cost_usd ?? 0,
   224→        turns_used: beforeTest?.turns_used,
   225→        duration_ms: beforeTest?.duration_ms,
   226→        detection_rate: beforeTest?.detection_rate,
   227→        tool_summary: beforeToolSummary,
   228→      },
   229→      after: {
   230→        passed: afterTest.passed,
   231→        cost_usd: afterTest.cost_usd,
   232→        turns_used: afterTest.turns_used,
   233→        duration_ms: afterTest.duration_ms,
   234→        detection_rate: afterTest.detection_rate,
   235→        tool_summary: afterToolSummary,
   236→      },
   237→      status_change: statusChange,
   238→    });
   239→
   240→    beforeMap.delete(afterTest.name);
   241→  }
   242→
   243→  // Tests that were in before but not in after (removed tests)
   244→  for (const [name, beforeTest] of beforeMap) {
   245→    const beforeToolSummary = beforeTest.transcript ? extractToolSummary(beforeTest.transcript) : {};
   246→    const beforeToolCount = Object.values(beforeToolSummary).reduce((a, b) => a + b, 0);
   247→    toolCountBefore += beforeToolCount;
   248→    unchanged++;
   249→    deltas.push({
   250→      name: `${name} (removed)`,
   251→      before: {
   252→        passed: beforeTest.passed,
   253→        cost_usd: beforeTest.cost_usd,
   254→        turns_used: beforeTest.turns_used,
   255→        duration_ms: beforeTest.duration_ms,
   256→        detection_rate: beforeTest.detection_rate,
   257→        tool_summary: beforeToolSummary,
   258→      },
   259→      after: { passed: false, cost_usd: 0, tool_summary: {} },
   260→      status_change: 'unchanged',
   261→    });
   262→  }
   263→
   264→  return {
   265→    before_file: beforeFile,
   266→    after_file: afterFile,
   267→    before_branch: before.branch,
   268→    after_branch: after.branch,
   269→    before_timestamp: before.timestamp,
   270→    after_timestamp: after.timestamp,
   271→    deltas,
   272→    total_cost_delta: after.total_cost_usd - before.total_cost_usd,
   273→    total_duration_delta: after.total_duration_ms - before.total_duration_ms,
   274→    improved,
   275→    regressed,
   276→    unchanged,
   277→    tool_count_before: toolCountBefore,
   278→    tool_count_after: toolCountAfter,
   279→  };
   280→}
   281→
   282→/**
   283→ * Format a ComparisonResult as a readable string.
   284→ */
   285→export function formatComparison(c: ComparisonResult): string {
   286→  const lines: string[] = [];
   287→  const ts = c.before_timestamp ? c.before_timestamp.replace('T', ' ').slice(0, 16) : 'unknown';
   288→  lines.push(`\nvs previous: ${c.before_branch}/${c.deltas.length ? 'eval' : ''} (${ts})`);
   289→  lines.push('─'.repeat(70));
   290→
   291→  // Per-test deltas
   292→  for (const d of c.deltas) {
   293→    const arrow = d.status_change === 'improved' ? '↑' : d.status_change === 'regressed' ? '↓' : '=';
   294→    const beforeStatus = d.before.passed ? 'PASS' : 'FAIL';
   295→    const afterStatus = d.after.passed ? 'PASS' : 'FAIL';
   296→
   297→    // Turns delta
   298→    let turnsDelta = '';
   299→    if (d.before.turns_used !== undefined && d.after.turns_used !== undefined) {
   300→      const td = d.after.turns_used - d.before.turns_used;
   301→      turnsDelta = ` ${d.before.turns_used}→${d.after.turns_used}t`;
   302→      if (td !== 0) turnsDelta += `(${td > 0 ? '+' : ''}${td})`;
   303→    } else if (d.after.turns_used !== undefined) {
   304→      turnsDelta = ` ${d.after.turns_used}t`;
   305→    }
   306→
   307→    // Duration delta
   308→    let durDelta = '';
   309→    if (d.before.duration_ms !== undefined && d.after.duration_ms !== undefined) {
   310→      const bs = Math.round(d.before.duration_ms / 1000);
   311→      const as = Math.round(d.after.duration_ms / 1000);
   312→      const dd = as - bs;
   313→      durDelta = ` ${bs}→${as}s`;
   314→      if (dd !== 0) durDelta += `(${dd > 0 ? '+' : ''}${dd})`;
   315→    } else if (d.after.duration_ms !== undefined) {
   316→      durDelta = ` ${Math.round(d.after.duration_ms / 1000)}s`;
   317→    }
   318→
   319→    let detail = '';
   320→    if (d.before.detection_rate !== undefined || d.after.detection_rate !== undefined) {
   321→      detail = ` ${d.before.detection_rate ?? '?'}→${d.after.detection_rate ?? '?'} det`;
   322→    } else {
   323→      const costBefore = d.before.cost_usd.toFixed(2);
   324→      const costAfter = d.after.cost_usd.toFixed(2);
   325→      detail = ` $${costBefore}→$${costAfter}`;
   326→    }
   327→
   328→    const name = d.name.length > 30 ? d.name.slice(0, 27) + '...' : d.name.padEnd(30);
   329→    lines.push(`  ${name}  ${beforeStatus.padEnd(5)} → ${afterStatus.padEnd(5)}  ${arrow}${detail}${turnsDelta}${durDelta}`);
   330→  }
   331→
   332→  lines.push('─'.repeat(70));
   333→
   334→  // Totals
   335→  const parts: string[] = [];
   336→  if (c.improved > 0) parts.push(`${c.improved} improved`);
   337→  if (c.regressed > 0) parts.push(`${c.regressed} regressed`);
   338→  if (c.unchanged > 0) parts.push(`${c.unchanged} unchanged`);
   339→  lines.push(`  Status: ${parts.join(', ')}`);
   340→
   341→  const costSign = c.total_cost_delta >= 0 ? '+' : '';
   342→  lines.push(`  Cost:   ${costSign}$${c.total_cost_delta.toFixed(2)}`);
   343→
   344→  const durDelta = Math.round(c.total_duration_delta / 1000);
   345→  const durSign = durDelta >= 0 ? '+' : '';
   346→  lines.push(`  Duration: ${durSign}${durDelta}s`);
   347→
   348→  const toolDelta = c.tool_count_after - c.tool_count_before;
   349→  const toolSign = toolDelta >= 0 ? '+' : '';
   350→  lines.push(`  Tool calls: ${c.tool_count_before} → ${c.tool_count_after} (${toolSign}${toolDelta})`);
   351→
   352→  // Tool breakdown (show tools that changed)
   353→  const allTools = new Set<string>();
   354→  for (const d of c.deltas) {
   355→    for (const t of Object.keys(d.before.tool_summary || {})) allTools.add(t);
   356→    for (const t of Object.keys(d.after.tool_summary || {})) allTools.add(t);
   357→  }
   358→
   359→  if (allTools.size > 0) {
   360→    // Aggregate tool counts across all tests
   361→    const totalBefore: Record<string, number> = {};
   362→    const totalAfter: Record<string, number> = {};
   363→    for (const d of c.deltas) {
   364→      for (const [t, n] of Object.entries(d.before.tool_summary || {})) {
   365→        totalBefore[t] = (totalBefore[t] || 0) + n;
   366→      }
   367→      for (const [t, n] of Object.entries(d.after.tool_summary || {})) {
   368→        totalAfter[t] = (totalAfter[t] || 0) + n;
   369→      }
   370→    }
   371→
   372→    for (const tool of [...allTools].sort()) {
   373→      const b = totalBefore[tool] || 0;
   374→      const a = totalAfter[tool] || 0;
   375→      if (b !== a) {
   376→        const d = a - b;
   377→        lines.push(`    ${tool}: ${b} → ${a} (${d >= 0 ? '+' : ''}${d})`);
   378→      }
   379→    }
   380→  }
   381→
   382→  // Commentary — interpret what the deltas mean
   383→  const commentary = generateCommentary(c);
   384→  if (commentary.length > 0) {
   385→    lines.push('');
   386→    lines.push('  Takeaway:');
   387→    for (const line of commentary) {
   388→      lines.push(`    ${line}`);
   389→    }
   390→  }
   391→
   392→  return lines.join('\n');
   393→}
   394→
   395→/**
   396→ * Generate human-readable commentary interpreting comparison deltas.
   397→ * Pure function — analyzes the numbers and explains what they mean.
   398→ */
   399→export function generateCommentary(c: ComparisonResult): string[] {
   400→  const notes: string[] = [];
   401→
   402→  // 1. Regressions are the most important signal — call them out first
   403→  const regressions = c.deltas.filter(d => d.status_change === 'regressed');
   404→  if (regressions.length > 0) {
   405→    for (const d of regressions) {
   406→      notes.push(`REGRESSION: "${d.name}" was passing, now fails. Investigate immediately.`);
   407→    }
   408→  }
   409→
   410→  // 2. Improvements
   411→  const improvements = c.deltas.filter(d => d.status_change === 'improved');
   412→  for (const d of improvements) {
   413→    notes.push(`Fixed: "${d.name}" now passes.`);
   414→  }
   415→
   416→  // 3. Per-test efficiency changes (only for unchanged-status tests — regressions/improvements are already noted)
   417→  const stable = c.deltas.filter(d => d.status_change === 'unchanged' && d.after.passed);
   418→  for (const d of stable) {
   419→    const insights: string[] = [];
   420→
   421→    // Turns
   422→    if (d.before.turns_used !== undefined && d.after.turns_used !== undefined && d.before.turns_used > 0) {
   423→      const turnsDelta = d.after.turns_used - d.before.turns_used;
   424→      const turnsPct = Math.round((turnsDelta / d.before.turns_used) * 100);
   425→      if (Math.abs(turnsPct) >= 20 && Math.abs(turnsDelta) >= 2) {
   426→        if (turnsDelta < 0) {
   427→          insights.push(`${Math.abs(turnsDelta)} fewer turns (${Math.abs(turnsPct)}% more efficient)`);
   428→        } else {
   429→          insights.push(`${turnsDelta} more turns (${turnsPct}% less efficient)`);
   430→        }
   431→      }
   432→    }
   433→
   434→    // Duration
   435→    if (d.before.duration_ms !== undefined && d.after.duration_ms !== undefined && d.before.duration_ms > 0) {
   436→      const durDelta = d.after.duration_ms - d.before.duration_ms;
   437→      const durPct = Math.round((durDelta / d.before.duration_ms) * 100);
   438→      if (Math.abs(durPct) >= 20 && Math.abs(durDelta) >= 5000) {
   439→        if (durDelta < 0) {
   440→          insights.push(`${Math.round(Math.abs(durDelta) / 1000)}s faster`);
   441→        } else {
   442→          insights.push(`${Math.round(durDelta / 1000)}s slower`);
   443→        }
   444→      }
   445→    }
   446→
   447→    // Detection rate
   448→    if (d.before.detection_rate !== undefined && d.after.detection_rate !== undefined) {
   449→      const detDelta = d.after.detection_rate - d.before.detection_rate;
   450→      if (detDelta !== 0) {
   451→        if (detDelta > 0) {
   452→          insights.push(`detecting ${detDelta} more bug${detDelta > 1 ? 's' : ''}`);
   453→        } else {
   454→          insights.push(`detecting ${Math.abs(detDelta)} fewer bug${Math.abs(detDelta) > 1 ? 's' : ''} — check prompt quality`);
   455→        }
   456→      }
   457→    }
   458→
   459→    // Cost
   460→    if (d.before.cost_usd > 0) {
   461→      const costDelta = d.after.cost_usd - d.before.cost_usd;
   462→      const costPct = Math.round((costDelta / d.before.cost_usd) * 100);
   463→      if (Math.abs(costPct) >= 30 && Math.abs(costDelta) >= 0.05) {
   464→        if (costDelta < 0) {
   465→          insights.push(`${Math.abs(costPct)}% cheaper`);
   466→        } else {
   467→          insights.push(`${costPct}% more expensive`);
   468→        }
   469→      }
   470→    }
   471→
   472→    if (insights.length > 0) {
   473→      notes.push(`"${d.name}": ${insights.join(', ')}.`);
   474→    }
   475→  }
   476→
   477→  // 4. Overall summary
   478→  if (c.deltas.length >= 3 && regressions.length === 0) {
   479→    const overallParts: string[] = [];
   480→
   481→    // Total cost
   482→    const totalBefore = c.deltas.reduce((s, d) => s + d.before.cost_usd, 0);
   483→    if (totalBefore > 0) {
   484→      const costPct = Math.round((c.total_cost_delta / totalBefore) * 100);
   485→      if (Math.abs(costPct) >= 10) {
   486→        overallParts.push(`${Math.abs(costPct)}% ${costPct < 0 ? 'cheaper' : 'more expensive'} overall`);
   487→      }
   488→    }
   489→
   490→    // Total duration
   491→    const totalDurBefore = c.deltas.reduce((s, d) => s + (d.before.duration_ms || 0), 0);
   492→    if (totalDurBefore > 0) {
   493→      const durPct = Math.round((c.total_duration_delta / totalDurBefore) * 100);
   494→      if (Math.abs(durPct) >= 10) {
   495→        overallParts.push(`${Math.abs(durPct)}% ${durPct < 0 ? 'faster' : 'slower'}`);
   496→      }
   497→    }
   498→
   499→    // Total turns
   500→    const turnsBefore = c.deltas.reduce((s, d) => s + (d.before.turns_used || 0), 0);
   501→    const turnsAfter = c.deltas.reduce((s, d) => s + (d.after.turns_used || 0), 0);
   502→    if (turnsBefore > 0) {
   503→      const turnsPct = Math.round(((turnsAfter - turnsBefore) / turnsBefore) * 100);
   504→      if (Math.abs(turnsPct) >= 10) {
   505→        overallParts.push(`${Math.abs(turnsPct)}% ${turnsPct < 0 ? 'fewer' : 'more'} turns`);
   506→      }
   507→    }
   508→
   509→    if (overallParts.length > 0) {
   510→      notes.push(`Overall: ${overallParts.join(', ')}. ${regressions.length === 0 ? 'No regressions.' : ''}`);
   511→    } else if (regressions.length === 0) {
   512→      notes.push('Stable run — no significant efficiency changes, no regressions.');
   513→    }
   514→  }
   515→
   516→  return notes;
   517→}
   518→
   519→// --- EvalCollector ---
   520→
   521→function getGitInfo(): { branch: string; sha: string } {
   522→  try {
   523→    const branch = spawnSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { stdio: 'pipe', timeout: 5000 });
   524→    const sha = spawnSync('git', ['rev-parse', '--short', 'HEAD'], { stdio: 'pipe', timeout: 5000 });
   525→    return {
   526→      branch: branch.stdout?.toString().trim() || 'unknown',
   527→      sha: sha.stdout?.toString().trim() || 'unknown',
   528→    };
   529→  } catch {
   530→    return { branch: 'unknown', sha: 'unknown' };
   531→  }
   532→}
   533→
   534→function getVersion(): string {
   535→  try {
   536→    const pkgPath = path.resolve(__dirname, '..', '..', 'package.json');
   537→    const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf-8'));
   538→    return pkg.version || 'unknown';
   539→  } catch {
   540→    return 'unknown';
   541→  }
   542→}
   543→
   544→export class EvalCollector {
   545→  private tier: 'e2e' | 'llm-judge';
   546→  private tests: EvalTestEntry[] = [];
   547→  private finalized = false;
   548→  private evalDir: string;
   549→
   550→  constructor(tier: 'e2e' | 'llm-judge', evalDir?: string) {
   551→    this.tier = tier;
   552→    this.evalDir = evalDir || DEFAULT_EVAL_DIR;
   553→  }
   554→
   555→  addTest(entry: EvalTestEntry): void {
   556→    this.tests.push(entry);
   557→    this.savePartial();
   558→  }
   559→
   560→  /** Write incremental results after each test. Atomic write, non-fatal. */
   561→  savePartial(): void {
   562→    try {
   563→      const git = getGitInfo();
   564→      const version = getVersion();
   565→      const totalCost = this.tests.reduce((s, t) => s + t.cost_usd, 0);
   566→      const totalDuration = this.tests.reduce((s, t) => s + t.duration_ms, 0);
   567→      const passed = this.tests.filter(t => t.passed).length;
   568→
   569→      const partial: EvalResult = {
   570→        schema_version: SCHEMA_VERSION,
   571→        version,
   572→        branch: git.branch,
   573→        git_sha: git.sha,
   574→        timestamp: new Date().toISOString(),
   575→        hostname: os.hostname(),
   576→        tier: this.tier,
   577→        total_tests: this.tests.length,
   578→        passed,
   579→        failed: this.tests.length - passed,
   580→        total_cost_usd: Math.round(totalCost * 100) / 100,
   581→        total_duration_ms: totalDuration,
   582→        tests: this.tests,
   583→        _partial: true,
   584→      };
   585→
   586→      fs.mkdirSync(this.evalDir, { recursive: true });
   587→      const partialPath = path.join(this.evalDir, '_partial-e2e.json');
   588→      const tmp = partialPath + '.tmp';
   589→      fs.writeFileSync(tmp, JSON.stringify(partial, null, 2) + '\n');
   590→      fs.renameSync(tmp, partialPath);
   591→    } catch { /* non-fatal — partial saves are best-effort */ }
   592→  }
   593→
   594→  async finalize(): Promise<string> {
   595→    if (this.finalized) return '';
   596→    this.finalized = true;
   597→
   598→    const git = getGitInfo();
   599→    const version = getVersion();
   600→    const timestamp = new Date().toISOString();
   601→    const totalCost = this.tests.reduce((s, t) => s + t.cost_usd, 0);
   602→    const totalDuration = this.tests.reduce((s, t) => s + t.duration_ms, 0);
   603→    const passed = this.tests.filter(t => t.passed).length;
   604→
   605→    const result: EvalResult = {
   606→      schema_version: SCHEMA_VERSION,
   607→      version,
   608→      branch: git.branch,
   609→      git_sha: git.sha,
   610→      timestamp,
   611→      hostname: os.hostname(),
   612→      tier: this.tier,
   613→      total_tests: this.tests.length,
   614→      passed,
   615→      failed: this.tests.length - passed,
   616→      total_cost_usd: Math.round(totalCost * 100) / 100,
   617→      total_duration_ms: totalDuration,
   618→      tests: this.tests,
   619→    };
   620→
   621→    // Write eval file
   622→    fs.mkdirSync(this.evalDir, { recursive: true });
   623→    const dateStr = timestamp.replace(/[:.]/g, '').replace('T', '-').slice(0, 15);
   624→    const safeBranch = git.branch.replace(/[^a-zA-Z0-9._-]/g, '-');
   625→    const filename = `${version}-${safeBranch}-${this.tier}-${dateStr}.json`;
   626→    const filepath = path.join(this.evalDir, filename);
   627→    fs.writeFileSync(filepath, JSON.stringify(result, null, 2) + '\n');
   628→
   629→    // Print summary table
   630→    this.printSummary(result, filepath, git);
   631→
   632→    // Auto-compare with previous run
   633→    try {
   634→      const prevFile = findPreviousRun(this.evalDir, this.tier, git.branch, filepath);
   635→      if (prevFile) {
   636→        const prevResult: EvalResult = JSON.parse(fs.readFileSync(prevFile, 'utf-8'));
   637→        const comparison = compareEvalResults(prevResult, result, prevFile, filepath);
   638→        process.stderr.write(formatComparison(comparison) + '\n');
   639→      } else {
   640→        process.stderr.write('\nFirst run — no comparison available.\n');
   641→      }
   642→    } catch (err: any) {
   643→      process.stderr.write(`\nCompare error: ${err.message}\n`);
   644→    }
   645→
   646→    return filepath;
   647→  }
   648→
   649→  private printSummary(result: EvalResult, filepath: string, git: { branch: string; sha: string }): void {
   650→    const lines: string[] = [];
   651→    lines.push('');
   652→    lines.push(`Eval Results — v${result.version} @ ${git.branch} (${git.sha}) — ${this.tier}`);
   653→    lines.push('═'.repeat(70));
   654→
   655→    for (const t of this.tests) {
   656→      const status = t.passed ? ' PASS ' : ' FAIL ';
   657→      const cost = `$${t.cost_usd.toFixed(2)}`;
   658→      const dur = t.duration_ms ? `${Math.round(t.duration_ms / 1000)}s` : '';
   659→      const turns = t.turns_used !== undefined ? `${t.turns_used}t` : '';
   660→
   661→      let detail = '';
   662→      if (t.detection_rate !== undefined) {
   663→        detail = `${t.detection_rate}/${(t.detected_bugs?.length || 0) + (t.missed_bugs?.length || 0)} det`;
   664→      } else if (t.judge_scores) {
   665→        const scores = Object.entries(t.judge_scores).map(([k, v]) => `${k[0]}:${v}`).join(' ');
   666→        detail = scores;
   667→      }
   668→
   669→      const name = t.name.length > 35 ? t.name.slice(0, 32) + '...' : t.name.padEnd(35);
   670→      lines.push(`  ${name}  ${status}  ${cost.padStart(6)}  ${turns.padStart(4)}  ${dur.padStart(5)}  ${detail}`);
   671→    }
   672→
   673→    lines.push('─'.repeat(70));
   674→    const totalCost = `$${result.total_cost_usd.toFixed(2)}`;
   675→    const totalDur = `${Math.round(result.total_duration_ms / 1000)}s`;
   676→    lines.push(`  Total: ${result.passed}/${result.total_tests} passed${' '.repeat(20)}${totalCost.padStart(6)}  ${totalDur}`);
   677→    lines.push(`Saved: ${filepath}`);
   678→
   679→    process.stderr.write(lines.join('\n') + '\n');
   680→  }
   681→}
   682→


Now let me read the test files to understand how evals work: