[Hackathon] fix(create-batch): throttle expiry dispatches a run after the wait deadline #485

import assert from 'node:assert/strict';
import { mkdtemp, writeFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { pathToFileURL } from 'node:url';

// Deterministic companion to reproduce-create-batch-throttle-deadline.mjs.
// Same injection style as the repository's MAJOR 2 / ROUND-2 tests: Date.now
// advances by the exact requested sleep, with no jump past the deadline.
// This is a controlled-clock offline command-handler test, not real elapsed
// time and not hosted observation. HTTP and clock are the only substitutions.
const root = resolve(process.argv[2] ?? 'work/testsprite-cli');
const { runCreateBatch } = await import(pathToFileURL(join(root, 'dist/commands/test.js')));
const temp = await mkdtemp(join(tmpdir(), 'testsprite-throttle-clock-'));
const realNow = Date.now;
const json = (body, status = 200, headers = {}) => new Response(JSON.stringify(body), {
  status, headers: { 'content-type': 'application/json', ...headers },
});
const results = [];
try {
  for (const c of [
    { count: 50, timeout: 60, expectedTriggers: 2, expectedStatus: 'timeout' },
    { count: 50, timeout: 2, expectedTriggers: 1, expectedStatus: 'timeout' },
    { count: 50, timeout: 61, expectedTriggers: 2, expectedStatus: 'passed' },
    { count: 49, timeout: 60, expectedTriggers: 2, expectedStatus: 'passed' },
  ]) {
    const start = Date.parse('2026-10-17T00:00:00Z'); let now = start;
    Date.now = () => now;
    const ids = Array.from({ length: c.count }, (_, i) => 'test_' + i), lastId = ids.at(-1);
    const plans = join(temp, `${c.count}-${c.timeout}.jsonl`);
    await writeFile(plans, ids.map(id => JSON.stringify({ projectId: 'project', type: 'frontend', name: id,
      planSteps: [{ type: 'assertion', description: 'Home is visible' }] })).join('\n') + '\n');
    const stdout = [], trace = [], sleeps = [], attempts = new Map(), acceptedAt = new Map();
    let exitCode = 0;
    try {
      await runCreateBatch({ profile: 'default', output: 'json', debug: false, dryRun: false,
        plans, run: true, wait: true, maxConcurrency: 1, timeoutSeconds: c.timeout }, {
        env: { TESTSPRITE_API_KEY: 'sk-user-offline-fixture', TESTSPRITE_API_URL: 'https://mock.example.com' },
        credentialsPath: join(temp, 'absent'), stdout: s => stdout.push(s), stderr: () => {},
        sleep: ms => { sleeps.push(ms); now += ms; return Promise.resolve(); },
        fetchImpl: async (input, init = {}) => {
          const u = new URL(typeof input === 'string' ? input : input.url ?? input.href);
          assert.equal(u.hostname, 'mock.example.com');
          const path = u.pathname.replace('/api/cli/v1', ''), method = init.method ?? 'GET';
          trace.push({ path, method, elapsedMs: now - start });
          if (path === '/projects/project/env') return json({ environments: [
            { name: 'default', url: 'https://example.com', isDefault: true, isTemporary: false },
          ] });
          if (path === '/tests/batch') return json({ results: ids.map((testId, specIndex) =>
            ({ testId, specIndex, status: 'created' })), summary: { total: ids.length, created: ids.length, failed: 0 } });
          const trigger = /^\/tests\/(test_\d+)\/runs$/.exec(path);
          if (trigger) {
            const id = trigger[1], n = (attempts.get(id) ?? 0) + 1; attempts.set(id, n);
            if (id === lastId && n === 1) return json({ error: { code: 'RATE_LIMITED',
              message: 'Run trigger rate limit exceeded: 60 triggers per minute per key.',
              nextAction: 'Wait.', requestId: 'fixture', details: { retryAfterSeconds: 1 } } },
              429, { 'retry-after': '1' });
            acceptedAt.set(id, now);
            return json({ runId: 'run_' + id, status: 'queued', codeVersion: 'v1',
              enqueuedAt: new Date(now).toISOString(), targetUrl: 'https://example.com' });
          }
          const poll = /^\/runs\/run_(test_\d+)$/.exec(path);
          if (poll) {
            const stamp = new Date(acceptedAt.get(poll[1])).toISOString();
            return json({ runId: 'run_' + poll[1], testId: poll[1], projectId: 'project', userId: 'fixture',
              status: 'passed', source: 'cli', createdAt: stamp, startedAt: stamp, finishedAt: stamp,
              codeVersion: 'v1', targetUrl: 'https://example.com', createdFrom: 'cli', failedStepIndex: null,
              failureKind: null, error: null, videoUrl: null,
              stepSummary: { total: 1, completed: 1, passedCount: 1, failedCount: 0 } });
          }
          throw Error('Unexpected route ' + path);
        },
      });
    } catch (e) { exitCode = e.exitCode ?? 1; }
    const payload = JSON.parse(stdout.join('\n'));
    const last = payload.results.find(r => r.testId === lastId);
    assert.equal(payload.results.filter(r => r.testId !== lastId && r.status === 'passed').length, c.count - 1);
    const triggers = trace.filter(r => r.path === '/tests/' + lastId + '/runs');
    const polls = trace.filter(r => r.path === '/runs/run_' + lastId);
    assert.equal(triggers.length, c.expectedTriggers); assert.equal(last.status, c.expectedStatus);
    assert.equal(polls.length, c.expectedStatus === 'passed' ? 1 : 0);
    if (c.count === 50 && c.timeout === 60) {
      assert.deepEqual(sleeps, [1000, 59000]);
      assert.deepEqual(triggers.map(x => x.elapsedMs), [0, 60000]);
      assert.equal(last.runId, 'run_test_49');
    }
    results.push({ ...c, exitCode, last, sleeps, lastTriggers: triggers, lastPolls: polls.length });
    Date.now = realNow;
  }
  console.log(JSON.stringify({ evidence: 'offline controlled-clock handler; Date.now advances exactly by requested sleep', results }, null, 2));
} finally { Date.now = realNow; await rm(temp, { recursive: true, force: true }); }

import assert from 'node:assert/strict';
import { mkdtemp, writeFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { pathToFileURL } from 'node:url';
import { createRequire } from 'node:module';

// Real 60-second rate window and real timers. No clock/sleep/throttle changes.
// Actual CLI parser, client, rate throttle and poller; only HTTP is offline.
const root = resolve(process.argv[2] ?? 'work/testsprite-cli');
const require = createRequire(join(root, 'package.json'));
const { Command } = require('commander');
const { createTestCommand } = await import(pathToFileURL(join(root, 'dist/commands/test.js')));
const temp = await mkdtemp(join(tmpdir(), 'testsprite-throttle-deadline-'));
const json = (body, status = 200, headers = {}) => new Response(JSON.stringify(body), {
  status, headers: { 'content-type': 'application/json', ...headers },
});
const cases = [
  { name: 'short-budget-no-late-trigger-control', count: 50, timeout: 2, retry: true },
  { name: 'window-boundary-expired-retry', count: 50, timeout: 60, retry: true },
  { name: 'window-boundary-expired-retry-repeat', count: 50, timeout: 60, retry: true },
  { name: 'extra-budget-retry-control', count: 50, timeout: 61, retry: true },
  { name: 'no-rate-limit-control', count: 50, timeout: 60, retry: false },
  { name: 'spare-throttle-slot-control', count: 49, timeout: 60, retry: true },
];
try {
  const results = await Promise.all(cases.map(async c => {
    const ids = Array.from({ length: c.count }, (_, i) => 'test_' + i);
    const lastId = ids.at(-1), plans = join(temp, c.name + '.jsonl');
    await writeFile(plans, ids.map(id => JSON.stringify({ projectId: 'project', type: 'frontend',
      name: id, planSteps: [{ type: 'assertion', description: 'Home is visible' }] })).join('\n') + '\n');
    const stdout = [], stderr = [], trace = [], sleeps = [], first = new Map(), acceptedAt = new Map(), attempts = new Map();
    const deps = {
      env: { TESTSPRITE_API_KEY: 'sk-user-offline-fixture', TESTSPRITE_API_URL: 'https://mock.example.com' },
      credentialsPath: join(temp, 'absent'), stdout: x => stdout.push(x), stderr: x => stderr.push(x),
      // Same real setTimeout used by the CLI's default sleep. Record an upper
      // bound for the deadline at a clamped sleep, without changing clock/time.
      sleep: ms => new Promise(resolve => {
        const s = { startedAt: Date.now(), ms }; sleeps.push(s);
        setTimeout(() => { s.settledAt = Date.now(); resolve(); }, ms);
      }),
      fetchImpl: async (input, init = {}) => {
        const u = new URL(typeof input === 'string' ? input : input.url ?? input.href);
        assert.equal(u.hostname, 'mock.example.com');
        const path = u.pathname.replace('/api/cli/v1', ''), method = init.method ?? 'GET';
        const event = { method, path, at: Date.now() }; trace.push(event);
        if (path === '/projects/project/env') return json({ environments: [
          { name: 'default', url: 'https://example.com', isDefault: true, isTemporary: false },
        ] });
        if (path === '/tests/batch' && method === 'POST') return json({
          results: ids.map((testId, specIndex) => ({ specIndex, testId, status: 'created' })),
          summary: { total: ids.length, created: ids.length, failed: 0 },
        });
        const trigger = /^\/tests\/(test_\d+)\/runs$/.exec(path);
        if (trigger && method === 'POST') {
          const id = trigger[1], attempt = (attempts.get(id) ?? 0) + 1;
          attempts.set(id, attempt); if (!first.has(id)) first.set(id, event.at);
          Object.assign(event, { attempt, idempotencyKey: new Headers(init.headers).get('idempotency-key'),
            sinceFirstTriggerMs: event.at - first.get(id) });
          if (c.retry && id === lastId && attempt === 1) return json({ error: {
            code: 'RATE_LIMITED', message: 'Run trigger rate limit exceeded: 60 triggers per minute per key.',
            nextAction: 'Wait and retry.', requestId: 'fixture', details: { retryAfterSeconds: 1 },
          } }, 429, { 'retry-after': '1' });
          acceptedAt.set(id, event.at);
          return json({ runId: 'run_' + id, status: 'queued', enqueuedAt: new Date(event.at).toISOString(),
            codeVersion: 'v1', targetUrl: 'https://example.com' });
        }
        const poll = /^\/runs\/run_(test_\d+)$/.exec(path);
        if (poll) {
          const id = poll[1], stamp = new Date(acceptedAt.get(id)).toISOString();
          return json({ runId: 'run_' + id, testId: id, projectId: 'project', userId: 'fixture',
            status: 'passed', source: 'cli', createdAt: stamp, startedAt: stamp, finishedAt: stamp,
            codeVersion: 'v1', targetUrl: 'https://example.com', createdFrom: 'cli',
            failedStepIndex: null, failureKind: null, error: null, videoUrl: null,
            stepSummary: { total: 1, completed: 1, passedCount: 1, failedCount: 0 } });
        }
        throw Error('Unexpected route ' + method + ' ' + path);
      },
    };
    const args = ['test', 'create-batch', '--plans', plans, '--run', '--wait',
      '--timeout', String(c.timeout), '--max-concurrency', '50'];
    let exitCode = 0, caught;
    const start = Date.now();
    try { await new Command('testsprite').option('--output <mode>').addCommand(createTestCommand(deps))
      .parseAsync(['--output', 'json', ...args], { from: 'user' }); }
    catch (e) { exitCode = e.exitCode ?? 1; caught = { name: e.name, message: e.message, exitCode }; }
    const payload = JSON.parse(stdout.join('\n'));
    const triggers = trace.filter(x => x.path === '/tests/' + lastId + '/runs');
    const polls = trace.filter(x => x.path === '/runs/run_' + lastId);
    const last = payload.results.find(x => x.testId === lastId);
    assert.equal(payload.results.length, c.count);
    assert.equal(payload.results.filter(x => x.testId !== lastId && x.status === 'passed').length, c.count - 1);
    const throttleSleep = sleeps.find(s => s.ms > 10000);
    const clampedDeadlineUpperBound = c.name.includes('boundary') && throttleSleep
      ? throttleSleep.startedAt + throttleSleep.ms : null;
    const lateTriggerConfirmed = clampedDeadlineUpperBound !== null && triggers.length > 1
      && triggers[1].at >= clampedDeadlineUpperBound;
    if (c.name.includes('boundary')) {
      // Natural Node timer jitter may wake just before or after the boundary.
      // Keep either result; only a POST at/after the measured upper bound is
      // evidence of a late dispatch. Do not turn timing uncertainty into proof.
      assert.equal(triggers.length, 2);
      assert.equal(last.runId, 'run_' + lastId);
      assert.ok(['timeout', 'passed'].includes(last.status));
      if (lateTriggerConfirmed) { assert.equal(polls.length, 0); assert.equal(last.status, 'timeout'); }
    } else if (c.timeout === 2) {
      assert.equal(triggers.length, 1); assert.equal(polls.length, 0);
      assert.equal(last.status, 'timeout'); assert.equal(last.runId, '');
    } else {
      assert.equal(triggers.length, c.retry ? 2 : 1); assert.equal(polls.length, 1);
      assert.equal(exitCode, 0); assert.equal(last.status, 'passed');
      if (c.retry) assert.ok(triggers[1].sinceFirstTriggerMs < c.timeout * 1000);
    }
    if (triggers.length === 2) assert.equal(triggers[0].idempotencyKey, triggers[1].idempotencyKey);
    return { ...c, args, exitCode, error: caught, elapsedMs: Date.now() - start,
      last, lastTriggers: triggers, lastPollCount: polls.length, sleeps,
      clampedDeadlineUpperBound, lateTriggerConfirmed, payload, trace, stderr };
  }));
  console.log(JSON.stringify({ baseline: '988c0355bb05631e9c6c1785e443937ff2a88c0e',
    node: process.version, scope: 'offline HTTP only; real 60s timers; no hosted runs', results }, null, 2));
} finally { await rm(temp, { recursive: true, force: true }); }