[Hackathon] fix(create-batch): rounding remaining wait budget down causes premature timeouts #480

import assert from 'node:assert/strict';
import {mkdtemp,writeFile,rm} from 'node:fs/promises';
import {tmpdir} from 'node:os';
import {join,resolve} from 'node:path';
import {pathToFileURL} from 'node:url';
import {createRequire} from 'node:module';

// Offline HTTP fixture with actual timers: emulate documented long polling,
// returning at min(waitSeconds, completion). Commander/client/poller unchanged.
const root=resolve(process.argv[2]??'work/testsprite-cli');
const require=createRequire(join(root,'package.json'));
const {Command}=require('commander');
const {createTestCommand}=await import(pathToFileURL(join(root,'dist/commands/test.js')));
const temp=await mkdtemp(join(tmpdir(),'testsprite-early-timeout-'));
const delay=ms=>new Promise(resolve=>setTimeout(resolve,Math.max(0,ms)));
const json=(body,status=200,headers={})=>new Response(JSON.stringify(body),{status,headers:{'content-type':'application/json',...headers}});
const scenarios=[
 {name:'fast-run-control',timeout:2,finishMs:800,verdict:'passed',expectedExit:0},
 {name:'budget-not-exhausted-pass',timeout:2,finishMs:1500,verdict:'passed',expectedExit:7},
 {name:'budget-not-exhausted-fail',timeout:2,finishMs:1500,verdict:'failed',expectedExit:7},
 {name:'larger-budget-control',timeout:3,finishMs:1500,verdict:'passed',expectedExit:0},
 {name:'two-member-batch',timeout:2,finishMs:1500,verdict:'passed',count:2,expectedExit:1},
 {name:'after-rate-retry',timeout:3,finishMs:2500,verdict:'passed',retry:true,expectedExit:7},
 {name:'retry-larger-budget-control',timeout:4,finishMs:2500,verdict:'passed',retry:true,expectedExit:0},
];
const results=[];
try{
 for(const s of scenarios){
  const ids=s.count===2?['a','b']:['a'];
  const plans=join(temp,s.name+'.jsonl');
  await writeFile(plans,ids.map(id=>JSON.stringify({projectId:'project',type:'frontend',name:'Case '+id,planSteps:[{type:'action',description:'Open home '+id}]})).join('\n')+'\n');
  const stdout=[],stderr=[],trace=[],starts=new Map(),attempts=new Map(),completedAt=new Map();
  let exitAt=0;
  const deps={env:{TESTSPRITE_API_KEY:'sk-user-offline-fixture',TESTSPRITE_API_URL:'https://mock.example.com'},credentialsPath:join(temp,'absent'),stdout:x=>stdout.push(x),stderr:x=>stderr.push(x),
   fetchImpl:async(input,init={})=>{
    const u=new URL(typeof input==='string'?input:input.url??input.href);assert.equal(u.hostname,'mock.example.com');
    const path=u.pathname.replace('/api/cli/v1',''),method=init.method??'GET';
    const event={method,path,query:Object.fromEntries(u.searchParams),at:Date.now()};trace.push(event);
    if(path==='/projects/project/env')return json({environments:[{name:'default',url:'https://example.com',isDefault:true,isTemporary:false}]});
    if(path==='/tests/batch'&&method==='POST')return json({results:ids.map((id,specIndex)=>({specIndex,testId:id,status:'created'})),summary:{total:ids.length,created:ids.length,failed:0}});
    const trigger=/^\/tests\/(a|b)\/runs$/.exec(path);
    if(trigger&&method==='POST'){
     const id=trigger[1],attempt=(attempts.get(id)??0)+1;attempts.set(id,attempt);
     if(!starts.has(id))starts.set(id,Date.now());
     event.attempt=attempt;event.idempotencyKey=new Headers(init.headers).get('idempotency-key');
     if(s.retry&&attempt===1)return json({error:{code:'RATE_LIMITED',message:'Run trigger rate limit exceeded: 60 triggers per minute per key.',nextAction:'Wait.',requestId:'fixture',details:{retryAfterSeconds:1}}},429,{'retry-after':'1'});
     await delay(250);
     const finishMs=s.count===2&&id==='a'?800:s.finishMs;
     completedAt.set(id,starts.get(id)+finishMs);
     return json({runId:'run_'+id,status:'queued',enqueuedAt:new Date(starts.get(id)).toISOString(),codeVersion:'v1',targetUrl:'https://example.com'});
    }
    const poll=/^\/runs\/run_(a|b)$/.exec(path);
    if(poll){
     const id=poll[1],start=starts.get(id),finish=completedAt.get(id);
     assert.ok(start&&finish);const waitSeconds=Number(u.searchParams.get('waitSeconds')??0);
     await delay(Math.min(Math.max(0,finish-Date.now()),waitSeconds*1000));
     const terminal=Date.now()>=finish,status=terminal?(s.count===2&&id==='a'?'passed':s.verdict):'running';
     event.returnAt=Date.now();event.status=status;
     return json({runId:'run_'+id,testId:id,projectId:'project',userId:'fixture',status,source:'cli',createdAt:new Date(start).toISOString(),startedAt:new Date(start+250).toISOString(),finishedAt:terminal?new Date(finish).toISOString():null,codeVersion:'v1',targetUrl:'https://example.com',createdFrom:'cli',failedStepIndex:status==='failed'?0:null,failureKind:status==='failed'?'assertion':null,error:null,videoUrl:null,stepSummary:{total:1,completed:terminal?1:0,passedCount:status==='passed'?1:0,failedCount:status==='failed'?1:0}});
    }
    throw Error('Unexpected fixture route '+method+' '+path);
   }};
  const args=['--output','json','test','create-batch','--plans',plans,'--run','--wait','--timeout',String(s.timeout),'--max-concurrency','2'];
  let exitCode=0,error;
  try{await new Command('testsprite').option('--output <mode>').addCommand(createTestCommand(deps)).parseAsync(args,{from:'user'});}
  catch(e){exitCode=e.exitCode??1;error={name:e.name,code:e.code,message:e.message,exitCode:e.exitCode};}
  exitAt=Date.now();assert.equal(exitCode,s.expectedExit,s.name+': '+JSON.stringify(error));
  const payload=JSON.parse(stdout.join('\n'));assert.equal(payload.results.length,ids.length);
  const timings=ids.map(id=>({testId:id,elapsedMs:exitAt-starts.get(id),budgetMs:s.timeout*1000,fixtureCompletionMs:completedAt.get(id)-starts.get(id),remainingAtExitMs:starts.get(id)+s.timeout*1000-exitAt}));
  for(const row of payload.results){
   assert.equal(row.runId,'run_'+row.testId);
   const affected=s.expectedExit!==0&&!(s.count===2&&row.testId==='a');
   assert.equal(row.status,affected?'timeout':'passed');
   if(affected){const timing=timings.find(t=>t.testId===row.testId);assert.ok(timing.remainingAtExitMs>400,JSON.stringify(timing));assert.ok(timing.elapsedMs<timing.fixtureCompletionMs);assert.ok(timing.fixtureCompletionMs<timing.budgetMs);}
  }
  if(s.retry){const triggers=trace.filter(x=>x.method==='POST'&&x.path.endsWith('/runs'));assert.equal(triggers.length,2);assert.equal(triggers[0].idempotencyKey,triggers[1].idempotencyKey);}
  // Read the same fixture after its real terminal instant, proving the eventual
  // status and run ID rather than relying only on a scheduled completion time.
  await delay(Math.max(...completedAt.values())-Date.now()+5);
  const observed=[];
  for(const id of ids){const response=await deps.fetchImpl('https://mock.example.com/api/cli/v1/runs/run_'+id);const row=await response.json();assert.equal(row.status,s.count===2&&id==='a'?'passed':s.verdict);observed.push({runId:row.runId,status:row.status,finishedAt:row.finishedAt});}
  results.push({name:s.name,args,exitCode,error,timings,payload,observed,trace,stderr});
 }
 console.log(JSON.stringify({baseline:'988c0355bb05631e9c6c1785e443937ff2a88c0e',node:process.version,scope:'offline real-timer long-poll fixture, not hosted observation',results},null,2));
}finally{await rm(temp,{recursive:true,force:true});}