Skip to content

Commit cced77a

Browse files
committed
test(chat): keep the E2E undeliverable setup undeliverable under disposition reads
Scenarios 10-12 cut only the worker's steer route. Reads now go to the disposition route, which said the ended run never got the message, so it went out as the next turn, correctly, and never showed Not delivered. The setup now cuts both routes, and scenario 13 proves the new path: steer unreachable, read reachable, sent once as the next turn.
1 parent 5530dd0 commit cced77a

1 file changed

Lines changed: 78 additions & 9 deletions

File tree

‎apps/sim/scripts/test-chat-steering-e2e.ts‎

Lines changed: 78 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,22 @@ const TURN_PATH = '/api/mothership/chat'
439439
const STEER_PATH = '/api/copilot/chat/steer'
440440
const DISMISS_PATH = '/api/copilot/chat/steer/dismiss'
441441
const STEER_WORKER_PATH = '/api/streams/steer/v2'
442+
const DISPOSITION_WORKER_PATH = '/api/streams/steer/v2/disposition'
443+
444+
/**
445+
* Cuts Sim off from every worker route that could deliver a steering message or say what became of
446+
* it. With only the steer route cut off, a turn's end still reads that the worker never got the
447+
* message and sends it on as a turn, which is scenario 13.
448+
*/
449+
function cutSteeringWorker() {
450+
proxy.refuse(STEER_WORKER_PATH)
451+
proxy.refuse(DISPOSITION_WORKER_PATH)
452+
}
453+
454+
function reachSteeringWorker() {
455+
proxy.allow(STEER_WORKER_PATH)
456+
proxy.allow(DISPOSITION_WORKER_PATH)
457+
}
442458

443459
async function openSurface(): Promise<Surface> {
444460
assert(context)
@@ -500,14 +516,15 @@ const notDelivered = (page: Page) => page.getByRole('status').filter({ hasText:
500516
const NOT_DELIVERED_TIMEOUT_MS = 240_000
501517

502518
/**
503-
* Starts a turn, steers `steer` into it with the worker's steer route unreachable, and lets the
504-
* turn finish, so the steer runs out of attempts and shows as not delivered.
519+
* Starts a turn, steers `steer` into it with the worker's steer and read routes unreachable, and
520+
* lets the turn finish, so nothing can say what became of it: its reads run out of attempts and it
521+
* shows as not delivered.
505522
*/
506523
async function undeliveredSteer(surface: Surface, first: string, steer: string, before: string) {
507524
const { page } = surface
508525
const gate = createGate()
509526
const chatId = await startTurn(surface, first, { text: before, gate })
510-
proxy.refuse(STEER_WORKER_PATH)
527+
cutSteeringWorker()
511528
await send(page, steer)
512529
await expect(page.getByText(steer, { exact: true })).toBeVisible({ timeout: 2_000 })
513530
await expect
@@ -1294,7 +1311,7 @@ const scenarios: Scenario[] = [
12941311
await page.screenshot({ path: shot })
12951312
evidence.screenshot = shot
12961313
model.enqueue({ text: reply })
1297-
proxy.allow(STEER_WORKER_PATH)
1314+
reachSteeringWorker()
12981315
await notDelivered(page).getByRole('button', { name: 'Retry', exact: true }).click()
12991316
await model.waitFor(steer)
13001317
await expect(page.getByText(reply)).toBeVisible()
@@ -1309,7 +1326,7 @@ const scenarios: Scenario[] = [
13091326
await assertOneBubble(page, steer)
13101327
await expect(notDelivered(page)).toHaveCount(0)
13111328
} finally {
1312-
proxy.allow(STEER_WORKER_PATH)
1329+
reachSteeringWorker()
13131330
}
13141331
},
13151332
},
@@ -1337,7 +1354,7 @@ const scenarios: Scenario[] = [
13371354
await assertOneBubble(page, steer)
13381355
assert.equal((await userRows(chatId, steer)).length, 1, 'It keeps its record')
13391356

1340-
proxy.allow(STEER_WORKER_PATH)
1357+
reachSteeringWorker()
13411358
const [answered] = await dismiss()
13421359
assert.equal(answered.status(), 200)
13431360
evidence.dismissOutcome = await answered.json()
@@ -1373,7 +1390,7 @@ const scenarios: Scenario[] = [
13731390
await expect(page.getByText(steer, { exact: true })).toHaveCount(0)
13741391
await assertOneBubble(page, later)
13751392
} finally {
1376-
proxy.allow(STEER_WORKER_PATH)
1393+
reachSteeringWorker()
13771394
}
13781395
},
13791396
},
@@ -1410,7 +1427,7 @@ const scenarios: Scenario[] = [
14101427
assert.equal(model.containing(failed).length, 0, 'A reload does not send it')
14111428

14121429
model.enqueue({ text: retryReply })
1413-
proxy.allow(STEER_WORKER_PATH)
1430+
reachSteeringWorker()
14141431
await notDelivered(page).getByRole('button', { name: 'Retry', exact: true }).click()
14151432
await model.waitFor(failed)
14161433
await expect(page.getByText(retryReply)).toBeVisible()
@@ -1428,7 +1445,59 @@ const scenarios: Scenario[] = [
14281445
await assertOneBubble(page, later)
14291446
await expect(notDelivered(page)).toHaveCount(0)
14301447
} finally {
1431-
proxy.allow(STEER_WORKER_PATH)
1448+
reachSteeringWorker()
1449+
}
1450+
},
1451+
},
1452+
{
1453+
id: '13',
1454+
name: 'unreachable steer: the read says the run never got it, so it goes out once as the next turn',
1455+
run: async (surface, evidence) => {
1456+
const { page } = surface
1457+
const first = 'S13 start the turn'
1458+
const steer = 'S13 the steer never reaches the worker'
1459+
const after = 'S13-answered-as-its-own-turn'
1460+
const gate = createGate()
1461+
const chatId = await startTurn(surface, first, { text: 'S13-before', gate })
1462+
try {
1463+
proxy.refuse(STEER_WORKER_PATH)
1464+
model.enqueue({ text: after })
1465+
await send(page, steer)
1466+
await expect(page.getByText(steer, { exact: true })).toBeVisible({ timeout: 2_000 })
1467+
await expect
1468+
.poll(() => surface.steerPosts.find((post) => post.body.content === steer)?.status)
1469+
.toBeGreaterThanOrEqual(500)
1470+
gate.release()
1471+
await model.waitFor(steer)
1472+
await expect(page.getByText(after)).toBeVisible()
1473+
await waitForIdle(page)
1474+
await expect(notDelivered(page)).toHaveCount(0)
1475+
await assertDeliveredAsTurn(surface, chatId, steer)
1476+
const reads = proxy.calls.filter(
1477+
(call) => call.path === DISPOSITION_WORKER_PATH && !call.refused
1478+
)
1479+
evidence.dispositionReads = reads.map((call) => toRecord(call.response).status)
1480+
assert(
1481+
reads.some((call) => toRecord(call.response).status === 'not_applied'),
1482+
'The worker read it as never applied'
1483+
)
1484+
assert(
1485+
proxy.steers(steer).every((call) => call.refused),
1486+
'No steer of it reached the worker'
1487+
)
1488+
assert.equal(
1489+
surface.turnPosts.find((post) => post.body.message === steer)?.body.steeringFallback,
1490+
true
1491+
)
1492+
assert.deepEqual(
1493+
(await userRows(chatId)).map((candidate) => candidate.content),
1494+
[first, steer]
1495+
)
1496+
await reload(page)
1497+
await assertOneBubble(page, steer)
1498+
await expect(notDelivered(page)).toHaveCount(0)
1499+
} finally {
1500+
reachSteeringWorker()
14321501
}
14331502
},
14341503
},

0 commit comments

Comments
 (0)