Skip to content

Commit 5038691

Browse files
waleedlatif1waleedlatif1
committed
test(chat): prove steering end to end across a real browser, app, worker and databases
Only the model's words are faked, by a local Anthropic-compatible server that holds a response open until a check releases it. A recording proxy between Sim and the worker logs every call and SSE frame and can refuse a path. Each check asserts on the DOM, copilot_messages rows and the model's received requests. chat-steering-e2e.sh boots the disposable stack.
1 parent 4dc38d5 commit 5038691

3 files changed

Lines changed: 1784 additions & 0 deletions

File tree

‎apps/sim/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
"start": "next start",
3131
"test": "vitest run",
3232
"test:scim:e2e": "bun run scripts/test-scim-e2e.ts",
33+
"test:chat-steering:e2e": "bash scripts/chat-steering-e2e.sh run",
3334
"test:sim-links:e2e": "bun run scripts/test-sim-links-e2e.ts",
3435
"test:people:e2e": "bun run scripts/test-people-e2e.ts",
3536
"test:browser-extension:e2e": "bun --no-env-file scripts/test-browser-extension-e2e.ts",

‎apps/sim/scripts/chat-steering-e2e.sh‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
#!/usr/bin/env bash
2+
# Disposable stack for scripts/test-chat-steering-e2e.ts: Postgres and Redis containers, the real
3+
# Mothership worker from a local checkout, and `next dev` for this app. The test itself owns the
4+
# fake model and a recording proxy between the app and the worker.
5+
#
6+
# Usage: chat-steering-e2e.sh <run|up|test|down>
7+
# run up, then the test, then down. Exits with the test's status.
8+
# up leaves the stack running, for iterating on the test.
9+
# test runs the test against a stack that is up. CHAT_STEERING_E2E_ONLY=1,7 picks checks.
10+
# down stops every process and removes every container this stack started.
11+
#
12+
# Required: MOTHERSHIP_WORKER_ROOT (the worker/ directory of a mothership checkout) and
13+
# CHAT_STEERING_E2E_REPORT_PATH. Every child gets a scrubbed environment, so no shell-exported
14+
# provider key or database URL reaches it; the worker's only Anthropic key is a placeholder and its
15+
# base URL is the test's fake model.
16+
set -euo pipefail
17+
18+
command=${1:?usage: chat-steering-e2e.sh <run|up|test|down>}
19+
worker_root=${MOTHERSHIP_WORKER_ROOT:?set MOTHERSHIP_WORKER_ROOT to a mothership worker/ directory}
20+
report_path=${CHAT_STEERING_E2E_REPORT_PATH:?set CHAT_STEERING_E2E_REPORT_PATH}
21+
22+
app_dir=$(cd "$(dirname "$0")/.." && pwd)
23+
db_dir=$(cd "$app_dir/../../packages/db" && pwd)
24+
state_dir=${CHAT_STEERING_E2E_STATE_DIR:-${TMPDIR:-/tmp}/chat-steering-e2e}
25+
mkdir -p "$state_dir"
26+
27+
pg_port=${CHAT_STEERING_E2E_PG_PORT:-55491}
28+
redis_port=${CHAT_STEERING_E2E_REDIS_PORT:-56391}
29+
app_port=${CHAT_STEERING_E2E_APP_PORT:-3471}
30+
worker_port=${CHAT_STEERING_E2E_WORKER_PORT:-3472}
31+
proxy_port=${CHAT_STEERING_E2E_PROXY_PORT:-3473}
32+
model_port=${CHAT_STEERING_E2E_MODEL_PORT:-3474}
33+
34+
pg_name=chat-steering-e2e-pg-$pg_port
35+
redis_name=chat-steering-e2e-redis-$redis_port
36+
app_url=http://127.0.0.1:$app_port
37+
database_url=postgresql://postgres:postgres@127.0.0.1:$pg_port/chat_steering_test
38+
worker_database_url=postgresql://postgres:postgres@127.0.0.1:$pg_port/chat_steering_worker_test
39+
redis_url=redis://127.0.0.1:$redis_port
40+
auth_secret=chat-steering-e2e-secret-at-least-32-characters
41+
internal_secret=chat-steering-e2e-internal-secret-at-least-32
42+
copilot_key=chat-steering-e2e-copilot-key
43+
44+
scrubbed() {
45+
env -i PATH="$PATH" HOME="$HOME" TMPDIR="${TMPDIR:-/tmp}" LANG="${LANG:-en_US.UTF-8}" "$@"
46+
}
47+
48+
stop_group() {
49+
local pid_file=$state_dir/$1.pid
50+
[ -f "$pid_file" ] || return 0
51+
local pid
52+
pid=$(cat "$pid_file")
53+
kill -TERM -- "-$pid" 2>/dev/null || true
54+
for _ in $(seq 1 40); do
55+
kill -0 -- "-$pid" 2>/dev/null || break
56+
sleep 0.25
57+
done
58+
kill -KILL -- "-$pid" 2>/dev/null || true
59+
rm -f "$pid_file"
60+
}
61+
62+
down() {
63+
stop_group app
64+
stop_group worker
65+
docker rm -f "$pg_name" "$redis_name" >/dev/null 2>&1 || true
66+
}
67+
68+
wait_for() {
69+
local label=$1 url=$2 pid_file=$3 seconds=$4
70+
for _ in $(seq 1 "$seconds"); do
71+
if curl -fsS -o /dev/null --max-time 5 "$url"; then return 0; fi
72+
if ! kill -0 "$(cat "$pid_file")" 2>/dev/null; then
73+
echo "$label exited before it became ready; see $state_dir" >&2
74+
return 1
75+
fi
76+
sleep 1
77+
done
78+
echo "$label was not ready after ${seconds}s; see $state_dir" >&2
79+
return 1
80+
}
81+
82+
up() {
83+
docker run -d --rm --name "$pg_name" -e POSTGRES_PASSWORD=postgres \
84+
-p "127.0.0.1:$pg_port:5432" pgvector/pgvector:pg17 -c max_connections=200 >/dev/null
85+
docker run -d --rm --name "$redis_name" -p "127.0.0.1:$redis_port:6379" redis:7-alpine >/dev/null
86+
for _ in $(seq 1 60); do
87+
docker exec "$pg_name" pg_isready -U postgres -h 127.0.0.1 >/dev/null 2>&1 && break
88+
sleep 1
89+
done
90+
docker exec "$pg_name" psql -U postgres -c 'CREATE DATABASE chat_steering_test' >/dev/null
91+
docker exec "$pg_name" psql -U postgres -c 'CREATE DATABASE chat_steering_worker_test' >/dev/null
92+
docker exec "$pg_name" psql -U postgres -d chat_steering_test \
93+
-c 'CREATE EXTENSION vector' -c 'CREATE EXTENSION btree_gin' -c 'CREATE EXTENSION pg_trgm' >/dev/null
94+
95+
(cd "$db_dir" && scrubbed DATABASE_URL="$database_url" bun --no-env-file ./scripts/migrate.ts) \
96+
>"$state_dir/migrate-sim.log" 2>&1
97+
(cd "$worker_root" && scrubbed DATABASE_URL="$worker_database_url" \
98+
bun --no-env-file packages/db/scripts/migrate.ts) >"$state_dir/migrate-worker.log" 2>&1
99+
100+
# Job control gives each background child its own process group, which is what stop_group signals.
101+
set -m
102+
(
103+
cd "$worker_root"
104+
exec nice -n 15 env -i PATH="$PATH" HOME="$HOME" TMPDIR="${TMPDIR:-/tmp}" \
105+
PORT="$worker_port" \
106+
DATABASE_URL="$worker_database_url" \
107+
REDIS_URL="$redis_url" \
108+
SIM_ENDPOINT="$app_url" \
109+
INTERNAL_API_SECRET="$internal_secret" \
110+
COPILOT_INBOUND_API_KEY="$copilot_key" \
111+
ANTHROPIC_API_KEY=fake-model-key-never-sent-to-a-provider \
112+
COPILOT_ANTHROPIC_PROVIDER=anthropic \
113+
COPILOT_ANTHROPIC_BASE_URL="http://127.0.0.1:$model_port/v1" \
114+
COPILOT_MODEL=claude-opus-5-5 \
115+
COPILOT_SUBAGENT_MODEL=claude-opus-5-5 \
116+
COPILOT_TITLE_MODEL=claude-haiku-5-5 \
117+
COPILOT_CONDENSER_MODEL=claude-haiku-5-5 \
118+
COPILOT_SEARCH_MODEL=claude-opus-5-5 \
119+
COPILOT_SHUTDOWN_HANDOFF_MS=1000 \
120+
OTEL_EXPORTER_OTLP_ENDPOINT= \
121+
bun --no-env-file apps/server/src/index.ts
122+
) >"$state_dir/worker.log" 2>&1 &
123+
echo $! >"$state_dir/worker.pid"
124+
wait_for worker "http://127.0.0.1:$worker_port/health" "$state_dir/worker.pid" 60
125+
126+
rm -rf "$app_dir/.next/dev"
127+
(
128+
cd "$app_dir"
129+
exec nice -n 15 env -i PATH="$PATH" HOME="$HOME" TMPDIR="${TMPDIR:-/tmp}" \
130+
NODE_OPTIONS=--max-old-space-size=8192 \
131+
NEXT_TELEMETRY_DISABLED=1 DISABLE_TELEMETRY=true \
132+
DATABASE_URL="$database_url" \
133+
REDIS_URL="$redis_url" \
134+
BETTER_AUTH_SECRET="$auth_secret" \
135+
BETTER_AUTH_URL="$app_url" \
136+
NEXT_PUBLIC_APP_URL="$app_url" \
137+
ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 \
138+
INTERNAL_API_SECRET="$internal_secret" \
139+
COPILOT_API_KEY="$copilot_key" \
140+
SIM_AGENT_API_URL="http://127.0.0.1:$proxy_port" \
141+
MOTHERSHIP_SIM_TRANSPORT=direct \
142+
node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port "$app_port"
143+
) >"$state_dir/app.log" 2>&1 &
144+
echo $! >"$state_dir/app.pid"
145+
set +m
146+
wait_for app "$app_url/api/health" "$state_dir/app.pid" 600
147+
}
148+
149+
run_test() {
150+
cd "$app_dir"
151+
scrubbed \
152+
CHAT_STEERING_E2E_BASE_URL="$app_url" \
153+
CHAT_STEERING_E2E_WORKER_URL="http://127.0.0.1:$worker_port" \
154+
CHAT_STEERING_E2E_PROXY_PORT="$proxy_port" \
155+
CHAT_STEERING_E2E_MODEL_PORT="$model_port" \
156+
CHAT_STEERING_E2E_DATABASE_URL="$database_url" \
157+
CHAT_STEERING_E2E_WORKER_DATABASE_URL="$worker_database_url" \
158+
CHAT_STEERING_E2E_AUTH_SECRET="$auth_secret" \
159+
CHAT_STEERING_E2E_REPORT_PATH="$report_path" \
160+
CHAT_STEERING_E2E_UPLOADS_PATH="$app_dir/uploads" \
161+
${CHAT_STEERING_E2E_ONLY:+CHAT_STEERING_E2E_ONLY="$CHAT_STEERING_E2E_ONLY"} \
162+
bun --no-env-file scripts/test-chat-steering-e2e.ts
163+
}
164+
165+
case "$command" in
166+
up) up ;;
167+
down) down ;;
168+
test) run_test ;;
169+
run)
170+
trap down EXIT
171+
up
172+
run_test
173+
;;
174+
*)
175+
echo "usage: chat-steering-e2e.sh <run|up|test|down>" >&2
176+
exit 2
177+
;;
178+
esac

0 commit comments

Comments
 (0)