From d78fd03134bd6d9f89791e312e83bed36d6c9baf Mon Sep 17 00:00:00 2001 From: developmentconexus Date: Sun, 8 Nov 2026 16:25:40 -0300 Subject: [PATCH 1/9] =?UTF-8?q?ci:=20sonda=20tempor=C3=A1ria=20do=20arquiv?= =?UTF-8?q?o=20que=20n=C3=A3o=20sai=20(removida=20antes=20do=20PR)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/hang-probe.yml | 78 ++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .github/workflows/hang-probe.yml diff --git a/.github/workflows/hang-probe.yml b/.github/workflows/hang-probe.yml new file mode 100644 index 000000000..92d004a0d --- /dev/null +++ b/.github/workflows/hang-probe.yml @@ -0,0 +1,78 @@ +name: hang probe (temporary) + +# Temporary: catches the agents.eval.test.ts child that never exits and dumps its threads. Removed +# before the pull request is opened. +on: + push: + branches: [feat/conexus-ci-server-hang] + +permissions: + contents: read + +env: + CONEXUS_TEST_RUNTIME_PASSWORD: ci-runtime + +jobs: + probe: + runs-on: ubuntu-24.04 + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + lane: [1, 2] + services: + postgres: + image: postgres:18@sha256:74935e72241653ca55e0414067e6d8763aceb8a810eb51b452253ec3dcfc4336 + env: + POSTGRES_PASSWORD: ci-superuser + ports: + - 5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/install # zizmor: ignore[self-repository] probe + - uses: ./.github/actions/test-database # zizmor: ignore[self-repository] probe + with: + port: ${{ job.services.postgres.ports['5432'] }} + - run: sudo apt-get install -y gdb >/dev/null 2>&1 || true + - run: npm run build -w @conexus/server + - name: loop the file + working-directory: apps/server + env: + MASTRA_AGENT_THREAD_LEASE_TTL_MS: '3000' + run: | + hangs=0 + for i in $(seq 1 60); do + rm -rf /tmp/ncc-probe + export NODE_COMPILE_CACHE=/tmp/ncc-probe + NODE_OPTIONS="--trace-exit --report-on-signal --report-signal=SIGUSR2 --report-directory=/tmp/reports" \ + node --test --test-concurrency=1 --test-force-exit --test-timeout=120000 \ + --test-reporter=../../scripts/test-reporter.ts src/builder/agents/agents.eval.test.ts \ + > /tmp/run.log 2>&1 & + pid=$! + for s in $(seq 1 40); do kill -0 $pid 2>/dev/null || break; sleep 1; done + if kill -0 $pid 2>/dev/null; then + hangs=$((hangs+1)) + echo "::group::HANG run $i" + cat /tmp/run.log + for c in $(pgrep -P $pid); do + echo "== child $c"; grep -E 'State|Threads' /proc/$c/status + for t in /proc/$c/task/*; do echo "$(basename $t) $(cat $t/comm) $(awk '{print $3}' $t/stat) wchan=$(cat $t/wchan)"; done + sudo bash -c "for t in /proc/$c/task/*; do echo \"-- kernel stack \$t\"; cat \$t/stack; done" + sudo gdb -p $c -batch -ex 'thread apply all bt' 2>&1 | tail -150 + ls -l /proc/$c/fd + kill -USR2 $c; sleep 3; ls /tmp/reports || true + done + echo "::endgroup::" + pkill -KILL -P $pid; kill -KILL $pid + fi + wait $pid 2>/dev/null + grep -c 'Exited the environment' /tmp/run.log | sed "s/^/run $i trace-exit lines: /" + done + echo "hangs=$hangs/60" From ebb05e16960bf799f196a85a693d7474ef3a1997 Mon Sep 17 00:00:00 2001 From: developmentconexus Date: Sun, 8 Nov 2026 16:27:32 -0300 Subject: [PATCH 2/9] =?UTF-8?q?ci:=20sonda,=20o=20la=C3=A7o=20segue=20sem?= =?UTF-8?q?=20achado=20do=20grep?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/hang-probe.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/hang-probe.yml b/.github/workflows/hang-probe.yml index 92d004a0d..98b1dee42 100644 --- a/.github/workflows/hang-probe.yml +++ b/.github/workflows/hang-probe.yml @@ -47,6 +47,7 @@ jobs: env: MASTRA_AGENT_THREAD_LEASE_TTL_MS: '3000' run: | + set +e hangs=0 for i in $(seq 1 60); do rm -rf /tmp/ncc-probe @@ -73,6 +74,7 @@ jobs: pkill -KILL -P $pid; kill -KILL $pid fi wait $pid 2>/dev/null - grep -c 'Exited the environment' /tmp/run.log | sed "s/^/run $i trace-exit lines: /" + echo "run $i trace-exit lines: $(grep -c 'Exited the environment' /tmp/run.log || true)" + [ $i = 1 ] && cat /tmp/run.log done echo "hangs=$hangs/60" From 63ba1a0b5052342240b1fb6bc5eecaf99bdd239f Mon Sep 17 00:00:00 2001 From: developmentconexus Date: Sun, 8 Nov 2026 16:34:07 -0300 Subject: [PATCH 3/9] =?UTF-8?q?ci:=20sonda=20roda=20a=20fatia=203/6=20e=20?= =?UTF-8?q?despeja=20pai=20e=20filho=20no=20sil=C3=AAncio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/hang-probe.yml | 72 ++++++++++++++++---------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/.github/workflows/hang-probe.yml b/.github/workflows/hang-probe.yml index 98b1dee42..5dffd88b6 100644 --- a/.github/workflows/hang-probe.yml +++ b/.github/workflows/hang-probe.yml @@ -1,6 +1,6 @@ name: hang probe (temporary) -# Temporary: catches the agents.eval.test.ts child that never exits and dumps its threads. Removed +# Temporary: catches the shard that stops printing and dumps the runner and its child. Removed # before the pull request is opened. on: push: @@ -11,15 +11,16 @@ permissions: env: CONEXUS_TEST_RUNTIME_PASSWORD: ci-runtime + NODE_COMPILE_CACHE: /tmp/node-compile-cache jobs: probe: runs-on: ubuntu-24.04 - timeout-minutes: 40 + timeout-minutes: 45 strategy: fail-fast: false matrix: - lane: [1, 2] + lane: [full-1, full-2, full-3, head-1, head-2] services: postgres: image: postgres:18@sha256:74935e72241653ca55e0414067e6d8763aceb8a810eb51b452253ec3dcfc4336 @@ -41,40 +42,41 @@ jobs: with: port: ${{ job.services.postgres.ports['5432'] }} - run: sudo apt-get install -y gdb >/dev/null 2>&1 || true - - run: npm run build -w @conexus/server - - name: loop the file - working-directory: apps/server + - name: run the shard, dump on silence env: - MASTRA_AGENT_THREAD_LEASE_TTL_MS: '3000' + LANE: ${{ matrix.lane }} + TEST_SHARD: 3/6 run: | set +e - hangs=0 - for i in $(seq 1 60); do - rm -rf /tmp/ncc-probe - export NODE_COMPILE_CACHE=/tmp/ncc-probe - NODE_OPTIONS="--trace-exit --report-on-signal --report-signal=SIGUSR2 --report-directory=/tmp/reports" \ - node --test --test-concurrency=1 --test-force-exit --test-timeout=120000 \ - --test-reporter=../../scripts/test-reporter.ts src/builder/agents/agents.eval.test.ts \ - > /tmp/run.log 2>&1 & - pid=$! - for s in $(seq 1 40); do kill -0 $pid 2>/dev/null || break; sleep 1; done - if kill -0 $pid 2>/dev/null; then - hangs=$((hangs+1)) - echo "::group::HANG run $i" - cat /tmp/run.log - for c in $(pgrep -P $pid); do - echo "== child $c"; grep -E 'State|Threads' /proc/$c/status - for t in /proc/$c/task/*; do echo "$(basename $t) $(cat $t/comm) $(awk '{print $3}' $t/stat) wchan=$(cat $t/wchan)"; done - sudo bash -c "for t in /proc/$c/task/*; do echo \"-- kernel stack \$t\"; cat \$t/stack; done" - sudo gdb -p $c -batch -ex 'thread apply all bt' 2>&1 | tail -150 - ls -l /proc/$c/fd - kill -USR2 $c; sleep 3; ls /tmp/reports || true - done - echo "::endgroup::" - pkill -KILL -P $pid; kill -KILL $pid + dump() { + local root=$1 + echo "::group::SILENT $2" + tail -5 /tmp/run.log + ps -eo pid,ppid,stat,etime,wchan:30,comm --forest | grep -E 'node|npm|sh|MainThread' | grep -v grep + for p in $(pgrep -f -- '--test-reporter' ) $(pgrep -f 'NODE_TEST_CONTEXT' ); do :; done + for p in $(pgrep node); do + echo "== pid $p ppid $(awk '{print $4}' /proc/$p/stat) $(tr '\0' ' ' < /proc/$p/cmdline | cut -c1-150)" + grep -E 'State|Threads|SigPnd|ShdPnd|SigBlk|SigCgt' /proc/$p/status + for t in /proc/$p/task/*; do echo " $(basename $t) $(cat $t/comm) $(awk '{print $3}' $t/stat) wchan=$(cat $t/wchan)"; done + sudo bash -c "for t in /proc/$p/task/*; do echo \"-- kernel stack \$t\"; cat \$t/stack; done" | head -80 + sudo gdb -p $p -batch -ex 'thread apply all bt' 2>&1 | grep -v '^\[New LWP' | tail -120 + ls -l /proc/$p/fd | tail -40 + done + echo "::endgroup::" + } + runs=40; [ "${LANE#full}" != "$LANE" ] && runs=2 + for i in $(seq 1 $runs); do + rm -rf /tmp/node-compile-cache + if [ "${LANE#full}" != "$LANE" ]; then + npm test -w @conexus/server > /tmp/run.log 2>&1 & + else + (cd apps/server && npm run build >/dev/null 2>&1 && MASTRA_AGENT_THREAD_LEASE_TTL_MS=3000 node --test --test-concurrency=1 --test-force-exit --test-timeout=120000 --test-reporter=../../scripts/test-reporter.ts $(node ../../scripts/server-shards.ts 3/6 | head -4)) > /tmp/run.log 2>&1 & fi - wait $pid 2>/dev/null - echo "run $i trace-exit lines: $(grep -c 'Exited the environment' /tmp/run.log || true)" - [ $i = 1 ] && cat /tmp/run.log + pid=$! + while kill -0 $pid 2>/dev/null; do + sleep 5 + age=$(( $(date +%s) - $(stat -c %Y /tmp/run.log) )) + if [ $age -gt 150 ]; then dump $pid "run $i"; pkill -KILL -f -- '--test'; break; fi + done + wait $pid; echo "run $i exit=$? lines=$(wc -l < /tmp/run.log)" done - echo "hangs=$hangs/60" From af1ded9daa63f81017c9b92d00a020b75ba80a8f Mon Sep 17 00:00:00 2001 From: developmentconexus Date: Sun, 8 Nov 2026 17:20:31 -0300 Subject: [PATCH 4/9] ci: sonda despeja os processos MainThread --- .github/workflows/hang-probe.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/hang-probe.yml b/.github/workflows/hang-probe.yml index 5dffd88b6..6cb6ea4d2 100644 --- a/.github/workflows/hang-probe.yml +++ b/.github/workflows/hang-probe.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - lane: [full-1, full-2, full-3, head-1, head-2] + lane: [head-1, head-2, head-3] services: postgres: image: postgres:18@sha256:74935e72241653ca55e0414067e6d8763aceb8a810eb51b452253ec3dcfc4336 @@ -53,8 +53,7 @@ jobs: echo "::group::SILENT $2" tail -5 /tmp/run.log ps -eo pid,ppid,stat,etime,wchan:30,comm --forest | grep -E 'node|npm|sh|MainThread' | grep -v grep - for p in $(pgrep -f -- '--test-reporter' ) $(pgrep -f 'NODE_TEST_CONTEXT' ); do :; done - for p in $(pgrep node); do + for p in $(pgrep -x MainThread); do echo "== pid $p ppid $(awk '{print $4}' /proc/$p/stat) $(tr '\0' ' ' < /proc/$p/cmdline | cut -c1-150)" grep -E 'State|Threads|SigPnd|ShdPnd|SigBlk|SigCgt' /proc/$p/status for t in /proc/$p/task/*; do echo " $(basename $t) $(cat $t/comm) $(awk '{print $3}' $t/stat) wchan=$(cat $t/wchan)"; done @@ -64,7 +63,7 @@ jobs: done echo "::endgroup::" } - runs=40; [ "${LANE#full}" != "$LANE" ] && runs=2 + runs=25; [ "${LANE#full}" != "$LANE" ] && runs=2 for i in $(seq 1 $runs); do rm -rf /tmp/node-compile-cache if [ "${LANE#full}" != "$LANE" ]; then @@ -76,7 +75,7 @@ jobs: while kill -0 $pid 2>/dev/null; do sleep 5 age=$(( $(date +%s) - $(stat -c %Y /tmp/run.log) )) - if [ $age -gt 150 ]; then dump $pid "run $i"; pkill -KILL -f -- '--test'; break; fi + if [ $age -gt 60 ]; then dump $pid "run $i"; pkill -KILL -f -- '--test'; break; fi done wait $pid; echo "run $i exit=$? lines=$(wc -l < /tmp/run.log)" done From f4238e091cc0ca51430bfc4bf004a0e094523198 Mon Sep 17 00:00:00 2001 From: developmentconexus Date: Sun, 8 Nov 2026 17:44:38 -0300 Subject: [PATCH 5/9] ci: sonda com --no-concurrent-sparkplug --- .github/workflows/hang-probe.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/hang-probe.yml b/.github/workflows/hang-probe.yml index 6cb6ea4d2..b3c83ea05 100644 --- a/.github/workflows/hang-probe.yml +++ b/.github/workflows/hang-probe.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - lane: [head-1, head-2, head-3] + lane: [head-1, head-2, head-3, head-4] services: postgres: image: postgres:18@sha256:74935e72241653ca55e0414067e6d8763aceb8a810eb51b452253ec3dcfc4336 @@ -69,7 +69,7 @@ jobs: if [ "${LANE#full}" != "$LANE" ]; then npm test -w @conexus/server > /tmp/run.log 2>&1 & else - (cd apps/server && npm run build >/dev/null 2>&1 && MASTRA_AGENT_THREAD_LEASE_TTL_MS=3000 node --test --test-concurrency=1 --test-force-exit --test-timeout=120000 --test-reporter=../../scripts/test-reporter.ts $(node ../../scripts/server-shards.ts 3/6 | head -4)) > /tmp/run.log 2>&1 & + (cd apps/server && npm run build >/dev/null 2>&1 && MASTRA_AGENT_THREAD_LEASE_TTL_MS=3000 node --no-concurrent-sparkplug --test --test-concurrency=1 --test-force-exit --test-timeout=120000 --test-reporter=../../scripts/test-reporter.ts $(node ../../scripts/server-shards.ts 3/6 | head -4)) > /tmp/run.log 2>&1 & fi pid=$! while kill -0 $pid 2>/dev/null; do From 97bcc7944032c0b51e15f29c630478d491580c53 Mon Sep 17 00:00:00 2001 From: developmentconexus Date: Sun, 8 Nov 2026 17:48:53 -0300 Subject: [PATCH 6/9] =?UTF-8?q?ci:=20sonda=20compara=20as=20duas=20flags?= =?UTF-8?q?=20com=20a=20sa=C3=ADda=20sem=20force-exit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 5 +++++ .github/workflows/hang-probe.yml | 12 ++++++------ apps/server/package.json | 2 +- package.json | 2 +- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d75d3a17..760034af7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,6 +136,11 @@ jobs: # and migrate.test.ts changes the roles of the whole server. The shards are the parallelism. # `--test-force-exit` ends a file's process once its tests are done: a failed test that leaves a # turn parked on a card keeps a timer alive, and its file used to hold the shard to the timeout. + # That exit is `process.exit()`, which on Node 24.21 can deadlock joining a V8 worker parked + # in a concurrent Sparkplug compile until the main thread collects garbage (nodejs/node#54918): + # agents.eval.test.ts, which exits right after loading, held shard 3 to the timeout in about one + # run in eleven. `--no-concurrent-sparkplug` in the `test` and `e2e` commands compiles on the + # main thread instead. Revalidate when nodejs/node#66171 lands in the pinned Node. - run: npm test -w @conexus/server env: TEST_SHARD: ${{ matrix.shard }}/6 diff --git a/.github/workflows/hang-probe.yml b/.github/workflows/hang-probe.yml index b3c83ea05..d02e24623 100644 --- a/.github/workflows/hang-probe.yml +++ b/.github/workflows/hang-probe.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - lane: [head-1, head-2, head-3, head-4] + lane: [flags-1, flags-2, flags-3, noexit-1, noexit-2, noexit-3, noexit-4, noexit-5, noexit-6] services: postgres: image: postgres:18@sha256:74935e72241653ca55e0414067e6d8763aceb8a810eb51b452253ec3dcfc4336 @@ -63,19 +63,19 @@ jobs: done echo "::endgroup::" } - runs=25; [ "${LANE#full}" != "$LANE" ] && runs=2 + runs=25; [ "${LANE#noexit}" != "$LANE" ] && runs=1 for i in $(seq 1 $runs); do rm -rf /tmp/node-compile-cache - if [ "${LANE#full}" != "$LANE" ]; then - npm test -w @conexus/server > /tmp/run.log 2>&1 & + if [ "${LANE#noexit}" != "$LANE" ]; then + (cd apps/server && npm run build >/dev/null 2>&1 && MASTRA_AGENT_THREAD_LEASE_TTL_MS=3000 node --test --test-concurrency=1 --test-timeout=120000 --test-reporter=../../scripts/test-reporter.ts $(node ../../scripts/server-shards.ts ${LANE#noexit-}/6)) > /tmp/run.log 2>&1 & else - (cd apps/server && npm run build >/dev/null 2>&1 && MASTRA_AGENT_THREAD_LEASE_TTL_MS=3000 node --no-concurrent-sparkplug --test --test-concurrency=1 --test-force-exit --test-timeout=120000 --test-reporter=../../scripts/test-reporter.ts $(node ../../scripts/server-shards.ts 3/6 | head -4)) > /tmp/run.log 2>&1 & + (cd apps/server && npm run build >/dev/null 2>&1 && MASTRA_AGENT_THREAD_LEASE_TTL_MS=3000 node --no-concurrent-sparkplug --no-maglev-build-code-on-background --test --test-concurrency=1 --test-force-exit --test-timeout=120000 --test-reporter=../../scripts/test-reporter.ts $(node ../../scripts/server-shards.ts 3/6 | head -4)) > /tmp/run.log 2>&1 & fi pid=$! while kill -0 $pid 2>/dev/null; do sleep 5 age=$(( $(date +%s) - $(stat -c %Y /tmp/run.log) )) - if [ $age -gt 60 ]; then dump $pid "run $i"; pkill -KILL -f -- '--test'; break; fi + if [ $age -gt 120 ]; then dump $pid "run $i"; pkill -KILL -f -- '--test'; break; fi done wait $pid; echo "run $i exit=$? lines=$(wc -l < /tmp/run.log)" done diff --git a/apps/server/package.json b/apps/server/package.json index 981f1ca80..f86ae0d9d 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -10,7 +10,7 @@ "pretest": "npm run build", "generate": "drizzle-kit generate", "typecheck": "tsc -p tsconfig.json", - "test": "MASTRA_AGENT_THREAD_LEASE_TTL_MS=3000 node --test --test-concurrency=1 --test-force-exit --test-timeout=120000 --test-reporter=../../scripts/test-reporter.ts $(node ../../scripts/server-shards.ts ${TEST_SHARD:-all})" + "test": "MASTRA_AGENT_THREAD_LEASE_TTL_MS=3000 node --no-concurrent-sparkplug --test --test-concurrency=1 --test-force-exit --test-timeout=120000 --test-reporter=../../scripts/test-reporter.ts $(node ../../scripts/server-shards.ts ${TEST_SHARD:-all})" }, "dependencies": { "@ai-sdk/anthropic": "3.0.133", diff --git a/package.json b/package.json index 61bc3fb45..98dcb337f 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "build": "npm run build --workspaces --if-present", "app:pack": "node scripts/app-pack.ts tests/apps/recados dist/apps/recados-1 && node scripts/app-pack.ts tests/apps/recados dist/apps/recados-2 tests/apps/recados/cor-2 && node scripts/app-pack.ts tests/apps/recados dist/apps/recados-nao-inicia tests/apps/recados-nao-inicia && node scripts/app-pack.ts tests/apps/recados dist/apps/recados-defeito tests/apps/recados-defeito && node scripts/app-pack.ts tests/apps/recados dist/apps/recados-laco tests/apps/recados-laco", "pree2e": "npm run build -w @conexus/server && npm run build -w @conexus/web && npm run app:pack && npm exec -w @conexus/web -- playwright-core install chromium-headless-shell", - "e2e": "node --test --test-force-exit --test-concurrency=1 --test-timeout=300000 --test-reporter=./scripts/test-reporter.ts tests/e2e/invitation.test.ts tests/e2e/client.test.ts tests/e2e/projects.test.ts tests/e2e/two-companies.test.ts tests/e2e/remove-person.test.ts tests/e2e/app.test.ts tests/e2e/app-open.test.ts tests/e2e/versions.test.ts tests/e2e/refusals.test.ts tests/e2e/model-accounts.test.ts tests/e2e/conversation.test.ts tests/e2e/defaults.test.ts tests/e2e/house.test.ts tests/e2e/builder-writes.test.ts tests/e2e/builder-builds.test.ts tests/e2e/builder-native.test.ts tests/e2e/first-request.test.ts tests/e2e/phone-rail.test.ts", + "e2e": "node --no-concurrent-sparkplug --test --test-force-exit --test-concurrency=1 --test-timeout=300000 --test-reporter=./scripts/test-reporter.ts tests/e2e/invitation.test.ts tests/e2e/client.test.ts tests/e2e/projects.test.ts tests/e2e/two-companies.test.ts tests/e2e/remove-person.test.ts tests/e2e/app.test.ts tests/e2e/app-open.test.ts tests/e2e/versions.test.ts tests/e2e/refusals.test.ts tests/e2e/model-accounts.test.ts tests/e2e/conversation.test.ts tests/e2e/defaults.test.ts tests/e2e/house.test.ts tests/e2e/builder-writes.test.ts tests/e2e/builder-builds.test.ts tests/e2e/builder-native.test.ts tests/e2e/first-request.test.ts tests/e2e/phone-rail.test.ts", "test:docs:mercado-livre": "npm run test:docs:mercado-livre -w @conexus/connectors", "test:image": "node --test --test-timeout=120000 --test-reporter=./scripts/test-reporter.ts tests/e2e/invitation-image.test.ts", "test:scripts": "node --test --test-timeout=120000 --test-reporter=./scripts/test-reporter.ts scripts/*.test.ts" From ec1458debb0852d989a2ad42fa8908cde43ab59e Mon Sep 17 00:00:00 2001 From: developmentconexus Date: Sun, 8 Nov 2026 18:26:17 -0300 Subject: [PATCH 7/9] =?UTF-8?q?test(server):=20os=20arquivos=20saem=20sozi?= =?UTF-8?q?nhos,=20sem=20--test-force-exit;=20within=20n=C3=A3o=20segura?= =?UTF-8?q?=20o=20processo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 10 +++------- apps/server/package.json | 2 +- apps/server/src/builder/conversation/test-questions.ts | 7 +++++-- package.json | 2 +- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 760034af7..551355954 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,13 +134,9 @@ jobs: port: ${{ job.services.postgres.ports['5432'] }} # One file at a time in a job: files that share CPU starve each other's real-time margins # and migrate.test.ts changes the roles of the whole server. The shards are the parallelism. - # `--test-force-exit` ends a file's process once its tests are done: a failed test that leaves a - # turn parked on a card keeps a timer alive, and its file used to hold the shard to the timeout. - # That exit is `process.exit()`, which on Node 24.21 can deadlock joining a V8 worker parked - # in a concurrent Sparkplug compile until the main thread collects garbage (nodejs/node#54918): - # agents.eval.test.ts, which exits right after loading, held shard 3 to the timeout in about one - # run in eleven. `--no-concurrent-sparkplug` in the `test` and `e2e` commands compiles on the - # main thread instead. Revalidate when nodejs/node#66171 lands in the pinned Node. + # A file's process ends when nothing holds it: no `--test-force-exit`. Its `process.exit()` can + # deadlock on Node 24.21 joining a V8 compile worker parked until the main thread collects + # garbage (nodejs/node#54918), and held shard 3 to the timeout in about one run in eleven. - run: npm test -w @conexus/server env: TEST_SHARD: ${{ matrix.shard }}/6 diff --git a/apps/server/package.json b/apps/server/package.json index f86ae0d9d..e1c99b71e 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -10,7 +10,7 @@ "pretest": "npm run build", "generate": "drizzle-kit generate", "typecheck": "tsc -p tsconfig.json", - "test": "MASTRA_AGENT_THREAD_LEASE_TTL_MS=3000 node --no-concurrent-sparkplug --test --test-concurrency=1 --test-force-exit --test-timeout=120000 --test-reporter=../../scripts/test-reporter.ts $(node ../../scripts/server-shards.ts ${TEST_SHARD:-all})" + "test": "MASTRA_AGENT_THREAD_LEASE_TTL_MS=3000 node --test --test-concurrency=1 --test-timeout=120000 --test-reporter=../../scripts/test-reporter.ts $(node ../../scripts/server-shards.ts ${TEST_SHARD:-all})" }, "dependencies": { "@ai-sdk/anthropic": "3.0.133", diff --git a/apps/server/src/builder/conversation/test-questions.ts b/apps/server/src/builder/conversation/test-questions.ts index 48a01bf15..453c5043e 100644 --- a/apps/server/src/builder/conversation/test-questions.ts +++ b/apps/server/src/builder/conversation/test-questions.ts @@ -138,9 +138,12 @@ export function invocations(parts: MastraMessagePart[], toolCallId: string) { /** The parked runs in Mastra's database: one run keeps a snapshot of its loop and of its execution. */ export const PARKED = `select distinct run_id from mastra.mastra_workflow_snapshot where snapshot->>'status' = 'suspended'` -/** What `said` resolves to within `ms`, or `undefined`: a stream that says nothing fails the test, not hangs it. */ +/** + * What `said` resolves to within `ms`, or `undefined`: a stream that says nothing fails the test, not + * hangs it. The deadline does not hold the file's process once `said` has answered. + */ export async function within(ms: number, said: Promise): Promise { - return Promise.race([said, sleep(ms).then(() => undefined)]) + return Promise.race([said, sleep(ms, undefined, { ref: false })]) } /** What a call that Mastra closed without an answer reads as: the states of its `tool-invocation` part. */ diff --git a/package.json b/package.json index 98dcb337f..61bc3fb45 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "build": "npm run build --workspaces --if-present", "app:pack": "node scripts/app-pack.ts tests/apps/recados dist/apps/recados-1 && node scripts/app-pack.ts tests/apps/recados dist/apps/recados-2 tests/apps/recados/cor-2 && node scripts/app-pack.ts tests/apps/recados dist/apps/recados-nao-inicia tests/apps/recados-nao-inicia && node scripts/app-pack.ts tests/apps/recados dist/apps/recados-defeito tests/apps/recados-defeito && node scripts/app-pack.ts tests/apps/recados dist/apps/recados-laco tests/apps/recados-laco", "pree2e": "npm run build -w @conexus/server && npm run build -w @conexus/web && npm run app:pack && npm exec -w @conexus/web -- playwright-core install chromium-headless-shell", - "e2e": "node --no-concurrent-sparkplug --test --test-force-exit --test-concurrency=1 --test-timeout=300000 --test-reporter=./scripts/test-reporter.ts tests/e2e/invitation.test.ts tests/e2e/client.test.ts tests/e2e/projects.test.ts tests/e2e/two-companies.test.ts tests/e2e/remove-person.test.ts tests/e2e/app.test.ts tests/e2e/app-open.test.ts tests/e2e/versions.test.ts tests/e2e/refusals.test.ts tests/e2e/model-accounts.test.ts tests/e2e/conversation.test.ts tests/e2e/defaults.test.ts tests/e2e/house.test.ts tests/e2e/builder-writes.test.ts tests/e2e/builder-builds.test.ts tests/e2e/builder-native.test.ts tests/e2e/first-request.test.ts tests/e2e/phone-rail.test.ts", + "e2e": "node --test --test-force-exit --test-concurrency=1 --test-timeout=300000 --test-reporter=./scripts/test-reporter.ts tests/e2e/invitation.test.ts tests/e2e/client.test.ts tests/e2e/projects.test.ts tests/e2e/two-companies.test.ts tests/e2e/remove-person.test.ts tests/e2e/app.test.ts tests/e2e/app-open.test.ts tests/e2e/versions.test.ts tests/e2e/refusals.test.ts tests/e2e/model-accounts.test.ts tests/e2e/conversation.test.ts tests/e2e/defaults.test.ts tests/e2e/house.test.ts tests/e2e/builder-writes.test.ts tests/e2e/builder-builds.test.ts tests/e2e/builder-native.test.ts tests/e2e/first-request.test.ts tests/e2e/phone-rail.test.ts", "test:docs:mercado-livre": "npm run test:docs:mercado-livre -w @conexus/connectors", "test:image": "node --test --test-timeout=120000 --test-reporter=./scripts/test-reporter.ts tests/e2e/invitation-image.test.ts", "test:scripts": "node --test --test-timeout=120000 --test-reporter=./scripts/test-reporter.ts scripts/*.test.ts" From 8de839bfba2a8681e36efab0f3a8e9839ec26098 Mon Sep 17 00:00:00 2001 From: developmentconexus Date: Sun, 8 Nov 2026 18:26:53 -0300 Subject: [PATCH 8/9] =?UTF-8?q?ci:=20sonda=20prova=20as=20fatias,=20a=20ca?= =?UTF-8?q?be=C3=A7a=20da=203=20e=20o=20e2e=20sem=20force-exit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/hang-probe.yml | 68 +++++++++++++++----------------- 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/.github/workflows/hang-probe.yml b/.github/workflows/hang-probe.yml index d02e24623..0c3536af2 100644 --- a/.github/workflows/hang-probe.yml +++ b/.github/workflows/hang-probe.yml @@ -1,7 +1,7 @@ name: hang probe (temporary) -# Temporary: catches the shard that stops printing and dumps the runner and its child. Removed -# before the pull request is opened. +# Temporary: proves the shards and e2e without --test-force-exit. Removed before the pull request +# is opened. on: push: branches: [feat/conexus-ci-server-hang] @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - lane: [flags-1, flags-2, flags-3, noexit-1, noexit-2, noexit-3, noexit-4, noexit-5, noexit-6] + lane: [head-1, head-2, head-3, shard-1, shard-2, shard-3, shard-4, shard-5, shard-6, e2e] services: postgres: image: postgres:18@sha256:74935e72241653ca55e0414067e6d8763aceb8a810eb51b452253ec3dcfc4336 @@ -41,41 +41,35 @@ jobs: - uses: ./.github/actions/test-database # zizmor: ignore[self-repository] probe with: port: ${{ job.services.postgres.ports['5432'] }} - - run: sudo apt-get install -y gdb >/dev/null 2>&1 || true - - name: run the shard, dump on silence + - name: run env: LANE: ${{ matrix.lane }} - TEST_SHARD: 3/6 run: | set +e - dump() { - local root=$1 - echo "::group::SILENT $2" - tail -5 /tmp/run.log - ps -eo pid,ppid,stat,etime,wchan:30,comm --forest | grep -E 'node|npm|sh|MainThread' | grep -v grep - for p in $(pgrep -x MainThread); do - echo "== pid $p ppid $(awk '{print $4}' /proc/$p/stat) $(tr '\0' ' ' < /proc/$p/cmdline | cut -c1-150)" - grep -E 'State|Threads|SigPnd|ShdPnd|SigBlk|SigCgt' /proc/$p/status - for t in /proc/$p/task/*; do echo " $(basename $t) $(cat $t/comm) $(awk '{print $3}' $t/stat) wchan=$(cat $t/wchan)"; done - sudo bash -c "for t in /proc/$p/task/*; do echo \"-- kernel stack \$t\"; cat \$t/stack; done" | head -80 - sudo gdb -p $p -batch -ex 'thread apply all bt' 2>&1 | grep -v '^\[New LWP' | tail -120 - ls -l /proc/$p/fd | tail -40 - done - echo "::endgroup::" - } - runs=25; [ "${LANE#noexit}" != "$LANE" ] && runs=1 - for i in $(seq 1 $runs); do - rm -rf /tmp/node-compile-cache - if [ "${LANE#noexit}" != "$LANE" ]; then - (cd apps/server && npm run build >/dev/null 2>&1 && MASTRA_AGENT_THREAD_LEASE_TTL_MS=3000 node --test --test-concurrency=1 --test-timeout=120000 --test-reporter=../../scripts/test-reporter.ts $(node ../../scripts/server-shards.ts ${LANE#noexit-}/6)) > /tmp/run.log 2>&1 & - else - (cd apps/server && npm run build >/dev/null 2>&1 && MASTRA_AGENT_THREAD_LEASE_TTL_MS=3000 node --no-concurrent-sparkplug --no-maglev-build-code-on-background --test --test-concurrency=1 --test-force-exit --test-timeout=120000 --test-reporter=../../scripts/test-reporter.ts $(node ../../scripts/server-shards.ts 3/6 | head -4)) > /tmp/run.log 2>&1 & - fi - pid=$! - while kill -0 $pid 2>/dev/null; do - sleep 5 - age=$(( $(date +%s) - $(stat -c %Y /tmp/run.log) )) - if [ $age -gt 120 ]; then dump $pid "run $i"; pkill -KILL -f -- '--test'; break; fi - done - wait $pid; echo "run $i exit=$? lines=$(wc -l < /tmp/run.log)" - done + case "$LANE" in + e2e) + sed -i 's/"e2e": "node --test --test-force-exit /"e2e": "node --test /' package.json + echo "force-exit left in package.json: $(grep -c 'force-exit' package.json)" + start=$(date +%s); npm run e2e; echo "e2e exit=$? wall=$(( $(date +%s)-start ))s" ;; + shard-*) + export GITHUB_STEP_SUMMARY=/tmp/summary.md TEST_SHARD=${LANE#shard-}/6 + start=$(date +%s); npm test -w @conexus/server > /tmp/run.log 2>&1; rc=$? + wall=$(( $(date +%s)-start )) + files=$(grep -cE '^\| [0-9.]+ \| src/' /tmp/run.log) + tests=$(grep -E '^\| [0-9.]+ \| src/' /tmp/run.log | awk -F'|' '{s+=$2} END {print int(s)}') + tail -70 /tmp/run.log + echo "shard $TEST_SHARD exit=$rc wall=${wall}s files=$files test-seconds=$tests" ;; + head-*) + hangs=0 + for i in $(seq 1 25); do + rm -rf /tmp/node-compile-cache + (cd apps/server && npm run build >/dev/null 2>&1 && MASTRA_AGENT_THREAD_LEASE_TTL_MS=3000 node --test --test-concurrency=1 --test-timeout=120000 --test-reporter=../../scripts/test-reporter.ts $(node ../../scripts/server-shards.ts 3/6 | head -4)) > /tmp/run.log 2>&1 & + pid=$! + while kill -0 $pid 2>/dev/null; do + sleep 5 + if [ $(( $(date +%s) - $(stat -c %Y /tmp/run.log) )) -gt 120 ]; then hangs=$((hangs+1)); echo "HANG run $i"; tail -5 /tmp/run.log; pkill -KILL -f -- '--test'; break; fi + done + wait $pid; echo "run $i exit=$?" + done + echo "hangs=$hangs/25" ;; + esac From 975826e8f6df1118e2487f19f6c2d2dc5c0d94c8 Mon Sep 17 00:00:00 2001 From: developmentconexus Date: Sun, 8 Nov 2026 18:50:23 -0300 Subject: [PATCH 9/9] =?UTF-8?q?test(server):=20os=20prazos=20de=20conversa?= =?UTF-8?q?tion-preview=20e=20code-tool=20n=C3=A3o=20seguram=20o=20process?= =?UTF-8?q?o;=20sai=20a=20sonda=20tempor=C3=A1ria?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/hang-probe.yml | 75 ------------------- .../src/builder/agents/code-tool.test.ts | 4 +- .../conversation/conversation-preview.test.ts | 2 +- 3 files changed, 4 insertions(+), 77 deletions(-) delete mode 100644 .github/workflows/hang-probe.yml diff --git a/.github/workflows/hang-probe.yml b/.github/workflows/hang-probe.yml deleted file mode 100644 index 0c3536af2..000000000 --- a/.github/workflows/hang-probe.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: hang probe (temporary) - -# Temporary: proves the shards and e2e without --test-force-exit. Removed before the pull request -# is opened. -on: - push: - branches: [feat/conexus-ci-server-hang] - -permissions: - contents: read - -env: - CONEXUS_TEST_RUNTIME_PASSWORD: ci-runtime - NODE_COMPILE_CACHE: /tmp/node-compile-cache - -jobs: - probe: - runs-on: ubuntu-24.04 - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - lane: [head-1, head-2, head-3, shard-1, shard-2, shard-3, shard-4, shard-5, shard-6, e2e] - services: - postgres: - image: postgres:18@sha256:74935e72241653ca55e0414067e6d8763aceb8a810eb51b452253ec3dcfc4336 - env: - POSTGRES_PASSWORD: ci-superuser - ports: - - 5432 - options: >- - --health-cmd "pg_isready -U postgres" - --health-interval 5s - --health-timeout 5s - --health-retries 10 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: ./.github/actions/install # zizmor: ignore[self-repository] probe - - uses: ./.github/actions/test-database # zizmor: ignore[self-repository] probe - with: - port: ${{ job.services.postgres.ports['5432'] }} - - name: run - env: - LANE: ${{ matrix.lane }} - run: | - set +e - case "$LANE" in - e2e) - sed -i 's/"e2e": "node --test --test-force-exit /"e2e": "node --test /' package.json - echo "force-exit left in package.json: $(grep -c 'force-exit' package.json)" - start=$(date +%s); npm run e2e; echo "e2e exit=$? wall=$(( $(date +%s)-start ))s" ;; - shard-*) - export GITHUB_STEP_SUMMARY=/tmp/summary.md TEST_SHARD=${LANE#shard-}/6 - start=$(date +%s); npm test -w @conexus/server > /tmp/run.log 2>&1; rc=$? - wall=$(( $(date +%s)-start )) - files=$(grep -cE '^\| [0-9.]+ \| src/' /tmp/run.log) - tests=$(grep -E '^\| [0-9.]+ \| src/' /tmp/run.log | awk -F'|' '{s+=$2} END {print int(s)}') - tail -70 /tmp/run.log - echo "shard $TEST_SHARD exit=$rc wall=${wall}s files=$files test-seconds=$tests" ;; - head-*) - hangs=0 - for i in $(seq 1 25); do - rm -rf /tmp/node-compile-cache - (cd apps/server && npm run build >/dev/null 2>&1 && MASTRA_AGENT_THREAD_LEASE_TTL_MS=3000 node --test --test-concurrency=1 --test-timeout=120000 --test-reporter=../../scripts/test-reporter.ts $(node ../../scripts/server-shards.ts 3/6 | head -4)) > /tmp/run.log 2>&1 & - pid=$! - while kill -0 $pid 2>/dev/null; do - sleep 5 - if [ $(( $(date +%s) - $(stat -c %Y /tmp/run.log) )) -gt 120 ]; then hangs=$((hangs+1)); echo "HANG run $i"; tail -5 /tmp/run.log; pkill -KILL -f -- '--test'; break; fi - done - wait $pid; echo "run $i exit=$?" - done - echo "hangs=$hangs/25" ;; - esac diff --git a/apps/server/src/builder/agents/code-tool.test.ts b/apps/server/src/builder/agents/code-tool.test.ts index 6b95e9a7c..9bc91e11b 100644 --- a/apps/server/src/builder/agents/code-tool.test.ts +++ b/apps/server/src/builder/agents/code-tool.test.ts @@ -162,7 +162,9 @@ test('o turno parado enquanto o Worker sobe de novo solta a subida: o sinal do t const turn = on.conversation.say('E setembro?') await held.arrived assert.equal((await stopped(on.hub, on.ana, on.url)).statusCode, 200) - const patience = new Promise((resolve) => setTimeout(resolve, 3000, 'STILL_WAITING')) + const patience = new Promise((resolve) => + setTimeout(resolve, 3000, 'STILL_WAITING').unref(), + ) assert.equal(await Promise.race([held.ended, patience]), 'ABORTED', 'the upload kept waiting') assert.equal(endOf(await turn), 'aborted') assert.deepEqual(on.hub.tools.workers(), [], 'the stopped upload left nothing in the namespace') diff --git a/apps/server/src/builder/conversation/conversation-preview.test.ts b/apps/server/src/builder/conversation/conversation-preview.test.ts index 8a3543360..9389abdee 100644 --- a/apps/server/src/builder/conversation/conversation-preview.test.ts +++ b/apps/server/src/builder/conversation/conversation-preview.test.ts @@ -63,7 +63,7 @@ test('a versão não espera o envio: com o envio parado, o fluxo já tem a vers while (!oursIn(before).includes('data-conexus-version')) before.push(await stream.next()) })() await held.arrived - await Promise.race([reading, setTimeout(5_000)]) + await Promise.race([reading, setTimeout(5_000, undefined, { ref: false })]) assert.ok(oursIn(before).includes('data-conexus-version'), 'told while the send is held') assert.ok(oursIn(before).includes('preview:sending')) assert.ok(!oursIn(before).includes('agent_end'))