From fe4c131cf5eb00dc267009dd99935480febde1bf Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Mon, 2 Nov 2026 06:22:01 +0800 Subject: [PATCH] hotfix: Gracefully stop interrupted native app commands Let interrupted native applications finish bounded cleanup and publish output before exact-container forced removal. Preserve existing live-run recovery, stdin/EOF, argv, exit status and public mapped guards. Add a real packaged-CLI six-case regression to Linux CI and retain the separate RDA consumer request. --- ...+native-command-graceful-cancellation.yaml | 2 + .github/workflows/ci.yml | 13 ++ docs/BACKLOG.md | 10 +- docs/consumers/README.md | 1 + docs/consumers/rda.md | 96 +++++++++ .../rda/reploy-cancellation-hotfix-request.md | 72 +++++++ internal/dockerdeploy/compose.go | 1 + internal/dockerdeploy/config.go | 70 ++++++- .../dockerdeploy/current_app_command_run.go | 1 + .../current_app_command_run_test.go | 22 ++ .../native_command_cancellation_test.go | 129 ++++++++++++ noxfile.py | 1 + tools/e2e/native_cancellation.py | 198 ++++++++++++++++++ 13 files changed, 609 insertions(+), 7 deletions(-) create mode 100644 .changes/unreleased/+native-command-graceful-cancellation.yaml create mode 100644 docs/consumers/rda.md create mode 100644 docs/consumers/supporting/rda/reploy-cancellation-hotfix-request.md create mode 100644 internal/dockerdeploy/native_command_cancellation_test.go create mode 100644 tools/e2e/native_cancellation.py diff --git a/.changes/unreleased/+native-command-graceful-cancellation.yaml b/.changes/unreleased/+native-command-graceful-cancellation.yaml new file mode 100644 index 00000000..3ad305ec --- /dev/null +++ b/.changes/unreleased/+native-command-graceful-cancellation.yaml @@ -0,0 +1,2 @@ +kind: Fixed +body: Give interrupted native app commands a bounded graceful container stop before forced cleanup, so applications can finish cleanup and retain results in the output directory. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5090727..c53c7896 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,19 @@ jobs: go test -v -count=1 -timeout 15m ./internal/dockerdeploy -run '^TestReployControlledSessionContractDockerIntegration$' + - name: Prove native launcher cancellation and stdin through the packaged CLI + run: | + python tools/build_reploy --target linux-amd64 --outdir "$RUNNER_TEMP/native-cancellation-bin" + python tools/e2e/native_cancellation.py --reploy "$RUNNER_TEMP/native-cancellation-bin/linux-amd64/reploy" --output-dir "$RUNNER_TEMP/native-cancellation-evidence" + + - name: Retain native launcher cancellation evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: native-cancellation-${{ github.sha }} + if-no-files-found: ignore + path: ${{ runner.temp }}/native-cancellation-evidence/ + - name: Retain external portable-tool observations if: always() uses: actions/upload-artifact@v4 diff --git a/docs/BACKLOG.md b/docs/BACKLOG.md index 9ec16d45..49da6bd2 100644 --- a/docs/BACKLOG.md +++ b/docs/BACKLOG.md @@ -1,6 +1,6 @@ --- status: Active -updated: 2026-10-30 +updated: 2026-11-01 summary: Active planning surface for Reploy design and implementation gaps. --- @@ -34,6 +34,14 @@ This file is the day-to-day queue for design and implementation gaps. ## Pre-release +- [ ] Deliver the RDA graceful launcher-cancellation hotfix after F/G design + and slice finalization, before implementing milestones F and G. + Follow [the consumer request and acceptance checks](consumers/rda.md). + Diagnose signal delivery and teardown ordering; retain application output + during bounded cleanup, then force containment when necessary. Use an + independent hotfix PR and complete its review/check gates before returning + to milestone delivery. Delivery is active; the F/G design PR is approved. + - [ ] `P2` Evaluate an operator-authorized host-session service profile for Flux packaged SSH. Follow [the consumer requirements and evidence](consumers/flux-ssh.md). Agree the smallest disposable-host qualification scope before design; diff --git a/docs/consumers/README.md b/docs/consumers/README.md index 2eb3f689..2cb10e3f 100644 --- a/docs/consumers/README.md +++ b/docs/consumers/README.md @@ -5,6 +5,7 @@ it into a subdirectory only when that improves navigation. | Consumer | Current need | Status | Next step | | --- | --- | --- | --- | +| [RDA](rda.md) | Graceful launcher cancellation that permits application cleanup and result publication | Failure reproduced; native-only fix under verification | Complete real-CLI regressions, independent review and standalone PR approval before F/G delivery | | [Flux SSH](flux-ssh.md) | Distribution-maintained packaged SSH with direct host sessions | Request evaluated; separate host-authority capability, design unapproved | Agree minimum disposable-host scope and required authority; preserve E priority | | [OmegaFlow](omegaflow.md) | Public preparation inspection and admission of the inspected workload; separate private-endpoint dependency | Requirements/HLD approved; LLD unfinished and paused for E | Resume detailed engineering when prioritized | | [Dingo](dingo.md) | Development workspace and robot deployment capabilities | Historical needs recorded; current gaps need revalidation | Evaluate and prioritize against current Reploy behavior | diff --git a/docs/consumers/rda.md b/docs/consumers/rda.md new file mode 100644 index 00000000..f968ff94 --- /dev/null +++ b/docs/consumers/rda.md @@ -0,0 +1,96 @@ +--- +status: Native-only cancellation fix under verification; independent PR pending +updated: 2026-11-02 +summary: Preserve bounded native-application cancellation cleanup and result publication when the actual Reploy launcher is interrupted. +--- + +# RDA + +## Requested outcome + +When the operator interrupts the actual Reploy CLI, signal the native container +workload through its init process and retain its output mount while it handles +termination, cleans up and atomically publishes its result. Allow a finite grace +period, then forcibly contain a nonresponsive workload. The interrupted launcher +must exit nonzero and leave no live container. + +Source: [retained cancellation hotfix request](supporting/rda/reploy-cancellation-hotfix-request.md), +received on 2026-11-01 from +[the RDA checkout](/home/omry/dev/rda/docs/plans/rda/supporting/implementation-plan/reploy-cancellation-hotfix-request.md). +The retained request is a byte-for-byte copy, SHA-256 +`b41bba760e85b4d5804df1f88534596ba98f5efbfa0e563c60be66458cb78f24`. +It is separate from the previously merged stdin hotfix. The source document has +since moved and added later reproductions and RDA's POC disposition. The retained +original stays unchanged; the requested Reploy behavior is the same. + +## Consumer-reported evidence + +RDA reports the failure against Reploy source +`ae1459aaba98be7223b05f2632504dbf6aced2d6`, using the qualified executable +and hashes recorded in the retained request: + +- `headless-cancel-09`: run the native blueprint command through + `reploy app --dir --output-dir run`, with application + entry `tini -- /usr/local/bin/python -m rda`. After `step: connect`, send SIGINT + to the actual Reploy launcher PID. The launcher exits `1` after approximately + `0.414` seconds; `run/result.json` is absent. +- `application-cancel-16`: signalling the application supervisor directly while + leaving Reploy running produces `reason_code: cancelled`, null action/artifact, + and exit `130` in approximately `1.76` seconds. This is a positive control for + application cleanup, not acceptance of launcher interruption. + +The app reserves five seconds for owned cleanup and result publication. That is +consumer evidence, not a selected Reploy-wide grace-period default. The raw records are +located under RDA's `temp/delivery-preparation/rda3/`, as listed in the retained +request. Reploy has now inspected those records and reproduced the +failure with an isolated native fixture. Cancellation killed the attached engine +CLI, then forced removal before the application could finalize its output. + +## Acceptance checks for the hotfix + +Use a credential-free native fixture under `tini` and the normal output mount: + +1. Interrupt the actual Reploy CLI, without directly signalling the application. + Verify signal delivery through init, child cleanup, atomic result publication, + retained output, a nonzero CLI result and removal of the owned container. +2. Repeat with a workload that ignores graceful termination. Verify forced + containment completes within a finite bound and leaves no live container. +3. Preserve piped stdin, EOF, native argv and exit-status behavior. Keep genuine + launcher failures distinguishable from a completed application cancellation. + +An external coordinator, direct application signalling, or a prewritten fake +cancellation result does not satisfy the launcher-interruption check. Retain +failed or uncertain cleanup as an explicit failure, rather than reporting +successful containment or inventing an application result. + +## Status and next action + +The request is recorded in [the backlog](../BACKLOG.md). A native-only lifecycle +fix passes six actual-CLI cases: SIGINT, SIGTERM, terminal process-group SIGINT, +an application that ignores termination, piped input and empty-input EOF. Graceful +cases permit child cleanup and atomic result publication; interrupted commands +exit nonzero and their owned containers are removed. Native argv and exit status +remain intact. Independent review and PR approval remain pending. + +Run the repeatable fixture with +`python3 tools/e2e/native_cancellation.py --reploy /path/to/packaged/reploy --output-dir /new/evidence/path`. +It uses a Python image and `tini`, without credentials or an RDA deployment. +The tested setup is Linux amd64 inside WSL2, kernel +`6.18.54.1-microsoft-standard-WSL2`, Docker Desktop `4.93.0 (240920)`, +Docker client/engine `29.8.1`, and Go `1.26.8`. These POSIX signal tests do not +qualify native Windows, macOS or Podman interruption. + +The fix requests the engine's ordinary container stop before forced removal. +It preserves the configured stop signal/grace, with a 30-second host limit on +the stop request and a separate five-second removal limit. It does not establish +a five-second application grace default. `--output-dir` keeps application-written +files after cancellation. Failed commands still do not publish `--output-file`. + +On 2026-11-01, the maintainer set the continuation order: finish the F/G design +and implementation slices, deliver this hotfix, then deliver milestones F and G. +The F/G design PR is now approved. The hotfix is based on main independently of +that design commit. Preserve the approved F/G work while completing this hotfix. + +Complete an independent hotfix PR with its required review and exact-head CI gates. No merge, +release or live-service change is authorized. Design decisions and qualification +gates remain required; the sequence does not pre-approve unresolved behavior. diff --git a/docs/consumers/supporting/rda/reploy-cancellation-hotfix-request.md b/docs/consumers/supporting/rda/reploy-cancellation-hotfix-request.md new file mode 100644 index 00000000..3cb3bec3 --- /dev/null +++ b/docs/consumers/supporting/rda/reploy-cancellation-hotfix-request.md @@ -0,0 +1,72 @@ +# Reploy hotfix request: allow native app cancellation cleanup + +## Observed problem + +Interrupting the qualified Reploy launcher during an RDA connection stops the +run before the application supervisor can publish its cancellation result. +The launcher exits nonzero and contains the container, but `run/result.json` +does not exist. This prevents acceptance of RDA's application cancellation +contract through the actual launcher. + +This is separate from the previously merged stdin fix. Reploy has not been +modified for this investigation. + +## Reproduction evidence + +- Reploy source: `ae1459aaba98be7223b05f2632504dbf6aced2d6`. +- Qualified executable SHA-256: + `32a39bfbb83470dbd30080b5c19f45151842ff08d817045aa18dd02bafbc1530`. +- Command: the native `run` command from `runtime/rda.blueprint.yaml`, launched + with `reploy app --dir --output-dir run`. +- Application entry: `tini -- /usr/local/bin/python -m rda`. +- Case: `headless-cancel-09`, isolated Windows mock only. +- Injection: SIGINT sent to the actual Reploy launcher PID after the app + published `step: connect`. No application PID was signalled directly. +- Observed launcher exit: `1`, approximately `0.414` seconds after SIGINT. +- Observed application result: absent. No verified artifact was claimed. +- Local records: + `temp/delivery-preparation/rda3/headless-cancel-09-verification.json` and + `temp/delivery-preparation/rda3/headless-cancel-09.log`. + +The RDA supervisor installs SIGINT/SIGTERM handlers, uses an independent +execution clock, and reserves five seconds for owned cleanup and atomic result +publication. Its offline cancellation tests pass. A live positive control, +`application-cancel-16`, sent SIGINT directly to the application supervisor +while leaving Reploy running. That control published `reason_code: cancelled` +with null action/artifact and exited `130` in about `1.76` seconds. Its record +is `temp/delivery-preparation/rda3/application-cancel-16-verification.json`. +Direct application signalling does not resolve or substitute for the actual +launcher-SIGINT failure. The launcher test does not establish that the app's +handlers were reached. + +## Requested behavior + +On launcher interruption, deliver a graceful termination signal to the native +container workload through its init process. Keep its output mount available +while the application handles the signal, cleans up and publishes its result. +Allow a finite grace period, then forcibly contain a nonresponsive workload. +Preserve a nonzero interrupted launcher result and leave no live container. + +The exact lifecycle implementation belongs to Reploy. The current evidence +suggests cancellation or teardown completes before application finalization; +it does not identify the precise failing internal call. + +## Suggested regression checks + +Use a credential-free native test app under `tini`. Its termination handler +should clean up a child and atomically write a small result under the normal +output mount. Interrupt the actual Reploy CLI. Verify the signal is delivered, +the final result survives, the CLI exits nonzero, and the container is removed. +Also test a workload that ignores termination: forced cleanup must remain +bounded. Keep piped stdin, EOF, native argv and exit-status checks passing. + +## RDA disposition + +External gap in graceful launcher cancellation; no acceptance of that behavior +is claimed. Whether it blocks the POC delivery is awaiting owner disposition, +because the plan also recognises launcher failures and unsupported interruption +exit handling. This classification does not fix the missing application result. +The RDA implementation +continues to use its one in-container supervisor and worker. An extra external +coordinator or a prewritten fake cancellation result is not a truth fix. +Other failure, timeout, headless and debug checks can proceed independently. diff --git a/internal/dockerdeploy/compose.go b/internal/dockerdeploy/compose.go index 950e2fe1..845dcd9d 100644 --- a/internal/dockerdeploy/compose.go +++ b/internal/dockerdeploy/compose.go @@ -33,6 +33,7 @@ type CommandSpec struct { type RunOptions struct { workloadLifecycle *workloadLifecycleV1 + gracefulCancellation bool Context context.Context Stdin io.Reader Stdout io.Writer diff --git a/internal/dockerdeploy/config.go b/internal/dockerdeploy/config.go index d50cc12e..265b0186 100644 --- a/internal/dockerdeploy/config.go +++ b/internal/dockerdeploy/config.go @@ -2,6 +2,7 @@ package dockerdeploy import ( "context" + "errors" "fmt" "io" "os" @@ -182,7 +183,13 @@ func runTemporaryContainerCommandWithCleanupHookV1( if parent == nil { parent = context.Background() } - ctx, cancel := context.WithCancel(parent) + // A native application's attached CLI must survive host cancellation long + // enough for the engine to stop its init and the app to finalize output. + attachedParent := parent + if runOptions.gracefulCancellation { + attachedParent = context.WithoutCancel(parent) + } + ctx, cancel := context.WithCancel(attachedParent) defer cancel() temporaryRunOptions := runOptions temporaryRunOptions.Context = ctx @@ -192,15 +199,31 @@ func runTemporaryContainerCommandWithCleanupHookV1( done := make(chan error, 1) go func() { done <- run(runSpec, temporaryRunOptions) }() var runErr error + parentDone := parent.Done() + if !runOptions.gracefulCancellation { + parentDone = nil + } select { case runErr = <-done: + // The attached engine CLI can also receive a terminal's process-group + // signal, or fail before the host cancellation notification is handled. + // Its failure does not prove that the container has stopped. + if runErr != nil && runOptions.gracefulCancellation { + runErr = errors.Join(runErr, stopNativeCommandContainerV1(run, cleanupSpec, runOptions, parent)) + } case sig := <-signals: - cancel() - if err := <-done; err != nil { - runErr = fmt.Errorf("interrupted by %s: %w", sig, err) + if runOptions.gracefulCancellation { + runErr = stopInterruptedNativeCommandV1(run, cleanupSpec, runOptions, parent, cancel, done) } else { - runErr = fmt.Errorf("interrupted by %s", sig) + cancel() + if err := <-done; err != nil { + runErr = fmt.Errorf("interrupted by %s: %w", sig, err) + } else { + runErr = fmt.Errorf("interrupted by %s", sig) + } } + case <-parentDone: + runErr = stopInterruptedNativeCommandV1(run, cleanupSpec, runOptions, parent, cancel, done) } var cleanupErr error var cleanup func(context.Context) error @@ -217,7 +240,13 @@ func runTemporaryContainerCommandWithCleanupHookV1( if beforeCleanup != nil { beforeCleanup(runErr, cleanup) } else { - cleanupErr = cleanup(context.Background()) + cleanupContext := context.Background() + if runOptions.gracefulCancellation { + var stop context.CancelFunc + cleanupContext, stop = context.WithTimeout(context.WithoutCancel(parent), defaultDockerPreflightTimeout) + defer stop() + } + cleanupErr = cleanup(cleanupContext) } } if runErr != nil && cleanupErr != nil { @@ -229,6 +258,35 @@ func runTemporaryContainerCommandWithCleanupHookV1( return cleanupErr } +// Use the existing engine stop policy (including its configured signal and +// grace), capped by a host deadline if the engine does not respond. Forced +// removal still follows, and its failures retain normal live-run recovery. +func stopInterruptedNativeCommandV1(run temporaryCommandRunner, cleanupSpec CommandSpec, options RunOptions, parent context.Context, cancel context.CancelFunc, done <-chan error) error { + stopErr := stopNativeCommandContainerV1(run, cleanupSpec, options, parent) + cancel() + attachedErr := <-done + return errors.Join(errors.New("native app command interrupted"), stopErr, attachedErr) +} + +func stopNativeCommandContainerV1(run temporaryCommandRunner, cleanupSpec CommandSpec, options RunOptions, parent context.Context) error { + var stopErr error + if len(cleanupSpec.Args) > 0 && reflectDockerCleanupCommandV1(cleanupSpec, cleanupSpec.Args[len(cleanupSpec.Args)-1]) { + stopSpec := cleanupSpec + stopSpec.Args = TemporaryContainerStopCommand(cleanupSpec.Args[len(cleanupSpec.Args)-1]).Args + stopContext, stop := context.WithTimeout(context.WithoutCancel(parent), 30*time.Second) + options.Context, options.Stdin, options.Stdout, options.Stderr = stopContext, nil, nil, nil + stopErr = run(stopSpec, options) + stop() + if stopErr != nil && isMissingContainerCleanupError(stopErr) { + stopErr = nil + } + } + if stopErr != nil { + stopErr = fmt.Errorf("graceful native command stop: %w", stopErr) + } + return stopErr +} + func AppCommandList(options AppCommandListOptions) (AppCommandListResult, error) { if options.Dir == "" { options.Dir = DefaultDeploymentDir diff --git a/internal/dockerdeploy/current_app_command_run.go b/internal/dockerdeploy/current_app_command_run.go index 3cc26fb0..eada32eb 100644 --- a/internal/dockerdeploy/current_app_command_run.go +++ b/internal/dockerdeploy/current_app_command_run.go @@ -217,6 +217,7 @@ func runCurrentAppCommandV1(ctx context.Context, input CurrentAppCommandRunInput runOptions.Stderr = newDeploymentOutputWriter(runOptions.Stderr, label, color) } runOptions.Context = ctx + runOptions.gracefulCancellation = command.Native published := PublishedRuntimeContainerInput{ Operation: operation, Store: store, Environment: document.Environment.ID, DeploymentDir: dir, DockerPlan: planned.Docker, Invocation: invocation, diff --git a/internal/dockerdeploy/current_app_command_run_test.go b/internal/dockerdeploy/current_app_command_run_test.go index 27a113e7..9ec465b5 100644 --- a/internal/dockerdeploy/current_app_command_run_test.go +++ b/internal/dockerdeploy/current_app_command_run_test.go @@ -50,6 +50,28 @@ func TestRunCurrentAppCommandV1OrdersStaleCheckOutputGateAndContainer(t *testing } } +func TestRunCurrentAppCommandV1SelectsGracefulCancellationOnlyForNativeCommands(t *testing.T) { + for _, native := range []bool{false, true} { + dir := t.TempDir() + current, _ := runtimeCurrentBuildFixture(t) + var order []string + backend := currentAppCommandRunTestBackend(t, dir, current, CurrentRuntimePlanV1{}, &order) + backend.planCommand = func(CurrentRuntimePlanV1, []providers.RealizedOutput, []string, bool) (ResolvedEnvironmentCommand, error) { + return ResolvedEnvironmentCommand{Name: "export", Native: native, Argv: []string{"/opt/demo"}}, nil + } + original := backend.runAdmitted + backend.runAdmitted = func(ctx context.Context, dir string, operation *deploy.OperationLock, id string, execution TransientContainerExecutionV1, options RunOptions) error { + if options.gracefulCancellation != native { + t.Fatalf("native=%t graceful cancellation=%t", native, options.gracefulCancellation) + } + return original(ctx, dir, operation, id, execution, options) + } + if err := runCurrentAppCommandV1(t.Context(), CurrentAppCommandRunInputV1{DeploymentDir: dir, Arguments: []string{"export"}, RunOptions: RunOptions{Stdin: strings.NewReader("input"), Stdout: io.Discard}}, backend); err != nil { + t.Fatal(err) + } + } +} + func TestRunCurrentAppCommandV1PreservesTerminalStreamsAndLabelsOtherOutput(t *testing.T) { t.Setenv("REPLOY_COLOR", "never") for _, tc := range []struct { diff --git a/internal/dockerdeploy/native_command_cancellation_test.go b/internal/dockerdeploy/native_command_cancellation_test.go new file mode 100644 index 00000000..78234e2a --- /dev/null +++ b/internal/dockerdeploy/native_command_cancellation_test.go @@ -0,0 +1,129 @@ +package dockerdeploy + +import ( + "context" + "errors" + "io" + "reflect" + "strings" + "testing" + "testing/synctest" + "time" +) + +func TestNativeCommandCancellationStopsBeforeDetachingAndRemoving(t *testing.T) { + for _, failure := range []string{"", "stop", "missing", "cleanup"} { + t.Run(failure, func(t *testing.T) { + parent, cancel := context.WithCancel(t.Context()) + defer cancel() + attached := make(chan context.Context, 1) + var attachedContext context.Context + var order []string + want := errors.New("owned cleanup failure") + run := func(spec CommandSpec, options RunOptions) error { + switch spec.Args[0] { + case "start": + attachedContext = options.Context + attached <- options.Context + <-options.Context.Done() + return errors.New("attached CLI cancelled") + case "container": + if options.Context.Err() != nil || options.Stdin != nil || options.Stdout != nil || options.Stderr != nil { + t.Error("stop/cleanup inherited cancellation or application streams") + } + if _, ok := options.Context.Deadline(); !ok { + t.Error("stop/cleanup lacks finite deadline") + } + order = append(order, spec.Args[1]) + if spec.Args[1] == "stop" { + if attachedContext.Err() != nil { + t.Error("attached CLI was cancelled before graceful stop") + } + if !reflect.DeepEqual(spec.Args, []string{"container", "stop", "owned"}) || !reflect.DeepEqual(spec.Env, []string{"DOCKER_HOST=unix:///bound", "DOCKER_CONTEXT="}) { + t.Error("stop changed the target or endpoint") + } + if failure == "stop" { + return want + } + if failure == "missing" { + return errors.New("No such container: owned") + } + } else if failure == "cleanup" { + return want + } + } + return nil + } + cleanup := TemporaryContainerCleanupCommand("owned") + cleanup.Env = []string{"DOCKER_HOST=unix:///bound", "DOCKER_CONTEXT="} + done := make(chan error, 1) + go func() { + done <- runTemporaryContainerCommand(run, CommandSpec{Name: "docker", Args: []string{"start", "owned"}}, cleanup, + RunOptions{Context: parent, gracefulCancellation: true, Stdin: strings.NewReader("input"), Stdout: io.Discard, Stderr: io.Discard}) + }() + ctx := <-attached + cancel() + err := <-done + if ctx.Err() == nil || err == nil || !strings.Contains(err.Error(), "native app command interrupted") || + !reflect.DeepEqual(order, []string{"stop", "rm"}) { + t.Fatalf("cancellation error=%v order=%v attached=%v", err, order, ctx.Err()) + } + if (failure == "stop" || failure == "cleanup") != errors.Is(err, want) { + t.Fatalf("stop/cleanup failure lost: %v", err) + } + var cleanupFailure *temporaryContainerCleanupFailureV1 + if errors.As(err, &cleanupFailure) != (failure == "cleanup") { + t.Fatalf("cleanup recovery ownership lost: %v", err) + } + }) + } +} + +func TestNativeCommandCancellationBoundsUnresponsiveEngine(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + started := make(chan struct{}) + run := func(spec CommandSpec, options RunOptions) error { + if spec.Args[0] == "start" { + close(started) + <-options.Context.Done() + return options.Context.Err() + } + <-options.Context.Done() + return options.Context.Err() + } + done := make(chan error, 1) + go func() { + done <- runTemporaryContainerCommand(run, CommandSpec{Name: "docker", Args: []string{"start", "owned"}}, TemporaryContainerCleanupCommand("owned"), RunOptions{Context: ctx, gracefulCancellation: true}) + }() + <-started + start := time.Now() + cancel() + err := <-done + if !errors.Is(err, context.DeadlineExceeded) || time.Since(start) != 30*time.Second+defaultDockerPreflightTimeout { + t.Fatalf("unbounded cleanup: elapsed=%s error=%v", time.Since(start), err) + } + }) +} + +func TestNativeCommandCompletionAndLaunchFailureRemainDistinctFromCancellation(t *testing.T) { + for _, runErr := range []error{nil, errors.New("genuine launcher failure")} { + var order []string + run := func(spec CommandSpec, _ RunOptions) error { + order = append(order, strings.Join(spec.Args, " ")) + if spec.Args[0] == "start" { + return runErr + } + return nil + } + err := runTemporaryContainerCommand(run, CommandSpec{Name: "docker", Args: []string{"start", "owned"}}, TemporaryContainerCleanupCommand("owned"), RunOptions{Context: t.Context(), gracefulCancellation: true}) + want := []string{"start owned"} + if runErr != nil { + want = append(want, "container stop owned", "container rm --force --volumes owned") + } + if !errors.Is(err, runErr) || !reflect.DeepEqual(order, want) || err != nil && strings.Contains(err.Error(), "interrupted") { + t.Fatalf("completion/failure changed: order=%v error=%v", order, err) + } + } +} diff --git a/noxfile.py b/noxfile.py index 7ce06f82..2998bd3e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -24,6 +24,7 @@ "tools/e2e/smoke", "tools/e2e/shell_pty.py", "tools/e2e/docker_interrupts.py", + "tools/e2e/native_cancellation.py", "packaging/python/hatch_build.py", "tests/e2e/python/packages/smoke-suite/src/smoke_suite/cli.py", "tests/e2e/python/packages/smoke-suite/src/smoke_suite/__init__.py", diff --git a/tools/e2e/native_cancellation.py b/tools/e2e/native_cancellation.py new file mode 100644 index 00000000..2cd06f04 --- /dev/null +++ b/tools/e2e/native_cancellation.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""Bounded real-CLI native cancellation proof. Uses no credentials or live app.""" + +import argparse +import json +import os +from pathlib import Path +import signal +import subprocess +import tempfile +import time +import uuid + + +PAYLOAD = r''' +import json, os, signal, subprocess, sys, time +from pathlib import Path +out = Path(os.environ["REPLOY_OUTPUT_DIR"]) +mode = sys.argv[1] +if mode == "stdin": + value = {"stdin": sys.stdin.read(), "argv": sys.argv[2:], "tty": sys.stdin.isatty()} + (out / "result.json").write_text(json.dumps(value)) + print("NATIVE_STDOUT", flush=True) + sys.exit(37) +child = subprocess.Popen(["/bin/sleep", "300"]) +def finish(sig, frame): + child.terminate() + child.wait(timeout=2) + time.sleep(1) + value = {"cancelled": True, "signal": sig, "child_reaped": child.poll() is not None} + tmp = out / "result.tmp" + tmp.write_text(json.dumps(value)) + tmp.replace(out / "result.json") + print("NATIVE_FINALIZED", flush=True) + sys.exit(130) +for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, signal.SIG_IGN if mode == "ignore" else finish) +print("CANCEL_READY", flush=True) +while True: + time.sleep(.1) +''' + + +def command(args, log, timeout=300): + with log.open("w") as stream: + result = subprocess.run(args, stdout=stream, stderr=subprocess.STDOUT, timeout=timeout) + if result.returncode: + raise RuntimeError(f"{args[1]} failed ({result.returncode}): {log.read_text()[-4000:]}") + + +def owned_containers(environment): + result = subprocess.run( + ["docker", "ps", "--all", "--filter", f"name={environment}-", "--format", "{{.ID}}"], + check=True, capture_output=True, text=True, timeout=10, + ) + return result.stdout.split() + + +def require_removed(containers): + for container in containers: + result = subprocess.run( + ["docker", "container", "inspect", container], capture_output=True, text=True, timeout=10, + ) + if result.returncode == 0 or not any( + text in result.stderr for text in ("No such container", "No such object") + ): + raise AssertionError(f"container removal not verified: {container}: {result.stderr}") + + +def cancellation_case(executable, stage, root, environment, name, host_signal, group=False): + output = root / name + output.mkdir() + log = output / "cli.log" + containers = [] + with log.open("w") as stream: + cli = subprocess.Popen( + [executable, "app", "--dir", str(stage), "--output-dir", str(output), "run", "--", + "ignore" if name == "forced" else "graceful"], + stdin=subprocess.DEVNULL, stdout=stream, stderr=subprocess.STDOUT, + start_new_session=group, + ) + try: + deadline = time.monotonic() + 40 + while "CANCEL_READY" not in log.read_text(): + if cli.poll() is not None or time.monotonic() > deadline: + raise AssertionError(f"workload did not become ready: {log.read_text()}") + time.sleep(.05) + containers = owned_containers(environment) + if len(containers) != 1: + raise AssertionError(f"expected one isolated fixture container: {containers}") + start = time.monotonic() + if group: + os.killpg(cli.pid, host_signal) + else: + cli.send_signal(host_signal) + code = cli.wait(timeout=45) + elapsed = time.monotonic() - start + finally: + if cli.poll() is None: + cli.kill() + cli.wait(timeout=10) + if code == 0: + raise AssertionError("interrupted CLI reported success") + require_removed(containers) + result = output / "result.json" + value = json.loads(result.read_text()) if result.exists() else None + if name == "forced": + if value is not None or elapsed >= 40: + raise AssertionError(f"forced containment failed: {value}, elapsed={elapsed}") + else: + if value != {"cancelled": True, "signal": 15, "child_reaped": True}: + raise AssertionError(f"application cleanup not established: {value}") + if (output / "result.tmp").exists(): + raise AssertionError("result was not atomically published") + return {"case": name, "exit_code": code, "seconds_after_interrupt": elapsed, + "result": value, "removed_containers": containers} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--reploy", required=True, type=Path, help="packaged CLI with embedded runtime") + parser.add_argument("--output-dir", required=True, type=Path, help="new evidence directory") + parser.add_argument("--base-image", default="python:3.13-slim") + args = parser.parse_args() + if os.name != "posix": + parser.error("this harness injects POSIX signals; it does not qualify native Windows interruption") + executable = str(args.reploy.resolve(strict=True)) + root = args.output_dir.resolve() + root.mkdir(parents=True, exist_ok=False) + environment = "native-cancel-" + uuid.uuid4().hex[:10] + with tempfile.TemporaryDirectory(prefix="reploy-native-cancel-") as work: + workspace = Path(work) + stage = workspace / "stage" + blueprint = workspace / "fixture.blueprint.yaml" + blueprint.write_text(f'''blueprint: + schema: 1 + version: 0.1.0 + compatibility: + platforms: [linux/amd64, linux/arm64] +environment: + id: {environment} + base: + image: {json.dumps(args.base_image)} + applications: + fixture: + packages: + os: + - package: tini + exports: + launcher: + executable: /usr/bin/tini + executables: + launcher: + source: os + binary: launcher + argv_prefix: [--, /usr/local/bin/python, -c] + commands: + run: + executable: fixture.launcher + trigger: [run] + native_command: true + argv: [{json.dumps(PAYLOAD)}] +docker: {{}} +''') + (root / "fixture.blueprint.yaml").write_bytes(blueprint.read_bytes()) + results = [] + try: + command([executable, "stage", str(blueprint), "--dir", str(stage)], root / "stage.log") + command([executable, "build", "--dir", str(stage)], root / "build.log") + for name, sig, group in [("sigint", signal.SIGINT, False), + ("sigterm", signal.SIGTERM, False), + ("terminal-group", signal.SIGINT, True), + ("forced", signal.SIGINT, False)]: + results.append(cancellation_case(executable, stage, root, environment, name, sig, group)) + (root / "results.json").write_text(json.dumps(results, indent=2) + "\n") + for name, data in [("stdin", "dummy input\nsecond line\n"), ("eof", "")]: + output = root / name + output.mkdir() + args_tail = ["literal with spaces", "dollar-$value", "semi;colon"] + run = subprocess.run( + [executable, "app", "--dir", str(stage), "--output-dir", str(output), + "run", "--", "stdin", *args_tail], input=data, capture_output=True, text=True, timeout=40, + ) + (output / "cli.log").write_text(run.stdout + run.stderr) + value = json.loads((output / "result.json").read_text()) + if run.returncode != 37 or value != {"stdin": data, "argv": args_tail, "tty": False}: + raise AssertionError(f"stdin/EOF/argv/exit-status regression: {run.returncode}, {value}") + if "NATIVE_STDOUT" not in run.stdout or owned_containers(environment): + raise AssertionError("stdout or completed container cleanup regressed") + results.append({"case": name, "exit_code": run.returncode, "result": value}) + (root / "results.json").write_text(json.dumps(results, indent=2) + "\n") + finally: + command([executable, "stage", "--remove", "--dir", str(stage), "--force"], root / "remove.log", timeout=60) + print(json.dumps({"passed": len(results), "results": str(root / "results.json")})) + + +if __name__ == "__main__": + main()