Use docker compose yaml for e2e tests against oss, add easy local run script

.github/workflows/pull_request.yml

@@ -14,7 +14,7 @@ on:
   workflow_dispatch:
     inputs:
       oss_conductor_version:
-        description: 'OSS Conductor image tag (falls back to E2E_TEST_OSS_CONDUCTOR_VERSION org var, then to a pinned default)'
+        description: 'OSS Conductor image tag (falls back to the E2E_TEST_OSS_CONDUCTOR_VERSION org var, then to the default in scripts/docker-compose-oss.yaml on fork PRs)'
         required: false
         type: string
 
@@ -173,50 +173,52 @@ jobs:
     runs-on: ubuntu-latest
     env:
       CONDUCTOR_SERVER_URL: http://localhost:8080/api
-      # The literal fallback is not redundant: GitHub withholds org/repo
-      # variables from pull_request runs on forks exactly as it withholds
-      # secrets, so vars.* resolves to "" there. This job needs no secrets —
-      # only a tag — so it pins one and keeps running on fork PRs instead of
-      # failing with nothing to pull.
-      OSS_CONDUCTOR_VERSION: ${{ inputs.oss_conductor_version || vars.E2E_TEST_OSS_CONDUCTOR_VERSION || '3.32.3' }}
     steps:
-      - name: Show OSS Conductor version
-        run: echo "Using conductoross/conductor:$OSS_CONDUCTOR_VERSION"
+      # OSS_CONDUCTOR_VERSION is resolved here rather than in the job `env` so
+      # that the two ways it can come back empty get different treatment:
+      #
+      #   - Fork PR: GitHub withholds org/repo variables from pull_request runs
+      #     on forks exactly as it withholds secrets, so vars.* is always "" for
+      #     an outside contributor (observed in csharp-sdk#178). This job needs
+      #     no secrets, only a tag, so leave the var unset and let the default
+      #     baked into the `image:` line of scripts/docker-compose-oss.yaml
+      #     apply. That is the same tag a plain local run of
+      #     scripts/run-integration-oss.sh gets, and the one place it is
+      #     written -- no second copy to drift out of sync here.
+      #   - Anything else: the org variable is genuinely missing or its
+      #     repository access policy no longer covers this repo. Fail loudly
+      #     rather than silently drifting onto the default.
+      - name: Resolve OSS Conductor version
+        env:
+          REQUESTED_VERSION: ${{ inputs.oss_conductor_version || vars.E2E_TEST_OSS_CONDUCTOR_VERSION }}
+          IS_FORK_PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}
+        run: |
+          if [ -n "$REQUESTED_VERSION" ]; then
+            echo "OSS_CONDUCTOR_VERSION=${REQUESTED_VERSION}" >> "$GITHUB_ENV"
+          elif [ "$IS_FORK_PR" = "true" ]; then
+            echo "::notice::Fork PR: org variables are withheld, falling back to the default tag in scripts/docker-compose-oss.yaml"
+          else
+            echo "::error::No Conductor OSS image tag resolved. Set the E2E_TEST_OSS_CONDUCTOR_VERSION organization variable (and ensure its repository access policy includes this repo), or pass the oss_conductor_version input via workflow_dispatch."
+            exit 1
+          fi
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Write docker-compose file
+      # `docker compose up` only pulls an image when it is missing locally. On a
+      # GitHub-hosted runner the VM is ephemeral and starts with no cached copy
+      # of this image, so `up` would pull anyway and this step is redundant
+      # today. It is here deliberately: it costs no extra network pull (`up`
+      # then finds the image locally), it separates "couldn't pull the image"
+      # from "the stack didn't come up" into two distinct red steps, and it is
+      # what keeps a mutable tag from going stale if this job ever moves to a
+      # self-hosted runner with a warm Docker daemon -- the same reason
+      # scripts/run-integration-oss.sh pulls. It also prints the tag actually in
+      # use, which for a fork PR comes from the compose file's default.
+      - name: Pull Conductor OSS image
         run: |
-          cat <<'EOF' > docker-compose-oss.yaml
-          services:
-            conductor-server:
-              image: conductoross/conductor:${{ env.OSS_CONDUCTOR_VERSION }}
-              environment:
-                - CONFIG_PROP=config-postgres.properties
-              ports:
-                - "8080:8080"
-              healthcheck:
-                test: ["CMD", "curl", "-I", "-XGET", "http://localhost:8080/health"]
-                interval: 10s
-                timeout: 10s
-                retries: 20
-              links:
-                - conductor-postgres:postgresdb
-              depends_on:
-                conductor-postgres:
-                  condition: service_healthy
-            conductor-postgres:
-              image: postgres:16
-              environment:
-                - POSTGRES_USER=conductor
-                - POSTGRES_PASSWORD=conductor
-              healthcheck:
-                test: timeout 5 bash -c 'cat < /dev/null > /dev/tcp/localhost/5432'
-                interval: 5s
-                timeout: 5s
-                retries: 12
-          EOF
+          echo "Using $(docker compose -f scripts/docker-compose-oss.yaml config --images | grep -m1 '^conductoross/conductor:')"
+          docker compose -f scripts/docker-compose-oss.yaml pull conductor-server
       - name: Start Conductor OSS stack
-        run: docker compose -f docker-compose-oss.yaml up -d
+        run: docker compose -f scripts/docker-compose-oss.yaml up -d
       - name: Wait for Conductor to be healthy
         run: timeout 120 bash -c 'until curl -sf http://localhost:8080/health; do sleep 5; done'
       - name: Setup .NET
@@ -231,4 +233,4 @@ jobs:
           -l "console;verbosity=normal"
       - name: Dump Conductor logs
         if: failure()
-        run: docker compose -f docker-compose-oss.yaml logs conductor-server
+        run: docker compose -f scripts/docker-compose-oss.yaml logs conductor-server

Tests/Integration/Environment/EnvironmentVariableTests.cs

@@ -18,6 +18,10 @@ namespace Tests.Integration.Environment
 {
     [Collection("Integration")]
     [Trait("Category", "Integration")]
+    // Orkes-only: OSS exposes the environment API read-only. Its EnvironmentResource declares
+    // only GET /api/environment and GET /api/environment/{key} (added by
+    // conductor-oss/conductor#1251, first released in 3.32.0-rc.5 and 3.31.2), so the
+    // CreateOrUpdateEnvVariable calls below (PUT /environment/{key}) return 405 against OSS.
     [Trait("ServerType", "Orkes")]
     public class EnvironmentVariableTests : IClassFixture<ConductorFixture>
     {

Tests/Integration/Helpers/TestPrefix.cs

@@ -17,15 +17,28 @@ namespace Tests.Integration.Helpers
     /// <summary>
     /// Generates unique resource names per test run to avoid conflicts
     /// between concurrent runs or leftover data from previous runs.
-    /// Format: csharp_sdk_{shortRunId}_{name}
+    /// Format: csharp_sdk_{shortRunId}[_{attempt}]_{name}
     /// </summary>
     public static class TestPrefix
     {
-        private static readonly string RunId = SysEnv.GetEnvironmentVariable("GITHUB_RUN_ID")
-            ?? System.Guid.NewGuid().ToString("N")[..8];
+        private static readonly string RunId = ResolveRunId();
 
         public static string Prefix => $"csharp_sdk_{RunId}";
 
         public static string Name(string name) => $"{Prefix}_{name}";
+
+        // GITHUB_RUN_ID is stable across attempts of the same run, so on a re-run the task
+        // queues would still be named after the failed attempt — and a task it orphaned (e.g.
+        // a StartWorkflow whose response was lost) is still sitting in one, ready to be polled
+        // by a test that did not create it. Including the attempt gives each try its own names.
+        private static string ResolveRunId()
+        {
+            var runId = SysEnv.GetEnvironmentVariable("GITHUB_RUN_ID");
+            if (string.IsNullOrEmpty(runId))
+                return System.Guid.NewGuid().ToString("N")[..8];
+
+            var attempt = SysEnv.GetEnvironmentVariable("GITHUB_RUN_ATTEMPT");
+            return string.IsNullOrEmpty(attempt) ? runId : $"{runId}_{attempt}";
+        }
     }
 }

Tests/Integration/Task/TaskPollTests.cs

@@ -95,9 +95,12 @@ private void Cleanup(string id)
         {
             try
             {
+                // Report each task against the workflow it actually belongs to — a batch poll by
+                // task type can return tasks from other executions, and passing `id` for those
+                // would complete them against the wrong workflow.
                 var tasks = _taskClient.BatchPoll(_taskName, WorkerId, domain: null, count: 10);
                 foreach (var t in tasks ?? new List<Conductor.Client.Models.Task>())
-                    _taskClient.UpdateTask(new TaskResult { TaskId = t.TaskId, WorkflowInstanceId = id, Status = TaskResult.StatusEnum.COMPLETED });
+                    _taskClient.UpdateTask(new TaskResult { TaskId = t.TaskId, WorkflowInstanceId = t.WorkflowInstanceId, Status = TaskResult.StatusEnum.COMPLETED });
             }
             catch { }
             try { _workflowClient.Terminate(id); } catch { }

Tests/Integration/Task/TaskUpdateTests.cs

@@ -54,34 +54,48 @@ public TaskUpdateTests(ConductorFixture fixture)
         public void CompleteTask_WorkflowCompletes()
         {
             var id = StartWorkflow();
-            var task = PollTask(id);
-
-            _taskClient.UpdateTask(new TaskResult
+            try
             {
-                TaskId = task.TaskId,
-                WorkflowInstanceId = id,
-                Status = TaskResult.StatusEnum.COMPLETED,
-                OutputData = new Dictionary<string, object> { { "result", "ok" } }
-            });
+                var task = PollTask(id);
 
-            Assert.Equal(Conductor.Client.Models.Workflow.StatusEnum.COMPLETED, GetWorkflowStatus(id));
+                _taskClient.UpdateTask(new TaskResult
+                {
+                    TaskId = task.TaskId,
+                    WorkflowInstanceId = id,
+                    Status = TaskResult.StatusEnum.COMPLETED,
+                    OutputData = new Dictionary<string, object> { { "result", "ok" } }
+                });
+
+                Assert.Equal(Conductor.Client.Models.Workflow.StatusEnum.COMPLETED, GetWorkflowStatus(id));
+            }
+            finally
+            {
+                Cleanup(id);
+            }
         }
 
         [Fact]
         public void FailTask_WorkflowFails()
         {
             var id = StartWorkflow();
-            var task = PollTask(id);
-
-            _taskClient.UpdateTask(new TaskResult
+            try
             {
-                TaskId = task.TaskId,
-                WorkflowInstanceId = id,
-                Status = TaskResult.StatusEnum.FAILED,
-                ReasonForIncompletion = "deliberate test failure"
-            });
+                var task = PollTask(id);
+
+                _taskClient.UpdateTask(new TaskResult
+                {
+                    TaskId = task.TaskId,
+                    WorkflowInstanceId = id,
+                    Status = TaskResult.StatusEnum.FAILED,
+                    ReasonForIncompletion = "deliberate test failure"
+                });
 
-            Assert.Equal(Conductor.Client.Models.Workflow.StatusEnum.FAILED, GetWorkflowStatus(id));
+                Assert.Equal(Conductor.Client.Models.Workflow.StatusEnum.FAILED, GetWorkflowStatus(id));
+            }
+            finally
+            {
+                Cleanup(id);
+            }
         }
 
         [Fact]
@@ -118,16 +132,26 @@ public void GetTask_ReturnsTaskDetails()
         private string StartWorkflow() =>
             _workflowClient.StartWorkflow(new StartWorkflowRequest(name: _workflowName));
 
+        // Polling is by task type, so the queue can hand back a task from another execution of
+        // the same definition. Writing our result to it would leave our workflow stuck and
+        // stamp this test's outcome on an unrelated one, so skip foreign tasks — leaving them
+        // claimed keeps them out of the queue — and keep polling for our own.
         private Conductor.Client.Models.Task PollTask(string workflowId)
         {
-            Conductor.Client.Models.Task task = null;
-            for (var i = 0; i < 10 && task == null; i++)
+            for (var i = 0; i < 20; i++)
             {
-                task = _taskClient.Poll(_taskName, WorkerId);
-                if (task == null) System.Threading.Thread.Sleep(500);
+                var task = _taskClient.Poll(_taskName, WorkerId);
+                if (task == null)
+                {
+                    System.Threading.Thread.Sleep(500);
+                    continue;
+                }
+                if (task.WorkflowInstanceId == workflowId)
+                    return task;
             }
-            Assert.NotNull(task);
-            return task;
+
+            Assert.True(false, $"No task for workflow {workflowId} appeared in queue {_taskName}");
+            return null;
         }
 
         private Conductor.Client.Models.Workflow.StatusEnum? GetWorkflowStatus(string id)
@@ -145,6 +169,11 @@ private Conductor.Client.Models.Task PollTask(string workflowId)
         private void Cleanup(string workflowId, string taskId)
         {
             try { _taskClient.UpdateTask(new TaskResult { TaskId = taskId, WorkflowInstanceId = workflowId, Status = TaskResult.StatusEnum.COMPLETED }); } catch { }
+            Cleanup(workflowId);
+        }
+
+        private void Cleanup(string workflowId)
+        {
             try { _workflowClient.Terminate(workflowId); } catch { }
             try { _workflowClient.Delete(workflowId); } catch { }
         }