fix: remove orphaned runtimes when the executor starts

app/http.php

@@ -38,6 +38,9 @@
         /** @var Container $container */
         global $container;
 
+        /* Remove runtimes left behind by a previous executor process */
+        $maintenance->removeOrphans();
+
         /* Fetch own container information */
         $hostname = gethostname() ?: throw new \RuntimeException('Could not determine hostname');
         $selfContainer = $orchestration->list(['name' => $hostname])[0] ?? throw new \RuntimeException('Own container not found');

src/Executor/Runner/Docker.php

@@ -17,8 +17,6 @@
 use Utopia\Storage\Device\Local;
 use Utopia\System\System;
 
-use function Swoole\Coroutine\batch;
-
 class Docker extends Adapter
 {
     /**
@@ -1142,41 +1140,6 @@ public function createExecution(
         return $execution;
     }
 
-    public function cleanup(): void
-    {
-        Console::log('Cleaning up containers and networks...');
-
-        $functionsToRemove = $this->orchestration->list(['label' => 'openruntimes-executor=' . System::getHostname()]);
-
-        if ($functionsToRemove === []) {
-            Console::info('No containers found to clean up.');
-        }
-
-        $jobsRuntimes = [];
-        foreach ($functionsToRemove as $container) {
-            $jobsRuntimes[] = function () use ($container): void {
-                try {
-                    $this->orchestration->remove($container->getId(), true);
-
-                    $activeRuntimeId = $container->getName();
-
-                    if ($this->runtimes->exists($activeRuntimeId)) {
-                        $this->runtimes->remove($activeRuntimeId);
-                    }
-
-                    Console::success('Removed container ' . $container->getName());
-                } catch (\Throwable $throwable) {
-                    Console::error('Failed to remove container: ' . $container->getName());
-                    Console::error($throwable);
-                }
-            };
-        }
-
-        batch($jobsRuntimes);
-
-        Console::success('Cleanup finished.');
-    }
-
     public function getRuntimes(): mixed
     {
         $runtimes = [];

src/Executor/Runner/Maintenance.php

@@ -6,14 +6,15 @@
 use Swoole\Timer;
 use Throwable;
 use Utopia\Console;
+use Utopia\Orchestration\Container;
 use Utopia\Orchestration\Orchestration;
 use Utopia\Storage\Device\Local;
 use Utopia\System\System;
 
 use function Swoole\Coroutine\batch;
 
 /**
- * Handles periodic cleanup of inactive runtimes and orphaned temp directories.
+ * Handles cleanup of inactive and orphaned runtimes and their temp directories.
  */
 class Maintenance
 {
@@ -39,6 +40,36 @@ public function start(int $intervalSeconds, int $inactiveSeconds): void
         Console::info(sprintf('[Maintenance] Started task on interval %d seconds.', $intervalSeconds));
     }
 
+    /**
+     * Removes runtime containers this executor no longer tracks, such as those
+     * left running by a previous executor process. Their temp directories get
+     * cleaned up, so a restart would leave them crash looping without code.
+     */
+    public function removeOrphans(): void
+    {
+        $containers = $this->orchestration->list(['label' => 'openruntimes-executor=' . System::getHostname()]);
+        $orphans = array_filter(
+            $containers,
+            fn (Container $container): bool => !$this->runtimes->exists($container->getName())
+        );
+
+        $jobs = array_map(
+            fn (Container $orphan): \Closure => function () use ($orphan): bool {
+                try {
+                    return $this->orchestration->remove($orphan->getId(), force: true);
+                } catch (Throwable $throwable) {
+                    Console::error(sprintf('[Maintenance] Failed to remove orphan %s: %s', $orphan->getName(), $throwable->getMessage()));
+
+                    return false;
+                }
+            },
+            $orphans
+        );
+        $removed = \count(array_filter(batch($jobs)));
+
+        Console::info(sprintf('[Maintenance] Removed %d/', $removed) . \count($orphans) . ' orphaned runtimes.');
+    }
+
     /**
      * Stops the maintenance loop. No-op if already stopped.
      */

tests/e2e/ExecutorTest.php

@@ -1576,4 +1576,40 @@ public function testLogStreamPersistent(): void
         $response = $this->client->call(Client::METHOD_DELETE, "/runtimes/test-log-stream-persistent", [], []);
         $this->assertEquals(200, $response['headers']['status-code']);
     }
+
+    public function testRestartRemovesOrphanedRuntimes(): void
+    {
+        $response = $this->client->call(Client::METHOD_POST, '/runtimes', [], [
+            'runtimeId' => 'test-orphan',
+            'remove' => false,
+            'image' => 'openruntimes/php:v5-8.1',
+            'entrypoint' => 'tail -f /dev/null',
+            'restartPolicy' => 'always',
+        ]);
+        $this->assertEquals(201, $response['headers']['status-code']);
+
+        $output = [];
+        \exec('docker ps -aq --filter name=^executor-test-orphan$', $output);
+        $this->assertCount(1, $output);
+
+        $output = [];
+        \exec('docker restart $(docker ps -q --filter label=com.docker.compose.service=openruntimes-executor)', $output, $status);
+        $this->assertSame(0, $status);
+
+        $healthy = false;
+        for ($attempt = 0; $attempt < 60 && !$healthy; $attempt++) {
+            \sleep(2);
+            try {
+                $healthy = $this->client->call(Client::METHOD_GET, '/health', [], [])['headers']['status-code'] === 200;
+            } catch (\Throwable) {
+                // Still restarting
+            }
+        }
+
+        $this->assertTrue($healthy);
+
+        $output = [];
+        \exec('docker ps -aq --filter name=^executor-test-orphan$', $output);
+        $this->assertSame([], $output);
+    }
 }