diff --git a/app/http.php b/app/http.php index 3d38724..09d47b2 100644 --- a/app/http.php +++ b/app/http.php @@ -38,6 +38,9 @@ /** @var Container $container */ global $container; + /* Remove runtimes left behind by a previous executor process */ + $maintenance->removeOrphans(); + /* Fetch own container information */ $hostname = gethostname() ?: throw new \RuntimeException('Could not determine hostname'); $selfContainer = $orchestration->list(['name' => $hostname])[0] ?? throw new \RuntimeException('Own container not found'); diff --git a/src/Executor/Runner/Docker.php b/src/Executor/Runner/Docker.php index db90fe8..54ba38f 100644 --- a/src/Executor/Runner/Docker.php +++ b/src/Executor/Runner/Docker.php @@ -17,8 +17,6 @@ use Utopia\Storage\Device\Local; use Utopia\System\System; -use function Swoole\Coroutine\batch; - class Docker extends Adapter { /** @@ -1142,41 +1140,6 @@ public function createExecution( return $execution; } - public function cleanup(): void - { - Console::log('Cleaning up containers and networks...'); - - $functionsToRemove = $this->orchestration->list(['label' => 'openruntimes-executor=' . System::getHostname()]); - - if ($functionsToRemove === []) { - Console::info('No containers found to clean up.'); - } - - $jobsRuntimes = []; - foreach ($functionsToRemove as $container) { - $jobsRuntimes[] = function () use ($container): void { - try { - $this->orchestration->remove($container->getId(), true); - - $activeRuntimeId = $container->getName(); - - if ($this->runtimes->exists($activeRuntimeId)) { - $this->runtimes->remove($activeRuntimeId); - } - - Console::success('Removed container ' . $container->getName()); - } catch (\Throwable $throwable) { - Console::error('Failed to remove container: ' . $container->getName()); - Console::error($throwable); - } - }; - } - - batch($jobsRuntimes); - - Console::success('Cleanup finished.'); - } - public function getRuntimes(): mixed { $runtimes = []; diff --git a/src/Executor/Runner/Maintenance.php b/src/Executor/Runner/Maintenance.php index be23d99..44fe800 100644 --- a/src/Executor/Runner/Maintenance.php +++ b/src/Executor/Runner/Maintenance.php @@ -6,6 +6,7 @@ use Swoole\Timer; use Throwable; use Utopia\Console; +use Utopia\Orchestration\Container; use Utopia\Orchestration\Orchestration; use Utopia\Storage\Device\Local; use Utopia\System\System; @@ -13,7 +14,7 @@ use function Swoole\Coroutine\batch; /** - * Handles periodic cleanup of inactive runtimes and orphaned temp directories. + * Handles cleanup of inactive and orphaned runtimes and their temp directories. */ class Maintenance { @@ -39,6 +40,36 @@ public function start(int $intervalSeconds, int $inactiveSeconds): void Console::info(sprintf('[Maintenance] Started task on interval %d seconds.', $intervalSeconds)); } + /** + * Removes runtime containers this executor no longer tracks, such as those + * left running by a previous executor process. Their temp directories get + * cleaned up, so a restart would leave them crash looping without code. + */ + public function removeOrphans(): void + { + $containers = $this->orchestration->list(['label' => 'openruntimes-executor=' . System::getHostname()]); + $orphans = array_filter( + $containers, + fn (Container $container): bool => !$this->runtimes->exists($container->getName()) + ); + + $jobs = array_map( + fn (Container $orphan): \Closure => function () use ($orphan): bool { + try { + return $this->orchestration->remove($orphan->getId(), force: true); + } catch (Throwable $throwable) { + Console::error(sprintf('[Maintenance] Failed to remove orphan %s: %s', $orphan->getName(), $throwable->getMessage())); + + return false; + } + }, + $orphans + ); + $removed = \count(array_filter(batch($jobs))); + + Console::info(sprintf('[Maintenance] Removed %d/', $removed) . \count($orphans) . ' orphaned runtimes.'); + } + /** * Stops the maintenance loop. No-op if already stopped. */ diff --git a/tests/e2e/ExecutorTest.php b/tests/e2e/ExecutorTest.php index 73b39f6..17c6738 100644 --- a/tests/e2e/ExecutorTest.php +++ b/tests/e2e/ExecutorTest.php @@ -1576,4 +1576,40 @@ public function testLogStreamPersistent(): void $response = $this->client->call(Client::METHOD_DELETE, "/runtimes/test-log-stream-persistent", [], []); $this->assertEquals(200, $response['headers']['status-code']); } + + public function testRestartRemovesOrphanedRuntimes(): void + { + $response = $this->client->call(Client::METHOD_POST, '/runtimes', [], [ + 'runtimeId' => 'test-orphan', + 'remove' => false, + 'image' => 'openruntimes/php:v5-8.1', + 'entrypoint' => 'tail -f /dev/null', + 'restartPolicy' => 'always', + ]); + $this->assertEquals(201, $response['headers']['status-code']); + + $output = []; + \exec('docker ps -aq --filter name=^executor-test-orphan$', $output); + $this->assertCount(1, $output); + + $output = []; + \exec('docker restart $(docker ps -q --filter label=com.docker.compose.service=openruntimes-executor)', $output, $status); + $this->assertSame(0, $status); + + $healthy = false; + for ($attempt = 0; $attempt < 60 && !$healthy; $attempt++) { + \sleep(2); + try { + $healthy = $this->client->call(Client::METHOD_GET, '/health', [], [])['headers']['status-code'] === 200; + } catch (\Throwable) { + // Still restarting + } + } + + $this->assertTrue($healthy); + + $output = []; + \exec('docker ps -aq --filter name=^executor-test-orphan$', $output); + $this->assertSame([], $output); + } }