From eb549ed28594d739a871455e54979dfeb0b389b9 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sun, 11 Oct 2026 11:45:13 +0100 Subject: [PATCH] refactor: send Target requests through utopia-php/client Target::call built and ran raw curl handles with no timeouts, so a remote that stopped answering held the migration open forever. Requests now go through utopia-php/client, which owns the transport: a 10s connect and 30s total timeout, redirects followed as before, and transport failures surfaced as an Exception with CODE_INTERNAL. The Appwrite SDK clients get the same limits. storage moves to 4.4 for client 0.6. --- composer.json | 5 +- composer.lock | 140 ++++++++++++------------ src/Migration/Destinations/Appwrite.php | 4 +- src/Migration/Sources/Appwrite.php | 4 +- src/Migration/Target.php | 123 +++++++++------------ 5 files changed, 130 insertions(+), 146 deletions(-) diff --git a/composer.json b/composer.json index 72d0ee0c..8a915e88 100644 --- a/composer.json +++ b/composer.json @@ -36,7 +36,10 @@ "utopia-php/database": "^8.0", "utopia-php/storage": "4.*", "utopia-php/dsn": "0.2.*", - "halaxa/json-machine": "^1.2" + "halaxa/json-machine": "^1.2", + "psr/http-client": "^1.0", + "utopia-php/client": "^0.6", + "utopia-php/psr7": "^0.2.1" }, "require-dev": { "ext-pdo": "*", diff --git a/composer.lock b/composer.lock index 56d2bde8..480732bf 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "62990e7e2ea096454bc1d3e81a6e0480", + "content-hash": "d8dee0b68c34008cd9d6a230b705881d", "packages": [ { "name": "adhocore/jwt", @@ -172,16 +172,16 @@ }, { "name": "composer/semver", - "version": "3.4.4", + "version": "3.5.1", "source": { "type": "git", "url": "https://github.com/composer/semver.git", - "reference": "198166618906cb2de69b95d7d47e5fa8aa1b2b95" + "reference": "528948210cc46f967a4b98e83b19e76f2b504993" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/semver/zipball/198166618906cb2de69b95d7d47e5fa8aa1b2b95", - "reference": "198166618906cb2de69b95d7d47e5fa8aa1b2b95", + "url": "https://api.github.com/repos/composer/semver/zipball/528948210cc46f967a4b98e83b19e76f2b504993", + "reference": "528948210cc46f967a4b98e83b19e76f2b504993", "shasum": "" }, "require": { @@ -223,7 +223,7 @@ "homepage": "http://robbast.nl" } ], - "description": "Semver library that offers utilities, version constraint parsing and validation.", + "description": "Version comparison library that offers utilities, version constraint parsing and validation.", "keywords": [ "semantic", "semver", @@ -233,7 +233,7 @@ "support": { "irc": "ircs://irc.libera.chat:6697/composer", "issues": "https://github.com/composer/semver/issues", - "source": "https://github.com/composer/semver/tree/3.4.4" + "source": "https://github.com/composer/semver/tree/3.5.1" }, "funding": [ { @@ -245,7 +245,7 @@ "type": "github" } ], - "time": "2025-08-20T19:15:30+00:00" + "time": "2026-10-05T13:12:12+00:00" }, { "name": "google/protobuf", @@ -1544,16 +1544,16 @@ }, { "name": "symfony/http-client", - "version": "v7.4.19", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "3a523f38dc399337ec45b2ab9628e8c37ceb23c7" + "reference": "750a420b8dd0af3d6e1c6c1412802abf2cdd2b6c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/3a523f38dc399337ec45b2ab9628e8c37ceb23c7", - "reference": "3a523f38dc399337ec45b2ab9628e8c37ceb23c7", + "url": "https://api.github.com/repos/symfony/http-client/zipball/750a420b8dd0af3d6e1c6c1412802abf2cdd2b6c", + "reference": "750a420b8dd0af3d6e1c6c1412802abf2cdd2b6c", "shasum": "" }, "require": { @@ -1621,7 +1621,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.4.19" + "source": "https://github.com/symfony/http-client/tree/v7.4.20" }, "funding": [ { @@ -1641,7 +1641,7 @@ "type": "tidelift" } ], - "time": "2026-09-08T13:28:43+00:00" + "time": "2026-09-24T11:02:13+00:00" }, { "name": "symfony/http-client-contracts", @@ -1727,16 +1727,16 @@ }, { "name": "symfony/polyfill-mbstring", - "version": "v1.38.2", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6" + "reference": "4d5d331bf731e0aedbc6b2f5e61135b7b7996a72" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6", - "reference": "d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/4d5d331bf731e0aedbc6b2f5e61135b7b7996a72", + "reference": "4d5d331bf731e0aedbc6b2f5e61135b7b7996a72", "shasum": "" }, "require": { @@ -1788,7 +1788,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.38.2" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.43.0" }, "funding": [ { @@ -1808,20 +1808,20 @@ "type": "tidelift" } ], - "time": "2026-05-27T06:59:30+00:00" + "time": "2026-09-24T15:19:04+00:00" }, { "name": "symfony/polyfill-php82", - "version": "v1.38.1", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php82.git", - "reference": "002dc0cfe5fd4ed6033d48f27d4f19a486c4b04b" + "reference": "843bbe8b2a7934307b749751eede0f111f5d1cfc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php82/zipball/002dc0cfe5fd4ed6033d48f27d4f19a486c4b04b", - "reference": "002dc0cfe5fd4ed6033d48f27d4f19a486c4b04b", + "url": "https://api.github.com/repos/symfony/polyfill-php82/zipball/843bbe8b2a7934307b749751eede0f111f5d1cfc", + "reference": "843bbe8b2a7934307b749751eede0f111f5d1cfc", "shasum": "" }, "require": { @@ -1868,7 +1868,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php82/tree/v1.38.1" + "source": "https://github.com/symfony/polyfill-php82/tree/v1.43.0" }, "funding": [ { @@ -1888,20 +1888,20 @@ "type": "tidelift" } ], - "time": "2026-05-26T12:45:58+00:00" + "time": "2026-09-25T15:50:05+00:00" }, { "name": "symfony/polyfill-php83", - "version": "v1.41.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php83.git", - "reference": "5ea99087fb99c273a9b9236ed4c31e78b16103c6" + "reference": "80ccff923a8d61f73ebe9da0aa94e25232f004ce" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php83/zipball/5ea99087fb99c273a9b9236ed4c31e78b16103c6", - "reference": "5ea99087fb99c273a9b9236ed4c31e78b16103c6", + "url": "https://api.github.com/repos/symfony/polyfill-php83/zipball/80ccff923a8d61f73ebe9da0aa94e25232f004ce", + "reference": "80ccff923a8d61f73ebe9da0aa94e25232f004ce", "shasum": "" }, "require": { @@ -1948,7 +1948,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php83/tree/v1.41.0" + "source": "https://github.com/symfony/polyfill-php83/tree/v1.43.0" }, "funding": [ { @@ -1968,7 +1968,7 @@ "type": "tidelift" } ], - "time": "2026-07-01T12:47:55+00:00" + "time": "2026-09-25T15:50:05+00:00" }, { "name": "symfony/service-contracts", @@ -2292,16 +2292,16 @@ }, { "name": "utopia-php/client", - "version": "0.4.5", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/client.git", - "reference": "d1e0d25ef602816961ee9fdcd3f62588929e37ce" + "reference": "00eae2ba5c87b95abfb507733e55a4268281ec72" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/client/zipball/d1e0d25ef602816961ee9fdcd3f62588929e37ce", - "reference": "d1e0d25ef602816961ee9fdcd3f62588929e37ce", + "url": "https://api.github.com/repos/utopia-php/client/zipball/00eae2ba5c87b95abfb507733e55a4268281ec72", + "reference": "00eae2ba5c87b95abfb507733e55a4268281ec72", "shasum": "" }, "require": { @@ -2324,7 +2324,7 @@ "type": "library", "autoload": { "psr-4": { - "Utopia\\": "src/" + "Utopia\\Client\\": "src/" } }, "notification-url": "https://packagist.org/downloads/", @@ -2343,9 +2343,9 @@ ], "support": { "issues": "https://github.com/utopia-php/client/issues", - "source": "https://github.com/utopia-php/client/tree/0.4.5" + "source": "https://github.com/utopia-php/client/tree/0.6.0" }, - "time": "2026-09-19T22:12:56+00:00" + "time": "2026-10-03T08:02:08+00:00" }, { "name": "utopia-php/console", @@ -2717,22 +2717,22 @@ }, { "name": "utopia-php/span", - "version": "4.2.0", + "version": "4.4.0", "source": { "type": "git", "url": "https://github.com/utopia-php/span.git", - "reference": "8505a2331772bd84ca60333e01ce6b6d068ee1dc" + "reference": "fd0958184567f8d14bf3696f176ce009a94f6317" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/span/zipball/8505a2331772bd84ca60333e01ce6b6d068ee1dc", - "reference": "8505a2331772bd84ca60333e01ce6b6d068ee1dc", + "url": "https://api.github.com/repos/utopia-php/span/zipball/fd0958184567f8d14bf3696f176ce009a94f6317", + "reference": "fd0958184567f8d14bf3696f176ce009a94f6317", "shasum": "" }, "require": { "php": ">=8.4", "psr/http-client": "^1.0", - "utopia-php/client": "^0.2 || ^0.3 || ^0.4", + "utopia-php/client": "^0.6", "utopia-php/psr7": "^0.1 || ^0.2" }, "require-dev": { @@ -2744,7 +2744,7 @@ "type": "library", "autoload": { "psr-4": { - "Utopia\\Span\\": "src/Span/" + "Utopia\\Span\\": "src/" } }, "notification-url": "https://packagist.org/downloads/", @@ -2754,22 +2754,22 @@ "description": "Simple span tracing library for PHP with coroutine support", "support": { "issues": "https://github.com/utopia-php/span/issues", - "source": "https://github.com/utopia-php/span/tree/4.2.0" + "source": "https://github.com/utopia-php/span/tree/4.4.0" }, - "time": "2026-08-28T00:40:32+00:00" + "time": "2026-10-07T17:04:58+00:00" }, { "name": "utopia-php/storage", - "version": "4.2.0", + "version": "4.4.0", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "5ecd5cb74bfcd6609c08fc31d6f444a706271d05" + "reference": "f4eca7ad115ce411727987347e62d1033e4115da" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/5ecd5cb74bfcd6609c08fc31d6f444a706271d05", - "reference": "5ecd5cb74bfcd6609c08fc31d6f444a706271d05", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/f4eca7ad115ce411727987347e62d1033e4115da", + "reference": "f4eca7ad115ce411727987347e62d1033e4115da", "shasum": "" }, "require": { @@ -2779,7 +2779,7 @@ "php": ">=8.5", "psr/http-client": "^1.0", "psr/http-message": "^1.1 || ^2.0", - "utopia-php/client": "^0.4", + "utopia-php/client": "^0.6", "utopia-php/psr7": "0.2.*", "utopia-php/telemetry": "^0.4.6", "utopia-php/validators": "^1.0" @@ -2787,7 +2787,7 @@ "type": "library", "autoload": { "psr-4": { - "Utopia\\Storage\\": "src/Storage" + "Utopia\\Storage\\": "src/" } }, "notification-url": "https://packagist.org/downloads/", @@ -2804,22 +2804,22 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/4.2.0" + "source": "https://github.com/utopia-php/storage/tree/4.4.0" }, - "time": "2026-09-19T22:42:19+00:00" + "time": "2026-10-07T17:04:58+00:00" }, { "name": "utopia-php/telemetry", - "version": "0.4.11", + "version": "0.4.12", "source": { "type": "git", "url": "https://github.com/utopia-php/telemetry.git", - "reference": "852e76c89d0c20fc6560ac9687b15457a97b3704" + "reference": "e628707fe262dc0ca418fbe6f5e421f2db719907" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/telemetry/zipball/852e76c89d0c20fc6560ac9687b15457a97b3704", - "reference": "852e76c89d0c20fc6560ac9687b15457a97b3704", + "url": "https://api.github.com/repos/utopia-php/telemetry/zipball/e628707fe262dc0ca418fbe6f5e421f2db719907", + "reference": "e628707fe262dc0ca418fbe6f5e421f2db719907", "shasum": "" }, "require": { @@ -2840,7 +2840,7 @@ "type": "library", "autoload": { "psr-4": { - "Utopia\\Telemetry\\": "src/Telemetry" + "Utopia\\Telemetry\\": "src/" } }, "notification-url": "https://packagist.org/downloads/", @@ -2855,9 +2855,9 @@ ], "support": { "issues": "https://github.com/utopia-php/telemetry/issues", - "source": "https://github.com/utopia-php/telemetry/tree/0.4.11" + "source": "https://github.com/utopia-php/telemetry/tree/0.4.12" }, - "time": "2026-09-16T10:12:28+00:00" + "time": "2026-09-28T10:14:23+00:00" }, { "name": "utopia-php/validators", @@ -4969,16 +4969,16 @@ }, { "name": "symfony/polyfill-php80", - "version": "v1.37.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "dfb55726c3a76ea3b6459fcfda1ec2d80a682411" + "reference": "9c6a5d6b01ca51d486c813e9a9c0ed55f208bb74" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/dfb55726c3a76ea3b6459fcfda1ec2d80a682411", - "reference": "dfb55726c3a76ea3b6459fcfda1ec2d80a682411", + "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/9c6a5d6b01ca51d486c813e9a9c0ed55f208bb74", + "reference": "9c6a5d6b01ca51d486c813e9a9c0ed55f208bb74", "shasum": "" }, "require": { @@ -5029,7 +5029,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/v1.37.0" + "source": "https://github.com/symfony/polyfill-php80/tree/v1.43.0" }, "funding": [ { @@ -5049,7 +5049,7 @@ "type": "tidelift" } ], - "time": "2026-04-10T16:19:22+00:00" + "time": "2026-09-25T15:50:05+00:00" }, { "name": "theseer/tokenizer", @@ -5106,12 +5106,12 @@ "version": "v5.6.4", "source": { "type": "git", - "url": "https://github.com/vlucas/phpdotenv.git", + "url": "https://github.com/phpdotenv/phpdotenv.git", "reference": "416df702837983f8d5ff48c9c3fee4f5f57b980b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/vlucas/phpdotenv/zipball/416df702837983f8d5ff48c9c3fee4f5f57b980b", + "url": "https://api.github.com/repos/phpdotenv/phpdotenv/zipball/416df702837983f8d5ff48c9c3fee4f5f57b980b", "reference": "416df702837983f8d5ff48c9c3fee4f5f57b980b", "shasum": "" }, diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 5af540ff..01035649 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -283,7 +283,9 @@ public function __construct( $this->client = (new Client()) ->setEndpoint($endpoint) ->setProject($project) - ->setKey($key); + ->setKey($key) + ->setConnectTimeout(static::CONNECT_TIMEOUT) + ->setTimeout(static::REQUEST_TIMEOUT); $this->functions = new Functions($this->client); $this->messaging = new Messaging($this->client); diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index d64d39d4..4b10c1ba 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -143,7 +143,9 @@ public function __construct( $this->client = (new Client()) ->setEndpoint($endpoint) ->setProject($projectId) - ->setKey($key); + ->setKey($key) + ->setConnectTimeout(static::CONNECT_TIMEOUT) + ->setTimeout(static::REQUEST_TIMEOUT); $this->users = new Users($this->client); $this->teams = new Teams($this->client); diff --git a/src/Migration/Target.php b/src/Migration/Target.php index b855d3ce..f31976f2 100644 --- a/src/Migration/Target.php +++ b/src/Migration/Target.php @@ -2,8 +2,25 @@ namespace Utopia\Migration; +use Psr\Http\Client\ClientExceptionInterface; +use Utopia\Client\Adapter\Curl\Client as CurlAdapter; +use Utopia\Client\Client; +use Utopia\Psr7\ContentType; +use Utopia\Psr7\Request\Factory as RequestFactory; + abstract class Target { + /** + * Seconds to wait for a remote to accept the connection. + */ + public const int CONNECT_TIMEOUT = 10; + + /** + * Seconds a request may take in total. Requests are single API pages and + * file chunks, so a remote that has stopped answering fails within this. + */ + public const int REQUEST_TIMEOUT = 30; + /** * Global Headers * @@ -31,6 +48,8 @@ abstract class Target protected string $endpoint = ''; + private ?Client $client = null; + protected string $rootResourceId = ''; protected string $rootResourceType = ''; @@ -88,101 +107,59 @@ protected function call( array &$responseHeaders = [] ): array|string { $headers = \array_merge($this->headers, $headers); - $ch = \curl_init(( - \str_contains($path, 'http') - ? $path.(($method == 'GET' && ! empty($params)) ? '?'.\http_build_query($params) : '') - : $this->endpoint.$path.( - ($method == 'GET' && ! empty($params)) - ? '?'.\http_build_query($params) - : '' - ) - )); - - $query = match ($headers['Content-Type']) { - 'application/json' => \json_encode($params), - 'multipart/form-data' => $this->flatten($params), - default => \http_build_query($params), - }; - - foreach ($headers as $i => $header) { - $headers[] = $i.':'.$header; - unset($headers[$i]); - } + $contentType = $headers['Content-Type'] ?? ''; + unset($headers['Content-Type']); - if ($method === 'HEAD') { - \curl_setopt($ch, CURLOPT_NOBODY, true); - } else { - \curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method); - } - \curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); - \curl_setopt($ch, CURLOPT_USERAGENT, php_uname('s').'-'.php_uname('r').':php-'.phpversion()); - \curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); - \curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - \curl_setopt($ch, CURLOPT_HEADERFUNCTION, function ($curl, $header) use (&$responseHeaders) { - $len = strlen($header); - $header = explode(':', strtolower($header), 2); - - if (\count($header) < 2) { // ignore invalid headers - return $len; - } + // An empty value has always meant "do not send this header". + $headers = \array_filter($headers, fn (string $value) => $value !== ''); + $headers['User-Agent'] = \php_uname('s').'-'.\php_uname('r').':php-'.\phpversion(); - $responseHeaders[\strtolower(\trim($header[0]))] = \trim($header[1]); + $uri = \str_contains($path, 'http') ? $path : $this->endpoint.$path; + $requests = new RequestFactory(); - return $len; - }); + $request = match (true) { + $method === 'GET' || $method === 'HEAD' => $requests->query($method, $uri, $params, $headers), + $contentType === ContentType::JSON => $requests->json($method, $uri, $params, $headers), + default => $requests->form($method, $uri, $params, $headers), + }; - if ($method != 'GET') { - \curl_setopt($ch, CURLOPT_POSTFIELDS, $query); + try { + $response = $this->client()->sendRequest($request); + } catch (ClientExceptionInterface $error) { + throw new \Exception($error->getMessage(), Exception::CODE_INTERNAL, $error); } - $responseBody = curl_exec($ch); - - $responseType = $responseHeaders['Content-Type'] ?? $responseHeaders['content-type'] ?? ''; - $responseStatus = \curl_getinfo($ch, CURLINFO_HTTP_CODE); - - switch (\substr($responseType, 0, \strpos($responseType, ';'))) { - case 'application/json': - $responseBody = \json_decode($responseBody, true); - break; + foreach (\array_keys($response->getHeaders()) as $name) { + $responseHeaders[\strtolower($name)] = $response->getHeaderLine($name); } - if (\curl_errno($ch)) { - throw new \Exception(\curl_error($ch), Exception::CODE_INTERNAL); - } + $responseBody = (string) $response->getBody(); + $responseStatus = $response->getStatusCode(); + if (\str_starts_with(\strtolower($response->getHeaderLine('Content-Type')), ContentType::JSON)) { + $responseBody = \json_decode($responseBody, true); + } if ($responseStatus >= 400) { if (\is_array($responseBody)) { throw new \Exception(\json_encode($responseBody), $responseStatus); - } else { - throw new \Exception($responseStatus.': '.$responseBody, $responseStatus); } + + throw new \Exception($responseStatus.': '.$responseBody, $responseStatus); } return $responseBody; } /** - * Flatten params array to PHP multiple format - * - * @param array $data - * @return array + * The HTTP client for {@see call()}, built once per target so connections are reused. */ - protected function flatten(array $data, string $prefix = ''): array + protected function client(): Client { - $output = []; - - foreach ($data as $key => $value) { - $finalKey = $prefix ? "{$prefix}[{$key}]" : $key; - - if (\is_array($value)) { - $output += $this->flatten($value, $finalKey); - } else { - $output[$finalKey] = $value; - } - } - - return $output; + return $this->client ??= (new Client(new CurlAdapter())) + ->withFollowRedirects() + ->withConnectTimeout(static::CONNECT_TIMEOUT) + ->withTimeout(static::REQUEST_TIMEOUT); } /**