From 3dfefdf4f56ce31bb9ccb1a467665543e80cd3b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20de=20Oliveira?= Date: Fri, 30 Oct 2026 23:13:33 -0400 Subject: [PATCH 1/2] Fixes for 4.5.5 - X/Twitter oEmbed from publish.twitter.com with the requested status URL (#520, #536) - Keep text after a "<" that cannot open a tag (#372) - Keep inner spaces in URLs (#485) - Use the final URL for media detected by content type (#524) - Release curl handles right away on PHP 8 (#283) --- CHANGELOG.md | 15 ++ src/Adapters/Twitter/Detectors/Title.php | 11 ++ src/Adapters/Twitter/Extractor.php | 20 +++ src/Adapters/Twitter/OEmbed.php | 48 ++++++ src/ApiTrait.php | 24 ++- src/Detectors/Code.php | 3 +- src/Http/CurlDispatcher.php | 16 +- src/functions.php | 4 + tests/CurlClientTest.php | 25 +++ tests/FunctionsTest.php | 12 ++ tests/PatchFixesTest.php | 143 ++++++++++++++++++ tests/fixtures/i.imgur.com.x6rkcc5.jpg.php | 2 +- ...om.pepephone-status-436461658601713664.php | 1 - 13 files changed, 319 insertions(+), 5 deletions(-) create mode 100644 src/Adapters/Twitter/OEmbed.php create mode 100644 tests/PatchFixesTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 62ba94d1..dd3e5c41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - NetEase Music adapter: songs, albums and playlists on `music.163.com` get an `outchain/player` iframe, and the author is read from the description. URLs with a `#/` route are requested without it. MV pages (`/mv?id=` and `/#/mv?id=`) get metadata only, because NetEase has no official embeddable MV player [#243] - `Extractor::normalizeUri()`, used by adapters to rewrite a URL before it is requested +## [4.5.5] - 2026-10-30 +### Fixed +- X/Twitter status URLs get their oEmbed from `publish.twitter.com` with the status URL that was requested, even when logged-out visitors are redirected to `x.com/home`, the `x.com/x/migrate` form or a login wall. `mobile.twitter.com` and `/statuses/` URLs are accepted, query strings like `?s=20` are dropped, and the title is no longer "x.com" [#520] [#536] +- A `<` that cannot open a tag (like in `Brytere =<250A`) no longer makes the rest of a title or description disappear [#372] +- Image and other URLs with inner spaces (like `1024-576 logo.jpg`) keep them, encoded as `%20`, instead of being collapsed [#485] +- Images and other media detected by content type use the final URL after redirects as their `src` [#524] +- On PHP 8, curl handles (and their file descriptors) are released as soon as a request finishes, instead of waiting for the garbage collector, so long running workers no longer hit "Too many open files" [#283] + ## [4.5.4] - 2026-10-30 ### Security - TLS verification is on by default. `CurlClient` now sends `CURLOPT_SSL_VERIFYPEER` true and `CURLOPT_SSL_VERIFYHOST` 2 unless told otherwise, so self-signed, expired and wrong-host certificates are rejected (curl error 60). Before, both checks were off when the settings were missing [#575] @@ -356,10 +364,17 @@ Full library refactoring. [#584]: https://github.com/php-embed/Embed/issues/584 [#243]: https://github.com/php-embed/Embed/issues/243 [#575]: https://github.com/php-embed/Embed/issues/575 +[#283]: https://github.com/php-embed/Embed/issues/283 +[#372]: https://github.com/php-embed/Embed/issues/372 +[#485]: https://github.com/php-embed/Embed/issues/485 +[#520]: https://github.com/php-embed/Embed/issues/520 +[#524]: https://github.com/php-embed/Embed/issues/524 +[#536]: https://github.com/php-embed/Embed/issues/536 [#574]: https://github.com/oscarotero/Embed/issues/574 [#577]: https://github.com/oscarotero/Embed/issues/577 [4.6.0]: https://github.com/php-embed/Embed/compare/v4.5.4...v4.6.0 +[4.5.5]: https://github.com/php-embed/Embed/compare/v4.5.4...v4.5.5 [4.5.4]: https://github.com/php-embed/Embed/compare/v4.5.3...v4.5.4 [4.5.3]: https://github.com/php-embed/Embed/compare/v4.5.2...v4.5.3 [4.5.2]: https://github.com/php-embed/Embed/compare/v4.5.1...v4.5.2 diff --git a/src/Adapters/Twitter/Detectors/Title.php b/src/Adapters/Twitter/Detectors/Title.php index 6774f7af..b5fa2647 100644 --- a/src/Adapters/Twitter/Detectors/Title.php +++ b/src/Adapters/Twitter/Detectors/Title.php @@ -20,6 +20,17 @@ public function detect(): ?string return "Tweet by $name"; } + // After a redirect to a login or migration page the HTML title is + // not about the tweet, so use the oEmbed author instead. + $status = $this->extractor->getStatusUri(); + $final = $this->extractor->getUri(); + if ($status !== null && strpos($final->getPath(), $status->getPath()) !== 0) { + $author = $this->extractor->getOEmbed()->str('author_name'); + if ($author !== null) { + return "Tweet by $author"; + } + } + return parent::detect(); } } diff --git a/src/Adapters/Twitter/Extractor.php b/src/Adapters/Twitter/Extractor.php index d505e5ee..ad4ae37a 100644 --- a/src/Adapters/Twitter/Extractor.php +++ b/src/Adapters/Twitter/Extractor.php @@ -24,6 +24,26 @@ public function __construct( ) { parent::__construct($uri, $request, $response, $crawler); $this->api = new Api($this); + $this->oembed = new OEmbed($this); + } + + /** + * The status URL of the tweet, taken from the requested URL first because + * the final URL can be a login or migration page after the redirects. + */ + public function getStatusUri(): ?UriInterface + { + foreach ([$this->getRequest()->getUri(), $this->getUri()] as $uri) { + $host = strtolower($uri->getHost()); + if (preg_match('/^(?:(?:www|mobile)\\.)?(twitter|x)\\.com$/', $host, $site) !== 1) { + continue; + } + if (preg_match('#^/([^/]+)/status(?:es)?/(\\d+)#', $uri->getPath(), $match) === 1) { + return $this->getCrawler()->createUri("https://{$site[1]}.com/{$match[1]}/status/{$match[2]}"); + } + } + + return null; } public function getApi(): Api diff --git a/src/Adapters/Twitter/OEmbed.php b/src/Adapters/Twitter/OEmbed.php new file mode 100644 index 00000000..69480dd6 --- /dev/null +++ b/src/Adapters/Twitter/OEmbed.php @@ -0,0 +1,48 @@ + + */ + protected function fetchData(): array + { + $data = parent::fetchData(); + + if ($data !== [] && $this->statusUrl !== null) { + $data += ['url' => $this->statusUrl]; + } + + return $data; + } + + protected function detectEndpoint(): ?UriInterface + { + $status = $this->extractor instanceof Extractor ? $this->extractor->getStatusUri() : null; + + if ($status === null) { + return parent::detectEndpoint(); + } + + $this->statusUrl = (string) $status; + + return $this->extractor->getCrawler() + ->createUri(self::ENDPOINT) + ->withQuery(http_build_query($this->getOembedQueryParameters((string) $status))); + } +} diff --git a/src/ApiTrait.php b/src/ApiTrait.php index d4891c85..d077bdc1 100644 --- a/src/ApiTrait.php +++ b/src/ApiTrait.php @@ -104,7 +104,7 @@ public function int(string ...$keys): ?int public function url(string ...$keys): ?UriInterface { - $url = $this->str(...$keys); + $url = $this->rawUrl(...$keys); try { return $url !== null ? $this->extractor->resolveUri($url) : null; @@ -113,6 +113,28 @@ public function url(string ...$keys): ?UriInterface } } + /** + * Inner spaces are part of the URL, so they are kept (and later encoded) + * instead of being collapsed like in str(). Tabs and line breaks are + * removed, as browsers do. + */ + private function rawUrl(string ...$keys): ?string + { + $value = $this->get(...$keys); + + if (is_array($value)) { + $value = array_shift($value); + } + + if (!is_string($value)) { + return null; + } + + $value = trim(str_replace(["\t", "\r", "\n"], '', $value)); + + return $value === '' ? null : $value; + } + public function time(string ...$keys): ?DateTime { $time = $this->str(...$keys); diff --git a/src/Detectors/Code.php b/src/Detectors/Code.php index b693d314..e11b80d4 100644 --- a/src/Detectors/Code.php +++ b/src/Detectors/Code.php @@ -226,7 +226,8 @@ private function detectFromContentType(): ?EmbedCode return null; } - $url = $this->extractor->getRequest()->getUri(); + // The final URL, so a file behind a redirect is embedded from where it is. + $url = $this->extractor->getUri(); if (strpos($contentType, 'video/') === 0 || $contentType === 'application/mp4') { $code = html('video', [ diff --git a/src/Http/CurlDispatcher.php b/src/Http/CurlDispatcher.php index 14175c48..bcd3e0a8 100644 --- a/src/Http/CurlDispatcher.php +++ b/src/Http/CurlDispatcher.php @@ -704,8 +704,22 @@ private function error(string $message, int $code): void private function closeCurl(): void { - if ($this->closed || PHP_VERSION_ID >= 80000) { + if ($this->closed) { + return; + } + + if (PHP_VERSION_ID >= 80000) { + // The header and body callbacks point back to $this, so the handle + // and this object keep each other alive until the garbage + // collector runs, and every request leaves its sockets open. + // Replacing them with static callbacks breaks that cycle, so the + // handle is freed as soon as the response is built. $this->closed = true; + $this->setopt(CURLOPT_HEADERFUNCTION, static fn ($curl, string $data): int => strlen($data)); + $this->setopt(CURLOPT_WRITEFUNCTION, static fn ($curl, string $data): int => strlen($data)); + if (PHP_VERSION_ID >= 80400 && defined('CURLOPT_PREREQFUNCTION')) { + $this->setopt(CURLOPT_PREREQFUNCTION, static fn (): int => CURL_PREREQFUNC_OK); + } return; } diff --git a/src/functions.php b/src/functions.php index 09569e05..8118b4d0 100644 --- a/src/functions.php +++ b/src/functions.php @@ -11,7 +11,11 @@ function clean(string $value, bool $allowHTML = false): ?string if (!$allowHTML) { $value = html_entity_decode($value, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401); + // A "<" before a digit, space or "=" (like in "=<250A") is plain text, but + // strip_tags() would drop everything after it. + $value = preg_replace('/<(?=[\d\s=]|$)/', "\u{E000}", $value) ?? $value; $value = strip_tags($value); + $value = str_replace("\u{E000}", '<', $value); } $replaced = preg_replace('/\s+/u', ' ', $value); diff --git a/tests/CurlClientTest.php b/tests/CurlClientTest.php index b1811ed1..a26fdd1a 100644 --- a/tests/CurlClientTest.php +++ b/tests/CurlClientTest.php @@ -439,6 +439,31 @@ public function testDnsRebindingDoesNotMoveTheConnection(): void $this->assertSame(['/landed'], self::$server->paths()); } + /** + * @see https://github.com/php-embed/Embed/issues/283 + */ + public function testRequestsDoNotLeaveFileDescriptorsOpen(): void + { + if (!is_dir('/dev/fd')) { + $this->markTestSkipped('/dev/fd is not available'); + } + + $client = $this->client(); + $factory = new Psr17Factory(); + $client->sendRequest($this->request($this->url('/landed'))); + $before = count((array) glob('/dev/fd/*')); + + for ($i = 0; $i < 10; ++$i) { + $client->sendRequest($this->request($this->url('/chain?i=0&n=1'))); + $client->sendRequests( + $factory->createRequest('GET', $this->url('/landed')), + $factory->createRequest('GET', $this->url('/landed')) + ); + } + + $this->assertLessThanOrEqual($before + 2, count((array) glob('/dev/fd/*'))); + } + public function testParallelRequestsFollowRedirects(): void { $factory = new Psr17Factory(); diff --git a/tests/FunctionsTest.php b/tests/FunctionsTest.php index a6a5b800..7f7643bf 100644 --- a/tests/FunctionsTest.php +++ b/tests/FunctionsTest.php @@ -3,6 +3,7 @@ namespace Embed\Tests; +use function Embed\clean; use function Embed\isHttp; use function Embed\resolveUri; use Nyholm\Psr7\Factory\Psr17Factory; @@ -52,4 +53,15 @@ public function testResolveUriKeepsTheBasePort(): void $other = resolveUri($base, $factory->createUri('https://other.example:9/z')); $this->assertSame('https://other.example:9/z', (string) $other); } + + /** + * @see https://github.com/php-embed/Embed/issues/372 + */ + public function testCleanKeepsALessThanSignThatIsNotATag(): void + { + $this->assertSame('Brytere =<250A | Shop', clean('Brytere =<250A | Shop')); + $this->assertSame('1 < 2 and 3 <= 4', clean('1 < 2 and 3 <= 4')); + $this->assertSame('Bold text', clean('Bold text')); + $this->assertSame('alert(1)Hi', clean('<script>alert(1)</script>Hi')); + } } diff --git a/tests/PatchFixesTest.php b/tests/PatchFixesTest.php new file mode 100644 index 00000000..18f01264 --- /dev/null +++ b/tests/PatchFixesTest.php @@ -0,0 +1,143 @@ +embed(static fn (): array => [ + 'text/html; charset=utf-8', + 'Tilbehør for brytere =<250A | Effektbrytere', + null, + ])->get('https://shop.public.test/category'); + + $this->assertSame('Tilbehør for brytere =<250A | Effektbrytere', $info->title); + } + + /** + * @see https://github.com/php-embed/Embed/issues/485 + */ + public function testSpacesInsideAnImageUrlAreKept(): void + { + $info = $this->embed(static fn (): array => [ + 'text/html; charset=utf-8', + '', + null, + ])->get('https://www.public.test/'); + + $this->assertSame('https://www.public.test/user/pages/01.home/1024-576%20%20logo.jpg', (string) $info->image); + } + + /** + * @see https://github.com/php-embed/Embed/issues/524 + */ + public function testImageBehindARedirectIsEmbeddedFromTheFinalUrl(): void + { + $info = $this->embed(static fn (): array => [ + 'image/jpeg', + '', + 'https://cdn.public.test/hug.jpeg', + ])->get('https://cdn.public.test/301/hug.jpeg'); + + $this->assertSame('https://cdn.public.test/hug.jpeg', (string) $info->url); + $this->assertNotNull($info->code); + $this->assertSame('', $info->code->html); + } + + /** + * @return array + */ + public function twitterRedirectProvider(): array + { + return [ + 'x.com to /home' => ['https://x.com/pepephone/status/436461658601713664', 'https://x.com/home'], + 'twitter.com to the migrate form' => ['https://twitter.com/pepephone/status/436461658601713664', 'https://x.com/x/migrate'], + 'mobile twitter.com to a login wall' => ['https://mobile.twitter.com/pepephone/status/436461658601713664?s=20', 'https://x.com/i/flow/login'], + ]; + } + + /** + * @dataProvider twitterRedirectProvider + * @see https://github.com/php-embed/Embed/issues/520 + * @see https://github.com/php-embed/Embed/issues/536 + */ + public function testTwitterUsesTheRequestedStatusForOEmbed(string $url, string $final): void + { + $oembed = (string) json_encode([ + 'url' => 'https://twitter.com/pepephone/status/436461658601713664', + 'author_name' => 'Pepephone', + 'author_url' => 'https://twitter.com/pepephone', + 'html' => '', + 'width' => 550, + 'type' => 'rich', + 'version' => '1.0', + ], JSON_UNESCAPED_SLASHES); + + $client = null; + $embed = $this->embed(static function (RequestInterface $request) use ($oembed, $final): array { + if ($request->getUri()->getHost() === 'publish.twitter.com') { + return ['application/json', $oembed, null]; + } + + return ['text/html; charset=utf-8', 'XLog in', $final]; + }, $client); + + $info = $embed->get($url); + + $this->assertNotNull($client); + $oembedRequests = array_values(array_filter( + $client->requests, + static fn (RequestInterface $request): bool => $request->getUri()->getHost() === 'publish.twitter.com' + )); + $this->assertCount(1, $oembedRequests); + parse_str($oembedRequests[0]->getUri()->getQuery(), $query); + $this->assertStringEndsWith('/pepephone/status/436461658601713664', (string) $query['url']); + $this->assertStringNotContainsString('home', (string) $query['url']); + + $this->assertNotNull($info->code); + $this->assertStringContainsString('twitter-tweet', $info->code->html); + $this->assertSame('Pepephone', $info->authorName); + $this->assertSame('Tweet by Pepephone', $info->title); + $this->assertSame('https://twitter.com/pepephone/status/436461658601713664', (string) $info->url); + } + + /** + * @param callable(RequestInterface): array{0: string, 1: string, 2: ?string} $handler + * @param-out RecordingClient $client + */ + private function embed(callable $handler, ?RecordingClient &$client = null): Embed + { + $client = new RecordingClient(static function (RequestInterface $request) use ($handler): ResponseInterface { + [$type, $body, $final] = $handler($request); + $factory = new Psr17Factory(); + + return $factory->createResponse(200) + ->withHeader('Content-Type', $type) + ->withHeader('Content-Location', $final ?? (string) $request->getUri()) + ->withBody($factory->createStream($body)); + }); + + $crawler = new Crawler($client); + $crawler->setUrlPolicy(UrlPolicy::default()->withResolver(static function (string $host): array { + return ['1.1.1.1']; + })); + + return new Embed($crawler); + } +} diff --git a/tests/fixtures/i.imgur.com.x6rkcc5.jpg.php b/tests/fixtures/i.imgur.com.x6rkcc5.jpg.php index 523fd36f..3570cb69 100644 --- a/tests/fixtures/i.imgur.com.x6rkcc5.jpg.php +++ b/tests/fixtures/i.imgur.com.x6rkcc5.jpg.php @@ -6,7 +6,7 @@ 'authorUrl' => null, 'cms' => null, 'code' => [ - 'html' => '', + 'html' => '', 'width' => null, 'height' => null, 'ratio' => null diff --git a/tests/fixtures/twitter.com.pepephone-status-436461658601713664.php b/tests/fixtures/twitter.com.pepephone-status-436461658601713664.php index 2b5b0fcb..fbccc744 100644 --- a/tests/fixtures/twitter.com.pepephone-status-436461658601713664.php +++ b/tests/fixtures/twitter.com.pepephone-status-436461658601713664.php @@ -33,7 +33,6 @@ 'author_url' => 'https://twitter.com/pepephone', 'html' => ' - ', 'width' => 550, 'height' => null, From 6d1b0e3fd8cb9712018b1c9c837e1ddb73719a85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20de=20Oliveira?= Date: Fri, 30 Oct 2026 23:39:36 -0400 Subject: [PATCH 2/2] Target 4.6.1 --- CHANGELOG.md | 18 +++++++++--------- tests/PatchFixesTest.php | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd3e5c41..f5af927f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [4.6.1] - 2026-10-30 +### Fixed +- X/Twitter status URLs get their oEmbed from `publish.twitter.com` with the status URL that was requested, even when logged-out visitors are redirected to `x.com/home`, the `x.com/x/migrate` form or a login wall. `mobile.twitter.com` and `/statuses/` URLs are accepted, query strings like `?s=20` are dropped, and the title is no longer "x.com" [#520] [#536] +- A `<` that cannot open a tag (like in `Brytere =<250A`) no longer makes the rest of a title or description disappear [#372] +- Image and other URLs with inner spaces (like `1024-576 logo.jpg`) keep them, encoded as `%20`, instead of being collapsed [#485] +- Images and other media detected by content type use the final URL after redirects as their `src` [#524] +- On PHP 8, curl handles (and their file descriptors) are released as soon as a request finishes, instead of waiting for the garbage collector, so long running workers no longer hit "Too many open files" [#283] + ## [4.6.0] - 2026-10-30 ### Added - `mastodon:iframe` setting (off by default). Mastodon oEmbed html (`blockquote.mastodon-embed` with `data-embed-url`) becomes a sandboxed `