From 04fd5b10375aa8e528a2766a583b2744cd540d06 Mon Sep 17 00:00:00 2001 From: Konstantin Obenland Date: Tue, 13 Oct 2026 14:54:15 -0500 Subject: [PATCH 01/10] FreeScout: Keep WordPress.org's copy of conversations up to date The plugin directory reads a copy of helpdesk conversations, wporg_helpscout, which HelpScout's webhook keeps. FreeScout's webhook now keeps it too: WPOrgWebhooks' events carry the conversation and the threads webhook.php needs to find the plugins and themes it mentions, and webhook.php writes the copy, one event at a time per conversation, in a transaction. The HelpScout import can point the copy at the imported conversations once a mailbox moves, through a new replace-copies.php, after which HelpScout's webhook leaves them alone. Split out of #1035, so that one can concentrate on the reviewers' tools. Co-Authored-By: Claude Opus 5.5 --- .../public_html/dotorg/freescout/common.php | 325 +++++++++++ .../dotorg/freescout/replace-copies.php | 152 +++++ .../dotorg/freescout/tests/bootstrap.php | 10 + .../freescout/tests/class-copy-plan-test.php | 289 ++++++++++ .../freescout/tests/class-email-row-test.php | 134 +++++ .../public_html/dotorg/freescout/webhook.php | 242 +++++++- .../public_html/dotorg/helpscout/common.php | 32 ++ environments/freescout/mock-api/index.php | 6 + .../plugin-directory/bin/database-tables.sql | 20 +- freescout.wordpress.net/AGENTS.md | 4 +- ..._to_wporghelpscoutimport_conversations.php | 74 +++ .../Entities/ImportedConversation.php | 2 +- .../Http/Controllers/ImportController.php | 48 ++ .../WPOrgHelpScoutImport/Http/routes.php | 1 + .../WPOrgHelpScoutImport/Jobs/ImportPage.php | 6 +- .../Jobs/ReplaceCopies.php | 120 ++++ .../WPOrgHelpScoutImportServiceProvider.php | 69 +++ .../Modules/WPOrgHelpScoutImport/README.md | 11 +- .../Resources/views/index.blade.php | 45 ++ .../WPOrgHelpScoutImport/Services/Copies.php | 531 ++++++++++++++++++ .../Services/Importer.php | 22 + .../WPOrgHelpScoutImport/tests/CopiesTest.php | 418 ++++++++++++++ .../tests/ImporterTest.php | 52 ++ .../Modules/WPOrgWebhooks/Jobs/SendEvent.php | 57 +- .../WPOrgWebhooksServiceProvider.php | 78 ++- .../Modules/WPOrgWebhooks/README.md | 17 +- .../Modules/WPOrgWebhooks/Services/Client.php | 71 ++- .../WPOrgWebhooks/Services/EventPayload.php | 322 ++++++++++- .../Services/PayloadTooLargeException.php | 19 + .../tests/EventForwardingTest.php | 261 +++++++++ .../WPOrgWebhooks/tests/SendEventTest.php | 80 ++- 31 files changed, 3458 insertions(+), 60 deletions(-) create mode 100644 api.wordpress.org/public_html/dotorg/freescout/replace-copies.php create mode 100644 api.wordpress.org/public_html/dotorg/freescout/tests/class-copy-plan-test.php create mode 100644 api.wordpress.org/public_html/dotorg/freescout/tests/class-email-row-test.php create mode 100644 freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Database/Migrations/2026_10_10_000000_add_mailbox_to_wporghelpscoutimport_conversations.php create mode 100644 freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ReplaceCopies.php create mode 100644 freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php create mode 100644 freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php create mode 100644 freescout.wordpress.net/Modules/WPOrgWebhooks/Services/PayloadTooLargeException.php diff --git a/api.wordpress.org/public_html/dotorg/freescout/common.php b/api.wordpress.org/public_html/dotorg/freescout/common.php index c6f452476c..53984d0f5b 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/common.php +++ b/api.wordpress.org/public_html/dotorg/freescout/common.php @@ -176,6 +176,44 @@ function send_panel( array $blocks, array $extra = array() ): never { exit; } +/** + * A conversation's row in WordPress.org's copy of helpdesk emails, which the plugin directory reads. + * + * The copy began with HelpScout's emails, so its columns, and the table's name, are HelpScout's; the values are the + * same for FreeScout's conversations, but for the ID. + * + * @param object $request Webhook payload, with the conversation in `email`; see WPOrgWebhooks' EventPayload. + * @param object|null $row The conversation's row so far, if it has one. + * @param int $user_id The sender's WordPress.org user ID, 0 if unknown. + * @return array + */ +function get_email_row( object $request, ?object $row, int $user_id ): array { + $email = $request->email ?? new \stdClass(); + $sender = $email->sender ?? new \stdClass(); + $address = (string) ( $sender->email ?? '' ); + $name = trim( ( $sender->first_name ?? '' ) . ' ' . ( $sender->last_name ?? '' ) ); + + $created = (int) strtotime( (string) ( $email->created_at ?? '' ) ); + $created = $created ? $created : (int) strtotime( (string) ( $row->created ?? '' ) ); + $closed = (int) strtotime( (string) ( $email->closed_at ?? '' ) ); + $updated = (int) strtotime( (string) ( $email->user_updated_at ?? '' ) ); + $times = array_filter( array( $created, $updated, $closed ) ); + + return array( + 'id' => (int) ( $request->conversation->id ?? 0 ), + 'number' => (int) ( $email->number ?? 0 ), + 'user_id' => $user_id, + 'mailbox' => get_copy_mailbox_slug( $request ), + 'status' => (string) ( $email->status ?? '' ), + 'email' => $address ? ( $name ? "{$name} <{$address}>" : $address ) : (string) ( $row->email ?? '' ), + 'subject' => (string) ( $email->subject ?? ( $row->subject ?? '' ) ), + 'preview' => (string) ( $email->preview ?? ( $row->preview ?? '' ) ), + 'created' => gmdate( 'Y-m-d H:i:s', $created ? $created : time() ), + 'closed' => $closed ? gmdate( 'Y-m-d H:i:s', $closed ) : '', + 'modified' => gmdate( 'Y-m-d H:i:s', $times ? max( $times ) : time() ), + ); +} + /** * A status badge; WPOrgSidebar's stylesheet colors it by tone. * @@ -214,6 +252,17 @@ function get_mailbox_slug( object $request ): string { return sanitize_title( (string) ( $request->mailbox->name ?? '' ) ); } +/** + * The slug of the mailbox WordPress.org's copy keeps a conversation in: the one it's in now, which the event carries in + * `email`, rather than the one the event happened in, which stats count. + * + * @param object $request Webhook payload. + * @return string + */ +function get_copy_mailbox_slug( object $request ): string { + return sanitize_title( (string) ( $request->email->mailbox->name ?? ( $request->mailbox->name ?? '' ) ) ); +} + /** * Gets the plain text of a thread body. * @@ -623,3 +672,279 @@ function get_plugin_slugs_by_title( string $title ): array { return wp_list_pluck( $plugins, 'post_name' ); } + +/** + * Meta key naming the helpdesk a copied conversation is in, `freescout`; HelpScout's copies have none. + * + * @var string + */ +const HELPDESK_META_KEY = 'helpdesk'; + +/** + * Meta key of a FreeScout conversation's copy naming a HelpScout conversation whose copy it replaced. + * + * HelpScout's webhook leaves alone a HelpScout conversation a copy names, so work done there since doesn't bring its + * copy back. It goes with the copy that names it. + * + * @var string + */ +const REPLACED_HELPSCOUT_META_KEY = 'helpscout_conversation'; + +/** + * What WordPress.org's copy does with a conversation's event: it's written, or deleted. + * + * @var string + */ +const COPY_DONE = 'done'; + +/** + * What WordPress.org's copy does with a conversation's event: nothing yet, as another write held the conversation up, + * or the write failed; FreeScout sends the event again. + * + * @var string + */ +const COPY_RETRY = 'retry'; + +/** + * What WordPress.org's copy does with a conversation's event: nothing yet, as it has no copy of the conversation, and + * the event only carries its newest threads; FreeScout sends it again with all of them. + * + * @var string + */ +const COPY_ALL_THREADS = 'all_threads'; + +/** + * How long a write to WordPress.org's copy of a conversation waits for another to finish, in seconds. + * + * Well under FreeScout's 10-second timeout for an event, so a held-up event is answered, and sent again, rather than + * written after FreeScout gave up on it, and sent again anyway. + * + * @var int + */ +const LOCK_TIMEOUT = 3; + +/** + * Takes the lock on WordPress.org's copy of a conversation, which writes to it hold, one at a time. + * + * The copy's meta table has no unique key to keep a pair from being added twice. The lock, and what's read while holding + * it, come from the primary database, not a replica. + * + * @param int $id Conversation ID. + * @return bool False if another write held it for too long. + */ +function lock_email( int $id ): bool { + global $wpdb; + + if ( is_callable( array( $wpdb, 'send_reads_to_masters' ) ) ) { + $wpdb->send_reads_to_masters(); + } + + return (bool) $wpdb->get_var( $wpdb->prepare( 'SELECT GET_LOCK( %s, %d )', 'wporg_helpscout_' . $id, LOCK_TIMEOUT ) ); +} + +/** + * Releases the lock on WordPress.org's copy of a conversation. + * + * @param int $id Conversation ID. + * @return void + */ +function unlock_email( int $id ): void { + global $wpdb; + + $wpdb->query( $wpdb->prepare( 'SELECT RELEASE_LOCK( %s )', 'wporg_helpscout_' . $id ) ); +} + +/** + * Deletes conversations from WordPress.org's copy, with their meta, and so the HelpScout conversations they name. + * + * @param int[] $ids Conversation IDs; HelpScout's for the copies of its conversations. + * @return void + * + * @throws \RuntimeException If a deletion failed. + */ +function delete_emails( array $ids ): void { + global $wpdb; + + if ( ! $ids ) { + return; + } + + // phpcs:disable WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare -- The placeholders are added for each ID. + check_write( $wpdb->query( $wpdb->prepare( 'DELETE FROM %i WHERE id IN (' . implode( ',', array_fill( 0, count( $ids ), '%d' ) ) . ')', array_merge( array( "{$wpdb->base_prefix}helpscout" ), $ids ) ) ) ); + check_write( $wpdb->query( $wpdb->prepare( 'DELETE FROM %i WHERE helpscout_id IN (' . implode( ',', array_fill( 0, count( $ids ), '%d' ) ) . ')', array_merge( array( "{$wpdb->base_prefix}helpscout_meta" ), $ids ) ) ) ); + // phpcs:enable +} + +/** + * The copies an event's conversation replaces: one merged into it, and the HelpScout conversations it, or the one + * merged into it, were imported from. + * + * @param object $request Webhook payload. + * @return array[] FreeScout's IDs, under `freescout`, and HelpScout's, under `helpscout`. + */ +function get_replaced_ids( object $request ): array { + return array( + 'freescout' => array_values( array_filter( array( (int) ( $request->merged_id ?? 0 ) ) ) ), + 'helpscout' => array_values( + array_unique( + array_filter( + array( + (int) ( $request->helpscout_id ?? 0 ), + (int) ( $request->merged_helpscout_id ?? 0 ), + ) + ) + ) + ), + ); +} + +/** + * Whether WordPress.org needs all of a conversation's threads before it copies it: it has no copy of it yet, nor of + * the HelpScout conversation it was imported from, which would have what the threads mention, and the event carries + * only the newest. + * + * @param object $request Webhook payload. + * @param object|null $row The conversation's row in the copy, if it has one. + * @param bool $has_helpscout Whether the copy has the HelpScout conversation it was imported from. + * @return bool + */ +function needs_all_threads( object $request, ?object $row, bool $has_helpscout ): bool { + return ! $row && ! $has_helpscout && empty( $request->email->all_threads ); +} + +/** + * What writing a conversation's copy adds to its meta, and which copies it deletes. + * + * It takes over what the copies it replaces mention, names the HelpScout conversations it replaces, and adds what its + * threads mention. A HelpScout conversation it names already was replaced by an earlier event, so it isn't again. + * + * @param int $id Conversation ID. + * @param array[] $replaced The copies it replaces, see get_replaced_ids(). + * @param array[] $all_meta The meta of it and of the copies it replaces: helpscout_id, meta_key, and meta_value. + * @param array[] $mentioned The plugins' and themes' slugs its threads and subject mention, by type. + * @return array Meta to add, as meta_key and meta_value pairs, under `meta`; IDs of the copies to delete, under `delete`. + */ +function plan_email_write( int $id, array $replaced, array $all_meta, array $mentioned ): array { + $own = array(); + $by_copy = array(); + $replaces = array(); + foreach ( $all_meta as $pair ) { + $key = $pair['meta_key'] . '|' . $pair['meta_value']; + if ( (int) $pair['helpscout_id'] === $id ) { + $own[ $key ] = true; + } else { + $by_copy[ (int) $pair['helpscout_id'] ][ $key ] = $pair; + } + + if ( REPLACED_HELPSCOUT_META_KEY === $pair['meta_key'] && (int) $pair['helpscout_id'] === $id ) { + $replaces[ (int) $pair['meta_value'] ] = true; + } + } + + // A HelpScout conversation the copy names was replaced already. + $helpscout = array_values( + array_filter( + $replaced['helpscout'], + static function ( int $helpscout_id ) use ( $replaces ): bool { + return ! isset( $replaces[ $helpscout_id ] ); + } + ) + ); + $delete = array_merge( $replaced['freescout'], $helpscout ); + + $meta = array(); + foreach ( $delete as $copy_id ) { + foreach ( $by_copy[ $copy_id ] ?? array() as $key => $pair ) { + $meta[ $key ] = array( $pair['meta_key'], $pair['meta_value'] ); + } + } + + $meta[ HELPDESK_META_KEY . '|freescout' ] = array( HELPDESK_META_KEY, 'freescout' ); + foreach ( $replaced['helpscout'] as $helpscout_id ) { + $meta[ REPLACED_HELPSCOUT_META_KEY . '|' . $helpscout_id ] = array( REPLACED_HELPSCOUT_META_KEY, (string) $helpscout_id ); + } + + foreach ( $mentioned as $type => $slugs ) { + foreach ( $slugs as $slug ) { + $meta[ $type . '|' . $slug ] = array( (string) $type, (string) $slug ); + } + } + + return array( + 'meta' => array_values( array_diff_key( $meta, $own ) ), + 'delete' => $delete, + ); +} + +/** + * What pointing WordPress.org at an imported conversation does to its copy. + * + * A conversation merged into another before the switch is sent as the one it went into, `merged`: its HelpScout one's + * copy is merged into that one's, whose own HelpScout copy may not be renamed yet, so it takes over what that copy + * mentions, as a merge after the switch does, rather than standing in for it. + * + * @param object $copy The imported conversation: helpscout_id, id, number, delete, and merged. + * @param bool $has_helpscout Whether the copy has the HelpScout conversation. + * @param bool $has_freescout Whether it has the FreeScout one. + * @param bool $names_it Whether the FreeScout one's copy names the HelpScout one, as it replaced it already. + * @return string `skip`; `delete` both, as the conversation was deleted, or is spam; `rename` the HelpScout one's copy + * into the FreeScout one's; `merge` it into the FreeScout one's copy; or `name` it in that copy. + */ +function plan_copy_replacement( object $copy, bool $has_helpscout, bool $has_freescout, bool $names_it ): string { + $helpscout_id = (int) ( $copy->helpscout_id ?? 0 ); + $id = (int) ( $copy->id ?? 0 ); + if ( $helpscout_id <= 0 || $id <= 0 || $helpscout_id === $id ) { + return 'skip'; + } + + if ( ! empty( $copy->delete ) ) { + return $has_helpscout || $has_freescout ? 'delete' : 'skip'; + } + + if ( $has_helpscout ) { + return $has_freescout || ! empty( $copy->merged ) ? 'merge' : 'rename'; + } + + // Its own events wrote the FreeScout one's copy, while HelpScout's was gone: it still keeps HelpScout's webhook out. + return $has_freescout && ! $names_it ? 'name' : 'skip'; +} + +/** + * Runs writes to WordPress.org's copy in a transaction, so none of them are seen, or kept, without the others. + * + * @param callable $write Writes; throws if one fails, see check_write(). + * @return void + * + * @throws \Throwable What the writes threw, once they're rolled back. + */ +function in_transaction( callable $write ): void { + global $wpdb; + + check_write( $wpdb->query( 'START TRANSACTION' ) ); + + try { + $write(); + } catch ( \Throwable $e ) { + $wpdb->query( 'ROLLBACK' ); + + throw $e; + } + + check_write( $wpdb->query( 'COMMIT' ) ); +} + +/** + * Throws if a write failed: wpdb returns false instead. + * + * @param mixed $result What wpdb returned. + * @return void + * + * @throws \RuntimeException If it failed. + */ +function check_write( mixed $result ): void { + global $wpdb; + + if ( false === $result ) { + throw new \RuntimeException( 'Could not write the copy: ' . (string) $wpdb->last_error ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- Logged, not shown. + } +} diff --git a/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php b/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php new file mode 100644 index 0000000000..4146641470 --- /dev/null +++ b/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php @@ -0,0 +1,152 @@ +base_prefix}helpscout"; + $meta_table = "{$wpdb->base_prefix}helpscout_meta"; + $replaced = 0; + + foreach ( array_slice( $copies, 0, MAX_COPIES ) as $copy ) { + $copy = (object) $copy; + $helpscout_id = (int) ( $copy->helpscout_id ?? 0 ); + $id = (int) ( $copy->id ?? 0 ); + if ( $helpscout_id <= 0 || $id <= 0 || $helpscout_id === $id ) { + continue; + } + + if ( ! lock_email( $id ) ) { + return null; + } + + try { + $has_helpscout = (bool) $wpdb->get_var( $wpdb->prepare( 'SELECT 1 FROM %i WHERE id = %d', $emails_table, $helpscout_id ) ); + $has_freescout = (bool) $wpdb->get_var( $wpdb->prepare( 'SELECT 1 FROM %i WHERE id = %d', $emails_table, $id ) ); + $names_it = (bool) $wpdb->get_var( $wpdb->prepare( 'SELECT 1 FROM %i WHERE helpscout_id = %d AND meta_key = %s AND meta_value = %s', $meta_table, $id, REPLACED_HELPSCOUT_META_KEY, (string) $helpscout_id ) ); + $action = plan_copy_replacement( $copy, $has_helpscout, $has_freescout, $names_it ); + if ( 'skip' === $action ) { + continue; + } + + in_transaction( + static function () use ( $wpdb, $emails_table, $meta_table, $action, $copy, $helpscout_id, $id, $names_it ): void { + if ( 'delete' === $action ) { + delete_emails( array( $helpscout_id, $id ) ); + + return; + } + + /* + * The FreeScout copy's marks first: its helpdesk, and the HelpScout conversation it replaces. A meta row is + * unique, so nothing it has already is written again. + */ + $marks = array( REPLACED_HELPSCOUT_META_KEY => (string) $helpscout_id ); + if ( 'rename' === $action ) { + $marks[ HELPDESK_META_KEY ] = 'freescout'; + } + foreach ( $marks as $meta_key => $meta_value ) { + check_write( + $wpdb->query( + $wpdb->prepare( + 'INSERT INTO %i ( helpscout_id, meta_key, meta_value ) + SELECT %d, %s, %s FROM DUAL + WHERE NOT EXISTS ( SELECT 1 FROM %i WHERE helpscout_id = %d AND meta_key = %s AND meta_value = %s )', + $meta_table, + $id, + $meta_key, + $meta_value, + $meta_table, + $id, + $meta_key, + $meta_value + ) + ) + ); + } + + if ( 'name' === $action ) { + return; + } + + // It takes the plugins and themes it doesn't have yet, like those of a conversation merged into it. + check_write( + $wpdb->query( + $wpdb->prepare( + 'INSERT INTO %i ( helpscout_id, meta_key, meta_value ) + SELECT %d, hs.meta_key, hs.meta_value FROM %i hs + WHERE hs.helpscout_id = %d + AND NOT EXISTS ( SELECT 1 FROM %i fs WHERE fs.helpscout_id = %d AND fs.meta_key = hs.meta_key AND fs.meta_value = hs.meta_value )', + $meta_table, + $id, + $meta_table, + $helpscout_id, + $meta_table, + $id + ) + ) + ); + + if ( 'rename' === $action ) { + check_write( $wpdb->delete( $meta_table, array( 'helpscout_id' => $helpscout_id ) ) ); + check_write( $wpdb->update( $emails_table, array( 'id' => $id, 'number' => (int) ( $copy->number ?? 0 ) ), array( 'id' => $helpscout_id ) ) ); // phpcs:ignore WordPress.Arrays.ArrayDeclarationSpacing.AssociativeArrayFound -- Two columns. + } else { + delete_emails( array( $helpscout_id ) ); + } + } + ); + + ++$replaced; + } catch ( \Throwable $e ) { + trigger_error( 'FreeScout copy of conversation ' . $id . ' failed: ' . $e->getMessage(), E_USER_WARNING ); // phpcs:ignore WordPress.PHP.DevelopmentFunctions, WordPress.Security.EscapeOutput.OutputNotEscaped -- Logged, not shown. + + return null; + } finally { + unlock_email( $id ); + } + } + + return $replaced; +} + +$request = get_request( basename( __FILE__ ) ); +$replaced = replace_copies( (array) ( $request->copies ?? array() ) ); + +if ( null === $replaced ) { + status_header( 503 ); + header( 'Retry-After: ' . LOCK_TIMEOUT ); + exit; +} + +wp_send_json( array( 'replaced' => $replaced ) ); diff --git a/api.wordpress.org/public_html/dotorg/freescout/tests/bootstrap.php b/api.wordpress.org/public_html/dotorg/freescout/tests/bootstrap.php index 37e977908b..604c39000b 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/tests/bootstrap.php +++ b/api.wordpress.org/public_html/dotorg/freescout/tests/bootstrap.php @@ -51,6 +51,16 @@ function wp_strip_all_tags( string $text ): string { return trim( (string) preg_replace( '/<[^>]*>/', '', $text ) ); } +/** + * Makes a slug, like WordPress's sanitize_title(), for the plain titles the tests use. + * + * @param string $title Title. + * @return string + */ +function sanitize_title( string $title ): string { + return trim( (string) preg_replace( '/[^a-z0-9]+/', '-', strtolower( $title ) ), '-' ); +} + /** * Encodes JSON, like WordPress's wp_json_encode(). * diff --git a/api.wordpress.org/public_html/dotorg/freescout/tests/class-copy-plan-test.php b/api.wordpress.org/public_html/dotorg/freescout/tests/class-copy-plan-test.php new file mode 100644 index 0000000000..38f9c62eae --- /dev/null +++ b/api.wordpress.org/public_html/dotorg/freescout/tests/class-copy-plan-test.php @@ -0,0 +1,289 @@ +assertSame( + array( + 'freescout' => array( self::MERGED_ID ), + 'helpscout' => array( self::HELPSCOUT_ID, self::MERGED_HELPSCOUT_ID ), + ), + get_replaced_ids( $this->request() ) + ); + + $this->assertSame( + array( + 'freescout' => array(), + 'helpscout' => array(), + ), + get_replaced_ids( (object) array( 'event' => 'conversation.user_replied' ) ) + ); + } + + /** + * Writing takes over the replaced copies' plugins and themes, names its helpdesk and the HelpScout conversations it + * replaces, adds what it mentions, and deletes the replaced copies; nothing it has already is added again. + * + * @return void + */ + public function test_takes_over_the_copies_it_replaces(): void { + $plan = plan_email_write( + self::ID, + get_replaced_ids( $this->request() ), + array( + $this->meta( self::ID, 'plugins', 'my-plugin' ), + $this->meta( self::MERGED_ID, 'plugins', 'merged-plugin' ), + $this->meta( self::MERGED_ID, HELPDESK_META_KEY, 'freescout' ), + $this->meta( self::HELPSCOUT_ID, 'themes', 'old-theme' ), + $this->meta( self::HELPSCOUT_ID, 'plugins', 'my-plugin' ), + ), + array( 'plugins' => array( 'my-plugin', 'new-plugin' ) ) + ); + + $this->assertEqualsCanonicalizing( + array( + array( 'plugins', 'merged-plugin' ), + array( 'themes', 'old-theme' ), + array( HELPDESK_META_KEY, 'freescout' ), + array( REPLACED_HELPSCOUT_META_KEY, (string) self::HELPSCOUT_ID ), + array( REPLACED_HELPSCOUT_META_KEY, (string) self::MERGED_HELPSCOUT_ID ), + array( 'plugins', 'new-plugin' ), + ), + $plan['meta'] + ); + $this->assertSame( array( self::MERGED_ID, self::HELPSCOUT_ID, self::MERGED_HELPSCOUT_ID ), $plan['delete'] ); + } + + /** + * Once the copy names a HelpScout conversation, later events, and the same event sent again, leave it alone: nothing + * is deleted again, and nothing more is added. + * + * @return void + */ + public function test_leaves_helpscout_conversations_it_replaced_alone(): void { + $request = (object) array( 'helpscout_id' => self::HELPSCOUT_ID ); + + $plan = plan_email_write( + self::ID, + get_replaced_ids( $request ), + array( + $this->meta( self::ID, HELPDESK_META_KEY, 'freescout' ), + $this->meta( self::ID, REPLACED_HELPSCOUT_META_KEY, (string) self::HELPSCOUT_ID ), + ), + array() + ); + + $this->assertSame( + array( + 'meta' => array(), + 'delete' => array(), + ), + $plan + ); + } + + /** + * A copy that names HelpScout conversations keeps naming them as its meta is written again. + * + * @return void + */ + public function test_keeps_naming_what_it_replaced(): void { + $plan = plan_email_write( + self::ID, + get_replaced_ids( (object) array() ), + array( $this->meta( self::ID, REPLACED_HELPSCOUT_META_KEY, (string) self::HELPSCOUT_ID ) ), + array( 'plugins' => array( 'my-plugin' ) ) + ); + + $this->assertSame( array( array( HELPDESK_META_KEY, 'freescout' ), array( 'plugins', 'my-plugin' ) ), $plan['meta'] ); + $this->assertSame( array(), $plan['delete'] ); + } + + /** + * A conversation merged into another hands it the HelpScout conversation it named. + * + * @return void + */ + public function test_merging_hands_over_what_was_replaced(): void { + $plan = plan_email_write( + self::ID, + get_replaced_ids( (object) array( 'merged_id' => self::MERGED_ID ) ), + array( $this->meta( self::MERGED_ID, REPLACED_HELPSCOUT_META_KEY, (string) self::MERGED_HELPSCOUT_ID ) ), + array() + ); + + $this->assertContains( array( REPLACED_HELPSCOUT_META_KEY, (string) self::MERGED_HELPSCOUT_ID ), $plan['meta'] ); + $this->assertSame( array( self::MERGED_ID ), $plan['delete'] ); + } + + /** + * Only a conversation the copy has nothing of yet needs all of its threads, unless the event carries them. + * + * @return void + */ + public function test_asks_for_all_threads_only_for_a_new_copy(): void { + $request = (object) array( 'email' => (object) array( 'all_threads' => false ) ); + + $this->assertTrue( needs_all_threads( $request, null, false ) ); + $this->assertFalse( needs_all_threads( $request, (object) array( 'id' => self::ID ), false ) ); + $this->assertFalse( needs_all_threads( $request, null, true ) ); + + $request->email->all_threads = true; + $this->assertFalse( needs_all_threads( $request, null, false ) ); + } + + /** + * Pointing the copy at an imported conversation renames HelpScout's copy, or merges it into FreeScout's; one + * deleted, or spam, goes; one already pointed at is left alone. + * + * @return void + */ + public function test_plans_replacing_a_copy(): void { + $copy = (object) array( + 'helpscout_id' => self::HELPSCOUT_ID, + 'id' => self::ID, + 'number' => 1234, + 'delete' => false, + ); + + $this->assertSame( 'rename', plan_copy_replacement( $copy, true, false, false ) ); + $this->assertSame( 'merge', plan_copy_replacement( $copy, true, true, false ) ); + $this->assertSame( 'name', plan_copy_replacement( $copy, false, true, false ) ); + $this->assertSame( 'skip', plan_copy_replacement( $copy, false, true, true ) ); + $this->assertSame( 'skip', plan_copy_replacement( $copy, false, false, false ) ); + + $copy->delete = true; + $this->assertSame( 'delete', plan_copy_replacement( $copy, true, false, false ) ); + $this->assertSame( 'delete', plan_copy_replacement( $copy, false, true, true ) ); + $this->assertSame( 'skip', plan_copy_replacement( $copy, false, false, false ) ); + + $copy->id = self::HELPSCOUT_ID; + $this->assertSame( 'skip', plan_copy_replacement( $copy, true, true, false ) ); + } + + /** + * A conversation merged into another before the switch has its HelpScout copy merged into that one's, whether that + * one's copy is there yet or not, so it keeps what the copy mentions; one the copy names already is left alone. + * + * @return void + */ + public function test_plans_merging_a_conversation_merged_away(): void { + $copy = (object) array( + 'helpscout_id' => self::MERGED_HELPSCOUT_ID, + 'id' => self::ID, + 'number' => 1234, + 'delete' => false, + 'merged' => true, + ); + + $this->assertSame( 'merge', plan_copy_replacement( $copy, true, false, false ) ); + $this->assertSame( 'merge', plan_copy_replacement( $copy, true, true, false ) ); + $this->assertSame( 'skip', plan_copy_replacement( $copy, false, true, true ) ); + + $copy->delete = true; + $this->assertSame( 'delete', plan_copy_replacement( $copy, true, true, false ) ); + } + + /** + * The copy keeps a conversation in the mailbox it's in now; events from before it moved still name theirs, which + * the stats count. + * + * @return void + */ + public function test_copy_takes_the_current_mailbox(): void { + $request = json_decode( + (string) wp_json_encode( + array( + 'mailbox' => array( 'name' => 'Plugins' ), + 'email' => array( 'mailbox' => array( 'name' => 'Themes' ) ), + ) + ) + ); + + $this->assertSame( 'themes', get_copy_mailbox_slug( $request ) ); + $this->assertSame( 'plugins', get_mailbox_slug( $request ) ); + + unset( $request->email->mailbox ); + $this->assertSame( 'plugins', get_copy_mailbox_slug( $request ) ); + } + + /** + * A merge event of a conversation imported from HelpScout. + * + * @return object + */ + private function request(): object { + return (object) array( + 'event' => 'conversation.merged', + 'conversation' => (object) array( 'id' => self::ID ), + 'merged_id' => self::MERGED_ID, + 'helpscout_id' => self::HELPSCOUT_ID, + 'merged_helpscout_id' => self::MERGED_HELPSCOUT_ID, + ); + } + + /** + * A row of the copy's meta. + * + * @param int $id Conversation ID. + * @param string $key Meta key. + * @param string $value Meta value. + * @return array + */ + private function meta( int $id, string $key, string $value ): array { + return array( + 'helpscout_id' => (string) $id, + 'meta_key' => $key, // phpcs:ignore WordPress.DB.SlowDBQuery -- Not a query. + 'meta_value' => $value, // phpcs:ignore WordPress.DB.SlowDBQuery -- Not a query. + ); + } +} diff --git a/api.wordpress.org/public_html/dotorg/freescout/tests/class-email-row-test.php b/api.wordpress.org/public_html/dotorg/freescout/tests/class-email-row-test.php new file mode 100644 index 0000000000..0014dfed43 --- /dev/null +++ b/api.wordpress.org/public_html/dotorg/freescout/tests/class-email-row-test.php @@ -0,0 +1,134 @@ +assertSame( + array( + 'id' => 42, + 'number' => 1234, + 'user_id' => 7, + 'mailbox' => 'plugins', + 'status' => 'closed', + 'email' => 'Jane Author ', + 'subject' => '[WordPress Plugin Directory] Review in Progress: My Plugin', + 'preview' => 'Thanks for your submission', + 'created' => '2026-08-01 10:00:00', + 'closed' => '2026-08-03 12:00:00', + 'modified' => '2026-08-03 12:00:00', + ), + get_email_row( $this->request(), null, 7 ) + ); + } + + /** + * What the event doesn't say is kept from the row, and a conversation that was never closed has no closing date. + * + * @return void + */ + public function test_keeps_what_the_event_does_not_say(): void { + $request = $this->request(); + unset( $request->email->sender, $request->email->created_at, $request->email->subject ); + $request->email->closed_at = null; + $request->email->user_updated_at = '2026-08-02T08:00:00Z'; + + $row = get_email_row( + $request, + (object) array( + 'email' => 'Old ', + 'subject' => 'Old subject', + 'created' => '2026-07-01 00:00:00', + ), + 0 + ); + + $this->assertSame( array( 'Old ', 'Old subject' ), array( $row['email'], $row['subject'] ) ); + $this->assertSame( array( '2026-07-01 00:00:00', '', '2026-08-02 08:00:00' ), array( $row['created'], $row['closed'], $row['modified'] ) ); + } + + /** + * A sender without a name is their address. + * + * @return void + */ + public function test_sender_without_a_name_is_their_address(): void { + $request = $this->request(); + $request->email->sender->first_name = ''; + $request->email->sender->last_name = ''; + + $this->assertSame( 'jane@example.org', get_email_row( $request, null, 0 )['email'] ); + } + + /** + * The row is in the mailbox the conversation is in now, rather than the one a late event happened in. + * + * @return void + */ + public function test_row_is_in_the_current_mailbox(): void { + $request = $this->request(); + $request->email->mailbox = (object) array( + 'id' => 4, + 'name' => 'Themes', + ); + + $this->assertSame( 'themes', get_email_row( $request, null, 0 )['mailbox'] ); + } + + /** + * A webhook payload, as WPOrgWebhooks' EventPayload builds it. + * + * @return object + */ + private function request(): object { + return json_decode( + (string) wp_json_encode( + array( + 'event' => 'conversation.user_replied', + 'conversation' => array( 'id' => 42 ), + 'mailbox' => array( + 'id' => 3, + 'name' => 'Plugins', + ), + 'email' => array( + 'number' => 1234, + 'subject' => '[WordPress Plugin Directory] Review in Progress: My Plugin', + 'status' => 'closed', + 'preview' => 'Thanks for your submission', + 'created_at' => '2026-08-01T10:00:00Z', + 'closed_at' => '2026-08-03T12:00:00Z', + 'user_updated_at' => '2026-08-02T08:00:00Z', + 'sender' => array( + 'email' => 'jane@example.org', + 'emails' => array( 'jane@example.org' ), + 'first_name' => 'Jane', + 'last_name' => 'Author', + ), + 'threads' => array(), + ), + ) + ) + ); + } +} diff --git a/api.wordpress.org/public_html/dotorg/freescout/webhook.php b/api.wordpress.org/public_html/dotorg/freescout/webhook.php index 7c5380a07d..bb38c6b920 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/webhook.php +++ b/api.wordpress.org/public_html/dotorg/freescout/webhook.php @@ -1,6 +1,6 @@ agent->id ) ) { + if ( empty( $request->agent->id ) || in_array( $event, UNCREDITED_EVENTS, true ) ) { return; } $username = (string) ( $request->agent->wporg_username ?? '' ); $wporg_user = $username ? get_user_by( 'login', $username ) : false; - $stat_user = $wporg_user ? $wporg_user->user_nicename : 'FS-' . (int) $request->agent->id; - $mailbox = get_mailbox_slug( $request ); - $fields = array( 'total' ); + if ( $wporg_user && in_array( (int) $wporg_user->ID, AUTOMATION_USER_IDS, true ) ) { + return; + } + + $stat_user = $wporg_user ? $wporg_user->user_nicename : 'FS-' . (int) $request->agent->id; + $mailbox = get_mailbox_slug( $request ); + $fields = array( 'total' ); if ( in_array( $event, REPLY_EVENTS, true ) ) { $fields[] = 'replies'; @@ -52,7 +72,217 @@ function contributor_stats( object $request ): void { } } -contributor_stats( get_request( basename( __FILE__ ) ) ); +/** + * Keeps WordPress.org's copy of a conversation, and the plugins and themes it mentions, as HelpScout's webhook did. + * + * The plugin directory reads the copy for a plugin's emails. A conversation replaces the copies of one merged into it, + * and of the HelpScout conversations it, or that one, was imported from, taking over their plugins and themes: slug + * changes update those, and the emails still name the old slug. + * + * @param object $request Request payload. + * @return string COPY_DONE; COPY_RETRY if another event for the conversation held it up, or the write failed; or + * COPY_ALL_THREADS if it needs all of the conversation's threads. FreeScout sends it again for both. + */ +function log_email( object $request ): string { + $id = (int) ( $request->conversation->id ?? 0 ); + $email = $request->email ?? null; + if ( ! $id || ! is_object( $email ) ) { + return COPY_DONE; + } + + /* + * Events for the conversation are written one at a time, see lock_email(). Each carries the conversation as it was + * when it was sent, so the copy ends up as the last one written says, which is current unless events crossed. + */ + if ( ! lock_email( $id ) ) { + return COPY_RETRY; + } + + try { + // Deleted conversations and spam aren't kept; a deletion undone since is sent as published. + $deleted = isset( $email->state ) ? 'deleted' === $email->state : 'conversation.deleted' === ( $request->event ?? '' ); + if ( $deleted || 'spam' === ( $email->status ?? '' ) ) { + in_transaction( + static function () use ( $request, $id ): void { + delete_emails( array_merge( array( $id ), ...array_values( get_replaced_ids( $request ) ) ) ); + } + ); + + return COPY_DONE; + } + + return write_email( $request, $id ); + } catch ( \Throwable $e ) { + trigger_error( 'FreeScout copy of conversation ' . $id . ' failed: ' . $e->getMessage(), E_USER_WARNING ); // phpcs:ignore WordPress.PHP.DevelopmentFunctions, WordPress.Security.EscapeOutput.OutputNotEscaped -- Logged, not shown. + + return COPY_RETRY; + } finally { + unlock_email( $id ); + } +} + +/** + * Writes WordPress.org's copy of a conversation, while holding its lock; see log_email(). + * + * @param object $request Request payload. + * @param int $id Conversation ID. + * @return string COPY_DONE, or COPY_ALL_THREADS if the copy has nothing of the conversation yet, and the event doesn't + * carry all of its threads. + */ +function write_email( object $request, int $id ): string { + global $wpdb; + + $email = $request->email; + $emails_table = "{$wpdb->base_prefix}helpscout"; + $meta_table = "{$wpdb->base_prefix}helpscout_meta"; + $replaced = get_replaced_ids( $request ); + $helpscout_id = (int) ( $request->helpscout_id ?? 0 ); + + $row = $wpdb->get_row( $wpdb->prepare( 'SELECT * FROM %i WHERE id = %d', $emails_table, $id ) ); + $has_helpscout = ! $row && $helpscout_id && $wpdb->get_var( $wpdb->prepare( 'SELECT 1 FROM %i WHERE id = %d', $emails_table, $helpscout_id ) ); + if ( needs_all_threads( $request, $row, (bool) $has_helpscout ) ) { + return COPY_ALL_THREADS; + } + + $ids = array_merge( array( $id ), ...array_values( $replaced ) ); + $all_meta = $wpdb->get_results( + $wpdb->prepare( + 'SELECT helpscout_id, meta_key, meta_value FROM %i WHERE helpscout_id IN (' . implode( ',', array_fill( 0, count( $ids ), '%d' ) ) . ')', // phpcs:ignore WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare -- The placeholders are added for each ID. + array_merge( array( $meta_table ), $ids ) + ), + ARRAY_A + ); + + // What the subject mentions was added when the copy got it; looking it up again takes queries for every event. + $mailbox = $request->email->mailbox ?? ( $request->mailbox ?? null ); + $subject = (string) ( $email->subject ?? '' ); + $threads = (array) ( $email->threads ?? array() ); + $seen = $row && $subject === (string) $row->subject && get_copy_mailbox_slug( $request ) === (string) $row->mailbox; + $mentioned = $seen && ! $threads ? array() : get_existing_slugs( + get_plugin_or_theme_from_email( + (object) array( + 'conversation' => (object) array( 'subject' => $seen ? '' : $subject ), + 'mailbox' => $mailbox, + 'threads' => $threads, + ) + ) + ); + + $plan = plan_email_write( $id, $replaced, (array) $all_meta, $mentioned ); + + $user_id = (int) ( $row->user_id ?? 0 ); + if ( ! $user_id ) { + $user = get_sender_user( (object) array( 'sender' => $email->sender ?? null ) ); + $user_id = $user ? (int) $user->ID : 0; + } + + $data = get_email_row( $request, $row, $user_id ); + $columns = array_keys( $data ); + + in_transaction( + static function () use ( $wpdb, $emails_table, $meta_table, $id, $data, $columns, $plan ): void { + // Its meta first, which names its helpdesk, and the HelpScout conversations it replaces. + foreach ( $plan['meta'] as list( $meta_key, $meta_value ) ) { + check_write( + $wpdb->insert( + $meta_table, + array( + 'helpscout_id' => $id, + 'meta_key' => $meta_key, // phpcs:ignore WordPress.DB.SlowDBQuery -- Not a query. + 'meta_value' => $meta_value, // phpcs:ignore WordPress.DB.SlowDBQuery -- Not a query. + ) + ) + ); + } + + // phpcs:disable WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholders are added for each column; one statement, so two events for a new conversation at once don't both insert it. + check_write( + $wpdb->query( + $wpdb->prepare( + 'INSERT INTO %i (' . implode( ', ', array_fill( 0, count( $columns ), '%i' ) ) . ') VALUES (' . implode( ', ', array_fill( 0, count( $data ), '%s' ) ) . ') ON DUPLICATE KEY UPDATE ' . implode( ', ', array_fill( 0, count( $columns ) - 1, '%i = VALUES( %i )' ) ), + array_merge( + array( $emails_table ), + $columns, + array_values( $data ), + // Every column but the ID, twice: name, and name again for VALUES(). + ...array_map( + static function ( string $column ): array { + return array( $column, $column ); + }, + array_values( array_diff( $columns, array( 'id' ) ) ) + ) + ) + ) + ) + ); + // phpcs:enable + + delete_emails( $plan['delete'] ); + } + ); + + return COPY_DONE; +} + +/** + * The slugs of plugins and themes that exist, as the copy only keeps those. + * + * @param array $mentioned Slugs, by type: plugins, and themes. + * @return array + */ +function get_existing_slugs( array $mentioned ): array { + global $wpdb; + + $sites = array( + 'plugins' => array( WPORG_PLUGIN_DIRECTORY_BLOGID, 'plugin' ), + 'themes' => array( WPORG_THEME_DIRECTORY_BLOGID, 'repopackage' ), + ); + + foreach ( $mentioned as $type => $slugs ) { + if ( ! isset( $sites[ $type ] ) || ! $slugs ) { + unset( $mentioned[ $type ] ); + continue; + } + + $slugs = array_values( array_unique( array_map( 'strval', $slugs ) ) ); + + /* + * Not get_posts(): its "any" status only means statuses this request registers, and the directories' own, like a + * plugin in review's, aren't registered here. + */ + switch_to_blog( $sites[ $type ][0] ); + // phpcs:disable WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholders are added for each slug. + $mentioned[ $type ] = $wpdb->get_col( + $wpdb->prepare( + "SELECT post_name FROM %i WHERE post_type = %s AND post_status NOT IN ( 'trash', 'auto-draft', 'inherit' ) AND post_name IN (" . implode( ',', array_fill( 0, count( $slugs ), '%s' ) ) . ')', + array_merge( array( $wpdb->posts, $sites[ $type ][1] ), $slugs ) + ) + ); + // phpcs:enable + restore_current_blog(); + } + + return array_filter( $mentioned ); +} + +$request = get_request( basename( __FILE__ ) ); + +// Before the stats, which an event sent again would count twice. +$copied = log_email( $request ); +if ( COPY_RETRY === $copied ) { + status_header( 503 ); + header( 'Retry-After: ' . LOCK_TIMEOUT ); + exit; +} header( 'Content-Type: application/json; charset=utf-8' ); + +// FreeScout sends the event again at once, with all of the conversation's threads; that one counts. +if ( COPY_ALL_THREADS === $copied ) { + echo '{"threads":"all"}'; + exit; +} + +contributor_stats( $request ); + echo '{}'; diff --git a/api.wordpress.org/public_html/dotorg/helpscout/common.php b/api.wordpress.org/public_html/dotorg/helpscout/common.php index 92fc0c12f3..96aba7980a 100644 --- a/api.wordpress.org/public_html/dotorg/helpscout/common.php +++ b/api.wordpress.org/public_html/dotorg/helpscout/common.php @@ -543,9 +543,37 @@ function get_mailbox_name( $mailbox_id_or_request ) { return sanitize_title( $mailbox->name ); } +/** + * Whether a HelpScout conversation moved to FreeScout, whose copy replaced HelpScout's. + * + * FreeScout's copy of a conversation imported from HelpScout records the HelpScout conversation it replaced, under the + * `helpscout_conversation` meta key; see api.wordpress.org/public_html/dotorg/freescout/. Anything still done to the + * conversation in HelpScout, like an agent or a workflow closing it, would otherwise add HelpScout's copy back, next + * to FreeScout's, and the plugin directory could reply to an upload in HelpScout, where no one looks any more. + * + * One lookup by meta key and value, like the plugin directory's lookup of a plugin's emails by slug. + * + * @param int $helpscout_id HelpScout conversation ID. + * @return bool + */ +function is_moved_to_freescout( $helpscout_id ) { + global $wpdb; + + return (bool) $wpdb->get_var( + $wpdb->prepare( + 'SELECT 1 FROM %i WHERE meta_key = %s AND meta_value = %s LIMIT 1', + "{$wpdb->base_prefix}helpscout_meta", + 'helpscout_conversation', + (string) $helpscout_id + ) + ); +} + /** * Keep a cached copy of the received emails in the database for querying. * + * Conversations that moved to FreeScout are left alone, see is_moved_to_freescout(). + * * @param string $event Event name. * @param object $request Helpscout request object / Conversation object. */ @@ -560,6 +588,10 @@ function log_email( $event, $request ) { return; } + if ( is_moved_to_freescout( $request->id ) ) { + return; + } + $row = $wpdb->get_row( $wpdb->prepare( 'SELECT * FROM %i WHERE id = %d', "{$wpdb->base_prefix}helpscout", $request->id ) ); $meta = $row ? $wpdb->get_results( $wpdb->prepare( 'SELECT meta_key, meta_value FROM %i WHERE helpscout_id = %d', "{$wpdb->base_prefix}helpscout_meta", $request->id ), ARRAY_A ) : []; diff --git a/environments/freescout/mock-api/index.php b/environments/freescout/mock-api/index.php index aa60b03451..418b60559e 100644 --- a/environments/freescout/mock-api/index.php +++ b/environments/freescout/mock-api/index.php @@ -88,6 +88,12 @@ function respond( int $status, array|object $data ): void { respond( 200, new \stdClass() ); } +if ( 'replace-copies.php' === $endpoint ) { + $copies = (array) ( $request->copies ?? array() ); + file_put_contents( 'php://stderr', sprintf( "[replace-copies] %d conversations\n", count( $copies ) ) ); + respond( 200, array( 'replaced' => count( $copies ) ) ); +} + /* * What the real endpoints sent for obenland's WordPress.org account, whatever the sender, though searches are for the * sender's email, as they are there. Download links lose their review info, and the forum note and privacy request diff --git a/environments/plugin-directory/bin/database-tables.sql b/environments/plugin-directory/bin/database-tables.sql index 37306e9907..eb2a6e0a0d 100644 --- a/environments/plugin-directory/bin/database-tables.sql +++ b/environments/plugin-directory/bin/database-tables.sql @@ -1,12 +1,28 @@ -- Stub tables for local development. -- These tables exist outside WordPress on production but are needed locally. +-- WordPress.org's copy of helpdesk conversations, which api.wordpress.org's webhooks fill. +CREATE TABLE IF NOT EXISTS `wp_helpscout` ( + `id` bigint(20) unsigned NOT NULL, + `number` int(10) unsigned NOT NULL DEFAULT 0, + `user_id` bigint(20) unsigned NOT NULL DEFAULT 0, + `mailbox` varchar(50) NOT NULL DEFAULT '', + `status` varchar(20) NOT NULL DEFAULT '', + `email` varchar(255) NOT NULL DEFAULT '', + `subject` varchar(255) NOT NULL DEFAULT '', + `preview` text, + `created` varchar(19) NOT NULL DEFAULT '', + `closed` varchar(19) NOT NULL DEFAULT '', + `modified` varchar(19) NOT NULL DEFAULT '', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + CREATE TABLE IF NOT EXISTS `wp_helpscout_meta` ( `helpscout_id` bigint(20) unsigned NOT NULL, `meta_key` varchar(255) NOT NULL DEFAULT '', `meta_value` varchar(255) NOT NULL DEFAULT '', - KEY `helpscout_id` (`helpscout_id`), - KEY `meta_key_value` (`meta_key`(191),`meta_value`(191)) + UNIQUE KEY `helpscout_id` (`helpscout_id`,`meta_key`(191),`meta_value`(191)), + KEY `meta_key` (`meta_key`(191),`meta_value`(191)) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE IF NOT EXISTS `wp_svn_access` ( diff --git a/freescout.wordpress.net/AGENTS.md b/freescout.wordpress.net/AGENTS.md index 478ed2d786..eb6ad95414 100644 --- a/freescout.wordpress.net/AGENTS.md +++ b/freescout.wordpress.net/AGENTS.md @@ -10,11 +10,11 @@ Context for AI coding agents and developers working on the WordPress.org FreeSco - `Modules//` — one directory per module (`module.json`, `Providers/`, `Http/`, `Resources/`, `Database/Migrations/`, `Public/`, `tests/`). - `WPOrgAkismet` — checks new conversations from senders with Akismet (`rest.akismet.com`, its own key), and reports agents' corrections. - - `WPOrgHelpScoutImport` — imports HelpScout mailboxes' conversations, with their tags and custom fields, and their saved replies, from HelpScout's API, page by page on the queue, without the events live email fires. Administrators run it under Manage » HelpScout Import. Remove it once every mailbox has moved. + - `WPOrgHelpScoutImport` — imports HelpScout mailboxes' conversations, with their tags and custom fields, and their saved replies, from HelpScout's API, page by page on the queue, without the events live email fires, and points WordPress.org's copy of conversations at the imported ones as each mailbox switches. Administrators run it under Manage » HelpScout Import. Remove it once every mailbox has moved. - `WPOrgSidebar` — WordPress.org panels in the conversation sidebar, loaded over AJAX from `api.wordpress.org/dotorg/freescout/`. - `WPOrgSite` — tweaks for how WordPress.org runs FreeScout, rather than features; add new ones here instead of starting a module. So far, it adds the WordPress mark, lists only installed modules on the Modules page, refuses to update or delete them there, and lays their cards out in columns. - `WPOrgSSO` — logs agents in through login.wordpress.org's SAML identity provider (wp-saml-idp, in the private dotorg repository), and connects every user to a WordPress.org account. Its SAML library is committed in its `vendor/`: FreeScout doesn't install module dependencies. After changing its `composer.json`, run `composer install --no-dev` in the module and commit `vendor/` (`git add -f`: the root `.gitignore` ignores it). - - `WPOrgWebhooks` — queues conversation events to `api.wordpress.org/dotorg/freescout/webhook.php`, which records contributor stats. + - `WPOrgWebhooks` — queues conversation events to `api.wordpress.org/dotorg/freescout/webhook.php`, which records contributor stats, and keeps the copy of conversations (`wporg_helpscout`) the plugin directory reads. - `tests/` — shared PHPUnit bootstrap and base `TestCase`. Premium (paid) modules must never be committed here. diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Database/Migrations/2026_10_10_000000_add_mailbox_to_wporghelpscoutimport_conversations.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Database/Migrations/2026_10_10_000000_add_mailbox_to_wporghelpscoutimport_conversations.php new file mode 100644 index 0000000000..b5e784fd26 --- /dev/null +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Database/Migrations/2026_10_10_000000_add_mailbox_to_wporghelpscoutimport_conversations.php @@ -0,0 +1,74 @@ +unsignedInteger( 'mailbox_id' )->nullable()->index(); + } + ); + + // Those imported already; those deleted for good since can't be told anymore. + foreach ( \DB::table( 'conversations' )->distinct()->pluck( 'mailbox_id' ) as $mailbox_id ) { + \DB::table( self::TABLE ) + ->whereIn( + 'conversation_id', + static function ( Builder $query ) use ( $mailbox_id ): void { + $query->select( 'id' )->from( 'conversations' )->where( 'mailbox_id', $mailbox_id ); + } + ) + ->update( array( 'mailbox_id' => (int) $mailbox_id ) ); + } + } + + /** + * Reverts the migration. + * + * @return void + */ + public function down(): void { + if ( Schema::hasTable( self::TABLE ) && Schema::hasColumn( self::TABLE, 'mailbox_id' ) ) { + Schema::table( + self::TABLE, + static function ( Blueprint $table ): void { + $table->dropIndex( array( 'mailbox_id' ) ); + $table->dropColumn( 'mailbox_id' ); + } + ); + } + } +} diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Entities/ImportedConversation.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Entities/ImportedConversation.php index b7059ec246..3d249a007b 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Entities/ImportedConversation.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Entities/ImportedConversation.php @@ -42,5 +42,5 @@ final class ImportedConversation extends Model { * * @var array */ - protected $fillable = array( 'helpscout_id', 'helpscout_number', 'conversation_id', 'creator_id', 'assignee_id', 'closer_id', 'tags', 'custom_fields', 'written_tags', 'written_values' ); + protected $fillable = array( 'helpscout_id', 'helpscout_number', 'conversation_id', 'mailbox_id', 'creator_id', 'assignee_id', 'closer_id', 'tags', 'custom_fields', 'written_tags', 'written_values' ); } diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Http/Controllers/ImportController.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Http/Controllers/ImportController.php index dbb63cc444..698dde8892 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Http/Controllers/ImportController.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Http/Controllers/ImportController.php @@ -18,6 +18,7 @@ use Illuminate\View\View; use Modules\WPOrgHelpScoutImport\Entities\Run; use Modules\WPOrgHelpScoutImport\Jobs\ImportPage; +use Modules\WPOrgHelpScoutImport\Services\Copies; use Modules\WPOrgHelpScoutImport\Services\HelpScout; use Modules\WPOrgHelpScoutImport\Services\People; use Modules\WPOrgHelpScoutImport\Services\SavedReplies; @@ -82,10 +83,57 @@ public function index(): View { 'numbering' => $sources ? self::numbering( $helpscout ) : null, // Imports bring saved replies only while there's somewhere to put them. 'replies' => SavedReplies::available(), + 'copies' => self::copies(), ) ); } + /** + * Points WordPress.org's copies of a mailbox's HelpScout conversations at the FreeScout conversations imported from + * them, when the mailbox switches to FreeScout; or carries on where that stopped. + * + * @param Request $request Request. + * @return RedirectResponse + */ + public function point( Request $request ): RedirectResponse { + $mailbox = Mailbox::find( (int) $request->input( 'mailbox_id' ) ); + if ( ! $mailbox ) { + return self::back_with_error( __( 'Choose a FreeScout mailbox.' ) ); + } + + if ( ! Copies::available() ) { + return self::back_with_error( __( 'Pointing WordPress.org at FreeScout needs the WP.org Webhooks module, set up with WPORG_API_SECRET.' ) ); + } + + if ( ! Copies::start( (int) $mailbox->id ) ) { + return self::back_with_error( __( 'WordPress.org is already being pointed at :name.', array( 'name' => $mailbox->name ) ) ); + } + + return redirect() + ->route( 'wporghelpscoutimport.index' ) + ->with( 'flash_success', __( 'Pointing WordPress.org at :name’s imported conversations.', array( 'name' => $mailbox->name ) ) ); + } + + /** + * The mailboxes conversations were imported into, and whether WordPress.org points at them. + * + * @return array[] Mailbox, total imported, and the switch, see Copies::get(). + */ + private static function copies(): array { + $copies = array(); + $ids = Run::query()->distinct()->pluck( 'mailbox_id' )->map( 'intval' )->all(); + + foreach ( Mailbox::query()->whereIn( 'id', $ids ? $ids : array( 0 ) )->orderBy( 'name' )->get() as $mailbox ) { + $copies[] = array( + 'mailbox' => $mailbox, + 'total' => Copies::total( (int) $mailbox->id ), + 'state' => Copies::get( (int) $mailbox->id ), + ); + } + + return $copies; + } + /** * Starts importing a HelpScout mailbox: everything, or what changed since its last finished import into the mailbox. * diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Http/routes.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Http/routes.php index 15993d7898..5255005e21 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Http/routes.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Http/routes.php @@ -22,6 +22,7 @@ static function (): void { Route::post( '/helpscout-import/users/refresh', 'AgentsController@refresh' )->name( 'wporghelpscoutimport.agents.refresh' ); Route::get( '/helpscout-import/users/export', 'AgentsController@export' )->name( 'wporghelpscoutimport.agents.export' ); Route::post( '/helpscout-import/users/connect', 'AgentsController@connect' )->name( 'wporghelpscoutimport.agents.connect' ); + Route::post( '/helpscout-import/copies', 'ImportController@point' )->name( 'wporghelpscoutimport.copies' ); Route::post( '/helpscout-import/{id}/pause', 'ImportController@pause' )->name( 'wporghelpscoutimport.pause' ); Route::post( '/helpscout-import/{id}/resume', 'ImportController@resume' )->name( 'wporghelpscoutimport.resume' ); Route::post( '/helpscout-import/{id}/cancel', 'ImportController@cancel' )->name( 'wporghelpscoutimport.cancel' ); diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ImportPage.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ImportPage.php index ddf9b27140..1ff57174bb 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ImportPage.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ImportPage.php @@ -17,6 +17,7 @@ use Modules\WPOrgHelpScoutImport\Entities\Run; use Modules\WPOrgHelpScoutImport\Exceptions\ApiError; use Modules\WPOrgHelpScoutImport\Exceptions\RateLimited; +use Modules\WPOrgHelpScoutImport\Services\Copies; use Modules\WPOrgHelpScoutImport\Services\HelpScout; use Modules\WPOrgHelpScoutImport\Services\Importer; use Modules\WPOrgHelpScoutImport\Services\People; @@ -260,7 +261,10 @@ static function () use ( $attempts ): bool { if ( $run->is_retry() || ! $page['conversations'] ) { $run->status = Run::STATUS_DONE; $run->finished_at = Carbon::now(); - $this->save( $run ); + if ( $this->save( $run ) ) { + // Once WordPress.org points at the mailbox, it points at what this imported too. + Copies::catch_up( (int) $run->mailbox_id, (string) $run->started_at ); + } return; } diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ReplaceCopies.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ReplaceCopies.php new file mode 100644 index 0000000000..1bf25819b5 --- /dev/null +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ReplaceCopies.php @@ -0,0 +1,120 @@ +mailbox_id = $mailbox_id; + } + + /** + * Sends the next batch, and queues the one after. + * + * Failures don't throw: the worker would retry anything thrown, and the switch would be left running. + * + * @return void + */ + public function handle(): void { + $state = Copies::get( $this->mailbox_id ); + if ( ! $state || Copies::STATUS_RUNNING !== $state['status'] ) { + return; + } + + try { + $batch = Copies::batch( $this->mailbox_id, (int) $state['after_id'], self::BATCH, $state['since'] ?? null ); + + if ( $batch->isEmpty() ) { + // Imports that finished while it ran may want another. + if ( Copies::finish( $this->mailbox_id ) ) { + self::dispatch( $this->mailbox_id ); + } + + return; + } + + Client::from_config( 60 )->post( + self::ENDPOINT, + array( 'copies' => $batch->map( array( Copies::class, 'request' ) )->values()->all() ) + ); + } catch ( \Throwable $e ) { + if ( $this->attempts() < $this->tries ) { + $this->release( self::RETRY_DELAY * $this->attempts() ); + + return; + } + + \Log::error( '[WPOrgHelpScoutImport] Could not point WordPress.org at mailbox ' . $this->mailbox_id . '\'s conversations: ' . $e->getMessage() ); + Copies::fail( $this->mailbox_id, $e->getMessage() ); + + return; + } + + Copies::advance( $this->mailbox_id, (int) $batch->last()->id, $batch->count() ); + + self::dispatch( $this->mailbox_id ); + } +} diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Providers/WPOrgHelpScoutImportServiceProvider.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Providers/WPOrgHelpScoutImportServiceProvider.php index bbefa23b57..cc0669ae9a 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Providers/WPOrgHelpScoutImportServiceProvider.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Providers/WPOrgHelpScoutImportServiceProvider.php @@ -9,7 +9,10 @@ namespace Modules\WPOrgHelpScoutImport\Providers; +use App\Conversation; use Illuminate\Support\ServiceProvider; +use Modules\WPOrgHelpScoutImport\Entities\ImportedConversation; +use Modules\WPOrgHelpScoutImport\Services\Copies; use Modules\WPOrgHelpScoutImport\Services\HelpScout; /** @@ -119,6 +122,72 @@ static function ( $javascripts = array() ) { } ); + // WordPress.org's copy of an imported conversation replaces the copy of its HelpScout conversation. + \Eventy::addFilter( + 'wporgwebhooks.helpscout_id', + static function ( mixed $helpscout_id = 0, mixed $conversation = null, mixed $conversation_id = 0 ): mixed { + // Deleted for good, the conversation is gone, but its ID still names what was imported. + $id = $conversation instanceof Conversation ? (int) $conversation->id : (int) $conversation_id; + + try { + if ( ! $helpscout_id && $id ) { + $helpscout_id = (int) ImportedConversation::query()->where( 'conversation_id', $id )->value( 'helpscout_id' ); + } + } catch ( \Throwable $e ) { + \Log::error( '[WPOrgHelpScoutImport] Could not find the HelpScout conversation of ' . $id . ': ' . $e->getMessage() ); + } + + return $helpscout_id; + }, + 10, + 3 + ); + + // For pointing WordPress.org at what an import brought, once it points at the mailbox. + \Eventy::addAction( + 'wporghelpscoutimport.conversation_imported', + static function ( mixed $conversation = null ): void { + try { + if ( $conversation instanceof Conversation ) { + Copies::touch( $conversation ); + } + } catch ( \Throwable $e ) { + \Log::error( '[WPOrgHelpScoutImport] Could not mark conversation ' . $conversation->id . ' as imported again: ' . $e->getMessage() ); + } + } + ); + + // Imported conversations' events leave WordPress.org's copy alone until it points at their mailbox. + \Eventy::addFilter( + 'wporgwebhooks.copy', + static function ( mixed $copy = true, mixed $conversation = null, mixed $conversation_id = 0, mixed $mailbox_id = 0 ): mixed { + $id = $conversation instanceof Conversation ? (int) $conversation->id : (int) $conversation_id; + $mailbox = $conversation instanceof Conversation ? (int) $conversation->mailbox_id : (int) $mailbox_id; + + try { + return $copy && ( ! $id || Copies::sends( $id, $mailbox ) ); + } catch ( \Throwable $e ) { + \Log::error( '[WPOrgHelpScoutImport] Could not tell whether to copy conversation ' . $id . ': ' . $e->getMessage() ); + + return $copy; + } + }, + 10, + 4 + ); + + // Deleted for good, an imported conversation's mailbox is gone with it; pointing WordPress.org at it needs it. + \Eventy::addAction( + 'conversations.before_delete_forever', + static function ( mixed $conversation_ids = array() ): void { + try { + Copies::remember_mailboxes( (array) $conversation_ids ); + } catch ( \Throwable $e ) { + \Log::error( '[WPOrgHelpScoutImport] Could not note the mailboxes of conversations deleted for good: ' . $e->getMessage() ); + } + } + ); + // Highlights Manage while the pages are open. \Eventy::addFilter( 'menu.selected', diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/README.md b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/README.md index 20ee8b3021..ada65fa976 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/README.md +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/README.md @@ -39,7 +39,7 @@ Copies a HelpScout mailbox's conversations into a FreeScout mailbox, so a team's - Someone with a FreeScout user under another email gets a second one unless they're chosen on the Users page first: FreeScout can't merge users. Choosing someone after an import credits what's imported for them to the user chosen, keeping HelpScout's dates; with the Reports module on, it computes those conversations' metrics again. - What HelpScout did itself, without a user, is credited to "HelpScout Import", a disabled robot user. The Reports module lists it among the users it reports on, for what it's credited with. - **Teams:** conversations assigned to a HelpScout team are assigned to the FreeScout team chosen on the Users page, or else to the one with the same name. FreeScout's teams come from its Teams module: create them there first. Without either, they're imported unassigned, and choosing a team later, or the next import finding one with the same name, assigns them. The Users page flags teams without access to the mailboxes their conversations are in. -- **Nothing reacts to it, except scheduled workflows:** conversations are marked as imported, and written without the events new email fires. Nothing is sent, nobody is notified, workflows aren't triggered, and `WPOrgAkismet` and `WPOrgWebhooks` leave them alone. New senders aren't announced either, like to the API & Webhooks module's webhooks. +- **Nothing reacts to it, except scheduled workflows:** conversations are marked as imported, and written without the events new email fires. Nothing is sent, nobody is notified, workflows aren't triggered, and `WPOrgAkismet` leaves them alone. New senders aren't announced either, like to the API & Webhooks module's webhooks. Each conversation imported or updated fires `wporghelpscoutimport.conversation_imported`, with the conversation and its HelpScout ID. - The Workflows module also runs automatic workflows every hour, over the conversations they apply to: all of them with **Apply to previous conversations**, and with a date condition, like how long it's been waiting since the last reply, every conversation created after the workflow. Imported conversations keep HelpScout's dates, so those workflows act on them as on any other: they can email senders, close conversations, or delete them for good. The import can't stop them. See Setup. - The same goes for conversations still being worked on in HelpScout, while both run. - **HelpScout's rate limit:** the whole HelpScout account shares one limit, and wppluginsteam.org and the reviewers' tools use it too. The import leaves 100 requests a minute to them, and waits for the next minute when only those are left. A conversation too big to read in one minute's share is read again on its next try, waiting for the limit as it goes, but only while no outgoing email is waiting in FreeScout's queue. @@ -56,6 +56,15 @@ Copies a HelpScout mailbox's conversations into a FreeScout mailbox, so a team's - FreeScout's queue worker must be running. - Conversation numbers: under Manage » Settings » General, set **Conversation Number** to **Custom…** (`APP_CUSTOM_NUMBER=true`), or FreeScout shows and searches its internal IDs instead. Before the first import, set the **Next Conversation #** that appears well above HelpScout's numbers, like 2,000,000: HelpScout keeps numbering the mailboxes that haven't moved yet, and conversations whose number FreeScout has given away already keep a number of FreeScout's. The import page warns until both are set, and doesn't start an import before the Next Conversation # is. Imports leave that setting alone. - Teams: install the Teams module, and create the teams, before importing mailboxes with conversations assigned to teams. Tick the mailboxes each team works in on its page: a team only has a folder, and can be assigned again, in those, even when the import assigns it a conversation elsewhere. +- **WordPress.org:** WordPress.org keeps a copy of conversations, which the plugin directory reads. It points at HelpScout until a mailbox switches. To switch one, in this order: + 1. In HelpScout, stop the mailbox's events going to `api.wordpress.org/dotorg/helpscout/webhook.php`, its webhook under **Manage » Apps**: scope it to the mailboxes still in HelpScout, or turn it off once none is left. While it runs for the mailbox, anything done there writes HelpScout's copy of a conversation back, and counts in HelpScout's stats. It leaves alone the HelpScout conversations whose copies FreeScout's replaced, as each copy names the one it replaced, but not the others. + 2. Run the last import of the mailbox. + 3. Together with moving the mailbox's email to FreeScout, press **Point WordPress.org at FreeScout** for it, under **WordPress.org** on the import page. That sends WordPress.org which FreeScout conversation each HelpScout one was imported as, a thousand at a time, on the queue (`api.wordpress.org/dotorg/freescout/replace-copies.php`), without importing anything again. The copies of conversations deleted in FreeScout, for good or not, or marked spam, are deleted instead. If WordPress.org doesn't take a batch, **Carry on** sends the rest; sending a batch again changes nothing. + 4. After the cutover, import the mailbox again, for what HelpScout got in between. Imports that finish after the switch send WordPress.org what they imported, new or again; one that finishes while a send runs is sent after it. + + Until a mailbox switches, `WPOrgWebhooks` sends imported conversations' events for stats, but not the conversations, so WordPress.org goes on pointing at HelpScout; after, it sends them, naming the HelpScout conversation each replaces. + + HelpScout conversations that were never imported, or never replaced, like those HelpScout deleted, keep HelpScout's copies, which have no `helpdesk` meta. Clean those up in WordPress.org's copy before HelpScout closes. - Saved replies, tags, and custom fields: switch on the Saved Replies, Tags, and Custom Fields modules before importing. - Workflows: before importing, give every automatic workflow with **Apply to previous conversations** or a date condition the condition **Imported** is **No**, unless it's meant for HelpScout's history too, and the same to those added later. It goes by the conversation's latest thread, so the workflow applies to a conversation again once agents add a reply, note, or change in FreeScout. Or keep the Workflows module off until after the last import, and add the condition before switching it on. - After the last mailbox has moved, switch the module off, and remove its code in a later deploy. Imported conversations keep HelpScout's numbers. Its tables map HelpScout's conversation IDs to FreeScout's: convert anything that still links to HelpScout before removing it. diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Resources/views/index.blade.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Resources/views/index.blade.php index b275d956be..bb23105dc0 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Resources/views/index.blade.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Resources/views/index.blade.php @@ -213,6 +213,51 @@ @endif + + @if ( count( $copies ) ) +

{{ __('WordPress.org') }}

+

{{ __('WordPress.org keeps a copy of conversations, which the plugin directory links to, and replies to uploads in. It points at HelpScout until a mailbox switches to FreeScout: once HelpScout’s webhook leaves the mailbox alone, and after its last import, point WordPress.org at the conversations imported from HelpScout, together with moving the mailbox’s email. Until then, what happens to imported conversations in FreeScout isn’t sent to WordPress.org; after, it is, and later imports are pointed at as they finish. See the module’s README for every step.') }}

+ + + + + + + + + + + @foreach ( $copies as $copy ) + @php( $state = $copy['state'] ) + + + + + + + @endforeach + +
{{ __('Mailbox') }}{{ __('Imported') }}{{ __('WordPress.org points at') }}
{{ $copy['mailbox']->name }}{{ number_format( $copy['total'] ) }} + @if ( ! $state ) + {{ __('HelpScout') }} + @elseif ( Modules\WPOrgHelpScoutImport\Services\Copies::STATUS_RUNNING === $state['status'] ) + {{ __('FreeScout, :sent of :total so far', [ 'sent' => number_format( $state['sent'] ), 'total' => number_format( $copy['total'] ) ]) }} + @elseif ( Modules\WPOrgHelpScoutImport\Services\Copies::STATUS_FAILED === $state['status'] ) + {{ __('FreeScout, :sent of :total so far', [ 'sent' => number_format( $state['sent'] ), 'total' => number_format( $copy['total'] ) ]) }} +
{{ $state['error'] }} + @else + {{ __('FreeScout, since :date', [ 'date' => App\User::dateFormat( $state['started_at'], 'M j, Y H:i' ) ]) }} + @endif + 		+ @if ( ! $state || Modules\WPOrgHelpScoutImport\Services\Copies::STATUS_FAILED === $state['status'] ) +
+ {{ csrf_field() }} + + +
+ @endif +
+ @endif @endsection diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php new file mode 100644 index 0000000000..c14a8067ef --- /dev/null +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php @@ -0,0 +1,531 @@ +is_configured(); + } + + /** + * Starts sending a mailbox's imported conversations to WordPress.org, or carries on where a failed send stopped. + * + * Under a lock on its state, so a press of the button and an import finishing at once don't both start sending. + * + * @param int $mailbox_id FreeScout mailbox ID. + * @return bool False if it's already running. + */ + public static function start( int $mailbox_id ): bool { + $started = self::change( + $mailbox_id, + static function ( ?array $state ): ?array { + if ( $state && self::STATUS_RUNNING === $state['status'] ) { + return null; + } + + return array( + 'status' => self::STATUS_RUNNING, + 'after_id' => (int) ( $state['after_id'] ?? 0 ), + 'sent' => (int) ( $state['sent'] ?? 0 ), + 'since' => $state['since'] ?? null, + 'started_at' => (string) ( $state['started_at'] ?? Carbon::now()->toDateTimeString() ), + 'finished_at' => null, + 'error' => '', + ) + array_intersect_key( (array) $state, array( 'pending' => true ) ); + } + ); + + if ( $started ) { + ReplaceCopies::dispatch( $mailbox_id ); + } + + return (bool) $started; + } + + /** + * Sends what an import added, or imported again, since a mailbox switched, if it has. Never throws: it runs as an + * import finishes. + * + * Imported again, a conversation's copy may be HelpScout's again: HelpScout's webhook writes it while anyone works on + * it there. While a send runs, the conversations are sent after it, from the start, as they may be behind it; a + * send that failed is started again, from the start, with them, so neither its conversations nor these are left out. + * + * @param int $mailbox_id FreeScout mailbox ID. + * @param string $since When the import started. + * @return void + */ + public static function catch_up( int $mailbox_id, string $since ): void { + try { + if ( ! self::get( $mailbox_id ) || ! self::available() ) { + return; + } + + $restarted = false; + self::change( + $mailbox_id, + static function ( ?array $state ) use ( $since, &$restarted ): ?array { + if ( ! $state ) { + return null; + } + + if ( self::STATUS_RUNNING === $state['status'] ) { + $state['pending'] = array_key_exists( 'pending', $state ) ? self::wider( $state['pending'], $since ) : $since; + + return $state; + } + + // A failed send hadn't sent everything since its own start; done, it had. + $wider = self::STATUS_FAILED === $state['status'] ? self::wider( $state['since'] ?? null, $since ) : $since; + if ( array_key_exists( 'pending', $state ) ) { + $wider = self::wider( $state['pending'], $wider ); + } + + $restarted = true; + + return self::restart( $state, $wider ); + } + ); + + if ( $restarted ) { + ReplaceCopies::dispatch( $mailbox_id ); + } + } catch ( \Throwable $e ) { + \Log::error( '[WPOrgHelpScoutImport] Could not point WordPress.org at mailbox ' . $mailbox_id . '\'s new conversations: ' . $e->getMessage() ); + } + } + + /** + * Notes how far a send got. + * + * @param int $mailbox_id FreeScout mailbox ID. + * @param int $after_id The last imported conversation sent. + * @param int $count How many the batch sent. + * @return void + */ + public static function advance( int $mailbox_id, int $after_id, int $count ): void { + self::change( + $mailbox_id, + static function ( ?array $state ) use ( $after_id, $count ): ?array { + if ( ! $state || self::STATUS_RUNNING !== $state['status'] ) { + return null; + } + + $state['after_id'] = $after_id; + $state['sent'] = (int) $state['sent'] + $count; + + return $state; + } + ); + } + + /** + * Ends a send that has nothing left, or starts the one imports asked for while it ran. + * + * @param int $mailbox_id FreeScout mailbox ID. + * @return bool Whether another send started, whose batches are to be queued. + */ + public static function finish( int $mailbox_id ): bool { + $state = self::change( + $mailbox_id, + static function ( ?array $state ): ?array { + if ( ! $state || self::STATUS_RUNNING !== $state['status'] ) { + return null; + } + + if ( array_key_exists( 'pending', $state ) ) { + return self::restart( $state, $state['pending'] ); + } + + $state['status'] = self::STATUS_DONE; + $state['finished_at'] = Carbon::now()->toDateTimeString(); + + return $state; + } + ); + + return $state && self::STATUS_RUNNING === $state['status']; + } + + /** + * Stops a send WordPress.org didn't take, which can carry on where it stopped. + * + * @param int $mailbox_id FreeScout mailbox ID. + * @param string $error What went wrong. + * @return void + */ + public static function fail( int $mailbox_id, string $error ): void { + self::change( + $mailbox_id, + static function ( ?array $state ) use ( $error ): ?array { + if ( ! $state || self::STATUS_RUNNING !== $state['status'] ) { + return null; + } + + $state['status'] = self::STATUS_FAILED; + $state['error'] = $error; + + return $state; + } + ); + } + + /** + * A send started again from the first imported conversation. + * + * @param array $state The switch. + * @param string|null $since Send the conversations imported, or imported again, since then; null for all. + * @return array + */ + private static function restart( array $state, ?string $since ): array { + unset( $state['pending'] ); + + return array( + 'status' => self::STATUS_RUNNING, + 'after_id' => 0, + 'sent' => 0, + 'since' => $since, + 'finished_at' => null, + 'error' => '', + ) + $state; + } + + /** + * The earlier of two times, which takes in more imported conversations; null, for all of them, takes in any. + * + * @param string|null $one A time; null for all. + * @param string|null $other Another. + * @return string|null + */ + private static function wider( ?string $one, ?string $other ): ?string { + if ( null === $one || null === $other ) { + return null; + } + + return Carbon::parse( $one )->lessThan( Carbon::parse( $other ) ) ? $one : $other; + } + + /** + * Changes a mailbox's switch under a lock on it, so the queue's jobs, imports, and the button don't undo each other. + * + * @param int $mailbox_id FreeScout mailbox ID. + * @param callable $change Takes the switch, null if it hasn't switched, and returns it changed, or null to leave + * it. + * @return array|null The switch as changed; null if it wasn't. + */ + private static function change( int $mailbox_id, callable $change ): ?array { + $name = self::OPTION . $mailbox_id; + + try { + Option::query()->firstOrCreate( array( 'name' => $name ), array( 'value' => '' ) ); + } catch ( QueryException $e ) { + // Created at the same time by another. + unset( $e ); + } + + return \DB::transaction( + static function () use ( $name, $change ): ?array { + $option = Option::query()->where( 'name', $name )->lockForUpdate()->first(); + if ( ! $option ) { + return null; + } + + $state = Option::maybeUnserialize( $option->value ); + $changed = $change( is_array( $state ) ? $state : null ); + if ( null === $changed ) { + return null; + } + + $option->value = Option::maybeSerialize( $changed ); + $option->save(); + + return $changed; + } + ); + } + + /** + * Notes the mailboxes of imported conversations about to be deleted for good, so a switch of their mailbox sends + * WordPress.org their deletion. + * + * @param int[] $conversation_ids Conversation IDs. + * @return void + */ + public static function remember_mailboxes( array $conversation_ids ): void { + foreach ( array_chunk( array_map( 'intval', $conversation_ids ), 500 ) as $ids ) { + $by_mailbox = Conversation::query()->whereIn( 'id', $ids )->get( array( 'id', 'mailbox_id' ) )->groupBy( 'mailbox_id' ); + + foreach ( $by_mailbox as $mailbox_id => $conversations ) { + // Not as imported again, which its updated_at says. + ImportedConversation::query() + ->toBase() + ->whereIn( 'conversation_id', $conversations->pluck( 'id' )->all() ) + ->update( array( 'mailbox_id' => (int) $mailbox_id ) ); + } + } + } + + /** + * Marks a conversation as imported again, for catch_up(). + * + * @param Conversation $conversation Conversation. + * @return void + */ + public static function touch( Conversation $conversation ): void { + ImportedConversation::query()->where( 'conversation_id', $conversation->id )->update( array( 'updated_at' => Carbon::now() ) ); + } + + /** + * How many conversations were imported into a mailbox. + * + * @param int $mailbox_id FreeScout mailbox ID. + * @return int + */ + public static function total( int $mailbox_id ): int { + return self::imported( $mailbox_id )->count(); + } + + /** + * The next conversations imported into a mailbox to send, in the order they were imported. + * + * @param int $mailbox_id FreeScout mailbox ID. + * @param int $after_id ID of the last one sent. + * @param int $limit How many. + * @param string|null $since Only those imported, or imported again, since then. + * @return Collection Imported conversations, with their ID, helpscout_id, conversation_id, and their conversation's + * found_id (null if it was deleted for good), number, state, and status; see request(). + */ + public static function batch( int $mailbox_id, int $after_id, int $limit, ?string $since = null ): Collection { + $table = ( new ImportedConversation() )->getTable(); + + return self::imported( $mailbox_id ) + ->where( $table . '.id', '>', $after_id ) + ->when( + null !== $since, + static function ( Builder $query ) use ( $table, $since ): Builder { + return $query->where( $table . '.updated_at', '>=', $since ); + } + ) + ->orderBy( $table . '.id' ) + ->limit( $limit ) + ->get( + array( + $table . '.id', + $table . '.helpscout_id', + $table . '.conversation_id', + 'conversations.id as found_id', + 'conversations.number', + 'conversations.state', + 'conversations.status', + ) + ); + } + + /** + * What WordPress.org is told about an imported conversation: which FreeScout conversation replaces the copy of its + * HelpScout one, or that the copy goes, as the conversation was deleted, for good or not, or is spam. + * + * One merged into another, which deletes it, has its HelpScout one's copy merged into the other's instead, which + * takes over the plugins and themes it mentions, like a merge after the switch. + * + * @param ImportedConversation $imported An imported conversation, as batch() reads it. + * @return array HelpScout's ID, FreeScout's ID and number, whether to delete the copy, and whether it's merged into + * the conversation that ID names. + */ + public static function request( ImportedConversation $imported ): array { + $request = array( + 'helpscout_id' => (int) $imported->helpscout_id, + 'id' => (int) $imported->conversation_id, + 'number' => (int) $imported->number, + 'delete' => ! $imported->found_id || self::gone( (int) $imported->state, (int) $imported->status ), + 'merged' => false, + ); + + $into = $imported->found_id && Conversation::STATE_DELETED === (int) $imported->state ? self::merged_into( (int) $imported->conversation_id ) : null; + if ( $into ) { + $request = array( + 'id' => (int) $into->id, + 'number' => (int) $into->number, + 'delete' => self::gone( (int) $into->state, (int) $into->status ), + 'merged' => true, + ) + $request; + } + + return $request; + } + + /** + * Whether a conversation's copy goes: it's deleted, or spam. + * + * @param int $state Its state. + * @param int $status Its status. + * @return bool + */ + private static function gone( int $state, int $status ): bool { + return Conversation::STATE_DELETED === $state || Conversation::STATUS_SPAM === $status; + } + + /** + * The conversation one deleted by a merge was merged into, following it on if that one was merged away too. + * + * Core notes the merge on the conversation merged away, in a line item's `META_MERGED_INTO_CONV` meta. + * + * @param int $conversation_id Conversation ID. + * @return Conversation|null Null if it wasn't merged away, or the one it went into was deleted for good. + */ + private static function merged_into( int $conversation_id ): ?Conversation { + $into = null; + + // Few hops: conversations are merged by hand. + for ( $hops = 0; $hops < 10; $hops++ ) { + $merges = Thread::query() + ->where( 'conversation_id', $conversation_id ) + ->where( 'type', Thread::TYPE_LINEITEM ) + ->where( 'action_type', Thread::ACTION_TYPE_MERGED ) + ->orderBy( 'id', 'desc' ) + ->get(); + + $next = 0; + foreach ( $merges as $merge ) { + $next = (int) $merge->getMeta( Thread::META_MERGED_INTO_CONV, 0 ); + if ( $next ) { + break; + } + } + + $next_conversation = $next ? Conversation::find( $next ) : null; + if ( ! $next_conversation ) { + return $into; + } + + $into = $next_conversation; + $conversation_id = $next; + if ( Conversation::STATE_DELETED !== (int) $into->state ) { + return $into; + } + } + + return $into; + } + + /** + * The conversations imported into a mailbox: those in it, and those deleted for good from it. + * + * @param int $mailbox_id FreeScout mailbox ID. + * @return Builder + */ + private static function imported( int $mailbox_id ): Builder { + $table = ( new ImportedConversation() )->getTable(); + + return ImportedConversation::query() + ->leftJoin( 'conversations', 'conversations.id', '=', $table . '.conversation_id' ) + ->where( + static function ( Builder $query ) use ( $table, $mailbox_id ): void { + $query->where( 'conversations.mailbox_id', $mailbox_id ) + ->orWhere( + static function ( Builder $gone ) use ( $table, $mailbox_id ): void { + $gone->whereNull( 'conversations.id' )->where( $table . '.mailbox_id', $mailbox_id ); + } + ); + } + ); + } + + /** + * Whether a conversation's events carry it for WordPress.org's copy: an imported one's only once its mailbox switched. + * + * @param int $conversation_id Conversation ID. + * @param int $mailbox_id Its mailbox's ID. + * @return bool + */ + public static function sends( int $conversation_id, int $mailbox_id ): bool { + return null !== self::get( $mailbox_id ) + || ! ImportedConversation::query()->where( 'conversation_id', $conversation_id )->exists(); + } +} diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Importer.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Importer.php index 739e8ca947..cf21384fdf 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Importer.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Importer.php @@ -271,6 +271,7 @@ public function import( array $source, Mailbox $mailbox ): string { $conversation->save(); $conversation->timestamps = true; self::update_counters( $conversation->mailbox ); + $this->announce( $conversation, $helpscout_id ); return self::UPDATED; } @@ -348,6 +349,7 @@ function () use ( &$conversation, $imported, $source, $mailbox, $sender, $thread $record = array( 'helpscout_number' => (int) ( $source['number'] ?? 0 ), 'conversation_id' => (int) $conversation->id, + 'mailbox_id' => (int) $conversation->mailbox_id, 'creator_id' => 'user' === ( $source['createdBy']['type'] ?? '' ) ? self::person_id( $source['createdBy'] ) : null, 'assignee_id' => self::person_id( $source['assignee'] ?? null ), 'closer_id' => self::person_id( $source['closedByUser'] ?? null ), @@ -392,6 +394,8 @@ function () use ( &$conversation, $imported, $source, $mailbox, $sender, $thread self::update_counters( $other_mailbox ); } + $this->announce( $conversation, $helpscout_id ); + return $imported ? self::UPDATED : self::IMPORTED; } @@ -414,6 +418,24 @@ public static function update_counters( Mailbox $mailbox ): void { } } + /** + * Tells other modules a conversation was imported or updated, as that happens without core's events: pointing + * WordPress.org at its mailbox sends it again. + * + * Never throws: the conversation is imported by now. + * + * @param Conversation $conversation Conversation. + * @param int $helpscout_id HelpScout's ID for it. + * @return void + */ + private function announce( Conversation $conversation, int $helpscout_id ): void { + try { + \Eventy::action( 'wporghelpscoutimport.conversation_imported', $conversation, $helpscout_id ); + } catch ( \Throwable $e ) { + \Log::error( '[WPOrgHelpScoutImport] Could not announce conversation ' . $conversation->id . ': ' . $e->getMessage() ); + } + } + /** * Closes, and so deletes, the temporary files a conversation's attachments and images were downloaded into. * diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php new file mode 100644 index 0000000000..5fe3eccd57 --- /dev/null +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php @@ -0,0 +1,418 @@ +app->register( WPOrgWebhooksServiceProvider::class ); + $this->app->register( WPOrgHelpScoutImportServiceProvider::class ); + $this->app['router']->getRoutes()->refreshNameLookups(); + + Queue::fake(); + \Option::$cache = array(); + $this->mailbox = $this->create_mailbox(); + $this->actingAs( $this->create_user( User::ROLE_ADMIN ) ); + } + + /** + * Forgets the switches the test saved, which the database forgets with it. + * + * @return void + */ + protected function tearDown(): void { + \Option::$cache = array(); + + parent::tearDown(); + } + + /** + * Imported conversations' events leave WordPress.org's copy alone until their mailbox switches; others' don't. + * + * @return void + */ + public function test_imported_conversations_are_copied_once_their_mailbox_switched(): void { + $imported = $this->import( 1000000028 ); + $new = $this->create_conversation( $this->mailbox, $this->create_sender() ); + + $this->assertFalse( \Eventy::filter( 'wporgwebhooks.copy', true, $imported ) ); + $this->assertTrue( \Eventy::filter( 'wporgwebhooks.copy', true, $new ) ); + + $this->post( route( 'wporghelpscoutimport.copies' ), array( 'mailbox_id' => $this->mailbox->id ) ); + + $this->assertTrue( \Eventy::filter( 'wporgwebhooks.copy', true, $imported ) ); + Queue::assertPushed( ReplaceCopies::class, 1 ); + } + + /** + * Each batch is sent, signed, to api.wordpress.org, which the local environment's mock answers, and the next is + * queued until none are left. + * + * @return void + */ + public function test_sends_batches_until_done(): void { + config( + array( + 'wporgwebhooks.api_url' => 'http://mock-api:8080/', + 'wporgwebhooks.secret' => 'local-secret', + ) + ); + $this->import( 1000000028 ); + $this->import( 1000000029 ); + Copies::start( (int) $this->mailbox->id ); + + ( new ReplaceCopies( (int) $this->mailbox->id ) )->handle(); + + $state = Copies::get( (int) $this->mailbox->id ); + $this->assertSame( array( Copies::STATUS_RUNNING, 2 ), array( $state['status'], $state['sent'] ) ); + Queue::assertPushed( ReplaceCopies::class, 2 ); + + ( new ReplaceCopies( (int) $this->mailbox->id ) )->handle(); + + $this->assertSame( Copies::STATUS_DONE, Copies::get( (int) $this->mailbox->id )['status'] ); + } + + /** + * A batch WordPress.org doesn't take, after its last try, stops the switch, which can carry on where it stopped. + * + * @return void + */ + public function test_failed_batches_stop_the_switch_until_carried_on(): void { + $this->import( 1000000028 ); + Copies::start( (int) $this->mailbox->id ); + + $queue_job = $this->createMock( DatabaseJob::class ); + $queue_job->method( 'attempts' )->willReturn( 3 ); + $job = new ReplaceCopies( (int) $this->mailbox->id ); + $job->setJob( $queue_job ); + $job->handle(); + + $state = Copies::get( (int) $this->mailbox->id ); + $this->assertSame( array( Copies::STATUS_FAILED, 0 ), array( $state['status'], $state['after_id'] ) ); + $this->assertNotSame( '', $state['error'] ); + + $this->assertTrue( Copies::start( (int) $this->mailbox->id ) ); + $this->assertFalse( Copies::start( (int) $this->mailbox->id ) ); + } + + /** + * Imports that finish after the switch send what they brought, new or imported again, from the start; before it, + * nothing is sent. + * + * @return void + */ + public function test_catches_up_on_imports_after_the_switch(): void { + $this->import( 1000000027 ); + $again = $this->import( 1000000028 ); + ImportedConversation::query()->update( array( 'updated_at' => '2026-10-01 09:00:00' ) ); + + Copies::catch_up( (int) $this->mailbox->id, '2026-10-02 10:00:00' ); + Queue::assertNotPushed( ReplaceCopies::class ); + + Copies::save( + (int) $this->mailbox->id, + array( + 'status' => Copies::STATUS_DONE, + 'after_id' => 99, + 'sent' => 2, + 'since' => null, + 'started_at' => '2026-10-01 10:00:00', + 'finished_at' => '2026-10-01 10:05:00', + 'error' => '', + ) + ); + $since = (string) now(); + \Eventy::action( 'wporghelpscoutimport.conversation_imported', $again, 1000000028 ); + $new = $this->import( 1000000029 ); + + Copies::catch_up( (int) $this->mailbox->id, $since ); + + Queue::assertPushed( ReplaceCopies::class, 1 ); + $state = Copies::get( (int) $this->mailbox->id ); + $this->assertSame( array( Copies::STATUS_RUNNING, 0, $since ), array( $state['status'], $state['after_id'], $state['since'] ) ); + $this->assertSame( + array( (int) $again->id, (int) $new->id ), + Copies::batch( (int) $this->mailbox->id, 0, 10, $since )->pluck( 'conversation_id' )->map( 'intval' )->all() + ); + } + + /** + * Starting while it's running starts nothing more, as two chains of batches would send everything twice. + * + * @return void + */ + public function test_starts_once(): void { + $this->assertTrue( Copies::start( (int) $this->mailbox->id ) ); + $this->assertFalse( Copies::start( (int) $this->mailbox->id ) ); + + Queue::assertPushed( ReplaceCopies::class, 1 ); + } + + /** + * An import that finishes while a send runs has what it brought sent after it, from the start, as the conversations + * it imported again may be behind it; a second one widens that. + * + * @return void + */ + public function test_catching_up_while_sending_sends_after_it(): void { + $sent = $this->answer(); + $first = $this->import( 1000000027 ); + $again = $this->import( 1000000028 ); + ImportedConversation::query()->update( array( 'updated_at' => '2026-10-01 09:00:00' ) ); + Copies::start( (int) $this->mailbox->id ); + + Copies::catch_up( (int) $this->mailbox->id, '2026-10-03 10:00:00' ); + Copies::catch_up( (int) $this->mailbox->id, '2026-10-02 10:00:00' ); + + Queue::assertPushed( ReplaceCopies::class, 1 ); + $this->assertSame( '2026-10-02 10:00:00', Copies::get( (int) $this->mailbox->id )['pending'] ); + + // Imported again after the send went past it. + $this->run_batches(); + ImportedConversation::query()->where( 'conversation_id', $again->id )->update( array( 'updated_at' => '2026-10-02 11:00:00' ) ); + $this->run_batches(); + $this->run_batches(); + $this->run_batches(); + + $this->assertCount( 2, $sent ); + $this->assertSame( array( (int) $first->id, (int) $again->id ), array_column( $sent[0]['copies'], 'id' ) ); + $this->assertSame( array( (int) $again->id ), array_column( $sent[1]['copies'], 'id' ) ); + + $state = Copies::get( (int) $this->mailbox->id ); + $this->assertSame( array( Copies::STATUS_DONE, '2026-10-02 10:00:00' ), array( $state['status'], $state['since'] ) ); + $this->assertArrayNotHasKey( 'pending', $state ); + } + + /** + * Catching up on a send that failed starts it again from the start, with the wider of the two, so what it hadn't + * sent yet isn't left out. + * + * @return void + */ + public function test_catching_up_after_a_failure_sends_everything_it_had_not(): void { + $this->answer(); + $this->import( 1000000028 ); + foreach ( array( null, '2026-10-01 10:00:00' ) as $failed_since ) { + $this->save_state( Copies::STATUS_FAILED, $failed_since ); + + Copies::catch_up( (int) $this->mailbox->id, '2026-10-02 10:00:00' ); + + $state = Copies::get( (int) $this->mailbox->id ); + $this->assertSame( array( Copies::STATUS_RUNNING, 0, 0, $failed_since ), array( $state['status'], $state['after_id'], $state['sent'], $state['since'] ) ); + } + + Queue::assertPushed( ReplaceCopies::class, 2 ); + } + + /** + * Conversations deleted, for good or not, or spam, have their copies deleted, HelpScout's too; the others are + * pointed at. Those deleted for good from another mailbox are left to its switch. + * + * @return void + */ + public function test_deleted_and_spam_conversations_have_their_copies_deleted(): void { + $sent = $this->answer(); + $kept = $this->import( 1000000025 ); + $trashed = $this->import( 1000000026 ); + $spam = $this->import( 1000000027 ); + $gone = $this->import( 1000000028 ); + $other = $this->create_conversation( $this->create_mailbox( 'Themes' ), $this->create_sender() ); + $this->import( 1000000029, $other ); + + $trashed->state = Conversation::STATE_DELETED; + $trashed->save(); + $spam->status = Conversation::STATUS_SPAM; + $spam->save(); + Conversation::deleteConversationsForever( array( (int) $gone->id, (int) $other->id ) ); + + Copies::start( (int) $this->mailbox->id ); + $this->run_batches(); + + $this->assertSame( + array( + (int) $kept->id => false, + (int) $trashed->id => true, + (int) $spam->id => true, + (int) $gone->id => true, + ), + array_column( $sent[0]['copies'], 'delete', 'id' ) + ); + $this->assertSame( 1000000028, array_column( $sent[0]['copies'], 'helpscout_id', 'id' )[ (int) $gone->id ] ); + } + + /** + * A conversation merged into another before the switch is sent as the one it went into, followed on if that was + * merged away too, so its HelpScout copy is merged into that one's rather than deleted; one merged into a + * conversation deleted since goes. + * + * @return void + */ + public function test_conversations_merged_away_are_sent_as_the_one_they_went_into(): void { + $sent = $this->answer(); + $agent = $this->create_user(); + $kept = $this->import( 1000000025 ); + $merged = $this->import( 1000000026 ); + $twice = $this->import( 1000000027 ); + $trashed = $this->import( 1000000028 ); + $lost = $this->import( 1000000029 ); + + $twice->mergeConversations( $merged, $agent ); + $kept->mergeConversations( $twice, $agent ); + $trashed->mergeConversations( $lost, $agent ); + $trashed->state = Conversation::STATE_DELETED; + $trashed->save(); + + Copies::start( (int) $this->mailbox->id ); + $this->run_batches(); + + $copies = array_column( $sent[0]['copies'], null, 'helpscout_id' ); + foreach ( array( 1000000026, 1000000027 ) as $helpscout_id ) { + $this->assertSame( + array( + 'id' => (int) $kept->id, + 'number' => (int) $kept->number, + 'delete' => false, + 'merged' => true, + ), + array_intersect_key( $copies[ $helpscout_id ], array_flip( array( 'id', 'number', 'delete', 'merged' ) ) ) + ); + } + $this->assertSame( array( (int) $kept->id, false, false ), array( $copies[1000000025]['id'], $copies[1000000025]['delete'], $copies[1000000025]['merged'] ) ); + $this->assertSame( array( (int) $trashed->id, true, true ), array( $copies[1000000029]['id'], $copies[1000000029]['delete'], $copies[1000000029]['merged'] ) ); + } + + /** + * Events of an imported conversation deleted for good before they went out leave the copy alone until its mailbox + * switched, as for any imported conversation. + * + * @return void + */ + public function test_copy_filter_knows_conversations_deleted_since(): void { + $gone = $this->import( 1000000028 ); + $id = (int) $gone->id; + Conversation::deleteConversationsForever( array( $id ) ); + + $this->assertFalse( \Eventy::filter( 'wporgwebhooks.copy', true, null, $id, (int) $this->mailbox->id ) ); + $this->assertSame( 1000000028, \Eventy::filter( 'wporgwebhooks.helpscout_id', 0, null, $id ) ); + + $this->save_state( Copies::STATUS_DONE, null ); + $this->assertTrue( \Eventy::filter( 'wporgwebhooks.copy', true, null, $id, (int) $this->mailbox->id ) ); + } + + /** + * Runs the next batch, as the queue would. + * + * @return void + */ + private function run_batches(): void { + ( new ReplaceCopies( (int) $this->mailbox->id ) )->handle(); + } + + /** + * Saves the mailbox's switch. + * + * @param string $status Status. + * @param string|null $since Only conversations imported since then; null for all. + * @return void + */ + private function save_state( string $status, ?string $since ): void { + Copies::save( + (int) $this->mailbox->id, + array( + 'status' => $status, + 'after_id' => 5, + 'sent' => 5, + 'since' => $since, + 'started_at' => '2026-10-01 10:00:00', + 'finished_at' => null, + 'error' => Copies::STATUS_FAILED === $status ? 'Request to replace-copies.php got a 500 response.' : '', + ) + ); + } + + /** + * Answers WordPress.org's requests, every one taken. + * + * @return \ArrayObject The payloads it was sent, as they arrive. + */ + private function answer(): \ArrayObject { + $sent = new \ArrayObject(); + $this->app->instance( + Client::class, + new Client( + 'https://api.wordpress.test/', + 'test-secret', + 5, + static function ( RequestInterface $request, array $options ) use ( $sent ): PromiseInterface { + $sent[] = json_decode( (string) $request->getBody(), true ); + + return ( new MockHandler( array( new Response( 200, array(), '{"replaced":1}' ) ) ) )( $request, $options ); + } + ) + ); + + return $sent; + } + + /** + * A conversation imported from HelpScout, into the mailbox unless it's given one. + * + * @param int $helpscout_id HelpScout's ID for it. + * @param Conversation|null $conversation The conversation it was imported as; a new one in the mailbox if null. + * @return Conversation + */ + private function import( int $helpscout_id, ?Conversation $conversation = null ): Conversation { + $conversation = $conversation ?? $this->create_conversation( $this->mailbox, $this->create_sender() ); + ImportedConversation::query()->create( + array( + 'helpscout_id' => $helpscout_id, + 'helpscout_number' => (int) $conversation->number, + 'conversation_id' => (int) $conversation->id, + 'mailbox_id' => (int) $conversation->mailbox_id, + ) + ); + + return $conversation; + } +} diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImporterTest.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImporterTest.php index 2394eb36a0..706b8848c0 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImporterTest.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImporterTest.php @@ -21,6 +21,7 @@ use Modules\WPOrgHelpScoutImport\Entities\ImportedConversation; use Modules\WPOrgHelpScoutImport\Entities\ImportedThread; use Modules\WPOrgHelpScoutImport\Entities\Person; +use Modules\WPOrgHelpScoutImport\Providers\WPOrgHelpScoutImportServiceProvider; use Modules\WPOrgHelpScoutImport\Services\HelpScout; use Modules\WPOrgHelpScoutImport\Services\Importer; use Modules\WPOrgHelpScoutImport\Services\People; @@ -394,6 +395,57 @@ public function test_conversation_staying_in_its_mailbox_updates_its_counters(): $this->assertSame( 0, (int) $themes->folders()->where( 'type', Folder::TYPE_UNASSIGNED )->value( 'active_count' ) ); } + /** + * Each conversation imported, or updated, is announced with its HelpScout ID, as that happens without core's events. + * + * @return void + */ + public function test_imported_conversations_are_announced(): void { + $announced = array(); + \Eventy::addAction( + 'wporghelpscoutimport.conversation_imported', + static function ( $conversation, $helpscout_id ) use ( &$announced ): void { + $announced[] = array( (int) $conversation->id, $helpscout_id ); + }, + 20, + 2 + ); + $source = $this->conversation(); + + $this->importer->import( $source, $this->mailbox ); + $source['status'] = 'spam'; + $this->importer->import( $source, $this->mailbox ); + + $id = (int) $this->imported_conversation()->id; + $this->assertSame( array( array( $id, (int) $source['id'] ), array( $id, (int) $source['id'] ) ), $announced ); + } + + /** + * An imported conversation's events name its HelpScout conversation, for WordPress.org's copy. + * + * @return void + */ + public function test_names_the_helpscout_conversation_of_imported_ones(): void { + $this->app->register( WPOrgHelpScoutImportServiceProvider::class ); + $source = $this->conversation(); + $this->importer->import( $source, $this->mailbox ); + + $this->assertSame( (int) $source['id'], \Eventy::filter( 'wporgwebhooks.helpscout_id', 0, $this->imported_conversation() ) ); + } + + /** + * What was imported keeps the mailbox it went into, which pointing WordPress.org at the mailbox reads once the + * conversation is deleted for good. + * + * @return void + */ + public function test_keeps_the_mailbox_of_imported_ones(): void { + $source = $this->conversation(); + $this->importer->import( $source, $this->mailbox ); + + $this->assertSame( (int) $this->mailbox->id, (int) ImportedConversation::query()->where( 'helpscout_id', (int) $source['id'] )->value( 'mailbox_id' ) ); + } + /** * A HelpScout user without a FreeScout user gets one, which can log in while HelpScout lists them. * diff --git a/freescout.wordpress.net/Modules/WPOrgWebhooks/Jobs/SendEvent.php b/freescout.wordpress.net/Modules/WPOrgWebhooks/Jobs/SendEvent.php index d77c39b45d..aab83da05f 100644 --- a/freescout.wordpress.net/Modules/WPOrgWebhooks/Jobs/SendEvent.php +++ b/freescout.wordpress.net/Modules/WPOrgWebhooks/Jobs/SendEvent.php @@ -14,7 +14,9 @@ use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Modules\WPOrgWebhooks\Services\Client; +use Modules\WPOrgWebhooks\Services\EventPayload; use Modules\WPOrgWebhooks\Services\NotDeliveredException; +use Modules\WPOrgWebhooks\Services\PayloadTooLargeException; /** * Posts an event to the WordPress.org webhook endpoint. @@ -59,14 +61,20 @@ public function __construct( array $payload ) { /** * Sends the event, and tries again later if api.wordpress.org didn't get it. * - * Other failures aren't retried, since webhook.php may have counted the event already. They fail the job rather - * than throw: the worker would retry anything thrown until the job runs out of tries. + * When WordPress.org has no copy of the conversation yet, it takes nothing, and asks for all of its threads, which + * the event then carries. Other failures aren't retried, since webhook.php may have counted the event already. They + * fail the job rather than throw: the worker would retry anything thrown until the job runs out of tries. * * @return void */ public function handle(): void { try { - Client::from_config()->post( (string) config( 'wporgwebhooks.endpoint' ), $this->payload ); + $client = Client::from_config(); + $response = self::send( $client, self::current( $this->payload ) ); + + if ( 'all' === ( $response['threads'] ?? '' ) && empty( $this->payload['all_threads'] ) ) { + self::send( $client, self::current( array( 'all_threads' => true ) + $this->payload ) ); + } } catch ( NotDeliveredException $e ) { if ( $this->attempts() < $this->tries ) { $this->release( self::RETRY_DELAY * $this->attempts() ); @@ -79,4 +87,47 @@ public function handle(): void { $this->fail( $e ); } } + + /** + * Posts an event; one too large for api.wordpress.org's web server goes again without its threads, so it still + * counts, and the copy still gets what else it says. + * + * @param Client $client Client. + * @param array $payload Event payload, as it's sent. + * @return array What webhook.php answered. + * + * @throws PayloadTooLargeException If it's too large without its threads too. + */ + private static function send( Client $client, array $payload ): array { + $endpoint = (string) config( 'wporgwebhooks.endpoint' ); + + try { + return $client->post( $endpoint, $payload ); + } catch ( PayloadTooLargeException $e ) { + if ( empty( $payload['email']['threads'] ) ) { + throw $e; + } + + \Log::error( '[WPOrgWebhooks] Sending conversation ' . (int) ( $payload['conversation']['id'] ?? 0 ) . '\'s event without its threads: ' . $e->getMessage() ); + $payload['email']['threads'] = array(); + + return $client->post( $endpoint, $payload ); + } + } + + /** + * The payload with the conversation as it is now; as built, if that can't be read. + * + * @param array $payload Event payload. + * @return array + */ + private static function current( array $payload ): array { + try { + return EventPayload::refresh( $payload ); + } catch ( \Throwable $e ) { + \Log::error( '[WPOrgWebhooks] Could not refresh an event: ' . $e->getMessage() ); + + return $payload; + } + } } diff --git a/freescout.wordpress.net/Modules/WPOrgWebhooks/Providers/WPOrgWebhooksServiceProvider.php b/freescout.wordpress.net/Modules/WPOrgWebhooks/Providers/WPOrgWebhooksServiceProvider.php index 6225603a6a..3f6e3d3e72 100644 --- a/freescout.wordpress.net/Modules/WPOrgWebhooks/Providers/WPOrgWebhooksServiceProvider.php +++ b/freescout.wordpress.net/Modules/WPOrgWebhooks/Providers/WPOrgWebhooksServiceProvider.php @@ -91,7 +91,7 @@ static function ( $conversation = null, $thread = null ) use ( $event ): void { self::forward( $event, static function () use ( $conversation, $thread ): ?array { - return self::imported( $thread ) || $conversation->isSpam() ? null : array( $conversation, null ); + return self::imported( $thread ) || $conversation->isSpam() ? null : array( $conversation, null, self::thread( $thread ) ); } ); }, @@ -106,7 +106,7 @@ static function ( $conversation = null, $thread = null ): void { self::forward( 'conversation.created_by_user', static function () use ( $conversation, $thread ): ?array { - return ! self::imported( $thread ) && self::sent( $thread ) ? array( $conversation, $thread->created_by_user ) : null; + return ! self::imported( $thread ) && self::sent( $thread ) ? array( $conversation, $thread->created_by_user, self::thread( $thread ) ) : null; } ); }, @@ -120,7 +120,7 @@ static function ( $conversation = null, $thread = null ): void { self::forward( 'conversation.user_replied', static function () use ( $conversation, $thread ): ?array { - return ! self::imported( $thread ) && self::sent( $thread ) ? array( $conversation, $thread->created_by_user ) : null; + return ! self::imported( $thread ) && self::sent( $thread ) ? array( $conversation, $thread->created_by_user, self::thread( $thread ) ) : null; } ); }, @@ -128,7 +128,39 @@ static function () use ( $conversation, $thread ): ?array { 2 ); - foreach ( array( 'conversation.user_changed', 'conversation.status_changed', 'conversation.moved' ) as $event ) { + // Out of spam, the conversation is new to WordPress.org's copy again, which took it out. + \Eventy::addAction( + 'conversation.status_changed', + static function ( $conversation = null, $user = null, $changed_on_reply = false, $prev_status = null ): void { + self::forward( + 'conversation.status_changed', + static function () use ( $conversation, $user, $prev_status ): array { + $unspammed = Conversation::STATUS_SPAM === (int) $prev_status && $conversation instanceof Conversation && ! $conversation->isSpam(); + + return array( $conversation, $user, $unspammed ? array( 'unspammed' => true ) : array() ); + } + ); + }, + 20, + 4 + ); + + // For the plugins and themes it mentions; WordPress.org doesn't credit it. + \Eventy::addAction( + 'conversation.note_added', + static function ( $conversation = null, $thread = null ): void { + self::forward( + 'conversation.note_added', + static function () use ( $conversation, $thread ): ?array { + return self::imported( $thread ) ? null : array( $conversation, $thread->created_by_user ?? null, self::thread( $thread ) ); + } + ); + }, + 20, + 2 + ); + + foreach ( array( 'conversation.user_changed', 'conversation.moved' ) as $event ) { \Eventy::addAction( $event, static function ( $conversation = null, $user = null ) use ( $event ): void { @@ -149,8 +181,26 @@ static function () use ( $conversation, $user ): array { static function ( $conversation = null, $second = null, $user = null ): void { self::forward( 'conversation.merged', - static function () use ( $conversation, $user ): array { - return array( $conversation, $user ); + static function () use ( $conversation, $second, $user ): array { + // The other conversation's threads are in this one now; WordPress.org drops its copy of it. + return array( $conversation, $user, array( 'merged_id' => $second instanceof Conversation ? (int) $second->id : 0 ) ); + } + ); + }, + 20, + 3 + ); + + // Restoring from Deleted: WordPress.org's copy took the conversation out when it was deleted. + \Eventy::addAction( + 'conversation.state_changed', + static function ( $conversation = null, $user = null, $previous_state = null ): void { + self::forward( + 'conversation.restored', + static function () use ( $conversation, $user, $previous_state ): ?array { + $restored = Conversation::STATE_DELETED === (int) $previous_state && Conversation::STATE_PUBLISHED === (int) $conversation->state; + + return $restored ? array( $conversation, $user ) : null; } ); }, @@ -197,6 +247,16 @@ private static function is_robot( User $user ): bool { return defined( User::class . '::TYPE_ROBOT' ) && User::TYPE_ROBOT === (int) $user->type; } + /** + * The reply an event is about, so WordPress.org reads the threads up to it, however late the event is sent. + * + * @param Thread|null $thread Reply. + * @return array Empty without one. + */ + private static function thread( ?Thread $thread ): array { + return $thread ? array( 'thread_id' => (int) $thread->id ) : array(); + } + /** * Whether a reply was sent and hasn't been counted yet. * @@ -235,7 +295,8 @@ private static function merged_away( Conversation $conversation ): bool { * Never throws: a failure here must not interrupt the core action that fired the event. * * @param string $event Event name. - * @param callable $resolve Returns the conversation and the agent who caused the event (or null), or null to skip it. + * @param callable $resolve Returns the conversation, the agent who caused the event (or null), and optionally more about + * the event for the payload; or null to skip it. * @return void */ private static function forward( string $event, callable $resolve ): void { @@ -250,13 +311,14 @@ private static function forward( string $event, callable $resolve ): void { } list( $conversation, $agent ) = $args; + $extra = (array) ( $args[2] ?? array() ); // Automations have no WordPress.org account to credit. if ( ! $agent instanceof User || self::is_robot( $agent ) ) { $agent = null; } - SendEvent::dispatch( EventPayload::build( $event, $conversation, $agent ) ); + SendEvent::dispatch( EventPayload::build( $event, $conversation, $agent, $extra ) ); } catch ( \Throwable $e ) { \Log::error( '[WPOrgWebhooks] Could not queue ' . $event . ': ' . $e->getMessage() ); } diff --git a/freescout.wordpress.net/Modules/WPOrgWebhooks/README.md b/freescout.wordpress.net/Modules/WPOrgWebhooks/README.md index a433fbfd96..73c85690e6 100644 --- a/freescout.wordpress.net/Modules/WPOrgWebhooks/README.md +++ b/freescout.wordpress.net/Modules/WPOrgWebhooks/README.md @@ -1,6 +1,6 @@ # WP.org Webhooks -Sends conversation events to WordPress.org, which records them as contributor stats for the agents who handled them. +Sends conversation events to WordPress.org, which records them as contributor stats for the agents who handled them, and keeps a copy of the conversations that the plugin directory reads. ## What it sends @@ -9,16 +9,25 @@ An event, the conversation, its mailbox, and the agent with the WordPress.org ac - a conversation comes in, or an agent starts one; - the sender or an agent replies (an undone reply isn't counted); - a conversation is assigned, changes status, or moves to another mailbox; -- conversations are merged, or one is moved to Deleted (a conversation merged into another counts as the merge). +- conversations are merged, or one is moved to Deleted (a conversation merged into another counts as the merge), or restored from it. +- an agent adds a note, which WordPress.org doesn't credit to them, but reads for the plugins and themes it mentions. + +Events of conversations `WPOrgHelpScoutImport` imported count for stats, but don't carry the conversation for WordPress.org's copy until WordPress.org is pointed at their mailbox (the `wporgwebhooks.copy` filter); see its README. Not sent: -- imported conversations, which the service they came from counted already (what agents do with them in FreeScout is sent); +- imported threads as they're written, which the service they came from counted already (what agents do with them in FreeScout is sent); - spam from senders (agents marking spam is sent). Automations, like Workflows, have no WordPress.org account, so their events are sent without an agent. -Nothing from the emails themselves is sent. Events go out from FreeScout's queue, and are retried if WordPress.org can't be reached. +Each event also carries the conversation as WordPress.org keeps it, read when the event is sent, so a late or retried one says what's current: its mailbox, state, number, subject, status, preview, and dates, and the sender. The event's own mailbox, which stats count, stays the one it happened in. + +For the plugins and themes they mention, a reply also carries the threads from the previous reply up to it (itself, and the notes before it), a note itself, and a conversation new to the copy, restored, taken out of spam, or moved to another mailbox, its newest 50 threads; other events carry none, as the copy keeps what earlier ones mentioned. When WordPress.org has no copy of a conversation yet, it takes nothing from an event without all of its threads, and asks for them: the event goes again at once with them, and only that one counts. Threads go as text, with their links' addresses, up to 256 KB an event, cutting the oldest short; an event api.wordpress.org's web server still refuses for its size goes again without them. + +A merge carries the ID of the conversation merged away, and every event of an imported conversation the HelpScout ID whose copy it replaces, as does a merge for the conversation merged away (from the `wporgwebhooks.helpscout_id` filter, which `WPOrgHelpScoutImport` answers, also for conversations deleted for good since). The copy names the HelpScout conversations it replaced, which keeps HelpScout's webhook from writing them back; later events don't replace them again. Whether an event carries the conversation at all is up to the `wporgwebhooks.copy` filter, which `WPOrgHelpScoutImport` answers for imported conversations. That copy is the `wporg_helpscout` table HelpScout's webhook filled, which the plugin directory lists a plugin's emails from, and finds review conversations in; deleted conversations, and spam, are taken out of it. + +Events go out from FreeScout's queue, and are retried if WordPress.org can't be reached. ## Setup diff --git a/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/Client.php b/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/Client.php index 8b29274f3b..74cad7fd15 100644 --- a/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/Client.php +++ b/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/Client.php @@ -32,6 +32,13 @@ final class Client { */ private const NOT_DELIVERED_STATUSES = array( 502, 503 ); + /** + * Status the web server sends for a request bigger than it takes, which it answers without passing it on. + * + * @var int + */ + private const PAYLOAD_TOO_LARGE = 413; + /** * Base URL of the endpoints, with a trailing slash. * @@ -53,26 +60,39 @@ final class Client { */ private $timeout; + /** + * Guzzle handler, for tests to answer requests. + * + * @var callable|null + */ + private $handler; + /** * Constructor. * - * @param string $base_url Base URL of the endpoints. - * @param string $secret Shared signing secret. - * @param int $timeout Request timeout in seconds. + * @param string $base_url Base URL of the endpoints. + * @param string $secret Shared signing secret. + * @param int $timeout Request timeout in seconds. + * @param callable|null $handler Guzzle handler; the default sends real requests. */ - public function __construct( string $base_url, string $secret, int $timeout = 10 ) { + public function __construct( string $base_url, string $secret, int $timeout = 10, ?callable $handler = null ) { $this->base_url = rtrim( $base_url, '/' ) . '/'; $this->secret = $secret; $this->timeout = $timeout; + $this->handler = $handler; } /** - * Creates a client from the module configuration. + * Creates a client from the module configuration, or returns the one tests bound. * * @param int $timeout Request timeout in seconds. * @return self */ public static function from_config( int $timeout = 10 ): self { + if ( app()->bound( self::class ) ) { + return app( self::class ); + } + return new self( (string) config( 'wporgwebhooks.api_url' ), (string) config( 'wporgwebhooks.secret' ), @@ -118,19 +138,19 @@ public static function encode( array $payload, string $endpoint ): string { /** * Posts a signed payload to an endpoint. * - * Only a 2xx response counts as delivered, whatever its body. Redirects aren't followed, since they'd turn the - * request into a GET. + * Only a 2xx response counts as delivered. Redirects aren't followed, since they'd turn the request into a GET. * * @param string $endpoint Endpoint path relative to the base URL. * @param array $payload Request payload. - * @return void + * @return array The response's JSON object; empty if it has none. * - * @throws NotDeliveredException If api.wordpress.org certainly didn't get the request: it couldn't be reached, or - * its web server answered for it. - * @throws RuntimeException If the client is not configured, or the request failed in a way that may have - * reached webhook.php, like timing out after it was sent. + * @throws NotDeliveredException If api.wordpress.org certainly didn't get the request: it couldn't be reached, + * or its web server answered for it. + * @throws PayloadTooLargeException If its web server refused the request for its size, without passing it on. + * @throws RuntimeException If the client is not configured, or the request failed in a way that may have + * reached webhook.php, like timing out after it was sent. */ - public function post( string $endpoint, array $payload ): void { + public function post( string $endpoint, array $payload ): array { if ( ! $this->is_configured() ) { throw new RuntimeException( 'WPORG_API_SECRET is not configured.' ); } @@ -139,15 +159,16 @@ public function post( string $endpoint, array $payload ): void { try { // Whatever APP_CURL_SSL_VERIFYPEER says. - $http = new \GuzzleHttp\Client( - \Helper::setGuzzleDefaultOptions( - array( - 'timeout' => $this->timeout, - 'allow_redirects' => false, - 'verify' => true, - ) - ) + $options = array( + 'timeout' => $this->timeout, + 'allow_redirects' => false, + 'verify' => true, ); + if ( $this->handler ) { + $options['handler'] = \GuzzleHttp\HandlerStack::create( $this->handler ); + } + + $http = new \GuzzleHttp\Client( \Helper::setGuzzleDefaultOptions( $options ) ); $response = $http->post( $this->base_url . ltrim( $endpoint, '/' ), array( @@ -172,8 +193,16 @@ public function post( string $endpoint, array $payload ): void { throw new NotDeliveredException( 'Request to ' . $endpoint . ' got a ' . $status . ' response.' ); } + if ( self::PAYLOAD_TOO_LARGE === $status ) { + throw new PayloadTooLargeException( 'Request to ' . $endpoint . ' was too large: ' . strlen( $body ) . ' bytes.' ); + } + if ( $status < 200 || $status > 299 ) { throw new RuntimeException( 'Request to ' . $endpoint . ' got a ' . $status . ' response.' ); } + + $data = json_decode( trim( (string) $response->getBody() ), true ); + + return is_array( $data ) ? $data : array(); } } diff --git a/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/EventPayload.php b/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/EventPayload.php index d05dd701d5..431e72719b 100644 --- a/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/EventPayload.php +++ b/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/EventPayload.php @@ -10,36 +10,110 @@ namespace Modules\WPOrgWebhooks\Services; use App\Conversation; +use App\Thread; use App\User; +use Carbon\Carbon; use Modules\WPOrgSSO\Entities\Account; /** * Builds the webhook payload for a conversation event. * - * Read by api.wordpress.org/dotorg/freescout/webhook.php; keep the two in sync. + * Read by api.wordpress.org/dotorg/freescout/webhook.php; keep the two in sync. Besides the event, which contributor + * stats count, it carries the conversation as WordPress.org keeps a copy of it, which the plugin directory reads. */ final class EventPayload { + /** + * Most threads sent, newest first; WordPress.org reads them for the plugins and themes they mention. + * + * @var int + */ + private const MAX_THREADS = 50; + + /** + * Most bytes of thread text sent with an event; the oldest threads are cut short, or left out, past it. + * + * WordPress.org only looks for the plugins and themes threads mention, so they're sent as text, which keeps most + * events far below it; this bounds the rest, like a long conversation moved to another mailbox. + * + * @var int + */ + private const MAX_TEXT_BYTES = 256 * 1024; + + /** + * Events after which WordPress.org reads all of the conversation's threads: it's new to the copy, or in another + * mailbox, which changes what it mentions. So do status changes out of spam, which the copy doesn't keep, and + * events WordPress.org asked to have all of them for, as it had no copy yet. + * + * @var string[] + */ + private const ALL_THREADS_EVENTS = array( + 'conversation.created_by_customer', + 'conversation.created_by_user', + 'conversation.restored', + 'conversation.moved', + ); + + /** + * Events after which WordPress.org reads the threads since the previous reply: the reply, and notes before it. + * + * Others change nothing the threads mention, so none are sent; the copy keeps what earlier ones mentioned. + * + * @var string[] + */ + private const NEW_THREADS_EVENTS = array( + 'conversation.customer_replied', + 'conversation.user_replied', + ); + + /** + * Events after which WordPress.org reads only the thread they're about: a note. + * + * @var string[] + */ + private const OWN_THREAD_EVENTS = array( 'conversation.note_added' ); + + /** + * Statuses, as WordPress.org's copy names them. + * + * @var string[] + */ + private const STATUSES = array( + Conversation::STATUS_ACTIVE => 'active', + Conversation::STATUS_PENDING => 'pending', + Conversation::STATUS_CLOSED => 'closed', + Conversation::STATUS_SPAM => 'spam', + ); + + /** + * Thread types, by name. + * + * @var string[] + */ + private const THREAD_TYPES = array( + Thread::TYPE_CUSTOMER => 'customer', + Thread::TYPE_MESSAGE => 'message', + Thread::TYPE_NOTE => 'note', + ); + /** * Builds the payload. * + * The conversation, as WordPress.org keeps a copy of it, is added when the event is sent; see refresh(). + * * @param string $event Event name: the FreeScout hook that fired. * @param Conversation $conversation Conversation the event happened on. * @param User|null $agent Agent who caused the event, if any. + * @param array $extra More about the event, like the conversation merged into this one. * @return array */ - public static function build( string $event, Conversation $conversation, ?User $agent ): array { - $mailbox = $conversation->mailbox; - - return array( + public static function build( string $event, Conversation $conversation, ?User $agent, array $extra = array() ): array { + return $extra + array( 'event' => $event, 'conversation' => array( 'id' => (int) $conversation->id, ), - 'mailbox' => array( - 'id' => (int) $conversation->mailbox_id, - 'name' => $mailbox ? (string) $mailbox->name : '', - ), + 'mailbox' => self::mailbox( $conversation ), 'agent' => $agent ? array( 'id' => (int) $agent->id, 'email' => (string) $agent->email, @@ -48,6 +122,236 @@ public static function build( string $event, Conversation $conversation, ?User $ ); } + /** + * The payload with the conversation as it is now, for sending it. + * + * Events can be sent late, and out of order, when one is retried: WordPress.org's copy takes whatever the latest + * says, so each says what's current, and a conversation deleted since says so. The event's own mailbox stays, for + * the stats; the copy takes the conversation's mailbox now. + * + * @param array $payload The payload, as built, and `all_threads` when WordPress.org asked for all of them. + * @return array + */ + public static function refresh( array $payload ): array { + $id = (int) ( $payload['conversation']['id'] ?? 0 ); + $conversation = Conversation::find( $id ); + $mailbox_id = $conversation ? (int) $conversation->mailbox_id : (int) ( $payload['mailbox']['id'] ?? 0 ); + + /** + * Filters whether an event carries the conversation for WordPress.org's copy, like an imported one's doesn't + * while WordPress.org still points at its HelpScout conversation; it still counts for stats. + * + * @param bool $copy Whether it does. + * @param Conversation|null $conversation Conversation; null if it was deleted for good since the event. + * @param int $conversation_id Its ID. + * @param int $mailbox_id Its mailbox's ID; the event's, if it was deleted for good. + */ + if ( ! \Eventy::filter( 'wporgwebhooks.copy', true, $conversation, $id, $mailbox_id ) ) { + unset( $payload['email'] ); + + return $payload; + } + + $helpscout_id = self::helpscout_id( $conversation, $id ); + if ( $helpscout_id && empty( $payload['helpscout_id'] ) ) { + $payload['helpscout_id'] = $helpscout_id; + } + + // A conversation merged into this one may have been imported, and its HelpScout conversation copied too. + $merged_id = (int) ( $payload['merged_id'] ?? 0 ); + if ( $merged_id && empty( $payload['merged_helpscout_id'] ) ) { + $merged_helpscout_id = self::helpscout_id( Conversation::find( $merged_id ), $merged_id ); + if ( $merged_helpscout_id ) { + $payload['merged_helpscout_id'] = $merged_helpscout_id; + } + } + + $payload['email'] = $conversation ? self::email( $conversation, $payload ) : array( 'state' => 'deleted' ); + + return $payload; + } + + /** + * The ID of the HelpScout conversation a conversation was imported from, whose copy its own replaces. + * + * @param Conversation|null $conversation Conversation; null if it was deleted for good. + * @param int $conversation_id Its ID. + * @return int 0 if it wasn't imported. + */ + private static function helpscout_id( ?Conversation $conversation, int $conversation_id ): int { + /** + * Filters the ID of the HelpScout conversation a conversation was imported from, whose copy its own replaces. + * + * @param int $helpscout_id HelpScout's ID; 0 if it wasn't imported. + * @param Conversation|null $conversation Conversation; null if it was deleted for good. + * @param int $conversation_id Its ID. + */ + return (int) \Eventy::filter( 'wporgwebhooks.helpscout_id', 0, $conversation, $conversation_id ); + } + + /** + * The conversation's mailbox. + * + * @param Conversation $conversation Conversation. + * @return array + */ + private static function mailbox( Conversation $conversation ): array { + $mailbox = $conversation->mailbox; + + return array( + 'id' => (int) $conversation->mailbox_id, + 'name' => $mailbox ? (string) $mailbox->name : '', + ); + } + + /** + * The conversation, as WordPress.org keeps a copy of it. + * + * @param Conversation $conversation Conversation. + * @param array $payload The event's payload. + * @return array + */ + private static function email( Conversation $conversation, array $payload ): array { + $customer = $conversation->customer; + $all = self::reads_all( $payload ); + + return array( + 'mailbox' => self::mailbox( $conversation ), + 'state' => Conversation::STATE_DELETED === (int) $conversation->state ? 'deleted' : 'published', + 'number' => (int) $conversation->number, + 'subject' => (string) $conversation->subject, + 'status' => self::STATUSES[ (int) $conversation->status ] ?? 'active', + 'preview' => (string) $conversation->preview, + 'created_at' => self::date( $conversation->created_at ), + 'closed_at' => self::date( $conversation->closed_at ), + 'user_updated_at' => self::date( $conversation->user_updated_at ), + 'sender' => array( + 'email' => (string) $conversation->customer_email, + 'emails' => $customer ? $customer->emails->pluck( 'email' )->values()->all() : array(), + 'first_name' => $customer ? (string) $customer->first_name : '', + 'last_name' => $customer ? (string) $customer->last_name : '', + ), + 'all_threads' => $all, + 'threads' => self::threads( $conversation, $payload, $all ), + ); + } + + /** + * Whether WordPress.org reads all of a conversation's threads after an event. + * + * @param array $payload The event's payload. + * @return bool + */ + private static function reads_all( array $payload ): bool { + return ! empty( $payload['all_threads'] ) + || ! empty( $payload['unspammed'] ) + || in_array( (string) ( $payload['event'] ?? '' ), self::ALL_THREADS_EVENTS, true ); + } + + /** + * The threads WordPress.org reads after an event, newest first, as text. + * + * @param Conversation $conversation Conversation. + * @param array $payload The event's payload: its name, and the reply it's about. + * @param bool $all Whether WordPress.org reads all of them; see reads_all(). + * @return array[] Type, and body, as text; see text(). + */ + private static function threads( Conversation $conversation, array $payload, bool $all ): array { + $event = (string) ( $payload['event'] ?? '' ); + $thread_id = (int) ( $payload['thread_id'] ?? 0 ); + $own = ! $all && in_array( $event, self::OWN_THREAD_EVENTS, true ); + if ( ! $all && ! $own && ! in_array( $event, self::NEW_THREADS_EVENTS, true ) ) { + return array(); + } + + $query = $conversation->threads() + ->where( 'state', Thread::STATE_PUBLISHED ) + ->whereIn( 'type', array_keys( self::THREAD_TYPES ) ); + + // A reply's threads end with it, however many came after it before the event was sent. + if ( $own ) { + $query->where( 'id', $thread_id ); + } elseif ( ! $all && $thread_id ) { + $query->where( 'id', '<=', $thread_id )->orderBy( 'id', 'desc' ); + } else { + $query->orderBy( 'created_at', 'desc' )->orderBy( 'id', 'desc' ); + } + + $sent = array(); + $replies = 0; + $budget = self::MAX_TEXT_BYTES; + foreach ( $query->limit( self::MAX_THREADS )->get( array( 'type', 'body' ) ) as $thread ) { + $is_reply = Thread::TYPE_NOTE !== (int) $thread->type; + if ( ( ! $all && $is_reply && ++$replies > 1 ) || $budget <= 0 ) { + break; + } + + $text = mb_strcut( self::text( (string) $thread->body ), 0, $budget, 'UTF-8' ); + $budget -= strlen( $text ); + $sent[] = array( + 'type' => self::THREAD_TYPES[ (int) $thread->type ], + 'body' => $text, + ); + } + + return $sent; + } + + /** + * A thread's HTML as text, for finding the plugins and themes it mentions: a line for each block, and the links' + * addresses on lines of their own after it. + * + * Angle brackets go too, so WordPress.org stripping tags from it can't take text along. + * + * @param string $html The thread's body. + * @return string + */ + public static function text( string $html ): string { + $html = mb_scrub( $html, 'UTF-8' ); + $html = (string) preg_replace( '#<(script|style)\b[^>]*>.*?#is', '', $html ); + + preg_match_all( '#\bhref\s*=\s*(["\'])(.*?)\1#is', $html, $links ); + + $text = (string) preg_replace( '#]*>|#i', "\n", $html ); + $text = self::decode( strip_tags( $text ) ); + + $urls = array(); + foreach ( $links[2] as $url ) { + $url = trim( self::decode( $url ) ); + if ( '' !== $url && ! str_contains( $text, $url ) ) { + $urls[ $url ] = true; + } + } + + $text = (string) preg_replace( '/[ \t\x{00A0}]+/u', ' ', $text . "\n" . implode( "\n", array_keys( $urls ) ) ); + + return trim( (string) preg_replace( '/ *\n\s*/', "\n", $text ) ); + } + + /** + * Decodes HTML entities, and takes out angle brackets; see text(). + * + * @param string $text Text with entities. + * @return string + */ + private static function decode( string $text ): string { + return str_replace( array( '<', '>' ), ' ', html_entity_decode( $text, ENT_QUOTES | ENT_HTML5, 'UTF-8' ) ); + } + + /** + * A stored time, in UTC, ISO 8601. + * + * @param mixed $date Time in the app's timezone, as a Carbon instance or a string; empty for none. + * @return string|null + */ + private static function date( $date ): ?string { + if ( ! $date ) { + return null; + } + + return Carbon::parse( (string) $date, config( 'app.timezone' ) )->setTimezone( 'UTC' )->format( 'Y-m-d\TH:i:s\Z' ); + } + /** * The WordPress.org account WPOrgSSO connected an agent to, which contributor stats credit. * diff --git a/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/PayloadTooLargeException.php b/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/PayloadTooLargeException.php new file mode 100644 index 0000000000..9401a6bcee --- /dev/null +++ b/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/PayloadTooLargeException.php @@ -0,0 +1,19 @@ +assert_queued( 'conversation.merged', (string) $this->agent->email ); Queue::assertPushed( SendEvent::class, 1 ); + + // WordPress.org drops its copy of the merged conversation. + $this->assertSame( (int) $second->id, $this->payload()['merged_id'] ); + } + + /** + * Restoring a conversation from Deleted is sent, so WordPress.org copies it again; other state changes aren't. + * + * @return void + */ + public function test_restoring_a_conversation_is_forwarded(): void { + \Eventy::action( 'conversation.state_changed', $this->conversation, $this->agent, Conversation::STATE_DRAFT ); + Queue::assertNotPushed( SendEvent::class ); + + \Eventy::action( 'conversation.state_changed', $this->conversation, $this->agent, Conversation::STATE_DELETED ); + $this->assert_queued( 'conversation.restored', (string) $this->agent->email ); + } + + /** + * Events carry the conversation as WordPress.org keeps a copy of it: its details, and sender; one that doesn't add a + * thread, no threads. + * + * @return void + */ + public function test_events_carry_the_conversation(): void { + $this->conversation->update( + array( + 'subject' => '[WordPress Plugin Directory] Review in Progress: My Plugin', + 'status' => Conversation::STATUS_CLOSED, + ) + ); + $this->create_thread( $this->conversation, Thread::TYPE_CUSTOMER, 'Please review', null, '2026-01-01 10:00:00' ); + $this->create_thread( $this->conversation, Thread::TYPE_NOTE, 'https://wordpress.org/plugins/my-plugin/', $this->agent, '2026-01-02 10:00:00' ); + + \Eventy::action( 'conversation.status_changed', $this->conversation->fresh(), $this->agent, false, Conversation::STATUS_ACTIVE ); + + $email = $this->payload()['email']; + $this->assertSame( (int) $this->conversation->number, $email['number'] ); + $this->assertSame( '[WordPress Plugin Directory] Review in Progress: My Plugin', $email['subject'] ); + $this->assertSame( 'closed', $email['status'] ); + $this->assertMatchesRegularExpression( '/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$/', (string) $email['created_at'] ); + $this->assertSame( $this->conversation->customer_email, $email['sender']['email'] ); + $this->assertSame( array( 'Jane', 'Sender' ), array( $email['sender']['first_name'], $email['sender']['last_name'] ) ); + $this->assertSame( array(), $email['threads'] ); + } + + /** + * A reply carries the threads since the previous reply, newest first: itself, and the notes before it. + * + * @return void + */ + public function test_replies_carry_the_threads_since_the_previous_reply(): void { + $this->create_thread( $this->conversation, Thread::TYPE_CUSTOMER, 'Please review', null, '2026-01-01 10:00:00' ); + $this->create_thread( $this->conversation, Thread::TYPE_NOTE, 'https://wordpress.org/plugins/my-plugin/', $this->agent, '2026-01-02 10:00:00' ); + $reply = $this->create_thread( $this->conversation, Thread::TYPE_MESSAGE, 'Thanks', $this->agent, '2026-01-03 10:00:00' ); + + \Eventy::action( 'conversation.user_replied', $this->conversation, $reply ); + + $this->assertSame( + array( + array( + 'type' => 'message', + 'body' => 'Thanks', + ), + array( + 'type' => 'note', + 'body' => 'https://wordpress.org/plugins/my-plugin/', + ), + ), + $this->payload()['email']['threads'] + ); + } + + /** + * A note goes out with itself, for the plugins and themes it mentions. + * + * @return void + */ + public function test_notes_carry_themselves(): void { + $this->create_thread( $this->conversation, Thread::TYPE_CUSTOMER, 'Please review', null, '2026-01-01 10:00:00' ); + $note = $this->create_thread( $this->conversation, Thread::TYPE_NOTE, 'https://wordpress.org/plugins/my-plugin/', $this->agent, '2026-01-02 10:00:00' ); + + \Eventy::action( 'conversation.note_added', $this->conversation, $note ); + + $this->assert_queued( 'conversation.note_added', (string) $this->agent->email ); + $this->assertSame( array( 'https://wordpress.org/plugins/my-plugin/' ), array_column( $this->payload()['email']['threads'], 'body' ) ); + } + + /** + * A reply's threads end with it, even when replies came after it before the event was sent. + * + * @return void + */ + public function test_late_replies_carry_their_own_threads(): void { + $this->create_thread( $this->conversation, Thread::TYPE_MESSAGE, 'Hello', $this->agent, '2026-01-01 10:00:00' ); + $reply = $this->create_thread( $this->conversation, Thread::TYPE_CUSTOMER, 'https://wordpress.org/plugins/my-plugin/', null, '2026-01-02 10:00:00' ); + + \Eventy::action( 'conversation.customer_replied', $this->conversation, $reply ); + $this->create_thread( $this->conversation, Thread::TYPE_MESSAGE, 'Thanks', $this->agent, '2026-01-03 10:00:00' ); + + $this->assertSame( array( 'https://wordpress.org/plugins/my-plugin/' ), array_column( $this->payload()['email']['threads'], 'body' ) ); + } + + /** + * Events the filter keeps out of WordPress.org's copy, like an imported conversation's before WordPress.org points at + * it, still go out for stats, without the conversation. + * + * @return void + */ + public function test_events_kept_out_of_the_copy_still_count(): void { + $filter = static function (): bool { + return false; + }; + \Eventy::addFilter( 'wporgwebhooks.copy', $filter ); + + \Eventy::action( 'conversation.status_changed', $this->conversation, $this->agent, false, Conversation::STATUS_ACTIVE ); + $payload = $this->payload(); + \Eventy::removeFilter( 'wporgwebhooks.copy', $filter ); + + $this->assertSame( (string) $this->agent->email, $payload['agent']['email'] ); + $this->assertArrayNotHasKey( 'email', $payload ); + } + + /** + * Every event of an imported conversation names the HelpScout conversation whose copy it replaces. + * + * @return void + */ + public function test_events_name_the_helpscout_conversation_they_replace(): void { + $filter = static function (): int { + return 1000000028; + }; + \Eventy::addFilter( 'wporgwebhooks.helpscout_id', $filter ); + + \Eventy::action( 'conversation.status_changed', $this->conversation, $this->agent, false, Conversation::STATUS_ACTIVE ); + $payload = $this->payload(); + \Eventy::removeFilter( 'wporgwebhooks.helpscout_id', $filter ); + + $this->assertSame( 1000000028, $payload['helpscout_id'] ); + } + + /** + * A late event counts for the mailbox it happened in; the copy takes the one the conversation is in now. + * + * @return void + */ + public function test_late_events_keep_their_mailbox_for_stats(): void { + \Eventy::action( 'conversation.status_changed', $this->conversation, $this->agent, false, Conversation::STATUS_ACTIVE ); + $this->conversation->mailbox_id = $this->create_mailbox( 'Themes' )->id; + $this->conversation->save(); + + $payload = $this->payload(); + + $this->assertSame( 'Plugins', $payload['mailbox']['name'] ); + $this->assertSame( 'Themes', $payload['email']['mailbox']['name'] ); + } + + /** + * A conversation deleted for good before its event went out is deleted from the copy, with the HelpScout + * conversation it was imported from, which the filter is asked about by ID. + * + * @return void + */ + public function test_events_of_conversations_deleted_since_name_their_helpscout_conversation(): void { + $id = (int) $this->conversation->id; + $filter = static function ( $helpscout_id = 0, $conversation = null, $conversation_id = 0 ) use ( $id ): int { + return null === $conversation && $id === (int) $conversation_id ? 1000000028 : (int) $helpscout_id; + }; + \Eventy::addFilter( 'wporgwebhooks.helpscout_id', $filter, 10, 3 ); + + \Eventy::action( 'conversation.status_changed', $this->conversation, $this->agent, false, Conversation::STATUS_ACTIVE ); + Conversation::deleteConversationsForever( array( $id ) ); + $payload = $this->payload(); + \Eventy::removeFilter( 'wporgwebhooks.helpscout_id', $filter ); + + $this->assertSame( array( 'state' => 'deleted' ), $payload['email'] ); + $this->assertSame( 1000000028, $payload['helpscout_id'] ); + } + + /** + * A merge names the HelpScout conversation the conversation merged away was imported from, whose copy goes too. + * + * @return void + */ + public function test_merges_name_the_helpscout_conversation_of_the_one_merged_away(): void { + $second = $this->create_conversation( $this->conversation->mailbox, $this->create_sender() ); + $filter = static function ( $helpscout_id = 0, $conversation = null, $conversation_id = 0 ) use ( $second ): int { + return (int) $second->id === (int) $conversation_id ? 1000000029 : (int) $helpscout_id; + }; + \Eventy::addFilter( 'wporgwebhooks.helpscout_id', $filter, 10, 3 ); + + \Eventy::action( 'conversation.merged', $this->conversation, $second, $this->agent ); + $payload = $this->payload(); + \Eventy::removeFilter( 'wporgwebhooks.helpscout_id', $filter ); + + $this->assertSame( array( (int) $second->id, 1000000029 ), array( $payload['merged_id'], $payload['merged_helpscout_id'] ) ); + $this->assertArrayNotHasKey( 'helpscout_id', $payload ); + } + + /** + * Asked for all of the threads, an event carries them, and says so, whatever it is. + * + * @return void + */ + public function test_events_carry_all_threads_when_asked(): void { + $this->create_thread( $this->conversation, Thread::TYPE_CUSTOMER, 'Please review', null, '2026-01-01 10:00:00' ); + $this->create_thread( $this->conversation, Thread::TYPE_NOTE, 'https://wordpress.org/plugins/my-plugin/', $this->agent, '2026-01-02 10:00:00' ); + + \Eventy::action( 'conversation.status_changed', $this->conversation, $this->agent, false, Conversation::STATUS_ACTIVE ); + $job = Queue::pushed( SendEvent::class )->first(); + + $this->assertFalse( EventPayload::refresh( $job->payload )['email']['all_threads'] ); + + $email = EventPayload::refresh( array( 'all_threads' => true ) + $job->payload )['email']; + $this->assertTrue( $email['all_threads'] ); + $this->assertSame( array( 'https://wordpress.org/plugins/my-plugin/', 'Please review' ), array_column( $email['threads'], 'body' ) ); + } + + /** + * Threads go as text, with a line for each block, and the addresses of their links, which is what WordPress.org + * looks for plugins and themes in. + * + * @return void + */ + public function test_threads_go_as_text(): void { + $this->assertSame( + "Hi Jane,\nSee your plugin and\nPlugin: my-plugin\nUse wp_enqueue script &\nhttps://wordpress.org/plugins/my-plugin/", + EventPayload::text( '

Hi Jane,

See your plugin and
Plugin: my-plugin

Use wp_enqueue <script> &
' ) + ); + } + + /** + * A conversation's threads, sent as it moves, stop at a size WordPress.org takes, cutting the oldest short. + * + * @return void + */ + public function test_threads_stop_at_a_size(): void { + $long = str_repeat( 'x', 100 * 1024 ); + foreach ( array( 1, 2, 3, 4 ) as $day ) { + $this->create_thread( $this->conversation, Thread::TYPE_CUSTOMER, $long, null, '2026-01-0' . $day . ' 10:00:00' ); + } + + \Eventy::action( 'conversation.moved', $this->conversation, $this->agent ); + $threads = $this->payload()['email']['threads']; + + $this->assertCount( 3, $threads ); + $this->assertSame( array( 100 * 1024, 100 * 1024, 56 * 1024 ), array_map( 'strlen', array_column( $threads, 'body' ) ) ); } /** @@ -256,6 +504,19 @@ public function test_nothing_is_queued_without_secret(): void { Queue::assertNotPushed( SendEvent::class ); } + /** + * The payload of the one event queued. + * + * @return array + */ + private function payload(): array { + $jobs = Queue::pushed( SendEvent::class ); + $this->assertCount( 1, $jobs ); + + // As it's sent. + return EventPayload::refresh( $jobs->first()->payload ); + } + /** * Asserts an event was queued for the conversation. * diff --git a/freescout.wordpress.net/Modules/WPOrgWebhooks/tests/SendEventTest.php b/freescout.wordpress.net/Modules/WPOrgWebhooks/tests/SendEventTest.php index bbd17ff879..65b1174e31 100644 --- a/freescout.wordpress.net/Modules/WPOrgWebhooks/tests/SendEventTest.php +++ b/freescout.wordpress.net/Modules/WPOrgWebhooks/tests/SendEventTest.php @@ -9,9 +9,16 @@ namespace Modules\WPOrgWebhooks\Tests; +use App\Thread; +use GuzzleHttp\Handler\MockHandler; +use GuzzleHttp\Promise\PromiseInterface; +use GuzzleHttp\Psr7\Response; use Illuminate\Queue\Jobs\DatabaseJob; use Modules\WPOrgWebhooks\Jobs\SendEvent; use Modules\WPOrgWebhooks\Providers\WPOrgWebhooksServiceProvider; +use Modules\WPOrgWebhooks\Services\Client; +use Modules\WPOrgWebhooks\Services\EventPayload; +use Psr\Http\Message\RequestInterface; use WordPressdotorg\FreeScout\Tests\TestCase; /** @@ -70,6 +77,74 @@ public function test_other_failures_are_not_retried(): void { $this->send_event( $queue_job ); } + /** + * WordPress.org asks for all of the threads of a conversation it has no copy of, and gets the event again with them; + * only that one counts. + * + * @return void + */ + public function test_sends_all_threads_when_asked(): void { + $conversation = $this->create_conversation( $this->create_mailbox(), $this->create_sender() ); + $this->create_thread( $conversation, Thread::TYPE_CUSTOMER, 'Please review', null, '2026-01-01 10:00:00' ); + $this->create_thread( $conversation, Thread::TYPE_NOTE, 'https://wordpress.org/plugins/my-plugin/', $this->create_user(), '2026-01-02 10:00:00' ); + $sent = $this->answer( new Response( 200, array(), '{"threads":"all"}' ), new Response( 200, array(), '{}' ) ); + + $queue_job = $this->queue_job( 1 ); + $queue_job->expects( $this->never() )->method( 'release' ); + $this->send_event( $queue_job, EventPayload::build( 'conversation.status_changed', $conversation, null ) ); + + $this->assertCount( 2, $sent ); + $this->assertSame( array( false, array() ), array( $sent[0]['email']['all_threads'], $sent[0]['email']['threads'] ) ); + $this->assertTrue( $sent[1]['email']['all_threads'] ); + $this->assertSame( array( 'https://wordpress.org/plugins/my-plugin/', 'Please review' ), array_column( $sent[1]['email']['threads'], 'body' ) ); + } + + /** + * An event too large for api.wordpress.org's web server goes again without its threads. + * + * @return void + */ + public function test_too_large_events_go_without_their_threads(): void { + $conversation = $this->create_conversation( $this->create_mailbox(), $this->create_sender() ); + $this->create_thread( $conversation, Thread::TYPE_CUSTOMER, 'Please review', null, '2026-01-01 10:00:00' ); + $sent = $this->answer( new Response( 413 ), new Response( 200, array(), '{}' ) ); + + $queue_job = $this->queue_job( 1 ); + $queue_job->expects( $this->never() )->method( 'markAsFailed' ); + $this->send_event( $queue_job, EventPayload::build( 'conversation.moved', $conversation, null ) ); + + $this->assertCount( 2, $sent ); + $this->assertCount( 1, $sent[0]['email']['threads'] ); + $this->assertSame( array(), $sent[1]['email']['threads'] ); + $this->assertSame( 'conversation.moved', $sent[1]['event'] ); + } + + /** + * Answers api.wordpress.org's requests in turn. + * + * @param Response ...$responses What it answers. + * @return \ArrayObject The payloads it was sent, as they arrive. + */ + private function answer( Response ...$responses ): \ArrayObject { + $sent = new \ArrayObject(); + $handler = new MockHandler( $responses ); + $this->app->instance( + Client::class, + new Client( + 'https://api.wordpress.test/', + 'test-secret', + 5, + static function ( RequestInterface $request, array $options ) use ( $sent, $handler ): PromiseInterface { + $sent[] = json_decode( (string) $request->getBody(), true ); + + return $handler( $request, $options ); + } + ) + ); + + return $sent; + } + /** * Mocks the queued job an event runs as; FreeScout queues in the database. * @@ -87,10 +162,11 @@ private function queue_job( int $attempts ): DatabaseJob { * Runs an event job as the queue worker would. * * @param DatabaseJob $queue_job Queued job the event runs as. + * @param array $payload Event payload. * @return void */ - private function send_event( DatabaseJob $queue_job ): void { - $event = new SendEvent( array( 'event' => 'conversation.user_replied' ) ); + private function send_event( DatabaseJob $queue_job, array $payload = array( 'event' => 'conversation.user_replied' ) ): void { + $event = new SendEvent( $payload ); $event->setJob( $queue_job ); $event->handle(); } From 9e768c2e0f54fa22f9654003ae9b2f4c4d4c4308 Mon Sep 17 00:00:00 2001 From: Konstantin Obenland Date: Tue, 13 Oct 2026 14:54:15 -0500 Subject: [PATCH 02/10] FreeScout: Bump WPOrgWebhooks and WPOrgHelpScoutImport to 1.1.0 Co-Authored-By: Claude Opus 5.5 --- .../Modules/WPOrgHelpScoutImport/module.json | 2 +- freescout.wordpress.net/Modules/WPOrgWebhooks/module.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/module.json b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/module.json index b92540bd58..640b3ca3e9 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/module.json +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/module.json @@ -3,7 +3,7 @@ "alias": "wporghelpscoutimport", "img": "/modules/wporghelpscoutimport/img/icon.svg", "description": "Imports HelpScout mailboxes' conversations, so their history moves to FreeScout with them.", - "version": "1.0.0", + "version": "1.1.0", "detailsUrl": "https://github.com/WordPress/wordpress.org/tree/trunk/freescout.wordpress.net/Modules/WPOrgHelpScoutImport", "author": "WordPress.org", "authorUrl": "https://wordpress.org/", diff --git a/freescout.wordpress.net/Modules/WPOrgWebhooks/module.json b/freescout.wordpress.net/Modules/WPOrgWebhooks/module.json index 7e935b672e..375dbd5afa 100644 --- a/freescout.wordpress.net/Modules/WPOrgWebhooks/module.json +++ b/freescout.wordpress.net/Modules/WPOrgWebhooks/module.json @@ -3,7 +3,7 @@ "alias": "wporgwebhooks", "img": "/modules/wporgwebhooks/img/icon.svg", "description": "Sends conversation events to WordPress.org, which records them as contributor stats.", - "version": "1.0.0", + "version": "1.1.0", "detailsUrl": "https://github.com/WordPress/wordpress.org/tree/trunk/freescout.wordpress.net/Modules/WPOrgWebhooks", "author": "WordPress.org", "authorUrl": "https://wordpress.org/", From 7760b2a155746d6975ae5f7734f5e7fedd22bf0c Mon Sep 17 00:00:00 2001 From: Konstantin Obenland Date: Tue, 13 Oct 2026 15:25:10 -0500 Subject: [PATCH 03/10] FreeScout: Write the copy without a lock, as its meta is unique wporg_helpscout_meta has a unique key on helpscout_id, meta_key and meta_value on production, as HelpScout's ON DUPLICATE KEY UPDATE relies on, so writes add their meta with INSERT IGNORE inside their transaction instead of taking a lock per conversation. A failed write throws, and the endpoint answers 503 for FreeScout to send it again. The local table matches production's. Also drops replace-copies.php's cap on a batch, which only FreeScout sends, names the automation account, and moves the shared meta keys to the top of common.php. Co-Authored-By: Claude Opus 5.5 --- .../public_html/dotorg/freescout/common.php | 98 ++++--------------- .../dotorg/freescout/replace-copies.php | 24 ++--- .../public_html/dotorg/freescout/webhook.php | 90 +++++++---------- .../plugin-directory/bin/database-tables.sql | 15 +-- 4 files changed, 71 insertions(+), 156 deletions(-) diff --git a/api.wordpress.org/public_html/dotorg/freescout/common.php b/api.wordpress.org/public_html/dotorg/freescout/common.php index 53984d0f5b..3adf6411d1 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/common.php +++ b/api.wordpress.org/public_html/dotorg/freescout/common.php @@ -48,6 +48,23 @@ function load_wordpress( string $wp_init_host = '' ): void { */ const MAX_CLOCK_SKEW = 10; +/** + * Meta key naming the helpdesk a copied conversation is in, `freescout`; HelpScout's copies have none. + * + * @var string + */ +const HELPDESK_META_KEY = 'helpdesk'; + +/** + * Meta key of a FreeScout conversation's copy naming a HelpScout conversation whose copy it replaced. + * + * HelpScout's webhook leaves alone a HelpScout conversation a copy names, so work done there since doesn't bring its + * copy back. It goes with the copy that names it. + * + * @var string + */ +const REPLACED_HELPSCOUT_META_KEY = 'helpscout_conversation'; + /** * Retrieves the incoming payload, and verifies FreeScout signed it for this endpoint. * @@ -673,87 +690,6 @@ function get_plugin_slugs_by_title( string $title ): array { return wp_list_pluck( $plugins, 'post_name' ); } -/** - * Meta key naming the helpdesk a copied conversation is in, `freescout`; HelpScout's copies have none. - * - * @var string - */ -const HELPDESK_META_KEY = 'helpdesk'; - -/** - * Meta key of a FreeScout conversation's copy naming a HelpScout conversation whose copy it replaced. - * - * HelpScout's webhook leaves alone a HelpScout conversation a copy names, so work done there since doesn't bring its - * copy back. It goes with the copy that names it. - * - * @var string - */ -const REPLACED_HELPSCOUT_META_KEY = 'helpscout_conversation'; - -/** - * What WordPress.org's copy does with a conversation's event: it's written, or deleted. - * - * @var string - */ -const COPY_DONE = 'done'; - -/** - * What WordPress.org's copy does with a conversation's event: nothing yet, as another write held the conversation up, - * or the write failed; FreeScout sends the event again. - * - * @var string - */ -const COPY_RETRY = 'retry'; - -/** - * What WordPress.org's copy does with a conversation's event: nothing yet, as it has no copy of the conversation, and - * the event only carries its newest threads; FreeScout sends it again with all of them. - * - * @var string - */ -const COPY_ALL_THREADS = 'all_threads'; - -/** - * How long a write to WordPress.org's copy of a conversation waits for another to finish, in seconds. - * - * Well under FreeScout's 10-second timeout for an event, so a held-up event is answered, and sent again, rather than - * written after FreeScout gave up on it, and sent again anyway. - * - * @var int - */ -const LOCK_TIMEOUT = 3; - -/** - * Takes the lock on WordPress.org's copy of a conversation, which writes to it hold, one at a time. - * - * The copy's meta table has no unique key to keep a pair from being added twice. The lock, and what's read while holding - * it, come from the primary database, not a replica. - * - * @param int $id Conversation ID. - * @return bool False if another write held it for too long. - */ -function lock_email( int $id ): bool { - global $wpdb; - - if ( is_callable( array( $wpdb, 'send_reads_to_masters' ) ) ) { - $wpdb->send_reads_to_masters(); - } - - return (bool) $wpdb->get_var( $wpdb->prepare( 'SELECT GET_LOCK( %s, %d )', 'wporg_helpscout_' . $id, LOCK_TIMEOUT ) ); -} - -/** - * Releases the lock on WordPress.org's copy of a conversation. - * - * @param int $id Conversation ID. - * @return void - */ -function unlock_email( int $id ): void { - global $wpdb; - - $wpdb->query( $wpdb->prepare( 'SELECT RELEASE_LOCK( %s )', 'wporg_helpscout_' . $id ) ); -} - /** * Deletes conversations from WordPress.org's copy, with their meta, and so the HelpScout conversations they name. * diff --git a/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php b/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php index 4146641470..7002660d63 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php +++ b/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php @@ -11,13 +11,6 @@ require __DIR__ . '/common.php'; -/** - * Most copies a request may replace; FreeScout's WPOrgHelpScoutImport sends a thousand at a time. - * - * @var int - */ -const MAX_COPIES = 1000; - /** * Swaps the copies of HelpScout's conversations for those of the FreeScout conversations imported from them. * @@ -29,8 +22,7 @@ * * @param array $copies Each a HelpScout ID, the FreeScout conversation's ID and number, whether its copy goes, and * whether it was merged into that conversation. - * @return int|null How many were swapped, or deleted; null if another write held one up, or one failed, so the batch - * should be sent again. + * @return int|null How many were swapped, or deleted; null if one failed, so the batch should be sent again. */ function replace_copies( array $copies ): ?int { global $wpdb; @@ -39,7 +31,12 @@ function replace_copies( array $copies ): ?int { $meta_table = "{$wpdb->base_prefix}helpscout_meta"; $replaced = 0; - foreach ( array_slice( $copies, 0, MAX_COPIES ) as $copy ) { + // What the swaps read comes from the primary database, not a replica. + if ( is_callable( array( $wpdb, 'send_reads_to_masters' ) ) ) { + $wpdb->send_reads_to_masters(); + } + + foreach ( $copies as $copy ) { $copy = (object) $copy; $helpscout_id = (int) ( $copy->helpscout_id ?? 0 ); $id = (int) ( $copy->id ?? 0 ); @@ -47,10 +44,6 @@ function replace_copies( array $copies ): ?int { continue; } - if ( ! lock_email( $id ) ) { - return null; - } - try { $has_helpscout = (bool) $wpdb->get_var( $wpdb->prepare( 'SELECT 1 FROM %i WHERE id = %d', $emails_table, $helpscout_id ) ); $has_freescout = (bool) $wpdb->get_var( $wpdb->prepare( 'SELECT 1 FROM %i WHERE id = %d', $emails_table, $id ) ); @@ -132,8 +125,6 @@ static function () use ( $wpdb, $emails_table, $meta_table, $action, $copy, $hel trigger_error( 'FreeScout copy of conversation ' . $id . ' failed: ' . $e->getMessage(), E_USER_WARNING ); // phpcs:ignore WordPress.PHP.DevelopmentFunctions, WordPress.Security.EscapeOutput.OutputNotEscaped -- Logged, not shown. return null; - } finally { - unlock_email( $id ); } } @@ -145,7 +136,6 @@ static function () use ( $wpdb, $emails_table, $meta_table, $action, $copy, $hel if ( null === $replaced ) { status_header( 503 ); - header( 'Retry-After: ' . LOCK_TIMEOUT ); exit; } diff --git a/api.wordpress.org/public_html/dotorg/freescout/webhook.php b/api.wordpress.org/public_html/dotorg/freescout/webhook.php index bb38c6b920..c81eda743a 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/webhook.php +++ b/api.wordpress.org/public_html/dotorg/freescout/webhook.php @@ -32,7 +32,7 @@ * * @var int[] */ -const AUTOMATION_USER_IDS = array( 5911429 ); +const AUTOMATION_USER_IDS = array( 5911429 ); // wordpressdotorg. /** * Records contributor stats for an event. @@ -79,57 +79,51 @@ function contributor_stats( object $request ): void { * and of the HelpScout conversations it, or that one, was imported from, taking over their plugins and themes: slug * changes update those, and the emails still name the old slug. * + * Each event carries the conversation as it was when it was sent, so the copy ends up as the last one written says. + * * @param object $request Request payload. - * @return string COPY_DONE; COPY_RETRY if another event for the conversation held it up, or the write failed; or - * COPY_ALL_THREADS if it needs all of the conversation's threads. FreeScout sends it again for both. + * @return bool Whether it needs all of the conversation's threads, which FreeScout sends it again with. + * + * @throws \Throwable If the write failed, and was rolled back. */ -function log_email( object $request ): string { +function log_email( object $request ): bool { + global $wpdb; + $id = (int) ( $request->conversation->id ?? 0 ); $email = $request->email ?? null; if ( ! $id || ! is_object( $email ) ) { - return COPY_DONE; + return false; } - /* - * Events for the conversation are written one at a time, see lock_email(). Each carries the conversation as it was - * when it was sent, so the copy ends up as the last one written says, which is current unless events crossed. - */ - if ( ! lock_email( $id ) ) { - return COPY_RETRY; + // What the write reads comes from the primary database, not a replica. + if ( is_callable( array( $wpdb, 'send_reads_to_masters' ) ) ) { + $wpdb->send_reads_to_masters(); } - try { - // Deleted conversations and spam aren't kept; a deletion undone since is sent as published. - $deleted = isset( $email->state ) ? 'deleted' === $email->state : 'conversation.deleted' === ( $request->event ?? '' ); - if ( $deleted || 'spam' === ( $email->status ?? '' ) ) { - in_transaction( - static function () use ( $request, $id ): void { - delete_emails( array_merge( array( $id ), ...array_values( get_replaced_ids( $request ) ) ) ); - } - ); - - return COPY_DONE; - } - - return write_email( $request, $id ); - } catch ( \Throwable $e ) { - trigger_error( 'FreeScout copy of conversation ' . $id . ' failed: ' . $e->getMessage(), E_USER_WARNING ); // phpcs:ignore WordPress.PHP.DevelopmentFunctions, WordPress.Security.EscapeOutput.OutputNotEscaped -- Logged, not shown. + // Deleted conversations and spam aren't kept; a deletion undone since is sent as published. + $deleted = isset( $email->state ) ? 'deleted' === $email->state : 'conversation.deleted' === ( $request->event ?? '' ); + if ( $deleted || 'spam' === ( $email->status ?? '' ) ) { + in_transaction( + static function () use ( $request, $id ): void { + delete_emails( array_merge( array( $id ), ...array_values( get_replaced_ids( $request ) ) ) ); + } + ); - return COPY_RETRY; - } finally { - unlock_email( $id ); + return false; } + + return write_email( $request, $id ); } /** - * Writes WordPress.org's copy of a conversation, while holding its lock; see log_email(). + * Writes WordPress.org's copy of a conversation; see log_email(). * * @param object $request Request payload. * @param int $id Conversation ID. - * @return string COPY_DONE, or COPY_ALL_THREADS if the copy has nothing of the conversation yet, and the event doesn't - * carry all of its threads. + * @return bool Whether it needs all of the conversation's threads: the copy has nothing of the conversation yet, and the + * event doesn't carry all of them. */ -function write_email( object $request, int $id ): string { +function write_email( object $request, int $id ): bool { global $wpdb; $email = $request->email; @@ -141,7 +135,7 @@ function write_email( object $request, int $id ): string { $row = $wpdb->get_row( $wpdb->prepare( 'SELECT * FROM %i WHERE id = %d', $emails_table, $id ) ); $has_helpscout = ! $row && $helpscout_id && $wpdb->get_var( $wpdb->prepare( 'SELECT 1 FROM %i WHERE id = %d', $emails_table, $helpscout_id ) ); if ( needs_all_threads( $request, $row, (bool) $has_helpscout ) ) { - return COPY_ALL_THREADS; + return true; } $ids = array_merge( array( $id ), ...array_values( $replaced ) ); @@ -181,18 +175,9 @@ function write_email( object $request, int $id ): string { in_transaction( static function () use ( $wpdb, $emails_table, $meta_table, $id, $data, $columns, $plan ): void { - // Its meta first, which names its helpdesk, and the HelpScout conversations it replaces. + // Its meta first, which names its helpdesk, and the HelpScout conversations it replaces; pairs are unique. foreach ( $plan['meta'] as list( $meta_key, $meta_value ) ) { - check_write( - $wpdb->insert( - $meta_table, - array( - 'helpscout_id' => $id, - 'meta_key' => $meta_key, // phpcs:ignore WordPress.DB.SlowDBQuery -- Not a query. - 'meta_value' => $meta_value, // phpcs:ignore WordPress.DB.SlowDBQuery -- Not a query. - ) - ) - ); + check_write( $wpdb->query( $wpdb->prepare( 'INSERT IGNORE INTO %i ( helpscout_id, meta_key, meta_value ) VALUES ( %d, %s, %s )', $meta_table, $id, $meta_key, $meta_value ) ) ); } // phpcs:disable WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholders are added for each column; one statement, so two events for a new conversation at once don't both insert it. @@ -221,7 +206,7 @@ static function ( string $column ): array { } ); - return COPY_DONE; + return false; } /** @@ -267,18 +252,19 @@ function get_existing_slugs( array $mentioned ): array { $request = get_request( basename( __FILE__ ) ); -// Before the stats, which an event sent again would count twice. -$copied = log_email( $request ); -if ( COPY_RETRY === $copied ) { +// Before the stats, which an event sent again would count twice. FreeScout sends it again after a 503. +try { + $needs_all_threads = log_email( $request ); +} catch ( \Throwable $e ) { + trigger_error( 'FreeScout copy of conversation ' . (int) ( $request->conversation->id ?? 0 ) . ' failed: ' . $e->getMessage(), E_USER_WARNING ); // phpcs:ignore WordPress.PHP.DevelopmentFunctions, WordPress.Security.EscapeOutput.OutputNotEscaped -- Logged, not shown. status_header( 503 ); - header( 'Retry-After: ' . LOCK_TIMEOUT ); exit; } header( 'Content-Type: application/json; charset=utf-8' ); // FreeScout sends the event again at once, with all of the conversation's threads; that one counts. -if ( COPY_ALL_THREADS === $copied ) { +if ( $needs_all_threads ) { echo '{"threads":"all"}'; exit; } diff --git a/environments/plugin-directory/bin/database-tables.sql b/environments/plugin-directory/bin/database-tables.sql index eb2a6e0a0d..44e35393d8 100644 --- a/environments/plugin-directory/bin/database-tables.sql +++ b/environments/plugin-directory/bin/database-tables.sql @@ -17,13 +17,16 @@ CREATE TABLE IF NOT EXISTS `wp_helpscout` ( PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +-- As on production. CREATE TABLE IF NOT EXISTS `wp_helpscout_meta` ( - `helpscout_id` bigint(20) unsigned NOT NULL, - `meta_key` varchar(255) NOT NULL DEFAULT '', - `meta_value` varchar(255) NOT NULL DEFAULT '', - UNIQUE KEY `helpscout_id` (`helpscout_id`,`meta_key`(191),`meta_value`(191)), - KEY `meta_key` (`meta_key`(191),`meta_value`(191)) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `helpscout_id` bigint(20) NOT NULL, + `meta_key` varchar(32) NOT NULL, + `meta_value` varchar(255) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `helpscout_id` (`helpscout_id`,`meta_key`,`meta_value`), + KEY `meta_key` (`meta_key`,`meta_value`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; CREATE TABLE IF NOT EXISTS `wp_svn_access` ( `path` varchar(255) NOT NULL DEFAULT '', From 33cdf61686f755be00c9e5fe4473bd072c31acc8 Mon Sep 17 00:00:00 2001 From: Konstantin Obenland Date: Tue, 13 Oct 2026 15:37:37 -0500 Subject: [PATCH 04/10] FreeScout: Trim the copy sync's comments Drop comments that restate the code or another docblock, and shorten the rest to what the code can't say. Co-Authored-By: Claude Opus 5.5 --- .../public_html/dotorg/freescout/common.php | 44 ++++------ .../dotorg/freescout/replace-copies.php | 18 ++--- .../freescout/tests/class-copy-plan-test.php | 3 +- .../freescout/tests/class-email-row-test.php | 5 +- .../public_html/dotorg/freescout/webhook.php | 25 +++--- .../public_html/dotorg/helpscout/common.php | 11 +-- ..._to_wporghelpscoutimport_conversations.php | 5 +- .../Http/Controllers/ImportController.php | 5 +- .../WPOrgHelpScoutImport/Jobs/ImportPage.php | 2 +- .../Jobs/ReplaceCopies.php | 8 +- .../WPOrgHelpScoutImportServiceProvider.php | 10 +-- .../WPOrgHelpScoutImport/Services/Copies.php | 80 ++++++++----------- .../Services/Importer.php | 6 +- .../WPOrgHelpScoutImport/tests/CopiesTest.php | 24 +++--- .../tests/ImporterTest.php | 3 +- .../Modules/WPOrgWebhooks/Jobs/SendEvent.php | 9 +-- .../WPOrgWebhooksServiceProvider.php | 12 +-- .../Modules/WPOrgWebhooks/Services/Client.php | 2 +- .../WPOrgWebhooks/Services/EventPayload.php | 51 +++++------- .../Services/PayloadTooLargeException.php | 3 +- .../tests/EventForwardingTest.php | 13 ++- 21 files changed, 133 insertions(+), 206 deletions(-) diff --git a/api.wordpress.org/public_html/dotorg/freescout/common.php b/api.wordpress.org/public_html/dotorg/freescout/common.php index 3adf6411d1..c39574a18a 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/common.php +++ b/api.wordpress.org/public_html/dotorg/freescout/common.php @@ -49,17 +49,14 @@ function load_wordpress( string $wp_init_host = '' ): void { const MAX_CLOCK_SKEW = 10; /** - * Meta key naming the helpdesk a copied conversation is in, `freescout`; HelpScout's copies have none. + * Meta key marking a copy as FreeScout's; HelpScout's copies have none. * * @var string */ const HELPDESK_META_KEY = 'helpdesk'; /** - * Meta key of a FreeScout conversation's copy naming a HelpScout conversation whose copy it replaced. - * - * HelpScout's webhook leaves alone a HelpScout conversation a copy names, so work done there since doesn't bring its - * copy back. It goes with the copy that names it. + * Meta key naming the HelpScout conversation a FreeScout copy replaced, which HelpScout's webhook then leaves alone. * * @var string */ @@ -194,13 +191,10 @@ function send_panel( array $blocks, array $extra = array() ): never { } /** - * A conversation's row in WordPress.org's copy of helpdesk emails, which the plugin directory reads. - * - * The copy began with HelpScout's emails, so its columns, and the table's name, are HelpScout's; the values are the - * same for FreeScout's conversations, but for the ID. + * A conversation's row in WordPress.org's copy, whose columns and table name are HelpScout's. * * @param object $request Webhook payload, with the conversation in `email`; see WPOrgWebhooks' EventPayload. - * @param object|null $row The conversation's row so far, if it has one. + * @param object|null $row Its current row, if any. * @param int $user_id The sender's WordPress.org user ID, 0 if unknown. * @return array */ @@ -270,8 +264,7 @@ function get_mailbox_slug( object $request ): string { } /** - * The slug of the mailbox WordPress.org's copy keeps a conversation in: the one it's in now, which the event carries in - * `email`, rather than the one the event happened in, which stats count. + * The slug of the conversation's current mailbox, which the copy keeps, rather than the event's, which stats count. * * @param object $request Webhook payload. * @return string @@ -691,7 +684,7 @@ function get_plugin_slugs_by_title( string $title ): array { } /** - * Deletes conversations from WordPress.org's copy, with their meta, and so the HelpScout conversations they name. + * Deletes conversations from WordPress.org's copy, with their meta. * * @param int[] $ids Conversation IDs; HelpScout's for the copies of its conversations. * @return void @@ -735,9 +728,8 @@ function get_replaced_ids( object $request ): array { } /** - * Whether WordPress.org needs all of a conversation's threads before it copies it: it has no copy of it yet, nor of - * the HelpScout conversation it was imported from, which would have what the threads mention, and the event carries - * only the newest. + * Whether copying a conversation needs all of its threads: the copy has neither it nor its HelpScout conversation, and + * the event carries only the newest. * * @param object $request Webhook payload. * @param object|null $row The conversation's row in the copy, if it has one. @@ -751,8 +743,8 @@ function needs_all_threads( object $request, ?object $row, bool $has_helpscout ) /** * What writing a conversation's copy adds to its meta, and which copies it deletes. * - * It takes over what the copies it replaces mention, names the HelpScout conversations it replaces, and adds what its - * threads mention. A HelpScout conversation it names already was replaced by an earlier event, so it isn't again. + * It takes over the replaced copies' meta, names the HelpScout conversations it replaces, and adds what its threads + * mention. HelpScout conversations it names already aren't replaced again. * * @param int $id Conversation ID. * @param array[] $replaced The copies it replaces, see get_replaced_ids(). @@ -777,7 +769,6 @@ function plan_email_write( int $id, array $replaced, array $all_meta, array $men } } - // A HelpScout conversation the copy names was replaced already. $helpscout = array_values( array_filter( $replaced['helpscout'], @@ -813,18 +804,17 @@ static function ( int $helpscout_id ) use ( $replaces ): bool { } /** - * What pointing WordPress.org at an imported conversation does to its copy. + * What switching an imported conversation does to its copy. * - * A conversation merged into another before the switch is sent as the one it went into, `merged`: its HelpScout one's - * copy is merged into that one's, whose own HelpScout copy may not be renamed yet, so it takes over what that copy - * mentions, as a merge after the switch does, rather than standing in for it. + * One merged away before the switch arrives as the conversation it went into, flagged `merged`: its HelpScout copy is + * merged into that one's rather than renamed, as that one's own HelpScout copy may still need renaming. * * @param object $copy The imported conversation: helpscout_id, id, number, delete, and merged. * @param bool $has_helpscout Whether the copy has the HelpScout conversation. * @param bool $has_freescout Whether it has the FreeScout one. * @param bool $names_it Whether the FreeScout one's copy names the HelpScout one, as it replaced it already. - * @return string `skip`; `delete` both, as the conversation was deleted, or is spam; `rename` the HelpScout one's copy - * into the FreeScout one's; `merge` it into the FreeScout one's copy; or `name` it in that copy. + * @return string `skip`; `delete` both copies (deleted or spam); `rename` the HelpScout copy to the FreeScout ID; + * `merge` it into the FreeScout copy; or `name` it in that copy. */ function plan_copy_replacement( object $copy, bool $has_helpscout, bool $has_freescout, bool $names_it ): string { $helpscout_id = (int) ( $copy->helpscout_id ?? 0 ); @@ -841,12 +831,12 @@ function plan_copy_replacement( object $copy, bool $has_helpscout, bool $has_fre return $has_freescout || ! empty( $copy->merged ) ? 'merge' : 'rename'; } - // Its own events wrote the FreeScout one's copy, while HelpScout's was gone: it still keeps HelpScout's webhook out. + // The FreeScout copy exists without the HelpScout one; naming it still keeps HelpScout's webhook out. return $has_freescout && ! $names_it ? 'name' : 'skip'; } /** - * Runs writes to WordPress.org's copy in a transaction, so none of them are seen, or kept, without the others. + * Runs writes to WordPress.org's copy in a transaction. * * @param callable $write Writes; throws if one fails, see check_write(). * @return void diff --git a/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php b/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php index 7002660d63..7c0754aed7 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php +++ b/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php @@ -12,13 +12,8 @@ require __DIR__ . '/common.php'; /** - * Swaps the copies of HelpScout's conversations for those of the FreeScout conversations imported from them. - * - * A copy keeps its plugins and themes, is marked as FreeScout's, and names the HelpScout conversation it replaces, which - * keeps HelpScout's webhook from bringing it back; one FreeScout's webhook added already takes over the HelpScout copy's - * plugins and themes instead, as does the copy of a conversation another was merged into, which is sent for that one. - * Copies of conversations deleted since, or spam, go. Each is swapped in a transaction, and copies already swapped are - * left alone, so a batch can be sent again. + * Swaps the copies of HelpScout's conversations for those of the FreeScout conversations imported from them; see + * plan_copy_replacement(). Each swap is a transaction, and swapped copies are skipped, so a batch can be sent again. * * @param array $copies Each a HelpScout ID, the FreeScout conversation's ID and number, whether its copy goes, and * whether it was merged into that conversation. @@ -31,7 +26,7 @@ function replace_copies( array $copies ): ?int { $meta_table = "{$wpdb->base_prefix}helpscout_meta"; $replaced = 0; - // What the swaps read comes from the primary database, not a replica. + // Read from the primary database, not a lagging replica. if ( is_callable( array( $wpdb, 'send_reads_to_masters' ) ) ) { $wpdb->send_reads_to_masters(); } @@ -61,10 +56,7 @@ static function () use ( $wpdb, $emails_table, $meta_table, $action, $copy, $hel return; } - /* - * The FreeScout copy's marks first: its helpdesk, and the HelpScout conversation it replaces. A meta row is - * unique, so nothing it has already is written again. - */ + // Mark the FreeScout copy: its helpdesk, and the HelpScout conversation it replaces. $marks = array( REPLACED_HELPSCOUT_META_KEY => (string) $helpscout_id ); if ( 'rename' === $action ) { $marks[ HELPDESK_META_KEY ] = 'freescout'; @@ -93,7 +85,7 @@ static function () use ( $wpdb, $emails_table, $meta_table, $action, $copy, $hel return; } - // It takes the plugins and themes it doesn't have yet, like those of a conversation merged into it. + // Take over the HelpScout copy's plugins and themes. check_write( $wpdb->query( $wpdb->prepare( diff --git a/api.wordpress.org/public_html/dotorg/freescout/tests/class-copy-plan-test.php b/api.wordpress.org/public_html/dotorg/freescout/tests/class-copy-plan-test.php index 38f9c62eae..c3177a6e86 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/tests/class-copy-plan-test.php +++ b/api.wordpress.org/public_html/dotorg/freescout/tests/class-copy-plan-test.php @@ -1,7 +1,6 @@ send_reads_to_masters(); } - // Deleted conversations and spam aren't kept; a deletion undone since is sent as published. + // Spam and deleted conversations aren't kept; the current state wins over the event, as a deletion may be undone. $deleted = isset( $email->state ) ? 'deleted' === $email->state : 'conversation.deleted' === ( $request->event ?? '' ); if ( $deleted || 'spam' === ( $email->status ?? '' ) ) { in_transaction( @@ -120,8 +116,7 @@ static function () use ( $request, $id ): void { * * @param object $request Request payload. * @param int $id Conversation ID. - * @return bool Whether it needs all of the conversation's threads: the copy has nothing of the conversation yet, and the - * event doesn't carry all of them. + * @return bool Whether it needs all of the conversation's threads; see needs_all_threads(). */ function write_email( object $request, int $id ): bool { global $wpdb; @@ -147,7 +142,7 @@ function write_email( object $request, int $id ): bool { ARRAY_A ); - // What the subject mentions was added when the copy got it; looking it up again takes queries for every event. + // The subject's mentions were added with the copy; the lookup costs queries, so only redo it when it changed. $mailbox = $request->email->mailbox ?? ( $request->mailbox ?? null ); $subject = (string) ( $email->subject ?? '' ); $threads = (array) ( $email->threads ?? array() ); @@ -175,7 +170,7 @@ function write_email( object $request, int $id ): bool { in_transaction( static function () use ( $wpdb, $emails_table, $meta_table, $id, $data, $columns, $plan ): void { - // Its meta first, which names its helpdesk, and the HelpScout conversations it replaces; pairs are unique. + // Pairs are unique; one an overlapping event added is skipped. foreach ( $plan['meta'] as list( $meta_key, $meta_value ) ) { check_write( $wpdb->query( $wpdb->prepare( 'INSERT IGNORE INTO %i ( helpscout_id, meta_key, meta_value ) VALUES ( %d, %s, %s )', $meta_table, $id, $meta_key, $meta_value ) ) ); } diff --git a/api.wordpress.org/public_html/dotorg/helpscout/common.php b/api.wordpress.org/public_html/dotorg/helpscout/common.php index 96aba7980a..8c352b2e59 100644 --- a/api.wordpress.org/public_html/dotorg/helpscout/common.php +++ b/api.wordpress.org/public_html/dotorg/helpscout/common.php @@ -544,14 +544,9 @@ function get_mailbox_name( $mailbox_id_or_request ) { } /** - * Whether a HelpScout conversation moved to FreeScout, whose copy replaced HelpScout's. + * Whether FreeScout's copy replaced a HelpScout conversation's, naming it under `helpscout_conversation`. * - * FreeScout's copy of a conversation imported from HelpScout records the HelpScout conversation it replaced, under the - * `helpscout_conversation` meta key; see api.wordpress.org/public_html/dotorg/freescout/. Anything still done to the - * conversation in HelpScout, like an agent or a workflow closing it, would otherwise add HelpScout's copy back, next - * to FreeScout's, and the plugin directory could reply to an upload in HelpScout, where no one looks any more. - * - * One lookup by meta key and value, like the plugin directory's lookup of a plugin's emails by slug. + * Work still done on it in HelpScout must not bring HelpScout's copy back. * * @param int $helpscout_id HelpScout conversation ID. * @return bool @@ -572,8 +567,6 @@ function is_moved_to_freescout( $helpscout_id ) { /** * Keep a cached copy of the received emails in the database for querying. * - * Conversations that moved to FreeScout are left alone, see is_moved_to_freescout(). - * * @param string $event Event name. * @param object $request Helpscout request object / Conversation object. */ diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Database/Migrations/2026_10_10_000000_add_mailbox_to_wporghelpscoutimport_conversations.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Database/Migrations/2026_10_10_000000_add_mailbox_to_wporghelpscoutimport_conversations.php index b5e784fd26..b5f3ae7299 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Database/Migrations/2026_10_10_000000_add_mailbox_to_wporghelpscoutimport_conversations.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Database/Migrations/2026_10_10_000000_add_mailbox_to_wporghelpscoutimport_conversations.php @@ -13,8 +13,7 @@ use Illuminate\Support\Facades\Schema; /** - * Which mailbox each imported conversation is in, so pointing WordPress.org at a mailbox's conversations still finds - * those deleted for good, whose copies go. + * Records each imported conversation's mailbox, so switching a mailbox still finds those deleted for good. */ class AddMailboxToWporghelpscoutimportConversations extends Migration { @@ -42,7 +41,7 @@ static function ( Blueprint $table ): void { } ); - // Those imported already; those deleted for good since can't be told anymore. + // Backfills earlier imports; those deleted for good since are lost. foreach ( \DB::table( 'conversations' )->distinct()->pluck( 'mailbox_id' ) as $mailbox_id ) { \DB::table( self::TABLE ) ->whereIn( diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Http/Controllers/ImportController.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Http/Controllers/ImportController.php index 698dde8892..bf0248c095 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Http/Controllers/ImportController.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Http/Controllers/ImportController.php @@ -89,8 +89,7 @@ public function index(): View { } /** - * Points WordPress.org's copies of a mailbox's HelpScout conversations at the FreeScout conversations imported from - * them, when the mailbox switches to FreeScout; or carries on where that stopped. + * Switches a mailbox's conversations in WordPress.org's copy to FreeScout, or resumes a failed switch. * * @param Request $request Request. * @return RedirectResponse @@ -115,7 +114,7 @@ public function point( Request $request ): RedirectResponse { } /** - * The mailboxes conversations were imported into, and whether WordPress.org points at them. + * The mailboxes conversations were imported into, and their switch. * * @return array[] Mailbox, total imported, and the switch, see Copies::get(). */ diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ImportPage.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ImportPage.php index 1ff57174bb..8e67c956f2 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ImportPage.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ImportPage.php @@ -262,7 +262,7 @@ static function () use ( $attempts ): bool { $run->status = Run::STATUS_DONE; $run->finished_at = Carbon::now(); if ( $this->save( $run ) ) { - // Once WordPress.org points at the mailbox, it points at what this imported too. + // Send what this run imported, if the mailbox switched. Copies::catch_up( (int) $run->mailbox_id, (string) $run->started_at ); } diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ReplaceCopies.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ReplaceCopies.php index 1bf25819b5..1c082c9a10 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ReplaceCopies.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ReplaceCopies.php @@ -17,10 +17,8 @@ use Modules\WPOrgWebhooks\Services\Client; /** - * Tells WordPress.org which FreeScout conversation each of a batch of HelpScout's was imported as, or that its copy goes, - * and queues the next batch. - * - * WordPress.org swaps the IDs in its copy, so running a batch again does nothing more. + * Sends WordPress.org a batch of imported conversations for its copy, and queues the next. Sending a batch again + * changes nothing. */ final class ReplaceCopies implements ShouldQueue { use Dispatchable; @@ -88,7 +86,7 @@ public function handle(): void { $batch = Copies::batch( $this->mailbox_id, (int) $state['after_id'], self::BATCH, $state['since'] ?? null ); if ( $batch->isEmpty() ) { - // Imports that finished while it ran may want another. + // finish() starts the send that imports queued meanwhile, if any. if ( Copies::finish( $this->mailbox_id ) ) { self::dispatch( $this->mailbox_id ); } diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Providers/WPOrgHelpScoutImportServiceProvider.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Providers/WPOrgHelpScoutImportServiceProvider.php index cc0669ae9a..d9cc2fb5d8 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Providers/WPOrgHelpScoutImportServiceProvider.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Providers/WPOrgHelpScoutImportServiceProvider.php @@ -122,11 +122,11 @@ static function ( $javascripts = array() ) { } ); - // WordPress.org's copy of an imported conversation replaces the copy of its HelpScout conversation. + // Names the HelpScout conversation whose copy an imported one's replaces. \Eventy::addFilter( 'wporgwebhooks.helpscout_id', static function ( mixed $helpscout_id = 0, mixed $conversation = null, mixed $conversation_id = 0 ): mixed { - // Deleted for good, the conversation is gone, but its ID still names what was imported. + // Deleted for good, the conversation is null, but its ID still finds the import. $id = $conversation instanceof Conversation ? (int) $conversation->id : (int) $conversation_id; try { @@ -143,7 +143,7 @@ static function ( mixed $helpscout_id = 0, mixed $conversation = null, mixed $co 3 ); - // For pointing WordPress.org at what an import brought, once it points at the mailbox. + // Marks updated conversations for Copies::catch_up(). \Eventy::addAction( 'wporghelpscoutimport.conversation_imported', static function ( mixed $conversation = null ): void { @@ -157,7 +157,7 @@ static function ( mixed $conversation = null ): void { } ); - // Imported conversations' events leave WordPress.org's copy alone until it points at their mailbox. + // Imported conversations' events leave the copy alone until their mailbox switches. \Eventy::addFilter( 'wporgwebhooks.copy', static function ( mixed $copy = true, mixed $conversation = null, mixed $conversation_id = 0, mixed $mailbox_id = 0 ): mixed { @@ -176,7 +176,7 @@ static function ( mixed $copy = true, mixed $conversation = null, mixed $convers 4 ); - // Deleted for good, an imported conversation's mailbox is gone with it; pointing WordPress.org at it needs it. + // Deleting for good loses the conversation's mailbox, which switching needs. \Eventy::addAction( 'conversations.before_delete_forever', static function ( mixed $conversation_ids = array() ): void { diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php index c14a8067ef..c1f080ae98 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php @@ -21,18 +21,14 @@ use Modules\WPOrgWebhooks\Services\Client; /** - * WordPress.org keeps a copy of conversations, which the plugin directory links to, and replies to uploads in. Until a - * mailbox switches to FreeScout, the copies of its conversations are HelpScout's; switching sends WordPress.org which - * FreeScout conversation each was imported as, in batches, on the queue, and which were deleted or are spam since, whose - * copies go. + * Switches a mailbox's conversations in WordPress.org's copy from HelpScout to FreeScout, in batches on the queue. * - * Until then, imported conversations' events don't carry them for the copy, so WordPress.org goes on pointing at - * HelpScout; after, they do, with the HelpScout ID whose copy they replace, as do conversations imported later. + * Until a mailbox switches, its imported conversations' events leave the copy alone, so it keeps pointing at HelpScout. */ final class Copies { /** - * Option keeping a mailbox's switch, followed by its ID: status, how far it got, and when. + * Option name of a mailbox's switch, before the mailbox ID. * * @var string */ @@ -53,7 +49,7 @@ final class Copies { public const STATUS_DONE = 'done'; /** - * Status of a switch WordPress.org didn't take, which can carry on where it stopped. + * Status of a switch WordPress.org refused; start() resumes it. * * @var string */ @@ -63,12 +59,11 @@ final class Copies { * A mailbox's switch. * * @param int $mailbox_id FreeScout mailbox ID. - * @return array|null Status, after_id (the last imported conversation sent), sent, since (only conversations imported - * since then; null for all), started_at, finished_at, error, and pending, when imports asked for a - * send while one ran: the since of the send after it. Null if it hasn't switched. + * @return array|null Null if it hasn't switched. Keys: status, after_id (last sent), sent, since (null for all), + * started_at, finished_at, error, and pending (the since of a send queued behind this one). */ public static function get( int $mailbox_id ): ?array { - // Not from the cache, which saving doesn't update, and the queue's jobs change. + // Bypasses the cache, which other processes' saves don't update. $state = Option::get( self::OPTION . $mailbox_id, null, true, false ); return is_array( $state ) ? $state : null; @@ -86,7 +81,7 @@ public static function save( int $mailbox_id, array $state ): void { } /** - * Whether WordPress.org can be sent copies at all: WPOrgWebhooks signs the requests. + * Whether requests to WordPress.org can be signed, which WPOrgWebhooks does. * * @return bool */ @@ -95,9 +90,7 @@ public static function available(): bool { } /** - * Starts sending a mailbox's imported conversations to WordPress.org, or carries on where a failed send stopped. - * - * Under a lock on its state, so a press of the button and an import finishing at once don't both start sending. + * Starts sending a mailbox's imported conversations to WordPress.org, or resumes a failed send. * * @param int $mailbox_id FreeScout mailbox ID. * @return bool False if it's already running. @@ -130,12 +123,10 @@ static function ( ?array $state ): ?array { } /** - * Sends what an import added, or imported again, since a mailbox switched, if it has. Never throws: it runs as an - * import finishes. + * Sends what an import added or updated, if the mailbox switched. Never throws: it runs as an import finishes. * - * Imported again, a conversation's copy may be HelpScout's again: HelpScout's webhook writes it while anyone works on - * it there. While a send runs, the conversations are sent after it, from the start, as they may be behind it; a - * send that failed is started again, from the start, with them, so neither its conversations nor these are left out. + * Updated conversations are sent again, as HelpScout's webhook may have rewritten their copies. A running send gets + * them queued behind it; a failed one restarts with them. * * @param int $mailbox_id FreeScout mailbox ID. * @param string $since When the import started. @@ -161,7 +152,7 @@ static function ( ?array $state ) use ( $since, &$restarted ): ?array { return $state; } - // A failed send hadn't sent everything since its own start; done, it had. + // A failed send still owes its own range; a finished one doesn't. $wider = self::STATUS_FAILED === $state['status'] ? self::wider( $state['since'] ?? null, $since ) : $since; if ( array_key_exists( 'pending', $state ) ) { $wider = self::wider( $state['pending'], $wider ); @@ -206,10 +197,10 @@ static function ( ?array $state ) use ( $after_id, $count ): ?array { } /** - * Ends a send that has nothing left, or starts the one imports asked for while it ran. + * Ends a finished send, or starts the one queued behind it. * * @param int $mailbox_id FreeScout mailbox ID. - * @return bool Whether another send started, whose batches are to be queued. + * @return bool Whether a queued send started. */ public static function finish( int $mailbox_id ): bool { $state = self::change( @@ -234,7 +225,7 @@ static function ( ?array $state ): ?array { } /** - * Stops a send WordPress.org didn't take, which can carry on where it stopped. + * Marks a send WordPress.org refused as failed; start() resumes it. * * @param int $mailbox_id FreeScout mailbox ID. * @param string $error What went wrong. @@ -260,7 +251,7 @@ static function ( ?array $state ) use ( $error ): ?array { * A send started again from the first imported conversation. * * @param array $state The switch. - * @param string|null $since Send the conversations imported, or imported again, since then; null for all. + * @param string|null $since Only conversations imported or updated since then; null for all. * @return array */ private static function restart( array $state, ?string $since ): array { @@ -277,7 +268,7 @@ private static function restart( array $state, ?string $since ): array { } /** - * The earlier of two times, which takes in more imported conversations; null, for all of them, takes in any. + * The wider of two since values: the earlier time, or null for all. * * @param string|null $one A time; null for all. * @param string|null $other Another. @@ -292,11 +283,10 @@ private static function wider( ?string $one, ?string $other ): ?string { } /** - * Changes a mailbox's switch under a lock on it, so the queue's jobs, imports, and the button don't undo each other. + * Changes a mailbox's switch under a row lock, so jobs, imports, and the button don't undo each other. * * @param int $mailbox_id FreeScout mailbox ID. - * @param callable $change Takes the switch, null if it hasn't switched, and returns it changed, or null to leave - * it. + * @param callable $change Takes the switch, or null, and returns it changed, or null to leave it. * @return array|null The switch as changed; null if it wasn't. */ private static function change( int $mailbox_id, callable $change ): ?array { @@ -305,7 +295,7 @@ private static function change( int $mailbox_id, callable $change ): ?array { try { Option::query()->firstOrCreate( array( 'name' => $name ), array( 'value' => '' ) ); } catch ( QueryException $e ) { - // Created at the same time by another. + // Another process created it first. unset( $e ); } @@ -331,8 +321,7 @@ static function () use ( $name, $change ): ?array { } /** - * Notes the mailboxes of imported conversations about to be deleted for good, so a switch of their mailbox sends - * WordPress.org their deletion. + * Notes the mailboxes of imported conversations about to be deleted for good, so a switch still sends their deletion. * * @param int[] $conversation_ids Conversation IDs. * @return void @@ -342,7 +331,7 @@ public static function remember_mailboxes( array $conversation_ids ): void { $by_mailbox = Conversation::query()->whereIn( 'id', $ids )->get( array( 'id', 'mailbox_id' ) )->groupBy( 'mailbox_id' ); foreach ( $by_mailbox as $mailbox_id => $conversations ) { - // Not as imported again, which its updated_at says. + // toBase() leaves updated_at alone, which marks updates. ImportedConversation::query() ->toBase() ->whereIn( 'conversation_id', $conversations->pluck( 'id' )->all() ) @@ -352,7 +341,7 @@ public static function remember_mailboxes( array $conversation_ids ): void { } /** - * Marks a conversation as imported again, for catch_up(). + * Marks an imported conversation as updated, for catch_up(). * * @param Conversation $conversation Conversation. * @return void @@ -377,9 +366,9 @@ public static function total( int $mailbox_id ): int { * @param int $mailbox_id FreeScout mailbox ID. * @param int $after_id ID of the last one sent. * @param int $limit How many. - * @param string|null $since Only those imported, or imported again, since then. - * @return Collection Imported conversations, with their ID, helpscout_id, conversation_id, and their conversation's - * found_id (null if it was deleted for good), number, state, and status; see request(). + * @param string|null $since Only those imported or updated since then. + * @return Collection With helpscout_id, conversation_id, and their conversation's found_id (null if deleted for + * good), number, state, and status. */ public static function batch( int $mailbox_id, int $after_id, int $limit, ?string $since = null ): Collection { $table = ( new ImportedConversation() )->getTable(); @@ -408,11 +397,10 @@ static function ( Builder $query ) use ( $table, $since ): Builder { } /** - * What WordPress.org is told about an imported conversation: which FreeScout conversation replaces the copy of its - * HelpScout one, or that the copy goes, as the conversation was deleted, for good or not, or is spam. + * What WordPress.org is told about an imported conversation: the FreeScout conversation that replaces its HelpScout + * copy, or that the copy goes, as it's deleted or spam. * - * One merged into another, which deletes it, has its HelpScout one's copy merged into the other's instead, which - * takes over the plugins and themes it mentions, like a merge after the switch. + * One merged away is sent as the conversation it went into, which takes over its copy's plugins and themes. * * @param ImportedConversation $imported An imported conversation, as batch() reads it. * @return array HelpScout's ID, FreeScout's ID and number, whether to delete the copy, and whether it's merged into @@ -452,9 +440,7 @@ private static function gone( int $state, int $status ): bool { } /** - * The conversation one deleted by a merge was merged into, following it on if that one was merged away too. - * - * Core notes the merge on the conversation merged away, in a line item's `META_MERGED_INTO_CONV` meta. + * The conversation one was merged into, following further merges. * * @param int $conversation_id Conversation ID. * @return Conversation|null Null if it wasn't merged away, or the one it went into was deleted for good. @@ -462,7 +448,7 @@ private static function gone( int $state, int $status ): bool { private static function merged_into( int $conversation_id ): ?Conversation { $into = null; - // Few hops: conversations are merged by hand. + // Merges are manual, so chains are short. for ( $hops = 0; $hops < 10; $hops++ ) { $merges = Thread::query() ->where( 'conversation_id', $conversation_id ) @@ -518,7 +504,7 @@ static function ( Builder $gone ) use ( $table, $mailbox_id ): void { } /** - * Whether a conversation's events carry it for WordPress.org's copy: an imported one's only once its mailbox switched. + * Whether a conversation's events carry it for WordPress.org's copy: an imported one's only once its mailbox switches. * * @param int $conversation_id Conversation ID. * @param int $mailbox_id Its mailbox's ID. diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Importer.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Importer.php index cf21384fdf..d78ba7b727 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Importer.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Importer.php @@ -419,10 +419,8 @@ public static function update_counters( Mailbox $mailbox ): void { } /** - * Tells other modules a conversation was imported or updated, as that happens without core's events: pointing - * WordPress.org at its mailbox sends it again. - * - * Never throws: the conversation is imported by now. + * Tells other modules a conversation was imported or updated, which happens without core's events. Never throws: the + * conversation is imported by now. * * @param Conversation $conversation Conversation. * @param int $helpscout_id HelpScout's ID for it. diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php index 5fe3eccd57..852186c90c 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php @@ -114,7 +114,7 @@ public function test_sends_batches_until_done(): void { } /** - * A batch WordPress.org doesn't take, after its last try, stops the switch, which can carry on where it stopped. + * A batch WordPress.org refuses on its last try fails the switch, which can resume. * * @return void */ @@ -137,8 +137,7 @@ public function test_failed_batches_stop_the_switch_until_carried_on(): void { } /** - * Imports that finish after the switch send what they brought, new or imported again, from the start; before it, - * nothing is sent. + * Imports finishing after the switch send what they added or updated; before it, nothing is sent. * * @return void */ @@ -190,8 +189,7 @@ public function test_starts_once(): void { } /** - * An import that finishes while a send runs has what it brought sent after it, from the start, as the conversations - * it imported again may be behind it; a second one widens that. + * An import finishing during a send queues its conversations behind it, from the start; a second one widens that. * * @return void */ @@ -208,7 +206,7 @@ public function test_catching_up_while_sending_sends_after_it(): void { Queue::assertPushed( ReplaceCopies::class, 1 ); $this->assertSame( '2026-10-02 10:00:00', Copies::get( (int) $this->mailbox->id )['pending'] ); - // Imported again after the send went past it. + // Updated after the send passed it. $this->run_batches(); ImportedConversation::query()->where( 'conversation_id', $again->id )->update( array( 'updated_at' => '2026-10-02 11:00:00' ) ); $this->run_batches(); @@ -225,8 +223,7 @@ public function test_catching_up_while_sending_sends_after_it(): void { } /** - * Catching up on a send that failed starts it again from the start, with the wider of the two, so what it hadn't - * sent yet isn't left out. + * Catching up on a failed send restarts it with the wider range, so nothing it hadn't sent is left out. * * @return void */ @@ -246,8 +243,8 @@ public function test_catching_up_after_a_failure_sends_everything_it_had_not(): } /** - * Conversations deleted, for good or not, or spam, have their copies deleted, HelpScout's too; the others are - * pointed at. Those deleted for good from another mailbox are left to its switch. + * Copies of conversations deleted or spam go, HelpScout's too; others are pointed at. Those deleted for good from + * another mailbox are left to its switch. * * @return void */ @@ -282,9 +279,8 @@ public function test_deleted_and_spam_conversations_have_their_copies_deleted(): } /** - * A conversation merged into another before the switch is sent as the one it went into, followed on if that was - * merged away too, so its HelpScout copy is merged into that one's rather than deleted; one merged into a - * conversation deleted since goes. + * A conversation merged away before the switch is sent as the one it went into, following further merges; one + * merged into a conversation deleted since goes. * * @return void */ @@ -324,7 +320,7 @@ public function test_conversations_merged_away_are_sent_as_the_one_they_went_int /** * Events of an imported conversation deleted for good before they went out leave the copy alone until its mailbox - * switched, as for any imported conversation. + * switches. * * @return void */ diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImporterTest.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImporterTest.php index 706b8848c0..579ce45f54 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImporterTest.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImporterTest.php @@ -434,8 +434,7 @@ public function test_names_the_helpscout_conversation_of_imported_ones(): void { } /** - * What was imported keeps the mailbox it went into, which pointing WordPress.org at the mailbox reads once the - * conversation is deleted for good. + * What was imported keeps the mailbox it went into, for switching once the conversation is deleted for good. * * @return void */ diff --git a/freescout.wordpress.net/Modules/WPOrgWebhooks/Jobs/SendEvent.php b/freescout.wordpress.net/Modules/WPOrgWebhooks/Jobs/SendEvent.php index aab83da05f..747bbf6cfe 100644 --- a/freescout.wordpress.net/Modules/WPOrgWebhooks/Jobs/SendEvent.php +++ b/freescout.wordpress.net/Modules/WPOrgWebhooks/Jobs/SendEvent.php @@ -61,9 +61,9 @@ public function __construct( array $payload ) { /** * Sends the event, and tries again later if api.wordpress.org didn't get it. * - * When WordPress.org has no copy of the conversation yet, it takes nothing, and asks for all of its threads, which - * the event then carries. Other failures aren't retried, since webhook.php may have counted the event already. They - * fail the job rather than throw: the worker would retry anything thrown until the job runs out of tries. + * If WordPress.org asks for all of the conversation's threads, it's sent again with them. Other failures aren't + * retried, since webhook.php may have counted the event already. They fail the job rather than throw: the worker + * would retry anything thrown until the job runs out of tries. * * @return void */ @@ -89,8 +89,7 @@ public function handle(): void { } /** - * Posts an event; one too large for api.wordpress.org's web server goes again without its threads, so it still - * counts, and the copy still gets what else it says. + * Posts an event; one too large for the web server goes again without its threads. * * @param Client $client Client. * @param array $payload Event payload, as it's sent. diff --git a/freescout.wordpress.net/Modules/WPOrgWebhooks/Providers/WPOrgWebhooksServiceProvider.php b/freescout.wordpress.net/Modules/WPOrgWebhooks/Providers/WPOrgWebhooksServiceProvider.php index 3f6e3d3e72..69fb07837a 100644 --- a/freescout.wordpress.net/Modules/WPOrgWebhooks/Providers/WPOrgWebhooksServiceProvider.php +++ b/freescout.wordpress.net/Modules/WPOrgWebhooks/Providers/WPOrgWebhooksServiceProvider.php @@ -128,7 +128,7 @@ static function () use ( $conversation, $thread ): ?array { 2 ); - // Out of spam, the conversation is new to WordPress.org's copy again, which took it out. + // Leaving spam, the conversation is copied in full again. \Eventy::addAction( 'conversation.status_changed', static function ( $conversation = null, $user = null, $changed_on_reply = false, $prev_status = null ): void { @@ -145,7 +145,7 @@ static function () use ( $conversation, $user, $prev_status ): array { 4 ); - // For the plugins and themes it mentions; WordPress.org doesn't credit it. + // For the plugins and themes it mentions; not credited. \Eventy::addAction( 'conversation.note_added', static function ( $conversation = null, $thread = null ): void { @@ -182,7 +182,7 @@ static function ( $conversation = null, $second = null, $user = null ): void { self::forward( 'conversation.merged', static function () use ( $conversation, $second, $user ): array { - // The other conversation's threads are in this one now; WordPress.org drops its copy of it. + // WordPress.org folds the merged conversation's copy into this one. return array( $conversation, $user, array( 'merged_id' => $second instanceof Conversation ? (int) $second->id : 0 ) ); } ); @@ -191,7 +191,7 @@ static function () use ( $conversation, $second, $user ): array { 3 ); - // Restoring from Deleted: WordPress.org's copy took the conversation out when it was deleted. + // The copy dropped the conversation when it was deleted. \Eventy::addAction( 'conversation.state_changed', static function ( $conversation = null, $user = null, $previous_state = null ): void { @@ -295,8 +295,8 @@ private static function merged_away( Conversation $conversation ): bool { * Never throws: a failure here must not interrupt the core action that fired the event. * * @param string $event Event name. - * @param callable $resolve Returns the conversation, the agent who caused the event (or null), and optionally more about - * the event for the payload; or null to skip it. + * @param callable $resolve Returns the conversation, the agent who caused the event (or null), and optionally event + * details; or null to skip it. * @return void */ private static function forward( string $event, callable $resolve ): void { diff --git a/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/Client.php b/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/Client.php index 74cad7fd15..7f19b24d36 100644 --- a/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/Client.php +++ b/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/Client.php @@ -33,7 +33,7 @@ final class Client { private const NOT_DELIVERED_STATUSES = array( 502, 503 ); /** - * Status the web server sends for a request bigger than it takes, which it answers without passing it on. + * Status the web server refuses an oversized request with, before webhook.php sees it. * * @var int */ diff --git a/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/EventPayload.php b/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/EventPayload.php index 431e72719b..464fcffc92 100644 --- a/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/EventPayload.php +++ b/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/EventPayload.php @@ -18,32 +18,28 @@ /** * Builds the webhook payload for a conversation event. * - * Read by api.wordpress.org/dotorg/freescout/webhook.php; keep the two in sync. Besides the event, which contributor - * stats count, it carries the conversation as WordPress.org keeps a copy of it, which the plugin directory reads. + * Read by api.wordpress.org/dotorg/freescout/webhook.php; keep the two in sync. Carries the event, for contributor + * stats, and the conversation, for WordPress.org's copy of it. */ final class EventPayload { /** - * Most threads sent, newest first; WordPress.org reads them for the plugins and themes they mention. + * Most threads an event carries. * * @var int */ private const MAX_THREADS = 50; /** - * Most bytes of thread text sent with an event; the oldest threads are cut short, or left out, past it. - * - * WordPress.org only looks for the plugins and themes threads mention, so they're sent as text, which keeps most - * events far below it; this bounds the rest, like a long conversation moved to another mailbox. + * Most bytes of thread text an event carries; past it, the oldest threads are cut short or left out. * * @var int */ private const MAX_TEXT_BYTES = 256 * 1024; /** - * Events after which WordPress.org reads all of the conversation's threads: it's new to the copy, or in another - * mailbox, which changes what it mentions. So do status changes out of spam, which the copy doesn't keep, and - * events WordPress.org asked to have all of them for, as it had no copy yet. + * Events that carry all of a conversation's threads: the copy has none of it yet, or it moved to another mailbox, + * which changes what it mentions. See reads_all() for the rest. * * @var string[] */ @@ -55,9 +51,8 @@ final class EventPayload { ); /** - * Events after which WordPress.org reads the threads since the previous reply: the reply, and notes before it. - * - * Others change nothing the threads mention, so none are sent; the copy keeps what earlier ones mentioned. + * Events that carry the threads since the previous reply: the reply, and the notes before it. Events not listed carry + * none. * * @var string[] */ @@ -67,7 +62,7 @@ final class EventPayload { ); /** - * Events after which WordPress.org reads only the thread they're about: a note. + * Events that carry only the thread they're about. * * @var string[] */ @@ -99,12 +94,12 @@ final class EventPayload { /** * Builds the payload. * - * The conversation, as WordPress.org keeps a copy of it, is added when the event is sent; see refresh(). + * The conversation is added when the event is sent; see refresh(). * * @param string $event Event name: the FreeScout hook that fired. * @param Conversation $conversation Conversation the event happened on. * @param User|null $agent Agent who caused the event, if any. - * @param array $extra More about the event, like the conversation merged into this one. + * @param array $extra Event details, like the ID of a conversation merged into this one. * @return array */ public static function build( string $event, Conversation $conversation, ?User $agent, array $extra = array() ): array { @@ -123,13 +118,10 @@ public static function build( string $event, Conversation $conversation, ?User $ } /** - * The payload with the conversation as it is now, for sending it. - * - * Events can be sent late, and out of order, when one is retried: WordPress.org's copy takes whatever the latest - * says, so each says what's current, and a conversation deleted since says so. The event's own mailbox stays, for - * the stats; the copy takes the conversation's mailbox now. + * Adds the conversation as it is now: events can be sent late or out of order, and the copy takes what the latest + * says. The event's own mailbox stays, for stats. * - * @param array $payload The payload, as built, and `all_threads` when WordPress.org asked for all of them. + * @param array $payload The payload; `all_threads` if WordPress.org asked for them. * @return array */ public static function refresh( array $payload ): array { @@ -138,8 +130,7 @@ public static function refresh( array $payload ): array { $mailbox_id = $conversation ? (int) $conversation->mailbox_id : (int) ( $payload['mailbox']['id'] ?? 0 ); /** - * Filters whether an event carries the conversation for WordPress.org's copy, like an imported one's doesn't - * while WordPress.org still points at its HelpScout conversation; it still counts for stats. + * Filters whether an event carries the conversation for WordPress.org's copy; it counts for stats either way. * * @param bool $copy Whether it does. * @param Conversation|null $conversation Conversation; null if it was deleted for good since the event. @@ -157,7 +148,7 @@ public static function refresh( array $payload ): array { $payload['helpscout_id'] = $helpscout_id; } - // A conversation merged into this one may have been imported, and its HelpScout conversation copied too. + // The conversation merged into this one may have a HelpScout copy too. $merged_id = (int) ( $payload['merged_id'] ?? 0 ); if ( $merged_id && empty( $payload['merged_helpscout_id'] ) ) { $merged_helpscout_id = self::helpscout_id( Conversation::find( $merged_id ), $merged_id ); @@ -180,7 +171,7 @@ public static function refresh( array $payload ): array { */ private static function helpscout_id( ?Conversation $conversation, int $conversation_id ): int { /** - * Filters the ID of the HelpScout conversation a conversation was imported from, whose copy its own replaces. + * Filters the ID of the HelpScout conversation a conversation was imported from. * * @param int $helpscout_id HelpScout's ID; 0 if it wasn't imported. * @param Conversation|null $conversation Conversation; null if it was deleted for good. @@ -268,7 +259,7 @@ private static function threads( Conversation $conversation, array $payload, boo ->where( 'state', Thread::STATE_PUBLISHED ) ->whereIn( 'type', array_keys( self::THREAD_TYPES ) ); - // A reply's threads end with it, however many came after it before the event was sent. + // A reply's threads end at it, even if more came before the event was sent. if ( $own ) { $query->where( 'id', $thread_id ); } elseif ( ! $all && $thread_id ) { @@ -298,10 +289,10 @@ private static function threads( Conversation $conversation, array $payload, boo } /** - * A thread's HTML as text, for finding the plugins and themes it mentions: a line for each block, and the links' - * addresses on lines of their own after it. + * A thread's HTML as text, for finding the plugins and themes it mentions: a line per block, and link addresses on + * lines of their own. * - * Angle brackets go too, so WordPress.org stripping tags from it can't take text along. + * Angle brackets are removed, so stripping tags on WordPress.org can't swallow text. * * @param string $html The thread's body. * @return string diff --git a/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/PayloadTooLargeException.php b/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/PayloadTooLargeException.php index 9401a6bcee..98bfb900a3 100644 --- a/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/PayloadTooLargeException.php +++ b/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/PayloadTooLargeException.php @@ -12,8 +12,7 @@ use RuntimeException; /** - * Thrown when api.wordpress.org's web server refused a request for its size: webhook.php never saw it, so a smaller - * one can be sent instead. + * Thrown when api.wordpress.org's web server refused a request for its size, before webhook.php saw it. */ final class PayloadTooLargeException extends RuntimeException { } diff --git a/freescout.wordpress.net/Modules/WPOrgWebhooks/tests/EventForwardingTest.php b/freescout.wordpress.net/Modules/WPOrgWebhooks/tests/EventForwardingTest.php index 5e9da8db91..d2c1540f91 100644 --- a/freescout.wordpress.net/Modules/WPOrgWebhooks/tests/EventForwardingTest.php +++ b/freescout.wordpress.net/Modules/WPOrgWebhooks/tests/EventForwardingTest.php @@ -172,7 +172,6 @@ public function test_merge_is_not_also_counted_as_a_deletion(): void { $this->assert_queued( 'conversation.merged', (string) $this->agent->email ); Queue::assertPushed( SendEvent::class, 1 ); - // WordPress.org drops its copy of the merged conversation. $this->assertSame( (int) $second->id, $this->payload()['merged_id'] ); } @@ -275,8 +274,7 @@ public function test_late_replies_carry_their_own_threads(): void { } /** - * Events the filter keeps out of WordPress.org's copy, like an imported conversation's before WordPress.org points at - * it, still go out for stats, without the conversation. + * Events the filter keeps out of the copy still go out for stats, without the conversation. * * @return void */ @@ -329,8 +327,8 @@ public function test_late_events_keep_their_mailbox_for_stats(): void { } /** - * A conversation deleted for good before its event went out is deleted from the copy, with the HelpScout - * conversation it was imported from, which the filter is asked about by ID. + * A conversation deleted for good before its event went out is deleted from the copy, with its HelpScout one, which + * the filter is asked about by ID. * * @return void */ @@ -390,8 +388,7 @@ public function test_events_carry_all_threads_when_asked(): void { } /** - * Threads go as text, with a line for each block, and the addresses of their links, which is what WordPress.org - * looks for plugins and themes in. + * Threads go as text: a line per block, and their links' addresses, where WordPress.org looks for plugins and themes. * * @return void */ @@ -513,7 +510,7 @@ private function payload(): array { $jobs = Queue::pushed( SendEvent::class ); $this->assertCount( 1, $jobs ); - // As it's sent. + // As SendEvent sends it. return EventPayload::refresh( $jobs->first()->payload ); } From 41047b72a024c3dcc00dec05896106b63f4ce1e7 Mon Sep 17 00:00:00 2001 From: Konstantin Obenland Date: Tue, 13 Oct 2026 16:08:06 -0500 Subject: [PATCH 05/10] FreeScout: Leave copies of conversations deleted for good to the cleanup Recording each imported conversation's mailbox only served conversations deleted for good before their mailbox switched, so that the switch could delete their HelpScout copies. Those copies are left to the cleanup of HelpScout copies nothing replaced, which happens before HelpScout closes anyway, so the column, its migration, and the delete hook go. Co-Authored-By: Claude Opus 5.5 --- ..._to_wporghelpscoutimport_conversations.php | 73 ------------------- .../Entities/ImportedConversation.php | 2 +- .../WPOrgHelpScoutImportServiceProvider.php | 12 --- .../Modules/WPOrgHelpScoutImport/README.md | 4 +- .../WPOrgHelpScoutImport/Services/Copies.php | 43 ++--------- .../Services/Importer.php | 1 - .../WPOrgHelpScoutImport/tests/CopiesTest.php | 19 ++--- .../tests/ImporterTest.php | 11 --- 8 files changed, 14 insertions(+), 151 deletions(-) delete mode 100644 freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Database/Migrations/2026_10_10_000000_add_mailbox_to_wporghelpscoutimport_conversations.php diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Database/Migrations/2026_10_10_000000_add_mailbox_to_wporghelpscoutimport_conversations.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Database/Migrations/2026_10_10_000000_add_mailbox_to_wporghelpscoutimport_conversations.php deleted file mode 100644 index b5f3ae7299..0000000000 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Database/Migrations/2026_10_10_000000_add_mailbox_to_wporghelpscoutimport_conversations.php +++ /dev/null @@ -1,73 +0,0 @@ -unsignedInteger( 'mailbox_id' )->nullable()->index(); - } - ); - - // Backfills earlier imports; those deleted for good since are lost. - foreach ( \DB::table( 'conversations' )->distinct()->pluck( 'mailbox_id' ) as $mailbox_id ) { - \DB::table( self::TABLE ) - ->whereIn( - 'conversation_id', - static function ( Builder $query ) use ( $mailbox_id ): void { - $query->select( 'id' )->from( 'conversations' )->where( 'mailbox_id', $mailbox_id ); - } - ) - ->update( array( 'mailbox_id' => (int) $mailbox_id ) ); - } - } - - /** - * Reverts the migration. - * - * @return void - */ - public function down(): void { - if ( Schema::hasTable( self::TABLE ) && Schema::hasColumn( self::TABLE, 'mailbox_id' ) ) { - Schema::table( - self::TABLE, - static function ( Blueprint $table ): void { - $table->dropIndex( array( 'mailbox_id' ) ); - $table->dropColumn( 'mailbox_id' ); - } - ); - } - } -} diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Entities/ImportedConversation.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Entities/ImportedConversation.php index 3d249a007b..b7059ec246 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Entities/ImportedConversation.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Entities/ImportedConversation.php @@ -42,5 +42,5 @@ final class ImportedConversation extends Model { * * @var array */ - protected $fillable = array( 'helpscout_id', 'helpscout_number', 'conversation_id', 'mailbox_id', 'creator_id', 'assignee_id', 'closer_id', 'tags', 'custom_fields', 'written_tags', 'written_values' ); + protected $fillable = array( 'helpscout_id', 'helpscout_number', 'conversation_id', 'creator_id', 'assignee_id', 'closer_id', 'tags', 'custom_fields', 'written_tags', 'written_values' ); } diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Providers/WPOrgHelpScoutImportServiceProvider.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Providers/WPOrgHelpScoutImportServiceProvider.php index d9cc2fb5d8..b64736a214 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Providers/WPOrgHelpScoutImportServiceProvider.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Providers/WPOrgHelpScoutImportServiceProvider.php @@ -176,18 +176,6 @@ static function ( mixed $copy = true, mixed $conversation = null, mixed $convers 4 ); - // Deleting for good loses the conversation's mailbox, which switching needs. - \Eventy::addAction( - 'conversations.before_delete_forever', - static function ( mixed $conversation_ids = array() ): void { - try { - Copies::remember_mailboxes( (array) $conversation_ids ); - } catch ( \Throwable $e ) { - \Log::error( '[WPOrgHelpScoutImport] Could not note the mailboxes of conversations deleted for good: ' . $e->getMessage() ); - } - } - ); - // Highlights Manage while the pages are open. \Eventy::addFilter( 'menu.selected', diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/README.md b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/README.md index ada65fa976..25691a2e31 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/README.md +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/README.md @@ -59,12 +59,12 @@ Copies a HelpScout mailbox's conversations into a FreeScout mailbox, so a team's - **WordPress.org:** WordPress.org keeps a copy of conversations, which the plugin directory reads. It points at HelpScout until a mailbox switches. To switch one, in this order: 1. In HelpScout, stop the mailbox's events going to `api.wordpress.org/dotorg/helpscout/webhook.php`, its webhook under **Manage » Apps**: scope it to the mailboxes still in HelpScout, or turn it off once none is left. While it runs for the mailbox, anything done there writes HelpScout's copy of a conversation back, and counts in HelpScout's stats. It leaves alone the HelpScout conversations whose copies FreeScout's replaced, as each copy names the one it replaced, but not the others. 2. Run the last import of the mailbox. - 3. Together with moving the mailbox's email to FreeScout, press **Point WordPress.org at FreeScout** for it, under **WordPress.org** on the import page. That sends WordPress.org which FreeScout conversation each HelpScout one was imported as, a thousand at a time, on the queue (`api.wordpress.org/dotorg/freescout/replace-copies.php`), without importing anything again. The copies of conversations deleted in FreeScout, for good or not, or marked spam, are deleted instead. If WordPress.org doesn't take a batch, **Carry on** sends the rest; sending a batch again changes nothing. + 3. Together with moving the mailbox's email to FreeScout, press **Point WordPress.org at FreeScout** for it, under **WordPress.org** on the import page. That sends WordPress.org which FreeScout conversation each HelpScout one was imported as, a thousand at a time, on the queue (`api.wordpress.org/dotorg/freescout/replace-copies.php`), without importing anything again. The copies of conversations in FreeScout's Deleted folder, or marked spam, are deleted instead. If WordPress.org doesn't take a batch, **Carry on** sends the rest; sending a batch again changes nothing. 4. After the cutover, import the mailbox again, for what HelpScout got in between. Imports that finish after the switch send WordPress.org what they imported, new or again; one that finishes while a send runs is sent after it. Until a mailbox switches, `WPOrgWebhooks` sends imported conversations' events for stats, but not the conversations, so WordPress.org goes on pointing at HelpScout; after, it sends them, naming the HelpScout conversation each replaces. - HelpScout conversations that were never imported, or never replaced, like those HelpScout deleted, keep HelpScout's copies, which have no `helpdesk` meta. Clean those up in WordPress.org's copy before HelpScout closes. + HelpScout conversations that were never imported, or never replaced, like those HelpScout deleted, or those deleted for good in FreeScout before the switch, keep HelpScout's copies, which have no `helpdesk` meta. Clean those up in WordPress.org's copy before HelpScout closes. - Saved replies, tags, and custom fields: switch on the Saved Replies, Tags, and Custom Fields modules before importing. - Workflows: before importing, give every automatic workflow with **Apply to previous conversations** or a date condition the condition **Imported** is **No**, unless it's meant for HelpScout's history too, and the same to those added later. It goes by the conversation's latest thread, so the workflow applies to a conversation again once agents add a reply, note, or change in FreeScout. Or keep the Workflows module off until after the last import, and add the condition before switching it on. - After the last mailbox has moved, switch the module off, and remove its code in a later deploy. Imported conversations keep HelpScout's numbers. Its tables map HelpScout's conversation IDs to FreeScout's: convert anything that still links to HelpScout before removing it. diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php index c1f080ae98..27b04ccd90 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php @@ -320,26 +320,6 @@ static function () use ( $name, $change ): ?array { ); } - /** - * Notes the mailboxes of imported conversations about to be deleted for good, so a switch still sends their deletion. - * - * @param int[] $conversation_ids Conversation IDs. - * @return void - */ - public static function remember_mailboxes( array $conversation_ids ): void { - foreach ( array_chunk( array_map( 'intval', $conversation_ids ), 500 ) as $ids ) { - $by_mailbox = Conversation::query()->whereIn( 'id', $ids )->get( array( 'id', 'mailbox_id' ) )->groupBy( 'mailbox_id' ); - - foreach ( $by_mailbox as $mailbox_id => $conversations ) { - // toBase() leaves updated_at alone, which marks updates. - ImportedConversation::query() - ->toBase() - ->whereIn( 'conversation_id', $conversations->pluck( 'id' )->all() ) - ->update( array( 'mailbox_id' => (int) $mailbox_id ) ); - } - } - } - /** * Marks an imported conversation as updated, for catch_up(). * @@ -367,8 +347,7 @@ public static function total( int $mailbox_id ): int { * @param int $after_id ID of the last one sent. * @param int $limit How many. * @param string|null $since Only those imported or updated since then. - * @return Collection With helpscout_id, conversation_id, and their conversation's found_id (null if deleted for - * good), number, state, and status. + * @return Collection With helpscout_id, conversation_id, and their conversation's number, state, and status. */ public static function batch( int $mailbox_id, int $after_id, int $limit, ?string $since = null ): Collection { $table = ( new ImportedConversation() )->getTable(); @@ -388,7 +367,6 @@ static function ( Builder $query ) use ( $table, $since ): Builder { $table . '.id', $table . '.helpscout_id', $table . '.conversation_id', - 'conversations.id as found_id', 'conversations.number', 'conversations.state', 'conversations.status', @@ -411,11 +389,11 @@ public static function request( ImportedConversation $imported ): array { 'helpscout_id' => (int) $imported->helpscout_id, 'id' => (int) $imported->conversation_id, 'number' => (int) $imported->number, - 'delete' => ! $imported->found_id || self::gone( (int) $imported->state, (int) $imported->status ), + 'delete' => self::gone( (int) $imported->state, (int) $imported->status ), 'merged' => false, ); - $into = $imported->found_id && Conversation::STATE_DELETED === (int) $imported->state ? self::merged_into( (int) $imported->conversation_id ) : null; + $into = Conversation::STATE_DELETED === (int) $imported->state ? self::merged_into( (int) $imported->conversation_id ) : null; if ( $into ) { $request = array( 'id' => (int) $into->id, @@ -481,7 +459,7 @@ private static function merged_into( int $conversation_id ): ?Conversation { } /** - * The conversations imported into a mailbox: those in it, and those deleted for good from it. + * The imported conversations in a mailbox. * * @param int $mailbox_id FreeScout mailbox ID. * @return Builder @@ -490,17 +468,8 @@ private static function imported( int $mailbox_id ): Builder { $table = ( new ImportedConversation() )->getTable(); return ImportedConversation::query() - ->leftJoin( 'conversations', 'conversations.id', '=', $table . '.conversation_id' ) - ->where( - static function ( Builder $query ) use ( $table, $mailbox_id ): void { - $query->where( 'conversations.mailbox_id', $mailbox_id ) - ->orWhere( - static function ( Builder $gone ) use ( $table, $mailbox_id ): void { - $gone->whereNull( 'conversations.id' )->where( $table . '.mailbox_id', $mailbox_id ); - } - ); - } - ); + ->join( 'conversations', 'conversations.id', '=', $table . '.conversation_id' ) + ->where( 'conversations.mailbox_id', $mailbox_id ); } /** diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Importer.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Importer.php index d78ba7b727..cb4f9a50a2 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Importer.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Importer.php @@ -349,7 +349,6 @@ function () use ( &$conversation, $imported, $source, $mailbox, $sender, $thread $record = array( 'helpscout_number' => (int) ( $source['number'] ?? 0 ), 'conversation_id' => (int) $conversation->id, - 'mailbox_id' => (int) $conversation->mailbox_id, 'creator_id' => 'user' === ( $source['createdBy']['type'] ?? '' ) ? self::person_id( $source['createdBy'] ) : null, 'assignee_id' => self::person_id( $source['assignee'] ?? null ), 'closer_id' => self::person_id( $source['closedByUser'] ?? null ), diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php index 852186c90c..6b2666c18c 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php @@ -243,8 +243,7 @@ public function test_catching_up_after_a_failure_sends_everything_it_had_not(): } /** - * Copies of conversations deleted or spam go, HelpScout's too; others are pointed at. Those deleted for good from - * another mailbox are left to its switch. + * Copies of conversations deleted or spam go, HelpScout's too; others are pointed at. * * @return void */ @@ -253,15 +252,11 @@ public function test_deleted_and_spam_conversations_have_their_copies_deleted(): $kept = $this->import( 1000000025 ); $trashed = $this->import( 1000000026 ); $spam = $this->import( 1000000027 ); - $gone = $this->import( 1000000028 ); - $other = $this->create_conversation( $this->create_mailbox( 'Themes' ), $this->create_sender() ); - $this->import( 1000000029, $other ); $trashed->state = Conversation::STATE_DELETED; $trashed->save(); $spam->status = Conversation::STATUS_SPAM; $spam->save(); - Conversation::deleteConversationsForever( array( (int) $gone->id, (int) $other->id ) ); Copies::start( (int) $this->mailbox->id ); $this->run_batches(); @@ -271,11 +266,9 @@ public function test_deleted_and_spam_conversations_have_their_copies_deleted(): (int) $kept->id => false, (int) $trashed->id => true, (int) $spam->id => true, - (int) $gone->id => true, ), array_column( $sent[0]['copies'], 'delete', 'id' ) ); - $this->assertSame( 1000000028, array_column( $sent[0]['copies'], 'helpscout_id', 'id' )[ (int) $gone->id ] ); } /** @@ -392,20 +385,18 @@ static function ( RequestInterface $request, array $options ) use ( $sent ): Pro } /** - * A conversation imported from HelpScout, into the mailbox unless it's given one. + * A conversation imported from HelpScout into the mailbox. * - * @param int $helpscout_id HelpScout's ID for it. - * @param Conversation|null $conversation The conversation it was imported as; a new one in the mailbox if null. + * @param int $helpscout_id HelpScout's ID for it. * @return Conversation */ - private function import( int $helpscout_id, ?Conversation $conversation = null ): Conversation { - $conversation = $conversation ?? $this->create_conversation( $this->mailbox, $this->create_sender() ); + private function import( int $helpscout_id ): Conversation { + $conversation = $this->create_conversation( $this->mailbox, $this->create_sender() ); ImportedConversation::query()->create( array( 'helpscout_id' => $helpscout_id, 'helpscout_number' => (int) $conversation->number, 'conversation_id' => (int) $conversation->id, - 'mailbox_id' => (int) $conversation->mailbox_id, ) ); diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImporterTest.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImporterTest.php index 579ce45f54..a9cde4a5b3 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImporterTest.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImporterTest.php @@ -433,17 +433,6 @@ public function test_names_the_helpscout_conversation_of_imported_ones(): void { $this->assertSame( (int) $source['id'], \Eventy::filter( 'wporgwebhooks.helpscout_id', 0, $this->imported_conversation() ) ); } - /** - * What was imported keeps the mailbox it went into, for switching once the conversation is deleted for good. - * - * @return void - */ - public function test_keeps_the_mailbox_of_imported_ones(): void { - $source = $this->conversation(); - $this->importer->import( $source, $this->mailbox ); - - $this->assertSame( (int) $this->mailbox->id, (int) ImportedConversation::query()->where( 'helpscout_id', (int) $source['id'] )->value( 'mailbox_id' ) ); - } /** * A HelpScout user without a FreeScout user gets one, which can log in while HelpScout lists them. From f312c892cedbf6547418f7856acf0b3aa95991bb Mon Sep 17 00:00:00 2001 From: Konstantin Obenland Date: Wed, 14 Oct 2026 08:35:57 -0500 Subject: [PATCH 06/10] FreeScout: Write the copy without transactions The copy's tables may sit behind a different connection than a table-less START TRANSACTION, so the transactions are gone. Every write now goes in an order that the event, or batch, sent again after a failure finishes. Taking over a replaced copy's plugins and themes is an UPDATE IGNORE, and the new meta one INSERT IGNORE, as the meta is unique. replace-copies.php reads which copies exist once per batch, and renames a HelpScout copy whenever FreeScout's isn't there yet, merged away or not, so a merge into a conversation without a copy keeps its plugins. Co-Authored-By: Claude Opus 5.5 --- .../public_html/dotorg/freescout/common.php | 132 ++++++--------- .../dotorg/freescout/replace-copies.php | 132 +++++---------- .../freescout/tests/class-copy-plan-test.php | 155 +----------------- .../public_html/dotorg/freescout/webhook.php | 76 ++++----- .../Modules/WPOrgWebhooks/README.md | 2 +- 5 files changed, 128 insertions(+), 369 deletions(-) diff --git a/api.wordpress.org/public_html/dotorg/freescout/common.php b/api.wordpress.org/public_html/dotorg/freescout/common.php index c39574a18a..c73791694f 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/common.php +++ b/api.wordpress.org/public_html/dotorg/freescout/common.php @@ -741,82 +741,69 @@ function needs_all_threads( object $request, ?object $row, bool $has_helpscout ) } /** - * What writing a conversation's copy adds to its meta, and which copies it deletes. - * - * It takes over the replaced copies' meta, names the HelpScout conversations it replaces, and adds what its threads - * mention. HelpScout conversations it names already aren't replaced again. - * - * @param int $id Conversation ID. - * @param array[] $replaced The copies it replaces, see get_replaced_ids(). - * @param array[] $all_meta The meta of it and of the copies it replaces: helpscout_id, meta_key, and meta_value. - * @param array[] $mentioned The plugins' and themes' slugs its threads and subject mention, by type. - * @return array Meta to add, as meta_key and meta_value pairs, under `meta`; IDs of the copies to delete, under `delete`. - */ -function plan_email_write( int $id, array $replaced, array $all_meta, array $mentioned ): array { - $own = array(); - $by_copy = array(); - $replaces = array(); - foreach ( $all_meta as $pair ) { - $key = $pair['meta_key'] . '|' . $pair['meta_value']; - if ( (int) $pair['helpscout_id'] === $id ) { - $own[ $key ] = true; - } else { - $by_copy[ (int) $pair['helpscout_id'] ][ $key ] = $pair; - } + * Moves the plugins and themes of copies to the copy that replaces them. Pairs it has already stay behind, and go with + * their copy. + * + * @param int $id ID of the copy that replaces them. + * @param int[] $ids IDs of the copies it replaces. + * @return void + * + * @throws \RuntimeException If the write failed. + */ +function take_over_meta( int $id, array $ids ): void { + global $wpdb; - if ( REPLACED_HELPSCOUT_META_KEY === $pair['meta_key'] && (int) $pair['helpscout_id'] === $id ) { - $replaces[ (int) $pair['meta_value'] ] = true; - } + if ( ! $ids ) { + return; } - $helpscout = array_values( - array_filter( - $replaced['helpscout'], - static function ( int $helpscout_id ) use ( $replaces ): bool { - return ! isset( $replaces[ $helpscout_id ] ); - } - ) - ); - $delete = array_merge( $replaced['freescout'], $helpscout ); + // phpcs:ignore WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare -- The placeholders are added for each ID. + check_write( $wpdb->query( $wpdb->prepare( 'UPDATE IGNORE %i SET helpscout_id = %d WHERE helpscout_id IN (' . implode( ',', array_fill( 0, count( $ids ), '%d' ) ) . ')', array_merge( array( "{$wpdb->base_prefix}helpscout_meta", $id ), $ids ) ) ) ); +} - $meta = array(); - foreach ( $delete as $copy_id ) { - foreach ( $by_copy[ $copy_id ] ?? array() as $key => $pair ) { - $meta[ $key ] = array( $pair['meta_key'], $pair['meta_value'] ); - } - } +/** + * Adds a FreeScout copy's meta, skipping pairs it has: its helpdesk, the HelpScout conversations it replaces, and the + * plugins and themes it mentions. + * + * @param int $id Conversation ID. + * @param int[] $helpscout_ids HelpScout conversations it replaces. + * @param array[] $mentioned Plugins' and themes' slugs, by type. + * @return void + * + * @throws \RuntimeException If the write failed. + */ +function add_copy_meta( int $id, array $helpscout_ids, array $mentioned ): void { + global $wpdb; - $meta[ HELPDESK_META_KEY . '|freescout' ] = array( HELPDESK_META_KEY, 'freescout' ); - foreach ( $replaced['helpscout'] as $helpscout_id ) { - $meta[ REPLACED_HELPSCOUT_META_KEY . '|' . $helpscout_id ] = array( REPLACED_HELPSCOUT_META_KEY, (string) $helpscout_id ); + $pairs = array( array( HELPDESK_META_KEY, 'freescout' ) ); + foreach ( $helpscout_ids as $helpscout_id ) { + $pairs[] = array( REPLACED_HELPSCOUT_META_KEY, (string) $helpscout_id ); } - foreach ( $mentioned as $type => $slugs ) { foreach ( $slugs as $slug ) { - $meta[ $type . '|' . $slug ] = array( (string) $type, (string) $slug ); + $pairs[] = array( (string) $type, (string) $slug ); } } - return array( - 'meta' => array_values( array_diff_key( $meta, $own ) ), - 'delete' => $delete, - ); + $args = array( "{$wpdb->base_prefix}helpscout_meta" ); + foreach ( $pairs as list( $meta_key, $meta_value ) ) { + array_push( $args, $id, $meta_key, $meta_value ); + } + + // phpcs:ignore WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholders are added for each pair. + check_write( $wpdb->query( $wpdb->prepare( 'INSERT IGNORE INTO %i ( helpscout_id, meta_key, meta_value ) VALUES ' . implode( ', ', array_fill( 0, count( $pairs ), '( %d, %s, %s )' ) ), $args ) ) ); } /** * What switching an imported conversation does to its copy. * - * One merged away before the switch arrives as the conversation it went into, flagged `merged`: its HelpScout copy is - * merged into that one's rather than renamed, as that one's own HelpScout copy may still need renaming. - * - * @param object $copy The imported conversation: helpscout_id, id, number, delete, and merged. + * @param object $copy The imported conversation: helpscout_id, id, number, status, and delete. * @param bool $has_helpscout Whether the copy has the HelpScout conversation. * @param bool $has_freescout Whether it has the FreeScout one. - * @param bool $names_it Whether the FreeScout one's copy names the HelpScout one, as it replaced it already. - * @return string `skip`; `delete` both copies (deleted or spam); `rename` the HelpScout copy to the FreeScout ID; - * `merge` it into the FreeScout copy; or `name` it in that copy. + * @return string `skip`; `delete` both copies (deleted or spam); `rename` the HelpScout copy to the FreeScout ID; or + * `merge` it into the FreeScout copy. */ -function plan_copy_replacement( object $copy, bool $has_helpscout, bool $has_freescout, bool $names_it ): string { +function plan_copy_replacement( object $copy, bool $has_helpscout, bool $has_freescout ): string { $helpscout_id = (int) ( $copy->helpscout_id ?? 0 ); $id = (int) ( $copy->id ?? 0 ); if ( $helpscout_id <= 0 || $id <= 0 || $helpscout_id === $id ) { @@ -827,36 +814,11 @@ function plan_copy_replacement( object $copy, bool $has_helpscout, bool $has_fre return $has_helpscout || $has_freescout ? 'delete' : 'skip'; } - if ( $has_helpscout ) { - return $has_freescout || ! empty( $copy->merged ) ? 'merge' : 'rename'; - } - - // The FreeScout copy exists without the HelpScout one; naming it still keeps HelpScout's webhook out. - return $has_freescout && ! $names_it ? 'name' : 'skip'; -} - -/** - * Runs writes to WordPress.org's copy in a transaction. - * - * @param callable $write Writes; throws if one fails, see check_write(). - * @return void - * - * @throws \Throwable What the writes threw, once they're rolled back. - */ -function in_transaction( callable $write ): void { - global $wpdb; - - check_write( $wpdb->query( 'START TRANSACTION' ) ); - - try { - $write(); - } catch ( \Throwable $e ) { - $wpdb->query( 'ROLLBACK' ); - - throw $e; + if ( ! $has_helpscout ) { + return 'skip'; } - check_write( $wpdb->query( 'COMMIT' ) ); + return $has_freescout ? 'merge' : 'rename'; } /** diff --git a/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php b/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php index 7c0754aed7..21c4246b4a 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php +++ b/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php @@ -13,120 +13,72 @@ /** * Swaps the copies of HelpScout's conversations for those of the FreeScout conversations imported from them; see - * plan_copy_replacement(). Each swap is a transaction, and swapped copies are skipped, so a batch can be sent again. + * plan_copy_replacement(). Writes go in an order that the batch, sent again after one failed, finishes. * - * @param array $copies Each a HelpScout ID, the FreeScout conversation's ID and number, whether its copy goes, and - * whether it was merged into that conversation. - * @return int|null How many were swapped, or deleted; null if one failed, so the batch should be sent again. + * @param array $copies Each a HelpScout ID, the FreeScout conversation's ID and number, and whether its copy goes. + * @return int How many were swapped, or deleted. + * + * @throws \RuntimeException If a write failed. */ -function replace_copies( array $copies ): ?int { +function replace_copies( array $copies ): int { global $wpdb; - $emails_table = "{$wpdb->base_prefix}helpscout"; - $meta_table = "{$wpdb->base_prefix}helpscout_meta"; - $replaced = 0; + $copies = array_map( + static function ( mixed $copy ): object { + return (object) $copy; + }, + $copies + ); + $ids = array_values( array_filter( array_map( 'intval', array_merge( array_column( $copies, 'helpscout_id' ), array_column( $copies, 'id' ) ) ) ) ); + if ( ! $ids ) { + return 0; + } // Read from the primary database, not a lagging replica. if ( is_callable( array( $wpdb, 'send_reads_to_masters' ) ) ) { $wpdb->send_reads_to_masters(); } + $emails_table = "{$wpdb->base_prefix}helpscout"; + // phpcs:ignore WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare -- The placeholders are added for each ID. + $existing = array_fill_keys( array_map( 'intval', $wpdb->get_col( $wpdb->prepare( 'SELECT id FROM %i WHERE id IN (' . implode( ',', array_fill( 0, count( $ids ), '%d' ) ) . ')', array_merge( array( $emails_table ), $ids ) ) ) ), true ); + $replaced = 0; + foreach ( $copies as $copy ) { - $copy = (object) $copy; $helpscout_id = (int) ( $copy->helpscout_id ?? 0 ); $id = (int) ( $copy->id ?? 0 ); - if ( $helpscout_id <= 0 || $id <= 0 || $helpscout_id === $id ) { + $action = plan_copy_replacement( $copy, isset( $existing[ $helpscout_id ] ), isset( $existing[ $id ] ) ); + if ( 'skip' === $action ) { continue; } - try { - $has_helpscout = (bool) $wpdb->get_var( $wpdb->prepare( 'SELECT 1 FROM %i WHERE id = %d', $emails_table, $helpscout_id ) ); - $has_freescout = (bool) $wpdb->get_var( $wpdb->prepare( 'SELECT 1 FROM %i WHERE id = %d', $emails_table, $id ) ); - $names_it = (bool) $wpdb->get_var( $wpdb->prepare( 'SELECT 1 FROM %i WHERE helpscout_id = %d AND meta_key = %s AND meta_value = %s', $meta_table, $id, REPLACED_HELPSCOUT_META_KEY, (string) $helpscout_id ) ); - $action = plan_copy_replacement( $copy, $has_helpscout, $has_freescout, $names_it ); - if ( 'skip' === $action ) { - continue; - } - - in_transaction( - static function () use ( $wpdb, $emails_table, $meta_table, $action, $copy, $helpscout_id, $id, $names_it ): void { - if ( 'delete' === $action ) { - delete_emails( array( $helpscout_id, $id ) ); - - return; - } - - // Mark the FreeScout copy: its helpdesk, and the HelpScout conversation it replaces. - $marks = array( REPLACED_HELPSCOUT_META_KEY => (string) $helpscout_id ); - if ( 'rename' === $action ) { - $marks[ HELPDESK_META_KEY ] = 'freescout'; - } - foreach ( $marks as $meta_key => $meta_value ) { - check_write( - $wpdb->query( - $wpdb->prepare( - 'INSERT INTO %i ( helpscout_id, meta_key, meta_value ) - SELECT %d, %s, %s FROM DUAL - WHERE NOT EXISTS ( SELECT 1 FROM %i WHERE helpscout_id = %d AND meta_key = %s AND meta_value = %s )', - $meta_table, - $id, - $meta_key, - $meta_value, - $meta_table, - $id, - $meta_key, - $meta_value - ) - ) - ); - } - - if ( 'name' === $action ) { - return; - } - - // Take over the HelpScout copy's plugins and themes. - check_write( - $wpdb->query( - $wpdb->prepare( - 'INSERT INTO %i ( helpscout_id, meta_key, meta_value ) - SELECT %d, hs.meta_key, hs.meta_value FROM %i hs - WHERE hs.helpscout_id = %d - AND NOT EXISTS ( SELECT 1 FROM %i fs WHERE fs.helpscout_id = %d AND fs.meta_key = hs.meta_key AND fs.meta_value = hs.meta_value )', - $meta_table, - $id, - $meta_table, - $helpscout_id, - $meta_table, - $id - ) - ) - ); - - if ( 'rename' === $action ) { - check_write( $wpdb->delete( $meta_table, array( 'helpscout_id' => $helpscout_id ) ) ); - check_write( $wpdb->update( $emails_table, array( 'id' => $id, 'number' => (int) ( $copy->number ?? 0 ) ), array( 'id' => $helpscout_id ) ) ); // phpcs:ignore WordPress.Arrays.ArrayDeclarationSpacing.AssociativeArrayFound -- Two columns. - } else { - delete_emails( array( $helpscout_id ) ); - } - } - ); - - ++$replaced; - } catch ( \Throwable $e ) { - trigger_error( 'FreeScout copy of conversation ' . $id . ' failed: ' . $e->getMessage(), E_USER_WARNING ); // phpcs:ignore WordPress.PHP.DevelopmentFunctions, WordPress.Security.EscapeOutput.OutputNotEscaped -- Logged, not shown. + ++$replaced; + if ( 'delete' === $action ) { + delete_emails( array( $helpscout_id, $id ) ); + unset( $existing[ $helpscout_id ], $existing[ $id ] ); + continue; + } - return null; + take_over_meta( $id, array( $helpscout_id ) ); + add_copy_meta( $id, array( $helpscout_id ), array() ); + if ( 'rename' === $action ) { + check_write( $wpdb->update( $emails_table, array( 'id' => $id, 'number' => (int) ( $copy->number ?? 0 ) ), array( 'id' => $helpscout_id ) ) ); // phpcs:ignore WordPress.Arrays.ArrayDeclarationSpacing.AssociativeArrayFound -- Two columns. + $existing[ $id ] = true; } + + delete_emails( array( $helpscout_id ) ); + unset( $existing[ $helpscout_id ] ); } return $replaced; } -$request = get_request( basename( __FILE__ ) ); -$replaced = replace_copies( (array) ( $request->copies ?? array() ) ); +$request = get_request( basename( __FILE__ ) ); -if ( null === $replaced ) { +try { + $replaced = replace_copies( (array) ( $request->copies ?? array() ) ); +} catch ( \Throwable $e ) { + trigger_error( 'FreeScout copies failed: ' . $e->getMessage(), E_USER_WARNING ); // phpcs:ignore WordPress.PHP.DevelopmentFunctions, WordPress.Security.EscapeOutput.OutputNotEscaped -- Logged, not shown. status_header( 503 ); exit; } diff --git a/api.wordpress.org/public_html/dotorg/freescout/tests/class-copy-plan-test.php b/api.wordpress.org/public_html/dotorg/freescout/tests/class-copy-plan-test.php index c3177a6e86..8ecd4879a4 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/tests/class-copy-plan-test.php +++ b/api.wordpress.org/public_html/dotorg/freescout/tests/class-copy-plan-test.php @@ -12,8 +12,7 @@ use PHPUnit\Framework\TestCase; /** - * Tests for get_replaced_ids(), needs_all_threads(), plan_email_write(), plan_copy_replacement(), and - * get_copy_mailbox_slug(). + * Tests for get_replaced_ids(), needs_all_threads(), plan_copy_replacement(), and get_copy_mailbox_slug(). */ class Copy_Plan_Test extends TestCase { @@ -68,102 +67,6 @@ public function test_names_the_copies_an_event_replaces(): void { ); } - /** - * Writing takes over the replaced copies' plugins and themes, names its helpdesk and the HelpScout conversations it - * replaces, adds what it mentions, and deletes the replaced copies; nothing it has already is added again. - * - * @return void - */ - public function test_takes_over_the_copies_it_replaces(): void { - $plan = plan_email_write( - self::ID, - get_replaced_ids( $this->request() ), - array( - $this->meta( self::ID, 'plugins', 'my-plugin' ), - $this->meta( self::MERGED_ID, 'plugins', 'merged-plugin' ), - $this->meta( self::MERGED_ID, HELPDESK_META_KEY, 'freescout' ), - $this->meta( self::HELPSCOUT_ID, 'themes', 'old-theme' ), - $this->meta( self::HELPSCOUT_ID, 'plugins', 'my-plugin' ), - ), - array( 'plugins' => array( 'my-plugin', 'new-plugin' ) ) - ); - - $this->assertEqualsCanonicalizing( - array( - array( 'plugins', 'merged-plugin' ), - array( 'themes', 'old-theme' ), - array( HELPDESK_META_KEY, 'freescout' ), - array( REPLACED_HELPSCOUT_META_KEY, (string) self::HELPSCOUT_ID ), - array( REPLACED_HELPSCOUT_META_KEY, (string) self::MERGED_HELPSCOUT_ID ), - array( 'plugins', 'new-plugin' ), - ), - $plan['meta'] - ); - $this->assertSame( array( self::MERGED_ID, self::HELPSCOUT_ID, self::MERGED_HELPSCOUT_ID ), $plan['delete'] ); - } - - /** - * Once the copy names a HelpScout conversation, later events, and the same event sent again, leave it alone: nothing - * is deleted again, and nothing more is added. - * - * @return void - */ - public function test_leaves_helpscout_conversations_it_replaced_alone(): void { - $request = (object) array( 'helpscout_id' => self::HELPSCOUT_ID ); - - $plan = plan_email_write( - self::ID, - get_replaced_ids( $request ), - array( - $this->meta( self::ID, HELPDESK_META_KEY, 'freescout' ), - $this->meta( self::ID, REPLACED_HELPSCOUT_META_KEY, (string) self::HELPSCOUT_ID ), - ), - array() - ); - - $this->assertSame( - array( - 'meta' => array(), - 'delete' => array(), - ), - $plan - ); - } - - /** - * A copy that names HelpScout conversations keeps naming them as its meta is written again. - * - * @return void - */ - public function test_keeps_naming_what_it_replaced(): void { - $plan = plan_email_write( - self::ID, - get_replaced_ids( (object) array() ), - array( $this->meta( self::ID, REPLACED_HELPSCOUT_META_KEY, (string) self::HELPSCOUT_ID ) ), - array( 'plugins' => array( 'my-plugin' ) ) - ); - - $this->assertSame( array( array( HELPDESK_META_KEY, 'freescout' ), array( 'plugins', 'my-plugin' ) ), $plan['meta'] ); - $this->assertSame( array(), $plan['delete'] ); - } - - /** - * A conversation merged into another hands it the HelpScout conversation it named. - * - * @return void - */ - public function test_merging_hands_over_what_was_replaced(): void { - $plan = plan_email_write( - self::ID, - get_replaced_ids( (object) array( 'merged_id' => self::MERGED_ID ) ), - array( $this->meta( self::MERGED_ID, REPLACED_HELPSCOUT_META_KEY, (string) self::MERGED_HELPSCOUT_ID ) ), - array() - ); - - $this->assertContains( array( REPLACED_HELPSCOUT_META_KEY, (string) self::MERGED_HELPSCOUT_ID ), $plan['meta'] ); - $this->assertSame( array( self::MERGED_ID ), $plan['delete'] ); - } - /** * Only a conversation the copy has nothing of yet needs all of its threads, unless the event carries them. * @@ -194,42 +97,18 @@ public function test_plans_replacing_a_copy(): void { 'delete' => false, ); - $this->assertSame( 'rename', plan_copy_replacement( $copy, true, false, false ) ); - $this->assertSame( 'merge', plan_copy_replacement( $copy, true, true, false ) ); - $this->assertSame( 'name', plan_copy_replacement( $copy, false, true, false ) ); - $this->assertSame( 'skip', plan_copy_replacement( $copy, false, true, true ) ); - $this->assertSame( 'skip', plan_copy_replacement( $copy, false, false, false ) ); + $this->assertSame( 'rename', plan_copy_replacement( $copy, true, false ) ); + $this->assertSame( 'merge', plan_copy_replacement( $copy, true, true ) ); + $this->assertSame( 'skip', plan_copy_replacement( $copy, false, true ) ); + $this->assertSame( 'skip', plan_copy_replacement( $copy, false, false ) ); $copy->delete = true; - $this->assertSame( 'delete', plan_copy_replacement( $copy, true, false, false ) ); - $this->assertSame( 'delete', plan_copy_replacement( $copy, false, true, true ) ); - $this->assertSame( 'skip', plan_copy_replacement( $copy, false, false, false ) ); + $this->assertSame( 'delete', plan_copy_replacement( $copy, true, false ) ); + $this->assertSame( 'delete', plan_copy_replacement( $copy, false, true ) ); + $this->assertSame( 'skip', plan_copy_replacement( $copy, false, false ) ); $copy->id = self::HELPSCOUT_ID; - $this->assertSame( 'skip', plan_copy_replacement( $copy, true, true, false ) ); - } - - /** - * A conversation merged into another before the switch has its HelpScout copy merged into that one's, whether that - * one's copy is there yet or not, so it keeps what the copy mentions; one the copy names already is left alone. - * - * @return void - */ - public function test_plans_merging_a_conversation_merged_away(): void { - $copy = (object) array( - 'helpscout_id' => self::MERGED_HELPSCOUT_ID, - 'id' => self::ID, - 'number' => 1234, - 'delete' => false, - 'merged' => true, - ); - - $this->assertSame( 'merge', plan_copy_replacement( $copy, true, false, false ) ); - $this->assertSame( 'merge', plan_copy_replacement( $copy, true, true, false ) ); - $this->assertSame( 'skip', plan_copy_replacement( $copy, false, true, true ) ); - - $copy->delete = true; - $this->assertSame( 'delete', plan_copy_replacement( $copy, true, true, false ) ); + $this->assertSame( 'skip', plan_copy_replacement( $copy, true, true ) ); } /** @@ -269,20 +148,4 @@ private function request(): object { 'merged_helpscout_id' => self::MERGED_HELPSCOUT_ID, ); } - - /** - * A row of the copy's meta. - * - * @param int $id Conversation ID. - * @param string $key Meta key. - * @param string $value Meta value. - * @return array - */ - private function meta( int $id, string $key, string $value ): array { - return array( - 'helpscout_id' => (string) $id, - 'meta_key' => $key, // phpcs:ignore WordPress.DB.SlowDBQuery -- Not a query. - 'meta_value' => $value, // phpcs:ignore WordPress.DB.SlowDBQuery -- Not a query. - ); - } } diff --git a/api.wordpress.org/public_html/dotorg/freescout/webhook.php b/api.wordpress.org/public_html/dotorg/freescout/webhook.php index f01061bed5..54fb488180 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/webhook.php +++ b/api.wordpress.org/public_html/dotorg/freescout/webhook.php @@ -80,7 +80,7 @@ function contributor_stats( object $request ): void { * @param object $request Request payload. * @return bool Whether it needs all of the conversation's threads, which FreeScout sends it again with. * - * @throws \Throwable If the write failed, and was rolled back. + * @throws \RuntimeException If a write failed; the event sent again finishes it. */ function log_email( object $request ): bool { global $wpdb; @@ -99,11 +99,7 @@ function log_email( object $request ): bool { // Spam and deleted conversations aren't kept; the current state wins over the event, as a deletion may be undone. $deleted = isset( $email->state ) ? 'deleted' === $email->state : 'conversation.deleted' === ( $request->event ?? '' ); if ( $deleted || 'spam' === ( $email->status ?? '' ) ) { - in_transaction( - static function () use ( $request, $id ): void { - delete_emails( array_merge( array( $id ), ...array_values( get_replaced_ids( $request ) ) ) ); - } - ); + delete_emails( array_merge( array( $id ), ...array_values( get_replaced_ids( $request ) ) ) ); return false; } @@ -117,13 +113,14 @@ static function () use ( $request, $id ): void { * @param object $request Request payload. * @param int $id Conversation ID. * @return bool Whether it needs all of the conversation's threads; see needs_all_threads(). + * + * @throws \RuntimeException If a write failed. */ function write_email( object $request, int $id ): bool { global $wpdb; $email = $request->email; $emails_table = "{$wpdb->base_prefix}helpscout"; - $meta_table = "{$wpdb->base_prefix}helpscout_meta"; $replaced = get_replaced_ids( $request ); $helpscout_id = (int) ( $request->helpscout_id ?? 0 ); @@ -133,15 +130,6 @@ function write_email( object $request, int $id ): bool { return true; } - $ids = array_merge( array( $id ), ...array_values( $replaced ) ); - $all_meta = $wpdb->get_results( - $wpdb->prepare( - 'SELECT helpscout_id, meta_key, meta_value FROM %i WHERE helpscout_id IN (' . implode( ',', array_fill( 0, count( $ids ), '%d' ) ) . ')', // phpcs:ignore WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare -- The placeholders are added for each ID. - array_merge( array( $meta_table ), $ids ) - ), - ARRAY_A - ); - // The subject's mentions were added with the copy; the lookup costs queries, so only redo it when it changed. $mailbox = $request->email->mailbox ?? ( $request->mailbox ?? null ); $subject = (string) ( $email->subject ?? '' ); @@ -157,8 +145,6 @@ function write_email( object $request, int $id ): bool { ) ); - $plan = plan_email_write( $id, $replaced, (array) $all_meta, $mentioned ); - $user_id = (int) ( $row->user_id ?? 0 ); if ( ! $user_id ) { $user = get_sender_user( (object) array( 'sender' => $email->sender ?? null ) ); @@ -168,38 +154,34 @@ function write_email( object $request, int $id ): bool { $data = get_email_row( $request, $row, $user_id ); $columns = array_keys( $data ); - in_transaction( - static function () use ( $wpdb, $emails_table, $meta_table, $id, $data, $columns, $plan ): void { - // Pairs are unique; one an overlapping event added is skipped. - foreach ( $plan['meta'] as list( $meta_key, $meta_value ) ) { - check_write( $wpdb->query( $wpdb->prepare( 'INSERT IGNORE INTO %i ( helpscout_id, meta_key, meta_value ) VALUES ( %d, %s, %s )', $meta_table, $id, $meta_key, $meta_value ) ) ); - } - - // phpcs:disable WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholders are added for each column; one statement, so two events for a new conversation at once don't both insert it. - check_write( - $wpdb->query( - $wpdb->prepare( - 'INSERT INTO %i (' . implode( ', ', array_fill( 0, count( $columns ), '%i' ) ) . ') VALUES (' . implode( ', ', array_fill( 0, count( $data ), '%s' ) ) . ') ON DUPLICATE KEY UPDATE ' . implode( ', ', array_fill( 0, count( $columns ) - 1, '%i = VALUES( %i )' ) ), - array_merge( - array( $emails_table ), - $columns, - array_values( $data ), - // Every column but the ID, twice: name, and name again for VALUES(). - ...array_map( - static function ( string $column ): array { - return array( $column, $column ); - }, - array_values( array_diff( $columns, array( 'id' ) ) ) - ) - ) + // In an order that the event, sent again after a failed write, finishes. + $replaced_ids = array_merge( ...array_values( $replaced ) ); + take_over_meta( $id, $replaced_ids ); + add_copy_meta( $id, $replaced['helpscout'], $mentioned ); + + // phpcs:disable WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholders are added for each column; one statement, so two events for a new conversation at once don't both insert it. + check_write( + $wpdb->query( + $wpdb->prepare( + 'INSERT INTO %i (' . implode( ', ', array_fill( 0, count( $columns ), '%i' ) ) . ') VALUES (' . implode( ', ', array_fill( 0, count( $data ), '%s' ) ) . ') ON DUPLICATE KEY UPDATE ' . implode( ', ', array_fill( 0, count( $columns ) - 1, '%i = VALUES( %i )' ) ), + array_merge( + array( $emails_table ), + $columns, + array_values( $data ), + // Every column but the ID, twice: name, and name again for VALUES(). + ...array_map( + static function ( string $column ): array { + return array( $column, $column ); + }, + array_values( array_diff( $columns, array( 'id' ) ) ) ) ) - ); - // phpcs:enable - - delete_emails( $plan['delete'] ); - } + ) + ) ); + // phpcs:enable + + delete_emails( $replaced_ids ); return false; } diff --git a/freescout.wordpress.net/Modules/WPOrgWebhooks/README.md b/freescout.wordpress.net/Modules/WPOrgWebhooks/README.md index 73c85690e6..f70a094bcd 100644 --- a/freescout.wordpress.net/Modules/WPOrgWebhooks/README.md +++ b/freescout.wordpress.net/Modules/WPOrgWebhooks/README.md @@ -25,7 +25,7 @@ Each event also carries the conversation as WordPress.org keeps it, read when th For the plugins and themes they mention, a reply also carries the threads from the previous reply up to it (itself, and the notes before it), a note itself, and a conversation new to the copy, restored, taken out of spam, or moved to another mailbox, its newest 50 threads; other events carry none, as the copy keeps what earlier ones mentioned. When WordPress.org has no copy of a conversation yet, it takes nothing from an event without all of its threads, and asks for them: the event goes again at once with them, and only that one counts. Threads go as text, with their links' addresses, up to 256 KB an event, cutting the oldest short; an event api.wordpress.org's web server still refuses for its size goes again without them. -A merge carries the ID of the conversation merged away, and every event of an imported conversation the HelpScout ID whose copy it replaces, as does a merge for the conversation merged away (from the `wporgwebhooks.helpscout_id` filter, which `WPOrgHelpScoutImport` answers, also for conversations deleted for good since). The copy names the HelpScout conversations it replaced, which keeps HelpScout's webhook from writing them back; later events don't replace them again. Whether an event carries the conversation at all is up to the `wporgwebhooks.copy` filter, which `WPOrgHelpScoutImport` answers for imported conversations. That copy is the `wporg_helpscout` table HelpScout's webhook filled, which the plugin directory lists a plugin's emails from, and finds review conversations in; deleted conversations, and spam, are taken out of it. +A merge carries the ID of the conversation merged away, and every event of an imported conversation the HelpScout ID whose copy it replaces, as does a merge for the conversation merged away (from the `wporgwebhooks.helpscout_id` filter, which `WPOrgHelpScoutImport` answers, also for conversations deleted for good since). The copy names the HelpScout conversations it replaced, which keeps HelpScout's webhook from writing them back. Whether an event carries the conversation at all is up to the `wporgwebhooks.copy` filter, which `WPOrgHelpScoutImport` answers for imported conversations. That copy is the `wporg_helpscout` table HelpScout's webhook filled, which the plugin directory lists a plugin's emails from, and finds review conversations in; deleted conversations, and spam, are taken out of it. Events go out from FreeScout's queue, and are retried if WordPress.org can't be reached. From af4b179198e733a995d951efb0cd6e335929ac8e Mon Sep 17 00:00:00 2001 From: Konstantin Obenland Date: Wed, 14 Oct 2026 08:44:52 -0500 Subject: [PATCH 07/10] FreeScout: Send the switch again instead of resuming it Pointing WordPress.org at a mailbox's imported conversations is safe to repeat, so the switch no longer tracks ranges, queues imports behind a running send, or resumes one. Each send goes through every imported conversation, 250 at a time; pressing Send again, or an import finishing after the switch, starts a new one, which stops the one before. The import page still shows how many were sent, when it's done, and why WordPress.org refused a batch. Imported conversations are no longer announced, as nothing here needs it anymore. replace-copies.php isn't told about merges either, as it treats a conversation merged away like any other. Co-Authored-By: Claude Opus 5.5 --- .../Http/Controllers/ImportController.php | 6 +- .../WPOrgHelpScoutImport/Jobs/ImportPage.php | 2 +- .../Jobs/ReplaceCopies.php | 47 ++-- .../WPOrgHelpScoutImportServiceProvider.php | 14 - .../Modules/WPOrgHelpScoutImport/README.md | 6 +- .../Resources/views/index.blade.php | 31 ++- .../WPOrgHelpScoutImport/Services/Copies.php | 255 +++--------------- .../Services/Importer.php | 19 -- .../WPOrgHelpScoutImport/tests/CopiesTest.php | 183 ++++--------- .../tests/ImportControllerTest.php | 28 ++ .../tests/ImporterTest.php | 26 -- 11 files changed, 170 insertions(+), 447 deletions(-) diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Http/Controllers/ImportController.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Http/Controllers/ImportController.php index bf0248c095..6669cb47cf 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Http/Controllers/ImportController.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Http/Controllers/ImportController.php @@ -89,7 +89,7 @@ public function index(): View { } /** - * Switches a mailbox's conversations in WordPress.org's copy to FreeScout, or resumes a failed switch. + * Switches a mailbox's conversations in WordPress.org's copy to FreeScout, or sends them again. * * @param Request $request Request. * @return RedirectResponse @@ -104,9 +104,7 @@ public function point( Request $request ): RedirectResponse { return self::back_with_error( __( 'Pointing WordPress.org at FreeScout needs the WP.org Webhooks module, set up with WPORG_API_SECRET.' ) ); } - if ( ! Copies::start( (int) $mailbox->id ) ) { - return self::back_with_error( __( 'WordPress.org is already being pointed at :name.', array( 'name' => $mailbox->name ) ) ); - } + Copies::start( (int) $mailbox->id ); return redirect() ->route( 'wporghelpscoutimport.index' ) diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ImportPage.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ImportPage.php index 8e67c956f2..c9fb4c7e16 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ImportPage.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ImportPage.php @@ -263,7 +263,7 @@ static function () use ( $attempts ): bool { $run->finished_at = Carbon::now(); if ( $this->save( $run ) ) { // Send what this run imported, if the mailbox switched. - Copies::catch_up( (int) $run->mailbox_id, (string) $run->started_at ); + Copies::send_again( (int) $run->mailbox_id ); } return; diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ReplaceCopies.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ReplaceCopies.php index 1c082c9a10..4146f57e57 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ReplaceCopies.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Jobs/ReplaceCopies.php @@ -9,6 +9,7 @@ namespace Modules\WPOrgHelpScoutImport\Jobs; +use Carbon\Carbon; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; @@ -26,11 +27,11 @@ final class ReplaceCopies implements ShouldQueue { use Queueable; /** - * How many conversations go in a request. + * How many conversations go in a request; each takes WordPress.org a handful of queries. * * @var int */ - public const BATCH = 1000; + public const BATCH = 250; /** * The endpoint on api.wordpress.org. @@ -60,36 +61,50 @@ final class ReplaceCopies implements ShouldQueue { */ public $mailbox_id; + /** + * Token of the send it's part of; a later send stops it. + * + * @var string + */ + public $run; + + /** + * ID of the last imported conversation sent. + * + * @var int + */ + public $after_id; + /** * Constructor. * - * @param int $mailbox_id FreeScout mailbox ID. + * @param int $mailbox_id FreeScout mailbox ID. + * @param string $run Token of the send it's part of. + * @param int $after_id ID of the last imported conversation sent. */ - public function __construct( int $mailbox_id ) { + public function __construct( int $mailbox_id, string $run, int $after_id = 0 ) { $this->mailbox_id = $mailbox_id; + $this->run = $run; + $this->after_id = $after_id; } /** * Sends the next batch, and queues the one after. * - * Failures don't throw: the worker would retry anything thrown, and the switch would be left running. + * Failures don't throw, but are noted for the import page, which can send everything again. * * @return void */ public function handle(): void { $state = Copies::get( $this->mailbox_id ); - if ( ! $state || Copies::STATUS_RUNNING !== $state['status'] ) { + if ( ! $state || ( $state['run'] ?? '' ) !== $this->run ) { return; } try { - $batch = Copies::batch( $this->mailbox_id, (int) $state['after_id'], self::BATCH, $state['since'] ?? null ); - + $batch = Copies::batch( $this->mailbox_id, $this->after_id, self::BATCH ); if ( $batch->isEmpty() ) { - // finish() starts the send that imports queued meanwhile, if any. - if ( Copies::finish( $this->mailbox_id ) ) { - self::dispatch( $this->mailbox_id ); - } + Copies::note( $this->mailbox_id, $this->run, array( 'finished_at' => Carbon::now()->toDateTimeString() ) ); return; } @@ -106,13 +121,13 @@ public function handle(): void { } \Log::error( '[WPOrgHelpScoutImport] Could not point WordPress.org at mailbox ' . $this->mailbox_id . '\'s conversations: ' . $e->getMessage() ); - Copies::fail( $this->mailbox_id, $e->getMessage() ); + Copies::note( $this->mailbox_id, $this->run, array( 'error' => $e->getMessage() ) ); return; } - Copies::advance( $this->mailbox_id, (int) $batch->last()->id, $batch->count() ); - - self::dispatch( $this->mailbox_id ); + if ( Copies::note( $this->mailbox_id, $this->run, array( 'sent' => (int) $state['sent'] + $batch->count() ) ) ) { + self::dispatch( $this->mailbox_id, $this->run, (int) $batch->last()->id ); + } } } diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Providers/WPOrgHelpScoutImportServiceProvider.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Providers/WPOrgHelpScoutImportServiceProvider.php index b64736a214..7c489e25b5 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Providers/WPOrgHelpScoutImportServiceProvider.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Providers/WPOrgHelpScoutImportServiceProvider.php @@ -143,20 +143,6 @@ static function ( mixed $helpscout_id = 0, mixed $conversation = null, mixed $co 3 ); - // Marks updated conversations for Copies::catch_up(). - \Eventy::addAction( - 'wporghelpscoutimport.conversation_imported', - static function ( mixed $conversation = null ): void { - try { - if ( $conversation instanceof Conversation ) { - Copies::touch( $conversation ); - } - } catch ( \Throwable $e ) { - \Log::error( '[WPOrgHelpScoutImport] Could not mark conversation ' . $conversation->id . ' as imported again: ' . $e->getMessage() ); - } - } - ); - // Imported conversations' events leave the copy alone until their mailbox switches. \Eventy::addFilter( 'wporgwebhooks.copy', diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/README.md b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/README.md index 25691a2e31..388dc9b8a3 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/README.md +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/README.md @@ -39,7 +39,7 @@ Copies a HelpScout mailbox's conversations into a FreeScout mailbox, so a team's - Someone with a FreeScout user under another email gets a second one unless they're chosen on the Users page first: FreeScout can't merge users. Choosing someone after an import credits what's imported for them to the user chosen, keeping HelpScout's dates; with the Reports module on, it computes those conversations' metrics again. - What HelpScout did itself, without a user, is credited to "HelpScout Import", a disabled robot user. The Reports module lists it among the users it reports on, for what it's credited with. - **Teams:** conversations assigned to a HelpScout team are assigned to the FreeScout team chosen on the Users page, or else to the one with the same name. FreeScout's teams come from its Teams module: create them there first. Without either, they're imported unassigned, and choosing a team later, or the next import finding one with the same name, assigns them. The Users page flags teams without access to the mailboxes their conversations are in. -- **Nothing reacts to it, except scheduled workflows:** conversations are marked as imported, and written without the events new email fires. Nothing is sent, nobody is notified, workflows aren't triggered, and `WPOrgAkismet` leaves them alone. New senders aren't announced either, like to the API & Webhooks module's webhooks. Each conversation imported or updated fires `wporghelpscoutimport.conversation_imported`, with the conversation and its HelpScout ID. +- **Nothing reacts to it, except scheduled workflows:** conversations are marked as imported, and written without the events new email fires. Nothing is sent, nobody is notified, workflows aren't triggered, and `WPOrgAkismet` leaves them alone. New senders aren't announced either, like to the API & Webhooks module's webhooks. - The Workflows module also runs automatic workflows every hour, over the conversations they apply to: all of them with **Apply to previous conversations**, and with a date condition, like how long it's been waiting since the last reply, every conversation created after the workflow. Imported conversations keep HelpScout's dates, so those workflows act on them as on any other: they can email senders, close conversations, or delete them for good. The import can't stop them. See Setup. - The same goes for conversations still being worked on in HelpScout, while both run. - **HelpScout's rate limit:** the whole HelpScout account shares one limit, and wppluginsteam.org and the reviewers' tools use it too. The import leaves 100 requests a minute to them, and waits for the next minute when only those are left. A conversation too big to read in one minute's share is read again on its next try, waiting for the limit as it goes, but only while no outgoing email is waiting in FreeScout's queue. @@ -59,8 +59,8 @@ Copies a HelpScout mailbox's conversations into a FreeScout mailbox, so a team's - **WordPress.org:** WordPress.org keeps a copy of conversations, which the plugin directory reads. It points at HelpScout until a mailbox switches. To switch one, in this order: 1. In HelpScout, stop the mailbox's events going to `api.wordpress.org/dotorg/helpscout/webhook.php`, its webhook under **Manage » Apps**: scope it to the mailboxes still in HelpScout, or turn it off once none is left. While it runs for the mailbox, anything done there writes HelpScout's copy of a conversation back, and counts in HelpScout's stats. It leaves alone the HelpScout conversations whose copies FreeScout's replaced, as each copy names the one it replaced, but not the others. 2. Run the last import of the mailbox. - 3. Together with moving the mailbox's email to FreeScout, press **Point WordPress.org at FreeScout** for it, under **WordPress.org** on the import page. That sends WordPress.org which FreeScout conversation each HelpScout one was imported as, a thousand at a time, on the queue (`api.wordpress.org/dotorg/freescout/replace-copies.php`), without importing anything again. The copies of conversations in FreeScout's Deleted folder, or marked spam, are deleted instead. If WordPress.org doesn't take a batch, **Carry on** sends the rest; sending a batch again changes nothing. - 4. After the cutover, import the mailbox again, for what HelpScout got in between. Imports that finish after the switch send WordPress.org what they imported, new or again; one that finishes while a send runs is sent after it. + 3. Together with moving the mailbox's email to FreeScout, press **Point WordPress.org at FreeScout** for it, under **WordPress.org** on the import page. That sends WordPress.org which FreeScout conversation each HelpScout one was imported as, 250 at a time, on the queue (`api.wordpress.org/dotorg/freescout/replace-copies.php`), without importing anything again. The copies of conversations in FreeScout's Deleted folder, or marked spam, are deleted instead. The import page shows how many were sent, and when it's done. If WordPress.org doesn't take a batch, it says why, and **Send again** sends them all again; what was sent already doesn't change. + 4. After the cutover, import the mailbox again, for what HelpScout got in between. Every import that finishes after the switch sends all of the mailbox's imported conversations again. Until a mailbox switches, `WPOrgWebhooks` sends imported conversations' events for stats, but not the conversations, so WordPress.org goes on pointing at HelpScout; after, it sends them, naming the HelpScout conversation each replaces. diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Resources/views/index.blade.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Resources/views/index.blade.php index bb23105dc0..ade6fe9903 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Resources/views/index.blade.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Resources/views/index.blade.php @@ -216,7 +216,7 @@ @if ( count( $copies ) )

{{ __('WordPress.org') }}

-

{{ __('WordPress.org keeps a copy of conversations, which the plugin directory links to, and replies to uploads in. It points at HelpScout until a mailbox switches to FreeScout: once HelpScout’s webhook leaves the mailbox alone, and after its last import, point WordPress.org at the conversations imported from HelpScout, together with moving the mailbox’s email. Until then, what happens to imported conversations in FreeScout isn’t sent to WordPress.org; after, it is, and later imports are pointed at as they finish. See the module’s README for every step.') }}

+

{{ __('WordPress.org keeps a copy of conversations, which the plugin directory links to, and replies to uploads in. It points at HelpScout until a mailbox switches to FreeScout: once HelpScout’s webhook leaves the mailbox alone, and after its last import, point WordPress.org at the conversations imported from HelpScout, together with moving the mailbox’s email. Until then, what happens to imported conversations in FreeScout isn’t sent to WordPress.org; after, it is, and every import that finishes sends them all again. Sending them again changes nothing. See the module’s README for every step.') }}

@@ -235,23 +235,26 @@ @endforeach diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php index 27b04ccd90..139041853b 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php @@ -16,6 +16,7 @@ use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\QueryException; +use Illuminate\Support\Str; use Modules\WPOrgHelpScoutImport\Entities\ImportedConversation; use Modules\WPOrgHelpScoutImport\Jobs\ReplaceCopies; use Modules\WPOrgWebhooks\Services\Client; @@ -34,33 +35,12 @@ final class Copies { */ private const OPTION = 'wporghelpscoutimport_copies_'; - /** - * Status of a switch being sent. - * - * @var string - */ - public const STATUS_RUNNING = 'running'; - - /** - * Status of a switch sent in full. - * - * @var string - */ - public const STATUS_DONE = 'done'; - - /** - * Status of a switch WordPress.org refused; start() resumes it. - * - * @var string - */ - public const STATUS_FAILED = 'failed'; - /** * A mailbox's switch. * * @param int $mailbox_id FreeScout mailbox ID. - * @return array|null Null if it hasn't switched. Keys: status, after_id (last sent), sent, since (null for all), - * started_at, finished_at, error, and pending (the since of a send queued behind this one). + * @return array|null Null if it hasn't switched. Keys: switched_at; and of the latest send, run (its token), sent, + * finished_at (null until it's done), and error (why WordPress.org refused a batch, which ends it). */ public static function get( int $mailbox_id ): ?array { // Bypasses the cache, which other processes' saves don't update. @@ -69,17 +49,6 @@ public static function get( int $mailbox_id ): ?array { return is_array( $state ) ? $state : null; } - /** - * Saves a mailbox's switch. - * - * @param int $mailbox_id FreeScout mailbox ID. - * @param array $state Its state, see get(). - * @return void - */ - public static function save( int $mailbox_id, array $state ): void { - Option::set( self::OPTION . $mailbox_id, $state ); - } - /** * Whether requests to WordPress.org can be signed, which WPOrgWebhooks does. * @@ -90,82 +59,41 @@ public static function available(): bool { } /** - * Starts sending a mailbox's imported conversations to WordPress.org, or resumes a failed send. + * Switches a mailbox, and sends all of its imported conversations to WordPress.org; again if it switched already, + * which stops the send before. * * @param int $mailbox_id FreeScout mailbox ID. - * @return bool False if it's already running. + * @return void */ - public static function start( int $mailbox_id ): bool { - $started = self::change( + public static function start( int $mailbox_id ): void { + $run = Str::random( 16 ); + self::change( $mailbox_id, - static function ( ?array $state ): ?array { - if ( $state && self::STATUS_RUNNING === $state['status'] ) { - return null; - } - + static function ( ?array $state ) use ( $run ): array { return array( - 'status' => self::STATUS_RUNNING, - 'after_id' => (int) ( $state['after_id'] ?? 0 ), - 'sent' => (int) ( $state['sent'] ?? 0 ), - 'since' => $state['since'] ?? null, - 'started_at' => (string) ( $state['started_at'] ?? Carbon::now()->toDateTimeString() ), + 'switched_at' => (string) ( $state['switched_at'] ?? Carbon::now()->toDateTimeString() ), + 'run' => $run, + 'sent' => 0, 'finished_at' => null, 'error' => '', - ) + array_intersect_key( (array) $state, array( 'pending' => true ) ); + ); } ); - if ( $started ) { - ReplaceCopies::dispatch( $mailbox_id ); - } - - return (bool) $started; + ReplaceCopies::dispatch( $mailbox_id, $run ); } /** - * Sends what an import added or updated, if the mailbox switched. Never throws: it runs as an import finishes. + * Sends a switched mailbox's imported conversations again, for what an import added or updated. Never throws: it + * runs as an import finishes. * - * Updated conversations are sent again, as HelpScout's webhook may have rewritten their copies. A running send gets - * them queued behind it; a failed one restarts with them. - * - * @param int $mailbox_id FreeScout mailbox ID. - * @param string $since When the import started. + * @param int $mailbox_id FreeScout mailbox ID. * @return void */ - public static function catch_up( int $mailbox_id, string $since ): void { + public static function send_again( int $mailbox_id ): void { try { - if ( ! self::get( $mailbox_id ) || ! self::available() ) { - return; - } - - $restarted = false; - self::change( - $mailbox_id, - static function ( ?array $state ) use ( $since, &$restarted ): ?array { - if ( ! $state ) { - return null; - } - - if ( self::STATUS_RUNNING === $state['status'] ) { - $state['pending'] = array_key_exists( 'pending', $state ) ? self::wider( $state['pending'], $since ) : $since; - - return $state; - } - - // A failed send still owes its own range; a finished one doesn't. - $wider = self::STATUS_FAILED === $state['status'] ? self::wider( $state['since'] ?? null, $since ) : $since; - if ( array_key_exists( 'pending', $state ) ) { - $wider = self::wider( $state['pending'], $wider ); - } - - $restarted = true; - - return self::restart( $state, $wider ); - } - ); - - if ( $restarted ) { - ReplaceCopies::dispatch( $mailbox_id ); + if ( self::get( $mailbox_id ) && self::available() ) { + self::start( $mailbox_id ); } } catch ( \Throwable $e ) { \Log::error( '[WPOrgHelpScoutImport] Could not point WordPress.org at mailbox ' . $mailbox_id . '\'s new conversations: ' . $e->getMessage() ); @@ -173,117 +101,24 @@ static function ( ?array $state ) use ( $since, &$restarted ): ?array { } /** - * Notes how far a send got. - * - * @param int $mailbox_id FreeScout mailbox ID. - * @param int $after_id The last imported conversation sent. - * @param int $count How many the batch sent. - * @return void - */ - public static function advance( int $mailbox_id, int $after_id, int $count ): void { - self::change( - $mailbox_id, - static function ( ?array $state ) use ( $after_id, $count ): ?array { - if ( ! $state || self::STATUS_RUNNING !== $state['status'] ) { - return null; - } - - $state['after_id'] = $after_id; - $state['sent'] = (int) $state['sent'] + $count; - - return $state; - } - ); - } - - /** - * Ends a finished send, or starts the one queued behind it. - * - * @param int $mailbox_id FreeScout mailbox ID. - * @return bool Whether a queued send started. - */ - public static function finish( int $mailbox_id ): bool { - $state = self::change( - $mailbox_id, - static function ( ?array $state ): ?array { - if ( ! $state || self::STATUS_RUNNING !== $state['status'] ) { - return null; - } - - if ( array_key_exists( 'pending', $state ) ) { - return self::restart( $state, $state['pending'] ); - } - - $state['status'] = self::STATUS_DONE; - $state['finished_at'] = Carbon::now()->toDateTimeString(); - - return $state; - } - ); - - return $state && self::STATUS_RUNNING === $state['status']; - } - - /** - * Marks a send WordPress.org refused as failed; start() resumes it. + * Notes how far a send got, or that it's done, or failed. * * @param int $mailbox_id FreeScout mailbox ID. - * @param string $error What went wrong. - * @return void + * @param string $run The send's token. + * @param array $changes Keys of the switch to change: sent, finished_at, or error. + * @return bool False if a later send took over. */ - public static function fail( int $mailbox_id, string $error ): void { - self::change( + public static function note( int $mailbox_id, string $run, array $changes ): bool { + return null !== self::change( $mailbox_id, - static function ( ?array $state ) use ( $error ): ?array { - if ( ! $state || self::STATUS_RUNNING !== $state['status'] ) { - return null; - } - - $state['status'] = self::STATUS_FAILED; - $state['error'] = $error; - - return $state; + static function ( ?array $state ) use ( $run, $changes ): ?array { + return $state && ( $state['run'] ?? '' ) === $run ? $changes + $state : null; } ); } /** - * A send started again from the first imported conversation. - * - * @param array $state The switch. - * @param string|null $since Only conversations imported or updated since then; null for all. - * @return array - */ - private static function restart( array $state, ?string $since ): array { - unset( $state['pending'] ); - - return array( - 'status' => self::STATUS_RUNNING, - 'after_id' => 0, - 'sent' => 0, - 'since' => $since, - 'finished_at' => null, - 'error' => '', - ) + $state; - } - - /** - * The wider of two since values: the earlier time, or null for all. - * - * @param string|null $one A time; null for all. - * @param string|null $other Another. - * @return string|null - */ - private static function wider( ?string $one, ?string $other ): ?string { - if ( null === $one || null === $other ) { - return null; - } - - return Carbon::parse( $one )->lessThan( Carbon::parse( $other ) ) ? $one : $other; - } - - /** - * Changes a mailbox's switch under a row lock, so jobs, imports, and the button don't undo each other. + * Changes a mailbox's switch under a row lock, so a send and the next one's start don't undo each other. * * @param int $mailbox_id FreeScout mailbox ID. * @param callable $change Takes the switch, or null, and returns it changed, or null to leave it. @@ -320,16 +155,6 @@ static function () use ( $name, $change ): ?array { ); } - /** - * Marks an imported conversation as updated, for catch_up(). - * - * @param Conversation $conversation Conversation. - * @return void - */ - public static function touch( Conversation $conversation ): void { - ImportedConversation::query()->where( 'conversation_id', $conversation->id )->update( array( 'updated_at' => Carbon::now() ) ); - } - /** * How many conversations were imported into a mailbox. * @@ -343,23 +168,16 @@ public static function total( int $mailbox_id ): int { /** * The next conversations imported into a mailbox to send, in the order they were imported. * - * @param int $mailbox_id FreeScout mailbox ID. - * @param int $after_id ID of the last one sent. - * @param int $limit How many. - * @param string|null $since Only those imported or updated since then. + * @param int $mailbox_id FreeScout mailbox ID. + * @param int $after_id ID of the last one sent. + * @param int $limit How many. * @return Collection With helpscout_id, conversation_id, and their conversation's number, state, and status. */ - public static function batch( int $mailbox_id, int $after_id, int $limit, ?string $since = null ): Collection { + public static function batch( int $mailbox_id, int $after_id, int $limit ): Collection { $table = ( new ImportedConversation() )->getTable(); return self::imported( $mailbox_id ) ->where( $table . '.id', '>', $after_id ) - ->when( - null !== $since, - static function ( Builder $query ) use ( $table, $since ): Builder { - return $query->where( $table . '.updated_at', '>=', $since ); - } - ) ->orderBy( $table . '.id' ) ->limit( $limit ) ->get( @@ -381,8 +199,7 @@ static function ( Builder $query ) use ( $table, $since ): Builder { * One merged away is sent as the conversation it went into, which takes over its copy's plugins and themes. * * @param ImportedConversation $imported An imported conversation, as batch() reads it. - * @return array HelpScout's ID, FreeScout's ID and number, whether to delete the copy, and whether it's merged into - * the conversation that ID names. + * @return array HelpScout's ID, FreeScout's ID and number, and whether to delete the copy. */ public static function request( ImportedConversation $imported ): array { $request = array( @@ -390,7 +207,6 @@ public static function request( ImportedConversation $imported ): array { 'id' => (int) $imported->conversation_id, 'number' => (int) $imported->number, 'delete' => self::gone( (int) $imported->state, (int) $imported->status ), - 'merged' => false, ); $into = Conversation::STATE_DELETED === (int) $imported->state ? self::merged_into( (int) $imported->conversation_id ) : null; @@ -399,7 +215,6 @@ public static function request( ImportedConversation $imported ): array { 'id' => (int) $into->id, 'number' => (int) $into->number, 'delete' => self::gone( (int) $into->state, (int) $into->status ), - 'merged' => true, ) + $request; } diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Importer.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Importer.php index cb4f9a50a2..739e8ca947 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Importer.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Importer.php @@ -271,7 +271,6 @@ public function import( array $source, Mailbox $mailbox ): string { $conversation->save(); $conversation->timestamps = true; self::update_counters( $conversation->mailbox ); - $this->announce( $conversation, $helpscout_id ); return self::UPDATED; } @@ -393,8 +392,6 @@ function () use ( &$conversation, $imported, $source, $mailbox, $sender, $thread self::update_counters( $other_mailbox ); } - $this->announce( $conversation, $helpscout_id ); - return $imported ? self::UPDATED : self::IMPORTED; } @@ -417,22 +414,6 @@ public static function update_counters( Mailbox $mailbox ): void { } } - /** - * Tells other modules a conversation was imported or updated, which happens without core's events. Never throws: the - * conversation is imported by now. - * - * @param Conversation $conversation Conversation. - * @param int $helpscout_id HelpScout's ID for it. - * @return void - */ - private function announce( Conversation $conversation, int $helpscout_id ): void { - try { - \Eventy::action( 'wporghelpscoutimport.conversation_imported', $conversation, $helpscout_id ); - } catch ( \Throwable $e ) { - \Log::error( '[WPOrgHelpScoutImport] Could not announce conversation ' . $conversation->id . ': ' . $e->getMessage() ); - } - } - /** * Closes, and so deletes, the temporary files a conversation's attachments and images were downloaded into. * diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php index 6b2666c18c..3f6a3e5a6c 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php @@ -87,7 +87,7 @@ public function test_imported_conversations_are_copied_once_their_mailbox_switch /** * Each batch is sent, signed, to api.wordpress.org, which the local environment's mock answers, and the next is - * queued until none are left. + * queued after it until none are left, counting what was sent. * * @return void */ @@ -101,143 +101,71 @@ public function test_sends_batches_until_done(): void { $this->import( 1000000028 ); $this->import( 1000000029 ); Copies::start( (int) $this->mailbox->id ); + $run = $this->send_token(); + $last = (int) ImportedConversation::query()->max( 'id' ); - ( new ReplaceCopies( (int) $this->mailbox->id ) )->handle(); + ( new ReplaceCopies( (int) $this->mailbox->id, $run ) )->handle(); - $state = Copies::get( (int) $this->mailbox->id ); - $this->assertSame( array( Copies::STATUS_RUNNING, 2 ), array( $state['status'], $state['sent'] ) ); - Queue::assertPushed( ReplaceCopies::class, 2 ); + Queue::assertPushed( + ReplaceCopies::class, + static function ( ReplaceCopies $job ) use ( $run, $last ): bool { + return $run === $job->run && $last === $job->after_id; + } + ); + $this->assertSame( array( 2, null ), array( $this->state()['sent'], $this->state()['finished_at'] ) ); - ( new ReplaceCopies( (int) $this->mailbox->id ) )->handle(); + ( new ReplaceCopies( (int) $this->mailbox->id, $run, $last ) )->handle(); - $this->assertSame( Copies::STATUS_DONE, Copies::get( (int) $this->mailbox->id )['status'] ); + Queue::assertPushed( ReplaceCopies::class, 2 ); + $this->assertNotNull( $this->state()['finished_at'] ); } /** - * A batch WordPress.org refuses on its last try fails the switch, which can resume. + * A batch WordPress.org refuses on its last try ends the send, noting why until it's sent again. * * @return void */ - public function test_failed_batches_stop_the_switch_until_carried_on(): void { + public function test_failed_batches_are_noted_until_sent_again(): void { $this->import( 1000000028 ); Copies::start( (int) $this->mailbox->id ); $queue_job = $this->createMock( DatabaseJob::class ); $queue_job->method( 'attempts' )->willReturn( 3 ); - $job = new ReplaceCopies( (int) $this->mailbox->id ); + $job = new ReplaceCopies( (int) $this->mailbox->id, $this->send_token() ); $job->setJob( $queue_job ); $job->handle(); - $state = Copies::get( (int) $this->mailbox->id ); - $this->assertSame( array( Copies::STATUS_FAILED, 0 ), array( $state['status'], $state['after_id'] ) ); - $this->assertNotSame( '', $state['error'] ); - - $this->assertTrue( Copies::start( (int) $this->mailbox->id ) ); - $this->assertFalse( Copies::start( (int) $this->mailbox->id ) ); - } - - /** - * Imports finishing after the switch send what they added or updated; before it, nothing is sent. - * - * @return void - */ - public function test_catches_up_on_imports_after_the_switch(): void { - $this->import( 1000000027 ); - $again = $this->import( 1000000028 ); - ImportedConversation::query()->update( array( 'updated_at' => '2026-10-01 09:00:00' ) ); - - Copies::catch_up( (int) $this->mailbox->id, '2026-10-02 10:00:00' ); - Queue::assertNotPushed( ReplaceCopies::class ); - - Copies::save( - (int) $this->mailbox->id, - array( - 'status' => Copies::STATUS_DONE, - 'after_id' => 99, - 'sent' => 2, - 'since' => null, - 'started_at' => '2026-10-01 10:00:00', - 'finished_at' => '2026-10-01 10:05:00', - 'error' => '', - ) - ); - $since = (string) now(); - \Eventy::action( 'wporghelpscoutimport.conversation_imported', $again, 1000000028 ); - $new = $this->import( 1000000029 ); - - Copies::catch_up( (int) $this->mailbox->id, $since ); - + $this->assertNotSame( '', $this->state()['error'] ); Queue::assertPushed( ReplaceCopies::class, 1 ); - $state = Copies::get( (int) $this->mailbox->id ); - $this->assertSame( array( Copies::STATUS_RUNNING, 0, $since ), array( $state['status'], $state['after_id'], $state['since'] ) ); - $this->assertSame( - array( (int) $again->id, (int) $new->id ), - Copies::batch( (int) $this->mailbox->id, 0, 10, $since )->pluck( 'conversation_id' )->map( 'intval' )->all() - ); - } - /** - * Starting while it's running starts nothing more, as two chains of batches would send everything twice. - * - * @return void - */ - public function test_starts_once(): void { - $this->assertTrue( Copies::start( (int) $this->mailbox->id ) ); - $this->assertFalse( Copies::start( (int) $this->mailbox->id ) ); - - Queue::assertPushed( ReplaceCopies::class, 1 ); - } - - /** - * An import finishing during a send queues its conversations behind it, from the start; a second one widens that. - * - * @return void - */ - public function test_catching_up_while_sending_sends_after_it(): void { - $sent = $this->answer(); - $first = $this->import( 1000000027 ); - $again = $this->import( 1000000028 ); - ImportedConversation::query()->update( array( 'updated_at' => '2026-10-01 09:00:00' ) ); Copies::start( (int) $this->mailbox->id ); - Copies::catch_up( (int) $this->mailbox->id, '2026-10-03 10:00:00' ); - Copies::catch_up( (int) $this->mailbox->id, '2026-10-02 10:00:00' ); - - Queue::assertPushed( ReplaceCopies::class, 1 ); - $this->assertSame( '2026-10-02 10:00:00', Copies::get( (int) $this->mailbox->id )['pending'] ); - - // Updated after the send passed it. - $this->run_batches(); - ImportedConversation::query()->where( 'conversation_id', $again->id )->update( array( 'updated_at' => '2026-10-02 11:00:00' ) ); - $this->run_batches(); - $this->run_batches(); - $this->run_batches(); - - $this->assertCount( 2, $sent ); - $this->assertSame( array( (int) $first->id, (int) $again->id ), array_column( $sent[0]['copies'], 'id' ) ); - $this->assertSame( array( (int) $again->id ), array_column( $sent[1]['copies'], 'id' ) ); - - $state = Copies::get( (int) $this->mailbox->id ); - $this->assertSame( array( Copies::STATUS_DONE, '2026-10-02 10:00:00' ), array( $state['status'], $state['since'] ) ); - $this->assertArrayNotHasKey( 'pending', $state ); + $this->assertSame( '', $this->state()['error'] ); + Queue::assertPushed( ReplaceCopies::class, 2 ); } /** - * Catching up on a failed send restarts it with the wider range, so nothing it hadn't sent is left out. + * Imports finishing after the switch send everything again, which stops the send before and keeps the switch's + * date; before it, nothing is sent. * * @return void */ - public function test_catching_up_after_a_failure_sends_everything_it_had_not(): void { - $this->answer(); + public function test_imports_after_the_switch_send_everything_again(): void { + config( array( 'wporgwebhooks.secret' => 'test-secret' ) ); $this->import( 1000000028 ); - foreach ( array( null, '2026-10-01 10:00:00' ) as $failed_since ) { - $this->save_state( Copies::STATUS_FAILED, $failed_since ); - Copies::catch_up( (int) $this->mailbox->id, '2026-10-02 10:00:00' ); + Copies::send_again( (int) $this->mailbox->id ); + Queue::assertNotPushed( ReplaceCopies::class ); - $state = Copies::get( (int) $this->mailbox->id ); - $this->assertSame( array( Copies::STATUS_RUNNING, 0, 0, $failed_since ), array( $state['status'], $state['after_id'], $state['sent'], $state['since'] ) ); - } + Copies::start( (int) $this->mailbox->id ); + $before = $this->state(); + Copies::send_again( (int) $this->mailbox->id ); + + $this->assertNotSame( $before['run'], $this->send_token() ); + $this->assertSame( $before['switched_at'], $this->state()['switched_at'] ); + Queue::assertPushed( ReplaceCopies::class, 2 ); + + ( new ReplaceCopies( (int) $this->mailbox->id, $before['run'] ) )->handle(); Queue::assertPushed( ReplaceCopies::class, 2 ); } @@ -302,13 +230,12 @@ public function test_conversations_merged_away_are_sent_as_the_one_they_went_int 'id' => (int) $kept->id, 'number' => (int) $kept->number, 'delete' => false, - 'merged' => true, ), - array_intersect_key( $copies[ $helpscout_id ], array_flip( array( 'id', 'number', 'delete', 'merged' ) ) ) + array_intersect_key( $copies[ $helpscout_id ], array_flip( array( 'id', 'number', 'delete' ) ) ) ); } - $this->assertSame( array( (int) $kept->id, false, false ), array( $copies[1000000025]['id'], $copies[1000000025]['delete'], $copies[1000000025]['merged'] ) ); - $this->assertSame( array( (int) $trashed->id, true, true ), array( $copies[1000000029]['id'], $copies[1000000029]['delete'], $copies[1000000029]['merged'] ) ); + $this->assertSame( array( (int) $kept->id, false ), array( $copies[1000000025]['id'], $copies[1000000025]['delete'] ) ); + $this->assertSame( array( (int) $trashed->id, true ), array( $copies[1000000029]['id'], $copies[1000000029]['delete'] ) ); } /** @@ -325,7 +252,7 @@ public function test_copy_filter_knows_conversations_deleted_since(): void { $this->assertFalse( \Eventy::filter( 'wporgwebhooks.copy', true, null, $id, (int) $this->mailbox->id ) ); $this->assertSame( 1000000028, \Eventy::filter( 'wporgwebhooks.helpscout_id', 0, null, $id ) ); - $this->save_state( Copies::STATUS_DONE, null ); + Copies::start( (int) $this->mailbox->id ); $this->assertTrue( \Eventy::filter( 'wporgwebhooks.copy', true, null, $id, (int) $this->mailbox->id ) ); } @@ -335,29 +262,25 @@ public function test_copy_filter_knows_conversations_deleted_since(): void { * @return void */ private function run_batches(): void { - ( new ReplaceCopies( (int) $this->mailbox->id ) )->handle(); + ( new ReplaceCopies( (int) $this->mailbox->id, $this->send_token() ) )->handle(); } /** - * Saves the mailbox's switch. + * The mailbox's switch. * - * @param string $status Status. - * @param string|null $since Only conversations imported since then; null for all. - * @return void + * @return array */ - private function save_state( string $status, ?string $since ): void { - Copies::save( - (int) $this->mailbox->id, - array( - 'status' => $status, - 'after_id' => 5, - 'sent' => 5, - 'since' => $since, - 'started_at' => '2026-10-01 10:00:00', - 'finished_at' => null, - 'error' => Copies::STATUS_FAILED === $status ? 'Request to replace-copies.php got a 500 response.' : '', - ) - ); + private function state(): array { + return (array) Copies::get( (int) $this->mailbox->id ); + } + + /** + * The token of the mailbox's latest send. + * + * @return string + */ + private function send_token(): string { + return (string) ( $this->state()['run'] ?? '' ); } /** diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImportControllerTest.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImportControllerTest.php index 66943527c5..00157ee1fc 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImportControllerTest.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImportControllerTest.php @@ -16,6 +16,7 @@ use Modules\WPOrgHelpScoutImport\Entities\Agent; use Modules\WPOrgHelpScoutImport\Entities\Run; use Modules\WPOrgHelpScoutImport\Jobs\ImportPage; +use Modules\WPOrgHelpScoutImport\Services\Copies; require_once __DIR__ . '/ImportTestCase.php'; @@ -301,6 +302,33 @@ public function test_cancel_frees_the_mailbox(): void { $this->assertSame( 2, Run::query()->count() ); } + /** + * The page shows how far pointing WordPress.org at a mailbox's imported conversations got, and why it stopped. + * + * @return void + */ + public function test_shows_how_far_wordpress_org_was_pointed(): void { + $this->post( route( 'wporghelpscoutimport.start' ), $this->start_form() ); + $mailbox_id = (int) Run::query()->firstOrFail()->mailbox_id; + Copies::start( $mailbox_id ); + + $this->assertStringContainsString( '0 of 0 sent so far', $this->get( route( 'wporghelpscoutimport.index' ) )->getContent() ); + + Copies::note( + $mailbox_id, + (string) Copies::get( $mailbox_id )['run'], + array( + 'sent' => 3, + 'finished_at' => '2026-10-02 10:00:00', + 'error' => 'WordPress.org answered 503.', + ) + ); + $page = $this->get( route( 'wporghelpscoutimport.index' ) )->getContent(); + + $this->assertStringContainsString( '3 sent, done', $page ); + $this->assertStringContainsString( 'WordPress.org answered 503.', $page ); + } + /** * A run that hasn't saved progress for a while is shown stalled, and can be resumed. * diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImporterTest.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImporterTest.php index a9cde4a5b3..51064dad44 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImporterTest.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/ImporterTest.php @@ -395,31 +395,6 @@ public function test_conversation_staying_in_its_mailbox_updates_its_counters(): $this->assertSame( 0, (int) $themes->folders()->where( 'type', Folder::TYPE_UNASSIGNED )->value( 'active_count' ) ); } - /** - * Each conversation imported, or updated, is announced with its HelpScout ID, as that happens without core's events. - * - * @return void - */ - public function test_imported_conversations_are_announced(): void { - $announced = array(); - \Eventy::addAction( - 'wporghelpscoutimport.conversation_imported', - static function ( $conversation, $helpscout_id ) use ( &$announced ): void { - $announced[] = array( (int) $conversation->id, $helpscout_id ); - }, - 20, - 2 - ); - $source = $this->conversation(); - - $this->importer->import( $source, $this->mailbox ); - $source['status'] = 'spam'; - $this->importer->import( $source, $this->mailbox ); - - $id = (int) $this->imported_conversation()->id; - $this->assertSame( array( array( $id, (int) $source['id'] ), array( $id, (int) $source['id'] ) ), $announced ); - } - /** * An imported conversation's events name its HelpScout conversation, for WordPress.org's copy. * @@ -433,7 +408,6 @@ public function test_names_the_helpscout_conversation_of_imported_ones(): void { $this->assertSame( (int) $source['id'], \Eventy::filter( 'wporgwebhooks.helpscout_id', 0, $this->imported_conversation() ) ); } - /** * A HelpScout user without a FreeScout user gets one, which can log in while HelpScout lists them. * From 46a70644ac6cd5104e0588e4f62d03fa0f057222 Mon Sep 17 00:00:00 2001 From: Konstantin Obenland Date: Wed, 14 Oct 2026 08:46:09 -0500 Subject: [PATCH 08/10] FreeScout: Keep a renamed copy's status, and spot long subjects unchanged Renaming a HelpScout copy now writes FreeScout's status too: HelpScout's webhook stops for the mailbox before the last import, so the status it wrote last can be out of date, and the plugin directory reads it. The copy's subject column keeps 255 characters, so a longer subject never matched it, and every event looked its plugins up again. Co-Authored-By: Claude Opus 5.5 --- .../public_html/dotorg/freescout/replace-copies.php | 11 +++++++++-- .../public_html/dotorg/freescout/webhook.php | 7 +++++-- .../Modules/WPOrgHelpScoutImport/Services/Copies.php | 5 ++++- .../Modules/WPOrgHelpScoutImport/tests/CopiesTest.php | 3 ++- .../Modules/WPOrgWebhooks/Services/EventPayload.php | 2 +- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php b/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php index 21c4246b4a..43b613adb8 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php +++ b/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php @@ -15,7 +15,8 @@ * Swaps the copies of HelpScout's conversations for those of the FreeScout conversations imported from them; see * plan_copy_replacement(). Writes go in an order that the batch, sent again after one failed, finishes. * - * @param array $copies Each a HelpScout ID, the FreeScout conversation's ID and number, and whether its copy goes. + * @param array $copies Each a HelpScout ID, the FreeScout conversation's ID, number, and status, and whether its copy + * goes. * @return int How many were swapped, or deleted. * * @throws \RuntimeException If a write failed. @@ -62,7 +63,13 @@ static function ( mixed $copy ): object { take_over_meta( $id, array( $helpscout_id ) ); add_copy_meta( $id, array( $helpscout_id ), array() ); if ( 'rename' === $action ) { - check_write( $wpdb->update( $emails_table, array( 'id' => $id, 'number' => (int) ( $copy->number ?? 0 ) ), array( 'id' => $helpscout_id ) ) ); // phpcs:ignore WordPress.Arrays.ArrayDeclarationSpacing.AssociativeArrayFound -- Two columns. + // What HelpScout's webhook last wrote may be out of date, as it no longer runs for the mailbox. + $columns = array( + 'id' => $id, + 'number' => (int) ( $copy->number ?? 0 ), + 'status' => (string) ( $copy->status ?? '' ), + ); + check_write( $wpdb->update( $emails_table, $columns, array( 'id' => $helpscout_id ) ) ); $existing[ $id ] = true; } diff --git a/api.wordpress.org/public_html/dotorg/freescout/webhook.php b/api.wordpress.org/public_html/dotorg/freescout/webhook.php index 54fb488180..3134e9bcb9 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/webhook.php +++ b/api.wordpress.org/public_html/dotorg/freescout/webhook.php @@ -130,11 +130,14 @@ function write_email( object $request, int $id ): bool { return true; } - // The subject's mentions were added with the copy; the lookup costs queries, so only redo it when it changed. + /* + * The subject's mentions were added with the copy; the lookup costs queries, so only redo it when it changed. The + * column keeps 255 characters. + */ $mailbox = $request->email->mailbox ?? ( $request->mailbox ?? null ); $subject = (string) ( $email->subject ?? '' ); $threads = (array) ( $email->threads ?? array() ); - $seen = $row && $subject === (string) $row->subject && get_copy_mailbox_slug( $request ) === (string) $row->mailbox; + $seen = $row && mb_substr( $subject, 0, 255 ) === (string) $row->subject && get_copy_mailbox_slug( $request ) === (string) $row->mailbox; $mentioned = $seen && ! $threads ? array() : get_existing_slugs( get_plugin_or_theme_from_email( (object) array( diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php index 139041853b..a6fffb120f 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/Services/Copies.php @@ -20,6 +20,7 @@ use Modules\WPOrgHelpScoutImport\Entities\ImportedConversation; use Modules\WPOrgHelpScoutImport\Jobs\ReplaceCopies; use Modules\WPOrgWebhooks\Services\Client; +use Modules\WPOrgWebhooks\Services\EventPayload; /** * Switches a mailbox's conversations in WordPress.org's copy from HelpScout to FreeScout, in batches on the queue. @@ -199,13 +200,14 @@ public static function batch( int $mailbox_id, int $after_id, int $limit ): Coll * One merged away is sent as the conversation it went into, which takes over its copy's plugins and themes. * * @param ImportedConversation $imported An imported conversation, as batch() reads it. - * @return array HelpScout's ID, FreeScout's ID and number, and whether to delete the copy. + * @return array HelpScout's ID, FreeScout's ID, number, and status, and whether to delete the copy. */ public static function request( ImportedConversation $imported ): array { $request = array( 'helpscout_id' => (int) $imported->helpscout_id, 'id' => (int) $imported->conversation_id, 'number' => (int) $imported->number, + 'status' => EventPayload::STATUSES[ (int) $imported->status ] ?? 'active', 'delete' => self::gone( (int) $imported->state, (int) $imported->status ), ); @@ -214,6 +216,7 @@ public static function request( ImportedConversation $imported ): array { $request = array( 'id' => (int) $into->id, 'number' => (int) $into->number, + 'status' => EventPayload::STATUSES[ (int) $into->status ] ?? 'active', 'delete' => self::gone( (int) $into->state, (int) $into->status ), ) + $request; } diff --git a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php index 3f6a3e5a6c..c13c3b2587 100644 --- a/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php +++ b/freescout.wordpress.net/Modules/WPOrgHelpScoutImport/tests/CopiesTest.php @@ -229,9 +229,10 @@ public function test_conversations_merged_away_are_sent_as_the_one_they_went_int array( 'id' => (int) $kept->id, 'number' => (int) $kept->number, + 'status' => 'active', 'delete' => false, ), - array_intersect_key( $copies[ $helpscout_id ], array_flip( array( 'id', 'number', 'delete' ) ) ) + array_intersect_key( $copies[ $helpscout_id ], array_flip( array( 'id', 'number', 'status', 'delete' ) ) ) ); } $this->assertSame( array( (int) $kept->id, false ), array( $copies[1000000025]['id'], $copies[1000000025]['delete'] ) ); diff --git a/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/EventPayload.php b/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/EventPayload.php index 464fcffc92..5d765e2006 100644 --- a/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/EventPayload.php +++ b/freescout.wordpress.net/Modules/WPOrgWebhooks/Services/EventPayload.php @@ -73,7 +73,7 @@ final class EventPayload { * * @var string[] */ - private const STATUSES = array( + public const STATUSES = array( Conversation::STATUS_ACTIVE => 'active', Conversation::STATUS_PENDING => 'pending', Conversation::STATUS_CLOSED => 'closed', From 958a3c854a4814044dede3b54a0f6695e141ab70 Mon Sep 17 00:00:00 2001 From: Konstantin Obenland Date: Wed, 14 Oct 2026 10:50:51 -0500 Subject: [PATCH 09/10] FreeScout: Keep the copy's code in its own file Everything that writes WordPress.org's copy of conversations, which only webhook.php and replace-copies.php use, moves out of common.php and webhook.php into copies.php, so it can go in one piece once nothing reads the copy. The two meta keys, each used once, are inlined. Co-Authored-By: Claude Opus 5.5 --- .../public_html/dotorg/freescout/common.php | 213 ---------- .../public_html/dotorg/freescout/copies.php | 369 ++++++++++++++++++ .../dotorg/freescout/replace-copies.php | 1 + .../dotorg/freescout/tests/bootstrap.php | 1 + .../public_html/dotorg/freescout/webhook.php | 160 +------- 5 files changed, 372 insertions(+), 372 deletions(-) create mode 100644 api.wordpress.org/public_html/dotorg/freescout/copies.php diff --git a/api.wordpress.org/public_html/dotorg/freescout/common.php b/api.wordpress.org/public_html/dotorg/freescout/common.php index c73791694f..c6f452476c 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/common.php +++ b/api.wordpress.org/public_html/dotorg/freescout/common.php @@ -48,20 +48,6 @@ function load_wordpress( string $wp_init_host = '' ): void { */ const MAX_CLOCK_SKEW = 10; -/** - * Meta key marking a copy as FreeScout's; HelpScout's copies have none. - * - * @var string - */ -const HELPDESK_META_KEY = 'helpdesk'; - -/** - * Meta key naming the HelpScout conversation a FreeScout copy replaced, which HelpScout's webhook then leaves alone. - * - * @var string - */ -const REPLACED_HELPSCOUT_META_KEY = 'helpscout_conversation'; - /** * Retrieves the incoming payload, and verifies FreeScout signed it for this endpoint. * @@ -190,41 +176,6 @@ function send_panel( array $blocks, array $extra = array() ): never { exit; } -/** - * A conversation's row in WordPress.org's copy, whose columns and table name are HelpScout's. - * - * @param object $request Webhook payload, with the conversation in `email`; see WPOrgWebhooks' EventPayload. - * @param object|null $row Its current row, if any. - * @param int $user_id The sender's WordPress.org user ID, 0 if unknown. - * @return array - */ -function get_email_row( object $request, ?object $row, int $user_id ): array { - $email = $request->email ?? new \stdClass(); - $sender = $email->sender ?? new \stdClass(); - $address = (string) ( $sender->email ?? '' ); - $name = trim( ( $sender->first_name ?? '' ) . ' ' . ( $sender->last_name ?? '' ) ); - - $created = (int) strtotime( (string) ( $email->created_at ?? '' ) ); - $created = $created ? $created : (int) strtotime( (string) ( $row->created ?? '' ) ); - $closed = (int) strtotime( (string) ( $email->closed_at ?? '' ) ); - $updated = (int) strtotime( (string) ( $email->user_updated_at ?? '' ) ); - $times = array_filter( array( $created, $updated, $closed ) ); - - return array( - 'id' => (int) ( $request->conversation->id ?? 0 ), - 'number' => (int) ( $email->number ?? 0 ), - 'user_id' => $user_id, - 'mailbox' => get_copy_mailbox_slug( $request ), - 'status' => (string) ( $email->status ?? '' ), - 'email' => $address ? ( $name ? "{$name} <{$address}>" : $address ) : (string) ( $row->email ?? '' ), - 'subject' => (string) ( $email->subject ?? ( $row->subject ?? '' ) ), - 'preview' => (string) ( $email->preview ?? ( $row->preview ?? '' ) ), - 'created' => gmdate( 'Y-m-d H:i:s', $created ? $created : time() ), - 'closed' => $closed ? gmdate( 'Y-m-d H:i:s', $closed ) : '', - 'modified' => gmdate( 'Y-m-d H:i:s', $times ? max( $times ) : time() ), - ); -} - /** * A status badge; WPOrgSidebar's stylesheet colors it by tone. * @@ -263,16 +214,6 @@ function get_mailbox_slug( object $request ): string { return sanitize_title( (string) ( $request->mailbox->name ?? '' ) ); } -/** - * The slug of the conversation's current mailbox, which the copy keeps, rather than the event's, which stats count. - * - * @param object $request Webhook payload. - * @return string - */ -function get_copy_mailbox_slug( object $request ): string { - return sanitize_title( (string) ( $request->email->mailbox->name ?? ( $request->mailbox->name ?? '' ) ) ); -} - /** * Gets the plain text of a thread body. * @@ -682,157 +623,3 @@ function get_plugin_slugs_by_title( string $title ): array { return wp_list_pluck( $plugins, 'post_name' ); } - -/** - * Deletes conversations from WordPress.org's copy, with their meta. - * - * @param int[] $ids Conversation IDs; HelpScout's for the copies of its conversations. - * @return void - * - * @throws \RuntimeException If a deletion failed. - */ -function delete_emails( array $ids ): void { - global $wpdb; - - if ( ! $ids ) { - return; - } - - // phpcs:disable WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare -- The placeholders are added for each ID. - check_write( $wpdb->query( $wpdb->prepare( 'DELETE FROM %i WHERE id IN (' . implode( ',', array_fill( 0, count( $ids ), '%d' ) ) . ')', array_merge( array( "{$wpdb->base_prefix}helpscout" ), $ids ) ) ) ); - check_write( $wpdb->query( $wpdb->prepare( 'DELETE FROM %i WHERE helpscout_id IN (' . implode( ',', array_fill( 0, count( $ids ), '%d' ) ) . ')', array_merge( array( "{$wpdb->base_prefix}helpscout_meta" ), $ids ) ) ) ); - // phpcs:enable -} - -/** - * The copies an event's conversation replaces: one merged into it, and the HelpScout conversations it, or the one - * merged into it, were imported from. - * - * @param object $request Webhook payload. - * @return array[] FreeScout's IDs, under `freescout`, and HelpScout's, under `helpscout`. - */ -function get_replaced_ids( object $request ): array { - return array( - 'freescout' => array_values( array_filter( array( (int) ( $request->merged_id ?? 0 ) ) ) ), - 'helpscout' => array_values( - array_unique( - array_filter( - array( - (int) ( $request->helpscout_id ?? 0 ), - (int) ( $request->merged_helpscout_id ?? 0 ), - ) - ) - ) - ), - ); -} - -/** - * Whether copying a conversation needs all of its threads: the copy has neither it nor its HelpScout conversation, and - * the event carries only the newest. - * - * @param object $request Webhook payload. - * @param object|null $row The conversation's row in the copy, if it has one. - * @param bool $has_helpscout Whether the copy has the HelpScout conversation it was imported from. - * @return bool - */ -function needs_all_threads( object $request, ?object $row, bool $has_helpscout ): bool { - return ! $row && ! $has_helpscout && empty( $request->email->all_threads ); -} - -/** - * Moves the plugins and themes of copies to the copy that replaces them. Pairs it has already stay behind, and go with - * their copy. - * - * @param int $id ID of the copy that replaces them. - * @param int[] $ids IDs of the copies it replaces. - * @return void - * - * @throws \RuntimeException If the write failed. - */ -function take_over_meta( int $id, array $ids ): void { - global $wpdb; - - if ( ! $ids ) { - return; - } - - // phpcs:ignore WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare -- The placeholders are added for each ID. - check_write( $wpdb->query( $wpdb->prepare( 'UPDATE IGNORE %i SET helpscout_id = %d WHERE helpscout_id IN (' . implode( ',', array_fill( 0, count( $ids ), '%d' ) ) . ')', array_merge( array( "{$wpdb->base_prefix}helpscout_meta", $id ), $ids ) ) ) ); -} - -/** - * Adds a FreeScout copy's meta, skipping pairs it has: its helpdesk, the HelpScout conversations it replaces, and the - * plugins and themes it mentions. - * - * @param int $id Conversation ID. - * @param int[] $helpscout_ids HelpScout conversations it replaces. - * @param array[] $mentioned Plugins' and themes' slugs, by type. - * @return void - * - * @throws \RuntimeException If the write failed. - */ -function add_copy_meta( int $id, array $helpscout_ids, array $mentioned ): void { - global $wpdb; - - $pairs = array( array( HELPDESK_META_KEY, 'freescout' ) ); - foreach ( $helpscout_ids as $helpscout_id ) { - $pairs[] = array( REPLACED_HELPSCOUT_META_KEY, (string) $helpscout_id ); - } - foreach ( $mentioned as $type => $slugs ) { - foreach ( $slugs as $slug ) { - $pairs[] = array( (string) $type, (string) $slug ); - } - } - - $args = array( "{$wpdb->base_prefix}helpscout_meta" ); - foreach ( $pairs as list( $meta_key, $meta_value ) ) { - array_push( $args, $id, $meta_key, $meta_value ); - } - - // phpcs:ignore WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholders are added for each pair. - check_write( $wpdb->query( $wpdb->prepare( 'INSERT IGNORE INTO %i ( helpscout_id, meta_key, meta_value ) VALUES ' . implode( ', ', array_fill( 0, count( $pairs ), '( %d, %s, %s )' ) ), $args ) ) ); -} - -/** - * What switching an imported conversation does to its copy. - * - * @param object $copy The imported conversation: helpscout_id, id, number, status, and delete. - * @param bool $has_helpscout Whether the copy has the HelpScout conversation. - * @param bool $has_freescout Whether it has the FreeScout one. - * @return string `skip`; `delete` both copies (deleted or spam); `rename` the HelpScout copy to the FreeScout ID; or - * `merge` it into the FreeScout copy. - */ -function plan_copy_replacement( object $copy, bool $has_helpscout, bool $has_freescout ): string { - $helpscout_id = (int) ( $copy->helpscout_id ?? 0 ); - $id = (int) ( $copy->id ?? 0 ); - if ( $helpscout_id <= 0 || $id <= 0 || $helpscout_id === $id ) { - return 'skip'; - } - - if ( ! empty( $copy->delete ) ) { - return $has_helpscout || $has_freescout ? 'delete' : 'skip'; - } - - if ( ! $has_helpscout ) { - return 'skip'; - } - - return $has_freescout ? 'merge' : 'rename'; -} - -/** - * Throws if a write failed: wpdb returns false instead. - * - * @param mixed $result What wpdb returned. - * @return void - * - * @throws \RuntimeException If it failed. - */ -function check_write( mixed $result ): void { - global $wpdb; - - if ( false === $result ) { - throw new \RuntimeException( 'Could not write the copy: ' . (string) $wpdb->last_error ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- Logged, not shown. - } -} diff --git a/api.wordpress.org/public_html/dotorg/freescout/copies.php b/api.wordpress.org/public_html/dotorg/freescout/copies.php new file mode 100644 index 0000000000..56bad9fe52 --- /dev/null +++ b/api.wordpress.org/public_html/dotorg/freescout/copies.php @@ -0,0 +1,369 @@ +conversation->id ?? 0 ); + $email = $request->email ?? null; + if ( ! $id || ! is_object( $email ) ) { + return false; + } + + // Read from the primary database, not a lagging replica. + if ( is_callable( array( $wpdb, 'send_reads_to_masters' ) ) ) { + $wpdb->send_reads_to_masters(); + } + + // Spam and deleted conversations aren't kept; the current state wins over the event, as a deletion may be undone. + $deleted = isset( $email->state ) ? 'deleted' === $email->state : 'conversation.deleted' === ( $request->event ?? '' ); + if ( $deleted || 'spam' === ( $email->status ?? '' ) ) { + delete_emails( array_merge( array( $id ), ...array_values( get_replaced_ids( $request ) ) ) ); + + return false; + } + + return write_email( $request, $id ); +} + +/** + * Writes WordPress.org's copy of a conversation; see log_email(). + * + * @param object $request Request payload. + * @param int $id Conversation ID. + * @return bool Whether it needs all of the conversation's threads; see needs_all_threads(). + * + * @throws \RuntimeException If a write failed. + */ +function write_email( object $request, int $id ): bool { + global $wpdb; + + $email = $request->email; + $emails_table = "{$wpdb->base_prefix}helpscout"; + $replaced = get_replaced_ids( $request ); + $helpscout_id = (int) ( $request->helpscout_id ?? 0 ); + + $row = $wpdb->get_row( $wpdb->prepare( 'SELECT * FROM %i WHERE id = %d', $emails_table, $id ) ); + $has_helpscout = ! $row && $helpscout_id && $wpdb->get_var( $wpdb->prepare( 'SELECT 1 FROM %i WHERE id = %d', $emails_table, $helpscout_id ) ); + if ( needs_all_threads( $request, $row, (bool) $has_helpscout ) ) { + return true; + } + + /* + * The subject's mentions were added with the copy; the lookup costs queries, so only redo it when it changed. The + * column keeps 255 characters. + */ + $mailbox = $request->email->mailbox ?? ( $request->mailbox ?? null ); + $subject = (string) ( $email->subject ?? '' ); + $threads = (array) ( $email->threads ?? array() ); + $seen = $row && mb_substr( $subject, 0, 255 ) === (string) $row->subject && get_copy_mailbox_slug( $request ) === (string) $row->mailbox; + $mentioned = $seen && ! $threads ? array() : get_existing_slugs( + get_plugin_or_theme_from_email( + (object) array( + 'conversation' => (object) array( 'subject' => $seen ? '' : $subject ), + 'mailbox' => $mailbox, + 'threads' => $threads, + ) + ) + ); + + $user_id = (int) ( $row->user_id ?? 0 ); + if ( ! $user_id ) { + $user = get_sender_user( (object) array( 'sender' => $email->sender ?? null ) ); + $user_id = $user ? (int) $user->ID : 0; + } + + $data = get_email_row( $request, $row, $user_id ); + $columns = array_keys( $data ); + + // In an order that the event, sent again after a failed write, finishes. + $replaced_ids = array_merge( ...array_values( $replaced ) ); + take_over_meta( $id, $replaced_ids ); + add_copy_meta( $id, $replaced['helpscout'], $mentioned ); + + // phpcs:disable WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholders are added for each column; one statement, so two events for a new conversation at once don't both insert it. + check_write( + $wpdb->query( + $wpdb->prepare( + 'INSERT INTO %i (' . implode( ', ', array_fill( 0, count( $columns ), '%i' ) ) . ') VALUES (' . implode( ', ', array_fill( 0, count( $data ), '%s' ) ) . ') ON DUPLICATE KEY UPDATE ' . implode( ', ', array_fill( 0, count( $columns ) - 1, '%i = VALUES( %i )' ) ), + array_merge( + array( $emails_table ), + $columns, + array_values( $data ), + // Every column but the ID, twice: name, and name again for VALUES(). + ...array_map( + static function ( string $column ): array { + return array( $column, $column ); + }, + array_values( array_diff( $columns, array( 'id' ) ) ) + ) + ) + ) + ) + ); + // phpcs:enable + + delete_emails( $replaced_ids ); + + return false; +} + +/** + * The slugs of plugins and themes that exist, as the copy only keeps those. + * + * @param array $mentioned Slugs, by type: plugins, and themes. + * @return array + */ +function get_existing_slugs( array $mentioned ): array { + global $wpdb; + + $sites = array( + 'plugins' => array( WPORG_PLUGIN_DIRECTORY_BLOGID, 'plugin' ), + 'themes' => array( WPORG_THEME_DIRECTORY_BLOGID, 'repopackage' ), + ); + + foreach ( $mentioned as $type => $slugs ) { + if ( ! isset( $sites[ $type ] ) || ! $slugs ) { + unset( $mentioned[ $type ] ); + continue; + } + + $slugs = array_values( array_unique( array_map( 'strval', $slugs ) ) ); + + /* + * Not get_posts(): its "any" status only means statuses this request registers, and the directories' own, like a + * plugin in review's, aren't registered here. + */ + switch_to_blog( $sites[ $type ][0] ); + // phpcs:disable WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholders are added for each slug. + $mentioned[ $type ] = $wpdb->get_col( + $wpdb->prepare( + "SELECT post_name FROM %i WHERE post_type = %s AND post_status NOT IN ( 'trash', 'auto-draft', 'inherit' ) AND post_name IN (" . implode( ',', array_fill( 0, count( $slugs ), '%s' ) ) . ')', + array_merge( array( $wpdb->posts, $sites[ $type ][1] ), $slugs ) + ) + ); + // phpcs:enable + restore_current_blog(); + } + + return array_filter( $mentioned ); +} + +/** + * A conversation's row in WordPress.org's copy, whose columns and table name are HelpScout's. + * + * @param object $request Webhook payload, with the conversation in `email`; see WPOrgWebhooks' EventPayload. + * @param object|null $row Its current row, if any. + * @param int $user_id The sender's WordPress.org user ID, 0 if unknown. + * @return array + */ +function get_email_row( object $request, ?object $row, int $user_id ): array { + $email = $request->email ?? new \stdClass(); + $sender = $email->sender ?? new \stdClass(); + $address = (string) ( $sender->email ?? '' ); + $name = trim( ( $sender->first_name ?? '' ) . ' ' . ( $sender->last_name ?? '' ) ); + + $created = (int) strtotime( (string) ( $email->created_at ?? '' ) ); + $created = $created ? $created : (int) strtotime( (string) ( $row->created ?? '' ) ); + $closed = (int) strtotime( (string) ( $email->closed_at ?? '' ) ); + $updated = (int) strtotime( (string) ( $email->user_updated_at ?? '' ) ); + $times = array_filter( array( $created, $updated, $closed ) ); + + return array( + 'id' => (int) ( $request->conversation->id ?? 0 ), + 'number' => (int) ( $email->number ?? 0 ), + 'user_id' => $user_id, + 'mailbox' => get_copy_mailbox_slug( $request ), + 'status' => (string) ( $email->status ?? '' ), + 'email' => $address ? ( $name ? "{$name} <{$address}>" : $address ) : (string) ( $row->email ?? '' ), + 'subject' => (string) ( $email->subject ?? ( $row->subject ?? '' ) ), + 'preview' => (string) ( $email->preview ?? ( $row->preview ?? '' ) ), + 'created' => gmdate( 'Y-m-d H:i:s', $created ? $created : time() ), + 'closed' => $closed ? gmdate( 'Y-m-d H:i:s', $closed ) : '', + 'modified' => gmdate( 'Y-m-d H:i:s', $times ? max( $times ) : time() ), + ); +} + +/** + * The slug of the conversation's current mailbox, which the copy keeps, rather than the event's, which stats count. + * + * @param object $request Webhook payload. + * @return string + */ +function get_copy_mailbox_slug( object $request ): string { + return sanitize_title( (string) ( $request->email->mailbox->name ?? ( $request->mailbox->name ?? '' ) ) ); +} + +/** + * The copies an event's conversation replaces: one merged into it, and the HelpScout conversations it, or the one + * merged into it, were imported from. + * + * @param object $request Webhook payload. + * @return array[] FreeScout's IDs, under `freescout`, and HelpScout's, under `helpscout`. + */ +function get_replaced_ids( object $request ): array { + return array( + 'freescout' => array_values( array_filter( array( (int) ( $request->merged_id ?? 0 ) ) ) ), + 'helpscout' => array_values( + array_unique( + array_filter( + array( + (int) ( $request->helpscout_id ?? 0 ), + (int) ( $request->merged_helpscout_id ?? 0 ), + ) + ) + ) + ), + ); +} + +/** + * Whether copying a conversation needs all of its threads: the copy has neither it nor its HelpScout conversation, and + * the event carries only the newest. + * + * @param object $request Webhook payload. + * @param object|null $row The conversation's row in the copy, if it has one. + * @param bool $has_helpscout Whether the copy has the HelpScout conversation it was imported from. + * @return bool + */ +function needs_all_threads( object $request, ?object $row, bool $has_helpscout ): bool { + return ! $row && ! $has_helpscout && empty( $request->email->all_threads ); +} + +/** + * Moves the plugins and themes of copies to the copy that replaces them. Pairs it has already stay behind, and go with + * their copy. + * + * @param int $id ID of the copy that replaces them. + * @param int[] $ids IDs of the copies it replaces. + * @return void + * + * @throws \RuntimeException If the write failed. + */ +function take_over_meta( int $id, array $ids ): void { + global $wpdb; + + if ( ! $ids ) { + return; + } + + // phpcs:ignore WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare -- The placeholders are added for each ID. + check_write( $wpdb->query( $wpdb->prepare( 'UPDATE IGNORE %i SET helpscout_id = %d WHERE helpscout_id IN (' . implode( ',', array_fill( 0, count( $ids ), '%d' ) ) . ')', array_merge( array( "{$wpdb->base_prefix}helpscout_meta", $id ), $ids ) ) ) ); +} + +/** + * Adds a FreeScout copy's meta, skipping pairs it has: its helpdesk, the HelpScout conversations it replaces, and the + * plugins and themes it mentions. + * + * @param int $id Conversation ID. + * @param int[] $helpscout_ids HelpScout conversations it replaces. + * @param array[] $mentioned Plugins' and themes' slugs, by type. + * @return void + * + * @throws \RuntimeException If the write failed. + */ +function add_copy_meta( int $id, array $helpscout_ids, array $mentioned ): void { + global $wpdb; + + $pairs = array( array( 'helpdesk', 'freescout' ) ); + foreach ( $helpscout_ids as $helpscout_id ) { + $pairs[] = array( 'helpscout_conversation', (string) $helpscout_id ); + } + foreach ( $mentioned as $type => $slugs ) { + foreach ( $slugs as $slug ) { + $pairs[] = array( (string) $type, (string) $slug ); + } + } + + $args = array( "{$wpdb->base_prefix}helpscout_meta" ); + foreach ( $pairs as list( $meta_key, $meta_value ) ) { + array_push( $args, $id, $meta_key, $meta_value ); + } + + // phpcs:ignore WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholders are added for each pair. + check_write( $wpdb->query( $wpdb->prepare( 'INSERT IGNORE INTO %i ( helpscout_id, meta_key, meta_value ) VALUES ' . implode( ', ', array_fill( 0, count( $pairs ), '( %d, %s, %s )' ) ), $args ) ) ); +} + +/** + * Deletes conversations from WordPress.org's copy, with their meta. + * + * @param int[] $ids Conversation IDs; HelpScout's for the copies of its conversations. + * @return void + * + * @throws \RuntimeException If a deletion failed. + */ +function delete_emails( array $ids ): void { + global $wpdb; + + if ( ! $ids ) { + return; + } + + // phpcs:disable WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare -- The placeholders are added for each ID. + check_write( $wpdb->query( $wpdb->prepare( 'DELETE FROM %i WHERE id IN (' . implode( ',', array_fill( 0, count( $ids ), '%d' ) ) . ')', array_merge( array( "{$wpdb->base_prefix}helpscout" ), $ids ) ) ) ); + check_write( $wpdb->query( $wpdb->prepare( 'DELETE FROM %i WHERE helpscout_id IN (' . implode( ',', array_fill( 0, count( $ids ), '%d' ) ) . ')', array_merge( array( "{$wpdb->base_prefix}helpscout_meta" ), $ids ) ) ) ); + // phpcs:enable +} + +/** + * What switching an imported conversation does to its copy. + * + * @param object $copy The imported conversation: helpscout_id, id, number, status, and delete. + * @param bool $has_helpscout Whether the copy has the HelpScout conversation. + * @param bool $has_freescout Whether it has the FreeScout one. + * @return string `skip`; `delete` both copies (deleted or spam); `rename` the HelpScout copy to the FreeScout ID; or + * `merge` it into the FreeScout copy. + */ +function plan_copy_replacement( object $copy, bool $has_helpscout, bool $has_freescout ): string { + $helpscout_id = (int) ( $copy->helpscout_id ?? 0 ); + $id = (int) ( $copy->id ?? 0 ); + if ( $helpscout_id <= 0 || $id <= 0 || $helpscout_id === $id ) { + return 'skip'; + } + + if ( ! empty( $copy->delete ) ) { + return $has_helpscout || $has_freescout ? 'delete' : 'skip'; + } + + if ( ! $has_helpscout ) { + return 'skip'; + } + + return $has_freescout ? 'merge' : 'rename'; +} + +/** + * Throws if a write failed: wpdb returns false instead. + * + * @param mixed $result What wpdb returned. + * @return void + * + * @throws \RuntimeException If it failed. + */ +function check_write( mixed $result ): void { + global $wpdb; + + if ( false === $result ) { + throw new \RuntimeException( 'Could not write the copy: ' . (string) $wpdb->last_error ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- Logged, not shown. + } +} diff --git a/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php b/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php index 43b613adb8..c2d49cc819 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php +++ b/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php @@ -10,6 +10,7 @@ namespace WordPressdotorg\API\FreeScout; require __DIR__ . '/common.php'; +require __DIR__ . '/copies.php'; /** * Swaps the copies of HelpScout's conversations for those of the FreeScout conversations imported from them; see diff --git a/api.wordpress.org/public_html/dotorg/freescout/tests/bootstrap.php b/api.wordpress.org/public_html/dotorg/freescout/tests/bootstrap.php index 604c39000b..ad27668857 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/tests/bootstrap.php +++ b/api.wordpress.org/public_html/dotorg/freescout/tests/bootstrap.php @@ -114,3 +114,4 @@ function wp_cache_get( string $key, string $group = '', bool $force = false, ?bo } require_once dirname( __DIR__ ) . '/common.php'; +require_once dirname( __DIR__ ) . '/copies.php'; diff --git a/api.wordpress.org/public_html/dotorg/freescout/webhook.php b/api.wordpress.org/public_html/dotorg/freescout/webhook.php index 3134e9bcb9..055692cb08 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/webhook.php +++ b/api.wordpress.org/public_html/dotorg/freescout/webhook.php @@ -10,6 +10,7 @@ namespace WordPressdotorg\API\FreeScout; require __DIR__ . '/common.php'; +require __DIR__ . '/copies.php'; /** * Events that count as a reply sent by the agent. @@ -71,165 +72,6 @@ function contributor_stats( object $request ): void { } } -/** - * Keeps WordPress.org's copy of a conversation, and the plugins and themes it mentions, as HelpScout's webhook did. - * - * A conversation replaces the copies of one merged into it, and of the HelpScout conversations they were imported from, - * taking over their plugins and themes, which slug changes kept up to date. - * - * @param object $request Request payload. - * @return bool Whether it needs all of the conversation's threads, which FreeScout sends it again with. - * - * @throws \RuntimeException If a write failed; the event sent again finishes it. - */ -function log_email( object $request ): bool { - global $wpdb; - - $id = (int) ( $request->conversation->id ?? 0 ); - $email = $request->email ?? null; - if ( ! $id || ! is_object( $email ) ) { - return false; - } - - // Read from the primary database, not a lagging replica. - if ( is_callable( array( $wpdb, 'send_reads_to_masters' ) ) ) { - $wpdb->send_reads_to_masters(); - } - - // Spam and deleted conversations aren't kept; the current state wins over the event, as a deletion may be undone. - $deleted = isset( $email->state ) ? 'deleted' === $email->state : 'conversation.deleted' === ( $request->event ?? '' ); - if ( $deleted || 'spam' === ( $email->status ?? '' ) ) { - delete_emails( array_merge( array( $id ), ...array_values( get_replaced_ids( $request ) ) ) ); - - return false; - } - - return write_email( $request, $id ); -} - -/** - * Writes WordPress.org's copy of a conversation; see log_email(). - * - * @param object $request Request payload. - * @param int $id Conversation ID. - * @return bool Whether it needs all of the conversation's threads; see needs_all_threads(). - * - * @throws \RuntimeException If a write failed. - */ -function write_email( object $request, int $id ): bool { - global $wpdb; - - $email = $request->email; - $emails_table = "{$wpdb->base_prefix}helpscout"; - $replaced = get_replaced_ids( $request ); - $helpscout_id = (int) ( $request->helpscout_id ?? 0 ); - - $row = $wpdb->get_row( $wpdb->prepare( 'SELECT * FROM %i WHERE id = %d', $emails_table, $id ) ); - $has_helpscout = ! $row && $helpscout_id && $wpdb->get_var( $wpdb->prepare( 'SELECT 1 FROM %i WHERE id = %d', $emails_table, $helpscout_id ) ); - if ( needs_all_threads( $request, $row, (bool) $has_helpscout ) ) { - return true; - } - - /* - * The subject's mentions were added with the copy; the lookup costs queries, so only redo it when it changed. The - * column keeps 255 characters. - */ - $mailbox = $request->email->mailbox ?? ( $request->mailbox ?? null ); - $subject = (string) ( $email->subject ?? '' ); - $threads = (array) ( $email->threads ?? array() ); - $seen = $row && mb_substr( $subject, 0, 255 ) === (string) $row->subject && get_copy_mailbox_slug( $request ) === (string) $row->mailbox; - $mentioned = $seen && ! $threads ? array() : get_existing_slugs( - get_plugin_or_theme_from_email( - (object) array( - 'conversation' => (object) array( 'subject' => $seen ? '' : $subject ), - 'mailbox' => $mailbox, - 'threads' => $threads, - ) - ) - ); - - $user_id = (int) ( $row->user_id ?? 0 ); - if ( ! $user_id ) { - $user = get_sender_user( (object) array( 'sender' => $email->sender ?? null ) ); - $user_id = $user ? (int) $user->ID : 0; - } - - $data = get_email_row( $request, $row, $user_id ); - $columns = array_keys( $data ); - - // In an order that the event, sent again after a failed write, finishes. - $replaced_ids = array_merge( ...array_values( $replaced ) ); - take_over_meta( $id, $replaced_ids ); - add_copy_meta( $id, $replaced['helpscout'], $mentioned ); - - // phpcs:disable WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholders are added for each column; one statement, so two events for a new conversation at once don't both insert it. - check_write( - $wpdb->query( - $wpdb->prepare( - 'INSERT INTO %i (' . implode( ', ', array_fill( 0, count( $columns ), '%i' ) ) . ') VALUES (' . implode( ', ', array_fill( 0, count( $data ), '%s' ) ) . ') ON DUPLICATE KEY UPDATE ' . implode( ', ', array_fill( 0, count( $columns ) - 1, '%i = VALUES( %i )' ) ), - array_merge( - array( $emails_table ), - $columns, - array_values( $data ), - // Every column but the ID, twice: name, and name again for VALUES(). - ...array_map( - static function ( string $column ): array { - return array( $column, $column ); - }, - array_values( array_diff( $columns, array( 'id' ) ) ) - ) - ) - ) - ) - ); - // phpcs:enable - - delete_emails( $replaced_ids ); - - return false; -} - -/** - * The slugs of plugins and themes that exist, as the copy only keeps those. - * - * @param array $mentioned Slugs, by type: plugins, and themes. - * @return array - */ -function get_existing_slugs( array $mentioned ): array { - global $wpdb; - - $sites = array( - 'plugins' => array( WPORG_PLUGIN_DIRECTORY_BLOGID, 'plugin' ), - 'themes' => array( WPORG_THEME_DIRECTORY_BLOGID, 'repopackage' ), - ); - - foreach ( $mentioned as $type => $slugs ) { - if ( ! isset( $sites[ $type ] ) || ! $slugs ) { - unset( $mentioned[ $type ] ); - continue; - } - - $slugs = array_values( array_unique( array_map( 'strval', $slugs ) ) ); - - /* - * Not get_posts(): its "any" status only means statuses this request registers, and the directories' own, like a - * plugin in review's, aren't registered here. - */ - switch_to_blog( $sites[ $type ][0] ); - // phpcs:disable WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholders are added for each slug. - $mentioned[ $type ] = $wpdb->get_col( - $wpdb->prepare( - "SELECT post_name FROM %i WHERE post_type = %s AND post_status NOT IN ( 'trash', 'auto-draft', 'inherit' ) AND post_name IN (" . implode( ',', array_fill( 0, count( $slugs ), '%s' ) ) . ')', - array_merge( array( $wpdb->posts, $sites[ $type ][1] ), $slugs ) - ) - ); - // phpcs:enable - restore_current_blog(); - } - - return array_filter( $mentioned ); -} - $request = get_request( basename( __FILE__ ) ); // Before the stats, which an event sent again would count twice. FreeScout sends it again after a 503. From 467aedfac37c67152c66044f94e584b85d926cda Mon Sep 17 00:00:00 2001 From: Konstantin Obenland Date: Wed, 14 Oct 2026 13:14:52 -0500 Subject: [PATCH 10/10] FreeScout: Write each batch of copies in a handful of statements Point WordPress.org at FreeScout ran five writes per conversation, about 1,250 per batch of 250. The batch is now planned first, then written in five statements: move the meta, add the new pairs, rename the rows with CASE, and delete what's left. The end state is the same, and so is the order that lets a batch sent again after a failure finish. Co-Authored-By: Claude Opus 5.5 --- .../public_html/dotorg/freescout/copies.php | 53 ++++++++++++++----- .../dotorg/freescout/replace-copies.php | 47 ++++++++++++---- 2 files changed, 76 insertions(+), 24 deletions(-) diff --git a/api.wordpress.org/public_html/dotorg/freescout/copies.php b/api.wordpress.org/public_html/dotorg/freescout/copies.php index 56bad9fe52..464948fb99 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/copies.php +++ b/api.wordpress.org/public_html/dotorg/freescout/copies.php @@ -98,7 +98,7 @@ function write_email( object $request, int $id ): bool { // In an order that the event, sent again after a failed write, finishes. $replaced_ids = array_merge( ...array_values( $replaced ) ); - take_over_meta( $id, $replaced_ids ); + take_over_meta( array_fill_keys( $replaced_ids, $id ) ); add_copy_meta( $id, $replaced['helpscout'], $mentioned ); // phpcs:disable WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholders are added for each column; one statement, so two events for a new conversation at once don't both insert it. @@ -251,24 +251,29 @@ function needs_all_threads( object $request, ?object $row, bool $has_helpscout ) } /** - * Moves the plugins and themes of copies to the copy that replaces them. Pairs it has already stay behind, and go with - * their copy. + * Moves the plugins and themes of copies to the copies that replace them. Pairs a copy has already stay behind, and go + * with the copy it replaces. * - * @param int $id ID of the copy that replaces them. - * @param int[] $ids IDs of the copies it replaces. + * @param int[] $into ID of the copy that replaces each, by the ID of the copy it replaces. * @return void * * @throws \RuntimeException If the write failed. */ -function take_over_meta( int $id, array $ids ): void { +function take_over_meta( array $into ): void { global $wpdb; - if ( ! $ids ) { + if ( ! $into ) { return; } - // phpcs:ignore WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare -- The placeholders are added for each ID. - check_write( $wpdb->query( $wpdb->prepare( 'UPDATE IGNORE %i SET helpscout_id = %d WHERE helpscout_id IN (' . implode( ',', array_fill( 0, count( $ids ), '%d' ) ) . ')', array_merge( array( "{$wpdb->base_prefix}helpscout_meta", $id ), $ids ) ) ) ); + $args = array( "{$wpdb->base_prefix}helpscout_meta" ); + foreach ( $into as $replaced_id => $id ) { + array_push( $args, $replaced_id, $id ); + } + array_push( $args, ...array_keys( $into ) ); + + // phpcs:ignore WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholders are added for each ID. + check_write( $wpdb->query( $wpdb->prepare( 'UPDATE IGNORE %i SET helpscout_id = CASE helpscout_id ' . implode( ' ', array_fill( 0, count( $into ), 'WHEN %d THEN %d' ) ) . ' END WHERE helpscout_id IN (' . implode( ',', array_fill( 0, count( $into ), '%d' ) ) . ')', $args ) ) ); } /** @@ -295,13 +300,33 @@ function add_copy_meta( int $id, array $helpscout_ids, array $mentioned ): void } } - $args = array( "{$wpdb->base_prefix}helpscout_meta" ); - foreach ( $pairs as list( $meta_key, $meta_value ) ) { - array_push( $args, $id, $meta_key, $meta_value ); + insert_meta( + array_map( + static function ( array $pair ) use ( $id ): array { + return array( $id, $pair[0], $pair[1] ); + }, + $pairs + ) + ); +} + +/** + * Adds meta to copies, skipping pairs they have. + * + * @param array[] $rows Each a copy's ID, a meta key, and its value. + * @return void + * + * @throws \RuntimeException If the write failed. + */ +function insert_meta( array $rows ): void { + global $wpdb; + + if ( ! $rows ) { + return; } - // phpcs:ignore WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholders are added for each pair. - check_write( $wpdb->query( $wpdb->prepare( 'INSERT IGNORE INTO %i ( helpscout_id, meta_key, meta_value ) VALUES ' . implode( ', ', array_fill( 0, count( $pairs ), '( %d, %s, %s )' ) ), $args ) ) ); + // phpcs:ignore WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholders are added for each row. + check_write( $wpdb->query( $wpdb->prepare( 'INSERT IGNORE INTO %i ( helpscout_id, meta_key, meta_value ) VALUES ' . implode( ', ', array_fill( 0, count( $rows ), '( %d, %s, %s )' ) ), array_merge( array( "{$wpdb->base_prefix}helpscout_meta" ), ...$rows ) ) ) ); } /** diff --git a/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php b/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php index c2d49cc819..8473961d91 100644 --- a/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php +++ b/api.wordpress.org/public_html/dotorg/freescout/replace-copies.php @@ -14,7 +14,8 @@ /** * Swaps the copies of HelpScout's conversations for those of the FreeScout conversations imported from them; see - * plan_copy_replacement(). Writes go in an order that the batch, sent again after one failed, finishes. + * plan_copy_replacement(). Writes go a statement for the whole batch at a time, in an order that the batch, sent again + * after one failed, finishes. * * @param array $copies Each a HelpScout ID, the FreeScout conversation's ID, number, and status, and whether its copy * goes. @@ -45,6 +46,10 @@ static function ( mixed $copy ): object { // phpcs:ignore WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare -- The placeholders are added for each ID. $existing = array_fill_keys( array_map( 'intval', $wpdb->get_col( $wpdb->prepare( 'SELECT id FROM %i WHERE id IN (' . implode( ',', array_fill( 0, count( $ids ), '%d' ) ) . ')', array_merge( array( $emails_table ), $ids ) ) ) ), true ); $replaced = 0; + $into = array(); + $meta = array(); + $renames = array(); + $gone = array(); foreach ( $copies as $copy ) { $helpscout_id = (int) ( $copy->helpscout_id ?? 0 ); @@ -55,28 +60,50 @@ static function ( mixed $copy ): object { } ++$replaced; + unset( $existing[ $helpscout_id ] ); + $gone[] = $helpscout_id; if ( 'delete' === $action ) { - delete_emails( array( $helpscout_id, $id ) ); - unset( $existing[ $helpscout_id ], $existing[ $id ] ); + unset( $existing[ $id ] ); + $gone[] = $id; continue; } - take_over_meta( $id, array( $helpscout_id ) ); - add_copy_meta( $id, array( $helpscout_id ), array() ); + $into[ $helpscout_id ] = $id; + array_push( $meta, array( $id, 'helpdesk', 'freescout' ), array( $id, 'helpscout_conversation', (string) $helpscout_id ) ); if ( 'rename' === $action ) { // What HelpScout's webhook last wrote may be out of date, as it no longer runs for the mailbox. - $columns = array( + $renames[ $helpscout_id ] = array( 'id' => $id, 'number' => (int) ( $copy->number ?? 0 ), 'status' => (string) ( $copy->status ?? '' ), ); - check_write( $wpdb->update( $emails_table, $columns, array( 'id' => $helpscout_id ) ) ); - $existing[ $id ] = true; + $existing[ $id ] = true; } + } - delete_emails( array( $helpscout_id ) ); - unset( $existing[ $helpscout_id ] ); + take_over_meta( $into ); + insert_meta( $meta ); + if ( $renames ) { + $args = array( $emails_table ); + $sets = array(); + // The ID goes last: later assignments see the row's new values. + $formats = array( + 'number' => '%d', + 'status' => '%s', + 'id' => '%d', + ); + foreach ( $formats as $column => $format ) { + $sets[] = "{$column} = CASE id " . implode( ' ', array_fill( 0, count( $renames ), "WHEN %d THEN {$format}" ) ) . ' END'; + foreach ( $renames as $helpscout_id => $columns ) { + array_push( $args, $helpscout_id, $columns[ $column ] ); + } + } + array_push( $args, ...array_keys( $renames ) ); + + // phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared, WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholders are added for each copy; $sets holds placeholders and column names. + check_write( $wpdb->query( $wpdb->prepare( 'UPDATE %i SET ' . implode( ', ', $sets ) . ' WHERE id IN (' . implode( ',', array_fill( 0, count( $renames ), '%d' ) ) . ')', $args ) ) ); } + delete_emails( $gone ); return $replaced; }
@if ( ! $state ) {{ __('HelpScout') }} - @elseif ( Modules\WPOrgHelpScoutImport\Services\Copies::STATUS_RUNNING === $state['status'] ) - {{ __('FreeScout, :sent of :total so far', [ 'sent' => number_format( $state['sent'] ), 'total' => number_format( $copy['total'] ) ]) }} - @elseif ( Modules\WPOrgHelpScoutImport\Services\Copies::STATUS_FAILED === $state['status'] ) - {{ __('FreeScout, :sent of :total so far', [ 'sent' => number_format( $state['sent'] ), 'total' => number_format( $copy['total'] ) ]) }} -
{{ $state['error'] }} @else - {{ __('FreeScout, since :date', [ 'date' => App\User::dateFormat( $state['started_at'], 'M j, Y H:i' ) ]) }} + {{ __('FreeScout, since :date', [ 'date' => App\User::dateFormat( $state['switched_at'], 'M j, Y H:i' ) ]) }} +
+ @if ( $state['finished_at'] ) + {{ __(':sent sent, done :date', [ 'sent' => number_format( $state['sent'] ), 'date' => App\User::dateFormat( $state['finished_at'], 'M j, Y H:i' ) ]) }} + @else + {{ __(':sent of :total sent so far', [ 'sent' => number_format( $state['sent'] ), 'total' => number_format( $copy['total'] ) ]) }} + @endif + + @if ( '' !== $state['error'] ) +
{{ $state['error'] }} + @endif @endif 		- @if ( ! $state || Modules\WPOrgHelpScoutImport\Services\Copies::STATUS_FAILED === $state['status'] ) -
- {{ csrf_field() }} - - -
- @endif +
+ {{ csrf_field() }} + + +