From 2d52fc4c54c5a739fe139104d50336f458f5f6c5 Mon Sep 17 00:00:00 2001 From: EquityBot <333319867+EquityBot@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:46:26 +1100 Subject: [PATCH 01/10] fix(register): recheck drifted evidence and decode obfuscated contact emails Recheck every entry in the monthly drift report: update 13 evidence hashes, correct Lead Lists' and Dymocks' privacy contacts, and drop evidence for 3 pages that now serve a script-only bot check. The channel check now decodes Cloudflare-protected addresses and searches an entry's sources without evidence before reporting an address as missing. Closes #26 --- docs/provenance.md | 12 +++- registry/README.md | 2 +- registry/au/canva-au.toml | 6 +- registry/au/coles-au.toml | 12 +--- registry/au/dymocks-au.toml | 17 ++---- registry/au/equifax-au.toml | 10 +-- registry/au/equifax-marketing-au.toml | 6 +- registry/au/everyday-rewards-au.toml | 10 +-- registry/au/harvey-norman-au.toml | 7 +-- registry/au/lead-lists-au.toml | 15 +++-- registry/au/michael-page-au.toml | 6 +- registry/au/records-finder-au.toml | 22 +++---- xtask/src/main.rs | 87 ++++++++++++++++++++------- xtask/src/text.rs | 43 +++++++++++++ 14 files changed, 169 insertions(+), 86 deletions(-) diff --git a/docs/provenance.md b/docs/provenance.md index ca2d975..bc63b9d 100644 --- a/docs/provenance.md +++ b/docs/provenance.md @@ -110,7 +110,7 @@ Levers: every entry with an ABN lists `app11-destroy` (sent only when the user n | `chandler-macleod-au` | Maintainer's request | Privacy policy and contact pages on www.chandlermacleod.com, www.rgfstaffing.com.au | 84 603 568 403: RGF STAFFING ANZ PTY LTD, active from 06 Jan 2015 | | `charterhouse-au` | Maintainer's request | Privacy policy and contact pages on www.charterhouse.com.au | 40 078 713 774: CHARTERHOUSE RECRUITMENT PTY LTD, active from 01 Mar 2000 | | `coles-au` | Large Australian firm | Privacy policy and contact pages on www.coles.com.au | 11 004 089 936: COLES GROUP LIMITED, active from 09 Feb 2000 | -| `dymocks-au` | Large Australian data breach | Privacy policy and contact pages on www.dymocks.group | 59 008 453 110: DYMOCKS HOLDINGS PTY LTD, active from 01 Nov 1999 | +| `dymocks-au` | Large Australian data breach | Privacy policy on www.dymocks.com.au (rechecked 2 November 2026; see below) | 64 074 571 083: DYMOCKS PTY LTD, active from 05 Dec 1999 (ABN Lookup, 2 November 2026) | | `ebay-au` | Large Australian firm | Privacy policy and contact pages on www.ebay.com.au, www.ebayinc.com | 64 652 016 681: eBay Marketplaces GmbH, active from 01 Jun 2018 | | `harvey-norman-au` | Large Australian firm | Privacy policy and contact pages on www.harveynorman.com.au, www.harveynormanholdings.com.au | 14 818 873 384: THE TRUSTEE FOR THE ECOMM STORE NO 2 TRUST, active from 08 Sep 2010 | | `hays-au` | Maintainer's request | Privacy policy and contact pages on www.hays.com.au | 47 001 407 281: HAYS SPECIALIST RECRUITMENT (AUSTRALIA) PTY LIMITED, active from 16 Feb 2000 | @@ -166,6 +166,16 @@ Evidence: 52 `[[evidence]]` hashes across 33 of these entries, taken on 26 Octob - `telstra-au`, `optus-au`, `tpg-telecom-au`, `aussie-broadband-au`: members of the Telecommunications Industry Ombudsman, which `escalation` cannot yet name. - Rejected: MyDeal (closed; the site refused connections), PeopleIN (its policy covers its own staff and host employers, not its agencies' candidates). +### Evidence drift recheck (2 November 2026) + +The first monthly drift check ([issue 26](https://github.com/EquityBot/GhostSoap/issues/26)) was worked through on 2 November 2026 by an AI agent for the maintainer, from public pages only: no form was submitted, no account created and no one signed in. Each flagged page was read in a browser (Firecrawl for 3 pages behind a bot check) and its entry's claims were matched against the text; new hashes were taken twice with `cargo run -p xtask -- evidence-hash` and matched. + +- Hash updated, claims unchanged: `canva-au`, `equifax-au` (contact and credit-ban pages), `equifax-marketing-au` (donotcontact.com.au), `everyday-rewards-au` (both HTML pages; the policy page still links the August 2026 PDF, whose hash is unchanged), `michael-page-au` and `records-finder-au` (all 5 pages; the Australian search page shows a 'Databases Updated' date, so it may drift every month). +- `lead-lists-au`: the privacy policy now sends requests to a new Privacy Requests page, updated 28 October 2026, which gives `suppressions@leadlists.com.au` (was `suppression@`) and a postal address in Chadstone. The page answers a plain fetch with a bot check, so it has no hash. +- `dymocks-au`: www.dymocks.group now redirects to the policy on www.dymocks.com.au, which names Dymocks Pty Ltd, gives `privacy@dymocks.com.au` (was `privacy@dymocks.group`) and the same Level 6, 428 George Street address, and links a data deletion request form. That page answers a plain fetch with 403, so the entry has no hash. +- Evidence removed: the 2 Coles HTML pages and Harvey Norman's policy now answer a plain fetch with a script-only bot check (0 characters of main text). Their claims still match the pages in a browser; Coles's PDF policy keeps its hash. +- Contact channels: the addresses of `aussie-broadband-au`, `careerone-au`, `hays-au`, `manpowergroup-au`, `robert-half-au` and `smrtr-au` are on their pages behind Cloudflare's email obfuscation, which the check now decodes; those of `chandler-macleod-au` and `peoplebank-au` are on the RGF Staffing policy and Thryv's on its privacy policy, sources without evidence, which the check now searches. + ### Rejected candidates (M1) | Candidate | Reason | diff --git a/registry/README.md b/registry/README.md index 7962322..fecd87f 100644 --- a/registry/README.md +++ b/registry/README.md @@ -51,7 +51,7 @@ A pack has `id` (equal to the file name), `name`, `description` and `entries`, a Each `[[evidence]]` table records a source page's state when the entry was verified. A scheduled workflow (`.github/workflows/evidence-drift.yml`) re-fetches every evidence page on the first of each month and opens or updates one issue, 'Register evidence drift', listing pages that changed or could not be read within 30 seconds, and closes that issue once every page matches again. It never commits. When a page changes, recheck the entry against it and update the hash and `checked_at`. -The same run checks the contact channels of every entry that is not obsolete. It looks up each email address's domain through Google Public DNS (`dns.google`) and reports one that does not exist, has no mail or address record, or publishes a null MX record saying it accepts no mail. It also requests each web-form address and reports an HTTP error or a host public DNS says does not exist. A 401, 403, 405, 429 or 5xx, a dropped connection or a failed DNS lookup proves nothing, because sites also answer scripts that way when the form works and such failures pass, so those go to the workflow log and not the issue: a form or domain behind one of them is not confirmed to work. It also reports an email address that appears on none of its entry's evidence pages, searching each page's raw HTML and decoded main text in any letter case; a missing address goes to the workflow log instead when its entry has no evidence page, or has one that failed or is not HTML. These checks never send an email or submit a form, and a passing domain does not prove the mailbox exists. +The same run checks the contact channels of every entry that is not obsolete. It looks up each email address's domain through Google Public DNS (`dns.google`) and reports one that does not exist, has no mail or address record, or publishes a null MX record saying it accepts no mail. It also requests each web-form address and reports an HTTP error or a host public DNS says does not exist. A 401, 403, 405, 429 or 5xx, a dropped connection or a failed DNS lookup proves nothing, because sites also answer scripts that way when the form works and such failures pass, so those go to the workflow log and not the issue: a form or domain behind one of them is not confirmed to work. It also reports an email address that appears on none of its entry's source pages. It searches the evidence pages first and fetches the entry's other sources only when an address is not on them, searching each page's raw HTML, its decoded main text and any address hidden by Cloudflare's email obfuscation, in any letter case; a missing address goes to the workflow log instead when one of those pages failed, has under 500 characters of main text or is not HTML. These checks never send an email or submit a form, and a passing domain does not prove the mailbox exists. On a pull request, CI compares the register with the base branch's and adds a warning annotation for each email address an entry gains, and for every address of an entry whose id is new, so a renamed entry is flagged too (`cargo run -p xtask -- email-changes registry/au`). Domains compare case-insensitively; local parts do not. The check passes either way, but fails on an entry that does not parse: check each flagged address on the entity's own pages before merging. diff --git a/registry/au/canva-au.toml b/registry/au/canva-au.toml index f66b9b3..805e197 100644 --- a/registry/au/canva-au.toml +++ b/registry/au/canva-au.toml @@ -5,7 +5,7 @@ abn = "80 158 929 938" kinds = ["online-service"] priority = "high" quality = "verified" -verified_at = 2026-10-26 +verified_at = 2026-11-02 sources = ["https://www.canva.com/policies/privacy-policy/", "https://www.canva.com/help/legal-form/", "https://www.canva.com/settings/email-preferences"] levers = ["app7-no-marketing", "app7-source", "app11-destroy", "app13", "app13-notify-recipients", "app13-statement"] opt_in_levers = ["app12"] @@ -31,5 +31,5 @@ levers = ["app7-no-marketing"] [[evidence]] url = "https://www.canva.com/policies/privacy-policy/" -checked_at = 2026-10-26 -sha256 = "95df1957be8349f36e6521af6567318fc594643149a84da04919bf98c47df73b" +checked_at = 2026-11-02 +sha256 = "b51d2e8073e901dbbbc0a76031ff92589e3da591ef367a4b41df145cd7eff0db" diff --git a/registry/au/coles-au.toml b/registry/au/coles-au.toml index b6a0887..72b179c 100644 --- a/registry/au/coles-au.toml +++ b/registry/au/coles-au.toml @@ -5,7 +5,7 @@ abn = "11 004 089 936" kinds = ["retail"] priority = "standard" quality = "verified" -verified_at = 2026-10-26 +verified_at = 2026-11-02 sources = ["https://www.coles.com.au/content/dam/coles/cusp/about-coles/documents/coles_group_privacy_policy_september_2025.pdf", "https://www.coles.com.au/privacy", "https://www.coles.com.au/help/accounts/delete-your-coles-account"] levers = ["app7-no-marketing", "app7-source", "app11-destroy", "app13", "app13-notify-recipients", "app13-statement"] opt_in_levers = ["app12"] @@ -33,13 +33,3 @@ levers = [] url = "https://www.coles.com.au/content/dam/coles/cusp/about-coles/documents/coles_group_privacy_policy_september_2025.pdf" checked_at = 2026-10-26 sha256 = "29adbc5d5c12d46560b01b06c12ca7069edb071647790378426043bf15b82316" - -[[evidence]] -url = "https://www.coles.com.au/privacy" -checked_at = 2026-10-26 -sha256 = "dd1b09e2a5e277cd06f930fb7a6eeb82a3f1f8204a95690e293fc9cd82e31c53" - -[[evidence]] -url = "https://www.coles.com.au/help/accounts/delete-your-coles-account" -checked_at = 2026-10-26 -sha256 = "a7cff50450d9b61599ce0a0ed1217e34ca3f574814807cd417ce27edcabe3a40" diff --git a/registry/au/dymocks-au.toml b/registry/au/dymocks-au.toml index 9d8a341..6da5678 100644 --- a/registry/au/dymocks-au.toml +++ b/registry/au/dymocks-au.toml @@ -1,12 +1,12 @@ id = "dymocks-au" name = "Dymocks" -legal_entity = "Dymocks Holdings Pty Ltd" -abn = "59 008 453 110" +legal_entity = "Dymocks Pty Ltd" +abn = "64 074 571 083" kinds = ["retail"] priority = "high" quality = "verified" -verified_at = 2026-10-26 -sources = ["https://www.dymocks.group/privacy-policy/"] +verified_at = 2026-11-02 +sources = ["https://www.dymocks.com.au/privacy-policy/"] levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app11-destroy", "app13", "app13-notify-recipients", "app13-statement"] opt_in_levers = ["app12"] identity_fields = ["full_name", "email", "phone", "address"] @@ -16,15 +16,10 @@ effort = "easy" [[channels]] kind = "email" -address = "privacy@dymocks.group" +address = "privacy@dymocks.com.au" levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app11-destroy", "app13", "app13-notify-recipients", "app13-statement", "app12"] [[channels]] kind = "post" -address = "Privacy Officer, Level 6, 428 George Street, Sydney NSW 2000" +address = "Privacy Officer, Dymocks Pty Ltd, Level 6, 428 George Street, Sydney NSW 2000" levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app11-destroy", "app13", "app13-notify-recipients", "app13-statement", "app12"] - -[[evidence]] -url = "https://www.dymocks.group/privacy-policy/" -checked_at = 2026-10-26 -sha256 = "69afc5faf986fb4a1e015daee40340efc3cb834b5b217b049ffd017e38de2df3" diff --git a/registry/au/equifax-au.toml b/registry/au/equifax-au.toml index 4d0f914..07f92a0 100644 --- a/registry/au/equifax-au.toml +++ b/registry/au/equifax-au.toml @@ -5,7 +5,7 @@ abn = "26 000 602 862" kinds = ["credit-reporting-body"] priority = "crucial" quality = "verified" -verified_at = 2026-10-24 +verified_at = 2026-11-02 sources = ["https://www.equifax.com.au/credit-reporting-policy", "https://www.equifax.com.au/contact", "https://www.equifax.com.au/personal/products/credit-and-identity-products", "https://www.equifax.com.au/personal/corrections-portal", "https://www.equifax.com.au/eform/submit/credit-ban", "https://www.equifax.com.au/eform/submit/complaints-form"] levers = ["s20t-correction"] opt_in_levers = ["s20r-access", "s20k-ban"] @@ -54,8 +54,8 @@ sha256 = "7778d413ce89cada85159267cb40976f9189b088cd77cbf0dee865e5ea7dfd72" [[evidence]] url = "https://www.equifax.com.au/contact" -checked_at = 2026-10-24 -sha256 = "b3859a399f75a35d75fe8c2041cf4eb57b4dd0bf6511489db51136c62102bf81" +checked_at = 2026-11-02 +sha256 = "6031976a8530c3fa32eb7430e5a41a11cc95f76fc4e333c5f0f594e4284289fc" [[evidence]] url = "https://www.equifax.com.au/personal/products/credit-and-identity-products" @@ -69,5 +69,5 @@ sha256 = "a59e502c73dc52851831e5553e8ee57dccb653459cdb8896271b7f6ae635fa56" [[evidence]] url = "https://www.equifax.com.au/eform/submit/credit-ban" -checked_at = 2026-10-24 -sha256 = "52c91f70960defd41b361559e6b4921439daec6794386499dd552532e0db8f4f" +checked_at = 2026-11-02 +sha256 = "1d077c96fdb342a1c7ef11aa385ba1a90697a845b80c55c4fa9141bdd7beb848" diff --git a/registry/au/equifax-marketing-au.toml b/registry/au/equifax-marketing-au.toml index 4a9e5d9..12eb70b 100644 --- a/registry/au/equifax-marketing-au.toml +++ b/registry/au/equifax-marketing-au.toml @@ -5,7 +5,7 @@ abn = "88 071 215 328" kinds = ["marketing-list"] priority = "high" quality = "verified" -verified_at = 2026-10-24 +verified_at = 2026-11-02 sources = ["https://www.equifax.com.au/privacy", "https://www.donotcontact.com.au/"] levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app13", "app13-notify-recipients", "app13-statement"] opt_in_levers = ["app12"] @@ -37,5 +37,5 @@ sha256 = "aaeaf4225ee4cbade081ece48db356b14608b81ff2148ae93b5b8f705ff04257" [[evidence]] url = "https://www.donotcontact.com.au/" -checked_at = 2026-10-24 -sha256 = "3acc7039a6121135ee06d56a54aad9a5a811181d2eabd699947fccdc86ba996b" +checked_at = 2026-11-02 +sha256 = "736dd3915c42f1a772efd5ee100c050f61d106524e148b1af409c617ef644ce7" diff --git a/registry/au/everyday-rewards-au.toml b/registry/au/everyday-rewards-au.toml index ad5ea20..d6efc87 100644 --- a/registry/au/everyday-rewards-au.toml +++ b/registry/au/everyday-rewards-au.toml @@ -5,7 +5,7 @@ abn = "88 000 014 675" kinds = ["loyalty"] priority = "standard" quality = "verified" -verified_at = 2026-10-24 +verified_at = 2026-11-02 sources = ["https://www.woolworthsgroup.com.au/au/en/privacy/privacy-policy.html", "https://www.woolworthsgroup.com.au/content/dam/wwg/privacy/2026/FINAL_Woolworths-Group-Privacy-Policy-August-2026.pdf", "https://www.woolworthsgroup.com.au/au/en/privacy.html"] levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app13", "app13-notify-recipients", "app13-statement"] opt_in_levers = ["app12"] @@ -32,8 +32,8 @@ levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app13", " [[evidence]] url = "https://www.woolworthsgroup.com.au/au/en/privacy/privacy-policy.html" -checked_at = 2026-10-24 -sha256 = "3d937e82f1c70a519fdf63dcb421ea14eb0aef3ceba370f5b79695c90c99d99b" +checked_at = 2026-11-02 +sha256 = "853190e0b024601f2c102f0b6ed97bb18b8c51632bdaf6ac02b205aff8a7c3e4" [[evidence]] url = "https://www.woolworthsgroup.com.au/content/dam/wwg/privacy/2026/FINAL_Woolworths-Group-Privacy-Policy-August-2026.pdf" @@ -42,5 +42,5 @@ sha256 = "70408cddb998cbffabe6b110d3deccf7cb7b84595f41f5f9e6e74dd2984f8e0b" [[evidence]] url = "https://www.woolworthsgroup.com.au/au/en/privacy.html" -checked_at = 2026-10-24 -sha256 = "859fb2ef90dadea90d7f6d149580c5d53bd796c562528eea6d89ebcab7ac4a3a" +checked_at = 2026-11-02 +sha256 = "b801887a60188d5a87dfff8319b91c8d3d2e7a462a892fc17741aada30d273b9" diff --git a/registry/au/harvey-norman-au.toml b/registry/au/harvey-norman-au.toml index 7f7a3a8..0b01062 100644 --- a/registry/au/harvey-norman-au.toml +++ b/registry/au/harvey-norman-au.toml @@ -5,7 +5,7 @@ abn = "14 818 873 384" kinds = ["retail"] priority = "standard" quality = "verified" -verified_at = 2026-10-26 +verified_at = 2026-11-02 sources = ["https://www.harveynorman.com.au/privacy-policy", "https://www.harveynormanholdings.com.au/privacy-policy"] levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app11-destroy", "app13", "app13-notify-recipients", "app13-statement"] opt_in_levers = ["app12"] @@ -28,8 +28,3 @@ levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app11-des kind = "phone" address = "02 9201 6111" levers = [] - -[[evidence]] -url = "https://www.harveynorman.com.au/privacy-policy" -checked_at = 2026-10-26 -sha256 = "db0c4e5daafd56008fda9aa126c41662892d93f3d89cddf7a12d8be12e0cc4d9" diff --git a/registry/au/lead-lists-au.toml b/registry/au/lead-lists-au.toml index 5fbea79..afe45df 100644 --- a/registry/au/lead-lists-au.toml +++ b/registry/au/lead-lists-au.toml @@ -5,8 +5,8 @@ abn = "53 634 428 470" kinds = ["marketing-list"] priority = "high" quality = "verified" -verified_at = 2026-10-24 -sources = ["https://leadlists.com.au/privacy-policy/", "https://leadlists.com.au/contact-us/"] +verified_at = 2026-11-02 +sources = ["https://leadlists.com.au/privacy-policy/", "https://leadlists.com.au/privacy-requests/", "https://leadlists.com.au/contact-us/"] levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app13", "app13-notify-recipients", "app13-statement"] opt_in_levers = ["app12"] identity_fields = ["full_name", "email", "phone", "address"] @@ -16,10 +16,15 @@ effort = "easy" [[channels]] kind = "email" -address = "suppression@leadlists.com.au" +address = "suppressions@leadlists.com.au" levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app13", "app13-notify-recipients", "app13-statement", "app12"] preferred = true +[[channels]] +kind = "post" +address = "Privacy Officer, Lead Lists Data Solutions, Suite 60, Level 2, 1341 Dandenong Rd, Chadstone VIC 3148" +levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app13", "app13-notify-recipients", "app13-statement", "app12"] + [[channels]] kind = "phone" address = "(03) 9007 0355" @@ -27,5 +32,5 @@ levers = [] [[evidence]] url = "https://leadlists.com.au/privacy-policy/" -checked_at = 2026-10-24 -sha256 = "910c2ea7301eec5d996d78b3a6acc4c3ecd17853b7509498eee8c710daa39fc2" +checked_at = 2026-11-02 +sha256 = "679c8de7a59a4872c7589a0fe78fdbfea13b4258b896970b58265583ba0eec7b" diff --git a/registry/au/michael-page-au.toml b/registry/au/michael-page-au.toml index fc29087..4d1af03 100644 --- a/registry/au/michael-page-au.toml +++ b/registry/au/michael-page-au.toml @@ -5,7 +5,7 @@ abn = "58 002 872 264" kinds = ["recruitment"] priority = "high" quality = "verified" -verified_at = 2026-10-26 +verified_at = 2026-11-02 sources = ["https://www.michaelpage.com.au/privacy-policy"] levers = ["app7-no-marketing", "app7-source", "app11-destroy", "app13", "app13-notify-recipients", "app13-statement"] opt_in_levers = ["app12"] @@ -31,5 +31,5 @@ levers = [] [[evidence]] url = "https://www.michaelpage.com.au/privacy-policy" -checked_at = 2026-10-26 -sha256 = "d5d2d1c28b5fa4d58a1a98158f89ead4fbdb4e75ce4ebdd13497ea4b880eca32" +checked_at = 2026-11-02 +sha256 = "a666427dccee8a8c573ebc94aca1c040638592b413cc9515d727d26af5f757b9" diff --git a/registry/au/records-finder-au.toml b/registry/au/records-finder-au.toml index 07bda2c..44796c9 100644 --- a/registry/au/records-finder-au.toml +++ b/registry/au/records-finder-au.toml @@ -4,7 +4,7 @@ legal_entity = "InfoPay, Inc." kinds = ["people-search"] priority = "standard" quality = "verified" -verified_at = 2026-10-25 +verified_at = 2026-11-02 sources = ["https://recordsfinder.com/privacy/", "https://recordsfinder.com/terms/", "https://recordsfinder.com/optout/", "https://recordsfinder.com/do-not-sell-share-my-personal-information/", "https://recordsfinder.com/australia/"] identity_fields = ["full_name"] exemption_risk = "overseas" @@ -21,25 +21,25 @@ url = "https://recordsfinder.com/australia/" [[evidence]] url = "https://recordsfinder.com/privacy/" -checked_at = 2026-10-25 -sha256 = "572946dbdda3efecf0b16fc347081530267df9d33c5d22deac923ab300abca87" +checked_at = 2026-11-02 +sha256 = "0a84a58297247e0c0edc9165d5259986779ade2b5b980a0be7bd1fba626fab0b" [[evidence]] url = "https://recordsfinder.com/terms/" -checked_at = 2026-10-25 -sha256 = "ec8f39257958f057bc7f9124a08c155f12c93bde766913e56a8b3b765883a4f8" +checked_at = 2026-11-02 +sha256 = "1d27469e8e1bdf9b377a41d38932231a867708501e68124e8398d16c13959c99" [[evidence]] url = "https://recordsfinder.com/optout/" -checked_at = 2026-10-25 -sha256 = "c88024ac830586c5ac152a30327dd01f977fc282ea75ff5de621232082d3dd78" +checked_at = 2026-11-02 +sha256 = "3c92ba03b5906df0786c6aa79ec8391153461adc563f0d5500081576f886c604" [[evidence]] url = "https://recordsfinder.com/do-not-sell-share-my-personal-information/" -checked_at = 2026-10-25 -sha256 = "ae99872996c5d8e13dec966599146e3a4c4fdf0b8c3a907e40390a9291abba7f" +checked_at = 2026-11-02 +sha256 = "5bb3cad71d41e919aa4fa683c045678f5bae92bc219a6451e6e13b740d9ab8a8" [[evidence]] url = "https://recordsfinder.com/australia/" -checked_at = 2026-10-25 -sha256 = "1864c827332dd7c8bc46ab22526f532ca858c0f6022b7ab5a209440ff1626a89" +checked_at = 2026-11-02 +sha256 = "82c868efaaa940aa19fb41e4d2e04e4280e2587d67dd1b570b54546f44382593" diff --git a/xtask/src/main.rs b/xtask/src/main.rs index b795e2d..07e32b7 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -33,8 +33,9 @@ fn agent(timeout: Duration) -> ureq::Agent { /// A fetched evidence page. struct Page { sha256: String, - /// Lowercase raw HTML followed by the lowercase main text, for finding an email - /// address in a link, a footer or entity-encoded text; `None` for a non-HTML page. + /// Lowercase raw HTML, main text and Cloudflare-protected addresses, for finding an + /// email address in a link, a footer, or entity-encoded or obfuscated text; `None` for + /// a non-HTML page. text: Option, } @@ -65,7 +66,8 @@ fn evidence_page(agent: &ureq::Agent, url: &str) -> Result { main.chars().count() )); } - let searchable = format!("{html}\n{main}").to_lowercase(); + let hidden = text::cloudflare_emails(&html).join("\n"); + let searchable = format!("{html}\n{main}\n{hidden}").to_lowercase(); (Sha256::digest(main.as_bytes()), Some(searchable)) } else { (Sha256::digest(&bytes), None) @@ -140,12 +142,8 @@ fn drift(dir: &Path) -> ExitCode { if entry.obsolete.is_none() { collect_channels(&entry, &mut mail_domains, &mut forms); let ids = std::slice::from_ref(&entry.id); - for channel in &entry.channels { - if channel.kind == ChannelKind::Email { - let what = format!("email {}", channel.address); - let check = email_on_pages(&channel.address, &pages); - record(&mut emails, &what, ids, check); - } + for (address, check) in email_checks(&agent, &entry, pages) { + record(&mut emails, &format!("email {address}"), ids, check); } } } @@ -209,7 +207,36 @@ fn record(lines: &mut Vec, what: &str, ids: &[String], check: Check) { } } -/// Whether `address` appears on one of its entry's evidence pages, given each page's +/// Check each email channel of `entry` against its source pages. `pages` holds the +/// searchable text of its evidence pages; its other sources, such as a page behind a bot +/// check that cannot have evidence, are fetched only when an address is not on those. +fn email_checks( + agent: &ureq::Agent, + entry: &Entry, + mut pages: Vec>, +) -> Vec<(String, Check)> { + let addresses: Vec<&str> = entry + .channels + .iter() + .filter(|c| c.kind == ChannelKind::Email) + .map(|c| c.address.as_str()) + .collect(); + if addresses + .iter() + .any(|a| email_on_pages(a, &pages) != Check::Fine) + { + let tracked = |url: &String| entry.evidence.iter().any(|e| &e.url == url); + for url in entry.sources.iter().filter(|url| !tracked(url)) { + pages.push(evidence_page(agent, url).ok().and_then(|page| page.text)); + } + } + addresses + .into_iter() + .map(|a| (a.to_owned(), email_on_pages(a, &pages))) + .collect() +} + +/// Whether `address` appears on one of its entry's source pages, given each page's /// searchable text (`None` for a page that failed or is not HTML). It is missing only when /// every page could be searched. fn email_on_pages(address: &str, pages: &[Option]) -> Check { @@ -217,11 +244,11 @@ fn email_on_pages(address: &str, pages: &[Option]) -> Check { if pages.iter().flatten().any(|text| text.contains(&address)) { Check::Fine } else if pages.is_empty() { - Check::Inconclusive("the entry has no evidence page".to_owned()) + Check::Inconclusive("the entry has no source page".to_owned()) } else if pages.iter().any(Option::is_none) { - Check::Inconclusive("not on the evidence pages that could be searched".to_owned()) + Check::Inconclusive("not on the source pages that could be searched".to_owned()) } else { - Check::Dead("appears on none of the entry's evidence pages".to_owned()) + Check::Dead("appears on none of the entry's source pages".to_owned()) } } @@ -662,7 +689,7 @@ levers = [] } #[test] - fn an_email_must_appear_on_an_evidence_page() { + fn an_email_must_appear_on_a_source_page() { let page = |text: &str| Some(text.to_owned()); let found = [page("other"), page("write to privacy@example.com.au.")]; assert_eq!( @@ -672,7 +699,7 @@ levers = [] let missing = [page("write to privacy@elsewhere.example")]; assert!(matches!( email_on_pages("privacy@example.com.au", &missing), - Check::Dead(p) if p.contains("none of the entry's evidence pages") + Check::Dead(p) if p.contains("none of the entry's source pages") )); let unsearched = [page("nothing here"), None]; assert!(matches!( @@ -685,13 +712,10 @@ levers = [] )); } - #[test] - fn a_page_is_searched_in_its_markup_and_its_decoded_text() { + /// Serve one HTML page whose main element holds 500 or more characters, then `main`. + fn serve_html(main: &str) -> String { let filler = "Privacy policy text. ".repeat(30); - let html = format!( - "
{filler} Email privacy@example.com.au.
\ - " - ); + let html = format!("
{filler} {main}
"); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let url = format!("http://{}/", listener.local_addr().unwrap()); std::thread::spawn(move || { @@ -704,6 +728,17 @@ levers = [] ); let _ = stream.write_all(reply.as_bytes()); }); + url + } + + #[test] + fn a_page_is_searched_in_its_markup_and_its_decoded_text() { + let url = serve_html( + "Email privacy@example.com.au. \ + Contact \ + [email protected]", + ); let page = evidence_page(&agent(Duration::from_secs(5)), &url).unwrap(); let pages = [page.text]; assert_eq!( @@ -711,12 +746,22 @@ levers = [] Check::Fine ); assert_eq!(email_on_pages("help@example.com.au", &pages), Check::Fine); + assert_eq!(email_on_pages("hidden@example.com.au", &pages), Check::Fine); assert!(matches!( email_on_pages("other@example.com.au", &pages), Check::Dead(_) )); } + #[test] + fn an_email_is_searched_on_sources_without_evidence() { + let url = serve_html("Email privacy@example.com.au."); + let entry: Entry = + toml::from_str(&ENTRY.replace("https://example.com.au/privacy", &url)).unwrap(); + let checks = email_checks(&agent(Duration::from_secs(5)), &entry, Vec::new()); + assert_eq!(checks, [("privacy@Example.com.au".to_owned(), Check::Fine)]); + } + fn source(text: &str) -> SourceFile { SourceFile { name: "example-au.toml".to_owned(), diff --git a/xtask/src/text.rs b/xtask/src/text.rs index 460d100..aa3c9e8 100644 --- a/xtask/src/text.rs +++ b/xtask/src/text.rs @@ -240,6 +240,36 @@ fn after(lower: &str, from: usize, needle: &str) -> usize { .map_or(lower.len(), |j| from + j + needle.len()) } +/// Email addresses hidden by Cloudflare's email obfuscation, which replaces each address in +/// a page with `data-cfemail=""` or a `/cdn-cgi/l/email-protection#` link. The +/// first byte of the hex is a key XORed with each following byte. +pub fn cloudflare_emails(html: &str) -> Vec { + let mut emails = Vec::new(); + for marker in ["data-cfemail=\"", "/cdn-cgi/l/email-protection#"] { + for (i, _) in html.match_indices(marker) { + let rest = &html[i + marker.len()..]; + let end = rest + .find(|c: char| !c.is_ascii_hexdigit()) + .unwrap_or(rest.len()); + emails.extend(decode_cloudflare(&rest[..end])); + } + } + emails +} + +fn decode_cloudflare(hex: &str) -> Option { + if hex.len() < 4 || hex.len() % 2 != 0 { + return None; + } + let bytes = (0..hex.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&hex[i..i + 2], 16)) + .collect::, _>>() + .ok()?; + let (key, rest) = bytes.split_first()?; + String::from_utf8(rest.iter().map(|b| b ^ key).collect()).ok() +} + fn decode_entities(text: &str) -> String { let mut out = String::with_capacity(text.len()); let mut rest = text; @@ -403,4 +433,17 @@ mod tests { assert!(!is_html(Some("application/pdf"), page)); assert!(!is_html(None, b"%PDF-1.7")); } + + #[test] + fn decodes_cloudflare_protected_emails() { + let html = "[email protected] \ + Help \ + "; + assert_eq!( + cloudflare_emails(html), + ["privacy@example.com.au", "help@example.com.au"] + ); + assert!(cloudflare_emails("

privacy@example.com.au

").is_empty()); + } } From 212f19f3568098bb76284173853e9b0260d9e6d3 Mon Sep 17 00:00:00 2001 From: EquityBot <333319867+EquityBot@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:47:02 +1100 Subject: [PATCH 02/10] fix(xtask): use is_multiple_of for the hex length check --- xtask/src/text.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xtask/src/text.rs b/xtask/src/text.rs index aa3c9e8..1803648 100644 --- a/xtask/src/text.rs +++ b/xtask/src/text.rs @@ -258,7 +258,7 @@ pub fn cloudflare_emails(html: &str) -> Vec { } fn decode_cloudflare(hex: &str) -> Option { - if hex.len() < 4 || hex.len() % 2 != 0 { + if hex.len() < 4 || !hex.len().is_multiple_of(2) { return None; } let bytes = (0..hex.len()) From ca58e2099cd6b6e146e7445f66bcc5b452274738 Mon Sep 17 00:00:00 2001 From: EquityBot <333319867+EquityBot@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:55:51 +1100 Subject: [PATCH 03/10] fix(register): drop Equifax evidence whose date example changes daily --- docs/provenance.md | 4 ++-- registry/au/equifax-au.toml | 10 ---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/docs/provenance.md b/docs/provenance.md index bc63b9d..2bd92c6 100644 --- a/docs/provenance.md +++ b/docs/provenance.md @@ -170,10 +170,10 @@ Evidence: 52 `[[evidence]]` hashes across 33 of these entries, taken on 26 Octob The first monthly drift check ([issue 26](https://github.com/EquityBot/GhostSoap/issues/26)) was worked through on 2 November 2026 by an AI agent for the maintainer, from public pages only: no form was submitted, no account created and no one signed in. Each flagged page was read in a browser (Firecrawl for 3 pages behind a bot check) and its entry's claims were matched against the text; new hashes were taken twice with `cargo run -p xtask -- evidence-hash` and matched. -- Hash updated, claims unchanged: `canva-au`, `equifax-au` (contact and credit-ban pages), `equifax-marketing-au` (donotcontact.com.au), `everyday-rewards-au` (both HTML pages; the policy page still links the August 2026 PDF, whose hash is unchanged), `michael-page-au` and `records-finder-au` (all 5 pages; the Australian search page shows a 'Databases Updated' date, so it may drift every month). +- Hash updated, claims unchanged: `canva-au`, `equifax-marketing-au` (donotcontact.com.au), `everyday-rewards-au` (both HTML pages; the policy page still links the August 2026 PDF, whose hash is unchanged), `michael-page-au` and `records-finder-au` (all 5 pages; the Australian search page shows a 'Databases Updated' date, so it may drift every month). - `lead-lists-au`: the privacy policy now sends requests to a new Privacy Requests page, updated 28 October 2026, which gives `suppressions@leadlists.com.au` (was `suppression@`) and a postal address in Chadstone. The page answers a plain fetch with a bot check, so it has no hash. - `dymocks-au`: www.dymocks.group now redirects to the policy on www.dymocks.com.au, which names Dymocks Pty Ltd, gives `privacy@dymocks.com.au` (was `privacy@dymocks.group`) and the same Level 6, 428 George Street address, and links a data deletion request form. That page answers a plain fetch with 403, so the entry has no hash. -- Evidence removed: the 2 Coles HTML pages and Harvey Norman's policy now answer a plain fetch with a script-only bot check (0 characters of main text). Their claims still match the pages in a browser; Coles's PDF policy keeps its hash. +- Evidence removed: the 2 Coles HTML pages and Harvey Norman's policy now answer a plain fetch with a script-only bot check (0 characters of main text); Equifax's contact page and credit ban form show the current date as a date field example ('E.g., 02/11/2026'), so their hashes change every day. Their claims still match the pages in a browser; Coles's PDF policy and Equifax's other 3 pages keep their hashes. - Contact channels: the addresses of `aussie-broadband-au`, `careerone-au`, `hays-au`, `manpowergroup-au`, `robert-half-au` and `smrtr-au` are on their pages behind Cloudflare's email obfuscation, which the check now decodes; those of `chandler-macleod-au` and `peoplebank-au` are on the RGF Staffing policy and Thryv's on its privacy policy, sources without evidence, which the check now searches. ### Rejected candidates (M1) diff --git a/registry/au/equifax-au.toml b/registry/au/equifax-au.toml index 07f92a0..daa9d20 100644 --- a/registry/au/equifax-au.toml +++ b/registry/au/equifax-au.toml @@ -52,11 +52,6 @@ url = "https://www.equifax.com.au/credit-reporting-policy" checked_at = 2026-10-24 sha256 = "7778d413ce89cada85159267cb40976f9189b088cd77cbf0dee865e5ea7dfd72" -[[evidence]] -url = "https://www.equifax.com.au/contact" -checked_at = 2026-11-02 -sha256 = "6031976a8530c3fa32eb7430e5a41a11cc95f76fc4e333c5f0f594e4284289fc" - [[evidence]] url = "https://www.equifax.com.au/personal/products/credit-and-identity-products" checked_at = 2026-10-24 @@ -66,8 +61,3 @@ sha256 = "c20f72dfd65dbed954440fb3494f30d4fa1c11e057effd2e551a7d9d63509068" url = "https://www.equifax.com.au/personal/corrections-portal" checked_at = 2026-10-24 sha256 = "a59e502c73dc52851831e5553e8ee57dccb653459cdb8896271b7f6ae635fa56" - -[[evidence]] -url = "https://www.equifax.com.au/eform/submit/credit-ban" -checked_at = 2026-11-02 -sha256 = "1d077c96fdb342a1c7ef11aa385ba1a90697a845b80c55c4fa9141bdd7beb848" From f32649dab19bbf4e46b9f97cd9ea021ddefb47d2 Mon Sep 17 00:00:00 2001 From: EquityBot <333319867+EquityBot@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:35:10 +1100 Subject: [PATCH 04/10] fix(xtask): match whole email addresses and reject malformed Cloudflare payloads --- xtask/src/main.rs | 42 +++++++++++++++++++++++++++++++++++++++++- xtask/src/text.rs | 10 +++++++--- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 07e32b7..4c7ea4a 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -241,7 +241,11 @@ fn email_checks( /// every page could be searched. fn email_on_pages(address: &str, pages: &[Option]) -> Check { let address = address.to_lowercase(); - if pages.iter().flatten().any(|text| text.contains(&address)) { + if pages + .iter() + .flatten() + .any(|text| has_address(text, &address)) + { Check::Fine } else if pages.is_empty() { Check::Inconclusive("the entry has no source page".to_owned()) @@ -252,6 +256,19 @@ fn email_on_pages(address: &str, pages: &[Option]) -> Check { } } +/// Whether `text` contains `address` as a whole address, not inside a longer one such as +/// `not` + address or address + `.au`. A sentence's full stop may follow it. +fn has_address(text: &str, address: &str) -> bool { + let local = |c: char| c.is_ascii_alphanumeric() || "._%+-".contains(c); + let domain = |c: char| c.is_ascii_alphanumeric() || c == '-'; + text.match_indices(address).any(|(i, _)| { + let after = &text[i + address.len()..]; + !text[..i].ends_with(local) + && !after.starts_with(domain) + && !(after.starts_with('.') && after[1..].starts_with(domain)) + }) +} + /// Email addresses in `head` that the same entry in `base` did not have, as `(file, message)`. /// A change that repoints a privacy address would send users' requests elsewhere, so a /// reviewer must see each one. Every address of an entry whose id is not in `base` is listed, @@ -696,6 +713,13 @@ levers = [] email_on_pages("Privacy@Example.com.au", &found), Check::Fine ); + let longer = [page( + "notprivacy@example.com.au, privacy@example.com.au.example.org", + )]; + assert!(matches!( + email_on_pages("privacy@example.com.au", &longer), + Check::Dead(_) + )); let missing = [page("write to privacy@elsewhere.example")]; assert!(matches!( email_on_pages("privacy@example.com.au", &missing), @@ -762,6 +786,22 @@ levers = [] assert_eq!(checks, [("privacy@Example.com.au".to_owned(), Check::Fine)]); } + #[test] + fn an_email_missing_from_evidence_is_searched_on_the_other_sources() { + let url = serve_html("Email privacy@example.com.au."); + let sources = format!("sources = [\"https://example.com.au/privacy\", \"{url}\"]"); + let evidence = format!( + "\n[[evidence]]\nurl = \"https://example.com.au/privacy\"\n\ + checked_at = 2026-10-30\nsha256 = \"{}\"\n", + "0".repeat(64) + ); + let toml = ENTRY.replace("sources = [\"https://example.com.au/privacy\"]", &sources); + let entry: Entry = toml::from_str(&(toml + &evidence)).unwrap(); + let evidence_page = vec![Some("a policy that names no address".to_owned())]; + let checks = email_checks(&agent(Duration::from_secs(5)), &entry, evidence_page); + assert_eq!(checks, [("privacy@Example.com.au".to_owned(), Check::Fine)]); + } + fn source(text: &str) -> SourceFile { SourceFile { name: "example-au.toml".to_owned(), diff --git a/xtask/src/text.rs b/xtask/src/text.rs index 1803648..e9cc322 100644 --- a/xtask/src/text.rs +++ b/xtask/src/text.rs @@ -242,7 +242,8 @@ fn after(lower: &str, from: usize, needle: &str) -> usize { /// Email addresses hidden by Cloudflare's email obfuscation, which replaces each address in /// a page with `data-cfemail=""` or a `/cdn-cgi/l/email-protection#` link. The -/// first byte of the hex is a key XORed with each following byte. +/// first byte of the hex is a key XORed with each following byte. Either form ends at the +/// attribute's closing quote; any other character there makes the payload malformed. pub fn cloudflare_emails(html: &str) -> Vec { let mut emails = Vec::new(); for marker in ["data-cfemail=\"", "/cdn-cgi/l/email-protection#"] { @@ -251,7 +252,9 @@ pub fn cloudflare_emails(html: &str) -> Vec { let end = rest .find(|c: char| !c.is_ascii_hexdigit()) .unwrap_or(rest.len()); - emails.extend(decode_cloudflare(&rest[..end])); + if rest[end..].starts_with(['"', '\'']) { + emails.extend(decode_cloudflare(&rest[..end])); + } } } emails @@ -439,7 +442,8 @@ mod tests { let html = "[email protected] \ Help \ - "; + \ + "; assert_eq!( cloudflare_emails(html), ["privacy@example.com.au", "help@example.com.au"] From 09343b80b75dcded6e162880f859107b743caeb8 Mon Sep 17 00:00:00 2001 From: EquityBot <333319867+EquityBot@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:38:29 +1100 Subject: [PATCH 05/10] fix(register): drop Records Finder evidence whose update date changes daily --- docs/provenance.md | 4 ++-- registry/au/records-finder-au.toml | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/provenance.md b/docs/provenance.md index 2bd92c6..33cfec4 100644 --- a/docs/provenance.md +++ b/docs/provenance.md @@ -170,10 +170,10 @@ Evidence: 52 `[[evidence]]` hashes across 33 of these entries, taken on 26 Octob The first monthly drift check ([issue 26](https://github.com/EquityBot/GhostSoap/issues/26)) was worked through on 2 November 2026 by an AI agent for the maintainer, from public pages only: no form was submitted, no account created and no one signed in. Each flagged page was read in a browser (Firecrawl for 3 pages behind a bot check) and its entry's claims were matched against the text; new hashes were taken twice with `cargo run -p xtask -- evidence-hash` and matched. -- Hash updated, claims unchanged: `canva-au`, `equifax-marketing-au` (donotcontact.com.au), `everyday-rewards-au` (both HTML pages; the policy page still links the August 2026 PDF, whose hash is unchanged), `michael-page-au` and `records-finder-au` (all 5 pages; the Australian search page shows a 'Databases Updated' date, so it may drift every month). +- Hash updated, claims unchanged: `canva-au`, `equifax-marketing-au` (donotcontact.com.au), `everyday-rewards-au` (both HTML pages; the policy page still links the August 2026 PDF, whose hash is unchanged), `michael-page-au` and `records-finder-au` (4 pages). - `lead-lists-au`: the privacy policy now sends requests to a new Privacy Requests page, updated 28 October 2026, which gives `suppressions@leadlists.com.au` (was `suppression@`) and a postal address in Chadstone. The page answers a plain fetch with a bot check, so it has no hash. - `dymocks-au`: www.dymocks.group now redirects to the policy on www.dymocks.com.au, which names Dymocks Pty Ltd, gives `privacy@dymocks.com.au` (was `privacy@dymocks.group`) and the same Level 6, 428 George Street address, and links a data deletion request form. That page answers a plain fetch with 403, so the entry has no hash. -- Evidence removed: the 2 Coles HTML pages and Harvey Norman's policy now answer a plain fetch with a script-only bot check (0 characters of main text); Equifax's contact page and credit ban form show the current date as a date field example ('E.g., 02/11/2026'), so their hashes change every day. Their claims still match the pages in a browser; Coles's PDF policy and Equifax's other 3 pages keep their hashes. +- Evidence removed: the 2 Coles HTML pages and Harvey Norman's policy now answer a plain fetch with a script-only bot check (0 characters of main text); Equifax's contact page and credit ban form show the current date as a date field example ('E.g., 02/11/2026'), so their hashes change every day, and Records Finder's Australian search page shows the date its database was last updated, which changed within the day. Their claims still match the pages in a browser; Coles's PDF policy and Equifax's other 3 pages keep their hashes. - Contact channels: the addresses of `aussie-broadband-au`, `careerone-au`, `hays-au`, `manpowergroup-au`, `robert-half-au` and `smrtr-au` are on their pages behind Cloudflare's email obfuscation, which the check now decodes; those of `chandler-macleod-au` and `peoplebank-au` are on the RGF Staffing policy and Thryv's on its privacy policy, sources without evidence, which the check now searches. ### Rejected candidates (M1) diff --git a/registry/au/records-finder-au.toml b/registry/au/records-finder-au.toml index 44796c9..4e2002c 100644 --- a/registry/au/records-finder-au.toml +++ b/registry/au/records-finder-au.toml @@ -38,8 +38,3 @@ sha256 = "3c92ba03b5906df0786c6aa79ec8391153461adc563f0d5500081576f886c604" url = "https://recordsfinder.com/do-not-sell-share-my-personal-information/" checked_at = 2026-11-02 sha256 = "5bb3cad71d41e919aa4fa683c045678f5bae92bc219a6451e6e13b740d9ab8a8" - -[[evidence]] -url = "https://recordsfinder.com/australia/" -checked_at = 2026-11-02 -sha256 = "82c868efaaa940aa19fb41e4d2e04e4280e2587d67dd1b570b54546f44382593" From 9f29b850ab62502c058ec778f8e15fee85dfd17d Mon Sep 17 00:00:00 2001 From: EquityBot <333319867+EquityBot@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:45:54 +1100 Subject: [PATCH 06/10] fix(xtask): allow a percent-escape before an address and require data-cfemail's double quote --- xtask/src/main.rs | 13 +++++++++++-- xtask/src/text.rs | 11 ++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 4c7ea4a..c3950f6 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -257,13 +257,15 @@ fn email_on_pages(address: &str, pages: &[Option]) -> Check { } /// Whether `text` contains `address` as a whole address, not inside a longer one such as -/// `not` + address or address + `.au`. A sentence's full stop may follow it. +/// `not` + address or address + `.au`. A sentence's full stop may follow it, and a +/// percent-escape such as `%20` in a link may precede it. fn has_address(text: &str, address: &str) -> bool { let local = |c: char| c.is_ascii_alphanumeric() || "._%+-".contains(c); let domain = |c: char| c.is_ascii_alphanumeric() || c == '-'; + let escaped = |before: &[u8]| matches!(before, [.., b'%', h, l] if h.is_ascii_hexdigit() && l.is_ascii_hexdigit()); text.match_indices(address).any(|(i, _)| { let after = &text[i + address.len()..]; - !text[..i].ends_with(local) + (!text[..i].ends_with(local) || escaped(&text.as_bytes()[..i])) && !after.starts_with(domain) && !(after.starts_with('.') && after[1..].starts_with(domain)) }) @@ -709,6 +711,13 @@ levers = [] fn an_email_must_appear_on_a_source_page() { let page = |text: &str| Some(text.to_owned()); let found = [page("other"), page("write to privacy@example.com.au.")]; + let linked = [page( + "", + )]; + assert_eq!( + email_on_pages("privacy@example.com.au", &linked), + Check::Fine + ); assert_eq!( email_on_pages("Privacy@Example.com.au", &found), Check::Fine diff --git a/xtask/src/text.rs b/xtask/src/text.rs index e9cc322..d5ad984 100644 --- a/xtask/src/text.rs +++ b/xtask/src/text.rs @@ -246,13 +246,17 @@ fn after(lower: &str, from: usize, needle: &str) -> usize { /// attribute's closing quote; any other character there makes the payload malformed. pub fn cloudflare_emails(html: &str) -> Vec { let mut emails = Vec::new(); - for marker in ["data-cfemail=\"", "/cdn-cgi/l/email-protection#"] { + let forms: [(&str, &[char]); 2] = [ + ("data-cfemail=\"", &['"']), + ("/cdn-cgi/l/email-protection#", &['"', '\'']), + ]; + for (marker, quotes) in forms { for (i, _) in html.match_indices(marker) { let rest = &html[i + marker.len()..]; let end = rest .find(|c: char| !c.is_ascii_hexdigit()) .unwrap_or(rest.len()); - if rest[end..].starts_with(['"', '\'']) { + if rest[end..].starts_with(quotes) { emails.extend(decode_cloudflare(&rest[..end])); } } @@ -443,7 +447,8 @@ mod tests { data-cfemail=\"4232302b3423213b02273a232f322e276c212d2f6c2337\">[email protected] \ Help \ \ - "; + \ + "; assert_eq!( cloudflare_emails(html), ["privacy@example.com.au", "help@example.com.au"] From 983bcadb72629d03bca8af87e274c43f5d05be4b Mon Sep 17 00:00:00 2001 From: EquityBot <333319867+EquityBot@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:54:23 +1100 Subject: [PATCH 07/10] fix(xtask): count only escapes of non-address characters as boundaries and require double quotes on Cloudflare links --- xtask/src/main.rs | 20 +++++++++++++++++--- xtask/src/text.rs | 16 +++++++--------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/xtask/src/main.rs b/xtask/src/main.rs index c3950f6..176024a 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -258,14 +258,21 @@ fn email_on_pages(address: &str, pages: &[Option]) -> Check { /// Whether `text` contains `address` as a whole address, not inside a longer one such as /// `not` + address or address + `.au`. A sentence's full stop may follow it, and a -/// percent-escape such as `%20` in a link may precede it. +/// percent-escape in a link, such as `%20`, may precede it unless it encodes a character an +/// address can contain, such as `%2B`. fn has_address(text: &str, address: &str) -> bool { let local = |c: char| c.is_ascii_alphanumeric() || "._%+-".contains(c); let domain = |c: char| c.is_ascii_alphanumeric() || c == '-'; - let escaped = |before: &[u8]| matches!(before, [.., b'%', h, l] if h.is_ascii_hexdigit() && l.is_ascii_hexdigit()); + let escaped_boundary = |before: &[u8]| match before { + [.., b'%', h, l] if h.is_ascii_hexdigit() && l.is_ascii_hexdigit() => { + let hex = |d: u8| char::from(d).to_digit(16).unwrap_or(0) as u8; + !local(char::from(hex(*h) * 16 + hex(*l))) + } + _ => false, + }; text.match_indices(address).any(|(i, _)| { let after = &text[i + address.len()..]; - (!text[..i].ends_with(local) || escaped(&text.as_bytes()[..i])) + (!text[..i].ends_with(local) || escaped_boundary(&text.as_bytes()[..i])) && !after.starts_with(domain) && !(after.starts_with('.') && after[1..].starts_with(domain)) }) @@ -718,6 +725,13 @@ levers = [] email_on_pages("privacy@example.com.au", &linked), Check::Fine ); + for escape in ["%2b", "%2e", "%61"] { + let longer = [page(&format!("?to=user{escape}privacy@example.com.au"))]; + assert!(matches!( + email_on_pages("privacy@example.com.au", &longer), + Check::Dead(_) + )); + } assert_eq!( email_on_pages("Privacy@Example.com.au", &found), Check::Fine diff --git a/xtask/src/text.rs b/xtask/src/text.rs index d5ad984..30fd3db 100644 --- a/xtask/src/text.rs +++ b/xtask/src/text.rs @@ -242,21 +242,18 @@ fn after(lower: &str, from: usize, needle: &str) -> usize { /// Email addresses hidden by Cloudflare's email obfuscation, which replaces each address in /// a page with `data-cfemail=""` or a `/cdn-cgi/l/email-protection#` link. The -/// first byte of the hex is a key XORed with each following byte. Either form ends at the -/// attribute's closing quote; any other character there makes the payload malformed. +/// first byte of the hex is a key XORed with each following byte. Cloudflare writes both +/// forms in double-quoted attributes, so the hex must end at a double quote; any other +/// character there makes the payload malformed. pub fn cloudflare_emails(html: &str) -> Vec { let mut emails = Vec::new(); - let forms: [(&str, &[char]); 2] = [ - ("data-cfemail=\"", &['"']), - ("/cdn-cgi/l/email-protection#", &['"', '\'']), - ]; - for (marker, quotes) in forms { + for marker in ["data-cfemail=\"", "/cdn-cgi/l/email-protection#"] { for (i, _) in html.match_indices(marker) { let rest = &html[i + marker.len()..]; let end = rest .find(|c: char| !c.is_ascii_hexdigit()) .unwrap_or(rest.len()); - if rest[end..].starts_with(quotes) { + if rest[end..].starts_with('"') { emails.extend(decode_cloudflare(&rest[..end])); } } @@ -448,7 +445,8 @@ mod tests { Help \ \ \ - "; + \ + "; assert_eq!( cloudflare_emails(html), ["privacy@example.com.au", "help@example.com.au"] From 62132337c5b338fa9cd12024dc41eaa526c24828 Mon Sep 17 00:00:00 2001 From: EquityBot <333319867+EquityBot@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:06:33 +1100 Subject: [PATCH 08/10] feat(xtask): log refused evidence pages and keep each run's page text A 401, 403 or 429 on an evidence page now goes to the workflow log, as it already does for web forms, so runner-only bot blocks no longer hold the drift issue open. The drift run saves each HTML page's main text, one sentence per line, and the workflow uploads it as a 90-day artifact the issue links to, so a changed page can be diffed against an earlier run. --- .github/workflows/evidence-drift.yml | 13 ++- registry/README.md | 2 +- xtask/src/main.rs | 136 +++++++++++++++++++++++---- 3 files changed, 131 insertions(+), 20 deletions(-) diff --git a/.github/workflows/evidence-drift.yml b/.github/workflows/evidence-drift.yml index aca5c1d..6a4f7cc 100644 --- a/.github/workflows/evidence-drift.yml +++ b/.github/workflows/evidence-drift.yml @@ -26,12 +26,23 @@ jobs: persist-credentials: false # rustup reads the pinned version from rust-toolchain.toml. - run: rustup toolchain install - - run: cargo run --locked -p xtask -- evidence-drift registry/au > drift.md + - run: cargo run --locked -p xtask -- evidence-drift registry/au evidence-text > drift.md + # Each page's text, so a changed page can be diffed against an earlier run's copy. + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: evidence-text + path: evidence-text/ + retention-days: 90 + if-no-files-found: warn - name: Open, update or close the drift issue env: GH_TOKEN: ${{ github.token }} TITLE: Register evidence drift run: | + if [ -s drift.md ]; then + printf '\nPage text from this run, for diffing against an earlier run: %s\n' \ + "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> drift.md + fi number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --limit 100 \ --search "\"$TITLE\" in:title" --json number,title \ --jq "map(select(.title == env.TITLE)) | .[0].number // empty") diff --git a/registry/README.md b/registry/README.md index fecd87f..44ec843 100644 --- a/registry/README.md +++ b/registry/README.md @@ -49,7 +49,7 @@ A pack has `id` (equal to the file name), `name`, `description` and `entries`, a ## Evidence hashes and drift -Each `[[evidence]]` table records a source page's state when the entry was verified. A scheduled workflow (`.github/workflows/evidence-drift.yml`) re-fetches every evidence page on the first of each month and opens or updates one issue, 'Register evidence drift', listing pages that changed or could not be read within 30 seconds, and closes that issue once every page matches again. It never commits. When a page changes, recheck the entry against it and update the hash and `checked_at`. +Each `[[evidence]]` table records a source page's state when the entry was verified. A scheduled workflow (`.github/workflows/evidence-drift.yml`) re-fetches every evidence page on the first of each month and opens or updates one issue, 'Register evidence drift', listing pages that changed or could not be read within 30 seconds, and closes that issue once every page matches again. A page that answers 401, 403 or 429 goes to the workflow log instead, because sites answer scripts that way when the page is up: its hash is not confirmed that month. It never commits. Each run keeps every HTML page's main text, one sentence per line, as the `evidence-text` artifact for 90 days, and the issue links to it. When a page changes, diff its file against an earlier run's copy, recheck the entry against the page and update the hash and `checked_at`. The same run checks the contact channels of every entry that is not obsolete. It looks up each email address's domain through Google Public DNS (`dns.google`) and reports one that does not exist, has no mail or address record, or publishes a null MX record saying it accepts no mail. It also requests each web-form address and reports an HTTP error or a host public DNS says does not exist. A 401, 403, 405, 429 or 5xx, a dropped connection or a failed DNS lookup proves nothing, because sites also answer scripts that way when the form works and such failures pass, so those go to the workflow log and not the issue: a form or domain behind one of them is not confirmed to work. It also reports an email address that appears on none of its entry's source pages. It searches the evidence pages first and fetches the entry's other sources only when an address is not on them, searching each page's raw HTML, its decoded main text and any address hidden by Cloudflare's email obfuscation, in any letter case; a missing address goes to the workflow log instead when one of those pages failed, has under 500 characters of main text or is not HTML. These checks never send an email or submit a form, and a passing domain does not prove the mailbox exists. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 176024a..b8e6e4d 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -3,7 +3,9 @@ //! //! ```text //! cargo run -p xtask -- evidence-hash ... print the evidence hash of each page -//! cargo run -p xtask -- evidence-drift [registry/au] report pages whose hash has changed +//! cargo run -p xtask -- evidence-drift [registry/au [text dir]] +//! report pages whose hash has changed, +//! optionally saving each page's text //! cargo run -p xtask -- email-changes warn of email addresses a change added //! ``` @@ -33,20 +35,41 @@ fn agent(timeout: Duration) -> ureq::Agent { /// A fetched evidence page. struct Page { sha256: String, + /// The normalised main text the hash covers; `None` for a non-HTML page. + main: Option, /// Lowercase raw HTML, main text and Cloudflare-protected addresses, for finding an /// email address in a link, a footer, or entity-encoded or obfuscated text; `None` for /// a non-HTML page. text: Option, } +/// Why an evidence page could not be hashed. +#[derive(Debug)] +enum PageError { + /// 401, 403 or 429: sites answer scripts this way even when the page is up. + Refused(String), + Failed(String), +} + +impl std::fmt::Display for PageError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Refused(e) | Self::Failed(e) => f.write_str(e), + } + } +} + /// Fetch `url`. Its evidence hash is SHA-256 of its normalised main text, or of the raw /// bytes for a non-HTML response. -fn evidence_page(agent: &ureq::Agent, url: &str) -> Result { +fn evidence_page(agent: &ureq::Agent, url: &str) -> Result { let mut response = agent .get(url) .header("User-Agent", USER_AGENT) .call() - .map_err(|e| e.to_string())?; + .map_err(|e| match e { + ureq::Error::StatusCode(401 | 403 | 429) => PageError::Refused(e.to_string()), + _ => PageError::Failed(e.to_string()), + })?; let content_type = response .headers() .get("content-type") @@ -56,28 +79,50 @@ fn evidence_page(agent: &ureq::Agent, url: &str) -> Result { .with_config() .limit(20 * 1024 * 1024) .read_to_vec() - .map_err(|e| e.to_string())?; - let (digest, text) = if text::is_html(content_type.as_deref(), &bytes) { + .map_err(|e| PageError::Failed(e.to_string()))?; + let (digest, main, text) = if text::is_html(content_type.as_deref(), &bytes) { let html = String::from_utf8_lossy(&bytes); let main = text::main_text(&html); if main.chars().count() < MIN_TEXT { - return Err(format!( + return Err(PageError::Failed(format!( "only {} characters of main text; the page may need a browser", main.chars().count() - )); + ))); } let hidden = text::cloudflare_emails(&html).join("\n"); let searchable = format!("{html}\n{main}\n{hidden}").to_lowercase(); - (Sha256::digest(main.as_bytes()), Some(searchable)) + ( + Sha256::digest(main.as_bytes()), + Some(main), + Some(searchable), + ) } else { - (Sha256::digest(&bytes), None) + (Sha256::digest(&bytes), None, None) }; Ok(Page { - sha256: digest.iter().map(|b| format!("{b:02x}")).collect(), + sha256: hex(&digest), + main, text, }) } +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// Save a page's main text as `-.txt` under `dir`, so a later run's +/// copy can be diffed against it. The URL is the first line; after a blank line each +/// sentence starts a line, and joining those lines with spaces gives the hashed text. +fn save_text(dir: &Path, id: &str, url: &str, main: &str) { + let name = format!("{id}-{}.txt", hex(&Sha256::digest(url.as_bytes())[..4])); + let body = format!("{url}\n\n{}\n", main.replace(". ", ".\n")); + if let Err(e) = + std::fs::create_dir_all(dir).and_then(|()| std::fs::write(dir.join(&name), body)) + { + eprintln!("warning: cannot save the text of {url}: {e}"); + } +} + fn hash(urls: &[String]) -> ExitCode { let agent = agent(TIMEOUT); let mut failed = false; @@ -98,8 +143,9 @@ fn hash(urls: &[String]) -> ExitCode { } /// Prints a Markdown report of changed and failing evidence pages, or nothing if every -/// page still matches. Exits 0 either way; a register that cannot be read exits 1. -fn drift(dir: &Path) -> ExitCode { +/// page still matches, and saves each HTML page's text under `text_dir` if given. Exits 0 +/// either way; a register that cannot be read exits 1. +fn drift(dir: &Path, text_dir: Option<&Path>) -> ExitCode { let files = match ghostsoap::io::register::load(dir) { Ok(files) => files, Err(e) => { @@ -121,6 +167,7 @@ fn drift(dir: &Path) -> ExitCode { continue; } }; + let ids = std::slice::from_ref(&entry.id); let mut pages = Vec::new(); for evidence in &entry.evidence { match evidence_page(&agent, &evidence.url) { @@ -131,17 +178,23 @@ fn drift(dir: &Path) -> ExitCode { entry.id, evidence.url, evidence.checked_at )); } + if let (Some(dir), Some(main)) = (text_dir, &page.main) { + save_text(dir, &entry.id, &evidence.url, main); + } pages.push(page.text); } - Err(e) => { - failing.push(format!("- `{}`: {}: {e}", entry.id, evidence.url)); + Err(PageError::Refused(e)) => { + record(&mut failing, &evidence.url, ids, Check::Inconclusive(e)); + pages.push(None); + } + Err(PageError::Failed(e)) => { + record(&mut failing, &evidence.url, ids, Check::Dead(e)); pages.push(None); } } } if entry.obsolete.is_none() { collect_channels(&entry, &mut mail_domains, &mut forms); - let ids = std::slice::from_ref(&entry.id); for (address, check) in email_checks(&agent, &entry, pages) { record(&mut emails, &format!("email {address}"), ids, check); } @@ -504,15 +557,17 @@ fn main() -> ExitCode { let args: Vec = std::env::args().skip(1).collect(); match args.split_first() { Some((command, urls)) if command == "evidence-hash" && !urls.is_empty() => hash(urls), - Some((command, rest)) if command == "evidence-drift" && rest.len() <= 1 => drift( + Some((command, rest)) if command == "evidence-drift" && rest.len() <= 2 => drift( Path::new(rest.first().map_or("registry/au", String::as_str)), + rest.get(1).map(Path::new), ), Some((command, [base, head])) if command == "email-changes" => { emails_changed(Path::new(base), Path::new(head)) } _ => { eprintln!( - "usage: xtask evidence-hash ... | xtask evidence-drift [registry dir] \ + "usage: xtask evidence-hash ... \ + | xtask evidence-drift [registry dir [page text dir]] \ | xtask email-changes " ); ExitCode::from(2) @@ -825,6 +880,51 @@ levers = [] assert_eq!(checks, [("privacy@Example.com.au".to_owned(), Check::Fine)]); } + #[test] + fn a_page_refused_to_scripts_is_not_a_failure() { + let agent = agent(Duration::from_secs(5)); + for status in ["401 Unauthorized", "403 Forbidden", "429 Too Many Requests"] { + let page = evidence_page(&agent, &serve_once(status)).map(|p| p.sha256); + assert!( + matches!(page, Err(PageError::Refused(_))), + "{status}: {page:?}" + ); + } + for status in ["404 Not Found", "410 Gone", "500 Internal Server Error"] { + let page = evidence_page(&agent, &serve_once(status)).map(|p| p.sha256); + assert!( + matches!(page, Err(PageError::Failed(_))), + "{status}: {page:?}" + ); + } + } + + #[test] + fn saved_text_rejoins_to_the_hashed_text() { + let url = serve_html("First sentence. Second sentence. Third."); + let page = evidence_page(&agent(Duration::from_secs(5)), &url).unwrap(); + let main = page.main.unwrap(); + let dir = std::env::temp_dir().join(format!("xtask-text-{}", std::process::id())); + save_text(&dir, "example-au", &url, &main); + let files: Vec<_> = std::fs::read_dir(&dir) + .unwrap() + .map(|f| f.unwrap().path()) + .collect(); + assert_eq!(files.len(), 1); + let name = files[0].file_name().unwrap().to_string_lossy().into_owned(); + assert!( + name.starts_with("example-au-") && name.ends_with(".txt"), + "{name}" + ); + let saved = std::fs::read_to_string(&files[0]).unwrap(); + std::fs::remove_dir_all(&dir).unwrap(); + let (head, body) = saved.split_once("\n\n").unwrap(); + assert_eq!(head, url); + assert!(body.contains("Second sentence.\nThird."), "{body}"); + let rejoined = body.trim_end_matches('\n').replace('\n', " "); + assert_eq!(hex(&Sha256::digest(rejoined.as_bytes())), page.sha256); + } + fn source(text: &str) -> SourceFile { SourceFile { name: "example-au.toml".to_owned(), @@ -894,7 +994,7 @@ levers = [] let start = Instant::now(); let result = evidence_page(&agent(Duration::from_millis(300)), &url).map(|p| p.sha256); assert!( - result.as_ref().is_err_and(|e| e.contains("timeout")), + matches!(&result, Err(PageError::Failed(e)) if e.contains("timeout")), "{result:?}" ); assert!(start.elapsed() < Duration::from_secs(10)); From 9ac8b6fdaa993075bdaf630342d566fc2d9a0bb4 Mon Sep 17 00:00:00 2001 From: EquityBot <333319867+EquityBot@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:06:34 +1100 Subject: [PATCH 09/10] fix(register): rehash Canva's privacy policy after its 2 November revision Only the EEA representative's address changed; the register's claims hold. --- docs/provenance.md | 3 ++- registry/au/canva-au.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/provenance.md b/docs/provenance.md index 33cfec4..388adb8 100644 --- a/docs/provenance.md +++ b/docs/provenance.md @@ -170,7 +170,8 @@ Evidence: 52 `[[evidence]]` hashes across 33 of these entries, taken on 26 Octob The first monthly drift check ([issue 26](https://github.com/EquityBot/GhostSoap/issues/26)) was worked through on 2 November 2026 by an AI agent for the maintainer, from public pages only: no form was submitted, no account created and no one signed in. Each flagged page was read in a browser (Firecrawl for 3 pages behind a bot check) and its entry's claims were matched against the text; new hashes were taken twice with `cargo run -p xtask -- evidence-hash` and matched. -- Hash updated, claims unchanged: `canva-au`, `equifax-marketing-au` (donotcontact.com.au), `everyday-rewards-au` (both HTML pages; the policy page still links the August 2026 PDF, whose hash is unchanged), `michael-page-au` and `records-finder-au` (4 pages). +- `canva-au`: hash updated, claims unchanged. Canva published a new policy version later the same day ('last updated on November 02, 2026', was August 25); a sentence diff of the 2 versions shows only its EEA representative's address changed, which the register does not record, so the hash was taken again (3 matching fetches). +- Hash updated, claims unchanged: `equifax-marketing-au` (donotcontact.com.au), `everyday-rewards-au` (both HTML pages; the policy page still links the August 2026 PDF, whose hash is unchanged), `michael-page-au` and `records-finder-au` (4 pages). - `lead-lists-au`: the privacy policy now sends requests to a new Privacy Requests page, updated 28 October 2026, which gives `suppressions@leadlists.com.au` (was `suppression@`) and a postal address in Chadstone. The page answers a plain fetch with a bot check, so it has no hash. - `dymocks-au`: www.dymocks.group now redirects to the policy on www.dymocks.com.au, which names Dymocks Pty Ltd, gives `privacy@dymocks.com.au` (was `privacy@dymocks.group`) and the same Level 6, 428 George Street address, and links a data deletion request form. That page answers a plain fetch with 403, so the entry has no hash. - Evidence removed: the 2 Coles HTML pages and Harvey Norman's policy now answer a plain fetch with a script-only bot check (0 characters of main text); Equifax's contact page and credit ban form show the current date as a date field example ('E.g., 02/11/2026'), so their hashes change every day, and Records Finder's Australian search page shows the date its database was last updated, which changed within the day. Their claims still match the pages in a browser; Coles's PDF policy and Equifax's other 3 pages keep their hashes. diff --git a/registry/au/canva-au.toml b/registry/au/canva-au.toml index 805e197..33404d5 100644 --- a/registry/au/canva-au.toml +++ b/registry/au/canva-au.toml @@ -32,4 +32,4 @@ levers = ["app7-no-marketing"] [[evidence]] url = "https://www.canva.com/policies/privacy-policy/" checked_at = 2026-11-02 -sha256 = "b51d2e8073e901dbbbc0a76031ff92589e3da591ef367a4b41df145cd7eff0db" +sha256 = "c06899afe969c0503724158b07be3f46bc002fade3161d321820ef59c3f7f93f" From df13df33fe07c3e2a3acfabcf4a65dc137848764 Mon Sep 17 00:00:00 2001 From: EquityBot <333319867+EquityBot@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:09:01 +1100 Subject: [PATCH 10/10] test(xtask): check saved page text without a temporary directory --- xtask/src/main.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/xtask/src/main.rs b/xtask/src/main.rs index b8e6e4d..1fa89fa 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -114,8 +114,7 @@ fn hex(bytes: &[u8]) -> String { /// copy can be diffed against it. The URL is the first line; after a blank line each /// sentence starts a line, and joining those lines with spaces gives the hashed text. fn save_text(dir: &Path, id: &str, url: &str, main: &str) { - let name = format!("{id}-{}.txt", hex(&Sha256::digest(url.as_bytes())[..4])); - let body = format!("{url}\n\n{}\n", main.replace(". ", ".\n")); + let (name, body) = text_file(id, url, main); if let Err(e) = std::fs::create_dir_all(dir).and_then(|()| std::fs::write(dir.join(&name), body)) { @@ -123,6 +122,12 @@ fn save_text(dir: &Path, id: &str, url: &str, main: &str) { } } +/// The file name and contents `save_text` writes. +fn text_file(id: &str, url: &str, main: &str) -> (String, String) { + let name = format!("{id}-{}.txt", hex(&Sha256::digest(url.as_bytes())[..4])); + (name, format!("{url}\n\n{}\n", main.replace(". ", ".\n"))) +} + fn hash(urls: &[String]) -> ExitCode { let agent = agent(TIMEOUT); let mut failed = false; @@ -904,20 +909,15 @@ levers = [] let url = serve_html("First sentence. Second sentence. Third."); let page = evidence_page(&agent(Duration::from_secs(5)), &url).unwrap(); let main = page.main.unwrap(); - let dir = std::env::temp_dir().join(format!("xtask-text-{}", std::process::id())); - save_text(&dir, "example-au", &url, &main); - let files: Vec<_> = std::fs::read_dir(&dir) - .unwrap() - .map(|f| f.unwrap().path()) - .collect(); - assert_eq!(files.len(), 1); - let name = files[0].file_name().unwrap().to_string_lossy().into_owned(); + let (name, saved) = text_file("example-au", &url, &main); assert!( name.starts_with("example-au-") && name.ends_with(".txt"), "{name}" ); - let saved = std::fs::read_to_string(&files[0]).unwrap(); - std::fs::remove_dir_all(&dir).unwrap(); + assert_ne!( + name, + text_file("example-au", "https://example.com.au/", &main).0 + ); let (head, body) = saved.split_once("\n\n").unwrap(); assert_eq!(head, url); assert!(body.contains("Second sentence.\nThird."), "{body}");