diff --git a/.github/workflows/evidence-drift.yml b/.github/workflows/evidence-drift.yml index aca5c1d..6a4f7cc 100644 --- a/.github/workflows/evidence-drift.yml +++ b/.github/workflows/evidence-drift.yml @@ -26,12 +26,23 @@ jobs: persist-credentials: false # rustup reads the pinned version from rust-toolchain.toml. - run: rustup toolchain install - - run: cargo run --locked -p xtask -- evidence-drift registry/au > drift.md + - run: cargo run --locked -p xtask -- evidence-drift registry/au evidence-text > drift.md + # Each page's text, so a changed page can be diffed against an earlier run's copy. + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: evidence-text + path: evidence-text/ + retention-days: 90 + if-no-files-found: warn - name: Open, update or close the drift issue env: GH_TOKEN: ${{ github.token }} TITLE: Register evidence drift run: | + if [ -s drift.md ]; then + printf '\nPage text from this run, for diffing against an earlier run: %s\n' \ + "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> drift.md + fi number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --limit 100 \ --search "\"$TITLE\" in:title" --json number,title \ --jq "map(select(.title == env.TITLE)) | .[0].number // empty") diff --git a/docs/provenance.md b/docs/provenance.md index ca2d975..388adb8 100644 --- a/docs/provenance.md +++ b/docs/provenance.md @@ -110,7 +110,7 @@ Levers: every entry with an ABN lists `app11-destroy` (sent only when the user n | `chandler-macleod-au` | Maintainer's request | Privacy policy and contact pages on www.chandlermacleod.com, www.rgfstaffing.com.au | 84 603 568 403: RGF STAFFING ANZ PTY LTD, active from 06 Jan 2015 | | `charterhouse-au` | Maintainer's request | Privacy policy and contact pages on www.charterhouse.com.au | 40 078 713 774: CHARTERHOUSE RECRUITMENT PTY LTD, active from 01 Mar 2000 | | `coles-au` | Large Australian firm | Privacy policy and contact pages on www.coles.com.au | 11 004 089 936: COLES GROUP LIMITED, active from 09 Feb 2000 | -| `dymocks-au` | Large Australian data breach | Privacy policy and contact pages on www.dymocks.group | 59 008 453 110: DYMOCKS HOLDINGS PTY LTD, active from 01 Nov 1999 | +| `dymocks-au` | Large Australian data breach | Privacy policy on www.dymocks.com.au (rechecked 2 November 2026; see below) | 64 074 571 083: DYMOCKS PTY LTD, active from 05 Dec 1999 (ABN Lookup, 2 November 2026) | | `ebay-au` | Large Australian firm | Privacy policy and contact pages on www.ebay.com.au, www.ebayinc.com | 64 652 016 681: eBay Marketplaces GmbH, active from 01 Jun 2018 | | `harvey-norman-au` | Large Australian firm | Privacy policy and contact pages on www.harveynorman.com.au, www.harveynormanholdings.com.au | 14 818 873 384: THE TRUSTEE FOR THE ECOMM STORE NO 2 TRUST, active from 08 Sep 2010 | | `hays-au` | Maintainer's request | Privacy policy and contact pages on www.hays.com.au | 47 001 407 281: HAYS SPECIALIST RECRUITMENT (AUSTRALIA) PTY LIMITED, active from 16 Feb 2000 | @@ -166,6 +166,17 @@ Evidence: 52 `[[evidence]]` hashes across 33 of these entries, taken on 26 Octob - `telstra-au`, `optus-au`, `tpg-telecom-au`, `aussie-broadband-au`: members of the Telecommunications Industry Ombudsman, which `escalation` cannot yet name. - Rejected: MyDeal (closed; the site refused connections), PeopleIN (its policy covers its own staff and host employers, not its agencies' candidates). +### Evidence drift recheck (2 November 2026) + +The first monthly drift check ([issue 26](https://github.com/EquityBot/GhostSoap/issues/26)) was worked through on 2 November 2026 by an AI agent for the maintainer, from public pages only: no form was submitted, no account created and no one signed in. Each flagged page was read in a browser (Firecrawl for 3 pages behind a bot check) and its entry's claims were matched against the text; new hashes were taken twice with `cargo run -p xtask -- evidence-hash` and matched. + +- `canva-au`: hash updated, claims unchanged. Canva published a new policy version later the same day ('last updated on November 02, 2026', was August 25); a sentence diff of the 2 versions shows only its EEA representative's address changed, which the register does not record, so the hash was taken again (3 matching fetches). +- Hash updated, claims unchanged: `equifax-marketing-au` (donotcontact.com.au), `everyday-rewards-au` (both HTML pages; the policy page still links the August 2026 PDF, whose hash is unchanged), `michael-page-au` and `records-finder-au` (4 pages). +- `lead-lists-au`: the privacy policy now sends requests to a new Privacy Requests page, updated 28 October 2026, which gives `suppressions@leadlists.com.au` (was `suppression@`) and a postal address in Chadstone. The page answers a plain fetch with a bot check, so it has no hash. +- `dymocks-au`: www.dymocks.group now redirects to the policy on www.dymocks.com.au, which names Dymocks Pty Ltd, gives `privacy@dymocks.com.au` (was `privacy@dymocks.group`) and the same Level 6, 428 George Street address, and links a data deletion request form. That page answers a plain fetch with 403, so the entry has no hash. +- Evidence removed: the 2 Coles HTML pages and Harvey Norman's policy now answer a plain fetch with a script-only bot check (0 characters of main text); Equifax's contact page and credit ban form show the current date as a date field example ('E.g., 02/11/2026'), so their hashes change every day, and Records Finder's Australian search page shows the date its database was last updated, which changed within the day. Their claims still match the pages in a browser; Coles's PDF policy and Equifax's other 3 pages keep their hashes. +- Contact channels: the addresses of `aussie-broadband-au`, `careerone-au`, `hays-au`, `manpowergroup-au`, `robert-half-au` and `smrtr-au` are on their pages behind Cloudflare's email obfuscation, which the check now decodes; those of `chandler-macleod-au` and `peoplebank-au` are on the RGF Staffing policy and Thryv's on its privacy policy, sources without evidence, which the check now searches. + ### Rejected candidates (M1) | Candidate | Reason | diff --git a/registry/README.md b/registry/README.md index 7962322..44ec843 100644 --- a/registry/README.md +++ b/registry/README.md @@ -49,9 +49,9 @@ A pack has `id` (equal to the file name), `name`, `description` and `entries`, a ## Evidence hashes and drift -Each `[[evidence]]` table records a source page's state when the entry was verified. A scheduled workflow (`.github/workflows/evidence-drift.yml`) re-fetches every evidence page on the first of each month and opens or updates one issue, 'Register evidence drift', listing pages that changed or could not be read within 30 seconds, and closes that issue once every page matches again. It never commits. When a page changes, recheck the entry against it and update the hash and `checked_at`. +Each `[[evidence]]` table records a source page's state when the entry was verified. A scheduled workflow (`.github/workflows/evidence-drift.yml`) re-fetches every evidence page on the first of each month and opens or updates one issue, 'Register evidence drift', listing pages that changed or could not be read within 30 seconds, and closes that issue once every page matches again. A page that answers 401, 403 or 429 goes to the workflow log instead, because sites answer scripts that way when the page is up: its hash is not confirmed that month. It never commits. Each run keeps every HTML page's main text, one sentence per line, as the `evidence-text` artifact for 90 days, and the issue links to it. When a page changes, diff its file against an earlier run's copy, recheck the entry against the page and update the hash and `checked_at`. -The same run checks the contact channels of every entry that is not obsolete. It looks up each email address's domain through Google Public DNS (`dns.google`) and reports one that does not exist, has no mail or address record, or publishes a null MX record saying it accepts no mail. It also requests each web-form address and reports an HTTP error or a host public DNS says does not exist. A 401, 403, 405, 429 or 5xx, a dropped connection or a failed DNS lookup proves nothing, because sites also answer scripts that way when the form works and such failures pass, so those go to the workflow log and not the issue: a form or domain behind one of them is not confirmed to work. It also reports an email address that appears on none of its entry's evidence pages, searching each page's raw HTML and decoded main text in any letter case; a missing address goes to the workflow log instead when its entry has no evidence page, or has one that failed or is not HTML. These checks never send an email or submit a form, and a passing domain does not prove the mailbox exists. +The same run checks the contact channels of every entry that is not obsolete. It looks up each email address's domain through Google Public DNS (`dns.google`) and reports one that does not exist, has no mail or address record, or publishes a null MX record saying it accepts no mail. It also requests each web-form address and reports an HTTP error or a host public DNS says does not exist. A 401, 403, 405, 429 or 5xx, a dropped connection or a failed DNS lookup proves nothing, because sites also answer scripts that way when the form works and such failures pass, so those go to the workflow log and not the issue: a form or domain behind one of them is not confirmed to work. It also reports an email address that appears on none of its entry's source pages. It searches the evidence pages first and fetches the entry's other sources only when an address is not on them, searching each page's raw HTML, its decoded main text and any address hidden by Cloudflare's email obfuscation, in any letter case; a missing address goes to the workflow log instead when one of those pages failed, has under 500 characters of main text or is not HTML. These checks never send an email or submit a form, and a passing domain does not prove the mailbox exists. On a pull request, CI compares the register with the base branch's and adds a warning annotation for each email address an entry gains, and for every address of an entry whose id is new, so a renamed entry is flagged too (`cargo run -p xtask -- email-changes registry/au`). Domains compare case-insensitively; local parts do not. The check passes either way, but fails on an entry that does not parse: check each flagged address on the entity's own pages before merging. diff --git a/registry/au/canva-au.toml b/registry/au/canva-au.toml index f66b9b3..33404d5 100644 --- a/registry/au/canva-au.toml +++ b/registry/au/canva-au.toml @@ -5,7 +5,7 @@ abn = "80 158 929 938" kinds = ["online-service"] priority = "high" quality = "verified" -verified_at = 2026-10-26 +verified_at = 2026-11-02 sources = ["https://www.canva.com/policies/privacy-policy/", "https://www.canva.com/help/legal-form/", "https://www.canva.com/settings/email-preferences"] levers = ["app7-no-marketing", "app7-source", "app11-destroy", "app13", "app13-notify-recipients", "app13-statement"] opt_in_levers = ["app12"] @@ -31,5 +31,5 @@ levers = ["app7-no-marketing"] [[evidence]] url = "https://www.canva.com/policies/privacy-policy/" -checked_at = 2026-10-26 -sha256 = "95df1957be8349f36e6521af6567318fc594643149a84da04919bf98c47df73b" +checked_at = 2026-11-02 +sha256 = "c06899afe969c0503724158b07be3f46bc002fade3161d321820ef59c3f7f93f" diff --git a/registry/au/coles-au.toml b/registry/au/coles-au.toml index b6a0887..72b179c 100644 --- a/registry/au/coles-au.toml +++ b/registry/au/coles-au.toml @@ -5,7 +5,7 @@ abn = "11 004 089 936" kinds = ["retail"] priority = "standard" quality = "verified" -verified_at = 2026-10-26 +verified_at = 2026-11-02 sources = ["https://www.coles.com.au/content/dam/coles/cusp/about-coles/documents/coles_group_privacy_policy_september_2025.pdf", "https://www.coles.com.au/privacy", "https://www.coles.com.au/help/accounts/delete-your-coles-account"] levers = ["app7-no-marketing", "app7-source", "app11-destroy", "app13", "app13-notify-recipients", "app13-statement"] opt_in_levers = ["app12"] @@ -33,13 +33,3 @@ levers = [] url = "https://www.coles.com.au/content/dam/coles/cusp/about-coles/documents/coles_group_privacy_policy_september_2025.pdf" checked_at = 2026-10-26 sha256 = "29adbc5d5c12d46560b01b06c12ca7069edb071647790378426043bf15b82316" - -[[evidence]] -url = "https://www.coles.com.au/privacy" -checked_at = 2026-10-26 -sha256 = "dd1b09e2a5e277cd06f930fb7a6eeb82a3f1f8204a95690e293fc9cd82e31c53" - -[[evidence]] -url = "https://www.coles.com.au/help/accounts/delete-your-coles-account" -checked_at = 2026-10-26 -sha256 = "a7cff50450d9b61599ce0a0ed1217e34ca3f574814807cd417ce27edcabe3a40" diff --git a/registry/au/dymocks-au.toml b/registry/au/dymocks-au.toml index 9d8a341..6da5678 100644 --- a/registry/au/dymocks-au.toml +++ b/registry/au/dymocks-au.toml @@ -1,12 +1,12 @@ id = "dymocks-au" name = "Dymocks" -legal_entity = "Dymocks Holdings Pty Ltd" -abn = "59 008 453 110" +legal_entity = "Dymocks Pty Ltd" +abn = "64 074 571 083" kinds = ["retail"] priority = "high" quality = "verified" -verified_at = 2026-10-26 -sources = ["https://www.dymocks.group/privacy-policy/"] +verified_at = 2026-11-02 +sources = ["https://www.dymocks.com.au/privacy-policy/"] levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app11-destroy", "app13", "app13-notify-recipients", "app13-statement"] opt_in_levers = ["app12"] identity_fields = ["full_name", "email", "phone", "address"] @@ -16,15 +16,10 @@ effort = "easy" [[channels]] kind = "email" -address = "privacy@dymocks.group" +address = "privacy@dymocks.com.au" levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app11-destroy", "app13", "app13-notify-recipients", "app13-statement", "app12"] [[channels]] kind = "post" -address = "Privacy Officer, Level 6, 428 George Street, Sydney NSW 2000" +address = "Privacy Officer, Dymocks Pty Ltd, Level 6, 428 George Street, Sydney NSW 2000" levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app11-destroy", "app13", "app13-notify-recipients", "app13-statement", "app12"] - -[[evidence]] -url = "https://www.dymocks.group/privacy-policy/" -checked_at = 2026-10-26 -sha256 = "69afc5faf986fb4a1e015daee40340efc3cb834b5b217b049ffd017e38de2df3" diff --git a/registry/au/equifax-au.toml b/registry/au/equifax-au.toml index 4d0f914..daa9d20 100644 --- a/registry/au/equifax-au.toml +++ b/registry/au/equifax-au.toml @@ -5,7 +5,7 @@ abn = "26 000 602 862" kinds = ["credit-reporting-body"] priority = "crucial" quality = "verified" -verified_at = 2026-10-24 +verified_at = 2026-11-02 sources = ["https://www.equifax.com.au/credit-reporting-policy", "https://www.equifax.com.au/contact", "https://www.equifax.com.au/personal/products/credit-and-identity-products", "https://www.equifax.com.au/personal/corrections-portal", "https://www.equifax.com.au/eform/submit/credit-ban", "https://www.equifax.com.au/eform/submit/complaints-form"] levers = ["s20t-correction"] opt_in_levers = ["s20r-access", "s20k-ban"] @@ -52,11 +52,6 @@ url = "https://www.equifax.com.au/credit-reporting-policy" checked_at = 2026-10-24 sha256 = "7778d413ce89cada85159267cb40976f9189b088cd77cbf0dee865e5ea7dfd72" -[[evidence]] -url = "https://www.equifax.com.au/contact" -checked_at = 2026-10-24 -sha256 = "b3859a399f75a35d75fe8c2041cf4eb57b4dd0bf6511489db51136c62102bf81" - [[evidence]] url = "https://www.equifax.com.au/personal/products/credit-and-identity-products" checked_at = 2026-10-24 @@ -66,8 +61,3 @@ sha256 = "c20f72dfd65dbed954440fb3494f30d4fa1c11e057effd2e551a7d9d63509068" url = "https://www.equifax.com.au/personal/corrections-portal" checked_at = 2026-10-24 sha256 = "a59e502c73dc52851831e5553e8ee57dccb653459cdb8896271b7f6ae635fa56" - -[[evidence]] -url = "https://www.equifax.com.au/eform/submit/credit-ban" -checked_at = 2026-10-24 -sha256 = "52c91f70960defd41b361559e6b4921439daec6794386499dd552532e0db8f4f" diff --git a/registry/au/equifax-marketing-au.toml b/registry/au/equifax-marketing-au.toml index 4a9e5d9..12eb70b 100644 --- a/registry/au/equifax-marketing-au.toml +++ b/registry/au/equifax-marketing-au.toml @@ -5,7 +5,7 @@ abn = "88 071 215 328" kinds = ["marketing-list"] priority = "high" quality = "verified" -verified_at = 2026-10-24 +verified_at = 2026-11-02 sources = ["https://www.equifax.com.au/privacy", "https://www.donotcontact.com.au/"] levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app13", "app13-notify-recipients", "app13-statement"] opt_in_levers = ["app12"] @@ -37,5 +37,5 @@ sha256 = "aaeaf4225ee4cbade081ece48db356b14608b81ff2148ae93b5b8f705ff04257" [[evidence]] url = "https://www.donotcontact.com.au/" -checked_at = 2026-10-24 -sha256 = "3acc7039a6121135ee06d56a54aad9a5a811181d2eabd699947fccdc86ba996b" +checked_at = 2026-11-02 +sha256 = "736dd3915c42f1a772efd5ee100c050f61d106524e148b1af409c617ef644ce7" diff --git a/registry/au/everyday-rewards-au.toml b/registry/au/everyday-rewards-au.toml index ad5ea20..d6efc87 100644 --- a/registry/au/everyday-rewards-au.toml +++ b/registry/au/everyday-rewards-au.toml @@ -5,7 +5,7 @@ abn = "88 000 014 675" kinds = ["loyalty"] priority = "standard" quality = "verified" -verified_at = 2026-10-24 +verified_at = 2026-11-02 sources = ["https://www.woolworthsgroup.com.au/au/en/privacy/privacy-policy.html", "https://www.woolworthsgroup.com.au/content/dam/wwg/privacy/2026/FINAL_Woolworths-Group-Privacy-Policy-August-2026.pdf", "https://www.woolworthsgroup.com.au/au/en/privacy.html"] levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app13", "app13-notify-recipients", "app13-statement"] opt_in_levers = ["app12"] @@ -32,8 +32,8 @@ levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app13", " [[evidence]] url = "https://www.woolworthsgroup.com.au/au/en/privacy/privacy-policy.html" -checked_at = 2026-10-24 -sha256 = "3d937e82f1c70a519fdf63dcb421ea14eb0aef3ceba370f5b79695c90c99d99b" +checked_at = 2026-11-02 +sha256 = "853190e0b024601f2c102f0b6ed97bb18b8c51632bdaf6ac02b205aff8a7c3e4" [[evidence]] url = "https://www.woolworthsgroup.com.au/content/dam/wwg/privacy/2026/FINAL_Woolworths-Group-Privacy-Policy-August-2026.pdf" @@ -42,5 +42,5 @@ sha256 = "70408cddb998cbffabe6b110d3deccf7cb7b84595f41f5f9e6e74dd2984f8e0b" [[evidence]] url = "https://www.woolworthsgroup.com.au/au/en/privacy.html" -checked_at = 2026-10-24 -sha256 = "859fb2ef90dadea90d7f6d149580c5d53bd796c562528eea6d89ebcab7ac4a3a" +checked_at = 2026-11-02 +sha256 = "b801887a60188d5a87dfff8319b91c8d3d2e7a462a892fc17741aada30d273b9" diff --git a/registry/au/harvey-norman-au.toml b/registry/au/harvey-norman-au.toml index 7f7a3a8..0b01062 100644 --- a/registry/au/harvey-norman-au.toml +++ b/registry/au/harvey-norman-au.toml @@ -5,7 +5,7 @@ abn = "14 818 873 384" kinds = ["retail"] priority = "standard" quality = "verified" -verified_at = 2026-10-26 +verified_at = 2026-11-02 sources = ["https://www.harveynorman.com.au/privacy-policy", "https://www.harveynormanholdings.com.au/privacy-policy"] levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app11-destroy", "app13", "app13-notify-recipients", "app13-statement"] opt_in_levers = ["app12"] @@ -28,8 +28,3 @@ levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app11-des kind = "phone" address = "02 9201 6111" levers = [] - -[[evidence]] -url = "https://www.harveynorman.com.au/privacy-policy" -checked_at = 2026-10-26 -sha256 = "db0c4e5daafd56008fda9aa126c41662892d93f3d89cddf7a12d8be12e0cc4d9" diff --git a/registry/au/lead-lists-au.toml b/registry/au/lead-lists-au.toml index 5fbea79..afe45df 100644 --- a/registry/au/lead-lists-au.toml +++ b/registry/au/lead-lists-au.toml @@ -5,8 +5,8 @@ abn = "53 634 428 470" kinds = ["marketing-list"] priority = "high" quality = "verified" -verified_at = 2026-10-24 -sources = ["https://leadlists.com.au/privacy-policy/", "https://leadlists.com.au/contact-us/"] +verified_at = 2026-11-02 +sources = ["https://leadlists.com.au/privacy-policy/", "https://leadlists.com.au/privacy-requests/", "https://leadlists.com.au/contact-us/"] levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app13", "app13-notify-recipients", "app13-statement"] opt_in_levers = ["app12"] identity_fields = ["full_name", "email", "phone", "address"] @@ -16,10 +16,15 @@ effort = "easy" [[channels]] kind = "email" -address = "suppression@leadlists.com.au" +address = "suppressions@leadlists.com.au" levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app13", "app13-notify-recipients", "app13-statement", "app12"] preferred = true +[[channels]] +kind = "post" +address = "Privacy Officer, Lead Lists Data Solutions, Suite 60, Level 2, 1341 Dandenong Rd, Chadstone VIC 3148" +levers = ["app7-no-marketing", "app7-no-facilitation", "app7-source", "app13", "app13-notify-recipients", "app13-statement", "app12"] + [[channels]] kind = "phone" address = "(03) 9007 0355" @@ -27,5 +32,5 @@ levers = [] [[evidence]] url = "https://leadlists.com.au/privacy-policy/" -checked_at = 2026-10-24 -sha256 = "910c2ea7301eec5d996d78b3a6acc4c3ecd17853b7509498eee8c710daa39fc2" +checked_at = 2026-11-02 +sha256 = "679c8de7a59a4872c7589a0fe78fdbfea13b4258b896970b58265583ba0eec7b" diff --git a/registry/au/michael-page-au.toml b/registry/au/michael-page-au.toml index fc29087..4d1af03 100644 --- a/registry/au/michael-page-au.toml +++ b/registry/au/michael-page-au.toml @@ -5,7 +5,7 @@ abn = "58 002 872 264" kinds = ["recruitment"] priority = "high" quality = "verified" -verified_at = 2026-10-26 +verified_at = 2026-11-02 sources = ["https://www.michaelpage.com.au/privacy-policy"] levers = ["app7-no-marketing", "app7-source", "app11-destroy", "app13", "app13-notify-recipients", "app13-statement"] opt_in_levers = ["app12"] @@ -31,5 +31,5 @@ levers = [] [[evidence]] url = "https://www.michaelpage.com.au/privacy-policy" -checked_at = 2026-10-26 -sha256 = "d5d2d1c28b5fa4d58a1a98158f89ead4fbdb4e75ce4ebdd13497ea4b880eca32" +checked_at = 2026-11-02 +sha256 = "a666427dccee8a8c573ebc94aca1c040638592b413cc9515d727d26af5f757b9" diff --git a/registry/au/records-finder-au.toml b/registry/au/records-finder-au.toml index 07bda2c..4e2002c 100644 --- a/registry/au/records-finder-au.toml +++ b/registry/au/records-finder-au.toml @@ -4,7 +4,7 @@ legal_entity = "InfoPay, Inc." kinds = ["people-search"] priority = "standard" quality = "verified" -verified_at = 2026-10-25 +verified_at = 2026-11-02 sources = ["https://recordsfinder.com/privacy/", "https://recordsfinder.com/terms/", "https://recordsfinder.com/optout/", "https://recordsfinder.com/do-not-sell-share-my-personal-information/", "https://recordsfinder.com/australia/"] identity_fields = ["full_name"] exemption_risk = "overseas" @@ -21,25 +21,20 @@ url = "https://recordsfinder.com/australia/" [[evidence]] url = "https://recordsfinder.com/privacy/" -checked_at = 2026-10-25 -sha256 = "572946dbdda3efecf0b16fc347081530267df9d33c5d22deac923ab300abca87" +checked_at = 2026-11-02 +sha256 = "0a84a58297247e0c0edc9165d5259986779ade2b5b980a0be7bd1fba626fab0b" [[evidence]] url = "https://recordsfinder.com/terms/" -checked_at = 2026-10-25 -sha256 = "ec8f39257958f057bc7f9124a08c155f12c93bde766913e56a8b3b765883a4f8" +checked_at = 2026-11-02 +sha256 = "1d27469e8e1bdf9b377a41d38932231a867708501e68124e8398d16c13959c99" [[evidence]] url = "https://recordsfinder.com/optout/" -checked_at = 2026-10-25 -sha256 = "c88024ac830586c5ac152a30327dd01f977fc282ea75ff5de621232082d3dd78" +checked_at = 2026-11-02 +sha256 = "3c92ba03b5906df0786c6aa79ec8391153461adc563f0d5500081576f886c604" [[evidence]] url = "https://recordsfinder.com/do-not-sell-share-my-personal-information/" -checked_at = 2026-10-25 -sha256 = "ae99872996c5d8e13dec966599146e3a4c4fdf0b8c3a907e40390a9291abba7f" - -[[evidence]] -url = "https://recordsfinder.com/australia/" -checked_at = 2026-10-25 -sha256 = "1864c827332dd7c8bc46ab22526f532ca858c0f6022b7ab5a209440ff1626a89" +checked_at = 2026-11-02 +sha256 = "5bb3cad71d41e919aa4fa683c045678f5bae92bc219a6451e6e13b740d9ab8a8" diff --git a/xtask/src/main.rs b/xtask/src/main.rs index b795e2d..1fa89fa 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -3,7 +3,9 @@ //! //! ```text //! cargo run -p xtask -- evidence-hash ... print the evidence hash of each page -//! cargo run -p xtask -- evidence-drift [registry/au] report pages whose hash has changed +//! cargo run -p xtask -- evidence-drift [registry/au [text dir]] +//! report pages whose hash has changed, +//! optionally saving each page's text //! cargo run -p xtask -- email-changes warn of email addresses a change added //! ``` @@ -33,19 +35,41 @@ fn agent(timeout: Duration) -> ureq::Agent { /// A fetched evidence page. struct Page { sha256: String, - /// Lowercase raw HTML followed by the lowercase main text, for finding an email - /// address in a link, a footer or entity-encoded text; `None` for a non-HTML page. + /// The normalised main text the hash covers; `None` for a non-HTML page. + main: Option, + /// Lowercase raw HTML, main text and Cloudflare-protected addresses, for finding an + /// email address in a link, a footer, or entity-encoded or obfuscated text; `None` for + /// a non-HTML page. text: Option, } +/// Why an evidence page could not be hashed. +#[derive(Debug)] +enum PageError { + /// 401, 403 or 429: sites answer scripts this way even when the page is up. + Refused(String), + Failed(String), +} + +impl std::fmt::Display for PageError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Refused(e) | Self::Failed(e) => f.write_str(e), + } + } +} + /// Fetch `url`. Its evidence hash is SHA-256 of its normalised main text, or of the raw /// bytes for a non-HTML response. -fn evidence_page(agent: &ureq::Agent, url: &str) -> Result { +fn evidence_page(agent: &ureq::Agent, url: &str) -> Result { let mut response = agent .get(url) .header("User-Agent", USER_AGENT) .call() - .map_err(|e| e.to_string())?; + .map_err(|e| match e { + ureq::Error::StatusCode(401 | 403 | 429) => PageError::Refused(e.to_string()), + _ => PageError::Failed(e.to_string()), + })?; let content_type = response .headers() .get("content-type") @@ -55,27 +79,55 @@ fn evidence_page(agent: &ureq::Agent, url: &str) -> Result { .with_config() .limit(20 * 1024 * 1024) .read_to_vec() - .map_err(|e| e.to_string())?; - let (digest, text) = if text::is_html(content_type.as_deref(), &bytes) { + .map_err(|e| PageError::Failed(e.to_string()))?; + let (digest, main, text) = if text::is_html(content_type.as_deref(), &bytes) { let html = String::from_utf8_lossy(&bytes); let main = text::main_text(&html); if main.chars().count() < MIN_TEXT { - return Err(format!( + return Err(PageError::Failed(format!( "only {} characters of main text; the page may need a browser", main.chars().count() - )); + ))); } - let searchable = format!("{html}\n{main}").to_lowercase(); - (Sha256::digest(main.as_bytes()), Some(searchable)) + let hidden = text::cloudflare_emails(&html).join("\n"); + let searchable = format!("{html}\n{main}\n{hidden}").to_lowercase(); + ( + Sha256::digest(main.as_bytes()), + Some(main), + Some(searchable), + ) } else { - (Sha256::digest(&bytes), None) + (Sha256::digest(&bytes), None, None) }; Ok(Page { - sha256: digest.iter().map(|b| format!("{b:02x}")).collect(), + sha256: hex(&digest), + main, text, }) } +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// Save a page's main text as `-.txt` under `dir`, so a later run's +/// copy can be diffed against it. The URL is the first line; after a blank line each +/// sentence starts a line, and joining those lines with spaces gives the hashed text. +fn save_text(dir: &Path, id: &str, url: &str, main: &str) { + let (name, body) = text_file(id, url, main); + if let Err(e) = + std::fs::create_dir_all(dir).and_then(|()| std::fs::write(dir.join(&name), body)) + { + eprintln!("warning: cannot save the text of {url}: {e}"); + } +} + +/// The file name and contents `save_text` writes. +fn text_file(id: &str, url: &str, main: &str) -> (String, String) { + let name = format!("{id}-{}.txt", hex(&Sha256::digest(url.as_bytes())[..4])); + (name, format!("{url}\n\n{}\n", main.replace(". ", ".\n"))) +} + fn hash(urls: &[String]) -> ExitCode { let agent = agent(TIMEOUT); let mut failed = false; @@ -96,8 +148,9 @@ fn hash(urls: &[String]) -> ExitCode { } /// Prints a Markdown report of changed and failing evidence pages, or nothing if every -/// page still matches. Exits 0 either way; a register that cannot be read exits 1. -fn drift(dir: &Path) -> ExitCode { +/// page still matches, and saves each HTML page's text under `text_dir` if given. Exits 0 +/// either way; a register that cannot be read exits 1. +fn drift(dir: &Path, text_dir: Option<&Path>) -> ExitCode { let files = match ghostsoap::io::register::load(dir) { Ok(files) => files, Err(e) => { @@ -119,6 +172,7 @@ fn drift(dir: &Path) -> ExitCode { continue; } }; + let ids = std::slice::from_ref(&entry.id); let mut pages = Vec::new(); for evidence in &entry.evidence { match evidence_page(&agent, &evidence.url) { @@ -129,23 +183,25 @@ fn drift(dir: &Path) -> ExitCode { entry.id, evidence.url, evidence.checked_at )); } + if let (Some(dir), Some(main)) = (text_dir, &page.main) { + save_text(dir, &entry.id, &evidence.url, main); + } pages.push(page.text); } - Err(e) => { - failing.push(format!("- `{}`: {}: {e}", entry.id, evidence.url)); + Err(PageError::Refused(e)) => { + record(&mut failing, &evidence.url, ids, Check::Inconclusive(e)); + pages.push(None); + } + Err(PageError::Failed(e)) => { + record(&mut failing, &evidence.url, ids, Check::Dead(e)); pages.push(None); } } } if entry.obsolete.is_none() { collect_channels(&entry, &mut mail_domains, &mut forms); - let ids = std::slice::from_ref(&entry.id); - for channel in &entry.channels { - if channel.kind == ChannelKind::Email { - let what = format!("email {}", channel.address); - let check = email_on_pages(&channel.address, &pages); - record(&mut emails, &what, ids, check); - } + for (address, check) in email_checks(&agent, &entry, pages) { + record(&mut emails, &format!("email {address}"), ids, check); } } } @@ -209,22 +265,77 @@ fn record(lines: &mut Vec, what: &str, ids: &[String], check: Check) { } } -/// Whether `address` appears on one of its entry's evidence pages, given each page's +/// Check each email channel of `entry` against its source pages. `pages` holds the +/// searchable text of its evidence pages; its other sources, such as a page behind a bot +/// check that cannot have evidence, are fetched only when an address is not on those. +fn email_checks( + agent: &ureq::Agent, + entry: &Entry, + mut pages: Vec>, +) -> Vec<(String, Check)> { + let addresses: Vec<&str> = entry + .channels + .iter() + .filter(|c| c.kind == ChannelKind::Email) + .map(|c| c.address.as_str()) + .collect(); + if addresses + .iter() + .any(|a| email_on_pages(a, &pages) != Check::Fine) + { + let tracked = |url: &String| entry.evidence.iter().any(|e| &e.url == url); + for url in entry.sources.iter().filter(|url| !tracked(url)) { + pages.push(evidence_page(agent, url).ok().and_then(|page| page.text)); + } + } + addresses + .into_iter() + .map(|a| (a.to_owned(), email_on_pages(a, &pages))) + .collect() +} + +/// Whether `address` appears on one of its entry's source pages, given each page's /// searchable text (`None` for a page that failed or is not HTML). It is missing only when /// every page could be searched. fn email_on_pages(address: &str, pages: &[Option]) -> Check { let address = address.to_lowercase(); - if pages.iter().flatten().any(|text| text.contains(&address)) { + if pages + .iter() + .flatten() + .any(|text| has_address(text, &address)) + { Check::Fine } else if pages.is_empty() { - Check::Inconclusive("the entry has no evidence page".to_owned()) + Check::Inconclusive("the entry has no source page".to_owned()) } else if pages.iter().any(Option::is_none) { - Check::Inconclusive("not on the evidence pages that could be searched".to_owned()) + Check::Inconclusive("not on the source pages that could be searched".to_owned()) } else { - Check::Dead("appears on none of the entry's evidence pages".to_owned()) + Check::Dead("appears on none of the entry's source pages".to_owned()) } } +/// Whether `text` contains `address` as a whole address, not inside a longer one such as +/// `not` + address or address + `.au`. A sentence's full stop may follow it, and a +/// percent-escape in a link, such as `%20`, may precede it unless it encodes a character an +/// address can contain, such as `%2B`. +fn has_address(text: &str, address: &str) -> bool { + let local = |c: char| c.is_ascii_alphanumeric() || "._%+-".contains(c); + let domain = |c: char| c.is_ascii_alphanumeric() || c == '-'; + let escaped_boundary = |before: &[u8]| match before { + [.., b'%', h, l] if h.is_ascii_hexdigit() && l.is_ascii_hexdigit() => { + let hex = |d: u8| char::from(d).to_digit(16).unwrap_or(0) as u8; + !local(char::from(hex(*h) * 16 + hex(*l))) + } + _ => false, + }; + text.match_indices(address).any(|(i, _)| { + let after = &text[i + address.len()..]; + (!text[..i].ends_with(local) || escaped_boundary(&text.as_bytes()[..i])) + && !after.starts_with(domain) + && !(after.starts_with('.') && after[1..].starts_with(domain)) + }) +} + /// Email addresses in `head` that the same entry in `base` did not have, as `(file, message)`. /// A change that repoints a privacy address would send users' requests elsewhere, so a /// reviewer must see each one. Every address of an entry whose id is not in `base` is listed, @@ -451,15 +562,17 @@ fn main() -> ExitCode { let args: Vec = std::env::args().skip(1).collect(); match args.split_first() { Some((command, urls)) if command == "evidence-hash" && !urls.is_empty() => hash(urls), - Some((command, rest)) if command == "evidence-drift" && rest.len() <= 1 => drift( + Some((command, rest)) if command == "evidence-drift" && rest.len() <= 2 => drift( Path::new(rest.first().map_or("registry/au", String::as_str)), + rest.get(1).map(Path::new), ), Some((command, [base, head])) if command == "email-changes" => { emails_changed(Path::new(base), Path::new(head)) } _ => { eprintln!( - "usage: xtask evidence-hash ... | xtask evidence-drift [registry dir] \ + "usage: xtask evidence-hash ... \ + | xtask evidence-drift [registry dir [page text dir]] \ | xtask email-changes " ); ExitCode::from(2) @@ -662,17 +775,38 @@ levers = [] } #[test] - fn an_email_must_appear_on_an_evidence_page() { + fn an_email_must_appear_on_a_source_page() { let page = |text: &str| Some(text.to_owned()); let found = [page("other"), page("write to privacy@example.com.au.")]; + let linked = [page( + "", + )]; + assert_eq!( + email_on_pages("privacy@example.com.au", &linked), + Check::Fine + ); + for escape in ["%2b", "%2e", "%61"] { + let longer = [page(&format!("?to=user{escape}privacy@example.com.au"))]; + assert!(matches!( + email_on_pages("privacy@example.com.au", &longer), + Check::Dead(_) + )); + } assert_eq!( email_on_pages("Privacy@Example.com.au", &found), Check::Fine ); + let longer = [page( + "notprivacy@example.com.au, privacy@example.com.au.example.org", + )]; + assert!(matches!( + email_on_pages("privacy@example.com.au", &longer), + Check::Dead(_) + )); let missing = [page("write to privacy@elsewhere.example")]; assert!(matches!( email_on_pages("privacy@example.com.au", &missing), - Check::Dead(p) if p.contains("none of the entry's evidence pages") + Check::Dead(p) if p.contains("none of the entry's source pages") )); let unsearched = [page("nothing here"), None]; assert!(matches!( @@ -685,13 +819,10 @@ levers = [] )); } - #[test] - fn a_page_is_searched_in_its_markup_and_its_decoded_text() { + /// Serve one HTML page whose main element holds 500 or more characters, then `main`. + fn serve_html(main: &str) -> String { let filler = "Privacy policy text. ".repeat(30); - let html = format!( - "
{filler} Email privacy@example.com.au.
\ - " - ); + let html = format!("
{filler} {main}
"); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let url = format!("http://{}/", listener.local_addr().unwrap()); std::thread::spawn(move || { @@ -704,6 +835,17 @@ levers = [] ); let _ = stream.write_all(reply.as_bytes()); }); + url + } + + #[test] + fn a_page_is_searched_in_its_markup_and_its_decoded_text() { + let url = serve_html( + "Email privacy@example.com.au. \ + Contact \ + [email protected]", + ); let page = evidence_page(&agent(Duration::from_secs(5)), &url).unwrap(); let pages = [page.text]; assert_eq!( @@ -711,12 +853,78 @@ levers = [] Check::Fine ); assert_eq!(email_on_pages("help@example.com.au", &pages), Check::Fine); + assert_eq!(email_on_pages("hidden@example.com.au", &pages), Check::Fine); assert!(matches!( email_on_pages("other@example.com.au", &pages), Check::Dead(_) )); } + #[test] + fn an_email_is_searched_on_sources_without_evidence() { + let url = serve_html("Email privacy@example.com.au."); + let entry: Entry = + toml::from_str(&ENTRY.replace("https://example.com.au/privacy", &url)).unwrap(); + let checks = email_checks(&agent(Duration::from_secs(5)), &entry, Vec::new()); + assert_eq!(checks, [("privacy@Example.com.au".to_owned(), Check::Fine)]); + } + + #[test] + fn an_email_missing_from_evidence_is_searched_on_the_other_sources() { + let url = serve_html("Email privacy@example.com.au."); + let sources = format!("sources = [\"https://example.com.au/privacy\", \"{url}\"]"); + let evidence = format!( + "\n[[evidence]]\nurl = \"https://example.com.au/privacy\"\n\ + checked_at = 2026-10-30\nsha256 = \"{}\"\n", + "0".repeat(64) + ); + let toml = ENTRY.replace("sources = [\"https://example.com.au/privacy\"]", &sources); + let entry: Entry = toml::from_str(&(toml + &evidence)).unwrap(); + let evidence_page = vec![Some("a policy that names no address".to_owned())]; + let checks = email_checks(&agent(Duration::from_secs(5)), &entry, evidence_page); + assert_eq!(checks, [("privacy@Example.com.au".to_owned(), Check::Fine)]); + } + + #[test] + fn a_page_refused_to_scripts_is_not_a_failure() { + let agent = agent(Duration::from_secs(5)); + for status in ["401 Unauthorized", "403 Forbidden", "429 Too Many Requests"] { + let page = evidence_page(&agent, &serve_once(status)).map(|p| p.sha256); + assert!( + matches!(page, Err(PageError::Refused(_))), + "{status}: {page:?}" + ); + } + for status in ["404 Not Found", "410 Gone", "500 Internal Server Error"] { + let page = evidence_page(&agent, &serve_once(status)).map(|p| p.sha256); + assert!( + matches!(page, Err(PageError::Failed(_))), + "{status}: {page:?}" + ); + } + } + + #[test] + fn saved_text_rejoins_to_the_hashed_text() { + let url = serve_html("First sentence. Second sentence. Third."); + let page = evidence_page(&agent(Duration::from_secs(5)), &url).unwrap(); + let main = page.main.unwrap(); + let (name, saved) = text_file("example-au", &url, &main); + assert!( + name.starts_with("example-au-") && name.ends_with(".txt"), + "{name}" + ); + assert_ne!( + name, + text_file("example-au", "https://example.com.au/", &main).0 + ); + let (head, body) = saved.split_once("\n\n").unwrap(); + assert_eq!(head, url); + assert!(body.contains("Second sentence.\nThird."), "{body}"); + let rejoined = body.trim_end_matches('\n').replace('\n', " "); + assert_eq!(hex(&Sha256::digest(rejoined.as_bytes())), page.sha256); + } + fn source(text: &str) -> SourceFile { SourceFile { name: "example-au.toml".to_owned(), @@ -786,7 +994,7 @@ levers = [] let start = Instant::now(); let result = evidence_page(&agent(Duration::from_millis(300)), &url).map(|p| p.sha256); assert!( - result.as_ref().is_err_and(|e| e.contains("timeout")), + matches!(&result, Err(PageError::Failed(e)) if e.contains("timeout")), "{result:?}" ); assert!(start.elapsed() < Duration::from_secs(10)); diff --git a/xtask/src/text.rs b/xtask/src/text.rs index 460d100..30fd3db 100644 --- a/xtask/src/text.rs +++ b/xtask/src/text.rs @@ -240,6 +240,40 @@ fn after(lower: &str, from: usize, needle: &str) -> usize { .map_or(lower.len(), |j| from + j + needle.len()) } +/// Email addresses hidden by Cloudflare's email obfuscation, which replaces each address in +/// a page with `data-cfemail=""` or a `/cdn-cgi/l/email-protection#` link. The +/// first byte of the hex is a key XORed with each following byte. Cloudflare writes both +/// forms in double-quoted attributes, so the hex must end at a double quote; any other +/// character there makes the payload malformed. +pub fn cloudflare_emails(html: &str) -> Vec { + let mut emails = Vec::new(); + for marker in ["data-cfemail=\"", "/cdn-cgi/l/email-protection#"] { + for (i, _) in html.match_indices(marker) { + let rest = &html[i + marker.len()..]; + let end = rest + .find(|c: char| !c.is_ascii_hexdigit()) + .unwrap_or(rest.len()); + if rest[end..].starts_with('"') { + emails.extend(decode_cloudflare(&rest[..end])); + } + } + } + emails +} + +fn decode_cloudflare(hex: &str) -> Option { + if hex.len() < 4 || !hex.len().is_multiple_of(2) { + return None; + } + let bytes = (0..hex.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&hex[i..i + 2], 16)) + .collect::, _>>() + .ok()?; + let (key, rest) = bytes.split_first()?; + String::from_utf8(rest.iter().map(|b| b ^ key).collect()).ok() +} + fn decode_entities(text: &str) -> String { let mut out = String::with_capacity(text.len()); let mut rest = text; @@ -403,4 +437,20 @@ mod tests { assert!(!is_html(Some("application/pdf"), page)); assert!(!is_html(None, b"%PDF-1.7")); } + + #[test] + fn decodes_cloudflare_protected_emails() { + let html = "[email protected] \ + Help \ + \ + \ + \ + "; + assert_eq!( + cloudflare_emails(html), + ["privacy@example.com.au", "help@example.com.au"] + ); + assert!(cloudflare_emails("

privacy@example.com.au

").is_empty()); + } }