From 60572dfa8ff0b0e0333aef7845578a514cf84ef7 Mon Sep 17 00:00:00 2001 From: Brigs Date: Sat, 1 Aug 2026 02:09:12 -0400 Subject: [PATCH] ci: guard artifact name/description against unsupported claims Adds admin/scripts/check_claim_language.py, ported from the canonical iLEAPP implementation, and wires it into the existing lint workflow as an added step in the same job. The check reads each scripts/artifacts/*.py module with ast, evaluates its __artifacts_v2__ literal, and matches the name and description of every entry against a vocabulary of phrasing that past audits found to assert what the parsed data does not establish: completeness words, attributions of an act to "the user", certainty words. Those two fields reach the examiner through the HTML report and the LAVA manifest, so a claim there is quoted in casework; PR #48 fixed them by hand and nothing has watched them since. It also fails on a stale allowlist entry, so an entry cannot outlive the description it was written for and silently shield the next claim under the same key, and prints NOT CHECKED modules on every run so the coverage hole stays visible rather than being assumed empty. Ten fields fired on first run. Nine are reworded to what the data shows; in each the trigger word was removable without losing information -- three cases of "complete " describing one column, six leading "All"/"Every" over a set the following clause already scopes. The tenth is allowlisted: discordCacheRecords' match is inside its own hedge ("a partial record ... rather than a complete one"). Known coverage gap, documented in the docstring: robloxWindows.py builds __artifacts_v2__ through a helper rather than a literal, so its four Windows artifacts are not read. Their descriptions are inherited from the macOS modules, which are checked. Co-Authored-By: Claude Fable 5 --- .github/workflows/python_lint.yml | 7 + admin/scripts/check_claim_language.py | 312 ++++++++++++++++++++++++++ scripts/artifacts/discordContacts.py | 2 +- scripts/artifacts/discordMedia.py | 2 +- scripts/artifacts/robloxCookies.py | 4 +- scripts/artifacts/robloxLogs.py | 2 +- scripts/artifacts/robloxWebView2.py | 6 +- scripts/artifacts/signalMessages.py | 2 +- scripts/artifacts/wireAppLog.py | 4 +- scripts/artifacts/wireIndexedDb.py | 4 +- 10 files changed, 332 insertions(+), 13 deletions(-) create mode 100644 admin/scripts/check_claim_language.py diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml index ccd6cba..a03e292 100644 --- a/.github/workflows/python_lint.yml +++ b/.github/workflows/python_lint.yml @@ -35,6 +35,13 @@ jobs: if: steps.changed-files-py.outputs.any_changed == 'true' run: python -m pip install -r requirements.txt + # Artifact name/description fields ship to the HTML report and the LAVA manifest + # and get quoted in casework, so they must not assert what the data means in the + # real world. See the script's docstring for the allowlist workflow. + - name: Guard against unsupported claim language + if: steps.changed-files-py.outputs.any_changed == 'true' + run: python admin/scripts/check_claim_language.py + # Fails only on warnings this pull request introduces. dleapp.py and # dleappGUI.py carry pre-existing warnings that are structural rather than # fixable -- wildcard imports are how those modules are put together -- so diff --git a/admin/scripts/check_claim_language.py b/admin/scripts/check_claim_language.py new file mode 100644 index 0000000..c999040 --- /dev/null +++ b/admin/scripts/check_claim_language.py @@ -0,0 +1,312 @@ +"""Guard the examiner-facing artifact metadata against unsupported claims. + +Every artifact module declares an ``__artifacts_v2__`` dict. Its ``name`` and +``description`` fields are not developer notes: they are rendered into the HTML +report, written into the LAVA manifest, and from there they get pasted into +examination notes and quoted in court. A description that says an artifact holds +"every file the user opened" is a statement about a person's conduct that the +underlying database does not make. The standard for these fields is therefore: + + Say what the data is and where it came from. Do not say what it means about + the world, or who did it, unless the data itself establishes that or the + description cites a source that documents it. + +The failure mode this check exists to stop is a fix that goes stale. An audit of +all 34 artifact modules (merged as PR #48) reworded the claims it found in +docstrings, notes and these two fields alike. Nothing then watched the fields, so +the next description written by hand could reintroduce the same phrasing without +anyone noticing, and a later edit to a docstring could leave the `description` +above it asserting what the docstring no longer does. Prose corrections decay +exactly where nothing is checking them. + +The check parses each artifact module with ``ast``, evaluates its +``__artifacts_v2__`` literal, and matches the ``name`` and ``description`` of +every entry against a vocabulary of phrasing that has historically signalled an +unsupported claim (completeness words, attributions of an act to "the user", +certainty words). + +Two ways a match gets resolved: + +* The wording overstates what the parser can show. Reword it to what the data + shows -- name the table, file, or key, and drop the actor and the completeness + word. This is the common case. +* The match is a false positive: a product feature literally named "All Files", a + verbatim database enum value, a UI path reproduced from the app, or a + cautionary sentence whose matched word is part of the hedge. Add a + ``(filename, artifact_key, field)`` tuple to ALLOWLIST **with an inline comment + saying why**. The allowlist is a record of decisions someone made on purpose; + it is not a place to park a description nobody wanted to rewrite. + +Two things the check reports rather than hides, because both are ways it can +quietly stop doing its job: + +* An ALLOWLIST entry that no longer matches anything. It means the description + was reworded or the artifact key changed, and the entry now shields nothing -- + except the next claim that lands under the same key. Stale entries fail the run + and must be deleted. +* A module whose ``__artifacts_v2__`` is not a static literal (built by a helper, + or absent). Its fields cannot be read without importing the module, so they are + never checked. Those modules are printed as NOT CHECKED on every run, so the + coverage hole stays visible. + +Known coverage gap: ``scripts/artifacts/robloxWindows.py`` builds its four +entries through a ``_windows_artifact()`` helper that copies and mutates the dict +imported from the macOS parser module, so ``__artifacts_v2__`` there is a call +rather than a literal and this check cannot read it. Those four Windows artifacts +(``robloxWindowsPresence``, ``robloxWindowsNotifications``, +``robloxWindowsGameJoins``, ``robloxWindowsAccount``) inherit the ``description`` +of their macOS counterparts, and a ``name`` derived from it, unchecked. The +macOS originals in ``robloxActivity.py``, ``robloxLogs.py`` and +``robloxAccount.py`` are checked, which covers the inherited description text but +not the substitution the helper performs on the name. + +Usage: + python3 admin/scripts/check_claim_language.py # CI mode, exits 1 on a violation + python3 admin/scripts/check_claim_language.py --list # every match, allowlisted included + python3 admin/scripts/check_claim_language.py --verbose # coverage and allowlist counts +""" + +import argparse +import ast +import os +import re +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +ARTIFACTS_DIR = REPO_ROOT / "scripts" / "artifacts" + +# The claim vocabulary. Each alternative is a phrasing that has, in past audits, +# turned out to be an assertion the parsed data does not support: +# - completeness claims over a source that is rotated, cached, or truncated +# ("all", "every", "complete", "full list", "entire") +# - attributing an act to a person when the record only shows a stored value +# ("the user viewed", "user-created", "searched by", "manually") +# - certainty and inference-about-conduct words ("proves", "definitively", +# "always", "reliable", "visited", "habits") +# +# Every alternative is anchored with an explicit \b. Do NOT express a boundary as +# a trailing space ("all ", "every ", "always "): that spelling matches inside +# "call log", "calllog.db" and similar, which is enough noise to make the check +# worth ignoring. Word boundaries also keep the hedges quiet -- neither +# "unreliable" nor "incomplete" trips, because there is no boundary mid-word. +# +# The stems below are left UNCLOSED on purpose so inflections still match: +# \bcomplete -> complete, completeness, completely +# \breliable -> reliable, reliably and its compounds +# \bhabit -> habit, habits, habitual, habitually +# "habitual" is the inference word most likely to appear in a description of app +# usage data, so the open stem is worth its one known false positive, "habitat": +# no artifact description or name in this repository contains that word today +# (verified by grep over scripts/artifacts). If a habitat-related artifact ever +# lands, close the stem to \bhabits?\b rather than allowlisting the artifact. +CLAIM_PATTERN = re.compile( + r"\ball\b" + r"|\bevery\b" + r"|\bcomplete" + r"|\bfull list\b" + r"|\bentire\b" + r"|\bthe user (?:searched|typed|viewed|visited|opened|selected|deleted|read|sent" + r"|created|hid|chose)\b" + r"|\buser[- ](?:created|entered|typed|searched|selected|initiated)\b" + r"|\bsearched by\b" + r"|\btyped by\b" + r"|\bviewed by\b" + r"|\bread by\b" + r"|\bmanually\b" + r"|\bproves?\b" + r"|\bdefinitively\b" + r"|\balways\b" + r"|\breliable" + r"|\bvisited\b" + r"|\bhabit", + re.IGNORECASE) + +# Fields that reach the examiner through the report and the LAVA manifest. +CHECKED_FIELDS = ("name", "description") + +# Reviewed exceptions, keyed by (filename, artifact_key, field). Every entry +# needs a comment justifying it. See the module docstring before adding one. +ALLOWLIST = { + # The match is inside the hedge itself: the description closes with "the + # cache evicts over time, so this index is a partial record of what the + # client fetched rather than a complete one". Removing the word would remove + # the caution it belongs to. + ("discordCacheRecords.py", "discordCacheRecords", "description"), +} + +STANDARD_NOTE = ( + "Artifact name/description reach the examiner through the HTML report and the LAVA\n" + "manifest and get quoted in casework. State what the data is and where it came from;\n" + "do not state what it means in the real world, or who performed an act, unless the\n" + "data establishes it or a cited source documents it.\n" + "Reword to what the data shows, or -- if the match is a product name, a verbatim\n" + "schema value, a UI path, or part of a hedge -- add it to ALLOWLIST in\n" + "admin/scripts/check_claim_language.py with a comment saying why." +) + + +def find_artifacts_dict(tree): + """Return the AST node assigned to `__artifacts_v2__`, or None.""" + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + for target in node.targets: + if isinstance(target, ast.Name) and target.id == "__artifacts_v2__": + return node.value + return None + + +def load_artifacts(path): + """Return (artifacts_dict, skip_reason). Exactly one of the two is None.""" + try: + source = path.read_text(encoding="utf-8") + except OSError as ex: + return None, f"could not read file: {ex}" + + try: + tree = ast.parse(source, filename=str(path)) + except SyntaxError as ex: + return None, f"could not parse module: {ex}" + + node = find_artifacts_dict(tree) + if node is None: + return None, "no __artifacts_v2__ assignment" + + # Modules that build the dict dynamically cannot be evaluated statically. + try: + artifacts = ast.literal_eval(node) + except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError) as ex: + return None, f"__artifacts_v2__ is not a literal: {ex}" + + if not isinstance(artifacts, dict): + return None, "__artifacts_v2__ is not a dict" + return artifacts, None + + +def scan_file(path): + """Return (matches, skip_reason) for one artifact module. + + Each match is a (path, artifact_key, field, text, matched_terms, allowlisted) + tuple. + """ + artifacts, skip_reason = load_artifacts(path) + if artifacts is None: + return [], skip_reason + + matches = [] + for artifact_key, entry in artifacts.items(): + if not isinstance(entry, dict): + continue + for field in CHECKED_FIELDS: + text = entry.get(field) + if not isinstance(text, str): + continue + terms = CLAIM_PATTERN.findall(text) + if not terms: + continue + allowlisted = (path.name, str(artifact_key), field) in ALLOWLIST + matches.append((path, str(artifact_key), field, text, terms, allowlisted)) + return matches, None + + +def format_match(match): + """Render one match as `path:artifact_key:field: `.""" + path, artifact_key, field, text, terms, _ = match + collapsed = " ".join(text.split()) + if len(collapsed) > 300: + collapsed = collapsed[:297] + "..." + quoted = ", ".join(sorted({term.lower() for term in terms})) + return f"{path}:{artifact_key}:{field}: {collapsed}\n matched: {quoted}" + + +def main(): + """Scan the artifact modules and report unallowlisted claim language.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--list", action="store_true", dest="list_all", + help="print every match, including allowlisted ones") + parser.add_argument("--verbose", action="store_true", + help="also report modules whose __artifacts_v2__ could not be read") + args = parser.parse_args() + + paths = sorted(ARTIFACTS_DIR.glob("*.py")) + if not paths: + print(f"No artifact modules found under {ARTIFACTS_DIR}", file=sys.stderr) + return 2 + + violations = [] + allowlisted = [] + skipped = [] + fired = set() + for path in paths: + rel_path = os.path.relpath(path, REPO_ROOT) + matches, skip_reason = scan_file(path) + if skip_reason: + skipped.append((rel_path, skip_reason)) + continue + for match in matches: + entry = (rel_path,) + match[1:] + fired.add((path.name, match[1], match[2])) + if match[5]: + allowlisted.append(entry) + else: + violations.append(entry) + + # An allowlist entry that no longer matches anything is either a fixed + # description or a stale key, and it hides the next real claim behind a name + # nobody rechecks. Surface it so the allowlist stays a list of live decisions. + stale = sorted(ALLOWLIST - fired) + + # A module whose __artifacts_v2__ cannot be evaluated statically is a real + # coverage hole: its fields are never checked. Report it rather than hide it. + if skipped: + print(f"NOT CHECKED -- {len(skipped)} module(s) have no statically readable " + f"__artifacts_v2__:") + for rel_path, reason in skipped: + print(f" {rel_path}: {reason}") + print() + + if args.verbose: + print(f"Scanned {len(paths)} module(s); {len(paths) - len(skipped)} checked, " + f"{len(skipped)} skipped.") + print(f"Allowlist holds {len(ALLOWLIST)} entr(ies); {len(allowlisted)} fired " + f"this run.") + print() + + if stale: + print(f"Stale ALLOWLIST entr(ies) ({len(stale)}) -- these no longer match " + f"anything and should be deleted:") + for entry in stale: + print(f" {entry[0]}:{entry[1]}:{entry[2]}") + print() + + if args.list_all and allowlisted: + print(f"Allowlisted matches ({len(allowlisted)}):") + for match in allowlisted: + print(f" {format_match(match)}") + print() + + if violations: + print(f"Unsupported claim language in examiner-facing artifact fields " + f"({len(violations)}):") + for match in violations: + print(f" {format_match(match)}") + print() + print(STANDARD_NOTE) + return 1 + + if stale: + print("Remove the stale entr(ies) above from ALLOWLIST in " + "admin/scripts/check_claim_language.py.") + return 1 + + summary = (f"Checked {len(paths) - len(skipped)} artifact module(s): no unsupported " + f"claim language ({len(allowlisted)} reviewed exception(s) allowlisted).") + if skipped: + summary += f" {len(skipped)} module(s) NOT checked, listed above." + print(summary) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/artifacts/discordContacts.py b/scripts/artifacts/discordContacts.py index 91c1c83..61bff53 100644 --- a/scripts/artifacts/discordContacts.py +++ b/scripts/artifacts/discordContacts.py @@ -1,7 +1,7 @@ __artifacts_v2__ = { "discordUsers": { "name": "Discord Users Seen", - "description": "Every Discord account seen in the cached responses this " + "description": "Discord accounts seen in the cached responses this " "parser decodes: message authors, mentioned users, DM " "recipients, reaction users, profiles and invite " "creators. User IDs are snowflakes, so each account's " diff --git a/scripts/artifacts/discordMedia.py b/scripts/artifacts/discordMedia.py index 838d3d1..8904fd8 100644 --- a/scripts/artifacts/discordMedia.py +++ b/scripts/artifacts/discordMedia.py @@ -1,7 +1,7 @@ __artifacts_v2__ = { "discordRecoveredMedia": { "name": "Discord Recovered Media", - "description": "Every cached file this parser could identify as Discord " + "description": "Cached files this parser could identify as Discord " "media and decode, extracted and embedded in the report: " "images, video, avatars, emoji, stickers and server " "icons. Attachment URLs carry the channel ID and an " diff --git a/scripts/artifacts/robloxCookies.py b/scripts/artifacts/robloxCookies.py index 7b7a71f..b048788 100644 --- a/scripts/artifacts/robloxCookies.py +++ b/scripts/artifacts/robloxCookies.py @@ -3,10 +3,10 @@ "name": "Roblox Cookies", "description": "Cookies from Roblox Desktop's macOS binary cookie store, " "including domain, name, path, creation, expiry, last-access " - "time, flags and the complete stored value.", + "time, flags and the stored value.", "author": "@AlexisBrignoni, Codex", "creation_date": "2026-07-28", - "last_update_date": "2026-07-29", + "last_update_date": "2026-08-01", "requirements": "none", "category": "Roblox (macOS)", "notes": "All values are reported verbatim for evidentiary analysis. The " diff --git a/scripts/artifacts/robloxLogs.py b/scripts/artifacts/robloxLogs.py index 1807a9e..fd6add3 100644 --- a/scripts/artifacts/robloxLogs.py +++ b/scripts/artifacts/robloxLogs.py @@ -52,7 +52,7 @@ }, "robloxPlayerLog": { "name": "Roblox Player Log", - "description": "All structured Roblox Player log events with the " + "description": "Structured Roblox Player log events with the " "timestamp as written in the log (UTC where the line " "carries a Z suffix), process-relative elapsed time, " "severity, logging component and message.", diff --git a/scripts/artifacts/robloxWebView2.py b/scripts/artifacts/robloxWebView2.py index c1043c1..154a41c 100644 --- a/scripts/artifacts/robloxWebView2.py +++ b/scripts/artifacts/robloxWebView2.py @@ -3,7 +3,7 @@ "name": "Roblox Windows Cookie Vault", "description": "The Roblox Player CookiesData vault retained in " "LocalStorage/RobloxCookies.dat, including the format version " - "and complete DPAPI-protected blob.", + "and the DPAPI-protected blob.", "author": "@AlexisBrignoni, Codex", "creation_date": "2026-07-28", "last_update_date": "2026-08-01", @@ -26,11 +26,11 @@ }, "robloxWebView2Cookies": { "name": "Roblox WebView2 Cookies", - "description": "Cookie metadata and complete encrypted values from Roblox's " + "description": "Cookie metadata and encrypted values from Roblox's " "Windows WebView2 profile.", "author": "@AlexisBrignoni, Codex", "creation_date": "2026-07-28", - "last_update_date": "2026-07-29", + "last_update_date": "2026-08-01", "requirements": "none", "category": "Roblox (Windows)", "notes": "Chromium v10 cookie values depend on the WebView2 Local State " diff --git a/scripts/artifacts/signalMessages.py b/scripts/artifacts/signalMessages.py index 93247a8..ed99e84 100644 --- a/scripts/artifacts/signalMessages.py +++ b/scripts/artifacts/signalMessages.py @@ -48,7 +48,7 @@ "description": "Files shared in Signal conversations, decrypted from " "attachments.noindex. Each stored file is encrypted with " "its own key held in the database, so the files cannot be " - "read without it. Every recovered file is checked against " + "read without it. Each recovered file is checked against " "the message authentication code and, where the database " "recorded one, its SHA-256.", "author": "@AlexisBrignoni", diff --git a/scripts/artifacts/wireAppLog.py b/scripts/artifacts/wireAppLog.py index d11e38b..aea738a 100644 --- a/scripts/artifacts/wireAppLog.py +++ b/scripts/artifacts/wireAppLog.py @@ -2,8 +2,8 @@ "wireDesktopLog": { "name": "Wire Desktop Log", "description": "Activity timeline parsed from the Wire desktop app log " - "(logs/electron.log and electron.old): every timestamped " - "line from the Wire desktop log except repetitive " + "(logs/electron.log and electron.old): timestamped " + "lines from the Wire desktop log except repetitive " "config-restore entries. Timestamps are the device's " "local time as written by the app.", "author": "@AlexisBrignoni", diff --git a/scripts/artifacts/wireIndexedDb.py b/scripts/artifacts/wireIndexedDb.py index 38ff7a0..cf7bcb4 100644 --- a/scripts/artifacts/wireIndexedDb.py +++ b/scripts/artifacts/wireIndexedDb.py @@ -20,12 +20,12 @@ }, "wireUsers": { "name": "Wire Users", - "description": "All Wire users (self and contacts) known to the client: " + "description": "Wire users (self and contacts) known to the client: " "user id, handle, display name, email, phone, domain, team " "and profile-picture asset keys.", "author": "@AlexisBrignoni", "creation_date": "2026-07-23", - "last_update_date": "2026-07-23", + "last_update_date": "2026-08-01", "requirements": "none", "category": "Wire (Windows)", "notes": "",