From 7b831c652e47800e9d25c9ffba9985be3ace7ae7 Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:37:20 -0500 Subject: [PATCH 01/10] ci(measure): a scorer or schema step whose inputs are unchanged takes its output from a score cache, output unchanged (#1237) Measure re-ran every scorer and schema check on every model for head and for base, though most pull requests move the export of a few models. tools/ci/measure-cache.py keys each step on its name and arguments, the content of its file arguments (the rvt-rs IFC without the FILE_NAME and IFCOWNERHISTORY lines that carry the export time), the scorer and every tools/re and tools/ci module it reaches, the environment variables they read, the files they read beside their arguments, the Python, platform and pinned package versions and RVT_IFC_FLAGS, and keeps the output of a step that finished with status 0. A step whose closure does something the key cannot see is not cached, and the held-out models and BIMcollab's are never looked up or stored. Every run restores the newest cache; only the workflow dispatched from main on main's own code saves it, and score_cache= gives a run of any ref a cache of its own. --- .github/workflows/measure.yml | 87 +++++ docs/maintaining.md | 60 +++ tools/ci/measure-cache.py | 530 +++++++++++++++++++++++++++ tools/ci/measure-reference-models.sh | 73 +++- tools/maintainer/clean_caches.py | 9 +- 5 files changed, 747 insertions(+), 12 deletions(-) create mode 100644 tools/ci/measure-cache.py diff --git a/.github/workflows/measure.yml b/.github/workflows/measure.yml index a967e4d25..0519e1f67 100644 --- a/.github/workflows/measure.yml +++ b/.github/workflows/measure.yml @@ -64,6 +64,14 @@ # probe-.txt and probe_score-.txt: # # gh workflow run measure.yml -f ref=my-branch -f base=none -f nordic=true -f models="nordic-office-*" -f scorers=none -f probe="probe_a probe_b" -f probe_scorer="a_vs_ifc -" +# +# A scorer or schema step whose every input is as it was in an earlier run is +# not run again: its output comes from the score cache (tools/ci/measure-cache.py; +# its key, what is never cached and how it is saved are in docs/maintaining.md). +# Exports, rvt-info, rvt-gltf, the witness exports, diagnostics and probes always +# run. `score_cache=off` runs every step, as before. +# +# gh workflow run measure.yml -f ref=my-branch -f base=main -f score_cache=off name: Measure @@ -150,6 +158,10 @@ on: description: Only these steps on the models, tools/re scorers and info, gltf, schema, witness (e.g. "qtos_vs_ifc witness"; a tools/re script in no scorer list runs as an IFC scorer), none for none, empty for all; every model is still exported and hashed required: false default: "" + score_cache: + description: 'The score cache (tools/ci/measure-cache.py, docs/maintaining.md): empty for the shared one, off to run every step, or a name for a cache of its own that this run saves to from any ref (the shared one is read as well)' + required: false + default: "" # A change to the measurement itself measures its own PR against main. pull_request: paths: @@ -235,6 +247,64 @@ jobs: python-version: "3.12" - name: Install IfcOpenShell run: python -m pip install --require-hashes -r tools/ci/requirements-ifc.txt + # The score cache (tools/ci/measure-cache.py, docs/maintaining.md): a + # scorer or schema step whose every input is as it was in an earlier run + # takes that run's output. Every run restores the newest cache, before + # the measured code builds; it is saved only where the cache above is + # (the workflow dispatched from main and the code measured main's, so a + # branch's build scripts can never write what a later main run reads), or + # under a name of its own that `score_cache` gives a run from any ref, + # which main's runs never read. Entries are named by what they were made + # from, so the models measured, the sides and the flags share them. + # The held-out models and BIMcollab's are never cached (RE-252). + - name: Score cache key + id: score-cache-key + shell: bash + env: + SCORE_CACHE: ${{ inputs.score_cache }} + SAVE_FROM_MAIN: ${{ github.ref == 'refs/heads/main' && (matrix.side == 'base' && (inputs.base == '' || inputs.base == 'main') || matrix.side != 'base' && inputs.ref == '') }} + run: | + enabled=true + save=false + case "$SCORE_CACHE" in + off) + enabled=false + ;; + "") + prefix=measure-scores-v1-main- + restore=$prefix + [ "$SAVE_FROM_MAIN" = true ] && save=true + ;; + *) + if ! [[ "$SCORE_CACHE" =~ ^[A-Za-z0-9._]{1,40}$ ]]; then + echo "::error::score_cache is off, or a name of up to 40 letters, digits, dots and underscores" + exit 1 + fi + prefix="measure-scores-v1-x-$SCORE_CACHE-" + restore="$prefix"$'\n'measure-scores-v1-main- + save=true + ;; + esac + { + echo "enabled=$enabled" + echo "save=$save" + echo "prefix=$prefix" + echo "restore<> "$GITHUB_OUTPUT" + if [ "$enabled" = true ]; then + echo "MEASURE_CACHE_DIR=$GITHUB_WORKSPACE/_score-cache" >> "$GITHUB_ENV" + echo "MEASURE_CACHE_WORK=$RUNNER_TEMP/score-cache-work" >> "$GITHUB_ENV" + fi + echo "score cache: enabled=$enabled save=$save key prefix ${prefix:-none}" + - name: Restore the score cache + if: steps.score-cache-key.outputs.enabled == 'true' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: _score-cache + key: ${{ steps.score-cache-key.outputs.prefix }}${{ matrix.side }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: ${{ steps.score-cache-key.outputs.restore }} - name: Build working-directory: measured run: | @@ -370,6 +440,23 @@ jobs: # The runner's CPU, so the compare job can say which machines the # wall times came from. grep -m1 'model name' /proc/cpuinfo | cut -d: -f2- | xargs > results/runner.txt + # Entries not used for 21 days go, then the oldest past 256 MiB; the cache + # is saved only if this run kept an entry, pruned one or refreshed one not + # used for three days (tools/maintainer/clean_caches.py deletes the + # generations it supersedes). + - name: Finish the score cache + id: score-cache + if: ${{ !cancelled() && steps.score-cache-key.outputs.enabled == 'true' }} + shell: bash + run: | + mkdir -p results + python tools/ci/measure-cache.py finish --report results/score-cache.log + - name: Save the score cache + if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' && steps.score-cache.outputs.changed == 'true' && steps.score-cache-key.outputs.save == 'true' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: _score-cache + key: ${{ steps.score-cache-key.outputs.prefix }}${{ matrix.side }}-${{ github.run_id }}-${{ github.run_attempt }} # The held-out models and BIMcollab's carry no licence: only their # counts leave the runner (RE-252). - name: Keep only the unlicensed models' counts diff --git a/docs/maintaining.md b/docs/maintaining.md index b20bc8ed0..1b58ee594 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -91,6 +91,66 @@ when a release ships. ([`changelog.d/`](../changelog.d/README.md)) change in the same pull request as the capability they describe. +## Measure's score cache + +Measure runs about 40 scorers and four schema checks on every model, and most +pull requests move the export of only a few of them. A step whose inputs are as +they were in an earlier run takes that run's output from the score cache +(`tools/ci/measure-cache.py`) instead of running; exports, `rvt-info`, +`rvt-gltf`, the witness exports, diagnostics and probes always run. A hit +writes the cached output and a ` cached` line to the model's +`timings.log`; every output is the one a run would have written. + +- **Key.** The sha256 of the step's name and arguments; the content of every + file argument (the rvt-rs IFC without its `FILE_NAME` and `IFCOWNERHISTORY` + lines, which carry the export time, so a rebuild that writes the same model + hits; Revit's export; the glTF; the validation findings); the scorer and + every module of `tools/re` and `tools/ci` it reaches by import, importlib + path or name; the value of every environment variable they read (apart from + the two that size the mesh cache), which is `PSETS_MISSING` for + `psets_vs_ifc` and `qtos_vs_ifc`; the files a scorer reads that its + arguments do not name (`qtos_vs_ifc`'s `diagnostics.json`, + `unfilled_openings_vs_ifc`'s sibling Revit exports); the Python, platform + and installed version of every package `tools/ci/requirements-ifc.txt` + pins; and `RVT_IFC_FLAGS`. +- **What is never cached.** A step whose closure lists or walks a directory, + takes a directory from an argument, opens a file by a literal path, runs a + program, draws a number or a date, reads the IFC's export time or reads an + environment variable by a computed name: the cache tells the reason in + `score-cache.log` and runs it. A new scorer that reads anything but the + files it is given is run every time until `EXTRA_INPUTS` in + `measure-cache.py` names what it reads. A step is kept only if it finished + with status 0 and printed nothing that shows a failure a rerun could mend. + The held-out models and BIMcollab's (`holdout-*`, `bimcollab-*`), of which + only counts leave the runner (`tools/ci/scrub-holdout-results.sh`), are never + looked up or stored, so no per-element result of theirs is in the cache. +- **Where it lives.** Every run restores the newest `measure-scores-v1-main-*` + cache before the measured code builds. It is saved only where rust-cache is + saved: the workflow dispatched from `main` and the side measuring `main`'s + own code (`inputs.ref` empty, or `base` `main`), because the build scripts of + any other ref run on the runner and must not write what a later `main` run + reads. A pull request run, and a run of any other ref, only reads. A run + dispatched with `-f score_cache=` reads and saves a cache of its own, + `measure-scores-v1-x--*`, from any ref (GitHub scopes a branch's + caches to that branch, and `main`'s runs never read the name); this is how a + change to the cache itself is proved, and `-f score_cache=off` runs every + step as before. +- **Bounds.** Entries not used for 21 days are removed, then the oldest past + 256 MiB; a run saves a new cache only if it kept an entry, pruned one or + refreshed one unused for three days, and + `tools/maintainer/clean_caches.py` deletes the generations a newer one + supersedes. An entry is a scorer's text output (kilobytes), so the cache is + a few MiB, not GiB. `results/score-cache.log` in the artifact lists every + hit, miss, kept and uncached step with the first twelve characters of its key. +- **Checking it.** The run after a change to an export is the check that the + key follows the IFC: its changed models miss and the rest hit. To check a + change to the cache or to a scorer, run Measure twice on one branch with + `-f score_cache=` (cold, then warm) and once with `-f + score_cache=off`, and `diff -r` the three results apart from `*.log`, + `diagnostics.json`; a third warm run with `-f + psets_missing=Pset_WallCommon.Reference` must miss `psets_vs_ifc` on every + model and hit everything else. + ## Dependencies and the MSRV `rust-version` in `Cargo.toml` is the oldest stable Rust the crate builds on, and diff --git a/tools/ci/measure-cache.py b/tools/ci/measure-cache.py new file mode 100644 index 000000000..64f498ac9 --- /dev/null +++ b/tools/ci/measure-cache.py @@ -0,0 +1,530 @@ +#!/usr/bin/env python3 +"""Measure's score cache: the output of a scorer or schema step, kept under a +key made of everything the step reads, so a step whose inputs have not changed +is not run again (.github/workflows/measure.yml, tools/ci/measure-reference-models.sh). + + measure-cache.py lookup --step S --dir D --dest FILE [--extra FILE]... -- SCRIPT ARG... + measure-cache.py store --step S --dir D --dest FILE [--extra FILE]... [--status N] + measure-cache.py finish [--days N] [--max-mb N] [--report FILE] + +`lookup` exits 0 on a hit (FILE and the extras are written from the cache and +`S cached` is added to D/timings.log), 1 on a miss (the key is remembered for +`store`) and 2 when the step is not cached, saying why in the report. `store` +keeps FILE and the extras (files the step also writes, such as the validation +findings) when the step finished with status 0 and its output shows no sign of +a failure that a rerun could mend. `finish` prunes the cache, prints what the +run did and, for the workflow, sets `changed` in $GITHUB_OUTPUT: whether a new +cache is worth saving. + +The key is a sha256 over: + - the step's name and the scorer's name as given, and every argument as given; + - the content of each argument that is a file: the rvt-rs IFC and the glTF + (the files the run writes beside the step, in D) and everything else + (Revit's export, the validation findings, the property set definitions). + An IFC in D is hashed without its FILE_NAME and IFCOWNERHISTORY lines, + which carry the export time (the hash tools/ci/measure-reference-models.sh + writes to ifc.sha256), so a rebuild that writes the same model hits; + - the scorer and every module of tools/re and tools/ci it can reach, by + import, importlib path or name (a module it only mentions is included); + - the value of every environment variable the closure reads, except the two + that only size the mesh cache (RVT_MESH_CACHE, RVT_MESH_WORKERS); + - the files a scorer reads that its arguments do not name (EXTRA_INPUTS); + - the Python and platform, the version of every package + tools/ci/requirements-ifc.txt pins as installed, and RVT_IFC_FLAGS. + +A step is not cached when its closure does something the key cannot see: lists +or walks a directory, takes a directory name from an argument, reads a file +it names itself, runs a program, draws a random number or a date, reads the +IFC's export time, or reads an environment variable by a computed name. The +models that Measure keeps only counts of (held out, BIMcollab's) are never +cached. Only status 0 is kept: a scorer that crashed, ran out of time or of +memory is run again. +""" + +import argparse +import ast +import fnmatch +import hashlib +import importlib.metadata +import json +import os +import platform +import re +import shutil +import sys +import time + +CI_DIR = os.path.dirname(os.path.abspath(__file__)) +RE_DIR = os.path.join(os.path.dirname(CI_DIR), "re") +REQUIREMENTS = os.path.join(CI_DIR, "requirements-ifc.txt") + +FORMAT = 1 +NEUTRAL_ENV = {"RVT_MESH_CACHE", "RVT_MESH_WORKERS"} +HELD_OUT = ("holdout-*", "bimcollab-*") +MAX_ENTRY_BYTES = 4 * 1024 * 1024 +STALE_SECONDS = 3 * 24 * 3600 +# Output that shows a failure a rerun could mend, whatever the status says. +TRANSIENT = ("unreadable:", "a worker died", "Traceback (most recent call last)", "MemoryError", "BrokenProcessPool") +IFC_TIME_LINES = re.compile(rb"^FILE_NAME\(|IFCOWNERHISTORY") + +BAD_MODULES = {"subprocess", "socket", "urllib", "requests", "http", "random", "uuid", "datetime", "tempfile", "glob"} +BAD_ATTRIBUTES = { + "listdir": "fs-listing", + "scandir": "fs-listing", + "iterdir": "fs-listing", + "rglob": "fs-listing", + "walk": "fs-listing", + "getcwd": "process", + "expanduser": "process", + "system": "process", + "popen": "process", + "OwnerHistory": "export-time", + "CreationDate": "export-time", + "file_name": "export-time", + "time_stamp": "export-time", + "TimeStamp": "export-time", + "creation_date": "export-time", +} +OWN_DIRECTORY = {"os.path.abspath(__file__)", "__file__"} +# ifc_common lists the mesh cache directory that RVT_MESH_CACHE names, which +# only saves meshing and changes no scorer's output. +# gen_standard_property_sets fetches the property set definitions when the +# file is absent, and checks them against the sha256 it pins; its source is in +# the key, so the pin is. +ALLOWED_IN = {"ifc_common": {"fs-listing"}, "gen_standard_property_sets": {"process"}} + + +class Uncacheable(Exception): + pass + + +def work_dir(): + path = os.environ.get("MEASURE_CACHE_WORK") or os.path.join(os.environ.get("RUNNER_TEMP", "/tmp"), "measure-cache-work") + os.makedirs(path, exist_ok=True) + return path + + +def cache_dir(): + path = os.environ.get("MEASURE_CACHE_DIR") + if path: + os.makedirs(path, exist_ok=True) + return path + + +def log(line): + with open(os.path.join(work_dir(), "report.log"), "a", encoding="utf-8") as report: + report.write(line + "\n") + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def atomic_write(path, data): + scratch = f"{path}.{os.getpid()}.{time.time_ns()}" + with open(scratch, "wb") as out: + out.write(data) + os.replace(scratch, path) + + +# ---- file digests ----------------------------------------------------------- + + +def file_digest(path, ifc_without_export_time): + """The sha256 of the file, memoised for the run; for an IFC the run wrote, + of its lines other than FILE_NAME and IFCOWNERHISTORY.""" + stat = os.stat(path) + memo = os.path.join(work_dir(), "digests") + os.makedirs(memo, exist_ok=True) + name = sha(f"{os.path.realpath(path)} {stat.st_size} {stat.st_mtime_ns} {ifc_without_export_time}".encode()) + remembered = os.path.join(memo, name) + if os.path.exists(remembered): + with open(remembered, encoding="ascii") as handle: + return handle.read() + digest = hashlib.sha256() + with open(path, "rb") as handle: + if ifc_without_export_time: + for line in handle: + if not IFC_TIME_LINES.search(line): + digest.update(line if line.endswith(b"\n") else line + b"\n") + else: + for block in iter(lambda: handle.read(1 << 20), b""): + digest.update(block) + value = digest.hexdigest() + atomic_write(remembered, value.encode("ascii")) + return value + + +# ---- the scorer's closure --------------------------------------------------- + + +def local_modules(): + found = {} + for directory in (RE_DIR, CI_DIR): + for name in sorted(os.listdir(directory)): + if name.endswith(".py"): + found.setdefault(name[:-3], []).append(os.path.join(directory, name)) + return found + + +def parse(path): + try: + with open(path, "rb") as handle: + return ast.parse(handle.read(), filename=path) + except (OSError, SyntaxError) as error: + raise Uncacheable(f"cannot read {os.path.basename(path)}: {type(error).__name__}") + + +def docstrings(tree): + ids = set() + for node in ast.walk(tree): + if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + body = node.body + if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant) and isinstance(body[0].value.value, str): + ids.add(id(body[0].value)) + return ids + + +def references(tree, known): + skip = docstrings(tree) + names = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom): + if node.module: + names.add(node.module.split(".")[0]) + names.update(alias.name for alias in node.names) + elif isinstance(node, ast.Constant) and isinstance(node.value, str) and id(node) not in skip: + base = os.path.basename(node.value) + names.add(base[:-3] if base.endswith(".py") else node.value) + return names & set(known) + + +def closure(script): + """The script and every local module it reaches, as {path: tree}.""" + known = local_modules() + script = os.path.abspath(script) + if not os.path.isfile(script): + raise Uncacheable("the scorer is not a file") + seen = {} + queue = [script] + while queue: + path = queue.pop() + if path in seen: + continue + seen[path] = parse(path) + for name in references(seen[path], known): + own = os.path.join(os.path.dirname(path), name + ".py") + queue.append(own if os.path.isfile(own) else known[name][0]) + return seen + + +def audit(trees): + """What the closure does that the key could not see, and the environment + variables it reads (a set of names; None for a computed name).""" + kinds = set() + env = set() + for path, tree in trees.items(): + found = set() + skip = docstrings(tree) + handled = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + found.update("process" for alias in node.names if alias.name.split(".")[0] in BAD_MODULES) + elif isinstance(node, ast.ImportFrom): + if node.module and node.module.split(".")[0] in BAD_MODULES: + found.add("process") + if node.module == "os" and any(alias.name in ("environ", "getenv") for alias in node.names): + env.add(None) + elif isinstance(node, ast.Attribute): + if node.attr in BAD_ATTRIBUTES: + found.add(BAD_ATTRIBUTES[node.attr]) + elif isinstance(node, ast.Constant) and isinstance(node.value, str) and id(node) not in skip: + if re.search(r"FILE_NAME|IFCOWNERHISTORY", node.value, re.I): + found.add("export-time") + if isinstance(node, ast.Call): + called = ast.unparse(node.func) + first = node.args[0] if node.args else None + if called in ("os.environ.get", "os.getenv", "os.environ.setdefault", "os.environ.pop"): + env.add(first.value if isinstance(first, ast.Constant) and isinstance(first.value, str) else None) + if called.startswith("os.environ."): + handled.add(id(node.func.value)) + elif called in ("os.path.dirname", "os.path.split"): + if first is None or ast.unparse(first) not in OWN_DIRECTORY: + found.add("dirname") + elif called == "open" and isinstance(first, ast.Constant) and isinstance(first.value, str): + found.add("literal-path") + elif isinstance(node, ast.Subscript) and ast.unparse(node.value) == "os.environ": + key = node.slice + env.add(key.value if isinstance(key, ast.Constant) and isinstance(key.value, str) else None) + handled.add(id(node.value)) + for node in ast.walk(tree): + if isinstance(node, ast.Attribute) and ast.unparse(node) == "os.environ" and id(node) not in handled: + env.add(None) + kinds |= found - ALLOWED_IN.get(os.path.basename(path)[:-3], set()) + return kinds, env + + +def closure_digest(trees): + parts = [] + for path in sorted(trees): + with open(path, "rb") as handle: + parts.append(f"{os.path.relpath(path, os.path.dirname(CI_DIR))} {sha(handle.read())}") + return sha("\n".join(parts).encode()) + + +# ---- files a scorer reads that its arguments do not name -------------------- + + +def declared_siblings(script, trees, ref): + """unfilled_openings_vs_ifc's `siblings(path)`: the other Revit exports in + the reference export's directory that share its first four words.""" + tree = trees[os.path.abspath(script)] + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == "siblings": + namespace = {"os": os} + exec(compile(ast.Module([node], []), script, "exec"), namespace) + return namespace["siblings"](ref) + raise Uncacheable("the scorer no longer has the siblings() Measure's cache reads") + + +# scorer -> (what it reads beside its arguments, found from the file arguments) +EXTRA_INPUTS = { + # The export's diagnostics, beside the IFC (args[0]). + "qtos_vs_ifc": lambda script, trees, files: [os.path.join(os.path.dirname(os.path.abspath(files[0])), "diagnostics.json")], + "unfilled_openings_vs_ifc": lambda script, trees, files: declared_siblings(script, trees, files[1]), +} + + +# ---- the key ---------------------------------------------------------------- + + +def fingerprint(): + packages = [] + with open(REQUIREMENTS, encoding="utf-8") as handle: + for line in handle: + match = re.match(r"^([A-Za-z0-9_.\-]+)==", line) + if match: + try: + packages.append((match.group(1), importlib.metadata.version(match.group(1)))) + except importlib.metadata.PackageNotFoundError: + packages.append((match.group(1), None)) + return { + "format": FORMAT, + "python": sys.version, + "platform": [platform.system(), platform.machine(), *platform.libc_ver()], + "packages": packages, + "ifc_flags": os.environ.get("RVT_IFC_FLAGS", ""), + } + + +def make_key(args, script, script_args): + outputs = {os.path.abspath(path) for path in args.extra} + if os.path.abspath(args.dest) in outputs: + raise Uncacheable("the output is also an input") + directory = os.path.abspath(args.dir) + trees = closure(script) + kinds, names = audit(trees) + stem = os.path.basename(script)[:-3] + declared = EXTRA_INPUTS.get(stem) + if declared: + kinds.discard("dirname") + kinds.discard("fs-listing") + if None in names: + kinds.add("computed-env") + if kinds: + raise Uncacheable("reads " + ", ".join(sorted(kinds))) + tokens = [] + files = [] + for arg in script_args: + full = os.path.abspath(arg) + if full in outputs: + tokens.append(["output", arg]) + elif os.path.isfile(arg): + own = os.path.dirname(full) == directory + tokens.append(["file", arg, file_digest(arg, own and arg.endswith(".ifc"))]) + files.append(arg) + else: + tokens.append(["literal", arg]) + extra = [] + if declared: + for path in declared(script, trees, files): + extra.append([os.path.basename(path), file_digest(path, False) if os.path.isfile(path) else None]) + environment = sorted((name, os.environ.get(name)) for name in names if name not in NEUTRAL_ENV) + parts = [fingerprint(), args.step, script, closure_digest(trees), environment, tokens, extra] + return sha(json.dumps(parts, sort_keys=True).encode()) + + +def keyfile(args): + return os.path.join(work_dir(), "keys", sha(f"{os.path.abspath(args.dir)} {args.step}".encode())) + + +def entry_path(root, key): + return os.path.join(root, key[:2], key) + + +# ---- commands --------------------------------------------------------------- + + +def lookup(args): + root = cache_dir() + label = f"{os.path.basename(os.path.abspath(args.dir))} {args.step}" + if not root: + return 2 + if any(fnmatch.fnmatch(os.path.basename(os.path.abspath(args.dir)), pattern) for pattern in HELD_OUT): + log(f"not cached {label}: held out, only its counts leave the runner") + return 2 + script, *script_args = args.command + try: + key = make_key(args, script, script_args) + except Uncacheable as why: + log(f"not cached {label}: {why}") + return 2 + os.makedirs(os.path.dirname(keyfile(args)), exist_ok=True) + atomic_write(keyfile(args), key.encode("ascii")) + entry = entry_path(root, key) + if os.path.isfile(os.path.join(entry, "out")): + shutil.copyfile(os.path.join(entry, "out"), args.dest) + for path in args.extra: + kept = os.path.join(entry, "extra-" + os.path.basename(path)) + if os.path.isfile(kept): + shutil.copyfile(kept, path) + stale = time.time() - os.stat(entry).st_mtime > STALE_SECONDS + os.utime(entry) + with open(os.path.join(args.dir, "timings.log"), "a", encoding="utf-8") as timings: + timings.write(f"{args.step} cached\n") + log(f"hit {label} {key[:12]}{' stale' if stale else ''}") + return 0 + log(f"miss {label} {key[:12]}") + return 1 + + +def store(args): + root = cache_dir() + label = f"{os.path.basename(os.path.abspath(args.dir))} {args.step}" + if not root or not os.path.exists(keyfile(args)): + return 0 + with open(keyfile(args), encoding="ascii") as handle: + key = handle.read() + with open(args.dest, "rb") as handle: + out = handle.read() + status = args.status + if status is None: + last = out.rstrip(b"\n").rsplit(b"\n", 1)[-1] + status = int(last[5:]) if last.startswith(b"exit ") and last[5:].isdigit() else -1 + text = out.decode("utf-8", "replace") + if status != 0: + log(f"not kept {label} {key[:12]}: status {status}") + return 0 + if len(out) > MAX_ENTRY_BYTES or any(mark in text for mark in TRANSIENT): + log(f"not kept {label} {key[:12]}: output too large or shows a failure to repeat") + return 0 + entry = entry_path(root, key) + if os.path.isdir(entry): + return 0 + scratch = os.path.join(root, f".new-{os.getpid()}-{time.time_ns()}") + os.makedirs(scratch) + shutil.copyfile(args.dest, os.path.join(scratch, "out")) + for path in args.extra: + if os.path.isfile(path): + shutil.copyfile(path, os.path.join(scratch, "extra-" + os.path.basename(path))) + os.makedirs(os.path.dirname(entry), exist_ok=True) + try: + os.rename(scratch, entry) + except OSError: + shutil.rmtree(scratch, ignore_errors=True) + return 0 + log(f"kept {label} {key[:12]} {len(out)} bytes") + return 0 + + +def entries(root): + for shard in sorted(os.listdir(root)): + shard_path = os.path.join(root, shard) + if len(shard) == 2 and os.path.isdir(shard_path): + for name in sorted(os.listdir(shard_path)): + path = os.path.join(shard_path, name) + if os.path.isdir(path): + yield path + + +def size_of(path): + return sum(os.path.getsize(os.path.join(base, name)) for base, _, names in os.walk(path) for name in names) + + +def finish(args): + root = cache_dir() + report = os.path.join(work_dir(), "report.log") + lines = [] + if os.path.exists(report): + with open(report, encoding="utf-8") as handle: + lines = handle.read().splitlines() + count = {kind: sum(line.startswith(kind + " ") for line in lines) for kind in ("hit", "miss", "kept", "not cached", "not kept")} + stale = sum(line.startswith("hit ") and line.endswith(" stale") for line in lines) + pruned = 0 + total = 0 + kept_entries = 0 + if root: + now = time.time() + survivors = [] + for path in list(entries(root)): + if now - os.stat(path).st_mtime > args.days * 86400: + shutil.rmtree(path) + pruned += 1 + else: + survivors.append((os.stat(path).st_mtime, path, size_of(path))) + survivors.sort() + total = sum(size for _, _, size in survivors) + while survivors and total > args.max_mb * 1024 * 1024: + _, path, size = survivors.pop(0) + shutil.rmtree(path) + total -= size + pruned += 1 + kept_entries = len(survivors) + for shard in os.listdir(root): + path = os.path.join(root, shard) + if os.path.isdir(path) and not os.listdir(path): + os.rmdir(path) + elif shard.startswith(".new-"): + shutil.rmtree(path, ignore_errors=True) + changed = count["kept"] > 0 or pruned > 0 or stale > 0 + summary = ( + f"score cache: {count['hit']} hits, {count['miss']} misses ({count['kept']} kept), " + f"{count['not cached']} not cached; {kept_entries} entries, {total / 1048576:.1f} MiB; " + f"{pruned} pruned, {stale} refreshed; changed={str(changed).lower()}" + ) + print(summary) + if args.report: + with open(args.report, "w", encoding="utf-8") as out: + out.write("\n".join(lines) + "\n" + summary + "\n") + output = os.environ.get("GITHUB_OUTPUT") + if output: + with open(output, "a", encoding="utf-8") as handle: + handle.write(f"changed={str(changed).lower()}\n") + return 0 + + +def main(argv): + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + sub = parser.add_subparsers(dest="command_name", required=True) + for name in ("lookup", "store"): + p = sub.add_parser(name) + p.add_argument("--step", required=True) + p.add_argument("--dir", required=True) + p.add_argument("--dest", required=True) + p.add_argument("--extra", action="append", default=[]) + if name == "store": + p.add_argument("--status", type=int) + else: + p.add_argument("command", nargs="+") + p = sub.add_parser("finish") + p.add_argument("--days", type=int, default=21) + p.add_argument("--max-mb", type=int, default=256) + p.add_argument("--report") + args = parser.parse_args(argv) + return {"lookup": lookup, "store": store, "finish": finish}[args.command_name](args) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tools/ci/measure-reference-models.sh b/tools/ci/measure-reference-models.sh index 573808a3a..04bb68fb6 100755 --- a/tools/ci/measure-reference-models.sh +++ b/tools/ci/measure-reference-models.sh @@ -59,6 +59,12 @@ # another release and the same project saved by that one, the sample first, # into probe-pair.txt, ending with its exit status: the cross-release # comparisons of #740 and #742. +# When MEASURE_CACHE_DIR names a directory, a scorer or schema step whose every +# input is as it was in an earlier run takes that run's output +# (tools/ci/measure-cache.py, docs/maintaining.md); its ` cached` line +# stands for its time in timings.log, and OUT_DIR/score-cache.log (written by the +# workflow) lists each hit and miss. Exports, rvt-info, rvt-gltf, the witness +# exports and probes always run. # A scorer that fails is recorded, not fatal: its output is the evidence. # These runs check what the exports hold; their times are not benchmarks # (the workflow's `timing` job times both refs on one runner, @@ -203,6 +209,48 @@ score() { # name args... echo "exit $?" >> "$out" } +# The score cache (tools/ci/measure-cache.py), on when MEASURE_CACHE_DIR names +# its directory: a step whose every input is as it was in an earlier run takes +# that run's output, and ` cached` stands for its time in timings.log. A +# step is kept only if it finished with status 0, and the models whose results +# only leave the runner as counts are never cached. +CACHE_TOOL="$(dirname "$0")/measure-cache.py" +cscore() { # name args...: `score`, from the cache when the scorer and its inputs are unchanged + local out="$1" dir step found=2; shift + dir=$(dirname "$out"); step=$(basename "$out" .txt) + if [ -n "${MEASURE_CACHE_DIR:-}" ]; then + python "$CACHE_TOOL" lookup --step "$step" --dir "$dir" --dest "$out" -- "$@" + found=$? + [ "$found" = 0 ] && return 0 + fi + score "$out" "$@" + [ "$found" = 1 ] && python "$CACHE_TOOL" store --step "$step" --dir "$dir" --dest "$out" + return 0 +} + +# A schema step's output on stdout, the cache's when the script and the file +# are unchanged, else the step's own under its time limit. A file the step also +# writes is named by CACHE_EXTRA and kept with the output. +cached() { # dir step limit script args... + local dir="$1" step="$2" limit="$3" script="$4" tmp status found=2; shift 4 + tmp="${RUNNER_TEMP:-/tmp}/cached-$$-$(basename "$dir")-$step" + if [ -n "${MEASURE_CACHE_DIR:-}" ]; then + python "$CACHE_TOOL" lookup --step "$step" --dir "$dir" --dest "$tmp" ${CACHE_EXTRA:+--extra "$CACHE_EXTRA"} -- "$script" "$@" + found=$? + if [ "$found" = 0 ]; then + cat "$tmp" + rm -f "$tmp" + return 0 + fi + fi + timed "$dir" "$step" timeout "$limit" python "$script" "$@" > "$tmp" 2>&1 + status=$? + cat "$tmp" + [ "$found" = 1 ] && python "$CACHE_TOOL" store --step "$step" --dir "$dir" --dest "$tmp" --status "$status" ${CACHE_EXTRA:+--extra "$CACHE_EXTRA"} + rm -f "$tmp" + return "$status" +} + info() { # file dir "$BIN/rvt-info" "$1" --redact --json > "$2/info.json" 2> "$2/info.log" echo "exit $?" > "$2/info.exit" @@ -216,19 +264,19 @@ PSET_DEFINITIONS="${RUNNER_TEMP:-/tmp}/ifc4-pset-definitions.csv" schema() { # dir max_bytes { echo "arity:" - timed "$1" schema-arity timeout 600 python "$(dirname "$0")/ifc_schema_arity.py" "$1/model.ifc" 2>&1 \ + cached "$1" schema-arity 600 "$(dirname "$0")/ifc_schema_arity.py" "$1/model.ifc" \ | grep -v '^IFC schema conformance passed\|^ instances checked' || true if [ "$(stat -c %s "$1/model.ifc")" -le "$2" ]; then echo "validate:" - timed "$1" schema-validate timeout "$STEP_TIMEOUT" python "$(dirname "$0")/ifc_validate_summary.py" "$1/model.ifc" "$1/validation.jsonl" 2>&1 + CACHE_EXTRA="$1/validation.jsonl" cached "$1" schema-validate "$STEP_TIMEOUT" "$(dirname "$0")/ifc_validate_summary.py" "$1/model.ifc" "$1/validation.jsonl" echo "rules:" - timed "$1" schema-rules timeout "$STEP_TIMEOUT" python "$(dirname "$0")/ifc_validate_summary.py" --rules "$1/model.ifc" 2>&1 + cached "$1" schema-rules "$STEP_TIMEOUT" "$(dirname "$0")/ifc_validate_summary.py" --rules "$1/model.ifc" else echo "validate: skipped, over $(( $2 / 1048576 )) MB" fi echo "property sets:" - timed "$1" schema-psets timeout "$STEP_TIMEOUT" python "$(dirname "$0")/ifc4_property_sets.py" "$1/model.ifc" \ - --definitions "$PSET_DEFINITIONS" 2>&1 + cached "$1" schema-psets "$STEP_TIMEOUT" "$(dirname "$0")/ifc4_property_sets.py" "$1/model.ifc" \ + --definitions "$PSET_DEFINITIONS" } > "$1/schema.txt" } @@ -305,7 +353,7 @@ model_task() { # name rvt ref task if [ "$ref" != "-" ] && wanted ports_vs_ifc; then extra=() [ -f "$dir/validation.jsonl" ] && extra=(--validation "$dir/validation.jsonl") - score "$dir/ports_vs_ifc.txt" "$SCORERS/ports_vs_ifc.py" "$dir/model.ifc" "$MODELS/$ref" "${extra[@]}" + cscore "$dir/ports_vs_ifc.txt" "$SCORERS/ports_vs_ifc.py" "$dir/model.ifc" "$MODELS/$ref" "${extra[@]}" fi ;; witness) @@ -322,18 +370,23 @@ model_task() { # name rvt ref task if [ "$ref" != "-" ]; then for s in $GLB_SCORERS; do wanted "$s" || continue - score "$dir/$s.txt" "$SCORERS/$s.py" "$dir/model.glb" "$MODELS/$ref" + cscore "$dir/$s.txt" "$SCORERS/$s.py" "$dir/model.glb" "$MODELS/$ref" done fi ;; ifc:*) s=${task#ifc:} - score "$dir/$s.txt" "$SCORERS/$s.py" "$dir/model.ifc" "$MODELS/$ref" + # A research lister named in MEASURE_SCORERS, in neither list, is not cached. + if listed "$s"; then + cscore "$dir/$s.txt" "$SCORERS/$s.py" "$dir/model.ifc" "$MODELS/$ref" + else + score "$dir/$s.txt" "$SCORERS/$s.py" "$dir/model.ifc" "$MODELS/$ref" + fi ;; meshes) for s in saved_meshes_vs_ifc mesh_bodies_vs_ifc; do wanted "$s" || continue - RVT_MESH_CACHE="$dir/mesh-cache" score "$dir/$s.txt" "$SCORERS/$s.py" "$dir/model.ifc" "$MODELS/$ref" + RVT_MESH_CACHE="$dir/mesh-cache" cscore "$dir/$s.txt" "$SCORERS/$s.py" "$dir/model.ifc" "$MODELS/$ref" done echo "$name: mesh cache $(du -sh "$dir/mesh-cache" 2>/dev/null | cut -f1), removed" rm -rf "$dir/mesh-cache" @@ -351,7 +404,7 @@ model_task() { # name rvt ref task timed "$dir" rvt-ifc-revit-sets "$BIN/rvt-ifc" "$MODELS/$rvt" -o "$dir/revit-sets.ifc" ${RVT_IFC_FLAGS:-} "${flags[@]}" \ > "$dir/rvt-ifc-revit-sets.log" 2>&1 echo "exit $?" >> "$dir/rvt-ifc-revit-sets.log" - score "$dir/revit_property_sets_vs_ifc.txt" "$SCORERS/revit_property_sets_vs_ifc.py" "$dir/revit-sets.ifc" "$MODELS/$sets" --list + cscore "$dir/revit_property_sets_vs_ifc.txt" "$SCORERS/revit_property_sets_vs_ifc.py" "$dir/revit-sets.ifc" "$MODELS/$sets" --list rm -f "$dir/revit-sets.ifc" ;; esac diff --git a/tools/maintainer/clean_caches.py b/tools/maintainer/clean_caches.py index 113ff2da8..d9b96171a 100644 --- a/tools/maintainer/clean_caches.py +++ b/tools/maintainer/clean_caches.py @@ -7,7 +7,9 @@ again, apart from the newest few that a build may still restore from; - on `main`, the superseded overlay base databases of each CodeQL language and version: every analysis of `main` saves one under its commit, and the next - analysis on that version reads only the newest. + analysis on that version reads only the newest; +- on `main`, the superseded generations of Measure's score cache, which a run + restores from the newest of its prefix and saves anew. The caches of live branches, of open pull requests, of tags, and `main`'s newest generations are kept. The repository's cache limit is 10 GB and GitHub evicts the @@ -39,6 +41,9 @@ # highest enabled CLI version among the cached bases, so each version keeps its # newest ones. CODEQL_OVERLAY_KEY = re.compile(r"(codeql-overlay-base-database-\d+-[0-9a-f]+-[a-z]+-[0-9.]+)-[0-9a-f]{40}-\d+-\d+") +# Measure's score cache (tools/ci/measure-cache.py): measure-scores-v-
---. +# A run restores the newest of its prefix, so only the newest generations are read. +SCORE_CACHE_KEY = re.compile(r"(measure-scores-v\d+-.+)-\d+-\d+") def gh(*args): @@ -76,7 +81,7 @@ def unusable(caches, open_prs, live_branches, keep): if name != "main" and name not in live_branches: doomed.append((c, f"branch {name} was deleted")) elif name == "main": - k = RUST_CACHE_KEY.fullmatch(c["key"]) or CODEQL_OVERLAY_KEY.fullmatch(c["key"]) + k = RUST_CACHE_KEY.fullmatch(c["key"]) or CODEQL_OVERLAY_KEY.fullmatch(c["key"]) or SCORE_CACHE_KEY.fullmatch(c["key"]) if k: generations.setdefault(k.group(1), []).append(c) for family, lst in generations.items(): From 5a9840078263f64a03b35a6ed849d0b5d1b81f1b Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:42:49 -0500 Subject: [PATCH 02/10] ci(measure): a score cache lookup or store that fails runs the step as before (#1237) An exception in the helper exited 1, which the Measure script reads as a miss and would then store under a stale key. A failed lookup now says not cached (2), a failed store keeps nothing, and a lookup forgets its key file first. --- tools/ci/measure-cache.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tools/ci/measure-cache.py b/tools/ci/measure-cache.py index 64f498ac9..3aad2f0be 100644 --- a/tools/ci/measure-cache.py +++ b/tools/ci/measure-cache.py @@ -523,7 +523,22 @@ def main(argv): p.add_argument("--max-mb", type=int, default=256) p.add_argument("--report") args = parser.parse_args(argv) - return {"lookup": lookup, "store": store, "finish": finish}[args.command_name](args) + if args.command_name == "finish": + return finish(args) + # Whatever goes wrong with the cache, the step runs as it always did: a + # lookup that fails says "not cached" (2), never "miss" (1), and a store + # that fails keeps nothing. + failed = 2 if args.command_name == "lookup" else 0 + try: + if args.command_name == "lookup": + os.remove(keyfile(args)) + except OSError: + pass + try: + return {"lookup": lookup, "store": store}[args.command_name](args) + except Exception as error: + log(f"not cached {os.path.basename(os.path.abspath(args.dir))} {args.step}: {args.command_name} failed, {type(error).__name__}: {error}") + return failed if __name__ == "__main__": From 74d91bada881b86dc34b3ef37f1cf0cbbc150765 Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:46:57 -0500 Subject: [PATCH 03/10] ci(measure): the schema validation and the EXPRESS rules check read one parse of the IFC, output unchanged (#1237) ifc_validate_summary.py --both MODEL STATEMENTS RULES.txt runs the default validation and the rules check on one ifcopenshell.open and writes each exactly as its own run does, an unreadable file or a failing check being the finding of both or of that one as before. The schema function runs it as one step, schema-validate-rules, cached with the validation findings and the rules summary. --- tools/ci/ifc_validate_summary.py | 54 +++++++++++++++++++++++++--- tools/ci/measure-reference-models.sh | 19 ++++++---- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/tools/ci/ifc_validate_summary.py b/tools/ci/ifc_validate_summary.py index 29f94d515..cd9339322 100644 --- a/tools/ci/ifc_validate_summary.py +++ b/tools/ci/ifc_validate_summary.py @@ -22,10 +22,18 @@ findings : (at most 30) +With --both MODEL.ifc STATEMENTS.jsonl RULES.txt it does both from one parse +of the file: the default validation's summary on stdout and its findings in +STATEMENTS.jsonl, and what --rules prints in RULES.txt, each exactly as its own +run writes it. + Usage: ifc_validate_summary.py [--rules] MODEL.ifc [STATEMENTS.jsonl] + ifc_validate_summary.py --both MODEL.ifc STATEMENTS.jsonl RULES.txt """ import collections +import contextlib +import io import json import re import sys @@ -43,18 +51,49 @@ def describe(statement): return f"{entity} {attribute}: {message}"[:300] -def rules(path): +def rules(path, model=None): logger = ifcopenshell.validate.json_logger() - ifcopenshell.express.rule_executor.run(ifcopenshell.open(path), logger) + ifcopenshell.express.rule_executor.run(model if model is not None else ifcopenshell.open(path), logger) kinds = collections.Counter(re.sub(r"#\d+", "#", describe(s)) for s in logger.statements) print(f"findings {len(logger.statements)}") for kind, count in sorted(kinds.items(), key=lambda item: (-item[1], item[0]))[:30]: print(f"{count} {kind}") -def main(path, statements=None): +def unreadable(error): + return f"unreadable: {type(error).__name__}: {json.dumps(str(error))[:300]}" + + +def both(path, statements, rules_out): + """The default validation on stdout and the rules check in RULES_OUT, each + exactly what its own run prints, from one parse of the file: a file that + does not open, or a check that fails, is the finding of both or of that + one, as it is when they run apart.""" + try: + model = ifcopenshell.open(path) + except Exception as error: + print(unreadable(error)) + with open(rules_out, "w", encoding="utf-8") as out: + out.write(unreadable(error) + "\n") + return + try: + main(path, statements, model) + except Exception as error: + print(unreadable(error)) + captured = io.StringIO() + failure = None + with contextlib.redirect_stdout(captured): + try: + rules(path, model) + except Exception as error: + failure = error + with open(rules_out, "w", encoding="utf-8") as out: + out.write(captured.getvalue() + (unreadable(failure) + "\n" if failure else "")) + + +def main(path, statements=None, model=None): logger = ifcopenshell.validate.json_logger() - ifcopenshell.validate.validate(ifcopenshell.open(path), logger) + ifcopenshell.validate.validate(model if model is not None else ifcopenshell.open(path), logger) if statements: with open(statements, "w", encoding="utf-8") as out: for statement in logger.statements: @@ -67,6 +106,11 @@ def main(path, statements=None): if __name__ == "__main__": args = sys.argv[1:] + if args[:1] == ["--both"]: + if len(args) != 4: + sys.exit(__doc__) + both(args[1], args[2], args[3]) + sys.exit(0) with_rules = args[:1] == ["--rules"] if with_rules: args = args[1:] @@ -78,4 +122,4 @@ def main(path, statements=None): else: main(args[0], args[1] if len(args) == 2 else None) except Exception as error: # an unreadable file is a finding too - print(f"unreadable: {type(error).__name__}: {json.dumps(str(error))[:300]}") + print(unreadable(error)) diff --git a/tools/ci/measure-reference-models.sh b/tools/ci/measure-reference-models.sh index 04bb68fb6..120a57d8f 100755 --- a/tools/ci/measure-reference-models.sh +++ b/tools/ci/measure-reference-models.sh @@ -229,13 +229,14 @@ cscore() { # name args...: `score`, from the cache when the scorer and its input } # A schema step's output on stdout, the cache's when the script and the file -# are unchanged, else the step's own under its time limit. A file the step also -# writes is named by CACHE_EXTRA and kept with the output. +# are unchanged, else the step's own under its time limit. The files the step +# also writes are named by CACHE_EXTRA, and kept with the output. cached() { # dir step limit script args... - local dir="$1" step="$2" limit="$3" script="$4" tmp status found=2; shift 4 + local dir="$1" step="$2" limit="$3" script="$4" tmp status found=2 extra=() path; shift 4 tmp="${RUNNER_TEMP:-/tmp}/cached-$$-$(basename "$dir")-$step" + for path in ${CACHE_EXTRA:-}; do extra+=(--extra "$path"); done if [ -n "${MEASURE_CACHE_DIR:-}" ]; then - python "$CACHE_TOOL" lookup --step "$step" --dir "$dir" --dest "$tmp" ${CACHE_EXTRA:+--extra "$CACHE_EXTRA"} -- "$script" "$@" + python "$CACHE_TOOL" lookup --step "$step" --dir "$dir" --dest "$tmp" ${extra[@]+"${extra[@]}"} -- "$script" "$@" found=$? if [ "$found" = 0 ]; then cat "$tmp" @@ -246,7 +247,7 @@ cached() { # dir step limit script args... timed "$dir" "$step" timeout "$limit" python "$script" "$@" > "$tmp" 2>&1 status=$? cat "$tmp" - [ "$found" = 1 ] && python "$CACHE_TOOL" store --step "$step" --dir "$dir" --dest "$tmp" --status "$status" ${CACHE_EXTRA:+--extra "$CACHE_EXTRA"} + [ "$found" = 1 ] && python "$CACHE_TOOL" store --step "$step" --dir "$dir" --dest "$tmp" --status "$status" ${extra[@]+"${extra[@]}"} rm -f "$tmp" return "$status" } @@ -267,10 +268,14 @@ schema() { # dir max_bytes cached "$1" schema-arity 600 "$(dirname "$0")/ifc_schema_arity.py" "$1/model.ifc" \ | grep -v '^IFC schema conformance passed\|^ instances checked' || true if [ "$(stat -c %s "$1/model.ifc")" -le "$2" ]; then + # The validation and the EXPRESS rules read one parse of the file; the + # rules' summary comes back in a file, to follow the validation's. echo "validate:" - CACHE_EXTRA="$1/validation.jsonl" cached "$1" schema-validate "$STEP_TIMEOUT" "$(dirname "$0")/ifc_validate_summary.py" "$1/model.ifc" "$1/validation.jsonl" + CACHE_EXTRA="$1/validation.jsonl $1/schema-rules.part" cached "$1" schema-validate-rules $(( 2 * STEP_TIMEOUT )) \ + "$(dirname "$0")/ifc_validate_summary.py" --both "$1/model.ifc" "$1/validation.jsonl" "$1/schema-rules.part" echo "rules:" - cached "$1" schema-rules "$STEP_TIMEOUT" "$(dirname "$0")/ifc_validate_summary.py" --rules "$1/model.ifc" + cat "$1/schema-rules.part" 2> /dev/null || true + rm -f "$1/schema-rules.part" else echo "validate: skipped, over $(( $2 / 1048576 )) MB" fi From 840af99f1b39446cf9441a7047e69f5ada5452c0 Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:51:45 -0500 Subject: [PATCH 04/10] ci(measure): each score cache entry names its model, a run removes any that names a held-out one, and the key adds the library versions IfcOpenShell and numpy report (#1237) The Actions cache of a public repository can be restored by a workflow that a fork's pull request changes, so nothing tools/ci/scrub-holdout-results.sh removes may be in it. holdout-* and bimcollab-* were never looked up or stored; now store refuses them as well, every entry carries its model in meta.json, and finish removes an entry that names a held-out model or none and prints the entries by model prefix. The key also hashes requirements-ifc.txt and the versions ifcopenshell and numpy report, worked out once a run. --- docs/maintaining.md | 26 ++++++++--- tools/ci/measure-cache.py | 91 +++++++++++++++++++++++++++++++-------- 2 files changed, 93 insertions(+), 24 deletions(-) diff --git a/docs/maintaining.md b/docs/maintaining.md index 1b58ee594..32f5a8761 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -110,9 +110,13 @@ writes the cached output and a ` cached` line to the model's the two that size the mesh cache), which is `PSETS_MISSING` for `psets_vs_ifc` and `qtos_vs_ifc`; the files a scorer reads that its arguments do not name (`qtos_vs_ifc`'s `diagnostics.json`, - `unfilled_openings_vs_ifc`'s sibling Revit exports); the Python, platform - and installed version of every package `tools/ci/requirements-ifc.txt` - pins; and `RVT_IFC_FLAGS`. + `unfilled_openings_vs_ifc`'s sibling Revit exports); the Python, the + platform, the sha256 of `tools/ci/requirements-ifc.txt`, the installed + version of every package it pins, the versions IfcOpenShell and numpy + report, and `RVT_IFC_FLAGS`. Each entry therefore hashes the rvt-rs IFC, + Revit's reference IFC (or the glTF), the command-line arguments, the scorer + and its transitive `tools/re` and `tools/ci` imports, the environment + variables they read and the library versions. - **What is never cached.** A step whose closure lists or walks a directory, takes a directory from an argument, opens a file by a literal path, runs a program, draws a number or a date, reads the IFC's export time or reads an @@ -123,13 +127,25 @@ writes the cached output and a ` cached` line to the model's with status 0 and printed nothing that shows a failure a rerun could mend. The held-out models and BIMcollab's (`holdout-*`, `bimcollab-*`), of which only counts leave the runner (`tools/ci/scrub-holdout-results.sh`), are never - looked up or stored, so no per-element result of theirs is in the cache. + looked up or stored, because the Actions cache of a public repository can be + restored by a workflow that a fork's pull request changes: nothing the scrub + removes may be in it. Each entry names its model, and the end of every run + removes any entry that names one of those models or none and prints the + entries by model prefix, so the saved cache is shown to hold none. That gives + up caching a share of the work, not a small one: in the unfiltered run + 41748430211 the 18 held-out models took 32.8 of the 119 task-minutes of + model steps (28%); they run only with `-f holdout=true`. - **Where it lives.** Every run restores the newest `measure-scores-v1-main-*` cache before the measured code builds. It is saved only where rust-cache is saved: the workflow dispatched from `main` and the side measuring `main`'s own code (`inputs.ref` empty, or `base` `main`), because the build scripts of any other ref run on the runner and must not write what a later `main` run - reads. A pull request run, and a run of any other ref, only reads. A run + reads. A pull request run, and a run of any other ref, only reads. The scorers + in the key make a poisoned entry reachable only by a run whose scorers, files + and libraries hash alike, but not by construction: a build script of the + measured ref could rewrite the installed IfcOpenShell, which the key names + only by version, and the cache then holds what that run produced. That is why + only main's own code saves it. A run dispatched with `-f score_cache=` reads and saves a cache of its own, `measure-scores-v1-x--*`, from any ref (GitHub scopes a branch's caches to that branch, and `main`'s runs never read the name); this is how a diff --git a/tools/ci/measure-cache.py b/tools/ci/measure-cache.py index 3aad2f0be..9c2b9e1e0 100644 --- a/tools/ci/measure-cache.py +++ b/tools/ci/measure-cache.py @@ -29,20 +29,26 @@ - the value of every environment variable the closure reads, except the two that only size the mesh cache (RVT_MESH_CACHE, RVT_MESH_WORKERS); - the files a scorer reads that its arguments do not name (EXTRA_INPUTS); - - the Python and platform, the version of every package - tools/ci/requirements-ifc.txt pins as installed, and RVT_IFC_FLAGS. + - the Python and platform, the sha256 of tools/ci/requirements-ifc.txt, the + installed version of every package it pins, the versions IfcOpenShell and + numpy report, and RVT_IFC_FLAGS. A step is not cached when its closure does something the key cannot see: lists or walks a directory, takes a directory name from an argument, reads a file it names itself, runs a program, draws a random number or a date, reads the IFC's export time, or reads an environment variable by a computed name. The -models that Measure keeps only counts of (held out, BIMcollab's) are never -cached. Only status 0 is kept: a scorer that crashed, ran out of time or of +models that Measure keeps only counts of (`holdout-*`, `bimcollab-*`, the +directories tools/ci/scrub-holdout-results.sh reduces) are never looked up or +stored, because the Actions cache of a public repository can be restored by a +workflow a fork's pull request changes, so what the scrub removes must not be in +it; each entry names its model, and `finish` removes any that names one of +those or none. Only status 0 is kept: a scorer that crashed, ran out of time or of memory is run again. """ import argparse import ast +import collections import fnmatch import hashlib import importlib.metadata @@ -98,6 +104,12 @@ class Uncacheable(Exception): pass +def is_held_out(model): + """The models tools/ci/scrub-holdout-results.sh reduces to counts before + anything leaves the runner (its `holdout-*` and `bimcollab-*` directories).""" + return any(fnmatch.fnmatch(model, pattern) for pattern in HELD_OUT) + + def work_dir(): path = os.environ.get("MEASURE_CACHE_WORK") or os.path.join(os.environ.get("RUNNER_TEMP", "/tmp"), "measure-cache-work") os.makedirs(path, exist_ok=True) @@ -301,22 +313,41 @@ def declared_siblings(script, trees, ref): def fingerprint(): - packages = [] - with open(REQUIREMENTS, encoding="utf-8") as handle: - for line in handle: + """The interpreter and the libraries the scorers run on, worked out once a + run: Python, the platform, requirements-ifc.txt and the installed version of + every package it pins, and the versions IfcOpenShell and numpy report.""" + remembered = os.path.join(work_dir(), "fingerprint.json") + if os.path.exists(remembered): + with open(remembered, encoding="utf-8") as handle: + found = json.load(handle) + else: + packages = [] + with open(REQUIREMENTS, "rb") as handle: + requirements = handle.read() + for line in requirements.decode("utf-8").splitlines(): match = re.match(r"^([A-Za-z0-9_.\-]+)==", line) if match: try: packages.append((match.group(1), importlib.metadata.version(match.group(1)))) except importlib.metadata.PackageNotFoundError: packages.append((match.group(1), None)) - return { - "format": FORMAT, - "python": sys.version, - "platform": [platform.system(), platform.machine(), *platform.libc_ver()], - "packages": packages, - "ifc_flags": os.environ.get("RVT_IFC_FLAGS", ""), - } + reported = {} + for name in ("ifcopenshell", "numpy"): + try: + module = __import__(name) + reported[name] = str(getattr(module, "version", None) or getattr(module, "__version__", None)) + except Exception as error: + reported[name] = f"unavailable: {type(error).__name__}" + found = { + "format": FORMAT, + "python": sys.version, + "platform": [platform.system(), platform.machine(), *platform.libc_ver()], + "requirements": sha(requirements), + "packages": packages, + "reported": reported, + } + atomic_write(remembered, json.dumps(found, sort_keys=True).encode("utf-8")) + return {**found, "ifc_flags": os.environ.get("RVT_IFC_FLAGS", "")} def make_key(args, script, script_args): @@ -372,7 +403,7 @@ def lookup(args): label = f"{os.path.basename(os.path.abspath(args.dir))} {args.step}" if not root: return 2 - if any(fnmatch.fnmatch(os.path.basename(os.path.abspath(args.dir)), pattern) for pattern in HELD_OUT): + if is_held_out(os.path.basename(os.path.abspath(args.dir))): log(f"not cached {label}: held out, only its counts leave the runner") return 2 script, *script_args = args.command @@ -403,7 +434,7 @@ def lookup(args): def store(args): root = cache_dir() label = f"{os.path.basename(os.path.abspath(args.dir))} {args.step}" - if not root or not os.path.exists(keyfile(args)): + if not root or is_held_out(os.path.basename(os.path.abspath(args.dir))) or not os.path.exists(keyfile(args)): return 0 with open(keyfile(args), encoding="ascii") as handle: key = handle.read() @@ -429,6 +460,8 @@ def store(args): for path in args.extra: if os.path.isfile(path): shutil.copyfile(path, os.path.join(scratch, "extra-" + os.path.basename(path))) + with open(os.path.join(scratch, "meta.json"), "w", encoding="utf-8") as meta: + json.dump({"model": os.path.basename(os.path.abspath(args.dir)), "step": args.step}, meta) os.makedirs(os.path.dirname(entry), exist_ok=True) try: os.rename(scratch, entry) @@ -465,15 +498,33 @@ def finish(args): pruned = 0 total = 0 kept_entries = 0 + held = 0 + unknown = 0 + by_model = collections.Counter() if root: now = time.time() survivors = [] for path in list(entries(root)): - if now - os.stat(path).st_mtime > args.days * 86400: + model = None + try: + with open(os.path.join(path, "meta.json"), encoding="utf-8") as meta: + model = json.load(meta)["model"] + except (OSError, ValueError, KeyError): + pass + # An entry that does not name its model, or names a held-out one, + # is removed: nothing of those models is kept. + if model is None: + shutil.rmtree(path) + unknown += 1 + elif is_held_out(model): + shutil.rmtree(path) + held += 1 + elif now - os.stat(path).st_mtime > args.days * 86400: shutil.rmtree(path) pruned += 1 else: survivors.append((os.stat(path).st_mtime, path, size_of(path))) + by_model[model.split("-")[0]] += 1 survivors.sort() total = sum(size for _, _, size in survivors) while survivors and total > args.max_mb * 1024 * 1024: @@ -488,11 +539,13 @@ def finish(args): os.rmdir(path) elif shard.startswith(".new-"): shutil.rmtree(path, ignore_errors=True) - changed = count["kept"] > 0 or pruned > 0 or stale > 0 + changed = count["kept"] > 0 or pruned > 0 or stale > 0 or held > 0 or unknown > 0 summary = ( f"score cache: {count['hit']} hits, {count['miss']} misses ({count['kept']} kept), " f"{count['not cached']} not cached; {kept_entries} entries, {total / 1048576:.1f} MiB; " - f"{pruned} pruned, {stale} refreshed; changed={str(changed).lower()}" + f"{pruned} pruned, {stale} refreshed; changed={str(changed).lower()}\n" + f"score cache entries by model prefix: {', '.join(f'{name} {n}' for name, n in sorted(by_model.items())) or 'none'}; " + f"held out ({', '.join(HELD_OUT)}): {held} found and removed; without a model name: {unknown} removed" ) print(summary) if args.report: From 6994bb5d0d683efa05fdd0b15a0849a668ba7352 Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:53:55 -0500 Subject: [PATCH 05/10] ci(measure): the cache cleaner reads the score cache key's shard and measured SHA (#1237) --- tools/maintainer/clean_caches.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/maintainer/clean_caches.py b/tools/maintainer/clean_caches.py index d9b96171a..5e075c028 100644 --- a/tools/maintainer/clean_caches.py +++ b/tools/maintainer/clean_caches.py @@ -41,9 +41,10 @@ # highest enabled CLI version among the cached bases, so each version keeps its # newest ones. CODEQL_OVERLAY_KEY = re.compile(r"(codeql-overlay-base-database-\d+-[0-9a-f]+-[a-z]+-[0-9.]+)-[0-9a-f]{40}-\d+-\d+") -# Measure's score cache (tools/ci/measure-cache.py): measure-scores-v-
---. -# A run restores the newest of its prefix, so only the newest generations are read. -SCORE_CACHE_KEY = re.compile(r"(measure-scores-v\d+-.+)-\d+-\d+") +# Measure's score cache (tools/ci/measure-cache.py): +# measure-scores-v-
-----. +# A run restores the newest of its shard's prefix, so only the newest generations are read. +SCORE_CACHE_KEY = re.compile(r"(measure-scores-v\d+-.+?)(?:-[0-9a-f]{40})?-\d+-\d+") def gh(*args): From 0a2493268c02ed0113a6ffa13d64e5878e842d97 Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:58:04 -0500 Subject: [PATCH 06/10] docs(ci): the score cache's per-shard log and caches, and the custody rule in the Measure script's header (#1237) --- docs/maintaining.md | 38 +++++++++++++++------------- tools/ci/measure-reference-models.sh | 9 ++++--- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/docs/maintaining.md b/docs/maintaining.md index 32f5a8761..2ded31e94 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -121,7 +121,7 @@ writes the cached output and a ` cached` line to the model's takes a directory from an argument, opens a file by a literal path, runs a program, draws a number or a date, reads the IFC's export time or reads an environment variable by a computed name: the cache tells the reason in - `score-cache.log` and runs it. A new scorer that reads anything but the + `score-cache-.log` and runs it. A new scorer that reads anything but the files it is given is run every time until `EXTRA_INPUTS` in `measure-cache.py` names what it reads. A step is kept only if it finished with status 0 and printed nothing that shows a failure a rerun could mend. @@ -135,28 +135,30 @@ writes the cached output and a ` cached` line to the model's up caching a share of the work, not a small one: in the unfiltered run 41748430211 the 18 held-out models took 32.8 of the 119 task-minutes of model steps (28%); they run only with `-f holdout=true`. -- **Where it lives.** Every run restores the newest `measure-scores-v1-main-*` - cache before the measured code builds. It is saved only where rust-cache is - saved: the workflow dispatched from `main` and the side measuring `main`'s - own code (`inputs.ref` empty, or `base` `main`), because the build scripts of - any other ref run on the runner and must not write what a later `main` run - reads. A pull request run, and a run of any other ref, only reads. The scorers - in the key make a poisoned entry reachable only by a run whose scorers, files - and libraries hash alike, but not by construction: a build script of the - measured ref could rewrite the installed IfcOpenShell, which the key names - only by version, and the cache then holds what that run produced. That is why - only main's own code saves it. A run - dispatched with `-f score_cache=` reads and saves a cache of its own, - `measure-scores-v1-x--*`, from any ref (GitHub scopes a branch's - caches to that branch, and `main`'s runs never read the name); this is how a - change to the cache itself is proved, and `-f score_cache=off` runs every - step as before. +- **Where it lives.** Each shard (`office-as-built`, `office-permit`, + `samples`, `rest`) restores the newest `measure-scores-v1-main--*` + cache before the measured code builds, and a run given `models` (the shard + `all`) restores all of them. It is saved only where rust-cache is saved: the + workflow dispatched from `main` and the side measuring `main`'s own code + (`inputs.ref` empty, or `base` `main`), because the build scripts of any + other ref run on the runner and must not write what a later `main` run + reads. A pull request run, and a run of any other ref, only reads. Keying on + the scorers makes an entry reachable only by a run whose scorers, files and + libraries hash alike, but that is not safe by construction: a build script of + the measured ref could rewrite the installed IfcOpenShell, which the key + names only by version, and the cache would then hold what that run produced. + That is why only `main`'s own code saves it. A run dispatched with `-f + score_cache=` reads and saves a cache of its own, + `measure-scores-v1-x---*`, from any ref (GitHub scopes a + branch's caches to that branch, and the shared cache never reads the name); + this is how a change to the cache itself is proved. `-f score_cache=off` + runs every step as before. - **Bounds.** Entries not used for 21 days are removed, then the oldest past 256 MiB; a run saves a new cache only if it kept an entry, pruned one or refreshed one unused for three days, and `tools/maintainer/clean_caches.py` deletes the generations a newer one supersedes. An entry is a scorer's text output (kilobytes), so the cache is - a few MiB, not GiB. `results/score-cache.log` in the artifact lists every + a few MiB, not GiB. `results/score-cache-.log` in the artifact lists every hit, miss, kept and uncached step with the first twelve characters of its key. - **Checking it.** The run after a change to an export is the check that the key follows the IFC: its changed models miss and the rest hit. To check a diff --git a/tools/ci/measure-reference-models.sh b/tools/ci/measure-reference-models.sh index 96f25511b..e188a5e5b 100755 --- a/tools/ci/measure-reference-models.sh +++ b/tools/ci/measure-reference-models.sh @@ -64,9 +64,12 @@ # When MEASURE_CACHE_DIR names a directory, a scorer or schema step whose every # input is as it was in an earlier run takes that run's output # (tools/ci/measure-cache.py, docs/maintaining.md); its ` cached` line -# stands for its time in timings.log, and OUT_DIR/score-cache.log (written by the -# workflow) lists each hit and miss. Exports, rvt-info, rvt-gltf, the witness -# exports and probes always run. +# stands for its time in timings.log, and OUT_DIR/score-cache-.log (written +# by the workflow) lists each hit and miss. Exports, rvt-info, rvt-gltf, the +# witness exports and probes always run, and the held-out models and BIMcollab's +# (holdout-*, bimcollab-*), of which only counts leave the runner, are never +# looked up or stored: the Actions cache is not private to the workflows that +# may change it. # A scorer that fails is recorded, not fatal: its output is the evidence. # These runs check what the exports hold; their times are not benchmarks # (the workflow's `timing` job times both refs on one runner, From c0afd26522c1b783b8f825a7f8082b4eff5416fe Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:10:26 -0500 Subject: [PATCH 07/10] ci(measure): the score cache is keyed on the shard's name whatever it is, and a run given models restores only its own (#1237) The shard names are the plan job's to choose, so no step lists them. A shard's prefix ends in a dot that no shard name holds, so one shard's prefix is never another's, and the extra restores of the four fixed shards for a run given models are gone: that run keeps a cache of its own, built by earlier runs of its kind. --- .github/workflows/measure.yml | 56 +++++--------------------------- docs/maintaining.md | 10 +++--- tools/maintainer/clean_caches.py | 2 +- 3 files changed, 14 insertions(+), 54 deletions(-) diff --git a/.github/workflows/measure.yml b/.github/workflows/measure.yml index 6fa420daf..03d91e993 100644 --- a/.github/workflows/measure.yml +++ b/.github/workflows/measure.yml @@ -330,9 +330,10 @@ jobs: # main's, so a branch's build scripts can never write what a later main # run reads), or under a name of its own that `score_cache` gives a run # from any ref, which main's runs never read. Entries are named by what - # they were made from, so the sides and the flags share them; a shard - # holds its own models' entries, and a run given `models` (the shard - # `all`) also restores the other shards'. + # they were made from, so the sides and the flags share them. Each shard + # keeps a cache of its own, named by the shard whatever the shard is + # called, and ends its name with a `.` that no shard name holds, so one + # shard's prefix is never another's. # The held-out models and BIMcollab's are never cached (RE-252). - name: Score cache key id: score-cache-key @@ -365,19 +366,17 @@ jobs: { echo "enabled=$enabled" echo "save=$save" - echo "own=$own" - echo "main=$main" - echo "prefix=$own$SHARD-" + echo "prefix=$own$SHARD." echo "restore<> "$GITHUB_OUTPUT" if [ "$enabled" = true ]; then echo "MEASURE_CACHE_DIR=$GITHUB_WORKSPACE/_score-cache" >> "$GITHUB_ENV" echo "MEASURE_CACHE_WORK=$RUNNER_TEMP/score-cache-work" >> "$GITHUB_ENV" fi - echo "score cache: enabled=$enabled save=$save key prefix $own$SHARD-" + echo "score cache: enabled=$enabled save=$save key prefix $own$SHARD." - name: Restore the score cache if: steps.score-cache-key.outputs.enabled == 'true' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -385,45 +384,6 @@ jobs: path: _score-cache key: ${{ steps.score-cache-key.outputs.prefix }}${{ matrix.side }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: ${{ steps.score-cache-key.outputs.restore }} - # A run given `models` is one shard, `all`, whatever models it names: it - # also takes what the other shards saved. Each restore adds its entries - # beside the others (they are named by their content). - - name: Restore the rest shard's score cache - if: steps.score-cache-key.outputs.enabled == 'true' && matrix.shard == 'all' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: _score-cache - key: ${{ steps.score-cache-key.outputs.own }}rest-${{ matrix.side }}-${{ github.run_id }}-${{ github.run_attempt }} - restore-keys: | - ${{ steps.score-cache-key.outputs.own }}rest- - ${{ steps.score-cache-key.outputs.main }}rest- - - name: Restore the as-built office shard's score cache - if: steps.score-cache-key.outputs.enabled == 'true' && matrix.shard == 'all' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: _score-cache - key: ${{ steps.score-cache-key.outputs.own }}office-as-built-${{ matrix.side }}-${{ github.run_id }}-${{ github.run_attempt }} - restore-keys: | - ${{ steps.score-cache-key.outputs.own }}office-as-built- - ${{ steps.score-cache-key.outputs.main }}office-as-built- - - name: Restore the permit office shard's score cache - if: steps.score-cache-key.outputs.enabled == 'true' && matrix.shard == 'all' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: _score-cache - key: ${{ steps.score-cache-key.outputs.own }}office-permit-${{ matrix.side }}-${{ github.run_id }}-${{ github.run_attempt }} - restore-keys: | - ${{ steps.score-cache-key.outputs.own }}office-permit- - ${{ steps.score-cache-key.outputs.main }}office-permit- - - name: Restore the samples shard's score cache - if: steps.score-cache-key.outputs.enabled == 'true' && matrix.shard == 'all' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: _score-cache - key: ${{ steps.score-cache-key.outputs.own }}samples-${{ matrix.side }}-${{ github.run_id }}-${{ github.run_attempt }} - restore-keys: | - ${{ steps.score-cache-key.outputs.own }}samples- - ${{ steps.score-cache-key.outputs.main }}samples- - name: Build working-directory: measured run: | diff --git a/docs/maintaining.md b/docs/maintaining.md index 2ded31e94..46b6c4f45 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -135,10 +135,10 @@ writes the cached output and a ` cached` line to the model's up caching a share of the work, not a small one: in the unfiltered run 41748430211 the 18 held-out models took 32.8 of the 119 task-minutes of model steps (28%); they run only with `-f holdout=true`. -- **Where it lives.** Each shard (`office-as-built`, `office-permit`, - `samples`, `rest`) restores the newest `measure-scores-v1-main--*` - cache before the measured code builds, and a run given `models` (the shard - `all`) restores all of them. It is saved only where rust-cache is saved: the +- **Where it lives.** Each shard of the run, whatever it is called, restores + the newest `measure-scores-v1-main-.*` cache before the measured code + builds (a run given `models` is the shard `all`, with a cache of its own, + built by earlier runs of that kind). It is saved only where rust-cache is saved: the workflow dispatched from `main` and the side measuring `main`'s own code (`inputs.ref` empty, or `base` `main`), because the build scripts of any other ref run on the runner and must not write what a later `main` run @@ -149,7 +149,7 @@ writes the cached output and a ` cached` line to the model's names only by version, and the cache would then hold what that run produced. That is why only `main`'s own code saves it. A run dispatched with `-f score_cache=` reads and saves a cache of its own, - `measure-scores-v1-x---*`, from any ref (GitHub scopes a + `measure-scores-v1-x--.*`, from any ref (GitHub scopes a branch's caches to that branch, and the shared cache never reads the name); this is how a change to the cache itself is proved. `-f score_cache=off` runs every step as before. diff --git a/tools/maintainer/clean_caches.py b/tools/maintainer/clean_caches.py index 5e075c028..2e5365fbb 100644 --- a/tools/maintainer/clean_caches.py +++ b/tools/maintainer/clean_caches.py @@ -42,7 +42,7 @@ # newest ones. CODEQL_OVERLAY_KEY = re.compile(r"(codeql-overlay-base-database-\d+-[0-9a-f]+-[a-z]+-[0-9.]+)-[0-9a-f]{40}-\d+-\d+") # Measure's score cache (tools/ci/measure-cache.py): -# measure-scores-v-
-----. +# measure-scores-v-
-.---. # A run restores the newest of its shard's prefix, so only the newest generations are read. SCORE_CACHE_KEY = re.compile(r"(measure-scores-v\d+-.+?)(?:-[0-9a-f]{40})?-\d+-\d+") From e31f887bc0b40ac0ea497f96631519c7771ebf5c Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:24:42 -0500 Subject: [PATCH 08/10] ci(measure): the three binaries are cached by the measured commit's SHA, the toolchain and the profile (#1237) The shards of a run, and every run that measures main's commit as its base, built the same three binaries (about 70 seconds of each job). rvt-bin-v1-----ci holds rvt-ifc, rvt-gltf and rvt-info; a hit skips their cargo build, a ref that is not a full SHA is built each time, the probe examples always are, a run of a pull request saves nothing, and clean_caches.py keeps the newest sixteen. --- .github/workflows/measure.yml | 46 +++++++++++++++++++++++++++++++- docs/maintaining.md | 10 +++++++ tools/maintainer/clean_caches.py | 18 ++++++++++--- 3 files changed, 70 insertions(+), 4 deletions(-) diff --git a/.github/workflows/measure.yml b/.github/workflows/measure.yml index 03d91e993..d1ce9e49b 100644 --- a/.github/workflows/measure.yml +++ b/.github/workflows/measure.yml @@ -384,11 +384,42 @@ jobs: path: _score-cache key: ${{ steps.score-cache-key.outputs.prefix }}${{ matrix.side }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: ${{ steps.score-cache-key.outputs.restore }} + # The three binaries every shard and every run of a commit builds alike + # (about 70 s of each job) are kept under the commit's full SHA, the + # toolchain and the profile: what the cache holds is what that code + # builds to, so a run of any ref may save it, and only a run of that + # commit restores it. A ref that is not a full SHA is built each time. + # The probe examples are always built. + - name: Binary cache key + id: binaries-key + shell: bash + env: + MEASURED: ${{ matrix.side == 'base' && needs.plan.outputs.base_sha || needs.plan.outputs.head_sha }} + run: | + if [[ "$MEASURED" =~ ^[0-9a-f]{40}$ ]]; then + echo "key=rvt-bin-v1-$MEASURED-$RUNNER_OS-$RUNNER_ARCH-$(rustc -vV | sha256sum | cut -c1-16)-ci" >> "$GITHUB_OUTPUT" + else + echo "the measured ref is not a full SHA, so its binaries are built each time" + fi + - name: Restore the binaries + id: binaries + if: steps.binaries-key.outputs.key != '' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + measured/target/ci/rvt-ifc + measured/target/ci/rvt-gltf + measured/target/ci/rvt-info + key: ${{ steps.binaries-key.outputs.key }} - name: Build working-directory: measured run: | git log -1 --format='%H %s' > ../measured-commit.txt - cargo build --profile ci --bin rvt-ifc --bin rvt-gltf --bin rvt-info + if [ "$BINARIES_CACHED" = true ]; then + echo "rvt-ifc, rvt-gltf and rvt-info restored from the binary cache" + else + cargo build --profile ci --bin rvt-ifc --bin rvt-gltf --bin rvt-info + fi # A base older than a probe has no such example; it is skipped there. # Probes live in the rvt-probes crate (probes/examples) since it was # split out; a ref from before that has them in examples/. Both build @@ -410,6 +441,19 @@ jobs: fi env: PROBE: ${{ inputs.probe }} + BINARIES_CACHED: ${{ steps.binaries.outputs.cache-hit }} + # Every shard of a commit misses at once and saves the same key; the one + # that reserves it first stores it and the others are told it is taken. + # A run from a pull request saves nothing, its cache being its own. + - name: Save the binaries + if: ${{ success() && github.event_name == 'workflow_dispatch' && steps.binaries-key.outputs.key != '' && steps.binaries.outputs.cache-hit != 'true' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + measured/target/ci/rvt-ifc + measured/target/ci/rvt-gltf + measured/target/ci/rvt-info + key: ${{ steps.binaries-key.outputs.key }} - name: Fetch the reference models run: tools/fetch-reference-models.sh models # The office models' host being down is NOT MEASURED; a file that no diff --git a/docs/maintaining.md b/docs/maintaining.md index 46b6c4f45..46b96c551 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -169,6 +169,16 @@ writes the cached output and a ` cached` line to the model's psets_missing=Pset_WallCommon.Reference` must miss `psets_vs_ifc` on every model and hit everything else. +Measure also keeps the three binaries its Build step makes (`rvt-ifc`, +`rvt-gltf`, `rvt-info`) under `rvt-bin-v1-----ci`, so the shards of a run, and every run that measures `main`'s commit +as its base, build them once (the build is about 70 seconds of each job). What +the cache holds is a function of its key, the code of that commit built by that +toolchain, and only a run of that commit restores it, so a run of any ref may +save it (a pull request's run does not: its caches are its own). A ref that is +not a full SHA is built each time, the probe examples always are, and +`clean_caches.py` keeps the newest sixteen. + ## Dependencies and the MSRV `rust-version` in `Cargo.toml` is the oldest stable Rust the crate builds on, and diff --git a/tools/maintainer/clean_caches.py b/tools/maintainer/clean_caches.py index 2e5365fbb..86807db5d 100644 --- a/tools/maintainer/clean_caches.py +++ b/tools/maintainer/clean_caches.py @@ -9,7 +9,9 @@ version: every analysis of `main` saves one under its commit, and the next analysis on that version reads only the newest; - on `main`, the superseded generations of Measure's score cache, which a run - restores from the newest of its prefix and saves anew. + restores from the newest of its prefix and saves anew, and all but the newest + sixteen of its binary caches, one per commit built, which only runs of that + commit read. The caches of live branches, of open pull requests, of tags, and `main`'s newest generations are kept. The repository's cache limit is 10 GB and GitHub evicts the @@ -45,6 +47,11 @@ # measure-scores-v-
-.---. # A run restores the newest of its shard's prefix, so only the newest generations are read. SCORE_CACHE_KEY = re.compile(r"(measure-scores-v\d+-.+?)(?:-[0-9a-f]{40})?-\d+-\d+") +# Measure's binary cache: rvt-bin-v-----, one +# per commit built. A commit's key is read only by runs of that commit, which are +# the last few: the newest BINARY_GENERATIONS are kept. +BINARY_CACHE_KEY = re.compile(r"(rvt-bin-v\d+)-[0-9a-f]{40}-.+") +BINARY_GENERATIONS = 16 def gh(*args): @@ -82,12 +89,17 @@ def unusable(caches, open_prs, live_branches, keep): if name != "main" and name not in live_branches: doomed.append((c, f"branch {name} was deleted")) elif name == "main": - k = RUST_CACHE_KEY.fullmatch(c["key"]) or CODEQL_OVERLAY_KEY.fullmatch(c["key"]) or SCORE_CACHE_KEY.fullmatch(c["key"]) + k = ( + RUST_CACHE_KEY.fullmatch(c["key"]) + or CODEQL_OVERLAY_KEY.fullmatch(c["key"]) + or SCORE_CACHE_KEY.fullmatch(c["key"]) + or BINARY_CACHE_KEY.fullmatch(c["key"]) + ) if k: generations.setdefault(k.group(1), []).append(c) for family, lst in generations.items(): lst.sort(key=lambda c: c["last_accessed_at"], reverse=True) - for c in lst[keep:]: + for c in lst[(BINARY_GENERATIONS if family.startswith("rvt-bin-") else keep):]: doomed.append((c, f"superseded generation of {family}")) return doomed From d5e0b7c6800e214faf54746eea2071e777acad1f Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:03:32 -0500 Subject: [PATCH 09/10] ci(measure): the score cache key holds the Python minor version, not its patch, and the versions IfcOpenShell and numpy report (#1237) In warm run 41769277902 two of six shards missed every step: their jobs landed on a newer runner image (20261028.624 against 20261021.618) whose Python was 3.12.16, not 3.12.15, and the key held sys.version and, through numpy.version being the submodule, a path with the patch in it. The libraries that decide a scorer's output are pinned by hash in requirements-ifc.txt and hashed in the key by their installed versions, so the patch only costs hits. --- docs/maintaining.md | 5 +++-- tools/ci/measure-cache.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/maintaining.md b/docs/maintaining.md index 46b96c551..4f8093aaa 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -110,8 +110,9 @@ writes the cached output and a ` cached` line to the model's the two that size the mesh cache), which is `PSETS_MISSING` for `psets_vs_ifc` and `qtos_vs_ifc`; the files a scorer reads that its arguments do not name (`qtos_vs_ifc`'s `diagnostics.json`, - `unfilled_openings_vs_ifc`'s sibling Revit exports); the Python, the - platform, the sha256 of `tools/ci/requirements-ifc.txt`, the installed + `unfilled_openings_vs_ifc`'s sibling Revit exports); the Python minor + version (not the patch: a runner image moves it, 3.12.15 to 3.12.16 in run + 41769277902, which cost two of six shards every hit), the platform, the sha256 of `tools/ci/requirements-ifc.txt`, the installed version of every package it pins, the versions IfcOpenShell and numpy report, and `RVT_IFC_FLAGS`. Each entry therefore hashes the rvt-rs IFC, Revit's reference IFC (or the glTF), the command-line arguments, the scorer diff --git a/tools/ci/measure-cache.py b/tools/ci/measure-cache.py index 9c2b9e1e0..409e0b4b3 100644 --- a/tools/ci/measure-cache.py +++ b/tools/ci/measure-cache.py @@ -332,15 +332,19 @@ def fingerprint(): except importlib.metadata.PackageNotFoundError: packages.append((match.group(1), None)) reported = {} - for name in ("ifcopenshell", "numpy"): + for name, attribute in (("ifcopenshell", "version"), ("numpy", "__version__")): try: - module = __import__(name) - reported[name] = str(getattr(module, "version", None) or getattr(module, "__version__", None)) + value = getattr(__import__(name), attribute, None) + reported[name] = value if isinstance(value, str) else None except Exception as error: reported[name] = f"unavailable: {type(error).__name__}" found = { "format": FORMAT, - "python": sys.version, + # The minor release, not the patch: the runner image a job lands on + # moves the patch (3.12.15 to 3.12.16 in run 41769277902, which cost + # two of its six shards every hit), and the libraries that decide + # the scorers' output are pinned by hash in requirements-ifc.txt. + "python": f"{sys.version_info.major}.{sys.version_info.minor}", "platform": [platform.system(), platform.machine(), *platform.libc_ver()], "requirements": sha(requirements), "packages": packages, From cd89fe05db5e7d7b47734a81ef3bdb5ba1a26918 Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:32:10 -0500 Subject: [PATCH 10/10] ci(measure): the score cache keeps no failure it could replay, no pair that was ever held out, and restores after the build (#1237) A check that raised (a MemoryError in the rules) wrote 'unreadable:' into schema-rules.part and exited 0, so the entry was stored and replayed until the export changed. ifc_validate_summary.py --both now exits 1 when any check failed, store scans every file the step wrote for the markers and the size cap, an entry that lacks a file is a miss, and the format and the key prefix go to 2 (the helper's own hash joins the key as well). spent-* joins holdout-* and bimcollab-* as never cached: BIM Whale's AdvancedProject went spent and back to held out on 2026-11-03. The key also holds every non-Python file of tools/re and tools/ci, a directory argument is not cached, validation.jsonl is kept for models only, and the score cache is restored after the Build step so a build script never sees it. --- .github/workflows/measure.yml | 23 ++++---- docs/maintaining.md | 76 +++++++++++++++---------- tools/ci/ifc_validate_summary.py | 12 ++-- tools/ci/measure-cache.py | 83 +++++++++++++++++++--------- tools/ci/measure-reference-models.sh | 17 ++++-- 5 files changed, 136 insertions(+), 75 deletions(-) diff --git a/.github/workflows/measure.yml b/.github/workflows/measure.yml index e42e8d118..454672984 100644 --- a/.github/workflows/measure.yml +++ b/.github/workflows/measure.yml @@ -341,7 +341,8 @@ jobs: # keeps a cache of its own, named by the shard whatever the shard is # called, and ends its name with a `.` that no shard name holds, so one # shard's prefix is never another's. - # The held-out models and BIMcollab's are never cached (RE-252). + # The held-out models, BIMcollab's and the pairs once held out (spent-*) + # are never cached (RE-252). - name: Score cache key id: score-cache-key shell: bash @@ -352,7 +353,7 @@ jobs: run: | enabled=true save=false - main=measure-scores-v1-main- + main=measure-scores-v2-main- case "$SCORE_CACHE" in off) enabled=false @@ -366,7 +367,7 @@ jobs: echo "::error::score_cache is off, or a name of up to 40 letters, digits, dots and underscores" exit 1 fi - own="measure-scores-v1-x-$SCORE_CACHE-" + own="measure-scores-v2-x-$SCORE_CACHE-" save=true ;; esac @@ -384,13 +385,6 @@ jobs: echo "MEASURE_CACHE_WORK=$RUNNER_TEMP/score-cache-work" >> "$GITHUB_ENV" fi echo "score cache: enabled=$enabled save=$save key prefix $own$SHARD." - - name: Restore the score cache - if: steps.score-cache-key.outputs.enabled == 'true' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: _score-cache - key: ${{ steps.score-cache-key.outputs.prefix }}${{ matrix.side }}-${{ github.run_id }}-${{ github.run_attempt }} - restore-keys: ${{ steps.score-cache-key.outputs.restore }} # The three binaries every shard and every run of a commit builds alike # (about 70 s of each job) are kept under the commit's full SHA, the # toolchain and the profile: what the cache holds is what that code @@ -461,6 +455,15 @@ jobs: measured/target/ci/rvt-gltf measured/target/ci/rvt-info key: ${{ steps.binaries-key.outputs.key }} + # Restored after the Build step, so that no build script of the measured + # ref ever sees a writable score cache. + - name: Restore the score cache + if: steps.score-cache-key.outputs.enabled == 'true' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: _score-cache + key: ${{ steps.score-cache-key.outputs.prefix }}${{ matrix.side }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: ${{ steps.score-cache-key.outputs.restore }} - name: Fetch the reference models run: tools/fetch-reference-models.sh models # The office models' host being down is NOT MEASURED; a file that no diff --git a/docs/maintaining.md b/docs/maintaining.md index 4f8093aaa..2b8f51b76 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -106,39 +106,50 @@ writes the cached output and a ` cached` line to the model's lines, which carry the export time, so a rebuild that writes the same model hits; Revit's export; the glTF; the validation findings); the scorer and every module of `tools/re` and `tools/ci` it reaches by import, importlib - path or name; the value of every environment variable they read (apart from - the two that size the mesh cache), which is `PSETS_MISSING` for - `psets_vs_ifc` and `qtos_vs_ifc`; the files a scorer reads that its - arguments do not name (`qtos_vs_ifc`'s `diagnostics.json`, - `unfilled_openings_vs_ifc`'s sibling Revit exports); the Python minor - version (not the patch: a runner image moves it, 3.12.15 to 3.12.16 in run - 41769277902, which cost two of six shards every hit), the platform, the sha256 of `tools/ci/requirements-ifc.txt`, the installed + path or name, and every file of `tools/re` and `tools/ci` that is not Python + (the Measure script, the requirements, the schemas); the value of every + environment variable the scorers read (apart from the two that size the mesh + cache), which is `PSETS_MISSING` for `psets_vs_ifc` and `qtos_vs_ifc`; the + files a scorer reads that its arguments do not name (`qtos_vs_ifc`'s + `diagnostics.json`, `unfilled_openings_vs_ifc`'s sibling Revit exports); the + Python minor version (not the patch: a runner image moves it, 3.12.15 to + 3.12.16 in run 41769277902, which cost two of six shards every hit), the + platform, the sha256 of `tools/ci/requirements-ifc.txt`, the installed version of every package it pins, the versions IfcOpenShell and numpy - report, and `RVT_IFC_FLAGS`. Each entry therefore hashes the rvt-rs IFC, - Revit's reference IFC (or the glTF), the command-line arguments, the scorer - and its transitive `tools/re` and `tools/ci` imports, the environment - variables they read and the library versions. + report, `RVT_IFC_FLAGS`, and the sha256 of `measure-cache.py` itself, so a + change to what is kept or looked up starts a cache. Each entry therefore + hashes the rvt-rs IFC, Revit's reference IFC (or the glTF), the command-line + arguments, the scorer and its transitive `tools/re` and `tools/ci` imports, + the environment variables they read and the library versions. - **What is never cached.** A step whose closure lists or walks a directory, takes a directory from an argument, opens a file by a literal path, runs a program, draws a number or a date, reads the IFC's export time or reads an - environment variable by a computed name: the cache tells the reason in - `score-cache-.log` and runs it. A new scorer that reads anything but the - files it is given is run every time until `EXTRA_INPUTS` in - `measure-cache.py` names what it reads. A step is kept only if it finished - with status 0 and printed nothing that shows a failure a rerun could mend. - The held-out models and BIMcollab's (`holdout-*`, `bimcollab-*`), of which - only counts leave the runner (`tools/ci/scrub-holdout-results.sh`), are never - looked up or stored, because the Actions cache of a public repository can be - restored by a workflow that a fork's pull request changes: nothing the scrub - removes may be in it. Each entry names its model, and the end of every run - removes any entry that names one of those models or none and prints the - entries by model prefix, so the saved cache is shown to hold none. That gives - up caching a share of the work, not a small one: in the unfiltered run - 41748430211 the 18 held-out models took 32.8 of the 119 task-minutes of - model steps (28%); they run only with `-f holdout=true`. + environment variable by a computed name, or one given a directory as an + argument: the cache tells the reason in `score-cache-.log` and runs + it. A new scorer that reads anything but the files it is given is run every + time until `EXTRA_INPUTS` in `measure-cache.py` names what it reads. A step + is kept only if it finished with status 0 and neither its output nor any file + it also wrote (the validation findings, the rules summary) is over 4 MiB or + shows a failure a rerun could mend (`unreadable:`, a traceback, a + `MemoryError`, a dead worker); `ifc_validate_summary.py --both` exits 1 when + a check failed, where its separate runs print the failure and exit 0. An + entry that lacks a file the step wrote is a miss. + The held-out models, BIMcollab's and the pairs that were held out + (`holdout-*`, `bimcollab-*`, `spent-*`: BIM Whale's AdvancedProject was spent + and held out again on 2026-11-03), of which only counts leave the runner + (`tools/ci/scrub-holdout-results.sh`), are never looked up or stored, because + the Actions cache of a public repository can be restored by a workflow that a + fork's pull request changes: nothing the scrub removes may be in it, and a + pair's results do not become safe by being spent. Each entry names its model, + and the end of every run removes any entry that names one of those models or + none and prints the entries by model prefix, so the saved cache is shown to + hold none. That gives up caching a share of the work, not a small one: in + the unfiltered run 41748430211 the 18 held-out models took 32.8 of the 119 + task-minutes of model steps (28%); they run only with `-f holdout=true`. - **Where it lives.** Each shard of the run, whatever it is called, restores - the newest `measure-scores-v1-main-.*` cache before the measured code - builds (a run given `models` is the shard `all`, with a cache of its own, + the newest `measure-scores-v2-main-.*` cache after the measured code + has built (a build script never sees a writable cache) (a run given `models` + is the shard `all`, with a cache of its own, built by earlier runs of that kind). It is saved only where rust-cache is saved: the workflow dispatched from `main` and the side measuring `main`'s own code (`inputs.ref` empty, or `base` `main`), because the build scripts of any @@ -150,7 +161,7 @@ writes the cached output and a ` cached` line to the model's names only by version, and the cache would then hold what that run produced. That is why only `main`'s own code saves it. A run dispatched with `-f score_cache=` reads and saves a cache of its own, - `measure-scores-v1-x--.*`, from any ref (GitHub scopes a + `measure-scores-v2-x--.*`, from any ref (GitHub scopes a branch's caches to that branch, and the shared cache never reads the name); this is how a change to the cache itself is proved. `-f score_cache=off` runs every step as before. @@ -166,7 +177,12 @@ writes the cached output and a ` cached` line to the model's change to the cache or to a scorer, run Measure twice on one branch with `-f score_cache=` (cold, then warm) and once with `-f score_cache=off`, and `diff -r` the three results apart from `*.log`, - `diagnostics.json`; a third warm run with `-f + `commit.txt` and `runner*.txt` (`diagnostics.json` is not left out: it is + byte-identical across runs of one commit, and `qtos_vs_ifc` reads it); to + check that a changed input misses, run the same name at two commits whose + exports differ for some model, and the second must miss on the models that + moved and hit on the rest, and diff clean against a run with the cache off + at that commit; a warm run with `-f psets_missing=Pset_WallCommon.Reference` must miss `psets_vs_ifc` on every model and hit everything else. diff --git a/tools/ci/ifc_validate_summary.py b/tools/ci/ifc_validate_summary.py index cd9339322..8139c026a 100644 --- a/tools/ci/ifc_validate_summary.py +++ b/tools/ci/ifc_validate_summary.py @@ -68,18 +68,22 @@ def both(path, statements, rules_out): """The default validation on stdout and the rules check in RULES_OUT, each exactly what its own run prints, from one parse of the file: a file that does not open, or a check that fails, is the finding of both or of that - one, as it is when they run apart.""" + one, as it is when they run apart. It returns 1 when anything failed, which + the apart runs do not (they print the failure and exit 0), so that Measure's + cache never keeps a failure that a rerun could mend, a MemoryError say.""" try: model = ifcopenshell.open(path) except Exception as error: print(unreadable(error)) with open(rules_out, "w", encoding="utf-8") as out: out.write(unreadable(error) + "\n") - return + return 1 + failed = 0 try: main(path, statements, model) except Exception as error: print(unreadable(error)) + failed = 1 captured = io.StringIO() failure = None with contextlib.redirect_stdout(captured): @@ -89,6 +93,7 @@ def both(path, statements, rules_out): failure = error with open(rules_out, "w", encoding="utf-8") as out: out.write(captured.getvalue() + (unreadable(failure) + "\n" if failure else "")) + return 1 if failed or failure else 0 def main(path, statements=None, model=None): @@ -109,8 +114,7 @@ def main(path, statements=None, model=None): if args[:1] == ["--both"]: if len(args) != 4: sys.exit(__doc__) - both(args[1], args[2], args[3]) - sys.exit(0) + sys.exit(both(args[1], args[2], args[3])) with_rules = args[:1] == ["--rules"] if with_rules: args = args[1:] diff --git a/tools/ci/measure-cache.py b/tools/ci/measure-cache.py index 409e0b4b3..5d92db389 100644 --- a/tools/ci/measure-cache.py +++ b/tools/ci/measure-cache.py @@ -25,25 +25,31 @@ which carry the export time (the hash tools/ci/measure-reference-models.sh writes to ifc.sha256), so a rebuild that writes the same model hits; - the scorer and every module of tools/re and tools/ci it can reach, by - import, importlib path or name (a module it only mentions is included); + import, importlib path or name (a module it only mentions is included), + and every file under tools/re and tools/ci that is not Python (the Measure + script, the requirements, the schemas), whichever the scorer reads; - the value of every environment variable the closure reads, except the two that only size the mesh cache (RVT_MESH_CACHE, RVT_MESH_WORKERS); - the files a scorer reads that its arguments do not name (EXTRA_INPUTS); - - the Python and platform, the sha256 of tools/ci/requirements-ifc.txt, the - installed version of every package it pins, the versions IfcOpenShell and - numpy report, and RVT_IFC_FLAGS. + - the Python minor version and platform, the sha256 of + tools/ci/requirements-ifc.txt, the installed version of every package it + pins, the versions IfcOpenShell and numpy report, RVT_IFC_FLAGS, and this + file's own sha256 (so a change to what is kept or looked up starts a cache). A step is not cached when its closure does something the key cannot see: lists or walks a directory, takes a directory name from an argument, reads a file it names itself, runs a program, draws a random number or a date, reads the -IFC's export time, or reads an environment variable by a computed name. The -models that Measure keeps only counts of (`holdout-*`, `bimcollab-*`, the -directories tools/ci/scrub-holdout-results.sh reduces) are never looked up or -stored, because the Actions cache of a public repository can be restored by a -workflow a fork's pull request changes, so what the scrub removes must not be in -it; each entry names its model, and `finish` removes any that names one of -those or none. Only status 0 is kept: a scorer that crashed, ran out of time or of -memory is run again. +IFC's export time, or reads an environment variable by a computed name; or when +an argument is a directory. The models that Measure keeps only counts of, or +once did (`holdout-*`, `bimcollab-*`, the directories +tools/ci/scrub-holdout-results.sh reduces, and `spent-*`, a pair that left the +held-out list and came back), are never looked up or stored, because the Actions +cache of a public repository can be restored by a workflow a fork's pull request +changes, so what the scrub removes must not be in it; each entry names its model, +and `finish` removes any that names one of those or none. Only status 0 is kept, +and only if neither the output nor any file the step also wrote shows a failure a +rerun could mend, or is over 4 MiB: a scorer that crashed, ran out of time or of +memory is run again. An entry that lacks a file the step wrote is a miss. """ import argparse @@ -64,9 +70,11 @@ RE_DIR = os.path.join(os.path.dirname(CI_DIR), "re") REQUIREMENTS = os.path.join(CI_DIR, "requirements-ifc.txt") -FORMAT = 1 +FORMAT = 2 NEUTRAL_ENV = {"RVT_MESH_CACHE", "RVT_MESH_WORKERS"} -HELD_OUT = ("holdout-*", "bimcollab-*") +# BIM Whale's AdvancedProject was spent, then held out again (2026-11-03, +# research/holdout-archive-models.tsv): a spent pair's results are not safe either. +HELD_OUT = ("holdout-*", "bimcollab-*", "spent-*") MAX_ENTRY_BYTES = 4 * 1024 * 1024 STALE_SECONDS = 3 * 24 * 3600 # Output that shows a failure a rerun could mend, whatever the status says. @@ -338,8 +346,21 @@ def fingerprint(): reported[name] = value if isinstance(value, str) else None except Exception as error: reported[name] = f"unavailable: {type(error).__name__}" + with open(os.path.abspath(__file__), "rb") as own: + helper = sha(own.read()) + others = [] + for directory in (RE_DIR, CI_DIR): + for base, names, files in os.walk(directory): + names.sort() + for name in sorted(files): + if not name.endswith(".py") and "__pycache__" not in base: + path = os.path.join(base, name) + with open(path, "rb") as handle: + others.append(f"{os.path.relpath(path, os.path.dirname(CI_DIR))} {sha(handle.read())}") found = { "format": FORMAT, + "helper": helper, + "other files": sha("\n".join(others).encode()), # The minor release, not the patch: the runner image a job lands on # moves the patch (3.12.15 to 3.12.16 in run 41769277902, which cost # two of its six shards every hit), and the libraries that decide @@ -376,6 +397,8 @@ def make_key(args, script, script_args): full = os.path.abspath(arg) if full in outputs: tokens.append(["output", arg]) + elif os.path.isdir(arg): + raise Uncacheable("an argument is a directory") elif os.path.isfile(arg): own = os.path.dirname(full) == directory tokens.append(["file", arg, file_digest(arg, own and arg.endswith(".ifc"))]) @@ -408,7 +431,7 @@ def lookup(args): if not root: return 2 if is_held_out(os.path.basename(os.path.abspath(args.dir))): - log(f"not cached {label}: held out, only its counts leave the runner") + log(f"not cached {label}: held out now or once, only counts of it leave the runner") return 2 script, *script_args = args.command try: @@ -419,12 +442,13 @@ def lookup(args): os.makedirs(os.path.dirname(keyfile(args)), exist_ok=True) atomic_write(keyfile(args), key.encode("ascii")) entry = entry_path(root, key) - if os.path.isfile(os.path.join(entry, "out")): + complete = os.path.isfile(os.path.join(entry, "out")) and all( + os.path.isfile(os.path.join(entry, "extra-" + os.path.basename(path))) for path in args.extra + ) + if complete: shutil.copyfile(os.path.join(entry, "out"), args.dest) for path in args.extra: - kept = os.path.join(entry, "extra-" + os.path.basename(path)) - if os.path.isfile(kept): - shutil.copyfile(kept, path) + shutil.copyfile(os.path.join(entry, "extra-" + os.path.basename(path)), path) stale = time.time() - os.stat(entry).st_mtime > STALE_SECONDS os.utime(entry) with open(os.path.join(args.dir, "timings.log"), "a", encoding="utf-8") as timings: @@ -448,13 +472,23 @@ def store(args): if status is None: last = out.rstrip(b"\n").rsplit(b"\n", 1)[-1] status = int(last[5:]) if last.startswith(b"exit ") and last[5:].isdigit() else -1 - text = out.decode("utf-8", "replace") if status != 0: log(f"not kept {label} {key[:12]}: status {status}") return 0 - if len(out) > MAX_ENTRY_BYTES or any(mark in text for mark in TRANSIENT): - log(f"not kept {label} {key[:12]}: output too large or shows a failure to repeat") - return 0 + # The output and every file the step also wrote: each must be there, under + # the size cap, and show no failure a rerun could mend (a check that raised + # prints it into its file and may still exit 0). + contents = [("output", out)] + for path in args.extra: + if not os.path.isfile(path): + log(f"not kept {label} {key[:12]}: {os.path.basename(path)} was not written") + return 0 + with open(path, "rb") as handle: + contents.append((os.path.basename(path), handle.read())) + for name, data in contents: + if len(data) > MAX_ENTRY_BYTES or any(mark in data.decode("utf-8", "replace") for mark in TRANSIENT): + log(f"not kept {label} {key[:12]}: {name} is over the size cap or shows a failure to repeat") + return 0 entry = entry_path(root, key) if os.path.isdir(entry): return 0 @@ -462,8 +496,7 @@ def store(args): os.makedirs(scratch) shutil.copyfile(args.dest, os.path.join(scratch, "out")) for path in args.extra: - if os.path.isfile(path): - shutil.copyfile(path, os.path.join(scratch, "extra-" + os.path.basename(path))) + shutil.copyfile(path, os.path.join(scratch, "extra-" + os.path.basename(path))) with open(os.path.join(scratch, "meta.json"), "w", encoding="utf-8") as meta: json.dump({"model": os.path.basename(os.path.abspath(args.dir)), "step": args.step}, meta) os.makedirs(os.path.dirname(entry), exist_ok=True) diff --git a/tools/ci/measure-reference-models.sh b/tools/ci/measure-reference-models.sh index e188a5e5b..7657bb211 100755 --- a/tools/ci/measure-reference-models.sh +++ b/tools/ci/measure-reference-models.sh @@ -66,10 +66,10 @@ # (tools/ci/measure-cache.py, docs/maintaining.md); its ` cached` line # stands for its time in timings.log, and OUT_DIR/score-cache-.log (written # by the workflow) lists each hit and miss. Exports, rvt-info, rvt-gltf, the -# witness exports and probes always run, and the held-out models and BIMcollab's -# (holdout-*, bimcollab-*), of which only counts leave the runner, are never -# looked up or stored: the Actions cache is not private to the workflows that -# may change it. +# witness exports and probes always run, and the held-out models, BIMcollab's +# and the pairs once held out (holdout-*, bimcollab-*, spent-*), of which only +# counts leave the runner, are never looked up or stored: the Actions cache is +# not private to the workflows that may change it. # A scorer that fails is recorded, not fatal: its output is the evidence. # These runs check what the exports hold; their times are not benchmarks # (the workflow's `timing` job times both refs on one runner, @@ -223,7 +223,8 @@ score() { # name args... # its directory: a step whose every input is as it was in an earlier run takes # that run's output, and ` cached` stands for its time in timings.log. A # step is kept only if it finished with status 0, and the models whose results -# only leave the runner as counts are never cached. +# only leave the runner as counts (holdout-*, bimcollab-*), and the pairs that +# were once held out (spent-*), are never cached. CACHE_TOOL="$(dirname "$0")/measure-cache.py" cscore() { # name args...: `score`, from the cache when the scorer and its inputs are unchanged local out="$1" dir step found=2; shift @@ -280,8 +281,12 @@ schema() { # dir max_bytes if [ "$(stat -c %s "$1/model.ifc")" -le "$2" ]; then # The validation and the EXPRESS rules read one parse of the file; the # rules' summary comes back in a file, to follow the validation's. + # (validation.jsonl is kept for ports_vs_ifc, which a sample has none of, + # and which measure_sample deletes.) + local extras="$1/validation.jsonl $1/schema-rules.part" + case "$(basename "$1")" in sample-*) extras="$1/schema-rules.part" ;; esac echo "validate:" - CACHE_EXTRA="$1/validation.jsonl $1/schema-rules.part" cached "$1" schema-validate-rules $(( 2 * STEP_TIMEOUT )) \ + CACHE_EXTRA="$extras" cached "$1" schema-validate-rules $(( 2 * STEP_TIMEOUT )) \ "$(dirname "$0")/ifc_validate_summary.py" --both "$1/model.ifc" "$1/validation.jsonl" "$1/schema-rules.part" echo "rules:" cat "$1/schema-rules.part" 2> /dev/null || true