diff --git a/.github/workflows/measure.yml b/.github/workflows/measure.yml index 5c3352c45..98e75490d 100644 --- a/.github/workflows/measure.yml +++ b/.github/workflows/measure.yml @@ -64,6 +64,14 @@ # probe-.txt and probe_score-.txt: # # gh workflow run measure.yml -f ref=my-branch -f base=none -f nordic=true -f models="nordic-office-*" -f scorers=none -f probe="probe_a probe_b" -f probe_scorer="a_vs_ifc -" +# +# A scorer or schema step whose every input is as it was in an earlier run is +# not run again: its output comes from the score cache (tools/ci/measure-cache.py; +# its key, what is never cached and how it is saved are in docs/maintaining.md). +# Exports, rvt-info, rvt-gltf, the witness exports, diagnostics and probes always +# run. `score_cache=off` runs every step, as before. +# +# gh workflow run measure.yml -f ref=my-branch -f base=main -f score_cache=off name: Measure @@ -150,6 +158,10 @@ on: description: Only these steps on the models, tools/re scorers and info, gltf, schema, witness (e.g. "qtos_vs_ifc witness"; a tools/re script in no scorer list runs as an IFC scorer), none for none, empty for all; every model is still exported and hashed required: false default: "" + score_cache: + description: 'The score cache (tools/ci/measure-cache.py, docs/maintaining.md): empty for the shared one, off to run every step, or a name for a cache of its own that this run saves to from any ref (the shared one is read as well)' + required: false + default: "" # A change to the measurement itself measures its own PR against main. pull_request: paths: @@ -317,11 +329,98 @@ jobs: python-version: "3.12" - name: Install IfcOpenShell run: python -m pip install --require-hashes -r tools/ci/requirements-ifc.txt + # The score cache (tools/ci/measure-cache.py, docs/maintaining.md): a + # scorer or schema step whose every input is as it was in an earlier run + # takes that run's output. Every run restores the newest cache of its + # shard, before the measured code builds; it is saved only where the + # Rust cache is (the workflow dispatched from main and the code measured + # main's, so a branch's build scripts can never write what a later main + # run reads), or under a name of its own that `score_cache` gives a run + # from any ref, which main's runs never read. Entries are named by what + # they were made from, so the sides and the flags share them. Each shard + # keeps a cache of its own, named by the shard whatever the shard is + # called, and ends its name with a `.` that no shard name holds, so one + # shard's prefix is never another's. + # The held-out models, BIMcollab's and the pairs once held out (spent-*) + # are never cached (RE-252). + - name: Score cache key + id: score-cache-key + shell: bash + env: + SCORE_CACHE: ${{ inputs.score_cache }} + SHARD: ${{ matrix.shard }} + SAVE_FROM_MAIN: ${{ github.ref == 'refs/heads/main' && (matrix.side == 'base' && (inputs.base == '' || inputs.base == 'main') || matrix.side != 'base' && inputs.ref == '') }} + run: | + enabled=true + save=false + main=measure-scores-v2-main- + case "$SCORE_CACHE" in + off) + enabled=false + ;; + "") + own=$main + [ "$SAVE_FROM_MAIN" = true ] && save=true + ;; + *) + if ! [[ "$SCORE_CACHE" =~ ^[A-Za-z0-9._]{1,40}$ ]]; then + echo "::error::score_cache is off, or a name of up to 40 letters, digits, dots and underscores" + exit 1 + fi + own="measure-scores-v2-x-$SCORE_CACHE-" + save=true + ;; + esac + { + echo "enabled=$enabled" + echo "save=$save" + echo "prefix=$own$SHARD." + echo "restore<> "$GITHUB_OUTPUT" + if [ "$enabled" = true ]; then + echo "MEASURE_CACHE_DIR=$GITHUB_WORKSPACE/_score-cache" >> "$GITHUB_ENV" + echo "MEASURE_CACHE_WORK=$RUNNER_TEMP/score-cache-work" >> "$GITHUB_ENV" + fi + echo "score cache: enabled=$enabled save=$save key prefix $own$SHARD." + # The three binaries every shard and every run of a commit builds alike + # (about 70 s of each job) are kept under the commit's full SHA, the + # toolchain and the profile: what the cache holds is what that code + # builds to, so a run of any ref may save it, and only a run of that + # commit restores it. A ref that is not a full SHA is built each time. + # The probe examples are always built. + - name: Binary cache key + id: binaries-key + shell: bash + env: + MEASURED: ${{ matrix.side == 'base' && needs.plan.outputs.base_sha || needs.plan.outputs.head_sha }} + run: | + if [[ "$MEASURED" =~ ^[0-9a-f]{40}$ ]]; then + echo "key=rvt-bin-v1-$MEASURED-$RUNNER_OS-$RUNNER_ARCH-$(rustc -vV | sha256sum | cut -c1-16)-ci" >> "$GITHUB_OUTPUT" + else + echo "the measured ref is not a full SHA, so its binaries are built each time" + fi + - name: Restore the binaries + id: binaries + if: steps.binaries-key.outputs.key != '' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + measured/target/ci/rvt-ifc + measured/target/ci/rvt-gltf + measured/target/ci/rvt-info + key: ${{ steps.binaries-key.outputs.key }} - name: Build working-directory: measured run: | git log -1 --format='%H %s' > ../measured-commit.txt - cargo build --profile ci --bin rvt-ifc --bin rvt-gltf --bin rvt-info + if [ "$BINARIES_CACHED" = true ]; then + echo "rvt-ifc, rvt-gltf and rvt-info restored from the binary cache" + else + cargo build --profile ci --bin rvt-ifc --bin rvt-gltf --bin rvt-info + fi # A base older than a probe has no such example; it is skipped there. # Probes live in the rvt-probes crate (probes/examples) since it was # split out; a ref from before that has them in examples/. Both build @@ -343,6 +442,28 @@ jobs: fi env: PROBE: ${{ inputs.probe }} + BINARIES_CACHED: ${{ steps.binaries.outputs.cache-hit }} + # Every shard of a commit misses at once and saves the same key; the one + # that reserves it first stores it and the others are told it is taken. + # A run from a pull request saves nothing, its cache being its own. + - name: Save the binaries + if: ${{ success() && github.event_name == 'workflow_dispatch' && steps.binaries-key.outputs.key != '' && steps.binaries.outputs.cache-hit != 'true' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + measured/target/ci/rvt-ifc + measured/target/ci/rvt-gltf + measured/target/ci/rvt-info + key: ${{ steps.binaries-key.outputs.key }} + # Restored after the Build step, so that no build script of the measured + # ref ever sees a writable score cache. + - name: Restore the score cache + if: steps.score-cache-key.outputs.enabled == 'true' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: _score-cache + key: ${{ steps.score-cache-key.outputs.prefix }}${{ matrix.side }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: ${{ steps.score-cache-key.outputs.restore }} - name: Fetch the reference models run: tools/fetch-reference-models.sh models # The office models' host being down is NOT MEASURED; a file that no @@ -507,6 +628,25 @@ jobs: # The runner's CPU, so the compare job can say which machines the # wall times came from. grep -m1 'model name' /proc/cpuinfo | cut -d: -f2- | xargs > "results/runner-$SHARD.txt" + # Entries not used for 21 days go, then the oldest past 256 MiB; the cache + # is saved only if this run kept an entry, pruned one or refreshed one not + # used for three days (tools/maintainer/clean_caches.py deletes the + # generations it supersedes). + - name: Finish the score cache + id: score-cache + if: ${{ !cancelled() && steps.score-cache-key.outputs.enabled == 'true' }} + shell: bash + env: + SHARD: ${{ matrix.shard }} + run: | + mkdir -p results + python tools/ci/measure-cache.py finish --report "results/score-cache-$SHARD.log" + - name: Save the score cache + if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' && steps.score-cache.outputs.changed == 'true' && steps.score-cache-key.outputs.save == 'true' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: _score-cache + key: ${{ steps.score-cache-key.outputs.prefix }}${{ matrix.side }}-${{ matrix.side == 'base' && needs.plan.outputs.base_sha || needs.plan.outputs.head_sha }}-${{ github.run_id }}-${{ github.run_attempt }} # The held-out models and BIMcollab's carry no licence: only their # counts leave the runner (RE-252). - name: Keep only the unlicensed models' counts diff --git a/docs/maintaining.md b/docs/maintaining.md index 0ccff7c4e..6130ac183 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -94,6 +94,111 @@ when a release ships. ([`changelog.d/`](../changelog.d/README.md)) change in the same pull request as the capability they describe. +## Measure's score cache + +Measure runs about 40 scorers and four schema checks on every model, and most +pull requests move the export of only a few of them. A step whose inputs are as +they were in an earlier run takes that run's output from the score cache +(`tools/ci/measure-cache.py`) instead of running; exports, `rvt-info`, +`rvt-gltf`, the witness exports, diagnostics and probes always run. A hit +writes the cached output and a ` cached` line to the model's +`timings.log`; every output is the one a run would have written. + +- **Key.** The sha256 of the step's name and arguments; the content of every + file argument (the rvt-rs IFC without its `FILE_NAME` and `IFCOWNERHISTORY` + lines, which carry the export time, so a rebuild that writes the same model + hits; Revit's export; the glTF; the validation findings); the scorer and + every module of `tools/re` and `tools/ci` it reaches by import, importlib + path or name, and every file of `tools/re` and `tools/ci` that is not Python + (the Measure script, the requirements, the schemas); the value of every + environment variable the scorers read (apart from the two that size the mesh + cache), which is `PSETS_MISSING` for `psets_vs_ifc` and `qtos_vs_ifc`; the + files a scorer reads that its arguments do not name (`qtos_vs_ifc`'s + `diagnostics.json`, `unfilled_openings_vs_ifc`'s sibling Revit exports); the + Python minor version (not the patch: a runner image moves it, 3.12.15 to + 3.12.16 in run 41769277902, which cost two of six shards every hit), the + platform, the sha256 of `tools/ci/requirements-ifc.txt`, the installed + version of every package it pins, the versions IfcOpenShell and numpy + report, `RVT_IFC_FLAGS`, and the sha256 of `measure-cache.py` itself, so a + change to what is kept or looked up starts a cache. Each entry therefore + hashes the rvt-rs IFC, Revit's reference IFC (or the glTF), the command-line + arguments, the scorer and its transitive `tools/re` and `tools/ci` imports, + the environment variables they read and the library versions. +- **What is never cached.** A step whose closure lists or walks a directory, + takes a directory from an argument, opens a file by a literal path, runs a + program, draws a number or a date, reads the IFC's export time or reads an + environment variable by a computed name, or one given a directory as an + argument: the cache tells the reason in `score-cache-.log` and runs + it. A new scorer that reads anything but the files it is given is run every + time until `EXTRA_INPUTS` in `measure-cache.py` names what it reads. A step + is kept only if it finished with status 0 and neither its output nor any file + it also wrote (the validation findings, the rules summary) is over 4 MiB or + shows a failure a rerun could mend (`unreadable:`, a traceback, a + `MemoryError`, a dead worker); `ifc_validate_summary.py --both` exits 1 when + a check failed, where its separate runs print the failure and exit 0. An + entry that lacks a file the step wrote is a miss. + The held-out models, BIMcollab's and the pairs that were held out + (`holdout-*`, `bimcollab-*`, `spent-*`: BIM Whale's AdvancedProject was spent + and held out again on 2026-11-03), of which only counts leave the runner + (`tools/ci/scrub-holdout-results.sh`), are never looked up or stored, because + the Actions cache of a public repository can be restored by a workflow that a + fork's pull request changes: nothing the scrub removes may be in it, and a + pair's results do not become safe by being spent. Each entry names its model, + and the end of every run removes any entry that names one of those models or + none and prints the entries by model prefix, so the saved cache is shown to + hold none. That gives up caching a share of the work, not a small one: in + the unfiltered run 41748430211 the 18 held-out models took 32.8 of the 119 + task-minutes of model steps (28%); they run only with `-f holdout=true`. +- **Where it lives.** Each shard of the run, whatever it is called, restores + the newest `measure-scores-v2-main-.*` cache after the measured code + has built (a build script never sees a writable cache) (a run given `models` + is the shard `all`, with a cache of its own, + built by earlier runs of that kind). It is saved only where rust-cache is saved: the + workflow dispatched from `main` and the side measuring `main`'s own code + (`inputs.ref` empty, or `base` `main`), because the build scripts of any + other ref run on the runner and must not write what a later `main` run + reads. A pull request run, and a run of any other ref, only reads. Keying on + the scorers makes an entry reachable only by a run whose scorers, files and + libraries hash alike, but that is not safe by construction: a build script of + the measured ref could rewrite the installed IfcOpenShell, which the key + names only by version, and the cache would then hold what that run produced. + That is why only `main`'s own code saves it. A run dispatched with `-f + score_cache=` reads and saves a cache of its own, + `measure-scores-v2-x--.*`, from any ref (GitHub scopes a + branch's caches to that branch, and the shared cache never reads the name); + this is how a change to the cache itself is proved. `-f score_cache=off` + runs every step as before. +- **Bounds.** Entries not used for 21 days are removed, then the oldest past + 256 MiB; a run saves a new cache only if it kept an entry, pruned one or + refreshed one unused for three days, and + `tools/maintainer/clean_caches.py` deletes the generations a newer one + supersedes. An entry is a scorer's text output (kilobytes), so the cache is + a few MiB, not GiB. `results/score-cache-.log` in the artifact lists every + hit, miss, kept and uncached step with the first twelve characters of its key. +- **Checking it.** The run after a change to an export is the check that the + key follows the IFC: its changed models miss and the rest hit. To check a + change to the cache or to a scorer, run Measure twice on one branch with + `-f score_cache=` (cold, then warm) and once with `-f + score_cache=off`, and `diff -r` the three results apart from `*.log`, + `commit.txt` and `runner*.txt` (`diagnostics.json` is not left out: it is + byte-identical across runs of one commit, and `qtos_vs_ifc` reads it); to + check that a changed input misses, run the same name at two commits whose + exports differ for some model, and the second must miss on the models that + moved and hit on the rest, and diff clean against a run with the cache off + at that commit; a warm run with `-f + psets_missing=Pset_WallCommon.Reference` must miss `psets_vs_ifc` on every + model and hit everything else. + +Measure also keeps the three binaries its Build step makes (`rvt-ifc`, +`rvt-gltf`, `rvt-info`) under `rvt-bin-v1-----ci`, so the shards of a run, and every run that measures `main`'s commit +as its base, build them once (the build is about 70 seconds of each job). What +the cache holds is a function of its key, the code of that commit built by that +toolchain, and only a run of that commit restores it, so a run of any ref may +save it (a pull request's run does not: its caches are its own). A ref that is +not a full SHA is built each time, the probe examples always are, and +`clean_caches.py` keeps the newest sixteen. + ## Dependencies and the MSRV `rust-version` in `Cargo.toml` is the oldest stable Rust the crate builds on, and diff --git a/tools/ci/ifc_validate_summary.py b/tools/ci/ifc_validate_summary.py index 29f94d515..8139c026a 100644 --- a/tools/ci/ifc_validate_summary.py +++ b/tools/ci/ifc_validate_summary.py @@ -22,10 +22,18 @@ findings : (at most 30) +With --both MODEL.ifc STATEMENTS.jsonl RULES.txt it does both from one parse +of the file: the default validation's summary on stdout and its findings in +STATEMENTS.jsonl, and what --rules prints in RULES.txt, each exactly as its own +run writes it. + Usage: ifc_validate_summary.py [--rules] MODEL.ifc [STATEMENTS.jsonl] + ifc_validate_summary.py --both MODEL.ifc STATEMENTS.jsonl RULES.txt """ import collections +import contextlib +import io import json import re import sys @@ -43,18 +51,54 @@ def describe(statement): return f"{entity} {attribute}: {message}"[:300] -def rules(path): +def rules(path, model=None): logger = ifcopenshell.validate.json_logger() - ifcopenshell.express.rule_executor.run(ifcopenshell.open(path), logger) + ifcopenshell.express.rule_executor.run(model if model is not None else ifcopenshell.open(path), logger) kinds = collections.Counter(re.sub(r"#\d+", "#", describe(s)) for s in logger.statements) print(f"findings {len(logger.statements)}") for kind, count in sorted(kinds.items(), key=lambda item: (-item[1], item[0]))[:30]: print(f"{count} {kind}") -def main(path, statements=None): +def unreadable(error): + return f"unreadable: {type(error).__name__}: {json.dumps(str(error))[:300]}" + + +def both(path, statements, rules_out): + """The default validation on stdout and the rules check in RULES_OUT, each + exactly what its own run prints, from one parse of the file: a file that + does not open, or a check that fails, is the finding of both or of that + one, as it is when they run apart. It returns 1 when anything failed, which + the apart runs do not (they print the failure and exit 0), so that Measure's + cache never keeps a failure that a rerun could mend, a MemoryError say.""" + try: + model = ifcopenshell.open(path) + except Exception as error: + print(unreadable(error)) + with open(rules_out, "w", encoding="utf-8") as out: + out.write(unreadable(error) + "\n") + return 1 + failed = 0 + try: + main(path, statements, model) + except Exception as error: + print(unreadable(error)) + failed = 1 + captured = io.StringIO() + failure = None + with contextlib.redirect_stdout(captured): + try: + rules(path, model) + except Exception as error: + failure = error + with open(rules_out, "w", encoding="utf-8") as out: + out.write(captured.getvalue() + (unreadable(failure) + "\n" if failure else "")) + return 1 if failed or failure else 0 + + +def main(path, statements=None, model=None): logger = ifcopenshell.validate.json_logger() - ifcopenshell.validate.validate(ifcopenshell.open(path), logger) + ifcopenshell.validate.validate(model if model is not None else ifcopenshell.open(path), logger) if statements: with open(statements, "w", encoding="utf-8") as out: for statement in logger.statements: @@ -67,6 +111,10 @@ def main(path, statements=None): if __name__ == "__main__": args = sys.argv[1:] + if args[:1] == ["--both"]: + if len(args) != 4: + sys.exit(__doc__) + sys.exit(both(args[1], args[2], args[3])) with_rules = args[:1] == ["--rules"] if with_rules: args = args[1:] @@ -78,4 +126,4 @@ def main(path, statements=None): else: main(args[0], args[1] if len(args) == 2 else None) except Exception as error: # an unreadable file is a finding too - print(f"unreadable: {type(error).__name__}: {json.dumps(str(error))[:300]}") + print(unreadable(error)) diff --git a/tools/ci/measure-cache.py b/tools/ci/measure-cache.py new file mode 100644 index 000000000..5d92db389 --- /dev/null +++ b/tools/ci/measure-cache.py @@ -0,0 +1,635 @@ +#!/usr/bin/env python3 +"""Measure's score cache: the output of a scorer or schema step, kept under a +key made of everything the step reads, so a step whose inputs have not changed +is not run again (.github/workflows/measure.yml, tools/ci/measure-reference-models.sh). + + measure-cache.py lookup --step S --dir D --dest FILE [--extra FILE]... -- SCRIPT ARG... + measure-cache.py store --step S --dir D --dest FILE [--extra FILE]... [--status N] + measure-cache.py finish [--days N] [--max-mb N] [--report FILE] + +`lookup` exits 0 on a hit (FILE and the extras are written from the cache and +`S cached` is added to D/timings.log), 1 on a miss (the key is remembered for +`store`) and 2 when the step is not cached, saying why in the report. `store` +keeps FILE and the extras (files the step also writes, such as the validation +findings) when the step finished with status 0 and its output shows no sign of +a failure that a rerun could mend. `finish` prunes the cache, prints what the +run did and, for the workflow, sets `changed` in $GITHUB_OUTPUT: whether a new +cache is worth saving. + +The key is a sha256 over: + - the step's name and the scorer's name as given, and every argument as given; + - the content of each argument that is a file: the rvt-rs IFC and the glTF + (the files the run writes beside the step, in D) and everything else + (Revit's export, the validation findings, the property set definitions). + An IFC in D is hashed without its FILE_NAME and IFCOWNERHISTORY lines, + which carry the export time (the hash tools/ci/measure-reference-models.sh + writes to ifc.sha256), so a rebuild that writes the same model hits; + - the scorer and every module of tools/re and tools/ci it can reach, by + import, importlib path or name (a module it only mentions is included), + and every file under tools/re and tools/ci that is not Python (the Measure + script, the requirements, the schemas), whichever the scorer reads; + - the value of every environment variable the closure reads, except the two + that only size the mesh cache (RVT_MESH_CACHE, RVT_MESH_WORKERS); + - the files a scorer reads that its arguments do not name (EXTRA_INPUTS); + - the Python minor version and platform, the sha256 of + tools/ci/requirements-ifc.txt, the installed version of every package it + pins, the versions IfcOpenShell and numpy report, RVT_IFC_FLAGS, and this + file's own sha256 (so a change to what is kept or looked up starts a cache). + +A step is not cached when its closure does something the key cannot see: lists +or walks a directory, takes a directory name from an argument, reads a file +it names itself, runs a program, draws a random number or a date, reads the +IFC's export time, or reads an environment variable by a computed name; or when +an argument is a directory. The models that Measure keeps only counts of, or +once did (`holdout-*`, `bimcollab-*`, the directories +tools/ci/scrub-holdout-results.sh reduces, and `spent-*`, a pair that left the +held-out list and came back), are never looked up or stored, because the Actions +cache of a public repository can be restored by a workflow a fork's pull request +changes, so what the scrub removes must not be in it; each entry names its model, +and `finish` removes any that names one of those or none. Only status 0 is kept, +and only if neither the output nor any file the step also wrote shows a failure a +rerun could mend, or is over 4 MiB: a scorer that crashed, ran out of time or of +memory is run again. An entry that lacks a file the step wrote is a miss. +""" + +import argparse +import ast +import collections +import fnmatch +import hashlib +import importlib.metadata +import json +import os +import platform +import re +import shutil +import sys +import time + +CI_DIR = os.path.dirname(os.path.abspath(__file__)) +RE_DIR = os.path.join(os.path.dirname(CI_DIR), "re") +REQUIREMENTS = os.path.join(CI_DIR, "requirements-ifc.txt") + +FORMAT = 2 +NEUTRAL_ENV = {"RVT_MESH_CACHE", "RVT_MESH_WORKERS"} +# BIM Whale's AdvancedProject was spent, then held out again (2026-11-03, +# research/holdout-archive-models.tsv): a spent pair's results are not safe either. +HELD_OUT = ("holdout-*", "bimcollab-*", "spent-*") +MAX_ENTRY_BYTES = 4 * 1024 * 1024 +STALE_SECONDS = 3 * 24 * 3600 +# Output that shows a failure a rerun could mend, whatever the status says. +TRANSIENT = ("unreadable:", "a worker died", "Traceback (most recent call last)", "MemoryError", "BrokenProcessPool") +IFC_TIME_LINES = re.compile(rb"^FILE_NAME\(|IFCOWNERHISTORY") + +BAD_MODULES = {"subprocess", "socket", "urllib", "requests", "http", "random", "uuid", "datetime", "tempfile", "glob"} +BAD_ATTRIBUTES = { + "listdir": "fs-listing", + "scandir": "fs-listing", + "iterdir": "fs-listing", + "rglob": "fs-listing", + "walk": "fs-listing", + "getcwd": "process", + "expanduser": "process", + "system": "process", + "popen": "process", + "OwnerHistory": "export-time", + "CreationDate": "export-time", + "file_name": "export-time", + "time_stamp": "export-time", + "TimeStamp": "export-time", + "creation_date": "export-time", +} +OWN_DIRECTORY = {"os.path.abspath(__file__)", "__file__"} +# ifc_common lists the mesh cache directory that RVT_MESH_CACHE names, which +# only saves meshing and changes no scorer's output. +# gen_standard_property_sets fetches the property set definitions when the +# file is absent, and checks them against the sha256 it pins; its source is in +# the key, so the pin is. +ALLOWED_IN = {"ifc_common": {"fs-listing"}, "gen_standard_property_sets": {"process"}} + + +class Uncacheable(Exception): + pass + + +def is_held_out(model): + """The models tools/ci/scrub-holdout-results.sh reduces to counts before + anything leaves the runner (its `holdout-*` and `bimcollab-*` directories).""" + return any(fnmatch.fnmatch(model, pattern) for pattern in HELD_OUT) + + +def work_dir(): + path = os.environ.get("MEASURE_CACHE_WORK") or os.path.join(os.environ.get("RUNNER_TEMP", "/tmp"), "measure-cache-work") + os.makedirs(path, exist_ok=True) + return path + + +def cache_dir(): + path = os.environ.get("MEASURE_CACHE_DIR") + if path: + os.makedirs(path, exist_ok=True) + return path + + +def log(line): + with open(os.path.join(work_dir(), "report.log"), "a", encoding="utf-8") as report: + report.write(line + "\n") + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def atomic_write(path, data): + scratch = f"{path}.{os.getpid()}.{time.time_ns()}" + with open(scratch, "wb") as out: + out.write(data) + os.replace(scratch, path) + + +# ---- file digests ----------------------------------------------------------- + + +def file_digest(path, ifc_without_export_time): + """The sha256 of the file, memoised for the run; for an IFC the run wrote, + of its lines other than FILE_NAME and IFCOWNERHISTORY.""" + stat = os.stat(path) + memo = os.path.join(work_dir(), "digests") + os.makedirs(memo, exist_ok=True) + name = sha(f"{os.path.realpath(path)} {stat.st_size} {stat.st_mtime_ns} {ifc_without_export_time}".encode()) + remembered = os.path.join(memo, name) + if os.path.exists(remembered): + with open(remembered, encoding="ascii") as handle: + return handle.read() + digest = hashlib.sha256() + with open(path, "rb") as handle: + if ifc_without_export_time: + for line in handle: + if not IFC_TIME_LINES.search(line): + digest.update(line if line.endswith(b"\n") else line + b"\n") + else: + for block in iter(lambda: handle.read(1 << 20), b""): + digest.update(block) + value = digest.hexdigest() + atomic_write(remembered, value.encode("ascii")) + return value + + +# ---- the scorer's closure --------------------------------------------------- + + +def local_modules(): + found = {} + for directory in (RE_DIR, CI_DIR): + for name in sorted(os.listdir(directory)): + if name.endswith(".py"): + found.setdefault(name[:-3], []).append(os.path.join(directory, name)) + return found + + +def parse(path): + try: + with open(path, "rb") as handle: + return ast.parse(handle.read(), filename=path) + except (OSError, SyntaxError) as error: + raise Uncacheable(f"cannot read {os.path.basename(path)}: {type(error).__name__}") + + +def docstrings(tree): + ids = set() + for node in ast.walk(tree): + if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + body = node.body + if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant) and isinstance(body[0].value.value, str): + ids.add(id(body[0].value)) + return ids + + +def references(tree, known): + skip = docstrings(tree) + names = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom): + if node.module: + names.add(node.module.split(".")[0]) + names.update(alias.name for alias in node.names) + elif isinstance(node, ast.Constant) and isinstance(node.value, str) and id(node) not in skip: + base = os.path.basename(node.value) + names.add(base[:-3] if base.endswith(".py") else node.value) + return names & set(known) + + +def closure(script): + """The script and every local module it reaches, as {path: tree}.""" + known = local_modules() + script = os.path.abspath(script) + if not os.path.isfile(script): + raise Uncacheable("the scorer is not a file") + seen = {} + queue = [script] + while queue: + path = queue.pop() + if path in seen: + continue + seen[path] = parse(path) + for name in references(seen[path], known): + own = os.path.join(os.path.dirname(path), name + ".py") + queue.append(own if os.path.isfile(own) else known[name][0]) + return seen + + +def audit(trees): + """What the closure does that the key could not see, and the environment + variables it reads (a set of names; None for a computed name).""" + kinds = set() + env = set() + for path, tree in trees.items(): + found = set() + skip = docstrings(tree) + handled = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + found.update("process" for alias in node.names if alias.name.split(".")[0] in BAD_MODULES) + elif isinstance(node, ast.ImportFrom): + if node.module and node.module.split(".")[0] in BAD_MODULES: + found.add("process") + if node.module == "os" and any(alias.name in ("environ", "getenv") for alias in node.names): + env.add(None) + elif isinstance(node, ast.Attribute): + if node.attr in BAD_ATTRIBUTES: + found.add(BAD_ATTRIBUTES[node.attr]) + elif isinstance(node, ast.Constant) and isinstance(node.value, str) and id(node) not in skip: + if re.search(r"FILE_NAME|IFCOWNERHISTORY", node.value, re.I): + found.add("export-time") + if isinstance(node, ast.Call): + called = ast.unparse(node.func) + first = node.args[0] if node.args else None + if called in ("os.environ.get", "os.getenv", "os.environ.setdefault", "os.environ.pop"): + env.add(first.value if isinstance(first, ast.Constant) and isinstance(first.value, str) else None) + if called.startswith("os.environ."): + handled.add(id(node.func.value)) + elif called in ("os.path.dirname", "os.path.split"): + if first is None or ast.unparse(first) not in OWN_DIRECTORY: + found.add("dirname") + elif called == "open" and isinstance(first, ast.Constant) and isinstance(first.value, str): + found.add("literal-path") + elif isinstance(node, ast.Subscript) and ast.unparse(node.value) == "os.environ": + key = node.slice + env.add(key.value if isinstance(key, ast.Constant) and isinstance(key.value, str) else None) + handled.add(id(node.value)) + for node in ast.walk(tree): + if isinstance(node, ast.Attribute) and ast.unparse(node) == "os.environ" and id(node) not in handled: + env.add(None) + kinds |= found - ALLOWED_IN.get(os.path.basename(path)[:-3], set()) + return kinds, env + + +def closure_digest(trees): + parts = [] + for path in sorted(trees): + with open(path, "rb") as handle: + parts.append(f"{os.path.relpath(path, os.path.dirname(CI_DIR))} {sha(handle.read())}") + return sha("\n".join(parts).encode()) + + +# ---- files a scorer reads that its arguments do not name -------------------- + + +def declared_siblings(script, trees, ref): + """unfilled_openings_vs_ifc's `siblings(path)`: the other Revit exports in + the reference export's directory that share its first four words.""" + tree = trees[os.path.abspath(script)] + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == "siblings": + namespace = {"os": os} + exec(compile(ast.Module([node], []), script, "exec"), namespace) + return namespace["siblings"](ref) + raise Uncacheable("the scorer no longer has the siblings() Measure's cache reads") + + +# scorer -> (what it reads beside its arguments, found from the file arguments) +EXTRA_INPUTS = { + # The export's diagnostics, beside the IFC (args[0]). + "qtos_vs_ifc": lambda script, trees, files: [os.path.join(os.path.dirname(os.path.abspath(files[0])), "diagnostics.json")], + "unfilled_openings_vs_ifc": lambda script, trees, files: declared_siblings(script, trees, files[1]), +} + + +# ---- the key ---------------------------------------------------------------- + + +def fingerprint(): + """The interpreter and the libraries the scorers run on, worked out once a + run: Python, the platform, requirements-ifc.txt and the installed version of + every package it pins, and the versions IfcOpenShell and numpy report.""" + remembered = os.path.join(work_dir(), "fingerprint.json") + if os.path.exists(remembered): + with open(remembered, encoding="utf-8") as handle: + found = json.load(handle) + else: + packages = [] + with open(REQUIREMENTS, "rb") as handle: + requirements = handle.read() + for line in requirements.decode("utf-8").splitlines(): + match = re.match(r"^([A-Za-z0-9_.\-]+)==", line) + if match: + try: + packages.append((match.group(1), importlib.metadata.version(match.group(1)))) + except importlib.metadata.PackageNotFoundError: + packages.append((match.group(1), None)) + reported = {} + for name, attribute in (("ifcopenshell", "version"), ("numpy", "__version__")): + try: + value = getattr(__import__(name), attribute, None) + reported[name] = value if isinstance(value, str) else None + except Exception as error: + reported[name] = f"unavailable: {type(error).__name__}" + with open(os.path.abspath(__file__), "rb") as own: + helper = sha(own.read()) + others = [] + for directory in (RE_DIR, CI_DIR): + for base, names, files in os.walk(directory): + names.sort() + for name in sorted(files): + if not name.endswith(".py") and "__pycache__" not in base: + path = os.path.join(base, name) + with open(path, "rb") as handle: + others.append(f"{os.path.relpath(path, os.path.dirname(CI_DIR))} {sha(handle.read())}") + found = { + "format": FORMAT, + "helper": helper, + "other files": sha("\n".join(others).encode()), + # The minor release, not the patch: the runner image a job lands on + # moves the patch (3.12.15 to 3.12.16 in run 41769277902, which cost + # two of its six shards every hit), and the libraries that decide + # the scorers' output are pinned by hash in requirements-ifc.txt. + "python": f"{sys.version_info.major}.{sys.version_info.minor}", + "platform": [platform.system(), platform.machine(), *platform.libc_ver()], + "requirements": sha(requirements), + "packages": packages, + "reported": reported, + } + atomic_write(remembered, json.dumps(found, sort_keys=True).encode("utf-8")) + return {**found, "ifc_flags": os.environ.get("RVT_IFC_FLAGS", "")} + + +def make_key(args, script, script_args): + outputs = {os.path.abspath(path) for path in args.extra} + if os.path.abspath(args.dest) in outputs: + raise Uncacheable("the output is also an input") + directory = os.path.abspath(args.dir) + trees = closure(script) + kinds, names = audit(trees) + stem = os.path.basename(script)[:-3] + declared = EXTRA_INPUTS.get(stem) + if declared: + kinds.discard("dirname") + kinds.discard("fs-listing") + if None in names: + kinds.add("computed-env") + if kinds: + raise Uncacheable("reads " + ", ".join(sorted(kinds))) + tokens = [] + files = [] + for arg in script_args: + full = os.path.abspath(arg) + if full in outputs: + tokens.append(["output", arg]) + elif os.path.isdir(arg): + raise Uncacheable("an argument is a directory") + elif os.path.isfile(arg): + own = os.path.dirname(full) == directory + tokens.append(["file", arg, file_digest(arg, own and arg.endswith(".ifc"))]) + files.append(arg) + else: + tokens.append(["literal", arg]) + extra = [] + if declared: + for path in declared(script, trees, files): + extra.append([os.path.basename(path), file_digest(path, False) if os.path.isfile(path) else None]) + environment = sorted((name, os.environ.get(name)) for name in names if name not in NEUTRAL_ENV) + parts = [fingerprint(), args.step, script, closure_digest(trees), environment, tokens, extra] + return sha(json.dumps(parts, sort_keys=True).encode()) + + +def keyfile(args): + return os.path.join(work_dir(), "keys", sha(f"{os.path.abspath(args.dir)} {args.step}".encode())) + + +def entry_path(root, key): + return os.path.join(root, key[:2], key) + + +# ---- commands --------------------------------------------------------------- + + +def lookup(args): + root = cache_dir() + label = f"{os.path.basename(os.path.abspath(args.dir))} {args.step}" + if not root: + return 2 + if is_held_out(os.path.basename(os.path.abspath(args.dir))): + log(f"not cached {label}: held out now or once, only counts of it leave the runner") + return 2 + script, *script_args = args.command + try: + key = make_key(args, script, script_args) + except Uncacheable as why: + log(f"not cached {label}: {why}") + return 2 + os.makedirs(os.path.dirname(keyfile(args)), exist_ok=True) + atomic_write(keyfile(args), key.encode("ascii")) + entry = entry_path(root, key) + complete = os.path.isfile(os.path.join(entry, "out")) and all( + os.path.isfile(os.path.join(entry, "extra-" + os.path.basename(path))) for path in args.extra + ) + if complete: + shutil.copyfile(os.path.join(entry, "out"), args.dest) + for path in args.extra: + shutil.copyfile(os.path.join(entry, "extra-" + os.path.basename(path)), path) + stale = time.time() - os.stat(entry).st_mtime > STALE_SECONDS + os.utime(entry) + with open(os.path.join(args.dir, "timings.log"), "a", encoding="utf-8") as timings: + timings.write(f"{args.step} cached\n") + log(f"hit {label} {key[:12]}{' stale' if stale else ''}") + return 0 + log(f"miss {label} {key[:12]}") + return 1 + + +def store(args): + root = cache_dir() + label = f"{os.path.basename(os.path.abspath(args.dir))} {args.step}" + if not root or is_held_out(os.path.basename(os.path.abspath(args.dir))) or not os.path.exists(keyfile(args)): + return 0 + with open(keyfile(args), encoding="ascii") as handle: + key = handle.read() + with open(args.dest, "rb") as handle: + out = handle.read() + status = args.status + if status is None: + last = out.rstrip(b"\n").rsplit(b"\n", 1)[-1] + status = int(last[5:]) if last.startswith(b"exit ") and last[5:].isdigit() else -1 + if status != 0: + log(f"not kept {label} {key[:12]}: status {status}") + return 0 + # The output and every file the step also wrote: each must be there, under + # the size cap, and show no failure a rerun could mend (a check that raised + # prints it into its file and may still exit 0). + contents = [("output", out)] + for path in args.extra: + if not os.path.isfile(path): + log(f"not kept {label} {key[:12]}: {os.path.basename(path)} was not written") + return 0 + with open(path, "rb") as handle: + contents.append((os.path.basename(path), handle.read())) + for name, data in contents: + if len(data) > MAX_ENTRY_BYTES or any(mark in data.decode("utf-8", "replace") for mark in TRANSIENT): + log(f"not kept {label} {key[:12]}: {name} is over the size cap or shows a failure to repeat") + return 0 + entry = entry_path(root, key) + if os.path.isdir(entry): + return 0 + scratch = os.path.join(root, f".new-{os.getpid()}-{time.time_ns()}") + os.makedirs(scratch) + shutil.copyfile(args.dest, os.path.join(scratch, "out")) + for path in args.extra: + shutil.copyfile(path, os.path.join(scratch, "extra-" + os.path.basename(path))) + with open(os.path.join(scratch, "meta.json"), "w", encoding="utf-8") as meta: + json.dump({"model": os.path.basename(os.path.abspath(args.dir)), "step": args.step}, meta) + os.makedirs(os.path.dirname(entry), exist_ok=True) + try: + os.rename(scratch, entry) + except OSError: + shutil.rmtree(scratch, ignore_errors=True) + return 0 + log(f"kept {label} {key[:12]} {len(out)} bytes") + return 0 + + +def entries(root): + for shard in sorted(os.listdir(root)): + shard_path = os.path.join(root, shard) + if len(shard) == 2 and os.path.isdir(shard_path): + for name in sorted(os.listdir(shard_path)): + path = os.path.join(shard_path, name) + if os.path.isdir(path): + yield path + + +def size_of(path): + return sum(os.path.getsize(os.path.join(base, name)) for base, _, names in os.walk(path) for name in names) + + +def finish(args): + root = cache_dir() + report = os.path.join(work_dir(), "report.log") + lines = [] + if os.path.exists(report): + with open(report, encoding="utf-8") as handle: + lines = handle.read().splitlines() + count = {kind: sum(line.startswith(kind + " ") for line in lines) for kind in ("hit", "miss", "kept", "not cached", "not kept")} + stale = sum(line.startswith("hit ") and line.endswith(" stale") for line in lines) + pruned = 0 + total = 0 + kept_entries = 0 + held = 0 + unknown = 0 + by_model = collections.Counter() + if root: + now = time.time() + survivors = [] + for path in list(entries(root)): + model = None + try: + with open(os.path.join(path, "meta.json"), encoding="utf-8") as meta: + model = json.load(meta)["model"] + except (OSError, ValueError, KeyError): + pass + # An entry that does not name its model, or names a held-out one, + # is removed: nothing of those models is kept. + if model is None: + shutil.rmtree(path) + unknown += 1 + elif is_held_out(model): + shutil.rmtree(path) + held += 1 + elif now - os.stat(path).st_mtime > args.days * 86400: + shutil.rmtree(path) + pruned += 1 + else: + survivors.append((os.stat(path).st_mtime, path, size_of(path))) + by_model[model.split("-")[0]] += 1 + survivors.sort() + total = sum(size for _, _, size in survivors) + while survivors and total > args.max_mb * 1024 * 1024: + _, path, size = survivors.pop(0) + shutil.rmtree(path) + total -= size + pruned += 1 + kept_entries = len(survivors) + for shard in os.listdir(root): + path = os.path.join(root, shard) + if os.path.isdir(path) and not os.listdir(path): + os.rmdir(path) + elif shard.startswith(".new-"): + shutil.rmtree(path, ignore_errors=True) + changed = count["kept"] > 0 or pruned > 0 or stale > 0 or held > 0 or unknown > 0 + summary = ( + f"score cache: {count['hit']} hits, {count['miss']} misses ({count['kept']} kept), " + f"{count['not cached']} not cached; {kept_entries} entries, {total / 1048576:.1f} MiB; " + f"{pruned} pruned, {stale} refreshed; changed={str(changed).lower()}\n" + f"score cache entries by model prefix: {', '.join(f'{name} {n}' for name, n in sorted(by_model.items())) or 'none'}; " + f"held out ({', '.join(HELD_OUT)}): {held} found and removed; without a model name: {unknown} removed" + ) + print(summary) + if args.report: + with open(args.report, "w", encoding="utf-8") as out: + out.write("\n".join(lines) + "\n" + summary + "\n") + output = os.environ.get("GITHUB_OUTPUT") + if output: + with open(output, "a", encoding="utf-8") as handle: + handle.write(f"changed={str(changed).lower()}\n") + return 0 + + +def main(argv): + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + sub = parser.add_subparsers(dest="command_name", required=True) + for name in ("lookup", "store"): + p = sub.add_parser(name) + p.add_argument("--step", required=True) + p.add_argument("--dir", required=True) + p.add_argument("--dest", required=True) + p.add_argument("--extra", action="append", default=[]) + if name == "store": + p.add_argument("--status", type=int) + else: + p.add_argument("command", nargs="+") + p = sub.add_parser("finish") + p.add_argument("--days", type=int, default=21) + p.add_argument("--max-mb", type=int, default=256) + p.add_argument("--report") + args = parser.parse_args(argv) + if args.command_name == "finish": + return finish(args) + # Whatever goes wrong with the cache, the step runs as it always did: a + # lookup that fails says "not cached" (2), never "miss" (1), and a store + # that fails keeps nothing. + failed = 2 if args.command_name == "lookup" else 0 + try: + if args.command_name == "lookup": + os.remove(keyfile(args)) + except OSError: + pass + try: + return {"lookup": lookup, "store": store}[args.command_name](args) + except Exception as error: + log(f"not cached {os.path.basename(os.path.abspath(args.dir))} {args.step}: {args.command_name} failed, {type(error).__name__}: {error}") + return failed + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tools/ci/measure-reference-models.sh b/tools/ci/measure-reference-models.sh index 0d4f06c79..7657bb211 100755 --- a/tools/ci/measure-reference-models.sh +++ b/tools/ci/measure-reference-models.sh @@ -61,6 +61,15 @@ # another release and the same project saved by that one, the sample first, # into probe-pair.txt, ending with its exit status: the cross-release # comparisons of #740 and #742. +# When MEASURE_CACHE_DIR names a directory, a scorer or schema step whose every +# input is as it was in an earlier run takes that run's output +# (tools/ci/measure-cache.py, docs/maintaining.md); its ` cached` line +# stands for its time in timings.log, and OUT_DIR/score-cache-.log (written +# by the workflow) lists each hit and miss. Exports, rvt-info, rvt-gltf, the +# witness exports and probes always run, and the held-out models, BIMcollab's +# and the pairs once held out (holdout-*, bimcollab-*, spent-*), of which only +# counts leave the runner, are never looked up or stored: the Actions cache is +# not private to the workflows that may change it. # A scorer that fails is recorded, not fatal: its output is the evidence. # These runs check what the exports hold; their times are not benchmarks # (the workflow's `timing` job times both refs on one runner, @@ -210,6 +219,50 @@ score() { # name args... echo "exit $?" >> "$out" } +# The score cache (tools/ci/measure-cache.py), on when MEASURE_CACHE_DIR names +# its directory: a step whose every input is as it was in an earlier run takes +# that run's output, and ` cached` stands for its time in timings.log. A +# step is kept only if it finished with status 0, and the models whose results +# only leave the runner as counts (holdout-*, bimcollab-*), and the pairs that +# were once held out (spent-*), are never cached. +CACHE_TOOL="$(dirname "$0")/measure-cache.py" +cscore() { # name args...: `score`, from the cache when the scorer and its inputs are unchanged + local out="$1" dir step found=2; shift + dir=$(dirname "$out"); step=$(basename "$out" .txt) + if [ -n "${MEASURE_CACHE_DIR:-}" ]; then + python "$CACHE_TOOL" lookup --step "$step" --dir "$dir" --dest "$out" -- "$@" + found=$? + [ "$found" = 0 ] && return 0 + fi + score "$out" "$@" + [ "$found" = 1 ] && python "$CACHE_TOOL" store --step "$step" --dir "$dir" --dest "$out" + return 0 +} + +# A schema step's output on stdout, the cache's when the script and the file +# are unchanged, else the step's own under its time limit. The files the step +# also writes are named by CACHE_EXTRA, and kept with the output. +cached() { # dir step limit script args... + local dir="$1" step="$2" limit="$3" script="$4" tmp status found=2 extra=() path; shift 4 + tmp="${RUNNER_TEMP:-/tmp}/cached-$$-$(basename "$dir")-$step" + for path in ${CACHE_EXTRA:-}; do extra+=(--extra "$path"); done + if [ -n "${MEASURE_CACHE_DIR:-}" ]; then + python "$CACHE_TOOL" lookup --step "$step" --dir "$dir" --dest "$tmp" ${extra[@]+"${extra[@]}"} -- "$script" "$@" + found=$? + if [ "$found" = 0 ]; then + cat "$tmp" + rm -f "$tmp" + return 0 + fi + fi + timed "$dir" "$step" timeout "$limit" python "$script" "$@" > "$tmp" 2>&1 + status=$? + cat "$tmp" + [ "$found" = 1 ] && python "$CACHE_TOOL" store --step "$step" --dir "$dir" --dest "$tmp" --status "$status" ${extra[@]+"${extra[@]}"} + rm -f "$tmp" + return "$status" +} + info() { # file dir "$BIN/rvt-info" "$1" --redact --json > "$2/info.json" 2> "$2/info.log" echo "exit $?" > "$2/info.exit" @@ -223,19 +276,27 @@ PSET_DEFINITIONS="${RUNNER_TEMP:-/tmp}/ifc4-pset-definitions.csv" schema() { # dir max_bytes { echo "arity:" - timed "$1" schema-arity timeout 600 python "$(dirname "$0")/ifc_schema_arity.py" "$1/model.ifc" 2>&1 \ + cached "$1" schema-arity 600 "$(dirname "$0")/ifc_schema_arity.py" "$1/model.ifc" \ | grep -v '^IFC schema conformance passed\|^ instances checked' || true if [ "$(stat -c %s "$1/model.ifc")" -le "$2" ]; then + # The validation and the EXPRESS rules read one parse of the file; the + # rules' summary comes back in a file, to follow the validation's. + # (validation.jsonl is kept for ports_vs_ifc, which a sample has none of, + # and which measure_sample deletes.) + local extras="$1/validation.jsonl $1/schema-rules.part" + case "$(basename "$1")" in sample-*) extras="$1/schema-rules.part" ;; esac echo "validate:" - timed "$1" schema-validate timeout "$STEP_TIMEOUT" python "$(dirname "$0")/ifc_validate_summary.py" "$1/model.ifc" "$1/validation.jsonl" 2>&1 + CACHE_EXTRA="$extras" cached "$1" schema-validate-rules $(( 2 * STEP_TIMEOUT )) \ + "$(dirname "$0")/ifc_validate_summary.py" --both "$1/model.ifc" "$1/validation.jsonl" "$1/schema-rules.part" echo "rules:" - timed "$1" schema-rules timeout "$STEP_TIMEOUT" python "$(dirname "$0")/ifc_validate_summary.py" --rules "$1/model.ifc" 2>&1 + cat "$1/schema-rules.part" 2> /dev/null || true + rm -f "$1/schema-rules.part" else echo "validate: skipped, over $(( $2 / 1048576 )) MB" fi echo "property sets:" - timed "$1" schema-psets timeout "$STEP_TIMEOUT" python "$(dirname "$0")/ifc4_property_sets.py" "$1/model.ifc" \ - --definitions "$PSET_DEFINITIONS" 2>&1 + cached "$1" schema-psets "$STEP_TIMEOUT" "$(dirname "$0")/ifc4_property_sets.py" "$1/model.ifc" \ + --definitions "$PSET_DEFINITIONS" } > "$1/schema.txt" } @@ -312,7 +373,7 @@ model_task() { # name rvt ref task if [ "$ref" != "-" ] && wanted ports_vs_ifc; then extra=() [ -f "$dir/validation.jsonl" ] && extra=(--validation "$dir/validation.jsonl") - score "$dir/ports_vs_ifc.txt" "$SCORERS/ports_vs_ifc.py" "$dir/model.ifc" "$MODELS/$ref" "${extra[@]}" + cscore "$dir/ports_vs_ifc.txt" "$SCORERS/ports_vs_ifc.py" "$dir/model.ifc" "$MODELS/$ref" "${extra[@]}" fi ;; witness) @@ -329,18 +390,23 @@ model_task() { # name rvt ref task if [ "$ref" != "-" ]; then for s in $GLB_SCORERS; do wanted "$s" || continue - score "$dir/$s.txt" "$SCORERS/$s.py" "$dir/model.glb" "$MODELS/$ref" + cscore "$dir/$s.txt" "$SCORERS/$s.py" "$dir/model.glb" "$MODELS/$ref" done fi ;; ifc:*) s=${task#ifc:} - score "$dir/$s.txt" "$SCORERS/$s.py" "$dir/model.ifc" "$MODELS/$ref" + # A research lister named in MEASURE_SCORERS, in neither list, is not cached. + if listed "$s"; then + cscore "$dir/$s.txt" "$SCORERS/$s.py" "$dir/model.ifc" "$MODELS/$ref" + else + score "$dir/$s.txt" "$SCORERS/$s.py" "$dir/model.ifc" "$MODELS/$ref" + fi ;; meshes) for s in saved_meshes_vs_ifc mesh_bodies_vs_ifc; do wanted "$s" || continue - RVT_MESH_CACHE="$dir/mesh-cache" score "$dir/$s.txt" "$SCORERS/$s.py" "$dir/model.ifc" "$MODELS/$ref" + RVT_MESH_CACHE="$dir/mesh-cache" cscore "$dir/$s.txt" "$SCORERS/$s.py" "$dir/model.ifc" "$MODELS/$ref" done echo "$name: mesh cache $(du -sh "$dir/mesh-cache" 2>/dev/null | cut -f1), removed" rm -rf "$dir/mesh-cache" @@ -358,7 +424,7 @@ model_task() { # name rvt ref task timed "$dir" rvt-ifc-revit-sets "$BIN/rvt-ifc" "$MODELS/$rvt" -o "$dir/revit-sets.ifc" ${RVT_IFC_FLAGS:-} "${flags[@]}" \ > "$dir/rvt-ifc-revit-sets.log" 2>&1 echo "exit $?" >> "$dir/rvt-ifc-revit-sets.log" - score "$dir/revit_property_sets_vs_ifc.txt" "$SCORERS/revit_property_sets_vs_ifc.py" "$dir/revit-sets.ifc" "$MODELS/$sets" --list + cscore "$dir/revit_property_sets_vs_ifc.txt" "$SCORERS/revit_property_sets_vs_ifc.py" "$dir/revit-sets.ifc" "$MODELS/$sets" --list rm -f "$dir/revit-sets.ifc" ;; esac diff --git a/tools/maintainer/clean_caches.py b/tools/maintainer/clean_caches.py index 113ff2da8..86807db5d 100644 --- a/tools/maintainer/clean_caches.py +++ b/tools/maintainer/clean_caches.py @@ -7,7 +7,11 @@ again, apart from the newest few that a build may still restore from; - on `main`, the superseded overlay base databases of each CodeQL language and version: every analysis of `main` saves one under its commit, and the next - analysis on that version reads only the newest. + analysis on that version reads only the newest; +- on `main`, the superseded generations of Measure's score cache, which a run + restores from the newest of its prefix and saves anew, and all but the newest + sixteen of its binary caches, one per commit built, which only runs of that + commit read. The caches of live branches, of open pull requests, of tags, and `main`'s newest generations are kept. The repository's cache limit is 10 GB and GitHub evicts the @@ -39,6 +43,15 @@ # highest enabled CLI version among the cached bases, so each version keeps its # newest ones. CODEQL_OVERLAY_KEY = re.compile(r"(codeql-overlay-base-database-\d+-[0-9a-f]+-[a-z]+-[0-9.]+)-[0-9a-f]{40}-\d+-\d+") +# Measure's score cache (tools/ci/measure-cache.py): +# measure-scores-v-
-.---. +# A run restores the newest of its shard's prefix, so only the newest generations are read. +SCORE_CACHE_KEY = re.compile(r"(measure-scores-v\d+-.+?)(?:-[0-9a-f]{40})?-\d+-\d+") +# Measure's binary cache: rvt-bin-v-----, one +# per commit built. A commit's key is read only by runs of that commit, which are +# the last few: the newest BINARY_GENERATIONS are kept. +BINARY_CACHE_KEY = re.compile(r"(rvt-bin-v\d+)-[0-9a-f]{40}-.+") +BINARY_GENERATIONS = 16 def gh(*args): @@ -76,12 +89,17 @@ def unusable(caches, open_prs, live_branches, keep): if name != "main" and name not in live_branches: doomed.append((c, f"branch {name} was deleted")) elif name == "main": - k = RUST_CACHE_KEY.fullmatch(c["key"]) or CODEQL_OVERLAY_KEY.fullmatch(c["key"]) + k = ( + RUST_CACHE_KEY.fullmatch(c["key"]) + or CODEQL_OVERLAY_KEY.fullmatch(c["key"]) + or SCORE_CACHE_KEY.fullmatch(c["key"]) + or BINARY_CACHE_KEY.fullmatch(c["key"]) + ) if k: generations.setdefault(k.group(1), []).append(c) for family, lst in generations.items(): lst.sort(key=lambda c: c["last_accessed_at"], reverse=True) - for c in lst[keep:]: + for c in lst[(BINARY_GENERATIONS if family.startswith("rvt-bin-") else keep):]: doomed.append((c, f"superseded generation of {family}")) return doomed