From 2e2a50adbd64ac86499574686d89b32d524a01e1 Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Wed, 14 Oct 2026 16:31:11 +0200 Subject: [PATCH 1/2] Add a sync of shared fields across the Interra boards I want the fields that the Interra boards, the PROTEUS roadmap and the issues share to stay equal, so that a change in one place shows up in the others. Start date, target date and priority are synced in all four places: the Interra Management board, the Interra Collaboration board, the PROTEUS roadmap and the org issue fields of each issue. Workload is synced on the three boards and sets the Effort issue field, and status, category, sub-team and release are synced between the two Interra boards. The package reads all places, decides per issue and field which value changed last, and writes that value to the other places. A state file in a private repository records what each place showed in the previous run, so a cleared value can be told apart from one that was never set. A scheduled workflow runs it every 30 minutes as a GitHub App; the public log shows counts, field names and error types only. --- .github/workflows/sync.yml | 89 ++++ .github/workflows/tests.yml | 34 ++ .gitignore | 12 + README.md | 92 ++++- board_sync/__init__.py | 1 + board_sync/__main__.py | 332 +++++++++++++++ board_sync/config.py | 80 ++++ board_sync/core.py | 462 +++++++++++++++++++++ board_sync/github.py | 396 ++++++++++++++++++ pyproject.toml | 16 + scripts/store_state.sh | 42 ++ tests/conftest.py | 71 ++++ tests/test_cli.py | 180 +++++++++ tests/test_core.py | 224 +++++++++++ tests/test_github.py | 415 +++++++++++++++++++ tests/test_store_state.py | 133 ++++++ tests/test_sync_rules.py | 779 ++++++++++++++++++++++++++++++++++++ 17 files changed, 3357 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/sync.yml create mode 100644 .github/workflows/tests.yml create mode 100644 .gitignore create mode 100644 board_sync/__init__.py create mode 100644 board_sync/__main__.py create mode 100644 board_sync/config.py create mode 100644 board_sync/core.py create mode 100644 board_sync/github.py create mode 100644 pyproject.toml create mode 100755 scripts/store_state.sh create mode 100644 tests/conftest.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_core.py create mode 100644 tests/test_github.py create mode 100644 tests/test_store_state.py create mode 100644 tests/test_sync_rules.py diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml new file mode 100644 index 0000000..1841640 --- /dev/null +++ b/.github/workflows/sync.yml @@ -0,0 +1,89 @@ +name: Board sync + +on: + schedule: + - cron: "7,37 * * * *" + workflow_dispatch: + inputs: + apply: + description: Write the planned values (off prints the plan only) + type: boolean + default: false + allow_bulk: + description: Apply even when the plan exceeds the change limit + type: boolean + default: false + +concurrency: + group: board-sync + cancel-in-progress: false + +permissions: + contents: read + +jobs: + sync: + if: github.repository == 'Interra-Project/board-sync' && github.ref == 'refs/heads/main' + environment: board-sync + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Check out the state repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: Interra-Project/board-sync-state + ssh-key: ${{ secrets.STATE_DEPLOY_KEY }} + path: state + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Token for Interra-Project + id: interra + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.BOARD_SYNC_CLIENT_ID }} + private-key: ${{ secrets.BOARD_SYNC_PRIVATE_KEY }} + owner: Interra-Project + + - name: Token for FormingWorlds + id: formingworlds + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.BOARD_SYNC_CLIENT_ID }} + private-key: ${{ secrets.BOARD_SYNC_PRIVATE_KEY }} + owner: FormingWorlds + + - name: Sync + timeout-minutes: 20 + env: + BOARD_SYNC_TOKEN_INTERRA_PROJECT: ${{ steps.interra.outputs.token }} + BOARD_SYNC_TOKEN_FORMINGWORLDS: ${{ steps.formingworlds.outputs.token }} + APPLY: ${{ github.event_name == 'schedule' || inputs.apply }} + BULK: ${{ github.event_name == 'workflow_dispatch' && inputs.allow_bulk }} + run: | + flags="--state state/state.json --snapshot-dir state --report state/report.txt --summary --pause 1" + if [ "$APPLY" = "true" ]; then flags="$flags --apply"; fi + if [ "$BULK" = "true" ]; then flags="$flags --allow-bulk"; fi + code=0 + python -m board_sync $flags || code=$? + if [ "$code" = 4 ]; then + echo "::warning::Some writes are parked after repeated failures; see report.txt in the state repository." + code=0 + fi + exit "$code" + + - name: Store state, snapshot and report + if: always() + env: + RUN_ID: ${{ github.run_id }} + GIT_AUTHOR_NAME: board-sync + GIT_AUTHOR_EMAIL: board-sync@users.noreply.github.com + GIT_COMMITTER_NAME: board-sync + GIT_COMMITTER_EMAIL: board-sync@users.noreply.github.com + run: bash scripts/store_state.sh state "$RUN_ID" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..a047ad7 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,34 @@ +name: Tests + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Install test tools + run: pip install pytest==9.1.1 coverage==7.16.2 ruff==0.16.9 + + - name: Lint + run: | + ruff check . + ruff format --check . + + - name: Test with coverage + run: | + coverage run -m pytest -q + coverage report --fail-under=95 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3bce80a --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +__pycache__/ +*.pyc +.coverage +.pytest_cache/ +.ruff_cache/ +*.tmp +# Files a local run writes: they hold private board data +state.json +state/ +snapshot/ +snapshot.json +report.txt diff --git a/README.md b/README.md index 3d4b5a2..e95becb 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,92 @@ # board-sync -Keeps dates, priority, workload and status equal across the Interra project boards and the org issue fields + +Keeps shared fields equal across 4 places: + +- the org issue fields on each issue (Interra-Project and FormingWorlds), +- the Interra Management board (Interra-Project project 1), +- the Interra Collaboration board (Interra-Project project 2), +- the PROTEUS Development Roadmap (FormingWorlds project 7). + +When a value changes in one place, the next run writes it to the others. + +| Field | Issue field | Management | Collaboration | PROTEUS | +|---|---|---|---|---| +| Start date | yes | yes | yes | yes | +| Target date | yes | yes | yes | yes | +| Priority | yes (Urgent to Low) | yes | yes | yes | +| Workload | Effort, written from Workload | yes | yes | yes | +| Category, Sub-team, Release | | yes | yes | | +| Status | | yes | yes | | + +The value names per place and the field list are in `board_sync/config.py`. +Status values `Import` and `Starter Projects` show as `TBD` on the Management board. +Effort follows Workload in one direction: a hand edit of Effort on an issue that has a Workload is overwritten, and a cleared Workload leaves Effort as it is. + +## Rules + +- The private repository `Interra-Project/board-sync-state` holds, per issue and field, the agreed value and the value last seen at each place. + A place whose value differs from its last seen value changed since the previous run. +- When several places changed, the newest change wins. + Board values and issue field values have a change time; a cleared issue field takes the time of its removal event. + A board does not record when a value was cleared, so a cleared board value loses to a value set at another place in the same interval. + The issue timeline read covers the last 100 field events; an older change has no time and loses. +- A value that every changed place already shows, after the name map, wins without a conflict. +- A cleared value is cleared in the other places too. + A field that is emptied as a whole at one place (all of its 5 or more values, see below) is filled again from the others. +- A field that is missing at a place is skipped there, never read as empty, and the run log names it; a value without a matching option is listed in `report.txt` as skipped. + A field with a new id (deleted and created again), or a field that held 5 or more values and now holds none, does not clear the other places. +- An issue that is not on a board, or cannot be read, keeps its record for that board. + When it is there again with the field, its empty fields receive the agreed values, an unchanged value counts as no change, and a different value wins only when it is newer than the agreed value (an agreed value of unknown time keeps its place). + On a field that was created again, the first value counts as a change, as on a new board. + An issue that is on no board for 90 days is dropped from the state. +- A list longer than one page of the API (fields of a board, values of an item, issue fields, issue field values) stops the read with an error, so a cut-off list is never read as empty values. +- A run that plans more than 20 writes stops and writes nothing (exit code 3). + Read the plan in `report.txt` in the state repository, then start the workflow by hand with `apply` and `allow_bulk`. + The state then holds the approved writes that are not done yet, and the scheduled runs write them, at most 400 values per run. + A later run stops again when its plan holds more than 20 new writes: writes outside the approved set that never failed before. + The approval ends when all approved writes are done, or after 4 runs in a row in which no write succeeds and none reaches parking; a new hand run with `allow_bulk` starts it again. +- After 5 failed writes in a row, or 10 minutes after the run started, a run stops writing; failed and unwritten values are written again by the next run, the failed ones after all others. + A write that failed in 3 runs is parked: it is listed in `report.txt` and tried again once a day, or at once when its value changes. + Parked writes give a warning in the run, not a failed run. +- `report.txt` holds the plan before the first write, and the results after the last. +- Each run stores the raw read of all places (`snapshot.json`) and the full report (`report.txt`) in the state repository, so lost values can be restored from its history. + The 3 files of a run replace the stored ones whole, and a run that changes nothing makes no commit. +- The job log of this public repository shows counts per board and field, field names and error types only. + Issue links, values and error text stay in the state repository. +- A run reads the places one after the other and then writes. + An edit made in the minutes of a run can be overwritten by that run; the next run then sees the written value as agreed. +- Exit codes: 0 done, 1 some writes failed, 2 the read, the state file, a token or the run failed, 3 too many changes, 4 done but some writes are parked (the workflow shows it as a warning), 143 or 130 stopped by a signal. + +## Setup + +1. Create a GitHub App that can be installed on any account (both orgs need it), with these permissions and no webhook: + - Repository: Issues read and write, Metadata read. + - Organization: Projects read and write, Issue fields read. +2. Install it on Interra-Project and on FormingWorlds, with access to all repositories whose issues are on the boards. +3. Protect `main` in this repository with a rule that requires a pull request with an approving review, includes administrators and blocks force pushes, because code on `main` runs with the App key and the deploy key. + Then create the environment `board-sync` and limit its deployment branches to `main`. + Do not add a required reviewer: it would hold every scheduled run too. +4. Create `Interra-Project/board-sync-state` as a private repository with one commit on `main`, for example a README. +5. Store all 3 values in the environment only, not as repository secrets: the App client id as the variable `BOARD_SYNC_CLIENT_ID`, the App private key as the secret `BOARD_SYNC_PRIVATE_KEY`, and the private key of an SSH deploy key with write access to `Interra-Project/board-sync-state` as the secret `STATE_DEPLOY_KEY`. + +The workflow runs at minute 7 and 37 of every hour on `main`; a run on another branch is skipped. +A run started by hand from the Actions tab only plans, unless `apply` is set. +GitHub turns off scheduled workflows in a public repository after 60 days without activity in that repository. +Check the date of the last run in the Actions tab of this repository, and enable the workflow again there when it is off. + +## Local run + +```bash +GH_TOKEN=$(gh auth token) python -m board_sync --state state.json --snapshot-dir snapshot +``` + +Without `--apply` the run changes nothing on GitHub; it writes only the snapshot and report files it is given. +These files hold private board data; `.gitignore` keeps them out of commits. +`scripts/store_state.sh` is the workflow step that stores the files of a run in the state repository. +`BOARD_SYNC_TOKEN_INTERRA_PROJECT` and `BOARD_SYNC_TOKEN_FORMINGWORLDS` set one token per org. + +## Tests + +```bash +python -m pytest +``` diff --git a/board_sync/__init__.py b/board_sync/__init__.py new file mode 100644 index 0000000..f9023b0 --- /dev/null +++ b/board_sync/__init__.py @@ -0,0 +1 @@ +"""Sync of fields across the Interra boards, the PROTEUS board and the org issue fields.""" diff --git a/board_sync/__main__.py b/board_sync/__main__.py new file mode 100644 index 0000000..930af90 --- /dev/null +++ b/board_sync/__main__.py @@ -0,0 +1,332 @@ +"""Sync shared fields across the Interra boards, the PROTEUS board and the org issue fields. + +Without ``--apply`` the run changes nothing on GitHub: it prints the plan and +writes only the snapshot and report files it is given. With ``--apply`` it +writes the plan, unless the plan has more than ``--max-changes`` writes; then +it stops with exit code 3. A person reads the plan and reruns with +``--allow-bulk``; the state then holds the approved writes, and the next runs +write them in batches of ``--limit``. +""" + +import argparse +import io +import json +import os +import signal +import sys +import time +import traceback +from collections import Counter +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from . import config, core, github + +TOKEN_ENV = { + "Interra-Project": "BOARD_SYNC_TOKEN_INTERRA_PROJECT", + "FormingWorlds": "BOARD_SYNC_TOKEN_FORMINGWORLDS", +} + +# Failed writes in a row after which a run stops and leaves the rest for later. +STOP_AFTER_FAILURES = 5 +# Runs a write may fail before it is parked; a parked write is tried once a day. +PARK_AFTER = 3 +PARK_RETRY = timedelta(days=1) +# Runs in a row without a successful write after which a bulk approval ends. +BULK_IDLE_RUNS = PARK_AFTER + 1 + + +class StateError(RuntimeError): + """The state file cannot be used.""" + + +def make_clients(env=os.environ): + """Build one GraphQL client per org from the environment. + + Parameters + ---------- + env : mapping + Environment; each org reads its own variable and falls back to + ``GH_TOKEN``. + + Returns + ------- + dict + Org login to ``github.Client``. + + Raises + ------ + SystemExit + With code 2 if an org has no token. + """ + clients = {} + for org, var in TOKEN_ENV.items(): + token = env.get(var) or env.get("GH_TOKEN") + if not token: + print(f"no token for {org}: set {var} or GH_TOKEN", file=sys.stderr) + raise SystemExit(2) + clients[org] = github.Client(token) + return clients + + +def load_state(path): + """Read the state file; an absent or empty file is an empty state. + + Raises + ------ + StateError + If the file is not a JSON object, so a damaged state never counts as a first run. + """ + p = Path(path) + if not p.exists() or not p.read_text().strip(): + return {} + try: + data = json.loads(p.read_text()) + except ValueError: + raise StateError(f"state file {path} is not valid JSON") from None + if not isinstance(data, dict): + raise StateError(f"state file {path} does not hold a JSON object") + return data + + +def write_file(path, text): + """Write a file through a temporary file, so a killed run leaves the old file whole.""" + tmp = Path(f"{path}.tmp") + tmp.write_text(text) + os.replace(tmp, path) + + +def save_state(path, state): + """Write the state file atomically.""" + write_file(path, json.dumps(state, indent=1, sort_keys=True)) + + +def describe(w): + """Format one planned write as one line.""" + where = config.BOARDS.get(w["place"], {}).get("title", "issue field") + value = "(clear)" if w["value"] is None else w["value"] + return f"{w['url']} {where} / {w['field']}: {value}" + + +def counts(writes): + """Count writes per place and field, as lines without issue data.""" + tally = Counter(f"{config.BOARDS.get(w['place'], {}).get('title', 'issue field')} / {w['field']}" for w in writes) + return [f" {n:5d} {k}" for k, n in sorted(tally.items())] + + +def warnings(snap): + """Lines about fields and items the run could not use, without issue data.""" + lines = [f"warning: {key} has no field {name!r}" for key, name in core.missing_fields(snap)] + for key, n in sorted(snap.get("skipped_items", {}).items()): + if n: + lines.append(f"note: {config.BOARDS[key]['title']}: {n} items are not readable issues") + for org, n in sorted(snap.get("unread_issues", {}).items()): + if n: + lines.append(f"warning: {n} {org} issues could not be read for their issue fields") + return lines + + +def emit(args, summary, detail, out): + """Write the full report file and print the summary, or everything.""" + if args.report: + write_file(args.report, "\n".join(summary + detail) + "\n") + print("\n".join(summary if args.summary else summary + detail), file=out) + + +def write_key(w): + """Key of one planned write, for failure counts and bulk approval.""" + return f"{w['issue']}|{w['place']}|{w['field']}|{w['value']}" + + +def is_parked(record, now): + """A write is parked after ``PARK_AFTER`` failures, until a day after its last one.""" + if not record or record["n"] < PARK_AFTER: + return False + return now - datetime.fromisoformat(record["at"]) < PARK_RETRY + + +def apply(args, clients, snap, result, active, parked, state, now, summary, detail): + """Write the planned values and save the state, also when the run is cut short. + + Parameters + ---------- + active, parked : list of dict + Writes to attempt, and writes held back after repeated failures. + state : dict + State of the previous run (failure records and bulk approval). + now : datetime + Time of this run, for the failure records. + + Returns + ------- + int + 0 when every attempted write succeeded, else 1. + """ + old_fails = state.get("failures", {}) + batch, later = active[: args.limit], active[args.limit :] + failed, done_keys, done, in_row, late = [], set(), 0, 0, False + try: + for n, w in enumerate(batch): + if in_row >= STOP_AFTER_FAILURES: + break + if time.monotonic() - args.started > args.deadline: + late = True + break + if n and args.pause: + time.sleep(args.pause) + try: + github.write(clients, snap, w) + done_keys.add(write_key(w)) + in_row = 0 + except Exception as exc: # noqa: BLE001 one refused write must not stop the others + failed.append((w, type(exc).__name__)) + in_row += 1 + detail.append(f"failed {describe(w)}: {exc}") + done = n + 1 + finally: + rest = batch[done:] + later + # Failed, unwritten and parked places keep their old record, so a later run plans them again. + core.undo_failed(result.state, [w for w, _ in failed] + rest + parked) + stamp = now.isoformat(timespec="seconds") + fails = {k: v for k, v in old_fails.items() if k in {write_key(w) for w in rest + parked}} + for w, _ in failed: + fails[write_key(w)] = {"n": old_fails.get(write_key(w), {}).get("n", 0) + 1, "at": stamp} + result.state["failures"] = fails + # A bulk approval covers the approved writes not yet done (parked ones too), until + # runs stop making progress; a write that reaches parking counts as progress. + planned = {write_key(w) for w in active + parked} + base = planned if args.allow_bulk else set(state.get("bulk_keys", [])) + left = sorted((base & planned) - done_keys) + progress = done_keys or any(r["n"] == PARK_AFTER for r in fails.values() if r["at"] == stamp) + idle = 0 if progress else (0 if args.allow_bulk else state.get("bulk_idle", 0)) + 1 + keep = left and idle < BULK_IDLE_RUNS + result.state["bulk_keys"], result.state["bulk_idle"] = (left, idle) if keep else ([], 0) + save_state(args.state, result.state) + if in_row >= STOP_AFTER_FAILURES: + summary.append(f"stopped after {in_row} failed writes in a row") + if late: + summary.append(f"stopped at the time limit of {args.deadline} s") + if rest: + summary.append(f"deferred: {len(rest)} writes to the next run") + if failed: + kinds = Counter(kind for _, kind in failed) + summary.append(f"failed: {len(failed)} ({', '.join(f'{k} {n}' for k, n in sorted(kinds.items()))})") + return 1 + return 0 + + +def plan_and_apply(args, clients, snap, out): + """Plan from a snapshot, apply when asked, and report. See ``run``.""" + if args.snapshot_dir: + # The read time stays out of the file, so an unchanged board gives an unchanged snapshot. + Path(args.snapshot_dir).mkdir(parents=True, exist_ok=True) + stored = {k: v for k, v in snap.items() if k != "read_at"} + write_file(Path(args.snapshot_dir) / "snapshot.json", json.dumps(stored, indent=1, sort_keys=True)) + state = load_state(args.state) + result = core.plan(snap, state) + now = datetime.now(UTC) + fails = state.get("failures", {}) + parked = [w for w in result.writes if is_parked(fails.get(write_key(w)), now)] + # Writes that failed before go last, so they cannot hold back the others. + active = sorted( + (w for w in result.writes if not is_parked(fails.get(write_key(w)), now)), key=lambda w: write_key(w) in fails + ) + detail = [f"conflict {c['url']} {c['fkey']}: {c['values']} -> {c['chosen']}" for c in result.conflicts] + detail += [f"skipped {s['url']} {s['place']} {s['fkey']}: no option {s['value']!r}" for s in result.skipped] + detail += ["parked " + describe(w) for w in parked] + detail += ["plan " + describe(w) for w in active] + summary = [f"issues: {len(snap['items'])} writes: {len(active)} conflicts: {len(result.conflicts)}"] + summary += counts(active) + warnings(snap) + summary += [f"reset: {key} / {fkey} is treated as new, its old values are ignored" for key, fkey in result.resets] + if result.skipped: + summary.append(f"skipped: {len(result.skipped)} values without an option at the place") + if parked: + summary.append(f"parked: {len(parked)} writes that failed in {PARK_AFTER} runs, listed in the report") + approved_keys = set(state.get("bulk_keys", [])) + # Writes that already failed were planned before; they never count as a new mass change. + outside = [w for w in active if write_key(w) not in approved_keys and write_key(w) not in fails] + approved = args.allow_bulk or len(outside) <= args.max_changes + code = 0 + if args.apply and len(active) > args.max_changes and not approved: + summary.append(f"stop: {len(active)} writes exceed --max-changes {args.max_changes}") + code = 3 + elif args.apply: + emit(args, summary, detail, io.StringIO()) # the report shows the plan while writes run + code = apply(args, clients, snap, result, active, parked, state, now, summary, detail) + emit(args, summary, detail, out) + return code or (4 if parked and args.apply else 0) + + +def safe_emit(args, line, detail, out): + """Report a failed run; if even that fails, print the line alone.""" + try: + emit(args, [line], [detail], out) + except Exception: # noqa: BLE001 never let a second error print the first one's text + print(line, file=out) + + +def run(args, clients, out=sys.stdout): + """Read all places, plan, and optionally apply. + + The detail lines name issues and values. With ``args.summary`` the + stream gets counts only, and the details go to ``args.report`` alone, + so a public job log shows no board data. An error prints only its type. + + Parameters + ---------- + args : argparse.Namespace + Parsed command line. + clients : dict + Org login to ``github.Client``. + out : file + Stream for the report. + + Returns + ------- + int + Exit code: 0 done, 1 some writes failed, 2 the read or the run failed, + 3 too many changes, 4 done but some writes are parked. + """ + try: + snap = github.read_all(clients) + except Exception as exc: # noqa: BLE001 the summary names the error type only + safe_emit(args, f"read failed: {type(exc).__name__}", f"read failed: {exc}", out) + return 2 + try: + return plan_and_apply(args, clients, snap, out) + except Exception as exc: # noqa: BLE001 the summary names the error type only + safe_emit(args, f"run failed: {type(exc).__name__}", traceback.format_exc(), out) + return 2 + + +def _terminate(signum, frame): + """Turn SIGTERM into an exit, so the state is saved on the way out.""" + raise SystemExit(128 + signum) + + +def main(argv=None, env=os.environ): + """Parse the command line and run the sync.""" + parser = argparse.ArgumentParser(prog="board_sync", description=__doc__.splitlines()[0]) + parser.add_argument("--state", default="state.json", help="state file of the previous run") + parser.add_argument("--snapshot-dir", help="folder for the raw snapshot of this run") + parser.add_argument("--apply", action="store_true", help="write the planned values") + parser.add_argument("--max-changes", type=int, default=20) + parser.add_argument("--allow-bulk", action="store_true", help="apply even above --max-changes") + parser.add_argument("--limit", type=int, default=400, help="most writes in one run; the rest wait") + parser.add_argument("--pause", type=float, default=0.0, help="seconds between writes") + parser.add_argument("--deadline", type=float, default=600.0, help="seconds from start after which writes stop") + parser.add_argument("--summary", action="store_true", help="print counts only, no issue data") + parser.add_argument("--report", help="file for the full report with issue data") + args = parser.parse_args(argv) + args.started = time.monotonic() + if env.get("GITHUB_ACTIONS") == "true": + args.summary = True # a job log can be public + old = signal.signal(signal.SIGTERM, _terminate) + try: + return run(args, make_clients(env)) + finally: + signal.signal(signal.SIGTERM, old) + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/board_sync/config.py b/board_sync/config.py new file mode 100644 index 0000000..5056f46 --- /dev/null +++ b/board_sync/config.py @@ -0,0 +1,80 @@ +"""Places and field maps for the board sync. + +A *place* is one location that holds a value for an issue: the org issue +fields on the issue itself (``issue``) or one of the project boards +(``b1``, ``b2``, ``b7``). Canonical values use the vocabulary of the +Interra Collaboration board (``b2``). +""" + +BOARDS = { + "b1": {"owner": "Interra-Project", "number": 1, "title": "Interra Management"}, + "b2": {"owner": "Interra-Project", "number": 2, "title": "Interra Collaboration"}, + "b7": {"owner": "FormingWorlds", "number": 7, "title": "PROTEUS Development Roadmap"}, +} + +# Orgs whose org issue fields take part in the sync. +ISSUE_ORGS = ("Interra-Project", "FormingWorlds") + +# Tie-break when two places changed at the same second. +PLACE_ORDER = ("issue", "b2", "b1", "b7") + +PRIORITY = ( + "Priority 1: critical", + "Priority 2: high", + "Priority 3: standard", + "Priority 4: low", +) + +# Per canonical field: the field name at each place, and the value map +# from canonical value to place value where the two differ. +FIELDS = { + "start": { + "type": "date", + "places": {"issue": "Start date", "b1": "Start date", "b2": "Start date", "b7": "Start date"}, + }, + "target": { + "type": "date", + "places": {"issue": "Target date", "b1": "Target date", "b2": "Target date", "b7": "Target date"}, + }, + "priority": { + "type": "select", + "places": {"issue": "Priority", "b1": "Priority", "b2": "Priority", "b7": "Priority"}, + "to_place": { + "issue": dict(zip(PRIORITY, ("Urgent", "High", "Medium", "Low"))), + "b1": dict(zip(PRIORITY, ("Priority: critical", "Priority: high", "Priority: normal", "Priority: low"))), + }, + }, + "workload": { + "type": "select", + "places": {"b1": "Workload", "b2": "Workload", "b7": "Workload"}, + }, + "category": {"type": "select", "places": {"b1": "Category", "b2": "Category"}}, + "subteam": { + "type": "select", + "places": {"b1": "Sub-team", "b2": "Sub-team"}, + "to_place": {"b1": {"Co-leads": "Co-Leads"}}, + }, + "release": {"type": "select", "places": {"b1": "Release", "b2": "Release"}}, + "status": { + "type": "select", + "places": {"b1": "Status", "b2": "Status"}, + "to_place": {"b1": {"Import": "TBD", "Starter Projects": "TBD"}}, + }, +} + +# Fields written from another field and never read as a source. +DERIVED = { + "effort": { + "source": "workload", + "place": "issue", + "field": "Effort", + "map": {"XL": "High", "L": "High", "M": "Medium", "S": "Low", "XS": "Low"}, + }, +} + +# A place whose field held at least this many values and now holds none is +# treated as a new, empty field, not as a mass clear. +RESET_MIN = 5 + +# An issue that is on no board for this many days is dropped from the state. +GONE_DAYS = 90 diff --git a/board_sync/core.py b/board_sync/core.py new file mode 100644 index 0000000..4ba6666 --- /dev/null +++ b/board_sync/core.py @@ -0,0 +1,462 @@ +"""Sync decision: which value wins for each issue and field, and what to write. + +The input is a snapshot of all places (see ``github.read_all``) and the state +of the previous run. The state holds, per issue and field, the agreed +canonical value ``c`` and the canonical value last seen at each place. A place +whose value differs from its last seen value changed since the previous run. +When several places changed, the newest change wins. +""" + +from dataclasses import dataclass, field +from datetime import datetime, timedelta + +from . import config + +_NEW = object() # marker: the place has no recorded value yet + + +@dataclass +class Plan: + """Result of one sync decision over all issues. + + Attributes + ---------- + writes : list of dict + One entry per value to write. Keys: ``issue``, ``url``, ``place``, + ``fkey`` (``None`` for a derived field), ``field``, ``value`` + (``None`` clears) and ``prev`` (the canonical value the place held + before the write). + state : dict + State to store after the writes; ``undo_failed`` restores the + records of writes that failed or were not made. + conflicts : list of dict + Fields where two places held different new values. + skipped : list of dict + Values that cannot be written because the place lacks the option. + resets : list of tuple + ``(place_key, fkey)`` pairs treated as new fields in this run. + """ + + writes: list = field(default_factory=list) + state: dict = field(default_factory=dict) + conflicts: list = field(default_factory=list) + skipped: list = field(default_factory=list) + resets: list = field(default_factory=list) + + +def to_place(fkey, place, value): + """Map a canonical value to the vocabulary of one place. + + Parameters + ---------- + fkey : str + Canonical field key, a key of ``config.FIELDS``. + place : str + Place key. + value : str or None + Canonical value. + + Returns + ------- + str or None + The value as the place names it. + """ + if value is None: + return None + return config.FIELDS[fkey].get("to_place", {}).get(place, {}).get(value, value) + + +def to_canon(fkey, place, value): + """Map a place value back to the canonical vocabulary. + + A place value that more than one canonical value maps to (for example + ``TBD`` on board 1) stays as it is. + + Parameters + ---------- + fkey : str + Canonical field key. + place : str + Place key. + value : str or None + Value as the place names it. + + Returns + ------- + str or None + Canonical value. + """ + if value is None: + return None + mapping = config.FIELDS[fkey].get("to_place", {}).get(place, {}) + sources = [c for c, v in mapping.items() if v == value] + return sources[0] if len(sources) == 1 else value + + +def place_fields(snap, item, place): + """Return the field definitions of one place for one issue. + + Parameters + ---------- + snap : dict + Snapshot of all places. + item : dict + One entry of ``snap["items"]``. + place : str + Place key. + + Returns + ------- + dict + Field name to definition; empty if the place has no fields. + """ + if place == "issue": + return snap["fields"].get(f"issue:{item.get('org')}", {}) + return snap["fields"].get(place, {}) + + +def scope(snap, item, fkey): + """List the places that hold this issue and this field. + + Parameters + ---------- + snap : dict + Snapshot of all places. + item : dict + One entry of ``snap["items"]``. + fkey : str + Canonical field key. + + Returns + ------- + list of str + Place keys in ``config.PLACE_ORDER`` order. + """ + names = config.FIELDS[fkey]["places"] + return [ + p for p in config.PLACE_ORDER if p in names and p in item["places"] and names[p] in place_fields(snap, item, p) + ] + + +def raw_value(item, place, name): + """Return ``(value, changed_at)`` of one field at one place. + + Parameters + ---------- + item : dict + One entry of ``snap["items"]``. + place : str + Place key. + name : str + Field name at that place. + + Returns + ------- + tuple + Value (or ``None``) and ISO change time (or ``None``). A board does not + record when a value was cleared, so an empty board value has no time. + """ + return tuple(item["places"][place]["values"].get(name, (None, None))) + + +def place_key(item, place): + """Return the key of a place's field set: ``issue:`` or the board key.""" + return f"issue:{item.get('org')}" if place == "issue" else place + + +def field_ids(snap, old=None): + """Return the id of every field at every place key, for the state. + + Ids of fields missing from this snapshot stay from ``old``, so a field + that is deleted and created again between two runs is still detected. + """ + ids = {k: dict(v) for k, v in (old or {}).items()} + for k, fields in snap["fields"].items(): + ids.setdefault(k, {}).update({name: f["id"] for name, f in fields.items()}) + return ids + + +def missing_fields(snap): + """List configured fields that a place does not have, as ``(place_key, name)``.""" + out = [] + for key in snap["fields"]: + place = "issue" if key.startswith("issue:") else key + names = [spec["places"].get(place) for spec in config.FIELDS.values()] + names += [spec["field"] for spec in config.DERIVED.values() if spec["place"] == place] + out += [(key, name) for name in names if name and name not in snap["fields"][key]] + return sorted(set(out)) + + +def reset_places(snap, state): + """Find place fields that were emptied as a whole or created again. + + A field counts as reset when its id differs from the id in the state + (deleted and created again), or when it held at least + ``config.RESET_MIN`` values in the state and holds none now. + + Parameters + ---------- + snap : dict + Snapshot of all places. + state : dict + State of the previous run. + + Returns + ------- + set of tuple + ``(place_key, fkey)`` pairs whose recorded values must be ignored. + """ + reset, seen, now = set(), {}, {} + old_ids = state.get("fields", {}) + for key, fields in snap["fields"].items(): + for fkey, spec in config.FIELDS.items(): + place = "issue" if key.startswith("issue:") else key + name = spec["places"].get(place) + if name in fields and name in old_ids.get(key, {}) and old_ids[key][name] != fields[name]["id"]: + reset.add((key, fkey)) + old = state.get("items", {}) + for iid, item in snap["items"].items(): + for fkey, spec in config.FIELDS.items(): + for place in scope(snap, item, fkey): + key = (place_key(item, place), fkey) + if raw_value(item, place, spec["places"][place])[0] is not None: + now[key] = now.get(key, 0) + 1 + if old.get(iid, {}).get(fkey, {}).get("seen", {}).get(place) is not None: + seen[key] = seen.get(key, 0) + 1 + reset |= {k for k, n in seen.items() if n >= config.RESET_MIN and now.get(k, 0) == 0} + return reset + + +def _newest(places, times): + """Pick the place with the newest change time; ties go by ``PLACE_ORDER``.""" + return max(places, key=lambda p: (times[p] or "", -config.PLACE_ORDER.index(p))) + + +def _pick(fkey, places, current, raw, times): + """Pick the winning value among places that each claim a new value. + + A value that every claiming place already shows (after mapping to that + place) wins without a conflict; ``Starter Projects`` on board 2 and + ``TBD`` on board 1 agree. Otherwise the newest change wins. + + Returns + ------- + tuple + Winning canonical value, the conflict dict (empty if none), and the + change time of the winner. + """ + agreeing = [p for p in places if all(raw[q] == to_place(fkey, q, current[p]) for q in places)] + if agreeing: + win = _newest(agreeing, times) + return current[win], {}, times[win] + win = _newest(places, times) + return current[win], {p: current[p] for p in places}, times[win] + + +def decide(fkey, places, current, raw, times, record): + """Decide the canonical value of one field of one issue. + + Parameters + ---------- + fkey : str + Canonical field key. + places : list of str + Places in scope. + current : dict + Place to current canonical value. + raw : dict + Place to current value as the place names it. + times : dict + Place to ISO change time or ``None``. + record : dict or None + State record ``{"c": ..., "t": ..., "seen": {...}, "gone": [...]}`` + of the previous run, or ``None`` for an issue and field seen for the + first time. ``t`` is the change time of the agreed value; ``gone`` + lists places that did not hold the issue or the field then. + + Returns + ------- + tuple + Canonical value, the conflicting values as a place-to-value dict + (empty when there is no conflict), and the change time of the value. + """ + if record is None: + filled = [p for p in places if current[p] is not None] + if not filled: + return None, {}, None + return _pick(fkey, filled, current, raw, times) + seen, gone = record.get("seen", {}), set(record.get("gone", ())) + changed = [] + for p in places: + before = seen.get(p, _NEW) + if p in gone and before is not _NEW: + # A returning place: empty receives, its old value is no change, and a new + # value counts only when it is newer than the agreed value (of known time). + newer = record.get("t") is not None and (times[p] or "") > record["t"] + if current[p] is not None and current[p] != before and newer: + changed.append(p) + elif before is _NEW: + # A place that joins with a value counts as a change; an empty one receives. + if current[p] is not None and raw[p] != to_place(fkey, p, record.get("c")): + changed.append(p) + elif current[p] != before: + changed.append(p) + if not changed: + return record.get("c"), {}, record.get("t") + return _pick(fkey, changed, current, raw, times) + + +def plan(snap, state): + """Compute all writes that bring every place to the agreed values. + + Parameters + ---------- + snap : dict + Snapshot of all places. + state : dict + State of the previous run (``{}`` on the first run). + + Returns + ------- + Plan + Writes, new state, conflicts and skipped values. + + Notes + ----- + An issue that is on no board for ``config.GONE_DAYS`` days (counted from + ``snap["read_at"]``) is dropped from the state. + """ + result = Plan(state={"items": {}, "fields": field_ids(snap, state.get("fields"))}) + reset = reset_places(snap, state) + result.resets = sorted(reset) + old = state.get("items", {}) + now = snap.get("read_at") + for iid, entry in old.items(): + if iid not in snap["items"]: + # Not readable or on no board this run: keep its records, every place gone. + kept = { + k: {**r, "gone": sorted(r.get("seen", {})), "gone_since": r.get("gone_since") or now} + for k, r in entry.items() + } + since = min((r["gone_since"] for r in kept.values() if r["gone_since"]), default=None) + if not (now and since and _days(since, now) > config.GONE_DAYS): + result.state["items"][iid] = kept + for iid, item in sorted(snap["items"].items()): + new_entry = {} + for fkey, spec in config.FIELDS.items(): + places = scope(snap, item, fkey) + if not places: + if fkey in old.get(iid, {}): + # The field is in no place: every place keeps its record and counts as gone. + r = old[iid][fkey] + new_entry[fkey] = {k: v for k, v in r.items() if k != "gone_since"} | { + "gone": sorted(r.get("seen", {})) + } + continue + raw, current, times = {}, {}, {} + for p in places: + raw[p], times[p] = raw_value(item, p, spec["places"][p]) + current[p] = to_canon(fkey, p, raw[p]) + record = old.get(iid, {}).get(fkey) + if record is not None: + record = { + "c": record.get("c"), + "seen": { + p: v for p, v in record.get("seen", {}).items() if (place_key(item, p), fkey) not in reset + }, + "gone": record.get("gone", []), + "t": record.get("t"), + } + c, clash, t = decide(fkey, places, current, raw, times, record) + if clash: + result.conflicts.append({"issue": iid, "url": item["url"], "fkey": fkey, "values": clash, "chosen": c}) + seen = {} + for p in places: + target = to_place(fkey, p, c) + seen[p] = current[p] + if raw[p] == target: + continue + fdef = place_fields(snap, item, p)[spec["places"][p]] + if target is not None and spec["type"] == "select" and target not in fdef.get("options", {}): + result.skipped.append({"issue": iid, "url": item["url"], "place": p, "fkey": fkey, "value": target}) + continue + result.writes.append( + { + "issue": iid, + "url": item["url"], + "place": p, + "fkey": fkey, + "field": spec["places"][p], + "value": target, + "prev": current[p], + } + ) + seen[p] = to_canon(fkey, p, target) + # Places out of scope keep their record; those without the issue are marked gone. + kept = {p: v for p, v in (record or {}).get("seen", {}).items() if p not in places} + # A place stays gone until it holds the issue and the field again. + gone = sorted(p for p in kept if p not in item["places"] or p in (record or {}).get("gone", ())) + new_entry[fkey] = {"c": c, "t": t, "seen": {**kept, **seen}, "gone": gone} + _derive(snap, item, iid, new_entry, result) + if new_entry: + result.state["items"][iid] = new_entry + return result + + +def _days(start, end): + """Days between two ISO times.""" + return (datetime.fromisoformat(end) - datetime.fromisoformat(start)) / timedelta(days=1) + + +def _derive(snap, item, iid, entry, result): + """Add writes for derived fields such as Effort from Workload.""" + for spec in config.DERIVED.values(): + source = entry.get(spec["source"], {}).get("c") + place = spec["place"] + fdef = place_fields(snap, item, place).get(spec["field"]) if place in item["places"] else None + if source is None or fdef is None: + continue + target = spec["map"].get(source) + if target is None: + continue + if target not in fdef.get("options", {}): + result.skipped.append( + {"issue": iid, "url": item["url"], "place": place, "fkey": spec["field"], "value": target} + ) + continue + if raw_value(item, place, spec["field"])[0] != target: + result.writes.append( + { + "issue": iid, + "url": item["url"], + "place": place, + "fkey": None, + "field": spec["field"], + "value": target, + "prev": None, + } + ) + + +def undo_failed(state, failed): + """Restore the state records of writes that failed. + + Parameters + ---------- + state : dict + New state from ``plan``; changed in place. + failed : list of dict + Write entries that the API refused. + + Returns + ------- + dict + The same state object. + """ + for w in failed: + if w["fkey"] is None: + continue + rec = state["items"].get(w["issue"], {}).get(w["fkey"]) + if rec is not None: + rec["seen"][w["place"]] = w["prev"] + return state diff --git a/board_sync/github.py b/board_sync/github.py new file mode 100644 index 0000000..04686f1 --- /dev/null +++ b/board_sync/github.py @@ -0,0 +1,396 @@ +"""Read and write the GitHub places: project boards and org issue fields.""" + +import http.client +import json +import time +import urllib.error +import urllib.request +from datetime import UTC, datetime + +from . import config + +API = "https://api.github.com/graphql" + +_FIELD_NAME = ( + "... on IssueFieldDate { name } ... on IssueFieldSingleSelect { name } " + "... on IssueFieldText { name } ... on IssueFieldNumber { name } " + "... on IssueFieldMultiSelect { name }" +) + +BOARD_QUERY = """ +query($owner: String!, $number: Int!, $after: String) { + organization(login: $owner) { projectV2(number: $number) { id + fields(first: 100) { pageInfo { hasNextPage } nodes { + ... on ProjectV2FieldCommon { id name dataType } + ... on ProjectV2SingleSelectField { options { id name } } } } + items(first: 100, after: $after) { pageInfo { hasNextPage endCursor } nodes { id + content { ... on Issue { id url repository { owner { login } } } } + fieldValues(first: 100) { pageInfo { hasNextPage } nodes { + ... on ProjectV2ItemFieldDateValue { date updatedAt field { ... on ProjectV2FieldCommon { name } } } + ... on ProjectV2ItemFieldSingleSelectValue { name updatedAt field { ... on ProjectV2FieldCommon { name } } } + } } } } } } +} +""" + +ISSUE_FIELDS_QUERY = """ +query($org: String!) { organization(login: $org) { issueFields(first: 100) { pageInfo { hasNextPage } nodes { + ... on IssueFieldDate { id name } + ... on IssueFieldSingleSelect { id name options { id name } } } } } } +""" + +ISSUE_VALUES_QUERY = ( + """ +query($ids: [ID!]!) { nodes(ids: $ids) { ... on Issue { id + issueFieldValues(first: 50) { pageInfo { hasNextPage } nodes { + ... on IssueFieldDateValue { value field { ... on IssueFieldDate { name } } } + ... on IssueFieldSingleSelectValue { name field { ... on IssueFieldSingleSelect { name } } } } } + timelineItems(last: 100, itemTypes: [ISSUE_FIELD_ADDED_EVENT, ISSUE_FIELD_CHANGED_EVENT, + ISSUE_FIELD_REMOVED_EVENT]) { nodes { + __typename + ... on IssueFieldAddedEvent { createdAt issueField { __FIELD_NAME__ } } + ... on IssueFieldChangedEvent { createdAt issueField { __FIELD_NAME__ } } + ... on IssueFieldRemovedEvent { createdAt issueField { __FIELD_NAME__ } } } } } } +} +""" +).replace("__FIELD_NAME__", _FIELD_NAME) + + +class GraphQLError(RuntimeError): + """The GraphQL API returned errors.""" + + +class BoardNotFound(RuntimeError): + """A configured board does not exist or the token cannot see it.""" + + +class TruncatedRead(RuntimeError): + """A list in a response was longer than the request asked for.""" + + +def _check_page(conn, what): + """Raise ``TruncatedRead`` if a connection has more nodes than were read.""" + if conn["pageInfo"]["hasNextPage"]: + raise TruncatedRead(what) + + +class Client: + """Minimal GraphQL client for one token. + + Parameters + ---------- + token : str + GitHub token (installation token or personal token). + opener : callable, optional + ``urllib.request.urlopen`` or a test replacement. + timeout : float + Seconds to wait for one response. + """ + + def __init__(self, token, opener=urllib.request.urlopen, timeout=60): + self.token = token + self.opener = opener + self.timeout = timeout + + def post(self, query, variables=None, retries=2, allow_not_found=False): + """Run one GraphQL request and return its ``data``. + + Parameters + ---------- + query : str + GraphQL document. + variables : dict, optional + Query variables. + retries : int + Retries after an HTTP 502, 503 or 504, a network error, or a rate + limit (HTTP 403 or 429, or a ``RATE_LIMITED`` error). + allow_not_found : bool + Return the partial ``data`` when every error is of type + ``NOT_FOUND`` (for example a deleted issue in a ``nodes`` list). + + Returns + ------- + dict + The ``data`` member of the response. + + Raises + ------ + GraphQLError + If the response holds errors. + urllib.error.URLError, TimeoutError, OSError, http.client.HTTPException + If the request still fails after the retries. + """ + body = json.dumps({"query": query, "variables": variables or {}}).encode() + req = urllib.request.Request( + API, + data=body, + headers={"Authorization": f"Bearer {self.token}", "Content-Type": "application/json"}, + ) + try: + with self.opener(req, timeout=self.timeout) as resp: + out = json.loads(resp.read()) + except urllib.error.HTTPError as exc: + try: + text = exc.read(65536).decode("utf-8", "replace") if exc.fp else "" + except (OSError, http.client.HTTPException): + text = "" + exc.close() + if retries > 0 and exc.code in (502, 503, 504): + time.sleep(5) + return self.post(query, variables, retries - 1, allow_not_found) + wait = _retry_after((exc.headers or {}).get("Retry-After")) + limited = exc.code == 429 or (exc.code == 403 and (wait is not None or "rate limit" in text.lower())) + if retries > 0 and limited: + time.sleep(60 if wait is None else wait) + return self.post(query, variables, retries - 1, allow_not_found) + raise + except (urllib.error.URLError, TimeoutError, ConnectionError, http.client.HTTPException): + if retries > 0: + time.sleep(5) + return self.post(query, variables, retries - 1, allow_not_found) + raise + errors = out.get("errors") + if errors and retries > 0 and any(e.get("type") == "RATE_LIMITED" for e in errors): + time.sleep(60) + return self.post(query, variables, retries - 1, allow_not_found) + if errors: + if allow_not_found and out.get("data") is not None and all(e.get("type") == "NOT_FOUND" for e in errors): + return out["data"] + raise GraphQLError(json.dumps(errors)[:2000]) + return out["data"] + + +def _retry_after(value): + """Return the ``Retry-After`` wait in seconds (0 to 120), or ``None`` if absent or not a number.""" + try: + return max(0, min(int(value), 120)) + except (TypeError, ValueError): + return None + + +def _field_type(data_type): + """Map a GraphQL field data type to ``date``, ``select`` or ``other``.""" + return {"DATE": "date", "SINGLE_SELECT": "select"}.get(data_type, "other") + + +def read_board(client, owner, number): + """Read the fields and the issue items of one project board. + + Parameters + ---------- + client : Client + Client with read access to the board. + owner : str + Org login that owns the board. + number : int + Board number. + + Returns + ------- + tuple + Project id, field definitions by name, items as a list of + ``(issue_id, url, org, item_id, values)``, and the number + of items skipped because they are not issues or cannot be read. + + Raises + ------ + BoardNotFound + If the board or its org does not exist or the token cannot see it. + TruncatedRead + If a board has more fields, or an item more values, than one page. + """ + after, items, fields, pid, skipped = None, [], None, None, 0 + while True: + # A missing board or org comes back as null data with a NOT_FOUND error. + data = client.post(BOARD_QUERY, {"owner": owner, "number": number, "after": after}, allow_not_found=True) + proj = (data.get("organization") or {}).get("projectV2") + if proj is None: + raise BoardNotFound(f"{owner} project {number}") + pid = proj["id"] + if fields is None: + _check_page(proj["fields"], f"fields of board {number}") + fields = { + f["name"]: { + "id": f["id"], + "type": _field_type(f["dataType"]), + "options": {o["name"]: o["id"] for o in f.get("options", [])}, + } + for f in proj["fields"]["nodes"] + if f + } + for node in proj["items"]["nodes"]: + content = (node or {}).get("content") or {} + if "id" not in content: + skipped += 1 # draft issue, pull request, or an issue the token cannot read + continue + _check_page(node["fieldValues"], f"values of an item on board {number}") + values = {} + for v in node["fieldValues"]["nodes"]: + if v and v.get("field"): + values[v["field"]["name"]] = [v.get("date", v.get("name")), v.get("updatedAt")] + org = content["repository"]["owner"]["login"] + items.append((content["id"], content["url"], org, node["id"], values)) + page = proj["items"]["pageInfo"] + if not page["hasNextPage"]: + return pid, fields, items, skipped + after = page["endCursor"] + + +def read_issue_fields(client, org): + """Read the org issue field definitions. + + Parameters + ---------- + client : Client + Client with access to the org. + org : str + Org login. + + Returns + ------- + dict + Field name to definition. + """ + conn = client.post(ISSUE_FIELDS_QUERY, {"org": org})["organization"]["issueFields"] + _check_page(conn, f"issue fields of {org}") + return { + n["name"]: { + "id": n["id"], + "type": "select" if "options" in n else "date", + "options": {o["name"]: o["id"] for o in n.get("options", [])}, + } + for n in conn["nodes"] + if n + } + + +def read_issue_values(client, ids, batch=25): + """Read org issue field values and their change times. + + A field whose latest event is a removal reads as ``[None, removed_at]``, + so a clear has its own time. + + Parameters + ---------- + client : Client + Client of the org that owns the issues. + ids : list of str + Issue node ids. + batch : int + Issues per request. + + Returns + ------- + tuple + Issue id to ``{field name: [value, changed_at]}``, and the number of + issues that could not be read. + """ + out, unread = {}, 0 + for i in range(0, len(ids), batch): + data = client.post(ISSUE_VALUES_QUERY, {"ids": ids[i : i + batch]}, allow_not_found=True) + for node in data["nodes"]: + if not node: + unread += 1 + continue + _check_page(node["issueFieldValues"], "issue field values") + times, removed = {}, set() + for ev in node["timelineItems"]["nodes"]: + name = ((ev or {}).get("issueField") or {}).get("name") + if name and ev["createdAt"] >= times.get(name, ""): + times[name] = ev["createdAt"] + if ev.get("__typename") == "IssueFieldRemovedEvent": + removed.add(name) + else: + removed.discard(name) + values = {name: [None, times[name]] for name in removed} + for v in node["issueFieldValues"]["nodes"]: + if v and v.get("field"): + name = v["field"]["name"] + values[name] = [v.get("value", v.get("name")), times.get(name)] + out[node["id"]] = values + return out, unread + + +def read_all(clients): + """Read every place into one snapshot. + + Parameters + ---------- + clients : dict + Org login to ``Client``. + + Returns + ------- + dict + Snapshot with keys ``fields``, ``projects``, ``items``, + ``skipped_items`` (per board) and ``unread_issues`` (per org). + """ + snap = { + "fields": {}, + "projects": {}, + "items": {}, + "skipped_items": {}, + "unread_issues": {}, + "read_at": datetime.now(UTC).isoformat(timespec="seconds"), + } + for key, board in config.BOARDS.items(): + pid, fields, items, skipped = read_board(clients[board["owner"]], board["owner"], board["number"]) + snap["projects"][key] = pid + snap["fields"][key] = fields + snap["skipped_items"][key] = skipped + for iid, url, org, item_id, values in items: + entry = snap["items"].setdefault(iid, {"url": url, "org": org, "places": {}}) + entry["places"][key] = {"item_id": item_id, "values": values} + for org in config.ISSUE_ORGS: + snap["fields"][f"issue:{org}"] = read_issue_fields(clients[org], org) + ids = sorted(i for i, e in snap["items"].items() if e["org"] == org) + values_by_issue, snap["unread_issues"][org] = read_issue_values(clients[org], ids) + for iid, values in values_by_issue.items(): + snap["items"][iid]["places"]["issue"] = {"values": values} + return snap + + +def write(clients, snap, w): + """Apply one planned write. + + Parameters + ---------- + clients : dict + Org login to ``Client``. + snap : dict + Snapshot the plan was made from. + w : dict + One entry of ``Plan.writes``. + """ + item = snap["items"][w["issue"]] + if w["place"] == "issue": + org = item["org"] + fdef = snap["fields"][f"issue:{org}"][w["field"]] + if w["value"] is None: + q = "mutation($i: ID!, $f: ID!) { deleteIssueFieldValue(input: {issueId: $i, fieldId: $f}) { clientMutationId } }" + clients[org].post(q, {"i": w["issue"], "f": fdef["id"]}) + return + val = {"fieldId": fdef["id"]} + if fdef["type"] == "date": + val["dateValue"] = w["value"] + else: + val["singleSelectOptionId"] = fdef["options"][w["value"]] + q = "mutation($i: ID!, $v: [IssueFieldCreateOrUpdateInput!]!) { setIssueFieldValue(input: {issueId: $i, issueFields: $v}) { clientMutationId } }" + clients[org].post(q, {"i": w["issue"], "v": [val]}) + return + board = config.BOARDS[w["place"]] + fdef = snap["fields"][w["place"]][w["field"]] + args = { + "p": snap["projects"][w["place"]], + "i": item["places"][w["place"]]["item_id"], + "f": fdef["id"], + } + if w["value"] is None: + q = "mutation($p: ID!, $i: ID!, $f: ID!) { clearProjectV2ItemFieldValue(input: {projectId: $p, itemId: $i, fieldId: $f}) { clientMutationId } }" + clients[board["owner"]].post(q, args) + return + if fdef["type"] == "date": + args["v"] = {"date": w["value"]} + else: + args["v"] = {"singleSelectOptionId": fdef["options"][w["value"]]} + q = "mutation($p: ID!, $i: ID!, $f: ID!, $v: ProjectV2FieldValue!) { updateProjectV2ItemFieldValue(input: {projectId: $p, itemId: $i, fieldId: $f, value: $v}) { clientMutationId } }" + clients[board["owner"]].post(q, args) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..10c9fbb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "board-sync" +version = "0.1.0" +description = "Keep dates, priority, workload and status equal across the Interra boards, the PROTEUS board and the org issue fields" +requires-python = ">=3.11" + +[tool.ruff] +line-length = 120 + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] + +[tool.coverage.run] +source = ["board_sync"] +branch = true diff --git a/scripts/store_state.sh b/scripts/store_state.sh new file mode 100755 index 0000000..2d88861 --- /dev/null +++ b/scripts/store_state.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Store the state, snapshot and report of a sync run in the state repository. +# Usage: store_state.sh +# The files of this run replace the stored ones whole; other files stay. +set -u +dir=$1 +run=$2 +# The checkout is absent, or not a repository, when an earlier step failed. +if ! cd "$dir" 2>/dev/null || [ ! -d .git ]; then + echo "no state checkout, nothing stored" + exit 0 +fi +keep=$(mktemp -d) +trap 'rm -rf "$keep"' EXIT +for f in state.json snapshot.json report.txt; do + if [ -f "$f" ]; then cp "$f" "$keep/"; fi +done +for attempt in 1 2 3; do + if git fetch -q origin main && git reset -q --hard FETCH_HEAD; then + cp "$keep"/* . 2>/dev/null || true + for f in state.json snapshot.json report.txt; do + if [ -f "$f" ] && ! git add "$f"; then + echo "state store failed: cannot stage $f" + exit 1 + fi + done + if git diff --cached --quiet; then + echo "state unchanged" + exit 0 + fi + if ! git commit -q -m "Sync run $run"; then + echo "state store failed: cannot commit" + exit 1 + fi + if git push -q origin HEAD:main; then + exit 0 + fi + fi + sleep "${STORE_RETRY_WAIT:-10}" +done +echo "state push failed after 3 attempts" +exit 1 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..d80e0e8 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,71 @@ +"""Shared builders for snapshots in the tests.""" + +import copy + +import pytest + +DATE = {"type": "date", "options": {}} + + +def select(*names): + """Return a select field definition with the given option names.""" + return {"type": "select", "options": {n: f"opt-{n}" for n in names}} + + +def fields(ids=None): + """Return field definitions for all places, with optional id overrides.""" + ids = ids or {} + status = select("In Progress", "Next Up", "TBD", "Done", "Import", "Starter Projects") + board = { + "Start date": dict(DATE), + "Target date": dict(DATE), + "Priority": select("Priority 1: critical", "Priority 2: high", "Priority 3: standard", "Priority 4: low"), + "Workload": select("XL", "L", "M", "S", "XS"), + "Category": select("Science", "Tech"), + "Sub-team": select("PIs", "Co-leads"), + "Release": select("Interra 2027"), + "Status": status, + } + b1 = dict(board) + b1["Priority"] = select("Priority: critical", "Priority: high", "Priority: normal", "Priority: low") + b1["Sub-team"] = select("PIs", "Co-Leads") + b1["Status"] = select("In Progress", "Next Up", "TBD", "Done") + b7 = {k: board[k] for k in ("Start date", "Target date", "Priority", "Workload")} + issue = { + "Start date": dict(DATE), + "Target date": dict(DATE), + "Priority": select("Urgent", "High", "Medium", "Low"), + "Effort": select("High", "Medium", "Low"), + } + out = {"b1": b1, "b2": board, "b7": b7, "issue:Interra-Project": issue, "issue:FormingWorlds": issue} + out = { + k: {n: {**copy.deepcopy(f), "id": ids.get((k, n), f"{k}/{n}")} for n, f in v.items()} for k, v in out.items() + } + return out + + +def item(org="Interra-Project", **places): + """Return one snapshot item; each keyword is a place with ``{field: (value, at)}``.""" + return { + "url": f"https://github.com/{org}/repo/issues/1", + "org": org, + "places": { + p: ({"values": {k: list(v) for k, v in vals.items()}} | ({} if p == "issue" else {"item_id": f"item-{p}"})) + for p, vals in places.items() + }, + } + + +def snap(items, ids=None): + """Return a snapshot with the given items keyed ``I1``, ``I2``, ...""" + return { + "fields": fields(ids), + "projects": {"b1": "P1", "b2": "P2", "b7": "P7"}, + "items": {f"I{n + 1}": it for n, it in enumerate(items)}, + } + + +@pytest.fixture +def mk(): + """Expose the builders to the tests.""" + return type("Builders", (), {"item": staticmethod(item), "snap": staticmethod(snap)}) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..b8f301d --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,180 @@ +"""Tests of the command line: dry run, apply, limits and failures.""" + +import argparse +import io +import json +import time + +import pytest + +import board_sync.__main__ as cli +from board_sync import github + + +def test_make_clients_reads_env(): + clients = cli.make_clients({"BOARD_SYNC_TOKEN_INTERRA_PROJECT": "a", "GH_TOKEN": "g"}) + assert clients["Interra-Project"].token == "a" + assert clients["FormingWorlds"].token == "g" + with pytest.raises(SystemExit): + cli.make_clients({}) + + +def test_load_state(tmp_path): + p = tmp_path / "s.json" + assert cli.load_state(p) == {} + p.write_text(" ") + assert cli.load_state(p) == {} + p.write_text('{"items": {}}') + assert cli.load_state(p) == {"items": {}} + + +@pytest.fixture +def world(mk, monkeypatch): + """Patch the reads to a snapshot with 2 writes due, and record writes.""" + s = mk.snap([mk.item(b1={}, b2={"Category": ("Tech", "t1"), "Start date": ("2026-01-01", "t1")})]) + monkeypatch.setattr(github, "read_all", lambda clients: s) + done = [] + + def fake_write(clients, snap, w): + done.append(w) + snap["items"][w["issue"]]["places"][w["place"]]["values"][w["field"]] = [w["value"], "t9"] + + monkeypatch.setattr(github, "write", fake_write) + return done + + +def args(tmp_path, **kw): + base = { + "state": str(tmp_path / "state.json"), + "snapshot_dir": None, + "apply": False, + "max_changes": 20, + "allow_bulk": False, + "limit": 400, + "pause": 0.0, + "deadline": 600.0, + "started": time.monotonic(), + "summary": False, + "report": None, + } + return argparse.Namespace(**{**base, **kw}) + + +def test_dry_run_writes_nothing(world, tmp_path): + out = io.StringIO() + assert cli.run(args(tmp_path, snapshot_dir=str(tmp_path / "snap")), {}, out) == 0 + assert world == [] + assert not (tmp_path / "state.json").exists() + assert (tmp_path / "snap" / "snapshot.json").exists() + text = out.getvalue() + assert "writes: 2" in text and "plan " in text and "Interra Management / Category: Tech" in text + assert " 1 Interra Management / Category" in text + + +def test_apply_writes_and_saves_state(world, tmp_path): + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True), {}, out) == 0 + assert len(world) == 2 + state = json.loads((tmp_path / "state.json").read_text()) + assert state["items"]["I1"]["category"]["seen"]["b1"] == "Tech" + + +def test_apply_stops_above_limit(world, tmp_path): + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True, max_changes=1), {}, out) == 3 + assert world == [] and "stop: 2 writes" in out.getvalue() + assert cli.run(args(tmp_path, apply=True, max_changes=1, allow_bulk=True), {}, io.StringIO()) == 0 + assert len(world) == 2 + + +def test_failed_write_is_retried_next_run(mk, monkeypatch, tmp_path): + s = mk.snap([mk.item(b1={}, b2={"Category": ("Tech", "t1")})]) + monkeypatch.setattr(github, "read_all", lambda clients: s) + + def refuse(clients, snap, w): + raise github.GraphQLError("no") + + monkeypatch.setattr(github, "write", refuse) + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True), {}, out) == 1 + assert "failed " in out.getvalue() + state = json.loads((tmp_path / "state.json").read_text()) + assert state["items"]["I1"]["category"]["seen"]["b1"] is None + + +def test_report_lists_conflicts_and_skips(mk, monkeypatch, tmp_path): + s = mk.snap( + [ + mk.item( + b1={"Category": ("Tech", "t1")}, b2={"Category": ("Science", "t2"), "Release": ("Interra 2099", "t2")} + ) + ] + ) + s["fields"]["b2"]["Release"]["options"]["Interra 2099"] = "z" + monkeypatch.setattr(github, "read_all", lambda clients: s) + out = io.StringIO() + cli.run(args(tmp_path), {}, out) + assert "conflict " in out.getvalue() and "skipped " in out.getvalue() + + +def test_describe_clear(): + w = {"url": "u", "place": "issue", "field": "Priority", "value": None} + assert cli.describe(w) == "u issue field / Priority: (clear)" + + +def test_main_parses(monkeypatch): + seen = {} + monkeypatch.setattr(cli, "make_clients", lambda env: {"c": 1}) + monkeypatch.setattr(cli, "run", lambda a, c: seen.update(a=a, c=c) or 0) + assert cli.main(["--apply", "--max-changes", "5", "--state", "s.json"], env={}) == 0 + assert seen["a"].apply and seen["a"].max_changes == 5 and seen["c"] == {"c": 1} and not seen["a"].summary + assert cli.main([], env={"GITHUB_ACTIONS": "true"}) == 0 + a = seen["a"] + assert (a.max_changes, a.limit, a.pause, a.summary, a.apply) == (20, 400, 0.0, True, False) + assert a.deadline == 600.0 and a.started > 0 + assert cli.main(["--deadline", "5"], env={}) == 0 and seen["a"].deadline == 5.0 + + +def test_limit_defers_the_rest_to_the_next_run(world, monkeypatch, tmp_path): + pauses = [] + monkeypatch.setattr(cli.time, "sleep", pauses.append) + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True, limit=1, pause=0.5), {}, out) == 0 + assert len(world) == 1 and pauses == [] and "deferred: 1 writes" in out.getvalue() + assert (world[0]["place"], world[0]["field"]) == ("b1", "Start date") + world.clear() + # The deferred place keeps its old record; a second plan writes it again. + assert cli.run(args(tmp_path, apply=True, limit=5, pause=0.5), {}, io.StringIO()) == 0 + assert [(w["place"], w["field"]) for w in world] == [("b1", "Category")] and pauses == [] + assert cli.run(args(tmp_path, apply=True), {}, io.StringIO()) == 0 + assert len(world) == 1 # nothing left to write + + +def test_pause_between_writes(world, monkeypatch, tmp_path): + pauses = [] + monkeypatch.setattr(cli.time, "sleep", pauses.append) + assert cli.run(args(tmp_path, apply=True, pause=0.5), {}, io.StringIO()) == 0 + assert len(world) == 2 and pauses == [0.5] + + +def test_summary_keeps_issue_data_out_of_the_log(world, tmp_path): + out = io.StringIO() + report = tmp_path / "report.txt" + assert cli.run(args(tmp_path, apply=True, summary=True, report=str(report)), {}, out) == 0 + assert "github.com" not in out.getvalue() and "writes: 2" in out.getvalue() + assert "plan https://github.com/Interra-Project/repo/issues/1 Interra Management / Category: Tech" in ( + report.read_text() + ) + + +def test_summary_counts_failures_by_kind(mk, monkeypatch, tmp_path): + s = mk.snap([mk.item(b1={}, b2={"Category": ("Tech", "t1")})]) + monkeypatch.setattr(github, "read_all", lambda clients: s) + + def refuse(clients, snap, w): + raise github.GraphQLError("secret detail") + + monkeypatch.setattr(github, "write", refuse) + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True, summary=True), {}, out) == 1 + assert "failed: 1 (GraphQLError 1)" in out.getvalue() and "secret detail" not in out.getvalue() diff --git a/tests/test_core.py b/tests/test_core.py new file mode 100644 index 0000000..c7d7ac5 --- /dev/null +++ b/tests/test_core.py @@ -0,0 +1,224 @@ +"""Tests of the sync decision.""" + +from board_sync import config, core + + +def writes(result): + """Return the writes as ``{(place, field): value}``.""" + return {(w["place"], w["field"]): w["value"] for w in result.writes} + + +def test_value_maps(): + assert core.to_place("priority", "b1", "Priority 3: standard") == "Priority: normal" + assert core.to_place("priority", "issue", "Priority 1: critical") == "Urgent" + assert core.to_place("start", "b1", "2026-01-01") == "2026-01-01" + assert core.to_place("status", "b1", None) is None + assert core.to_canon("priority", "b1", "Priority: normal") == "Priority 3: standard" + # TBD on board 1 stands for 3 canonical values, so it stays TBD. + assert core.to_canon("status", "b1", "TBD") == "TBD" + assert core.to_canon("subteam", "b1", "Co-Leads") == "Co-leads" + assert core.to_canon("status", "b1", None) is None + + +def test_scope_needs_item_and_field(mk): + s = mk.snap([mk.item(b2={}, issue={})]) + it = s["items"]["I1"] + assert core.scope(s, it, "start") == ["issue", "b2"] + del s["fields"]["b2"]["Start date"] + assert core.scope(s, it, "start") == ["issue"] + it["org"] = "someone" # a user repo has no org issue fields + assert core.scope(s, it, "start") == [] + + +def test_first_run_fills_empty_places(mk): + s = mk.snap([mk.item(b1={}, b2={"Target date": ("2026-12-31", "2026-10-01T00:00:00Z")}, b7={}, issue={})]) + r = core.plan(s, {}) + assert writes(r) == { + ("b1", "Target date"): "2026-12-31", + ("b7", "Target date"): "2026-12-31", + ("issue", "Target date"): "2026-12-31", + } + assert r.conflicts == [] + assert r.state["items"]["I1"]["target"]["c"] == "2026-12-31" + assert r.state["fields"]["b2"]["Target date"] == "b2/Target date" + + +def test_first_run_all_empty_writes_nothing(mk): + r = core.plan(mk.snap([mk.item(b1={}, b2={})]), {}) + assert r.writes == [] + assert r.state["items"]["I1"]["status"] == {"c": None, "t": None, "seen": {"b2": None, "b1": None}, "gone": []} + + +def test_first_run_newest_wins_with_conflict(mk): + s = mk.snap( + [ + mk.item( + b1={"Start date": ("2025-12-27", "2025-12-29T00:00:00Z")}, + b2={"Start date": ("2027-07-01", "2026-10-14T00:00:00Z")}, + ) + ] + ) + r = core.plan(s, {}) + assert writes(r) == {("b1", "Start date"): "2027-07-01"} + assert r.conflicts[0]["values"] == {"b2": "2027-07-01", "b1": "2025-12-27"} + + +def test_tie_goes_by_place_order(mk): + t = "2026-01-01T00:00:00Z" + s = mk.snap([mk.item(b1={"Category": ("Tech", t)}, b2={"Category": ("Science", t)})]) + assert writes(core.plan(s, {})) == {("b1", "Category"): "Science"} + + +def test_issue_time_unknown_loses_to_board(mk): + s = mk.snap( + [mk.item(b2={"Priority": ("Priority 2: high", "2026-01-01T00:00:00Z")}, issue={"Priority": ("Low", None)})] + ) + assert writes(core.plan(s, {})) == {("issue", "Priority"): "High"} + + +def test_lossy_status_agrees_without_conflict(mk): + s = mk.snap( + [ + mk.item( + b1={"Status": ("TBD", "2026-10-13T00:00:00Z")}, + b2={"Status": ("Starter Projects", "2026-01-01T00:00:00Z")}, + ) + ] + ) + r = core.plan(s, {}) + assert r.writes == [] and r.conflicts == [] + assert r.state["items"]["I1"]["status"]["c"] == "Starter Projects" + + +def test_mapped_values_agree(mk): + s = mk.snap( + [ + mk.item( + b1={"Priority": ("Priority: normal", "2026-01-02T00:00:00Z")}, + b2={"Priority": ("Priority 3: standard", "2026-01-01T00:00:00Z")}, + issue={"Priority": ("Medium", None)}, + ) + ] + ) + r = core.plan(s, {}) + assert r.writes == [] and r.conflicts == [] + + +def run_twice(mk, first, second): + """Plan ``first`` from scratch, then plan ``second`` against its state.""" + state = core.plan(mk.snap([first]), {}).state + return core.plan(mk.snap([second]), state) + + +def test_unchanged_second_run_writes_nothing(mk): + it = mk.item(b1={"Start date": ("2026-01-01", "t1")}, b2={"Start date": ("2026-01-01", "t1")}) + assert run_twice(mk, it, it).writes == [] + + +def test_change_at_one_place_propagates(mk): + a = mk.item(b1={"Start date": ("2026-01-01", "t1")}, b2={"Start date": ("2026-01-01", "t1")}) + b = mk.item(b1={"Start date": ("2026-01-01", "t1")}, b2={"Start date": ("2026-02-01", "t2")}) + r = run_twice(mk, a, b) + assert writes(r) == {("b1", "Start date"): "2026-02-01"} + assert r.state["items"]["I1"]["start"]["seen"] == {"b2": "2026-02-01", "b1": "2026-02-01"} + + +def test_clear_propagates(mk): + a = mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t1")}) + b = mk.item(b1={"Category": ("Tech", "t1")}, b2={}) + assert writes(run_twice(mk, a, b)) == {("b1", "Category"): None} + + +def test_two_changes_newest_wins(mk): + a = mk.item(b1={"Category": ("Tech", "2026-01-01")}, b2={"Category": ("Tech", "2026-01-01")}) + b = mk.item(b1={"Category": ("Science", "2026-03-01")}, b2={"Category": ("Management", "2026-02-01")}) + r = run_twice(mk, a, b) + assert writes(r) == {("b2", "Category"): "Science"} + assert r.conflicts[0]["chosen"] == "Science" + + +def test_new_place_empty_receives(mk): + a = mk.item(b2={"Workload": ("M", "t1")}, b1={"Workload": ("M", "t1")}) + b = mk.item(b2={"Workload": ("M", "t1")}, b1={"Workload": ("M", "t1")}, b7={}) + assert writes(run_twice(mk, a, b)) == {("b7", "Workload"): "M"} + + +def test_new_place_with_value_counts_as_change(mk): + a = mk.item(b2={"Workload": ("M", "t1")}, b1={"Workload": ("M", "t1")}) + b = mk.item(b2={"Workload": ("M", "t1")}, b1={"Workload": ("M", "t1")}, b7={"Workload": ("XL", "t9")}) + assert writes(run_twice(mk, a, b)) == {("b2", "Workload"): "XL", ("b1", "Workload"): "XL"} + + +def test_new_place_with_agreeing_value_is_no_change(mk): + a = mk.item(b2={"Workload": ("M", "t1")}, b1={"Workload": ("M", "t1")}) + b = mk.item(b2={"Workload": ("M", "t1")}, b1={"Workload": ("M", "t1")}, b7={"Workload": ("M", "t9")}) + assert run_twice(mk, a, b).writes == [] + + +def test_out_of_scope_places_keep_their_record(mk): + a = mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t1")}) + state = core.plan(mk.snap([a]), {}).state + s = mk.snap([mk.item(b1={"Category": ("Tech", "t1")}, b2={})]) + del s["fields"]["b2"]["Category"] # item still on board 2, field gone + rec = core.plan(s, state).state["items"]["I1"]["category"] + assert rec["seen"] == {"b1": "Tech", "b2": "Tech"} and rec["gone"] == [] + s = mk.snap([mk.item(b1={"Category": ("Tech", "t1")})]) # item left board 2 + rec = core.plan(s, state).state["items"]["I1"]["category"] + assert rec["seen"] == {"b1": "Tech", "b2": "Tech"} and rec["gone"] == ["b2"] + + +def test_recreated_field_does_not_clear(mk): + a = mk.item(b1={"Target date": ("2026-05-01", "t1")}, b2={"Target date": ("2026-05-01", "t1")}) + state = core.plan(mk.snap([a]), {}).state + s = mk.snap([mk.item(b1={"Target date": ("2026-05-01", "t1")}, b2={})], ids={("b2", "Target date"): "NEW"}) + r = core.plan(s, state) + assert writes(r) == {("b2", "Target date"): "2026-05-01"} + + +def test_mass_empty_field_does_not_clear(mk, monkeypatch): + monkeypatch.setattr(config, "RESET_MIN", 2) + full = [mk.item(b1={"Start date": ("2026-05-01", "t")}, b2={"Start date": ("2026-05-01", "t")}) for _ in range(2)] + state = core.plan(mk.snap(full), {}).state + empty = [mk.item(b1={"Start date": ("2026-05-01", "t")}, b2={}) for _ in range(2)] + r = core.plan(mk.snap(empty), state) + assert sorted(writes(r).values()) == ["2026-05-01"] + assert len(r.writes) == 2 + + +def test_missing_option_is_skipped(mk): + s = mk.snap([mk.item(b1={}, b2={"Release": ("Interra 2028+", "t1")})]) + s["fields"]["b2"]["Release"]["options"]["Interra 2028+"] = "x" + r = core.plan(s, {}) + assert r.writes == [] + assert r.skipped == [ + {"issue": "I1", "url": s["items"]["I1"]["url"], "place": "b1", "fkey": "release", "value": "Interra 2028+"} + ] + assert r.state["items"]["I1"]["release"]["seen"]["b1"] is None + + +def test_effort_follows_workload(mk): + s = mk.snap([mk.item(b2={"Workload": ("XL", "t1")}, issue={"Effort": ("Low", None)})]) + assert writes(core.plan(s, {})) == {("issue", "Effort"): "High"} + + +def test_effort_untouched_without_workload_or_match(mk): + s = mk.snap([mk.item(b2={}, issue={"Effort": ("Low", None)})]) + assert core.plan(s, {}).writes == [] + s = mk.snap([mk.item(b2={"Workload": ("S", "t1")}, issue={"Effort": ("Low", None)})]) + assert core.plan(s, {}).writes == [] + s = mk.snap([mk.item(b2={"Workload": ("S", "t1")}, issue={})]) + del s["fields"]["issue:Interra-Project"]["Effort"]["options"]["Low"] + assert core.plan(s, {}).writes == [] + s = mk.snap([mk.item(b2={"Workload": ("S", "t1")})]) # no issue place + assert core.plan(s, {}).writes == [] + + +def test_undo_failed_restores_seen(mk): + s = mk.snap([mk.item(b1={}, b2={"Category": ("Tech", "t1")}, issue={})]) + r = core.plan(s, {}) + failed = [w for w in r.writes if w["place"] == "b1"] + core.undo_failed(r.state, failed + [{"fkey": None}, {"fkey": "start", "issue": "I9", "place": "b1"}]) + assert r.state["items"]["I1"]["category"]["seen"]["b1"] is None + # The next run sees no change at b1 and writes the value again. + r2 = core.plan(s, r.state) + assert writes(r2) == {("b1", "Category"): "Tech"} diff --git a/tests/test_github.py b/tests/test_github.py new file mode 100644 index 0000000..f7d7303 --- /dev/null +++ b/tests/test_github.py @@ -0,0 +1,415 @@ +"""Tests of the GraphQL reads and writes, with a fake transport.""" + +import http.client +import io +import json +import urllib.error + +import pytest + +from board_sync import github + +DONE = {"hasNextPage": False} +MORE = {"hasNextPage": True} + + +class Resp(io.BytesIO): + """File-like HTTP response usable as a context manager.""" + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def opener_for(*bodies): + """Return an opener that answers with ``bodies`` in order and records requests.""" + calls = [] + + def opener(req, timeout=None): + calls.append({**json.loads(req.data), "timeout": timeout, "auth": req.get_header("Authorization")}) + body = bodies[len(calls) - 1] + if isinstance(body, BaseException): + raise body + return Resp(json.dumps(body).encode()) + + opener.calls = calls + return opener + + +def http_error(code, headers=None, body=b""): + return urllib.error.HTTPError(github.API, code, "x", headers or {}, io.BytesIO(body)) + + +@pytest.fixture +def slept(monkeypatch): + out = [] + monkeypatch.setattr(github.time, "sleep", out.append) + return out + + +def test_post_returns_data_and_sends_token(): + op = opener_for({"data": {"ok": 1}}) + assert github.Client("tok", op).post("q", {"a": 1}) == {"ok": 1} + assert op.calls == [{"query": "q", "variables": {"a": 1}, "timeout": 60, "auth": "Bearer tok"}] + + +def test_post_raises_on_errors(): + with pytest.raises(github.GraphQLError): + github.Client("t", opener_for({"errors": [{"message": "bad"}]})).post("q") + + +def test_post_retries_server_errors_and_rate_limits(slept): + assert github.Client("t", opener_for(http_error(502), {"data": {"ok": 2}})).post("q") == {"ok": 2} + op = opener_for(http_error(403, {"Retry-After": "7"}), http_error(429, {"Retry-After": "999"}), {"data": {"ok": 3}}) + assert github.Client("t", op).post("q") == {"ok": 3} + op = opener_for(http_error(429, {"Retry-After": "Wed, 21 Oct 2026 07:28:00 GMT"}), {"data": {"ok": 5}}) + assert github.Client("t", op).post("q") == {"ok": 5} + op = opener_for( + http_error(403, body=b'{"message": "You have exceeded a secondary rate limit"}'), {"data": {"ok": 6}} + ) + assert github.Client("t", op).post("q") == {"ok": 6} + op = opener_for({"errors": [{"type": "RATE_LIMITED"}]}, {"data": {"ok": 7}}) + assert github.Client("t", op).post("q") == {"ok": 7} + assert slept == [5, 7, 120, 60, 60, 60] + + +@pytest.mark.parametrize( + "failure", + [http_error(401), http_error(403, body=b'{"message": "Resource not accessible by integration"}')], +) +def test_post_does_not_retry_refusals(failure, slept): + op = opener_for(failure) + with pytest.raises(urllib.error.HTTPError): + github.Client("t", op).post("q") + assert len(op.calls) == 1 and slept == [] + + +def test_post_gives_up_after_the_retries(slept): + op = opener_for(*[http_error(503) for _ in range(3)]) + with pytest.raises(urllib.error.HTTPError): + github.Client("t", op).post("q") + assert len(op.calls) == 3 + with pytest.raises(github.GraphQLError): + github.Client("t", opener_for(*[{"errors": [{"type": "RATE_LIMITED"}]}] * 3)).post("q") + with pytest.raises(urllib.error.URLError): + github.Client("t", opener_for(*[urllib.error.URLError("down")] * 3)).post("q") + + +@pytest.mark.parametrize( + "failure", [urllib.error.URLError("reset"), TimeoutError(), ConnectionResetError(), http.client.IncompleteRead(b"")] +) +def test_post_retries_network_errors(failure, slept): + assert github.Client("t", opener_for(failure, {"data": {"ok": 4}})).post("q") == {"ok": 4} + + +def test_post_accepts_partial_not_found_only_when_asked(): + body = {"data": {"nodes": [None]}, "errors": [{"type": "NOT_FOUND", "message": "gone"}]} + assert github.Client("t", opener_for(body)).post("q", allow_not_found=True) == {"nodes": [None]} + with pytest.raises(github.GraphQLError): + github.Client("t", opener_for(body)).post("q") + for bad in ( + {"data": {"nodes": [None]}, "errors": [{"type": "FORBIDDEN"}]}, + {"data": {"nodes": [None]}, "errors": [{"type": "NOT_FOUND"}, {"type": "FORBIDDEN"}]}, + {"data": None, "errors": [{"type": "NOT_FOUND"}]}, + ): + with pytest.raises(github.GraphQLError): + github.Client("t", opener_for(bad)).post("q", allow_not_found=True) + + +class Fake: + """Client stand-in that answers by query name and records mutations and partial-read flags.""" + + def __init__(self, pages=None, fields=None, values=None, pid="PID"): + self.pages, self.fields, self.values = list(pages or []), fields or [], values or [] + self.sent, self.flags, self.pid = [], [], pid + + def post(self, query, variables=None, allow_not_found=False): + if "projectV2(number" in query: + self.flags.append(("board", allow_not_found)) + return {"organization": {"projectV2": {**self.pages.pop(0), "id": self.pid}}} + if "issueFields(first" in query: + self.flags.append(("issue_fields", allow_not_found)) + return {"organization": {"issueFields": {"pageInfo": DONE, "nodes": self.fields}}} + if "nodes(ids" in query: + self.flags.append(("issue_values", allow_not_found)) + found = [v for v in self.values if v and v["id"] in variables["ids"]] + return {"nodes": found + [None] * (len(variables["ids"]) - len(found))} + self.sent.append((query.split("{")[1].split("(")[0].strip(), variables)) + return {} + + +def page(items, more=False, field_id="F1"): + return { + "fields": { + "pageInfo": DONE, + "nodes": [ + {"id": field_id, "name": "Start date", "dataType": "DATE"}, + {"id": "F2", "name": "Status", "dataType": "SINGLE_SELECT", "options": [{"id": "o1", "name": "TBD"}]}, + {"id": "F3", "name": "Title", "dataType": "TITLE"}, + {}, + ], + }, + "items": {"pageInfo": {"hasNextPage": more, "endCursor": "C1"}, "nodes": items}, + } + + +BOARD_VALUES = {"Start date": ["2026-01-01", "t1"], "Status": ["TBD", "t2"]} + + +def board_item(iid, org="FormingWorlds"): + return { + "id": f"item-{iid}", + "content": {"id": iid, "url": f"u/{iid}", "repository": {"owner": {"login": org}}}, + "fieldValues": { + "pageInfo": DONE, + "nodes": [ + {"date": "2026-01-01", "updatedAt": "t1", "field": {"name": "Start date"}}, + {"name": "TBD", "updatedAt": "t2", "field": {"name": "Status"}}, + {}, + ], + }, + } + + +def issue_node(iid, values=(), events=()): + return { + "id": iid, + "issueFieldValues": {"pageInfo": DONE, "nodes": list(values)}, + "timelineItems": {"nodes": list(events)}, + } + + +def test_read_board_pages_and_skips_drafts(): + draft = {"id": "d", "content": {}, "fieldValues": {"pageInfo": DONE, "nodes": []}} + fake = Fake(pages=[page([board_item("A"), draft, None], more=True), page([board_item("B")])]) + pid, fields, items, skipped = github.read_board(fake, "FormingWorlds", 7) + assert pid == "PID" and skipped == 2 + assert fields["Status"] == {"id": "F2", "type": "select", "options": {"TBD": "o1"}} + assert fields["Title"]["type"] == "other" + assert [i[0] for i in items] == ["A", "B"] + assert items[0] == ("A", "u/A", "FormingWorlds", "item-A", BOARD_VALUES) + + +def test_read_issue_fields_and_values(): + fake = Fake( + fields=[ + {"id": "D", "name": "Start date"}, + {"id": "S", "name": "Priority", "options": [{"id": "h", "name": "High"}]}, + {}, + ], + values=[ + issue_node( + "A", + values=[ + {"value": "2026-02-01", "field": {"name": "Start date"}}, + {"name": "High", "field": {"name": "Priority"}}, + {}, + ], + events=[ + {"createdAt": "2026-01-01", "issueField": {"name": "Start date"}}, + {"createdAt": "2026-03-01", "issueField": {"name": "Start date"}}, + {"createdAt": "2026-02-01", "issueField": {"name": "Start date"}}, + {"createdAt": "2026-02-01", "issueField": None}, + None, + ], + ) + ], + ) + assert github.read_issue_fields(fake, "o") == { + "Start date": {"id": "D", "type": "date", "options": {}}, + "Priority": {"id": "S", "type": "select", "options": {"High": "h"}}, + } + values, unread = github.read_issue_values(fake, ["A", "GONE"]) + assert values == {"A": {"Start date": ["2026-02-01", "2026-03-01"], "Priority": ["High", None]}} + assert unread == 1 + + +def test_only_the_issue_values_read_accepts_partial_responses(): + interra = Fake(pages=[page([board_item("X", "Interra-Project")]), page([])], values=[issue_node("X")]) + fw = Fake(pages=[page([])]) + github.read_all({"Interra-Project": interra, "FormingWorlds": fw}) + assert set(interra.flags + fw.flags) == {("board", True), ("issue_fields", False), ("issue_values", True)} + + +def test_read_issue_values_batches(): + calls = [] + + class C: + def post(self, q, v, allow_not_found=False): + calls.append(v["ids"]) + return {"nodes": []} + + github.read_issue_values(C(), ["a", "b", "c"], batch=2) + assert calls == [["a", "b"], ["c"]] + + +def test_read_all_maps_each_board_to_its_own_project(): + interra = Fake( + pages=[page([board_item("X", "Interra-Project")]), page([board_item("X", "Interra-Project")])], + fields=[{"id": "D", "name": "Start date"}], + values=[issue_node("X")], + pid="P-interra", + ) + fw = Fake(pages=[page([board_item("Y"), None], field_id="F7")], pid="P-fw") + snap = github.read_all({"Interra-Project": interra, "FormingWorlds": fw}) + assert snap["projects"] == {"b1": "P-interra", "b2": "P-interra", "b7": "P-fw"} + assert snap["fields"]["b7"]["Start date"]["id"] == "F7" and snap["fields"]["b1"]["Start date"]["id"] == "F1" + assert set(snap["items"]["X"]["places"]) == {"b1", "b2", "issue"} + assert snap["items"]["X"]["places"]["b2"] == {"item_id": "item-X", "values": BOARD_VALUES} + assert set(snap["items"]["Y"]["places"]) == {"b7"} # Y is unreadable in the FormingWorlds issue read + assert snap["skipped_items"] == {"b1": 0, "b2": 0, "b7": 1} + assert snap["unread_issues"] == {"Interra-Project": 0, "FormingWorlds": 1} + assert snap["fields"]["issue:Interra-Project"]["Start date"]["id"] == "D" + + +def test_removed_issue_field_reads_as_clear_with_time(): + node = issue_node( + "A", + values=[{"value": "2026-01-01", "field": {"name": "Start date"}}], + events=[ + {"__typename": "IssueFieldAddedEvent", "createdAt": "2026-01-01", "issueField": {"name": "Target date"}}, + {"__typename": "IssueFieldRemovedEvent", "createdAt": "2026-02-01", "issueField": {"name": "Target date"}}, + {"__typename": "IssueFieldRemovedEvent", "createdAt": "2026-01-01", "issueField": {"name": "Start date"}}, + {"__typename": "IssueFieldAddedEvent", "createdAt": "2026-03-01", "issueField": {"name": "Start date"}}, + ], + ) + values, _ = github.read_issue_values(Fake(values=[node]), ["A"]) + assert values == {"A": {"Target date": [None, "2026-02-01"], "Start date": ["2026-01-01", "2026-03-01"]}} + + +@pytest.mark.parametrize("where", ["fields", "values", "issue_fields", "issue_values"]) +def test_truncated_lists_stop_the_read(where): + if where in ("fields", "values"): + p = page([board_item("A")]) + if where == "fields": + p["fields"]["pageInfo"] = MORE + else: + p["items"]["nodes"][0]["fieldValues"]["pageInfo"] = MORE + with pytest.raises(github.TruncatedRead): + github.read_board(Fake(pages=[p]), "o", 1) + elif where == "issue_fields": + + class C: + def post(self, q, v): + return {"organization": {"issueFields": {"pageInfo": MORE, "nodes": []}}} + + with pytest.raises(github.TruncatedRead): + github.read_issue_fields(C(), "o") + else: + node = issue_node("A") + node["issueFieldValues"]["pageInfo"] = MORE + with pytest.raises(github.TruncatedRead): + github.read_issue_values(Fake(values=[node]), ["A"]) + + +def test_missing_page_info_is_an_error(): + p = page([board_item("A")]) + del p["fields"]["pageInfo"] + with pytest.raises(KeyError): + github.read_board(Fake(pages=[p]), "o", 1) + + +SNAP = { + "projects": {"b2": "P2"}, + "fields": { + "b2": { + "Start date": {"id": "FD", "type": "date"}, + "Status": {"id": "FS", "type": "select", "options": {"TBD": "o"}}, + }, + "issue:FormingWorlds": { + "Target date": {"id": "ID", "type": "date"}, + "Priority": {"id": "IP", "type": "select", "options": {"High": "h"}}, + }, + }, + "items": {"I": {"org": "FormingWorlds", "places": {"b2": {"item_id": "IT"}}}}, +} + + +@pytest.mark.parametrize( + ("w", "name", "variables"), + [ + ( + {"place": "b2", "field": "Start date", "value": "2026-01-01"}, + "updateProjectV2ItemFieldValue", + {"p": "P2", "i": "IT", "f": "FD", "v": {"date": "2026-01-01"}}, + ), + ( + {"place": "b2", "field": "Status", "value": "TBD"}, + "updateProjectV2ItemFieldValue", + {"p": "P2", "i": "IT", "f": "FS", "v": {"singleSelectOptionId": "o"}}, + ), + ( + {"place": "b2", "field": "Status", "value": None}, + "clearProjectV2ItemFieldValue", + {"p": "P2", "i": "IT", "f": "FS"}, + ), + ( + {"place": "issue", "field": "Target date", "value": "2026-02-02"}, + "setIssueFieldValue", + {"i": "I", "v": [{"fieldId": "ID", "dateValue": "2026-02-02"}]}, + ), + ( + {"place": "issue", "field": "Priority", "value": "High"}, + "setIssueFieldValue", + {"i": "I", "v": [{"fieldId": "IP", "singleSelectOptionId": "h"}]}, + ), + ({"place": "issue", "field": "Priority", "value": None}, "deleteIssueFieldValue", {"i": "I", "f": "IP"}), + ], +) +def test_write(w, name, variables): + fw, interra = Fake(), Fake() + clients = {"FormingWorlds": fw, "Interra-Project": interra} + github.write(clients, SNAP, {"issue": "I", **w}) + sent = fw.sent if w["place"] == "issue" else interra.sent + assert sent == [(name, variables)] + + +def test_retry_after_edge_values(slept): + op = opener_for(http_error(429, {"Retry-After": "0"}), http_error(429, {"Retry-After": "-5"}), {"data": {"ok": 1}}) + assert github.Client("t", op).post("q") == {"ok": 1} + assert slept == [0, 0] + assert github._retry_after(None) is None and github._retry_after("soon") is None + + +def test_rate_limit_text_in_any_case_and_unreadable_bodies(slept): + op = opener_for(http_error(403, body=b"API Rate Limit exceeded"), {"data": {"ok": 1}}) + assert github.Client("t", op).post("q") == {"ok": 1} + + class Broken(io.BytesIO): + def read(self, *a): + raise http.client.IncompleteRead(b"") + + err = urllib.error.HTTPError(github.API, 502, "x", {}, Broken()) + assert github.Client("t", opener_for(err, {"data": {"ok": 2}})).post("q") == {"ok": 2} + no_body = urllib.error.HTTPError(github.API, 503, "x", {}, None) + assert github.Client("t", opener_for(no_body, {"data": {"ok": 3}})).post("q") == {"ok": 3} + + +def test_missing_board_is_named(): + class C: + def post(self, q, v, allow_not_found=False): + return {"organization": {"projectV2": None}} + + with pytest.raises(github.BoardNotFound, match="Interra-Project project 9"): + github.read_board(C(), "Interra-Project", 9) + + class D: + def post(self, q, v, allow_not_found=False): + return {"organization": None} + + with pytest.raises(github.BoardNotFound): + github.read_board(D(), "Nobody", 1) + + +def test_missing_board_from_the_real_response_shape(): + body = { + "data": {"organization": {"projectV2": None}}, + "errors": [{"type": "NOT_FOUND", "path": ["organization", "projectV2"], "message": "Could not resolve"}], + } + with pytest.raises(github.BoardNotFound, match="Interra-Project project 999"): + github.read_board(github.Client("t", opener_for(body)), "Interra-Project", 999) + no_org = {"data": {"organization": None}, "errors": [{"type": "NOT_FOUND", "path": ["organization"]}]} + with pytest.raises(github.BoardNotFound): + github.read_board(github.Client("t", opener_for(no_org)), "no-such-org", 1) diff --git a/tests/test_store_state.py b/tests/test_store_state.py new file mode 100644 index 0000000..c56fcc9 --- /dev/null +++ b/tests/test_store_state.py @@ -0,0 +1,133 @@ +"""Tests of the script that stores a run's files in the state repository, against local git repositories.""" + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "store_state.sh" +pytestmark = pytest.mark.skipif(not (shutil.which("git") and shutil.which("bash")), reason="needs git and bash") + +IDENTITY = { + "GIT_AUTHOR_NAME": "board-sync", + "GIT_AUTHOR_EMAIL": "board-sync@users.noreply.github.com", + "GIT_COMMITTER_NAME": "board-sync", + "GIT_COMMITTER_EMAIL": "board-sync@users.noreply.github.com", +} + + +def git(cwd, *args, env=None): + return subprocess.run(["git", *args], cwd=cwd, env=env, check=True, capture_output=True, text=True).stdout + + +@pytest.fixture +def repos(tmp_path): + """A bare remote with one README commit, a run checkout, and a clean environment without a git identity.""" + home = tmp_path / "home" + home.mkdir() + env = {"PATH": os.environ["PATH"], "HOME": str(home), "TMPDIR": str(tmp_path), "STORE_RETRY_WAIT": "0", **IDENTITY} + remote = tmp_path / "remote.git" + git(tmp_path, "init", "-q", "--bare", "-b", "main", str(remote), env=env) + seed = tmp_path / "seed" + git(tmp_path, "clone", "-q", str(remote), str(seed), env=env) + (seed / "README.md").write_text("readme\n") + git(seed, "add", ".", env=env) + git(seed, "commit", "-q", "-m", "init", env=env) + git(seed, "push", "-q", "origin", "HEAD:main", env=env) + run = tmp_path / "state" + git(tmp_path, "clone", "-q", "-b", "main", str(remote), str(run), env=env) + return {"tmp": tmp_path, "env": env, "remote": remote, "seed": seed, "run": run} + + +def store(r, run_id="1"): + return subprocess.run( + ["bash", str(SCRIPT), str(r["run"]), run_id], env=r["env"], capture_output=True, text=True, check=False + ) + + +def remote_files(r): + check = r["tmp"] / f"check{len(list(r['tmp'].glob('check*')))}" + git(r["tmp"], "clone", "-q", "-b", "main", str(r["remote"]), str(check), env=r["env"]) + return {p.name: p.read_text() for p in check.iterdir() if p.is_file()}, git( + check, "log", "--format=%s", env=r["env"] + ) + + +def test_first_run_without_a_state_file_stores_snapshot_and_report(repos): + (repos["run"] / "snapshot.json").write_text("{}\n") + (repos["run"] / "report.txt").write_text("stop: 1688 writes\n") + assert store(repos).returncode == 0 + files, log = remote_files(repos) + assert files["report.txt"] == "stop: 1688 writes\n" and "snapshot.json" in files and "state.json" not in files + assert log.splitlines()[0] == "Sync run 1" + + +def test_run_files_replace_a_changed_remote_whole_and_keep_other_files(repos): + (repos["seed"] / "state.json").write_text('{"v": "human"}\n') + (repos["seed"] / "README.md").write_text("readme 2\n") + git(repos["seed"], "add", ".", env=repos["env"]) + git(repos["seed"], "commit", "-q", "-m", "human", env=repos["env"]) + git(repos["seed"], "push", "-q", "origin", "HEAD:main", env=repos["env"]) + (repos["run"] / "state.json").write_text('{"v": "run"}\n') + (repos["run"] / "state.json.tmp").write_text("partial") + assert store(repos, "7").returncode == 0 + files, log = remote_files(repos) + assert files["state.json"] == '{"v": "run"}\n' and files["README.md"] == "readme 2\n" + assert "state.json.tmp" not in files and log.splitlines()[:2] == ["Sync run 7", "human"] + + +def test_nothing_changed_makes_no_commit(repos): + (repos["run"] / "report.txt").write_text("issues: 1\n") + assert store(repos).returncode == 0 + second = store(repos, "2") + assert second.returncode == 0 and "state unchanged" in second.stdout + assert len(remote_files(repos)[1].splitlines()) == 2 + + +def test_missing_checkout_is_not_an_error(repos): + result = subprocess.run(["bash", str(SCRIPT), str(repos["tmp"] / "absent"), "1"], env=repos["env"], check=False) + assert result.returncode == 0 + + +def test_push_failure_is_reported(repos): + (repos["run"] / "report.txt").write_text("x\n") + git(repos["run"], "remote", "set-url", "origin", str(repos["tmp"] / "nowhere.git"), env=repos["env"]) + result = store(repos) + assert result.returncode == 1 and "state push failed after 3 attempts" in result.stdout + + +def test_rejected_push_is_tried_again(repos): + hook = repos["remote"] / "hooks" / "pre-receive" + count = repos["tmp"] / "pushes" + hook.write_text( + f'#!/bin/sh\nn=$(cat "{count}" 2>/dev/null || echo 0)\necho $((n + 1)) > "{count}"\n[ "$n" -ge 1 ]\n' + ) + hook.chmod(0o755) + (repos["run"] / "report.txt").write_text("x\n") + result = store(repos) + assert result.returncode == 0 and count.read_text().strip() == "2" + files, log = remote_files(repos) + assert files["report.txt"] == "x\n" and log.splitlines()[0] == "Sync run 1" + + +def test_failed_commit_is_reported(repos): + hook = repos["run"] / ".git" / "hooks" / "pre-commit" + hook.write_text("#!/bin/sh\nexit 1\n") + hook.chmod(0o755) + (repos["run"] / "report.txt").write_text("x\n") + result = store(repos) + assert result.returncode == 1 and "cannot commit" in result.stdout + assert "report.txt" not in remote_files(repos)[0] + + +def test_a_folder_that_is_not_a_repository_is_left_alone(repos): + plain = repos["tmp"] / "plain" + plain.mkdir() + (plain / "report.txt").write_text("x\n") + result = subprocess.run( + ["bash", str(SCRIPT), str(plain), "1"], env=repos["env"], capture_output=True, text=True, check=False + ) + assert result.returncode == 0 and "no state checkout" in result.stdout + assert (plain / "report.txt").read_text() == "x\n" diff --git a/tests/test_sync_rules.py b/tests/test_sync_rules.py new file mode 100644 index 0000000..80a2344 --- /dev/null +++ b/tests/test_sync_rules.py @@ -0,0 +1,779 @@ +"""Regression tests for faults in the sync rules.""" + +import io +import json + +import pytest + +import board_sync.__main__ as cli +from board_sync import config, core, github +from tests.test_cli import args + + +def writes(result): + return {(w["place"], w["field"]): w["value"] for w in result.writes} + + +def test_bulk_drain_continues_on_scheduled_runs(mk, monkeypatch, tmp_path): + s = mk.snap( + [ + mk.item( + b1={}, + b2={ + "Category": ("Tech", "t1"), + "Start date": ("2026-01-01", "t1"), + "Target date": ("2026-02-01", "t1"), + }, + ) + ] + ) + monkeypatch.setattr(github, "read_all", lambda clients: s) + + done = [] + + def fake_write(clients, snap, w): + done.append(w["field"]) + snap["items"][w["issue"]]["places"][w["place"]]["values"][w["field"]] = [w["value"], "t9"] + + monkeypatch.setattr(github, "write", fake_write) + assert cli.run(args(tmp_path, apply=True, max_changes=1, limit=1, allow_bulk=True), {}, io.StringIO()) == 0 + # Scheduled runs (no allow_bulk) go on with the approved remainder until it is written. + assert cli.run(args(tmp_path, apply=True, max_changes=1, limit=1), {}, io.StringIO()) == 0 + assert cli.run(args(tmp_path, apply=True, max_changes=1, limit=1), {}, io.StringIO()) == 0 + assert sorted(done) == ["Category", "Start date", "Target date"] + + +def test_readded_item_receives_and_does_not_clear(mk): + a = mk.item(b2={"Priority": ("Priority 2: high", "t1")}, b7={"Priority": ("Priority 2: high", "t1")}) + state = core.plan(mk.snap([a]), {}).state + gone = mk.item(b2={"Priority": ("Priority 2: high", "t1")}) + state = core.plan(mk.snap([gone]), state).state + back = mk.item(b2={"Priority": ("Priority 2: high", "t1")}, b7={}) + assert writes(core.plan(mk.snap([back]), state)) == {("b7", "Priority"): "Priority 2: high"} + + +def test_field_missing_for_a_run_keeps_its_id(mk): + a = mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t1")}) + state = core.plan(mk.snap([a]), {}).state + s = mk.snap([mk.item(b1={"Category": ("Tech", "t1")}, b2={})]) + del s["fields"]["b2"]["Category"] + state = core.plan(s, state).state + s = mk.snap([mk.item(b1={"Category": ("Tech", "t1")}, b2={})], ids={("b2", "Category"): "NEW"}) + assert writes(core.plan(s, state)) == {("b2", "Category"): "Tech"} + + +def test_issue_field_clear_uses_its_removal_time(mk): + a = mk.item(b2={"Target date": ("2026-05-01", "2026-01-01")}, issue={"Target date": ("2026-05-01", None)}) + state = core.plan(mk.snap([a]), {}).state + b = mk.item(b2={"Target date": ("2026-06-01", "2026-02-01")}, issue={"Target date": (None, "2026-03-01")}) + assert writes(core.plan(mk.snap([b]), state)) == {("b2", "Target date"): None} + + +def test_board_clear_loses_to_a_value_set_elsewhere(mk): + a = mk.item(b1={"Category": ("Tech", "2026-01-01")}, b2={"Category": ("Tech", "2026-01-01")}) + state = core.plan(mk.snap([a]), {}).state + b = mk.item(b1={"Category": ("Science", "2026-02-01")}, b2={}) + r = core.plan(mk.snap([b]), state) + assert writes(r) == {("b2", "Category"): "Science"} and r.conflicts + alone = mk.item(b1={"Category": ("Tech", "2026-01-01")}, b2={}) + assert writes(core.plan(mk.snap([alone]), state)) == {("b1", "Category"): None} + + +def test_unreadable_place_returning_with_its_old_value_does_not_undo_a_newer_value(mk): + both = mk.item(b2={"Start date": ("2026-05-01", "t1")}, issue={"Start date": ("2026-05-01", "t1")}, b1={}) + s1 = mk.snap([both]) + state = core.plan(s1, {}).state + apply_to(s1, core.plan(s1, {})) + state = core.plan(s1, state).state + # Run 2: the issue is unreadable, and board 1 gets a new date. + s2 = mk.snap([mk.item(b2={"Start date": ("2026-05-01", "t1")}, b1={"Start date": ("2026-06-01", "t2")})]) + r2 = core.plan(s2, state) + assert writes(r2) == {("b2", "Start date"): "2026-06-01"} + apply_to(s2, r2) + # Run 3: the issue returns with its old date; the newer board date must win. + s3 = mk.snap( + [ + mk.item( + b2={"Start date": ("2026-06-01", "t3")}, + b1={"Start date": ("2026-06-01", "t2")}, + issue={"Start date": ("2026-05-01", "t1")}, + ) + ] + ) + assert writes(core.plan(s3, r2.state)) == {("issue", "Start date"): "2026-06-01"} + + +def test_issue_absent_from_every_board_keeps_its_records(mk): + a = mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t1")}) + state = core.plan(mk.snap([a]), {}).state + state = core.plan(mk.snap([]), state).state + assert state["items"]["I1"]["category"]["gone"] == ["b1", "b2"] + back = mk.item(b1={"Category": ("Tech", "t1")}, b2={}) + assert writes(core.plan(mk.snap([back]), state)) == {("b2", "Category"): "Tech"} + + +def test_field_on_no_place_keeps_its_record(mk): + a = mk.item(b1={"Status": ("In Progress", "t1")}, b2={"Status": ("In Progress", "t1")}, b7={}) + state = core.plan(mk.snap([a]), {}).state + s = mk.snap([mk.item(b7={})]) # only on board 7, which has no Status + state = core.plan(s, state).state + assert state["items"]["I1"]["status"]["c"] == "In Progress" + back = mk.item(b1={}, b2={"Status": ("In Progress", "t1")}, b7={}) + assert writes(core.plan(mk.snap([back]), state)) == {("b1", "Status"): "In Progress"} + + +def test_missing_fields_are_listed(mk): + s = mk.snap([]) + del s["fields"]["b7"]["Priority"] + del s["fields"]["issue:FormingWorlds"]["Target date"] + assert core.missing_fields(s) == [("b7", "Priority"), ("issue:FormingWorlds", "Target date")] + + +def test_reset_min_boundary(mk): + from board_sync import config + + n = config.RESET_MIN + for count, clears in ((n - 1, True), (n, False)): + full = [ + mk.item(b1={"Release": ("Interra 2027", "t")}, b2={"Release": ("Interra 2027", "t")}) for _ in range(count) + ] + state = core.plan(mk.snap(full), {}).state + empty = [mk.item(b1={"Release": ("Interra 2027", "t")}, b2={}) for _ in range(count)] + values = {w["value"] for w in core.plan(mk.snap(empty), state).writes} + assert values == ({None} if clears else {"Interra 2027"}) + + +def apply_to(snap, result): + """Apply planned writes to a snapshot as the API would (a clear removes the value).""" + for w in result.writes: + vals = snap["items"][w["issue"]]["places"][w["place"]]["values"] + if w["value"] is None: + vals.pop(w["field"], None) + else: + vals[w["field"]] = [w["value"], "t-new"] + + +def test_plans_converge(mk): + cases = [ + mk.item(b1={"Status": ("TBD", "t1")}, b2={"Status": ("Import", "t2")}), + mk.item(b1={"Priority": ("Priority: normal", "t3")}, b2={}, b7={}, issue={"Priority": ("High", "t1")}), + mk.item(b1={"Sub-team": ("Co-Leads", "t1")}, b2={}), + mk.item(b2={"Workload": ("XS", "t1")}, b7={}, issue={"Effort": ("High", None)}), + mk.item(b1={"Start date": ("2026-01-01", "t1")}, b2={"Start date": ("2026-02-01", "t2")}, b7={}, issue={}), + ] + s = mk.snap(cases) + r = core.plan(s, {}) + assert r.writes + apply_to(s, r) + r2 = core.plan(s, r.state) + assert r2.writes == [] and r2.conflicts == [] + + +def test_read_failure_keeps_details_out_of_the_log(monkeypatch, tmp_path): + def boom(clients): + raise github.GraphQLError('[{"message": "Could not resolve node PVTI_secret"}]') + + monkeypatch.setattr(github, "read_all", boom) + out = io.StringIO() + report = tmp_path / "r.txt" + assert cli.run(args(tmp_path, summary=True, report=str(report)), {}, out) == 2 + assert out.getvalue().strip() == "read failed: GraphQLError" + assert "PVTI_secret" in report.read_text() + + +def test_summary_hides_conflicts_and_skips(mk, monkeypatch, tmp_path): + s = mk.snap( + [ + mk.item( + b1={"Category": ("Tech", "t1")}, b2={"Category": ("Science", "t2"), "Release": ("Interra 2099", "t2")} + ) + ] + ) + s["fields"]["b2"]["Release"]["options"]["Interra 2099"] = "z" + monkeypatch.setattr(github, "read_all", lambda clients: s) + out = io.StringIO() + cli.run(args(tmp_path, summary=True), {}, out) + text = out.getvalue() + assert "conflicts: 1" in text and "skipped: 1" in text + assert "github.com" not in text and "Science" not in text and "Interra 2099" not in text + + +def test_crash_during_writes_saves_state(mk, monkeypatch, tmp_path): + s = mk.snap([mk.item(b1={}, b2={"Category": ("Tech", "t1"), "Start date": ("2026-01-01", "t1")})]) + monkeypatch.setattr(github, "read_all", lambda clients: s) + calls = [] + + def write(clients, snap, w): + calls.append(w) + if len(calls) == 2: + raise KeyboardInterrupt + + monkeypatch.setattr(github, "write", write) + try: + cli.run(args(tmp_path, apply=True), {}, io.StringIO()) + except KeyboardInterrupt: + pass + state = cli.load_state(tmp_path / "state.json") + seen = {f: state["items"]["I1"][f]["seen"]["b1"] for f in ("start", "category")} + assert seen == {"start": "2026-01-01", "category": None} # first write kept, second left for later + assert not (tmp_path / "state.json.tmp").exists() + + +def test_stop_after_failures_in_a_row(mk, monkeypatch, tmp_path): + items = [mk.item(b1={}, b2={"Category": ("Tech", "t1")}) for _ in range(cli.STOP_AFTER_FAILURES + 3)] + monkeypatch.setattr(github, "read_all", lambda clients: mk.snap(items)) + calls = [] + + def refuse(clients, snap, w): + calls.append(w) + raise github.GraphQLError("x") + + monkeypatch.setattr(github, "write", refuse) + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True, max_changes=99), {}, out) == 1 + assert len(calls) == cli.STOP_AFTER_FAILURES + assert "stopped after 5 failed writes in a row" in out.getvalue() and "deferred: 3 writes" in out.getvalue() + + +def test_damaged_state_stops_the_run(mk, monkeypatch, tmp_path): + p = tmp_path / "state.json" + for text in ('{"items": ', "[]"): + p.write_text(text) + with pytest.raises(cli.StateError): + cli.load_state(p) + monkeypatch.setattr(github, "read_all", lambda clients: mk.snap([])) + out = io.StringIO() + assert cli.run(args(tmp_path, summary=True), {}, out) == 2 + assert out.getvalue().strip() == "run failed: StateError" + + +def test_bulk_mark_clears_when_drained(mk, monkeypatch, tmp_path): + s = mk.snap([mk.item(b1={}, b2={"Category": ("Tech", "t1"), "Start date": ("2026-01-01", "t1")})]) + monkeypatch.setattr(github, "read_all", lambda clients: s) + monkeypatch.setattr(github, "write", lambda c, snap, w: apply_to(snap, type("R", (), {"writes": [w]}))) + cli.run(args(tmp_path, apply=True, max_changes=0, limit=1, allow_bulk=True), {}, io.StringIO()) + assert len(cli.load_state(tmp_path / "state.json")["bulk_keys"]) == 1 + cli.run(args(tmp_path, apply=True, max_changes=0, limit=1), {}, io.StringIO()) + assert cli.load_state(tmp_path / "state.json")["bulk_keys"] == [] + + +def test_plain_deferral_does_not_approve_a_bulk(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 3)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + monkeypatch.setattr(github, "write", lambda c, snap, w: apply_to(snap, type("R", (), {"writes": [w]}))) + assert cli.run(args(tmp_path, apply=True, limit=1), {}, io.StringIO()) == 0 + assert cli.load_state(tmp_path / "state.json")["bulk_keys"] == [] + + +def many(mk, n, field="Category", value="Tech"): + return [mk.item(b1={}, b2={field: (value, "t1")}) for _ in range(n)] + + +def test_bulk_approval_does_not_cover_a_new_mass_change(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 6)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + monkeypatch.setattr(github, "write", lambda c, snap, w: apply_to(snap, type("R", (), {"writes": [w]}))) + cli.run(args(tmp_path, apply=True, max_changes=1, limit=2, allow_bulk=True), {}, io.StringIO()) + assert len(cli.load_state(tmp_path / "state.json")["bulk_keys"]) == 4 + # A new change of the same size as the remainder is not covered by the approval. + for n, it in enumerate(many(mk, 4, "Release", "Interra 2027")): + s["items"][f"N{n}"] = it + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True, max_changes=1, limit=2), {}, out) == 3 + assert "stop: 8 writes" in out.getvalue() + + +def test_bulk_approval_ends_after_idle_runs(mk, monkeypatch, tmp_path): + monkeypatch.setattr(cli, "PARK_AFTER", 99) # no parking, so only the idle rule can end the approval + s = mk.snap(many(mk, 4)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + + def refuse(clients, snap, w): + raise github.GraphQLError("x") + + monkeypatch.setattr(github, "write", refuse) + cli.run(args(tmp_path, apply=True, max_changes=1, allow_bulk=True), {}, io.StringIO()) + for idle in range(1, cli.BULK_IDLE_RUNS): + state = cli.load_state(tmp_path / "state.json") + assert state["bulk_idle"] == idle and len(state["bulk_keys"]) == 4 + # The approval stays while runs fail, so the failures can reach parking. + assert cli.run(args(tmp_path, apply=True, max_changes=1), {}, io.StringIO()) == 1 + assert cli.load_state(tmp_path / "state.json")["bulk_keys"] == [] + # Writes that failed before are retried without approval; a new mass change still stops. + assert cli.run(args(tmp_path, apply=True, max_changes=1), {}, io.StringIO()) == 1 + for n, it in enumerate(many(mk, 3, "Release", "Interra 2027")): + s["items"][f"N{n}"] = it + assert cli.run(args(tmp_path, apply=True, max_changes=1), {}, io.StringIO()) == 3 + + +def test_persistent_failures_are_parked_and_do_not_block(mk, monkeypatch, tmp_path): + bad = [mk.item(b1={}, b2={"Category": ("Tech", "t1")}) for _ in range(cli.STOP_AFTER_FAILURES)] + good = [mk.item(b1={}, b2={"Release": ("Interra 2027", "t1")}) for _ in range(3)] + s = mk.snap(bad + good) # the failing writes sort first (I1 .. I5) + monkeypatch.setattr(github, "read_all", lambda clients: s) + written = [] + + def write(clients, snap, w): + if w["fkey"] == "category": + raise github.GraphQLError("locked") + written.append(w["issue"]) + apply_to(snap, type("R", (), {"writes": [w]})) + + monkeypatch.setattr(github, "write", write) + cli.run(args(tmp_path, apply=True, max_changes=99), {}, io.StringIO()) + assert written == [] # run 1: the 5 failures at the head stop the run + cli.run(args(tmp_path, apply=True, max_changes=99), {}, io.StringIO()) + assert sorted(written) == ["I6", "I7", "I8"] # run 2: writes that failed before go last + for _ in range(cli.PARK_AFTER - 2): + cli.run(args(tmp_path, apply=True, max_changes=99), {}, io.StringIO()) + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True, max_changes=99, report=str(tmp_path / "r.txt")), {}, out) == 4 + assert "parked: 5 writes" in out.getvalue() and "parked https://" in (tmp_path / "r.txt").read_text() + state = cli.load_state(tmp_path / "state.json") + assert {r["n"] for r in state["failures"].values()} == {cli.PARK_AFTER} # parked counts are kept + assert {state["items"][f"I{n}"]["category"]["seen"]["b1"] for n in range(1, 6)} == {None} # not written + + +def test_failure_count_resets_on_success(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 1)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + outcome = ["fail"] + + def write(clients, snap, w): + if outcome[0] == "fail": + raise github.GraphQLError("x") + apply_to(snap, type("R", (), {"writes": [w]})) + + monkeypatch.setattr(github, "write", write) + cli.run(args(tmp_path, apply=True), {}, io.StringIO()) + assert [r["n"] for r in cli.load_state(tmp_path / "state.json")["failures"].values()] == [1] + outcome[0] = "ok" + cli.run(args(tmp_path, apply=True), {}, io.StringIO()) + assert cli.load_state(tmp_path / "state.json")["failures"] == {} + + +def test_parked_write_is_tried_again_after_a_day(mk, monkeypatch, tmp_path): + from datetime import UTC, datetime, timedelta + + s = mk.snap(many(mk, 1)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + key = cli.write_key(core.plan(s, {}).writes[0]) + for age, tried in ((timedelta(hours=2), 0), (timedelta(days=2), 1)): + at = (datetime.now(UTC) - age).isoformat(timespec="seconds") + (tmp_path / "state.json").write_text(json.dumps({"failures": {key: {"n": cli.PARK_AFTER, "at": at}}})) + calls = [] + monkeypatch.setattr(github, "write", lambda c, snap, w, calls=calls: calls.append(w)) + assert cli.run(args(tmp_path, apply=True), {}, io.StringIO()) == (4 if tried == 0 else 0) + assert len(calls) == tried + assert not cli.is_parked({"n": cli.PARK_AFTER - 1, "at": datetime.now(UTC).isoformat()}, datetime.now(UTC)) + assert not cli.is_parked(None, datetime.now(UTC)) + + +def test_failures_in_a_row_reset_after_a_success(mk, monkeypatch, tmp_path): + n = 2 * cli.STOP_AFTER_FAILURES + s = mk.snap(many(mk, n)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + calls = [] + + def write(clients, snap, w): + calls.append(w) + if len(calls) % cli.STOP_AFTER_FAILURES: + raise github.GraphQLError("x") + + monkeypatch.setattr(github, "write", write) + cli.run(args(tmp_path, apply=True, max_changes=99), {}, io.StringIO()) + assert len(calls) == n + + +@pytest.mark.parametrize("kind", [ValueError, TypeError, RuntimeError]) +def test_any_write_error_counts_as_a_failed_write(kind, mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 3)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + calls = [] + + def write(clients, snap, w): + calls.append(w) + raise kind("x") + + monkeypatch.setattr(github, "write", write) + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True), {}, out) == 1 + assert len(calls) == 3 and f"failed: 3 ({kind.__name__} 3)" in out.getvalue() + + +def test_state_save_keeps_the_old_file_when_writing_fails(tmp_path, monkeypatch): + p = tmp_path / "state.json" + cli.save_state(p, {"items": {"old": {}}}) + + def boom(*a, **k): + raise OSError("disk full") + + monkeypatch.setattr(cli.os, "replace", boom) + with pytest.raises(OSError): + cli.save_state(p, {"items": {"new": {}}}) + assert cli.load_state(p) == {"items": {"old": {}}} + + +def test_run_failure_after_the_read_shows_the_type_only(mk, monkeypatch, tmp_path): + monkeypatch.setattr(github, "read_all", lambda clients: mk.snap([])) + + def boom(snap, state): + raise KeyError("PVTI_secret") + + monkeypatch.setattr(core, "plan", boom) + out = io.StringIO() + assert cli.run(args(tmp_path, summary=True, report=str(tmp_path / "r.txt")), {}, out) == 2 + assert out.getvalue().strip() == "run failed: KeyError" + assert "PVTI_secret" in (tmp_path / "r.txt").read_text() + + +def test_sigterm_becomes_an_exit(): + import signal + + with pytest.raises(SystemExit) as exc: + cli._terminate(signal.SIGTERM, None) + assert exc.value.code == 128 + signal.SIGTERM + + +def test_warnings_name_missing_fields_and_unread_items(mk, monkeypatch, tmp_path): + s = mk.snap([]) + del s["fields"]["b7"]["Workload"] + s["skipped_items"] = {"b2": 3, "b1": 0} + s["unread_issues"] = {"FormingWorlds": 2, "Interra-Project": 0} + monkeypatch.setattr(github, "read_all", lambda clients: s) + out = io.StringIO() + cli.run(args(tmp_path, summary=True), {}, out) + assert "warning: b7 has no field 'Workload'" in out.getvalue() + assert "note: Interra Collaboration: 3 items are not readable issues" in out.getvalue() + assert "warning: 2 FormingWorlds issues could not be read" in out.getvalue() + assert "Interra-Project issues" not in out.getvalue() + + +def test_resets_are_named_in_the_summary(mk, monkeypatch, tmp_path): + a = mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t1")}) + state = core.plan(mk.snap([a]), {}).state + (tmp_path / "state.json").write_text(json.dumps(state)) + s = mk.snap([mk.item(b1={"Category": ("Tech", "t1")}, b2={})], ids={("b2", "Category"): "NEW"}) + monkeypatch.setattr(github, "read_all", lambda clients: s) + out = io.StringIO() + cli.run(args(tmp_path, summary=True), {}, out) + assert "reset: b2 / category is treated as new" in out.getvalue() + + +def three_runs_with_issue_gone(mk, back_value, back_time): + """Agreed 05-01; the issue goes; board 1 changes to 06-01 at t2; the issue returns.""" + a = mk.item(b1={"Start date": ("2026-05-01", "t1")}, issue={"Start date": ("2026-05-01", "t1")}) + state = core.plan(mk.snap([a]), {}).state + s2 = mk.snap([mk.item(b1={"Start date": ("2026-06-01", "t2")})]) + state = core.plan(s2, state).state + back = mk.item(b1={"Start date": ("2026-06-01", "t2")}, issue={"Start date": (back_value, back_time)}) + return core.plan(mk.snap([back]), state) + + +def test_returning_place_with_a_newer_value_wins(mk): + r = three_runs_with_issue_gone(mk, "2026-07-01", "t3") + assert writes(r) == {("b1", "Start date"): "2026-07-01"} + + +def test_returning_place_with_an_older_value_receives(mk): + r = three_runs_with_issue_gone(mk, "2026-07-01", "t0") + assert writes(r) == {("issue", "Start date"): "2026-06-01"} + + +def test_returning_place_on_a_reset_field_agrees_without_a_change(mk): + a = mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t1")}) + state = core.plan(mk.snap([a]), {}).state + state = core.plan(mk.snap([mk.item(b1={"Category": ("Tech", "t1")})]), state).state + s = mk.snap( + [mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t9")})], ids={("b2", "Category"): "NEW"} + ) + r = core.plan(s, state) + assert r.writes == [] and r.conflicts == [] + + +def test_place_stays_gone_until_it_has_the_field_again(mk): + a = mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t1")}) + state = core.plan(mk.snap([a]), {}).state + state = core.plan(mk.snap([mk.item(b1={"Category": ("Tech", "t1")})]), state).state + s = mk.snap([mk.item(b1={"Category": ("Tech", "t1")}, b2={})]) + del s["fields"]["b2"]["Category"] # back on board 2, but the field is missing this run + state = core.plan(s, state).state + assert state["items"]["I1"]["category"]["gone"] == ["b2"] + back = mk.item(b1={"Category": ("Tech", "t1")}, b2={}) # field back, empty: receive, not clear + assert writes(core.plan(mk.snap([back]), state)) == {("b2", "Category"): "Tech"} + + +def test_issue_gone_for_long_is_dropped(mk): + a = mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t1")}) + state = core.plan(mk.snap([a]), {}).state + s = mk.snap([]) + s["read_at"] = "2026-01-01T00:00:00+00:00" + state = core.plan(s, state).state + assert state["items"]["I1"]["category"]["gone_since"] == "2026-01-01T00:00:00+00:00" + s["read_at"] = "2026-04-01T00:00:00+00:00" # exactly 90 days + state = core.plan(s, state).state + assert "I1" in state["items"] + s["read_at"] = "2026-04-01T00:00:01+00:00" + assert "I1" not in core.plan(s, state).state["items"] + + +def test_deadline_stops_the_writes(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 4)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + a = args(tmp_path, apply=True, deadline=10.0, started=1000.0) + clock = iter([1000.0, 1009.0, 1011.0, 1012.0]) # checked before each write + monkeypatch.setattr(cli.time, "monotonic", lambda: next(clock)) + calls = [] + monkeypatch.setattr(github, "write", lambda c, snap, w: calls.append(w)) + out = io.StringIO() + assert cli.run(a, {}, out) == 0 + assert len(calls) == 2 and "stopped at the time limit" in out.getvalue() and "deferred: 2 writes" in out.getvalue() + + +def test_report_shows_the_plan_while_writes_run(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 2)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + report = tmp_path / "r.txt" + seen = [] + monkeypatch.setattr(github, "write", lambda c, snap, w: seen.append(report.read_text())) + cli.run(args(tmp_path, apply=True, report=str(report)), {}, io.StringIO()) + assert "plan https://" in seen[0] + + +def test_sigterm_handler_is_installed_and_restored(monkeypatch): + import signal + + during = [] + monkeypatch.setattr(cli, "make_clients", lambda env: {}) + monkeypatch.setattr(cli, "run", lambda a, c: during.append(signal.getsignal(signal.SIGTERM)) or 0) + before = signal.getsignal(signal.SIGTERM) + assert cli.main([], env={}) == 0 + assert during == [cli._terminate] and signal.getsignal(signal.SIGTERM) == before + + +def test_exit_during_writes_saves_state(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 2)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + calls = [] + + def write(clients, snap, w): + calls.append(w) + if len(calls) == 2: + cli._terminate(15, None) + + monkeypatch.setattr(github, "write", write) + with pytest.raises(SystemExit): + cli.run(args(tmp_path, apply=True), {}, io.StringIO()) + state = cli.load_state(tmp_path / "state.json") + assert [state["items"][i]["category"]["seen"]["b1"] for i in ("I1", "I2")] == ["Tech", None] + + +def test_failing_report_does_not_print_the_error_text(monkeypatch, tmp_path): + def boom(clients): + raise github.GraphQLError("PVTI_secret") + + monkeypatch.setattr(github, "read_all", boom) + out = io.StringIO() + bad = tmp_path / "missing" / "r.txt" + assert cli.run(args(tmp_path, summary=True, report=str(bad)), {}, out) == 2 + assert out.getvalue().strip() == "read failed: GraphQLError" + + +def test_missing_token_exits_with_code_2(): + with pytest.raises(SystemExit) as exc: + cli.make_clients({}) + assert exc.value.code == 2 + + +def test_bulk_approval_keeps_failed_writes(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 4)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + + def write(clients, snap, w): + if w["issue"] == "I1": + raise github.GraphQLError("x") + apply_to(snap, type("R", (), {"writes": [w]})) + + monkeypatch.setattr(github, "write", write) + cli.run(args(tmp_path, apply=True, max_changes=1, limit=2, allow_bulk=True), {}, io.StringIO()) + keys = cli.load_state(tmp_path / "state.json")["bulk_keys"] + assert len(keys) == 3 and any(k.startswith("I1|") for k in keys) + + +def test_new_value_on_a_recreated_field_is_a_change(mk): + a = mk.item(b1={"Category": ("Tech", "t5")}, b2={"Category": ("Tech", "t5")}) + state = core.plan(mk.snap([a]), {}).state + state = core.plan(mk.snap([mk.item(b1={"Category": ("Tech", "t5")})]), state).state + s = mk.snap( + [mk.item(b1={"Category": ("Tech", "t5")}, b2={"Category": ("Science", "t1")})], ids={("b2", "Category"): "NEW"} + ) + assert writes(core.plan(s, state)) == {("b1", "Category"): "Science"} + + +def test_a_new_value_is_a_new_write_for_failures_and_parking(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 1)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + calls = [] + + def refuse(clients, snap, w): + calls.append(w["value"]) + raise github.GraphQLError("x") + + monkeypatch.setattr(github, "write", refuse) + for _ in range(cli.PARK_AFTER): + cli.run(args(tmp_path, apply=True), {}, io.StringIO()) + assert cli.run(args(tmp_path, apply=True), {}, io.StringIO()) == 4 # parked + s["items"]["I1"]["places"]["b2"]["values"]["Category"] = ["Science", "t2"] + assert cli.run(args(tmp_path, apply=True), {}, io.StringIO()) == 1 # tried at once + assert calls[-1] == "Science" + assert sorted(r["n"] for r in cli.load_state(tmp_path / "state.json")["failures"].values()) == [1] + + +def test_parked_writes_stay_approved_and_do_not_count_as_new(mk, monkeypatch, tmp_path): + from datetime import UTC, datetime, timedelta + + s = mk.snap(many(mk, 3)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + keys = [cli.write_key(w) for w in core.plan(s, {}).writes] + old = (datetime.now(UTC) - timedelta(days=2)).isoformat(timespec="seconds") + state = {"failures": {k: {"n": cli.PARK_AFTER, "at": old} for k in keys}} + (tmp_path / "state.json").write_text(json.dumps(state)) + calls = [] + monkeypatch.setattr(github, "write", lambda c, snap, w: calls.append(w)) + # 3 retried writes against a limit of 1: they failed before, so they are not a new mass change. + assert cli.run(args(tmp_path, apply=True, max_changes=1), {}, io.StringIO()) == 0 + assert len(calls) == 3 + + +def test_parked_writes_keep_their_bulk_approval(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 3)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + monkeypatch.setattr(cli, "STOP_AFTER_FAILURES", 99) + + def refuse(clients, snap, w): + raise github.GraphQLError("x") + + monkeypatch.setattr(github, "write", refuse) + cli.run(args(tmp_path, apply=True, max_changes=0, allow_bulk=True), {}, io.StringIO()) + for _ in range(cli.PARK_AFTER): + cli.run(args(tmp_path, apply=True, max_changes=0), {}, io.StringIO()) + state = cli.load_state(tmp_path / "state.json") + assert len(state["bulk_keys"]) == 3 + + +def test_reaching_parking_counts_as_progress_and_reapproval_resets_idle(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 2 * cli.STOP_AFTER_FAILURES)) # 10 writes that always fail + monkeypatch.setattr(github, "read_all", lambda clients: s) + + def refuse(clients, snap, w): + raise github.GraphQLError("x") + + monkeypatch.setattr(github, "write", refuse) + cli.run(args(tmp_path, apply=True, max_changes=1, allow_bulk=True), {}, io.StringIO()) + idles = [] + for _ in range(2 * cli.PARK_AFTER): + assert cli.run(args(tmp_path, apply=True, max_changes=1), {}, io.StringIO()) != 3 + idles.append(cli.load_state(tmp_path / "state.json")["bulk_idle"]) + assert 0 in idles[cli.PARK_AFTER - 2 :] # the run that parks a block resets the idle count + state = cli.load_state(tmp_path / "state.json") + state["bulk_idle"] = cli.BULK_IDLE_RUNS - 1 + (tmp_path / "state.json").write_text(json.dumps(state)) + cli.run(args(tmp_path, apply=True, max_changes=1, allow_bulk=True), {}, io.StringIO()) + assert cli.load_state(tmp_path / "state.json")["bulk_idle"] <= 1 + + +def test_returning_place_without_a_known_agreed_time_receives(mk): + a = mk.item(b1={"Start date": ("2026-05-01", None)}, issue={"Start date": ("2026-05-01", None)}) + state = core.plan(mk.snap([a]), {}).state + assert state["items"]["I1"]["start"]["t"] is None + state = core.plan(mk.snap([mk.item(b1={"Start date": ("2026-05-01", None)})]), state).state + back = mk.item(b1={"Start date": ("2026-05-01", None)}, issue={"Start date": ("2026-07-01", "t9")}) + assert writes(core.plan(mk.snap([back]), state)) == {("issue", "Start date"): "2026-05-01"} + + +def test_snapshot_does_not_change_when_the_boards_do_not(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 1)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + texts = [] + for when in ("2026-01-01T00:00:00+00:00", "2026-01-01T00:30:00+00:00"): + s["read_at"] = when + cli.run(args(tmp_path, snapshot_dir=str(tmp_path / "snap")), {}, io.StringIO()) + texts.append((tmp_path / "snap" / "snapshot.json").read_text()) + assert texts[0] == texts[1] and "read_at" not in texts[0] + + +def test_parked_writes_give_code_4_only_when_applying(mk, monkeypatch, tmp_path): + from datetime import UTC, datetime + + s = mk.snap(many(mk, 1)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + key = cli.write_key(core.plan(s, {}).writes[0]) + now = datetime.now(UTC).isoformat(timespec="seconds") + (tmp_path / "state.json").write_text(json.dumps({"failures": {key: {"n": cli.PARK_AFTER, "at": now}}})) + assert cli.run(args(tmp_path), {}, io.StringIO()) == 0 + assert cli.run(args(tmp_path, apply=True), {}, io.StringIO()) == 4 + + +def test_reapproval_starts_a_fresh_idle_count(mk, monkeypatch, tmp_path): + monkeypatch.setattr(cli, "PARK_AFTER", 99) # no parking, so no progress at all + s = mk.snap(many(mk, 3)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + + def refuse(clients, snap, w): + raise github.GraphQLError("x") + + monkeypatch.setattr(github, "write", refuse) + (tmp_path / "state.json").write_text('{"bulk_idle": 3, "bulk_keys": []}') + cli.run(args(tmp_path, apply=True, max_changes=0, allow_bulk=True), {}, io.StringIO()) + state = cli.load_state(tmp_path / "state.json") + assert state["bulk_idle"] == 1 and len(state["bulk_keys"]) == 3 + + +def test_output_files_are_written_atomically(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 1)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + written = [] + real = cli.write_file + monkeypatch.setattr(cli, "write_file", lambda path, text: written.append(str(path)) or real(path, text)) + monkeypatch.setattr(github, "write", lambda c, snap, w: None) + report, snapdir = tmp_path / "r.txt", tmp_path / "snap" + cli.run(args(tmp_path, apply=True, report=str(report), snapshot_dir=str(snapdir)), {}, io.StringIO()) + assert {str(report), str(snapdir / "snapshot.json"), str(tmp_path / "state.json")} <= set(written) + + +def test_derived_field_is_listed_when_missing_and_skips_are_reported(mk): + s = mk.snap([mk.item(b2={"Workload": ("S", "t1")}, issue={})]) + del s["fields"]["issue:Interra-Project"]["Effort"]["options"]["Low"] + r = core.plan(s, {}) + assert r.skipped == [ + {"issue": "I1", "url": s["items"]["I1"]["url"], "place": "issue", "fkey": "Effort", "value": "Low"} + ] + del s["fields"]["issue:FormingWorlds"]["Effort"] + assert ("issue:FormingWorlds", "Effort") in core.missing_fields(s) + + +def test_writes_that_failed_before_go_last(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 3)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + first = cli.write_key(core.plan(s, {}).writes[0]) + (tmp_path / "state.json").write_text(json.dumps({"failures": {first: {"n": 1, "at": "2026-01-01T00:00:00+00:00"}}})) + order = [] + monkeypatch.setattr(github, "write", lambda c, snap, w: order.append(cli.write_key(w))) + cli.run(args(tmp_path, apply=True), {}, io.StringIO()) + assert order[-1] == first and len(order) == 3 + + +def test_workload_without_an_effort_mapping_writes_no_effort(mk, monkeypatch): + monkeypatch.setitem(config.DERIVED["effort"], "map", {"XL": "High"}) + s = mk.snap([mk.item(b2={"Workload": ("S", "t1")}, issue={})]) + r = core.plan(s, {}) + assert r.writes == [] and r.skipped == [] + + +def test_skip_line_names_the_derived_field(mk, monkeypatch, tmp_path): + s = mk.snap([mk.item(b2={"Workload": ("S", "t1")}, issue={})]) + del s["fields"]["issue:Interra-Project"]["Effort"]["options"]["Low"] + monkeypatch.setattr(github, "read_all", lambda clients: s) + out = io.StringIO() + cli.run(args(tmp_path), {}, out) + assert "issue Effort: no option 'Low'" in out.getvalue() From 54a18c4d2e543b4f9eb387898753802a070681a0 Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Wed, 14 Oct 2026 17:57:08 +0200 Subject: [PATCH 2/2] Accept a NOT_FOUND board read only for a missing board A NOT_FOUND error deeper in a board read could null a value, which would then read as empty and spread as a clear. The board read now accepts NOT_FOUND only at the board or org itself, which is how a missing board answers. The tests also check that the page cursor is passed on, and the README setup step names the protection rule this repository uses. --- README.md | 2 +- board_sync/github.py | 15 ++++++++++++--- tests/test_github.py | 20 +++++++++++++++++--- tests/test_store_state.py | 4 ++-- tests/test_sync_rules.py | 2 +- 5 files changed, 33 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index e95becb..2b79886 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Effort follows Workload in one direction: a hand edit of Effort on an issue that - Repository: Issues read and write, Metadata read. - Organization: Projects read and write, Issue fields read. 2. Install it on Interra-Project and on FormingWorlds, with access to all repositories whose issues are on the boards. -3. Protect `main` in this repository with a rule that requires a pull request with an approving review, includes administrators and blocks force pushes, because code on `main` runs with the App key and the deploy key. +3. Protect `main` in this repository with a rule that requires a pull request with an approving review and the `test` check, and blocks force pushes and deletion, because code on `main` runs with the App key and the deploy key. Then create the environment `board-sync` and limit its deployment branches to `main`. Do not add a required reviewer: it would hold every scheduled run too. 4. Create `Interra-Project/board-sync-state` as a private repository with one commit on `main`, for example a README. diff --git a/board_sync/github.py b/board_sync/github.py index 04686f1..6fbe44d 100644 --- a/board_sync/github.py +++ b/board_sync/github.py @@ -59,6 +59,10 @@ class GraphQLError(RuntimeError): """The GraphQL API returned errors.""" +# Error paths of a missing board or org; any other NOT_FOUND in a board read stops the read. +BOARD_GONE = (("organization",), ("organization", "projectV2")) + + class BoardNotFound(RuntimeError): """A configured board does not exist or the token cannot see it.""" @@ -103,9 +107,10 @@ def post(self, query, variables=None, retries=2, allow_not_found=False): retries : int Retries after an HTTP 502, 503 or 504, a network error, or a rate limit (HTTP 403 or 429, or a ``RATE_LIMITED`` error). - allow_not_found : bool + allow_not_found : bool or tuple Return the partial ``data`` when every error is of type ``NOT_FOUND`` (for example a deleted issue in a ``nodes`` list). + A tuple of paths further limits this to errors at those paths. Returns ------- @@ -153,7 +158,11 @@ def post(self, query, variables=None, retries=2, allow_not_found=False): time.sleep(60) return self.post(query, variables, retries - 1, allow_not_found) if errors: - if allow_not_found and out.get("data") is not None and all(e.get("type") == "NOT_FOUND" for e in errors): + paths = allow_not_found if isinstance(allow_not_found, tuple) else None + found = all( + e.get("type") == "NOT_FOUND" and (paths is None or tuple(e.get("path") or ()) in paths) for e in errors + ) + if allow_not_found and out.get("data") is not None and found: return out["data"] raise GraphQLError(json.dumps(errors)[:2000]) return out["data"] @@ -201,7 +210,7 @@ def read_board(client, owner, number): after, items, fields, pid, skipped = None, [], None, None, 0 while True: # A missing board or org comes back as null data with a NOT_FOUND error. - data = client.post(BOARD_QUERY, {"owner": owner, "number": number, "after": after}, allow_not_found=True) + data = client.post(BOARD_QUERY, {"owner": owner, "number": number, "after": after}, allow_not_found=BOARD_GONE) proj = (data.get("organization") or {}).get("projectV2") if proj is None: raise BoardNotFound(f"{owner} project {number}") diff --git a/tests/test_github.py b/tests/test_github.py index f7d7303..6e565ff 100644 --- a/tests/test_github.py +++ b/tests/test_github.py @@ -123,11 +123,12 @@ class Fake: def __init__(self, pages=None, fields=None, values=None, pid="PID"): self.pages, self.fields, self.values = list(pages or []), fields or [], values or [] - self.sent, self.flags, self.pid = [], [], pid + self.sent, self.flags, self.pid, self.cursors = [], [], pid, [] def post(self, query, variables=None, allow_not_found=False): if "projectV2(number" in query: self.flags.append(("board", allow_not_found)) + self.cursors.append(variables["after"]) return {"organization": {"projectV2": {**self.pages.pop(0), "id": self.pid}}} if "issueFields(first" in query: self.flags.append(("issue_fields", allow_not_found)) @@ -185,7 +186,7 @@ def test_read_board_pages_and_skips_drafts(): draft = {"id": "d", "content": {}, "fieldValues": {"pageInfo": DONE, "nodes": []}} fake = Fake(pages=[page([board_item("A"), draft, None], more=True), page([board_item("B")])]) pid, fields, items, skipped = github.read_board(fake, "FormingWorlds", 7) - assert pid == "PID" and skipped == 2 + assert pid == "PID" and skipped == 2 and fake.cursors == [None, "C1"] assert fields["Status"] == {"id": "F2", "type": "select", "options": {"TBD": "o1"}} assert fields["Title"]["type"] == "other" assert [i[0] for i in items] == ["A", "B"] @@ -230,7 +231,11 @@ def test_only_the_issue_values_read_accepts_partial_responses(): interra = Fake(pages=[page([board_item("X", "Interra-Project")]), page([])], values=[issue_node("X")]) fw = Fake(pages=[page([])]) github.read_all({"Interra-Project": interra, "FormingWorlds": fw}) - assert set(interra.flags + fw.flags) == {("board", True), ("issue_fields", False), ("issue_values", True)} + assert set(interra.flags + fw.flags) == { + ("board", github.BOARD_GONE), + ("issue_fields", False), + ("issue_values", True), + } def test_read_issue_values_batches(): @@ -413,3 +418,12 @@ def test_missing_board_from_the_real_response_shape(): no_org = {"data": {"organization": None}, "errors": [{"type": "NOT_FOUND", "path": ["organization"]}]} with pytest.raises(github.BoardNotFound): github.read_board(github.Client("t", opener_for(no_org)), "no-such-org", 1) + + +def test_board_read_rejects_not_found_below_the_board(): + deep = { + "data": {"organization": {"projectV2": page([board_item("A")]) | {"id": "P"}}}, + "errors": [{"type": "NOT_FOUND", "path": ["organization", "projectV2", "items", "nodes", 0, "fieldValues"]}], + } + with pytest.raises(github.GraphQLError): + github.read_board(github.Client("t", opener_for(deep)), "o", 1) diff --git a/tests/test_store_state.py b/tests/test_store_state.py index c56fcc9..9326093 100644 --- a/tests/test_store_state.py +++ b/tests/test_store_state.py @@ -57,10 +57,10 @@ def remote_files(r): def test_first_run_without_a_state_file_stores_snapshot_and_report(repos): (repos["run"] / "snapshot.json").write_text("{}\n") - (repos["run"] / "report.txt").write_text("stop: 1688 writes\n") + (repos["run"] / "report.txt").write_text("stop: 30 writes\n") assert store(repos).returncode == 0 files, log = remote_files(repos) - assert files["report.txt"] == "stop: 1688 writes\n" and "snapshot.json" in files and "state.json" not in files + assert files["report.txt"] == "stop: 30 writes\n" and "snapshot.json" in files and "state.json" not in files assert log.splitlines()[0] == "Sync run 1" diff --git a/tests/test_sync_rules.py b/tests/test_sync_rules.py index 80a2344..ae956d9 100644 --- a/tests/test_sync_rules.py +++ b/tests/test_sync_rules.py @@ -1,4 +1,4 @@ -"""Regression tests for faults in the sync rules.""" +"""Tests of the sync rules across runs: bulk approval, parking, returning places and time limits.""" import io import json