diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml new file mode 100644 index 0000000..1841640 --- /dev/null +++ b/.github/workflows/sync.yml @@ -0,0 +1,89 @@ +name: Board sync + +on: + schedule: + - cron: "7,37 * * * *" + workflow_dispatch: + inputs: + apply: + description: Write the planned values (off prints the plan only) + type: boolean + default: false + allow_bulk: + description: Apply even when the plan exceeds the change limit + type: boolean + default: false + +concurrency: + group: board-sync + cancel-in-progress: false + +permissions: + contents: read + +jobs: + sync: + if: github.repository == 'Interra-Project/board-sync' && github.ref == 'refs/heads/main' + environment: board-sync + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Check out the state repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: Interra-Project/board-sync-state + ssh-key: ${{ secrets.STATE_DEPLOY_KEY }} + path: state + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Token for Interra-Project + id: interra + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.BOARD_SYNC_CLIENT_ID }} + private-key: ${{ secrets.BOARD_SYNC_PRIVATE_KEY }} + owner: Interra-Project + + - name: Token for FormingWorlds + id: formingworlds + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.BOARD_SYNC_CLIENT_ID }} + private-key: ${{ secrets.BOARD_SYNC_PRIVATE_KEY }} + owner: FormingWorlds + + - name: Sync + timeout-minutes: 20 + env: + BOARD_SYNC_TOKEN_INTERRA_PROJECT: ${{ steps.interra.outputs.token }} + BOARD_SYNC_TOKEN_FORMINGWORLDS: ${{ steps.formingworlds.outputs.token }} + APPLY: ${{ github.event_name == 'schedule' || inputs.apply }} + BULK: ${{ github.event_name == 'workflow_dispatch' && inputs.allow_bulk }} + run: | + flags="--state state/state.json --snapshot-dir state --report state/report.txt --summary --pause 1" + if [ "$APPLY" = "true" ]; then flags="$flags --apply"; fi + if [ "$BULK" = "true" ]; then flags="$flags --allow-bulk"; fi + code=0 + python -m board_sync $flags || code=$? + if [ "$code" = 4 ]; then + echo "::warning::Some writes are parked after repeated failures; see report.txt in the state repository." + code=0 + fi + exit "$code" + + - name: Store state, snapshot and report + if: always() + env: + RUN_ID: ${{ github.run_id }} + GIT_AUTHOR_NAME: board-sync + GIT_AUTHOR_EMAIL: board-sync@users.noreply.github.com + GIT_COMMITTER_NAME: board-sync + GIT_COMMITTER_EMAIL: board-sync@users.noreply.github.com + run: bash scripts/store_state.sh state "$RUN_ID" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..a047ad7 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,34 @@ +name: Tests + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Install test tools + run: pip install pytest==9.1.1 coverage==7.16.2 ruff==0.16.9 + + - name: Lint + run: | + ruff check . + ruff format --check . + + - name: Test with coverage + run: | + coverage run -m pytest -q + coverage report --fail-under=95 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3bce80a --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +__pycache__/ +*.pyc +.coverage +.pytest_cache/ +.ruff_cache/ +*.tmp +# Files a local run writes: they hold private board data +state.json +state/ +snapshot/ +snapshot.json +report.txt diff --git a/README.md b/README.md index 3d4b5a2..2b79886 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,92 @@ # board-sync -Keeps dates, priority, workload and status equal across the Interra project boards and the org issue fields + +Keeps shared fields equal across 4 places: + +- the org issue fields on each issue (Interra-Project and FormingWorlds), +- the Interra Management board (Interra-Project project 1), +- the Interra Collaboration board (Interra-Project project 2), +- the PROTEUS Development Roadmap (FormingWorlds project 7). + +When a value changes in one place, the next run writes it to the others. + +| Field | Issue field | Management | Collaboration | PROTEUS | +|---|---|---|---|---| +| Start date | yes | yes | yes | yes | +| Target date | yes | yes | yes | yes | +| Priority | yes (Urgent to Low) | yes | yes | yes | +| Workload | Effort, written from Workload | yes | yes | yes | +| Category, Sub-team, Release | | yes | yes | | +| Status | | yes | yes | | + +The value names per place and the field list are in `board_sync/config.py`. +Status values `Import` and `Starter Projects` show as `TBD` on the Management board. +Effort follows Workload in one direction: a hand edit of Effort on an issue that has a Workload is overwritten, and a cleared Workload leaves Effort as it is. + +## Rules + +- The private repository `Interra-Project/board-sync-state` holds, per issue and field, the agreed value and the value last seen at each place. + A place whose value differs from its last seen value changed since the previous run. +- When several places changed, the newest change wins. + Board values and issue field values have a change time; a cleared issue field takes the time of its removal event. + A board does not record when a value was cleared, so a cleared board value loses to a value set at another place in the same interval. + The issue timeline read covers the last 100 field events; an older change has no time and loses. +- A value that every changed place already shows, after the name map, wins without a conflict. +- A cleared value is cleared in the other places too. + A field that is emptied as a whole at one place (all of its 5 or more values, see below) is filled again from the others. +- A field that is missing at a place is skipped there, never read as empty, and the run log names it; a value without a matching option is listed in `report.txt` as skipped. + A field with a new id (deleted and created again), or a field that held 5 or more values and now holds none, does not clear the other places. +- An issue that is not on a board, or cannot be read, keeps its record for that board. + When it is there again with the field, its empty fields receive the agreed values, an unchanged value counts as no change, and a different value wins only when it is newer than the agreed value (an agreed value of unknown time keeps its place). + On a field that was created again, the first value counts as a change, as on a new board. + An issue that is on no board for 90 days is dropped from the state. +- A list longer than one page of the API (fields of a board, values of an item, issue fields, issue field values) stops the read with an error, so a cut-off list is never read as empty values. +- A run that plans more than 20 writes stops and writes nothing (exit code 3). + Read the plan in `report.txt` in the state repository, then start the workflow by hand with `apply` and `allow_bulk`. + The state then holds the approved writes that are not done yet, and the scheduled runs write them, at most 400 values per run. + A later run stops again when its plan holds more than 20 new writes: writes outside the approved set that never failed before. + The approval ends when all approved writes are done, or after 4 runs in a row in which no write succeeds and none reaches parking; a new hand run with `allow_bulk` starts it again. +- After 5 failed writes in a row, or 10 minutes after the run started, a run stops writing; failed and unwritten values are written again by the next run, the failed ones after all others. + A write that failed in 3 runs is parked: it is listed in `report.txt` and tried again once a day, or at once when its value changes. + Parked writes give a warning in the run, not a failed run. +- `report.txt` holds the plan before the first write, and the results after the last. +- Each run stores the raw read of all places (`snapshot.json`) and the full report (`report.txt`) in the state repository, so lost values can be restored from its history. + The 3 files of a run replace the stored ones whole, and a run that changes nothing makes no commit. +- The job log of this public repository shows counts per board and field, field names and error types only. + Issue links, values and error text stay in the state repository. +- A run reads the places one after the other and then writes. + An edit made in the minutes of a run can be overwritten by that run; the next run then sees the written value as agreed. +- Exit codes: 0 done, 1 some writes failed, 2 the read, the state file, a token or the run failed, 3 too many changes, 4 done but some writes are parked (the workflow shows it as a warning), 143 or 130 stopped by a signal. + +## Setup + +1. Create a GitHub App that can be installed on any account (both orgs need it), with these permissions and no webhook: + - Repository: Issues read and write, Metadata read. + - Organization: Projects read and write, Issue fields read. +2. Install it on Interra-Project and on FormingWorlds, with access to all repositories whose issues are on the boards. +3. Protect `main` in this repository with a rule that requires a pull request with an approving review and the `test` check, and blocks force pushes and deletion, because code on `main` runs with the App key and the deploy key. + Then create the environment `board-sync` and limit its deployment branches to `main`. + Do not add a required reviewer: it would hold every scheduled run too. +4. Create `Interra-Project/board-sync-state` as a private repository with one commit on `main`, for example a README. +5. Store all 3 values in the environment only, not as repository secrets: the App client id as the variable `BOARD_SYNC_CLIENT_ID`, the App private key as the secret `BOARD_SYNC_PRIVATE_KEY`, and the private key of an SSH deploy key with write access to `Interra-Project/board-sync-state` as the secret `STATE_DEPLOY_KEY`. + +The workflow runs at minute 7 and 37 of every hour on `main`; a run on another branch is skipped. +A run started by hand from the Actions tab only plans, unless `apply` is set. +GitHub turns off scheduled workflows in a public repository after 60 days without activity in that repository. +Check the date of the last run in the Actions tab of this repository, and enable the workflow again there when it is off. + +## Local run + +```bash +GH_TOKEN=$(gh auth token) python -m board_sync --state state.json --snapshot-dir snapshot +``` + +Without `--apply` the run changes nothing on GitHub; it writes only the snapshot and report files it is given. +These files hold private board data; `.gitignore` keeps them out of commits. +`scripts/store_state.sh` is the workflow step that stores the files of a run in the state repository. +`BOARD_SYNC_TOKEN_INTERRA_PROJECT` and `BOARD_SYNC_TOKEN_FORMINGWORLDS` set one token per org. + +## Tests + +```bash +python -m pytest +``` diff --git a/board_sync/__init__.py b/board_sync/__init__.py new file mode 100644 index 0000000..f9023b0 --- /dev/null +++ b/board_sync/__init__.py @@ -0,0 +1 @@ +"""Sync of fields across the Interra boards, the PROTEUS board and the org issue fields.""" diff --git a/board_sync/__main__.py b/board_sync/__main__.py new file mode 100644 index 0000000..930af90 --- /dev/null +++ b/board_sync/__main__.py @@ -0,0 +1,332 @@ +"""Sync shared fields across the Interra boards, the PROTEUS board and the org issue fields. + +Without ``--apply`` the run changes nothing on GitHub: it prints the plan and +writes only the snapshot and report files it is given. With ``--apply`` it +writes the plan, unless the plan has more than ``--max-changes`` writes; then +it stops with exit code 3. A person reads the plan and reruns with +``--allow-bulk``; the state then holds the approved writes, and the next runs +write them in batches of ``--limit``. +""" + +import argparse +import io +import json +import os +import signal +import sys +import time +import traceback +from collections import Counter +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from . import config, core, github + +TOKEN_ENV = { + "Interra-Project": "BOARD_SYNC_TOKEN_INTERRA_PROJECT", + "FormingWorlds": "BOARD_SYNC_TOKEN_FORMINGWORLDS", +} + +# Failed writes in a row after which a run stops and leaves the rest for later. +STOP_AFTER_FAILURES = 5 +# Runs a write may fail before it is parked; a parked write is tried once a day. +PARK_AFTER = 3 +PARK_RETRY = timedelta(days=1) +# Runs in a row without a successful write after which a bulk approval ends. +BULK_IDLE_RUNS = PARK_AFTER + 1 + + +class StateError(RuntimeError): + """The state file cannot be used.""" + + +def make_clients(env=os.environ): + """Build one GraphQL client per org from the environment. + + Parameters + ---------- + env : mapping + Environment; each org reads its own variable and falls back to + ``GH_TOKEN``. + + Returns + ------- + dict + Org login to ``github.Client``. + + Raises + ------ + SystemExit + With code 2 if an org has no token. + """ + clients = {} + for org, var in TOKEN_ENV.items(): + token = env.get(var) or env.get("GH_TOKEN") + if not token: + print(f"no token for {org}: set {var} or GH_TOKEN", file=sys.stderr) + raise SystemExit(2) + clients[org] = github.Client(token) + return clients + + +def load_state(path): + """Read the state file; an absent or empty file is an empty state. + + Raises + ------ + StateError + If the file is not a JSON object, so a damaged state never counts as a first run. + """ + p = Path(path) + if not p.exists() or not p.read_text().strip(): + return {} + try: + data = json.loads(p.read_text()) + except ValueError: + raise StateError(f"state file {path} is not valid JSON") from None + if not isinstance(data, dict): + raise StateError(f"state file {path} does not hold a JSON object") + return data + + +def write_file(path, text): + """Write a file through a temporary file, so a killed run leaves the old file whole.""" + tmp = Path(f"{path}.tmp") + tmp.write_text(text) + os.replace(tmp, path) + + +def save_state(path, state): + """Write the state file atomically.""" + write_file(path, json.dumps(state, indent=1, sort_keys=True)) + + +def describe(w): + """Format one planned write as one line.""" + where = config.BOARDS.get(w["place"], {}).get("title", "issue field") + value = "(clear)" if w["value"] is None else w["value"] + return f"{w['url']} {where} / {w['field']}: {value}" + + +def counts(writes): + """Count writes per place and field, as lines without issue data.""" + tally = Counter(f"{config.BOARDS.get(w['place'], {}).get('title', 'issue field')} / {w['field']}" for w in writes) + return [f" {n:5d} {k}" for k, n in sorted(tally.items())] + + +def warnings(snap): + """Lines about fields and items the run could not use, without issue data.""" + lines = [f"warning: {key} has no field {name!r}" for key, name in core.missing_fields(snap)] + for key, n in sorted(snap.get("skipped_items", {}).items()): + if n: + lines.append(f"note: {config.BOARDS[key]['title']}: {n} items are not readable issues") + for org, n in sorted(snap.get("unread_issues", {}).items()): + if n: + lines.append(f"warning: {n} {org} issues could not be read for their issue fields") + return lines + + +def emit(args, summary, detail, out): + """Write the full report file and print the summary, or everything.""" + if args.report: + write_file(args.report, "\n".join(summary + detail) + "\n") + print("\n".join(summary if args.summary else summary + detail), file=out) + + +def write_key(w): + """Key of one planned write, for failure counts and bulk approval.""" + return f"{w['issue']}|{w['place']}|{w['field']}|{w['value']}" + + +def is_parked(record, now): + """A write is parked after ``PARK_AFTER`` failures, until a day after its last one.""" + if not record or record["n"] < PARK_AFTER: + return False + return now - datetime.fromisoformat(record["at"]) < PARK_RETRY + + +def apply(args, clients, snap, result, active, parked, state, now, summary, detail): + """Write the planned values and save the state, also when the run is cut short. + + Parameters + ---------- + active, parked : list of dict + Writes to attempt, and writes held back after repeated failures. + state : dict + State of the previous run (failure records and bulk approval). + now : datetime + Time of this run, for the failure records. + + Returns + ------- + int + 0 when every attempted write succeeded, else 1. + """ + old_fails = state.get("failures", {}) + batch, later = active[: args.limit], active[args.limit :] + failed, done_keys, done, in_row, late = [], set(), 0, 0, False + try: + for n, w in enumerate(batch): + if in_row >= STOP_AFTER_FAILURES: + break + if time.monotonic() - args.started > args.deadline: + late = True + break + if n and args.pause: + time.sleep(args.pause) + try: + github.write(clients, snap, w) + done_keys.add(write_key(w)) + in_row = 0 + except Exception as exc: # noqa: BLE001 one refused write must not stop the others + failed.append((w, type(exc).__name__)) + in_row += 1 + detail.append(f"failed {describe(w)}: {exc}") + done = n + 1 + finally: + rest = batch[done:] + later + # Failed, unwritten and parked places keep their old record, so a later run plans them again. + core.undo_failed(result.state, [w for w, _ in failed] + rest + parked) + stamp = now.isoformat(timespec="seconds") + fails = {k: v for k, v in old_fails.items() if k in {write_key(w) for w in rest + parked}} + for w, _ in failed: + fails[write_key(w)] = {"n": old_fails.get(write_key(w), {}).get("n", 0) + 1, "at": stamp} + result.state["failures"] = fails + # A bulk approval covers the approved writes not yet done (parked ones too), until + # runs stop making progress; a write that reaches parking counts as progress. + planned = {write_key(w) for w in active + parked} + base = planned if args.allow_bulk else set(state.get("bulk_keys", [])) + left = sorted((base & planned) - done_keys) + progress = done_keys or any(r["n"] == PARK_AFTER for r in fails.values() if r["at"] == stamp) + idle = 0 if progress else (0 if args.allow_bulk else state.get("bulk_idle", 0)) + 1 + keep = left and idle < BULK_IDLE_RUNS + result.state["bulk_keys"], result.state["bulk_idle"] = (left, idle) if keep else ([], 0) + save_state(args.state, result.state) + if in_row >= STOP_AFTER_FAILURES: + summary.append(f"stopped after {in_row} failed writes in a row") + if late: + summary.append(f"stopped at the time limit of {args.deadline} s") + if rest: + summary.append(f"deferred: {len(rest)} writes to the next run") + if failed: + kinds = Counter(kind for _, kind in failed) + summary.append(f"failed: {len(failed)} ({', '.join(f'{k} {n}' for k, n in sorted(kinds.items()))})") + return 1 + return 0 + + +def plan_and_apply(args, clients, snap, out): + """Plan from a snapshot, apply when asked, and report. See ``run``.""" + if args.snapshot_dir: + # The read time stays out of the file, so an unchanged board gives an unchanged snapshot. + Path(args.snapshot_dir).mkdir(parents=True, exist_ok=True) + stored = {k: v for k, v in snap.items() if k != "read_at"} + write_file(Path(args.snapshot_dir) / "snapshot.json", json.dumps(stored, indent=1, sort_keys=True)) + state = load_state(args.state) + result = core.plan(snap, state) + now = datetime.now(UTC) + fails = state.get("failures", {}) + parked = [w for w in result.writes if is_parked(fails.get(write_key(w)), now)] + # Writes that failed before go last, so they cannot hold back the others. + active = sorted( + (w for w in result.writes if not is_parked(fails.get(write_key(w)), now)), key=lambda w: write_key(w) in fails + ) + detail = [f"conflict {c['url']} {c['fkey']}: {c['values']} -> {c['chosen']}" for c in result.conflicts] + detail += [f"skipped {s['url']} {s['place']} {s['fkey']}: no option {s['value']!r}" for s in result.skipped] + detail += ["parked " + describe(w) for w in parked] + detail += ["plan " + describe(w) for w in active] + summary = [f"issues: {len(snap['items'])} writes: {len(active)} conflicts: {len(result.conflicts)}"] + summary += counts(active) + warnings(snap) + summary += [f"reset: {key} / {fkey} is treated as new, its old values are ignored" for key, fkey in result.resets] + if result.skipped: + summary.append(f"skipped: {len(result.skipped)} values without an option at the place") + if parked: + summary.append(f"parked: {len(parked)} writes that failed in {PARK_AFTER} runs, listed in the report") + approved_keys = set(state.get("bulk_keys", [])) + # Writes that already failed were planned before; they never count as a new mass change. + outside = [w for w in active if write_key(w) not in approved_keys and write_key(w) not in fails] + approved = args.allow_bulk or len(outside) <= args.max_changes + code = 0 + if args.apply and len(active) > args.max_changes and not approved: + summary.append(f"stop: {len(active)} writes exceed --max-changes {args.max_changes}") + code = 3 + elif args.apply: + emit(args, summary, detail, io.StringIO()) # the report shows the plan while writes run + code = apply(args, clients, snap, result, active, parked, state, now, summary, detail) + emit(args, summary, detail, out) + return code or (4 if parked and args.apply else 0) + + +def safe_emit(args, line, detail, out): + """Report a failed run; if even that fails, print the line alone.""" + try: + emit(args, [line], [detail], out) + except Exception: # noqa: BLE001 never let a second error print the first one's text + print(line, file=out) + + +def run(args, clients, out=sys.stdout): + """Read all places, plan, and optionally apply. + + The detail lines name issues and values. With ``args.summary`` the + stream gets counts only, and the details go to ``args.report`` alone, + so a public job log shows no board data. An error prints only its type. + + Parameters + ---------- + args : argparse.Namespace + Parsed command line. + clients : dict + Org login to ``github.Client``. + out : file + Stream for the report. + + Returns + ------- + int + Exit code: 0 done, 1 some writes failed, 2 the read or the run failed, + 3 too many changes, 4 done but some writes are parked. + """ + try: + snap = github.read_all(clients) + except Exception as exc: # noqa: BLE001 the summary names the error type only + safe_emit(args, f"read failed: {type(exc).__name__}", f"read failed: {exc}", out) + return 2 + try: + return plan_and_apply(args, clients, snap, out) + except Exception as exc: # noqa: BLE001 the summary names the error type only + safe_emit(args, f"run failed: {type(exc).__name__}", traceback.format_exc(), out) + return 2 + + +def _terminate(signum, frame): + """Turn SIGTERM into an exit, so the state is saved on the way out.""" + raise SystemExit(128 + signum) + + +def main(argv=None, env=os.environ): + """Parse the command line and run the sync.""" + parser = argparse.ArgumentParser(prog="board_sync", description=__doc__.splitlines()[0]) + parser.add_argument("--state", default="state.json", help="state file of the previous run") + parser.add_argument("--snapshot-dir", help="folder for the raw snapshot of this run") + parser.add_argument("--apply", action="store_true", help="write the planned values") + parser.add_argument("--max-changes", type=int, default=20) + parser.add_argument("--allow-bulk", action="store_true", help="apply even above --max-changes") + parser.add_argument("--limit", type=int, default=400, help="most writes in one run; the rest wait") + parser.add_argument("--pause", type=float, default=0.0, help="seconds between writes") + parser.add_argument("--deadline", type=float, default=600.0, help="seconds from start after which writes stop") + parser.add_argument("--summary", action="store_true", help="print counts only, no issue data") + parser.add_argument("--report", help="file for the full report with issue data") + args = parser.parse_args(argv) + args.started = time.monotonic() + if env.get("GITHUB_ACTIONS") == "true": + args.summary = True # a job log can be public + old = signal.signal(signal.SIGTERM, _terminate) + try: + return run(args, make_clients(env)) + finally: + signal.signal(signal.SIGTERM, old) + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/board_sync/config.py b/board_sync/config.py new file mode 100644 index 0000000..5056f46 --- /dev/null +++ b/board_sync/config.py @@ -0,0 +1,80 @@ +"""Places and field maps for the board sync. + +A *place* is one location that holds a value for an issue: the org issue +fields on the issue itself (``issue``) or one of the project boards +(``b1``, ``b2``, ``b7``). Canonical values use the vocabulary of the +Interra Collaboration board (``b2``). +""" + +BOARDS = { + "b1": {"owner": "Interra-Project", "number": 1, "title": "Interra Management"}, + "b2": {"owner": "Interra-Project", "number": 2, "title": "Interra Collaboration"}, + "b7": {"owner": "FormingWorlds", "number": 7, "title": "PROTEUS Development Roadmap"}, +} + +# Orgs whose org issue fields take part in the sync. +ISSUE_ORGS = ("Interra-Project", "FormingWorlds") + +# Tie-break when two places changed at the same second. +PLACE_ORDER = ("issue", "b2", "b1", "b7") + +PRIORITY = ( + "Priority 1: critical", + "Priority 2: high", + "Priority 3: standard", + "Priority 4: low", +) + +# Per canonical field: the field name at each place, and the value map +# from canonical value to place value where the two differ. +FIELDS = { + "start": { + "type": "date", + "places": {"issue": "Start date", "b1": "Start date", "b2": "Start date", "b7": "Start date"}, + }, + "target": { + "type": "date", + "places": {"issue": "Target date", "b1": "Target date", "b2": "Target date", "b7": "Target date"}, + }, + "priority": { + "type": "select", + "places": {"issue": "Priority", "b1": "Priority", "b2": "Priority", "b7": "Priority"}, + "to_place": { + "issue": dict(zip(PRIORITY, ("Urgent", "High", "Medium", "Low"))), + "b1": dict(zip(PRIORITY, ("Priority: critical", "Priority: high", "Priority: normal", "Priority: low"))), + }, + }, + "workload": { + "type": "select", + "places": {"b1": "Workload", "b2": "Workload", "b7": "Workload"}, + }, + "category": {"type": "select", "places": {"b1": "Category", "b2": "Category"}}, + "subteam": { + "type": "select", + "places": {"b1": "Sub-team", "b2": "Sub-team"}, + "to_place": {"b1": {"Co-leads": "Co-Leads"}}, + }, + "release": {"type": "select", "places": {"b1": "Release", "b2": "Release"}}, + "status": { + "type": "select", + "places": {"b1": "Status", "b2": "Status"}, + "to_place": {"b1": {"Import": "TBD", "Starter Projects": "TBD"}}, + }, +} + +# Fields written from another field and never read as a source. +DERIVED = { + "effort": { + "source": "workload", + "place": "issue", + "field": "Effort", + "map": {"XL": "High", "L": "High", "M": "Medium", "S": "Low", "XS": "Low"}, + }, +} + +# A place whose field held at least this many values and now holds none is +# treated as a new, empty field, not as a mass clear. +RESET_MIN = 5 + +# An issue that is on no board for this many days is dropped from the state. +GONE_DAYS = 90 diff --git a/board_sync/core.py b/board_sync/core.py new file mode 100644 index 0000000..4ba6666 --- /dev/null +++ b/board_sync/core.py @@ -0,0 +1,462 @@ +"""Sync decision: which value wins for each issue and field, and what to write. + +The input is a snapshot of all places (see ``github.read_all``) and the state +of the previous run. The state holds, per issue and field, the agreed +canonical value ``c`` and the canonical value last seen at each place. A place +whose value differs from its last seen value changed since the previous run. +When several places changed, the newest change wins. +""" + +from dataclasses import dataclass, field +from datetime import datetime, timedelta + +from . import config + +_NEW = object() # marker: the place has no recorded value yet + + +@dataclass +class Plan: + """Result of one sync decision over all issues. + + Attributes + ---------- + writes : list of dict + One entry per value to write. Keys: ``issue``, ``url``, ``place``, + ``fkey`` (``None`` for a derived field), ``field``, ``value`` + (``None`` clears) and ``prev`` (the canonical value the place held + before the write). + state : dict + State to store after the writes; ``undo_failed`` restores the + records of writes that failed or were not made. + conflicts : list of dict + Fields where two places held different new values. + skipped : list of dict + Values that cannot be written because the place lacks the option. + resets : list of tuple + ``(place_key, fkey)`` pairs treated as new fields in this run. + """ + + writes: list = field(default_factory=list) + state: dict = field(default_factory=dict) + conflicts: list = field(default_factory=list) + skipped: list = field(default_factory=list) + resets: list = field(default_factory=list) + + +def to_place(fkey, place, value): + """Map a canonical value to the vocabulary of one place. + + Parameters + ---------- + fkey : str + Canonical field key, a key of ``config.FIELDS``. + place : str + Place key. + value : str or None + Canonical value. + + Returns + ------- + str or None + The value as the place names it. + """ + if value is None: + return None + return config.FIELDS[fkey].get("to_place", {}).get(place, {}).get(value, value) + + +def to_canon(fkey, place, value): + """Map a place value back to the canonical vocabulary. + + A place value that more than one canonical value maps to (for example + ``TBD`` on board 1) stays as it is. + + Parameters + ---------- + fkey : str + Canonical field key. + place : str + Place key. + value : str or None + Value as the place names it. + + Returns + ------- + str or None + Canonical value. + """ + if value is None: + return None + mapping = config.FIELDS[fkey].get("to_place", {}).get(place, {}) + sources = [c for c, v in mapping.items() if v == value] + return sources[0] if len(sources) == 1 else value + + +def place_fields(snap, item, place): + """Return the field definitions of one place for one issue. + + Parameters + ---------- + snap : dict + Snapshot of all places. + item : dict + One entry of ``snap["items"]``. + place : str + Place key. + + Returns + ------- + dict + Field name to definition; empty if the place has no fields. + """ + if place == "issue": + return snap["fields"].get(f"issue:{item.get('org')}", {}) + return snap["fields"].get(place, {}) + + +def scope(snap, item, fkey): + """List the places that hold this issue and this field. + + Parameters + ---------- + snap : dict + Snapshot of all places. + item : dict + One entry of ``snap["items"]``. + fkey : str + Canonical field key. + + Returns + ------- + list of str + Place keys in ``config.PLACE_ORDER`` order. + """ + names = config.FIELDS[fkey]["places"] + return [ + p for p in config.PLACE_ORDER if p in names and p in item["places"] and names[p] in place_fields(snap, item, p) + ] + + +def raw_value(item, place, name): + """Return ``(value, changed_at)`` of one field at one place. + + Parameters + ---------- + item : dict + One entry of ``snap["items"]``. + place : str + Place key. + name : str + Field name at that place. + + Returns + ------- + tuple + Value (or ``None``) and ISO change time (or ``None``). A board does not + record when a value was cleared, so an empty board value has no time. + """ + return tuple(item["places"][place]["values"].get(name, (None, None))) + + +def place_key(item, place): + """Return the key of a place's field set: ``issue:`` or the board key.""" + return f"issue:{item.get('org')}" if place == "issue" else place + + +def field_ids(snap, old=None): + """Return the id of every field at every place key, for the state. + + Ids of fields missing from this snapshot stay from ``old``, so a field + that is deleted and created again between two runs is still detected. + """ + ids = {k: dict(v) for k, v in (old or {}).items()} + for k, fields in snap["fields"].items(): + ids.setdefault(k, {}).update({name: f["id"] for name, f in fields.items()}) + return ids + + +def missing_fields(snap): + """List configured fields that a place does not have, as ``(place_key, name)``.""" + out = [] + for key in snap["fields"]: + place = "issue" if key.startswith("issue:") else key + names = [spec["places"].get(place) for spec in config.FIELDS.values()] + names += [spec["field"] for spec in config.DERIVED.values() if spec["place"] == place] + out += [(key, name) for name in names if name and name not in snap["fields"][key]] + return sorted(set(out)) + + +def reset_places(snap, state): + """Find place fields that were emptied as a whole or created again. + + A field counts as reset when its id differs from the id in the state + (deleted and created again), or when it held at least + ``config.RESET_MIN`` values in the state and holds none now. + + Parameters + ---------- + snap : dict + Snapshot of all places. + state : dict + State of the previous run. + + Returns + ------- + set of tuple + ``(place_key, fkey)`` pairs whose recorded values must be ignored. + """ + reset, seen, now = set(), {}, {} + old_ids = state.get("fields", {}) + for key, fields in snap["fields"].items(): + for fkey, spec in config.FIELDS.items(): + place = "issue" if key.startswith("issue:") else key + name = spec["places"].get(place) + if name in fields and name in old_ids.get(key, {}) and old_ids[key][name] != fields[name]["id"]: + reset.add((key, fkey)) + old = state.get("items", {}) + for iid, item in snap["items"].items(): + for fkey, spec in config.FIELDS.items(): + for place in scope(snap, item, fkey): + key = (place_key(item, place), fkey) + if raw_value(item, place, spec["places"][place])[0] is not None: + now[key] = now.get(key, 0) + 1 + if old.get(iid, {}).get(fkey, {}).get("seen", {}).get(place) is not None: + seen[key] = seen.get(key, 0) + 1 + reset |= {k for k, n in seen.items() if n >= config.RESET_MIN and now.get(k, 0) == 0} + return reset + + +def _newest(places, times): + """Pick the place with the newest change time; ties go by ``PLACE_ORDER``.""" + return max(places, key=lambda p: (times[p] or "", -config.PLACE_ORDER.index(p))) + + +def _pick(fkey, places, current, raw, times): + """Pick the winning value among places that each claim a new value. + + A value that every claiming place already shows (after mapping to that + place) wins without a conflict; ``Starter Projects`` on board 2 and + ``TBD`` on board 1 agree. Otherwise the newest change wins. + + Returns + ------- + tuple + Winning canonical value, the conflict dict (empty if none), and the + change time of the winner. + """ + agreeing = [p for p in places if all(raw[q] == to_place(fkey, q, current[p]) for q in places)] + if agreeing: + win = _newest(agreeing, times) + return current[win], {}, times[win] + win = _newest(places, times) + return current[win], {p: current[p] for p in places}, times[win] + + +def decide(fkey, places, current, raw, times, record): + """Decide the canonical value of one field of one issue. + + Parameters + ---------- + fkey : str + Canonical field key. + places : list of str + Places in scope. + current : dict + Place to current canonical value. + raw : dict + Place to current value as the place names it. + times : dict + Place to ISO change time or ``None``. + record : dict or None + State record ``{"c": ..., "t": ..., "seen": {...}, "gone": [...]}`` + of the previous run, or ``None`` for an issue and field seen for the + first time. ``t`` is the change time of the agreed value; ``gone`` + lists places that did not hold the issue or the field then. + + Returns + ------- + tuple + Canonical value, the conflicting values as a place-to-value dict + (empty when there is no conflict), and the change time of the value. + """ + if record is None: + filled = [p for p in places if current[p] is not None] + if not filled: + return None, {}, None + return _pick(fkey, filled, current, raw, times) + seen, gone = record.get("seen", {}), set(record.get("gone", ())) + changed = [] + for p in places: + before = seen.get(p, _NEW) + if p in gone and before is not _NEW: + # A returning place: empty receives, its old value is no change, and a new + # value counts only when it is newer than the agreed value (of known time). + newer = record.get("t") is not None and (times[p] or "") > record["t"] + if current[p] is not None and current[p] != before and newer: + changed.append(p) + elif before is _NEW: + # A place that joins with a value counts as a change; an empty one receives. + if current[p] is not None and raw[p] != to_place(fkey, p, record.get("c")): + changed.append(p) + elif current[p] != before: + changed.append(p) + if not changed: + return record.get("c"), {}, record.get("t") + return _pick(fkey, changed, current, raw, times) + + +def plan(snap, state): + """Compute all writes that bring every place to the agreed values. + + Parameters + ---------- + snap : dict + Snapshot of all places. + state : dict + State of the previous run (``{}`` on the first run). + + Returns + ------- + Plan + Writes, new state, conflicts and skipped values. + + Notes + ----- + An issue that is on no board for ``config.GONE_DAYS`` days (counted from + ``snap["read_at"]``) is dropped from the state. + """ + result = Plan(state={"items": {}, "fields": field_ids(snap, state.get("fields"))}) + reset = reset_places(snap, state) + result.resets = sorted(reset) + old = state.get("items", {}) + now = snap.get("read_at") + for iid, entry in old.items(): + if iid not in snap["items"]: + # Not readable or on no board this run: keep its records, every place gone. + kept = { + k: {**r, "gone": sorted(r.get("seen", {})), "gone_since": r.get("gone_since") or now} + for k, r in entry.items() + } + since = min((r["gone_since"] for r in kept.values() if r["gone_since"]), default=None) + if not (now and since and _days(since, now) > config.GONE_DAYS): + result.state["items"][iid] = kept + for iid, item in sorted(snap["items"].items()): + new_entry = {} + for fkey, spec in config.FIELDS.items(): + places = scope(snap, item, fkey) + if not places: + if fkey in old.get(iid, {}): + # The field is in no place: every place keeps its record and counts as gone. + r = old[iid][fkey] + new_entry[fkey] = {k: v for k, v in r.items() if k != "gone_since"} | { + "gone": sorted(r.get("seen", {})) + } + continue + raw, current, times = {}, {}, {} + for p in places: + raw[p], times[p] = raw_value(item, p, spec["places"][p]) + current[p] = to_canon(fkey, p, raw[p]) + record = old.get(iid, {}).get(fkey) + if record is not None: + record = { + "c": record.get("c"), + "seen": { + p: v for p, v in record.get("seen", {}).items() if (place_key(item, p), fkey) not in reset + }, + "gone": record.get("gone", []), + "t": record.get("t"), + } + c, clash, t = decide(fkey, places, current, raw, times, record) + if clash: + result.conflicts.append({"issue": iid, "url": item["url"], "fkey": fkey, "values": clash, "chosen": c}) + seen = {} + for p in places: + target = to_place(fkey, p, c) + seen[p] = current[p] + if raw[p] == target: + continue + fdef = place_fields(snap, item, p)[spec["places"][p]] + if target is not None and spec["type"] == "select" and target not in fdef.get("options", {}): + result.skipped.append({"issue": iid, "url": item["url"], "place": p, "fkey": fkey, "value": target}) + continue + result.writes.append( + { + "issue": iid, + "url": item["url"], + "place": p, + "fkey": fkey, + "field": spec["places"][p], + "value": target, + "prev": current[p], + } + ) + seen[p] = to_canon(fkey, p, target) + # Places out of scope keep their record; those without the issue are marked gone. + kept = {p: v for p, v in (record or {}).get("seen", {}).items() if p not in places} + # A place stays gone until it holds the issue and the field again. + gone = sorted(p for p in kept if p not in item["places"] or p in (record or {}).get("gone", ())) + new_entry[fkey] = {"c": c, "t": t, "seen": {**kept, **seen}, "gone": gone} + _derive(snap, item, iid, new_entry, result) + if new_entry: + result.state["items"][iid] = new_entry + return result + + +def _days(start, end): + """Days between two ISO times.""" + return (datetime.fromisoformat(end) - datetime.fromisoformat(start)) / timedelta(days=1) + + +def _derive(snap, item, iid, entry, result): + """Add writes for derived fields such as Effort from Workload.""" + for spec in config.DERIVED.values(): + source = entry.get(spec["source"], {}).get("c") + place = spec["place"] + fdef = place_fields(snap, item, place).get(spec["field"]) if place in item["places"] else None + if source is None or fdef is None: + continue + target = spec["map"].get(source) + if target is None: + continue + if target not in fdef.get("options", {}): + result.skipped.append( + {"issue": iid, "url": item["url"], "place": place, "fkey": spec["field"], "value": target} + ) + continue + if raw_value(item, place, spec["field"])[0] != target: + result.writes.append( + { + "issue": iid, + "url": item["url"], + "place": place, + "fkey": None, + "field": spec["field"], + "value": target, + "prev": None, + } + ) + + +def undo_failed(state, failed): + """Restore the state records of writes that failed. + + Parameters + ---------- + state : dict + New state from ``plan``; changed in place. + failed : list of dict + Write entries that the API refused. + + Returns + ------- + dict + The same state object. + """ + for w in failed: + if w["fkey"] is None: + continue + rec = state["items"].get(w["issue"], {}).get(w["fkey"]) + if rec is not None: + rec["seen"][w["place"]] = w["prev"] + return state diff --git a/board_sync/github.py b/board_sync/github.py new file mode 100644 index 0000000..6fbe44d --- /dev/null +++ b/board_sync/github.py @@ -0,0 +1,405 @@ +"""Read and write the GitHub places: project boards and org issue fields.""" + +import http.client +import json +import time +import urllib.error +import urllib.request +from datetime import UTC, datetime + +from . import config + +API = "https://api.github.com/graphql" + +_FIELD_NAME = ( + "... on IssueFieldDate { name } ... on IssueFieldSingleSelect { name } " + "... on IssueFieldText { name } ... on IssueFieldNumber { name } " + "... on IssueFieldMultiSelect { name }" +) + +BOARD_QUERY = """ +query($owner: String!, $number: Int!, $after: String) { + organization(login: $owner) { projectV2(number: $number) { id + fields(first: 100) { pageInfo { hasNextPage } nodes { + ... on ProjectV2FieldCommon { id name dataType } + ... on ProjectV2SingleSelectField { options { id name } } } } + items(first: 100, after: $after) { pageInfo { hasNextPage endCursor } nodes { id + content { ... on Issue { id url repository { owner { login } } } } + fieldValues(first: 100) { pageInfo { hasNextPage } nodes { + ... on ProjectV2ItemFieldDateValue { date updatedAt field { ... on ProjectV2FieldCommon { name } } } + ... on ProjectV2ItemFieldSingleSelectValue { name updatedAt field { ... on ProjectV2FieldCommon { name } } } + } } } } } } +} +""" + +ISSUE_FIELDS_QUERY = """ +query($org: String!) { organization(login: $org) { issueFields(first: 100) { pageInfo { hasNextPage } nodes { + ... on IssueFieldDate { id name } + ... on IssueFieldSingleSelect { id name options { id name } } } } } } +""" + +ISSUE_VALUES_QUERY = ( + """ +query($ids: [ID!]!) { nodes(ids: $ids) { ... on Issue { id + issueFieldValues(first: 50) { pageInfo { hasNextPage } nodes { + ... on IssueFieldDateValue { value field { ... on IssueFieldDate { name } } } + ... on IssueFieldSingleSelectValue { name field { ... on IssueFieldSingleSelect { name } } } } } + timelineItems(last: 100, itemTypes: [ISSUE_FIELD_ADDED_EVENT, ISSUE_FIELD_CHANGED_EVENT, + ISSUE_FIELD_REMOVED_EVENT]) { nodes { + __typename + ... on IssueFieldAddedEvent { createdAt issueField { __FIELD_NAME__ } } + ... on IssueFieldChangedEvent { createdAt issueField { __FIELD_NAME__ } } + ... on IssueFieldRemovedEvent { createdAt issueField { __FIELD_NAME__ } } } } } } +} +""" +).replace("__FIELD_NAME__", _FIELD_NAME) + + +class GraphQLError(RuntimeError): + """The GraphQL API returned errors.""" + + +# Error paths of a missing board or org; any other NOT_FOUND in a board read stops the read. +BOARD_GONE = (("organization",), ("organization", "projectV2")) + + +class BoardNotFound(RuntimeError): + """A configured board does not exist or the token cannot see it.""" + + +class TruncatedRead(RuntimeError): + """A list in a response was longer than the request asked for.""" + + +def _check_page(conn, what): + """Raise ``TruncatedRead`` if a connection has more nodes than were read.""" + if conn["pageInfo"]["hasNextPage"]: + raise TruncatedRead(what) + + +class Client: + """Minimal GraphQL client for one token. + + Parameters + ---------- + token : str + GitHub token (installation token or personal token). + opener : callable, optional + ``urllib.request.urlopen`` or a test replacement. + timeout : float + Seconds to wait for one response. + """ + + def __init__(self, token, opener=urllib.request.urlopen, timeout=60): + self.token = token + self.opener = opener + self.timeout = timeout + + def post(self, query, variables=None, retries=2, allow_not_found=False): + """Run one GraphQL request and return its ``data``. + + Parameters + ---------- + query : str + GraphQL document. + variables : dict, optional + Query variables. + retries : int + Retries after an HTTP 502, 503 or 504, a network error, or a rate + limit (HTTP 403 or 429, or a ``RATE_LIMITED`` error). + allow_not_found : bool or tuple + Return the partial ``data`` when every error is of type + ``NOT_FOUND`` (for example a deleted issue in a ``nodes`` list). + A tuple of paths further limits this to errors at those paths. + + Returns + ------- + dict + The ``data`` member of the response. + + Raises + ------ + GraphQLError + If the response holds errors. + urllib.error.URLError, TimeoutError, OSError, http.client.HTTPException + If the request still fails after the retries. + """ + body = json.dumps({"query": query, "variables": variables or {}}).encode() + req = urllib.request.Request( + API, + data=body, + headers={"Authorization": f"Bearer {self.token}", "Content-Type": "application/json"}, + ) + try: + with self.opener(req, timeout=self.timeout) as resp: + out = json.loads(resp.read()) + except urllib.error.HTTPError as exc: + try: + text = exc.read(65536).decode("utf-8", "replace") if exc.fp else "" + except (OSError, http.client.HTTPException): + text = "" + exc.close() + if retries > 0 and exc.code in (502, 503, 504): + time.sleep(5) + return self.post(query, variables, retries - 1, allow_not_found) + wait = _retry_after((exc.headers or {}).get("Retry-After")) + limited = exc.code == 429 or (exc.code == 403 and (wait is not None or "rate limit" in text.lower())) + if retries > 0 and limited: + time.sleep(60 if wait is None else wait) + return self.post(query, variables, retries - 1, allow_not_found) + raise + except (urllib.error.URLError, TimeoutError, ConnectionError, http.client.HTTPException): + if retries > 0: + time.sleep(5) + return self.post(query, variables, retries - 1, allow_not_found) + raise + errors = out.get("errors") + if errors and retries > 0 and any(e.get("type") == "RATE_LIMITED" for e in errors): + time.sleep(60) + return self.post(query, variables, retries - 1, allow_not_found) + if errors: + paths = allow_not_found if isinstance(allow_not_found, tuple) else None + found = all( + e.get("type") == "NOT_FOUND" and (paths is None or tuple(e.get("path") or ()) in paths) for e in errors + ) + if allow_not_found and out.get("data") is not None and found: + return out["data"] + raise GraphQLError(json.dumps(errors)[:2000]) + return out["data"] + + +def _retry_after(value): + """Return the ``Retry-After`` wait in seconds (0 to 120), or ``None`` if absent or not a number.""" + try: + return max(0, min(int(value), 120)) + except (TypeError, ValueError): + return None + + +def _field_type(data_type): + """Map a GraphQL field data type to ``date``, ``select`` or ``other``.""" + return {"DATE": "date", "SINGLE_SELECT": "select"}.get(data_type, "other") + + +def read_board(client, owner, number): + """Read the fields and the issue items of one project board. + + Parameters + ---------- + client : Client + Client with read access to the board. + owner : str + Org login that owns the board. + number : int + Board number. + + Returns + ------- + tuple + Project id, field definitions by name, items as a list of + ``(issue_id, url, org, item_id, values)``, and the number + of items skipped because they are not issues or cannot be read. + + Raises + ------ + BoardNotFound + If the board or its org does not exist or the token cannot see it. + TruncatedRead + If a board has more fields, or an item more values, than one page. + """ + after, items, fields, pid, skipped = None, [], None, None, 0 + while True: + # A missing board or org comes back as null data with a NOT_FOUND error. + data = client.post(BOARD_QUERY, {"owner": owner, "number": number, "after": after}, allow_not_found=BOARD_GONE) + proj = (data.get("organization") or {}).get("projectV2") + if proj is None: + raise BoardNotFound(f"{owner} project {number}") + pid = proj["id"] + if fields is None: + _check_page(proj["fields"], f"fields of board {number}") + fields = { + f["name"]: { + "id": f["id"], + "type": _field_type(f["dataType"]), + "options": {o["name"]: o["id"] for o in f.get("options", [])}, + } + for f in proj["fields"]["nodes"] + if f + } + for node in proj["items"]["nodes"]: + content = (node or {}).get("content") or {} + if "id" not in content: + skipped += 1 # draft issue, pull request, or an issue the token cannot read + continue + _check_page(node["fieldValues"], f"values of an item on board {number}") + values = {} + for v in node["fieldValues"]["nodes"]: + if v and v.get("field"): + values[v["field"]["name"]] = [v.get("date", v.get("name")), v.get("updatedAt")] + org = content["repository"]["owner"]["login"] + items.append((content["id"], content["url"], org, node["id"], values)) + page = proj["items"]["pageInfo"] + if not page["hasNextPage"]: + return pid, fields, items, skipped + after = page["endCursor"] + + +def read_issue_fields(client, org): + """Read the org issue field definitions. + + Parameters + ---------- + client : Client + Client with access to the org. + org : str + Org login. + + Returns + ------- + dict + Field name to definition. + """ + conn = client.post(ISSUE_FIELDS_QUERY, {"org": org})["organization"]["issueFields"] + _check_page(conn, f"issue fields of {org}") + return { + n["name"]: { + "id": n["id"], + "type": "select" if "options" in n else "date", + "options": {o["name"]: o["id"] for o in n.get("options", [])}, + } + for n in conn["nodes"] + if n + } + + +def read_issue_values(client, ids, batch=25): + """Read org issue field values and their change times. + + A field whose latest event is a removal reads as ``[None, removed_at]``, + so a clear has its own time. + + Parameters + ---------- + client : Client + Client of the org that owns the issues. + ids : list of str + Issue node ids. + batch : int + Issues per request. + + Returns + ------- + tuple + Issue id to ``{field name: [value, changed_at]}``, and the number of + issues that could not be read. + """ + out, unread = {}, 0 + for i in range(0, len(ids), batch): + data = client.post(ISSUE_VALUES_QUERY, {"ids": ids[i : i + batch]}, allow_not_found=True) + for node in data["nodes"]: + if not node: + unread += 1 + continue + _check_page(node["issueFieldValues"], "issue field values") + times, removed = {}, set() + for ev in node["timelineItems"]["nodes"]: + name = ((ev or {}).get("issueField") or {}).get("name") + if name and ev["createdAt"] >= times.get(name, ""): + times[name] = ev["createdAt"] + if ev.get("__typename") == "IssueFieldRemovedEvent": + removed.add(name) + else: + removed.discard(name) + values = {name: [None, times[name]] for name in removed} + for v in node["issueFieldValues"]["nodes"]: + if v and v.get("field"): + name = v["field"]["name"] + values[name] = [v.get("value", v.get("name")), times.get(name)] + out[node["id"]] = values + return out, unread + + +def read_all(clients): + """Read every place into one snapshot. + + Parameters + ---------- + clients : dict + Org login to ``Client``. + + Returns + ------- + dict + Snapshot with keys ``fields``, ``projects``, ``items``, + ``skipped_items`` (per board) and ``unread_issues`` (per org). + """ + snap = { + "fields": {}, + "projects": {}, + "items": {}, + "skipped_items": {}, + "unread_issues": {}, + "read_at": datetime.now(UTC).isoformat(timespec="seconds"), + } + for key, board in config.BOARDS.items(): + pid, fields, items, skipped = read_board(clients[board["owner"]], board["owner"], board["number"]) + snap["projects"][key] = pid + snap["fields"][key] = fields + snap["skipped_items"][key] = skipped + for iid, url, org, item_id, values in items: + entry = snap["items"].setdefault(iid, {"url": url, "org": org, "places": {}}) + entry["places"][key] = {"item_id": item_id, "values": values} + for org in config.ISSUE_ORGS: + snap["fields"][f"issue:{org}"] = read_issue_fields(clients[org], org) + ids = sorted(i for i, e in snap["items"].items() if e["org"] == org) + values_by_issue, snap["unread_issues"][org] = read_issue_values(clients[org], ids) + for iid, values in values_by_issue.items(): + snap["items"][iid]["places"]["issue"] = {"values": values} + return snap + + +def write(clients, snap, w): + """Apply one planned write. + + Parameters + ---------- + clients : dict + Org login to ``Client``. + snap : dict + Snapshot the plan was made from. + w : dict + One entry of ``Plan.writes``. + """ + item = snap["items"][w["issue"]] + if w["place"] == "issue": + org = item["org"] + fdef = snap["fields"][f"issue:{org}"][w["field"]] + if w["value"] is None: + q = "mutation($i: ID!, $f: ID!) { deleteIssueFieldValue(input: {issueId: $i, fieldId: $f}) { clientMutationId } }" + clients[org].post(q, {"i": w["issue"], "f": fdef["id"]}) + return + val = {"fieldId": fdef["id"]} + if fdef["type"] == "date": + val["dateValue"] = w["value"] + else: + val["singleSelectOptionId"] = fdef["options"][w["value"]] + q = "mutation($i: ID!, $v: [IssueFieldCreateOrUpdateInput!]!) { setIssueFieldValue(input: {issueId: $i, issueFields: $v}) { clientMutationId } }" + clients[org].post(q, {"i": w["issue"], "v": [val]}) + return + board = config.BOARDS[w["place"]] + fdef = snap["fields"][w["place"]][w["field"]] + args = { + "p": snap["projects"][w["place"]], + "i": item["places"][w["place"]]["item_id"], + "f": fdef["id"], + } + if w["value"] is None: + q = "mutation($p: ID!, $i: ID!, $f: ID!) { clearProjectV2ItemFieldValue(input: {projectId: $p, itemId: $i, fieldId: $f}) { clientMutationId } }" + clients[board["owner"]].post(q, args) + return + if fdef["type"] == "date": + args["v"] = {"date": w["value"]} + else: + args["v"] = {"singleSelectOptionId": fdef["options"][w["value"]]} + q = "mutation($p: ID!, $i: ID!, $f: ID!, $v: ProjectV2FieldValue!) { updateProjectV2ItemFieldValue(input: {projectId: $p, itemId: $i, fieldId: $f, value: $v}) { clientMutationId } }" + clients[board["owner"]].post(q, args) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..10c9fbb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "board-sync" +version = "0.1.0" +description = "Keep dates, priority, workload and status equal across the Interra boards, the PROTEUS board and the org issue fields" +requires-python = ">=3.11" + +[tool.ruff] +line-length = 120 + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] + +[tool.coverage.run] +source = ["board_sync"] +branch = true diff --git a/scripts/store_state.sh b/scripts/store_state.sh new file mode 100755 index 0000000..2d88861 --- /dev/null +++ b/scripts/store_state.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Store the state, snapshot and report of a sync run in the state repository. +# Usage: store_state.sh +# The files of this run replace the stored ones whole; other files stay. +set -u +dir=$1 +run=$2 +# The checkout is absent, or not a repository, when an earlier step failed. +if ! cd "$dir" 2>/dev/null || [ ! -d .git ]; then + echo "no state checkout, nothing stored" + exit 0 +fi +keep=$(mktemp -d) +trap 'rm -rf "$keep"' EXIT +for f in state.json snapshot.json report.txt; do + if [ -f "$f" ]; then cp "$f" "$keep/"; fi +done +for attempt in 1 2 3; do + if git fetch -q origin main && git reset -q --hard FETCH_HEAD; then + cp "$keep"/* . 2>/dev/null || true + for f in state.json snapshot.json report.txt; do + if [ -f "$f" ] && ! git add "$f"; then + echo "state store failed: cannot stage $f" + exit 1 + fi + done + if git diff --cached --quiet; then + echo "state unchanged" + exit 0 + fi + if ! git commit -q -m "Sync run $run"; then + echo "state store failed: cannot commit" + exit 1 + fi + if git push -q origin HEAD:main; then + exit 0 + fi + fi + sleep "${STORE_RETRY_WAIT:-10}" +done +echo "state push failed after 3 attempts" +exit 1 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..d80e0e8 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,71 @@ +"""Shared builders for snapshots in the tests.""" + +import copy + +import pytest + +DATE = {"type": "date", "options": {}} + + +def select(*names): + """Return a select field definition with the given option names.""" + return {"type": "select", "options": {n: f"opt-{n}" for n in names}} + + +def fields(ids=None): + """Return field definitions for all places, with optional id overrides.""" + ids = ids or {} + status = select("In Progress", "Next Up", "TBD", "Done", "Import", "Starter Projects") + board = { + "Start date": dict(DATE), + "Target date": dict(DATE), + "Priority": select("Priority 1: critical", "Priority 2: high", "Priority 3: standard", "Priority 4: low"), + "Workload": select("XL", "L", "M", "S", "XS"), + "Category": select("Science", "Tech"), + "Sub-team": select("PIs", "Co-leads"), + "Release": select("Interra 2027"), + "Status": status, + } + b1 = dict(board) + b1["Priority"] = select("Priority: critical", "Priority: high", "Priority: normal", "Priority: low") + b1["Sub-team"] = select("PIs", "Co-Leads") + b1["Status"] = select("In Progress", "Next Up", "TBD", "Done") + b7 = {k: board[k] for k in ("Start date", "Target date", "Priority", "Workload")} + issue = { + "Start date": dict(DATE), + "Target date": dict(DATE), + "Priority": select("Urgent", "High", "Medium", "Low"), + "Effort": select("High", "Medium", "Low"), + } + out = {"b1": b1, "b2": board, "b7": b7, "issue:Interra-Project": issue, "issue:FormingWorlds": issue} + out = { + k: {n: {**copy.deepcopy(f), "id": ids.get((k, n), f"{k}/{n}")} for n, f in v.items()} for k, v in out.items() + } + return out + + +def item(org="Interra-Project", **places): + """Return one snapshot item; each keyword is a place with ``{field: (value, at)}``.""" + return { + "url": f"https://github.com/{org}/repo/issues/1", + "org": org, + "places": { + p: ({"values": {k: list(v) for k, v in vals.items()}} | ({} if p == "issue" else {"item_id": f"item-{p}"})) + for p, vals in places.items() + }, + } + + +def snap(items, ids=None): + """Return a snapshot with the given items keyed ``I1``, ``I2``, ...""" + return { + "fields": fields(ids), + "projects": {"b1": "P1", "b2": "P2", "b7": "P7"}, + "items": {f"I{n + 1}": it for n, it in enumerate(items)}, + } + + +@pytest.fixture +def mk(): + """Expose the builders to the tests.""" + return type("Builders", (), {"item": staticmethod(item), "snap": staticmethod(snap)}) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..b8f301d --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,180 @@ +"""Tests of the command line: dry run, apply, limits and failures.""" + +import argparse +import io +import json +import time + +import pytest + +import board_sync.__main__ as cli +from board_sync import github + + +def test_make_clients_reads_env(): + clients = cli.make_clients({"BOARD_SYNC_TOKEN_INTERRA_PROJECT": "a", "GH_TOKEN": "g"}) + assert clients["Interra-Project"].token == "a" + assert clients["FormingWorlds"].token == "g" + with pytest.raises(SystemExit): + cli.make_clients({}) + + +def test_load_state(tmp_path): + p = tmp_path / "s.json" + assert cli.load_state(p) == {} + p.write_text(" ") + assert cli.load_state(p) == {} + p.write_text('{"items": {}}') + assert cli.load_state(p) == {"items": {}} + + +@pytest.fixture +def world(mk, monkeypatch): + """Patch the reads to a snapshot with 2 writes due, and record writes.""" + s = mk.snap([mk.item(b1={}, b2={"Category": ("Tech", "t1"), "Start date": ("2026-01-01", "t1")})]) + monkeypatch.setattr(github, "read_all", lambda clients: s) + done = [] + + def fake_write(clients, snap, w): + done.append(w) + snap["items"][w["issue"]]["places"][w["place"]]["values"][w["field"]] = [w["value"], "t9"] + + monkeypatch.setattr(github, "write", fake_write) + return done + + +def args(tmp_path, **kw): + base = { + "state": str(tmp_path / "state.json"), + "snapshot_dir": None, + "apply": False, + "max_changes": 20, + "allow_bulk": False, + "limit": 400, + "pause": 0.0, + "deadline": 600.0, + "started": time.monotonic(), + "summary": False, + "report": None, + } + return argparse.Namespace(**{**base, **kw}) + + +def test_dry_run_writes_nothing(world, tmp_path): + out = io.StringIO() + assert cli.run(args(tmp_path, snapshot_dir=str(tmp_path / "snap")), {}, out) == 0 + assert world == [] + assert not (tmp_path / "state.json").exists() + assert (tmp_path / "snap" / "snapshot.json").exists() + text = out.getvalue() + assert "writes: 2" in text and "plan " in text and "Interra Management / Category: Tech" in text + assert " 1 Interra Management / Category" in text + + +def test_apply_writes_and_saves_state(world, tmp_path): + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True), {}, out) == 0 + assert len(world) == 2 + state = json.loads((tmp_path / "state.json").read_text()) + assert state["items"]["I1"]["category"]["seen"]["b1"] == "Tech" + + +def test_apply_stops_above_limit(world, tmp_path): + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True, max_changes=1), {}, out) == 3 + assert world == [] and "stop: 2 writes" in out.getvalue() + assert cli.run(args(tmp_path, apply=True, max_changes=1, allow_bulk=True), {}, io.StringIO()) == 0 + assert len(world) == 2 + + +def test_failed_write_is_retried_next_run(mk, monkeypatch, tmp_path): + s = mk.snap([mk.item(b1={}, b2={"Category": ("Tech", "t1")})]) + monkeypatch.setattr(github, "read_all", lambda clients: s) + + def refuse(clients, snap, w): + raise github.GraphQLError("no") + + monkeypatch.setattr(github, "write", refuse) + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True), {}, out) == 1 + assert "failed " in out.getvalue() + state = json.loads((tmp_path / "state.json").read_text()) + assert state["items"]["I1"]["category"]["seen"]["b1"] is None + + +def test_report_lists_conflicts_and_skips(mk, monkeypatch, tmp_path): + s = mk.snap( + [ + mk.item( + b1={"Category": ("Tech", "t1")}, b2={"Category": ("Science", "t2"), "Release": ("Interra 2099", "t2")} + ) + ] + ) + s["fields"]["b2"]["Release"]["options"]["Interra 2099"] = "z" + monkeypatch.setattr(github, "read_all", lambda clients: s) + out = io.StringIO() + cli.run(args(tmp_path), {}, out) + assert "conflict " in out.getvalue() and "skipped " in out.getvalue() + + +def test_describe_clear(): + w = {"url": "u", "place": "issue", "field": "Priority", "value": None} + assert cli.describe(w) == "u issue field / Priority: (clear)" + + +def test_main_parses(monkeypatch): + seen = {} + monkeypatch.setattr(cli, "make_clients", lambda env: {"c": 1}) + monkeypatch.setattr(cli, "run", lambda a, c: seen.update(a=a, c=c) or 0) + assert cli.main(["--apply", "--max-changes", "5", "--state", "s.json"], env={}) == 0 + assert seen["a"].apply and seen["a"].max_changes == 5 and seen["c"] == {"c": 1} and not seen["a"].summary + assert cli.main([], env={"GITHUB_ACTIONS": "true"}) == 0 + a = seen["a"] + assert (a.max_changes, a.limit, a.pause, a.summary, a.apply) == (20, 400, 0.0, True, False) + assert a.deadline == 600.0 and a.started > 0 + assert cli.main(["--deadline", "5"], env={}) == 0 and seen["a"].deadline == 5.0 + + +def test_limit_defers_the_rest_to_the_next_run(world, monkeypatch, tmp_path): + pauses = [] + monkeypatch.setattr(cli.time, "sleep", pauses.append) + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True, limit=1, pause=0.5), {}, out) == 0 + assert len(world) == 1 and pauses == [] and "deferred: 1 writes" in out.getvalue() + assert (world[0]["place"], world[0]["field"]) == ("b1", "Start date") + world.clear() + # The deferred place keeps its old record; a second plan writes it again. + assert cli.run(args(tmp_path, apply=True, limit=5, pause=0.5), {}, io.StringIO()) == 0 + assert [(w["place"], w["field"]) for w in world] == [("b1", "Category")] and pauses == [] + assert cli.run(args(tmp_path, apply=True), {}, io.StringIO()) == 0 + assert len(world) == 1 # nothing left to write + + +def test_pause_between_writes(world, monkeypatch, tmp_path): + pauses = [] + monkeypatch.setattr(cli.time, "sleep", pauses.append) + assert cli.run(args(tmp_path, apply=True, pause=0.5), {}, io.StringIO()) == 0 + assert len(world) == 2 and pauses == [0.5] + + +def test_summary_keeps_issue_data_out_of_the_log(world, tmp_path): + out = io.StringIO() + report = tmp_path / "report.txt" + assert cli.run(args(tmp_path, apply=True, summary=True, report=str(report)), {}, out) == 0 + assert "github.com" not in out.getvalue() and "writes: 2" in out.getvalue() + assert "plan https://github.com/Interra-Project/repo/issues/1 Interra Management / Category: Tech" in ( + report.read_text() + ) + + +def test_summary_counts_failures_by_kind(mk, monkeypatch, tmp_path): + s = mk.snap([mk.item(b1={}, b2={"Category": ("Tech", "t1")})]) + monkeypatch.setattr(github, "read_all", lambda clients: s) + + def refuse(clients, snap, w): + raise github.GraphQLError("secret detail") + + monkeypatch.setattr(github, "write", refuse) + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True, summary=True), {}, out) == 1 + assert "failed: 1 (GraphQLError 1)" in out.getvalue() and "secret detail" not in out.getvalue() diff --git a/tests/test_core.py b/tests/test_core.py new file mode 100644 index 0000000..c7d7ac5 --- /dev/null +++ b/tests/test_core.py @@ -0,0 +1,224 @@ +"""Tests of the sync decision.""" + +from board_sync import config, core + + +def writes(result): + """Return the writes as ``{(place, field): value}``.""" + return {(w["place"], w["field"]): w["value"] for w in result.writes} + + +def test_value_maps(): + assert core.to_place("priority", "b1", "Priority 3: standard") == "Priority: normal" + assert core.to_place("priority", "issue", "Priority 1: critical") == "Urgent" + assert core.to_place("start", "b1", "2026-01-01") == "2026-01-01" + assert core.to_place("status", "b1", None) is None + assert core.to_canon("priority", "b1", "Priority: normal") == "Priority 3: standard" + # TBD on board 1 stands for 3 canonical values, so it stays TBD. + assert core.to_canon("status", "b1", "TBD") == "TBD" + assert core.to_canon("subteam", "b1", "Co-Leads") == "Co-leads" + assert core.to_canon("status", "b1", None) is None + + +def test_scope_needs_item_and_field(mk): + s = mk.snap([mk.item(b2={}, issue={})]) + it = s["items"]["I1"] + assert core.scope(s, it, "start") == ["issue", "b2"] + del s["fields"]["b2"]["Start date"] + assert core.scope(s, it, "start") == ["issue"] + it["org"] = "someone" # a user repo has no org issue fields + assert core.scope(s, it, "start") == [] + + +def test_first_run_fills_empty_places(mk): + s = mk.snap([mk.item(b1={}, b2={"Target date": ("2026-12-31", "2026-10-01T00:00:00Z")}, b7={}, issue={})]) + r = core.plan(s, {}) + assert writes(r) == { + ("b1", "Target date"): "2026-12-31", + ("b7", "Target date"): "2026-12-31", + ("issue", "Target date"): "2026-12-31", + } + assert r.conflicts == [] + assert r.state["items"]["I1"]["target"]["c"] == "2026-12-31" + assert r.state["fields"]["b2"]["Target date"] == "b2/Target date" + + +def test_first_run_all_empty_writes_nothing(mk): + r = core.plan(mk.snap([mk.item(b1={}, b2={})]), {}) + assert r.writes == [] + assert r.state["items"]["I1"]["status"] == {"c": None, "t": None, "seen": {"b2": None, "b1": None}, "gone": []} + + +def test_first_run_newest_wins_with_conflict(mk): + s = mk.snap( + [ + mk.item( + b1={"Start date": ("2025-12-27", "2025-12-29T00:00:00Z")}, + b2={"Start date": ("2027-07-01", "2026-10-14T00:00:00Z")}, + ) + ] + ) + r = core.plan(s, {}) + assert writes(r) == {("b1", "Start date"): "2027-07-01"} + assert r.conflicts[0]["values"] == {"b2": "2027-07-01", "b1": "2025-12-27"} + + +def test_tie_goes_by_place_order(mk): + t = "2026-01-01T00:00:00Z" + s = mk.snap([mk.item(b1={"Category": ("Tech", t)}, b2={"Category": ("Science", t)})]) + assert writes(core.plan(s, {})) == {("b1", "Category"): "Science"} + + +def test_issue_time_unknown_loses_to_board(mk): + s = mk.snap( + [mk.item(b2={"Priority": ("Priority 2: high", "2026-01-01T00:00:00Z")}, issue={"Priority": ("Low", None)})] + ) + assert writes(core.plan(s, {})) == {("issue", "Priority"): "High"} + + +def test_lossy_status_agrees_without_conflict(mk): + s = mk.snap( + [ + mk.item( + b1={"Status": ("TBD", "2026-10-13T00:00:00Z")}, + b2={"Status": ("Starter Projects", "2026-01-01T00:00:00Z")}, + ) + ] + ) + r = core.plan(s, {}) + assert r.writes == [] and r.conflicts == [] + assert r.state["items"]["I1"]["status"]["c"] == "Starter Projects" + + +def test_mapped_values_agree(mk): + s = mk.snap( + [ + mk.item( + b1={"Priority": ("Priority: normal", "2026-01-02T00:00:00Z")}, + b2={"Priority": ("Priority 3: standard", "2026-01-01T00:00:00Z")}, + issue={"Priority": ("Medium", None)}, + ) + ] + ) + r = core.plan(s, {}) + assert r.writes == [] and r.conflicts == [] + + +def run_twice(mk, first, second): + """Plan ``first`` from scratch, then plan ``second`` against its state.""" + state = core.plan(mk.snap([first]), {}).state + return core.plan(mk.snap([second]), state) + + +def test_unchanged_second_run_writes_nothing(mk): + it = mk.item(b1={"Start date": ("2026-01-01", "t1")}, b2={"Start date": ("2026-01-01", "t1")}) + assert run_twice(mk, it, it).writes == [] + + +def test_change_at_one_place_propagates(mk): + a = mk.item(b1={"Start date": ("2026-01-01", "t1")}, b2={"Start date": ("2026-01-01", "t1")}) + b = mk.item(b1={"Start date": ("2026-01-01", "t1")}, b2={"Start date": ("2026-02-01", "t2")}) + r = run_twice(mk, a, b) + assert writes(r) == {("b1", "Start date"): "2026-02-01"} + assert r.state["items"]["I1"]["start"]["seen"] == {"b2": "2026-02-01", "b1": "2026-02-01"} + + +def test_clear_propagates(mk): + a = mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t1")}) + b = mk.item(b1={"Category": ("Tech", "t1")}, b2={}) + assert writes(run_twice(mk, a, b)) == {("b1", "Category"): None} + + +def test_two_changes_newest_wins(mk): + a = mk.item(b1={"Category": ("Tech", "2026-01-01")}, b2={"Category": ("Tech", "2026-01-01")}) + b = mk.item(b1={"Category": ("Science", "2026-03-01")}, b2={"Category": ("Management", "2026-02-01")}) + r = run_twice(mk, a, b) + assert writes(r) == {("b2", "Category"): "Science"} + assert r.conflicts[0]["chosen"] == "Science" + + +def test_new_place_empty_receives(mk): + a = mk.item(b2={"Workload": ("M", "t1")}, b1={"Workload": ("M", "t1")}) + b = mk.item(b2={"Workload": ("M", "t1")}, b1={"Workload": ("M", "t1")}, b7={}) + assert writes(run_twice(mk, a, b)) == {("b7", "Workload"): "M"} + + +def test_new_place_with_value_counts_as_change(mk): + a = mk.item(b2={"Workload": ("M", "t1")}, b1={"Workload": ("M", "t1")}) + b = mk.item(b2={"Workload": ("M", "t1")}, b1={"Workload": ("M", "t1")}, b7={"Workload": ("XL", "t9")}) + assert writes(run_twice(mk, a, b)) == {("b2", "Workload"): "XL", ("b1", "Workload"): "XL"} + + +def test_new_place_with_agreeing_value_is_no_change(mk): + a = mk.item(b2={"Workload": ("M", "t1")}, b1={"Workload": ("M", "t1")}) + b = mk.item(b2={"Workload": ("M", "t1")}, b1={"Workload": ("M", "t1")}, b7={"Workload": ("M", "t9")}) + assert run_twice(mk, a, b).writes == [] + + +def test_out_of_scope_places_keep_their_record(mk): + a = mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t1")}) + state = core.plan(mk.snap([a]), {}).state + s = mk.snap([mk.item(b1={"Category": ("Tech", "t1")}, b2={})]) + del s["fields"]["b2"]["Category"] # item still on board 2, field gone + rec = core.plan(s, state).state["items"]["I1"]["category"] + assert rec["seen"] == {"b1": "Tech", "b2": "Tech"} and rec["gone"] == [] + s = mk.snap([mk.item(b1={"Category": ("Tech", "t1")})]) # item left board 2 + rec = core.plan(s, state).state["items"]["I1"]["category"] + assert rec["seen"] == {"b1": "Tech", "b2": "Tech"} and rec["gone"] == ["b2"] + + +def test_recreated_field_does_not_clear(mk): + a = mk.item(b1={"Target date": ("2026-05-01", "t1")}, b2={"Target date": ("2026-05-01", "t1")}) + state = core.plan(mk.snap([a]), {}).state + s = mk.snap([mk.item(b1={"Target date": ("2026-05-01", "t1")}, b2={})], ids={("b2", "Target date"): "NEW"}) + r = core.plan(s, state) + assert writes(r) == {("b2", "Target date"): "2026-05-01"} + + +def test_mass_empty_field_does_not_clear(mk, monkeypatch): + monkeypatch.setattr(config, "RESET_MIN", 2) + full = [mk.item(b1={"Start date": ("2026-05-01", "t")}, b2={"Start date": ("2026-05-01", "t")}) for _ in range(2)] + state = core.plan(mk.snap(full), {}).state + empty = [mk.item(b1={"Start date": ("2026-05-01", "t")}, b2={}) for _ in range(2)] + r = core.plan(mk.snap(empty), state) + assert sorted(writes(r).values()) == ["2026-05-01"] + assert len(r.writes) == 2 + + +def test_missing_option_is_skipped(mk): + s = mk.snap([mk.item(b1={}, b2={"Release": ("Interra 2028+", "t1")})]) + s["fields"]["b2"]["Release"]["options"]["Interra 2028+"] = "x" + r = core.plan(s, {}) + assert r.writes == [] + assert r.skipped == [ + {"issue": "I1", "url": s["items"]["I1"]["url"], "place": "b1", "fkey": "release", "value": "Interra 2028+"} + ] + assert r.state["items"]["I1"]["release"]["seen"]["b1"] is None + + +def test_effort_follows_workload(mk): + s = mk.snap([mk.item(b2={"Workload": ("XL", "t1")}, issue={"Effort": ("Low", None)})]) + assert writes(core.plan(s, {})) == {("issue", "Effort"): "High"} + + +def test_effort_untouched_without_workload_or_match(mk): + s = mk.snap([mk.item(b2={}, issue={"Effort": ("Low", None)})]) + assert core.plan(s, {}).writes == [] + s = mk.snap([mk.item(b2={"Workload": ("S", "t1")}, issue={"Effort": ("Low", None)})]) + assert core.plan(s, {}).writes == [] + s = mk.snap([mk.item(b2={"Workload": ("S", "t1")}, issue={})]) + del s["fields"]["issue:Interra-Project"]["Effort"]["options"]["Low"] + assert core.plan(s, {}).writes == [] + s = mk.snap([mk.item(b2={"Workload": ("S", "t1")})]) # no issue place + assert core.plan(s, {}).writes == [] + + +def test_undo_failed_restores_seen(mk): + s = mk.snap([mk.item(b1={}, b2={"Category": ("Tech", "t1")}, issue={})]) + r = core.plan(s, {}) + failed = [w for w in r.writes if w["place"] == "b1"] + core.undo_failed(r.state, failed + [{"fkey": None}, {"fkey": "start", "issue": "I9", "place": "b1"}]) + assert r.state["items"]["I1"]["category"]["seen"]["b1"] is None + # The next run sees no change at b1 and writes the value again. + r2 = core.plan(s, r.state) + assert writes(r2) == {("b1", "Category"): "Tech"} diff --git a/tests/test_github.py b/tests/test_github.py new file mode 100644 index 0000000..6e565ff --- /dev/null +++ b/tests/test_github.py @@ -0,0 +1,429 @@ +"""Tests of the GraphQL reads and writes, with a fake transport.""" + +import http.client +import io +import json +import urllib.error + +import pytest + +from board_sync import github + +DONE = {"hasNextPage": False} +MORE = {"hasNextPage": True} + + +class Resp(io.BytesIO): + """File-like HTTP response usable as a context manager.""" + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def opener_for(*bodies): + """Return an opener that answers with ``bodies`` in order and records requests.""" + calls = [] + + def opener(req, timeout=None): + calls.append({**json.loads(req.data), "timeout": timeout, "auth": req.get_header("Authorization")}) + body = bodies[len(calls) - 1] + if isinstance(body, BaseException): + raise body + return Resp(json.dumps(body).encode()) + + opener.calls = calls + return opener + + +def http_error(code, headers=None, body=b""): + return urllib.error.HTTPError(github.API, code, "x", headers or {}, io.BytesIO(body)) + + +@pytest.fixture +def slept(monkeypatch): + out = [] + monkeypatch.setattr(github.time, "sleep", out.append) + return out + + +def test_post_returns_data_and_sends_token(): + op = opener_for({"data": {"ok": 1}}) + assert github.Client("tok", op).post("q", {"a": 1}) == {"ok": 1} + assert op.calls == [{"query": "q", "variables": {"a": 1}, "timeout": 60, "auth": "Bearer tok"}] + + +def test_post_raises_on_errors(): + with pytest.raises(github.GraphQLError): + github.Client("t", opener_for({"errors": [{"message": "bad"}]})).post("q") + + +def test_post_retries_server_errors_and_rate_limits(slept): + assert github.Client("t", opener_for(http_error(502), {"data": {"ok": 2}})).post("q") == {"ok": 2} + op = opener_for(http_error(403, {"Retry-After": "7"}), http_error(429, {"Retry-After": "999"}), {"data": {"ok": 3}}) + assert github.Client("t", op).post("q") == {"ok": 3} + op = opener_for(http_error(429, {"Retry-After": "Wed, 21 Oct 2026 07:28:00 GMT"}), {"data": {"ok": 5}}) + assert github.Client("t", op).post("q") == {"ok": 5} + op = opener_for( + http_error(403, body=b'{"message": "You have exceeded a secondary rate limit"}'), {"data": {"ok": 6}} + ) + assert github.Client("t", op).post("q") == {"ok": 6} + op = opener_for({"errors": [{"type": "RATE_LIMITED"}]}, {"data": {"ok": 7}}) + assert github.Client("t", op).post("q") == {"ok": 7} + assert slept == [5, 7, 120, 60, 60, 60] + + +@pytest.mark.parametrize( + "failure", + [http_error(401), http_error(403, body=b'{"message": "Resource not accessible by integration"}')], +) +def test_post_does_not_retry_refusals(failure, slept): + op = opener_for(failure) + with pytest.raises(urllib.error.HTTPError): + github.Client("t", op).post("q") + assert len(op.calls) == 1 and slept == [] + + +def test_post_gives_up_after_the_retries(slept): + op = opener_for(*[http_error(503) for _ in range(3)]) + with pytest.raises(urllib.error.HTTPError): + github.Client("t", op).post("q") + assert len(op.calls) == 3 + with pytest.raises(github.GraphQLError): + github.Client("t", opener_for(*[{"errors": [{"type": "RATE_LIMITED"}]}] * 3)).post("q") + with pytest.raises(urllib.error.URLError): + github.Client("t", opener_for(*[urllib.error.URLError("down")] * 3)).post("q") + + +@pytest.mark.parametrize( + "failure", [urllib.error.URLError("reset"), TimeoutError(), ConnectionResetError(), http.client.IncompleteRead(b"")] +) +def test_post_retries_network_errors(failure, slept): + assert github.Client("t", opener_for(failure, {"data": {"ok": 4}})).post("q") == {"ok": 4} + + +def test_post_accepts_partial_not_found_only_when_asked(): + body = {"data": {"nodes": [None]}, "errors": [{"type": "NOT_FOUND", "message": "gone"}]} + assert github.Client("t", opener_for(body)).post("q", allow_not_found=True) == {"nodes": [None]} + with pytest.raises(github.GraphQLError): + github.Client("t", opener_for(body)).post("q") + for bad in ( + {"data": {"nodes": [None]}, "errors": [{"type": "FORBIDDEN"}]}, + {"data": {"nodes": [None]}, "errors": [{"type": "NOT_FOUND"}, {"type": "FORBIDDEN"}]}, + {"data": None, "errors": [{"type": "NOT_FOUND"}]}, + ): + with pytest.raises(github.GraphQLError): + github.Client("t", opener_for(bad)).post("q", allow_not_found=True) + + +class Fake: + """Client stand-in that answers by query name and records mutations and partial-read flags.""" + + def __init__(self, pages=None, fields=None, values=None, pid="PID"): + self.pages, self.fields, self.values = list(pages or []), fields or [], values or [] + self.sent, self.flags, self.pid, self.cursors = [], [], pid, [] + + def post(self, query, variables=None, allow_not_found=False): + if "projectV2(number" in query: + self.flags.append(("board", allow_not_found)) + self.cursors.append(variables["after"]) + return {"organization": {"projectV2": {**self.pages.pop(0), "id": self.pid}}} + if "issueFields(first" in query: + self.flags.append(("issue_fields", allow_not_found)) + return {"organization": {"issueFields": {"pageInfo": DONE, "nodes": self.fields}}} + if "nodes(ids" in query: + self.flags.append(("issue_values", allow_not_found)) + found = [v for v in self.values if v and v["id"] in variables["ids"]] + return {"nodes": found + [None] * (len(variables["ids"]) - len(found))} + self.sent.append((query.split("{")[1].split("(")[0].strip(), variables)) + return {} + + +def page(items, more=False, field_id="F1"): + return { + "fields": { + "pageInfo": DONE, + "nodes": [ + {"id": field_id, "name": "Start date", "dataType": "DATE"}, + {"id": "F2", "name": "Status", "dataType": "SINGLE_SELECT", "options": [{"id": "o1", "name": "TBD"}]}, + {"id": "F3", "name": "Title", "dataType": "TITLE"}, + {}, + ], + }, + "items": {"pageInfo": {"hasNextPage": more, "endCursor": "C1"}, "nodes": items}, + } + + +BOARD_VALUES = {"Start date": ["2026-01-01", "t1"], "Status": ["TBD", "t2"]} + + +def board_item(iid, org="FormingWorlds"): + return { + "id": f"item-{iid}", + "content": {"id": iid, "url": f"u/{iid}", "repository": {"owner": {"login": org}}}, + "fieldValues": { + "pageInfo": DONE, + "nodes": [ + {"date": "2026-01-01", "updatedAt": "t1", "field": {"name": "Start date"}}, + {"name": "TBD", "updatedAt": "t2", "field": {"name": "Status"}}, + {}, + ], + }, + } + + +def issue_node(iid, values=(), events=()): + return { + "id": iid, + "issueFieldValues": {"pageInfo": DONE, "nodes": list(values)}, + "timelineItems": {"nodes": list(events)}, + } + + +def test_read_board_pages_and_skips_drafts(): + draft = {"id": "d", "content": {}, "fieldValues": {"pageInfo": DONE, "nodes": []}} + fake = Fake(pages=[page([board_item("A"), draft, None], more=True), page([board_item("B")])]) + pid, fields, items, skipped = github.read_board(fake, "FormingWorlds", 7) + assert pid == "PID" and skipped == 2 and fake.cursors == [None, "C1"] + assert fields["Status"] == {"id": "F2", "type": "select", "options": {"TBD": "o1"}} + assert fields["Title"]["type"] == "other" + assert [i[0] for i in items] == ["A", "B"] + assert items[0] == ("A", "u/A", "FormingWorlds", "item-A", BOARD_VALUES) + + +def test_read_issue_fields_and_values(): + fake = Fake( + fields=[ + {"id": "D", "name": "Start date"}, + {"id": "S", "name": "Priority", "options": [{"id": "h", "name": "High"}]}, + {}, + ], + values=[ + issue_node( + "A", + values=[ + {"value": "2026-02-01", "field": {"name": "Start date"}}, + {"name": "High", "field": {"name": "Priority"}}, + {}, + ], + events=[ + {"createdAt": "2026-01-01", "issueField": {"name": "Start date"}}, + {"createdAt": "2026-03-01", "issueField": {"name": "Start date"}}, + {"createdAt": "2026-02-01", "issueField": {"name": "Start date"}}, + {"createdAt": "2026-02-01", "issueField": None}, + None, + ], + ) + ], + ) + assert github.read_issue_fields(fake, "o") == { + "Start date": {"id": "D", "type": "date", "options": {}}, + "Priority": {"id": "S", "type": "select", "options": {"High": "h"}}, + } + values, unread = github.read_issue_values(fake, ["A", "GONE"]) + assert values == {"A": {"Start date": ["2026-02-01", "2026-03-01"], "Priority": ["High", None]}} + assert unread == 1 + + +def test_only_the_issue_values_read_accepts_partial_responses(): + interra = Fake(pages=[page([board_item("X", "Interra-Project")]), page([])], values=[issue_node("X")]) + fw = Fake(pages=[page([])]) + github.read_all({"Interra-Project": interra, "FormingWorlds": fw}) + assert set(interra.flags + fw.flags) == { + ("board", github.BOARD_GONE), + ("issue_fields", False), + ("issue_values", True), + } + + +def test_read_issue_values_batches(): + calls = [] + + class C: + def post(self, q, v, allow_not_found=False): + calls.append(v["ids"]) + return {"nodes": []} + + github.read_issue_values(C(), ["a", "b", "c"], batch=2) + assert calls == [["a", "b"], ["c"]] + + +def test_read_all_maps_each_board_to_its_own_project(): + interra = Fake( + pages=[page([board_item("X", "Interra-Project")]), page([board_item("X", "Interra-Project")])], + fields=[{"id": "D", "name": "Start date"}], + values=[issue_node("X")], + pid="P-interra", + ) + fw = Fake(pages=[page([board_item("Y"), None], field_id="F7")], pid="P-fw") + snap = github.read_all({"Interra-Project": interra, "FormingWorlds": fw}) + assert snap["projects"] == {"b1": "P-interra", "b2": "P-interra", "b7": "P-fw"} + assert snap["fields"]["b7"]["Start date"]["id"] == "F7" and snap["fields"]["b1"]["Start date"]["id"] == "F1" + assert set(snap["items"]["X"]["places"]) == {"b1", "b2", "issue"} + assert snap["items"]["X"]["places"]["b2"] == {"item_id": "item-X", "values": BOARD_VALUES} + assert set(snap["items"]["Y"]["places"]) == {"b7"} # Y is unreadable in the FormingWorlds issue read + assert snap["skipped_items"] == {"b1": 0, "b2": 0, "b7": 1} + assert snap["unread_issues"] == {"Interra-Project": 0, "FormingWorlds": 1} + assert snap["fields"]["issue:Interra-Project"]["Start date"]["id"] == "D" + + +def test_removed_issue_field_reads_as_clear_with_time(): + node = issue_node( + "A", + values=[{"value": "2026-01-01", "field": {"name": "Start date"}}], + events=[ + {"__typename": "IssueFieldAddedEvent", "createdAt": "2026-01-01", "issueField": {"name": "Target date"}}, + {"__typename": "IssueFieldRemovedEvent", "createdAt": "2026-02-01", "issueField": {"name": "Target date"}}, + {"__typename": "IssueFieldRemovedEvent", "createdAt": "2026-01-01", "issueField": {"name": "Start date"}}, + {"__typename": "IssueFieldAddedEvent", "createdAt": "2026-03-01", "issueField": {"name": "Start date"}}, + ], + ) + values, _ = github.read_issue_values(Fake(values=[node]), ["A"]) + assert values == {"A": {"Target date": [None, "2026-02-01"], "Start date": ["2026-01-01", "2026-03-01"]}} + + +@pytest.mark.parametrize("where", ["fields", "values", "issue_fields", "issue_values"]) +def test_truncated_lists_stop_the_read(where): + if where in ("fields", "values"): + p = page([board_item("A")]) + if where == "fields": + p["fields"]["pageInfo"] = MORE + else: + p["items"]["nodes"][0]["fieldValues"]["pageInfo"] = MORE + with pytest.raises(github.TruncatedRead): + github.read_board(Fake(pages=[p]), "o", 1) + elif where == "issue_fields": + + class C: + def post(self, q, v): + return {"organization": {"issueFields": {"pageInfo": MORE, "nodes": []}}} + + with pytest.raises(github.TruncatedRead): + github.read_issue_fields(C(), "o") + else: + node = issue_node("A") + node["issueFieldValues"]["pageInfo"] = MORE + with pytest.raises(github.TruncatedRead): + github.read_issue_values(Fake(values=[node]), ["A"]) + + +def test_missing_page_info_is_an_error(): + p = page([board_item("A")]) + del p["fields"]["pageInfo"] + with pytest.raises(KeyError): + github.read_board(Fake(pages=[p]), "o", 1) + + +SNAP = { + "projects": {"b2": "P2"}, + "fields": { + "b2": { + "Start date": {"id": "FD", "type": "date"}, + "Status": {"id": "FS", "type": "select", "options": {"TBD": "o"}}, + }, + "issue:FormingWorlds": { + "Target date": {"id": "ID", "type": "date"}, + "Priority": {"id": "IP", "type": "select", "options": {"High": "h"}}, + }, + }, + "items": {"I": {"org": "FormingWorlds", "places": {"b2": {"item_id": "IT"}}}}, +} + + +@pytest.mark.parametrize( + ("w", "name", "variables"), + [ + ( + {"place": "b2", "field": "Start date", "value": "2026-01-01"}, + "updateProjectV2ItemFieldValue", + {"p": "P2", "i": "IT", "f": "FD", "v": {"date": "2026-01-01"}}, + ), + ( + {"place": "b2", "field": "Status", "value": "TBD"}, + "updateProjectV2ItemFieldValue", + {"p": "P2", "i": "IT", "f": "FS", "v": {"singleSelectOptionId": "o"}}, + ), + ( + {"place": "b2", "field": "Status", "value": None}, + "clearProjectV2ItemFieldValue", + {"p": "P2", "i": "IT", "f": "FS"}, + ), + ( + {"place": "issue", "field": "Target date", "value": "2026-02-02"}, + "setIssueFieldValue", + {"i": "I", "v": [{"fieldId": "ID", "dateValue": "2026-02-02"}]}, + ), + ( + {"place": "issue", "field": "Priority", "value": "High"}, + "setIssueFieldValue", + {"i": "I", "v": [{"fieldId": "IP", "singleSelectOptionId": "h"}]}, + ), + ({"place": "issue", "field": "Priority", "value": None}, "deleteIssueFieldValue", {"i": "I", "f": "IP"}), + ], +) +def test_write(w, name, variables): + fw, interra = Fake(), Fake() + clients = {"FormingWorlds": fw, "Interra-Project": interra} + github.write(clients, SNAP, {"issue": "I", **w}) + sent = fw.sent if w["place"] == "issue" else interra.sent + assert sent == [(name, variables)] + + +def test_retry_after_edge_values(slept): + op = opener_for(http_error(429, {"Retry-After": "0"}), http_error(429, {"Retry-After": "-5"}), {"data": {"ok": 1}}) + assert github.Client("t", op).post("q") == {"ok": 1} + assert slept == [0, 0] + assert github._retry_after(None) is None and github._retry_after("soon") is None + + +def test_rate_limit_text_in_any_case_and_unreadable_bodies(slept): + op = opener_for(http_error(403, body=b"API Rate Limit exceeded"), {"data": {"ok": 1}}) + assert github.Client("t", op).post("q") == {"ok": 1} + + class Broken(io.BytesIO): + def read(self, *a): + raise http.client.IncompleteRead(b"") + + err = urllib.error.HTTPError(github.API, 502, "x", {}, Broken()) + assert github.Client("t", opener_for(err, {"data": {"ok": 2}})).post("q") == {"ok": 2} + no_body = urllib.error.HTTPError(github.API, 503, "x", {}, None) + assert github.Client("t", opener_for(no_body, {"data": {"ok": 3}})).post("q") == {"ok": 3} + + +def test_missing_board_is_named(): + class C: + def post(self, q, v, allow_not_found=False): + return {"organization": {"projectV2": None}} + + with pytest.raises(github.BoardNotFound, match="Interra-Project project 9"): + github.read_board(C(), "Interra-Project", 9) + + class D: + def post(self, q, v, allow_not_found=False): + return {"organization": None} + + with pytest.raises(github.BoardNotFound): + github.read_board(D(), "Nobody", 1) + + +def test_missing_board_from_the_real_response_shape(): + body = { + "data": {"organization": {"projectV2": None}}, + "errors": [{"type": "NOT_FOUND", "path": ["organization", "projectV2"], "message": "Could not resolve"}], + } + with pytest.raises(github.BoardNotFound, match="Interra-Project project 999"): + github.read_board(github.Client("t", opener_for(body)), "Interra-Project", 999) + no_org = {"data": {"organization": None}, "errors": [{"type": "NOT_FOUND", "path": ["organization"]}]} + with pytest.raises(github.BoardNotFound): + github.read_board(github.Client("t", opener_for(no_org)), "no-such-org", 1) + + +def test_board_read_rejects_not_found_below_the_board(): + deep = { + "data": {"organization": {"projectV2": page([board_item("A")]) | {"id": "P"}}}, + "errors": [{"type": "NOT_FOUND", "path": ["organization", "projectV2", "items", "nodes", 0, "fieldValues"]}], + } + with pytest.raises(github.GraphQLError): + github.read_board(github.Client("t", opener_for(deep)), "o", 1) diff --git a/tests/test_store_state.py b/tests/test_store_state.py new file mode 100644 index 0000000..9326093 --- /dev/null +++ b/tests/test_store_state.py @@ -0,0 +1,133 @@ +"""Tests of the script that stores a run's files in the state repository, against local git repositories.""" + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "store_state.sh" +pytestmark = pytest.mark.skipif(not (shutil.which("git") and shutil.which("bash")), reason="needs git and bash") + +IDENTITY = { + "GIT_AUTHOR_NAME": "board-sync", + "GIT_AUTHOR_EMAIL": "board-sync@users.noreply.github.com", + "GIT_COMMITTER_NAME": "board-sync", + "GIT_COMMITTER_EMAIL": "board-sync@users.noreply.github.com", +} + + +def git(cwd, *args, env=None): + return subprocess.run(["git", *args], cwd=cwd, env=env, check=True, capture_output=True, text=True).stdout + + +@pytest.fixture +def repos(tmp_path): + """A bare remote with one README commit, a run checkout, and a clean environment without a git identity.""" + home = tmp_path / "home" + home.mkdir() + env = {"PATH": os.environ["PATH"], "HOME": str(home), "TMPDIR": str(tmp_path), "STORE_RETRY_WAIT": "0", **IDENTITY} + remote = tmp_path / "remote.git" + git(tmp_path, "init", "-q", "--bare", "-b", "main", str(remote), env=env) + seed = tmp_path / "seed" + git(tmp_path, "clone", "-q", str(remote), str(seed), env=env) + (seed / "README.md").write_text("readme\n") + git(seed, "add", ".", env=env) + git(seed, "commit", "-q", "-m", "init", env=env) + git(seed, "push", "-q", "origin", "HEAD:main", env=env) + run = tmp_path / "state" + git(tmp_path, "clone", "-q", "-b", "main", str(remote), str(run), env=env) + return {"tmp": tmp_path, "env": env, "remote": remote, "seed": seed, "run": run} + + +def store(r, run_id="1"): + return subprocess.run( + ["bash", str(SCRIPT), str(r["run"]), run_id], env=r["env"], capture_output=True, text=True, check=False + ) + + +def remote_files(r): + check = r["tmp"] / f"check{len(list(r['tmp'].glob('check*')))}" + git(r["tmp"], "clone", "-q", "-b", "main", str(r["remote"]), str(check), env=r["env"]) + return {p.name: p.read_text() for p in check.iterdir() if p.is_file()}, git( + check, "log", "--format=%s", env=r["env"] + ) + + +def test_first_run_without_a_state_file_stores_snapshot_and_report(repos): + (repos["run"] / "snapshot.json").write_text("{}\n") + (repos["run"] / "report.txt").write_text("stop: 30 writes\n") + assert store(repos).returncode == 0 + files, log = remote_files(repos) + assert files["report.txt"] == "stop: 30 writes\n" and "snapshot.json" in files and "state.json" not in files + assert log.splitlines()[0] == "Sync run 1" + + +def test_run_files_replace_a_changed_remote_whole_and_keep_other_files(repos): + (repos["seed"] / "state.json").write_text('{"v": "human"}\n') + (repos["seed"] / "README.md").write_text("readme 2\n") + git(repos["seed"], "add", ".", env=repos["env"]) + git(repos["seed"], "commit", "-q", "-m", "human", env=repos["env"]) + git(repos["seed"], "push", "-q", "origin", "HEAD:main", env=repos["env"]) + (repos["run"] / "state.json").write_text('{"v": "run"}\n') + (repos["run"] / "state.json.tmp").write_text("partial") + assert store(repos, "7").returncode == 0 + files, log = remote_files(repos) + assert files["state.json"] == '{"v": "run"}\n' and files["README.md"] == "readme 2\n" + assert "state.json.tmp" not in files and log.splitlines()[:2] == ["Sync run 7", "human"] + + +def test_nothing_changed_makes_no_commit(repos): + (repos["run"] / "report.txt").write_text("issues: 1\n") + assert store(repos).returncode == 0 + second = store(repos, "2") + assert second.returncode == 0 and "state unchanged" in second.stdout + assert len(remote_files(repos)[1].splitlines()) == 2 + + +def test_missing_checkout_is_not_an_error(repos): + result = subprocess.run(["bash", str(SCRIPT), str(repos["tmp"] / "absent"), "1"], env=repos["env"], check=False) + assert result.returncode == 0 + + +def test_push_failure_is_reported(repos): + (repos["run"] / "report.txt").write_text("x\n") + git(repos["run"], "remote", "set-url", "origin", str(repos["tmp"] / "nowhere.git"), env=repos["env"]) + result = store(repos) + assert result.returncode == 1 and "state push failed after 3 attempts" in result.stdout + + +def test_rejected_push_is_tried_again(repos): + hook = repos["remote"] / "hooks" / "pre-receive" + count = repos["tmp"] / "pushes" + hook.write_text( + f'#!/bin/sh\nn=$(cat "{count}" 2>/dev/null || echo 0)\necho $((n + 1)) > "{count}"\n[ "$n" -ge 1 ]\n' + ) + hook.chmod(0o755) + (repos["run"] / "report.txt").write_text("x\n") + result = store(repos) + assert result.returncode == 0 and count.read_text().strip() == "2" + files, log = remote_files(repos) + assert files["report.txt"] == "x\n" and log.splitlines()[0] == "Sync run 1" + + +def test_failed_commit_is_reported(repos): + hook = repos["run"] / ".git" / "hooks" / "pre-commit" + hook.write_text("#!/bin/sh\nexit 1\n") + hook.chmod(0o755) + (repos["run"] / "report.txt").write_text("x\n") + result = store(repos) + assert result.returncode == 1 and "cannot commit" in result.stdout + assert "report.txt" not in remote_files(repos)[0] + + +def test_a_folder_that_is_not_a_repository_is_left_alone(repos): + plain = repos["tmp"] / "plain" + plain.mkdir() + (plain / "report.txt").write_text("x\n") + result = subprocess.run( + ["bash", str(SCRIPT), str(plain), "1"], env=repos["env"], capture_output=True, text=True, check=False + ) + assert result.returncode == 0 and "no state checkout" in result.stdout + assert (plain / "report.txt").read_text() == "x\n" diff --git a/tests/test_sync_rules.py b/tests/test_sync_rules.py new file mode 100644 index 0000000..ae956d9 --- /dev/null +++ b/tests/test_sync_rules.py @@ -0,0 +1,779 @@ +"""Tests of the sync rules across runs: bulk approval, parking, returning places and time limits.""" + +import io +import json + +import pytest + +import board_sync.__main__ as cli +from board_sync import config, core, github +from tests.test_cli import args + + +def writes(result): + return {(w["place"], w["field"]): w["value"] for w in result.writes} + + +def test_bulk_drain_continues_on_scheduled_runs(mk, monkeypatch, tmp_path): + s = mk.snap( + [ + mk.item( + b1={}, + b2={ + "Category": ("Tech", "t1"), + "Start date": ("2026-01-01", "t1"), + "Target date": ("2026-02-01", "t1"), + }, + ) + ] + ) + monkeypatch.setattr(github, "read_all", lambda clients: s) + + done = [] + + def fake_write(clients, snap, w): + done.append(w["field"]) + snap["items"][w["issue"]]["places"][w["place"]]["values"][w["field"]] = [w["value"], "t9"] + + monkeypatch.setattr(github, "write", fake_write) + assert cli.run(args(tmp_path, apply=True, max_changes=1, limit=1, allow_bulk=True), {}, io.StringIO()) == 0 + # Scheduled runs (no allow_bulk) go on with the approved remainder until it is written. + assert cli.run(args(tmp_path, apply=True, max_changes=1, limit=1), {}, io.StringIO()) == 0 + assert cli.run(args(tmp_path, apply=True, max_changes=1, limit=1), {}, io.StringIO()) == 0 + assert sorted(done) == ["Category", "Start date", "Target date"] + + +def test_readded_item_receives_and_does_not_clear(mk): + a = mk.item(b2={"Priority": ("Priority 2: high", "t1")}, b7={"Priority": ("Priority 2: high", "t1")}) + state = core.plan(mk.snap([a]), {}).state + gone = mk.item(b2={"Priority": ("Priority 2: high", "t1")}) + state = core.plan(mk.snap([gone]), state).state + back = mk.item(b2={"Priority": ("Priority 2: high", "t1")}, b7={}) + assert writes(core.plan(mk.snap([back]), state)) == {("b7", "Priority"): "Priority 2: high"} + + +def test_field_missing_for_a_run_keeps_its_id(mk): + a = mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t1")}) + state = core.plan(mk.snap([a]), {}).state + s = mk.snap([mk.item(b1={"Category": ("Tech", "t1")}, b2={})]) + del s["fields"]["b2"]["Category"] + state = core.plan(s, state).state + s = mk.snap([mk.item(b1={"Category": ("Tech", "t1")}, b2={})], ids={("b2", "Category"): "NEW"}) + assert writes(core.plan(s, state)) == {("b2", "Category"): "Tech"} + + +def test_issue_field_clear_uses_its_removal_time(mk): + a = mk.item(b2={"Target date": ("2026-05-01", "2026-01-01")}, issue={"Target date": ("2026-05-01", None)}) + state = core.plan(mk.snap([a]), {}).state + b = mk.item(b2={"Target date": ("2026-06-01", "2026-02-01")}, issue={"Target date": (None, "2026-03-01")}) + assert writes(core.plan(mk.snap([b]), state)) == {("b2", "Target date"): None} + + +def test_board_clear_loses_to_a_value_set_elsewhere(mk): + a = mk.item(b1={"Category": ("Tech", "2026-01-01")}, b2={"Category": ("Tech", "2026-01-01")}) + state = core.plan(mk.snap([a]), {}).state + b = mk.item(b1={"Category": ("Science", "2026-02-01")}, b2={}) + r = core.plan(mk.snap([b]), state) + assert writes(r) == {("b2", "Category"): "Science"} and r.conflicts + alone = mk.item(b1={"Category": ("Tech", "2026-01-01")}, b2={}) + assert writes(core.plan(mk.snap([alone]), state)) == {("b1", "Category"): None} + + +def test_unreadable_place_returning_with_its_old_value_does_not_undo_a_newer_value(mk): + both = mk.item(b2={"Start date": ("2026-05-01", "t1")}, issue={"Start date": ("2026-05-01", "t1")}, b1={}) + s1 = mk.snap([both]) + state = core.plan(s1, {}).state + apply_to(s1, core.plan(s1, {})) + state = core.plan(s1, state).state + # Run 2: the issue is unreadable, and board 1 gets a new date. + s2 = mk.snap([mk.item(b2={"Start date": ("2026-05-01", "t1")}, b1={"Start date": ("2026-06-01", "t2")})]) + r2 = core.plan(s2, state) + assert writes(r2) == {("b2", "Start date"): "2026-06-01"} + apply_to(s2, r2) + # Run 3: the issue returns with its old date; the newer board date must win. + s3 = mk.snap( + [ + mk.item( + b2={"Start date": ("2026-06-01", "t3")}, + b1={"Start date": ("2026-06-01", "t2")}, + issue={"Start date": ("2026-05-01", "t1")}, + ) + ] + ) + assert writes(core.plan(s3, r2.state)) == {("issue", "Start date"): "2026-06-01"} + + +def test_issue_absent_from_every_board_keeps_its_records(mk): + a = mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t1")}) + state = core.plan(mk.snap([a]), {}).state + state = core.plan(mk.snap([]), state).state + assert state["items"]["I1"]["category"]["gone"] == ["b1", "b2"] + back = mk.item(b1={"Category": ("Tech", "t1")}, b2={}) + assert writes(core.plan(mk.snap([back]), state)) == {("b2", "Category"): "Tech"} + + +def test_field_on_no_place_keeps_its_record(mk): + a = mk.item(b1={"Status": ("In Progress", "t1")}, b2={"Status": ("In Progress", "t1")}, b7={}) + state = core.plan(mk.snap([a]), {}).state + s = mk.snap([mk.item(b7={})]) # only on board 7, which has no Status + state = core.plan(s, state).state + assert state["items"]["I1"]["status"]["c"] == "In Progress" + back = mk.item(b1={}, b2={"Status": ("In Progress", "t1")}, b7={}) + assert writes(core.plan(mk.snap([back]), state)) == {("b1", "Status"): "In Progress"} + + +def test_missing_fields_are_listed(mk): + s = mk.snap([]) + del s["fields"]["b7"]["Priority"] + del s["fields"]["issue:FormingWorlds"]["Target date"] + assert core.missing_fields(s) == [("b7", "Priority"), ("issue:FormingWorlds", "Target date")] + + +def test_reset_min_boundary(mk): + from board_sync import config + + n = config.RESET_MIN + for count, clears in ((n - 1, True), (n, False)): + full = [ + mk.item(b1={"Release": ("Interra 2027", "t")}, b2={"Release": ("Interra 2027", "t")}) for _ in range(count) + ] + state = core.plan(mk.snap(full), {}).state + empty = [mk.item(b1={"Release": ("Interra 2027", "t")}, b2={}) for _ in range(count)] + values = {w["value"] for w in core.plan(mk.snap(empty), state).writes} + assert values == ({None} if clears else {"Interra 2027"}) + + +def apply_to(snap, result): + """Apply planned writes to a snapshot as the API would (a clear removes the value).""" + for w in result.writes: + vals = snap["items"][w["issue"]]["places"][w["place"]]["values"] + if w["value"] is None: + vals.pop(w["field"], None) + else: + vals[w["field"]] = [w["value"], "t-new"] + + +def test_plans_converge(mk): + cases = [ + mk.item(b1={"Status": ("TBD", "t1")}, b2={"Status": ("Import", "t2")}), + mk.item(b1={"Priority": ("Priority: normal", "t3")}, b2={}, b7={}, issue={"Priority": ("High", "t1")}), + mk.item(b1={"Sub-team": ("Co-Leads", "t1")}, b2={}), + mk.item(b2={"Workload": ("XS", "t1")}, b7={}, issue={"Effort": ("High", None)}), + mk.item(b1={"Start date": ("2026-01-01", "t1")}, b2={"Start date": ("2026-02-01", "t2")}, b7={}, issue={}), + ] + s = mk.snap(cases) + r = core.plan(s, {}) + assert r.writes + apply_to(s, r) + r2 = core.plan(s, r.state) + assert r2.writes == [] and r2.conflicts == [] + + +def test_read_failure_keeps_details_out_of_the_log(monkeypatch, tmp_path): + def boom(clients): + raise github.GraphQLError('[{"message": "Could not resolve node PVTI_secret"}]') + + monkeypatch.setattr(github, "read_all", boom) + out = io.StringIO() + report = tmp_path / "r.txt" + assert cli.run(args(tmp_path, summary=True, report=str(report)), {}, out) == 2 + assert out.getvalue().strip() == "read failed: GraphQLError" + assert "PVTI_secret" in report.read_text() + + +def test_summary_hides_conflicts_and_skips(mk, monkeypatch, tmp_path): + s = mk.snap( + [ + mk.item( + b1={"Category": ("Tech", "t1")}, b2={"Category": ("Science", "t2"), "Release": ("Interra 2099", "t2")} + ) + ] + ) + s["fields"]["b2"]["Release"]["options"]["Interra 2099"] = "z" + monkeypatch.setattr(github, "read_all", lambda clients: s) + out = io.StringIO() + cli.run(args(tmp_path, summary=True), {}, out) + text = out.getvalue() + assert "conflicts: 1" in text and "skipped: 1" in text + assert "github.com" not in text and "Science" not in text and "Interra 2099" not in text + + +def test_crash_during_writes_saves_state(mk, monkeypatch, tmp_path): + s = mk.snap([mk.item(b1={}, b2={"Category": ("Tech", "t1"), "Start date": ("2026-01-01", "t1")})]) + monkeypatch.setattr(github, "read_all", lambda clients: s) + calls = [] + + def write(clients, snap, w): + calls.append(w) + if len(calls) == 2: + raise KeyboardInterrupt + + monkeypatch.setattr(github, "write", write) + try: + cli.run(args(tmp_path, apply=True), {}, io.StringIO()) + except KeyboardInterrupt: + pass + state = cli.load_state(tmp_path / "state.json") + seen = {f: state["items"]["I1"][f]["seen"]["b1"] for f in ("start", "category")} + assert seen == {"start": "2026-01-01", "category": None} # first write kept, second left for later + assert not (tmp_path / "state.json.tmp").exists() + + +def test_stop_after_failures_in_a_row(mk, monkeypatch, tmp_path): + items = [mk.item(b1={}, b2={"Category": ("Tech", "t1")}) for _ in range(cli.STOP_AFTER_FAILURES + 3)] + monkeypatch.setattr(github, "read_all", lambda clients: mk.snap(items)) + calls = [] + + def refuse(clients, snap, w): + calls.append(w) + raise github.GraphQLError("x") + + monkeypatch.setattr(github, "write", refuse) + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True, max_changes=99), {}, out) == 1 + assert len(calls) == cli.STOP_AFTER_FAILURES + assert "stopped after 5 failed writes in a row" in out.getvalue() and "deferred: 3 writes" in out.getvalue() + + +def test_damaged_state_stops_the_run(mk, monkeypatch, tmp_path): + p = tmp_path / "state.json" + for text in ('{"items": ', "[]"): + p.write_text(text) + with pytest.raises(cli.StateError): + cli.load_state(p) + monkeypatch.setattr(github, "read_all", lambda clients: mk.snap([])) + out = io.StringIO() + assert cli.run(args(tmp_path, summary=True), {}, out) == 2 + assert out.getvalue().strip() == "run failed: StateError" + + +def test_bulk_mark_clears_when_drained(mk, monkeypatch, tmp_path): + s = mk.snap([mk.item(b1={}, b2={"Category": ("Tech", "t1"), "Start date": ("2026-01-01", "t1")})]) + monkeypatch.setattr(github, "read_all", lambda clients: s) + monkeypatch.setattr(github, "write", lambda c, snap, w: apply_to(snap, type("R", (), {"writes": [w]}))) + cli.run(args(tmp_path, apply=True, max_changes=0, limit=1, allow_bulk=True), {}, io.StringIO()) + assert len(cli.load_state(tmp_path / "state.json")["bulk_keys"]) == 1 + cli.run(args(tmp_path, apply=True, max_changes=0, limit=1), {}, io.StringIO()) + assert cli.load_state(tmp_path / "state.json")["bulk_keys"] == [] + + +def test_plain_deferral_does_not_approve_a_bulk(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 3)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + monkeypatch.setattr(github, "write", lambda c, snap, w: apply_to(snap, type("R", (), {"writes": [w]}))) + assert cli.run(args(tmp_path, apply=True, limit=1), {}, io.StringIO()) == 0 + assert cli.load_state(tmp_path / "state.json")["bulk_keys"] == [] + + +def many(mk, n, field="Category", value="Tech"): + return [mk.item(b1={}, b2={field: (value, "t1")}) for _ in range(n)] + + +def test_bulk_approval_does_not_cover_a_new_mass_change(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 6)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + monkeypatch.setattr(github, "write", lambda c, snap, w: apply_to(snap, type("R", (), {"writes": [w]}))) + cli.run(args(tmp_path, apply=True, max_changes=1, limit=2, allow_bulk=True), {}, io.StringIO()) + assert len(cli.load_state(tmp_path / "state.json")["bulk_keys"]) == 4 + # A new change of the same size as the remainder is not covered by the approval. + for n, it in enumerate(many(mk, 4, "Release", "Interra 2027")): + s["items"][f"N{n}"] = it + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True, max_changes=1, limit=2), {}, out) == 3 + assert "stop: 8 writes" in out.getvalue() + + +def test_bulk_approval_ends_after_idle_runs(mk, monkeypatch, tmp_path): + monkeypatch.setattr(cli, "PARK_AFTER", 99) # no parking, so only the idle rule can end the approval + s = mk.snap(many(mk, 4)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + + def refuse(clients, snap, w): + raise github.GraphQLError("x") + + monkeypatch.setattr(github, "write", refuse) + cli.run(args(tmp_path, apply=True, max_changes=1, allow_bulk=True), {}, io.StringIO()) + for idle in range(1, cli.BULK_IDLE_RUNS): + state = cli.load_state(tmp_path / "state.json") + assert state["bulk_idle"] == idle and len(state["bulk_keys"]) == 4 + # The approval stays while runs fail, so the failures can reach parking. + assert cli.run(args(tmp_path, apply=True, max_changes=1), {}, io.StringIO()) == 1 + assert cli.load_state(tmp_path / "state.json")["bulk_keys"] == [] + # Writes that failed before are retried without approval; a new mass change still stops. + assert cli.run(args(tmp_path, apply=True, max_changes=1), {}, io.StringIO()) == 1 + for n, it in enumerate(many(mk, 3, "Release", "Interra 2027")): + s["items"][f"N{n}"] = it + assert cli.run(args(tmp_path, apply=True, max_changes=1), {}, io.StringIO()) == 3 + + +def test_persistent_failures_are_parked_and_do_not_block(mk, monkeypatch, tmp_path): + bad = [mk.item(b1={}, b2={"Category": ("Tech", "t1")}) for _ in range(cli.STOP_AFTER_FAILURES)] + good = [mk.item(b1={}, b2={"Release": ("Interra 2027", "t1")}) for _ in range(3)] + s = mk.snap(bad + good) # the failing writes sort first (I1 .. I5) + monkeypatch.setattr(github, "read_all", lambda clients: s) + written = [] + + def write(clients, snap, w): + if w["fkey"] == "category": + raise github.GraphQLError("locked") + written.append(w["issue"]) + apply_to(snap, type("R", (), {"writes": [w]})) + + monkeypatch.setattr(github, "write", write) + cli.run(args(tmp_path, apply=True, max_changes=99), {}, io.StringIO()) + assert written == [] # run 1: the 5 failures at the head stop the run + cli.run(args(tmp_path, apply=True, max_changes=99), {}, io.StringIO()) + assert sorted(written) == ["I6", "I7", "I8"] # run 2: writes that failed before go last + for _ in range(cli.PARK_AFTER - 2): + cli.run(args(tmp_path, apply=True, max_changes=99), {}, io.StringIO()) + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True, max_changes=99, report=str(tmp_path / "r.txt")), {}, out) == 4 + assert "parked: 5 writes" in out.getvalue() and "parked https://" in (tmp_path / "r.txt").read_text() + state = cli.load_state(tmp_path / "state.json") + assert {r["n"] for r in state["failures"].values()} == {cli.PARK_AFTER} # parked counts are kept + assert {state["items"][f"I{n}"]["category"]["seen"]["b1"] for n in range(1, 6)} == {None} # not written + + +def test_failure_count_resets_on_success(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 1)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + outcome = ["fail"] + + def write(clients, snap, w): + if outcome[0] == "fail": + raise github.GraphQLError("x") + apply_to(snap, type("R", (), {"writes": [w]})) + + monkeypatch.setattr(github, "write", write) + cli.run(args(tmp_path, apply=True), {}, io.StringIO()) + assert [r["n"] for r in cli.load_state(tmp_path / "state.json")["failures"].values()] == [1] + outcome[0] = "ok" + cli.run(args(tmp_path, apply=True), {}, io.StringIO()) + assert cli.load_state(tmp_path / "state.json")["failures"] == {} + + +def test_parked_write_is_tried_again_after_a_day(mk, monkeypatch, tmp_path): + from datetime import UTC, datetime, timedelta + + s = mk.snap(many(mk, 1)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + key = cli.write_key(core.plan(s, {}).writes[0]) + for age, tried in ((timedelta(hours=2), 0), (timedelta(days=2), 1)): + at = (datetime.now(UTC) - age).isoformat(timespec="seconds") + (tmp_path / "state.json").write_text(json.dumps({"failures": {key: {"n": cli.PARK_AFTER, "at": at}}})) + calls = [] + monkeypatch.setattr(github, "write", lambda c, snap, w, calls=calls: calls.append(w)) + assert cli.run(args(tmp_path, apply=True), {}, io.StringIO()) == (4 if tried == 0 else 0) + assert len(calls) == tried + assert not cli.is_parked({"n": cli.PARK_AFTER - 1, "at": datetime.now(UTC).isoformat()}, datetime.now(UTC)) + assert not cli.is_parked(None, datetime.now(UTC)) + + +def test_failures_in_a_row_reset_after_a_success(mk, monkeypatch, tmp_path): + n = 2 * cli.STOP_AFTER_FAILURES + s = mk.snap(many(mk, n)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + calls = [] + + def write(clients, snap, w): + calls.append(w) + if len(calls) % cli.STOP_AFTER_FAILURES: + raise github.GraphQLError("x") + + monkeypatch.setattr(github, "write", write) + cli.run(args(tmp_path, apply=True, max_changes=99), {}, io.StringIO()) + assert len(calls) == n + + +@pytest.mark.parametrize("kind", [ValueError, TypeError, RuntimeError]) +def test_any_write_error_counts_as_a_failed_write(kind, mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 3)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + calls = [] + + def write(clients, snap, w): + calls.append(w) + raise kind("x") + + monkeypatch.setattr(github, "write", write) + out = io.StringIO() + assert cli.run(args(tmp_path, apply=True), {}, out) == 1 + assert len(calls) == 3 and f"failed: 3 ({kind.__name__} 3)" in out.getvalue() + + +def test_state_save_keeps_the_old_file_when_writing_fails(tmp_path, monkeypatch): + p = tmp_path / "state.json" + cli.save_state(p, {"items": {"old": {}}}) + + def boom(*a, **k): + raise OSError("disk full") + + monkeypatch.setattr(cli.os, "replace", boom) + with pytest.raises(OSError): + cli.save_state(p, {"items": {"new": {}}}) + assert cli.load_state(p) == {"items": {"old": {}}} + + +def test_run_failure_after_the_read_shows_the_type_only(mk, monkeypatch, tmp_path): + monkeypatch.setattr(github, "read_all", lambda clients: mk.snap([])) + + def boom(snap, state): + raise KeyError("PVTI_secret") + + monkeypatch.setattr(core, "plan", boom) + out = io.StringIO() + assert cli.run(args(tmp_path, summary=True, report=str(tmp_path / "r.txt")), {}, out) == 2 + assert out.getvalue().strip() == "run failed: KeyError" + assert "PVTI_secret" in (tmp_path / "r.txt").read_text() + + +def test_sigterm_becomes_an_exit(): + import signal + + with pytest.raises(SystemExit) as exc: + cli._terminate(signal.SIGTERM, None) + assert exc.value.code == 128 + signal.SIGTERM + + +def test_warnings_name_missing_fields_and_unread_items(mk, monkeypatch, tmp_path): + s = mk.snap([]) + del s["fields"]["b7"]["Workload"] + s["skipped_items"] = {"b2": 3, "b1": 0} + s["unread_issues"] = {"FormingWorlds": 2, "Interra-Project": 0} + monkeypatch.setattr(github, "read_all", lambda clients: s) + out = io.StringIO() + cli.run(args(tmp_path, summary=True), {}, out) + assert "warning: b7 has no field 'Workload'" in out.getvalue() + assert "note: Interra Collaboration: 3 items are not readable issues" in out.getvalue() + assert "warning: 2 FormingWorlds issues could not be read" in out.getvalue() + assert "Interra-Project issues" not in out.getvalue() + + +def test_resets_are_named_in_the_summary(mk, monkeypatch, tmp_path): + a = mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t1")}) + state = core.plan(mk.snap([a]), {}).state + (tmp_path / "state.json").write_text(json.dumps(state)) + s = mk.snap([mk.item(b1={"Category": ("Tech", "t1")}, b2={})], ids={("b2", "Category"): "NEW"}) + monkeypatch.setattr(github, "read_all", lambda clients: s) + out = io.StringIO() + cli.run(args(tmp_path, summary=True), {}, out) + assert "reset: b2 / category is treated as new" in out.getvalue() + + +def three_runs_with_issue_gone(mk, back_value, back_time): + """Agreed 05-01; the issue goes; board 1 changes to 06-01 at t2; the issue returns.""" + a = mk.item(b1={"Start date": ("2026-05-01", "t1")}, issue={"Start date": ("2026-05-01", "t1")}) + state = core.plan(mk.snap([a]), {}).state + s2 = mk.snap([mk.item(b1={"Start date": ("2026-06-01", "t2")})]) + state = core.plan(s2, state).state + back = mk.item(b1={"Start date": ("2026-06-01", "t2")}, issue={"Start date": (back_value, back_time)}) + return core.plan(mk.snap([back]), state) + + +def test_returning_place_with_a_newer_value_wins(mk): + r = three_runs_with_issue_gone(mk, "2026-07-01", "t3") + assert writes(r) == {("b1", "Start date"): "2026-07-01"} + + +def test_returning_place_with_an_older_value_receives(mk): + r = three_runs_with_issue_gone(mk, "2026-07-01", "t0") + assert writes(r) == {("issue", "Start date"): "2026-06-01"} + + +def test_returning_place_on_a_reset_field_agrees_without_a_change(mk): + a = mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t1")}) + state = core.plan(mk.snap([a]), {}).state + state = core.plan(mk.snap([mk.item(b1={"Category": ("Tech", "t1")})]), state).state + s = mk.snap( + [mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t9")})], ids={("b2", "Category"): "NEW"} + ) + r = core.plan(s, state) + assert r.writes == [] and r.conflicts == [] + + +def test_place_stays_gone_until_it_has_the_field_again(mk): + a = mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t1")}) + state = core.plan(mk.snap([a]), {}).state + state = core.plan(mk.snap([mk.item(b1={"Category": ("Tech", "t1")})]), state).state + s = mk.snap([mk.item(b1={"Category": ("Tech", "t1")}, b2={})]) + del s["fields"]["b2"]["Category"] # back on board 2, but the field is missing this run + state = core.plan(s, state).state + assert state["items"]["I1"]["category"]["gone"] == ["b2"] + back = mk.item(b1={"Category": ("Tech", "t1")}, b2={}) # field back, empty: receive, not clear + assert writes(core.plan(mk.snap([back]), state)) == {("b2", "Category"): "Tech"} + + +def test_issue_gone_for_long_is_dropped(mk): + a = mk.item(b1={"Category": ("Tech", "t1")}, b2={"Category": ("Tech", "t1")}) + state = core.plan(mk.snap([a]), {}).state + s = mk.snap([]) + s["read_at"] = "2026-01-01T00:00:00+00:00" + state = core.plan(s, state).state + assert state["items"]["I1"]["category"]["gone_since"] == "2026-01-01T00:00:00+00:00" + s["read_at"] = "2026-04-01T00:00:00+00:00" # exactly 90 days + state = core.plan(s, state).state + assert "I1" in state["items"] + s["read_at"] = "2026-04-01T00:00:01+00:00" + assert "I1" not in core.plan(s, state).state["items"] + + +def test_deadline_stops_the_writes(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 4)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + a = args(tmp_path, apply=True, deadline=10.0, started=1000.0) + clock = iter([1000.0, 1009.0, 1011.0, 1012.0]) # checked before each write + monkeypatch.setattr(cli.time, "monotonic", lambda: next(clock)) + calls = [] + monkeypatch.setattr(github, "write", lambda c, snap, w: calls.append(w)) + out = io.StringIO() + assert cli.run(a, {}, out) == 0 + assert len(calls) == 2 and "stopped at the time limit" in out.getvalue() and "deferred: 2 writes" in out.getvalue() + + +def test_report_shows_the_plan_while_writes_run(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 2)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + report = tmp_path / "r.txt" + seen = [] + monkeypatch.setattr(github, "write", lambda c, snap, w: seen.append(report.read_text())) + cli.run(args(tmp_path, apply=True, report=str(report)), {}, io.StringIO()) + assert "plan https://" in seen[0] + + +def test_sigterm_handler_is_installed_and_restored(monkeypatch): + import signal + + during = [] + monkeypatch.setattr(cli, "make_clients", lambda env: {}) + monkeypatch.setattr(cli, "run", lambda a, c: during.append(signal.getsignal(signal.SIGTERM)) or 0) + before = signal.getsignal(signal.SIGTERM) + assert cli.main([], env={}) == 0 + assert during == [cli._terminate] and signal.getsignal(signal.SIGTERM) == before + + +def test_exit_during_writes_saves_state(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 2)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + calls = [] + + def write(clients, snap, w): + calls.append(w) + if len(calls) == 2: + cli._terminate(15, None) + + monkeypatch.setattr(github, "write", write) + with pytest.raises(SystemExit): + cli.run(args(tmp_path, apply=True), {}, io.StringIO()) + state = cli.load_state(tmp_path / "state.json") + assert [state["items"][i]["category"]["seen"]["b1"] for i in ("I1", "I2")] == ["Tech", None] + + +def test_failing_report_does_not_print_the_error_text(monkeypatch, tmp_path): + def boom(clients): + raise github.GraphQLError("PVTI_secret") + + monkeypatch.setattr(github, "read_all", boom) + out = io.StringIO() + bad = tmp_path / "missing" / "r.txt" + assert cli.run(args(tmp_path, summary=True, report=str(bad)), {}, out) == 2 + assert out.getvalue().strip() == "read failed: GraphQLError" + + +def test_missing_token_exits_with_code_2(): + with pytest.raises(SystemExit) as exc: + cli.make_clients({}) + assert exc.value.code == 2 + + +def test_bulk_approval_keeps_failed_writes(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 4)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + + def write(clients, snap, w): + if w["issue"] == "I1": + raise github.GraphQLError("x") + apply_to(snap, type("R", (), {"writes": [w]})) + + monkeypatch.setattr(github, "write", write) + cli.run(args(tmp_path, apply=True, max_changes=1, limit=2, allow_bulk=True), {}, io.StringIO()) + keys = cli.load_state(tmp_path / "state.json")["bulk_keys"] + assert len(keys) == 3 and any(k.startswith("I1|") for k in keys) + + +def test_new_value_on_a_recreated_field_is_a_change(mk): + a = mk.item(b1={"Category": ("Tech", "t5")}, b2={"Category": ("Tech", "t5")}) + state = core.plan(mk.snap([a]), {}).state + state = core.plan(mk.snap([mk.item(b1={"Category": ("Tech", "t5")})]), state).state + s = mk.snap( + [mk.item(b1={"Category": ("Tech", "t5")}, b2={"Category": ("Science", "t1")})], ids={("b2", "Category"): "NEW"} + ) + assert writes(core.plan(s, state)) == {("b1", "Category"): "Science"} + + +def test_a_new_value_is_a_new_write_for_failures_and_parking(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 1)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + calls = [] + + def refuse(clients, snap, w): + calls.append(w["value"]) + raise github.GraphQLError("x") + + monkeypatch.setattr(github, "write", refuse) + for _ in range(cli.PARK_AFTER): + cli.run(args(tmp_path, apply=True), {}, io.StringIO()) + assert cli.run(args(tmp_path, apply=True), {}, io.StringIO()) == 4 # parked + s["items"]["I1"]["places"]["b2"]["values"]["Category"] = ["Science", "t2"] + assert cli.run(args(tmp_path, apply=True), {}, io.StringIO()) == 1 # tried at once + assert calls[-1] == "Science" + assert sorted(r["n"] for r in cli.load_state(tmp_path / "state.json")["failures"].values()) == [1] + + +def test_parked_writes_stay_approved_and_do_not_count_as_new(mk, monkeypatch, tmp_path): + from datetime import UTC, datetime, timedelta + + s = mk.snap(many(mk, 3)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + keys = [cli.write_key(w) for w in core.plan(s, {}).writes] + old = (datetime.now(UTC) - timedelta(days=2)).isoformat(timespec="seconds") + state = {"failures": {k: {"n": cli.PARK_AFTER, "at": old} for k in keys}} + (tmp_path / "state.json").write_text(json.dumps(state)) + calls = [] + monkeypatch.setattr(github, "write", lambda c, snap, w: calls.append(w)) + # 3 retried writes against a limit of 1: they failed before, so they are not a new mass change. + assert cli.run(args(tmp_path, apply=True, max_changes=1), {}, io.StringIO()) == 0 + assert len(calls) == 3 + + +def test_parked_writes_keep_their_bulk_approval(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 3)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + monkeypatch.setattr(cli, "STOP_AFTER_FAILURES", 99) + + def refuse(clients, snap, w): + raise github.GraphQLError("x") + + monkeypatch.setattr(github, "write", refuse) + cli.run(args(tmp_path, apply=True, max_changes=0, allow_bulk=True), {}, io.StringIO()) + for _ in range(cli.PARK_AFTER): + cli.run(args(tmp_path, apply=True, max_changes=0), {}, io.StringIO()) + state = cli.load_state(tmp_path / "state.json") + assert len(state["bulk_keys"]) == 3 + + +def test_reaching_parking_counts_as_progress_and_reapproval_resets_idle(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 2 * cli.STOP_AFTER_FAILURES)) # 10 writes that always fail + monkeypatch.setattr(github, "read_all", lambda clients: s) + + def refuse(clients, snap, w): + raise github.GraphQLError("x") + + monkeypatch.setattr(github, "write", refuse) + cli.run(args(tmp_path, apply=True, max_changes=1, allow_bulk=True), {}, io.StringIO()) + idles = [] + for _ in range(2 * cli.PARK_AFTER): + assert cli.run(args(tmp_path, apply=True, max_changes=1), {}, io.StringIO()) != 3 + idles.append(cli.load_state(tmp_path / "state.json")["bulk_idle"]) + assert 0 in idles[cli.PARK_AFTER - 2 :] # the run that parks a block resets the idle count + state = cli.load_state(tmp_path / "state.json") + state["bulk_idle"] = cli.BULK_IDLE_RUNS - 1 + (tmp_path / "state.json").write_text(json.dumps(state)) + cli.run(args(tmp_path, apply=True, max_changes=1, allow_bulk=True), {}, io.StringIO()) + assert cli.load_state(tmp_path / "state.json")["bulk_idle"] <= 1 + + +def test_returning_place_without_a_known_agreed_time_receives(mk): + a = mk.item(b1={"Start date": ("2026-05-01", None)}, issue={"Start date": ("2026-05-01", None)}) + state = core.plan(mk.snap([a]), {}).state + assert state["items"]["I1"]["start"]["t"] is None + state = core.plan(mk.snap([mk.item(b1={"Start date": ("2026-05-01", None)})]), state).state + back = mk.item(b1={"Start date": ("2026-05-01", None)}, issue={"Start date": ("2026-07-01", "t9")}) + assert writes(core.plan(mk.snap([back]), state)) == {("issue", "Start date"): "2026-05-01"} + + +def test_snapshot_does_not_change_when_the_boards_do_not(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 1)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + texts = [] + for when in ("2026-01-01T00:00:00+00:00", "2026-01-01T00:30:00+00:00"): + s["read_at"] = when + cli.run(args(tmp_path, snapshot_dir=str(tmp_path / "snap")), {}, io.StringIO()) + texts.append((tmp_path / "snap" / "snapshot.json").read_text()) + assert texts[0] == texts[1] and "read_at" not in texts[0] + + +def test_parked_writes_give_code_4_only_when_applying(mk, monkeypatch, tmp_path): + from datetime import UTC, datetime + + s = mk.snap(many(mk, 1)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + key = cli.write_key(core.plan(s, {}).writes[0]) + now = datetime.now(UTC).isoformat(timespec="seconds") + (tmp_path / "state.json").write_text(json.dumps({"failures": {key: {"n": cli.PARK_AFTER, "at": now}}})) + assert cli.run(args(tmp_path), {}, io.StringIO()) == 0 + assert cli.run(args(tmp_path, apply=True), {}, io.StringIO()) == 4 + + +def test_reapproval_starts_a_fresh_idle_count(mk, monkeypatch, tmp_path): + monkeypatch.setattr(cli, "PARK_AFTER", 99) # no parking, so no progress at all + s = mk.snap(many(mk, 3)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + + def refuse(clients, snap, w): + raise github.GraphQLError("x") + + monkeypatch.setattr(github, "write", refuse) + (tmp_path / "state.json").write_text('{"bulk_idle": 3, "bulk_keys": []}') + cli.run(args(tmp_path, apply=True, max_changes=0, allow_bulk=True), {}, io.StringIO()) + state = cli.load_state(tmp_path / "state.json") + assert state["bulk_idle"] == 1 and len(state["bulk_keys"]) == 3 + + +def test_output_files_are_written_atomically(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 1)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + written = [] + real = cli.write_file + monkeypatch.setattr(cli, "write_file", lambda path, text: written.append(str(path)) or real(path, text)) + monkeypatch.setattr(github, "write", lambda c, snap, w: None) + report, snapdir = tmp_path / "r.txt", tmp_path / "snap" + cli.run(args(tmp_path, apply=True, report=str(report), snapshot_dir=str(snapdir)), {}, io.StringIO()) + assert {str(report), str(snapdir / "snapshot.json"), str(tmp_path / "state.json")} <= set(written) + + +def test_derived_field_is_listed_when_missing_and_skips_are_reported(mk): + s = mk.snap([mk.item(b2={"Workload": ("S", "t1")}, issue={})]) + del s["fields"]["issue:Interra-Project"]["Effort"]["options"]["Low"] + r = core.plan(s, {}) + assert r.skipped == [ + {"issue": "I1", "url": s["items"]["I1"]["url"], "place": "issue", "fkey": "Effort", "value": "Low"} + ] + del s["fields"]["issue:FormingWorlds"]["Effort"] + assert ("issue:FormingWorlds", "Effort") in core.missing_fields(s) + + +def test_writes_that_failed_before_go_last(mk, monkeypatch, tmp_path): + s = mk.snap(many(mk, 3)) + monkeypatch.setattr(github, "read_all", lambda clients: s) + first = cli.write_key(core.plan(s, {}).writes[0]) + (tmp_path / "state.json").write_text(json.dumps({"failures": {first: {"n": 1, "at": "2026-01-01T00:00:00+00:00"}}})) + order = [] + monkeypatch.setattr(github, "write", lambda c, snap, w: order.append(cli.write_key(w))) + cli.run(args(tmp_path, apply=True), {}, io.StringIO()) + assert order[-1] == first and len(order) == 3 + + +def test_workload_without_an_effort_mapping_writes_no_effort(mk, monkeypatch): + monkeypatch.setitem(config.DERIVED["effort"], "map", {"XL": "High"}) + s = mk.snap([mk.item(b2={"Workload": ("S", "t1")}, issue={})]) + r = core.plan(s, {}) + assert r.writes == [] and r.skipped == [] + + +def test_skip_line_names_the_derived_field(mk, monkeypatch, tmp_path): + s = mk.snap([mk.item(b2={"Workload": ("S", "t1")}, issue={})]) + del s["fields"]["issue:Interra-Project"]["Effort"]["options"]["Low"] + monkeypatch.setattr(github, "read_all", lambda clients: s) + out = io.StringIO() + cli.run(args(tmp_path), {}, out) + assert "issue Effort: no option 'Low'" in out.getvalue()