From e92a3500d4498c8ac5113bc1a302a37a111ae91a Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Thu, 22 Oct 2026 17:56:13 +0200 Subject: [PATCH 1/2] Stop a board read that comes back short instead of syncing a cut list The GraphQL API can return a project item page with no items, a totalCount of 0 and no next page, without an error. read_board took that as the end of the board, so a 225-item board could read as 200, 100 or 0 items. The read now asks for totalCount and treats a read as short when a later page reports another total, when the items read differ from the total, or when the board has no items. A short board is read up to 6 times in all, 10 seconds apart, and the run then stops with ShortBoardRead before any write. --- README.md | 3 +++ board_sync/github.py | 36 ++++++++++++++++++++++++++++- tests/test_github.py | 54 ++++++++++++++++++++++++++++++++++++++------ 3 files changed, 85 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 1874eef..99653f1 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,9 @@ Effort follows Workload in one direction: a hand edit of Effort on an issue that On a field that was created again, the first value counts as a change, as on a new board. An issue that is on no board for 90 days is dropped from the state. - A list longer than one page of the API (fields of a board, values of an item, issue fields, issue field values) stops the read with an error, so a cut-off list is never read as empty values. +- A board read is short when it finds no items, a number of items other than the board's total, or a total that changes between pages, since the API can return an empty page without an error. + The board is then read again, up to 6 reads in all, 10 seconds apart, and the run stops with exit code 2 before any write when all 6 are short. + So a board with no items stops every run, and the retries count against the 10 minutes of a run. - A run that plans more than 20 writes stops and writes nothing (exit code 3). Read the plan in `report.txt` in the state repository, then start the workflow by hand with `apply` and `allow_bulk`. The state then holds the approved writes that are not done yet, and the scheduled runs write them, at most 400 values per run. diff --git a/board_sync/github.py b/board_sync/github.py index 6fbe44d..56dd3f0 100644 --- a/board_sync/github.py +++ b/board_sync/github.py @@ -23,7 +23,7 @@ fields(first: 100) { pageInfo { hasNextPage } nodes { ... on ProjectV2FieldCommon { id name dataType } ... on ProjectV2SingleSelectField { options { id name } } } } - items(first: 100, after: $after) { pageInfo { hasNextPage endCursor } nodes { id + items(first: 100, after: $after) { totalCount pageInfo { hasNextPage endCursor } nodes { id content { ... on Issue { id url repository { owner { login } } } } fieldValues(first: 100) { pageInfo { hasNextPage } nodes { ... on ProjectV2ItemFieldDateValue { date updatedAt field { ... on ProjectV2FieldCommon { name } } } @@ -71,6 +71,15 @@ class TruncatedRead(RuntimeError): """A list in a response was longer than the request asked for.""" +class ShortBoardRead(RuntimeError): + """A board read gave no items, or a number of items other than the board total.""" + + +# Attempts of one board read, and seconds between them, when a read comes back short. +BOARD_READ_ATTEMPTS = 6 +BOARD_READ_WAIT = 10 + + def _check_page(conn, what): """Raise ``TruncatedRead`` if a connection has more nodes than were read.""" if conn["pageInfo"]["hasNextPage"]: @@ -184,6 +193,11 @@ def _field_type(data_type): def read_board(client, owner, number): """Read the fields and the issue items of one project board. + A read that finds no items, a number of items other than the board's + ``totalCount``, or a ``totalCount`` that changes between pages, is short: + the board is read up to ``BOARD_READ_ATTEMPTS`` times in all, since the + API can return an empty page with no error. + Parameters ---------- client : Client @@ -206,8 +220,21 @@ def read_board(client, owner, number): If the board or its org does not exist or the token cannot see it. TruncatedRead If a board has more fields, or an item more values, than one page. + ShortBoardRead + If every attempt reads no items, or a number of items other than the total. """ + for _ in range(BOARD_READ_ATTEMPTS - 1): + try: + return _read_board_once(client, owner, number) + except ShortBoardRead: + time.sleep(BOARD_READ_WAIT) + return _read_board_once(client, owner, number) + + +def _read_board_once(client, owner, number): + """Read one board once; see ``read_board``.""" after, items, fields, pid, skipped = None, [], None, None, 0 + total, seen = None, 0 while True: # A missing board or org comes back as null data with a NOT_FOUND error. data = client.post(BOARD_QUERY, {"owner": owner, "number": number, "after": after}, allow_not_found=BOARD_GONE) @@ -226,6 +253,11 @@ def read_board(client, owner, number): for f in proj["fields"]["nodes"] if f } + if total is None: + total = proj["items"]["totalCount"] + elif proj["items"]["totalCount"] != total: + raise ShortBoardRead(f"board {number}: total {proj['items']['totalCount']} on a later page, {total} first") + seen += len(proj["items"]["nodes"]) for node in proj["items"]["nodes"]: content = (node or {}).get("content") or {} if "id" not in content: @@ -240,6 +272,8 @@ def read_board(client, owner, number): items.append((content["id"], content["url"], org, node["id"], values)) page = proj["items"]["pageInfo"] if not page["hasNextPage"]: + if total == 0 or seen != total: + raise ShortBoardRead(f"board {number}: {seen} items read, total {total}") return pid, fields, items, skipped after = page["endCursor"] diff --git a/tests/test_github.py b/tests/test_github.py index 6e565ff..f42a691 100644 --- a/tests/test_github.py +++ b/tests/test_github.py @@ -141,7 +141,7 @@ def post(self, query, variables=None, allow_not_found=False): return {} -def page(items, more=False, field_id="F1"): +def page(items, more=False, field_id="F1", total=None): return { "fields": { "pageInfo": DONE, @@ -152,7 +152,11 @@ def page(items, more=False, field_id="F1"): {}, ], }, - "items": {"pageInfo": {"hasNextPage": more, "endCursor": "C1"}, "nodes": items}, + "items": { + "totalCount": len(items) if total is None else total, + "pageInfo": {"hasNextPage": more, "endCursor": "C1"}, + "nodes": items, + }, } @@ -184,7 +188,7 @@ def issue_node(iid, values=(), events=()): def test_read_board_pages_and_skips_drafts(): draft = {"id": "d", "content": {}, "fieldValues": {"pageInfo": DONE, "nodes": []}} - fake = Fake(pages=[page([board_item("A"), draft, None], more=True), page([board_item("B")])]) + fake = Fake(pages=[page([board_item("A"), draft, None], more=True, total=4), page([board_item("B")], total=4)]) pid, fields, items, skipped = github.read_board(fake, "FormingWorlds", 7) assert pid == "PID" and skipped == 2 and fake.cursors == [None, "C1"] assert fields["Status"] == {"id": "F2", "type": "select", "options": {"TBD": "o1"}} @@ -228,8 +232,8 @@ def test_read_issue_fields_and_values(): def test_only_the_issue_values_read_accepts_partial_responses(): - interra = Fake(pages=[page([board_item("X", "Interra-Project")]), page([])], values=[issue_node("X")]) - fw = Fake(pages=[page([])]) + interra = Fake(pages=[page([board_item("X", "Interra-Project")])] * 2, values=[issue_node("X")]) + fw = Fake(pages=[page([None])]) github.read_all({"Interra-Project": interra, "FormingWorlds": fw}) assert set(interra.flags + fw.flags) == { ("board", github.BOARD_GONE), @@ -284,16 +288,22 @@ def test_removed_issue_field_reads_as_clear_with_time(): assert values == {"A": {"Target date": [None, "2026-02-01"], "Start date": ["2026-01-01", "2026-03-01"]}} +def test_board_query_asks_for_the_item_total(): + assert "items(first: 100, after: $after) { totalCount pageInfo" in github.BOARD_QUERY + + @pytest.mark.parametrize("where", ["fields", "values", "issue_fields", "issue_values"]) -def test_truncated_lists_stop_the_read(where): +def test_truncated_lists_stop_the_read(where, slept): if where in ("fields", "values"): p = page([board_item("A")]) if where == "fields": p["fields"]["pageInfo"] = MORE else: p["items"]["nodes"][0]["fieldValues"]["pageInfo"] = MORE + fake = Fake(pages=[p]) with pytest.raises(github.TruncatedRead): - github.read_board(Fake(pages=[p]), "o", 1) + github.read_board(fake, "o", 1) + assert slept == [] and len(fake.cursors) == 1 # not read again elif where == "issue_fields": class C: @@ -309,6 +319,36 @@ def post(self, q, v): github.read_issue_values(Fake(values=[node]), ["A"]) +FULL = [board_item("A"), board_item("B")] + + +@pytest.mark.parametrize( + "short", + [ + [page([], total=0)], # the whole board empty + [page(FULL[:1], more=True, total=2), page([], total=0)], # a later page empty + [page(FULL[:1], more=True, total=2), page([], total=2)], # fewer items than the total + [page(FULL[:1], more=True, total=2), page(FULL[1:], total=3)], # the total changed + [page([], more=True, total=2), page(FULL[1:], total=2)], # an empty first page + [page(FULL + [board_item("C")], total=2)], # more items than the total + ], +) +def test_short_board_read_is_tried_again(short, slept): + fake = Fake(pages=[*short, page(FULL)]) + pid, fields, items, skipped = github.read_board(fake, "Interra-Project", 2) + assert [i[0] for i in items] == ["A", "B"] and skipped == 0 + assert slept == [github.BOARD_READ_WAIT] + + +def test_short_board_read_stops_after_the_attempts(slept): + fake = Fake(pages=[page([], total=0)] * github.BOARD_READ_ATTEMPTS) + with pytest.raises(github.ShortBoardRead, match="0 items read, total 0"): + github.read_board(fake, "Interra-Project", 1) + assert (github.BOARD_READ_ATTEMPTS, github.BOARD_READ_WAIT) == (6, 10) # the README states both + assert len(fake.cursors) == github.BOARD_READ_ATTEMPTS + assert slept == [github.BOARD_READ_WAIT] * (github.BOARD_READ_ATTEMPTS - 1) + + def test_missing_page_info_is_an_error(): p = page([board_item("A")]) del p["fields"]["pageInfo"] From 6f204a8c5d5f786b9971bc2a856bb71e7a01ffde Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Thu, 22 Oct 2026 17:57:21 +0200 Subject: [PATCH 2/2] Drop unused names in the short-read test --- tests/test_github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_github.py b/tests/test_github.py index f42a691..3fe64aa 100644 --- a/tests/test_github.py +++ b/tests/test_github.py @@ -335,7 +335,7 @@ def post(self, q, v): ) def test_short_board_read_is_tried_again(short, slept): fake = Fake(pages=[*short, page(FULL)]) - pid, fields, items, skipped = github.read_board(fake, "Interra-Project", 2) + _, _, items, skipped = github.read_board(fake, "Interra-Project", 2) assert [i[0] for i in items] == ["A", "B"] and skipped == 0 assert slept == [github.BOARD_READ_WAIT]