Stop a board read that comes back short

README.md

@@ -40,6 +40,9 @@ Effort follows Workload in one direction: a hand edit of Effort on an issue that
   On a field that was created again, the first value counts as a change, as on a new board.
   An issue that is on no board for 90 days is dropped from the state.
 - A list longer than one page of the API (fields of a board, values of an item, issue fields, issue field values) stops the read with an error, so a cut-off list is never read as empty values.
+- A board read is short when it finds no items, a number of items other than the board's total, or a total that changes between pages, since the API can return an empty page without an error.
+  The board is then read again, up to 6 reads in all, 10 seconds apart, and the run stops with exit code 2 before any write when all 6 are short.
+  So a board with no items stops every run, and the retries count against the 10 minutes of a run.
 - A run that plans more than 20 writes stops and writes nothing (exit code 3).
   Read the plan in `report.txt` in the state repository, then start the workflow by hand with `apply` and `allow_bulk`.
   The state then holds the approved writes that are not done yet, and the scheduled runs write them, at most 400 values per run.

board_sync/github.py

@@ -23,7 +23,7 @@
     fields(first: 100) { pageInfo { hasNextPage } nodes {
       ... on ProjectV2FieldCommon { id name dataType }
       ... on ProjectV2SingleSelectField { options { id name } } } }
-    items(first: 100, after: $after) { pageInfo { hasNextPage endCursor } nodes { id
+    items(first: 100, after: $after) { totalCount pageInfo { hasNextPage endCursor } nodes { id
       content { ... on Issue { id url repository { owner { login } } } }
       fieldValues(first: 100) { pageInfo { hasNextPage } nodes {
         ... on ProjectV2ItemFieldDateValue { date updatedAt field { ... on ProjectV2FieldCommon { name } } }
@@ -71,6 +71,15 @@ class TruncatedRead(RuntimeError):
     """A list in a response was longer than the request asked for."""
 
 
+class ShortBoardRead(RuntimeError):
+    """A board read gave no items, or a number of items other than the board total."""
+
+
+# Attempts of one board read, and seconds between them, when a read comes back short.
+BOARD_READ_ATTEMPTS = 6
+BOARD_READ_WAIT = 10
+
+
 def _check_page(conn, what):
     """Raise ``TruncatedRead`` if a connection has more nodes than were read."""
     if conn["pageInfo"]["hasNextPage"]:
@@ -184,6 +193,11 @@ def _field_type(data_type):
 def read_board(client, owner, number):
     """Read the fields and the issue items of one project board.
 
+    A read that finds no items, a number of items other than the board's
+    ``totalCount``, or a ``totalCount`` that changes between pages, is short:
+    the board is read up to ``BOARD_READ_ATTEMPTS`` times in all, since the
+    API can return an empty page with no error.
+
     Parameters
     ----------
     client : Client
@@ -206,8 +220,21 @@ def read_board(client, owner, number):
         If the board or its org does not exist or the token cannot see it.
     TruncatedRead
         If a board has more fields, or an item more values, than one page.
+    ShortBoardRead
+        If every attempt reads no items, or a number of items other than the total.
     """
+    for _ in range(BOARD_READ_ATTEMPTS - 1):
+        try:
+            return _read_board_once(client, owner, number)
+        except ShortBoardRead:
+            time.sleep(BOARD_READ_WAIT)
+    return _read_board_once(client, owner, number)
+
+
+def _read_board_once(client, owner, number):
+    """Read one board once; see ``read_board``."""
     after, items, fields, pid, skipped = None, [], None, None, 0
+    total, seen = None, 0
     while True:
         # A missing board or org comes back as null data with a NOT_FOUND error.
         data = client.post(BOARD_QUERY, {"owner": owner, "number": number, "after": after}, allow_not_found=BOARD_GONE)
@@ -226,6 +253,11 @@ def read_board(client, owner, number):
                 for f in proj["fields"]["nodes"]
                 if f
             }
+        if total is None:
+            total = proj["items"]["totalCount"]
+        elif proj["items"]["totalCount"] != total:
+            raise ShortBoardRead(f"board {number}: total {proj['items']['totalCount']} on a later page, {total} first")
+        seen += len(proj["items"]["nodes"])
         for node in proj["items"]["nodes"]:
             content = (node or {}).get("content") or {}
             if "id" not in content:
@@ -240,6 +272,8 @@ def read_board(client, owner, number):
             items.append((content["id"], content["url"], org, node["id"], values))
         page = proj["items"]["pageInfo"]
         if not page["hasNextPage"]:
+            if total == 0 or seen != total:
+                raise ShortBoardRead(f"board {number}: {seen} items read, total {total}")
             return pid, fields, items, skipped
         after = page["endCursor"]
 

tests/test_github.py

@@ -141,7 +141,7 @@ def post(self, query, variables=None, allow_not_found=False):
         return {}
 
 
-def page(items, more=False, field_id="F1"):
+def page(items, more=False, field_id="F1", total=None):
     return {
         "fields": {
             "pageInfo": DONE,
@@ -152,7 +152,11 @@ def page(items, more=False, field_id="F1"):
                 {},
             ],
         },
-        "items": {"pageInfo": {"hasNextPage": more, "endCursor": "C1"}, "nodes": items},
+        "items": {
+            "totalCount": len(items) if total is None else total,
+            "pageInfo": {"hasNextPage": more, "endCursor": "C1"},
+            "nodes": items,
+        },
     }
 
 
@@ -184,7 +188,7 @@ def issue_node(iid, values=(), events=()):
 
 def test_read_board_pages_and_skips_drafts():
     draft = {"id": "d", "content": {}, "fieldValues": {"pageInfo": DONE, "nodes": []}}
-    fake = Fake(pages=[page([board_item("A"), draft, None], more=True), page([board_item("B")])])
+    fake = Fake(pages=[page([board_item("A"), draft, None], more=True, total=4), page([board_item("B")], total=4)])
     pid, fields, items, skipped = github.read_board(fake, "FormingWorlds", 7)
     assert pid == "PID" and skipped == 2 and fake.cursors == [None, "C1"]
     assert fields["Status"] == {"id": "F2", "type": "select", "options": {"TBD": "o1"}}
@@ -228,8 +232,8 @@ def test_read_issue_fields_and_values():
 
 
 def test_only_the_issue_values_read_accepts_partial_responses():
-    interra = Fake(pages=[page([board_item("X", "Interra-Project")]), page([])], values=[issue_node("X")])
-    fw = Fake(pages=[page([])])
+    interra = Fake(pages=[page([board_item("X", "Interra-Project")])] * 2, values=[issue_node("X")])
+    fw = Fake(pages=[page([None])])
     github.read_all({"Interra-Project": interra, "FormingWorlds": fw})
     assert set(interra.flags + fw.flags) == {
         ("board", github.BOARD_GONE),
@@ -284,16 +288,22 @@ def test_removed_issue_field_reads_as_clear_with_time():
     assert values == {"A": {"Target date": [None, "2026-02-01"], "Start date": ["2026-01-01", "2026-03-01"]}}
 
 
+def test_board_query_asks_for_the_item_total():
+    assert "items(first: 100, after: $after) { totalCount pageInfo" in github.BOARD_QUERY
+
+
 @pytest.mark.parametrize("where", ["fields", "values", "issue_fields", "issue_values"])
-def test_truncated_lists_stop_the_read(where):
+def test_truncated_lists_stop_the_read(where, slept):
     if where in ("fields", "values"):
         p = page([board_item("A")])
         if where == "fields":
             p["fields"]["pageInfo"] = MORE
         else:
             p["items"]["nodes"][0]["fieldValues"]["pageInfo"] = MORE
+        fake = Fake(pages=[p])
         with pytest.raises(github.TruncatedRead):
-            github.read_board(Fake(pages=[p]), "o", 1)
+            github.read_board(fake, "o", 1)
+        assert slept == [] and len(fake.cursors) == 1  # not read again
     elif where == "issue_fields":
 
         class C:
@@ -309,6 +319,36 @@ def post(self, q, v):
             github.read_issue_values(Fake(values=[node]), ["A"])
 
 
+FULL = [board_item("A"), board_item("B")]
+
+
+@pytest.mark.parametrize(
+    "short",
+    [
+        [page([], total=0)],  # the whole board empty
+        [page(FULL[:1], more=True, total=2), page([], total=0)],  # a later page empty
+        [page(FULL[:1], more=True, total=2), page([], total=2)],  # fewer items than the total
+        [page(FULL[:1], more=True, total=2), page(FULL[1:], total=3)],  # the total changed
+        [page([], more=True, total=2), page(FULL[1:], total=2)],  # an empty first page
+        [page(FULL + [board_item("C")], total=2)],  # more items than the total
+    ],
+)
+def test_short_board_read_is_tried_again(short, slept):
+    fake = Fake(pages=[*short, page(FULL)])
+    _, _, items, skipped = github.read_board(fake, "Interra-Project", 2)
+    assert [i[0] for i in items] == ["A", "B"] and skipped == 0
+    assert slept == [github.BOARD_READ_WAIT]
+
+
+def test_short_board_read_stops_after_the_attempts(slept):
+    fake = Fake(pages=[page([], total=0)] * github.BOARD_READ_ATTEMPTS)
+    with pytest.raises(github.ShortBoardRead, match="0 items read, total 0"):
+        github.read_board(fake, "Interra-Project", 1)
+    assert (github.BOARD_READ_ATTEMPTS, github.BOARD_READ_WAIT) == (6, 10)  # the README states both
+    assert len(fake.cursors) == github.BOARD_READ_ATTEMPTS
+    assert slept == [github.BOARD_READ_WAIT] * (github.BOARD_READ_ATTEMPTS - 1)
+
+
 def test_missing_page_info_is_an_error():
     p = page([board_item("A")])
     del p["fields"]["pageInfo"]