From 7e165073180493a332a86d55f4c06447a6f55369 Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Fri, 23 Oct 2026 12:26:31 +0200 Subject: [PATCH 1/3] Check the issue timeline read against its total An issue timeline read can come back short or empty with no error, and a synced value then has no change time and loses to any other place. The timeline query now asks for totalCount: a read with a number of field events other than min(totalCount, 100) is read again like the other short reads. An issue with a synced value but no event for it, while the timeline holds 100 field events or fewer, counts as unread for the run, so its issue fields are not written. --- README.md | 3 ++- board_sync/github.py | 30 ++++++++++++++++++++++++------ tests/test_github.py | 37 ++++++++++++++++++++++++++++++++----- 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index d6e5e53..495efab 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,9 @@ Effort follows Workload in one direction: a hand edit of Effort on an issue that - A board read is short when it finds no items, a number of items other than the board's total, a total that changes between pages, or an issue item with no values or a number of values other than its total (every item holds at least its title), since the API can return an empty page without an error. The board is then read again, up to 6 reads in all, 10 seconds apart, and the run stops with exit code 2 before any write when all 6 are short. So a board with no items stops every run, and the retries count against the 10 minutes of a run. - A read of issue field values is short when it returns fewer issues than asked for, or an issue with a number of values other than its total; it is read again in the same way. + A read of issue field values is short when it returns fewer issues than asked for, an issue with a number of values other than its total, or an issue with a number of field events other than its total (at most the last 100); it is read again in the same way. An issue that lacks the value of a synced field whose latest event set it counts as unread for its issue fields. + So does an issue with a value of a synced field but no event for that field, when its timeline holds 100 field events or fewer; so an empty timeline read does not make a value timeless. - A board place of an issue whose synced fields held 2 or more values in the state and all read empty now is held back for the run, as if the place did not hold the issue, and `report.txt` lists it; fields that were created again or emptied as a whole do not count. The issue fields of an issue are held back in the same way when one synced field that held a value reads empty without a removal event, since a clear of an issue field always has one. So a read that drops all values of a board item that held 2 or more, or a value of an issue field, clears nothing; when the place reads the same in the next run, it receives the agreed values. diff --git a/board_sync/github.py b/board_sync/github.py index d45d0fa..4f319bc 100644 --- a/board_sync/github.py +++ b/board_sync/github.py @@ -38,21 +38,28 @@ ... on IssueFieldSingleSelect { id name options { id name } } } } } } """ +# Field events read per issue, the latest first. +TIMELINE_LAST = 100 + ISSUE_VALUES_QUERY = ( - """ + ( + """ query($ids: [ID!]!) { nodes(ids: $ids) { ... on Issue { id issueFieldValues(first: 50) { totalCount pageInfo { hasNextPage } nodes { ... on IssueFieldDateValue { value field { ... on IssueFieldDate { name } } } ... on IssueFieldSingleSelectValue { name field { ... on IssueFieldSingleSelect { name } } } } } - timelineItems(last: 100, itemTypes: [ISSUE_FIELD_ADDED_EVENT, ISSUE_FIELD_CHANGED_EVENT, - ISSUE_FIELD_REMOVED_EVENT]) { nodes { + timelineItems(last: __LAST__, itemTypes: [ISSUE_FIELD_ADDED_EVENT, ISSUE_FIELD_CHANGED_EVENT, + ISSUE_FIELD_REMOVED_EVENT]) { totalCount nodes { __typename ... on IssueFieldAddedEvent { createdAt issueField { __FIELD_NAME__ } } ... on IssueFieldChangedEvent { createdAt issueField { __FIELD_NAME__ } } ... on IssueFieldRemovedEvent { createdAt issueField { __FIELD_NAME__ } } } } } } } """ -).replace("__FIELD_NAME__", _FIELD_NAME) + ) + .replace("__FIELD_NAME__", _FIELD_NAME) + .replace("__LAST__", str(TIMELINE_LAST)) +) class GraphQLError(RuntimeError): @@ -339,7 +346,10 @@ def read_issue_values(client, ids, batch=25): so a clear has its own time. A batch that returns fewer issues than asked for, or an issue with fewer values than its ``totalCount``, is read again, up to ``READ_ATTEMPTS`` times in all. An issue that lacks - the value of a synced field whose latest event set it counts as unread. + the value of a synced field whose latest event set it counts as unread, + and so does one with a synced value but no event for it while the + timeline was read in full. A timeline with fewer events than its + ``totalCount`` (at most ``TIMELINE_LAST``) is read again. Parameters ---------- @@ -386,8 +396,13 @@ def _read_issue_batch(client, ids): _check_page(conn, "issue field values") if _count(conn["nodes"]) != conn["totalCount"]: raise ShortIssueRead(f"an issue has {_count(conn['nodes'])} field values, total {conn['totalCount']}") + timeline = node.get("timelineItems") + if timeline is None or timeline.get("nodes") is None: + raise ShortIssueRead("an issue has no timeline list") + if _count(timeline["nodes"]) != min(timeline["totalCount"], TIMELINE_LAST): + raise ShortIssueRead(f"an issue has {_count(timeline['nodes'])} events, total {timeline['totalCount']}") times, removed = {}, set() - for ev in node["timelineItems"]["nodes"]: + for ev in timeline["nodes"]: name = ((ev or {}).get("issueField") or {}).get("name") if name and ev["createdAt"] >= times.get(name, ""): times[name] = ev["createdAt"] @@ -403,6 +418,9 @@ def _read_issue_batch(client, ids): if any(name not in values for name in (ISSUE_NAMES & times.keys()) - removed): unread += 1 # a value that its latest event set is absent from the read continue + if timeline["totalCount"] <= TIMELINE_LAST and any(values[n][1] is None for n in ISSUE_NAMES & values.keys()): + unread += 1 # a synced value with no event in a complete timeline: the read lost events + continue out[node["id"]] = values return out, unread diff --git a/tests/test_github.py b/tests/test_github.py index 0e217bf..eb510da 100644 --- a/tests/test_github.py +++ b/tests/test_github.py @@ -183,7 +183,7 @@ def issue_node(iid, values=(), events=()): return { "id": iid, "issueFieldValues": {"totalCount": len(values), "pageInfo": DONE, "nodes": list(values)}, - "timelineItems": {"nodes": list(events)}, + "timelineItems": {"totalCount": len(events), "nodes": list(events)}, } @@ -218,7 +218,7 @@ def test_read_issue_fields_and_values(): {"createdAt": "2026-03-01", "issueField": {"name": "Start date"}}, {"createdAt": "2026-02-01", "issueField": {"name": "Start date"}}, {"createdAt": "2026-02-01", "issueField": None}, - None, + {"createdAt": "2026-01-05", "issueField": {"name": "Priority"}}, ], ) ], @@ -228,7 +228,7 @@ def test_read_issue_fields_and_values(): "Priority": {"id": "S", "type": "select", "options": {"High": "h"}}, } values, unread = github.read_issue_values(fake, ["A", "GONE"]) - assert values == {"A": {"Start date": ["2026-02-01", "2026-03-01"], "Priority": ["High", None]}} + assert values == {"A": {"Start date": ["2026-02-01", "2026-03-01"], "Priority": ["High", "2026-01-05"]}} assert unread == 1 @@ -372,20 +372,47 @@ def test_short_board_read_stops_after_the_attempts(slept): {"nodes": [issue_node("A") | {"issueFieldValues": {"totalCount": 0, "pageInfo": DONE, "nodes": None}}]}, {"nodes": [issue_node("A") | {"issueFieldValues": None}]}, {"nodes": [None, None]}, # more issues than asked for + {"nodes": [issue_node("A", values=[START]) | {"timelineItems": {"totalCount": 2, "nodes": [SET_START]}}]}, + {"nodes": [issue_node("A", values=[START]) | {"timelineItems": {"totalCount": 1, "nodes": [None]}}]}, + {"nodes": [issue_node("A", values=[START]) | {"timelineItems": {"totalCount": 1, "nodes": None}}]}, + {"nodes": [issue_node("A", values=[START]) | {"timelineItems": None}]}, + {"nodes": [issue_node("A", values=[START]) | {"timelineItems": {"totalCount": 0, "nodes": None}}]}, + { + "nodes": [ + issue_node("A", values=[START]) | {"timelineItems": {"totalCount": 150, "nodes": [SET_START] * 99}} + ] + }, ], ) def test_short_issue_read_is_tried_again(short, slept): - answers = [short, {"nodes": [issue_node("A", values=[START])]}] + answers = [short, {"nodes": [issue_node("A", values=[START], events=[SET_START])]}] class C: def post(self, q, v, allow_not_found=False): return answers.pop(0) values, unread = github.read_issue_values(C(), ["A"]) - assert values == {"A": {"Start date": ["2026-02-01", None]}} and unread == 0 + assert values == {"A": {"Start date": ["2026-02-01", "t1"]}} and unread == 0 assert slept == [github.READ_WAIT] +def test_synced_value_without_an_event_is_unread(): + effort = {"name": "High", "field": {"name": "Effort"}} + long_timeline = {"totalCount": 150, "nodes": [SET_START | {"issueField": {"name": "Notes"}}] * 100} + nodes = [ + issue_node("A", values=[START]), # the timeline came back empty + issue_node("B", values=[effort]), # Effort is not a synced source + issue_node("C", values=[START]) | {"timelineItems": long_timeline}, # the event is older than the read + issue_node("D", values=[START], events=[SET_START]), + issue_node("E", values=[START]) | {"timelineItems": long_timeline | {"totalCount": 100}}, # full, no event + ] + values, unread = github.read_issue_values(Fake(values=nodes), ["A", "B", "C", "D", "E"]) + assert unread == 2 and sorted(values) == ["B", "C", "D"] + assert values["B"] == {"Effort": ["High", None]} and values["C"] == {"Start date": ["2026-02-01", None]} + assert github.TIMELINE_LAST == 100 and "timelineItems(last: 100," in github.ISSUE_VALUES_QUERY + assert "ISSUE_FIELD_REMOVED_EVENT]) { totalCount nodes {" in github.ISSUE_VALUES_QUERY + + def test_short_issue_read_stops_after_the_attempts(slept): calls = [] From 0c7969708cb64c969b4db2e49d8c5e9353e81c89 Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Fri, 23 Oct 2026 12:33:21 +0200 Subject: [PATCH 2/3] Read the timeline again when its total is missing A null or absent totalCount on the issue timeline raised TypeError or KeyError and stopped the run without a second read; it now counts as a short read like the value list. Tests cover a timeline total of 101, two synced values of which only one has an event, more events than the total, and the query placeholders. --- board_sync/github.py | 11 ++++++----- tests/test_github.py | 11 +++++++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/board_sync/github.py b/board_sync/github.py index 4f319bc..b55b780 100644 --- a/board_sync/github.py +++ b/board_sync/github.py @@ -87,7 +87,7 @@ class ShortBoardRead(ShortRead): class ShortIssueRead(ShortRead): - """A read of issue field values gave fewer values or issues than asked for.""" + """A read of issue field values gave fewer values, events or issues than asked for.""" # Attempts of one read, and seconds between them, when a read comes back short. @@ -399,8 +399,9 @@ def _read_issue_batch(client, ids): timeline = node.get("timelineItems") if timeline is None or timeline.get("nodes") is None: raise ShortIssueRead("an issue has no timeline list") - if _count(timeline["nodes"]) != min(timeline["totalCount"], TIMELINE_LAST): - raise ShortIssueRead(f"an issue has {_count(timeline['nodes'])} events, total {timeline['totalCount']}") + total = timeline.get("totalCount") + if not isinstance(total, int) or _count(timeline["nodes"]) != min(total, TIMELINE_LAST): + raise ShortIssueRead(f"an issue has {_count(timeline['nodes'])} events, total {total}") times, removed = {}, set() for ev in timeline["nodes"]: name = ((ev or {}).get("issueField") or {}).get("name") @@ -418,8 +419,8 @@ def _read_issue_batch(client, ids): if any(name not in values for name in (ISSUE_NAMES & times.keys()) - removed): unread += 1 # a value that its latest event set is absent from the read continue - if timeline["totalCount"] <= TIMELINE_LAST and any(values[n][1] is None for n in ISSUE_NAMES & values.keys()): - unread += 1 # a synced value with no event in a complete timeline: the read lost events + if total <= TIMELINE_LAST and any(values[n][1] is None for n in ISSUE_NAMES & values.keys()): + unread += 1 # a synced value with no event although the timeline was read in full continue out[node["id"]] = values return out, unread diff --git a/tests/test_github.py b/tests/test_github.py index eb510da..0ce70fb 100644 --- a/tests/test_github.py +++ b/tests/test_github.py @@ -359,6 +359,7 @@ def test_short_board_read_stops_after_the_attempts(slept): START = {"value": "2026-02-01", "field": {"name": "Start date"}} +PRIORITY = {"name": "High", "field": {"name": "Priority"}} SET_START = {"__typename": "IssueFieldAddedEvent", "createdAt": "t1", "issueField": {"name": "Start date"}} @@ -376,6 +377,9 @@ def test_short_board_read_stops_after_the_attempts(slept): {"nodes": [issue_node("A", values=[START]) | {"timelineItems": {"totalCount": 1, "nodes": [None]}}]}, {"nodes": [issue_node("A", values=[START]) | {"timelineItems": {"totalCount": 1, "nodes": None}}]}, {"nodes": [issue_node("A", values=[START]) | {"timelineItems": None}]}, + {"nodes": [issue_node("A", values=[START]) | {"timelineItems": {"totalCount": None, "nodes": []}}]}, + {"nodes": [issue_node("A", values=[START]) | {"timelineItems": {"nodes": []}}]}, + {"nodes": [issue_node("A", values=[START]) | {"timelineItems": {"totalCount": 1, "nodes": [SET_START] * 2}}]}, {"nodes": [issue_node("A", values=[START]) | {"timelineItems": {"totalCount": 0, "nodes": None}}]}, { "nodes": [ @@ -405,11 +409,14 @@ def test_synced_value_without_an_event_is_unread(): issue_node("C", values=[START]) | {"timelineItems": long_timeline}, # the event is older than the read issue_node("D", values=[START], events=[SET_START]), issue_node("E", values=[START]) | {"timelineItems": long_timeline | {"totalCount": 100}}, # full, no event + issue_node("F", values=[START]) | {"timelineItems": long_timeline | {"totalCount": 101}}, # one event not read + issue_node("G", values=[START, PRIORITY], events=[SET_START]), # one of two values has no event ] - values, unread = github.read_issue_values(Fake(values=nodes), ["A", "B", "C", "D", "E"]) - assert unread == 2 and sorted(values) == ["B", "C", "D"] + values, unread = github.read_issue_values(Fake(values=nodes), ["A", "B", "C", "D", "E", "F", "G"]) + assert unread == 3 and sorted(values) == ["B", "C", "D", "F"] assert values["B"] == {"Effort": ["High", None]} and values["C"] == {"Start date": ["2026-02-01", None]} assert github.TIMELINE_LAST == 100 and "timelineItems(last: 100," in github.ISSUE_VALUES_QUERY + assert "__LAST__" not in github.ISSUE_VALUES_QUERY and "__FIELD_NAME__" not in github.ISSUE_VALUES_QUERY assert "ISSUE_FIELD_REMOVED_EVENT]) { totalCount nodes {" in github.ISSUE_VALUES_QUERY From b0f02b52063b5ad2c6f2b59d5b73978543b6b945 Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Fri, 23 Oct 2026 19:42:07 +0200 Subject: [PATCH 3/3] State which timeline reads are read again The docstrings now say that any event count other than the total, and a missing total, lead to a new read, and that an unread issue waits for the next run. The constant comment no longer claims an event order. --- README.md | 1 + board_sync/github.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 495efab..4873b55 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ Effort follows Workload in one direction: a hand edit of Effort on an issue that A read of issue field values is short when it returns fewer issues than asked for, an issue with a number of values other than its total, or an issue with a number of field events other than its total (at most the last 100); it is read again in the same way. An issue that lacks the value of a synced field whose latest event set it counts as unread for its issue fields. So does an issue with a value of a synced field but no event for that field, when its timeline holds 100 field events or fewer; so an empty timeline read does not make a value timeless. + An unread issue is not read again in the same run; the next run reads it again. - A board place of an issue whose synced fields held 2 or more values in the state and all read empty now is held back for the run, as if the place did not hold the issue, and `report.txt` lists it; fields that were created again or emptied as a whole do not count. The issue fields of an issue are held back in the same way when one synced field that held a value reads empty without a removal event, since a clear of an issue field always has one. So a read that drops all values of a board item that held 2 or more, or a value of an issue field, clears nothing; when the place reads the same in the next run, it receives the agreed values. diff --git a/board_sync/github.py b/board_sync/github.py index b55b780..dc084eb 100644 --- a/board_sync/github.py +++ b/board_sync/github.py @@ -38,7 +38,7 @@ ... on IssueFieldSingleSelect { id name options { id name } } } } } } """ -# Field events read per issue, the latest first. +# Field events read per issue: the most recent ones, up to this number. TIMELINE_LAST = 100 ISSUE_VALUES_QUERY = ( @@ -87,7 +87,7 @@ class ShortBoardRead(ShortRead): class ShortIssueRead(ShortRead): - """A read of issue field values gave fewer values, events or issues than asked for.""" + """A read of issue field values gave fewer issues than asked for, or a number of values or events other than the total.""" # Attempts of one read, and seconds between them, when a read comes back short. @@ -344,12 +344,14 @@ def read_issue_values(client, ids, batch=25): A field whose latest event is a removal reads as ``[None, removed_at]``, so a clear has its own time. A batch that returns fewer issues than - asked for, or an issue with fewer values than its ``totalCount``, is - read again, up to ``READ_ATTEMPTS`` times in all. An issue that lacks + asked for, an issue with a number of values other than its + ``totalCount``, or a timeline with a number of events other than its + ``totalCount`` (at most ``TIMELINE_LAST``) or with no ``totalCount``, + is read again, up to ``READ_ATTEMPTS`` times in all. An issue that lacks the value of a synced field whose latest event set it counts as unread, and so does one with a synced value but no event for it while the - timeline was read in full. A timeline with fewer events than its - ``totalCount`` (at most ``TIMELINE_LAST``) is read again. + timeline was read in full; an unread issue is not read again in the + same run. Parameters ----------