diff --git a/README.md b/README.md index d6e5e53..4873b55 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,10 @@ Effort follows Workload in one direction: a hand edit of Effort on an issue that - A board read is short when it finds no items, a number of items other than the board's total, a total that changes between pages, or an issue item with no values or a number of values other than its total (every item holds at least its title), since the API can return an empty page without an error. The board is then read again, up to 6 reads in all, 10 seconds apart, and the run stops with exit code 2 before any write when all 6 are short. So a board with no items stops every run, and the retries count against the 10 minutes of a run. - A read of issue field values is short when it returns fewer issues than asked for, or an issue with a number of values other than its total; it is read again in the same way. + A read of issue field values is short when it returns fewer issues than asked for, an issue with a number of values other than its total, or an issue with a number of field events other than its total (at most the last 100); it is read again in the same way. An issue that lacks the value of a synced field whose latest event set it counts as unread for its issue fields. + So does an issue with a value of a synced field but no event for that field, when its timeline holds 100 field events or fewer; so an empty timeline read does not make a value timeless. + An unread issue is not read again in the same run; the next run reads it again. - A board place of an issue whose synced fields held 2 or more values in the state and all read empty now is held back for the run, as if the place did not hold the issue, and `report.txt` lists it; fields that were created again or emptied as a whole do not count. The issue fields of an issue are held back in the same way when one synced field that held a value reads empty without a removal event, since a clear of an issue field always has one. So a read that drops all values of a board item that held 2 or more, or a value of an issue field, clears nothing; when the place reads the same in the next run, it receives the agreed values. diff --git a/board_sync/github.py b/board_sync/github.py index d45d0fa..dc084eb 100644 --- a/board_sync/github.py +++ b/board_sync/github.py @@ -38,21 +38,28 @@ ... on IssueFieldSingleSelect { id name options { id name } } } } } } """ +# Field events read per issue: the most recent ones, up to this number. +TIMELINE_LAST = 100 + ISSUE_VALUES_QUERY = ( - """ + ( + """ query($ids: [ID!]!) { nodes(ids: $ids) { ... on Issue { id issueFieldValues(first: 50) { totalCount pageInfo { hasNextPage } nodes { ... on IssueFieldDateValue { value field { ... on IssueFieldDate { name } } } ... on IssueFieldSingleSelectValue { name field { ... on IssueFieldSingleSelect { name } } } } } - timelineItems(last: 100, itemTypes: [ISSUE_FIELD_ADDED_EVENT, ISSUE_FIELD_CHANGED_EVENT, - ISSUE_FIELD_REMOVED_EVENT]) { nodes { + timelineItems(last: __LAST__, itemTypes: [ISSUE_FIELD_ADDED_EVENT, ISSUE_FIELD_CHANGED_EVENT, + ISSUE_FIELD_REMOVED_EVENT]) { totalCount nodes { __typename ... on IssueFieldAddedEvent { createdAt issueField { __FIELD_NAME__ } } ... on IssueFieldChangedEvent { createdAt issueField { __FIELD_NAME__ } } ... on IssueFieldRemovedEvent { createdAt issueField { __FIELD_NAME__ } } } } } } } """ -).replace("__FIELD_NAME__", _FIELD_NAME) + ) + .replace("__FIELD_NAME__", _FIELD_NAME) + .replace("__LAST__", str(TIMELINE_LAST)) +) class GraphQLError(RuntimeError): @@ -80,7 +87,7 @@ class ShortBoardRead(ShortRead): class ShortIssueRead(ShortRead): - """A read of issue field values gave fewer values or issues than asked for.""" + """A read of issue field values gave fewer issues than asked for, or a number of values or events other than the total.""" # Attempts of one read, and seconds between them, when a read comes back short. @@ -337,9 +344,14 @@ def read_issue_values(client, ids, batch=25): A field whose latest event is a removal reads as ``[None, removed_at]``, so a clear has its own time. A batch that returns fewer issues than - asked for, or an issue with fewer values than its ``totalCount``, is - read again, up to ``READ_ATTEMPTS`` times in all. An issue that lacks - the value of a synced field whose latest event set it counts as unread. + asked for, an issue with a number of values other than its + ``totalCount``, or a timeline with a number of events other than its + ``totalCount`` (at most ``TIMELINE_LAST``) or with no ``totalCount``, + is read again, up to ``READ_ATTEMPTS`` times in all. An issue that lacks + the value of a synced field whose latest event set it counts as unread, + and so does one with a synced value but no event for it while the + timeline was read in full; an unread issue is not read again in the + same run. Parameters ---------- @@ -386,8 +398,14 @@ def _read_issue_batch(client, ids): _check_page(conn, "issue field values") if _count(conn["nodes"]) != conn["totalCount"]: raise ShortIssueRead(f"an issue has {_count(conn['nodes'])} field values, total {conn['totalCount']}") + timeline = node.get("timelineItems") + if timeline is None or timeline.get("nodes") is None: + raise ShortIssueRead("an issue has no timeline list") + total = timeline.get("totalCount") + if not isinstance(total, int) or _count(timeline["nodes"]) != min(total, TIMELINE_LAST): + raise ShortIssueRead(f"an issue has {_count(timeline['nodes'])} events, total {total}") times, removed = {}, set() - for ev in node["timelineItems"]["nodes"]: + for ev in timeline["nodes"]: name = ((ev or {}).get("issueField") or {}).get("name") if name and ev["createdAt"] >= times.get(name, ""): times[name] = ev["createdAt"] @@ -403,6 +421,9 @@ def _read_issue_batch(client, ids): if any(name not in values for name in (ISSUE_NAMES & times.keys()) - removed): unread += 1 # a value that its latest event set is absent from the read continue + if total <= TIMELINE_LAST and any(values[n][1] is None for n in ISSUE_NAMES & values.keys()): + unread += 1 # a synced value with no event although the timeline was read in full + continue out[node["id"]] = values return out, unread diff --git a/tests/test_github.py b/tests/test_github.py index 0e217bf..0ce70fb 100644 --- a/tests/test_github.py +++ b/tests/test_github.py @@ -183,7 +183,7 @@ def issue_node(iid, values=(), events=()): return { "id": iid, "issueFieldValues": {"totalCount": len(values), "pageInfo": DONE, "nodes": list(values)}, - "timelineItems": {"nodes": list(events)}, + "timelineItems": {"totalCount": len(events), "nodes": list(events)}, } @@ -218,7 +218,7 @@ def test_read_issue_fields_and_values(): {"createdAt": "2026-03-01", "issueField": {"name": "Start date"}}, {"createdAt": "2026-02-01", "issueField": {"name": "Start date"}}, {"createdAt": "2026-02-01", "issueField": None}, - None, + {"createdAt": "2026-01-05", "issueField": {"name": "Priority"}}, ], ) ], @@ -228,7 +228,7 @@ def test_read_issue_fields_and_values(): "Priority": {"id": "S", "type": "select", "options": {"High": "h"}}, } values, unread = github.read_issue_values(fake, ["A", "GONE"]) - assert values == {"A": {"Start date": ["2026-02-01", "2026-03-01"], "Priority": ["High", None]}} + assert values == {"A": {"Start date": ["2026-02-01", "2026-03-01"], "Priority": ["High", "2026-01-05"]}} assert unread == 1 @@ -359,6 +359,7 @@ def test_short_board_read_stops_after_the_attempts(slept): START = {"value": "2026-02-01", "field": {"name": "Start date"}} +PRIORITY = {"name": "High", "field": {"name": "Priority"}} SET_START = {"__typename": "IssueFieldAddedEvent", "createdAt": "t1", "issueField": {"name": "Start date"}} @@ -372,20 +373,53 @@ def test_short_board_read_stops_after_the_attempts(slept): {"nodes": [issue_node("A") | {"issueFieldValues": {"totalCount": 0, "pageInfo": DONE, "nodes": None}}]}, {"nodes": [issue_node("A") | {"issueFieldValues": None}]}, {"nodes": [None, None]}, # more issues than asked for + {"nodes": [issue_node("A", values=[START]) | {"timelineItems": {"totalCount": 2, "nodes": [SET_START]}}]}, + {"nodes": [issue_node("A", values=[START]) | {"timelineItems": {"totalCount": 1, "nodes": [None]}}]}, + {"nodes": [issue_node("A", values=[START]) | {"timelineItems": {"totalCount": 1, "nodes": None}}]}, + {"nodes": [issue_node("A", values=[START]) | {"timelineItems": None}]}, + {"nodes": [issue_node("A", values=[START]) | {"timelineItems": {"totalCount": None, "nodes": []}}]}, + {"nodes": [issue_node("A", values=[START]) | {"timelineItems": {"nodes": []}}]}, + {"nodes": [issue_node("A", values=[START]) | {"timelineItems": {"totalCount": 1, "nodes": [SET_START] * 2}}]}, + {"nodes": [issue_node("A", values=[START]) | {"timelineItems": {"totalCount": 0, "nodes": None}}]}, + { + "nodes": [ + issue_node("A", values=[START]) | {"timelineItems": {"totalCount": 150, "nodes": [SET_START] * 99}} + ] + }, ], ) def test_short_issue_read_is_tried_again(short, slept): - answers = [short, {"nodes": [issue_node("A", values=[START])]}] + answers = [short, {"nodes": [issue_node("A", values=[START], events=[SET_START])]}] class C: def post(self, q, v, allow_not_found=False): return answers.pop(0) values, unread = github.read_issue_values(C(), ["A"]) - assert values == {"A": {"Start date": ["2026-02-01", None]}} and unread == 0 + assert values == {"A": {"Start date": ["2026-02-01", "t1"]}} and unread == 0 assert slept == [github.READ_WAIT] +def test_synced_value_without_an_event_is_unread(): + effort = {"name": "High", "field": {"name": "Effort"}} + long_timeline = {"totalCount": 150, "nodes": [SET_START | {"issueField": {"name": "Notes"}}] * 100} + nodes = [ + issue_node("A", values=[START]), # the timeline came back empty + issue_node("B", values=[effort]), # Effort is not a synced source + issue_node("C", values=[START]) | {"timelineItems": long_timeline}, # the event is older than the read + issue_node("D", values=[START], events=[SET_START]), + issue_node("E", values=[START]) | {"timelineItems": long_timeline | {"totalCount": 100}}, # full, no event + issue_node("F", values=[START]) | {"timelineItems": long_timeline | {"totalCount": 101}}, # one event not read + issue_node("G", values=[START, PRIORITY], events=[SET_START]), # one of two values has no event + ] + values, unread = github.read_issue_values(Fake(values=nodes), ["A", "B", "C", "D", "E", "F", "G"]) + assert unread == 3 and sorted(values) == ["B", "C", "D", "F"] + assert values["B"] == {"Effort": ["High", None]} and values["C"] == {"Start date": ["2026-02-01", None]} + assert github.TIMELINE_LAST == 100 and "timelineItems(last: 100," in github.ISSUE_VALUES_QUERY + assert "__LAST__" not in github.ISSUE_VALUES_QUERY and "__FIELD_NAME__" not in github.ISSUE_VALUES_QUERY + assert "ISSUE_FIELD_REMOVED_EVENT]) { totalCount nodes {" in github.ISSUE_VALUES_QUERY + + def test_short_issue_read_stops_after_the_attempts(slept): calls = []