diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dd6f5f..92905a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,6 +100,23 @@ things were settled in the same exchange. Nothing here changes a field of the sc them nothing: it read the stage's `bind:`, which a waiting stage cannot write, and not the question's `shows:`, which a `decide` asking the same question did read. Found by the first shared case with a waiting stage in it. +- **`everyone` means every entry of `asked-of:`, each answering separately.** The two cases for a + waiting stage merged in the entry above were wrong, and so was the reference: it took one answer + whatever `answered-by:` said, and the second case delivered one answer "from" two people, which + nothing could check. The schema's words are "how many of `asked-of:` must answer". Bud Agent + Flow, whose engine asks each entry, refused to build to the cases. Now: each entry is one + answerer (a person, or a team or channel one of whose members answers for it); all are asked at + once and each answers separately; one answer has one answerer; a second, different answer from + somebody who has answered, or one answer naming several people, is refused and recorded + (`refused` in a case's `expect`), changes nothing, and the wait stays open; the same answer + handed in again is a resend and counts once; one no is a no at once; the stage's fields are those of the answer that settled the wait (the + first no, or the answer that made it enough). Five cases replace the two (31 workflow cases, 55 cases of replayed + runs), on two workflows, `sign-off` and `sign-off-by-the-team`. In a case, a list under `people` + for a question is the answers as they arrive, and `shown` has one entry for each entry of + `asked-of:` that was asked. +- **A shared case runs on its own clock.** The replay kit ran a case's first leg on the wall clock + and its resumes on the case's own, so a wait could never run out by itself. It starts at 0 now. + No case's expected result changed. (Bud Agent Flow found the same mixture in its replay.) - Ruled, with no case yet: a workflow run a port ended (`if-still-running: stop`, `cancel-previous`) ends as `cancelled`. `stopped` stays "could not go on, and says why". diff --git a/README.md b/README.md index 2d696c9..0afc8f7 100644 --- a/README.md +++ b/README.md @@ -1000,11 +1000,11 @@ framework is an extra. |---|---| | **Design** | Thesis, 29 binding decisions, FRD (120 requirements), 60-row refusal ledger — complete | | **Research** | 14 source-grounded studies, ~15,750 lines, over 140 repos (~15 GB) + 57 papers | -| **Code** | Loader, diagnostics, schema engine, CLI, harness, resolver, evals, SLO — **5034 tests (1679 Rust + 3355 adapter), clippy clean, TypeScript type-checked** | +| **Code** | Loader, diagnostics, schema engine, CLI, harness, resolver, evals, SLO — **5043 tests (1679 Rust + 3364 adapter), clippy clean, TypeScript type-checked** | | **Adapters** | **All 7 named targets**, proven against one shared conformance suite | ```bash -./scripts/test-all.sh # 5034 tests, Rust + 7 adapters, fully offline +./scripts/test-all.sh # 5043 tests, Rust + 7 adapters, fully offline ``` **What is still owed** is not hidden — it lives in diff --git a/adapters/python/src/pact_adapters/workflow_runtime.py b/adapters/python/src/pact_adapters/workflow_runtime.py index f84791f..d2d3103 100644 --- a/adapters/python/src/pact_adapters/workflow_runtime.py +++ b/adapters/python/src/pact_adapters/workflow_runtime.py @@ -143,6 +143,15 @@ class WorkflowResult: #: its reason (`because:`), by stage: the run's "why this path". It is not #: part of the stage's record, which a binding reads. because: dict[str, str] = field(default_factory=dict) + #: Every time a question was put to somebody, in order: `{question, + #: shown}`, one for each entry of the question's `asked-of:`. A question + #: asked of two people is put twice, at once, whoever answers first. + asked: list[dict[str, Any]] = field(default_factory=list) + #: Answers that were handed in and did not count, with why + #: (`already-answered`: that person had answered this wait; + #: `several-people`: one answer named more than one answerer). Whoever + #: handed one in is told no. It is never taken as if it had counted. + refused: list[dict[str, Any]] = field(default_factory=list) #: Who answered a question, where the host says: a person's answer may carry @@ -496,7 +505,9 @@ async def decide(self, stage: StageSpec, scope: _Scope) -> tuple[str, Any]: if rung.asks: if standing is not None: continue - answer = await self.person(rung.asks, self.shown_for(rung.asks, scope), stage.name) + shown = self.shown_for(rung.asks, scope) + self.put(rung.asks, shown, stage.name) + answer = await self.person(rung.asks, shown, stage.name) who = answer.pop(ANSWERED_BY, None) label = _one_of(answer, labels) if label: @@ -623,14 +634,18 @@ def decides_alone(self, agent: str) -> bool: rows = (self._catalogue.get(str(m).strip()) for m in models) return all(row is not None and row.kind == "decision" for row in rows) + def question(self, name: str) -> Mapping[str, Any]: + """The question of this name: the workflow's own, else the workspace's.""" + mine = (self.document.get("workflows") or {}).get(self.spec.name) or {} + return (mine.get("questions") or {}).get(name) or ( + (self.document.get("questions") or {}).get(name) or {} + ) + def shown_for(self, question: str, scope: _Scope) -> dict[str, Any]: """What a person who decides is shown: the question's own `shows:` lines, read — `shows: [input.message]`. It was `{}`, so the person was asked which desk should take a message nobody showed them.""" - mine = (self.document.get("workflows") or {}).get(self.spec.name) or {} - asked = (mine.get("questions") or {}).get(question) or ( - (self.document.get("questions") or {}).get(question) or {} - ) + asked = self.question(question) shown: dict[str, Any] = {} for binding in asked.get("shows") or (): value = scope.read(str(binding)) @@ -638,32 +653,114 @@ def shown_for(self, question: str, scope: _Scope) -> dict[str, Any]: shown[str(binding)] = value return shown + def put(self, question: str, shown: dict[str, Any], at: str) -> None: + """Put `question` to everybody it is `asked-of:`, and say so on the + result: once for each entry, because each entry is one answerer and + each is asked, whoever answers first. With nobody there to ask, the + run parks and nobody has been asked.""" + if self.callers.person is None: + raise Parked(question, at) + entries = list(self.question(question).get("asked-of") or ()) or [None] + for _ in entries: + self.result.asked.append({"question": question, "shown": dict(shown)}) + async def person(self, question: str, shown: dict[str, Any], at: str) -> dict[str, Any]: + """One answer to a question that has been `put`.""" if self.callers.person is None: raise Parked(question, at) return _as_answer(await _maybe(self.callers.person(question, shown))) async def ask(self, stage: StageSpec, scope: _Scope) -> tuple[str, Any]: - try: - # What the question's own `shows:` lists, as a `decide` asking the - # same question shows it. A stage that waits showed the person - # nothing: it passed its `bind:`, which a waiting stage cannot - # write, so "May this fix be closed?" arrived with no fix. - shown = {**self.shown_for(stage.asks, scope), **self.inputs(stage, scope)} - answer = await self.person(stage.asks, shown, stage.name) - except NobodyAnswered: - return "nobody-answered", {} - who = answer.pop(ANSWERED_BY, None) - if who: - # A LIST on a wait, always, in the order they answered: a question - # can need several answers (`answered-by: everyone`), and the - # shape of `steps..answered-by` must not depend on how many - # people happened to answer. One name is a list of one. (On a - # `decide` it is one name: a decision is one person's.) - names = [str(w) for w in who] if isinstance(who, (list, tuple)) else [str(who)] - self.stage_words[id(stage)] = {ANSWERED_BY: names} - no = [k for k, v in answer.items() if isinstance(v, bool) and v is False] - return ("declined" if no else "answered"), answer + """A stage that waits for people. + + `answered-by:` on the question says how many of `asked-of:` must + answer: the first (unwritten, or `anyone`), `enough-is:` of them, or + every one. The count is over the ENTRIES of `asked-of:`, each of which + is one answerer — a person, or a team or channel one of whose members + answers for it. So each is asked and answers separately, and one + answer has one answerer. This took a single answer whatever the + question said, and passed on whatever name or list of names a host + put beside it; the first shared cases for it then delivered one answer + "from" two people, which no runtime could check. Bud Agent Flow, whose + engine asks each entry, refused to build to them. + + Everybody asked is asked at once (`put`), and the answers are taken + as they arrive. Any `no` is a no, at once: the rest are not waited + for. `steps..answered-by` is who answered, in the order their + answers arrived, a list also when one person did. + + An answer that does not count is REFUSED, and the wait stays open: a + second, different answer from somebody who has answered this wait + (nobody is "everyone" by answering for somebody else as well, and + nobody changes their answer by answering again), or one answer naming + several people. It is on the result as refused; it is never quietly + taken as if it had counted, because whoever handed it in would + believe it had. + + The SAME answer from the same person, handed in again, is not a second + answer. Hosts resend: a file of answers passed whole at every resume, + a retry after a timeout. It counts once and nothing is refused. + + The stage's fields are those of the answer that SETTLED the wait: the + first no, or the answer that made it enough. Several people can each + give a reason beside their yes, and one value has to be the stage's; + the one that settled it is the one the outcome came from. + """ + asked = self.question(stage.asks) + entries = [str(e) for e in asked.get("asked-of") or ()] + needs = str(asked.get("answered-by") or "anyone").strip() + if needs == "everyone": + wanted = max(len(entries), 1) + elif needs == "enough-of-them": + wanted = max(int(asked.get("enough-is") or 1), 1) + else: + wanted = 1 + # What the question's own `shows:` lists, as a `decide` asking the + # same question shows it. A stage that waits showed the person + # nothing: it passed its `bind:`, which a waiting stage cannot + # write, so "May this fix be closed?" arrived with no fix. + shown = {**self.shown_for(stage.asks, scope), **self.inputs(stage, scope)} + self.put(stage.asks, shown, stage.name) + names: list[str] = [] + gave: dict[str, dict[str, Any]] = {} + fields: dict[str, Any] = {} + heard = 0 + # Bounded, so a host that only ever hands in answers that do not + # count ends the run instead of spinning. + for _ in range(wanted + max(len(entries), 1) + 1): + if heard >= wanted: + break + try: + answer = await self.person(stage.asks, shown, stage.name) + except NobodyAnswered: + return "nobody-answered", {} + who = answer.pop(ANSWERED_BY, None) + if isinstance(who, (list, tuple)): + self.result.refused.append( + {"question": stage.asks, "answered-by": [str(w) for w in who], "why": "several-people"} + ) + continue + if who and str(who) in names: + if answer != gave[str(who)]: + self.result.refused.append( + {"question": stage.asks, "answered-by": str(who), "why": "already-answered"} + ) + continue + if who: + names.append(str(who)) + gave[str(who)] = dict(answer) + self.stage_words[id(stage)] = {ANSWERED_BY: list(names)} + heard += 1 + fields = answer + if any(isinstance(v, bool) and v is False for v in answer.values()): + return "declined", answer + if heard < wanted: + raise Stopped( + f"{stage.name!r} needs {wanted} of `asked-of:` to answer {stage.asks!r} and " + f"heard {heard}: every other answer handed in came from somebody who had " + f"answered already, or was refused ({len(self.result.refused)} refused)" + ) + return "answered", fields async def each(self, stage: StageSpec, scope: _Scope, where: str) -> dict[str, Any]: items = (scope.read(stage.over)) diff --git a/adapters/python/tests/conformance_kit.py b/adapters/python/tests/conformance_kit.py index d9d038a..c4cbf6c 100644 --- a/adapters/python/tests/conformance_kit.py +++ b/adapters/python/tests/conformance_kit.py @@ -32,6 +32,11 @@ that was told, which is one for each yes, the first request after it and no other. +* **The clock** — a case runs on its own clock and never on the wall clock: + the first leg at 0 seconds, each resume at the number of parks so far, and a + `"nobody"` resume one second after the wait has run out. A replay on the + wall clock is a different run each day: a wait with `answer-within:` is + inside its time today and past it tomorrow. * **`people`** — one entry for each time the run parks, in order: `"yes"` or `"no"`, given to every question of that park. `"nobody"` means no answer is given and the time the question allows (`answer-within:`) passes: the run is @@ -126,7 +131,12 @@ async def model_call(self, system, history, tools): # type: ignore[override] kw["checking_transport"] = ReferenceTransport(Script([Turn(str(case["judge"]))])) parks: list[dict[str, Any]] = [] - out = asyncio.run(run(spec, Lead(lead), case["says"], tools, **kw)) + # The case's own clock, from 0. This leg ran on the wall clock while every + # resume below ran on the case's, so a park was stamped with today's time + # and resumed at second 1: a wait could then never run out. No case's + # result depended on it. Bud Agent Flow found the same mixture in its own + # replay, where it did change a result. + out = asyncio.run(run(spec, Lead(lead), case["says"], tools, now=0.0, **kw)) for said in case.get("people") or (): if out.halted != "suspended": break @@ -178,7 +188,7 @@ def replay_workflow(case: dict[str, Any]) -> dict[str, Any]: doc = shown(REPO / case["tree"]) spec = WorkflowSpec.from_workflow(doc, case["workflow"]) given: list[dict[str, Any]] = [] - shown_to: list[dict[str, Any]] = [] + handed: dict[str, int] = {} called: list[dict[str, Any]] = [] def agent(name: str, inputs: dict[str, Any]) -> dict[str, Any]: @@ -194,15 +204,19 @@ def tool(name: str, action: str, inputs: dict[str, Any]) -> dict[str, Any]: return dict((case.get("tools") or {}).get(f"{name}/{action}", {})) def person(question: str, shown_: dict[str, Any]) -> dict[str, Any]: - shown_to.append({"question": question, "shown": dict(shown_)}) - return dict((case.get("people") or {}).get(question, {})) + handed[question] = handed.get(question, 0) + 1 + says = (case.get("people") or {}).get(question, {}) + if isinstance(says, list): # the answers as they arrive, the last repeating + says = says[min(handed[question], len(says)) - 1] + return dict(says) callers = Callers(agent=agent, tool=tool, person=person if case.get("people") is not None else None) out = asyncio.run(run_workflow(spec, case["input"], callers, document=doc)) return { "trace": list(out.trace), "given": given, - "shown": shown_to, + "shown": list(out.asked), + "refused": list(out.refused), "called": called, "answer": json.loads(json.dumps(out.answer, default=str)), "halted": out.halted, diff --git a/adapters/python/tests/test_the_shared_cases_are_what_this_runtime_does.py b/adapters/python/tests/test_the_shared_cases_are_what_this_runtime_does.py index e59214f..33f4ff1 100644 --- a/adapters/python/tests/test_the_shared_cases_are_what_this_runtime_does.py +++ b/adapters/python/tests/test_the_shared_cases_are_what_this_runtime_does.py @@ -113,15 +113,45 @@ def test_the_cases_say_the_things_they_are_there_to_say() -> None: asked = by_name["the-agent-is-not-sure-so-a-person-decides-and-is-shown-the-message"] assert asked["shown"] == [{"question": "which-desk", "shown": {"input.message": "The app crashes."}}] - one = by_name["who-answered-a-wait-is-a-list-of-one-when-one-person-did"] - # On a stage that waits for people, who answered is a list, also of one; - # and the person is shown what the question says to show. - assert one["answer"] == {"close": True, "by": ["dana"]} - assert one["shown"] == [ - {"question": "may-we-close", "shown": {"input.report": "Seat 14C will not recline."}} + team = by_name["a-team-is-asked-and-who-answered-is-a-list-of-the-one-who-did"] + # A team is one entry of `asked-of:`. The first answer decides; who in the + # team gave it is a list of one, and not the name that was asked. The + # person is shown what the question says to show. + assert team["answer"] == {"close": True, "by": ["dana"]} + assert team["shown"] == [ + {"question": "may-the-team-close", "shown": {"input.report": "Seat 14C will not recline."}} ] - both = by_name["who-answered-a-wait-is-everyone-who-did-in-the-order-they-answered"] - assert both["answer"]["by"] == ["lee", "dana"], "in the order they answered, not sorted" + both = by_name["everyone-asked-answers-separately-and-is-listed-in-the-order-they-answered"] + # `asked-of: [lee, dana]`, `answered-by: everyone`: both are asked, at + # once, and their answers are listed as they arrived, which is not the + # order they are written in. + assert len(both["shown"]) == 2 and both["answer"]["by"] == ["dana", "lee"] + assert both["refused"] == [] + assert both["answer"]["outcome"] == "answered" + # Each gave a note. The stage's is the note of the answer that settled it: + # here the last yes that was needed, lee's. + assert both["answer"]["note"] == "Parts are logged." + said_no = by_name["one-no-from-everyone-asked-is-a-no-and-the-rest-are-not-waited-for"] + assert said_no["answer"] == { + "close": False, "by": ["dana"], "outcome": "declined", + "note": "The recline cable is still loose.", + } + # Both were asked. dana's no ended it, and lee's answer was not waited for. + assert len(said_no["shown"]) == 2 + twice = by_name[ + "a-second-answer-from-somebody-who-has-answered-is-refused-and-the-wait-stays-open" + ] + # dana said yes, then handed in a no: the second is refused, on the + # record, and changes nothing. The wait stays open until lee answers, and + # the answer is yes. + assert twice["refused"] == [ + {"question": "may-we-close", "answered-by": "dana", "why": "already-answered"} + ] + assert twice["answer"]["by"] == ["dana", "lee"] and len(twice["shown"]) == 2 + assert twice["answer"]["close"] is True and twice["answer"]["outcome"] == "answered" + resent = by_name["the-same-answer-handed-in-again-is-not-a-second-answer"] + # The same answer from dana again is a resend. Nothing is refused. + assert resent["refused"] == [] and resent["answer"]["by"] == ["dana", "lee"] found = by_name["a-deciding-agent-is-given-what-an-earlier-stage-found-and-the-bound-name-wins"] # `kind` is triage's `engine`, not the `cabin` the report came in with; the # `secret` the assigner does not accept is not given to it. diff --git a/adapters/python/tests/test_who_answered_a_wait_is_a_list.py b/adapters/python/tests/test_who_answered_a_wait_is_a_list.py index e0cadaf..2834634 100644 --- a/adapters/python/tests/test_who_answered_a_wait_is_a_list.py +++ b/adapters/python/tests/test_who_answered_a_wait_is_a_list.py @@ -1,14 +1,17 @@ -"""Who answered a stage that waits for people, and what they were shown. +"""Who answers a stage that waits for people, and what they are shown. -Two things the reference workflow runtime did, found by the first shared case -with a waiting stage in it. +`answered-by:` on a question says how many of `asked-of:` must answer. The +count is over the ENTRIES of `asked-of:`: each is one answerer, a person or a +team one of whose members answers for it. So with `everyone` each is asked and +answers separately; one answer has one answerer. -* `steps..answered-by` was whatever the host put beside the answer: a - name when one person answered, a list when several did. A stage reading it - would fit its shape one day and not the next. On a wait it is a list, - always; on a `decide` it is one name, because a decision is one person's. -* The person was shown nothing. A waiting stage passed its `bind:`, which it - cannot write, and not the question's `shows:`. +The reference workflow runtime took one answer whatever the question said, and +kept whatever name or list of names a host put beside it. The first shared +cases written for it then delivered one answer "from" two people. Bud Agent +Flow, whose engine asks each entry, refused to build to them, and was right. + +And it showed the person nothing: a waiting stage passed its `bind:`, which it +cannot write, and not the question's `shows:`. """ from __future__ import annotations @@ -27,43 +30,107 @@ REPO = Path(__file__).resolve().parents[3] DOC = shown(REPO / "tests/conformance/trees/triaged-defects") +REPORT = {"report": "Seat 14C will not recline."} -def signed_off(who: Any) -> tuple[dict[str, Any], list[dict[str, Any]]]: +def asked(workflow: str, answers: list[dict[str, Any]]): + """Run `workflow`, handing its question each of `answers` in turn.""" seen: list[dict[str, Any]] = [] + left = list(answers) def person(question: str, shown_: dict[str, Any]) -> dict[str, Any]: seen.append({question: dict(shown_)}) - return {"close": True, **({"answered-by": who} if who is not None else {})} + return dict(left.pop(0) if len(left) > 1 else left[0]) - spec = WorkflowSpec.from_workflow(DOC, "sign-off") - out = asyncio.run( - run_workflow(spec, {"report": "Seat 14C will not recline."}, Callers(person=person), document=DOC) - ) + spec = WorkflowSpec.from_workflow(DOC, workflow) + out = asyncio.run(run_workflow(spec, REPORT, Callers(person=person), document=DOC)) + return out, seen # `seen`: each time the host was gone to for an answer + + +def yes(who: Any) -> dict[str, Any]: + return {"close": True, "answered-by": who} + + +def test_a_team_is_one_answerer_and_who_answered_is_a_list_of_one() -> None: + out, seen = asked("sign-off-by-the-team", [yes("dana")]) assert out.halted == "final", out.said - return out.answer, seen + assert out.answer == {"close": True, "by": ["dana"]} + assert len(seen) == 1, "the first answer decides" + assert len(out.asked) == 1, "one entry under `asked-of:`, so one ask" + + +def test_everyone_is_each_of_asked_of_answering_separately() -> None: + out, seen = asked("sign-off", [yes("dana"), yes("lee")]) + assert out.halted == "final", out.said + assert len(seen) == 2, "two are asked of, so two answers are waited for" + assert len(out.asked) == 2 and out.refused == [] + # In the order the answers arrived, which is not the order they are written. + assert out.answer["by"] == ["dana", "lee"] and out.answer["close"] is True + + +def test_one_no_is_a_no_and_the_rest_are_not_waited_for() -> None: + out, seen = asked("sign-off", [{"close": False, "answered-by": "dana"}, yes("lee")]) + assert out.answer == {"close": False, "by": ["dana"], "outcome": "declined"} + assert len(seen) == 1, "lee's answer is not waited for" + assert len(out.asked) == 2, "but lee was asked: both are, at once" + + +def test_the_stages_fields_are_those_of_the_answer_that_settled_it() -> None: + """Two people each add a note beside their yes. One value has to be the + stage's: the last yes that was needed. After a no, it is the no's.""" + out, _ = asked("sign-off", [{**yes("dana"), "note": "first"}, {**yes("lee"), "note": "second"}]) + assert out.answer["note"] == "second" + out, _ = asked( + "sign-off", + [{"close": False, "note": "not yet", "answered-by": "dana"}, {**yes("lee"), "note": "second"}], + ) + assert out.answer["note"] == "not yet" + + +def test_a_second_answer_from_somebody_who_answered_is_refused_and_the_wait_stays_open() -> None: + """Refused, and said so on the result. Not quietly counted once: whoever + handed it in would believe it had counted.""" + changed = {"close": False, "note": "On second thoughts, no.", "answered-by": "dana"} + out, seen = asked("sign-off", [yes("dana"), changed, yes("lee")]) + assert out.answer["by"] == ["dana", "lee"] and len(seen) == 3 + assert out.refused == [ + {"question": "may-we-close", "answered-by": "dana", "why": "already-answered"} + ] + # And nobody changes their answer by answering again: it is still a yes. + assert out.answer["close"] is True and out.answer["outcome"] == "answered" + + +def test_the_same_answer_handed_in_again_is_a_resend_and_refuses_nothing() -> None: + """Hosts resend: a file of answers passed whole at every resume, a retry + after a timeout. The same answer from the same person counts once.""" + out, seen = asked("sign-off", [yes("dana"), yes("dana"), yes("lee")]) + assert out.refused == [] and out.answer["by"] == ["dana", "lee"] and len(seen) == 3 -def test_one_person_is_a_list_of_one() -> None: - answer, _ = signed_off("dana") - assert answer == {"close": True, "by": ["dana"]} +def test_nobody_is_everyone_by_answering_again_and_again() -> None: + out, seen = asked("sign-off", [yes("dana")]) + assert out.halted == "stopped" + assert "came from somebody who had answered already" in out.said + assert len(seen) <= 5, "it does not go on asking for ever" + assert out.refused == [], "the same answer again is a resend, not a refusal" -def test_several_people_are_listed_in_the_order_they_answered() -> None: - answer, _ = signed_off(["lee", "dana"]) - assert answer["by"] == ["lee", "dana"] - answer, _ = signed_off(("lee", "dana")) - assert answer["by"] == ["lee", "dana"] +def test_one_answer_naming_two_people_is_refused_and_the_wait_stays_open() -> None: + out, _ = asked("sign-off", [yes(["lee", "dana"]), yes("dana"), yes("lee")]) + assert out.refused == [ + {"question": "may-we-close", "answered-by": ["lee", "dana"], "why": "several-people"} + ] + assert out.halted == "final" and out.answer["by"] == ["dana", "lee"] def test_who_answered_is_not_left_in_the_answer() -> None: - answer, _ = signed_off("dana") - assert "answered-by" not in answer + out, _ = asked("sign-off-by-the-team", [yes("dana")]) + assert "answered-by" not in out.answer def test_the_person_is_shown_what_the_question_says_to_show() -> None: - _, seen = signed_off("dana") - assert seen == [{"may-we-close": {"input.report": "Seat 14C will not recline."}}] + _, seen = asked("sign-off-by-the-team", [yes("dana")]) + assert seen == [{"may-the-team-close": {"input.report": "Seat 14C will not recline."}}] def test_on_a_decide_it_stays_one_name() -> None: diff --git a/docs/90-REVIEW.md b/docs/90-REVIEW.md index 6cbd16e..075ab5d 100644 --- a/docs/90-REVIEW.md +++ b/docs/90-REVIEW.md @@ -81,7 +81,7 @@ them and changes nothing. |---|---| | Schema | **43 groups, 262 field entries, 207 distinct field names** (core 123, expert 69, both 15), 40 closed enums, 27 cross-reference constraints, **0 fields missing `tier:` or `surface:`** | | Rust | 5 crates, **1679 tests**, `unsafe_code = forbid`, clippy-deny, edition 2024 | -| Python | **47 modules** (`find adapters/python/src -name "*.py" \| wc -l`), **3353 tests**, **nine transports** — seven model-bound (`anthropic`, `autogen`, `langchain`, `langgraph`, `ollama`, `openai_agents`, `pydantic_ai`), one bound to a remote *agent* (`a2a`), one deterministic mock; `_metering.py`, `_summarise.py` and `_tool_choice.py` are shared code the transports import, which is what the `_` says — offline by default | +| Python | **47 modules** (`find adapters/python/src -name "*.py" \| wc -l`), **3364 tests**, **nine transports** — seven model-bound (`anthropic`, `autogen`, `langchain`, `langgraph`, `ollama`, `openai_agents`, `pydantic_ai`), one bound to a remote *agent* (`a2a`), one deterministic mock; `_metering.py`, `_summarise.py` and `_tool_choice.py` are shared code the transports import, which is what the `_` says — offline by default | | TypeScript | 5 modules, 2,305 lines, a second port of **the stepping, the ceilings and the stage path** (`harness.ts:15-21`: *"AC-5.3's bar is the stage path"*) — **not** of the loop entire: no interceptor chain, no durable suspension record, no ledger (`notDoneHere`; a park returns `halted: "suspended"` and writes nothing down). This row is load-bearing for the Phase 3 pricing below (E4) | | CLI | **6 verbs**: `check`, `show`, `waits`, `discover`, `card`, `help` | | Worked example | 42 files, 1128 lines, **132 distinct authored keys**, `pact check` → OK (498 settings) | diff --git a/site-docs/guide/programs-in-the-language.md b/site-docs/guide/programs-in-the-language.md index d55b784..2f404fb 100644 --- a/site-docs/guide/programs-in-the-language.md +++ b/site-docs/guide/programs-in-the-language.md @@ -246,6 +246,11 @@ label stands when people pick different ones; `pact check` refuses it On a step that asks people, `.answered-by` is a list, in the order they answered, also when one person did. +`answered-by:` on the question counts the names under `asked-of:`. With +`everyone`, each of them is asked and answers on their own, and one "no" is a +no at once. A name there can be a person or a team; a team is answered for by +one of its members, and who that was is what `.answered-by` keeps. + ## Things at once ```text diff --git a/site-docs/index.md b/site-docs/index.md index add68bf..1ae2e52 100644 --- a/site-docs/index.md +++ b/site-docs/index.md @@ -47,7 +47,7 @@ so plainly rather than leaving it to be discovered. | | | |---|---| -| Tests | **5,034** — 1679 Rust, 3355 adapter | +| Tests | **5,043** — 1679 Rust, 3364 adapter | | Schema kinds | **69** | | Framework targets | **7**, over 2 runtimes | | Eve capabilities accounted for | **101 of 101** | diff --git a/site-docs/status/verified.md b/site-docs/status/verified.md index 5db9f1b..b74b453 100644 --- a/site-docs/status/verified.md +++ b/site-docs/status/verified.md @@ -13,8 +13,8 @@ from an earlier document. | Claim | Value | How to check | |---|---|---| | Rust tests | **1679** | `cargo test --workspace` | -| Adapter tests | **3353** | `cd adapters/python && uv run pytest tests/ -q` | -| Total | **5,034** | both of the above | +| Adapter tests | **3364** | `cd adapters/python && uv run pytest tests/ -q` | +| Total | **5,043** | both of the above | | TypeScript type-checks | clean | `cd adapters/typescript && npx tsc --noEmit` | | Lints | clean | `cargo clippy --all-targets` | | Worked example loads | 498 settings | `cargo run -p pact-cli -- check examples/refund-desk` | diff --git a/tests/conformance/digests.json b/tests/conformance/digests.json index a86b41b..03788f0 100644 --- a/tests/conformance/digests.json +++ b/tests/conformance/digests.json @@ -36,38 +36,42 @@ } }, "tests/conformance/trees/triaged-defects": { - "digest": "sha256:47a525825224e48e9637ba22a46cb48e07feeb54b1d1b63571bf8766240e5972", + "digest": "sha256:0634a2afd42db93355db2024735806a831e0a4124d6006e8029ddbf04a2ec501", "workflows": { "classify-only": { - "doc-digest": "sha256:07030d8f5c2089fb62da1f59049cfb3302da7b002cf56356d93d8424a9f1853b", + "doc-digest": "sha256:0470fe6f3f7de8eba44d7a9286071e6a6f57c28d8e1446b9f1aa2c854ebabc12", "contract-digest": "sha256:373c3db55151ef337d72e5aa2f767e3e19b907188b7c5b7dfdd62c37bcb19dc1" }, "draft-three-times": { - "doc-digest": "sha256:de737c26d7a73f9cba63916fea00641d31517155a6fbf4bc2e65e954862f8df4", + "doc-digest": "sha256:6c792aa92fed26a03a45ec78c237a50688df98b68745bc12b4430019abeb3b31", "contract-digest": "sha256:cc88efa5b071c1dd1763bdcc020c70abbb84103a922660589f4f2d80e24cc000" }, "draft-until-good": { - "doc-digest": "sha256:6e4d49e30ab09d8c37781040db51ae67eafacfc4680d02eece8527dba48fa822", + "doc-digest": "sha256:e4e31a1ebeb28e07a265775cdf1f6a1ea675f5001c5632b7d277a343326bcf07", "contract-digest": "sha256:1b9f8469dbe518f30be0660c5e440629cf5e6ef218ce951862b9ba70c4e04e82" }, "fix-item": { - "doc-digest": "sha256:56d8f0b3495c1ecc47f14ebd5e432fce6443477120923ece14e0cb195e87aaa7", + "doc-digest": "sha256:488bdd16a4969bf513c3be226ac99b16e96105ff78ce432f20e9cf06d6be8faa", "contract-digest": "sha256:8250dadd43a05efd7fb74f2704e1c072ac6d7de417a8b88e3fb5ebf8fe776936" }, "shelve": { - "doc-digest": "sha256:ee4a8473396bdd50eee38b3af86c18944c9084cdf2d4859e66471e009eed392d", + "doc-digest": "sha256:c31b89eba8be2489efa8ed9b8ec1249cfae6d8cd300ab541b222fecda11735ea", "contract-digest": "sha256:a3470d96b6e4b7c3ddc1b18e8673d70a81504ecbf800d71ad6adeda492ba84d5" }, "shelve-checked": { - "doc-digest": "sha256:8d83068b0f795509c9bded471ad27b4678001694963b568ca2d857922c84c8d0", + "doc-digest": "sha256:d6a85c179c52f5ca2f84df326b7634bd80da42eba409f745e02899dab78c9fda", "contract-digest": "sha256:d5f0555af2f5a9565c92d6900490948d071d442da170f0e493ef63496422f057" }, "shelve-twice": { - "doc-digest": "sha256:770476ad62ddc0c026521cc9d1565c053496373446c1bcbf3047e8251c10a830", + "doc-digest": "sha256:71721c73b13347d2250a23e5aed61ed5de07d0405a6aac85c03429e05f4c9ea5", "contract-digest": "sha256:a3470d96b6e4b7c3ddc1b18e8673d70a81504ecbf800d71ad6adeda492ba84d5" }, "sign-off": { - "doc-digest": "sha256:61c7e3c573c6603ecb99d820a072865beac733fcb4bf7b8630afdb44ac1f9a46", + "doc-digest": "sha256:8743d5a18c7369031478b133a38e8352e6bcfb753a68e148fb72abd22c7fe388", + "contract-digest": "sha256:43784b26caa17e0a651853fd8bf660ebc94dd00c32355fd5982ff1a3391d9a69" + }, + "sign-off-by-the-team": { + "doc-digest": "sha256:6856e75f81288d142bb52abcb1179230b0008c6468ed5e8e30394e2b1fddd046", "contract-digest": "sha256:e9594ce43d73dec2b1791b98eb1d6b433c5cf313957e421af97778fbc8225140" } } diff --git a/tests/conformance/trees/triaged-defects/defects.pact b/tests/conformance/trees/triaged-defects/defects.pact index 2c02fc1..7b99e7a 100644 --- a/tests/conformance/trees/triaged-defects/defects.pact +++ b/tests/conformance/trees/triaged-defects/defects.pact @@ -111,15 +111,25 @@ question which-shelf answer-within: 1h question may-we-close - description: Asked of the leads before a fix is closed. + description: Asked of both leads before a fix is closed. says: May this fix be closed? answer: close: yes or no + note: text, optional shows: [input.report] - asked-of: [leads] + asked-of: [lee, dana] answered-by: everyone answer-within: 1h +question may-the-team-close + description: Asked of the leads' team before a fix is closed. + says: May this fix be closed? + answer: + close: yes or no + shows: [input.report] + asked-of: [leads] + answer-within: 1h + agent checker description: Gives a second opinion on which shelf a note belongs on. takes message: text @@ -172,13 +182,28 @@ workflow classify-only takes report: text triage = call triager with report: input.report -# A stage that waits for people. Who answered is a list, always: a question -# can need several answers, and one name is a list of one. +# Stages that wait for people. Who answered is a list, always, in the order +# their answers arrived. +# +# Everyone asked must answer: each of `asked-of:` answers separately, and one +# no is a no. Each may add a note; the stage's note is that of the answer that +# settled it (the first no, or the last yes needed). workflow sign-off - description: Ask whether a fix may be closed, and say who answered. + description: Ask both leads whether a fix may be closed, and say who answered. takes report: text - gives close: yes or no, by: list of text + gives close: yes or no, by: list of text, outcome: text, note: text, optional approve = ask may-we-close + then: + declined: reply + reply = answer with close: approve.close, by: approve.answered-by, outcome: approve.outcome, note: approve.note + +# A team is asked and the first answer decides. Who in the team answered is +# the host's to say, so the name kept is not the name that was asked. +workflow sign-off-by-the-team + description: Ask the leads' team whether a fix may be closed, and say who answered. + takes report: text + gives close: yes or no, by: list of text + approve = ask may-the-team-close reply = answer with close: approve.close, by: approve.answered-by # A loop that is waiting for nothing: it runs its rounds and ends as any diff --git a/tests/conformance/workflows.json b/tests/conformance/workflows.json index 5de1dbd..154be08 100644 --- a/tests/conformance/workflows.json +++ b/tests/conformance/workflows.json @@ -1,5 +1,5 @@ { - "about": "Workflows: stages in the order they run, what each called agent and each person was given, and the answer. Replayed as conformance_kit.replay_workflow does: `agents` is what each agent answers when called (a list is one answer for each time it is asked, the last repeating); `tools` is what each `/` hands back; `people` is what a person answers to each question by name (absent: nobody is there to ask, and the run parks). In `expect`: `trace` is the stages that ran, nested ones as `outer/inner`; `given` is each agent called and its inputs, in order; `shown` is each question put to a person and what they were shown; `called` is each tool call. An agent a stage calls is an agent's run, under its own limits, approvals and reactions; here each is answered by the case, so what is compared is the workflow. A case expects only what the specification fixes: where the order things happen in is the runtime's to choose (an `each` with no `at-once-at-most:`), the tree a case runs on says `at-once-at-most: 1`. In `answer`, a field the run did not set is absent, not null; a run that stopped hands back an empty record, and why is in `said`. A trained classifier's answer carries `score` beside its label, a share of one: how sure it was of the label it picked, which `if-unsure-below:` is read against. A person's answer may carry `answered-by` beside the fields the question asks for: who answered, which the stage keeps as `steps..answered-by` and does not leave in the answer. On a `decide` that is one name; on a stage that waits for people (`ask`) it is a list, always, in the order they answered, so one name is a list of one.", + "about": "Workflows: stages in the order they run, what each called agent and each person was given, and the answer. Replayed as conformance_kit.replay_workflow does: `agents` is what each agent answers when called (a list is one answer for each time it is asked, the last repeating); `tools` is what each `/` hands back; `people` is what a person answers to each question by name (absent: nobody is there to ask, and the run parks). In `expect`: `trace` is the stages that ran, nested ones as `outer/inner`; `given` is each agent called and its inputs, in order; `shown` is each question put to somebody and what they were shown, once for each entry of the question's `asked-of:` (a question asked of two people is put twice, at once, whoever answers first; with nobody there to ask, nobody was asked); `refused` is each answer handed in that did not count, with who gave it and why (`already-answered`: that person had answered this wait; `several-people`: one answer named more than one answerer), and the wait stays open after one; the same answer from the same person handed in again is a resend, counts once, and is not refused; `called` is each tool call. An agent a stage calls is an agent's run, under its own limits, approvals and reactions; here each is answered by the case, so what is compared is the workflow. A case expects only what the specification fixes: where the order things happen in is the runtime's to choose (an `each` with no `at-once-at-most:`), the tree a case runs on says `at-once-at-most: 1`. In `answer`, a field the run did not set is absent, not null; a run that stopped hands back an empty record, and why is in `said`. A trained classifier's answer carries `score` beside its label, a share of one: how sure it was of the label it picked, which `if-unsure-below:` is read against. A person's answer may carry `answered-by` beside the fields the question asks for: who answered, which the stage keeps as `steps..answered-by` and does not leave in the answer. On a `decide` that is one name; on a stage that waits for people (`ask`) it is a list, always, in the order they answered, so one name is a list of one. `answered-by:` on the question counts the entries of `asked-of:`: with `everyone` each of them is asked and answers separately, one answer has one answerer, a second, different answer from somebody who has answered is refused and changes nothing, and one no is a no at once; the stage's fields are those of the answer that settled the wait, the first no or the answer that made it enough. In `people`, a list for a question is the answers to it as they arrive, in order, the last repeating; each carries who gave it.", "cases": [ { "name": "a-written-rule-decides-and-nobody-is-asked", @@ -41,6 +41,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -97,6 +98,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -153,6 +155,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [ { @@ -208,6 +211,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -250,6 +254,7 @@ "halted": "parked", "parked-at": "where", "question": "which-desk", + "refused": [], "said": "waiting on 'which-desk' at 'where'", "shown": [], "trace": [ @@ -328,6 +333,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -386,6 +392,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -492,6 +499,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -545,6 +553,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -605,6 +614,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -665,6 +675,7 @@ "halted": "stopped", "parked-at": "", "question": "", + "refused": [], "said": "the answer does not fit `answers-with:` — ideas: item 1 has 'idea', and idea has no part by that name\nitem 1 leaves out 'text', which should be some text", "shown": [], "trace": [ @@ -720,6 +731,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -772,6 +784,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -823,6 +836,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -873,6 +887,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -916,6 +931,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -964,6 +980,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -1010,6 +1027,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -1056,6 +1074,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [ { @@ -1109,6 +1128,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [ { @@ -1162,6 +1182,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -1199,6 +1220,7 @@ "halted": "parked", "parked-at": "where", "question": "which-shelf", + "refused": [], "said": "waiting on 'which-shelf' at 'where'", "shown": [], "trace": [ @@ -1251,6 +1273,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -1308,6 +1331,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -1348,6 +1372,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -1399,6 +1424,7 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [], "trace": [ @@ -1412,15 +1438,15 @@ } }, { - "name": "who-answered-a-wait-is-a-list-of-one-when-one-person-did", + "name": "a-team-is-asked-and-who-answered-is-a-list-of-the-one-who-did", "tree": "tests/conformance/trees/triaged-defects", - "workflow": "sign-off", + "workflow": "sign-off-by-the-team", "input": { "report": "Seat 14C will not recline." }, "agents": {}, "people": { - "may-we-close": { + "may-the-team-close": { "close": true, "answered-by": "dana" } @@ -1437,10 +1463,11 @@ "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [ { - "question": "may-we-close", + "question": "may-the-team-close", "shown": { "input.report": "Seat 14C will not recline." } @@ -1454,7 +1481,7 @@ } }, { - "name": "who-answered-a-wait-is-everyone-who-did-in-the-order-they-answered", + "name": "everyone-asked-answers-separately-and-is-listed-in-the-order-they-answered", "tree": "tests/conformance/trees/triaged-defects", "workflow": "sign-off", "input": { @@ -1462,29 +1489,231 @@ }, "agents": {}, "people": { - "may-we-close": { + "may-we-close": [ + { + "close": true, + "note": "Checked on the aircraft.", + "answered-by": "dana" + }, + { + "close": true, + "note": "Parts are logged.", + "answered-by": "lee" + } + ] + }, + "expect": { + "answer": { + "by": [ + "dana", + "lee" + ], "close": true, - "answered-by": [ - "lee", - "dana" - ] - } + "note": "Parts are logged.", + "outcome": "answered" + }, + "called": [], + "given": [], + "halted": "final", + "parked-at": "", + "question": "", + "refused": [], + "said": "", + "shown": [ + { + "question": "may-we-close", + "shown": { + "input.report": "Seat 14C will not recline." + } + }, + { + "question": "may-we-close", + "shown": { + "input.report": "Seat 14C will not recline." + } + } + ], + "trace": [ + "approve", + "reply" + ], + "unenforced": [] + } + }, + { + "name": "one-no-from-everyone-asked-is-a-no-and-the-rest-are-not-waited-for", + "tree": "tests/conformance/trees/triaged-defects", + "workflow": "sign-off", + "input": { + "report": "Seat 14C will not recline." + }, + "agents": {}, + "people": { + "may-we-close": [ + { + "close": false, + "note": "The recline cable is still loose.", + "answered-by": "dana" + }, + { + "close": true, + "note": "Parts are logged.", + "answered-by": "lee" + } + ] }, "expect": { "answer": { "by": [ - "lee", "dana" ], - "close": true + "close": false, + "note": "The recline cable is still loose.", + "outcome": "declined" }, "called": [], "given": [], "halted": "final", "parked-at": "", "question": "", + "refused": [], "said": "", "shown": [ + { + "question": "may-we-close", + "shown": { + "input.report": "Seat 14C will not recline." + } + }, + { + "question": "may-we-close", + "shown": { + "input.report": "Seat 14C will not recline." + } + } + ], + "trace": [ + "approve", + "reply" + ], + "unenforced": [] + } + }, + { + "name": "a-second-answer-from-somebody-who-has-answered-is-refused-and-the-wait-stays-open", + "tree": "tests/conformance/trees/triaged-defects", + "workflow": "sign-off", + "input": { + "report": "Seat 14C will not recline." + }, + "agents": {}, + "people": { + "may-we-close": [ + { + "close": true, + "answered-by": "dana" + }, + { + "close": false, + "note": "On second thoughts, no.", + "answered-by": "dana" + }, + { + "close": true, + "answered-by": "lee" + } + ] + }, + "expect": { + "answer": { + "by": [ + "dana", + "lee" + ], + "close": true, + "outcome": "answered" + }, + "called": [], + "given": [], + "halted": "final", + "parked-at": "", + "question": "", + "refused": [ + { + "answered-by": "dana", + "question": "may-we-close", + "why": "already-answered" + } + ], + "said": "", + "shown": [ + { + "question": "may-we-close", + "shown": { + "input.report": "Seat 14C will not recline." + } + }, + { + "question": "may-we-close", + "shown": { + "input.report": "Seat 14C will not recline." + } + } + ], + "trace": [ + "approve", + "reply" + ], + "unenforced": [] + } + }, + { + "name": "the-same-answer-handed-in-again-is-not-a-second-answer", + "tree": "tests/conformance/trees/triaged-defects", + "workflow": "sign-off", + "input": { + "report": "Seat 14C will not recline." + }, + "agents": {}, + "people": { + "may-we-close": [ + { + "close": true, + "answered-by": "dana" + }, + { + "close": true, + "answered-by": "dana" + }, + { + "close": true, + "answered-by": "lee" + } + ] + }, + "expect": { + "answer": { + "by": [ + "dana", + "lee" + ], + "close": true, + "outcome": "answered" + }, + "called": [], + "given": [], + "halted": "final", + "parked-at": "", + "question": "", + "refused": [], + "said": "", + "shown": [ + { + "question": "may-we-close", + "shown": { + "input.report": "Seat 14C will not recline." + } + }, { "question": "may-we-close", "shown": {