From a5de6579b981615410bda87544f4499100c1249c Mon Sep 17 00:00:00 2001 From: DABH Date: Thu, 10 Sep 2026 00:45:01 -0500 Subject: [PATCH 1/2] Pass pydantic_core through the workflow sandbox by default The OpenAI Agents replay tests intermittently failed on slow CI runners with "[TMPRL1101] Potential deadlock detected" and a stack showing the workflow thread inside the sandbox importer executing pydantic_core/core_schema.py. `pydantic` is in the default passthrough list but `pydantic_core` was not, and openai's `_compat.field_get_default` imports it lazily from the plugin's payload converter. So the first `resolve_activity` activation of every sandboxed workflow re-executed the ~4.6k-line pydantic_core package inside the sandbox, inside the 2s deadlock budget, serialized with every other workflow thread doing the same through the import lock and the GIL. Pass pydantic_core through alongside pydantic, for the same reason pydantic is passed through: Pydantic-based libraries import it lazily. It is pydantic's version-locked compiled core with no workflow state, and its extension objects were already shared across sandboxes. Three contrib plugins (strands, google_genai, deepagents) already pass it through explicitly. Unloaded, the first activation drops from 6-57ms to 1-2ms and no module is copied into the sandbox during activations anymore. 128 concurrent first activations went from 0.7s max latency (serialized import) to ~0. Under emulated CPU starvation (background QoS plus 72 CPU burners) the in-sandbox import took 0.5-0.9s per workflow; passthrough takes ~0. The replay tests now also assert that no module is imported into the sandbox after initial workflow load, so a reintroduced in-activation import fails deterministically instead of as a timing-dependent deadlock. --- CHANGELOG.md | 4 ++++ temporalio/worker/workflow_sandbox/_restrictions.py | 2 ++ tests/contrib/openai_agents/test_openai_replay.py | 12 ++++++++++++ 3 files changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40e806916..d4a84ef42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,10 @@ to include examples, links to docs, or any other relevant information. retried on its configured interval instead. - Nexus-context workflow/activity starts no longer set `on_conflict_options` when there are no links or callbacks to attach. +- The workflow sandbox now passes `pydantic_core` through by default, alongside `pydantic`. + Libraries built on Pydantic (e.g. `openai`) import it lazily, so every sandboxed workflow used to + re-import it during its first activation. That import counted toward the deadlock detection + timeout and could fail workflow tasks on slow or overloaded workers. ### Security diff --git a/temporalio/worker/workflow_sandbox/_restrictions.py b/temporalio/worker/workflow_sandbox/_restrictions.py index 23774fcd2..3e6e0d6ba 100644 --- a/temporalio/worker/workflow_sandbox/_restrictions.py +++ b/temporalio/worker/workflow_sandbox/_restrictions.py @@ -513,6 +513,8 @@ def with_child_unrestricted(self, *child_path: str) -> SandboxMatcher: # Due to how Pydantic is importing lazily inside of some classes, we choose # to always pass it through "pydantic", + # Same for its compiled core, which Pydantic-based libraries import lazily + "pydantic_core", } ) diff --git a/tests/contrib/openai_agents/test_openai_replay.py b/tests/contrib/openai_agents/test_openai_replay.py index 6db463392..7c9a586b2 100644 --- a/tests/contrib/openai_agents/test_openai_replay.py +++ b/tests/contrib/openai_agents/test_openai_replay.py @@ -1,3 +1,4 @@ +import warnings from pathlib import Path import pytest @@ -32,6 +33,10 @@ async def test_replay(file_name: str) -> None: with (Path(__file__).with_name("histories") / file_name).open("r") as f: history_json = f.read() + with warnings.catch_warnings(record=True) as recorder: + warnings.filterwarnings( + "always", message=r"Module .* was imported after initial workflow load" + ) await Replayer( workflows=[ ResearchWorkflow, @@ -44,3 +49,10 @@ async def test_replay(file_name: str) -> None: ], plugins=[OpenAIAgentsPlugin()], ).replay_workflow(WorkflowHistory.from_json("fake", history_json)) + + # Sandbox imports during an activation count toward the deadlock timeout + assert not [ + str(w.message) + for w in recorder + if "was imported after initial workflow load" in str(w.message) + ] From 5e533b314c9aad2bf41982d09bc90616e35e53ae Mon Sep 17 00:00:00 2001 From: David Hyde Date: Thu, 10 Sep 2026 03:02:12 -0500 Subject: [PATCH 2/2] clean up changelog --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4a84ef42..7769f776d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,9 +67,6 @@ to include examples, links to docs, or any other relevant information. - Nexus-context workflow/activity starts no longer set `on_conflict_options` when there are no links or callbacks to attach. - The workflow sandbox now passes `pydantic_core` through by default, alongside `pydantic`. - Libraries built on Pydantic (e.g. `openai`) import it lazily, so every sandboxed workflow used to - re-import it during its first activation. That import counted toward the deadlock detection - timeout and could fail workflow tasks on slow or overloaded workers. ### Security