From 4925cbd6fd64eebd741f87a5af77d0f6ddb10bb2 Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Mon, 13 Jul 2026 18:54:18 +0100 Subject: [PATCH 01/34] typo in events readme --- dagrunner/events.py | 2 +- docs/dagrunner.events.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dagrunner/events.py b/dagrunner/events.py index d0fece1..fa619b7 100644 --- a/dagrunner/events.py +++ b/dagrunner/events.py @@ -36,7 +36,7 @@ Only the non-ignored cubes (`cube1` and `cube3`) reach `Proc`; the ignored inputs are dropped. -### `event.IGNORE` +### `event.SKIP` The SKIP event differs from the IGNORE event in that if **any** input to a plugin is a SKIP event, node execution is skipped and it instead propagates this skip event so that all dependent nodes and their descendants aren't executed. diff --git a/docs/dagrunner.events.md b/docs/dagrunner.events.md index 5616830..eb7f512 100644 --- a/docs/dagrunner.events.md +++ b/docs/dagrunner.events.md @@ -35,7 +35,7 @@ Here, Input2 and Input4 return an IGNORE event, likely due to there being missin Only the non-ignored cubes (`cube1` and `cube3`) reach `Proc`; the ignored inputs are dropped. -### `event.IGNORE` +### `event.SKIP` The SKIP event differs from the IGNORE event in that if **any** input to a plugin is a SKIP event, node execution is skipped and it instead propagates this skip event so that all dependent nodes and their descendants aren't executed. From 603fdad1ec5bbd14a63a26314cb9b1d9aabb8388 Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Mon, 13 Jul 2026 19:02:32 +0100 Subject: [PATCH 02/34] update events docs with current names --- dagrunner/events.py | 35 +++++++++++++++++----------------- docs/dagrunner.events.md | 41 ++++++++++++++++++---------------------- 2 files changed, 35 insertions(+), 41 deletions(-) diff --git a/dagrunner/events.py b/dagrunner/events.py index fa619b7..a8c4519 100644 --- a/dagrunner/events.py +++ b/dagrunner/events.py @@ -7,28 +7,28 @@ DAGrunner defines two special singleton events that plugins can return to control the execution flow of their graph. -- event.IGNORE - removes a particular input from further processing. -- event.SKIP - aborts the execution of a plugin (and all downstream nodes) when any +- IGNORE_EVENT - removes a particular input from further processing. +- SKIP_EVENT - aborts the execution of a plugin (and all downstream nodes) when any input carries this event. -### event.IGNORE -When a plugin returns `event.IGNORE`, the immediate descendant node filters out that +### _IgnoreEvent: `IGNORE_EVENT` +When a plugin returns `IGNORE_EVENT`, the immediate descendant node filters out that input. The remaining inputs of that plugin are utilised by that node as normal. -If all inputs to a node are `event.IGNORE`, the node's execution is skipped, and a -`event.SKIP` event is returned instead, skipping execution through all descendants. +If all inputs to a node are `IGNORE_EVENT`, the node's execution is skipped, and a +`SKIP_EVENT` event is returned instead, skipping execution through all descendants. ```mermaid --- -title: event.IGNORE (filtering some) +title: IGNORE_EVENT (filtering some) --- graph cycle1{cycleX} cycle1 --> Input1 --> filepath --> Load1 --> cube1 --> Proc - cycle1 --> Input2 --> filepath --> Load2 --> event.IGNORE --> Proc + cycle1 --> Input2 --> filepath --> Load2 --> IGNORE_EVENT --> Proc cycle1 --> Input3 --> filepath --> Load3 --> cube3 --> Proc - cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc + cycle1 --> Input4 --> filepath --> Load4 --> IGNORE_EVENT --> Proc Proc --> cube --> Save Proc["Proc (cube1, cube3)"] ``` @@ -36,27 +36,26 @@ Only the non-ignored cubes (`cube1` and `cube3`) reach `Proc`; the ignored inputs are dropped. -### `event.SKIP` -The SKIP event differs from the IGNORE event in that if **any** input to a plugin is a -SKIP event, node execution is skipped and it instead propagates this skip event so that +### _SkipEvent: `SKIP_EVENT` +The skip event differs from the ignore event in that if **any** input to a plugin is +`SKIP_EVENT`, node execution is skipped and it instead propagates this skip event so that all dependent nodes and their descendants aren't executed. ```mermaid --- -title: event.SKIP +title: SKIP_EVENT --- graph cycle1{cycleX} cycle1 --> Input1 --> filepath --> Load1 --> cube --> Proc - cycle1 --> Input2 --> filepath --> Load2 --> event.SKIP --> Proc + cycle1 --> Input2 --> filepath --> Load2 --> SKIP_EVENT --> Proc cycle1 --> Input3 --> filepath --> Load3 --> cube --> Proc - cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc - Proc --> event.SKIP --> Save + cycle1 --> Input4 --> filepath --> Load4 --> IGNORE_EVENT --> Proc + Proc --> SKIP_EVENT --> Save ``` -Because `Input2` returns a SKIP event, the Proc node and everything that follows aren't +Because `Input2` returns `SKIP_EVENT`, the Proc node and everything that follows aren't executed and neither is the Save node since the skip is propagated along the execution graph. - """ from dagrunner.utils import Singleton diff --git a/docs/dagrunner.events.md b/docs/dagrunner.events.md index eb7f512..50b7468 100644 --- a/docs/dagrunner.events.md +++ b/docs/dagrunner.events.md @@ -6,28 +6,28 @@ DAGrunner defines two special singleton events that plugins can return to control the execution flow of their graph. -- event.IGNORE - removes a particular input from further processing. -- event.SKIP - aborts the execution of a plugin (and all downstream nodes) when any +- IGNORE_EVENT - removes a particular input from further processing. +- SKIP_EVENT - aborts the execution of a plugin (and all downstream nodes) when any input carries this event. -### event.IGNORE -When a plugin returns `event.IGNORE`, the immediate descendant node filters out that +### _IgnoreEvent: `IGNORE_EVENT` +When a plugin returns `IGNORE_EVENT`, the immediate descendant node filters out that input. The remaining inputs of that plugin are utilised by that node as normal. -If all inputs to a node are `event.IGNORE`, the node's execution is skipped, and a -`event.SKIP` event is returned instead, skipping execution through all descendants. +If all inputs to a node are `IGNORE_EVENT`, the node's execution is skipped, and a +`SKIP_EVENT` event is returned instead, skipping execution through all descendants. ```mermaid --- -title: event.IGNORE (filtering some) +title: IGNORE_EVENT (filtering some) --- graph cycle1{cycleX} cycle1 --> Input1 --> filepath --> Load1 --> cube1 --> Proc - cycle1 --> Input2 --> filepath --> Load2 --> event.IGNORE --> Proc + cycle1 --> Input2 --> filepath --> Load2 --> IGNORE_EVENT --> Proc cycle1 --> Input3 --> filepath --> Load3 --> cube3 --> Proc - cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc + cycle1 --> Input4 --> filepath --> Load4 --> IGNORE_EVENT --> Proc Proc --> cube --> Save Proc["Proc (cube1, cube3)"] ``` @@ -35,30 +35,25 @@ Here, Input2 and Input4 return an IGNORE event, likely due to there being missin Only the non-ignored cubes (`cube1` and `cube3`) reach `Proc`; the ignored inputs are dropped. -### `event.SKIP` -The SKIP event differs from the IGNORE event in that if **any** input to a plugin is a -SKIP event, node execution is skipped and it instead propagates this skip event so that +### _SkipEvent: `SKIP_EVENT` +The skip event differs from the ignore event in that if **any** input to a plugin is +`SKIP_EVENT`, node execution is skipped and it instead propagates this skip event so that all dependent nodes and their descendants aren't executed. ```mermaid --- -title: event.SKIP +title: SKIP_EVENT --- graph cycle1{cycleX} cycle1 --> Input1 --> filepath --> Load1 --> cube --> Proc - cycle1 --> Input2 --> filepath --> Load2 --> event.SKIP --> Proc + cycle1 --> Input2 --> filepath --> Load2 --> SKIP_EVENT --> Proc cycle1 --> Input3 --> filepath --> Load3 --> cube --> Proc - cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc - Proc --> event.SKIP --> Save + cycle1 --> Input4 --> filepath --> Load4 --> IGNORE_EVENT --> Proc + Proc --> SKIP_EVENT --> Save ``` -Because `Input2` returns a SKIP event, the Proc node and everything that follows aren't +Because `Input2` returns `SKIP_EVENT`, the Proc node and everything that follows aren't executed and neither is the Save node since the skip is propagated along the execution graph. -see [class: dagrunner.utils.Singleton](dagrunner.utils.md#class-singleton) - -## _IgnoreEvent: `IGNORE_EVENT` - -## _SkipEvent: `SKIP_EVENT` - +see [class: dagrunner.utils.Singleton](dagrunner.utils.md#class-singleton) \ No newline at end of file From beef41af2d6e9e39fb4849375ac206c5745fa079 Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Mon, 13 Jul 2026 19:07:22 +0100 Subject: [PATCH 03/34] If Load plugin receives an empty argument list, catch the error and allow it to return SKIP and IGNORE events, as if no files had been found. This is an edge case for when the EPP input_latest plugin returns None when it cannot find any files. --- dagrunner/plugin_framework.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dagrunner/plugin_framework.py b/dagrunner/plugin_framework.py index ba215e3..1ae5b11 100644 --- a/dagrunner/plugin_framework.py +++ b/dagrunner/plugin_framework.py @@ -133,6 +133,7 @@ def __call__(self, *args, **kwargs): Raises: - FileNotFoundError: If any of the files do not exist and `on_missing` is set to 'error'. + - ValueError: If `on_missing` is set to an invalid value. """ args = list(map(process_path, args)) @@ -143,7 +144,7 @@ def __call__(self, *args, **kwargs): raise ValueError( "Staging directory must be specified for loading remote files." ) - + try: if self._staging_dir and args: args = stage_to_dir( @@ -155,21 +156,23 @@ def __call__(self, *args, **kwargs): raise FileNotFoundError( f"No such file or directory: {', '.join(missing_files)}" ) - except FileNotFoundError as e: + elif len(args) == 0: + raise ValueError(f"Empty input arguments.") + + except (FileNotFoundError, ValueError) as e: if self._on_missing == "error": raise e elif self._on_missing == "ignore": - warnings.warn(str(e)) + warnings.warn(str(e) + " Ignoring node.") return events.IGNORE_EVENT elif self._on_missing == "skip": - warnings.warn(str(e)) + warnings.warn(str(e) + " Skipping node") return events.SKIP_EVENT else: raise ValueError( f"Invalid value for 'on_missing': {self._on_missing}. " "Accepted values are 'error', 'ignore', and 'skip'." ) - return self.load(*args, **kwargs) From 94f57c1e1cea9dafe67883ae2fea7089feac3124 Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Mon, 13 Jul 2026 19:12:31 +0100 Subject: [PATCH 04/34] Allow nodes with only one input to return IGNORE_EVENT. --- dagrunner/execute_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dagrunner/execute_graph.py b/dagrunner/execute_graph.py index 2df4ff1..6de36f2 100755 --- a/dagrunner/execute_graph.py +++ b/dagrunner/execute_graph.py @@ -149,7 +149,7 @@ def plugin_executor( call = as_iterable(call) # IGNORE_EVENT event handling - if len(args) > 1 and all((map(lambda x: x is events.IGNORE_EVENT, args))): + if len(args) > 0 and all((map(lambda x: x is events.IGNORE_EVENT, args))): # all args are IGNORE_EVENT, return IGNORE_EVENT (pass along) if verbose: print(f"Retuning 'IGNORE_EVENT' event {call[0]}") From 081dd94008880830de1c69a3338e1776b4035590 Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Mon, 13 Jul 2026 20:51:13 +0100 Subject: [PATCH 05/34] logging --- dagrunner/plugin_framework.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dagrunner/plugin_framework.py b/dagrunner/plugin_framework.py index 1ae5b11..2aa0af1 100644 --- a/dagrunner/plugin_framework.py +++ b/dagrunner/plugin_framework.py @@ -157,16 +157,16 @@ def __call__(self, *args, **kwargs): f"No such file or directory: {', '.join(missing_files)}" ) elif len(args) == 0: - raise ValueError(f"Empty input arguments.") + raise ValueError(f"Empty input arguments") except (FileNotFoundError, ValueError) as e: if self._on_missing == "error": raise e elif self._on_missing == "ignore": - warnings.warn(str(e) + " Ignoring node.") + warnings.warn(str(e) + "\nIgnoring node.") return events.IGNORE_EVENT elif self._on_missing == "skip": - warnings.warn(str(e) + " Skipping node") + warnings.warn(str(e) + "\nSkipping node.") return events.SKIP_EVENT else: raise ValueError( From 3f105f6c949bcf30c1b503126a2f99a45bb8d6a1 Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Tue, 14 Jul 2026 10:30:27 +0100 Subject: [PATCH 06/34] precommit --- dagrunner/events.py | 6 +++--- dagrunner/plugin_framework.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dagrunner/events.py b/dagrunner/events.py index a8c4519..70b5a50 100644 --- a/dagrunner/events.py +++ b/dagrunner/events.py @@ -37,9 +37,9 @@ dropped. ### _SkipEvent: `SKIP_EVENT` -The skip event differs from the ignore event in that if **any** input to a plugin is -`SKIP_EVENT`, node execution is skipped and it instead propagates this skip event so that -all dependent nodes and their descendants aren't executed. +The skip event differs from the ignore event in that if **any** input to a plugin is +`SKIP_EVENT`, node execution is skipped and it instead propagates this skip event so +that all dependent nodes and their descendants aren't executed. ```mermaid --- diff --git a/dagrunner/plugin_framework.py b/dagrunner/plugin_framework.py index 2aa0af1..9b4daae 100644 --- a/dagrunner/plugin_framework.py +++ b/dagrunner/plugin_framework.py @@ -144,7 +144,7 @@ def __call__(self, *args, **kwargs): raise ValueError( "Staging directory must be specified for loading remote files." ) - + try: if self._staging_dir and args: args = stage_to_dir( @@ -157,8 +157,8 @@ def __call__(self, *args, **kwargs): f"No such file or directory: {', '.join(missing_files)}" ) elif len(args) == 0: - raise ValueError(f"Empty input arguments") - + raise ValueError("Empty input arguments") + except (FileNotFoundError, ValueError) as e: if self._on_missing == "error": raise e From adc9d3faa48434855e65fe80f4a0c0b35eb611d7 Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Tue, 14 Jul 2026 13:59:43 +0100 Subject: [PATCH 07/34] tests for new load functionality --- dagrunner/tests/plugin_framework/test_LoadJson.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dagrunner/tests/plugin_framework/test_LoadJson.py b/dagrunner/tests/plugin_framework/test_LoadJson.py index d51d509..8545fff 100644 --- a/dagrunner/tests/plugin_framework/test_LoadJson.py +++ b/dagrunner/tests/plugin_framework/test_LoadJson.py @@ -50,6 +50,19 @@ def test_missing_data_skip(): assert LoadJson(on_missing="skip")("dummy-file") == SKIP_EVENT +def test_empty_args_error(): + with pytest.raises(ValueError, match="Empty input arguments"): + LoadJson(on_missing="error")() + + +def test_empty_args_ignore(): + assert LoadJson(on_missing="ignore")() == IGNORE_EVENT + + +def test_empty_args_skip(): + assert LoadJson(on_missing="skip")() == SKIP_EVENT + + def test_missing_data_unknown(): with pytest.raises( ValueError, From 3012692cc4de3bbac0fb0f37ed4fc1ed0dbe200a Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Tue, 14 Jul 2026 14:21:17 +0100 Subject: [PATCH 08/34] tests for single input events in plugin executor --- .../tests/execute_graph/test_plugin_executor.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dagrunner/tests/execute_graph/test_plugin_executor.py b/dagrunner/tests/execute_graph/test_plugin_executor.py index de218f8..71500f2 100644 --- a/dagrunner/tests/execute_graph/test_plugin_executor.py +++ b/dagrunner/tests/execute_graph/test_plugin_executor.py @@ -119,6 +119,12 @@ def test_ignore_event(): assert res == IGNORE_EVENT +def test_ignore_event_single_input(): + """Check what happens when the sole argument is IGNORE_EVENT.""" + res = plugin_executor(*(IGNORE_EVENT,), call=()) + assert res == IGNORE_EVENT + + def test_skip_event(): """Check what happens a subset of arguments are SKIP_EVENT.""" call = tuple([lambda x: x + 5]) @@ -126,6 +132,12 @@ def test_skip_event(): assert res == SKIP_EVENT +def test_skip_event_single_input(): + """Check what happens when the sole argument is SKIP_EVENT.""" + res = plugin_executor(*(SKIP_EVENT,), call=()) + assert res == SKIP_EVENT + + def test_pass_common_args(): """ Passing 'common args', some relevant to class init and some to call method. From f747ae93b64b7d09f47b02f0b9b690d8d2801cfe Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Wed, 15 Jul 2026 10:08:52 +0100 Subject: [PATCH 09/34] alter skip/ignore log --- dagrunner/plugin_framework.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dagrunner/plugin_framework.py b/dagrunner/plugin_framework.py index 9b4daae..981163c 100644 --- a/dagrunner/plugin_framework.py +++ b/dagrunner/plugin_framework.py @@ -163,10 +163,10 @@ def __call__(self, *args, **kwargs): if self._on_missing == "error": raise e elif self._on_missing == "ignore": - warnings.warn(str(e) + "\nIgnoring node.") + warnings.warn(f"Ignoring node due to error {str(e)!r}") return events.IGNORE_EVENT elif self._on_missing == "skip": - warnings.warn(str(e) + "\nSkipping node.") + warnings.warn(f"Skipping node due to error {str(e)!r}") return events.SKIP_EVENT else: raise ValueError( From e9cfb22d66f917acd571108ac64dbb77e4a4f6c5 Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Wed, 15 Jul 2026 10:37:08 +0100 Subject: [PATCH 10/34] additional update to events docs to accurately reflect what happens in plugin_executor. If all inputs to a node are 'IGNORE_EVENT', that node also returns 'IGNORE_EVENT'. --- dagrunner/events.py | 13 ++++++------- docs/dagrunner.events.md | 11 +++++------ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/dagrunner/events.py b/dagrunner/events.py index 70b5a50..5d0cbcf 100644 --- a/dagrunner/events.py +++ b/dagrunner/events.py @@ -14,10 +14,9 @@ ### _IgnoreEvent: `IGNORE_EVENT` When a plugin returns `IGNORE_EVENT`, the immediate descendant node filters out that -input. -The remaining inputs of that plugin are utilised by that node as normal. -If all inputs to a node are `IGNORE_EVENT`, the node's execution is skipped, and a -`SKIP_EVENT` event is returned instead, skipping execution through all descendants. +input. The remaining inputs of that plugin are utilised by that node as normal. +If all inputs to a node are `IGNORE_EVENT`, the node's execution is skipped, and +`IGNORE_EVENT` is returned instead. ```mermaid --- @@ -37,9 +36,9 @@ dropped. ### _SkipEvent: `SKIP_EVENT` -The skip event differs from the ignore event in that if **any** input to a plugin is -`SKIP_EVENT`, node execution is skipped and it instead propagates this skip event so -that all dependent nodes and their descendants aren't executed. +The skip event differs from the ignore event in that if **any** input to a plugin is +`SKIP_EVENT`, node execution is skipped and `SKIP_EVENT` is returned instead. This skip +propagates so that all dependent nodes and their descendants are not executed. ```mermaid --- diff --git a/docs/dagrunner.events.md b/docs/dagrunner.events.md index 50b7468..dc3b7c1 100644 --- a/docs/dagrunner.events.md +++ b/docs/dagrunner.events.md @@ -13,10 +13,9 @@ execution flow of their graph. ### _IgnoreEvent: `IGNORE_EVENT` When a plugin returns `IGNORE_EVENT`, the immediate descendant node filters out that -input. -The remaining inputs of that plugin are utilised by that node as normal. -If all inputs to a node are `IGNORE_EVENT`, the node's execution is skipped, and a -`SKIP_EVENT` event is returned instead, skipping execution through all descendants. +input. The remaining inputs of that plugin are utilised by that node as normal. +If all inputs to a node are `IGNORE_EVENT`, the node's execution is skipped, and +`IGNORE_EVENT` is returned instead. ```mermaid --- @@ -37,8 +36,8 @@ dropped. ### _SkipEvent: `SKIP_EVENT` The skip event differs from the ignore event in that if **any** input to a plugin is -`SKIP_EVENT`, node execution is skipped and it instead propagates this skip event so that -all dependent nodes and their descendants aren't executed. +`SKIP_EVENT`, node execution is skipped and `SKIP_EVENT` is returned instead. This skip +propagates so that all dependent nodes and their descendants are not executed. ```mermaid --- From 73ab9bf1ee9c3eba044183c904473e2e928e87c1 Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Wed, 15 Jul 2026 14:21:42 +0100 Subject: [PATCH 11/34] precommit --- dagrunner/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dagrunner/events.py b/dagrunner/events.py index 5d0cbcf..bc770f9 100644 --- a/dagrunner/events.py +++ b/dagrunner/events.py @@ -36,7 +36,7 @@ dropped. ### _SkipEvent: `SKIP_EVENT` -The skip event differs from the ignore event in that if **any** input to a plugin is +The skip event differs from the ignore event in that if **any** input to a plugin is `SKIP_EVENT`, node execution is skipped and `SKIP_EVENT` is returned instead. This skip propagates so that all dependent nodes and their descendants are not executed. From 8b2bd1d609cb4f4852ab59d2115fd69ac13b3801 Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Wed, 15 Jul 2026 14:58:28 +0100 Subject: [PATCH 12/34] reword warning message --- dagrunner/plugin_framework.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dagrunner/plugin_framework.py b/dagrunner/plugin_framework.py index 981163c..dbd509e 100644 --- a/dagrunner/plugin_framework.py +++ b/dagrunner/plugin_framework.py @@ -163,10 +163,10 @@ def __call__(self, *args, **kwargs): if self._on_missing == "error": raise e elif self._on_missing == "ignore": - warnings.warn(f"Ignoring node due to error {str(e)!r}") + warnings.warn(f"Ignoring node due to caught exception {str(e)!r}") return events.IGNORE_EVENT elif self._on_missing == "skip": - warnings.warn(f"Skipping node due to error {str(e)!r}") + warnings.warn(f"Skipping node due to caught exception {str(e)!r}") return events.SKIP_EVENT else: raise ValueError( From 10f7592cc628669a55b8d77f02642026e9ca32b9 Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Tue, 21 Jul 2026 15:51:46 +0100 Subject: [PATCH 13/34] Combine ignore tests with parametrize Co-authored-by: Carwyn Pelley --- .../tests/execute_graph/test_plugin_executor.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/dagrunner/tests/execute_graph/test_plugin_executor.py b/dagrunner/tests/execute_graph/test_plugin_executor.py index 71500f2..889a35b 100644 --- a/dagrunner/tests/execute_graph/test_plugin_executor.py +++ b/dagrunner/tests/execute_graph/test_plugin_executor.py @@ -122,7 +122,19 @@ def test_ignore_event(): def test_ignore_event_single_input(): """Check what happens when the sole argument is IGNORE_EVENT.""" res = plugin_executor(*(IGNORE_EVENT,), call=()) - assert res == IGNORE_EVENT +@pytest.mark.parametrize( + "args, expected", + [ + ((5, IGNORE_EVENT), 10), + ((IGNORE_EVENT, IGNORE_EVENT), IGNORE_EVENT), + ((IGNORE_EVENT,), IGNORE_EVENT), + ], +) +def test_ignore_event(args, expected): + """Check IGNORE_EVENT behaviour across mixed, all, and single-input cases.""" + call = tuple([lambda x: x + 5]) + res = plugin_executor(*args, call=call) + assert res == expected def test_skip_event(): From 26fb855df02e7d8ccb047e25a907a8559f11d35c Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Tue, 21 Jul 2026 15:52:57 +0100 Subject: [PATCH 14/34] Combine skip tests with parametrize Co-authored-by: Carwyn Pelley --- .../tests/execute_graph/test_plugin_executor.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/dagrunner/tests/execute_graph/test_plugin_executor.py b/dagrunner/tests/execute_graph/test_plugin_executor.py index 889a35b..c3e4656 100644 --- a/dagrunner/tests/execute_graph/test_plugin_executor.py +++ b/dagrunner/tests/execute_graph/test_plugin_executor.py @@ -137,16 +137,11 @@ def test_ignore_event(args, expected): assert res == expected -def test_skip_event(): - """Check what happens a subset of arguments are SKIP_EVENT.""" +@pytest.mark.parametrize("args", [(5, SKIP_EVENT), (SKIP_EVENT,)]) +def test_skip_event(args): + """Check what happens when any input argument is SKIP_EVENT.""" call = tuple([lambda x: x + 5]) - res = plugin_executor(*(5, SKIP_EVENT), call=call) - assert res == SKIP_EVENT - - -def test_skip_event_single_input(): - """Check what happens when the sole argument is SKIP_EVENT.""" - res = plugin_executor(*(SKIP_EVENT,), call=()) + res = plugin_executor(*args, call=call) assert res == SKIP_EVENT From 8f8ea2b86b5119cb90d423787c483657ccade87b Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Tue, 21 Jul 2026 15:55:50 +0100 Subject: [PATCH 15/34] clean up gh commit --- dagrunner/tests/execute_graph/test_plugin_executor.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/dagrunner/tests/execute_graph/test_plugin_executor.py b/dagrunner/tests/execute_graph/test_plugin_executor.py index c3e4656..72e6f2b 100644 --- a/dagrunner/tests/execute_graph/test_plugin_executor.py +++ b/dagrunner/tests/execute_graph/test_plugin_executor.py @@ -119,9 +119,6 @@ def test_ignore_event(): assert res == IGNORE_EVENT -def test_ignore_event_single_input(): - """Check what happens when the sole argument is IGNORE_EVENT.""" - res = plugin_executor(*(IGNORE_EVENT,), call=()) @pytest.mark.parametrize( "args, expected", [ From 720f601117cb85d11be7b6562e2f459c91370bcf Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Tue, 21 Jul 2026 15:56:42 +0100 Subject: [PATCH 16/34] typo --- dagrunner/execute_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dagrunner/execute_graph.py b/dagrunner/execute_graph.py index 6de36f2..bfad40f 100755 --- a/dagrunner/execute_graph.py +++ b/dagrunner/execute_graph.py @@ -152,7 +152,7 @@ def plugin_executor( if len(args) > 0 and all((map(lambda x: x is events.IGNORE_EVENT, args))): # all args are IGNORE_EVENT, return IGNORE_EVENT (pass along) if verbose: - print(f"Retuning 'IGNORE_EVENT' event {call[0]}") + print(f"Returning 'IGNORE_EVENT' event {call[0]}") return events.IGNORE_EVENT args = list(filter(lambda x: x is not events.IGNORE_EVENT, args)) From e31bd80d3d4cd2ef862a6d0deb695bfcee6aa35b Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Tue, 21 Jul 2026 16:00:02 +0100 Subject: [PATCH 17/34] remove obsolete test --- dagrunner/tests/execute_graph/test_plugin_executor.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/dagrunner/tests/execute_graph/test_plugin_executor.py b/dagrunner/tests/execute_graph/test_plugin_executor.py index 72e6f2b..42591bd 100644 --- a/dagrunner/tests/execute_graph/test_plugin_executor.py +++ b/dagrunner/tests/execute_graph/test_plugin_executor.py @@ -109,16 +109,6 @@ def test_pass_class_arg_kwargs(plugin, init_args, call_args, target): assert res == target -def test_ignore_event(): - """Check what happens a subset of arguments and all arguments are IGNORE_EVENT.""" - call = tuple([lambda x: x + 5]) - res = plugin_executor(*(5, IGNORE_EVENT), call=call) - assert res == 10 - - res = plugin_executor(*(IGNORE_EVENT, IGNORE_EVENT), call=call) - assert res == IGNORE_EVENT - - @pytest.mark.parametrize( "args, expected", [ From a6b404f64f69179923eac90d718cbdb75f06c4bb Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Tue, 21 Jul 2026 16:12:48 +0100 Subject: [PATCH 18/34] refactor if-else for clarity --- dagrunner/plugin_framework.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dagrunner/plugin_framework.py b/dagrunner/plugin_framework.py index dbd509e..9c671bd 100644 --- a/dagrunner/plugin_framework.py +++ b/dagrunner/plugin_framework.py @@ -150,14 +150,14 @@ def __call__(self, *args, **kwargs): args = stage_to_dir( *args, staging_dir=self._staging_dir, verbose=self._verbose ) + elif not args: + raise ValueError("Empty input arguments") else: missing_files = list(filter(lambda fpath: not glob(fpath), args)) if missing_files: raise FileNotFoundError( f"No such file or directory: {', '.join(missing_files)}" ) - elif len(args) == 0: - raise ValueError("Empty input arguments") except (FileNotFoundError, ValueError) as e: if self._on_missing == "error": From 1a5caf0ccaf9225cdb70c154046ead3b5ce78b60 Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Tue, 21 Jul 2026 16:44:25 +0100 Subject: [PATCH 19/34] Revert none handling by Load --- dagrunner/plugin_framework.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dagrunner/plugin_framework.py b/dagrunner/plugin_framework.py index 9c671bd..0c6cd51 100644 --- a/dagrunner/plugin_framework.py +++ b/dagrunner/plugin_framework.py @@ -133,10 +133,10 @@ def __call__(self, *args, **kwargs): Raises: - FileNotFoundError: If any of the files do not exist and `on_missing` is set to 'error'. - - ValueError: If `on_missing` is set to an invalid value. + - ValueError: If no args provided or `on_missing` is set to an invalid value. """ - args = list(map(process_path, args)) + args = list(map(process_path, args)) if ( any([arg.split(":")[0] for arg in args if ":" in arg]) and self._staging_dir is None @@ -150,16 +150,13 @@ def __call__(self, *args, **kwargs): args = stage_to_dir( *args, staging_dir=self._staging_dir, verbose=self._verbose ) - elif not args: - raise ValueError("Empty input arguments") else: missing_files = list(filter(lambda fpath: not glob(fpath), args)) if missing_files: raise FileNotFoundError( f"No such file or directory: {', '.join(missing_files)}" ) - - except (FileNotFoundError, ValueError) as e: + except FileNotFoundError as e: if self._on_missing == "error": raise e elif self._on_missing == "ignore": From 49e0c962b5d8810131a6653de67a0721caf70079 Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Tue, 21 Jul 2026 17:00:47 +0100 Subject: [PATCH 20/34] raise error if no args in Load --- dagrunner/plugin_framework.py | 4 +++- .../tests/plugin_framework/test_LoadJson.py | 18 ++++++------------ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/dagrunner/plugin_framework.py b/dagrunner/plugin_framework.py index 0c6cd51..28246b9 100644 --- a/dagrunner/plugin_framework.py +++ b/dagrunner/plugin_framework.py @@ -136,7 +136,9 @@ def __call__(self, *args, **kwargs): - ValueError: If no args provided or `on_missing` is set to an invalid value. """ - args = list(map(process_path, args)) + args = list(map(process_path, args)) + if not args: + raise ValueError(f"Invalid input arguments: {args}") if ( any([arg.split(":")[0] for arg in args if ":" in arg]) and self._staging_dir is None diff --git a/dagrunner/tests/plugin_framework/test_LoadJson.py b/dagrunner/tests/plugin_framework/test_LoadJson.py index 8545fff..252b0a0 100644 --- a/dagrunner/tests/plugin_framework/test_LoadJson.py +++ b/dagrunner/tests/plugin_framework/test_LoadJson.py @@ -10,7 +10,7 @@ import pytest -from dagrunner.events import IGNORE_EVENT, SKIP_EVENT +from dagrunner.events import IGNORE_EVENT, SKIP_EVENT, _EventBase from dagrunner.plugin_framework import LoadJson @@ -50,17 +50,11 @@ def test_missing_data_skip(): assert LoadJson(on_missing="skip")("dummy-file") == SKIP_EVENT -def test_empty_args_error(): - with pytest.raises(ValueError, match="Empty input arguments"): - LoadJson(on_missing="error")() - - -def test_empty_args_ignore(): - assert LoadJson(on_missing="ignore")() == IGNORE_EVENT - - -def test_empty_args_skip(): - assert LoadJson(on_missing="skip")() == SKIP_EVENT +@pytest.mark.parametrize("on_missing", [None, "error", "skip", "ignore"]) +def test_empty_args(on_missing: str | None): + plugin = LoadJson(on_missing=on_missing) if on_missing else LoadJson() + with pytest.raises(ValueError, match="Invalid input arguments"): + plugin() def test_missing_data_unknown(): From 9b5d641831e42cc6d0013631a891c118d0f39f8a Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Wed, 22 Jul 2026 16:20:58 +0100 Subject: [PATCH 21/34] edit err message --- dagrunner/plugin_framework.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dagrunner/plugin_framework.py b/dagrunner/plugin_framework.py index 28246b9..702b951 100644 --- a/dagrunner/plugin_framework.py +++ b/dagrunner/plugin_framework.py @@ -162,10 +162,10 @@ def __call__(self, *args, **kwargs): if self._on_missing == "error": raise e elif self._on_missing == "ignore": - warnings.warn(f"Ignoring node due to caught exception {str(e)!r}") + warnings.warn(f"Ignoring node - {str(e)}") return events.IGNORE_EVENT elif self._on_missing == "skip": - warnings.warn(f"Skipping node due to caught exception {str(e)!r}") + warnings.warn(f"Skipping node - {str(e)}") return events.SKIP_EVENT else: raise ValueError( From 8e809360393748bbdc8af86a0e06c7a94ce28de2 Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Thu, 23 Jul 2026 09:44:58 +0100 Subject: [PATCH 22/34] remove node awareness from err message --- dagrunner/plugin_framework.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dagrunner/plugin_framework.py b/dagrunner/plugin_framework.py index 702b951..a0c2338 100644 --- a/dagrunner/plugin_framework.py +++ b/dagrunner/plugin_framework.py @@ -133,12 +133,10 @@ def __call__(self, *args, **kwargs): Raises: - FileNotFoundError: If any of the files do not exist and `on_missing` is set to 'error'. - - ValueError: If no args provided or `on_missing` is set to an invalid value. + - ValueError: If `on_missing` is set to an invalid value. """ args = list(map(process_path, args)) - if not args: - raise ValueError(f"Invalid input arguments: {args}") if ( any([arg.split(":")[0] for arg in args if ":" in arg]) and self._staging_dir is None @@ -162,10 +160,10 @@ def __call__(self, *args, **kwargs): if self._on_missing == "error": raise e elif self._on_missing == "ignore": - warnings.warn(f"Ignoring node - {str(e)}") + warnings.warn(f"Ignoring load - {str(e)}") return events.IGNORE_EVENT elif self._on_missing == "skip": - warnings.warn(f"Skipping node - {str(e)}") + warnings.warn(f"Skipping load - {str(e)}") return events.SKIP_EVENT else: raise ValueError( From 5ba6eb4fdce640282905b55b0ee40b19d7b5ed84 Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Thu, 23 Jul 2026 09:50:33 +0100 Subject: [PATCH 23/34] revert doc changes --- dagrunner/events.py | 39 +++++++++++++++++----------------- docs/dagrunner.events.md | 45 ++++++++++++++++++++++------------------ 2 files changed, 45 insertions(+), 39 deletions(-) diff --git a/dagrunner/events.py b/dagrunner/events.py index bc770f9..36d225a 100644 --- a/dagrunner/events.py +++ b/dagrunner/events.py @@ -7,27 +7,28 @@ DAGrunner defines two special singleton events that plugins can return to control the execution flow of their graph. -- IGNORE_EVENT - removes a particular input from further processing. -- SKIP_EVENT - aborts the execution of a plugin (and all downstream nodes) when any +- event.IGNORE - removes a particular input from further processing. +- event.SKIP - aborts the execution of a plugin (and all downstream nodes) when any input carries this event. -### _IgnoreEvent: `IGNORE_EVENT` -When a plugin returns `IGNORE_EVENT`, the immediate descendant node filters out that -input. The remaining inputs of that plugin are utilised by that node as normal. -If all inputs to a node are `IGNORE_EVENT`, the node's execution is skipped, and -`IGNORE_EVENT` is returned instead. +### event.IGNORE +When a plugin returns `event.IGNORE`, the immediate descendant node filters out that +input. +The remaining inputs of that plugin are utilised by that node as normal. +If all inputs to a node are `event.IGNORE`, the node's execution is ignored, and a +`event.IGNORE` event is returned instead, ignoring execution through all descendants. ```mermaid --- -title: IGNORE_EVENT (filtering some) +title: event.IGNORE (filtering some) --- graph cycle1{cycleX} cycle1 --> Input1 --> filepath --> Load1 --> cube1 --> Proc - cycle1 --> Input2 --> filepath --> Load2 --> IGNORE_EVENT --> Proc + cycle1 --> Input2 --> filepath --> Load2 --> event.IGNORE --> Proc cycle1 --> Input3 --> filepath --> Load3 --> cube3 --> Proc - cycle1 --> Input4 --> filepath --> Load4 --> IGNORE_EVENT --> Proc + cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc Proc --> cube --> Save Proc["Proc (cube1, cube3)"] ``` @@ -35,24 +36,24 @@ Only the non-ignored cubes (`cube1` and `cube3`) reach `Proc`; the ignored inputs are dropped. -### _SkipEvent: `SKIP_EVENT` -The skip event differs from the ignore event in that if **any** input to a plugin is -`SKIP_EVENT`, node execution is skipped and `SKIP_EVENT` is returned instead. This skip -propagates so that all dependent nodes and their descendants are not executed. +### `event.IGNORE` +The SKIP event differs from the IGNORE event in that if **any** input to a plugin is a +SKIP event, node execution is skipped and it instead propagates this skip event so that +all dependent nodes and their descendants aren't executed. ```mermaid --- -title: SKIP_EVENT +title: event.SKIP --- graph cycle1{cycleX} cycle1 --> Input1 --> filepath --> Load1 --> cube --> Proc - cycle1 --> Input2 --> filepath --> Load2 --> SKIP_EVENT --> Proc + cycle1 --> Input2 --> filepath --> Load2 --> event.SKIP --> Proc cycle1 --> Input3 --> filepath --> Load3 --> cube --> Proc - cycle1 --> Input4 --> filepath --> Load4 --> IGNORE_EVENT --> Proc - Proc --> SKIP_EVENT --> Save + cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc + Proc --> event.SKIP --> Save ``` -Because `Input2` returns `SKIP_EVENT`, the Proc node and everything that follows aren't +Because `Input2` returns a SKIP event, the Proc node and everything that follows aren't executed and neither is the Save node since the skip is propagated along the execution graph. """ diff --git a/docs/dagrunner.events.md b/docs/dagrunner.events.md index dc3b7c1..0c7ff5c 100644 --- a/docs/dagrunner.events.md +++ b/docs/dagrunner.events.md @@ -6,27 +6,28 @@ DAGrunner defines two special singleton events that plugins can return to control the execution flow of their graph. -- IGNORE_EVENT - removes a particular input from further processing. -- SKIP_EVENT - aborts the execution of a plugin (and all downstream nodes) when any +- event.IGNORE - removes a particular input from further processing. +- event.SKIP - aborts the execution of a plugin (and all downstream nodes) when any input carries this event. -### _IgnoreEvent: `IGNORE_EVENT` -When a plugin returns `IGNORE_EVENT`, the immediate descendant node filters out that -input. The remaining inputs of that plugin are utilised by that node as normal. -If all inputs to a node are `IGNORE_EVENT`, the node's execution is skipped, and -`IGNORE_EVENT` is returned instead. +### event.IGNORE +When a plugin returns `event.IGNORE`, the immediate descendant node filters out that +input. +The remaining inputs of that plugin are utilised by that node as normal. +If all inputs to a node are `event.IGNORE`, the node's execution is ignored, and a +`event.IGNORE` event is returned instead, ignoring execution through all descendants. ```mermaid --- -title: IGNORE_EVENT (filtering some) +title: event.IGNORE (filtering some) --- graph cycle1{cycleX} cycle1 --> Input1 --> filepath --> Load1 --> cube1 --> Proc - cycle1 --> Input2 --> filepath --> Load2 --> IGNORE_EVENT --> Proc + cycle1 --> Input2 --> filepath --> Load2 --> event.IGNORE --> Proc cycle1 --> Input3 --> filepath --> Load3 --> cube3 --> Proc - cycle1 --> Input4 --> filepath --> Load4 --> IGNORE_EVENT --> Proc + cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc Proc --> cube --> Save Proc["Proc (cube1, cube3)"] ``` @@ -34,25 +35,29 @@ Here, Input2 and Input4 return an IGNORE event, likely due to there being missin Only the non-ignored cubes (`cube1` and `cube3`) reach `Proc`; the ignored inputs are dropped. -### _SkipEvent: `SKIP_EVENT` -The skip event differs from the ignore event in that if **any** input to a plugin is -`SKIP_EVENT`, node execution is skipped and `SKIP_EVENT` is returned instead. This skip -propagates so that all dependent nodes and their descendants are not executed. +### `event.IGNORE` +The SKIP event differs from the IGNORE event in that if **any** input to a plugin is a +SKIP event, node execution is skipped and it instead propagates this skip event so that +all dependent nodes and their descendants aren't executed. ```mermaid --- -title: SKIP_EVENT +title: event.SKIP --- graph cycle1{cycleX} cycle1 --> Input1 --> filepath --> Load1 --> cube --> Proc - cycle1 --> Input2 --> filepath --> Load2 --> SKIP_EVENT --> Proc + cycle1 --> Input2 --> filepath --> Load2 --> event.SKIP --> Proc cycle1 --> Input3 --> filepath --> Load3 --> cube --> Proc - cycle1 --> Input4 --> filepath --> Load4 --> IGNORE_EVENT --> Proc - Proc --> SKIP_EVENT --> Save + cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc + Proc --> event.SKIP --> Save ``` -Because `Input2` returns `SKIP_EVENT`, the Proc node and everything that follows aren't +Because `Input2` returns a SKIP event, the Proc node and everything that follows aren't executed and neither is the Save node since the skip is propagated along the execution graph. -see [class: dagrunner.utils.Singleton](dagrunner.utils.md#class-singleton) \ No newline at end of file +see [class: dagrunner.utils.Singleton](dagrunner.utils.md#class-singleton) + +## _IgnoreEvent: `IGNORE_EVENT` + +## _SkipEvent: `SKIP_EVENT` \ No newline at end of file From 8ba83516045218add19ab22f20eedfcd3d6f7d66 Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Thu, 23 Jul 2026 09:51:34 +0100 Subject: [PATCH 24/34] remove empty test --- dagrunner/tests/plugin_framework/test_LoadJson.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/dagrunner/tests/plugin_framework/test_LoadJson.py b/dagrunner/tests/plugin_framework/test_LoadJson.py index 252b0a0..9b51d81 100644 --- a/dagrunner/tests/plugin_framework/test_LoadJson.py +++ b/dagrunner/tests/plugin_framework/test_LoadJson.py @@ -50,13 +50,6 @@ def test_missing_data_skip(): assert LoadJson(on_missing="skip")("dummy-file") == SKIP_EVENT -@pytest.mark.parametrize("on_missing", [None, "error", "skip", "ignore"]) -def test_empty_args(on_missing: str | None): - plugin = LoadJson(on_missing=on_missing) if on_missing else LoadJson() - with pytest.raises(ValueError, match="Invalid input arguments"): - plugin() - - def test_missing_data_unknown(): with pytest.raises( ValueError, From c42236b196a5514f3d3ed112348eb8652f3cf606 Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Thu, 23 Jul 2026 09:52:02 +0100 Subject: [PATCH 25/34] remove typehint --- dagrunner/tests/plugin_framework/test_LoadJson.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dagrunner/tests/plugin_framework/test_LoadJson.py b/dagrunner/tests/plugin_framework/test_LoadJson.py index 9b51d81..d51d509 100644 --- a/dagrunner/tests/plugin_framework/test_LoadJson.py +++ b/dagrunner/tests/plugin_framework/test_LoadJson.py @@ -10,7 +10,7 @@ import pytest -from dagrunner.events import IGNORE_EVENT, SKIP_EVENT, _EventBase +from dagrunner.events import IGNORE_EVENT, SKIP_EVENT from dagrunner.plugin_framework import LoadJson From 77fcb230e9305d160d6daa1960e00e19a6f1d55c Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Thu, 23 Jul 2026 11:09:24 +0100 Subject: [PATCH 26/34] remove warning from Load --- dagrunner/plugin_framework.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dagrunner/plugin_framework.py b/dagrunner/plugin_framework.py index a0c2338..0f4e6b1 100644 --- a/dagrunner/plugin_framework.py +++ b/dagrunner/plugin_framework.py @@ -160,10 +160,8 @@ def __call__(self, *args, **kwargs): if self._on_missing == "error": raise e elif self._on_missing == "ignore": - warnings.warn(f"Ignoring load - {str(e)}") return events.IGNORE_EVENT elif self._on_missing == "skip": - warnings.warn(f"Skipping load - {str(e)}") return events.SKIP_EVENT else: raise ValueError( From 7c8dc6d3b50d42a8cd75d63b48552d1d9601af0e Mon Sep 17 00:00:00 2001 From: ryan-cocking-mo Date: Thu, 23 Jul 2026 11:14:05 +0100 Subject: [PATCH 27/34] precommit --- dagrunner/plugin_framework.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dagrunner/plugin_framework.py b/dagrunner/plugin_framework.py index 0f4e6b1..9859342 100644 --- a/dagrunner/plugin_framework.py +++ b/dagrunner/plugin_framework.py @@ -8,7 +8,6 @@ import pickle import string import subprocess -import warnings from abc import ABC, abstractmethod from glob import glob From f749145af713579b4dc224445ee7e37b5b168987 Mon Sep 17 00:00:00 2001 From: cpelley Date: Tue, 28 Jul 2026 12:10:55 +0100 Subject: [PATCH 28/34] zizmor usage --- .github/dependabot.yml | 15 +++++++++++ .github/workflows/tests.yml | 51 +++++++++++++++++++++++++------------ .pre-commit-config.yaml | 5 ++++ 3 files changed, 55 insertions(+), 16 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..db4e17b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +# Reference: +# - https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/keeping-your-actions-up-to-date-with-dependabot +# - https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/configuration-options-for-dependency-updates + +version: 2 +updates: + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + labels: + - "bot" + cooldown: + default-days: 7 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 06fd713..ee2b0cb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,6 +2,9 @@ name: Pull Request Tests on: [pull_request] +permissions: + contents: read + jobs: pre-commit: @@ -12,10 +15,12 @@ jobs: # SETUP ############################ - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 with: python-version: 3.x cache: 'pip' @@ -35,7 +40,7 @@ jobs: run: echo "PYSHA=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV - name: Cache pre-commit - uses: actions/cache@v3 + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 id: pre-commit-cache with: path: ~/.cache/pre-commit @@ -58,10 +63,12 @@ jobs: # SETUP ############################ - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 with: python-version: 3.9 cache: 'pip' @@ -96,10 +103,12 @@ jobs: # SETUP ############################ - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 with: python-version: 3.9 cache: 'pip' @@ -145,10 +154,12 @@ jobs: # SETUP ############################ - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 with: python-version: 3.x cache: 'pip' @@ -178,15 +189,16 @@ jobs: ############################ - name: Cache ref branch coverage report id: cache-ref-coverage - uses: actions/cache@v4 + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 with: path: ref/coverage_output.txt key: ref-${{ github.event.pull_request.base.sha }} - name: Checkout ref branch if: steps.cache-ref-coverage.outputs.cache-hit != 'true' - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: + persist-credentials: false path: ref ref: ${{ github.base_ref }} @@ -203,7 +215,7 @@ jobs: run: | set -e # ----- USER SETTINGS ------------------------------------------------- - CHECK_MISS_TOL=1 # 1 = evaluate the miss‑count difference, 0 = ignore it + CHECK_MISS_TOL=1 # 1 = evaluate the miss-count difference, 0 = ignore it CHECK_PERC_TOL=0 # 1 = evaluate the percentage difference, 0 = ignore it # Tolerances @@ -252,7 +264,7 @@ jobs: - name: Upload coverage report if: steps.comp-coverage.outputs.coverage_decreased == 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: coverage-report-pr path: | @@ -265,10 +277,17 @@ jobs: echo "Code coverage has decreased" echo "-----------------------------------------------" echo -e "Coverage totals\tstatements\tmissed\n\ - PR\t${{ steps.comp-coverage.outputs.pr_tot_stmts }}\t${{ steps.comp-coverage.outputs.pr_tot_miss }}\t${{ steps.comp-coverage.outputs.pr_perc_total }}%\n\ - Reference\t${{ steps.comp-coverage.outputs.ref_tot_stmts }}\t${{ steps.comp-coverage.outputs.ref_tot_miss }}\t${{ steps.comp-coverage.outputs.ref_perc_total }}%" \ + PR\t${PR_TOT_STMTS}\t${PR_TOT_MISS}\t${PR_PERC_TOTAL}%\n\ + Reference\t${REF_TOT_STMTS}\t${REF_TOT_MISS}\t${REF_PERC_TOTAL}%" \ | column -t -s $'\t' echo "-----------------------------------------------" echo "Report uploaded as artifact 'coverage-report-pr'" echo "Test coverage failure is non-blocking, but please consider improving test coverage before merging." - exit 1 \ No newline at end of file + exit 1 + env: + PR_TOT_STMTS: ${{ steps.comp-coverage.outputs.pr_tot_stmts }} + PR_TOT_MISS: ${{ steps.comp-coverage.outputs.pr_tot_miss }} + PR_PERC_TOTAL: ${{ steps.comp-coverage.outputs.pr_perc_total }} + REF_TOT_STMTS: ${{ steps.comp-coverage.outputs.ref_tot_stmts }} + REF_TOT_MISS: ${{ steps.comp-coverage.outputs.ref_tot_miss }} + REF_PERC_TOTAL: ${{ steps.comp-coverage.outputs.ref_perc_total }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3e4cb35..a3d99ba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,3 +15,8 @@ repos: files: '\.py$' language: system entry: ./copyright_check +- repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.28.0 + hooks: + - id: zizmor + args: [--fix] From 158df68e5f79180fc1548b3c41cbc0a4fbcddcc3 Mon Sep 17 00:00:00 2001 From: Carwyn Pelley Date: Fri, 24 Jul 2026 12:02:54 +0100 Subject: [PATCH 29/34] DOC: Update illustration of mermaid events handling Data flow was being represented as a node, giving rise to confusion. Also, ensured further clarity that Proc and Save isn't actually doing anything in the skip case. --- dagrunner/events.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/dagrunner/events.py b/dagrunner/events.py index 36d225a..5a9221b 100644 --- a/dagrunner/events.py +++ b/dagrunner/events.py @@ -25,11 +25,11 @@ --- graph cycle1{cycleX} - cycle1 --> Input1 --> filepath --> Load1 --> cube1 --> Proc - cycle1 --> Input2 --> filepath --> Load2 --> event.IGNORE --> Proc - cycle1 --> Input3 --> filepath --> Load3 --> cube3 --> Proc - cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc - Proc --> cube --> Save + cycle1 --> Input1 -- filepath --> Load1 --cube1 --> Proc + cycle1 --> Input2 -- filepath --> Load2 -- event.IGNORE --> Proc + cycle1 --> Input3 -- filepath --> Load3 -- cube3 --> Proc + cycle1 --> Input4 -- filepath --> Load4 -- event.IGNORE --> Proc + Proc -- cube --> Save Proc["Proc (cube1, cube3)"] ``` Here, Input2 and Input4 return an IGNORE event, likely due to there being missing data. @@ -47,11 +47,11 @@ --- graph cycle1{cycleX} - cycle1 --> Input1 --> filepath --> Load1 --> cube --> Proc - cycle1 --> Input2 --> filepath --> Load2 --> event.SKIP --> Proc - cycle1 --> Input3 --> filepath --> Load3 --> cube --> Proc - cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc - Proc --> event.SKIP --> Save + cycle1 --> Input1 -- filepath --> Load1 -- cube --> Proc + cycle1 --> Input2 -- filepath --> Load2 -- event.SKIP --> Proc + cycle1 --> Input3 -- filepath --> Load3 -- cube --> Proc + cycle1 --> Input4 -- filepath --> Load4 -- event.IGNORE --> Proc + Proc["Proc
(not called)"] -- event.SKIP --> Save["Save
(not called)"] ``` Because `Input2` returns a SKIP event, the Proc node and everything that follows aren't executed and neither is the Save node since the skip is propagated along the execution From 09a47e84f216eeed71b409cc2de2e80086cfa01f Mon Sep 17 00:00:00 2001 From: Carwyn Pelley Date: Fri, 24 Jul 2026 12:08:51 +0100 Subject: [PATCH 30/34] Update events.py --- dagrunner/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dagrunner/events.py b/dagrunner/events.py index 5a9221b..5e0b634 100644 --- a/dagrunner/events.py +++ b/dagrunner/events.py @@ -56,7 +56,7 @@ Because `Input2` returns a SKIP event, the Proc node and everything that follows aren't executed and neither is the Save node since the skip is propagated along the execution graph. -""" +""" # noqa: E501 from dagrunner.utils import Singleton From 1b32fa136d0c47b9ae2eba76aa24e2bed8735d86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:53:19 +0000 Subject: [PATCH 31/34] Bump actions/cache from 5.1.0 to 6.1.0 Bumps [actions/cache](https://github.com/actions/cache) from 5.1.0 to 6.1.0. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/caa296126883cff596d87d8935842f9db880ef25...55cc8345863c7cc4c66a329aec7e433d2d1c52a9) --- updated-dependencies: - dependency-name: actions/cache dependency-version: 6.1.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ee2b0cb..2cdf04b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,7 +40,7 @@ jobs: run: echo "PYSHA=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV - name: Cache pre-commit - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 id: pre-commit-cache with: path: ~/.cache/pre-commit @@ -189,7 +189,7 @@ jobs: ############################ - name: Cache ref branch coverage report id: cache-ref-coverage - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: path: ref/coverage_output.txt key: ref-${{ github.event.pull_request.base.sha }} From b8d67aeccc2028b55fba37930e894b24f33a1a18 Mon Sep 17 00:00:00 2001 From: cpelley Date: Thu, 30 Jul 2026 16:03:19 +0100 Subject: [PATCH 32/34] DOC: Building docutmentation via CI --- .github/workflows/docs.yml | 103 ++++++++++++++++++ README.md | 6 +- docs/{ => _build}/dagrunner.config.md | 16 +-- docs/{ => _build}/dagrunner.events.md | 25 +++-- docs/{ => _build}/dagrunner.execute_graph.md | 16 +-- docs/{ => _build}/dagrunner.md | 2 +- .../dagrunner.plugin_framework.md | 59 +++++----- docs/{ => _build}/dagrunner.runner.md | 2 +- .../dagrunner.runner.schedulers.asyncmp.md | 12 +- .../dagrunner.runner.schedulers.base.md | 4 + .../dagrunner.runner.schedulers.dask.md | 36 +++--- .../dagrunner.runner.schedulers.md | 2 +- docs/{ => _build}/dagrunner.utils.logger.md | 36 +++--- docs/{ => _build}/dagrunner.utils.md | 86 ++++++++------- docs/{ => _build}/dagrunner.utils.networkx.md | 8 +- .../dagrunner.utils.visualisation.md | 38 +++---- docs/{ => _build}/dagrunner_index.md | 1 + docs/dagrunner.runner.schedulers.base.md | 4 - 18 files changed, 288 insertions(+), 168 deletions(-) create mode 100644 .github/workflows/docs.yml rename docs/{ => _build}/dagrunner.config.md (88%) rename docs/{ => _build}/dagrunner.events.md (69%) rename docs/{ => _build}/dagrunner.execute_graph.md (93%) rename docs/{ => _build}/dagrunner.md (94%) rename docs/{ => _build}/dagrunner.plugin_framework.md (87%) rename docs/{ => _build}/dagrunner.runner.md (71%) rename docs/{ => _build}/dagrunner.runner.schedulers.asyncmp.md (84%) create mode 100644 docs/_build/dagrunner.runner.schedulers.base.md rename docs/{ => _build}/dagrunner.runner.schedulers.dask.md (80%) rename docs/{ => _build}/dagrunner.runner.schedulers.md (89%) rename docs/{ => _build}/dagrunner.utils.logger.md (90%) rename docs/{ => _build}/dagrunner.utils.md (83%) rename docs/{ => _build}/dagrunner.utils.networkx.md (95%) rename docs/{ => _build}/dagrunner.utils.visualisation.md (79%) rename docs/{ => _build}/dagrunner_index.md (98%) delete mode 100644 docs/dagrunner.runner.schedulers.base.md diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..099b958 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,103 @@ +name: Documentation + +on: + push: + branches: + - main + - release/* + - feature/* + +permissions: + contents: write + pull-requests: write + +jobs: + build: + if: github.repository_owner == 'MetOffice' && !contains(github.event.head_commit.message, '[skip ci]') + runs-on: ubuntu-latest + + steps: + # SETUP + ############################ + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ github.ref_name }} # Ensure branch is checked out, not detached state (so we can push a commit later) + token: ${{ secrets.GITHUB_TOKEN }} + persist-credentials: true # Ensure that the token is available for pushing changes + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: 3.x + + - name: Add checkout directory to PYTHONPATH + run: echo "PYTHONPATH=$(pwd):$PYTHONPATH" >> $GITHUB_ENV + + - name: Install dependencies + id: install-dependencies + run: | + pip install .[tests,dev] + pip uninstall dagrunner -y + + # DOCUMENTATION + ############################ + - name: Build documentation + run: | + rm -rf ./docs/_build + mkdir -p ./docs/_build + ./docs/gen_docs dagrunner ./docs/_build + git add -f docs/_build/ + + - name: Check if documentation has changed + id: check-docs + run: | + echo "changed=$(git diff --cached --quiet --exit-code || echo true)" | tee -a $GITHUB_OUTPUT + + - name: Create PR + if: steps.check-docs.outputs.changed == 'true' + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + + source_branch="${GITHUB_REF_NAME}" + DOCS_BRANCH="docs-update/${source_branch//\//-}" + sha="${{ github.sha }}" + short_sha="${sha::8}" + + git switch -c "$DOCS_BRANCH" + git commit -m "Automated reference documentation update for $short_sha [skip ci]" + git push origin "$DOCS_BRANCH" --force + + # Create a PR only if one is not already open for this branch + if ! gh pr list --head "$DOCS_BRANCH" --json number --jq '.[0].number' | grep -q .; then + gh pr create \ + --title "Automated reference documentation update for $source_branch" \ + --body "This PR was automatically generated by the documentation workflow for commit $short_sha [skip ci]" \ + --base "$source_branch" \ + --head "$DOCS_BRANCH" \ + --label "documentation" \ + --assignee ${GITHUB_ACTOR} + else + # comment on PR to say that we have pushed a new commit to the branch + pr_number=$(gh pr list --head "$DOCS_BRANCH" --json number --jq '.[0].number') + gh pr comment "$pr_number" --body "New documentation changes have been pushed to this branch for commit $short_sha [skip ci]" + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + + # # https://github.com/orgs/community/discussions/26560#discussioncomment-3531273 + # # This must be our very final step to ensure that it runs only on condition of + # # success of all previous steps. A pushed commit will not trigger the re-running + # # of this workflow. + # - name: Commit and push documentation changes + # if: steps.check-docs.outputs.changed == 'true' + # run: | + # git config user.name github-actions[bot] + # git config user.email 41898282+github-actions[bot]@users.noreply.github.com + # git add docs/. + # git commit -am "Automated reference documentation update for PR ${{ github.event.number }} [skip ci]" + # git push + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 5d7e83c..3676c40 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ DAGrunner takes advantage of the native markdown rendering support provided by g ## API -See [DAGrunner API](docs/dagrunner_index.md) +See [DAGrunner API](docs/_build/dagrunner_index.md) ## License/copyright @@ -52,7 +52,7 @@ See [docs/demo.ipynb](docs/demo.ipynb) DAGrunner concerns itself with graph execution and does not strictly require processing modules (plugins) to take any particular form. That is, you may or may not choose to use or subclass the plugins provided by DAGrunner. However, for convenience, DAGrunner does define some plugins which fall into two broad categories, some abstract and some for use as they are. -See [here](docs/dagrunner.plugin_framework.md) for more information. +See [here](docs/_build/dagrunner.plugin_framework.md) for more information. ## Schedulers @@ -63,7 +63,7 @@ These range from [dask](https://www.dask.org/), [ray](https://docs.ray.io/en/lat ## Logging and monitoring DAGrunner provides a script `dagrunner-logger` for running a TCP server. This enables logging to function across the network. Additionally, it will write logs to an sqlite database to aid in realtime monitoring from external tools. -See [logger](docs/dagrunner.utils.logger.md) for more information. +See [logger](docs/_build/dagrunner.utils.logger.md) for more information. ## Logo diff --git a/docs/dagrunner.config.md b/docs/_build/dagrunner.config.md similarity index 88% rename from docs/dagrunner.config.md rename to docs/_build/dagrunner.config.md index 266c896..db0bf6c 100644 --- a/docs/dagrunner.config.md +++ b/docs/_build/dagrunner.config.md @@ -1,6 +1,6 @@ # module: `dagrunner.config` -[Source](../dagrunner/config.py#L0) +[Source](../../dagrunner/config.py#L0) This module handles the run-time configuration of the dagrunner library. @@ -19,7 +19,7 @@ see [class: dagrunner.utils.Singleton](dagrunner.utils.md#class-singleton) ## class: `GlobalConfiguration` -[Source](../dagrunner/config.py#L28) +[Source](../../dagrunner/config.py#L28) ### Call Signature: @@ -63,7 +63,7 @@ options:: ### function: `__getitem__` -[Source](../dagrunner/config.py#L172) +[Source](../../dagrunner/config.py#L172) #### Call Signature: @@ -73,7 +73,7 @@ __getitem__(self, key) ### function: `__init__` -[Source](../dagrunner/config.py#L83) +[Source](../../dagrunner/config.py#L83) #### Call Signature: @@ -85,7 +85,7 @@ Initialize self. See help(type(self)) for accurate signature. ### function: `__repr__` -[Source](../dagrunner/config.py#L90) +[Source](../../dagrunner/config.py#L90) #### Call Signature: @@ -97,7 +97,7 @@ Return repr(self). ### function: `__setitem__` -[Source](../dagrunner/config.py#L175) +[Source](../../dagrunner/config.py#L175) #### Call Signature: @@ -107,7 +107,7 @@ __setitem__(self, key, value) ### function: `__str__` -[Source](../dagrunner/config.py#L87) +[Source](../../dagrunner/config.py#L87) #### Call Signature: @@ -119,7 +119,7 @@ Return str(self). ### function: `parse_configuration` -[Source](../dagrunner/config.py#L146) +[Source](../../dagrunner/config.py#L146) #### Call Signature: diff --git a/docs/dagrunner.events.md b/docs/_build/dagrunner.events.md similarity index 69% rename from docs/dagrunner.events.md rename to docs/_build/dagrunner.events.md index 0c7ff5c..750ec9c 100644 --- a/docs/dagrunner.events.md +++ b/docs/_build/dagrunner.events.md @@ -1,6 +1,6 @@ # module: `dagrunner.events` -[Source](../dagrunner/events.py#L0) +[Source](../../dagrunner/events.py#L0) ## Overview DAGrunner defines two special singleton events that plugins can return to control the @@ -24,11 +24,11 @@ title: event.IGNORE (filtering some) --- graph cycle1{cycleX} - cycle1 --> Input1 --> filepath --> Load1 --> cube1 --> Proc - cycle1 --> Input2 --> filepath --> Load2 --> event.IGNORE --> Proc - cycle1 --> Input3 --> filepath --> Load3 --> cube3 --> Proc - cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc - Proc --> cube --> Save + cycle1 --> Input1 -- filepath --> Load1 --cube1 --> Proc + cycle1 --> Input2 -- filepath --> Load2 -- event.IGNORE --> Proc + cycle1 --> Input3 -- filepath --> Load3 -- cube3 --> Proc + cycle1 --> Input4 -- filepath --> Load4 -- event.IGNORE --> Proc + Proc -- cube --> Save Proc["Proc (cube1, cube3)"] ``` Here, Input2 and Input4 return an IGNORE event, likely due to there being missing data. @@ -46,11 +46,11 @@ title: event.SKIP --- graph cycle1{cycleX} - cycle1 --> Input1 --> filepath --> Load1 --> cube --> Proc - cycle1 --> Input2 --> filepath --> Load2 --> event.SKIP --> Proc - cycle1 --> Input3 --> filepath --> Load3 --> cube --> Proc - cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc - Proc --> event.SKIP --> Save + cycle1 --> Input1 -- filepath --> Load1 -- cube --> Proc + cycle1 --> Input2 -- filepath --> Load2 -- event.SKIP --> Proc + cycle1 --> Input3 -- filepath --> Load3 -- cube --> Proc + cycle1 --> Input4 -- filepath --> Load4 -- event.IGNORE --> Proc + Proc["Proc
(not called)"] -- event.SKIP --> Save["Save
(not called)"] ``` Because `Input2` returns a SKIP event, the Proc node and everything that follows aren't executed and neither is the Save node since the skip is propagated along the execution @@ -60,4 +60,5 @@ see [class: dagrunner.utils.Singleton](dagrunner.utils.md#class-singleton) ## _IgnoreEvent: `IGNORE_EVENT` -## _SkipEvent: `SKIP_EVENT` \ No newline at end of file +## _SkipEvent: `SKIP_EVENT` + diff --git a/docs/dagrunner.execute_graph.md b/docs/_build/dagrunner.execute_graph.md similarity index 93% rename from docs/dagrunner.execute_graph.md rename to docs/_build/dagrunner.execute_graph.md index 1a03168..544089d 100644 --- a/docs/dagrunner.execute_graph.md +++ b/docs/_build/dagrunner.execute_graph.md @@ -1,6 +1,6 @@ # module: `dagrunner.execute_graph` -[Source](../dagrunner/execute_graph.py#L0) +[Source](../../dagrunner/execute_graph.py#L0) see [GlobalConfiguration: dagrunner.config.CONFIG](dagrunner.config.md#globalconfiguration-config) @@ -16,13 +16,15 @@ see [module: dagrunner.events](dagrunner.events.md#module-dagrunnerevents) see [function: dagrunner.utils.function_to_argparse_parse_args](dagrunner.utils.md#function-function_to_argparse_parse_args) +see [function: dagrunner.utils.get_object_dot_module_path](dagrunner.utils.md#function-get_object_dot_module_path) + see [module: dagrunner.utils.logger](dagrunner.utils.logger.md#module-dagrunnerutilslogger) see [function: dagrunner.utils.networkx.visualise_graph](dagrunner.utils.networkx.md#function-visualise_graph) ## class: `ExecuteGraph` -[Source](../dagrunner/execute_graph.py#L274) +[Source](../../dagrunner/execute_graph.py#L318) ### Call Signature: @@ -32,7 +34,7 @@ ExecuteGraph(networkx_graph: str, networkx_graph_kwargs: dict = None, Date: Mon, 3 Aug 2026 14:55:58 +0100 Subject: [PATCH 33/34] DOC: improved description of DAGrunner purpose --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3676c40..2d01343 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ # dagrunner_icon![](https://placehold.co/600x200/transparent/929292?text=DAGrunner) -DAGrunner serves as a Directed Acyclic Graph (DAG) runner, primarily designed to ensure a clear distinction between a project's graph definition (typically in native networkx format) and its execution method. In essence, DAGrunner offers various schedulers for executing the graph, but it firmly separates these operational concepts from the scientific configuration or recipe, i.e., the graph itself. Consequently, while DAGrunner currently provides convenient scheduling options, it remains adaptable to future changes or alternative solutions, ensuring that the scientific configuration can persist regardless of the technologies or tools employed, whether DAGrunner is utilized or not. +DAGrunner is an abstraction layer for executing Directed Acyclic Graphs (DAGs), providing portability and standardization across different schedulers. Rather than building your graph execution logic around a specific scheduler (such as Dask or Ray), DAGrunner enables you to define your graph once in networkx format and execute it with any supported scheduler without modifying your underlying graph definition. It decouples the scientific configuration and graph recipe from execution concerns, ensuring that your graph remains independent of scheduling technology choices. Additionally, DAGrunner provides a unified tooling layer that includes scheduler-agnostic graph visualization, scheduler-independent logging and monitoring, and other operational utilities—allowing you to maintain consistent workflows regardless of which backend scheduler you choose. ## Documentation From 930845a2e6c6f0032b7889066b220ef5f6c36fed Mon Sep 17 00:00:00 2001 From: cpelley Date: Mon, 3 Aug 2026 15:52:41 +0100 Subject: [PATCH 34/34] DOC: improved description of how the documentation is put together --- docs/README.md | 3 +++ docs/gen_docs | 51 ++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 docs/README.md diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..c4701e7 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,3 @@ +Documentation is built under `_build` automatically via github continuous integration (CI) using the [gen_docs](gen_docs) script (see [.github/workflows/docs.yml](../.github/workflows/docs.yml)). As `main`, `release/*` or `feature/*` branches are updated, pull requests are automatically created targetting these branches with updated documentation. + +Whilst running this script locally yourself might be useful for building the documentation, please defer from including such changes to the built documentation to the repository. This is to be handled by CI. \ No newline at end of file diff --git a/docs/gen_docs b/docs/gen_docs index 17f8530..5b09454 100755 --- a/docs/gen_docs +++ b/docs/gen_docs @@ -1,16 +1,45 @@ #!/usr/bin/env python3 """ -Generate reference documentation for a Python module. +Generate markdown reference documentation for a Python package Example usage: - python generate_docs.py my_module /path/to/docs + gen_docs python.module.path /path/to/output + +Each module will be converted into a markdown file, and an index file will be +generated for the package. +`.` will be written to `..md`, +and the index file `_index.md` will be extended to include a link to +this module and its contents. + +Note that markdown syntax is supported in docstrings, taken without modification. +This means that anything supported by your markdown renderer (e.g. vscode and GitHub) +will be supported in docstrings, including links to other documentation, images, +tables and other markdown features. + +Example usage: + +``` +def function(args1, keywarg1=default1): + \""" + Dummy function to demonstrate markdown docstrings. + + Args: + - `args1` (str): + Some positional argument. + - `keywarg1` (int, optional): + Some keyword argument with a default value. + + \""" + ... +``` + """ +import argparse import importlib import importlib.util import inspect import os -import sys def gen_source_link(obj, target_root): @@ -281,6 +310,16 @@ def generate_markdown_recursive(module_dot_path, target_root): if __name__ == "__main__": - module_path = sys.argv[1] - target_root = os.path.abspath(sys.argv[2]) - generate_markdown_recursive(module_path, target_root) + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "pymodule_path", help="Python module path, which must be on the PYTHONPATH" + ) + parser.add_argument( + "target_root", help="Target root directory for generated markdown" + ) + args = parser.parse_args() + pymodule_path = args.pymodule_path + target_root = os.path.abspath(args.target_root) + generate_markdown_recursive(pymodule_path, target_root)