diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000..db4e17b
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,15 @@
+# Reference:
+# - https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/keeping-your-actions-up-to-date-with-dependabot
+# - https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+
+ - package-ecosystem: "github-actions"
+ directory: "/"
+ schedule:
+ interval: "weekly"
+ labels:
+ - "bot"
+ cooldown:
+ default-days: 7
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
new file mode 100644
index 0000000..099b958
--- /dev/null
+++ b/.github/workflows/docs.yml
@@ -0,0 +1,103 @@
+name: Documentation
+
+on:
+ push:
+ branches:
+ - main
+ - release/*
+ - feature/*
+
+permissions:
+ contents: write
+ pull-requests: write
+
+jobs:
+ build:
+ if: github.repository_owner == 'MetOffice' && !contains(github.event.head_commit.message, '[skip ci]')
+ runs-on: ubuntu-latest
+
+ steps:
+ # SETUP
+ ############################
+ - name: Checkout code
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+ with:
+ ref: ${{ github.ref_name }} # Ensure branch is checked out, not detached state (so we can push a commit later)
+ token: ${{ secrets.GITHUB_TOKEN }}
+ persist-credentials: true # Ensure that the token is available for pushing changes
+
+ - name: Set up Python
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
+ with:
+ python-version: 3.x
+
+ - name: Add checkout directory to PYTHONPATH
+ run: echo "PYTHONPATH=$(pwd):$PYTHONPATH" >> $GITHUB_ENV
+
+ - name: Install dependencies
+ id: install-dependencies
+ run: |
+ pip install .[tests,dev]
+ pip uninstall dagrunner -y
+
+ # DOCUMENTATION
+ ############################
+ - name: Build documentation
+ run: |
+ rm -rf ./docs/_build
+ mkdir -p ./docs/_build
+ ./docs/gen_docs dagrunner ./docs/_build
+ git add -f docs/_build/
+
+ - name: Check if documentation has changed
+ id: check-docs
+ run: |
+ echo "changed=$(git diff --cached --quiet --exit-code || echo true)" | tee -a $GITHUB_OUTPUT
+
+ - name: Create PR
+ if: steps.check-docs.outputs.changed == 'true'
+ run: |
+ git config user.name github-actions[bot]
+ git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+
+ source_branch="${GITHUB_REF_NAME}"
+ DOCS_BRANCH="docs-update/${source_branch//\//-}"
+ sha="${{ github.sha }}"
+ short_sha="${sha::8}"
+
+ git switch -c "$DOCS_BRANCH"
+ git commit -m "Automated reference documentation update for $short_sha [skip ci]"
+ git push origin "$DOCS_BRANCH" --force
+
+ # Create a PR only if one is not already open for this branch
+ if ! gh pr list --head "$DOCS_BRANCH" --json number --jq '.[0].number' | grep -q .; then
+ gh pr create \
+ --title "Automated reference documentation update for $source_branch" \
+ --body "This PR was automatically generated by the documentation workflow for commit $short_sha [skip ci]" \
+ --base "$source_branch" \
+ --head "$DOCS_BRANCH" \
+ --label "documentation" \
+ --assignee ${GITHUB_ACTOR}
+ else
+ # comment on PR to say that we have pushed a new commit to the branch
+ pr_number=$(gh pr list --head "$DOCS_BRANCH" --json number --jq '.[0].number')
+ gh pr comment "$pr_number" --body "New documentation changes have been pushed to this branch for commit $short_sha [skip ci]"
+ fi
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+
+ # # https://github.com/orgs/community/discussions/26560#discussioncomment-3531273
+ # # This must be our very final step to ensure that it runs only on condition of
+ # # success of all previous steps. A pushed commit will not trigger the re-running
+ # # of this workflow.
+ # - name: Commit and push documentation changes
+ # if: steps.check-docs.outputs.changed == 'true'
+ # run: |
+ # git config user.name github-actions[bot]
+ # git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+ # git add docs/.
+ # git commit -am "Automated reference documentation update for PR ${{ github.event.number }} [skip ci]"
+ # git push
+ # env:
+ # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 06fd713..2cdf04b 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -2,6 +2,9 @@ name: Pull Request Tests
on: [pull_request]
+permissions:
+ contents: read
+
jobs:
pre-commit:
@@ -12,10 +15,12 @@ jobs:
# SETUP
############################
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+ with:
+ persist-credentials: false
- name: Set up Python
- uses: actions/setup-python@v5
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
with:
python-version: 3.x
cache: 'pip'
@@ -35,7 +40,7 @@ jobs:
run: echo "PYSHA=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
- name: Cache pre-commit
- uses: actions/cache@v3
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
id: pre-commit-cache
with:
path: ~/.cache/pre-commit
@@ -58,10 +63,12 @@ jobs:
# SETUP
############################
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+ with:
+ persist-credentials: false
- name: Set up Python
- uses: actions/setup-python@v5
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
with:
python-version: 3.9
cache: 'pip'
@@ -96,10 +103,12 @@ jobs:
# SETUP
############################
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+ with:
+ persist-credentials: false
- name: Set up Python
- uses: actions/setup-python@v5
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
with:
python-version: 3.9
cache: 'pip'
@@ -145,10 +154,12 @@ jobs:
# SETUP
############################
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+ with:
+ persist-credentials: false
- name: Set up Python
- uses: actions/setup-python@v5
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
with:
python-version: 3.x
cache: 'pip'
@@ -178,15 +189,16 @@ jobs:
############################
- name: Cache ref branch coverage report
id: cache-ref-coverage
- uses: actions/cache@v4
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
with:
path: ref/coverage_output.txt
key: ref-${{ github.event.pull_request.base.sha }}
- name: Checkout ref branch
if: steps.cache-ref-coverage.outputs.cache-hit != 'true'
- uses: actions/checkout@v4
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
+ persist-credentials: false
path: ref
ref: ${{ github.base_ref }}
@@ -203,7 +215,7 @@ jobs:
run: |
set -e
# ----- USER SETTINGS -------------------------------------------------
- CHECK_MISS_TOL=1 # 1 = evaluate the miss‑count difference, 0 = ignore it
+ CHECK_MISS_TOL=1 # 1 = evaluate the miss-count difference, 0 = ignore it
CHECK_PERC_TOL=0 # 1 = evaluate the percentage difference, 0 = ignore it
# Tolerances
@@ -252,7 +264,7 @@ jobs:
- name: Upload coverage report
if: steps.comp-coverage.outputs.coverage_decreased == 'true'
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
with:
name: coverage-report-pr
path: |
@@ -265,10 +277,17 @@ jobs:
echo "Code coverage has decreased"
echo "-----------------------------------------------"
echo -e "Coverage totals\tstatements\tmissed\n\
- PR\t${{ steps.comp-coverage.outputs.pr_tot_stmts }}\t${{ steps.comp-coverage.outputs.pr_tot_miss }}\t${{ steps.comp-coverage.outputs.pr_perc_total }}%\n\
- Reference\t${{ steps.comp-coverage.outputs.ref_tot_stmts }}\t${{ steps.comp-coverage.outputs.ref_tot_miss }}\t${{ steps.comp-coverage.outputs.ref_perc_total }}%" \
+ PR\t${PR_TOT_STMTS}\t${PR_TOT_MISS}\t${PR_PERC_TOTAL}%\n\
+ Reference\t${REF_TOT_STMTS}\t${REF_TOT_MISS}\t${REF_PERC_TOTAL}%" \
| column -t -s $'\t'
echo "-----------------------------------------------"
echo "Report uploaded as artifact 'coverage-report-pr'"
echo "Test coverage failure is non-blocking, but please consider improving test coverage before merging."
- exit 1
\ No newline at end of file
+ exit 1
+ env:
+ PR_TOT_STMTS: ${{ steps.comp-coverage.outputs.pr_tot_stmts }}
+ PR_TOT_MISS: ${{ steps.comp-coverage.outputs.pr_tot_miss }}
+ PR_PERC_TOTAL: ${{ steps.comp-coverage.outputs.pr_perc_total }}
+ REF_TOT_STMTS: ${{ steps.comp-coverage.outputs.ref_tot_stmts }}
+ REF_TOT_MISS: ${{ steps.comp-coverage.outputs.ref_tot_miss }}
+ REF_PERC_TOTAL: ${{ steps.comp-coverage.outputs.ref_perc_total }}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 3e4cb35..a3d99ba 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -15,3 +15,8 @@ repos:
files: '\.py$'
language: system
entry: ./copyright_check
+- repo: https://github.com/zizmorcore/zizmor-pre-commit
+ rev: v1.28.0
+ hooks:
+ - id: zizmor
+ args: [--fix]
diff --git a/README.md b/README.md
index 5d7e83c..2d01343 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@
#

-DAGrunner serves as a Directed Acyclic Graph (DAG) runner, primarily designed to ensure a clear distinction between a project's graph definition (typically in native networkx format) and its execution method. In essence, DAGrunner offers various schedulers for executing the graph, but it firmly separates these operational concepts from the scientific configuration or recipe, i.e., the graph itself. Consequently, while DAGrunner currently provides convenient scheduling options, it remains adaptable to future changes or alternative solutions, ensuring that the scientific configuration can persist regardless of the technologies or tools employed, whether DAGrunner is utilized or not.
+DAGrunner is an abstraction layer for executing Directed Acyclic Graphs (DAGs), providing portability and standardization across different schedulers. Rather than building your graph execution logic around a specific scheduler (such as Dask or Ray), DAGrunner enables you to define your graph once in networkx format and execute it with any supported scheduler without modifying your underlying graph definition. It decouples the scientific configuration and graph recipe from execution concerns, ensuring that your graph remains independent of scheduling technology choices. Additionally, DAGrunner provides a unified tooling layer that includes scheduler-agnostic graph visualization, scheduler-independent logging and monitoring, and other operational utilities—allowing you to maintain consistent workflows regardless of which backend scheduler you choose.
## Documentation
@@ -16,7 +16,7 @@ DAGrunner takes advantage of the native markdown rendering support provided by g
## API
-See [DAGrunner API](docs/dagrunner_index.md)
+See [DAGrunner API](docs/_build/dagrunner_index.md)
## License/copyright
@@ -52,7 +52,7 @@ See [docs/demo.ipynb](docs/demo.ipynb)
DAGrunner concerns itself with graph execution and does not strictly require processing modules (plugins) to take any particular form. That is, you may or may not choose to use or subclass the plugins provided by DAGrunner.
However, for convenience, DAGrunner does define some plugins which fall into two broad categories, some abstract and some for use as they are.
-See [here](docs/dagrunner.plugin_framework.md) for more information.
+See [here](docs/_build/dagrunner.plugin_framework.md) for more information.
## Schedulers
@@ -63,7 +63,7 @@ These range from [dask](https://www.dask.org/), [ray](https://docs.ray.io/en/lat
## Logging and monitoring
DAGrunner provides a script `dagrunner-logger` for running a TCP server. This enables logging to function across the network. Additionally, it will write logs to an sqlite database to aid in realtime monitoring from external tools.
-See [logger](docs/dagrunner.utils.logger.md) for more information.
+See [logger](docs/_build/dagrunner.utils.logger.md) for more information.
## Logo
diff --git a/dagrunner/events.py b/dagrunner/events.py
index d0fece1..5e0b634 100644
--- a/dagrunner/events.py
+++ b/dagrunner/events.py
@@ -16,8 +16,8 @@
When a plugin returns `event.IGNORE`, the immediate descendant node filters out that
input.
The remaining inputs of that plugin are utilised by that node as normal.
-If all inputs to a node are `event.IGNORE`, the node's execution is skipped, and a
-`event.SKIP` event is returned instead, skipping execution through all descendants.
+If all inputs to a node are `event.IGNORE`, the node's execution is ignored, and a
+`event.IGNORE` event is returned instead, ignoring execution through all descendants.
```mermaid
---
@@ -25,11 +25,11 @@
---
graph
cycle1{cycleX}
- cycle1 --> Input1 --> filepath --> Load1 --> cube1 --> Proc
- cycle1 --> Input2 --> filepath --> Load2 --> event.IGNORE --> Proc
- cycle1 --> Input3 --> filepath --> Load3 --> cube3 --> Proc
- cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc
- Proc --> cube --> Save
+ cycle1 --> Input1 -- filepath --> Load1 --cube1 --> Proc
+ cycle1 --> Input2 -- filepath --> Load2 -- event.IGNORE --> Proc
+ cycle1 --> Input3 -- filepath --> Load3 -- cube3 --> Proc
+ cycle1 --> Input4 -- filepath --> Load4 -- event.IGNORE --> Proc
+ Proc -- cube --> Save
Proc["Proc (cube1, cube3)"]
```
Here, Input2 and Input4 return an IGNORE event, likely due to there being missing data.
@@ -47,17 +47,16 @@
---
graph
cycle1{cycleX}
- cycle1 --> Input1 --> filepath --> Load1 --> cube --> Proc
- cycle1 --> Input2 --> filepath --> Load2 --> event.SKIP --> Proc
- cycle1 --> Input3 --> filepath --> Load3 --> cube --> Proc
- cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc
- Proc --> event.SKIP --> Save
+ cycle1 --> Input1 -- filepath --> Load1 -- cube --> Proc
+ cycle1 --> Input2 -- filepath --> Load2 -- event.SKIP --> Proc
+ cycle1 --> Input3 -- filepath --> Load3 -- cube --> Proc
+ cycle1 --> Input4 -- filepath --> Load4 -- event.IGNORE --> Proc
+ Proc["Proc
(not called)"] -- event.SKIP --> Save["Save
(not called)"]
```
Because `Input2` returns a SKIP event, the Proc node and everything that follows aren't
executed and neither is the Save node since the skip is propagated along the execution
graph.
-
-"""
+""" # noqa: E501
from dagrunner.utils import Singleton
diff --git a/dagrunner/execute_graph.py b/dagrunner/execute_graph.py
index 2df4ff1..bfad40f 100755
--- a/dagrunner/execute_graph.py
+++ b/dagrunner/execute_graph.py
@@ -149,10 +149,10 @@ def plugin_executor(
call = as_iterable(call)
# IGNORE_EVENT event handling
- if len(args) > 1 and all((map(lambda x: x is events.IGNORE_EVENT, args))):
+ if len(args) > 0 and all((map(lambda x: x is events.IGNORE_EVENT, args))):
# all args are IGNORE_EVENT, return IGNORE_EVENT (pass along)
if verbose:
- print(f"Retuning 'IGNORE_EVENT' event {call[0]}")
+ print(f"Returning 'IGNORE_EVENT' event {call[0]}")
return events.IGNORE_EVENT
args = list(filter(lambda x: x is not events.IGNORE_EVENT, args))
diff --git a/dagrunner/plugin_framework.py b/dagrunner/plugin_framework.py
index ba215e3..9859342 100644
--- a/dagrunner/plugin_framework.py
+++ b/dagrunner/plugin_framework.py
@@ -8,7 +8,6 @@
import pickle
import string
import subprocess
-import warnings
from abc import ABC, abstractmethod
from glob import glob
@@ -133,6 +132,7 @@ def __call__(self, *args, **kwargs):
Raises:
- FileNotFoundError: If any of the files do not exist and `on_missing` is set
to 'error'.
+ - ValueError: If `on_missing` is set to an invalid value.
"""
args = list(map(process_path, args))
@@ -159,17 +159,14 @@ def __call__(self, *args, **kwargs):
if self._on_missing == "error":
raise e
elif self._on_missing == "ignore":
- warnings.warn(str(e))
return events.IGNORE_EVENT
elif self._on_missing == "skip":
- warnings.warn(str(e))
return events.SKIP_EVENT
else:
raise ValueError(
f"Invalid value for 'on_missing': {self._on_missing}. "
"Accepted values are 'error', 'ignore', and 'skip'."
)
-
return self.load(*args, **kwargs)
diff --git a/dagrunner/tests/execute_graph/test_plugin_executor.py b/dagrunner/tests/execute_graph/test_plugin_executor.py
index de218f8..42591bd 100644
--- a/dagrunner/tests/execute_graph/test_plugin_executor.py
+++ b/dagrunner/tests/execute_graph/test_plugin_executor.py
@@ -109,20 +109,26 @@ def test_pass_class_arg_kwargs(plugin, init_args, call_args, target):
assert res == target
-def test_ignore_event():
- """Check what happens a subset of arguments and all arguments are IGNORE_EVENT."""
+@pytest.mark.parametrize(
+ "args, expected",
+ [
+ ((5, IGNORE_EVENT), 10),
+ ((IGNORE_EVENT, IGNORE_EVENT), IGNORE_EVENT),
+ ((IGNORE_EVENT,), IGNORE_EVENT),
+ ],
+)
+def test_ignore_event(args, expected):
+ """Check IGNORE_EVENT behaviour across mixed, all, and single-input cases."""
call = tuple([lambda x: x + 5])
- res = plugin_executor(*(5, IGNORE_EVENT), call=call)
- assert res == 10
-
- res = plugin_executor(*(IGNORE_EVENT, IGNORE_EVENT), call=call)
- assert res == IGNORE_EVENT
+ res = plugin_executor(*args, call=call)
+ assert res == expected
-def test_skip_event():
- """Check what happens a subset of arguments are SKIP_EVENT."""
+@pytest.mark.parametrize("args", [(5, SKIP_EVENT), (SKIP_EVENT,)])
+def test_skip_event(args):
+ """Check what happens when any input argument is SKIP_EVENT."""
call = tuple([lambda x: x + 5])
- res = plugin_executor(*(5, SKIP_EVENT), call=call)
+ res = plugin_executor(*args, call=call)
assert res == SKIP_EVENT
diff --git a/docs/README.md b/docs/README.md
new file mode 100644
index 0000000..c4701e7
--- /dev/null
+++ b/docs/README.md
@@ -0,0 +1,3 @@
+Documentation is built under `_build` automatically via github continuous integration (CI) using the [gen_docs](gen_docs) script (see [.github/workflows/docs.yml](../.github/workflows/docs.yml)). As `main`, `release/*` or `feature/*` branches are updated, pull requests are automatically created targetting these branches with updated documentation.
+
+Whilst running this script locally yourself might be useful for building the documentation, please defer from including such changes to the built documentation to the repository. This is to be handled by CI.
\ No newline at end of file
diff --git a/docs/dagrunner.config.md b/docs/_build/dagrunner.config.md
similarity index 88%
rename from docs/dagrunner.config.md
rename to docs/_build/dagrunner.config.md
index 266c896..db0bf6c 100644
--- a/docs/dagrunner.config.md
+++ b/docs/_build/dagrunner.config.md
@@ -1,6 +1,6 @@
# module: `dagrunner.config`
-[Source](../dagrunner/config.py#L0)
+[Source](../../dagrunner/config.py#L0)
This module handles the run-time configuration of the dagrunner library.
@@ -19,7 +19,7 @@ see [class: dagrunner.utils.Singleton](dagrunner.utils.md#class-singleton)
## class: `GlobalConfiguration`
-[Source](../dagrunner/config.py#L28)
+[Source](../../dagrunner/config.py#L28)
### Call Signature:
@@ -63,7 +63,7 @@ options::
### function: `__getitem__`
-[Source](../dagrunner/config.py#L172)
+[Source](../../dagrunner/config.py#L172)
#### Call Signature:
@@ -73,7 +73,7 @@ __getitem__(self, key)
### function: `__init__`
-[Source](../dagrunner/config.py#L83)
+[Source](../../dagrunner/config.py#L83)
#### Call Signature:
@@ -85,7 +85,7 @@ Initialize self. See help(type(self)) for accurate signature.
### function: `__repr__`
-[Source](../dagrunner/config.py#L90)
+[Source](../../dagrunner/config.py#L90)
#### Call Signature:
@@ -97,7 +97,7 @@ Return repr(self).
### function: `__setitem__`
-[Source](../dagrunner/config.py#L175)
+[Source](../../dagrunner/config.py#L175)
#### Call Signature:
@@ -107,7 +107,7 @@ __setitem__(self, key, value)
### function: `__str__`
-[Source](../dagrunner/config.py#L87)
+[Source](../../dagrunner/config.py#L87)
#### Call Signature:
@@ -119,7 +119,7 @@ Return str(self).
### function: `parse_configuration`
-[Source](../dagrunner/config.py#L146)
+[Source](../../dagrunner/config.py#L146)
#### Call Signature:
diff --git a/docs/dagrunner.events.md b/docs/_build/dagrunner.events.md
similarity index 66%
rename from docs/dagrunner.events.md
rename to docs/_build/dagrunner.events.md
index 5616830..750ec9c 100644
--- a/docs/dagrunner.events.md
+++ b/docs/_build/dagrunner.events.md
@@ -1,6 +1,6 @@
# module: `dagrunner.events`
-[Source](../dagrunner/events.py#L0)
+[Source](../../dagrunner/events.py#L0)
## Overview
DAGrunner defines two special singleton events that plugins can return to control the
@@ -15,8 +15,8 @@ execution flow of their graph.
When a plugin returns `event.IGNORE`, the immediate descendant node filters out that
input.
The remaining inputs of that plugin are utilised by that node as normal.
-If all inputs to a node are `event.IGNORE`, the node's execution is skipped, and a
-`event.SKIP` event is returned instead, skipping execution through all descendants.
+If all inputs to a node are `event.IGNORE`, the node's execution is ignored, and a
+`event.IGNORE` event is returned instead, ignoring execution through all descendants.
```mermaid
---
@@ -24,11 +24,11 @@ title: event.IGNORE (filtering some)
---
graph
cycle1{cycleX}
- cycle1 --> Input1 --> filepath --> Load1 --> cube1 --> Proc
- cycle1 --> Input2 --> filepath --> Load2 --> event.IGNORE --> Proc
- cycle1 --> Input3 --> filepath --> Load3 --> cube3 --> Proc
- cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc
- Proc --> cube --> Save
+ cycle1 --> Input1 -- filepath --> Load1 --cube1 --> Proc
+ cycle1 --> Input2 -- filepath --> Load2 -- event.IGNORE --> Proc
+ cycle1 --> Input3 -- filepath --> Load3 -- cube3 --> Proc
+ cycle1 --> Input4 -- filepath --> Load4 -- event.IGNORE --> Proc
+ Proc -- cube --> Save
Proc["Proc (cube1, cube3)"]
```
Here, Input2 and Input4 return an IGNORE event, likely due to there being missing data.
@@ -46,11 +46,11 @@ title: event.SKIP
---
graph
cycle1{cycleX}
- cycle1 --> Input1 --> filepath --> Load1 --> cube --> Proc
- cycle1 --> Input2 --> filepath --> Load2 --> event.SKIP --> Proc
- cycle1 --> Input3 --> filepath --> Load3 --> cube --> Proc
- cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc
- Proc --> event.SKIP --> Save
+ cycle1 --> Input1 -- filepath --> Load1 -- cube --> Proc
+ cycle1 --> Input2 -- filepath --> Load2 -- event.SKIP --> Proc
+ cycle1 --> Input3 -- filepath --> Load3 -- cube --> Proc
+ cycle1 --> Input4 -- filepath --> Load4 -- event.IGNORE --> Proc
+ Proc["Proc
(not called)"] -- event.SKIP --> Save["Save
(not called)"]
```
Because `Input2` returns a SKIP event, the Proc node and everything that follows aren't
executed and neither is the Save node since the skip is propagated along the execution
diff --git a/docs/dagrunner.execute_graph.md b/docs/_build/dagrunner.execute_graph.md
similarity index 93%
rename from docs/dagrunner.execute_graph.md
rename to docs/_build/dagrunner.execute_graph.md
index 1a03168..544089d 100644
--- a/docs/dagrunner.execute_graph.md
+++ b/docs/_build/dagrunner.execute_graph.md
@@ -1,6 +1,6 @@
# module: `dagrunner.execute_graph`
-[Source](../dagrunner/execute_graph.py#L0)
+[Source](../../dagrunner/execute_graph.py#L0)
see [GlobalConfiguration: dagrunner.config.CONFIG](dagrunner.config.md#globalconfiguration-config)
@@ -16,13 +16,15 @@ see [module: dagrunner.events](dagrunner.events.md#module-dagrunnerevents)
see [function: dagrunner.utils.function_to_argparse_parse_args](dagrunner.utils.md#function-function_to_argparse_parse_args)
+see [function: dagrunner.utils.get_object_dot_module_path](dagrunner.utils.md#function-get_object_dot_module_path)
+
see [module: dagrunner.utils.logger](dagrunner.utils.logger.md#module-dagrunnerutilslogger)
see [function: dagrunner.utils.networkx.visualise_graph](dagrunner.utils.networkx.md#function-visualise_graph)
## class: `ExecuteGraph`
-[Source](../dagrunner/execute_graph.py#L274)
+[Source](../../dagrunner/execute_graph.py#L318)
### Call Signature:
@@ -32,7 +34,7 @@ ExecuteGraph(networkx_graph: str, networkx_graph_kwargs: dict = None, .` will be written to `..md`,
+and the index file `_index.md` will be extended to include a link to
+this module and its contents.
+
+Note that markdown syntax is supported in docstrings, taken without modification.
+This means that anything supported by your markdown renderer (e.g. vscode and GitHub)
+will be supported in docstrings, including links to other documentation, images,
+tables and other markdown features.
+
+Example usage:
+
+```
+def function(args1, keywarg1=default1):
+ \"""
+ Dummy function to demonstrate markdown docstrings.
+
+ Args:
+ - `args1` (str):
+ Some positional argument.
+ - `keywarg1` (int, optional):
+ Some keyword argument with a default value.
+
+ \"""
+ ...
+```
+
"""
+import argparse
import importlib
import importlib.util
import inspect
import os
-import sys
def gen_source_link(obj, target_root):
@@ -281,6 +310,16 @@ def generate_markdown_recursive(module_dot_path, target_root):
if __name__ == "__main__":
- module_path = sys.argv[1]
- target_root = os.path.abspath(sys.argv[2])
- generate_markdown_recursive(module_path, target_root)
+ parser = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+ )
+ parser.add_argument(
+ "pymodule_path", help="Python module path, which must be on the PYTHONPATH"
+ )
+ parser.add_argument(
+ "target_root", help="Target root directory for generated markdown"
+ )
+ args = parser.parse_args()
+ pymodule_path = args.pymodule_path
+ target_root = os.path.abspath(args.target_root)
+ generate_markdown_recursive(pymodule_path, target_root)