DOC: improved description of how the documentation is put together

.github/dependabot.yml

@@ -0,0 +1,15 @@
+# Reference:
+# - https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/keeping-your-actions-up-to-date-with-dependabot
+# - https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    labels:
+      - "bot"
+    cooldown:
+      default-days: 7

.github/workflows/docs.yml

@@ -0,0 +1,103 @@
+name: Documentation
+
+on:
+  push:
+    branches:
+      - main
+      - release/*
+      - feature/*
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  build:
+    if: github.repository_owner == 'MetOffice' && !contains(github.event.head_commit.message, '[skip ci]')
+    runs-on: ubuntu-latest
+
+    steps:
+      # SETUP
+      ############################
+      - name: Checkout code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+        with:
+          ref: ${{ github.ref_name }}  # Ensure branch is checked out, not detached state (so we can push a commit later)
+          token: ${{ secrets.GITHUB_TOKEN }}
+          persist-credentials: true  # Ensure that the token is available for pushing changes
+
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
+        with:
+          python-version: 3.x
+
+      - name: Add checkout directory to PYTHONPATH
+        run: echo "PYTHONPATH=$(pwd):$PYTHONPATH" >> $GITHUB_ENV
+
+      - name: Install dependencies
+        id: install-dependencies
+        run: |
+          pip install .[tests,dev]
+          pip uninstall dagrunner -y
+
+      # DOCUMENTATION
+      ############################
+      - name: Build documentation
+        run: |
+          rm -rf ./docs/_build
+          mkdir -p ./docs/_build
+          ./docs/gen_docs dagrunner ./docs/_build
+          git add -f docs/_build/
+
+      - name: Check if documentation has changed
+        id: check-docs
+        run: |
+          echo "changed=$(git diff --cached --quiet --exit-code || echo true)" | tee -a $GITHUB_OUTPUT
+
+      - name: Create PR
+        if: steps.check-docs.outputs.changed == 'true'
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+
+          source_branch="${GITHUB_REF_NAME}"
+          DOCS_BRANCH="docs-update/${source_branch//\//-}"
+          sha="${{ github.sha }}"
+          short_sha="${sha::8}"
+
+          git switch -c "$DOCS_BRANCH"
+          git commit -m "Automated reference documentation update for $short_sha [skip ci]"
+          git push origin "$DOCS_BRANCH" --force
+
+          # Create a PR only if one is not already open for this branch
+          if ! gh pr list --head "$DOCS_BRANCH" --json number --jq '.[0].number' | grep -q .; then
+            gh pr create \
+              --title "Automated reference documentation update for $source_branch" \
+              --body "This PR was automatically generated by the documentation workflow for commit $short_sha [skip ci]" \
+              --base "$source_branch" \
+              --head "$DOCS_BRANCH" \
+              --label "documentation" \
+              --assignee ${GITHUB_ACTOR}
+          else
+            # comment on PR to say that we have pushed a new commit to the branch
+            pr_number=$(gh pr list --head "$DOCS_BRANCH" --json number --jq '.[0].number')
+            gh pr comment "$pr_number" --body "New documentation changes have been pushed to this branch for commit $short_sha [skip ci]"
+          fi
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+
+      # # https://github.com/orgs/community/discussions/26560#discussioncomment-3531273
+      # # This must be our very final step to ensure that it runs only on condition of
+      # # success of all previous steps.  A pushed commit will not trigger the re-running
+      # # of this workflow.
+      # - name: Commit and push documentation changes
+      #   if: steps.check-docs.outputs.changed == 'true'
+      #   run: |
+      #     git config user.name github-actions[bot]
+      #     git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+      #     git add docs/.
+      #     git commit -am "Automated reference documentation update for PR ${{ github.event.number }} [skip ci]"
+      #     git push
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/tests.yml

@@ -2,6 +2,9 @@ name: Pull Request Tests
 
 on: [pull_request]
 
+permissions:
+  contents: read
+
 
 jobs:
   pre-commit:
@@ -12,10 +15,12 @@ jobs:
       # SETUP
       ############################
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+        with:
+          persist-credentials: false
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
         with:
           python-version: 3.x
           cache: 'pip'
@@ -35,7 +40,7 @@ jobs:
         run: echo "PYSHA=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
 
       - name: Cache pre-commit
-        uses: actions/cache@v3
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
         id: pre-commit-cache
         with:
           path: ~/.cache/pre-commit
@@ -58,10 +63,12 @@ jobs:
       # SETUP
       ############################
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+        with:
+          persist-credentials: false
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
         with:
           python-version: 3.9
           cache: 'pip'
@@ -96,10 +103,12 @@ jobs:
       # SETUP
       ############################
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+        with:
+          persist-credentials: false
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
         with:
           python-version: 3.9
           cache: 'pip'
@@ -145,10 +154,12 @@ jobs:
       # SETUP
       ############################
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+        with:
+          persist-credentials: false
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
         with:
           python-version: 3.x
           cache: 'pip'
@@ -178,15 +189,16 @@ jobs:
       ############################
       - name: Cache ref branch coverage report
         id: cache-ref-coverage
-        uses: actions/cache@v4
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
         with:
           path: ref/coverage_output.txt
           key: ref-${{ github.event.pull_request.base.sha }}
 
       - name: Checkout ref branch
         if: steps.cache-ref-coverage.outputs.cache-hit != 'true'
-        uses: actions/checkout@v4
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
         with:
+          persist-credentials: false
           path: ref
           ref: ${{ github.base_ref }}
 
@@ -203,7 +215,7 @@ jobs:
         run: |
           set -e
           # ----- USER SETTINGS -------------------------------------------------
-          CHECK_MISS_TOL=1      # 1 = evaluate the miss‑count difference, 0 = ignore it
+          CHECK_MISS_TOL=1      # 1 = evaluate the miss-count difference, 0 = ignore it
           CHECK_PERC_TOL=0      # 1 = evaluate the percentage difference, 0 = ignore it
 
           # Tolerances
@@ -252,7 +264,7 @@ jobs:
 
       - name: Upload coverage report
         if: steps.comp-coverage.outputs.coverage_decreased == 'true'
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
         with:
           name: coverage-report-pr
           path: |
@@ -265,10 +277,17 @@ jobs:
             echo "Code coverage has decreased"
             echo "-----------------------------------------------"
             echo -e "Coverage totals\tstatements\tmissed\n\
-            PR\t${{ steps.comp-coverage.outputs.pr_tot_stmts }}\t${{ steps.comp-coverage.outputs.pr_tot_miss }}\t${{ steps.comp-coverage.outputs.pr_perc_total }}%\n\
-            Reference\t${{ steps.comp-coverage.outputs.ref_tot_stmts }}\t${{ steps.comp-coverage.outputs.ref_tot_miss }}\t${{ steps.comp-coverage.outputs.ref_perc_total }}%" \
+            PR\t${PR_TOT_STMTS}\t${PR_TOT_MISS}\t${PR_PERC_TOTAL}%\n\
+            Reference\t${REF_TOT_STMTS}\t${REF_TOT_MISS}\t${REF_PERC_TOTAL}%" \
             | column -t -s $'\t'
             echo "-----------------------------------------------"
             echo "Report uploaded as artifact 'coverage-report-pr'"
             echo "Test coverage failure is non-blocking, but please consider improving test coverage before merging."
-            exit 1
+            exit 1
+        env:
+          PR_TOT_STMTS: ${{ steps.comp-coverage.outputs.pr_tot_stmts }}
+          PR_TOT_MISS: ${{ steps.comp-coverage.outputs.pr_tot_miss }}
+          PR_PERC_TOTAL: ${{ steps.comp-coverage.outputs.pr_perc_total }}
+          REF_TOT_STMTS: ${{ steps.comp-coverage.outputs.ref_tot_stmts }}
+          REF_TOT_MISS: ${{ steps.comp-coverage.outputs.ref_tot_miss }}
+          REF_PERC_TOTAL: ${{ steps.comp-coverage.outputs.ref_perc_total }}

.pre-commit-config.yaml

@@ -15,3 +15,8 @@ repos:
       files: '\.py$'
       language: system
       entry: ./copyright_check
+- repo: https://github.com/zizmorcore/zizmor-pre-commit
+  rev: v1.28.0
+  hooks:
+    - id: zizmor
+      args: [--fix]

README.md

@@ -8,15 +8,15 @@
 
 # <img src="docs/logo.svg" alt="dagrunner_icon" width=200px>![](https://placehold.co/600x200/transparent/929292?text=DAGrunner)
 
-DAGrunner serves as a Directed Acyclic Graph (DAG) runner, primarily designed to ensure a clear distinction between a project's graph definition (typically in native networkx format) and its execution method. In essence, DAGrunner offers various schedulers for executing the graph, but it firmly separates these operational concepts from the scientific configuration or recipe, i.e., the graph itself. Consequently, while DAGrunner currently provides convenient scheduling options, it remains adaptable to future changes or alternative solutions, ensuring that the scientific configuration can persist regardless of the technologies or tools employed, whether DAGrunner is utilized or not.
+DAGrunner is an abstraction layer for executing Directed Acyclic Graphs (DAGs), providing portability and standardization across different schedulers. Rather than building your graph execution logic around a specific scheduler (such as Dask or Ray), DAGrunner enables you to define your graph once in networkx format and execute it with any supported scheduler without modifying your underlying graph definition. It decouples the scientific configuration and graph recipe from execution concerns, ensuring that your graph remains independent of scheduling technology choices. Additionally, DAGrunner provides a unified tooling layer that includes scheduler-agnostic graph visualization, scheduler-independent logging and monitoring, and other operational utilities—allowing you to maintain consistent workflows regardless of which backend scheduler you choose.
 
 ## Documentation
 
 DAGrunner takes advantage of the native markdown rendering support provided by github.  To that end, all documentation of DAGrunner resides in markdown files.
 
 ## API
 
-See [DAGrunner API](docs/dagrunner_index.md)
+See [DAGrunner API](docs/_build/dagrunner_index.md)
 
 ## License/copyright
 
@@ -52,7 +52,7 @@ See [docs/demo.ipynb](docs/demo.ipynb)
 DAGrunner concerns itself with graph execution and does not strictly require processing modules (plugins) to take any particular form.  That is, you may or may not choose to use or subclass the plugins provided by DAGrunner.
 However, for convenience, DAGrunner does define some plugins which fall into two broad categories, some abstract and some for use as they are.
 
-See [here](docs/dagrunner.plugin_framework.md) for more information.
+See [here](docs/_build/dagrunner.plugin_framework.md) for more information.
 
 ## Schedulers
 
@@ -63,7 +63,7 @@ These range from [dask](https://www.dask.org/), [ray](https://docs.ray.io/en/lat
 ## Logging and monitoring
 
 DAGrunner provides a script `dagrunner-logger` for running a TCP server.  This enables logging to function across the network.  Additionally, it will write logs to an sqlite database to aid in realtime monitoring from external tools.
-See [logger](docs/dagrunner.utils.logger.md) for more information.
+See [logger](docs/_build/dagrunner.utils.logger.md) for more information.
 
 ## Logo
 

dagrunner/events.py

@@ -16,20 +16,20 @@
 When a plugin returns `event.IGNORE`, the immediate descendant node filters out that
 input.
 The remaining inputs of that plugin are utilised by that node as normal.
-If all inputs to a node are `event.IGNORE`, the node's execution is skipped, and a
-`event.SKIP` event is returned instead, skipping execution through all descendants.
+If all inputs to a node are `event.IGNORE`, the node's execution is ignored, and a
+`event.IGNORE` event is returned instead, ignoring execution through all descendants.
 
 ```mermaid
 ---
 title: event.IGNORE (filtering some)
 ---
 graph
     cycle1{cycleX}
-    cycle1 --> Input1 --> filepath --> Load1 --> cube1 --> Proc
-    cycle1 --> Input2 --> filepath --> Load2 --> event.IGNORE --> Proc
-    cycle1 --> Input3 --> filepath --> Load3 --> cube3 --> Proc
-    cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc
-    Proc --> cube --> Save
+    cycle1 --> Input1 -- filepath --> Load1 --cube1 --> Proc
+    cycle1 --> Input2 -- filepath --> Load2 -- event.IGNORE --> Proc
+    cycle1 --> Input3 -- filepath --> Load3 -- cube3 --> Proc
+    cycle1 --> Input4 -- filepath --> Load4 -- event.IGNORE --> Proc
+    Proc -- cube --> Save
     Proc["Proc (cube1, cube3)"]
 ```
 Here, Input2 and Input4 return an IGNORE event, likely due to there being missing data.
@@ -47,17 +47,16 @@
 ---
 graph
     cycle1{cycleX}
-    cycle1 --> Input1 --> filepath --> Load1 --> cube --> Proc
-    cycle1 --> Input2 --> filepath --> Load2 --> event.SKIP --> Proc
-    cycle1 --> Input3 --> filepath --> Load3 --> cube --> Proc
-    cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc
-    Proc --> event.SKIP --> Save
+    cycle1 --> Input1 -- filepath --> Load1 -- cube --> Proc
+    cycle1 --> Input2 -- filepath --> Load2 -- event.SKIP --> Proc
+    cycle1 --> Input3 -- filepath --> Load3 -- cube --> Proc
+    cycle1 --> Input4 -- filepath --> Load4 -- event.IGNORE --> Proc
+    Proc["Proc<br><sup>(not called)"] -- event.SKIP --> Save["Save<br><sup>(not called)"]
 ```
 Because `Input2` returns a SKIP event, the Proc node and everything that follows aren't
 executed and neither is the Save node since the skip is propagated along the execution
 graph.
-
-"""
+"""  # noqa: E501
 
 from dagrunner.utils import Singleton
 

dagrunner/execute_graph.py

@@ -149,10 +149,10 @@ def plugin_executor(
     call = as_iterable(call)
 
     # IGNORE_EVENT event handling
-    if len(args) > 1 and all((map(lambda x: x is events.IGNORE_EVENT, args))):
+    if len(args) > 0 and all((map(lambda x: x is events.IGNORE_EVENT, args))):
         # all args are IGNORE_EVENT, return IGNORE_EVENT (pass along)
         if verbose:
-            print(f"Retuning 'IGNORE_EVENT' event {call[0]}")
+            print(f"Returning 'IGNORE_EVENT' event {call[0]}")
         return events.IGNORE_EVENT
     args = list(filter(lambda x: x is not events.IGNORE_EVENT, args))