feat(country-map): retire the notebook in favor of per-country build scripts

.github/workflows/update-country-maps.yml

@@ -2,7 +2,7 @@ name: Update Country Maps
 
 # Regenerates plugins/plugin-chart-country-map's GeoJSON files, countries.ts
 # and docs/data/countries.json from upstream Natural Earth + community geo
-# data, by running the same notebook `npm run update-maps` runs locally.
+# data, by running the same `build_maps.py` script `npm run update-maps` runs locally.
 #
 # Runs weekly and on workflow_dispatch, but only opens or updates a PR when
 # the regenerated output actually differs from what's on `master` - most
@@ -12,7 +12,7 @@ name: Update Country Maps
 # to a commit where practical; the live WHO ArcGIS query can't be pinned
 # (it's a live service, not a repo), so a run can still legitimately pick up
 # an unreviewed upstream change there. The job itself fails (separately from
-# whether a PR was opened) if the notebook hits a cell error, so a broken
+# whether a PR was opened) if build_maps.py errors, so a broken
 # upstream source doesn't just quietly go green.
 on:
   schedule:
@@ -45,86 +45,28 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Install notebook dependencies
+      - name: Install build dependencies
         run: |
           # Exact versions, with the full resolved set pinned via a constraints
           # file, so a diff only moves with upstream data and not with a
           # GeoPandas/Shapely/pyogrio change to simplification or serialization.
           pip install -c superset-frontend/plugins/plugin-chart-country-map/scripts/requirements-constraints.txt \
-            jupyter==1.1.1 nbconvert==7.16.6 ipykernel==6.29.5 \
             geopandas==1.1.1 shapely==2.1.1 matplotlib==3.10.3 requests==2.32.4
 
-      - name: Force speed_run (skip interactive preview rendering)
+      - name: Run build_maps.py
+        id: build
         working-directory: superset-frontend/plugins/plugin-chart-country-map/scripts
         run: |
-          python3 - <<'PYEOF'
-          import json
-
-          path = "Country Map GeoJSON Generator.ipynb"
-          with open(path, encoding="utf-8") as f:
-              nb = json.load(f)
-
-          for cell in nb["cells"]:
-              if cell["cell_type"] != "code":
-                  continue
-              if "".join(cell["source"]).strip().startswith("speed_run = False"):
-                  cell["source"] = [
-                      "speed_run = True\n",
-                      "# forced True for unattended CI regeneration; not committed back\n",
-                  ]
-                  break
-          else:
-              raise SystemExit("Could not find the speed_run cell to patch")
-
-          with open(".ci-run.ipynb", "w", encoding="utf-8") as f:
-              json.dump(nb, f)
-          PYEOF
-
-      - name: Run notebook
-        working-directory: superset-frontend/plugins/plugin-chart-country-map/scripts
-        run: |
-          jupyter nbconvert --to notebook --execute --inplace --allow-errors \
-            --ExecutePreprocessor.timeout=1200 .ci-run.ipynb
-
-      - name: Check for cell errors and record a summary
-        id: notebook
-        working-directory: superset-frontend/plugins/plugin-chart-country-map/scripts
-        run: |
-          python3 - <<'PYEOF' >> "$GITHUB_OUTPUT"
-          import json
-
-          with open(".ci-run.ipynb", encoding="utf-8") as f:
-              nb = json.load(f)
-
-          errors = []
-          for i, cell in enumerate(nb["cells"]):
-              for output in cell.get("outputs", []):
-                  if output.get("output_type") == "error":
-                      errors.append(f"cell {i}: {output.get('ename')}: {output.get('evalue')}")
-                  elif output.get("output_type") == "stream":
-                      # Some cells catch download failures and print them
-                      # instead of raising, so they never emit an error output.
-                      for line in "".join(output.get("text", [])).splitlines():
-                          if line.startswith("Unable to download"):
-                              errors.append(f"cell {i}: {line}")
-
-          if errors:
-              print("had_errors=true")
-              summary = "\n".join(f"- {e}" for e in errors[:20])
-          else:
-              print("had_errors=false")
-              summary = ""
-
-          with open("notebook-errors.txt", "w", encoding="utf-8") as f:
-              f.write(summary)
-          PYEOF
-          rm -f .ci-run.ipynb
-          if [ -s notebook-errors.txt ]; then
-            echo "::warning::Notebook hit cell errors; generated data may be stale."
-            {
-              echo "### Country map notebook cell errors"
-              cat notebook-errors.txt
-            } >> "$GITHUB_STEP_SUMMARY"
+          # Capture the exit status instead of failing here, so the drift/PR
+          # steps below still run; the final step fails the job on error.
+          set +e
+          python3 build_maps.py --data-dir "$RUNNER_TEMP/natural-earth"
+          status=$?
+          if [ "$status" -ne 0 ]; then
+            echo "had_errors=true" >> "$GITHUB_OUTPUT"
+            echo "::warning::build_maps.py failed (exit $status); generated data may be stale."
+          else
+            echo "had_errors=false" >> "$GITHUB_OUTPUT"
           fi
 
       - name: Check for drift
@@ -142,7 +84,7 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           BRANCH: chore/update-country-maps-automated
-          HAD_ERRORS: ${{ steps.notebook.outputs.had_errors }}
+          HAD_ERRORS: ${{ steps.build.outputs.had_errors }}
         run: |
           git config user.name "GitHub Action"
           git config user.email "action@github.com"
@@ -151,15 +93,14 @@ jobs:
             superset-frontend/plugins/plugin-chart-country-map/src/countries.ts \
             docs/data/countries.json
           git commit -m "chore(country-map): regenerate GeoJSON from upstream sources" \
-            -m "Automated run of the Country Map GeoJSON Generator notebook (npm run update-maps). Review the rendered diffs before merging - one source (WHO ArcGIS) is a live query, not a pinned commit."
+            -m "Automated run of build_maps.py (npm run update-maps). Review the rendered diffs before merging - one source (WHO ArcGIS) is a live query, not a pinned commit."
           git push --force origin "$BRANCH"
 
           BODY="Automated run of \`npm run update-maps\` against current \`master\`. **Please review the rendered map diffs before merging** - this pulls from several community geo sources plus a live WHO ArcGIS query that can't be pinned to a commit, so an upstream change there hasn't had human eyes on it yet."
           if [ "$HAD_ERRORS" = "true" ]; then
             BODY="$BODY
 
-          ⚠️ The notebook run hit cell errors (ran with --allow-errors, so it continued past them - affected countries' data may be stale rather than updated):
-          $(cat superset-frontend/plugins/plugin-chart-country-map/scripts/notebook-errors.txt)"
+          ⚠️ build_maps.py exited with an error, so the affected countries' data may be stale rather than updated. See the workflow run log."
           fi
 
           if gh pr list --repo "$GITHUB_REPOSITORY" --head "$BRANCH" --state open --json number --jq 'length > 0' | grep -q true; then
@@ -172,7 +113,7 @@ jobs:
           fi
 
       - name: Close stale sync PR when no drift remains
-        if: steps.diff.outputs.has_changes == 'false' && steps.notebook.outputs.had_errors != 'true'
+        if: steps.diff.outputs.has_changes == 'false' && steps.build.outputs.had_errors != 'true'
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           BRANCH: chore/update-country-maps-automated
@@ -182,8 +123,8 @@ jobs:
               --comment "Upstream sources no longer differ from \`master\`, so this regenerated data is stale. Closing automatically."
           fi
 
-      - name: Fail if the notebook hit cell errors
-        if: steps.notebook.outputs.had_errors == 'true'
+      - name: Fail if build_maps.py errored
+        if: steps.build.outputs.had_errors == 'true'
         run: |
-          echo "::error::Notebook hit cell errors - see the job summary above. Any resulting PR update still went out; the data for the affected countries may be stale rather than updated."
+          echo "::error::build_maps.py failed - see the Run build_maps.py step log. Any resulting PR update still went out; the data for the affected countries may be stale rather than updated."
           exit 1

docs/admin_docs/configuration/country-map-tools.mdx

@@ -33,8 +33,16 @@ The Country Maps visualization already ships with the maps for the following cou
 To add a new country to the list, you'd have to edit files in
 [@superset-ui/plugin-chart-country-map](https://github.com/apache/superset/tree/master/superset-frontend/plugins/plugin-chart-country-map).
 
-1. Generate a new GeoJSON file for your country following the guide in [this Jupyter notebook](https://github.com/apache/superset/blob/master/superset-frontend/plugins/plugin-chart-country-map/scripts/Country%20Map%20GeoJSON%20Generator.ipynb).
-2. Edit the countries list in [plugin-chart-country-map/src/countries.ts](https://github.com/apache/superset/blob/master/superset-frontend/plugins/plugin-chart-country-map/src/countries.ts).
-3. Install superset-frontend dependencies: `cd superset-frontend && npm install`
-4. Verify your countries in Superset plugins storybook: `npm run plugins:storybook`.
-5. Build and install Superset from source code.
+1. Most countries need no correction at all - check whether yours already resolves via a plain
+   lookup in [`scripts/maps/catalog.py`](https://github.com/apache/superset/blob/master/superset-frontend/plugins/plugin-chart-country-map/scripts/maps/catalog.py)
+   before writing anything.
+2. If it needs a fix (a renamed region, a relocated territory, a replacement data source, ...),
+   add a module under [`scripts/maps/countries/`](https://github.com/apache/superset/tree/master/superset-frontend/plugins/plugin-chart-country-map/scripts/maps/countries) -
+   the existing modules there are the best reference for the pattern. Then import the module and add it to
+   `REGISTRY` in `scripts/maps/countries/__init__.py`; unregistered modules are ignored by the build.
+3. Install the Python geo stack (`pip install geopandas shapely matplotlib requests`), then regenerate the GeoJSON files, `countries.ts` and `docs/data/countries.json`:
+   `cd superset-frontend && npm run update-maps`. Iterate on just your country with a rendered
+   preview via `python3 plugins/plugin-chart-country-map/scripts/build_maps.py --countries <name> --preview`.
+4. Install superset-frontend dependencies: `cd superset-frontend && npm install`
+5. Verify your countries in Superset plugins storybook: `npm run plugins:storybook`.
+6. Build and install Superset from source code.

pyproject.toml

@@ -531,6 +531,7 @@ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
 "superset/mcp_service/app.py" = ["S608", "E501"]  # LLM instruction text: SQL examples (S608) and long lines in multiline string (E501)
 "superset/mcp_service/*/tool/list_*.py" = ["E501"]  # LLM docstring examples show full request shapes which exceed line length
 "scripts/*" = ["TID251"]
+"superset-frontend/plugins/plugin-chart-country-map/scripts/**" = ["TID251"]  # standalone data-pipeline tooling, no superset package import
 "setup.py" = ["TID251"]
 "superset/config.py" = ["TID251"]
 "superset/cli/update.py" = ["TID251"]

superset-frontend/package.json

@@ -86,7 +86,7 @@
     "test": "cross-env NODE_ENV=test NODE_OPTIONS=\"--max-old-space-size=8192\" jest --silent",
     "test-loud": "cross-env NODE_ENV=test NODE_OPTIONS=\"--max-old-space-size=8192\" jest",
     "type": "cross-env NODE_OPTIONS=\"--max-old-space-size=8192\" tsc --noEmit",
-    "update-maps": "cd plugins/plugin-chart-country-map/scripts && jupyter nbconvert --to notebook --execute --inplace --allow-errors --ExecutePreprocessor.timeout=1200 'Country Map GeoJSON Generator.ipynb'",
+    "update-maps": "cd plugins/plugin-chart-country-map/scripts && python3 build_maps.py",
     "validate-release": "../RELEASING/validate_this_release.sh"
   },
   "dependencies": {

superset-frontend/plugins/plugin-chart-country-map/README.md

@@ -53,10 +53,24 @@ for more details.
 
 ### Update Map
 
-To update the country maps or add a new country, run scripts in the Jupyter notebook
-`scripts/Country Map GeoJSON Generator.ipynb`.
+To regenerate every country's GeoJSON from upstream Natural Earth and
+community sources, `countries.ts`, and `docs/data/countries.json`, run:
 
 ```bash
-pip install geopandas shapely matplotlib notebook
-jupyter notebook
+pip install geopandas shapely matplotlib requests
+# from the superset-frontend/ directory
+npm run update-maps
+```
+
+To add a new country, or fix a correction on an existing one, write a
+module under `scripts/maps/countries/` and add it to `REGISTRY` in
+`scripts/maps/countries/__init__.py` (unregistered modules are ignored) - see
+the existing modules there for the pattern (most countries need no module at all, just a plain
+Natural Earth lookup, handled by `scripts/maps/catalog.py`). Iterate on
+just your new module, with a rendered preview, via (from
+`superset-frontend/plugins/plugin-chart-country-map/`):
+
+```bash
+python3 scripts/build_maps.py --countries <your_country> --preview \
+  --out-dir /tmp/maps-preview
 ```