diff --git a/.agents/skills/ams-build/SKILL.md b/.agents/skills/ams-build/SKILL.md new file mode 100644 index 00000000..88752dc3 --- /dev/null +++ b/.agents/skills/ams-build/SKILL.md @@ -0,0 +1,252 @@ +--- +name: ams-build +description: >- + Build and install the AMS library (github.com/LLNL/AMS) from source on an HPC + cluster via CMake. Use this whenever the user wants to compile, configure, + build, or install AMS, or is choosing between build variants (with/without + RabbitMQ, GPU, MPI, Caliper, etc.). ALWAYS use this skill for AMS build/CMake + questions even when phrased loosely ("get AMS running on Tioga", "why can't + CMake find Torch", "AMS build with RabbitMQ"). It knows how to gather AMS's + dependencies two ways: via LLNL Livermore Computing's internal Spack + environment or by pointing CMake at manually provided libraries. +--- + +# Installing AMS from source + +AMS is a C++17 library built with CMake. The build is usually simple once the +dependency hints are correct. + +## Before Spack or LC setup + +In restricted environments, set these before any Spack command or before +sourcing `scripts/gitlab/setup-env.sh`: + +```bash +export SPACK_DISABLE_LOCAL_CONFIG=true +export SPACK_SKIP_MODULES=1 +export SPACK_USER_CACHE_PATH=/tmp +export XDG_CACHE_HOME=/tmp +``` + +## Dependency provisioning path + +Run this check before writing a CMake command: + +```bash +if [[ -d /usr/workspace/AMS/ams-spack-environments ]]; then + echo "LC cluster: use scripts/gitlab/setup-env.sh" +else + echo "Non-LC: provide dependencies manually (references/manual-deps.md)" +fi +``` + +On an LLNL Livermore Computing cluster, source the repo setup script. It loads +modules, activates the AMS Spack environment, and exports `AMS_TORCH_PATH`, +`AMS_HDF5_PATH`, `AMS_CALIPER_PATH`, `AMS_AMQPCPP_PATH`, +`AMS_NLOHMANN_JSON_DIR`, `AMS_FMT_DIR`, `AMS_TL_EXPECTED_DIR`, +`AMS_CATCH2_DIR`, and GPU arch variables. + +On any other cluster, install dependencies yourself and pass package hints or +`CMAKE_PREFIX_PATH`; see `references/manual-deps.md`. + +`$SYS_TYPE` is a secondary LC signal: `toss_4_x86_64_ib` is Dane/CTS-1, while +`toss_4_x86_64_ib_cray` is Tuolumne/Tioga/El Capitan-class ROCm. + +## LC Python virtual environments + +On LC machines, do not create workflow Python environments with plain +`python3 -m venv myenv`. The AMS Spack environments use a Python external, and +a normal venv can miss Spack-provided Python packages. Use the repository +helper so the venv is based on the Spack Python and can see the Python packages +from the active AMS Spack environment. + +Run this from the AMS repository root after the Spack cache exports and +`scripts/gitlab/setup-env.sh`: + +```bash +host=$(hostname) +host=${host//[0-9]/} +python3 scripts/make-spack-venv.py \ + --env "/usr/workspace/AMS/ams-spack-environments/1.1/${host}/" \ + --output "venv-${host}" \ + --with-system-flux-python +source "venv-${host}/bin/activate" +``` + +Use this venv before installing or running Python workflow pieces, for example +with `pip install -e .` or with builds that enable `-DENABLE_WORKFLOW=On`. +The helper links the active system Flux Python bindings through a venv-local shim, +records `flux version`, `which flux`, the Python version, `flux.__file__`, and +the shim path, and warns on activation when the active Flux version differs. +Recreate the venv after system Flux changes. + +For LC workflow CMake builds, pass this so pip uses the prepared venv instead +of an isolated build environment: + +```bash +-DAMS_PIP_INSTALL_ARGS="--no-build-isolation" +``` + +Leave `AMS_INSTALL_FLUX_PYTHON=Off` for this path. The prepared venv supplies +system Flux Python. For container or non-LC builds that need pip-managed Flux +Python, enable `-DAMS_INSTALL_FLUX_PYTHON=On` with `-DENABLE_WORKFLOW=On`; the +workflow install target uses the `flux-python` optional dependency. + +## Current dependencies and options + +Always required: HDF5, libTorch, nlohmann_json, fmt, tl-expected, Threads, and +a C++17 compiler. `fmt` and `tl-expected` can fall back to `FetchContent`, so +network-free builds should pass local package hints. + +Current CMake options: + +| Option | Purpose | +| --- | --- | +| `ENABLE_MPI` | Enable MPI support. | +| `ENABLE_CUDA` | Enable CUDA support. | +| `ENABLE_HIP` | Enable HIP / ROCm support. | +| `ENABLE_CALIPER` | Enable Caliper profiling. | +| `ENABLE_RMQ` | Enable RabbitMQ database support. | +| `ENABLE_PERFFLOWASPECT` | Enable PerfFlowAspect profiling. | +| `ENABLE_WORKFLOW` | Install Python workflow drivers. | +| `ENABLE_TESTS` | Build Catch2 tests. | +| `AMS_ENABLE_DEBUG` | Enable verbose debug messages. | +| `AMS_INSTALL_FLUX_PYTHON` | Install the Python workflow package with `flux-python` when `ENABLE_WORKFLOW=On`. | +| `AMS_PIP_INSTALL_ARGS` | Extra arguments passed to `pip install` when `ENABLE_WORKFLOW=On`. | + +`ENABLE_CUDA` and `ENABLE_HIP` are mutually exclusive. + +Important CMake hints: + +| Package | CMake hint | LC export | +| --- | --- | --- | +| Torch | `Torch_DIR` | `$AMS_TORCH_PATH` | +| HDF5 | `HDF5_DIR` | `$AMS_HDF5_PATH` | +| Caliper | `caliper_DIR` | `$AMS_CALIPER_PATH` | +| amqp-cpp | `amqpcpp_DIR` | `$AMS_AMQPCPP_PATH` | +| nlohmann_json | `nlohmann_json_DIR` | `$AMS_NLOHMANN_JSON_DIR` | +| fmt | `AMS_FMT_DIR` | `$AMS_FMT_DIR` | +| tl-expected | `tl-expected_DIR` | `$AMS_TL_EXPECTED_DIR` | +| Catch2, when `ENABLE_TESTS=On` | `AMS_CATCH2_DIR` | `$AMS_CATCH2_DIR` | +| CUDA arch | `CMAKE_CUDA_ARCHITECTURES` | `$AMS_CUDA_ARCH` | +| Zlib, if needed | `ZLIB_ROOT` or `ZLIB_DIR` | `$AMS_ZLIB_PATH` | + +## Common configure shapes + +### Minimal CPU on LC + +```bash +export SPACK_DISABLE_LOCAL_CONFIG=true +export SPACK_SKIP_MODULES=1 +export SPACK_USER_CACHE_PATH=/tmp +export XDG_CACHE_HOME=/tmp +source scripts/gitlab/setup-env.sh + +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=On \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=On \ + -DTorch_DIR="$AMS_TORCH_PATH" \ + -DHDF5_DIR="$AMS_HDF5_PATH" \ + -DAMS_FMT_DIR="$AMS_FMT_DIR" \ + -Dnlohmann_json_DIR="$AMS_NLOHMANN_JSON_DIR" \ + -Dtl-expected_DIR="$AMS_TL_EXPECTED_DIR" +``` + +### Validated HIP / ROCm path on Tuolumne + +Use `amdclang` and `amdclang++` on LC Cray/ROCm systems. The validated +network-free smoke build used `ENABLE_TESTS=Off`; CTest discovery then reports +no tests. To build the Catch2 tests on LC, switch to `ENABLE_TESTS=On` and pass +`-DAMS_CATCH2_DIR="$AMS_CATCH2_DIR"` from `scripts/gitlab/setup-env.sh`. + +```bash +export SPACK_DISABLE_LOCAL_CONFIG=true +export SPACK_SKIP_MODULES=1 +export SPACK_USER_CACHE_PATH=/tmp +export XDG_CACHE_HOME=/tmp +source scripts/gitlab/setup-env.sh + +cmake -S . -B codex-build \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=On \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=On \ + -DCMAKE_C_COMPILER=amdclang \ + -DCMAKE_CXX_COMPILER=amdclang++ \ + -DENABLE_HIP=On \ + -DENABLE_MPI=On \ + -DENABLE_CALIPER=On \ + -DENABLE_RMQ=Off \ + -DENABLE_WORKFLOW=Off \ + -DENABLE_TESTS=Off \ + -DAMS_ENABLE_DEBUG=On \ + -DTorch_DIR="$AMS_TORCH_PATH" \ + -DHDF5_DIR="$AMS_HDF5_PATH" \ + -Dcaliper_DIR="$AMS_CALIPER_PATH" \ + -DAMS_FMT_DIR="$AMS_FMT_DIR" \ + -Dnlohmann_json_DIR="$AMS_NLOHMANN_JSON_DIR" \ + -Dtl-expected_DIR="$AMS_TL_EXPECTED_DIR" + +cmake --build codex-build -j +ctest --test-dir codex-build -N +``` + +Expected CTest discovery for that configuration is `Total Tests: 0`. + +### RabbitMQ variant + +Add `-DENABLE_RMQ=On` and pass `-Damqpcpp_DIR="$AMS_AMQPCPP_PATH"` on LC or the +manual amqp-cpp hint off LC. RabbitMQ support compiles the AMQP client; it does +not start or provision a broker. + +## Helper script + +`scripts/ams-configure.sh` assembles the common CMake line and maps LC +`AMS_*` exports to current hints: + +```bash +scripts/ams-configure.sh +scripts/ams-configure.sh --mpi --rmq +scripts/ams-configure.sh --hip --mpi --caliper +scripts/ams-configure.sh --mpi --tests +scripts/ams-configure.sh --workflow --install-flux-python +scripts/ams-configure.sh --mpi --rmq --dry-run +``` + +When `--tests` is enabled and `AMS_CATCH2_DIR` is set, the helper forwards the +LC Spack-provided Catch2 package with `-DAMS_CATCH2_DIR="$AMS_CATCH2_DIR"`. + +Manual CMake is still needed when forcing LC Cray/ROCm compilers unless the +script is later extended with compiler options. + +## Build and install + +```bash +cmake --build build -j "$(nproc)" +cmake --install build +``` + +## Common failure modes + +- **`Could NOT find Torch` / `HDF5` / `nlohmann_json` / `fmt` / + `tl-expected`**: on LC, source `scripts/gitlab/setup-env.sh` after setting + the Spack cache variables above. Off LC, provide the corresponding package + hint or add the install prefix to `CMAKE_PREFIX_PATH`. +- **`Could NOT find amqpcpp` / `libevent`**: only appears with + `-DENABLE_RMQ=On`; provide `amqpcpp_DIR` and libevent/OpenSSL locations. +- **Both CUDA and HIP set**: CMake hard-errors; choose one. +- **Catch2 configure tries GitHub**: `ENABLE_TESTS=On` first tries Catch2 + package discovery. On LC, source `scripts/gitlab/setup-env.sh` and pass + `-DAMS_CATCH2_DIR="$AMS_CATCH2_DIR"` to use the Spack-provided Catch2 + package. If no package hint is provided and discovery fails, CMake falls back + to `FetchContent` for Catch2 v3.11.0; in network-free environments, provide a + local/package Catch2 config directory or configure with + `-DENABLE_TESTS=Off`. +- **LC setup emits GitHub clone warnings**: if the cache variables are set, the + warning can be non-fatal when the AMS environment still exports all package + paths. Do not request network access unless the user explicitly asks. + +## Files in this skill + +- `references/manual-deps.md` - how to obtain each dependency on a non-LC + cluster and which CMake variable points at it. diff --git a/.agents/skills/ams-build/agents/openai.yaml b/.agents/skills/ams-build/agents/openai.yaml new file mode 100644 index 00000000..41bdd5d2 --- /dev/null +++ b/.agents/skills/ams-build/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "AMS Build" + short_description: "Skill to build the AMS library" + default_prompt: "Use $ams-build to build AMS" diff --git a/.agents/skills/ams-build/references/manual-deps.md b/.agents/skills/ams-build/references/manual-deps.md new file mode 100644 index 00000000..6b8a3857 --- /dev/null +++ b/.agents/skills/ams-build/references/manual-deps.md @@ -0,0 +1,102 @@ +# Providing AMS dependencies on a non-LC cluster + +Off LLNL Livermore Computing systems there is no shared AMS Spack environment, +so install dependencies yourself and point CMake at them. + +## Spack + +If your site has Spack and an `ams` package in a reachable repo: + +```bash +export SPACK_DISABLE_LOCAL_CONFIG=true +export SPACK_SKIP_MODULES=1 +export SPACK_USER_CACHE_PATH=/tmp +export XDG_CACHE_HOME=/tmp + +spack install ams +spack load ams +``` + +For active development, use `spack dev-build ams` from a working copy. If your +Spack does not have an `ams` package, install the dependencies individually: + +```bash +spack install nlohmann-json hdf5 py-torch fmt tl-expected +spack install caliper # if ENABLE_CALIPER +spack install mpi # if ENABLE_MPI, or use site MPI +spack install amqp-cpp openssl libevent # if ENABLE_RMQ +spack install catch2 # if ENABLE_TESTS +``` + +Then locate each install and pass its CMake hint. + +## Module system + +Many clusters expose some dependencies as modules: + +```bash +module load cmake gcc hdf5 cuda openmpi +``` + +LibTorch, nlohmann_json, fmt, and tl-expected are less commonly available as +modules. Use Spack or manual installs for anything the module stack does not +provide. + +## Manual dependencies and hints + +| Dependency | Required? | CMake hint | +| --- | --- | --- | +| `nlohmann_json` | yes | `nlohmann_json_DIR=/lib/cmake/nlohmann_json` | +| `fmt` | yes | `AMS_FMT_DIR=/lib/cmake/fmt` or `fmt_DIR=/lib/cmake/fmt` | +| `tl::expected` | yes | `tl-expected_DIR=/share/cmake/tl-expected` | +| HDF5 | yes | `HDF5_DIR=` | +| libTorch | yes | `Torch_DIR=/share/cmake/Torch` | +| MPI | if `ENABLE_MPI` | compiler wrappers or `CMAKE_PREFIX_PATH` | +| CUDA | if `ENABLE_CUDA` | CUDA toolkit on path plus `CMAKE_CUDA_ARCHITECTURES=` | +| ROCm/HIP | if `ENABLE_HIP` | `ROCM_PATH`, `hip_DIR`, or `CMAKE_PREFIX_PATH` | +| Caliper | if `ENABLE_CALIPER` | `caliper_DIR=/share/cmake/caliper` | +| amqp-cpp | if `ENABLE_RMQ` | `amqpcpp_DIR=/cmake` | +| OpenSSL | if `ENABLE_RMQ` | `OPENSSL_ROOT_DIR=` or `CMAKE_PREFIX_PATH` | +| libevent | if `ENABLE_RMQ` | `CMAKE_PREFIX_PATH` | +| Catch2 | if `ENABLE_TESTS` | `AMS_CATCH2_DIR=/lib/cmake/Catch2` or package config dir | +| PerfFlowAspect | if `ENABLE_PERFFLOWASPECT` | `perfflowaspect_DIR=/share` | +| Zlib | if static HDF5 needs it | `ZLIB_ROOT=` or `ZLIB_DIR=` | + +Put hand-built installs under one prefix when possible: + +```bash +export CMAKE_PREFIX_PATH=/opt/ams-deps:$CMAKE_PREFIX_PATH +``` + +Then pass explicit `*_DIR` hints only for packages CMake still cannot find. +LibTorch usually needs `Torch_DIR` because its CMake package is nested. + +## Network-free builds + +Configure with local package hints for fmt and tl-expected. Otherwise AMS may +try `FetchContent` from GitHub for those packages. + +`ENABLE_TESTS=On` first tries Catch2 package discovery. For network-free test +builds, provide `AMS_CATCH2_DIR=/lib/cmake/Catch2` or another directory +containing `Catch2Config.cmake` or `catch2-config.cmake`. If no package hint is +provided and discovery fails, AMS falls back to `FetchContent` from GitHub for +Catch2 v3.11.0; use `-DENABLE_TESTS=Off` when no local Catch2 package is +available. + +## Sanity check + +Configure a minimal CPU build first: + +```bash +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=On \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=On \ + -DTorch_DIR=/opt/libtorch/share/cmake/Torch \ + -DHDF5_DIR=/opt/hdf5 \ + -DAMS_FMT_DIR=/opt/ams-deps/lib/cmake/fmt \ + -Dnlohmann_json_DIR=/opt/ams-deps/lib/cmake/nlohmann_json \ + -Dtl-expected_DIR=/opt/ams-deps/share/cmake/tl-expected +``` + +If that succeeds, add MPI, GPU, RabbitMQ, and profiling flags one at a time. diff --git a/.agents/skills/changelog/SKILL.md b/.agents/skills/changelog/SKILL.md new file mode 100644 index 00000000..0523e8ec --- /dev/null +++ b/.agents/skills/changelog/SKILL.md @@ -0,0 +1,105 @@ +--- +name: changelog +description: > + Maintain a CHANGELOG.md in Keep a Changelog format: record new features, changes, + and fixes under an Unreleased section, and cut releases. Use whenever a + user-facing change ships or the user says "update the changelog", "add a + changelog entry", "record this feature/fix", "cut a release", "what changed + since ", or mentions release notes or a CHANGELOG. Works for versioned and + unversioned projects — fall back to dates plus merge-request or commit IDs when + there are no version numbers. Trigger on any notable change worth recording, even + without the word "changelog". +--- + +# Changelog + +Maintain `CHANGELOG.md` at the repo root in **Keep a Changelog** format: a +human-readable, reverse-chronological list of notable changes, grouped by release. +A changelog is for humans — it is not a `git log` dump. The format follows +Keep a Changelog (https://keepachangelog.com). + +## Structure + +```markdown +# Changelog + +## [Unreleased] + +### Added +- User-facing description of a new feature (#123). + +## [1.2.0] - 2026-07-08 + +### Added +- ... +### Fixed +- ... +``` + +- Newest first. Keep an `## [Unreleased]` section at the top as a staging area. +- Dates are ISO 8601 (`YYYY-MM-DD`). +- Group each change under one of six headings; omit headings with no entries: + - **Added** — new features. + - **Changed** — changes to existing behavior. + - **Deprecated** — features slated for removal. + - **Removed** — features now removed. + - **Fixed** — bug fixes. + - **Security** — vulnerability fixes. + +Initialize on first use: if `CHANGELOG.md` is missing, create it with the header +above and an empty `## [Unreleased]`. Never overwrite existing history. + +## Adding an entry + +Add to `## [Unreleased]` when a change merges — not at release time — so nothing +is forgotten. + +1. Pick the right group heading (create it under Unreleased if absent). +2. Write one bullet per notable change in **plain, user-facing language**: what + changed and why it matters, not the commit subject. Rewrite + "fix: handle keydown in modal (#412)" as "Fixed the dialog not closing on Escape." +3. Reference the source for an audit trail: the PR/merge-request ID (`#123`, `!57`) + or a short commit SHA when there is no PR. Optionally lead with a bold name: + `- **CSV export:** feedback entries can now be exported to CSV (#234).` +4. Skip internal churn (refactors, formatting, test-only changes) unless it is + notable to users or integrators. + +Drafting from history is fine — inspect `git log ..HEAD` or the merged +PRs — but always curate and rewrite into user-facing wording; never paste raw +commit messages. + +## Cutting a release + +Move the `## [Unreleased]` entries into a new release section, then leave an empty +`## [Unreleased]` at the top. The release identifier is flexible: + +- **Versioned (SemVer):** `## [1.4.0] - YYYY-MM-DD`. Bump MAJOR for breaking + changes, MINOR for new features, PATCH for fixes. +- **Unversioned:** use a dated header, annotated with the merge request or commit + that marks the release point: + ``` + ## [2026-07-08] — mr !57 + ## [2026-07-08] — a1b2c3d + ``` + +Optionally, add comparison links at the bottom so headers are clickable. On a git +host these are compare URLs — by tag (`compare/v1.3.0...v1.4.0`) or, for +unversioned projects, by commit/MR range (`compare/...`): + +``` +[Unreleased]: https:////compare/...HEAD +[1.4.0]: https:////compare/v1.3.0...v1.4.0 +``` + +## Conventions + +- Write for readers who have never seen the code: no internal ticket shorthand or + component names without explanation. +- Be brief, we do not want the changelog to be millions of lines long. Not more than + one sentence for each change. +- Make "update the changelog" part of the definition of done — the person shipping + the change writes the entry, since they have the context. +- Do not reconstruct a changelog from memory; derive it from git history and curate. +- Do not update past entries of a changelog without very good reason and notify the user. +- If an architecture wiki is maintained (see the `codebase-map` skill), a change + notable enough for the changelog that also alters structure should update both. \ No newline at end of file diff --git a/.agents/skills/changelog/agents/openai.yaml b/.agents/skills/changelog/agents/openai.yaml new file mode 100644 index 00000000..bb65b823 --- /dev/null +++ b/.agents/skills/changelog/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Changelog" + short_description: "Skill to build and maintain a CHANGELOG file" + default_prompt: "Use $changelog to build and/or update the CHANGELOG of a repository" diff --git a/.agents/skills/codebase-map/SKILL.md b/.agents/skills/codebase-map/SKILL.md new file mode 100644 index 00000000..0c7940e2 --- /dev/null +++ b/.agents/skills/codebase-map/SKILL.md @@ -0,0 +1,217 @@ +--- +name: codebase-map +description: > + Build and maintain a living architecture wiki for a codebase: one markdown + article per module/subsystem plus an index that maps how they fit together. + Use whenever the user wants to document, map, or reason about the code's + architecture — "map the codebase", "update the architecture wiki", "what does + module X do", "how does X connect to Y", "reconstruct the architecture", or + onboarding to an unfamiliar repo. Trigger on any mention of an "architecture + map", "code wiki", "module map", or keeping architecture docs in sync with the + code, even without those exact words. +--- + +# Codebase architecture map + +Maintain a living wiki describing a codebase's architecture. The **code is the +source of truth** — read it, never invent — and the wiki is the compiled, +human-readable map you own and keep in sync. + +Principle: the LLM writes and maintains the map; the human reads it and asks +questions. Keep it at the architecture level — module responsibilities, interfaces, +and how things fit together — not a restatement of every function (that is what +the code and API docs are for). + +## Layout + +Under `architecture/` at the repo root (configurable; point elsewhere if it +clashes with generated docs): + +- `architecture/index.md` — the map: a header recording the commit the map was + last refreshed at, a short system overview, a module list with one-line + summaries grouped by layer/package (each row noting the commit that article + reflects), and a system-level Mermaid diagram of the main module dependencies / + data flow (see Diagrams below). +- `architecture/.md` — one article per module or subsystem. +- `architecture/log.md` — append-only log of updates. + +One level of articles only. A "module" is a package/directory or a cohesive +subsystem, not a single file. + +### Module article format + +```markdown +# + +- Path(s): `src/foo/`, `src/foo_utils.py` +- Reflects commit: Updated: YYYY-MM-DD + +## Purpose +What this module is responsible for, in 1-3 sentences. + +## Key files +- `path` — what it does. + +## Public interface +Classes / functions / endpoints other modules or users call into. + +## Depends on +- Internal: [Other Module](other-module.md) +- External: notable third-party libraries. + +## Used by +Modules that depend on this one (link them). + +## Data flow / interactions +What comes in, what goes out, how it talks to other modules. Add a Mermaid diagram +here when it makes the flow clearer than prose (see Diagrams below). + +## Gotchas / invariants +Non-obvious constraints, footguns, assumptions. +``` + +Initialize on first run: if missing, create `architecture/` with `index.md` +(heading `# Architecture Map`) and `log.md` (heading `# Architecture Log`). +Never overwrite existing files. + +## Recording the commit SHA + +Every write records the commit the repo was at, so you can tell which state of the +codebase the content was built from. Capture it once at the start of a Map or +Verify run: + +```bash +git rev-parse --short HEAD # e.g. a1b2c3d +git status --porcelain # if non-empty, the working tree is dirty +``` + +If the working tree has uncommitted changes, append `-dirty` (e.g. `a1b2c3d-dirty`) +so the SHA is not mistaken for a clean checkout; `git describe --always --dirty` is +a one-shot equivalent. Record the same value in three places: + +- each article's **Reflects commit** field — the state that article was written from; +- the **index header**, e.g. `Map reflects commit: — YYYY-MM-DD`, set to the + commit of the most recent Map run; +- every **log entry** (see below). + +If the project is not a git checkout, use `unknown` and note it. + +## Diagrams (Mermaid) + +Use Mermaid so diagrams live inside the markdown, stay diffable, and render on +GitHub/GitLab and in most wiki viewers. Put each diagram in a fenced ` ```mermaid ` +block. (If the wiki is published through Sphinx, enable a Mermaid extension such as +`sphinxcontrib-mermaid`; MkDocs needs the `mermaid2` plugin.) + +Where diagrams go: +- **index.md** — one system-level diagram: modules as nodes, dependencies or data + flow as edges, grouped into layers with `subgraph`. This is the visual map. +- **Module article** — a focused diagram for that module: its data flow, an + important call sequence, or its lifecycle. Prefer one clear diagram over several. + +Pick the type by what you are showing: +- **Module/dependency map or data flow** → `flowchart` (label edges with what flows). +- **Runtime interaction across components** → `sequenceDiagram`. +- **Lifecycle of a stateful component** → `stateDiagram-v2`. +- **Type/class relationships**, when they clarify → `classDiagram`. + +Grounding and legibility (same discipline as the prose): +- Nodes are real modules/files/symbols; edges are real dependencies, calls, or data + flows. Do not invent structure to make a diagram look complete. +- Keep node IDs stable and human-readable so diffs stay small as the code changes. +- Stay at the architecture level. If a diagram exceeds ~15-20 nodes, scope it to one + concern or split it rather than drawing the whole repo at once. + +Example — module data flow in an article: + +```mermaid +flowchart LR + caller[API layer] -->|request| svc[This module] + svc -->|reads / writes| db[(Store)] + svc -->|calls| dep[Other Module] +``` + +## Scope (what to read) + +Keep the scan cheap and focused on the real codebase: read only committed source, +and never walk build or generated output. + +- Enumerate files with `git ls-files` instead of walking the filesystem. It lists + exactly the tracked files and automatically excludes untracked files and anything + in `.gitignore` (build dirs, caches, artifacts). For strictly the state committed + at HEAD — excluding staged-but-uncommitted files — use + `git ls-tree -r --name-only HEAD`. +- Skip build/generated/vendored trees even when a project commits them: `build/`, + `dist/`, `out/`, `target/`, `node_modules/`, `.venv/`, `venv/`, `__pycache__/`, + `*.egg-info/`, `site-packages/`, minified assets, generated code, large data + files, lockfiles, and binaries — none of these describe architecture. +- Do not read every file. Per module, read the entry points and public interface + (`__init__.py`, headers, `main`, service entrypoints) and sample a few + representative implementation files; skip tests/fixtures unless they are the + clearest description of behavior. Use `git ls-files ` to list a module's + files, then open only what you need. +- If the project is not a git checkout, fall back to a filesystem walk but apply + the same ignore list and honor `.gitignore`. + +## Map (build / refresh) + +Scan the repo — or a named module — and create or update articles. + +1. Identify modules from the directory/package structure and entry points + (`pyproject.toml`, `CMakeLists.txt`, `__init__.py`, `main`, service configs). + List files with `git ls-files` and stay within Scope — do not walk the tree. +2. Read enough of the actual code to describe each module accurately. **Every + claim must trace to a real file or symbol** — read the code rather than guess; + if you cannot verify something, say so instead of inventing it. +3. Write/update the article in the format above, including a Mermaid diagram where + it aids understanding (see Diagrams). Record the commit the article reflects + (see Recording the commit SHA) and today's date. +4. Cascade: if a module's public interface, dependencies, or responsibilities + changed, update the "Depends on" / "Used by" sections of affected articles and + the index. Refresh the Updated date on every article you materially change. +5. Update `index.md` — refresh its `Map reflects commit` header, the module list, + summaries, and the system-level Mermaid diagram — and append to `log.md`: + ``` + ## [YYYY-MM-DD] map | | + ``` + +## Query + +Answer architecture questions from the wiki. + +1. Read `index.md` to locate relevant articles, then read those articles. +2. Prefer wiki content; if it is thin or possibly stale, fall back to reading the + code and note that you did so. +3. Cite articles and the underlying files, e.g. `[Module](architecture/module.md)` + and `src/foo/bar.py`. Answer in the conversation; do not write files unless asked. + +## Verify (lint against the code) + +Check the map against the real codebase (enumerate files with `git ls-files`; see +Scope). + +Auto-fix when unambiguous: +- Article references a path/symbol that moved → update it if there is exactly one + clear match; otherwise report. +- A module directory exists with no article → add a stub entry to the index. +- An index entry points to a missing article → mark `[MISSING]`; do not delete. + +Report only (needs judgment): +- Articles whose recorded commit is behind `HEAD` **and** whose paths changed since + → flag as possibly stale. Check with + `git log --oneline ..HEAD -- ` (non-empty = changed). +- Described modules/interfaces that no longer exist, or new ones undocumented. +- Mermaid diagram nodes referencing modules/files that no longer exist, or missing + edges for dependencies now present in the code. +- Contradictions between articles; missing cross-references. + +Append to `log.md`: +``` +## [YYYY-MM-DD] verify | | issues, auto-fixed +``` + +## Conventions + +- Standard markdown with relative links between articles. +- Update the map in the same PR that changes a module's structure or interface — + the map is only useful if it stays trustworthy. \ No newline at end of file diff --git a/.agents/skills/codebase-map/agents/openai.yaml b/.agents/skills/codebase-map/agents/openai.yaml new file mode 100644 index 00000000..19744a61 --- /dev/null +++ b/.agents/skills/codebase-map/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Codebase Map" + short_description: "Skill to build and maintain a knowledge base of a code repository" + default_prompt: "Use $codebase-map to build and/or update the code knowledge database" diff --git a/.agents/skills/flux/SKILL.md b/.agents/skills/flux/SKILL.md new file mode 100644 index 00000000..c8491938 --- /dev/null +++ b/.agents/skills/flux/SKILL.md @@ -0,0 +1,97 @@ +--- +name: flux +description: > + Commands for running and monitoring jobs on a Flux (flux-framework) HPC + scheduler: submitting jobs, checking whether a job is running, reading job + output and exit codes, and cancelling jobs. Use this whenever a task involves + submitting, monitoring, inspecting, or cancelling work on a Flux cluster, or + running builds / test suites / model serving on compute nodes rather than the + login node. Trigger it on any mention of `flux submit`, `flux run`, + `flux jobs`, `flux batch`, Flux job IDs, or "is my job running / done" on an + HPC system where Flux is the main scheduler. +--- + +# Flux job management + +Flux is a resource manager / scheduler used on HPC clusters. Use it to run +anything heavy — builds, test suites, model serving, data processing — as a job +on **compute nodes**. Do **not** run heavy work directly on the login node. + +Queue names, bank/account, and node counts are cluster-specific. This skill +covers the generic commands only; get the per-machine values (queue, bank, +typical `-N`/`-n`) from the project's machine-specific setup notes before +submitting. In general, you want to use `--exclusive` when you request nodes. + +## Interactive allocations + +```bash +flux alloc -B --exclusive -N1 -q pdebug -t 1h # open a new shell with the allocated nodes +flux alloc -B --exclusive -N4 -t 8h +``` + +## Submitting jobs + +```bash +flux submit ./script.sh # queue a job; prints a job ID and returns immediately +flux run ./script.sh # run interactively and block until it finishes +flux batch ./batch.sh # submit a batch script (directives via '# flux:' lines) +flux submit -N2 -n8 ./script.sh # request 2 nodes, 8 tasks +flux submit --queue= --name= ./script.sh # target a queue, name the job +``` + +A batch script declares its resources with `# flux:` directive lines, e.g.: + +```bash +#!/bin/sh +# flux: -N4 -n16 +flux run -n16 ./my_step.sh +``` + +## Checking whether a job is running + +```bash +flux jobs # your active jobs (pending + running) +flux jobs -a # include completed / inactive jobs +flux jobs --filter=running # only running jobs (also: pending, inactive) +flux job last # job ID of your most recent submission +``` + +## Reading output and exit code + +```bash +flux submit --watch ./script.sh # stream output live as it runs +flux job attach $(flux job last) # attach to / print output of the last job +flux submit --output=job-{{id}}.out ./script.sh # write stdout to a file named per job ID +flux jobs --no-header -o '{status}:{returncode}' # status + exit code of one job +``` + +A non-zero `returncode` means the job failed — inspect its output before assuming +the step succeeded. Do not report a job as "passed" until you have confirmed both +that it is `inactive` and that its return code is `0`. + +## Cancelling jobs + +```bash +flux cancel # cancel a single job +flux cancel --all # cancel all of your jobs +flux cancel --states=RUN # cancel jobs in a given state +``` + +## Inspecting resources + +```bash +flux resource list # nodes available to you and their state +flux uptime # is the Flux instance up, and for how long +flux overlay status # health of the Flux overlay network +``` + +## Notes for agents + +- Prefer `flux submit` (non-blocking) for long work, then poll with `flux jobs`; + use `flux run` only for quick interactive checks. +- Always capture the job ID from `flux submit` (or use `flux job last`) so you can + check status and output later. +- Never launch a full model-serving stack or a long test run on the login node — + submit it as a Flux job. + +Full command reference: https://flux-framework.org/cheat-sheet/ diff --git a/.agents/skills/flux/agents/openai.yaml b/.agents/skills/flux/agents/openai.yaml new file mode 100644 index 00000000..63feb0b6 --- /dev/null +++ b/.agents/skills/flux/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Flux Resource Manager" + short_description: "Skill to use the Flux resource manager" + default_prompt: "Use $flux to let the agents interact with Flux" diff --git a/.claude/skills/ams-build b/.claude/skills/ams-build new file mode 120000 index 00000000..4cc67435 --- /dev/null +++ b/.claude/skills/ams-build @@ -0,0 +1 @@ +../../.agents/skills/ams-build/ \ No newline at end of file diff --git a/.claude/skills/changelog b/.claude/skills/changelog new file mode 120000 index 00000000..5dffccaa --- /dev/null +++ b/.claude/skills/changelog @@ -0,0 +1 @@ +../../.agents/skills/changelog/ \ No newline at end of file diff --git a/.claude/skills/codebase-map b/.claude/skills/codebase-map new file mode 120000 index 00000000..0c5253c1 --- /dev/null +++ b/.claude/skills/codebase-map @@ -0,0 +1 @@ +../../.agents/skills/codebase-map/ \ No newline at end of file diff --git a/.claude/skills/flux b/.claude/skills/flux new file mode 120000 index 00000000..9a71d17e --- /dev/null +++ b/.claude/skills/flux @@ -0,0 +1 @@ +../../.agents/skills/flux/ \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4e05bcb..e53264e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -280,7 +280,7 @@ jobs: export AMS_RMQ_CONFIG=$(cat ./rmq.json) cmake \ -DBUILD_SHARED_LIBS=On \ - -DCMAKE_PREFIX_PATH=$INSTALL_DIR \ + -DCMAKE_PREFIX_PATH=. \ -DCMAKE_INSTALL_PREFIX=./install \ -DCMAKE_BUILD_TYPE=Release \ -DENABLE_CALIPER=On \ @@ -289,6 +289,7 @@ jobs: -DENABLE_TESTS=On \ -DAMS_ENABLE_DEBUG=On \ -DENABLE_WORKFLOW=On \ + -DAMS_INSTALL_FLUX_PYTHON=On \ -DENABLE_RMQ=On \ -DTorch_DIR=$AMS_TORCH_PATH \ -Dcaliper_DIR=$AMS_CALIPER_PATH \ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..f407e5a2 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,303 @@ +# AGENTS.md + +This file provides guidance to LLMs (Claude Code, Codex etc) when working with code in this repository. + +## Project Overview + +AMS (Autonomous MultiScale Library) is a library to simplify machine learning surrogate model integration in HPC codes. +It enables scientific applications to use ML models as surrogates for expensive physics computations with uncertainty quantification. + +**Key components:** +- **AMSLib (C++)**: Core library providing the AMS API for scientific applications +- **AMSWorkflow (Python)**: Workflow orchestration components (AMSBroker, AMSTrain, AMSDeploy, AMSStore, AMSOrchestrator, AMSDBStage) +- **ML Integration**: PyTorch-based surrogate models with uncertainty quantification +- **Data Management**: HDF5 and optional RabbitMQ backends for storing/retrieving training data + + +## Executable Commands +- **Load Dependencies on Livermore Computing**: `source scripts/gitlab/setup-env.sh` +- **Test**: `ctest --test-dir build --output-on-failure` +- **Lint**: `clang-tidy -p build src/**/*.cpp` +- **Format**: `find src/ -regex '.*\.\(cpp\|hpp\|cu\|cuh\|c\|h\)' -exec clang-format -i {} \;` + +## Spack + +Before running Spack commands or sourcing `scripts/gitlab/setup-env.sh`, keep +Spack and XDG caches out of the home directory: + +```bash +export SPACK_DISABLE_LOCAL_CONFIG=true +export SPACK_SKIP_MODULES=1 +export SPACK_USER_CACHE_PATH=/tmp +export XDG_CACHE_HOME=/tmp +``` + +## Build System + +AMS uses [BLT](https://github.com/llnl/blt) to build. + +### CMake Configuration + +Standard build on Dane or on machine **without** GPU: +```bash +mkdir build && cd build +cmake \ + -DENABLE_HIP=Off \ + -DENABLE_CALIPER=On \ + -Dcaliper_DIR=$AMS_CALIPER_PATH \ + -DTorch_DIR=$AMS_TORCH_PATH \ + -DENABLE_MPI=On \ + -DHDF5_DIR="$AMS_HDF5_PATH" \ + -DENABLE_RMQ=On \ + -Damqpcpp_DIR=$AMS_AMQPCPP_PATH \ + -DENABLE_TESTS=On \ + -DAMS_CATCH2_DIR="$AMS_CATCH2_DIR" \ + -DENABLE_WORKFLOW=On \ + -DAMS_ENABLE_DEBUG=On \ + -DAMS_FMT_DIR="$AMS_FMT_DIR" \ + -Dnlohmann_json_DIR="$AMS_NLOHMANN_JSON_DIR" \ + -Dtl-expected_DIR="$AMS_TL_EXPECTED_DIR" \ + .. +make -j6 +make install +``` + +Standard build on Tioga/Tuo or on machine with AMD GPUs: + +```bash +export SPACK_DISABLE_LOCAL_CONFIG=true +export SPACK_SKIP_MODULES=1 +export SPACK_USER_CACHE_PATH=/tmp +export XDG_CACHE_HOME=/tmp +source scripts/gitlab/setup-env.sh + +cmake \ + -DBUILD_SHARED_LIBS=On \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=On \ + -DCMAKE_C_COMPILER=amdclang \ + -DCMAKE_CXX_COMPILER=amdclang++ \ + -DENABLE_HIP=On \ + -DENABLE_CALIPER=On \ + -Dcaliper_DIR=$AMS_CALIPER_PATH \ + -DTorch_DIR=$AMS_TORCH_PATH \ + -DENABLE_MPI=On \ + -DHDF5_DIR="$AMS_HDF5_PATH" \ + -DENABLE_RMQ=Off \ + -DENABLE_TESTS=Off \ + -DENABLE_WORKFLOW=Off \ + -DAMS_ENABLE_DEBUG=On \ + -DAMS_FMT_DIR="$AMS_FMT_DIR" \ + -Dnlohmann_json_DIR="$AMS_NLOHMANN_JSON_DIR" \ + -Dtl-expected_DIR="$AMS_TL_EXPECTED_DIR" \ + .. +``` + +If you want to build on a system with NVIDIA GPU you can just use `-DENABLE_CUDA=On` and `-DCMAKE_CUDA_ARCHITECTURES="$AMS_CUDA_ARCH"`. + +Make sure to set `-DBUILD_SHARED_LIBS=On -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=On` to both `On` (dynamic library) or both `Off` (static library). + +The most minimal build will need: +``` + -DTorch_DIR=$AMS_TORCH_PATH \ + -DHDF5_DIR="$AMS_HDF5_PATH" \ + -DAMS_FMT_DIR="$AMS_FMT_DIR" \ + -Dnlohmann_json_DIR="$AMS_NLOHMANN_JSON_DIR" \ + -Dtl-expected_DIR="$AMS_TL_EXPECTED_DIR" \ +``` + +Test builds also need `-DAMS_CATCH2_DIR="$AMS_CATCH2_DIR"` when using the LC +Spack-provided Catch2 package. + +For some builds you might need to specify the correct Zlib with `-DZLIB_DIR="$AMS_ZLIB_PATH"` (if `AMS_ZLIB_PATH` is defined). + +#### Compilers + +On Tuolumne and Tioga machine from LC, you must use `amdclang` as your compiler. +Make sure to add to the CMake command line `-DCMAKE_C_COMPILER="amdclang" -DCMAKE_CXX_COMPILER="amdclang++"`. + +Unless specified otherwise you can use `gcc` and let CMake discover the correct compiler. + +### CMake Options + +Required dependencies: +- HDF5, Torch, nlohmann_json, fmt, tl-expected, Threads, and a C++17 compiler. + +Optional features: +- `ENABLE_MPI`: Enable MPI support +- `ENABLE_CUDA` / `ENABLE_HIP`: GPU acceleration (mutually exclusive) +- `ENABLE_CALIPER`: Caliper profiling support +- `ENABLE_PERFFLOWASPECT`: PerfFlowAspect profiling (requires PFA-enabled clang/llvm) +- `ENABLE_RMQ`: RabbitMQ backend for distributed data management +- `AMS_ENABLE_DEBUG`: Enable verbose debug output (defines `LIBAMS_VERBOSE` and `__AMS_DEBUG__`) +- `ENABLE_TESTS`: Build test suite (uses Catch2) +- `ENABLE_WORKFLOW`: Install Python workflow drivers + +## Running Tests + +```bash +cd build +make test +# or for detailed output: +ctest --output-on-failure +# or +CTEST_OUTPUT_ON_FAILURE=1 make test +# or to run a specific test +ctest --output-on-failure -R "testName" +``` + +Tests use Catch2 framework (v3.11.0). On LC systems, +`scripts/gitlab/setup-env.sh` exports `AMS_CATCH2_DIR` for the Spack-provided +Catch2 package. Pass `-DAMS_CATCH2_DIR="$AMS_CATCH2_DIR"` with +`-DENABLE_TESTS=On` to avoid network access. + +If Catch2 package discovery fails and no `AMS_CATCH2_DIR` hint is provided, +CMake falls back to `FetchContent` from GitHub. In network-free environments, +provide a local/package Catch2 config directory with `AMS_CATCH2_DIR` or +configure with `-DENABLE_TESTS=Off`. + +Test directory structure: +- `tests/AMSlib/ams_interface/`: End-to-end AMS interface tests +- `tests/AMSlib/db/`: Database backend tests (HDF5) +- `tests/AMSlib/torch/`: PyTorch model inference tests +- `tests/AMSlib/wf/`: Workflow component tests +- `tests/AMSlib/models/`: Test model generation scripts + +## Code Architecture + +### Core AMS API (`src/AMSlib/`) + +Main API is defined in `src/AMSlib/include/AMS.h`: + +1. **Initialization**: `AMSInit()` / `AMSFinalize()` - Setup and teardown +2. **Model Registration**: `AMSRegisterAbstractModel()` - Register a surrogate model with domain name, threshold, and model path +3. **Executor Creation**: `AMSCreateExecutor()` - Create an executor for a registered model +4. **Execution**: `AMSExecute()` / `AMSCExecute()` - Execute with surrogate model or physics fallback +5. **Cleanup**: `AMSDestroyExecutor()` - Destroy executor + +**Key concepts:** +- **Uncertainty Quantification**: Models return `Tuple[[Tensor[N, ...], Tensor[N, 1]]` where second tensor contains uncertainty scores (lower = more confident) +- **Threshold**: Controls when to use surrogate vs physics (based on uncertainty) +- **Hybrid Execution**: Automatically falls back to physics computation when uncertainty exceeds threshold + +### Workflow System (`src/AMSlib/wf/`) + +The `AMSWorkflow` class orchestrates hybrid execution: + +- **Evaluation Pipeline**: + 1. Predict using surrogate model + 2. Check uncertainty against threshold + 3. For high-uncertainty samples: execute physics and store data + 4. For low-uncertainty samples: use ML predictions + +- **Model Updates**: Supports dynamic model updates via RabbitMQ +- **Data Storage**: Stores training data to HDF5 or RabbitMQ backends +- **Distributed Execution**: MPI-aware for parallel processing + +Key files: +- `src/AMSlib/wf/workflow.hpp`: Main workflow class +- `src/AMSlib/wf/action.hpp`: Action concept for data transformations +- `src/AMSlib/wf/eval_context.hpp`: Evaluation context management +- `src/AMSlib/wf/basedb.hpp`: Database backend interface + +### ML Components (`src/AMSlib/ml/`) + +- `surrogate.hpp`: Surrogate model wrapper around PyTorch models +- `Model.hpp`: PyTorch model loading and inference +- `AbstractModel.hpp`: Abstract interface for ML models + +### Python Workflow (`src/AMSWorkflow/`) + +Components for outer training/deployment loop: +- `AMSBroker`: Message broker for distributed coordination +- `AMSTrain`: Training orchestration +- `AMSDeploy`: Model deployment +- `AMSStore`: Data storage management +- `AMSOrchestrator`: Workflow orchestration +- `AMSDBStage`: Database staging + +Install with: `pip install -e .` from project root + +## Code Style +- **Standards**: C++17, strictly. Prefer standard library over external dependencies where possible. +- **Ownership**: Use smart pointers or value semantics. NO raw `new`/`delete`. +- **Safety**: Use `tl::expected` for error handling; avoid raw exceptions in performance-critical paths. +- **Headers**: Prefer `#pragma once` over traditional include guards. +- **Formatting**: Strictly follow the project's `.clang-format`. Run it after every file modification. +- **Memory leaks**: Test the code with Valgrind if you suspect memroy leaks + +Format Python code with: +```bash +ruff format +``` + +## Development Workflow + +**Main branch**: `develop` (not `main`) + +**Creating PRs**: Always target `develop` as the base branch. + +**Python requirements**: Tests require `h5py` installed (`pip install h5py`) + +## Installation + +Recommended: Use Spack for dependency management: +```bash +spack install ams +# or for development: +spack dev-build ams +``` + +See INSTALL.md for manual installation details. + +## Repository Structure + +``` +src/ +├── AMSlib/ # C++ library +│ ├── include/ # Public API headers +│ ├── ml/ # ML model components +│ └── wf/ # Workflow system +└── AMSWorkflow/ # Python workflow tools + ├── ams/ # Python package + └── ams_wf/ # Workflow drivers + +tests/ +├── AMSlib/ # C++ tests (Catch2) +└── AMSWorkflow/ # Python tests + +examples/ +├── ideal_gas/ # Example: ideal gas law application +└── bnm_opt/ # Example: optimization application + +cmake/ # CMake modules +docs/ # Sphinx documentation +``` + +## Common Patterns + +**Type aliases in AMS.h:** +- `AMSExecutor`: Executor handle (int64_t) +- `AMSCAbstrModel`: Model handle (int) +- `DomainLambda`: C++ lambda callback type +- `DomainCFn`: C function pointer callback type + +**Device support:** +- AMS uses custom resource manager for memory management across CPU/GPU +- Set allocator: `AMSSetAllocator(AMSResourceType resource, const char* name)` +- Supported resources: Host, Device (CUDA/HIP) + +**Database configuration:** +- File system DB: `AMSConfigureFSDatabase(AMSDBType db_type, const char* db_path)` +- RabbitMQ DB: Enable with `-DENABLE_RMQ=On` at build time + +## Boundaries & Guardrails +- **Always**: Run tests that are impacted by your changes. For example, to re-run the + core tests: `ctest --output-on-failure -R "CORE::"` or `ctest --output-on-failure -R "CORE::TENSOR_INT"` + to re-run one specific test. +- **Always**: Run `./scripts/run-code-quality.sh --staged --clang-format --ruff --fix` before testing your changes +- **Ask First**: Before adding new external dependencies to `CMakeLists.txt`. +- **Never**: Use C-style casts; instead use `static_cast` or `reinterpret_cast`. +- **Never**: Over-engineer solutions with superfluous safety checking. +- **Never**: Use modifying git commands unless explicitly asked to by the user. +- **Never**: Run all the tests with `ctest` unless explicitly asked to by the user or before + commiting to a branch. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..d9763d28 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +## [Unreleased] + +### Changed + +- Workflow environments can now use active system Flux Python bindings instead + of installing `flux-python` through default AMS Python dependencies. +- Added `AMS_INSTALL_FLUX_PYTHON` so non-system workflow builds can opt into + installing the `flux-python` optional dependency through CMake. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 684b4de8..d65f0304 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,10 +44,15 @@ option(ENABLE_WORKFLOW "Install python drivers used by the outer workflow" OFF) option(ENABLE_RMQ "Use RabbitMQ as a database back end" OFF) option(ENABLE_PERFFLOWASPECT "Use PerfFlowAspect for profiling" OFF) option(AMS_ENABLE_DEBUG "Enable verbose AMS messages" OFF) +option(AMS_INSTALL_FLUX_PYTHON + "Install AMS Workflow Python package with the flux-python optional dependency" + OFF) option(AMS_DEFER_STATIC_TPL_RESOLUTION "Defer selected static TPL resolution to downstream final links when building shared AMS" OFF) +set(AMS_PIP_INSTALL_ARGS "" CACHE STRING + "Additional arguments passed to pip install when ENABLE_WORKFLOW is On") set(AMS_THIRDPARTY_DIR "${PROJECT_SOURCE_DIR}/thirdparty" CACHE PATH "Path to AMS-local third-party dependencies") set(AMS_FMT_DIR "" CACHE PATH diff --git a/INSTALL.md b/INSTALL.md index 2240b226..ef7535fb 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,85 +1,342 @@ # Setup and Build -AMSLib depends on the following packages: -* UMPIRE (Mandatory) -* MPI (Mandatory) -* CALIPER (Optional) -* FAISS (Optional) -* MFEM (Optional) -* PY-TORCH (Optional) -* MFEM (Optional) -* REDIS (Optional) -* HDF5 (Optional) -* CUDA (Optional) -* ADIAK (Optional) - -## Spack Installation - -AMS depends on multiple complex external libraries, our preferred and suggested mechanism to install AMS is through [spack](https://github.com/spack/spack) as follows: +AMSLib is a CMake (>= 3.18, C++17) project. The build itself is +straightforward; the effort is in providing dependencies so that +`find_package` succeeds. + +## Dependencies + +**Always required:** + +* HDF5 (C component) +* LibTorch (PyTorch C++ API) +* `nlohmann_json` +* `{fmt}` +* `tl::expected` +* A C++17 compiler and a threading library + +`fmt` and `tl::expected` can fall back to CMake `FetchContent` if package +discovery fails. For network-free builds, provide local packages through the +hint variables below. + +**Optional, enabled per build flag:** + +| Dependency | Enabled by | +| --- | --- | +| MPI | `ENABLE_MPI` | +| CUDA (NVIDIA) | `ENABLE_CUDA` | +| HIP / ROCm (AMD) | `ENABLE_HIP` | +| Caliper | `ENABLE_CALIPER` | +| amqp-cpp, OpenSSL, libevent | `ENABLE_RMQ` | +| PerfFlowAspect | `ENABLE_PERFFLOWASPECT` | +| Python workflow drivers | `ENABLE_WORKFLOW` | +| Catch2 tests | `ENABLE_TESTS` | + +`ENABLE_CUDA` and `ENABLE_HIP` are mutually exclusive. + +## Build Options + +| Option | Default | Description | +| --- | --- | --- | +| `ENABLE_MPI` | `OFF` | Enable MPI support. | +| `ENABLE_CUDA` | `OFF` | Enable CUDA support for NVIDIA GPUs. | +| `ENABLE_HIP` | `OFF` | Enable HIP support for AMD GPUs. | +| `ENABLE_CALIPER` | `OFF` | Enable Caliper profiling. | +| `ENABLE_TESTS` | `OFF` | Build the Catch2-based test suite. | +| `ENABLE_WORKFLOW` | `OFF` | Install the Python drivers used by the outer workflow. | +| `ENABLE_RMQ` | `OFF` | Enable the RabbitMQ database backend. | +| `ENABLE_PERFFLOWASPECT` | `OFF` | Enable PerfFlowAspect profiling. | +| `AMS_ENABLE_DEBUG` | `OFF` | Enable verbose AMS debug messages. | +| `AMS_INSTALL_FLUX_PYTHON` | `OFF` | Install the Python workflow package with the `flux-python` optional dependency when `ENABLE_WORKFLOW=On`. | +| `BUILD_SHARED_LIBS` | CMake default | Build shared libraries when `ON`; static when `OFF`. | +| `AMS_DEFER_STATIC_TPL_RESOLUTION` | `OFF` | Defer selected static TPL resolution to downstream final links when building shared AMS. | +| `AMS_PIP_INSTALL_ARGS` | empty | Extra arguments passed to `pip install` when `ENABLE_WORKFLOW=On`. | + +When building shared libraries, use +`-DBUILD_SHARED_LIBS=On -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=On`. For a static +build, set both to `Off`. + +## Dependency Location Hints + +When a dependency is installed outside default search paths, point CMake at its +config package or add its prefix to `CMAKE_PREFIX_PATH`. On LC systems, +`scripts/gitlab/setup-env.sh` exports the `AMS_*` variables shown below. + +| Package | CMake variable | LC export | +| --- | --- | --- | +| libTorch | `Torch_DIR` | `$AMS_TORCH_PATH` | +| HDF5 | `HDF5_DIR` | `$AMS_HDF5_PATH` | +| Caliper | `caliper_DIR` | `$AMS_CALIPER_PATH` | +| amqp-cpp | `amqpcpp_DIR` | `$AMS_AMQPCPP_PATH` | +| nlohmann_json | `nlohmann_json_DIR` | `$AMS_NLOHMANN_JSON_DIR` | +| fmt | `AMS_FMT_DIR` | `$AMS_FMT_DIR` | +| tl-expected | `tl-expected_DIR` | `$AMS_TL_EXPECTED_DIR` | +| Catch2, when `ENABLE_TESTS=On` | `AMS_CATCH2_DIR` | `$AMS_CATCH2_DIR` | +| CUDA arch | `CMAKE_CUDA_ARCHITECTURES` | `$AMS_CUDA_ARCH` | +| HIP arch | `CMAKE_HIP_ARCHITECTURES` | auto-detected or `$AMS_HIP_ARCH` | +| Zlib, when needed by static HDF5 | `ZLIB_ROOT` or `ZLIB_DIR` | `$AMS_ZLIB_PATH` | + +## Spack and LC Setup + +Before running Spack commands or sourcing the LC setup script in restricted +environments, keep Spack and XDG caches out of the home directory: + +```bash +export SPACK_DISABLE_LOCAL_CONFIG=true +export SPACK_SKIP_MODULES=1 +export SPACK_USER_CACHE_PATH=/tmp +export XDG_CACHE_HOME=/tmp +``` + +On LLNL Livermore Computing systems, source the repository setup script from +the repository root: + +```bash +source scripts/gitlab/setup-env.sh +``` + +The script loads the appropriate compiler, MPI, and ROCm modules, activates the +AMS Spack environment, and exports dependency locations used by CMake. + +### LC Workflow Python Environments + +On LC systems, do not create AMS Workflow environments with plain +`python3 -m venv`. The AMS Spack environments use a Python external and LC +Flux provides the compatible `flux` Python bindings for the active Flux +installation. Create the venv through the repository helper so it links both +Spack Python packages and a venv-local shim for system Flux Python: + +```bash +host=$(hostname) +host=${host//[0-9]/} +python3 scripts/make-spack-venv.py \ + --env "/usr/workspace/AMS/ams-spack-environments/1.1/${host}/" \ + --output "venv-${host}" \ + --with-system-flux-python +source "venv-${host}/bin/activate" +``` + +The helper writes system Flux metadata into the venv and warns on activation if the +active `flux version` differs from the recorded one. The shim exposes Flux +without putting unrelated LC Python packages ahead of AMS Spack packages. +Recreate the venv after system Flux changes. For LC workflow CMake builds, pass +`-DAMS_PIP_INSTALL_ARGS="--no-build-isolation"` so pip uses the prepared venv +instead of an isolated build environment. Do not enable +`AMS_INSTALL_FLUX_PYTHON` for this path; Flux Python is supplied by the +prepared system-backed venv, not by pip. + +Design choices for system Flux Python: + +- AMS does not install `flux-python` as a default Python dependency on LC. + The Python bindings must match the active LC `flux` command and runtime, + which can change independently of the AMS Spack environment. +- `scripts/make-spack-venv.py --with-system-flux-python` discovers the active + Flux binding with the venv Python first, then falls back to `flux python` + when Flux keeps its bindings outside the default Python import path. +- The helper writes `system-flux-python.pth` in the venv `site-packages` directory. + This file points at a venv-local `system_flux_python` shim, not at the whole LC + Python `site-packages` tree, so packages such as LC's system `numpy` do not + shadow the AMS Spack Python packages. +- The helper writes `system-flux-python.json` beside the `.pth` file. This records + `which flux`, `flux version`, the venv Python version, the `flux python` + version, the original `flux.__file__`, the original system Flux Python path, and + the shim path. +- The activation hook compares the current `flux version` against the recorded + metadata and warns when they differ. Treat that warning as a signal to + recreate the venv with `--with-system-flux-python`. + +### Non-System Flux Workflow Installs + +When building AMS Workflow in an environment that does not provide compatible +system Flux Python bindings, enable the workflow Flux extra: + +```bash +cmake -S . -B build \ + -DENABLE_WORKFLOW=On \ + -DAMS_INSTALL_FLUX_PYTHON=On +``` + +With this flag, the `PyAMS` target runs pip against the generated package tree +as `pip install [flux]`, which installs the `flux-python` optional +dependency declared by `pyproject.toml`. Use this for container or non-LC +builds that rely on pip-managed Flux Python. Leave it off when using +`scripts/make-spack-venv.py --with-system-flux-python`. + +## Convenience Configure Script + +`scripts/ams-configure.sh` assembles a standard CMake command and maps LC +`AMS_*` exports to the current `-D*_DIR` hints: ```bash -spack install ams +scripts/ams-configure.sh +scripts/ams-configure.sh --mpi --rmq +scripts/ams-configure.sh --hip --mpi --caliper +scripts/ams-configure.sh --mpi --tests +scripts/ams-configure.sh --workflow --install-flux-python +scripts/ams-configure.sh --mpi --rmq --dry-run ``` -If you are a developer and would like to extend AMS you can do so by using the `spack dev-build' command. -For more instructions look [here](https://spack-tutorial.readthedocs.io/en/lanl19/tutorial_developer_workflows.html) +When `--tests` is enabled and `AMS_CATCH2_DIR` is set, the helper forwards +`-DAMS_CATCH2_DIR="$AMS_CATCH2_DIR"` so LC builds use the Spack-provided +Catch2 package. +Manual CMake is still the clearest path when forcing LC Cray/ROCm compilers +such as `amdclang` and `amdclang++`, unless the helper script is extended with +compiler options. -## Manual cmake installation +## Manual CMake Installation -Below you can find a `cmake` command to configure to configure AMS, build and install it. +This representative LC command enables MPI, Caliper, RabbitMQ, and debug +messages with shared libraries: ```bash -$ mkdir build; cd build -$ cmake \ - -DWITH_DB=On -DWITH_RMQ=On \ - -Damqpcpp_DIR=$AMS_AMQPCPP_PATH \ +export SPACK_DISABLE_LOCAL_CONFIG=true +export SPACK_SKIP_MODULES=1 +export SPACK_USER_CACHE_PATH=/tmp +export XDG_CACHE_HOME=/tmp +source scripts/gitlab/setup-env.sh + +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=On \ - -DCMAKE_PREFIX_PATH=$INSTALL_DIR \ - -DWITH_CALIPER=On \ - -DWITH_HDF5=On \ - -DWITH_EXAMPLES=On \ - -DHDF5_Dir=$AMS_HDF5_PATH \ - -DCMAKE_INSTALL_PREFIX=./install \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=On \ + -DENABLE_MPI=On \ + -DENABLE_CALIPER=On \ + -DENABLE_RMQ=On \ + -DENABLE_WORKFLOW=On \ + -DAMS_PIP_INSTALL_ARGS="--no-build-isolation" \ + -DAMS_ENABLE_DEBUG=On \ + -DTorch_DIR="$AMS_TORCH_PATH" \ + -DHDF5_DIR="$AMS_HDF5_PATH" \ + -Dcaliper_DIR="$AMS_CALIPER_PATH" \ + -Damqpcpp_DIR="$AMS_AMQPCPP_PATH" \ + -DAMS_FMT_DIR="$AMS_FMT_DIR" \ + -Dnlohmann_json_DIR="$AMS_NLOHMANN_JSON_DIR" \ + -Dtl-expected_DIR="$AMS_TL_EXPECTED_DIR" + +cmake --build build -j 6 +cmake --install build +``` + +## Example Builds + +### 1. Minimal CPU Build on LC + +```bash +export SPACK_DISABLE_LOCAL_CONFIG=true +export SPACK_SKIP_MODULES=1 +export SPACK_USER_CACHE_PATH=/tmp +export XDG_CACHE_HOME=/tmp +source scripts/gitlab/setup-env.sh + +cmake -S . -B build \ -DCMAKE_BUILD_TYPE=Release \ - -DWITH_CUDA=On \ - -DUMPIRE_DIR=$AMS_UMPIRE_PATH \ - -DMFEM_DIR=$AMS_MFEM_PATH \ - -DWITH_FAISS=On \ - -DWITH_MPI=On \ - -DWITH_TORCH=On \ - -DWITH_TESTS=Off \ - -DTorch_DIR=$AMS_TORCH_PATH \ - -DFAISS_DIR=$AMS_FAISS_PATH \ - -DAMS_CUDA_ARCH=${AMS_CUDA_ARCH} \ - -DWITH_AMS_DEBUG=On \ - ../ - -$ make -j6 -$ make install + -DBUILD_SHARED_LIBS=On \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=On \ + -DTorch_DIR="$AMS_TORCH_PATH" \ + -DHDF5_DIR="$AMS_HDF5_PATH" \ + -DAMS_FMT_DIR="$AMS_FMT_DIR" \ + -Dnlohmann_json_DIR="$AMS_NLOHMANN_JSON_DIR" \ + -Dtl-expected_DIR="$AMS_TL_EXPECTED_DIR" ``` -Most of the compile time options are optional. +### 2. HIP / ROCm Build on Tuolumne or Tioga -## Building AMS with PerfFlowAspect +Use `amdclang` and `amdclang++` on LC Cray/ROCm machines. -To built AMS with [PFA](https://github.com/flux-framework/PerfFlowAspect) support you first need to install a PFA clang/llvm version and add it to `$PATH`. Next to configure, built and install perform the following: +```bash +export SPACK_DISABLE_LOCAL_CONFIG=true +export SPACK_SKIP_MODULES=1 +export SPACK_USER_CACHE_PATH=/tmp +export XDG_CACHE_HOME=/tmp +source scripts/gitlab/setup-env.sh +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=On \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=On \ + -DCMAKE_C_COMPILER=amdclang \ + -DCMAKE_CXX_COMPILER=amdclang++ \ + -DENABLE_HIP=On \ + -DENABLE_MPI=On \ + -DENABLE_CALIPER=On \ + -DENABLE_RMQ=Off \ + -DENABLE_WORKFLOW=Off \ + -DENABLE_TESTS=Off \ + -DAMS_ENABLE_DEBUG=On \ + -DTorch_DIR="$AMS_TORCH_PATH" \ + -DHDF5_DIR="$AMS_HDF5_PATH" \ + -Dcaliper_DIR="$AMS_CALIPER_PATH" \ + -DAMS_FMT_DIR="$AMS_FMT_DIR" \ + -Dnlohmann_json_DIR="$AMS_NLOHMANN_JSON_DIR" \ + -Dtl-expected_DIR="$AMS_TL_EXPECTED_DIR" + +cmake --build build -j +ctest --test-dir build -N ``` -$ cd $CODE_ROOT/setup -$ mkdir build; cd build -$ cmake \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_C_COMPILER=clang \ - -DMFEM_DIR=$AMS_MFEM_PATH \ - -DUMPIRE_DIR=$AMS_UMPIRE_PATH \ - -DWITH_CUDA=On \ - -DWITH_CALIPER=On \ - -DWITH_TORCH=On -DTorch_DIR=$AMS_TORCH_PATH \ - -DWITH_FAISS=On -DFAISS_DIR=$AMS_FAISS_PATH \ - -DWITH_PERFFLOWASPECT=On \ - -Dperfflowaspect_DIR=$AMS_PFA_PATH/share \ - ../ -$ make -j6 + +The Tuolumne validation used this shape with `ENABLE_TESTS=Off`; CTest +reported `Total Tests: 0`. To build the Catch2 tests on LC, switch to +`-DENABLE_TESTS=On` and add `-DAMS_CATCH2_DIR="$AMS_CATCH2_DIR"`. + +### 3. CUDA + Caliper on LC + +```bash +export SPACK_DISABLE_LOCAL_CONFIG=true +export SPACK_SKIP_MODULES=1 +export SPACK_USER_CACHE_PATH=/tmp +export XDG_CACHE_HOME=/tmp +source scripts/gitlab/setup-env.sh + +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=On \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=On \ + -DENABLE_CUDA=On \ + -DCMAKE_CUDA_ARCHITECTURES="$AMS_CUDA_ARCH" \ + -DENABLE_CALIPER=On \ + -Dcaliper_DIR="$AMS_CALIPER_PATH" \ + -DTorch_DIR="$AMS_TORCH_PATH" \ + -DHDF5_DIR="$AMS_HDF5_PATH" \ + -DAMS_FMT_DIR="$AMS_FMT_DIR" \ + -Dnlohmann_json_DIR="$AMS_NLOHMANN_JSON_DIR" \ + -Dtl-expected_DIR="$AMS_TL_EXPECTED_DIR" +``` + +### 4. Minimal CPU Build with Manual Dependencies + +```bash +export CMAKE_PREFIX_PATH=/opt/ams-deps:$CMAKE_PREFIX_PATH + +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=On \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=On \ + -DTorch_DIR=/opt/libtorch/share/cmake/Torch \ + -DHDF5_DIR=/opt/hdf5 \ + -DAMS_FMT_DIR=/opt/ams-deps/lib/cmake/fmt \ + -Dnlohmann_json_DIR=/opt/ams-deps/lib/cmake/nlohmann_json \ + -Dtl-expected_DIR=/opt/ams-deps/share/cmake/tl-expected ``` +Add feature flags such as `-DENABLE_RMQ=On`, `-DENABLE_MPI=On`, or +`-DENABLE_HIP=On` and the corresponding package hints as needed. + +## Tests and Catch2 + +`ENABLE_TESTS=On` enters `tests/AMSlib/CMakeLists.txt` and looks for a Catch2 +CMake package. On LC systems, `scripts/gitlab/setup-env.sh` exports +`AMS_CATCH2_DIR` pointing at the Spack-provided package directory; pass it with +`-DAMS_CATCH2_DIR="$AMS_CATCH2_DIR"` to avoid network access. + +If Catch2 package discovery fails and no `AMS_CATCH2_DIR` hint is provided, +CMake falls back to `FetchContent` from GitHub for Catch2 v3.11.0. For +network-free builds outside LC, provide a local/package Catch2 config directory +with `AMS_CATCH2_DIR`, or configure with `-DENABLE_TESTS=Off`. + +## Build and Install + +```bash +cmake --build build -j "$(nproc)" +cmake --install build +``` diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 694b63d9..b617156b 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -1,66 +1,218 @@ Installation ============ -This page provides detailed instructions for installing AMS on various systems. +AMS is built with CMake and C++17. The main build task is making sure CMake can +find the required packages. Requirements ------------ -AMS has several dependencies that need to be installed before building: +Core dependencies: + +* CMake >= 3.18 +* C++17 compatible compiler +* HDF5 +* LibTorch +* nlohmann_json +* fmt +* tl-expected + +Optional dependencies: + +* CUDA for NVIDIA GPU support +* HIP / ROCm for AMD GPU support +* MPI for distributed execution +* Caliper for profiling +* amqp-cpp, OpenSSL, and libevent for RabbitMQ support +* Catch2 for tests + +Current CMake Options +--------------------- + +Use the current ``ENABLE_*`` options: + +.. list-table:: + :header-rows: 1 + + * - Option + - Purpose + * - ``ENABLE_MPI`` + - Enable MPI support. + * - ``ENABLE_CUDA`` + - Enable CUDA support. + * - ``ENABLE_HIP`` + - Enable HIP / ROCm support. + * - ``ENABLE_CALIPER`` + - Enable Caliper profiling. + * - ``ENABLE_RMQ`` + - Enable RabbitMQ database support. + * - ``ENABLE_WORKFLOW`` + - Install Python workflow drivers. + * - ``ENABLE_TESTS`` + - Build Catch2-based tests. + * - ``AMS_ENABLE_DEBUG`` + - Enable verbose AMS debug messages. + * - ``AMS_INSTALL_FLUX_PYTHON`` + - Install the Python workflow package with the ``flux-python`` optional dependency when ``ENABLE_WORKFLOW=On``. + * - ``AMS_PIP_INSTALL_ARGS`` + - Extra arguments passed to ``pip install`` when ``ENABLE_WORKFLOW=On``. + +``ENABLE_CUDA`` and ``ENABLE_HIP`` are mutually exclusive. + +Dependency Hints +---------------- + +When packages are outside default CMake search paths, pass explicit hints: + +.. list-table:: + :header-rows: 1 + + * - Package + - CMake variable + - LC setup export + * - LibTorch + - ``Torch_DIR`` + - ``$AMS_TORCH_PATH`` + * - HDF5 + - ``HDF5_DIR`` + - ``$AMS_HDF5_PATH`` + * - Caliper + - ``caliper_DIR`` + - ``$AMS_CALIPER_PATH`` + * - amqp-cpp + - ``amqpcpp_DIR`` + - ``$AMS_AMQPCPP_PATH`` + * - nlohmann_json + - ``nlohmann_json_DIR`` + - ``$AMS_NLOHMANN_JSON_DIR`` + * - fmt + - ``AMS_FMT_DIR`` + - ``$AMS_FMT_DIR`` + * - tl-expected + - ``tl-expected_DIR`` + - ``$AMS_TL_EXPECTED_DIR`` + * - Catch2, when ``ENABLE_TESTS=On`` + - ``AMS_CATCH2_DIR`` + - ``$AMS_CATCH2_DIR`` + +LC HIP / ROCm Build +------------------- + +On Tuolumne, Tioga, and similar LC ROCm systems, keep Spack caches out of the +home directory, source the LC setup script, use ``amdclang``/``amdclang++``, +and pass the dependency exports: -Core Dependencies -~~~~~~~~~~~~~~~~~ +.. code-block:: bash -* **CMake** >= 3.25 -* **C++17** compatible compiler (GCC >= 8.5) -* **Python** >= 3.10 (for Python bindings) -* **PyTorch** >= 2.0 (for ML model support) -* **HDF5** (for data storage) + export SPACK_DISABLE_LOCAL_CONFIG=true + export SPACK_SKIP_MODULES=1 + export SPACK_USER_CACHE_PATH=/tmp + export XDG_CACHE_HOME=/tmp + source scripts/gitlab/setup-env.sh -Optional Dependencies -~~~~~~~~~~~~~~~~~~~~~ + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=On \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=On \ + -DCMAKE_C_COMPILER=amdclang \ + -DCMAKE_CXX_COMPILER=amdclang++ \ + -DENABLE_HIP=On \ + -DENABLE_MPI=On \ + -DENABLE_CALIPER=On \ + -DENABLE_RMQ=Off \ + -DENABLE_WORKFLOW=Off \ + -DENABLE_TESTS=Off \ + -DAMS_ENABLE_DEBUG=On \ + -DTorch_DIR="$AMS_TORCH_PATH" \ + -DHDF5_DIR="$AMS_HDF5_PATH" \ + -Dcaliper_DIR="$AMS_CALIPER_PATH" \ + -DAMS_FMT_DIR="$AMS_FMT_DIR" \ + -Dnlohmann_json_DIR="$AMS_NLOHMANN_JSON_DIR" \ + -Dtl-expected_DIR="$AMS_TL_EXPECTED_DIR" + + cmake --build build -j + ctest --test-dir build -N + +The validated HIP configuration used ``ENABLE_TESTS=Off`` and CTest reported +``Total Tests: 0``. To build Catch2 tests on LC, switch to +``-DENABLE_TESTS=On`` and add ``-DAMS_CATCH2_DIR="$AMS_CATCH2_DIR"``. + +LC Workflow Python Environments +------------------------------- + +On LC systems, do not create AMS Workflow environments with plain +``python3 -m venv``. The AMS Spack environments use a Python external, and LC +Flux provides the compatible ``flux`` Python bindings for the active Flux +installation. Create the venv through the repository helper so it links both +Spack Python packages and a venv-local shim for system Flux Python: -* **CUDA** >= 11.0 (for NVIDIA GPU support) -* **HIP** >= 6.4 (for AMD GPU support) -* **MPI** (for distributed computing) -* **RabbitMQ/AMQP-CPP** (for message queue support) -* **Caliper** (for performance profiling) +.. code-block:: bash -Installation Methods --------------------- + host=$(hostname) + host=${host//[0-9]/} + python3 scripts/make-spack-venv.py \ + --env "/usr/workspace/AMS/ams-spack-environments/1.1/${host}/" \ + --output "venv-${host}" \ + --with-system-flux-python + source "venv-${host}/bin/activate" + +The helper writes system Flux metadata into the venv and warns on activation if the +active ``flux version`` differs from the recorded one. The shim exposes Flux +without putting unrelated LC Python packages ahead of AMS Spack packages. +Recreate the venv after system Flux changes. For LC workflow CMake builds, pass +``-DAMS_PIP_INSTALL_ARGS="--no-build-isolation"`` so pip uses the prepared venv +instead of an isolated build environment. Do not enable +``AMS_INSTALL_FLUX_PYTHON`` for this path; Flux Python is supplied by +the prepared system-backed venv, not by pip. + +Non-System Flux Workflow Installs +--------------------------------- + +When building AMS Workflow in an environment that does not provide compatible +system Flux Python bindings, enable the workflow Flux extra: + +.. code-block:: bash -Using Spack -~~~~~~~~~~~ + cmake -S . -B build \ + -DENABLE_WORKFLOW=On \ + -DAMS_INSTALL_FLUX_PYTHON=On -TBD +With this flag, the ``PyAMS`` target runs pip against the generated package tree +as ``pip install [flux]``, which installs the ``flux-python`` +optional dependency declared by ``pyproject.toml``. Use this for container or +non-LC builds that rely on pip-managed Flux Python. Leave it off when using +``scripts/make-spack-venv.py --with-system-flux-python``. -Manual Build with CMake -~~~~~~~~~~~~~~~~~~~~~~~ +Convenience Script +------------------ -For a basic installation: +``scripts/ams-configure.sh`` can assemble the common CMake command: .. code-block:: bash - git clone https://github.com/LLNL/AMS.git - cd AMS - mkdir build && cd build - - cmake \ - -DWITH_RMQ=On \ - -Damqpcpp_DIR=$AMS_AMQPCPP_PATH \ - -DWITH_CALIPER=On \ - -DWITH_HDF5=On \ - -DHDF5_Dir=$AMS_HDF5_PATH \ - -DCMAKE_INSTALL_PREFIX=./install \ - -DCMAKE_BUILD_TYPE=Release \ - -DWITH_CUDA=On \ - -DWITH_MPI=On \ - -DWITH_TESTS=On \ - -DTorch_DIR=$AMS_TORCH_PATH - -DWITH_EXAMPLES=On \ - .. + scripts/ams-configure.sh --hip --mpi --caliper + scripts/ams-configure.sh --mpi --tests + scripts/ams-configure.sh --workflow --install-flux-python + scripts/ams-configure.sh --mpi --rmq --dry-run + +When ``--tests`` is enabled and ``AMS_CATCH2_DIR`` is set, the helper forwards +``-DAMS_CATCH2_DIR="$AMS_CATCH2_DIR"`` so LC builds use the Spack-provided +Catch2 package. + +Manual CMake is still needed when forcing LC Cray/ROCm compilers unless the +script is later extended with compiler options. + +Tests +----- + +``ENABLE_TESTS=On`` enters the Catch2 test tree and first looks for a Catch2 +CMake package. On LC systems, ``scripts/gitlab/setup-env.sh`` exports +``AMS_CATCH2_DIR`` for the Spack-provided Catch2 package; pass it with +``-DAMS_CATCH2_DIR="$AMS_CATCH2_DIR"``. - make -j 4 - make install +If package discovery fails and no ``AMS_CATCH2_DIR`` hint is provided, CMake +falls back to ``FetchContent`` from GitHub for Catch2 v3.11.0. In network-free +environments outside LC, provide a local/package Catch2 config directory with +``AMS_CATCH2_DIR`` or configure with ``-DENABLE_TESTS=Off``. -For complete installation instructions, see the repository's INSTALL.md file. +For more examples, see the repository ``INSTALL.md``. diff --git a/pyproject.toml b/pyproject.toml index d940a7b8..369b0176 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,15 +33,16 @@ classifiers = [ ] dependencies = [ "h5py", - "argparse", "SQLAlchemy", "mysql-connector-python", "pika>=1.3.0", - "numpy>=1.2.0", - "flux-python>=0.75.0" + "numpy>=1.2.0" ] [project.optional-dependencies] +flux = [ + "flux-python>=0.75.0", +] dev = [ "ruff", ] @@ -84,4 +85,3 @@ known-first-party = ["ams", "ams_wf"] [tool.ruff.format] quote-style = "double" indent-style = "space" - diff --git a/scripts/ams-configure.sh b/scripts/ams-configure.sh new file mode 100755 index 00000000..41d7bd15 --- /dev/null +++ b/scripts/ams-configure.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# +# ams-configure.sh — assemble and run the CMake configure step for AMS. +# +# We have two dependency-provisioning paths: +# * On an LLNL Livermore Computing (LC) cluster it sources the repo's own +# scripts/gitlab/setup-env.sh (internal Spack env) and maps the exported +# AMS_*_PATH variables onto the right -D..._DIR hints. +# * Elsewhere it relies on *_DIR / CMAKE_PREFIX_PATH you set yourself +# (see references/manual-deps.md). +# +# The full cmake command is always printed before it runs. Use --dry-run to +# print without configuring. +# +# Run from the AMS repo root, or pass --src . + +set -euo pipefail + +# ---- defaults -------------------------------------------------------------- +SRC="." +BUILD="build" +BUILD_TYPE="Release" +INSTALL_PREFIX="./install" +SHARED="On" +DRY_RUN=0 +FORCE_MODE="" # "lc" | "nolc" | "" (auto) + +# feature toggles (all off by default -> minimal CPU build) +ENABLE_MPI="Off" +ENABLE_CUDA="Off" +ENABLE_HIP="Off" +ENABLE_CALIPER="Off" +ENABLE_RMQ="Off" +ENABLE_PERFFLOWASPECT="Off" +ENABLE_WORKFLOW="Off" +INSTALL_FLUX_PYTHON="Off" +ENABLE_DEBUG="Off" +ENABLE_TESTS="Off" +CUDA_ARCH="" # override; else AMS_CUDA_ARCH after setup-env + +usage() { + cat <<'EOF' +Usage: ams-configure.sh [feature flags] [options] + +Feature flags (compose freely): + --mpi enable MPI (-DENABLE_MPI=On) + --cuda enable CUDA (NVIDIA) (-DENABLE_CUDA=On) + --hip enable HIP (AMD) (-DENABLE_HIP=On) [mutually exclusive with --cuda] + --caliper Caliper profiling (-DENABLE_CALIPER=On) + --rmq RabbitMQ back end (-DENABLE_RMQ=On) + --perfflowaspect PerfFlowAspect (-DENABLE_PERFFLOWASPECT=On) + --workflow Python drivers (-DENABLE_WORKFLOW=On) + --install-flux-python install flux-python (-DAMS_INSTALL_FLUX_PYTHON=On) + --debug verbose logging (-DAMS_ENABLE_DEBUG=On) + --tests build tests (-DENABLE_TESTS=On) + +Options: + --src PATH AMS repo root (default: .) + --build DIR build directory (default: build) + --build-type TYPE Release|Debug|RelWithDebInfo (default: Release) + --install-prefix PATH install prefix (default: ./install) + --static build static libs (default: shared) + --cuda-arch ARCH CUDA arch, e.g. 80,90 (default: $AMS_CUDA_ARCH on LC) + --lc | --no-lc force LC / non-LC dependency path (default: auto-detect) + --dry-run print the cmake command but do not run it + -h, --help this help + +Examples: + ams-configure.sh # minimal CPU build + ams-configure.sh --rmq --mpi # RabbitMQ + MPI + ams-configure.sh --cuda --caliper # GPU + profiling + ams-configure.sh --rmq --dry-run # show the command only +EOF +} + +# ---- parse args ------------------------------------------------------------ +while [[ $# -gt 0 ]]; do + case "$1" in + --mpi) ENABLE_MPI="On";; + --cuda) ENABLE_CUDA="On";; + --hip) ENABLE_HIP="On";; + --caliper) ENABLE_CALIPER="On";; + --rmq|--rabbitmq) ENABLE_RMQ="On";; + --perfflowaspect|--pfa) ENABLE_PERFFLOWASPECT="On";; + --workflow) ENABLE_WORKFLOW="On";; + --install-flux-python) INSTALL_FLUX_PYTHON="On";; + --debug) ENABLE_DEBUG="On";; + --tests) ENABLE_TESTS="On";; + --src) SRC="$2"; shift;; + --build) BUILD="$2"; shift;; + --build-type) BUILD_TYPE="$2"; shift;; + --install-prefix) INSTALL_PREFIX="$2"; shift;; + --static) SHARED="Off";; + --cuda-arch) CUDA_ARCH="$2"; shift;; + --lc) FORCE_MODE="lc";; + --no-lc) FORCE_MODE="nolc";; + --dry-run) DRY_RUN=1;; + -h|--help) usage; exit 0;; + *) echo "Unknown argument: $1" >&2; usage; exit 2;; + esac + shift +done + +if [[ "$ENABLE_CUDA" == "On" && "$ENABLE_HIP" == "On" ]]; then + echo "Error: --cuda and --hip are mutually exclusive." >&2 + exit 2 +fi + +if [[ ! -f "$SRC/CMakeLists.txt" ]]; then + echo "Error: '$SRC' does not look like the AMS repo root (no CMakeLists.txt)." >&2 + echo " cd into the AMS clone or pass --src ." >&2 + exit 2 +fi + +# ---- decide dependency path ------------------------------------------------ +LC_ENV_DIR="/usr/workspace/AMS/ams-spack-environments" +MODE="$FORCE_MODE" +if [[ -z "$MODE" ]]; then + if [[ -d "$LC_ENV_DIR" ]]; then MODE="lc"; else MODE="nolc"; fi +fi + +# extra -D hints accumulated from the environment +declare -a DEP_ARGS=() + +if [[ "$MODE" == "lc" ]]; then + echo ">> LC cluster detected — sourcing $SRC/scripts/gitlab/setup-env.sh" + # shellcheck disable=SC1091 + source "$SRC/scripts/gitlab/setup-env.sh" + + [[ -n "${AMS_TORCH_PATH:-}" ]] && DEP_ARGS+=("-DTorch_DIR=${AMS_TORCH_PATH}") + [[ -n "${AMS_HDF5_PATH:-}" ]] && DEP_ARGS+=("-DHDF5_DIR=${AMS_HDF5_PATH}") + if [[ "$ENABLE_CALIPER" == "On" && -n "${AMS_CALIPER_PATH:-}" ]]; then + DEP_ARGS+=("-Dcaliper_DIR=${AMS_CALIPER_PATH}") + fi + if [[ "$ENABLE_RMQ" == "On" && -n "${AMS_AMQPCPP_PATH:-}" ]]; then + DEP_ARGS+=("-Damqpcpp_DIR=${AMS_AMQPCPP_PATH}") + fi + if [[ "$ENABLE_TESTS" == "On" && -n "${AMS_CATCH2_DIR:-}" ]]; then + DEP_ARGS+=("-DAMS_CATCH2_DIR=${AMS_CATCH2_DIR}") + fi + if [[ "$ENABLE_CUDA" == "On" ]]; then + ARCH="${CUDA_ARCH:-${AMS_CUDA_ARCH:-}}" + [[ -n "$ARCH" ]] && DEP_ARGS+=("-DCMAKE_CUDA_ARCHITECTURES=${ARCH}") + fi +else + echo ">> Non-LC cluster — using your *_DIR / CMAKE_PREFIX_PATH hints." + echo " (see references/manual-deps.md; a bare configure will fail if" + echo " Torch/HDF5/nlohmann_json can't be found)" + # pass through anything the user already exported, if present + [[ -n "${Torch_DIR:-}" ]] && DEP_ARGS+=("-DTorch_DIR=${Torch_DIR}") + [[ -n "${AMS_HDF5_DIR:-}" ]] && DEP_ARGS+=("-DHDF5_DIR=${AMS_HDF5_DIR}") + [[ -n "${nlohmann_json_DIR:-}" ]] && DEP_ARGS+=("-Dnlohmann_json_DIR=${nlohmann_json_DIR}") + [[ "$ENABLE_CALIPER" == "On" && -n "${caliper_DIR:-}" ]] && DEP_ARGS+=("-Dcaliper_DIR=${caliper_DIR}") + [[ "$ENABLE_RMQ" == "On" && -n "${amqpcpp_DIR:-}" ]] && DEP_ARGS+=("-Damqpcpp_DIR=${amqpcpp_DIR}") + [[ "$ENABLE_TESTS" == "On" && -n "${AMS_CATCH2_DIR:-}" ]] && DEP_ARGS+=("-DAMS_CATCH2_DIR=${AMS_CATCH2_DIR}") + if [[ "$ENABLE_CUDA" == "On" && -n "$CUDA_ARCH" ]]; then + DEP_ARGS+=("-DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH}") + fi +fi + +# ---- assemble cmake command ------------------------------------------------ +CMAKE_ARGS=( + -S "$SRC" -B "$BUILD" + -DCMAKE_BUILD_TYPE="$BUILD_TYPE" + -DBUILD_SHARED_LIBS="$SHARED" + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH="$SHARED" + -DCMAKE_INSTALL_PREFIX="$INSTALL_PREFIX" + -DENABLE_MPI="$ENABLE_MPI" + -DENABLE_CUDA="$ENABLE_CUDA" + -DENABLE_HIP="$ENABLE_HIP" + -DENABLE_CALIPER="$ENABLE_CALIPER" + -DENABLE_RMQ="$ENABLE_RMQ" + -DENABLE_PERFFLOWASPECT="$ENABLE_PERFFLOWASPECT" + -DENABLE_WORKFLOW="$ENABLE_WORKFLOW" + -DAMS_INSTALL_FLUX_PYTHON="$INSTALL_FLUX_PYTHON" + -DAMS_ENABLE_DEBUG="$ENABLE_DEBUG" + -DENABLE_TESTS="$ENABLE_TESTS" + "${DEP_ARGS[@]}" +) + +echo +echo ">> cmake command:" +printf ' cmake' +for a in "${CMAKE_ARGS[@]}"; do printf ' \\\n %q' "$a"; done +printf '\n\n' + +if [[ "$DRY_RUN" == "1" ]]; then + echo ">> --dry-run set; not configuring." + exit 0 +fi + +cmake "${CMAKE_ARGS[@]}" + +echo +echo ">> Configured. Next:" +echo " cmake --build $BUILD -j \"\$(nproc)\"" +echo " cmake --install $BUILD" diff --git a/scripts/gitlab/ci-build-test.sh b/scripts/gitlab/ci-build-test.sh index 96b4432b..e8656db4 100755 --- a/scripts/gitlab/ci-build-test.sh +++ b/scripts/gitlab/ci-build-test.sh @@ -31,22 +31,33 @@ build_and_test() { cleanup - # We need custom Virtual env on Tuo because we use Spack python external + # We need custom Virtual env on LC machines because we use Spack python external export host=$(hostname) export host=${host//[0-9]/} - python3 ${CI_PROJECT_DIR}/scripts/make-spack-venv.py -e /usr/workspace/AMS/ams-spack-environments/1.1/${host}/ -o venv-${host} + venv_args=( + --env /usr/workspace/AMS/ams-spack-environments/1.1/${host}/ + --with-system-flux-python + --output venv-${host} + ) + + python3 ${CI_PROJECT_DIR}/scripts/make-spack-venv.py "${venv_args[@]}" source venv-${host}/bin/activate if [[ "$SYS_TYPE" == "toss_4_x86_64_ib_cray" ]]; then C_COMPILER=amdclang CXX_COMPILER=amdclang++ - # We cannot set CC and CXX as it will conflict with flux-python package.. elif [[ "$SYS_TYPE" == "toss_4_x86_64_ib" ]]; then C_COMPILER=gcc CXX_COMPILER=g++ fi + if [[ "${WITH_WORKFLOW}" == "On" ]]; then + AMS_WORKFLOW_PIP_ARGS="--no-build-isolation" + else + AMS_WORKFLOW_PIP_ARGS="" + fi + mkdir build pushd build @@ -58,6 +69,7 @@ build_and_test() { -DCMAKE_INSTALL_PREFIX=./install \ -DENABLE_RMQ=Off \ -DENABLE_WORKFLOW=${WITH_WORKFLOW} \ + -DAMS_PIP_INSTALL_ARGS="${AMS_WORKFLOW_PIP_ARGS}" \ -DENABLE_TESTS=On \ -DCUDA_ARCH=${AMS_CUDA_ARCH} \ -DENABLE_CUDA=${WITH_CUDA} \ diff --git a/scripts/make-spack-venv.py b/scripts/make-spack-venv.py index 0a23494f..3b8da799 100755 --- a/scripts/make-spack-venv.py +++ b/scripts/make-spack-venv.py @@ -10,7 +10,11 @@ import subprocess import sys from pathlib import Path -from typing import Dict, Iterable, List, Optional, Sequence, Set, Tuple +from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple + + +SYSTEM_FLUX_PTH_NAME = "system-flux-python.pth" +SYSTEM_FLUX_METADATA_NAME = "system-flux-python.json" def run( @@ -19,17 +23,19 @@ def run( check: bool = True, capture: bool = True, cwd: Optional[Path] = None, + env: Optional[Dict[str, str]] = None, ) -> subprocess.CompletedProcess: if capture: return subprocess.run( cmd, check=check, cwd=str(cwd) if cwd else None, + env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) - return subprocess.run(cmd, check=check, cwd=str(cwd) if cwd else None) + return subprocess.run(cmd, check=check, cwd=str(cwd) if cwd else None, env=env) def die(msg: str, code: int = 2) -> None: @@ -45,6 +51,14 @@ def info(msg: str) -> None: print(msg, file=sys.stderr) +def canonical_path(path: Path | str) -> Path: + p = Path(path).expanduser() + try: + return p.resolve(strict=False) + except RuntimeError: + return p.absolute() + + def find_spack() -> str: spack = shutil.which("spack") if not spack: @@ -55,27 +69,38 @@ def find_spack() -> str: def guess_spack_home(spack_exe: str) -> Path: env_root = os.environ.get("SPACK_HOME") or os.environ.get("SPACK_ROOT") if env_root: - return Path(env_root).expanduser().resolve() + return canonical_path(env_root) - p = Path(spack_exe).resolve() + p = canonical_path(spack_exe) if p.name == "spack" and p.parent.name == "bin": return p.parent.parent return p.parent def is_under(child: Path, parent: Path) -> bool: + child = canonical_path(child) + parent = canonical_path(parent) try: - child = child.resolve() - parent = parent.resolve() child.relative_to(parent) return True except Exception: + pass + + if not child.exists() or not parent.exists(): return False + for candidate in [child, *child.parents]: + try: + if candidate.samefile(parent): + return True + except OSError: + continue + return False + def find_env_dir(env_arg: Optional[str]) -> Path: if env_arg: - p = Path(env_arg).expanduser().resolve() + p = canonical_path(env_arg) if p.is_file() and p.name == "spack.yaml": return p.parent if p.is_dir(): @@ -120,7 +145,7 @@ def spack_location(spack: str, spec_ref: str) -> Optional[Path]: loc = (cp.stdout or "").strip() if not loc: return None - return Path(loc) + return canonical_path(loc) def load_yaml_modules(spack_yaml: Path) -> List[str]: @@ -252,7 +277,7 @@ def indent_of(s: str) -> int: if m_pref: p = m_pref.group(1).strip().strip("'").strip('"') if p: - prefix = Path(p).expanduser().resolve() + prefix = canonical_path(p) break return (ver, prefix) @@ -379,26 +404,31 @@ def ensure_python_shims(venv_dir: Path) -> None: py.symlink_to("python3" if py3.exists() else target.name) -def venv_sitepackages(venv_dir: Path) -> Path: +def venv_python(venv_dir: Path) -> Path: py = venv_dir / "bin" / "python" if not py.exists(): py = venv_dir / "bin" / "python3" + if not py.exists(): + die("venv python not found") + return py + + +def venv_sitepackages(venv_dir: Path) -> Path: + py = venv_python(venv_dir) if not py.exists(): die("venv python not found to query site-packages") cp = run([str(py), "-c", "import sysconfig; print(sysconfig.get_paths()['purelib'])"]) purelib = (cp.stdout or "").strip() if not purelib: die("Failed to query venv site-packages (purelib)") - p = Path(purelib) + p = canonical_path(purelib) if not p.exists(): die(f"Venv site-packages path does not exist: {p}") return p def venv_py_mm(venv_dir: Path) -> str: - py = venv_dir / "bin" / "python" - if not py.exists(): - py = venv_dir / "bin" / "python3" + py = venv_python(venv_dir) cp = run([str(py), "-c", "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}')"]) mm = (cp.stdout or "").strip() if not re.match(r"^\d+\.\d+$", mm): @@ -437,7 +467,7 @@ def find_site_packages_under(prefix: Path) -> List[Path]: out: List[Path] = [] seen: Set[str] = set() for p in candidates: - rp = p.resolve() + rp = canonical_path(p) s = str(rp) if s in seen: continue @@ -455,7 +485,7 @@ def filter_spack_sites( out: List[Path] = [] seen: Set[str] = set() for p in spack_sites: - rp = p.resolve() + rp = canonical_path(p) if not is_under(rp, spack_home): continue @@ -478,6 +508,197 @@ def write_pth(venv_site: Path, spack_sites: List[Path]) -> Path: return pth +FLUX_DISCOVER_SCRIPT = r""" +import json +import pathlib +import sys + +import flux +import flux.job # noqa: F401 +import flux.resource # noqa: F401 + +flux_file = pathlib.Path(flux.__file__).resolve() +flux_python_path = None +for parent in flux_file.parents: + if parent.name in {"site-packages", "dist-packages"}: + flux_python_path = parent + break + +if flux_python_path is None and flux_file.parent.name == "flux": + flux_python_path = flux_file.parent.parent + +if flux_python_path is None: + raise SystemExit(f"Could not derive Python import path from flux.__file__: {flux_file}") + +print( + json.dumps( + { + "python_executable": sys.executable, + "python_version": sys.version, + "flux_file": str(flux_file), + "flux_python_path": str(flux_python_path), + "sys_path": sys.path, + }, + sort_keys=True, + ) +) +""" + + +def parse_flux_discovery(cp: subprocess.CompletedProcess) -> Dict[str, str]: + try: + return json.loads(cp.stdout) + except json.JSONDecodeError as e: + die(f"Could not parse Flux Python discovery output as JSON: {e}\nOutput was:\n{cp.stdout}") + + +def probe_flux_import(python_exe: Path | str, python_path: Optional[Path] = None) -> subprocess.CompletedProcess: + env: Optional[Dict[str, str]] = None + if python_path is not None: + env = os.environ.copy() + existing = env.get("PYTHONPATH") + env["PYTHONPATH"] = str(python_path) if not existing else f"{python_path}{os.pathsep}{existing}" + return run([str(python_exe), "-c", FLUX_DISCOVER_SCRIPT], check=False, env=env) + + +def venv_can_import(venv_dir: Path, module: str) -> bool: + cp = run([str(venv_python(venv_dir)), "-c", f"import {module}"], check=False) + return cp.returncode == 0 + + +def find_module_entry(module: str, search_paths: List[Path]) -> Optional[Path]: + for root in search_paths: + package = root / module + if package.exists(): + return package + module_file = root / f"{module}.py" + if module_file.exists(): + return module_file + return None + + +def create_system_flux_python_shim( + venv_dir: Path, + venv_site: Path, + flux_python_path: Path, + flux_python_sys_path: List[Path], +) -> Path: + shim = venv_site / "system_flux_python" + shim.mkdir(parents=True, exist_ok=True) + + names: Set[str] = set() + symlink_sources: Dict[str, Path] = {} + for source in flux_python_path.iterdir(): + source_name = source.name + import_name = source.stem if source.is_file() else source_name + if import_name in {"flux", "_flux"} or re.match(r"^flux[A-Za-z0-9_]*$", import_name): + symlink_sources[source_name] = source + + for dependency in ["ply", "yaml"]: + if venv_can_import(venv_dir, dependency): + continue + source = find_module_entry(dependency, flux_python_sys_path) + if source is not None: + symlink_sources[source.name] = source + + if not symlink_sources: + die(f"Could not find Flux Python packages under: {flux_python_path}") + + for name, source in sorted(symlink_sources.items()): + target = shim / name + if target.exists() or target.is_symlink(): + die(f"Cannot create system Flux shim because target already exists: {target}") + target.symlink_to(source, target_is_directory=source.is_dir()) + + return shim + + +def verify_system_flux_python_path(venv_dir: Path, flux_python_path: Path) -> None: + cp = probe_flux_import(venv_python(venv_dir), flux_python_path) + if cp.returncode != 0: + die( + "The venv Python could not import Flux from the prepared system Flux Python path.\n" + f"Flux Python path: {flux_python_path}\n" + f"stdout:\n{cp.stdout}\n" + f"stderr:\n{cp.stderr}" + ) + + +def get_python_version(python_exe: Path) -> str: + cp = run([str(python_exe), "-c", "import sys; print(sys.version)"]) + return cp.stdout.strip() + + +def discover_system_flux_python(venv_dir: Path, spack_home: Path, spack_sites: List[Path]) -> Dict[str, Any]: + flux_exe = shutil.which("flux") + if not flux_exe: + die("--with-system-flux-python requires an active system Flux installation, but 'flux' was not found on PATH.") + + flux_version_cp = run([flux_exe, "version"], check=False) + if flux_version_cp.returncode != 0: + die( + "Could not run 'flux version' for --with-system-flux-python.\n" + f"stdout:\n{flux_version_cp.stdout}\n" + f"stderr:\n{flux_version_cp.stderr}" + ) + + py = venv_python(venv_dir) + cp = probe_flux_import(py) + if cp.returncode != 0: + flux_python_cp = run([flux_exe, "python", "-c", FLUX_DISCOVER_SCRIPT], check=False) + if flux_python_cp.returncode != 0: + die( + "Could not import the active system Flux Python bindings with the venv Python " + "or through 'flux python'.\n" + "Load the system Flux environment and recreate the venv with --with-system-flux-python.\n" + f"venv stdout:\n{cp.stdout}\n" + f"venv stderr:\n{cp.stderr}\n" + f"flux python stdout:\n{flux_python_cp.stdout}\n" + f"flux python stderr:\n{flux_python_cp.stderr}" + ) + imported = parse_flux_discovery(flux_python_cp) + else: + imported = parse_flux_discovery(cp) + + flux_python_sys_path = [ + str(canonical_path(path)) for path in imported.get("sys_path", []) if path + ] + + flux_site = canonical_path(imported["flux_python_path"]) + flux_file = canonical_path(imported["flux_file"]) + spack_site_keys = {str(canonical_path(p)) for p in spack_sites} + if str(flux_site) in spack_site_keys or is_under(flux_site, spack_home): + die( + "The active 'flux' Python module resolved to the AMS Spack environment, " + f"not system Flux Python: {flux_file}" + ) + + return { + "schema_version": 1, + "which_flux": str(canonical_path(flux_exe)), + "flux_version": flux_version_cp.stdout.strip(), + "python_executable": str(py), + "python_version": get_python_version(py), + "flux_python_executable": imported["python_executable"], + "flux_python_version": imported["python_version"], + "flux_file": str(flux_file), + "flux_site_packages": str(flux_site), + "flux_python_sys_path": flux_python_sys_path, + } + + +def write_system_flux_python_pth(venv_site: Path, flux_site_packages: Path) -> Path: + pth = venv_site / SYSTEM_FLUX_PTH_NAME + pth.write_text(f"{flux_site_packages}\n", encoding="utf-8") + return pth + + +def write_system_flux_metadata(venv_site: Path, metadata: Dict[str, Any]) -> Path: + metadata_path = venv_site / SYSTEM_FLUX_METADATA_NAME + metadata_path.write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return metadata_path + + def capture_env_vars(keys: Iterable[str]) -> Dict[str, str]: out: Dict[str, str] = {} for k in keys: @@ -538,6 +759,42 @@ def write_activate_hooks(venv_dir: Path, modules: List[str], envvars: Optional[D activate.write_text(act_txt + snippet, encoding="utf-8") +def write_system_flux_activation_hook(venv_dir: Path, metadata_path: Path, metadata: Dict[str, Any]) -> Path: + actived = venv_dir / "bin" / "activate.d" + actived.mkdir(parents=True, exist_ok=True) + hook = actived / "zz_system_flux_python.sh" + recorded_version = str(metadata["flux_version"]) + hook.write_text( + "\n".join( + [ + "#!/usr/bin/env bash", + "# Auto-generated. Warns when the active system Flux installation changed since venv creation.", + f"_ams_system_flux_metadata={shlex.quote(str(metadata_path))}", + f"_ams_system_flux_recorded_version={shlex.quote(recorded_version)}", + "if ! command -v flux >/dev/null 2>&1; then", + ' echo "spack-venv: flux command not found; load system Flux and recreate this venv" 1>&2', + "else", + ' _ams_system_flux_current_version="$(flux version 2>/dev/null || true)"', + ' if [ -n "$_ams_system_flux_current_version" ] && \\', + ' [ "$_ams_system_flux_current_version" != "$_ams_system_flux_recorded_version" ]; then', + ' echo "spack-venv: active flux version differs from recorded system Flux metadata" 1>&2', + ' echo "spack-venv: recreate this venv with ' + 'scripts/make-spack-venv.py --with-system-flux-python" 1>&2', + ' echo "spack-venv: metadata: ${_ams_system_flux_metadata}" 1>&2', + " fi", + "fi", + "unset _ams_system_flux_current_version", + "unset _ams_system_flux_recorded_version", + "unset _ams_system_flux_metadata", + ] + ) + + "\n", + encoding="utf-8", + ) + hook.chmod(0o755) + return hook + + def has_any_package(specs: List[dict], names: Set[str]) -> bool: for s in specs: n = s.get("name") @@ -588,6 +845,11 @@ def main(argv: Optional[Sequence[str]] = None) -> int: action="store_true", help="Do not restrict .pth entries to the venv Python major.minor (default: restrict).", ) + ap.add_argument( + "--with-system-flux-python", + action="store_true", + help="Link the active system Flux Python package path into the venv and warn when Flux changes.", + ) args = ap.parse_args(argv) spack = find_spack() @@ -599,7 +861,10 @@ def main(argv: Optional[Sequence[str]] = None) -> int: die(f"spack.yaml not found in env dir: {env_dir}") if not spack_env_is_active(spack): - warn("Spack env does not appear active (SPACK_ENV not set). Continuing, but spack queries may not match your intended env.") + warn( + "Spack env does not appear active (SPACK_ENV not set). " + "Continuing, but spack queries may not match your intended env." + ) modules = load_yaml_modules(spack_yaml) info(f"Modules found in spack.yaml: {modules if modules else 'none'}") @@ -642,6 +907,24 @@ def main(argv: Optional[Sequence[str]] = None) -> int: pth = write_pth(venv_site, uniq_sites) + system_flux_pth: Optional[Path] = None + system_flux_metadata_path: Optional[Path] = None + system_flux_metadata: Optional[Dict[str, Any]] = None + if args.with_system_flux_python: + system_flux_metadata = discover_system_flux_python(venv_dir, spack_home, uniq_sites) + system_flux_site = canonical_path(system_flux_metadata["flux_site_packages"]) + system_flux_sys_path = [canonical_path(path) for path in system_flux_metadata["flux_python_sys_path"]] + system_flux_shim = create_system_flux_python_shim( + venv_dir, + venv_site, + system_flux_site, + system_flux_sys_path, + ) + verify_system_flux_python_path(venv_dir, system_flux_shim) + system_flux_metadata["flux_shim_path"] = str(system_flux_shim) + system_flux_pth = write_system_flux_python_pth(venv_site, system_flux_shim) + system_flux_metadata_path = write_system_flux_metadata(venv_site, system_flux_metadata) + envvars: Optional[Dict[str, str]] = None if args.capture_envvars is not None: seen: Set[str] = set() @@ -655,9 +938,16 @@ def main(argv: Optional[Sequence[str]] = None) -> int: envvars = capture_env_vars(keys) write_activate_hooks(venv_dir, modules, envvars) + if system_flux_metadata_path is not None and system_flux_metadata is not None: + system_flux_hook = write_system_flux_activation_hook(venv_dir, system_flux_metadata_path, system_flux_metadata) + info(f"Wrote system Flux activation hook: {system_flux_hook}") info(f"Done. Venv: {venv_dir}") info(f"Wrote .pth: {pth}") + if system_flux_pth is not None: + info(f"Wrote system Flux .pth: {system_flux_pth}") + if system_flux_metadata_path is not None: + info(f"Wrote system Flux metadata: {system_flux_metadata_path}") return 0 diff --git a/src/AMSWorkflow/CMakeLists.txt b/src/AMSWorkflow/CMakeLists.txt index e1f377c5..ddd885d6 100644 --- a/src/AMSWorkflow/CMakeLists.txt +++ b/src/AMSWorkflow/CMakeLists.txt @@ -10,19 +10,27 @@ file(GLOB_RECURSE pyfiles *.py ams_wf/*.py ams/*.py) # detect virtualenv and set Pip args accordingly set(AMS_PY_APP "${CMAKE_BINARY_DIR}") +set(_ams_py_install_target "${AMS_PY_APP}") +if(AMS_INSTALL_FLUX_PYTHON) + set(_ams_py_install_target "${AMS_PY_APP}[flux]") +endif() if(DEFINED ENV{VIRTUAL_ENV} OR DEFINED ENV{CONDA_PREFIX}) set(_pip_args) else() set(_pip_args "--user") endif() +if(AMS_PIP_INSTALL_ARGS) + separate_arguments(_ams_extra_pip_args NATIVE_COMMAND "${AMS_PIP_INSTALL_ARGS}") + list(APPEND _pip_args ${_ams_extra_pip_args}) +endif() message(STATUS "AMS Python Source files are ${pyfiles}") -message(STATUS "AMS Python built cmd is : ${Python_EXECUTABLE} -m pip install ${_pip_args} ${AMS_PY_APP}") +message(STATUS "AMS Python built cmd is : ${Python_EXECUTABLE} -m pip install ${_pip_args} ${_ams_py_install_target}") add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/timestamp" COMMAND ${CMAKE_COMMAND} -E touch "${CMAKE_CURRENT_BINARY_DIR}/timestamp" - COMMAND ${Python_EXECUTABLE} -m pip install ${_pip_args} ${AMS_PY_APP} + COMMAND ${Python_EXECUTABLE} -m pip install ${_pip_args} "${_ams_py_install_target}" DEPENDS ${pyfiles} WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" COMMENT "Build AMS-WF Python Modules and Applications" @@ -32,4 +40,3 @@ add_custom_command( add_custom_target(PyAMS ALL DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/timestamp" ) - diff --git a/tests/AMSlib/CMakeLists.txt b/tests/AMSlib/CMakeLists.txt index 6aaa36ad..ddc0c3d4 100644 --- a/tests/AMSlib/CMakeLists.txt +++ b/tests/AMSlib/CMakeLists.txt @@ -39,7 +39,6 @@ if (NOT _ams_catch2_ctest_adapter) endif() set(AMS_TEST_ROOT "${CMAKE_CURRENT_BINARY_DIR}") -message(INFO "AMS_TEST_ROOT is ${AMS_TEST_ROOT}") add_subdirectory(models) add_subdirectory(torch) add_subdirectory(db) diff --git a/tests/AMSlib/db/CMakeLists.txt b/tests/AMSlib/db/CMakeLists.txt index 511a8bac..f858bdbb 100644 --- a/tests/AMSlib/db/CMakeLists.txt +++ b/tests/AMSlib/db/CMakeLists.txt @@ -15,7 +15,7 @@ function(BUILD_UNIT_TEST exe source) target_link_libraries(${exe} PRIVATE ${AMS_HDF5_LINK_TARGETS}) if (ENABLE_CALIPER) - message(STATUS "Building witth caliper ${exe}") + message(STATUS "Building with caliper ${exe}") target_link_libraries(${exe} PRIVATE caliper) endif()