From e778192d4c1cfd47e97be1881047532bfce7a7c0 Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Thu, 5 Nov 2026 14:45:50 +0800 Subject: [PATCH 01/42] test: remove stale pp_size fixture parameter (#544) ## Purpose Fix the missing `pp_size` fixture error in `test_npu_async_attention_allows_model_owned_sp` introduced by #539. Its parameterization was removed and pipeline parallel size fixed to 1, but the unused function argument remained. Only the stale argument is removed. The test body and assertions are preserved. ## Issue Follow-up to #539. [Buildkite #225](https://buildkite.com/vllm/afd-plugin/builds/225#01a1a9f1-f0bd-462f-8651-5dacde7b567f) reported 1430 passed, 47 skipped and this one setup error; the subsequent GPU routing command did not execute. ## Test Plan Run the existing `ready` L4 job, verifying both commands complete: - CPU-safe CI selection, with the CI image's runtime dependencies. - `pytest -sv tests/gpu/test_attention_routing.py`, including both renormalization cases without skips. ## Test Result - Before: reproduced `fixture 'pp_size' not found`. - After: the target and adjacent SP tests passed (4 cases) using CPU Torch and temporary upstream import stubs; real AFD construction code and test bodies executed. - Selected-file pre-commit and `git diff --check` passed. - CI validation is pending. The local adapter results do not establish native CUDA/NPU correctness. ## Docs Impact None: this fixes test setup only. ---
Essential PR Checklist - [x] Purpose is clear and linked to the failing CI. - [x] Scope is one unused test argument. - [x] Pinned runtime contracts are unchanged. - [x] No vLLM checkout or production code changes. - [x] No new classes, compatibility shims or monkey patches. - [x] Imports remain CPU-safe. - [x] Local validation and its limitations are recorded; native CI is pending. - [x] Documentation impact is stated.
Signed-off-by: Zhou ziheng --- .../unit/model_executor/models/test_deepseek_v2_construction.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/model_executor/models/test_deepseek_v2_construction.py b/tests/unit/model_executor/models/test_deepseek_v2_construction.py index 03756f275..ab846f7d9 100644 --- a/tests/unit/model_executor/models/test_deepseek_v2_construction.py +++ b/tests/unit/model_executor/models/test_deepseek_v2_construction.py @@ -764,7 +764,6 @@ def test_model_constructor_rejects_sequence_parallel_moe_before_allocation( def test_npu_async_attention_allows_model_owned_sp( monkeypatch, construction_env, - pp_size, ): vllm_config = _vllm_config() vllm_config.parallel_config.use_sequence_parallel_moe = True From 621587b02bd0fdb3cb8c99aba6f4a7bc98ac0c5c Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Thu, 5 Nov 2026 18:51:51 +0800 Subject: [PATCH 02/42] docs(skill): refresh review-pr against the current tree (#547) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Purpose Bring the `review-pr` skill back in line with the code it reviews. The skill carried pins, runner architecture, ownership, and module inventory from the vLLM 0.26 / dual-runner era; several of its statements were actively wrong for a reviewer following them literally, so it would have produced incorrect findings or missed real blockers. ## Issue - No linked issue. This is documentation maintenance for an agent skill; raise an RFC first if the skill's review workflow itself should change. ## Scope - In scope: `.agents/skills/review-pr/**` only. - Out of scope: production code, tests, design pages, CI configuration, and every other skill under `.agents/skills/`. ## Implementation Notes Corrections against the current tree: | Item | Skill said | Head says | | --- | --- | --- | | vLLM pin | `0.26.0` | `0.32.0` at `5df63a72` | | vLLM-Ascend pin | `80d8c194f` | `bd42eea0` | | CI image tag | `VLLM_BASE_TAG=v0.26.0` | `v0.32.0`, asserted in `docker/Dockerfile.ci` | | Runner variants | V1/V2 maintained in pairs; GPU V2 gated off | no V1 and no fallback; both roles require `VLLM_USE_V2_MODEL_RUNNER=1`, Attention runs MRV2, FFN stays connector-driven | | CODEOWNERS | one global rule, no path owners | global catch-all plus `@ronenkat` on GPU and E2E paths, `@francisleely` on `/csrc/npu/` | | Design-page status | all pages treated as draft | `plugin_boundary.md` and `e2e_testing.md` are `normative`; their `ENTRY-INV-*` / `CFG-INV-*` / `E2E-INV-*` are blockers | Structural changes: - **Pins are read, not remembered.** The skill now instructs the reviewer to read `TARGET_VLLM_VERSION`, the design-page `upstream_refs`, `docker/Dockerfile.ci`, `pyproject.toml`, and the README from the frozen head. The values quoted inside the skill are labelled as its refresh-time `main` values, with the reviewed head winning on disagreement, so the next pin move degrades it to a hint rather than to a false assertion. - **Three FFN execution paths** are separated by their real selection contract — `connector.control_plane is not None`, `issubclass(connector_cls, AFDRoutedExpertsConnector)`, and async CAM on NPU — instead of being described as one runner pair. Routing by connector *name* is now a finding. - **The patch contract now covers patched methods in `afd_plugin/v1/worker/**`**, not only `afd_plugin/compat/`. Those files hold the heaviest patch surface (`attention_model_runner_v2.py` has 9 `# ### PATCH START` blocks, the NPU twin 6), and `tests/unit/compat/patches/test_signatures.py` already owns signature alignment for both. A newly patched function with no table row is a P1. - **New `references/modules/e2e-harness.md`** owns `tests/e2e/**`, which previously had no primary contract even though `e2e_testing.md` is one of the two `normative` pages. It records the current gate sets, the four-device bound, sample thresholds, replay evidence rules, and the no-skip rule. - **New `references/features/dspark-draft.md`** covers the DSpark draft model, its speculator patch, target/draft layered FFN wiring, and its exclusion set (no DBO, no FFN two-stream). - **`skip_ci.py` is recorded as an evidence hazard.** A docs-only or skip-mark-only diff suppresses the whole default pipeline, so a green run on such a PR proves nothing about the code. The skill now says this explicitly instead of treating CI as free validation. Machine-readable metadata for the added references is unchanged: the skill keeps its `name` and `agents/openai.yaml` display name. ## Test Plan Static checks only; this change touches documentation. - Resolve every relative Markdown link inside the skill directory. - Confirm every reference file is reachable directly from `SKILL.md`. - Re-verify each path, symbol, E2E case ID, and test file named in the skill against the current tree. - Confirm no stale pin or runner-variant language remains. ## Test Result Run on `codex/refresh-review-pr-skill` at `6bfc44b5`, based on upstream `main` `e778192d`: | Check | Result | | --- | --- | | Markdown files | 25 files, 1577 lines | | Internal relative links | 0 broken | | Reference reachability from `SKILL.md` | 25 of 25 linked | | Stale `0.26` / `80d8c194` / V1-V2 claims | none remaining | | pre-commit (typos, end-of-file, mixed-line-ending, trailing-whitespace, merge-conflict) | passed | | markdownlint-cli2 | not applicable; `.agents/` is excluded by `.pre-commit-config.yaml` | Two errors were found and fixed during verification, before staging: - a reference to `tests/unit/model_executor/npu/`, which does not exist; - a claim that the DeepEP M2N scenarios run in weekly CI — the Qwen3 MoE module collects them, but CI selects node IDs explicitly and does not run them. No runtime, device, or E2E validation applies: no production code changed. ## Docs Impact - Files updated: `.agents/skills/review-pr/SKILL.md` and its `references/` tree (24 modified, 2 added). - No changes to `docs/` or user-facing documentation. The skill is agent instructions, not user documentation, and markdownlint excludes `.agents/` by design. ---
Essential PR Checklist - [x] Purpose is clear and linked to public context when possible. - [x] Scope is bounded: one skill directory, documentation only. - [x] Compatibility with vLLM v0.32.0 and the pinned vLLM-Ascend source is considered; the skill now reads the pin instead of hard-coding it. - [x] No changes are made to the vLLM source checkout. - [x] Plugin-owned classes or explicit dotted class paths are preferred over monkey patches (not applicable; no production change). - [x] Any compat shim or monkey patch is isolated, idempotent, version-guarded, documented, and tested (none added). - [x] Imports remain CPU-safe; CUDA-heavy work is delayed or GPU-gated (no code changes). - [x] Validation evidence is included; no GPU tests were skipped because none apply. - [x] Documentation impact is stated.
🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- .agents/skills/review-pr/SKILL.md | 143 ++++++++++++------ .../checks/model-addition-checklist.md | 28 +++- .../references/checks/perf-verification.md | 34 +++-- .../checks/test-quality-evaluation.md | 80 ++++++++-- .../references/checks/tests-docs-checklist.md | 57 +++++-- .../references/checks/verification.md | 10 +- .../references/delivery/delivery-style.md | 48 +++--- .../references/delivery/review-requests.md | 24 +-- .../features/async-cam-ubatching.md | 62 +++++--- .../features/disaggregation-topologies.md | 43 ++++-- .../references/features/dspark-draft.md | 46 ++++++ .../features/graphs-and-compilation.md | 53 ++++--- .../references/modules/compat-patches.md | 51 +++++-- .../references/modules/connectors.md | 71 ++++++--- .../modules/distributed-topology.md | 32 ++-- .../references/modules/e2e-harness.md | 64 ++++++++ .../modules/engine-orchestration.md | 39 +++-- .../references/modules/execution-platforms.md | 54 ++++--- .../references/modules/model-integration.md | 27 +++- .../references/modules/npu-compat.md | 23 ++- .../references/modules/plugin-config.md | 37 ++++- .../references/modules/workers-runners.md | 55 +++++-- .../references/process/design-contracts.md | 49 +++++- .../references/process/general-checks.md | 70 ++++++--- .../references/process/review-execution.md | 22 ++- .../references/process/review-routing.md | 29 ++-- 26 files changed, 915 insertions(+), 336 deletions(-) create mode 100644 .agents/skills/review-pr/references/features/dspark-draft.md create mode 100644 .agents/skills/review-pr/references/modules/e2e-harness.md diff --git a/.agents/skills/review-pr/SKILL.md b/.agents/skills/review-pr/SKILL.md index 81d94c95d..c1c0151f2 100644 --- a/.agents/skills/review-pr/SKILL.md +++ b/.agents/skills/review-pr/SKILL.md @@ -1,6 +1,6 @@ --- name: review-pr -description: Review pull requests and local branches for vllm-project/afd-plugin with a frozen snapshot, module-design ownership, feature overlays, patch-contract enforcement, targeted validation, and concise evidence-backed findings. Use for default, detailed, or repeat reviews; checking correctness, NPU/GPU compatibility, vLLM 0.26.0 patch discipline, connectors, distributed topology, workers and model runners, tests, benchmarks, or model-family additions; and identifying or explicitly requesting code-owner reviewers. Use run-e2e instead to execute E2E suites, and AGENTS.md for authoring rules. +description: Review pull requests and local branches for vllm-project/afd-plugin with a frozen snapshot, module-design ownership, feature overlays, patch-contract enforcement, targeted validation, and concise evidence-backed findings. Use for default, detailed, or repeat reviews; checking correctness, NPU/GPU compatibility, pinned vLLM/vLLM-Ascend patch discipline, ModelRunnerV2-only execution, connectors and routed-experts transports, distributed topology, workers and model runners, E2E gate structure, tests, benchmarks, or model-family additions; and identifying or explicitly requesting code-owner reviewers. Use run-e2e instead to execute E2E suites, and AGENTS.md plus docs/UT_MAINTENANCE.md for authoring and test rules. --- # Review AFD Plugin Pull Requests @@ -21,8 +21,8 @@ Make every finding: - **Calibrated:** match severity to user and maintainer impact. Do not report unrelated backlog, style already enforced by pre-commit (ruff, -mypy, SPDX headers, markdownlint, DCO), or a missing test that would not -protect changed behavior. +mypy, typos, SPDX headers, markdownlint, shellcheck, DCO), or a missing test +that would not protect changed behavior. ## Select the input and depth @@ -38,6 +38,25 @@ checkouts; use another skill for unrelated repositories. Default to maintainer brevity. A detailed or audit request expands coverage and lists `path:line` findings, but keeps the same confidence and severity bar. +## Pins are data, not constants + +Version pins move with every upstream upgrade. Read them from the frozen head, +never from this skill and never from memory: + +- `afd_plugin/compat/vllm.py` — `TARGET_VLLM_VERSION` (single pin shared by + both backends, matched exactly, not by range). +- `docs/design/module/index.md` — `upstream_refs` (vLLM commit and vLLM-Ascend + commit) and `last_reviewed`. +- `pyproject.toml` optional `vllm` extra, `docker/Dockerfile.ci` + (`VLLM_BASE_TAG` plus its version assertion), and `README.md`'s supported + runtime table. + +The values quoted in this skill are the `main` values as of its last refresh +(vLLM `0.32.0` at `5df63a72`, vLLM-Ascend `bd42eea0`); a mismatch between the +skill and the reviewed head is expected and the head wins. A PR that moves a pin +without moving the gate, the CI image tag, the docs, and the design pages +together is a finding. + ## Reference guide Load references in review order: process, one primary module contract, matching @@ -56,7 +75,7 @@ drift. | --- | --- | | [review-execution.md](references/process/review-execution.md) | Every review; freeze inputs, inspect safely, and deliver against the same snapshot. | | [general-checks.md](references/process/general-checks.md) | Every review; apply repository-wide correctness, patch-contract, and evidence rules. | -| [design-contracts.md](references/process/design-contracts.md) | Every production review; resolve branch-local module design status. | +| [design-contracts.md](references/process/design-contracts.md) | Every production review; resolve branch-local module design status and ownership. | | [review-routing.md](references/process/review-routing.md) | After the diff census; select one primary module and conditional overlays. | ### Primary module contract @@ -64,22 +83,24 @@ drift. | Reference | Read when | | --- | --- | | [plugin-config.md](references/modules/plugin-config.md) | Plugin registration, `AFDConfig` schema, parse/validation, extra-config, or env flags change. | -| [compat-patches.md](references/modules/compat-patches.md) | A monkey patch is added or changed under `afd_plugin/compat/patches/`, or the vLLM/vLLM-Ascend version gate moves. | -| [npu-compat.md](references/modules/npu-compat.md) | `afd_plugin/compat/npu/` runtime init, deferred NPU patch application, feature validation, or Ascend ops loading changes. | -| [connectors.md](references/modules/connectors.md) | An attention↔FFN connector backend, its control plane, or transfer metadata changes. | +| [compat-patches.md](references/modules/compat-patches.md) | A monkey patch is added or changed under `afd_plugin/compat/patches/`, `afd_plugin/v1/worker/`, or the vLLM/vLLM-Ascend version gate moves. | +| [npu-compat.md](references/modules/npu-compat.md) | `afd_plugin/compat/npu/` runtime init, deferred NPU patch application, feature validation, routing ids, or Ascend ops loading changes. | +| [connectors.md](references/modules/connectors.md) | An attention↔FFN connector backend, routed-experts transport, its control plane, or transfer metadata changes. | | [distributed-topology.md](references/modules/distributed-topology.md) | AFD process groups, rank mapping, or topology validation changes. | | [engine-orchestration.md](references/modules/engine-orchestration.md) | Async-DP engine behavior, cross-role request routing, DBO yield, or ubatching orchestration changes. | -| [workers-runners.md](references/modules/workers-runners.md) | Role workers, model runners (V1/V2), graph capture, or attention/FFN metadata change. | +| [workers-runners.md](references/modules/workers-runners.md) | Role workers, ModelRunnerV2 runners, routed-experts/async FFN runners, graph capture, or attention/FFN metadata change. | | [execution-platforms.md](references/modules/execution-platforms.md) | NPU/GPU selection, platform checks, Ascend op build, or `csrc/` kernels change. | | [model-integration.md](references/modules/model-integration.md) | Model registration, role-split wrappers, weight filtering, or a model family changes. | +| [e2e-harness.md](references/modules/e2e-harness.md) | `tests/e2e/**`, case IDs, gate selection, sample counts, device layout, or `run-e2e` skill behavior change. | ### Feature-design overlays | Reference | Read when | | --- | --- | -| [disaggregation-topologies.md](references/features/disaggregation-topologies.md) | 2A2F/2A1F, colocation vs disaggregation placement, or recipe launch scripts change. | -| [async-cam-ubatching.md](references/features/async-cam-ubatching.md) | The async CAM connector, two-stage MoE ubatching, or DSV3.2/DSV4 async CAM forward paths change. | +| [disaggregation-topologies.md](references/features/disaggregation-topologies.md) | 2A2F/2A1F/1A1F/4A2F/4A4F, colocation vs disaggregation, multi-pod placement, or recipe launch scripts change. | +| [async-cam-ubatching.md](references/features/async-cam-ubatching.md) | The async CAM connector, layered W4A8/W8A8 FFN execution, FFN two-stream, or DSV3.2/DSV4 async CAM forward paths change. | | [graphs-and-compilation.md](references/features/graphs-and-compilation.md) | CUDA graph, ACL graph, capture metadata, or graph/eager fallback behavior changes. | +| [dspark-draft.md](references/features/dspark-draft.md) | DSpark draft-model speculation, its speculator patch, or target/draft layered FFN wiring changes. | ### Evidence and quality checks @@ -102,30 +123,31 @@ drift. ### 1. Freeze and report the snapshot -Pin the base and head before reading source or running validation. Within 60 seconds, -report the pinned head, CI, mergeability, and preliminary findings in the host -conversation. Do not wait for CI or post this update to GitHub. +Pin the base and head before reading source or running validation. Within 60 +seconds, report the pinned head, CI, mergeability, and preliminary findings in +the host conversation. Do not wait for CI or post this update to GitHub. -If the target changes while fetching, discard the evidence and retry once. If -it changes again, report the churn and wait for a stable target. +If the target changes while fetching, discard the evidence and retry once. If it +changes again, report the churn and wait for a stable target. For a trusted PR head, materialize the pinned head in an isolated detached worktree. A worktree freezes identity but is not a security sandbox. Treat fork heads as untrusted unless the user and environment policy explicitly establish -otherwise: execute them only in a disposable, secret-free sandbox with restricted -filesystem, network, and resources; without one, use static SHA-addressed reads -and CI evidence only. For a local review, freeze the committed, index, worktree, -and NUL-safe in-scope untracked contents. Follow +otherwise: execute them only in a disposable, secret-free sandbox with +restricted filesystem, network, and resources; without one, use static +SHA-addressed reads and CI evidence only. For a local review, freeze the +committed, index, worktree, and NUL-safe in-scope untracked contents. Follow [review-execution.md](references/process/review-execution.md) for trust gates, state fingerprints, and byte-for-byte staleness checks. ### 2. Build the diff census -Group files into production code, tests, docs, configuration, build/CI, and -vendored artifacts (for example `afd_plugin/connectors/npu/bin/`). Map each -changed production file and test group to the PR goal. Compare the title/body -claims with the actual diff; use linked issues only when they define the -contract or reproduction. +Group files into production code, tests, docs, configuration, build/CI, agent +skills, and vendored artifacts (for example +`afd_plugin/connectors/npu/bin/*.run|*.whl` and the packaged +`afd_plugin/_cann_ops_custom` tree). Map each changed production file and test +group to the PR goal. Compare the title/body claims with the actual diff; use +linked issues only when they define the contract or reproduction. Mark unrelated scope and unexplained generated artifacts. Do not infer behavior from the PR description without tracing the live code. @@ -139,7 +161,8 @@ module contract, a second only for a real documented cross-boundary call path, and every matching feature and evidence overlay. Treat titles and paths as hints; live behavior and the frozen head's current design metadata are authoritative. For docs-, tests-, or CI-only changes, route to the production -contract they protect or use only the applicable evidence checks. +contract they protect or use only the applicable evidence checks; E2E harness +changes route to [e2e-harness.md](references/modules/e2e-harness.md). ### 4. Run the blocker scan @@ -156,24 +179,40 @@ vLLM engine/config ingress (--additional-config {"afd": ...}) -> sampling/output in the owning role -> shutdown/teardown ``` -Cover every applicable GPU/NPU, graph/eager, DBO on/off, 2A2F/2A1F, -runner V1/V2, colocation/disaggregation, and AFD-on/AFD-off path. Patched code -must stay transparent when AFD is inactive. Search bounded callers and sibling -implementations rather than assuming the changed hunk is the only path. +Cover every applicable GPU/NPU, graph/eager, DBO on/off, 2A2F/2A1F/1A1F, +colocation/disaggregation, and AFD-on/AFD-off path, and the per-family variants +that actually exist: + +- connector backend: CUDA P2P, DeepEP M2N (routed experts), Ascend CAMP2P, + Ascend CAM async; +- FFN execution: control-plane-driven, routed-experts, async layered GMM, + experimental FFN two-stream; +- draft model on/off (DSpark) where the suite supports it. + +There is no V1 fallback on either platform: both roles require +`VLLM_USE_V2_MODEL_RUNNER=1`, Attention runs ModelRunnerV2, and FFN stays +connector-driven. Patched code must stay transparent when AFD is inactive +(`is_afd_active` false). Search bounded callers and sibling implementations +rather than assuming the changed hunk is the only path. ### 5. Apply module, feature, and patch contracts Apply the reference set selected in step 3 and any matching repo-local skill. Read the exact module and feature pages in the frozen head, including status, -ownership, `primary_code_paths`, dependencies, and `last_reviewed`. Candidate -or draft rules are questions, not blockers, unless current code, tests, or -AGENTS.md enforce them. Inspect both sides of every config, registration, -connector, rank-mapping, and patch boundary. - -For every change under `afd_plugin/compat/`, enforce the AGENTS.md patch -contract: pinned upstream refs, full upstream copy with marked AFD differences, -identical signatures, `Patch reason`/`Patch functionality` comments, and -CPU-safe test coverage. +owners, `primary_code_paths`, `depends_on`, `validation_paths`, and +`last_reviewed`. Only `plugin_boundary.md` and `e2e_testing.md` are currently +`normative`; every other page is `draft`, so its rules are questions, not +blockers, unless current code, tests, or AGENTS.md enforce them. Inspect both +sides of every config, registration, connector, rank-mapping, and patch +boundary. + +For every patch change — under `afd_plugin/compat/` and the patched methods in +`afd_plugin/v1/worker/**` — enforce the AGENTS.md patch contract: pinned +upstream refs, full upstream copy with marked AFD differences, identical +signatures, `Patch reason:`/`Patch functionality:` comments, `# Upstream +source:` pointers, and CPU-safe test coverage. The single parametrized table in +`tests/unit/compat/patches/test_signatures.py` owns upstream signature +alignment; a new patched function without a table row is a finding. When a diff adds or expands a helper, class, fallback, compatibility branch, or public behavior, run a subtraction pass: remove out-of-scope behavior and check @@ -190,9 +229,20 @@ imports, tests, builds, hooks, or repo-configurable tooling from an untrusted head on the reviewer host. - Treat CI as status evidence; inspect only the first overlapping failure. - Buildkite `test-ready` runs unit tests plus the DeepSeek-V2-Lite E2E gate; - NPU has no in-repo CI, so NPU evidence is author-provided or manually run. -- For docs-only changes, use diff hygiene, links checks, and bounded live +- Buildkite `test-ready` (L2) and `test-merge` (L3) run the CPU unit selection + — `tests/unit` with `-m "not gpu and not vllm_runtime"` plus the four + CPU-only `tests/npu` files the pipeline names — then + `tests/gpu/test_attention_routing.py`, on `l4_1`; the DeepSeek-V2-Lite GPU + seven-case E2E gate runs on `l4_4`. `test-weekly` adds Qwen3 MoE, Qwen3.6 + MoE, and the DeepSeek-V2-Lite 2A1F Graph + DBO case on `h100_4`. +- CI mirrors `docker/Dockerfile.ci`, which builds on + `vllm/vllm-openai:` and asserts the installed vLLM pin. +- `skip_ci.py` suppresses all default CI when the diff is docs-only + (`docs/**`, any `*.md`, `mkdocs.yml`) or changes only pytest skip marks. For + such a PR, a green pipeline is not evidence about the change. +- NPU has no in-repo CI (`.buildkite/npu/` does not exist yet); NPU evidence is + author-provided or manually run. +- For docs-only changes, use diff hygiene, link checks, and bounded live contract verification instead of dependency setup or pytest. - For hardware-dependent paths, run available static/CPU checks (`pytest -m "not gpu and not vllm_runtime"`) and name the exact GPU/NPU gap. Never @@ -238,9 +288,16 @@ permission from a request to review the code. ## Related skills -- `run-e2e` — execute the DeepSeek-V2-Lite / Qwen3 MoE / Qwen3.6 E2E gates; use - when the review needs active GPU or NPU evidence. +- `run-e2e` — execute the DeepSeek-V2-Lite / Qwen3 MoE / Qwen3.6 MoE and + DeepSeek-V4 Flash E2E gates; use when the review needs active GPU or NPU + evidence. - `adapt-model` — author-side workflow for adapting a new model family; suggest to contributors alongside the model-addition checklist. - `upgrade-npu` / `upgrade-gpu-version` — the pinned-ref upgrade workflow; a PR that moves vLLM or vLLM-Ascend pins should follow them. +- `upgrade-layered-gmm-ops` — replaying the A3 layered GMM selection onto a new + ops-transformer revision. +- `deploy-afd-k8s` / `run-vllm-bench-k8s` / `deploy-dsv4-flash-async` — + deployment and benchmark workflows; a PR that changes their inputs, recipes, + or skill scripts needs the matching owner. +- `maintain-vllm-main` — incremental vLLM-main tracking between pin upgrades. diff --git a/.agents/skills/review-pr/references/checks/model-addition-checklist.md b/.agents/skills/review-pr/references/checks/model-addition-checklist.md index 55affad46..8d6aff4ca 100644 --- a/.agents/skills/review-pr/references/checks/model-addition-checklist.md +++ b/.agents/skills/review-pr/references/checks/model-addition-checklist.md @@ -12,7 +12,10 @@ skill; this checklist is the reviewer's evidence bar. only). Each role constructs only its role-required components. - **Registration.** Entries in the `register_afd()` registration maps with the correct upstream architecture names; NPU/GPU variant selection - (`torch_npu` branch) resolved at registration, not in forward code. + (`torch_npu` branch) resolved at registration, not in forward code. The maps + are frozen (`MappingProxyType`) and the AFD alias is only installed when + `is_afd_active` is true — non-AFD workers must keep the checkpoint + architecture and resolve to the native vLLM class. - **Weight-role filtering.** `_checkpoint_weight_roles` coverage so no role loads weights it never reads and none drops weights it later needs; verify against the checkpoint's full weight list, including gate/expert and @@ -20,19 +23,28 @@ skill; this checklist is the reviewer's evidence bar. - **Upstream fidelity.** Copied model code follows the patch contract: marked AFD differences, `# Upstream source:` pointers, identical signatures. - **Config compatibility.** Direct attribute access to upstream config; - documented assumptions about quantization, MLA/attention backend, and - MoE settings the family requires. + documented assumptions about quantization, MLA/attention backend, and MoE + settings the family requires. +- **Connector/execution fit.** State which connectors and topologies the family + actually supports, and make the unsupported combinations fail at startup + rather than silently degrade. Family restrictions are real: Qwen3.5/3.6 GDN + DBO is unsupported; DeepEP M2N requires an all-MoE family with no shared + expert; Async CAM is DeepSeek-V4 W4A8/W8A8 only. ## Evidence required in the PR - A unit suite under `tests/unit/model_executor/` (CPU-safe, checkpoint- or - fixture-driven) covering role construction and weight filtering. -- An E2E suite or scenario under `tests/e2e/models/` with the four gate - scenarios (or the suite's stated gate set) and GSM8K-7 accuracy evidence. + fixture-driven) covering role construction, proxy fidelity, and weight + filtering — the existing naming is `test__construction.py`, + `test__proxy.py`, `test__weight_policy.py`. +- An E2E suite or scenario under `tests/e2e/models//` with the suite's + gate scenarios and GSM8K accuracy evidence, respecting the four-device bound + (`E2E-INV-002`) and the no-skip rule (`E2E-INV-004`). - A recipe under `recipe/` for at least one working topology, with launch scripts consistent with the user guides. -- Docs: the relevant `docs/design/module/` page mention (model integration) - and, for user-facing support, a user guide or guide section. +- Docs: the relevant `docs/design/module/` page (model integration) and, for + user-facing support, a user guide or guide section; the design-page + registration table must gain the new row. - For NPU support: manual validation evidence per `docs/npu/TESTING.md`; there is no NPU CI. diff --git a/.agents/skills/review-pr/references/checks/perf-verification.md b/.agents/skills/review-pr/references/checks/perf-verification.md index afa4159a6..8fe3fbe23 100644 --- a/.agents/skills/review-pr/references/checks/perf-verification.md +++ b/.agents/skills/review-pr/references/checks/perf-verification.md @@ -1,8 +1,8 @@ # Performance and Accuracy Verification Use when a PR claims latency, throughput, memory, or accuracy improvement — -including "[PERF]" titles, ubatching/CAM control-path optimizations, graph -changes, and routing changes. +including `[PERF]` titles, layered GMM / CAM control-path optimizations, graph +changes, TP/SP scheduling changes, and routing changes. ## Comparable A/B contract @@ -10,20 +10,23 @@ Accept a claim only with base and head runs that hold constant: - Hardware pool and placement (CI presets: `l4_1`–`l4_4`, `h100_4`; NPU: documented manual environment per `docs/npu/TESTING.md`). -- Container image and dependency versions (CI image pins - `VLLM_BASE_TAG=v0.26.0`). -- Workload and datasets (`tools/benchmarks/decode_bench.py` or the named E2E - scenario; GSM8K-7 for accuracy gates). -- Graph/eager mode, DBO on/off, ubatching mode, and topology (2A2F/2A1F, - colocation/disaggregation). +- Container image and dependency versions (`docker/Dockerfile.ci` pins + `VLLM_BASE_TAG` and asserts the installed vLLM version). +- Workload and datasets (`tools/benchmarks/decode_bench.py`, + `tools/benchmarks/request_generator.sh`, or the named E2E scenario; GSM8K + for accuracy gates). +- Graph/eager mode, DBO on/off, ubatching mode, FFN execution path, and + topology (1A1F/2A1F/2A2F/4A2F/4A4F, colocation/disaggregation, multi-pod). - Warmup, repetitions, and the metric definition (mean vs median, percentiles, - tokenization boundaries). + tokenization boundaries). The NPU benchmark harnesses deliberately do not + pool or interpolate samples — preserve that when comparing. ## Isolate per claim One claim, one comparison. A PR claiming several improvements (transfer -overlap, MoE split, control-path overhead) needs evidence per mechanism or a -staged ablation; a single end-to-end delta cannot attribute multiple causes. +overlap, MoE split, layered scheduling, control-path overhead) needs evidence +per mechanism or a staged ablation; a single end-to-end delta cannot attribute +multiple causes. ## Evidence table @@ -32,18 +35,21 @@ staged ablation; a single end-to-end delta cannot attribute multiple causes. | Latency | Decode bench per-step or E2E per-request timing, same batch mix | | Throughput | Tokens/s at stated concurrency, steady-state window | | Memory | Peak allocator/device memory, same graph/capture settings | -| Accuracy | GSM8K-7 gate plus the model suite's accuracy scenario; state pass criteria | +| Accuracy | GSM8K gate plus the model suite's accuracy scenario; state pass criteria and sample count | ## Reviewer verification ladder 1. Check the A/B contract above against the PR's Test Result section; missing constants make the claim unproven, not merely weak. 2. When a runnable GPU path exists, offer to rerun the named benchmark or E2E - scenario via the `run-e2e` skill; otherwise mark the claim unverified and - name the exact missing run. + scenario via the `run-e2e` skill (or `run-vllm-bench-k8s` for a deployed + recipe); otherwise mark the claim unverified and name the exact missing run. 3. For NPU claims, require the manual-run evidence (environment, command, numbers) in the PR; there is no NPU CI, and unit tests never substitute for device measurements. +4. Watch for numbers carried over from a previous pin: evidence measured on an + older vLLM/vLLM-Ascend pair does not support a claim about the current pair. + The design pages mark such rows historical for the same reason. An unproven perf claim on a hot path is a P1 finding (missing blocking evidence), stated as the specific missing comparison — not as distrust. diff --git a/.agents/skills/review-pr/references/checks/test-quality-evaluation.md b/.agents/skills/review-pr/references/checks/test-quality-evaluation.md index 9ac0a7fb7..a10b34664 100644 --- a/.agents/skills/review-pr/references/checks/test-quality-evaluation.md +++ b/.agents/skills/review-pr/references/checks/test-quality-evaluation.md @@ -1,13 +1,14 @@ # Test Quality Evaluation Use when tests change in the diff, are absent for risky changed code, or may -pass without exercising production behavior. +pass without exercising production behavior. Standards live in AGENTS.md +("Test Conventions") and `docs/UT_MAINTENANCE.md`; this file is the reviewer's +application of them. ## CPU-safe imports first -The unit suite must run on a CPU-only host without torch. The recurring -regression class (fixed in `f5fa9fc`) is a test that imports a worker or -connector module before guarding its runtime dependencies. Enforce: +The unit suite must run on a CPU-only host without torch, torch_npu, or vLLM. +Enforce: - Runtime dependencies (`vllm`, `vllm_ascend`, `torch_npu`, `torch`) are gated with `pytest.importorskip` (or the `_require_npu_runtime()` helper in @@ -16,10 +17,13 @@ connector module before guarding its runtime dependencies. Enforce: import, never after. - Module-level imports in test files stay CPU-safe; a top-level `from afd_plugin.v1.worker... import ...` in a test file breaks collection - on CPU hosts even if every test body is guarded. + on CPU hosts even if every test body is guarded. This is the recurring + regression class; check it before anything else. - New tests pass in the CI selection: `pytest tests/unit -m "not gpu and not - vllm_runtime"`. Run (or reason through) the selection for the changed - directory. + vllm_runtime"`, plus the four CPU-only `tests/npu` modules CI names + (`test_async_cam_harness.py`, `test_async_cam_performance.py`, + `test_async_cam_golden.py`, `test_gmm_layered_performance.py`). Run (or + reason through) the selection for the changed directory. ## Markers and selection @@ -31,16 +35,62 @@ connector module before guarding its runtime dependencies. Enforce: | `eval` | Datasets/accuracy | Missing dataset guard | | `slow` | > 120 s | Unmarked long tests that inflate CI time | +Note that `tests/npu` is not uniformly device-gated: several modules there are +CPU host-harness checks that CI runs directly. Do not demand an `npu` marker on +those, and do not assume a `tests/npu` path implies hardware. + ## Quality bar +- Assert the seam, not the plumbing: return values, raised errors and their + messages, protocol messages sent to a peer, final file or environment state, + config-validation outcomes. Private attributes, internal-collaborator + arguments, and internal helper call counts are forbidden. Call order is a + valid assertion only when the order is itself the contract (connector + handshakes, service announce, DBO stage order). +- The refactor test: if the implementation is restructured without changing a + protocol, the test must stay green. A test that cannot fail when behavior + regresses is not coverage — flag it and ask for deletion rather than + retention. - Tests exercise production behavior, not a mock echo: patched functions get AFD-on and AFD-off assertions; state machines get abort/timeout paths; fail-fast contracts assert the error message, not just `pytest.raises`. -- Device-independent logic (planners, rank mapping, config validation) is - tested CPU-side with `parametrize`; device-only behavior is honestly - delegated to E2E scenarios rather than fake-simulated in unit tests. -- A risky change without a test that would protect it is a P2 finding (P1 if - the untested path already regressed before); name the specific behavior to - cover and the cheapest harness for it. -- Do not demand coverage the repo's structure already provides elsewhere - (E2E gates, package tests, upstream CI). +- Shared fakes live in `tests/unit/fakes/`, one module per upstream concept; a + builder defined in two test modules is a layout bug. A test patching more + than ~4 attributes of the module under test is testing its own mocks. +- Setup over ~15 lines belongs in a fixture or factory; a test body over ~50 + lines needs `# long-test: `; the third test with the same prefix and + setup becomes a `parametrize` table. +- Deterministic: no sleeps, no wall-clock assertions, no reliance on set + iteration order. Concurrency is coordinated with events or bounded + conditions. Numerical tolerances are justified. +- Text artifacts (build manifests, patch markers, docs links) may be asserted + as text. Production Python modules may not: no test reads + `afd_plugin/**/*.py` and asserts on a substring. +- File placement mirrors the source tree under `tests/unit//`, + and a new test file must fall under an existing `validation_paths` entry in + `docs/design/module/`. A test file owned by no module is a finding — or, if + it is a full-stack check, it belongs in `tests/e2e`. +- Upstream signature alignment stays in the single table in + `tests/unit/compat/patches/test_signatures.py`; inheritance alignment is + mypy's job, not pytest's. +- Device-independent logic (planners, rank mapping, config validation, layered + layout math) is tested CPU-side with `parametrize`; device-only behavior is + honestly delegated to E2E or `tests/gpu`/`tests/npu` rather than + fake-simulated in unit tests. + +## Lifecycle + +- A PR that removes behavior deletes its tests and now-unused helpers in the + same PR. Skipping is not retirement; a skip needs an issue link. +- Merging duplicate tests is acceptable only when the retained test still + detects the relevant failures; similar bodies or equal expected values are + not sufficient justification. +- Never make CI green with retries, weaker checks, relaxed tolerances, or + deleted necessary coverage. A gate case turned into a skip is a P1. + +## Finding severity + +A risky change without a test that would protect it is a P2 finding (P1 if the +untested path already regressed before); name the specific behavior to cover +and the cheapest harness for it. Do not demand coverage the repo's structure +already provides elsewhere (E2E gates, package tests, upstream CI). diff --git a/.agents/skills/review-pr/references/checks/tests-docs-checklist.md b/.agents/skills/review-pr/references/checks/tests-docs-checklist.md index 969acd35d..f1e83832b 100644 --- a/.agents/skills/review-pr/references/checks/tests-docs-checklist.md +++ b/.agents/skills/review-pr/references/checks/tests-docs-checklist.md @@ -5,15 +5,33 @@ technical review, before delivery. ## CI coverage -- GitHub Actions runs pre-commit only (ruff, format, mypy/ty, typos, - markdownlint, actionlint, shellcheck, SPDX headers, Buildkite YAML checks, - DCO). Treat lint findings as gate status, not review findings. -- Buildkite: `test-ready` (L2 premerge) runs unit tests (`l4_1`) plus the - DeepSeek-V2-Lite E2E gate (`l4_4`); `test-merge` (L3) repeats post-merge; - `test-weekly` (main or `weekly-test` label) adds Qwen3 MoE, Qwen3.6, and - DSV2-Lite 2A1F DBO on `h100_4`. -- A change touching an area no pipeline covers (NPU paths, non-gate model - suites, `csrc/` kernels) needs author-provided evidence; name the gap. +- GitHub Actions runs pre-commit only (check-yaml, merge-conflict, end-of-file, + mixed-line-ending, trailing-whitespace, ruff check/format, typos, + actionlint, markdownlint, mypy on changed files for 3.11 with 3.12/3.13 + manual-only, shellcheck, SPDX headers, Buildkite YAML validation, DCO). Treat + lint findings as gate status, not review findings. +- Pre-commit only inspects files changed in the push/PR (`--from-ref/--to-ref`); + a violation in an untouched file is not this PR's finding. +- Buildkite image: `docker/Dockerfile.ci` builds on + `vllm/vllm-openai:${VLLM_BASE_TAG}` (currently `v0.32.0`) and asserts the + installed vLLM version equals the pin. A PR that changes the pin without + changing that assertion leaves CI unable to catch drift. +- `test-ready` (L2) and `test-merge` (L3) run on `l4_1`: the CPU unit + selection (`pytest -m "not gpu and not vllm_runtime" tests/unit` plus + `tests/npu/test_async_cam_harness.py`, `test_async_cam_performance.py`, + `test_async_cam_golden.py`, `test_gmm_layered_performance.py`) and + `pytest tests/gpu/test_attention_routing.py`. The DeepSeek-V2-Lite + seven-case GPU E2E gate runs on `l4_4`. +- `test-weekly` (main + `WEEKLY=1`, or PR label `weekly-test`) adds Qwen3 MoE + (four cases), Qwen3.6 MoE (three cases), and DeepSeek-V2-Lite + `afd-graph-dbo-2a1f` on `h100_4`. +- `skip_ci.py` suppresses the whole default pipeline for docs-only + (`docs/**`, any `*.md`, `mkdocs.yml`) or skip-mark-only diffs; label-only CI + YAML edits gate per platform and level. Say what a green run does and does + not cover. +- A change touching an area no pipeline covers (NPU paths, `csrc/` kernels, + non-gate model suites, `tests/gpu/` device tests) needs author-provided + evidence; name the gap. ## PR template and checklist @@ -21,12 +39,14 @@ The template requires Purpose / Issue / Scope / Implementation Notes / Test Plan / Test Result / Docs Impact, plus the Essential PR Checklist. Verify the high-signal items rather than restating them: -- vLLM 0.26.0 compatibility considered; no vLLM source-checkout changes. -- Plugin-owned classes or dotted paths preferred over monkey patches; any - shim isolated, idempotent, version-guarded, documented, tested. -- Imports remain CPU-safe; CUDA-heavy work delayed or GPU-gated. +- vLLM and, for NPU changes, the pinned vLLM-Ascend revision compatibility + considered; no vLLM source-checkout changes. +- Plugin-owned classes or dotted paths preferred over monkey patches; any shim + isolated, idempotent, version-guarded, documented, and tested. +- Imports remain CPU-safe; CUDA/NPU-heavy work delayed or device-gated. - Validation evidence included, including skipped GPU tests and NPU manual - evidence where applicable. + evidence, with exact upstream revisions and backend-scoped results where + native builds or ABIs matter. - DCO `Signed-off-by` present (the pre-commit hook auto-adds it; a missing sign-off blocks the DCO gate). @@ -37,8 +57,13 @@ high-signal items rather than restating them: [design-contracts.md](../process/design-contracts.md)). - User-facing changes (config fields, connector usage, launch flows) update the relevant user guide (`docs/gpu/`, `docs/npu/`) or state why not. -- Topology/launch changes update `recipe/` scripts consistently; E2E gate - structure changes update `docs/design/module/e2e_testing.md`. +- Topology/launch changes update `recipe/` scripts and their READMEs + consistently; E2E gate structure changes update + `docs/design/module/e2e_testing.md`. +- `.agents/skills/**` are executable instructions, not prose: a change to + commands, case lists, pins, or workflows that leaves a SKILL.md stating the + old behavior is a P2 finding. Markdownlint excludes `.agents/`, so nothing + else catches it. - New source files carry SPDX headers; user guides keep the documented env vars and defaults in sync with `AFDConfig`. diff --git a/.agents/skills/review-pr/references/checks/verification.md b/.agents/skills/review-pr/references/checks/verification.md index e2708bfd0..fe506ad5e 100644 --- a/.agents/skills/review-pr/references/checks/verification.md +++ b/.agents/skills/review-pr/references/checks/verification.md @@ -8,6 +8,7 @@ and the review needs evidence beyond static reading and CI. | Available environment | Verification | | --- | --- | | GPU host with devices free | Narrowest changed unit tests, then the single most relevant E2E scenario via `run-e2e` | +| GPU host for connector device tests | `tests/gpu/test_deepep_m2n_gpu.py` on SM90+ with afdEP-comm and DeepGEMM; most cases need 4 GPUs, the Attention-TP cases need 6 | | NPU host | Manual per `docs/npu/TESTING.md`; only what the user explicitly asked to run | | CPU-only trusted checkout | Import preflight + `pytest tests/unit/ -m "not gpu and not vllm_runtime"` | | Static only (untrusted head, no sandbox) | SHA-addressed reads + existing CI evidence; report execution as a gap | @@ -19,13 +20,16 @@ and the review needs evidence beyond static reading and CI. 2. Start from the narrowest command that can change a conclusion; a full E2E suite is never the first run. 3. E2E runs go through the `run-e2e` skill with its prerequisite checks - (devices, model path, `HF_HOME`, mirror reachability); a missing - prerequisite fails the plan — it never becomes a silent skip. + (devices, model path, `HF_HOME`, mirror reachability); a missing prerequisite + fails the plan — it never becomes a silent skip. 4. Record every run: head SHA, snapshot fingerprint, command, result, Python - and platform, dependency fingerprint. + and platform, dependency fingerprint, and the vLLM/vLLM-Ascend revisions. 5. Classify failures before drawing conclusions: environment/setup failures are not product evidence; a product failure on the changed path is a finding; an unrelated pre-existing failure is context, cited not fixed. + Known open failures in the current tree include DeepSeek-V2-Lite DBO + shutdown and NPU graceful exit — do not present them as this PR's regression + without a base-commit comparison. Never simulate device evidence: no fabricated NPU numbers, no "should pass" substitutions for hardware gates. If the gap cannot be closed here, the diff --git a/.agents/skills/review-pr/references/delivery/delivery-style.md b/.agents/skills/review-pr/references/delivery/delivery-style.md index 2715a5af8..fdf4a7afc 100644 --- a/.agents/skills/review-pr/references/delivery/delivery-style.md +++ b/.agents/skills/review-pr/references/delivery/delivery-style.md @@ -7,24 +7,31 @@ direct engineering tone of its merged history — into comment craft. ## Recurring review behavior - Start from the changed contract, not style: pre-commit already owns ruff, - format, SPDX, markdownlint, and DCO; never restate them. + format, typos, SPDX, markdownlint, shellcheck, and DCO; never restate them. - Protect the boundaries that make the plugin maintainable: patch fidelity - against pinned refs, CPU-safe imports, role split, platform twins, AFD-off - transparency. These are the findings this repo's maintainers need most. + against pinned refs, CPU-safe imports, role split, platform twins, the + no-V1/ModelRunnerV2-only contract, AFD-off transparency, and E2E gate + integrity. These are the findings this repo's maintainers need most. - Challenge scope when a PR smuggles in an unexplained abstraction, a new - global, or a second mechanism for an existing one; ask for the smallest - version. -- Demand evidence exactly where it is load-bearing: perf claims, NPU - behavior, E2E gate impact — and nowhere else. -- Address named owners (`@hsliuustc0106`, `@jiangkuaixue123`) for - architectural questions, not a generic "PTAL". + global, a second mechanism for an existing one, or a permissive fallback + where the design says "fail at startup"; ask for the smallest version. +- Demand evidence exactly where it is load-bearing: perf claims, NPU behavior, + E2E gate impact — and nowhere else. +- Address the owner who actually holds the contract. CODEOWNERS is no longer a + single global rule: GPU paths add `@ronenkat`, `/csrc/npu/` adds + `@francisleely`, E2E paths add `@ronenkat`, and `e2e_testing.md` is owned by + `@yujuancao07`. Use the design page's `owners` plus the path-matched + CODEOWNERS entry, not a generic "PTAL". - Defer cleanup to follow-ups; do not block on debt the PR did not create. +- Known open failures are context, not ammunition: cite DeepSeek-V2-Lite DBO + shutdown or NPU graceful-exit limitations only to explain why the PR's + evidence differs from a clean pass. ## Write the finding Lead with the issue; no preamble, no praise sandwich. One sentence when the fix is obvious. Prefer root-cause comments ("this breaks patch comparability -at the next upgrade") over restating the diff. Hedge only when evidence is +at the next pin move") over restating the diff. Hedge only when evidence is genuinely incomplete, and say what would complete it. Format (severity tiers from @@ -35,19 +42,22 @@ Format (severity tiers from . . . ``` -Patch-contract findings name the exact missing element (marker, signature -parity, upstream source pointer, pinned ref) because that is what the next -upgrade will trip over. +Patch-contract findings name the exact missing element (marker, `Patch reason:` +/ `Patch functionality:` comment, signature parity, upstream source pointer, +pinned ref) because that is what the next upgrade will trip over. Gate findings +name the invariant (`E2E-INV-004`) and the case ID. Calibrate to roughly 1–5 short comments; treat that as calibration, not a -quota. Zero comments is a valid outcome — say so and name material -validation gaps. +quota. Zero comments is a valid outcome — say so and name material validation +gaps. ## Anti-patterns - Generic praise, "Nit:" prefixes, decorative severity labels. -- Audit-template artifacts: rule IDs, checklist restatements, comment-count - narration. -- Style findings pre-commit owns, or AGENTS.md wording quotes with no - concrete failure. +- Audit-template artifacts: rule IDs as decoration, checklist restatements, + comment-count narration. +- Style findings pre-commit owns, or AGENTS.md wording quotes with no concrete + failure. - Batching many small findings that share one root cause; merge them. +- Reporting a stale pin as a finding when the head deliberately moved it in an + upgrade PR that also moved the gate, image, docs, and design pages. diff --git a/.agents/skills/review-pr/references/delivery/review-requests.md b/.agents/skills/review-pr/references/delivery/review-requests.md index e6ee29b05..30203bf9b 100644 --- a/.agents/skills/review-pr/references/delivery/review-requests.md +++ b/.agents/skills/review-pr/references/delivery/review-requests.md @@ -7,24 +7,28 @@ authorization for this PR or session. ## Owners -`CODEOWNERS` is a single global rule: `* @hsliuustc0106 @jiangkuaixue123`. -The design pages under `docs/design/module/` declare the same owners. There -are no path-specific owners, so route by contract expertise instead: +`.github/CODEOWNERS` has a global catch-all +(`* @hsliuustc0106 @jiangkuaixue123`) plus path-specific entries: -| Changed contract | Ask for | +| Path | Adds | | --- | --- | -| Patches, version pins, platform selection | Owner with upstream vLLM/vLLM-Ascend upgrade context | -| Connectors, topology, async CAM | Owner with the connector/NPU runtime context | -| Model families, E2E gates | Owner with the model-suite context | -| CI / Buildkite pipelines | Owner with CI pool context | +| `/afd_plugin/connectors/gpu/`, `/csrc/gpu/`, `/recipe/gpu/`, `/tests/gpu/`, `/docs/gpu/`, `/.buildkite/cuda/` | `@ronenkat` | +| `/csrc/npu/` | `@francisleely` | +| `/tests/e2e/`, `/.agents/skills/run-e2e/`, `/.agents/skills/deploy-afd-k8s/` | `@ronenkat` | + +Design pages declare their own `owners` under `docs/design/module/`; most list +`@hsliuustc0106` and `@jiangkuaixue123`, while `e2e_testing.md` is owned by +`@yujuancao07` alone. Prefer the design page's owners for the *contract*, and +CODEOWNERS for the *paths*; when they disagree, say which one you are +following. ## Procedure 1. From the frozen head, map the diff census to the primary module contract and feature overlays. 2. Propose one to three focused reviewers with an explicit rationale naming - the contract ("patch contract + async CAM ubatching"), never a bare - "PTAL". + the contract ("patch contract + async CAM layered FFN", "E2E gate + selection"), never a bare "PTAL". 3. Check existing reviews and requests first; do not duplicate an owner already engaged on the PR threads. 4. When authorized to post, recheck the head SHA, then post at most one diff --git a/.agents/skills/review-pr/references/features/async-cam-ubatching.md b/.agents/skills/review-pr/references/features/async-cam-ubatching.md index 341d7e57c..57489f48b 100644 --- a/.agents/skills/review-pr/references/features/async-cam-ubatching.md +++ b/.agents/skills/review-pr/references/features/async-cam-ubatching.md @@ -1,34 +1,54 @@ -# Async CAM and Ubatching +# Async CAM, Layered FFN, and Ubatching -Load with the primary module contract for changes to the experimental async -CAM data path: `CAMAsyncAFDConnector`, AFD-managed two-stage MoE ubatching, -the NPU ubatch wrappers/metadata, DBO yield interplay, and the DSV3.2/DSV4 -async CAM forward modules under `afd_plugin/model_executor/models/npu/`. +Load with the primary module contract for changes to the async CAM data path: +`CAMAsyncAFDConnector` and `AFDAsyncTransferState`, AFD-managed native +ubatching, the NPU async FFN runner and its two-stream mode, DBO yield +interplay, the layered GMM executor, and the DSV3.2/DSV4 async CAM forward +modules under `afd_plugin/model_executor/models/npu/`. Designs: `docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md`, -`docs/design/module/connector_contracts.md`; recent work concentrates here -(DSV4 CAM FFN control path, `v0.26.0_camasync_dsv4`). +`docs/npu/ASYNC_FFN_RUNTIME.md`, `docs/npu/CAM_ASYNC_ROUTED_OPS.md`, +`docs/design/module/connector_contracts.md`, +`docs/design/module/ffn_runtime.md`. ## Feature checks -- Respect experimental status: async CAM must not become a default or silently - required path; the opt-in remains explicit in config/recipes, and the sync - CAM P2P path keeps working. -- Preserve MoE semantics across the two-stage split: gate computation - placement, shared-expert handling, and topk routing inputs must match the - single-stage semantics the model defines; the execution planner - (`model_executor/npu/async_cam_ubatching.py`) is pure planning — no device - side effects. +- Respect the layer's status. `CAMAsyncAFDConnector` itself is a supported + product path for DeepSeek-V4 W4A8/W8A8; the *experimental* parts are FFN + two-stream (`ffn_two_stream`, default off) and any DP layout beyond the + documented ones. Experimental flags must not become defaults or silently + required paths; the synchronous `CAMP2pAFDConnector` path keeps working, and + DSpark must not acquire DBO or two-stream. +- Layered GMM is the only Async CAM FFN compute path. Incompatible weights + fail startup rather than selecting another FFN path — a PR that adds a + fallback contradicts `ffn_runtime.md`. `dynamicQuant=1` is required; + `w13_scale_bias`/`w2_scale_bias` are required for W4A8; W8A8 uses INT8 NZ + tensors with a single shared zero bias; `swiglu_limit` is applied in both + eager and FULL while W8A8 keeps its unclipped computation. +- Keep `AsyncCAMLayeredExecutor` and `LayeredFFNWeights` pure about device + side effects where the design says so: planning (`async_cam_layout.py`) is + host-side and testable on CPU, device execution belongs to the executor. - Prove state-machine safety: `AFDAsyncTransferState`, async FFN work items, - and ubatch contexts handle abort, timeout, and shutdown at every pending - state; a dropped work item or a lost completion signal hangs a rank. + ubatch contexts, and two-stream slots handle abort, timeout, and shutdown at + every pending state; a dropped work item or a lost completion signal hangs a + rank. FULL retires a slot only after its send is observed complete; eager + slot reuse requires the same guarantee. - Keep hidden-state handoff correct at ubatch boundaries: residual streams, - layernorm state, and metadata keys (`AscendUbatchMetadata`, graph metadata) - must be consistent between attention and FFN sides and across graph capture. + layernorm state, and graph/ubatch metadata keys must be consistent between + attention and FFN sides and across capture. +- Both roles select `VLLM_USE_V2_MODEL_RUNNER=1`; Attention stays eager under + Async CAM (`--enforce-eager`) and may enable native DBO with exactly two + microbatches, while FFN never joins Attention's DBO thread schedule. Native + DBO on FFN is rejected. - Require evidence proportional to risk: control-path or scheduling changes - here carry perf implications (the DSV4 CAM FFN work was a `[PERF][NPU]` - change) — demand comparable base/head NPU measurements or an explicit + here carry perf implications and have historically shipped as `[PERF][NPU]` + changes — demand comparable base/head NPU measurements or an explicit no-impact rationale; unit tests alone do not prove device behavior. +- Cleanup is part of the contract: a pending CAM receive cannot be interrupted + cleanly, so the E2E harness uses a marker-based SIGKILL sweep with a 120 s + survivor window. Changes that touch receiver teardown must keep that path + working; a "fix" that makes teardown look clean by suppressing survivors is + a regression. Name the NPU evidence (who ran what, on which environment per `docs/npu/TESTING.md`) for any behavioral claim; CPU tests cover planning and diff --git a/.agents/skills/review-pr/references/features/disaggregation-topologies.md b/.agents/skills/review-pr/references/features/disaggregation-topologies.md index 210985c86..b5d920306 100644 --- a/.agents/skills/review-pr/references/features/disaggregation-topologies.md +++ b/.agents/skills/review-pr/references/features/disaggregation-topologies.md @@ -2,38 +2,51 @@ Load with the primary module contract whenever a review touches role placement, rank counts, or launch scripts: attention/FFN colocation vs -prefill-decode disaggregation, 2A2F vs 2A1F rank counts, and the recipe -scripts that encode them. +prefill-decode disaggregation, the A/F rank ratios, and the recipe scripts that +encode them. Designs: `docs/design/module/attention_runtime.md`, -`docs/design/module/ffn_runtime.md`; recipes under `recipe/gpu/` and -`recipe/npu/`. +`docs/design/module/ffn_runtime.md`, `docs/design/module/e2e_testing.md`; +recipes under `recipe/gpu/` and `recipe/npu/`. ## Topology map | Topology | Where it lives | | --- | --- | | Baseline (native DP/TP, no AFD) | `baseline-graph` E2E scenarios; native DP4/TP1/EP4 | -| GPU NCCL P2P, colocation + disaggregation | `recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/` | -| NPU CAM P2P disaggregation | `recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/` | -| NPU async CAM | `recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/` | -| 2A2F / 2A1F gates | `tests/e2e/models/*/` scenario matrix | +| CUDA NCCL P2P, colocation | `recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/` (2A2F and 4A4F, eager/graph, DBO variants, plus `multipod_2a_2f_graph_dbo_dp1tp2.sh`) | +| CUDA NCCL P2P, disaggregation | `recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_disaggregation/` (2P1A1F eager/graph DBO) | +| CUDA NCCL P2P, Qwen3.5 multi-pod | `recipe/gpu/P2pNcclAFDConnector/qwen3_5_122b_a10b_fp8/prefill_decode_colocation/multipod_2a_2a_2f_graph.sh` | +| CUDA DeepEP M2N (routed experts) | `recipe/gpu/DeepEPM2NAFDConnector/{deepseek_v2_lite,qwen3_30b_a3b,qwen3_235b_a22b_fp8}/` (2A2F, and 4A2F for the 235B FP8 case) | +| NPU CAM P2P disaggregation | `recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/` (attention/FFN/baseline scripts) | +| NPU async CAM | `recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/v0_26_accuracy/` and `deepseek_v4_flash/` (DSpark A/F) | +| Gate matrices | `tests/e2e/models/*/` scenario lists; `docs/design/module/e2e_testing.md` | ## Feature checks - Validate `num_attention_ranks` / `num_ffn_ranks` against the actual world - size on every rank before collectives; a 2A1F misconfiguration must fail at + size on every rank before collectives; a misconfiguration must fail at startup with a named mismatch, not deadlock mid-run. +- Keep the per-suite topology contract: DeepSeek-V2-Lite gate cases default to + the V2 DP2/TP2 families on GPU and 2A2F on NPU; Qwen3 MoE and Qwen3.6 MoE use + 2A1F; DeepEP M2N requires at least two FFN ranks for EP, so its DeepSeek and + Qwen3 cases run 2A2F (4A2F for 235B FP8). A new case that breaks the suite's + default needs an explicit reason in the design page. +- Keep the four-device bound (`E2E-INV-002`): a PR case must not exceed four + unique devices without changing that normative invariant. - Treat a launch-script change in `recipe/` as a user contract: env vars, - device counts, and model paths in recipes must stay consistent with the E2E - scenarios and user guides that document them. + device counts, and model paths must stay consistent with the E2E scenarios + and user guides that document them. `VLLM_USE_V2_MODEL_RUNNER=1` is required + on both roles in every current recipe — a recipe that drops it will fail at + startup validation. - Keep baseline comparability: a topology or launch change that affects E2E gates must state how `baseline-graph` and AFD scenarios remain comparable. - Cover the matrix edge the change implies (graph/eager, DBO on/off, DBO + - 2A1F) in at least one named scenario or an explicit follow-up. -- Cross-node setups additionally - name host/port and rank assumptions; flag silent assumptions about network - interfaces or device ordering. + 2A1F, TP variants) in at least one named scenario or an explicit follow-up. +- Cross-node setups additionally name host/port and rank assumptions; multi-pod + runners live in `tests/e2e/multi_pod/` and + `.agents/skills/run-e2e/resources/k8-multi-pod.md`. Flag silent assumptions + about network interfaces, NVLink domains, or device ordering. Name the scenario or recipe that exercises the changed topology; for a new topology, require the E2E case and design-page update together. diff --git a/.agents/skills/review-pr/references/features/dspark-draft.md b/.agents/skills/review-pr/references/features/dspark-draft.md new file mode 100644 index 000000000..902df0753 --- /dev/null +++ b/.agents/skills/review-pr/references/features/dspark-draft.md @@ -0,0 +1,46 @@ +# DSpark Draft-Model Speculation + +Load with the primary module contract when a change touches DSpark: the +draft-model registration (`DSparkDraftModel` → +`AFDDSparkDeepseekV4ForCausalLM`), the speculator seam +(`afd_plugin/v1/worker/npu/dspark_speculator.py`), the target/draft layered FFN +wiring (`models/npu/deepseek_v4_dspark.py`), or the DSpark E2E evidence +harnesses under `tests/e2e/models/deepseek_v4_flash/`. + +Designs: `docs/design/module/model_integration.md`, +`docs/design/module/execution_platforms.md`, +`docs/design/module/attention_runtime.md`; +guide: `recipe/npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md`. + +## Scope + +DSpark is an Ascend-only draft model paired with a DeepSeek-V4 target under +`CAMAsyncAFDConnector`. The attention role keeps native eager proposal; the FFN +role may independently use layered eager or `FULL` graph execution. Supported +combinations are enumerated in `execution_platforms.md`; everything else is +excluded rather than best-effort. + +## Feature checks + +- Respect the excluded set: DSpark does not support DBO or FFN two-stream, and + it does not extend to model families outside DeepSeek-V4. A PR that enables + either — even behind a flag — needs a design-page change and device evidence + first. +- The speculator patch (`AFDDSparkSpeculator.set_attn`, `.propose`) must follow + the patch contract against the pinned vLLM-Ascend ref, including signature + parity and `Patch reason:`/`Patch functionality:` comments. +- Target and draft must agree on layered FFN weight extraction + (`_extract_async_cam_layered_weights`) and on the layer set each side runs; + a mismatch fails at startup or, worse, silently runs a partial draft. +- The prefill/decode split used by the DSpark recipes is a launch contract: + `MODEL`, `DSPARK_CONFIG`, TP/DP/EP counts, and the upstream worker choice on + P must stay consistent between the recipe, the README, and the E2E harness. +- Evidence for a behavioral claim comes from the DSpark evidence harnesses + (`dspark_eager.py`, `dspark_evidence.DSparkEagerEvidence`, + `dspark_ffn_graph_evidence.DSparkFFNGraphEvidence`), which exercise SWA + boundaries and chunked prompts. A unit test alone cannot qualify draft + execution. + +Test the draft/target wiring CPU-side in +`tests/unit/model_executor/models/npu/test_deepseek_v4_dspark.py` and name the +NPU run for anything device-dependent. diff --git a/.agents/skills/review-pr/references/features/graphs-and-compilation.md b/.agents/skills/review-pr/references/features/graphs-and-compilation.md index 33dd8c361..84a48b4ba 100644 --- a/.agents/skills/review-pr/references/features/graphs-and-compilation.md +++ b/.agents/skills/review-pr/references/features/graphs-and-compilation.md @@ -1,9 +1,11 @@ # Graphs and Compilation Load with the primary module contract for changes to graph capture or -compilation paths: FULL_DECODE_ONLY CUDA graph (`cuda_graph.py`, GPU), -ACL graph via the `mla_graph` patches (NPU), `_AFDCaptureEventTracker`, graph -metadata preparation, and graph/eager fallback. +compilation paths: `FULL_DECODE_ONLY` / `FULL` CUDA graph +(`v1/worker/cuda_graph.py`, GPU), ACL graph via the `mla_graph` module and +patches (`v1/worker/npu/mla_graph.py`, `compat/patches/npu/mla_graph.py`, NPU), +`_AFDCaptureEventTracker`, graph metadata preparation, and graph/eager +fallback. Designs: `docs/design/module/attention_runtime.md`, `docs/design/module/execution_platforms.md`. @@ -11,22 +13,37 @@ Designs: `docs/design/module/attention_runtime.md`, ## Feature checks - Keep capture/replay parity: every input to a captured graph (metadata, - buffers, connector state, ubatch keys) must be identical between capture - and replay; flag any capture-time-only branch that replay can also reach. -- Respect decode-only scoping: CUDA graph paths must not trigger for prefill - or mixed batches; the FULL_DECODE_ONLY gate stays enforced where the - connector contract declares it. -- Keep ACL graph patches pinned: `mla_graph` patches - `vllm_ascend/attention/mla_v1.py` at commit `80d8c194f` — apply the patch - contract (markers, upstream source, signature parity) and re-verify the copy - against that ref. + buffers, connector state, ubatch keys, async CAM graph payload) must be + identical between capture and replay; flag any capture-time-only branch that + replay can also reach. `_AFDCaptureEventTracker` asserts that capture input + preparation ran exactly as expected — do not accept weakening or deleting + those assertions without a replacement guarantee. +- Respect mode scoping. GPU uses `FULL_DECODE_ONLY` where the connector + contract says so; Ascend CAM uses `FULL`, rejects piecewise and decode-only + modes, and requires plain MLA graph + DBO to use `FULL_DECODE_ONLY`. A change + that lets an unsupported mode through to capture is a P1. +- Keep ACL graph patches pinned: `compat/patches/npu/mla_graph.py` patches + `vllm_ascend/attention/mla_v1.py` at the pinned vLLM-Ascend ref — apply the + patch contract (markers, upstream source, signature parity) and re-verify the + copy against that ref. - Preserve the eager escape hatch: every graph path has a tested eager fallback, and fallback selection is explicit config or documented platform - behavior, not an accident of ordering. -- State resource implications: graph size, memory pools, and capture-count - changes belong in the PR description and the relevant user guide; a capture - change that shifts memory is a perf claim needing evidence. + behavior, not an accident of ordering. Async CAM explicitly has *no* + automatic eager fallback: unsupported FULL configurations and mismatched + transport parameters fail startup. +- Graph-owned resources are bounded: async CAM FULL owns four private graph + pools and four graphs; FFN two-stream FULL owns its own pools per slot. State + resource implications (graph size, memory pools, capture counts) in the PR + description and the relevant user guide; a capture change that shifts memory + is a perf claim needing evidence. +- Async CAM graph startup is a host-only handshake + (`afd_plugin/async_cam_startup.py`, `FFNGraphPlan`): the requested + eager/full mode, executor error, and signature must agree between roles + before HCCL creation, with `GRAPH_STARTUP_TIMEOUT_SECONDS` bounding the wait. + A change to those keys is a cross-role protocol change. Test graph gating and metadata parity CPU-side where possible -(`tests/unit/v1/worker/`); device behavior claims need the named GPU scenario -(`afd-graph-*`, `afd-graph-dbo-*`) or NPU run. +(`tests/unit/v1/worker/test_cuda_graph.py`, `test_npu_mla_graph.py`, +`test_async_ffn_graph.py`, `tests/unit/connectors/test_async_cam_startup.py`); +device behavior claims need the named GPU scenario (`afd-v2-graph-*`, +`afd-v2-graph-dbo-*`) or the NPU run recorded per `docs/npu/TESTING.md`. diff --git a/.agents/skills/review-pr/references/modules/compat-patches.md b/.agents/skills/review-pr/references/modules/compat-patches.md index b47fd0360..d6fa2b984 100644 --- a/.agents/skills/review-pr/references/modules/compat-patches.md +++ b/.agents/skills/review-pr/references/modules/compat-patches.md @@ -5,9 +5,14 @@ reviewed head. Authoring rules: `AGENTS.md` ("Patching Requirement"). Use for monkey patches under `afd_plugin/compat/patches/` (generic engine/GPU: `async_dp_engine.py`, `async_dp_forward_context.py`, -`config_validation.py`, `engine_core.py`) and the version gate in -`compat/vllm.py`. Does not own the deferred NPU runtime layer -([npu-compat.md](npu-compat.md)) or plugin-owned classes. +`config_validation.py`, `engine_core.py`, `gpu_mrv2_ubatching.py`, +`gpu_deepseek_v4_dbo.py`), the vLLM-Ascend patches under +`compat/patches/npu/` (`ascend_config.py`, `ascend_platform.py`, `mla_graph.py`, +`mrv2_ubatching.py`), the version gate in `compat/vllm.py`, and the patched +methods in `afd_plugin/v1/worker/**` (`attention_worker.py`, `ffn_worker.py`, +`ffn_model_runner.py`, `attention_model_runner_v2.py`, and the NPU twins). +Does not own the deferred NPU runtime layer ([npu-compat.md](npu-compat.md)) or +plugin-owned classes. ## Contract checks @@ -15,25 +20,41 @@ engine/GPU: `async_dp_engine.py`, `async_dp_forward_context.py`, new monkey patch; a new patch needs an architectural-review rationale: correct upstream target, minimal scope, understood performance impact, and a long-term upstream-or-removal plan. -- Copy the upstream function wholesale from the pinned ref (vLLM 0.26.0; - vLLM-Ascend commit `80d8c194f`) and mark only AFD-specific differences with - `# ### PATCH START:` / `# ### PATCH END:`; keep the marker text short and - specific. +- Copy the upstream function wholesale from the pinned ref and mark only + AFD-specific differences with `# ### PATCH START:` / `# ### PATCH END:`; + keep the marker text short and specific. - Keep a patched function's signature and return type identical to upstream; any added parameter must be documented in the comments immediately above the patch function. -- Require `Patch reason:` / `Patch functionality:` comments immediately above - every patch function, and keep `# Upstream source:` pointers accurate. +- Require, immediately above every patch function: `# Upstream source:` + (module, symbol, and the pinned ref — a bare `vLLM 0.32.0` label is weaker + than the commit but acceptable when it names the file and symbol), + `Patch reason:`, and `Patch functionality:`. `# Removal plan:` and + `# Expansion exception:` are expected where the patch delegates to + `_original_*` or expands a large upstream function; their absence there is a + finding. - Treat `_original_*` delegation as the exception, not the default non-AFD path; require the exception to be called out in the patch comments. - Keep patches idempotent and version-aware (guarded by - `TARGET_VLLM_VERSION`); re-applying or importing twice must not corrupt - state. + `TARGET_VLLM_VERSION` / `is_target_vllm_compatible`); re-applying or + importing twice must not corrupt state. - Do not flag upstream style inside patch copies — ruff ignores E501/N806/N807 - there so diffs stay comparable to upstream; do flag upstream drift (the copy - no longer matches the pinned ref). + under `afd_plugin/compat/patches/**` so diffs stay comparable to upstream; do + flag upstream drift (the copy no longer matches the pinned ref). - Patched behavior must be transparent when AFD is inactive: non-AFD requests execute the copied upstream logic unchanged. -Test every patch in `tests/unit/compat/patches/` with CPU-safe tests covering -idempotence, version guarding, and both AFD-on and AFD-off behavior. +## Signature alignment + +`tests/unit/compat/patches/test_signatures.py` is the single parametrized table +that owns upstream signature checks for every scoped runtime patch, including +the ones that live in `afd_plugin/v1/worker/**` (capture inputs, fullgraph +replay, workspace slot, ubatch prepare, CAMP2P dispatch) and the NPU +`vllm_ascend` installers. It reads the installed upstream definition from +source rather than comparing a patch to itself. A newly patched function with +no table row, or a row whose target symbol no longer exists upstream, is a P1 +finding: the next pin move will fail or, worse, silently stop being checked. + +Test every patch in `tests/unit/compat/patches/` (and +`tests/unit/compat/patches/npu/`) with CPU-safe tests covering idempotence, +version guarding, and both AFD-on and AFD-off behavior. diff --git a/.agents/skills/review-pr/references/modules/connectors.md b/.agents/skills/review-pr/references/modules/connectors.md index 954d541a2..3e9647452 100644 --- a/.agents/skills/review-pr/references/modules/connectors.md +++ b/.agents/skills/review-pr/references/modules/connectors.md @@ -1,21 +1,28 @@ # Connectors Primary design: `docs/design/module/connector_contracts.md` in the reviewed -head; backend guides in `docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md` and -`docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md` / `docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md`. +head; backend guides in `docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md`, +`docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md`, +`docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md`, and +`docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md`. -Use for `afd_plugin/connectors/`: `AFDConnectorBase`, `AFDControlPlane`, -`ConnectorExtraInfo`, `AFDConnectorFactory`, and the concrete backends — -`gpu/p2p.py` (NCCL P2P), `npu/camp2p.py` (HCCL/CAM P2P), `npu/async_cam.py` -(experimental async CAM). Connectors do not own process-group creation or rank -mapping ([distributed-topology.md](distributed-topology.md)), model-side -proxies, or role workers. +Use for `afd_plugin/connectors/`: `AFDConnectorBase`, +`AFDRoutedExpertsConnector`, `AFDControlPlane`, `ConnectorExtraInfo`, +`AFDConnectorFactory`, and the concrete backends — `gpu/p2p.py` (NCCL P2P), +`gpu/deepep_m2n/` (DeepEP routed-experts M2N), `npu/camp2p.py` +(HCCL/CAM P2P), and `npu/async_cam.py` (async CAM). Connectors do not own +process-group creation or rank mapping ([distributed-topology.md](distributed-topology.md)), +model-side proxies, or role workers. ## Contract checks - Register backends through `AFDConnectorFactory`; role workers, runners, and models must stay connector-agnostic and never branch on a concrete - connector type. + connector type. The factory stores lazy loaders, so importing it must not + import CUDA or Ascend implementations. Registration alone does not admit an + external connector: `AFDConfig` validates `connector` against its own + allow-list in `afd_plugin/config.py`, and the design page records that gap as + draft. A PR claiming "extensible connectors" must close both. - Change both sides of the transfer boundary together: `ConnectorExtraInfo` handshake, tensor shapes/dtypes/layout, and completion signaling must agree between attention and FFN implementations, including metadata versioning @@ -24,14 +31,42 @@ proxies, or role workers. cancellation, and close must surface as loud errors, not a silent hang of both roles; every state machine handles abort before completion. - Declare graph-capture compatibility per connector (for example - FULL_DECODE_ONLY CUDA graph for P2pNccl, ACL graph constraints for CAM) and - encode unsupported combinations in `feature_validation`, not runtime - surprises. -- Treat additions under `connectors/npu/bin/` (vendored `.run`/`.whl` - artifacts) as design-level changes: require source, version, and target - CANN/Ascend runtime justification; they are pre-commit-excluded by design, - which is not a license to skip review. + `FULL_DECODE_ONLY` CUDA graph constraints for the CUDA connectors, ACL graph + constraints for CAM) and encode unsupported combinations in + `feature_validation`, not runtime surprises. +- Keep `connector_extra_config` connector-owned: unknown keys must fail in the + selected connector's parser, not be silently ignored. Adding a key to one + backend is a user-contract change that needs the guide and design-page table + updated together. + +## Routed-experts transports + +`DeepEPM2NAFDConnector` extends the contract through +`AFDRoutedExpertsConnector`: routing runs on Attention, FFN ranks receive one +row per (token, local expert) pair, run only local experts via `run_experts()`, +and send weighted rows back. `synchronize_lanes()` joins connector-forked +streams at the end of each step. When reviewing changes here: + +- The FFN worker selects `GPUFFNRoutedExpertsModelRunner` from the connector + class, not from the connector name; keep that selection type-based. +- The runner pins vLLM's MoE backend to `triton` because other backends + reorder expert weights in place without changing dtype or shape, producing + wrong outputs with no error. A new backend needs a layout proof, not an + allow-list entry. +- Dispatch/combine are collectives over all ranks, including ranks with no + tokens for a layer; a zero-length split is valid, an empty list is not. +- Weight loading must report what each rank actually loaded through + `init_afd_connector(moe_layer_indices=, local_experts=)`; skipping the + post-load check silently admits mismatched expert placement. + +## Vendored Ascend artifacts + +Additions under `connectors/npu/bin/` (`CAM_ascend*_openEuler_*.run`, +`umdk_cam_op_lib-*.whl`) are design-level changes: require source, version, and +target CANN/Ascend runtime justification. They are pre-commit-excluded by +design, which is not a license to skip review. Test each backend in `tests/unit/connectors/` (state machines, handshake, -failure paths, CPU-safe) and name the E2E gate scenario that exercises the -real transport. +failure paths, CPU-safe), name the device suite (`tests/gpu/` for M2N, +`tests/npu/` for CAM) and the E2E gate scenario that exercises the real +transport. diff --git a/.agents/skills/review-pr/references/modules/distributed-topology.md b/.agents/skills/review-pr/references/modules/distributed-topology.md index 451ff398b..d5e4f9ff4 100644 --- a/.agents/skills/review-pr/references/modules/distributed-topology.md +++ b/.agents/skills/review-pr/references/modules/distributed-topology.md @@ -4,11 +4,12 @@ Primary design: role lifecycle pages `docs/design/module/attention_runtime.md` and `docs/design/module/ffn_runtime.md`; connector handoff in `docs/design/module/connector_contracts.md`. -Use for `afd_plugin/distributed/`: `init_afd_process_group` and the -`DefaultProcessGroupSwitcher` (`afd_process_group.py`), `AFDRankMapping`, -`validate_p2p_topology`, `resolve_role_rank`, and `build_rank_mapping` -(`topology.py`). Does not own connector transfer semantics or worker startup -ordering. +Use for `afd_plugin/distributed/`: `init_afd_process_group`, +`create_hccl_process_group_options`, and `DefaultProcessGroupSwitcher` +(`afd_process_group.py`), plus `AFDRankMapping`, `topology_from_config`, +`validate_p2p_topology`, `resolve_role_rank`, `subgroup_attention_block`, and +`build_rank_mapping` (`topology.py`). Does not own connector transfer +semantics or worker startup ordering. ## Contract checks @@ -16,15 +17,26 @@ ordering. connectors, and models must not create overlapping groups or bypass the switcher. - Keep rank mappings total and bijective across the attention and FFN worlds; - `validate_p2p_topology` must reject mismatched world sizes (2A2F/2A1F) - before any collective runs. + `validate_p2p_topology` must reject mismatched world sizes before any + collective runs. - Derive backend choice (HCCL vs NCCL) from the detected platform; no silent fallback to a different backend. - Resolve roles and ranks deterministically from the same inputs on every - rank; any divergence is a startup hang or collective mismatch. + rank; any divergence is a startup hang or collective mismatch. Per-process + role ranks are runtime state resolved from vLLM's parallel placement and are + deliberately absent from `AFDConfig` — a PR that stores one there is a + contract change. +- Topology assumptions differ per connector and must stay coherent with them: + P2P and CAM connectors use FFN-then-Attention world ordering with role ranks + equal to DP x TP; DeepEP M2N treats the AFD world as one symmetric EP group + with attention ranks holding dummy expert slots, requires `num_experts` to + be divisible by the FFN rank count, and lets attention TP ranks act as + separate union ranks. `afd_process_group_timeout_s` (default 120 s) is the + rendezvous knob; `hccl_buffer_size` sizes connector-owned HCCL domains only. - Release AFD groups and restored defaults on shutdown and engine restart; no leaked process groups or stale switcher state. Test mapping construction, validation rejections, and teardown in -`tests/unit/distributed/`; name the E2E topology (2A2F or 2A1F) that covers -real collectives. +`tests/unit/distributed/` (`test_topology.py`, `test_afd_process_group.py`); +name the E2E topology (1A1F, 2A1F, 2A2F, 4A2F, 4A4F, 2P1A1F, or multi-pod) +that covers real collectives. diff --git a/.agents/skills/review-pr/references/modules/e2e-harness.md b/.agents/skills/review-pr/references/modules/e2e-harness.md new file mode 100644 index 000000000..eec7ee331 --- /dev/null +++ b/.agents/skills/review-pr/references/modules/e2e-harness.md @@ -0,0 +1,64 @@ +# E2E Harness and Gate Selection + +Primary design: `docs/design/module/e2e_testing.md` in the reviewed head. That +page is `normative`; its `E2E-INV-001` … `E2E-INV-008` invariants are +blockers when broken. Author-side runner: the `run-e2e` skill. + +Use for `tests/e2e/**`, the Buildkite case selection in +`.buildkite/cuda/*.yml`, `.buildkite/common/scripts/skip_ci.py` as it affects +what runs, and the `run-e2e` / `deploy-dsv4-flash-async` skill text that +describes cases. + +## Contract checks + +- Case IDs are stable lower-kebab-case and unique (`E2E-INV-001`). Renaming a + case silently breaks every CI node ID and every recipe/README that names it; + require the rename to land with the CI, docs, and skill updates. +- Gate cases never skip (`E2E-INV-004`): no `skip`, `xfail`, or + success-on-empty. A new `pytest.importorskip` or a conditional `return` in a + gate case is a P1. Missing setup must fail. +- CI selects cases by pytest node ID and must not lower sample counts, + thresholds, or the required set (`E2E-INV-007`). Compare + `.buildkite/cuda/test-ready.yml` / `test-merge.yml` / `test-weekly.yml` + against the module's `SCENARIOS` tuple: a case added to the module but not + selected (or selected but renamed) is drift worth reporting. +- The current gate set, which a change must keep coherent: + - DeepSeek-V2-Lite GPU: seven cases — `baseline-graph`, `afd-v2-eager-dp2`, + `afd-v2-graph-dp2`, `afd-v2-eager-tp2`, `afd-v2-graph-tp2`, + `afd-v2-eager-dbo-dp2`, `afd-v2-graph-dbo-dp2`; the module also holds + local-only `afd-v2-{eager,graph}-1a1f`, `afd-v2-graph-dbo-tp2`, and the + `afd-{eager,graph,graph-dbo}-2a1f` cases. + - DeepSeek-V2-Lite NPU: `baseline-graph` plus `afd-eager-2a2f`, + `afd-graph-2a2f`, `afd-graph-dbo-2a2f` (selected by + `AFD_E2E_BACKEND=npu`). + - Qwen3 MoE GPU: `baseline-graph`, `afd-eager-2a1f`, `afd-graph-2a1f`, + `afd-graph-dbo-2a1f`. The module also collects the DeepEP M2N 2A2F + scenarios (`afd-deepep-m2n-{eager,graph,graph-dbo}-2a2f` and the two + Attention-TP variants) through `test_qwen3_moe_deepep_m2n`; CI selects node + IDs explicitly and does not run them. + - Qwen3.6 MoE GPU: `baseline-graph`, `afd-eager-2a1f`, `afd-graph-2a1f` + (GDN DBO unsupported). + - DeepSeek-V4 Flash: separate `test_gpu.py`, `test_async_cam_npu.py`, and + `test_sync_camp2p_npu.py` suites; DSV4 Flash W4A8 async CAM is a local-only + 16-NPU case. +- Accuracy plumbing: ordinary cases evaluate the first 7 GSM8K samples, DBO + cases require a 24-sample minimum, and the GPU DP2 eager/eager-DBO/graph-DBO + comparison keeps 128 samples, concurrency 12, `max_num_seqs=8`, and the 0.27 + accuracy threshold. A change here is a gate change. +- Execution evidence: GPU and NPU graph/DBO cases load a test-only worker + extension and must report completed replays or two-stage execution on every + participating rank; dummy, profile, capture, and empty-second-microbatch work + does not qualify. Pure eager and baseline cases must not load the extension, + and performance comparisons must omit it. +- Process hygiene: child processes use owned process groups; cancellation is + SIGTERM, bounded grace, then reaping. The NPU Async CAM marker-based SIGKILL + sweep and its 120 s survivor window are a documented test-scoped exception, + not a general strategy. +- Harness code is not unit-tested (`tests/unit` covers `afd_plugin` only); + validate harness changes by running the affected E2E cases or + `tests/e2e/test_runner.py` / `test_deploy_dsv4_flash_async_skill.py`. + Production code must never import the harness. + +A PR that changes a case, sample count, device layout, or gate selection +without updating `docs/design/module/e2e_testing.md` and the `run-e2e` skill +narrative is a P2 (P1 when the change weakens a blocking gate). diff --git a/.agents/skills/review-pr/references/modules/engine-orchestration.md b/.agents/skills/review-pr/references/modules/engine-orchestration.md index db4982984..80cb3ad11 100644 --- a/.agents/skills/review-pr/references/modules/engine-orchestration.md +++ b/.agents/skills/review-pr/references/modules/engine-orchestration.md @@ -5,10 +5,13 @@ Primary design: `docs/design/module/attention_runtime.md` and head. Use for cross-role request flow semantics: async-DP engine behavior from the -patched `EngineCore`/`DPEngineCoreProc` busy loops, AFD request routing -between attention and FFN, the DBO yield custom op, ubatching orchestration, -and the `AFDBalancedRoutingStrategy` routing simulator. Does not own the patch -mechanics ([compat-patches.md](compat-patches.md)) or worker internals +patched `EngineCore` / `EngineCoreProc` / `DPEngineCoreProc` busy loops, +`DPAsyncMPClient.add_request_async`, AFD request routing between attention and +FFN, the DBO yield custom op (`v1/worker/dbo.py` → +`maybe_apply_dbo_yield`), ubatching orchestration +(`v1/worker/ubatch_wrapper.py`), and the `AFDBalancedRoutingStrategy` routing +simulator. Does not own the patch mechanics +([compat-patches.md](compat-patches.md)) or worker internals ([workers-runners.md](workers-runners.md)). ## Contract checks @@ -16,19 +19,31 @@ mechanics ([compat-patches.md](compat-patches.md)) or worker internals - Keep one owner for attention→FFN routing; models and workers must not independently select a route or a peer. - Keep the async-DP patches externally transparent: non-AFD requests observe - upstream vLLM 0.26.0 scheduling, output order, and finish semantics - unchanged, on both `EngineCoreProc` and `DPEngineCoreProc` paths. + upstream scheduling, output order, and finish semantics unchanged, on both + `EngineCoreProc` and `DPEngineCoreProc` paths. - Preserve request identity and ordering across the two roles; correlate every connector result, ubatch work item, and DBO yield to the originating request, and make terminal state monotonic (no forwarding after finish, abort, or failure). - Define shutdown ordering between the worlds: FFN shutdown must not hang attention and vice versa; cancellation propagates to in-flight transfers and - queued ubatch items. -- Keep the balanced routing strategy benchmark-only and explicitly registered; - it must never become the default routing behavior silently. + queued ubatch items. DeepSeek-V2-Lite has known DBO shutdown failures — check + whether a PR claims to fix that honestly, with evidence, rather than + suppressing the symptom. +- Native DBO is a vLLM capability that AFD reuses: exactly two microbatches, + enabled on Attention only. Async CAM FFN consumes work items outside + Attention's DBO thread schedule. GPU DBO accepts Attention DP1 or DP > 1; + NPU graph DBO with plain MLA requires `FULL_DECODE_ONLY`. A change that adds + a second scheduling path or relaxes the two-ubatch bound is a design change. +- Keep the balanced routing strategy benchmark-only and explicitly registered + (gated by `AFD_FORCE_BALANCED_TOPK_IDS`); it must never become the default + routing behavior silently. Test engine-loop patch behavior (AFD on/off), ubatch state transitions, and -shutdown/cancellation in `tests/unit/compat/patches/` and -`tests/unit/model_executor/`; name the DBO E2E scenario that exercises the -real path. +shutdown/cancellation in `tests/unit/compat/patches/` +(`test_engine_core.py`, `test_async_dp_engine.py`, +`test_async_dp_forward_context.py`, `test_gpu_mrv2_ubatching.py`), +`tests/unit/v1/worker/test_dbo.py`, +`tests/unit/model_executor/test_async_cam_dbo.py`, and +`tests/unit/model_executor/test_routing_simulator.py`; name the DBO E2E +scenario that exercises the real path. diff --git a/.agents/skills/review-pr/references/modules/execution-platforms.md b/.agents/skills/review-pr/references/modules/execution-platforms.md index 00aaa7d89..00b40ae8b 100644 --- a/.agents/skills/review-pr/references/modules/execution-platforms.md +++ b/.agents/skills/review-pr/references/modules/execution-platforms.md @@ -4,12 +4,14 @@ Primary design: `docs/design/module/execution_platforms.md` in the reviewed head. Use for NPU/GPU selection and platform mechanics: `validation.py` -(`assert_compatible_afd_stack`, -`afd_worker_qualname_for_platform_default`), `setup.py` Ascend-op build -gating, and `csrc/npu/` (`ascend_kernels/` a2e/e2a ACLNN operators under the -`npu_op_*` build system, `pybind/` → `afd_plugin._C_ascend`, and the build -drivers in `scripts/`). Does not own connector behavior or the deferred NPU -patch application ([npu-compat.md](npu-compat.md)). +(`assert_compatible_afd_stack`, `afd_worker_qualname_for_platform_default`, +`validate_gpu_model_runner_v2_config`, +`validate_npu_model_runner_v2_config`), `setup.py` Ascend-op build gating, and +`csrc/npu/` (`ascend_kernels/` a2e/e2a and `grouped_matmul_layered` ACLNN +operators under the `npu_op_*` build system, `pybind/` → `afd_plugin._C_ascend`, +`build_aclnn.sh` installing into `afd_plugin/_cann_ops_custom`, and the drivers +in `scripts/`). Does not own connector behavior or the deferred NPU patch +application ([npu-compat.md](npu-compat.md)). ## Contract checks @@ -17,18 +19,30 @@ patch application ([npu-compat.md](npu-compat.md)). default worker classes), never hardcoded device strings; reject unsupported platforms (Ascend 310P/xlite) explicitly with an actionable error. - Keep the build contract: compile Ascend ops only in a detected Ascend - environment or with `AFD_BUILD_ASCEND_OPS=1`; honor - `AFD_SKIP_ACLNN_BUILD=1`; document `SOC_VERSION` defaults and any new build - knob in the install docs. -- Pair `csrc/` kernel changes (a2e/e2a hosts and kernels) with the torch - adapter/extension updates they require, and state accuracy and performance - impact; kernel behavior changes need NPU evidence, never simulation. -- Keep version pins coherent: vLLM 0.26.0 (`TARGET_VLLM_VERSION`) and - vLLM-Ascend `80d8c194f` move together across the compat gate, docker base - image, docs, and design-page `upstream_refs`, or the PR states why not. -- Keep GPU path parity visible: `csrc/gpu/` is reserved — a PR adding GPU - native ops must state where the GPU counterpart lives. + environment or with `AFD_BUILD_ASCEND_OPS=1` (unset/empty means auto-detect; + an invalid value must raise, not default); honor `AFD_SKIP_ACLNN_BUILD=1`; + document `SOC_VERSION` defaults and any new build knob in the install docs. +- Pair `csrc/` kernel changes (a2e/e2a and grouped-matmul-layered hosts and + kernels) with the torch adapter/extension updates they require, and state + accuracy and performance impact; kernel behavior changes need NPU evidence, + never simulation. Layered GMM selection changes belong to the + `upgrade-layered-gmm-ops` workflow. +- Keep version pins coherent: `TARGET_VLLM_VERSION` in `compat/vllm.py`, the + `vllm` extra in `pyproject.toml`, `VLLM_BASE_TAG` and its version assertion + in `docker/Dockerfile.ci`, README, and design-page `upstream_refs` move + together, or the PR states why not. Read the actual pin; do not trust the + value quoted in a review note or in this skill. +- Keep GPU path parity visible: `csrc/gpu/` contains only a README reserving + the directory — the plugin builds no GPU native extension. A PR adding GPU + native ops must state where the GPU counterpart lives and why it is not + plugin-owned today. +- Vendored `.run`/`.whl` artifacts under `afd_plugin/connectors/npu/bin/` and + the generated `_cann_ops_custom` tree are packaging, not source; a change + there needs version/CANN justification (see + [connectors.md](connectors.md#vendored-ascend-artifacts)). -Test platform selection and build gating in `tests/unit/v1/worker/` (runtime -classpath tests) and `tests/unit/package/`; kernel changes additionally name -the NPU run that produced their evidence. +Test platform selection and build gating in `tests/unit/v1/worker/` +(`test_runtime_classpaths.py`), `tests/unit/config/test_validation.py`, and +`tests/unit/package/` (`test_package.py`, `test_ascend_build_files.py`, +`test_ascend_op_host_build.py`, `test_cam_async_build.py`); kernel changes +additionally name the NPU run that produced their evidence. diff --git a/.agents/skills/review-pr/references/modules/model-integration.md b/.agents/skills/review-pr/references/modules/model-integration.md index 2db06fa89..450684a50 100644 --- a/.agents/skills/review-pr/references/modules/model-integration.md +++ b/.agents/skills/review-pr/references/modules/model-integration.md @@ -7,16 +7,19 @@ Use for `afd_plugin/model_executor/`: the registration maps in `afd_plugin/__init__.py` (`_DEEPSEEK_MODEL_REGISTRATIONS`, `_QWEN_MODEL_REGISTRATIONS`, `_QWEN3_5_MODEL_REGISTRATIONS`), role-split model wrappers (`RemoteFFNProxy`, `AFDAttentionFusedMoE`, `GateOnlyRemoteMoE`, the -`AFDDeepseekV2/V3/V4` and `AFDQwen3Moe`/`AFDQwen3_5` families), checkpoint -weight-role filtering, and the NPU-specific forward/gate modules under -`models/npu/`. Does not own runner execution, connector transport, or the -routing simulator's registration mechanics. +`AFDDeepseekV2/V3/V4` and `AFDQwen3Moe`/`AFDQwen3_5` families, plus +`AFDGlmMoeDsaForCausalLM` and `AFDDSparkDeepseekV4ForCausalLM`), checkpoint +weight-role filtering, the NPU-specific forward/gate modules under +`models/npu/`, and the layered executor under `model_executor/npu/`. Does not +own runner execution, connector transport, or the routing simulator's +registration mechanics. ## Contract checks - Keep registrations explicit in the `register_afd()` maps; select NPU vs GPU variants (for example `DeepseekV4ForCausalLM` branching on `torch_npu`) at - registration time, never deep inside forward paths. + registration time, never deep inside forward paths. The maps are frozen with + `MappingProxyType` — a change that mutates them at runtime is a finding. - Preserve the role construction contract: each role builds only role-required components, and weight filtering via `_checkpoint_weight_roles` must not silently drop weights that role later @@ -28,11 +31,19 @@ routing simulator's registration mechanics. `getattr`/`hasattr` guard around vLLM model fields hides the exact incompatibility the pinned-version contract exists to surface. - Mark copied upstream model code with the patch contract (markers, upstream - source pointers); the qwen3_5 and deepseek_v4 files are among the heaviest - patch surfaces. + source pointers); `qwen3_5.py`, `deepseek_v4.py`, and + `models/npu/deepseek_v4.py` are among the heaviest patch surfaces. +- Family-specific constraints are contracts, not suggestions: Qwen3.5/3.6 GDN + DBO is unsupported; DeepEP M2N covers Qwen3 MoE (all-MoE layers, no shared + expert); Async CAM covers DeepSeek-V4 W4A8/W8A8 with static EP and + Attention-local SP, and no longer supports DeepSeek-V2/V2-Lite. A PR that + widens a family's supported matrix without E2E and design-page evidence is a + finding. - Register new model families against the [model-addition-checklist.md](../checks/model-addition-checklist.md): both role wrappers, registration, recipe, E2E suite, and docs. Test wrappers and weight filtering in `tests/unit/model_executor/` -(CPU-safe, checkpoint-driven); name the E2E model suite covering the family. +(`models/test_*_construction.py`, `models/test_*_weight_policy.py`, +`models/test_*_proxy.py`, `models/npu/`, and `test_async_cam_*.py`); name the +E2E model suite covering the family. diff --git a/.agents/skills/review-pr/references/modules/npu-compat.md b/.agents/skills/review-pr/references/modules/npu-compat.md index 60976b510..a41d6698b 100644 --- a/.agents/skills/review-pr/references/modules/npu-compat.md +++ b/.agents/skills/review-pr/references/modules/npu-compat.md @@ -6,10 +6,11 @@ sections) in the reviewed head; platform init ordering lives in Use for `afd_plugin/compat/npu/`: deferred patch application (`runtime.py::apply_afd_ascend_patches_if_needed`), runtime config, forward -context, feature validation, Ascend ops loading (`ops.py`), and the NPU -profiler. Does not own the vLLM-Ascend patch files under -`compat/patches/npu/` ([compat-patches.md](compat-patches.md)) or the NPU -workers ([workers-runners.md](workers-runners.md)). +context, feature validation, Ascend ops loading (`ops.py`), the opaque +routing-id balancing boundary (`routing_ids.py`), and the NPU profiler. Does +not own the vLLM-Ascend patch files under `compat/patches/npu/` +([compat-patches.md](compat-patches.md)) or the NPU workers +([workers-runners.md](workers-runners.md)). ## Contract checks @@ -22,9 +23,19 @@ workers ([workers-runners.md](workers-runners.md)). silently continue when ops are missing. - Guard every `torch_npu`/`vllm_ascend` import so module import stays CPU-safe; runtime checks happen inside guarded functions. +- `routing_ids.py` is an opaque boundary registered during worker + initialization; keep profile routing decisions inside the op rather than + exporting them as Python-visible state. Standalone scripts import the module + after device init. Its fixture lives in + `tests/unit/fakes/routing_ids_op.py` and is registered from + `tests/unit/conftest.py`; a new consumer that bypasses the fixture will break + CPU collection. - Keep NPU behavior observable: a deferred patch or runtime tweak that changes execution must be loggable and attributable when troubleshooting per `docs/npu/TROUBLESHOOTING.md`. -Test in `tests/unit/compat/npu/`: application ordering, idempotence, feature -rejections, and op-presence failure paths — all CPU-safe via guarded imports. +Test in `tests/unit/compat/npu/` (`test_runtime_config.py`, +`test_feature_validation.py`, `test_ascend_config_lifecycle.py`, +`test_ascend_engine_core_binding.py`, `test_routing_ids.py`, +`test_profiler.py`): application ordering, idempotence, feature rejections, and +op-presence failure paths — all CPU-safe via guarded imports. diff --git a/.agents/skills/review-pr/references/modules/plugin-config.md b/.agents/skills/review-pr/references/modules/plugin-config.md index f1891c355..b0c0eaa17 100644 --- a/.agents/skills/review-pr/references/modules/plugin-config.md +++ b/.agents/skills/review-pr/references/modules/plugin-config.md @@ -1,27 +1,50 @@ # Plugin Entry and Config Primary design: `docs/design/module/plugin_boundary.md` in the reviewed head. +This page is `normative`; its `ENTRY-INV-*` and `CFG-INV-*` invariants are +blockers when broken. Use for plugin registration (`register_afd()`), the `AFDConfig` dataclass and -schema, `parse_afd_config`/`validate_afd_config`, extra-config coercion, and -`afd_plugin/envs.py` flags. Plugin config does not own platform/worker -selection, patch mechanics, or worker construction. +schema, `parse_afd_config` / `parse_optional_afd_config` / +`validate_afd_config`, extra-config coercion, `afd_plugin/envs.py` flags, and +`afd_plugin/async_cam_startup.py` (the host-only Async CAM graph startup +protocol). Plugin config does not own platform/worker selection, patch +mechanics, or worker construction. ## Contract checks - Keep `register_afd()` import-time work minimal, idempotent, and version-guarded; importing `afd_plugin` must stay CPU-safe (no torch, - torch_npu, or vLLM runtime imports at module level). + torch_npu, or vLLM runtime imports at module level). The package uses + module-level `__getattr__` for the runtime classes and + `_force_spawn_multiprocessing_if_requested()` at import time — that call is + opt-in behind `AFD_FORCE_SPAWN_MULTIPROCESSING=1` and must stay that way. - Parse AFD settings only from the `afd` key of `--additional-config`; route every derived consumption through validated `AFDConfig` fields, not raw dicts, and fail fast on unsupported combinations with an actionable message. + Short keys are canonical (`connector`, `role`, `port`, `host`, + `num_attention_ranks`, `num_ffn_ranks`, `compute_gate_on_attention`, + `async`→`async_dp`, `afd_process_group_timeout_s`, + `connector_extra_config`); the `afd_*` names survive only as read-only + compatibility aliases, and unknown keys belong under + `connector_extra_config`, not at the top level. - Give every new `AFDConfig` field a default, validation, a stated role and platform implication (attention vs FFN, GPU vs NPU), and a docs update; changing a default or meaning of an existing field is a user-contract change. + `compute_hash()` covers only graph-affecting fields — a new field that + affects graph capture must join it or capture will be reused wrongly. - Read `envs.py` flags at use time, keep them opt-in and documented, and never let an env flag silently override validated config in production paths. -- Keep the plugin entry point (`vllm.general_plugins`) the only installation - path; no side effects from importing submodules. + Today `envs.py` holds a single diagnostic flag + (`AFD_FORCE_BALANCED_TOPK_IDS`); a new runtime-behavior flag needs a design + and guide entry, not just a constant. +- Keep the plugin entry point (`vllm.general_plugins` → `afd_plugin:register_afd`) + the only installation path; no side effects from importing submodules. +- `async_cam_startup.py` is host-only and must not import device runtimes: it + is imported on both roles before and after HCCL setup. Test registration idempotence, config parse/validate rejections, defaults, and -CPU-safe import in `tests/unit/config/` and `tests/unit/package/`. +CPU-safe import in `tests/unit/config/` (`test_config.py`, +`test_validation.py`), `tests/unit/test_envs.py`, +`tests/unit/test_async_cam_startup_wiring.py`, and +`tests/unit/package/test_package.py`. diff --git a/.agents/skills/review-pr/references/modules/workers-runners.md b/.agents/skills/review-pr/references/modules/workers-runners.md index 2e2439ef9..8f50d50c2 100644 --- a/.agents/skills/review-pr/references/modules/workers-runners.md +++ b/.agents/skills/review-pr/references/modules/workers-runners.md @@ -4,33 +4,56 @@ Primary design: `docs/design/module/attention_runtime.md` and `docs/design/module/ffn_runtime.md` in the reviewed head. Use for `afd_plugin/v1/worker/` on both platforms: `AFDAttentionWorker` / -`AFDFFNWorker` and `AFDNPU*Worker`, the model runners (V1 and V2 variants), -attention/FFN metadata builders, `cuda_graph.py`, `dbo.py`, and the ubatch -wrappers. Does not own platform selection -([execution-platforms.md](execution-platforms.md)), connector transport, or -model wrappers ([model-integration.md](model-integration.md)). +`AFDFFNWorker` and `AFDNPUAttentionWorker` / `AFDNPUFFNWorker`, the +Attention ModelRunnerV2 variants (`AFDAttentionModelRunnerV2`, +`AFDNPUAttentionModelRunnerV2`), the FFN runners (`GPUFFNModelRunner`, +`GPUFFNRoutedExpertsModelRunner`, `AFDNPUFFNModelRunner`, +`AsyncNPUFFNModelRunner`), attention/FFN metadata builders, `cuda_graph.py`, +`dbo.py`, `ubatch_wrapper.py`, and the NPU `ubatch_runner.py`, +`mla_graph.py`, `async_ffn_two_stream.py`, and `dspark_speculator.py`. Does +not own platform selection ([execution-platforms.md](execution-platforms.md)), +connector transport, or model wrappers ([model-integration.md](model-integration.md)). ## Contract checks - Preserve the role split: FFN-side `execute_model()` is connector-driven and fails fast on scheduler-driven calls; attention-side runners never construct FFN components and vice versa. -- Keep runner V1/V2 in pairs: a change to shared runner behavior lands in both - variants or states the exclusion; GPU keeps V2 gated off - (`VLLM_USE_V2_MODEL_RUNNER=0` required) while NPU V2 exists — flag any change - that silently moves that gate. -- Enforce the patch contract on every patched runner method (the NPU - attention model runner is the heaviest patch site): marked AFD differences, - upstream-copy fidelity against the pinned ref, identical signatures. +- There is no V1 runner and no fallback. Both roles require + `VLLM_USE_V2_MODEL_RUNNER=1`; Attention runs native ModelRunnerV2 (GPU) or + the NPU V2 runner, while FFN keeps a connector-driven runner that is *not* + a V2 runner. `validate_gpu_model_runner_v2_config` and + `validate_npu_model_runner_v2_config` enforce the flag; do not accept a + change that turns the requirement into a warning or a silent default. +- Keep the three FFN execution paths distinct and selected by contract, not by + convenience: + | Path | Selected by | Runner | + | --- | --- | --- | + | Control-plane-driven | `connector.control_plane is not None` | `GPUFFNModelRunner` / `AFDNPUFFNModelRunner` | + | Routed experts | `issubclass(connector_cls, AFDRoutedExpertsConnector)` | `GPUFFNRoutedExpertsModelRunner` | + | Async CAM (no control plane) | `CAMAsyncAFDConnector` on NPU | `AsyncNPUFFNModelRunner` | + A change that collapses two of these, or routes by connector *name*, is a + finding. +- Enforce the patch contract on every patched worker/runner method — the + Attention runners are the heaviest patch sites on both platforms: marked AFD + differences, upstream-copy fidelity against the pinned ref, identical + signatures, `Patch reason:`/`Patch functionality:` comments, and a row in + `tests/unit/compat/patches/test_signatures.py`. - Keep graph capture sound: metadata must be prepared identically for capture and replay; capture-only state must not leak into eager paths; DBO yield and ubatch wrappers behave the same under graph and eager. -- Keep the lazy export map in `v1/worker/__init__.py` CPU-safe: dotted-path - resolution must not import runtime modules eagerly. + `_AFDCaptureEventTracker` exists to detect capture input-preparation drift — + a change that silences those assertions is worse than the drift. +- Keep the lazy export maps CPU-safe: `afd_plugin/v1/worker/__init__.py` and + `afd_plugin/v1/worker/npu/__init__.py` resolve dotted paths through + `__getattr__`; new runtime classes must be added there rather than imported + eagerly, or plugin import on a CPU host breaks. - Update the GPU and NPU twins together or prove platform neutrality; the NPU - runners (largest patch surface) and GPU runners drift independently. + runners (largest patch surface) and GPU runners drift independently, and the + NPU sync/async FFN runners diverge by design. Test role contracts, metadata building, and gating in -`tests/unit/v1/worker/` using the CPU-safe runtime-guard convention (see +`tests/unit/v1/worker/` and its `npu/` subtree using the CPU-safe runtime-guard +convention (see [test-quality-evaluation.md](../checks/test-quality-evaluation.md)); name the GPU/NPU gap for anything not run. diff --git a/.agents/skills/review-pr/references/process/design-contracts.md b/.agents/skills/review-pr/references/process/design-contracts.md index a9215355a..19a69f6a8 100644 --- a/.agents/skills/review-pr/references/process/design-contracts.md +++ b/.agents/skills/review-pr/references/process/design-contracts.md @@ -7,15 +7,25 @@ the ownership and status source of truth for the frozen head. ## Resolution order 1. Open `docs/design/module/index.md` in the reviewed checkout. It declares - the reading order and the dependency direction between module designs. + the reading order, the dependency direction, and a "Production path routing" + table that assigns every shipped path, native build path, and packaging + file to exactly one primary document. Tests, tools, recipes, generated + files, user guides, and design docs are outside that inventory; the E2E + harness is owned separately by `e2e_testing.md`. 2. Match the changed production paths against each design page's `primary_code_paths` (and `related_code_paths`) frontmatter to identify the owning design document; a path may appear in `related_code_paths` of several - pages while having one primary owner. -3. Read the matched page's `status`, `depends_on`, `validation_paths`, and - `last_reviewed` fields. `status: draft` means its boundaries and invariants - are candidates, not enforced contracts — treat them as questions, and raise - a blocking finding only when current code, tests, or AGENTS.md enforce them. + pages while having one primary owner. A new shipped file that appears in no + page is an ownership gap — that is a finding, not a reason to route by + guesswork. +3. Read the matched page's `status`, `owners`, `depends_on`, + `validation_paths`, `upstream_refs`, and `last_reviewed`. Today only + `plugin_boundary.md` and `e2e_testing.md` are `normative`; every other page + is `draft`. `status: draft` means its boundaries and invariants are + candidates, not enforced contracts — treat them as questions, and raise a + blocking finding only when current code, tests, or AGENTS.md enforce them. + A `normative` page's `MUST`/`MUST NOT` invariants (`ENTRY-INV-*`, + `CFG-INV-*`, `E2E-INV-*`) are blockers. 4. Prefer live code and tests over stale prose when they disagree; report the drift against the design page as a finding when the PR touches that area. 5. Honor the declared dependency direction: role modules (attention/FFN/model @@ -23,7 +33,13 @@ the ownership and status source of truth for the frozen head. boundary, execution platforms, compatibility). A PR that adds a dependency from a shared boundary back into a role implementation contradicts the documented architecture; question it. -6. Never import a newer design page into an older branch's review; release +6. Compare `upstream_refs` and `last_reviewed` against the pin in + `afd_plugin/compat/vllm.py` and `docs/design/module/index.md`. A page whose + `upstream_refs` still name the previous pair while the code moved is drift + worth reporting when the PR touches that area — but do not report a page as + stale merely because it records historical evidence; those rows are + explicitly historical. +7. Never import a newer design page into an older branch's review; release branches (for example `release/v0.19.1rc1`, `update/v0.28.0`) are judged against their own head's pages. @@ -36,8 +52,25 @@ A PR changes a documented contract, and must update the matching - changes connector, worker, platform, or compatibility behavior that a design page describes; - changes the E2E gate structure or pass criteria (`e2e_testing.md`); -- moves the pinned vLLM or vLLM-Ascend refs recorded in `upstream_refs`. +- moves the pinned vLLM or vLLM-Ascend refs recorded in `upstream_refs`, or + changes the `TARGET_VLLM_VERSION` gate in `afd_plugin/compat/vllm.py`; +- adds, retires, or renames an E2E case, or changes a gate's sample count, + device count, or required set (`e2e_testing.md`); +- changes behavior that an `.agents/skills/**` SKILL.md describes as a command, + case list, or pin. Doc-only updates to a design page route back to the production contract the page claims; verify the claim against code before accepting the page edit as the review's whole surface. + +## Ownership + +CODEOWNERS is no longer a single global rule. The global catch-all maps to +`@hsliuustc0106 @jiangkuaixue123`, with path-specific owners on top: +`/afd_plugin/connectors/gpu/`, `/csrc/gpu/`, `/recipe/gpu/`, `/tests/gpu/`, +`/docs/gpu/`, and `/.buildkite/cuda/` add `@ronenkat`; `/csrc/npu/` adds +`@francisleely`; `/tests/e2e/`, `/.agents/skills/run-e2e/`, and +`/.agents/skills/deploy-afd-k8s/` add `@ronenkat`. Design pages name their own +`owners`; `e2e_testing.md` is owned by `@yujuancao07` alone. When a design page +and CODEOWNERS disagree about who owns a contract, report the disagreement +rather than choosing one silently. diff --git a/.agents/skills/review-pr/references/process/general-checks.md b/.agents/skills/review-pr/references/process/general-checks.md index 8533a9e24..3b4abe72e 100644 --- a/.agents/skills/review-pr/references/process/general-checks.md +++ b/.agents/skills/review-pr/references/process/general-checks.md @@ -16,33 +16,56 @@ vLLM engine/config ingress (--additional-config {"afd": ...}) ``` - Prove the path in the frozen head, not from the PR description. -- Cover the matrix the change can affect: GPU/NPU, graph/eager, DBO on/off, - 2A2F/2A1F, runner V1/V2, colocation/disaggregation, AFD-on/AFD-off. +- Cover the matrix the change can affect: + + | Axis | Values in the current tree | + | --- | --- | + | Platform | CUDA, Ascend NPU | + | Execution | eager, `FULL`, `FULL_DECODE_ONLY` graph | + | Ubatching | DBO off, two native microbatches | + | Topology | 1A1F, 2A1F, 2A2F, 4A2F, 4A4F, 2P1A1F, multi-pod | + | Connector | `P2pNcclAFDConnector`, `DeepEPM2NAFDConnector`, `CAMP2pAFDConnector`, `CAMAsyncAFDConnector` | + | FFN execution | control-plane-driven, routed-experts, async layered GMM, FFN two-stream | + | Draft model | target-only, DSpark target + draft | + | AFD | on, off (transparent passthrough) | + + Not every cell exists; claim only the combinations the code and design pages + enumerate, and flag a change that silently widens the supported set. + +- There is no V1 model-runner fallback. Both roles require + `VLLM_USE_V2_MODEL_RUNNER=1`; Attention runs ModelRunnerV2 and FFN stays + connector-driven. `validate_gpu_model_runner_v2_config` / + `validate_npu_model_runner_v2_config` raise on a missing flag — a change that + relaxes or bypasses that check is a contract change, not a convenience. - Patched or wrapper code must be transparent when AFD is inactive - (`is_afd_active` false): a non-AFD request must keep upstream vLLM 0.26.0 - behavior exactly. + (`is_afd_active` false): a non-AFD request must keep upstream vLLM behavior + exactly, at the pinned ref. - Search bounded callers and sibling implementations (the GPU vs NPU twin of - every worker/model file) rather than assuming the changed hunk is the only - path; the two platforms drift independently. + every worker/model file, and the sync vs async NPU FFN runner pair) rather + than assuming the changed hunk is the only path; the platforms drift + independently. ## Module, API, and class design -These are the repo's own normative rules (AGENTS.md); enforce them as review -rules, not style nits: +These are the repo's own normative rules (AGENTS.md, docs/UT_MAINTENANCE.md); +enforce them as review rules, not style nits: 1. Access vLLM / vLLM-Ascend attributes directly; `getattr`/`hasattr` guards and proactive custom exceptions hide upstream-compatibility breakage that upgrades must surface. Flag them unless the change is genuinely optional. 2. Keep parameter types concrete; `Any`/`object` parameters hide contract - violations from mypy/pyright. + violations from mypy/ty. 3. No new magic numbers; name constants (`MAX_CONTEXT_LENGTH`, not `2048`). -4. No new mutable global state; constants and immutable config objects only. +4. No new mutable global state; constants and immutable config objects only + (`MappingProxyType` for the frozen registration maps is the house pattern). New mutable globals require approval and must be justified in the PR. 5. Do not split simple functions into helper layers; extract only for real complexity, real duplication, or an established local pattern. 6. Keep imports CPU-safe at module level: importing `afd_plugin` (or any unit - test module) must not import torch, torch_npu, or vLLM runtime modules. - Defer or `importorskip` them inside functions/tests. + test module) must not import torch, torch_npu, or vLLM runtime modules. The + package uses `__getattr__` lazy export maps in `afd_plugin/__init__.py`, + `afd_plugin/v1/worker/__init__.py`, and `afd_plugin/v1/worker/npu/__init__.py`; + keep new runtime classes behind those maps, not as eager imports. 7. New public behavior needs a named owner for its design page and a docs impact statement in the PR template. 8. Performance-affecting changes need a comparable A/B claim or an explicit @@ -53,16 +76,22 @@ rules, not style nits: | Risk | Prove before reporting | | --- | --- | | Correctness | A reachable input reaches the changed code and produces wrong output, a hang, or a crash. | -| Patch-contract drift | A patched function's signature, return type, or upstream copy diverges from the pinned ref, or missing/incorrect `# ### PATCH` markers, `Patch reason`/`Patch functionality` comments, or `# Upstream source:` pointers break upgrade comparability. | -| Version/compat | The change breaks the vLLM 0.26.0 / vLLM-Ascend `80d8c194f` contract, or moves a pin without updating the version gate, docker base, docs, and design pages together. | -| Platform divergence | The GPU or NPU twin (worker/runner/model variant) is not updated and the change is not provably platform-neutral. | +| Patch-contract drift | A patched function's signature, return type, or upstream copy diverges from the pinned ref, or missing/incorrect `# ### PATCH START/END` markers, `Patch reason:` / `Patch functionality:` / `# Upstream source:` comments break upgrade comparability. Patched methods now live in `afd_plugin/v1/worker/**` as well as `afd_plugin/compat/`, and `tests/unit/compat/patches/test_signatures.py` owns the signature table for both. | +| Version/compat | The change breaks the current `TARGET_VLLM_VERSION` / vLLM-Ascend pin contract, or moves a pin without updating the version gate, `docker/Dockerfile.ci` base tag and its assertion, `pyproject.toml`, README, and design-page `upstream_refs` together. Read the pin from `afd_plugin/compat/vllm.py`, not from memory. | +| Platform divergence | The GPU or NPU twin (worker/runner/model variant, sync vs async NPU FFN runner) is not updated and the change is not provably platform-neutral. | | CPU-safe imports | A new module-level import of torch/torch_npu/vLLM runtime breaks `pytest -m "not gpu and not vllm_runtime"` or plugin import on a CPU host. | -| Concurrency/async | Async-DP busy loops, ubatching state machines, DBO yield, or connector transfer states can deadlock, drop, reorder, or leak work items on abort/timeout. | +| Concurrency/async | Async-DP busy loops, ubatching state machines, DBO yield, async CAM work items, or two-stream slots can deadlock, drop, reorder, or leak work items on abort/timeout. | | Distributed/topology | Rank mapping is not bijective/validated, process groups are created outside the AFD owner, or teardown leaks groups across restarts. | -| Lifecycle | Graph capture state leaks into eager paths, shutdown ordering between attention and FFN worlds hangs either side, or connector close leaves a transfer half-open. | -| Security/validation | Untrusted `--additional-config`, env flags, or connector metadata are consumed without validation; host/port/rank inputs reach device code unchecked. | +| Lifecycle | Graph capture state leaks into eager paths, shutdown ordering between attention and FFN worlds hangs either side, or connector close leaves a transfer half-open. Known open failures (for example DeepSeek-V2-Lite DBO shutdown, NPU graceful exit) must be named as pre-existing context, not silently repaired by assertion changes. | +| Security/validation | Untrusted `--additional-config`, `connector_extra_config`, env flags, or connector metadata are consumed without validation; host/port/rank inputs reach device code unchecked. | +| Routed-expert contract | A routed-experts connector (`DeepEPM2NAFDConnector`) change assumes a weight layout the pinned vLLM MoE backend does not preserve. `ROUTED_MOE_BACKEND = "triton"` is pinned deliberately; a new backend needs an explicit layout proof, not a silent allow-list addition. | +| Quantization/layer contract | Async CAM layered FFN rejects incompatible weights at startup (`dynamicQuant=1`, homogeneous W4A8/W8A8, layering). A change that falls back to another FFN path instead of failing startup contradicts the design page. | +| E2E gate integrity | Case IDs, sample counts, device counts, or the required case set change without a design-page update; a gate case gains a `skip`/`xfail`; CI stops selecting a node ID it used to select. `docs/design/module/e2e_testing.md` is `normative`, so its `E2E-INV-*` items are blockers. | +| CI suppression | The diff is docs-only or skip-mark-only, so `skip_ci.py` suppresses all default CI. A green pipeline then proves nothing about the code; say so instead of treating it as validation. | +| Vendored artifacts | New or changed files under `afd_plugin/connectors/npu/bin/` (`.run`/`.whl`) are pre-commit-excluded by design; they still need source, version, and target CANN justification. | +| Agent/skill drift | A change to production paths, CI, or E2E gates that leaves `.agents/skills/**` stating stale behavior (pins, case counts, commands) is a P2 finding; those skills are executed instructions. | | Evidence | Perf/accuracy claims lack comparable base/head runs; NPU behavior claims lack manual evidence per `docs/npu/TESTING.md`. | -| User contract | `AFDConfig` fields, defaults, or recipe launch scripts change meaning without a docs update and migration note. | +| User contract | `AFDConfig` fields, defaults, `connector_extra_config` keys, or recipe launch scripts change meaning without a docs update and migration note. | ## Finding bar @@ -70,7 +99,8 @@ Report a finding only when you can name the changed `path:line`, the trigger or call path, the current behavior, the user or maintainer impact, and the smallest safe fix direction. Do not report: -- Style pre-commit already enforces (ruff, format, SPDX, markdownlint, DCO). +- Style pre-commit already enforces (ruff, format, typos, SPDX, markdownlint, + shellcheck, DCO). - Hypothetical issues with no reachable trigger in the supported matrix. - Missing tests that would not protect the changed behavior. - Debt the PR only modifies or removes. diff --git a/.agents/skills/review-pr/references/process/review-execution.md b/.agents/skills/review-pr/references/process/review-execution.md index 5436b37f7..0138528d5 100644 --- a/.agents/skills/review-pr/references/process/review-execution.md +++ b/.agents/skills/review-pr/references/process/review-execution.md @@ -113,6 +113,12 @@ Mark early findings as preliminary and continue. This is not a GitHub comment. blocks the verdict. Start with the first error, not the last cascade. - Treat the PR body as navigation. Commands, benchmark tables, and claimed tests become evidence only after their provenance and relevance are checked. +- Check what the pipeline actually selects before trusting a green run. + `.buildkite/common/scripts/skip_ci.py` suppresses the whole default pipeline + when every changed path is docs (`docs/**`, any `*.md`, `mkdocs.yml`) or a + pytest skip-mark-only test edit. On such a diff, "CI passed" is not evidence + about the code. Label-only CI files gate per platform and level; there is no + `.buildkite/npu/` pipeline yet. ## Run bounded validation @@ -129,14 +135,24 @@ paths have supported findings or explicit no-issue conclusions: 1. Import preflight on a trusted head: `python -c "import afd_plugin"` and the changed modules import without torch/torch_npu present. 2. Narrowest unit tests for the changed area, CPU-safe by default: - `pytest tests/unit/ -m "not gpu and not vllm_runtime"`. -3. Targeted GPU-gated tests only when a GPU and a trusted head exist. + `pytest tests/unit/ -m "not gpu and not vllm_runtime"`. Mirror the CI + selection when the change is broad: + `pytest -m "not gpu and not vllm_runtime" tests/unit tests/npu/test_async_cam_harness.py tests/npu/test_async_cam_performance.py tests/npu/test_async_cam_golden.py tests/npu/test_gmm_layered_performance.py`. +3. Targeted device tests only when the device and a trusted head exist: + `pytest tests/gpu/test_attention_routing.py` on CUDA (this one runs in CI), + `tests/gpu/test_deepep_m2n_gpu.py` on SM90+ with afdEP-comm and DeepGEMM + installed, and the `tests/npu/` device suites on Ascend. 4. E2E gate scenarios only through the `run-e2e` skill, only when the review genuinely needs active evidence and devices are free; otherwise request - author evidence. + author evidence. Gate cases must never be turned into skips to make a run + pass. 5. NPU paths have no in-repo CI. Require author-provided evidence per `docs/npu/TESTING.md`, or a manually executed run the user explicitly asks for. Never simulate device evidence. +6. When a pin moves, verify the coherent set: `TARGET_VLLM_VERSION` in + `afd_plugin/compat/vllm.py`, the `vllm` extra in `pyproject.toml`, + `VLLM_BASE_TAG` and its assertion in `docker/Dockerfile.ci`, `README.md`, + and every design page's `upstream_refs`. ## Deliver maintainer-style findings diff --git a/.agents/skills/review-pr/references/process/review-routing.md b/.agents/skills/review-pr/references/process/review-routing.md index 5cf7f349a..658493264 100644 --- a/.agents/skills/review-pr/references/process/review-routing.md +++ b/.agents/skills/review-pr/references/process/review-routing.md @@ -12,14 +12,15 @@ path (for example a connector change that also moves rank mapping). | Module | Live behavior signals | Read | | --- | --- | --- | | Plugin entry & config | `register_afd()`, `AFDConfig`, parse/validate, extra-config coercion, env flags | [plugin-config.md](../modules/plugin-config.md) | -| Compatibility patches | New or changed monkey patch under `afd_plugin/compat/patches/`, version gate, upstream copy | [compat-patches.md](../modules/compat-patches.md) | -| NPU runtime compat | `compat/npu/` deferred patch application, feature validation, Ascend ops loading | [npu-compat.md](../modules/npu-compat.md) | -| Connectors | Connector backend, control plane, transfer/metadata handshake | [connectors.md](../modules/connectors.md) | +| Compatibility patches | New or changed monkey patch under `afd_plugin/compat/patches/` or a patched method under `afd_plugin/v1/worker/`, version gate, upstream copy | [compat-patches.md](../modules/compat-patches.md) | +| NPU runtime compat | `compat/npu/` deferred patch application, feature validation, routing ids, Ascend ops loading | [npu-compat.md](../modules/npu-compat.md) | +| Connectors | Connector backend, routed-experts transport, control plane, transfer/metadata handshake | [connectors.md](../modules/connectors.md) | | Distributed topology | AFD process groups, `AFDRankMapping`, topology validation | [distributed-topology.md](../modules/distributed-topology.md) | | Engine orchestration | Async-DP engine loops, cross-role routing, DBO yield op, ubatch orchestration | [engine-orchestration.md](../modules/engine-orchestration.md) | -| Workers & runners | Role workers, model runners V1/V2, graph capture, attention/FFN metadata | [workers-runners.md](../modules/workers-runners.md) | +| Workers & runners | Role workers, ModelRunnerV2, routed-experts and async FFN runners, graph capture, attention/FFN metadata | [workers-runners.md](../modules/workers-runners.md) | | Execution platforms | NPU/GPU worker selection, Ascend op build, `csrc/` kernels | [execution-platforms.md](../modules/execution-platforms.md) | | Model integration | Model registrations, role-split wrappers, weight-role filtering | [model-integration.md](../modules/model-integration.md) | +| E2E harness | `tests/e2e/**`, case IDs, gate selection, device layout, accuracy plumbing | [e2e-harness.md](../modules/e2e-harness.md) | Docs-, tests-, or CI-only changes route to the production contract they protect; if none applies, use only the applicable evidence checks. @@ -28,9 +29,10 @@ protect; if none applies, use only the applicable evidence checks. | Overlay | Signals | Read | | --- | --- | --- | -| Disaggregation topologies | 2A2F/2A1F, colocation vs disaggregation, `recipe/` launch scripts | [disaggregation-topologies.md](../features/disaggregation-topologies.md) | -| Async CAM & ubatching | `CAMAsyncAFDConnector`, two-stage MoE ubatching, DSV3.2/DSV4 async CAM forwards | [async-cam-ubatching.md](../features/async-cam-ubatching.md) | +| Disaggregation topologies | 1A1F/2A1F/2A2F/4A2F/4A4F/2P1A1F, colocation vs disaggregation, multi-pod, `recipe/` launch scripts | [disaggregation-topologies.md](../features/disaggregation-topologies.md) | +| Async CAM & layered FFN | `CAMAsyncAFDConnector`, layered W4A8/W8A8 GMM, FFN two-stream, DSV3.2/DSV4 async CAM forwards | [async-cam-ubatching.md](../features/async-cam-ubatching.md) | | Graphs & compilation | CUDA graph, ACL graph, capture/replay metadata, eager fallback | [graphs-and-compilation.md](../features/graphs-and-compilation.md) | +| DSpark draft | `AFDDSparkSpeculator`, `DSparkDraftModel`, target/draft layered FFN | [dspark-draft.md](../features/dspark-draft.md) | ## Evidence and change overlays @@ -42,18 +44,23 @@ protect; if none applies, use only the applicable evidence checks. | CI, docs, PR evidence, checklist | [tests-docs-checklist.md](../checks/tests-docs-checklist.md) | — | | Hardware or runnable path available | [verification.md](../checks/verification.md) | `run-e2e` | | vLLM/vLLM-Ascend pin moves | [compat-patches.md](../modules/compat-patches.md) + [execution-platforms.md](../modules/execution-platforms.md) | `upgrade-npu` / `upgrade-gpu-version` | +| Layered GMM ops pinned to a new ops-transformer revision | [async-cam-ubatching.md](../features/async-cam-ubatching.md) + [execution-platforms.md](../modules/execution-platforms.md) | `upgrade-layered-gmm-ops` | +| Recipe, deployment, or benchmark flow changes | [disaggregation-topologies.md](../features/disaggregation-topologies.md) | `deploy-afd-k8s` / `run-vllm-bench-k8s` / `deploy-dsv4-flash-async` | +| `.agents/skills/**` changes | [tests-docs-checklist.md](../checks/tests-docs-checklist.md) | — | ## Calibrate findings - **P0:** security exposure, data corruption, an unusable plugin or engine start, a patch that silently changes non-AFD upstream behavior. Block merge. - **P1:** reachable runtime failure, wrong output, platform divergence, - patch-contract violation that breaks upgrade comparability, missing evidence - for a blocking claim. Block merge until fixed or explicitly waived by an - owner. + patch-contract violation that breaks upgrade comparability, a `normative` + design invariant broken (`plugin_boundary.md`, `e2e_testing.md`), missing + evidence for a blocking claim. Block merge until fixed or explicitly waived + by an owner. - **P2:** real defect with a concrete future failure mode, contract drift the - code does not yet enforce, missing docs/test for changed behavior. - Non-blocking; do not inflate to block. + code does not yet enforce, missing docs/test for changed behavior, stale + agent-skill instructions that would mislead the next run. Non-blocking; do + not inflate to block. Everything else is a question for the author, not a finding. One root cause yields one finding even if it surfaces at several `path:line` sites. From f5ee720fbc12bc0392a20fc8279e5c158c148f7d Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:21:57 +0200 Subject: [PATCH 03/42] [GPU] Standalone FFN service for Qwen3-30B-A3B over DeepEP M2N Add the gpu-deepep-m2n backend to the standalone FFN service (python -m afd_plugin.ffn_faas serve): BF16 Qwen3MoeForCausalLM routed experts over DeepEPM2NAFDConnector, eager, FULL_DECODE_ONLY graphs and two-lane DBO, without a vLLM Worker, EngineCore, scheduler, KV cache or complete CausalLM. - Host protocol: FFNBackend.execute_step() returns bool and gains cancel_receive(); the host cancels an idle receive before joining. The NPU backend returns True and keeps a no-op cancel. - GPUFFNModelRunner.execute_connector_driven_step(): the Worker loop body, now shared by AFDFFNWorker and the standalone backend. - GPUFFNRoutedExpertsModelRunner.attach_model() and mark_receiver_exited() for an externally loaded model. - ffn_faas/platforms/gpu: CPU-only adapter, settings and Qwen3 checkpoint description; GPU environment (F-only NCCL world, AFD architecture, Triton MoE pin); FFN-only Qwen3 model shell on vLLM's native loader; DeepEPM2NFFNBackend. - Tests: CPU unit tests, a 4-GPU service smoke in tests/gpu, and four Qwen3 M2N FFN-service E2E scenarios in the E2E runner. - Docs: GPU standalone guide and FFN runtime design update. Co-Authored-By: Claude Opus 5.5 Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- afd_plugin/ffn_faas/__init__.py | 2 +- afd_plugin/ffn_faas/interfaces.py | 19 +- afd_plugin/ffn_faas/platforms/__init__.py | 16 +- afd_plugin/ffn_faas/platforms/gpu/__init__.py | 15 + afd_plugin/ffn_faas/platforms/gpu/adapter.py | 153 +++++ afd_plugin/ffn_faas/platforms/gpu/backend.py | 194 +++++++ .../ffn_faas/platforms/gpu/environment.py | 231 ++++++++ .../ffn_faas/platforms/gpu/qwen3_model.py | 132 +++++ .../platforms/gpu/qwen3_routed_model.py | 218 +++++++ afd_plugin/ffn_faas/platforms/gpu/settings.py | 183 ++++++ afd_plugin/ffn_faas/platforms/npu/backend.py | 15 +- afd_plugin/ffn_faas/runner.py | 28 +- afd_plugin/v1/worker/ffn_model_runner.py | 67 +++ .../worker/ffn_routed_experts_model_runner.py | 77 ++- afd_plugin/v1/worker/ffn_worker.py | 47 +- docs/design/module/ffn_runtime.md | 40 +- docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md | 5 + docs/gpu/STANDALONE_FFN_SERVICE.md | 151 +++++ tests/e2e/models/qwen3_moe/test_qwen3_moe.py | 17 +- tests/e2e/runner.py | 223 ++++++- tests/gpu/test_ffn_faas_gpu.py | 546 ++++++++++++++++++ tests/unit/fakes/ffn_faas.py | 57 +- tests/unit/ffn_faas/test_gpu_adapter.py | 140 +++++ tests/unit/ffn_faas/test_gpu_qwen3_model.py | 74 +++ tests/unit/ffn_faas/test_gpu_settings.py | 115 ++++ tests/unit/ffn_faas/test_runner.py | 35 ++ tests/unit/v1/worker/test_ffn_model_runner.py | 59 +- .../test_ffn_routed_experts_model_runner.py | 67 +++ 28 files changed, 2857 insertions(+), 69 deletions(-) create mode 100644 afd_plugin/ffn_faas/platforms/gpu/__init__.py create mode 100644 afd_plugin/ffn_faas/platforms/gpu/adapter.py create mode 100644 afd_plugin/ffn_faas/platforms/gpu/backend.py create mode 100644 afd_plugin/ffn_faas/platforms/gpu/environment.py create mode 100644 afd_plugin/ffn_faas/platforms/gpu/qwen3_model.py create mode 100644 afd_plugin/ffn_faas/platforms/gpu/qwen3_routed_model.py create mode 100644 afd_plugin/ffn_faas/platforms/gpu/settings.py create mode 100644 docs/gpu/STANDALONE_FFN_SERVICE.md create mode 100644 tests/gpu/test_ffn_faas_gpu.py create mode 100644 tests/unit/ffn_faas/test_gpu_adapter.py create mode 100644 tests/unit/ffn_faas/test_gpu_qwen3_model.py create mode 100644 tests/unit/ffn_faas/test_gpu_settings.py diff --git a/afd_plugin/ffn_faas/__init__.py b/afd_plugin/ffn_faas/__init__.py index aaac2bd74..49e5f0c5a 100644 --- a/afd_plugin/ffn_faas/__init__.py +++ b/afd_plugin/ffn_faas/__init__.py @@ -1,3 +1,3 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project -"""Standalone FFN process using the existing Async CAM backend.""" +"""Standalone FFN process: Async CAM on NPU, DeepEP M2N on GPU.""" diff --git a/afd_plugin/ffn_faas/interfaces.py b/afd_plugin/ffn_faas/interfaces.py index 91f419d7b..85c647dce 100644 --- a/afd_plugin/ffn_faas/interfaces.py +++ b/afd_plugin/ffn_faas/interfaces.py @@ -21,7 +21,24 @@ def prepare(self, service: FFNService) -> None: ... def start_receiver(self) -> None: ... - def execute_step(self) -> None: ... + def execute_step(self) -> bool: + """Run one connector-driven step on the receiver thread. + + Returns ``True`` after a completed step. Returns ``False`` only when a + ``cancel_receive()`` request ended an idle receive, so the host loop + exits cleanly; every other failure raises. + """ + ... + + def cancel_receive(self) -> None: + """Unblock an idle receive after a host stop request. + + Called on the host thread while the receiver thread may be blocked in + ``execute_step()``. A backend that cannot cancel a pending receive + implements this as a no-op; the host then reports the pending receive + and retains every owner. + """ + ... def receiver_exited(self) -> None: ... diff --git a/afd_plugin/ffn_faas/platforms/__init__.py b/afd_plugin/ffn_faas/platforms/__init__.py index fa764be69..b1db5054c 100644 --- a/afd_plugin/ffn_faas/platforms/__init__.py +++ b/afd_plugin/ffn_faas/platforms/__init__.py @@ -6,15 +6,27 @@ from typing import TYPE_CHECKING +from afd_plugin.ffn_faas.platforms.gpu import GPU_DEEPEP_M2N_BACKEND + if TYPE_CHECKING: from afd_plugin.ffn_faas.interfaces import FFNPlatformAdapter -SUPPORTED_BACKENDS = ("ascend-cam",) +ASCEND_CAM_BACKEND = "ascend-cam" +SUPPORTED_BACKENDS = (ASCEND_CAM_BACKEND, GPU_DEEPEP_M2N_BACKEND) def get_platform_adapter(name: str) -> FFNPlatformAdapter: - if name == "ascend-cam": + """Import and construct the CPU-only adapter of one backend by name. + + Each adapter is imported only when selected, so choosing one platform + never imports another platform's modules. + """ + if name == ASCEND_CAM_BACKEND: from afd_plugin.ffn_faas.platforms.npu.adapter import AscendCAMAdapter return AscendCAMAdapter() + if name == GPU_DEEPEP_M2N_BACKEND: + from afd_plugin.ffn_faas.platforms.gpu.adapter import GPUDeepEPM2NAdapter + + return GPUDeepEPM2NAdapter() raise ValueError(f"unsupported standalone FFN backend: {name}") diff --git a/afd_plugin/ffn_faas/platforms/gpu/__init__.py b/afd_plugin/ffn_faas/platforms/gpu/__init__.py new file mode 100644 index 000000000..ec845d1e7 --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/gpu/__init__.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""GPU standalone FFN backend: Qwen3 MoE routed experts over DeepEP M2N. + +The adapter, settings and checkpoint description are CPU-only. The +environment, FFN-only model and backend import torch, vLLM and afdEP lazily, +after the adapter has validated the launch. +""" + +# NPU parallel: afd_plugin/ffn_faas/platforms/npu/__init__.py (get_model +# selects the one qualified NPU model). +# Reuse: NO - the GPU backend reads one checkpoint description directly and +# needs no model-adapter selection step. + +GPU_DEEPEP_M2N_BACKEND = "gpu-deepep-m2n" diff --git a/afd_plugin/ffn_faas/platforms/gpu/adapter.py b/afd_plugin/ffn_faas/platforms/gpu/adapter.py new file mode 100644 index 000000000..3f6f5197d --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/gpu/adapter.py @@ -0,0 +1,153 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""CPU-only launch adaptation for the GPU DeepEP M2N FFN service. + +Imports neither torch, vLLM nor afdEP: every check here runs before a device +or process group exists, so a bad launch exits with a usage error and +allocates nothing. Native imports happen in the backend factory. +""" + +from __future__ import annotations + +import os +from argparse import ArgumentParser, Namespace +from functools import partial +from typing import TYPE_CHECKING + +from afd_plugin.config import DEEPEP_M2N_CONNECTOR +from afd_plugin.ffn_faas.interfaces import FFNLaunch +from afd_plugin.ffn_faas.platforms.gpu.qwen3_model import Qwen3CheckpointDescription +from afd_plugin.ffn_faas.platforms.gpu.settings import ( + BF16_PAYLOAD, + DEFAULT_AFD_PORT, + DEFAULT_AFD_TIMEOUT_S, + GPUFFNConfig, + parse_connector_extra_config, +) + +if TYPE_CHECKING: + from afd_plugin.ffn_faas.interfaces import FFNBackend + +V2_MODEL_RUNNER_ENV = "VLLM_USE_V2_MODEL_RUNNER" + + +# NPU parallel: afd_plugin/ffn_faas/platforms/npu/adapter.py::create_backend +# (bootstraps the Ascend OPP environment, then imports the backend). +# Reuse: PARTIAL - the deferred-import factory shape is the same; GPU needs +# no environment bootstrap before importing torch. +def create_backend( + checkpoint: Qwen3CheckpointDescription, settings: GPUFFNConfig +) -> FFNBackend: + """Import the GPU backend only when the host loop constructs it. + + Called by ``FFNModelRunner.run`` after the management endpoint is up, so + torch, vLLM and afdEP load inside the launched F process and an import + failure is reported as a FAILED service rather than a usage error. + """ + from afd_plugin.ffn_faas.platforms.gpu.backend import DeepEPM2NFFNBackend + + return DeepEPM2NFFNBackend(checkpoint, settings) + + +# NPU parallel: afd_plugin/ffn_faas/platforms/npu/adapter.py::AscendCAMAdapter. +# Reuse: PARTIAL - the RANK/LOCAL_RANK/WORLD_SIZE parsing, the WORLD_SIZE == +# --ffn-ranks check and the ModelRunnerV2 environment check are the same; +# the option set differs because GPU adopts Attention TP, capacity and the +# bucket ladder from the M2N handshake. +# REFACTOR-CANDIDATE(R2): a shared F-only launcher-environment reader. +class GPUDeepEPM2NAdapter: + """Keep GPU, topology and connector options out of the host loop.""" + + def add_arguments(self, parser: ArgumentParser) -> None: + """Register the GPU backend's options on the shared ``serve`` parser. + + ``--host``/``--port`` must equal Attention's ``afd.host``/``afd.port``, + ``--attention-ranks``/``--ffn-ranks`` its rank counts, ``--dbo`` its + ``--enable-dbo`` and ``--connector-extra-config`` its must-match + connector keys. There is no Attention TP, batch or capture-size flag: + F learns them from the M2N handshake and control payloads. + """ + parser.add_argument("--host", default="127.0.0.1", help="AFD rendezvous host") + parser.add_argument( + "--port", type=int, default=DEFAULT_AFD_PORT, help="AFD rendezvous port" + ) + parser.add_argument("--attention-ranks", type=int, required=True) + parser.add_argument("--ffn-ranks", type=int, required=True) + parser.add_argument( + "--graph", + action="store_true", + help="capture FULL_DECODE_ONLY CUDA graphs on F (default: eager)", + ) + parser.add_argument( + "--dbo", + action="store_true", + help="two DeepEP lanes; required when Attention runs --enable-dbo", + ) + parser.add_argument( + "--connector-extra-config", + default="{}", + help="DeepEPM2NAFDConnector options as a JSON object", + ) + parser.add_argument( + "--afd-timeout", + type=int, + default=DEFAULT_AFD_TIMEOUT_S, + help="afd_process_group_timeout_s for the M2N rendezvous", + ) + + def configure(self, args: Namespace) -> FFNLaunch: + """Validate the launch on the CPU and describe the backend to build. + + Reads this rank from the ``torchrun`` environment and requires the + F-only world (``WORLD_SIZE == --ffn-ranks``) and + ``VLLM_USE_V2_MODEL_RUNNER=1`` (Attention runs ModelRunnerV2 and the + FFN runner validates the same deployment). Parses the connector JSON + structurally, reads the checkpoint description, and checks that the + experts divide across the F ranks. Raises ``ValueError`` or + ``OSError``, which the entry point reports as a usage error. + """ + rank, local_rank, world = ( + int(os.environ.get(key, default)) + for key, default in ( + ("RANK", "0"), + ("LOCAL_RANK", "0"), + ("WORLD_SIZE", "1"), + ) + ) + if world != args.ffn_ranks: + raise ValueError("F-only launcher WORLD_SIZE must equal --ffn-ranks") + if os.environ.get(V2_MODEL_RUNNER_ENV) != "1": + raise ValueError(f"set {V2_MODEL_RUNNER_ENV}=1 on both A and F") + settings = GPUFFNConfig( + model=args.model, + attention_ranks=args.attention_ranks, + ffn_ranks=args.ffn_ranks, + rank=rank, + local_rank=local_rank, + host=args.host, + port=args.port, + graph=args.graph, + dbo=args.dbo, + connector_extra_config=parse_connector_extra_config( + args.connector_extra_config + ), + afd_timeout_s=args.afd_timeout, + shutdown_timeout=args.shutdown_timeout, + ) + checkpoint = Qwen3CheckpointDescription.read(args.model) + checkpoint.local_experts(settings.ffn_ranks) + return FFNLaunch( + rank=rank, + local_rank=local_rank, + model_path=str(checkpoint.root), + backend_name=DEEPEP_M2N_CONNECTOR, + shutdown_timeout=settings.shutdown_timeout, + backend_factory=partial(create_backend, checkpoint, settings), + initial_status={ + "payload_format": BF16_PAYLOAD, + "num_lanes": settings.num_lanes, + "graph_mode": settings.graph_mode, + "dbo": settings.dbo, + "m2n_paired": False, + }, + ) diff --git a/afd_plugin/ffn_faas/platforms/gpu/backend.py b/afd_plugin/ffn_faas/platforms/gpu/backend.py new file mode 100644 index 000000000..7e250655f --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/gpu/backend.py @@ -0,0 +1,194 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""FFN-only Qwen3 routed experts composed with the existing GPU M2N runtime. + +Imported inside the externally launched F process, after the adapter has +validated the launch. No vLLM Worker, upstream ModelRunner, EngineCore, +scheduler, KV cache or complete CausalLM is constructed. Step, graph, DBO and +transport logic are not re-implemented: they belong to +``GPUFFNRoutedExpertsModelRunner`` and ``DeepEPM2NAFDConnector``. +""" + +from __future__ import annotations + +import threading + +from vllm.config import set_current_vllm_config + +from afd_plugin.connectors.gpu.deepep_m2n.connector import DeepEPM2NAFDConnector +from afd_plugin.ffn_faas.platforms.gpu.environment import GPUFFNEnvironment +from afd_plugin.ffn_faas.platforms.gpu.qwen3_model import Qwen3CheckpointDescription +from afd_plugin.ffn_faas.platforms.gpu.qwen3_routed_model import ( + Qwen3RoutedExpertsModel, + load_qwen3_routed_experts, + parameter_storage_bytes, +) +from afd_plugin.ffn_faas.platforms.gpu.settings import GPUFFNConfig +from afd_plugin.ffn_faas.service import FFNService, ServicePhase +from afd_plugin.v1.worker.ffn_routed_experts_model_runner import ( + GPUFFNRoutedExpertsModelRunner, +) + + +# NPU parallel: afd_plugin/ffn_faas/platforms/npu/backend.py::CAMFFNBackend. +# Reuse: PARTIAL - the same FFNBackend lifecycle (prepare phases, receiver +# thread activation, receiver-exit proof, release order runtime -> +# compute groups -> weights); the native pieces differ (NCCL/DeepEP M2N vs +# HCCL/CAM, lazy graph capture vs capture inside prepare) and GPU adds +# cancel_receive because the M2N control plane can end an idle receive. +# REFACTOR-CANDIDATE(R8): a backend base owning the phase sequence and the +# release-after-proof ordering, with platform hooks. +class DeepEPM2NFFNBackend: + """Keep weights and communication owners until receiver exit is proven.""" + + def __init__( + self, checkpoint: Qwen3CheckpointDescription, settings: GPUFFNConfig + ) -> None: + """Hold the CPU-side launch facts; native work starts in ``prepare``.""" + self.checkpoint = checkpoint + self.settings = settings + self.environment = GPUFFNEnvironment(settings) + self.runner: GPUFFNRoutedExpertsModelRunner | None = None + self.model: Qwen3RoutedExpertsModel | None = None + self._cancel_requested = threading.Event() + + def prepare(self, service: FFNService) -> None: + """Bring this F rank from launch to paired with Attention. + + Runs on the host thread, in the order that lets a bad deployment fail + before the expert weights are read: + + 1. select and check the GPU and the afdEP/DeepGEMM stack; + 2. build the F-only ``VllmConfig`` (AFD architecture, Triton MoE pin); + 3. create the F-only NCCL world and vLLM's model-parallel groups; + 4. construct the routed-experts runner, which runs the ModelRunnerV2 + and CUDA-graph validation and creates the connector (no + collective yet); + 5. load the FFN-only Qwen3 shell and attach it to the runner; + 6. start the connector: the collective M2N handshake with Attention, + the model-info check, the NCCL control group, the lanes' DeepEP + buffers and the dispatch-kernel precompile. + + Phases: ``loading`` through step 5 (with ``total_experts`` and, after + the load, ``loaded_experts``), ``connecting`` from step 5 on; the host + marks ``ready`` afterwards. CUDA graphs are not captured here: they + are captured lazily, driven by Attention's warmup and capture + payloads, exactly as on the Worker path. Every owner is assigned + before the next native step, so a failure leaves it reachable. + """ + environment = self.environment + environment.initialize_device() + config = environment.build_config() + environment.initialize_compute(config) + # Counted like the NPU backend: this rank's experts over every layer. + total_experts = self.checkpoint.num_layers * self.checkpoint.local_experts( + self.settings.ffn_ranks + ) + with set_current_vllm_config(config): + self.runner = GPUFFNRoutedExpertsModelRunner(config, environment.device) + connector = self.runner.connector + if not isinstance(connector, DeepEPM2NAFDConnector): + raise TypeError( + f"standalone GPU FFN needs DeepEPM2NAFDConnector, got " + f"{type(connector).__name__}" + ) + service.loading_progress(0, total_experts) + self.model = load_qwen3_routed_experts(config, environment.device) + # vLLM's loader reports no per-expert progress; this rank's experts + # are all loaded once it returns. + service.loading_progress(total_experts, total_experts) + parameter_bytes = parameter_storage_bytes(self.model) + self.runner.attach_model(self.model, model_memory_usage=parameter_bytes) + local_expert_ids = self.runner.local_experts[ + self.runner.moe_layer_indices[0] + ].expert_ids + service.phase( + ServicePhase.CONNECTING, + model_loaded=True, + parameter_bytes=parameter_bytes, + local_expert_first=local_expert_ids[0], + local_expert_last=local_expert_ids[-1], + ) + self.runner.initialize_afd_connector() + assert connector.attn_tp_size is not None + assert connector.capacity_tokens_per_rank is not None + assert connector.bucket_ladder is not None + assert connector.network_plan is not None + service.phase( + ServicePhase.CONNECTING, + m2n_paired=True, + attn_tp_size=connector.attn_tp_size, + capacity_tokens_per_rank=connector.capacity_tokens_per_rank, + num_buckets=len(connector.bucket_ladder), + network_hybrid=connector.network_plan.hybrid, + compute_ep_world_size=environment.ep_group.world_size, + ) + + # NPU parallel: CAMFFNBackend.start_receiver. + # Reuse: YES - identical delegation to the environment. + def start_receiver(self) -> None: + """Select the GPU once on the receiving thread, as the Worker does.""" + self.environment.activate_thread() + + # NPU parallel: CAMFFNBackend.execute_step. + # Reuse: PARTIAL - both delegate to the runner's + # execute_connector_driven_step; NPU adds its eager fence and never + # reports a cancel, GPU's runner already ends each step with a + # synchronize. + def execute_step(self) -> bool: + """Run one control payload's step through the shared GPU runner method. + + Returns ``False`` only when ``cancel_receive`` closed the control + plane while the receive was idle. A closed control plane without a + cancel request, and any other error (including a DeepEP timeout after + a peer failed), raises and fails the service with owners retained. + """ + assert self.runner is not None + return self.runner.execute_connector_driven_step( + cancel_requested=self._cancel_requested.is_set + ) + + # NPU parallel: CAMFFNBackend.cancel_receive (a no-op there). + # Reuse: NO - GPU-only: the M2N control plane owns an abortable group. + def cancel_receive(self) -> None: + """End an idle control receive so a host stop can exit cleanly. + + Records the request first, then closes the connector's control plane, + which aborts its dedicated NCCL control group; a blocked + ``recv_dp_metadata_list`` then raises ``AFDControlPlaneClosedError`` + and ``execute_step`` returns ``False``. The DeepEP lane buffers keep + their own group, so a step already in flight still completes. Same + mechanism as ``AFDFFNWorker.stop_ffn_server_loop``. + """ + self._cancel_requested.set() + if self.runner is not None: + control_plane = self.runner.connector.control_plane + if control_plane is not None: + control_plane.close() + + # NPU parallel: CAMFFNBackend.receiver_exited. + # Reuse: YES - identical delegation to the runner. + def receiver_exited(self) -> None: + """Record on the receiver thread that its loop ended cleanly.""" + assert self.runner is not None + self.runner.mark_receiver_exited() + + # NPU parallel: CAMFFNBackend.close. + # Reuse: PARTIAL - same release order (runtime, compute groups, weights). + def close(self) -> None: + """Release CUDA graphs, DeepEP buffers, compute groups and weights. + + Only after the receiver exit was proven. ``runner.shutdown`` resets + the captured graphs, drops their pool, clears vLLM's forward-context + registrations and the model reference, and closes the connector + (control group and lanes); then the environment destroys the F-only + groups, and the last reference to the FFN-only model is dropped. The + connector's gloo bootstrap group outlives this, which only matters + for an in-process restart, which is not supported. + """ + assert self.runner is not None + if not self.runner.receiver_exited: + raise RuntimeError("GPU FFN receiver has not exited; retain resources") + self.runner.shutdown() + self.environment.close() + self.model = None diff --git a/afd_plugin/ffn_faas/platforms/gpu/environment.py b/afd_plugin/ffn_faas/platforms/gpu/environment.py new file mode 100644 index 000000000..f22b5ef43 --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/gpu/environment.py @@ -0,0 +1,231 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Own the F-local GPU, the F-only vLLM config and its compute groups. + +The M2N world (bootstrap, control and lane groups) belongs to the connector. +The vLLM world, DP and EP groups created here span only the ``torchrun`` F +ranks; they exist so vLLM can construct FusedMoE and place experts, and carry +no per-step traffic. +""" + +from __future__ import annotations + +import importlib +import os +from typing import TYPE_CHECKING + +from afd_plugin.ffn_faas.platforms.gpu.settings import EXTERNAL_LAUNCHER, GPUFFNConfig + +if TYPE_CHECKING: + import torch + from vllm.config import VllmConfig + from vllm.distributed.parallel_state import GroupCoordinator + +# afdEP-comm transport and the DeepGEMM grouped GEMMs the connector runs; both +# are imported lazily by the connector, so a missing package would otherwise +# fail only after 27 GiB of experts were read. +AFDEP_TRANSPORT_MODULE = "afdep.deepep" +DEEPGEMM_MODULE = "deep_gemm" +FUSED_EXPERTS_BACKEND = "fused_experts" +# vLLM's GPU Worker gives DBO two workspace slots. +DBO_WORKSPACE_UBATCHES = 2 + + +# NPU parallel: afd_plugin/ffn_faas/platforms/npu/environment.py:: +# AscendFFNEnvironment. +# Reuse: PARTIAL - the lifecycle (launcher check, refuse borrowed groups, +# device, F-only world + EP group == F world, per-thread device, ordered +# teardown) is the same; NPU initializes HCCL explicitly plus Ascend groups +# and CPU binding, GPU reuses vLLM's GPU Worker distributed initializer, +# pins the routed MoE backend and selects the AFD model architecture. +# REFACTOR-CANDIDATE(R1): a platform-neutral F-only environment base +# (launcher check, ownership flags, EP-group check, teardown order). +class GPUFFNEnvironment: + """Never borrow unrelated groups or tear them down on a failed native seam.""" + + def __init__(self, settings: GPUFFNConfig) -> None: + """Record the settings; nothing native happens until ``initialize_device``.""" + self.settings = settings + self._device: torch.device | None = None + self._ep_group: GroupCoordinator | None = None + self._groups_owned = False + + @property + def device(self) -> torch.device: + """This rank's GPU, available after ``initialize_device``.""" + if self._device is None: + raise RuntimeError("GPU FFN device has not been initialized") + return self._device + + @property + def ep_group(self) -> GroupCoordinator: + """vLLM's EP group over the F-only world, after ``initialize_compute``.""" + if self._ep_group is None: + raise RuntimeError("GPU FFN expert group has not been initialized") + return self._ep_group + + # NPU parallel: AscendFFNEnvironment.initialize_device. + # Reuse: PARTIAL - launcher and borrowed-group checks are identical; + # the device and native preflight are platform-specific. + def initialize_device(self) -> None: + """Select this rank's GPU after checking the launcher and native stack. + + Requires the ``torchrun`` environment to still match the settings and + no pre-initialized ``torch.distributed`` or vLLM model-parallel state, + so this process never adopts groups it does not own. Imports afdEP's + DeepEP transport and, for the DeepGEMM expert backend, DeepGEMM, so a + missing package fails before any weight is read. Maps ``LOCAL_RANK`` + to the visible GPU the way vLLM's GPU Worker does (honoring + ``CUDA_VISIBLE_DEVICES``) and requires SM90 or newer. + """ + if self._groups_owned or self._device is not None: + raise RuntimeError("GPU FFN environment is already initialized") + settings = self.settings + try: + world, rank = int(os.environ["WORLD_SIZE"]), int(os.environ["RANK"]) + except (KeyError, ValueError) as error: + raise ValueError("GPU FFN requires an external F-only launcher") from error + if world != settings.ffn_ranks or rank != settings.rank: + raise ValueError("GPU FFN environment differs from the F-only launcher") + import torch + from vllm.distributed.parallel_state import model_parallel_is_initialized + from vllm.platforms import current_platform + + if torch.distributed.is_initialized() or model_parallel_is_initialized(): + raise RuntimeError("GPU FFN cannot borrow existing native groups") + importlib.import_module(AFDEP_TRANSPORT_MODULE) + expert_backend = settings.connector_extra_config.get("expert_backend") + if expert_backend != FUSED_EXPERTS_BACKEND: + importlib.import_module(DEEPGEMM_MODULE) + # Importable only after the afdEP preflight above. + from afd_plugin.connectors.gpu.deepep_m2n.connector import MIN_SM_MAJOR + + visible_index = current_platform.logical_device_id_to_visible_device_id( + settings.local_rank + ) + device = torch.device("cuda", visible_index) + major, minor = torch.cuda.get_device_capability(device) + if major < MIN_SM_MAJOR: + raise RuntimeError( + f"DeepEP M2N needs SM{MIN_SM_MAJOR}0 or newer, got SM{major}{minor}" + ) + torch.accelerator.set_device_index(device) + self._device = device + + # NPU parallel: CAMFFNBackend.prepare (EngineArgs, create_engine_config, + # re-asserted external_launcher). + # Reuse: PARTIAL - see REFACTOR-CANDIDATE(R4) in settings.engine_kwargs. + def build_config(self) -> VllmConfig: + """Build the F-only ``VllmConfig`` exactly as the GPU FFN Worker sees it. + + Creates the config from ``GPUFFNConfig.engine_kwargs`` through + ``EngineArgs.create_engine_config`` (with the AFD plugin's config + patches, which accept DBO without a DeepEP all2all backend), then + re-asserts the ``external_launcher`` executor, swaps in the + AFD-prefixed model architecture that ``AFDFFNWorker.init_device`` + selects, and pins vLLM's MoE backend to Triton as + ``AFDFFNWorker._make_model_runner`` does. The pin must happen before + any FusedMoE is constructed: FusedMoE picks its weight layout at + construction, and a reordered layout would be read silently wrong. + """ + from vllm.engine.arg_utils import EngineArgs + + from afd_plugin.model_executor.models.model_utils import get_afd_model_config + from afd_plugin.v1.worker.ffn_routed_experts_model_runner import ( + select_routed_moe_backend, + ) + + config = EngineArgs(**self.settings.engine_kwargs()).create_engine_config() + # The F world is the already launched torchrun world, whatever executor + # platform defaults would pick for an ordinary engine. + config.parallel_config.distributed_executor_backend = EXTERNAL_LAUNCHER + config.model_config = get_afd_model_config( + config.model_config, device_type="cuda" + ) + select_routed_moe_backend(config) + return config + + # NPU parallel: AscendFFNEnvironment.initialize_compute. + # Reuse: PARTIAL - ownership-before-collective and the EP-group check are + # identical; GPU calls vLLM's GPU Worker initializer instead of + # init_distributed_environment + initialize_model_parallel directly. + def initialize_compute(self, config: VllmConfig) -> None: + """Create the F-only NCCL world, model-parallel groups and workspace. + + Uses ``init_worker_distributed_environment``, the same function vLLM's + GPU Worker calls (batch invariance, custom all-reduce setting, world, + TP/PP/DP/EP groups), with the ``env://`` rendezvous ``torchrun`` + provides. With ``external_launcher`` vLLM keeps the launcher's world + size and rank, so EP spans exactly the F ranks; this is re-checked + because the connector's handshake assumes contiguous expert ranges + over that world. Ownership is recorded before the first collective so + a partial initialization is still torn down by ``close``. Finally + initializes vLLM's workspace manager with the Worker's micro-batch + count (two under DBO). + """ + if self._device is None or self._groups_owned: + raise RuntimeError("GPU FFN compute initialization requires its device") + from vllm.config import set_current_vllm_config + from vllm.distributed.parallel_state import get_ep_group + from vllm.platforms import current_platform + from vllm.v1.worker.gpu_worker import init_worker_distributed_environment + from vllm.v1.worker.workspace import init_workspace_manager + + settings = self.settings + with set_current_vllm_config(config): + self._groups_owned = True + init_worker_distributed_environment( + config, + settings.rank, + "env://", + settings.local_rank, + current_platform.dist_backend, + ) + self._ep_group = get_ep_group() + if ( + self._ep_group.world_size != settings.ffn_ranks + or self._ep_group.rank_in_group != settings.rank + ): + raise ValueError("vLLM expert group differs from the F-only world") + # Same slot count as Worker._init_workspace_and_model_runner. + init_workspace_manager( + self._device, + DBO_WORKSPACE_UBATCHES if config.parallel_config.enable_dbo else 1, + ) + + # NPU parallel: AscendFFNEnvironment.activate_thread. + # Reuse: YES (shape) - one call; only the device API differs. + def activate_thread(self) -> None: + """Select this rank's GPU on the calling (receiver) thread. + + CUDA's current device is per thread; graph capture and the connector's + lane streams run on the receiver thread. + """ + import torch + + torch.cuda.set_device(self.device) + + # NPU parallel: AscendFFNEnvironment.close. + # Reuse: PARTIAL - same order and ownership guard; NPU also destroys the + # Ascend model-parallel groups. + def close(self) -> None: + """Destroy the vLLM model-parallel groups and the F-only world. + + Only after the runner released CUDA graphs and DeepEP buffers; a failed + native seam never reaches here, so groups stay owned until process + isolation. + """ + if not self._groups_owned: + return + from vllm.distributed.parallel_state import ( + destroy_distributed_environment, + destroy_model_parallel, + ) + from vllm.v1.worker.workspace import reset_workspace_manager + + reset_workspace_manager() + destroy_model_parallel() + destroy_distributed_environment() + self._groups_owned = False + self._ep_group = None + self._device = None diff --git a/afd_plugin/ffn_faas/platforms/gpu/qwen3_model.py b/afd_plugin/ffn_faas/platforms/gpu/qwen3_model.py new file mode 100644 index 000000000..7f2d9710e --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/gpu/qwen3_model.py @@ -0,0 +1,132 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""CPU-only description of the Qwen3 MoE checkpoints the GPU FFN service serves. + +Reads ``config.json`` only. Rejects every Qwen3 shape the routed-experts M2N +path cannot serve before any device, process group or weight is touched. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +# NPU parallel: afd_plugin/ffn_faas/platforms/npu/v4_model.py:: +# V4CheckpointDescription (CPU-only checkpoint facts for /status and +# startup validation). +# Reuse: PARTIAL - the read-validate-describe pattern and the positive-int +# helper are the same; the V4 description also maps W4A8 safetensors names +# for its own loader, while Qwen3 loads through vLLM's native loader. +# REFACTOR-CANDIDATE(R6): a shared CPU checkpoint-config reader (config +# JSON parse, positive-int fields, dtype check). + +QWEN3_MOE_ARCHITECTURE = "Qwen3MoeForCausalLM" +BF16_DTYPE = "bfloat16" +BF16_BYTES = 2 +# gate_proj, up_proj and down_proj per routed expert. +PROJECTIONS_PER_EXPERT = 3 +# Qwen3 MoE places an MoE block on every layer whose index + 1 is a multiple of +# decoder_sparse_step; the routed-experts path requires every layer to be MoE. +DENSE_FREE_SPARSE_STEP = 1 + + +def _positive_int(config: dict[str, Any], name: str) -> int: + """Read a required positive integer field, rejecting bools and absent keys.""" + value = config.get(name) + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise ValueError(f"Qwen3 MoE checkpoint {name} must be a positive integer") + return value + + +@dataclass(frozen=True) +class Qwen3CheckpointDescription: + """Facts about one supported Qwen3 MoE checkpoint, read on the CPU.""" + + root: Path + num_layers: int + num_experts: int + hidden_size: int + moe_intermediate_size: int + top_k: int + + @property + def layer_indices(self) -> tuple[int, ...]: + """MoE layer indices; every layer is MoE for a supported checkpoint.""" + return tuple(range(self.num_layers)) + + @classmethod + def read(cls, model_path: str | Path) -> Qwen3CheckpointDescription: + """Parse ``config.json`` and accept only the routed-experts Qwen3 shape. + + Accepts ``Qwen3MoeForCausalLM`` in BF16 without quantization, with an + MoE block on every layer (no ``mlp_only_layers``, sparse step 1) and + no shared expert. ``shared_expert_intermediate_size`` that is absent or + ``null`` counts as zero, as vLLM reads it; the stock Qwen3-30B-A3B + config omits it. Newer configs name the dtype ``dtype`` instead of + ``torch_dtype``; either is read. Raises ``ValueError`` for anything + else, and ``OSError`` when the path or file is missing. + """ + root = Path(model_path).resolve(strict=True) + try: + config = json.loads((root / "config.json").read_text()) + except json.JSONDecodeError as error: + raise ValueError(f"invalid Qwen3 config.json: {error}") from error + if not isinstance(config, dict): + raise ValueError("Qwen3 config.json must be a JSON object") + if config.get("architectures") != [QWEN3_MOE_ARCHITECTURE]: + raise ValueError( + f"GPU standalone FFN supports only {QWEN3_MOE_ARCHITECTURE}, got " + f"{config.get('architectures')!r}" + ) + dtype = config.get("torch_dtype", config.get("dtype")) + if dtype != BF16_DTYPE: + raise ValueError( + f"GPU standalone FFN requires a BF16 checkpoint, got {dtype!r}" + ) + if config.get("quantization_config") is not None: + raise ValueError( + "GPU standalone FFN does not support quantized checkpoints" + ) + if config.get("mlp_only_layers"): + raise ValueError("GPU standalone FFN requires every layer to be MoE") + if config.get("decoder_sparse_step", DENSE_FREE_SPARSE_STEP) != ( + DENSE_FREE_SPARSE_STEP + ): + raise ValueError("GPU standalone FFN requires decoder_sparse_step=1") + if (config.get("shared_expert_intermediate_size") or 0) > 0: + raise ValueError("GPU standalone FFN does not support shared experts") + return cls( + root=root, + num_layers=_positive_int(config, "num_hidden_layers"), + num_experts=_positive_int(config, "num_experts"), + hidden_size=_positive_int(config, "hidden_size"), + moe_intermediate_size=_positive_int(config, "moe_intermediate_size"), + top_k=_positive_int(config, "num_experts_per_tok"), + ) + + def local_experts(self, ffn_ranks: int) -> int: + """Experts per F rank under linear placement; rejects an uneven split.""" + if ffn_ranks <= 0 or self.num_experts % ffn_ranks: + raise ValueError( + f"{self.num_experts} routed experts do not divide across " + f"{ffn_ranks} FFN ranks" + ) + return self.num_experts // ffn_ranks + + def estimated_local_expert_bytes(self, ffn_ranks: int) -> int: + """BF16 bytes of this rank's routed experts over every layer. + + ``layers * local experts * 3 projections * hidden * intermediate * 2``; + 48 x 64 x 3 x 2048 x 768 x 2 B is about 27 GiB for Qwen3-30B-A3B at 2F. + The MoE gate (``num_experts x hidden`` per layer) is not counted. + """ + return ( + self.num_layers + * self.local_experts(ffn_ranks) + * PROJECTIONS_PER_EXPERT + * self.hidden_size + * self.moe_intermediate_size + * BF16_BYTES + ) diff --git a/afd_plugin/ffn_faas/platforms/gpu/qwen3_routed_model.py b/afd_plugin/ffn_faas/platforms/gpu/qwen3_routed_model.py new file mode 100644 index 000000000..7561bf74a --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/gpu/qwen3_routed_model.py @@ -0,0 +1,218 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""FFN-only Qwen3 MoE model: the routed experts and nothing else. + +The Worker-path FFN loads ``AFDQwen3MoeForCausalLM`` with ``role="ffn"``. Its +decoder layers already skip Attention, but the CausalLM still builds the +embedding, final norm, LM head and logits processor (about 1.16 GiB per F +rank for Qwen3-30B-A3B, never used). This shell keeps the native module tree +only where the routed experts live, ``model.layers..mlp``, built from the +native ``Qwen3MoeSparseMoeBlock``, so vLLM's own loader, EP weight filter, +FusedMoE expert loading and ``process_weights_after_loading`` run unchanged. + +The MoE forward never runs here: the M2N connector's executor reads the +experts' parameters as views (``local_experts_from_vllm``). The native gate is +built because the block constructs it and the strict loader expects its +weight; it is never executed (Attention routes). +""" + +from __future__ import annotations + +import gc +from collections.abc import Iterable +from typing import TYPE_CHECKING + +import torch +from torch import nn +from vllm.config import VllmConfig +from vllm.model_executor.model_loader import get_model_loader +from vllm.model_executor.model_loader.utils import process_weights_after_loading +from vllm.model_executor.models import qwen3_moe as native +from vllm.model_executor.models.utils import AutoWeightsLoader +from vllm.utils.torch_utils import set_default_torch_dtype + +from afd_plugin.model_executor.models.qwen3_moe import _weight_layer_path + +if TYPE_CHECKING: + from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts + +ROUTED_STAGE = "mlp" +MODEL_PREFIX = "model" + + +def _is_routed_mlp_path(name: str) -> bool: + """Whether a checkpoint tensor belongs to a decoder layer's MoE block. + + Keeps ``model.layers..mlp.*`` (routed experts and the unused gate) and + drops embeddings, norms, Attention and the LM head, which this shell does + not construct. Reuses the AFD Qwen3 wrapper's path parser so both models + classify checkpoint names identically. + """ + layer_path = _weight_layer_path(name) + return layer_path is not None and layer_path[1] == ROUTED_STAGE + + +# NPU parallel: afd_plugin/ffn_faas/platforms/npu/model.py::RoutedWeightOwner +# via V4ModelAdapter.create_owner/create_layer (native routed modules built +# layer by layer from the pinned Ascend loader). +# Reuse: NO - NPU constructs and loads W4A8 layered weights through its own +# safetensors reader; GPU keeps vLLM's module tree and native loader. +class _Qwen3RoutedExpertsLayer(nn.Module): + """One decoder layer reduced to its native MoE block at ``.mlp``.""" + + def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: + """Construct the native ``Qwen3MoeSparseMoeBlock`` with its native prefix. + + The prefix keeps FusedMoE's ``layer_name`` and its + ``static_forward_context`` key identical to the Worker-path model. + """ + super().__init__() + self.mlp = native.Qwen3MoeSparseMoeBlock( + vllm_config=vllm_config, prefix=f"{prefix}.{ROUTED_STAGE}" + ) + + +class _Qwen3RoutedExpertsStack(nn.Module): + """The ``model`` submodule: only ``layers``, each holding its MoE block.""" + + # Upstream: vLLM v0.32.0, vllm/model_executor/models/qwen3_moe.py, + # Qwen3MoeModel.hf_to_vllm_mapper; reused, not copied. + hf_to_vllm_mapper = native.Qwen3MoeModel.hf_to_vllm_mapper + + def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: + """Build one ``_Qwen3RoutedExpertsLayer`` per decoder layer.""" + super().__init__() + num_layers = vllm_config.model_config.hf_text_config.num_hidden_layers + self.layers = nn.ModuleList( + _Qwen3RoutedExpertsLayer(vllm_config, f"{prefix}.layers.{layer_idx}") + for layer_idx in range(num_layers) + ) + + # Upstream source: vLLM v0.32.0 (5df63a7203cdb9328d824a27b682e77c7b9928be), + # vllm/model_executor/models/qwen3_moe.py, Qwen3MoeModel.load_weights. + # Copied unchanged: the same AutoWeightsLoader, ignored suffixes and + # checkpoint mapper, so expert stacking and EP sharding stay in FusedMoE's + # own loader. Re-copy on a vLLM upgrade. + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + """Load ``layers.*`` tensors through the native Qwen3 model loader path. + + ``AutoWeightsLoader`` walks the module tree and hands each MoE block's + ``experts.*`` tensors to FusedMoE's ``load_weights``, which stacks + per-expert gate/up/down projections into ``w13``/``w2`` and keeps + only this rank's experts. Returns the loaded parameter names for the + strict loaded-weights check. + """ + loader = AutoWeightsLoader( + self, + ignore_unexpected_suffixes=[ + ".bias", + "_bias", + ".weight_scale", + "_weight_scale", + ".input_scale", + "_input_scale", + ], + ) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) + + +class Qwen3RoutedExpertsModel(nn.Module): + """FFN-only Qwen3 MoE model consumed by ``GPUFFNRoutedExpertsModelRunner``. + + Implements ``RoutedExpertsModel``: ``get_experts_layer_indices`` and + ``get_routed_experts``, the same accessors ``AFDQwen3MoeForCausalLM`` + exposes on the Worker path. + """ + + # Same as native Qwen3MoeForCausalLM: load safetensors only, never fall + # back to PyTorch checkpoint files. + fall_back_to_pt_during_load = False + + def __init__(self, *, vllm_config: VllmConfig) -> None: + """Build ``model.layers..mlp`` per layer, named as in the native model. + + Must run under ``set_current_vllm_config`` and after + ``select_routed_moe_backend``: FusedMoE reads the current config and + picks its weight layout at construction. + """ + super().__init__() + self.model = _Qwen3RoutedExpertsStack(vllm_config, MODEL_PREFIX) + + # Upstream source: vLLM v0.32.0 (5df63a7203cdb9328d824a27b682e77c7b9928be), + # vllm/model_executor/models/qwen3_moe.py, Qwen3MoeForCausalLM.load_weights. + # Copied with one marked difference, the routed-path filter. + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + """Drop non-MoE checkpoint tensors, then load like the native CausalLM. + + The native CausalLM delegates to ``AutoWeightsLoader(self)``, which + recurses into ``model.load_weights``. Tensors this shell does not + construct (embeddings, norms, Attention, LM head) are filtered first, + as ``AFDQwen3MoeForCausalLM.load_weights`` filters by role, but + stricter: that wrapper still keeps non-layer tensors for its + embedding and LM head. + """ + # ### PATCH START: keep only routed MoE checkpoint paths. + weights = ( + (name, tensor) for name, tensor in weights if _is_routed_mlp_path(name) + ) + # ### PATCH END: keep only routed MoE checkpoint paths. + loader = AutoWeightsLoader(self) + return loader.load_weights(weights) + + def get_experts_layer_indices(self) -> tuple[int, ...]: + """Every decoder layer is an MoE layer for a supported checkpoint.""" + return tuple(range(len(self.model.layers))) + + def get_routed_experts(self, layer_idx: int) -> RoutedExperts: + """The vLLM module holding one MoE layer's local experts.""" + return self.model.layers[layer_idx].mlp.experts.routed_experts + + +# NPU parallel: afd_plugin/ffn_faas/platforms/npu/model.py::CAMRoutedModel.load +# (layer-by-layer safetensors load with per-layer progress, then storage +# accounting). +# Reuse: PARTIAL - storage accounting is the same computation +# (REFACTOR-CANDIDATE(R7)); loading differs: vLLM's loader exposes no +# per-layer callback, so progress is reported only before and after. +# Upstream source: vLLM v0.32.0 (5df63a7203cdb9328d824a27b682e77c7b9928be), +# vllm/model_executor/model_loader/base_loader.py, BaseModelLoader.load_model. +# Same steps; the model is constructed here instead of by initialize_model, +# and online quantization is not handled (BF16 only). +def load_qwen3_routed_experts( + vllm_config: VllmConfig, device: torch.device +) -> Qwen3RoutedExpertsModel: + """Construct the FFN-only shell on ``device`` and load its weights. + + Follows ``BaseModelLoader.load_model``: construct and load under the model + dtype and target device, run vLLM's ``DefaultModelLoader.load_weights`` + (EP weight filter, then the strict loaded-weights check), release loading + temporaries, run ``process_weights_after_loading`` (the Triton backend + keeps the FusedMoE layout), and return the model in eval mode. The caller + holds ``set_current_vllm_config``. + """ + model_config = vllm_config.model_config + loader = get_model_loader(vllm_config.load_config) + with set_default_torch_dtype(model_config.dtype), torch.device(device): + model = Qwen3RoutedExpertsModel(vllm_config=vllm_config) + loader.load_weights(model, model_config) + gc.collect() + torch.accelerator.empty_cache() + process_weights_after_loading(model, model_config, device) + gc.collect() + torch.accelerator.empty_cache() + return model.eval() + + +# NPU parallel: CAMRoutedModel._load_native storage accounting. +# Reuse: YES - same computation; REFACTOR-CANDIDATE(R7). +def parameter_storage_bytes(model: nn.Module) -> int: + """Bytes of the distinct storages behind ``model``'s parameters. + + Views that share storage (for example the connector's expert views) are + counted once. + """ + storage_sizes: dict[tuple[str, int], int] = {} + for parameter in model.parameters(): + storage = parameter.untyped_storage() + storage_sizes[(str(parameter.device), storage.data_ptr())] = storage.nbytes() + return sum(storage_sizes.values()) diff --git a/afd_plugin/ffn_faas/platforms/gpu/settings.py b/afd_plugin/ffn_faas/platforms/gpu/settings.py new file mode 100644 index 000000000..e97f3877e --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/gpu/settings.py @@ -0,0 +1,183 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""CPU-only launch settings for the standalone GPU DeepEP M2N FFN process. + +``GPUFFNConfig`` produces the same AFD config a Worker-path FFN recipe passes +to ``vllm serve``, plus the ``EngineArgs`` of an F-only world launched by +``torchrun``. Connector option values stay with ``DeepEPM2NConfig``; only the +options the FFN service itself decides (payload format, lane count) are +checked here. +""" + +from __future__ import annotations + +import json +import math +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path +from types import MappingProxyType +from typing import Any + +from afd_plugin.config import DEEPEP_M2N_CONNECTOR, AFDConfig + +# The only qualified payload: BF16 rows against BF16 expert weights. +BF16_PAYLOAD = "bf16" +SINGLE_LANE = 1 +# DBO runs two micro-batches, one DeepEP lane each. +DBO_LANES = 2 +FFN_GRAPH_MODE = "FULL_DECODE_ONLY" +EAGER_MODE = "eager" +EXTERNAL_LAUNCHER = "external_launcher" +MAX_PORT = 65535 +DEFAULT_AFD_PORT = AFDConfig.port +DEFAULT_AFD_TIMEOUT_S = AFDConfig.afd_process_group_timeout_s + + +# NPU parallel: afd_plugin/ffn_faas/platforms/npu/settings.py::FFNConfig. +# Reuse: PARTIAL - rank, host/port and shutdown-timeout validation are the same +# checks; NPU also owns the CAM dispatch envelope, Attention TP and core +# budgets, which GPU adopts from the M2N handshake instead. +# REFACTOR-CANDIDATE(R3): a shared base for the F-only launch fields and +# their validation. +@dataclass(frozen=True) +class GPUFFNConfig: + """Frozen settings of one F rank in a standalone DeepEP M2N deployment.""" + + model: Path + attention_ranks: int + ffn_ranks: int + rank: int = 0 + local_rank: int = 0 + host: str = AFDConfig.host + port: int = DEFAULT_AFD_PORT + graph: bool = False + dbo: bool = False + connector_extra_config: Mapping[str, Any] = field( + default_factory=lambda: MappingProxyType({}) + ) + afd_timeout_s: int = DEFAULT_AFD_TIMEOUT_S + shutdown_timeout: float = 5.0 + + def __post_init__(self) -> None: + """Reject launches no Attention peer could pair with, on the CPU. + + Checks rank counts, this rank, the rendezvous address, timeouts, and + the two connector options the service fixes: ``payload_format`` must + be ``bf16`` and an explicit ``num_lanes`` must equal the lane count + ``--dbo`` implies (1 without, 2 with). Every other connector option is + validated by ``DeepEPM2NConfig.from_mapping`` once the backend runs. + """ + if self.attention_ranks <= 0 or self.ffn_ranks <= 0: + raise ValueError("Attention and FFN rank counts must be positive") + if not 0 <= self.rank < self.ffn_ranks or self.local_rank < 0: + raise ValueError("invalid F rank or local rank") + if not self.host or not 0 < self.port <= MAX_PORT: + raise ValueError("DeepEP M2N rendezvous requires a host and a valid port") + if self.afd_timeout_s <= 0: + raise ValueError("--afd-timeout must be positive") + if not math.isfinite(self.shutdown_timeout) or self.shutdown_timeout <= 0: + raise ValueError("shutdown timeout must be finite and positive") + payload_format = self.connector_extra_config.get("payload_format", BF16_PAYLOAD) + if payload_format != BF16_PAYLOAD: + raise ValueError( + f"GPU standalone FFN supports only payload_format={BF16_PAYLOAD!r}, " + f"got {payload_format!r}" + ) + num_lanes = self.connector_extra_config.get("num_lanes") + if num_lanes is not None and num_lanes != self.num_lanes: + raise ValueError( + f"connector num_lanes={num_lanes!r} disagrees with " + f"{'--dbo' if self.dbo else 'no --dbo'} ({self.num_lanes} lane(s))" + ) + + @property + def num_lanes(self) -> int: + """DeepEP lanes this rank creates: two under DBO, otherwise one.""" + return DBO_LANES if self.dbo else SINGLE_LANE + + @property + def graph_mode(self) -> str: + """Status name of the FFN execution mode.""" + return FFN_GRAPH_MODE if self.graph else EAGER_MODE + + def afd_additional_config(self) -> dict[str, Any]: + """``additional_config`` with the same AFD shape a Worker-path FFN uses. + + Role ``ffn`` on ``DeepEPM2NAFDConnector`` with the Attention-side gate + (required by M2N), the rendezvous address, the rank counts, the + rendezvous timeout and the connector options passed through unchanged. + """ + return { + "afd": { + "role": "ffn", + "connector": DEEPEP_M2N_CONNECTOR, + "host": self.host, + "port": self.port, + "num_attention_ranks": self.attention_ranks, + "num_ffn_ranks": self.ffn_ranks, + "compute_gate_on_attention": True, + "afd_process_group_timeout_s": self.afd_timeout_s, + "connector_extra_config": dict(self.connector_extra_config), + } + } + + # NPU parallel: afd_plugin/ffn_faas/platforms/npu/backend.py:: + # CAMFFNBackend.prepare builds its EngineArgs inline. + # Reuse: PARTIAL - external_launcher, DP = F ranks, TP = 1 and expert + # parallelism are identical; NPU adds Ascend quantization and the CAM + # dispatch envelope, GPU adds graph mode, DBO and the EP weight filter. + # REFACTOR-CANDIDATE(R4): one F-only EngineArgs builder with + # platform-specific extras. + def engine_kwargs(self) -> dict[str, Any]: + """Keyword arguments for ``EngineArgs`` of this F-only rank. + + DP = F ranks and TP = 1 make vLLM's EP group the whole F world, with + ``data_parallel_rank`` = this rank (``resolve_role_rank`` then gives + the same M2N role rank as a Worker-path FFN). ``external_launcher`` + uses the ``torchrun`` world. Graph mode enables FULL_DECODE_ONLY, + which turns on the FFN graph cache; F needs no capture sizes because + graph keys come from Attention's control payloads. ``enable_dbo`` + gives vLLM two micro-batches, so the connector creates two lanes. + Scheduler limits keep vLLM defaults: F adopts the dispatch capacity + and bucket ladder from Attention in the M2N handshake. vLLM's + ``all2all_backend`` and ``moe_backend`` are not set, so the handshake + sees the idle default backend and ``select_routed_moe_backend`` can + pin ``auto`` to Triton. The EP weight filter skips non-local expert + tensors before they are read, as the Worker-path recipes do. + """ + return { + "model": str(self.model), + "dtype": "bfloat16", + "skip_tokenizer_init": True, + "tensor_parallel_size": 1, + "data_parallel_size": self.ffn_ranks, + "data_parallel_rank": self.rank, + "enable_expert_parallel": True, + "enable_ep_weight_filter": True, + "distributed_executor_backend": EXTERNAL_LAUNCHER, + "enforce_eager": not self.graph, + "compilation_config": ( + {"cudagraph_mode": FFN_GRAPH_MODE} if self.graph else {} + ), + "enable_dbo": self.dbo, + "additional_config": self.afd_additional_config(), + } + + +def parse_connector_extra_config(text: str) -> Mapping[str, Any]: + """Parse ``--connector-extra-config`` and require a JSON object. + + Only the structure is checked here; values are validated by the backend's + ``DeepEPM2NConfig.from_mapping``, which owns unknown keys and ranges. + Returns a read-only mapping so the frozen settings stay immutable. + """ + try: + value = json.loads(text) + except json.JSONDecodeError as error: + raise ValueError( + f"--connector-extra-config is not valid JSON: {error}" + ) from error + if not isinstance(value, dict): + raise ValueError("--connector-extra-config must be a JSON object") + return MappingProxyType(value) diff --git a/afd_plugin/ffn_faas/platforms/npu/backend.py b/afd_plugin/ffn_faas/platforms/npu/backend.py index 8f1e0a5b9..a18e40de2 100644 --- a/afd_plugin/ffn_faas/platforms/npu/backend.py +++ b/afd_plugin/ffn_faas/platforms/npu/backend.py @@ -97,12 +97,25 @@ def start_receiver(self) -> None: """Select the device once on the receiving thread, as the Worker does.""" self.environment.activate_thread() - def execute_step(self) -> None: + def execute_step(self) -> bool: + """Run one CAM submission batch; a CAM receive never reports a cancel. + + The return value follows ``FFNBackend.execute_step``: CAM cannot end a + pending receive, so a completed batch always returns ``True``. + """ assert self.runtime is not None self.runtime.execute_connector_driven_step() if not self.settings.full_graph and not self.settings.ffn_two_stream: # Preserve the mature eager loop's completion fence. torch.npu.synchronize() + return True + + def cancel_receive(self) -> None: + """No-op: a pending CAM receive cannot be cancelled. + + The host join then times out, reports the pending receive and retains + every owner, exactly as before ``cancel_receive`` existed. + """ def receiver_exited(self) -> None: assert self.runtime is not None diff --git a/afd_plugin/ffn_faas/runner.py b/afd_plugin/ffn_faas/runner.py index 199293ff5..b7b59aba7 100644 --- a/afd_plugin/ffn_faas/runner.py +++ b/afd_plugin/ffn_faas/runner.py @@ -35,11 +35,19 @@ def __init__( self._failure: BaseException | None = None def _serve(self) -> None: + """Receiver-thread loop: one backend step per control payload. + + The loop ends when the host requests a stop between steps, or when + ``execute_step()`` reports that ``cancel_receive()`` ended an idle + receive. Only then does the backend record its receiver exit, which is + the proof ``close()`` requires before releasing native resources. + """ assert self.backend is not None try: self.backend.start_receiver() while not self.service.stop_requested.is_set(): - self.backend.execute_step() + if not self.backend.execute_step(): + break self.service.completed_batches() self.backend.receiver_exited() except BaseException as error: @@ -48,6 +56,16 @@ def _serve(self) -> None: self._done.set() def run(self) -> None: + """Prepare the backend, serve until stopped, then release or retain. + + The backend is constructed before native initialization so a failed + preparation keeps its owners reachable. After READY the receiver runs + on a daemon thread; on a host stop the backend is asked to cancel an + idle receive, then the thread is joined with ``shutdown_timeout``. + Resources are closed only after a clean receiver exit; any failure, + including a receive that does not end, marks the service FAILED and + re-raises with every owner retained for process isolation. + """ try: if self.service.stop_requested.is_set(): self.service.phase(ServicePhase.STOPPED, resources_released=True) @@ -68,11 +86,17 @@ def run(self) -> None: while not self.service.stop_requested.wait(HOST_STATUS_POLL_SECONDS): if self._done.is_set(): break + if self.service.stop_requested.is_set() and not self._done.is_set(): + # The receiver may be blocked in an idle receive that only + # the backend can end; a backend without cancellation keeps + # it pending and the join below reports it. + self.backend.cancel_receive() self._thread.join(timeout=self.shutdown_timeout) if self._thread.is_alive(): raise RuntimeError( "FFN receive is pending; retain owners and isolate the " - "deployment. A host stop cannot cancel backend device work." + "deployment. The backend did not end its receive or " + "in-flight device work within the shutdown timeout." ) if self._failure is not None: raise RuntimeError( diff --git a/afd_plugin/v1/worker/ffn_model_runner.py b/afd_plugin/v1/worker/ffn_model_runner.py index bab6ac42a..ab23cbe0e 100644 --- a/afd_plugin/v1/worker/ffn_model_runner.py +++ b/afd_plugin/v1/worker/ffn_model_runner.py @@ -4,6 +4,7 @@ from __future__ import annotations +from collections.abc import Callable from contextlib import contextmanager from typing import TYPE_CHECKING, Any, cast @@ -30,6 +31,7 @@ AFDControlPlane, AFDDPMetadata, ) +from afd_plugin.connectors.base import AFDControlPlaneClosedError from afd_plugin.v1.worker.cuda_graph import ( AFDGraphRunMode, graph_run_mode, @@ -150,6 +152,71 @@ def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]: def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: return None + # NPU parallel: afd_plugin/v1/worker/npu/async_ffn_model_runner.py:: + # AsyncNPUFFNModelRunner.execute_connector_driven_step (same name and + # bool return; the CAM step submits layered work items instead of + # receiving one control payload). + # Reuse: NO - the step bodies share no code; only the host contract + # (one call per batch, a bool result) is common. + def execute_connector_driven_step( + self, *, cancel_requested: Callable[[], bool] + ) -> bool: + """Receive one control payload and run the FFN step it describes. + + Blocks in the control plane until Attention rank 0 sends the next + payload. Warmup and capture payloads go to ``capture_model`` when FFN + CUDA graphs are enabled; every other payload (eager, replay, or an + eager fallback for a replay without a captured key) goes to + ``execute_model``. The step ends with a device synchronize so the + caller only sees completed work. + + ``cancel_requested`` tells a cancelled receive apart from a broken + control plane: when the control plane was closed on purpose (the + caller set its stop flag before closing it) the method returns + ``False``; any other closed or failed receive raises unchanged, so a + peer or transport failure is never reported as a clean stop. + + Shared by ``AFDFFNWorker._run_ffn_server_loop`` and the standalone + FFN service backend, so both drive identical capture, replay and + synchronization order. + """ + control_plane = self.connector.control_plane + if control_plane is None: + raise NotImplementedError( + "GPU FFN only supports control-plane-driven connectors; " + "connectors without a control plane (control_plane is None) " + "are not supported.", + ) + + try: + payload = control_plane.recv_dp_metadata_list() + except AFDControlPlaneClosedError: + if not cancel_requested(): + raise + return False + dp_metadata_list = payload.dp_metadata_list + is_attn_graph_capturing = payload.is_graph_capturing + is_warmup = payload.is_warmup + is_graph_replaying = payload.is_graph_replaying + + if self.use_cuda_graph and (is_warmup or is_attn_graph_capturing): + self.capture_model( + dp_metadata_list=dp_metadata_list, + is_warmup=is_warmup, + is_attn_graph_capturing=is_attn_graph_capturing, + ) + else: + self.execute_model( + dp_metadata_list=dp_metadata_list, + is_graph_capturing=is_attn_graph_capturing, + is_warmup=is_warmup, + is_graph_replaying=is_graph_replaying, + ) + + if self.device.type == "cuda": + torch.cuda.synchronize() + return True + def execute_model( self, scheduler_output: SchedulerOutput | None = None, diff --git a/afd_plugin/v1/worker/ffn_routed_experts_model_runner.py b/afd_plugin/v1/worker/ffn_routed_experts_model_runner.py index 55c26f41d..7178163fe 100644 --- a/afd_plugin/v1/worker/ffn_routed_experts_model_runner.py +++ b/afd_plugin/v1/worker/ffn_routed_experts_model_runner.py @@ -4,7 +4,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Protocol import torch from vllm.config import VllmConfig @@ -92,6 +92,18 @@ def local_experts_from_vllm(routed_experts: RoutedExperts) -> FFNLocalExperts: ) +class RoutedExpertsModel(Protocol): + """What the routed runner reads from a model: MoE layers and their experts. + + Implemented by the AFD CausalLM models the Worker path loads and by the + standalone FFN service's FFN-only model shells. + """ + + def get_experts_layer_indices(self) -> tuple[int, ...]: ... + + def get_routed_experts(self, layer_idx: int) -> RoutedExperts: ... + + def _require_routed(connector: AFDConnectorBase) -> AFDRoutedExpertsConnector: if not isinstance(connector, AFDRoutedExpertsConnector): raise TypeError( @@ -121,25 +133,80 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device) -> None: _require_routed(self.connector) self.moe_layer_indices: tuple[int, ...] = () self.local_experts: dict[int, FFNLocalExperts] = {} + self._receiver_exited = False @property def routed_connector(self) -> AFDRoutedExpertsConnector: """The connector, typed as the routed-experts contract it must meet.""" return _require_routed(self.connector) + @property + def receiver_exited(self) -> bool: + """Whether ``mark_receiver_exited`` proved the receive loop finished.""" + return self._receiver_exited + def load_model(self, *, load_dummy_weights: bool = False, **kwargs: Any) -> None: """Load with the native loader, then take each MoE layer's experts. Runs again on ``reload_weights``, whose parameters replace the old ones. """ super().load_model(load_dummy_weights=load_dummy_weights, **kwargs) - model = self.get_model() + self._take_local_experts(self.get_model()) + + # NPU parallel: afd_plugin/v1/worker/npu/async_ffn_model_runner.py:: + # AsyncNPUFFNModelRunner.attach_model (same guard: before connector + # startup and at most once; NPU borrows layered W4A8 weight views). + # Reuse: PARTIAL - the attach-once/before-init guard is identical; the + # expert extraction is GPU-specific (FusedMoE Triton layout views). + # REFACTOR-CANDIDATE(R5): a shared attach guard for both runners. + def attach_model( + self, model: RoutedExpertsModel, *, model_memory_usage: int + ) -> None: + """Adopt a model loaded outside this runner, as ``load_model`` would. + + The standalone FFN service builds and loads its own FFN-only model. + This hands it to the runner and builds the per-layer local-expert + views the connector reads, through the same + ``local_experts_from_vllm`` check ``load_model`` uses (Triton layout). + Must run before ``initialize_afd_connector`` because the connector + all-gathers these views in its startup collective, and only once, + because the connector keeps no way to swap them afterwards. + """ + if self.model is not None or self.connector.is_initialized: + raise RuntimeError( + "a routed FFN model must be attached once, before connector startup" + ) + if model_memory_usage < 0: + raise ValueError("model memory usage must be non-negative") + self.model = model + self.model_memory_usage = model_memory_usage + self._take_local_experts(model) + + def _take_local_experts(self, model: RoutedExpertsModel) -> None: + """Record the model's MoE layers and this rank's expert views per layer.""" self.moe_layer_indices = tuple(model.get_experts_layer_indices()) self.local_experts = { layer_idx: local_experts_from_vllm(model.get_routed_experts(layer_idx)) for layer_idx in self.moe_layer_indices } + # NPU parallel: afd_plugin/v1/worker/npu/async_ffn_model_runner.py:: + # AsyncNPUFFNModelRunner.mark_receiver_exited (records the same proof; + # NPU rejects pending graph work, GPU fences the device instead). + # Reuse: PARTIAL - same contract, device-specific completion check. + def mark_receiver_exited(self) -> None: + """Record that the receive loop ended cleanly and its device work is done. + + Called on the receiver thread after its last step. Each step already + ends with a synchronize; the extra fence here also covers a step that + returned early because a stop cancelled its receive. The standalone FFN + backend releases weights, CUDA graphs and DeepEP buffers only after + this proof. + """ + if self.device.type == "cuda": + torch.cuda.synchronize(self.device) + self._receiver_exited = True + def initialize_afd_connector(self) -> None: self.routed_connector.init_afd_connector( moe_layer_indices=self.moe_layer_indices, @@ -189,4 +256,8 @@ def _ffn_forward( return ffn_output -__all__ = ["GPUFFNRoutedExpertsModelRunner", "local_experts_from_vllm"] +__all__ = [ + "GPUFFNRoutedExpertsModelRunner", + "RoutedExpertsModel", + "local_experts_from_vllm", +] diff --git a/afd_plugin/v1/worker/ffn_worker.py b/afd_plugin/v1/worker/ffn_worker.py index 4a998cc34..6686da273 100644 --- a/afd_plugin/v1/worker/ffn_worker.py +++ b/afd_plugin/v1/worker/ffn_worker.py @@ -15,7 +15,6 @@ from afd_plugin.config import parse_afd_config from afd_plugin.connectors import AFDConnectorFactory, AFDRoutedExpertsConnector -from afd_plugin.connectors.base import AFDControlPlaneClosedError from afd_plugin.model_executor.models.model_utils import get_afd_model_config from afd_plugin.v1.worker.ffn_model_runner import GPUFFNModelRunner from afd_plugin.validation import ( @@ -174,6 +173,13 @@ def ffn_worker_loop() -> None: self._ffn_thread.start() def _run_ffn_server_loop(self) -> None: + """Drive connector steps until the shutdown event stops the loop. + + Each iteration delegates one receive-and-execute step to the runner's + ``execute_connector_driven_step``. The loop ends when the event is set + between steps, or when ``stop_ffn_server_loop`` set the event and then + closed the control plane, which ends a receive blocked while idle. + """ event = self._ffn_shutdown_event if event is None: return @@ -182,43 +188,10 @@ def _run_ffn_server_loop(self) -> None: torch.cuda.set_device(self.device) while not event.is_set(): - control_plane = self.model_runner.connector.control_plane - if control_plane is None: - raise NotImplementedError( - "GPU FFN only supports control-plane-driven connectors; " - "connectors without a control plane (control_plane is None) " - "are not supported.", - ) - - try: - payload = control_plane.recv_dp_metadata_list() - except AFDControlPlaneClosedError: - if not event.is_set(): - raise - break - dp_metadata_list = payload.dp_metadata_list - is_attn_graph_capturing = payload.is_graph_capturing - is_warmup = payload.is_warmup - is_graph_replaying = payload.is_graph_replaying - - if self.model_runner.use_cuda_graph and ( - is_warmup or is_attn_graph_capturing + if not self.model_runner.execute_connector_driven_step( + cancel_requested=event.is_set ): - self.model_runner.capture_model( - dp_metadata_list=dp_metadata_list, - is_warmup=is_warmup, - is_attn_graph_capturing=is_attn_graph_capturing, - ) - else: - self.model_runner.execute_model( - dp_metadata_list=dp_metadata_list, - is_graph_capturing=is_attn_graph_capturing, - is_warmup=is_warmup, - is_graph_replaying=is_graph_replaying, - ) - - if self.device.type == "cuda": - torch.cuda.synchronize() + break def raise_ffn_loop_error_if_any(self) -> None: error = self._ffn_loop_error diff --git a/docs/design/module/ffn_runtime.md b/docs/design/module/ffn_runtime.md index e5301c1d3..5f30107eb 100644 --- a/docs/design/module/ffn_runtime.md +++ b/docs/design/module/ffn_runtime.md @@ -27,6 +27,7 @@ depends_on: - "compatibility_and_patches.md" validation_paths: - "tests/unit/ffn_faas/test_*.py" + - "tests/gpu/test_ffn_faas_gpu.py" - "tests/unit/model_executor/test_async_cam_layered.py" - "tests/npu/test_async_cam_layered_w8a8.py" - "tests/unit/v1/worker/test_ffn_metadata.py" @@ -61,7 +62,7 @@ related_issues: - "#105" - "#107" - "#129" -last_reviewed: 2026-10-26 +last_reviewed: 2026-11-05 --- # FFN runtime @@ -133,10 +134,42 @@ native work retains all owners until whole-pair process isolation. A host See the [standalone service guide](../../npu/STANDALONE_FFN_SERVICE.md) for supported configuration, launch and shutdown behavior. +Qwen3-30B-A3B (BF16) over `DeepEPM2NAFDConnector` has a GPU standalone F, +`--backend gpu-deepep-m2n`, under `ffn_faas/platforms/gpu/`: + +- `GPUDeepEPM2NAdapter`, `GPUFFNConfig` and `Qwen3CheckpointDescription` are + CPU-only: launcher world, ModelRunnerV2 environment, connector JSON + structure, BF16/no-shared-expert/all-MoE checkpoint shape and lane parity + with `--dbo` are checked before any device work. +- `GPUFFNEnvironment` selects the GPU, preflights afdEP and DeepGEMM, builds + the F-only `VllmConfig` (`external_launcher`, DP = F ranks, TP 1, EP), selects + the AFD architecture, pins vLLM's MoE backend to Triton before any FusedMoE + is built, and creates the F-only world through vLLM's + `init_worker_distributed_environment`. +- `Qwen3RoutedExpertsModel` keeps only `model.layers..mlp` (native + `Qwen3MoeSparseMoeBlock`) and loads through vLLM's `DefaultModelLoader`, so + the EP weight filter, FusedMoE expert loading and strict loaded-weights check + run unchanged; the embedding, final norm and LM head are not allocated. +- `DeepEPM2NFFNBackend` constructs `GPUFFNRoutedExpertsModelRunner`, attaches + the model (`attach_model`), starts the connector, and delegates each step to + `execute_connector_driven_step()`, the method the Worker loop also calls. + CUDA graphs are captured lazily from Attention's payloads, as on the Worker + path, so `/ready` means paired and receiving. + +`FFNBackend.execute_step()` returns `False` only when `cancel_receive()` ended +an idle receive; the host calls `cancel_receive()` after a stop request and +before joining the receiver. The GPU backend closes the M2N control plane +(aborting its own NCCL group); the NPU backend's `cancel_receive()` is a no-op. +Releasing DeepEP lanes is a barrier over every M2N rank, so a stopped GPU F +completes its release when Attention shuts down too. See the +[GPU standalone service guide](../../gpu/STANDALONE_FFN_SERVICE.md). + | Platform | Worker | Model runner | Current connectors | | --- | --- | --- | --- | | CUDA | `afd_plugin.v1.worker.AFDFFNWorker` | `GPUFFNModelRunner` | `P2pNcclAFDConnector` | | CUDA | `afd_plugin.v1.worker.AFDFFNWorker` | `GPUFFNRoutedExpertsModelRunner` | `DeepEPM2NAFDConnector` (any `AFDRoutedExpertsConnector`) | +| CUDA standalone (`ffn_faas`, no Worker) | `DeepEPM2NFFNBackend` | `GPUFFNRoutedExpertsModelRunner` | `DeepEPM2NAFDConnector` | +| NPU standalone (`ffn_faas`, no Worker) | `CAMFFNBackend` | `AsyncNPUFFNModelRunner` | `CAMAsyncAFDConnector` | | NPU | `afd_plugin.v1.worker.npu.AFDNPUFFNWorker` | `AFDNPUFFNModelRunner` (sync) or `AsyncNPUFFNModelRunner` | `CAMP2pAFDConnector`, `CAMAsyncAFDConnector` | GPU and NPU FFN runners are independent, connector-driven owners. The CUDA @@ -289,6 +322,11 @@ daemon thread: -> repeat ``` +On CUDA, one iteration is `GPUFFNModelRunner.execute_connector_driven_step()`, +shared by `AFDFFNWorker` and the standalone GPU FFN backend. It returns +`False` only for a receive cancelled by a requested stop; a closed control +plane without a stop request, and every other failure, raises. + ### Connector-driven loop The Ascend worker checks `connector.control_plane`. When it is `None`, as for diff --git a/docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md b/docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md index f5cbd55fb..cd5d882dc 100644 --- a/docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md +++ b/docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md @@ -269,6 +269,11 @@ Send API traffic only to the Attention server. For eager, DBO, Attention TP and FP8 examples, see the [DeepEP M2N recipes](../../recipe/gpu/DeepEPM2NAFDConnector/README.md). +For BF16 Qwen3-30B-A3B, the FFN side can instead run as the standalone FFN +service (`torchrun -m afd_plugin.ffn_faas serve --backend gpu-deepep-m2n`), +which loads only the routed experts and constructs no vLLM Worker; see the +[standalone GPU FFN service guide](STANDALONE_FFN_SERVICE.md). + ## Validation - CPU unit tests: diff --git a/docs/gpu/STANDALONE_FFN_SERVICE.md b/docs/gpu/STANDALONE_FFN_SERVICE.md new file mode 100644 index 000000000..d22058c29 --- /dev/null +++ b/docs/gpu/STANDALONE_FFN_SERVICE.md @@ -0,0 +1,151 @@ +# Standalone GPU FFN service + +Run the Qwen3-30B-A3B routed experts in independent F-only processes with +`torchrun -m afd_plugin.ffn_faas serve --backend gpu-deepep-m2n`. The service +loads only the routed experts, reuses vLLM's native loader and the existing +DeepEP M2N FFN runtime, and constructs no vLLM Worker, upstream ModelRunner, +EngineCore, scheduler, KV cache or complete CausalLM. Attention keeps its +existing `vllm serve` launch, model, connector and DBO path. + +## Supported configuration + +| Supported | Rejected at startup | +| --- | --- | +| `Qwen3MoeForCausalLM`, BF16, every layer MoE, no shared experts (`Qwen/Qwen3-30B-A3B`) | FP8 or any `quantization_config`, dense or partial-MoE Qwen3, shared experts, other models | +| `DeepEPM2NAFDConnector` with `payload_format: bf16` | P2P, afdEP-direct, flexible placement, FP8 payload | +| F TP=1, EP over all F ranks, linear expert placement | EPLB, elastic EP, redundant experts, PP/CP, LoRA, speculative decoding | +| Eager, FULL_DECODE_ONLY CUDA graphs, DBO with two lanes (eager and graph) | Other graph modes, more than two micro-batches | +| SM90 or newer (H100, H200, GB200) | Pre-Hopper GPUs | + +The routed experts must divide evenly across the F ranks. + +## Architecture + +```text +CPU service / host runner -> FFNBackend + DeepEPM2NFFNBackend (ffn_faas/platforms/gpu/backend.py) + GPUFFNEnvironment: GPU, F-only NCCL world and vLLM EP group + Qwen3RoutedExpertsModel: model.layers..mlp only, native vLLM loader + GPUFFNRoutedExpertsModelRunner: the Worker path's FFN runner + DeepEPM2NAFDConnector: handshake, control plane, lanes, experts +``` + +Two worlds per deployment: the `torchrun` F-only world (vLLM's world, DP and +EP groups, used only to build FusedMoE and place experts) and the AFD M2N +world `[F0 .. Fn-1, A0 .. Am-1]` the connector creates at `--host:--port`. +Each F step runs `GPUFFNModelRunner.execute_connector_driven_step()`, the same +method the Worker-path `AFDFFNWorker` loop calls, so the data path, buckets, +lanes, graphs and DBO order are identical to a `vllm serve` FFN. See the +[FFN runtime design](../design/module/ffn_runtime.md#runtime-selection). + +Compared with a `vllm serve` FFN, F does not allocate the embedding, final +norm and LM head (about 1.16 GiB per rank for Qwen3-30B-A3B). + +## Launch + +Keep the Attention half of an existing recipe under +`recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/` unchanged and replace only +the FFN half. Example: 2A2F, graph and DBO on one node, Attention on GPUs 0 +and 1, F on GPUs 2 and 3: + +```bash +export VLLM_USE_V2_MODEL_RUNNER=1 +CUDA_VISIBLE_DEVICES=0,1 vllm serve "$MODEL_PATH" \ + --data-parallel-size 2 --tensor-parallel-size 1 --enable-expert-parallel \ + --additional-config '{"afd":{"role":"attention","connector":"DeepEPM2NAFDConnector", + "host":"127.0.0.1","port":6239,"num_attention_ranks":2,"num_ffn_ranks":2, + "compute_gate_on_attention":true,"afd_process_group_timeout_s":600}}' \ + --max-num-seqs 64 --max-num-batched-tokens 4096 --max-model-len 4096 \ + --max-cudagraph-capture-size 64 \ + --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[8,16,32,64]}' \ + --enable-dbo --dbo-decode-token-threshold 2 --dbo-prefill-token-threshold 8 \ + --host 127.0.0.1 --port 18000 & + +CUDA_VISIBLE_DEVICES=2,3 torchrun --nproc-per-node=2 \ + --master-addr=127.0.0.1 --master-port=29500 \ + -m afd_plugin.ffn_faas serve "$MODEL_PATH" --backend gpu-deepep-m2n \ + --host 127.0.0.1 --port 6239 \ + --attention-ranks 2 --ffn-ranks 2 \ + --graph --dbo --afd-timeout 600 \ + --connector-extra-config '{}' \ + --status-port 29700 +``` + +Three ports: the `torchrun` master port (F-only world), the AFD rendezvous +`--port` (equal to Attention's `afd.port`) and the management port base. + +| Option | Meaning | +| --- | --- | +| `--host`, `--port` | AFD rendezvous; equal to Attention's `afd.host`/`afd.port` | +| `--attention-ranks`, `--ffn-ranks` | equal to Attention's AFD config; `WORLD_SIZE` must equal `--ffn-ranks` | +| `--graph` | FULL_DECODE_ONLY CUDA graphs on F (default eager) | +| `--dbo` | two DeepEP lanes; required exactly when Attention runs `--enable-dbo` | +| `--connector-extra-config` | `DeepEPM2NAFDConnector` options (JSON object) | +| `--afd-timeout` | `afd_process_group_timeout_s` for the M2N rendezvous (default 120) | + +F takes no Attention TP, batch size, model length or capture sizes: it adopts +Attention's TP size, dispatch capacity and bucket ladder in the M2N handshake, +and graph keys come from Attention's control payloads. + +### Configuration parity + +The handshake fails on both roles when they disagree: + +| Must agree | Attention | F | +| --- | --- | --- | +| Rank counts, AFD host/port | `afd` config | `--attention-ranks`, `--ffn-ranks`, `--host`, `--port` | +| `payload_format`, `network`, `bucket_granularity`, `cpu_sync_min_tokens` | `connector_extra_config` | `--connector-extra-config` | +| `num_lanes` | `--enable-dbo` | `--dbo` (an explicit `num_lanes` must agree) | +| `capacity_tokens_per_rank` | derived or explicit | adopted, or explicit and equal | +| Hidden size, experts, top-k, BF16 | checkpoint | the same checkpoint files | + +`expert_backend`, `expert_num_sms`, `num_sms`, `num_qps` and the DeepEP +timeouts are F-local tuning knobs. + +### Startup ordering + +Attention joins the M2N rendezvous at the end of its model load; F joins after +its expert load. Independently started services must pair within +`afd_process_group_timeout_s` on both roles, or both fail. Raise it on both +sides to cover the expected start skew (600 s above). + +## Management and readiness + +Each local F rank serves `--status-port + LOCAL_RANK` (`0` chooses a free +port; each rank prints its address at start). + +- `/status`: phase, `loaded_experts`/`total_experts` (this rank's experts over + all layers; vLLM's loader reports no per-expert progress, so the count + jumps from 0 to the total), `parameter_bytes`, `local_expert_first`/ + `local_expert_last`, `m2n_paired`, `attn_tp_size`, + `capacity_tokens_per_rank`, `num_buckets`, `network_hybrid`, + `compute_ep_world_size`, `num_lanes`, `graph_mode`, `dbo` and + `completed_batches` (one per control payload, including warmup, capture + and replay steps). +- `/ready`: paired with Attention and receiving. CUDA graphs are captured + later, driven by Attention's own capture; gate client traffic on + Attention's health endpoint, which comes up only after that. +- `/health`: host liveness. + +## Shutdown + +`POST /stop`, SIGINT and SIGTERM set a stop flag. An idle F cancels its +blocked control receive (the control plane aborts its own NCCL group), its +receive loop exits, and it releases CUDA graphs, DeepEP lanes, the F-only +groups and the experts, in that order. + +Releasing a DeepEP lane is a barrier over every M2N rank, so F finishes its +release only when Attention also shuts down (within the DeepEP GPU timeout, +120 s by default). Recommended order: + +1. drain client traffic; +2. `POST /stop` to every F rank; +3. stop Attention right away; +4. F exits 0 with `resources_released=true` in its status. + +A step that is in flight when the stop arrives completes first; it needs +Attention to finish the same step. If Attention dies or stops without F, +F's pending receive or release fails, F reports `failed` with +`resources_released=false` and exits non-zero. A restarted Attention needs a +new M2N world, so restart F with it. Hot restart, reconnect and graceful +in-flight drain are not supported. diff --git a/tests/e2e/models/qwen3_moe/test_qwen3_moe.py b/tests/e2e/models/qwen3_moe/test_qwen3_moe.py index ec02ccab9..c54493bf2 100644 --- a/tests/e2e/models/qwen3_moe/test_qwen3_moe.py +++ b/tests/e2e/models/qwen3_moe/test_qwen3_moe.py @@ -17,7 +17,7 @@ preserve_environment_variable, run_runner, ) -from tests.e2e.runner import DEEPEP_M2N_SCENARIOS +from tests.e2e.runner import DEEPEP_M2N_FAAS_SCENARIOS, DEEPEP_M2N_SCENARIOS GSM8K_DATASET_ID = "openai/gsm8k" GSM8K_DATASET_CONFIG = "main" @@ -37,7 +37,9 @@ # DeepEP M2N (routing on Attention) needs EP over at least two FFN ranks, so # its scenarios (DEEPEP_M2N_SCENARIOS) run 2A2F. CI selects test ids # explicitly and does not run them; running the whole module (pytest -m e2e -# tests/e2e/models/qwen3_moe) does, and they need DeepEP and 4 GPUs. +# tests/e2e/models/qwen3_moe) does, and they need DeepEP and 4 GPUs. The +# DEEPEP_M2N_FAAS_SCENARIOS run the same 2A2F deployments with the FFN side +# served by the standalone FFN service (torchrun) instead of `vllm serve`. def _required_env(name: str) -> str: @@ -82,7 +84,7 @@ def build_runner_command(scenario: str, gsm8k_output_path: Path) -> list[str]: else: ffn_device_count = ( DEEPEP_M2N_FFN_DEVICE_COUNT - if scenario in DEEPEP_M2N_SCENARIOS + if scenario in (*DEEPEP_M2N_SCENARIOS, *DEEPEP_M2N_FAAS_SCENARIOS) else AFD_FFN_DEVICE_COUNT ) attention_devices = devices[:ATTENTION_DEVICE_COUNT] @@ -139,3 +141,12 @@ def test_qwen3_moe(scenario: str, tmp_path: Path) -> None: def test_qwen3_moe_deepep_m2n(scenario: str, tmp_path: Path) -> None: command = build_runner_command(scenario, tmp_path / scenario) run_runner(command) + + +@pytest.mark.e2e +@pytest.mark.parametrize( + "scenario", DEEPEP_M2N_FAAS_SCENARIOS, ids=DEEPEP_M2N_FAAS_SCENARIOS +) +def test_qwen3_moe_deepep_m2n_ffn_service(scenario: str, tmp_path: Path) -> None: + command = build_runner_command(scenario, tmp_path / scenario) + run_runner(command) diff --git a/tests/e2e/runner.py b/tests/e2e/runner.py index 84c4966ba..584dc2fe5 100644 --- a/tests/e2e/runner.py +++ b/tests/e2e/runner.py @@ -81,10 +81,32 @@ ) # Attention runs DP1 x TP2: each TP rank dispatches its slice of the tokens. # FFN stays DP2 x TP1 (experts are not TP-sharded under DeepEP M2N). +# The same DeepEP M2N deployments with the FFN side served by the standalone +# FFN service (torchrun -m afd_plugin.ffn_faas serve --backend gpu-deepep-m2n) +# instead of `vllm serve`. Attention launches exactly as in the mirrored case. +DEEPEP_M2N_FAAS_SCENARIOS = ( + "afd-deepep-m2n-faas-eager-2a2f", + "afd-deepep-m2n-faas-graph-2a2f", + "afd-deepep-m2n-faas-graph-dbo-2a2f", + "afd-deepep-m2n-faas-graph-tp2-2a2f", +) DEEPEP_M2N_ATTENTION_TP_SCENARIOS = frozenset( - ("afd-deepep-m2n-eager-tp2-2a2f", "afd-deepep-m2n-graph-tp2-2a2f"), + ( + "afd-deepep-m2n-eager-tp2-2a2f", + "afd-deepep-m2n-graph-tp2-2a2f", + "afd-deepep-m2n-faas-graph-tp2-2a2f", + ), ) DEEPEP_M2N_ATTENTION_TP_SIZE = 2 +FFN_FAAS_BACKEND = "gpu-deepep-m2n" +# Independently started services must pair within the M2N rendezvous timeout +# on both roles; the standalone F loads its experts while Attention loads. +FFN_FAAS_AFD_TIMEOUT_S = 600 +FFN_FAAS_SHUTDOWN_TIMEOUT_S = 10 +FFN_FAAS_STOP_TIMEOUT_S = 120 +FFN_FAAS_STATUS_TIMEOUT_S = 5 +# The F-only torchrun world listens next to the AFD rendezvous port. +FFN_FAAS_MASTER_PORT_OFFSET = 1 V2_DBO_COMPARISON_SCENARIOS = frozenset( ("afd-v2-eager-dp2", "afd-v2-eager-dbo-dp2", "afd-v2-graph-dbo-dp2"), ) @@ -141,6 +163,8 @@ def main() -> int: mrv2_execution_evidence: list[tuple[str, str]] = [] mrv2_evidence_dir: tempfile.TemporaryDirectory[str] | None = None mrv2_evidence_start: Path | None = None + ffn_faas_dir: tempfile.TemporaryDirectory[str] | None = None + ffn_faas_status_file: Path | None = None def cleanup() -> None: ffn_process = processes_by_role.get("ffn") @@ -170,12 +194,17 @@ def cleanup() -> None: finally: if mrv2_evidence_dir is not None: mrv2_evidence_dir.cleanup() + if ffn_faas_dir is not None: + ffn_faas_dir.cleanup() launch_order: tuple[tuple[str, str], ...] with cancellable_run(cleanup): if needs_mrv2_execution_evidence(args): mrv2_evidence_dir = tempfile.TemporaryDirectory(prefix="afd-e2e-mrv2-") mrv2_evidence_start = Path(mrv2_evidence_dir.name) / "evaluation-start" + if uses_ffn_faas(args): + ffn_faas_dir = tempfile.TemporaryDirectory(prefix="afd-e2e-ffn-faas-") + ffn_faas_status_file = Path(ffn_faas_dir.name) / "ffn.json" if args.baseline: role_devices = {"baseline": attention_devices} launch_order = (("baseline", "BASELINE"),) @@ -192,11 +221,16 @@ def cleanup() -> None: launch_order = tuple(reversed(launch_order)) for role, label in launch_order: - command = ( - build_baseline_command(args) - if role == "baseline" - else build_vllm_command(args, role=role) - ) + if role == "baseline": + command = build_baseline_command(args) + elif role == "ffn" and ffn_faas_status_file is not None: + command = build_ffn_faas_command( + args, + ffn_faas_status_file, + args.afd_port + FFN_FAAS_MASTER_PORT_OFFSET, + ) + else: + command = build_vllm_command(args, role=role) visible_devices = ",".join(role_devices[role]) process_env = build_env( visible_devices, @@ -221,6 +255,8 @@ def cleanup() -> None: log_threads.append(stream_output(role, process, mrv2_execution_evidence)) ensure_alive(process, f"{label} process exited during startup") + if ffn_faas_status_file is not None: + wait_for_ffn_faas_ready(args, processes) wait_for_openai_api(args, processes) ensure_processes_alive(processes) @@ -262,6 +298,14 @@ def cleanup() -> None: ) ensure_processes_alive(processes) + if ffn_faas_status_file is not None: + # Recommended order: stop the idle FFN service, then Attention. + stop_ffn_faas( + args, + processes_by_role["ffn"], + processes_by_role["attention"], + ffn_faas_status_file, + ) print(f"\nE2E SCENARIO {args.scenario} PASSED") return 0 @@ -317,6 +361,7 @@ def add_scenario_arguments(parser: argparse.ArgumentParser) -> None: *DSV4_GPU_SCENARIOS, *V2_SCENARIOS, *DEEPEP_M2N_SCENARIOS, + *DEEPEP_M2N_FAAS_SCENARIOS, ], required=True, help="Fixed E2E scenario to run.", @@ -443,6 +488,10 @@ def configure_scenario(args: argparse.Namespace) -> None: "afd-deepep-m2n-graph-dbo-2a2f": (False, True, True, 2, 2), "afd-deepep-m2n-eager-tp2-2a2f": (False, False, False, 2, 2), "afd-deepep-m2n-graph-tp2-2a2f": (False, True, False, 2, 2), + "afd-deepep-m2n-faas-eager-2a2f": (False, False, False, 2, 2), + "afd-deepep-m2n-faas-graph-2a2f": (False, True, False, 2, 2), + "afd-deepep-m2n-faas-graph-dbo-2a2f": (False, True, True, 2, 2), + "afd-deepep-m2n-faas-graph-tp2-2a2f": (False, True, False, 2, 2), } for sync_scenario, sync_profile in DSV4_SYNC_SHAPES.items(): # The A5 script's native DBO stays off for these scenarios: a split batch @@ -489,7 +538,7 @@ def configure_scenario(args: argparse.Namespace) -> None: args.ffn_tp_size = 1 if not is_dsv4 and args.gsm8k_output_path is None: raise ValueError("--gsm8k-output-path is required for GSM8K scenarios") - if args.scenario in DEEPEP_M2N_SCENARIOS: + if args.scenario in DEEPEP_M2N_SCENARIOS or uses_ffn_faas(args): args.afd_connector = DEEPEP_M2N_CONNECTOR args.compute_gate_on_attention = True if args.scenario == DSV4_ASYNC_CAM_SCENARIO: @@ -559,7 +608,9 @@ def validate_topology( raise ValueError("ModelRunnerV2 E2E scenarios require GPU") if args.scenario == NPU_EAGER_DBO_SCENARIO and args.device_backend != "npu": raise ValueError("Lite eager DBO scenario requires NPU") - if args.scenario in DEEPEP_M2N_SCENARIOS and args.device_backend != "gpu": + if ( + args.scenario in DEEPEP_M2N_SCENARIOS or uses_ffn_faas(args) + ) and args.device_backend != "gpu": raise ValueError("DeepEP M2N E2E scenarios require GPU") if args.scenario in DSV4_SCENARIOS and args.device_backend != "npu": raise ValueError("DSV4 scenarios require NPU") @@ -672,6 +723,10 @@ def build_vllm_command( afd_config["afd"]["async"] = True if args.compute_gate_on_attention: afd_config["afd"]["compute_gate_on_attention"] = True + if uses_ffn_faas(args): + # Attention pairs with an independently started FFN service; keep + # its rendezvous timeout equal to the FFN service's --afd-timeout. + afd_config["afd"]["afd_process_group_timeout_s"] = FFN_FAAS_AFD_TIMEOUT_S connector_extra_config = parse_afd_connector_extra_config( args.afd_connector_extra_config, ) @@ -858,6 +913,150 @@ def uses_async_connector(args: argparse.Namespace) -> bool: return args.afd_connector == ASYNC_AFD_CONNECTOR +def uses_ffn_faas(args: argparse.Namespace) -> bool: + """Whether the FFN side runs as the standalone FFN service.""" + return args.scenario in DEEPEP_M2N_FAAS_SCENARIOS + + +def ffn_faas_status_port(args: argparse.Namespace) -> int: + """Management port of FFN service rank 0; rank r listens on this + r.""" + return ffn_api_port(args) + + +def build_ffn_faas_command( + args: argparse.Namespace, status_file: Path, master_port: int +) -> list[str]: + """Build the ``torchrun`` command of the standalone GPU FFN service. + + One process per FFN rank on this host (DP = FFN ranks, TP = 1). The AFD + rendezvous, rank counts, connector options, graph mode and DBO mirror the + FFN half ``build_vllm_command`` would produce for the same scenario, so + the unchanged Attention command pairs with it. The torchrun master port + is the F-only world; the AFD port is the M2N world. + """ + command = [ + sys.executable, + "-m", + "torch.distributed.run", + f"--nproc-per-node={args.num_ffn_ranks}", + "--nnodes=1", + f"--master-addr={args.afd_host}", + f"--master-port={master_port}", + "-m", + "afd_plugin.ffn_faas", + "serve", + args.model, + "--backend", + FFN_FAAS_BACKEND, + "--host", + args.afd_host, + "--port", + str(args.afd_port), + "--attention-ranks", + str(args.num_attention_ranks), + "--ffn-ranks", + str(args.num_ffn_ranks), + "--connector-extra-config", + json.dumps( + parse_afd_connector_extra_config(args.afd_connector_extra_config), + separators=(",", ":"), + ), + "--afd-timeout", + str(FFN_FAAS_AFD_TIMEOUT_S), + "--shutdown-timeout", + str(FFN_FAAS_SHUTDOWN_TIMEOUT_S), + "--status-host", + args.api_host, + "--status-port", + str(ffn_faas_status_port(args)), + "--status-file", + str(status_file), + ] + if args.cuda_graph_full_decode_only: + command.append("--graph") + if args.enable_dbo: + command.append("--dbo") + return command + + +def ffn_faas_status(args: argparse.Namespace, rank: int, path: str) -> int: + """HTTP status code of one FFN service rank's management endpoint.""" + url = f"http://{args.api_host}:{ffn_faas_status_port(args) + rank}{path}" + try: + with urllib.request.urlopen(url, timeout=FFN_FAAS_STATUS_TIMEOUT_S) as reply: + return int(reply.status) + except urllib.error.HTTPError as error: + return int(error.code) + + +def wait_for_ffn_faas_ready( + args: argparse.Namespace, processes: list[subprocess.Popen[str]] +) -> None: + """Wait until every FFN service rank reports ``/ready`` (paired with A). + + ``/ready`` means paired and receiving; CUDA graphs are captured later, + driven by Attention, whose API comes up only after that. + """ + deadline = time.monotonic() + args.startup_timeout + pending = set(range(args.num_ffn_ranks)) + last_error: BaseException | None = None + while pending and time.monotonic() < deadline: + ensure_processes_alive(processes) + for rank in sorted(pending): + try: + if ffn_faas_status(args, rank, "/ready") == 200: + pending.discard(rank) + except (OSError, urllib.error.URLError) as error: + last_error = error + time.sleep(PROCESS_POLL_INTERVAL_S) + if pending: + raise TimeoutError( + f"FFN service ranks {sorted(pending)} were not ready; " + f"last error={last_error!r}" + ) + print("\nFFN service ranks are ready") + + +def stop_ffn_faas( + args: argparse.Namespace, + process: subprocess.Popen[str], + attention_process: subprocess.Popen[str], + status_file: Path, +) -> None: + """Stop the idle pair, FFN service first, and require released resources. + + Posts ``/stop`` to every FFN rank while Attention is idle, which cancels + each rank's idle control receive; then shuts Attention down right away. + Releasing a DeepEP lane (``ElasticBuffer.destroy``) is a barrier over + every M2N rank, so the FFN service's release completes only once + Attention's shutdown destroys its lanes too. The launcher must then exit + 0 and every rank's status file must report ``stopped`` with + ``resources_released=true``. + """ + for rank in range(args.num_ffn_ranks): + request = urllib.request.Request( + f"http://{args.api_host}:{ffn_faas_status_port(args) + rank}/stop", + method="POST", + ) + with urllib.request.urlopen(request, timeout=FFN_FAAS_STATUS_TIMEOUT_S): + pass + terminate_processes( + [attention_process], + termination_timeout_s=process_termination_timeout(args), + ) + returncode = process.wait(timeout=FFN_FAAS_STOP_TIMEOUT_S) + if returncode != 0: + raise RuntimeError(f"FFN service exited with returncode={returncode}") + for rank in range(args.num_ffn_ranks): + rank_file = status_file.with_name(f"{status_file.stem}-rank-{rank}.json") + status = json.loads(rank_file.read_text()) + if status["phase"] != "stopped" or status["resources_released"] is not True: + raise RuntimeError( + f"FFN service rank {rank} did not stop cleanly: {status}" + ) + print("\nFFN service stopped cleanly with resources released") + + def uses_npu_async_process_cleanup(args: argparse.Namespace) -> bool: """Return whether E2E teardown must find and kill every FFN process.""" return args.device_backend == "npu" and args.scenario == DSV4_ASYNC_CAM_SCENARIO @@ -964,7 +1163,13 @@ def execution_evidence_roles(args: argparse.Namespace) -> tuple[str, ...]: return ("baseline",) # Async CAM stages belong to Attention; FFN consumes independent native # work items. Sync graph/DBO must prove the matching path on both roles. - return ("attention",) if uses_async_connector(args) else ("attention", "ffn") + # The standalone FFN service has no vLLM Worker to host the evidence + # extension; Attention's evidence covers the paired graph/DBO path. + return ( + ("attention",) + if uses_async_connector(args) or uses_ffn_faas(args) + else ("attention", "ffn") + ) def needs_mrv2_execution_evidence(args: argparse.Namespace) -> bool: diff --git a/tests/gpu/test_ffn_faas_gpu.py b/tests/gpu/test_ffn_faas_gpu.py new file mode 100644 index 000000000..4dd7dafb2 --- /dev/null +++ b/tests/gpu/test_ffn_faas_gpu.py @@ -0,0 +1,546 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""GPU smoke for the standalone FFN service (``--backend gpu-deepep-m2n``). + +Four GPUs: two synthetic Attention ranks built with the DeepEP M2N harness +(fake vLLM config, no model) and two FFN service ranks launched by +``torchrun -m afd_plugin.ffn_faas serve`` from a synthetic Qwen3 MoE +checkpoint. The checkpoint's experts are the harness's deterministic expert +weights (Qwen3-30B-A3B sizes, two layers), so each Attention rank checks the +combined output of every layer against the harness's dense MoE reference: +that covers the FFN-only model, vLLM's loader and EP weight filter, expert +placement, the M2N handshake, eager steps and a captured CUDA graph. + +Teardown: a ``/stop`` on an idle FFN service cancels its control receive; +the service releases its DeepEP lanes together with Attention (a lane's +``destroy`` is a barrier over every M2N rank) and exits 0. When Attention +dies first, the FFN service must fail and exit non-zero. +""" + +from __future__ import annotations + +import contextlib +import json +import os +import signal +import subprocess +import sys +import time +import traceback +import urllib.request +from pathlib import Path +from typing import Any + +import pytest + +torch = pytest.importorskip("torch") +pytest.importorskip("vllm") +pytest.importorskip("safetensors") + +import torch.distributed as dist # noqa: E402 +import torch.multiprocessing as mp # noqa: E402 + +from tests.gpu.deepep_m2n_harness import ( # noqa: E402 + Checker, + available_gpus, + free_ports, + make_connector, + make_expert_weights, + make_inputs, + run_attention_step, + stage_seed, +) +from tests.unit.fakes.vllm_config import ( # noqa: E402 + M2N_HIDDEN_SIZE, + M2N_INTERMEDIATE_SIZE, + M2N_NUM_EXPERTS, + M2N_TOP_K, + M2NTopology, +) + +pytestmark = pytest.mark.gpu + +REPO_ROOT = Path(__file__).resolve().parents[2] +TOPOLOGY = M2NTopology(ffn_size=2, attn_size=2) +NUM_LAYERS = 2 +CAPACITY = 256 +BUCKET = 64 +EAGER_STEPS = 2 +GRAPH_REPLAYS = 3 +HOST = "127.0.0.1" +EXTRA_CONFIG = {"capacity_tokens_per_rank": CAPACITY} +# Short DeepEP and rendezvous timeouts for the Attention-first failure case. +PEER_EXIT_EXTRA_CONFIG = {**EXTRA_CONFIG, "cpu_timeout_s": 10, "gpu_timeout_s": 10} +PEER_EXIT_AFD_TIMEOUT_S = 30 +STARTUP_TIMEOUT_S = 900 +STOP_TIMEOUT_S = 300 +POLL_S = 0.5 +ATTENTION_EXIT_CODE = 23 +MANAGEMENT_PREFIX = '{"rank"' + + +def _skip_without_gpus() -> None: + if available_gpus() < TOPOLOGY.world_size: + pytest.skip(f"needs {TOPOLOGY.world_size} GPUs") + if torch.cuda.get_device_capability()[0] < 9: + pytest.skip("DeepEP M2N needs SM90+") + + +def write_qwen3_m2n_checkpoint(root: Path) -> Path: + """A BF16 Qwen3 MoE checkpoint whose experts are the harness weights. + + Every layer holds the same experts, as the harness's dense reference + assumes. HF names: ``experts..{gate,up,down}_proj``; the harness's + ``w13`` is ``[gate; up]``. The MoE gate is zero: F builds it, never runs it. + """ + from safetensors.torch import save_file + + root.mkdir(parents=True, exist_ok=True) + w13, w2 = make_expert_weights(range(M2N_NUM_EXPERTS)) + w13, w2 = w13.cpu(), w2.cpu() + tensors: dict[str, Any] = {} + for layer in range(NUM_LAYERS): + prefix = f"model.layers.{layer}.mlp" + tensors[f"{prefix}.gate.weight"] = torch.zeros( + M2N_NUM_EXPERTS, M2N_HIDDEN_SIZE, dtype=torch.bfloat16 + ) + for expert in range(M2N_NUM_EXPERTS): + gate, up = w13[expert].chunk(2, dim=0) + name = f"{prefix}.experts.{expert}" + tensors[f"{name}.gate_proj.weight"] = gate.clone() + tensors[f"{name}.up_proj.weight"] = up.clone() + tensors[f"{name}.down_proj.weight"] = w2[expert].clone() + save_file(tensors, str(root / "model.safetensors"), metadata={"format": "pt"}) + config = { + "architectures": ["Qwen3MoeForCausalLM"], + "model_type": "qwen3_moe", + "torch_dtype": "bfloat16", + "hidden_act": "silu", + "hidden_size": M2N_HIDDEN_SIZE, + "intermediate_size": 4 * M2N_HIDDEN_SIZE, + "moe_intermediate_size": M2N_INTERMEDIATE_SIZE, + "num_hidden_layers": NUM_LAYERS, + "num_attention_heads": 32, + "num_key_value_heads": 4, + "head_dim": 128, + "num_experts": M2N_NUM_EXPERTS, + "num_experts_per_tok": M2N_TOP_K, + "norm_topk_prob": True, + "decoder_sparse_step": 1, + "mlp_only_layers": [], + "vocab_size": 1024, + "max_position_embeddings": 4096, + "rms_norm_eps": 1e-6, + "rope_theta": 1000000.0, + "tie_word_embeddings": False, + } + (root / "config.json").write_text(json.dumps(config)) + return root + + +@pytest.fixture(scope="module") +def checkpoint(tmp_path_factory): + """One synthetic checkpoint (about 2.3 GB) shared by this module's tests.""" + _skip_without_gpus() + return write_qwen3_m2n_checkpoint(tmp_path_factory.mktemp("qwen3-m2n")) + + +def _send_control(connector: Any, tokens: tuple[int, ...], **flags: bool) -> None: + """Send one step's control payload and plan it, as Attention's runner does.""" + from afd_plugin.connectors import AFDControlPayload, AFDDPMetadata + + payload = AFDControlPayload( + dp_metadata_list={ + 0: AFDDPMetadata(num_tokens_across_dp_cpu=torch.tensor(tokens)) + }, + is_graph_capturing=flags.get("is_graph_capturing", False), + is_warmup=flags.get("is_warmup", False), + is_graph_replaying=flags.get("is_graph_replaying", False), + ) + connector.control_plane.send_dp_metadata_list(payload) + connector.control_plane.update_state_from_dp_metadata(payload) + + +def attention_scenario(connector: Any, args: dict[str, Any]) -> dict[str, Any]: + """Drive the FFN service like vLLM's Attention runner and check combines. + + Eager steps first; with ``graph``, then Attention's capture sequence: a + warmup payload run eagerly, a capture payload captured into a CUDA graph + on both sides, and replays with fresh inputs. Every step runs all + ``NUM_LAYERS`` MoE layers, as the FFN service runs every layer per + payload. Returns the checker's worst errors and the payload count. + """ + checker = Checker(connector) + tokens = (BUCKET,) * TOPOLOGY.attn_size + rank = connector.role_rank + payloads = 0 + + def inputs(step: int, layer: int) -> tuple[Any, Any, Any]: + return make_inputs(stage_seed(step * NUM_LAYERS + layer, 0), rank, BUCKET) + + def eager_step(step: int, **flags: bool) -> None: + _send_control(connector, tokens, **flags) + for layer in range(NUM_LAYERS): + x, ids, w = inputs(step, layer) + out = torch.empty_like(x) + run_attention_step(connector, [(x, ids, w)], [out], layer) + checker.check_combined(out, x, ids, w) + + step = 0 + for _ in range(args["eager_steps"]): + eager_step(step) + step += 1 + payloads += 1 + if args["graph"]: + eager_step(step, is_warmup=True) + step += 1 + payloads += 1 + statics = [inputs(step, layer) for layer in range(NUM_LAYERS)] + outs = [torch.empty_like(x) for x, _, _ in statics] + _send_control(connector, tokens, is_graph_capturing=True) + payloads += 1 + graph = torch.cuda.CUDAGraph() + torch.cuda.synchronize() + with torch.cuda.graph(graph): + for layer, (static, out) in enumerate(zip(statics, outs, strict=True)): + run_attention_step(connector, [static], [out], layer) + torch.cuda.synchronize() + for _ in range(GRAPH_REPLAYS): + step += 1 + for layer, static in enumerate(statics): + for static_tensor, fresh in zip( + static, inputs(step, layer), strict=True + ): + static_tensor.copy_(fresh) + _send_control(connector, tokens, is_graph_replaying=True) + graph.replay() + torch.cuda.synchronize() + payloads += 1 + for (x, ids, w), out in zip(statics, outs, strict=True): + checker.check_combined(out, x, ids, w) + return {**checker.finish(), "control_payloads": payloads} + + +def _attention_rank_main( + role_rank: int, + args: dict[str, Any], + extra_config: dict[str, Any], + ports: tuple[int, int], + out_dir: str, +) -> None: + """One synthetic Attention rank: connect, run the scenario, then release. + + The default process group spans only the Attention ranks (the FFN ranks + are another launcher's world). After the scenario the rank writes a + ``scenario`` marker, then either exits without cleanup (``exit_early``, + simulating an Attention failure) or waits for the test's ``release`` + marker and closes its connector together with the FFN service. + """ + out = Path(out_dir) + result: dict[str, Any] = {"ok": True, "error": "", "metrics": {}} + connector = None + try: + torch.cuda.set_device(role_rank) + dist.init_process_group( + "gloo", + init_method=f"tcp://{HOST}:{ports[0]}", + world_size=TOPOLOGY.attn_size, + rank=role_rank, + ) + connector = make_connector( + TOPOLOGY, + TOPOLOGY.ffn_size + role_rank, + role_rank, + extra_config, + ports[1], + HOST, + ) + connector.init_afd_connector() + result["metrics"] = attention_scenario(connector, args) + (out / f"scenario{role_rank}").write_text(json.dumps(result["metrics"])) + if args["exit_early"]: + os._exit(ATTENTION_EXIT_CODE) + deadline = time.monotonic() + STOP_TIMEOUT_S + while not (out / "release").exists() and time.monotonic() < deadline: + time.sleep(POLL_S) + except Exception: + result["ok"] = False + result["error"] = traceback.format_exc() + (out / f"scenario{role_rank}").write_text("{}") + finally: + if connector is not None: + try: + connector.close() + except Exception: + result["ok"] = False + result["error"] += traceback.format_exc() + (out / f"rank{role_rank}.json").write_text(json.dumps(result, default=str)) + if dist.is_initialized(): + dist.destroy_process_group() + + +def _start_attention( + args: dict[str, Any], + extra_config: dict[str, Any], + ports: tuple[int, int], + out: Path, +) -> list[Any]: + """Spawn the Attention ranks on the first two visible GPUs.""" + ctx = mp.get_context("spawn") + procs = [ + ctx.Process( + target=_attention_rank_main, + args=(rank, args, extra_config, ports, str(out)), + ) + for rank in range(TOPOLOGY.attn_size) + ] + for proc in procs: + proc.start() + return procs + + +def _ffn_devices() -> str: + """The visible GPUs after the Attention ranks', as a CUDA_VISIBLE_DEVICES value.""" + visible = os.environ.get("CUDA_VISIBLE_DEVICES") + ids = ( + [item.strip() for item in visible.split(",") if item.strip()] + if visible + else [str(index) for index in range(available_gpus())] + ) + return ",".join(ids[TOPOLOGY.attn_size : TOPOLOGY.world_size]) + + +def _start_ffn_service( + checkpoint: Path, + *, + afd_port: int, + master_port: int, + status_file: Path, + log_path: Path, + graph: bool, + extra_config: dict[str, Any], + afd_timeout_s: int, +) -> subprocess.Popen[bytes]: + """Launch both FFN service ranks with torchrun, as an operator would.""" + env = { + **os.environ, + "CUDA_VISIBLE_DEVICES": _ffn_devices(), + "VLLM_USE_V2_MODEL_RUNNER": "1", + "VLLM_PLUGINS": "afd", + "PYTHONPATH": os.pathsep.join( + filter(None, (str(REPO_ROOT), os.environ.get("PYTHONPATH"))) + ), + } + command = [ + sys.executable, + "-m", + "torch.distributed.run", + f"--nproc-per-node={TOPOLOGY.ffn_size}", + "--nnodes=1", + f"--master-addr={HOST}", + f"--master-port={master_port}", + "-m", + "afd_plugin.ffn_faas", + "serve", + str(checkpoint), + "--backend", + "gpu-deepep-m2n", + "--host", + HOST, + "--port", + str(afd_port), + "--attention-ranks", + str(TOPOLOGY.attn_size), + "--ffn-ranks", + str(TOPOLOGY.ffn_size), + "--connector-extra-config", + json.dumps(extra_config), + "--afd-timeout", + str(afd_timeout_s), + "--status-host", + HOST, + # Port 0: each rank binds a free port and prints it. + "--status-port", + "0", + "--status-file", + str(status_file), + *(["--graph"] if graph else []), + ] + with log_path.open("wb") as log: + return subprocess.Popen( + command, + cwd=REPO_ROOT, + env=env, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + + +def _management_ports(log_path: Path) -> dict[int, int]: + """Management port per FFN rank, from the line each rank prints at start.""" + ports = {} + for line in log_path.read_text(errors="replace").splitlines(): + if line.startswith(MANAGEMENT_PREFIX): + announced = json.loads(line) + ports[int(announced["rank"])] = int(announced["management"][1]) + return ports + + +def _wait_for(predicate: Any, timeout_s: float, what: str, *procs: Any) -> None: + """Poll ``predicate`` until true; fail early if a watched process died.""" + deadline = time.monotonic() + timeout_s + while not predicate(): + for proc in procs: + code = proc.poll() if hasattr(proc, "poll") else proc.exitcode + if code is not None: + raise AssertionError(f"process exited with {code} while waiting {what}") + if time.monotonic() >= deadline: + raise AssertionError(f"timed out waiting {what}") + time.sleep(POLL_S) + + +def _status(port: int, path: str = "/status", method: str = "GET") -> dict[str, Any]: + request = urllib.request.Request(f"http://{HOST}:{port}{path}", method=method) + with urllib.request.urlopen(request, timeout=5) as reply: + return json.loads(reply.read()) + + +def _status_files(status_file: Path) -> dict[int, dict[str, Any]]: + files = {} + for rank in range(TOPOLOGY.ffn_size): + path = status_file.with_name(f"{status_file.stem}-rank-{rank}.json") + if path.exists(): + files[rank] = json.loads(path.read_text()) + return files + + +def _kill(service: subprocess.Popen[bytes], attention: list[Any]) -> None: + """Best-effort teardown so a failed test does not leak GPU processes.""" + if service.poll() is None: + with contextlib.suppress(ProcessLookupError): + os.killpg(service.pid, signal.SIGKILL) + service.wait(timeout=30) + for proc in attention: + if proc.is_alive(): + proc.kill() + proc.join(timeout=30) + + +@pytest.mark.parametrize("graph", [False, True], ids=["eager", "graph"]) +def test_ffn_service_serves_and_stops_cleanly( # long-test: two launchers + checkpoint, tmp_path, graph +): + attention_args = {"eager_steps": EAGER_STEPS, "graph": graph, "exit_early": False} + group_port, afd_port, master_port = free_ports(3) + status_file = tmp_path / "ffn.json" + log_path = tmp_path / "ffn.log" + service = _start_ffn_service( + checkpoint, + afd_port=afd_port, + master_port=master_port, + status_file=status_file, + log_path=log_path, + graph=graph, + extra_config=EXTRA_CONFIG, + afd_timeout_s=STARTUP_TIMEOUT_S, + ) + attention = _start_attention( + attention_args, EXTRA_CONFIG, (group_port, afd_port), tmp_path + ) + try: + _wait_for( + lambda: all( + (tmp_path / f"scenario{rank}").exists() + for rank in range(TOPOLOGY.attn_size) + ), + STARTUP_TIMEOUT_S, + "for the Attention scenario", + service, + *attention, + ) + metrics = [ + json.loads((tmp_path / f"scenario{rank}").read_text()) + for rank in range(TOPOLOGY.attn_size) + ] + assert all("control_payloads" in rank_metrics for rank_metrics in metrics), ( + (tmp_path / "rank0.json").read_text() + if (tmp_path / "rank0.json").exists() + else metrics + ) + payloads = metrics[0]["control_payloads"] + ports = _management_ports(log_path) + assert sorted(ports) == list(range(TOPOLOGY.ffn_size)) + # A step is counted after its device work completes. + _wait_for( + lambda: all( + _status(port)["completed_batches"] == payloads + for port in ports.values() + ), + STOP_TIMEOUT_S, + "for the FFN service to count every step", + service, + ) + experts_per_rank = M2N_NUM_EXPERTS // TOPOLOGY.ffn_size + for rank, port in ports.items(): + status = _status(port) + assert status["phase"] == "ready" + assert status["m2n_paired"] is True + assert status["compute_ep_world_size"] == TOPOLOGY.ffn_size + assert status["capacity_tokens_per_rank"] == CAPACITY + assert status["attn_tp_size"] == 1 + assert (status["local_expert_first"], status["local_expert_last"]) == ( + rank * experts_per_rank, + (rank + 1) * experts_per_rank - 1, + ) + assert status["loaded_experts"] == status["total_experts"] + assert status["total_experts"] == NUM_LAYERS * experts_per_rank + + for port in ports.values(): + _status(port, "/stop", "POST") + (tmp_path / "release").write_text("") + assert service.wait(timeout=STOP_TIMEOUT_S) == 0, log_path.read_text()[-4000:] + for proc in attention: + proc.join(timeout=STOP_TIMEOUT_S) + results = [ + json.loads((tmp_path / f"rank{rank}.json").read_text()) + for rank in range(TOPOLOGY.attn_size) + ] + assert all(result["ok"] for result in results), results + files = _status_files(status_file) + assert sorted(files) == list(range(TOPOLOGY.ffn_size)) + for status in files.values(): + assert status["phase"] == "stopped" + assert status["resources_released"] is True + finally: + _kill(service, attention) + + +def test_ffn_service_fails_when_attention_dies_first(checkpoint, tmp_path): + attention_args = {"eager_steps": 1, "graph": False, "exit_early": True} + group_port, afd_port, master_port = free_ports(3) + status_file = tmp_path / "ffn.json" + log_path = tmp_path / "ffn.log" + service = _start_ffn_service( + checkpoint, + afd_port=afd_port, + master_port=master_port, + status_file=status_file, + log_path=log_path, + graph=False, + extra_config=PEER_EXIT_EXTRA_CONFIG, + afd_timeout_s=PEER_EXIT_AFD_TIMEOUT_S, + ) + attention = _start_attention( + attention_args, PEER_EXIT_EXTRA_CONFIG, (group_port, afd_port), tmp_path + ) + try: + for proc in attention: + proc.join(timeout=STARTUP_TIMEOUT_S) + assert proc.exitcode == ATTENTION_EXIT_CODE + code = service.wait(timeout=STOP_TIMEOUT_S) + assert code != 0, log_path.read_text()[-4000:] + for status in _status_files(status_file).values(): + assert status["resources_released"] is False + finally: + _kill(service, attention) diff --git a/tests/unit/fakes/ffn_faas.py b/tests/unit/fakes/ffn_faas.py index 861f6e8e3..f71715076 100644 --- a/tests/unit/fakes/ffn_faas.py +++ b/tests/unit/fakes/ffn_faas.py @@ -72,18 +72,59 @@ def write_v4_checkpoint(root: Path, *, layers: int = 2, experts: int = 4) -> Pat return root +# The shape of Qwen/Qwen3-30B-A3B's config.json (no shared-expert field). +QWEN3_30B_A3B_CONFIG = { + "architectures": ["Qwen3MoeForCausalLM"], + "model_type": "qwen3_moe", + "torch_dtype": "bfloat16", + "hidden_size": 2048, + "num_hidden_layers": 48, + "num_experts": 128, + "num_experts_per_tok": 8, + "moe_intermediate_size": 768, + "decoder_sparse_step": 1, + "mlp_only_layers": [], + "norm_topk_prob": True, +} + + +def write_qwen3_checkpoint(root: Path, **overrides) -> Path: + """A Qwen3 MoE ``config.json`` only; ``None`` values remove a key.""" + root.mkdir(parents=True, exist_ok=True) + config = {**QWEN3_30B_A3B_CONFIG, **overrides} + config = {key: value for key, value in config.items() if value is not None} + (root / "config.json").write_text(json.dumps(config)) + return root + + class HostBackend: """Exercise host ownership rules, without simulating device completion.""" def __init__( - self, *, stop_during_prepare: bool = False, fail: str = "", block: bool = False + self, + *, + stop_during_prepare: bool = False, + fail: str = "", + block: bool = False, + cancellable: bool = False, + end_receive: bool = False, ) -> None: + """Configure one host-contract behavior. + + ``block`` keeps ``execute_step`` waiting like an idle receive; + ``cancellable`` lets ``cancel_receive`` end that wait the way the GPU + M2N control plane does; ``end_receive`` makes the first step report a + cancelled receive without any host stop. + """ self.stop_during_prepare = stop_during_prepare self.fail = fail self.block = block + self.cancellable = cancellable + self.end_receive = end_receive self.entered = threading.Event() self.release = threading.Event() self.finished = threading.Event() + self.cancelled = False self.released = False self.service: FFNService | None = None @@ -99,15 +140,27 @@ def start_receiver(self) -> None: if self.fail == "activate": raise RuntimeError("receiver activation failed") - def execute_step(self) -> None: + def execute_step(self) -> bool: + """Return ``False`` for a cancelled receive, else request a stop.""" assert self.service is not None self.entered.set() + if self.end_receive: + return False if self.block: self.release.wait() + if self.cancelled: + return False if self.fail == "execute": self.finished.set() raise RuntimeError("execute failed") self.service.request_stop() + return True + + def cancel_receive(self) -> None: + """End a blocked receive only when this fake models a cancellable one.""" + if self.cancellable: + self.cancelled = True + self.release.set() def receiver_exited(self) -> None: self.finished.set() diff --git a/tests/unit/ffn_faas/test_gpu_adapter.py b/tests/unit/ffn_faas/test_gpu_adapter.py new file mode 100644 index 000000000..0503916aa --- /dev/null +++ b/tests/unit/ffn_faas/test_gpu_adapter.py @@ -0,0 +1,140 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""GPU DeepEP M2N launch validation happens on the CPU, before any device.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from argparse import ArgumentParser +from pathlib import Path + +import pytest + +from afd_plugin.ffn_faas.platforms import get_platform_adapter +from tests.unit.fakes.ffn_faas import write_qwen3_checkpoint + +REPO_ROOT = Path(__file__).resolve().parents[3] +NATIVE_MODULES = ("torch", "vllm", "afdep", "deep_gemm") + + +@pytest.fixture +def launch_env(monkeypatch): + """A two-rank torchrun environment for F rank 1.""" + for key, value in { + "RANK": "1", + "LOCAL_RANK": "1", + "WORLD_SIZE": "2", + "VLLM_USE_V2_MODEL_RUNNER": "1", + }.items(): + monkeypatch.setenv(key, value) + + +@pytest.fixture +def configure(tmp_path): + """Parse a ``serve`` command line for the GPU adapter and configure it.""" + checkpoint = write_qwen3_checkpoint(tmp_path / "model") + + def run(*options, model=checkpoint): + adapter = get_platform_adapter("gpu-deepep-m2n") + parser = ArgumentParser() + parser.add_argument("model", type=Path) + parser.add_argument("--shutdown-timeout", type=float, default=5.0) + adapter.add_arguments(parser) + args = parser.parse_args( + [str(model), "--attention-ranks", "2", "--ffn-ranks", "2", *options] + ) + return adapter.configure(args) + + return run + + +def test_backend_selection_imports_no_native_stack(): + script = ( + "import json, sys; " + "from afd_plugin.ffn_faas.platforms import get_platform_adapter; " + "get_platform_adapter('gpu-deepep-m2n'); " + f"print(json.dumps([m for m in {NATIVE_MODULES!r} if m in sys.modules]))" + ) + result = subprocess.run( + [sys.executable, "-c", script], + cwd=REPO_ROOT, + capture_output=True, + text=True, + timeout=60, + check=True, + ) + assert json.loads(result.stdout) == [] + + +def test_valid_launch_describes_the_rank_and_its_lanes(launch_env, configure, tmp_path): + launch = configure("--graph", "--dbo", "--port", "6239") + + assert (launch.rank, launch.local_rank) == (1, 1) + assert launch.model_path == str((tmp_path / "model").resolve()) + assert launch.backend_name == "DeepEPM2NAFDConnector" + assert launch.initial_status == { + "payload_format": "bf16", + "num_lanes": 2, + "graph_mode": "FULL_DECODE_ONLY", + "dbo": True, + "m2n_paired": False, + } + + +@pytest.mark.parametrize( + "env, options, message", + [ + ({"WORLD_SIZE": "4"}, (), "WORLD_SIZE must equal --ffn-ranks"), + ({"VLLM_USE_V2_MODEL_RUNNER": "0"}, (), "VLLM_USE_V2_MODEL_RUNNER=1"), + ({}, ("--connector-extra-config", "{bad"), "not valid JSON"), + ({}, ("--connector-extra-config", '{"num_lanes": 2}'), "num_lanes"), + ], + ids=["world-size", "model-runner-v2", "malformed-json", "lanes-vs-dbo"], +) +def test_invalid_launch_is_rejected( + launch_env, configure, monkeypatch, env, options, message +): + for key, value in env.items(): + monkeypatch.setenv(key, value) + with pytest.raises(ValueError, match=message): + configure(*options) + + +def test_unsupported_checkpoint_is_rejected(launch_env, configure, tmp_path): + fp8 = write_qwen3_checkpoint( + tmp_path / "fp8", quantization_config={"quant_method": "fp8"} + ) + with pytest.raises(ValueError, match="quantized"): + configure(model=fp8) + + +def test_cli_reports_a_bad_launch_as_a_usage_error(tmp_path): + checkpoint = write_qwen3_checkpoint(tmp_path / "model") + env = {**os.environ, "WORLD_SIZE": "1", "VLLM_USE_V2_MODEL_RUNNER": "1"} + result = subprocess.run( + [ + sys.executable, + "-m", + "afd_plugin.ffn_faas", + "serve", + str(checkpoint), + "--backend", + "gpu-deepep-m2n", + "--attention-ranks", + "2", + "--ffn-ranks", + "2", + "--status-port", + "0", + ], + cwd=REPO_ROOT, + env=env, + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 2 + assert "WORLD_SIZE must equal --ffn-ranks" in result.stderr diff --git a/tests/unit/ffn_faas/test_gpu_qwen3_model.py b/tests/unit/ffn_faas/test_gpu_qwen3_model.py new file mode 100644 index 000000000..d234742b0 --- /dev/null +++ b/tests/unit/ffn_faas/test_gpu_qwen3_model.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""The GPU FFN service accepts only the routed-experts Qwen3 MoE shape.""" + +from __future__ import annotations + +import pytest + +from afd_plugin.ffn_faas.platforms.gpu.qwen3_model import Qwen3CheckpointDescription +from tests.unit.fakes.ffn_faas import write_qwen3_checkpoint + +# Qwen3-30B-A3B at 2F: 48 layers x 64 experts x 3 x 2048 x 768 x 2 B. +QWEN3_30B_2F_EXPERT_BYTES = 48 * 64 * 3 * 2048 * 768 * 2 + + +def test_stock_qwen3_30b_checkpoint_is_described(tmp_path): + checkpoint = Qwen3CheckpointDescription.read(write_qwen3_checkpoint(tmp_path)) + + assert checkpoint.root == tmp_path.resolve() + assert checkpoint.layer_indices == tuple(range(48)) + assert (checkpoint.num_experts, checkpoint.top_k) == (128, 8) + assert checkpoint.local_experts(2) == 64 + assert checkpoint.estimated_local_expert_bytes(2) == QWEN3_30B_2F_EXPERT_BYTES + + +@pytest.mark.parametrize( + "overrides", + [ + {"shared_expert_intermediate_size": 0}, + {"shared_expert_intermediate_size": None}, + {"torch_dtype": None, "dtype": "bfloat16"}, + ], + ids=["shared-zero", "shared-absent", "dtype-key"], +) +def test_equivalent_config_spellings_are_accepted(tmp_path, overrides): + checkpoint = Qwen3CheckpointDescription.read( + write_qwen3_checkpoint(tmp_path, **overrides) + ) + assert checkpoint.num_layers == 48 + + +@pytest.mark.parametrize( + "overrides, message", + [ + ({"architectures": ["DeepseekV2ForCausalLM"]}, "Qwen3MoeForCausalLM"), + ({"torch_dtype": "float16"}, "BF16"), + ( + {"quantization_config": {"quant_method": "fp8"}}, + "quantized", + ), + ({"mlp_only_layers": [0]}, "every layer to be MoE"), + ({"decoder_sparse_step": 2}, "decoder_sparse_step=1"), + ({"shared_expert_intermediate_size": 512}, "shared experts"), + ({"num_experts": 0}, "num_experts must be a positive integer"), + ], + ids=[ + "architecture", + "dtype", + "fp8", + "mlp-only-layers", + "sparse-step", + "shared-experts", + "no-experts", + ], +) +def test_unsupported_qwen3_shapes_fail_on_cpu(tmp_path, overrides, message): + with pytest.raises(ValueError, match=message): + Qwen3CheckpointDescription.read(write_qwen3_checkpoint(tmp_path, **overrides)) + + +def test_experts_must_divide_across_ffn_ranks(tmp_path): + checkpoint = Qwen3CheckpointDescription.read(write_qwen3_checkpoint(tmp_path)) + with pytest.raises(ValueError, match="do not divide across 3 FFN ranks"): + checkpoint.local_experts(3) diff --git a/tests/unit/ffn_faas/test_gpu_settings.py b/tests/unit/ffn_faas/test_gpu_settings.py new file mode 100644 index 000000000..06e31ccfb --- /dev/null +++ b/tests/unit/ffn_faas/test_gpu_settings.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""GPU standalone FFN settings produce the Worker-path DeepEP M2N FFN config.""" + +from __future__ import annotations + +from pathlib import Path +from types import MappingProxyType + +import pytest + +from afd_plugin.config import connector_extra_config_from_source, parse_afd_config +from afd_plugin.ffn_faas.platforms.gpu.settings import ( + GPUFFNConfig, + parse_connector_extra_config, +) + + +def _settings(**overrides): + return GPUFFNConfig( + **{ + "model": Path("/model"), + "attention_ranks": 2, + "ffn_ranks": 2, + "rank": 1, + **overrides, + } + ) + + +def test_afd_config_matches_a_worker_path_m2n_ffn(): + settings = _settings( + port=6239, + afd_timeout_s=600, + connector_extra_config=MappingProxyType({"expert_num_sms": 96}), + ) + config = settings.afd_additional_config() + + afd = parse_afd_config(config, expected_role="ffn") + assert afd.connector == "DeepEPM2NAFDConnector" + assert afd.compute_gate_on_attention + assert (afd.num_attention_ranks, afd.num_ffn_ranks) == (2, 2) + assert (afd.port, afd.afd_process_group_timeout_s) == (6239, 600) + assert connector_extra_config_from_source(config) == {"expert_num_sms": 96} + + +@pytest.mark.parametrize( + "graph, dbo, eager, compilation_config", + [ + (False, False, True, {}), + (True, False, False, {"cudagraph_mode": "FULL_DECODE_ONLY"}), + (True, True, False, {"cudagraph_mode": "FULL_DECODE_ONLY"}), + ], + ids=["eager", "graph", "graph-dbo"], +) +def test_engine_args_describe_an_f_only_ep_rank(graph, dbo, eager, compilation_config): + kwargs = _settings(graph=graph, dbo=dbo).engine_kwargs() + + assert kwargs["distributed_executor_backend"] == "external_launcher" + assert (kwargs["data_parallel_size"], kwargs["data_parallel_rank"]) == (2, 1) + assert kwargs["tensor_parallel_size"] == 1 + assert kwargs["enable_expert_parallel"] and kwargs["enable_ep_weight_filter"] + assert kwargs["enforce_eager"] is eager + assert kwargs["compilation_config"] == compilation_config + assert kwargs["enable_dbo"] is dbo + # vLLM's idle default all2all backend is what the M2N handshake requires. + assert "all2all_backend" not in kwargs + + +@pytest.mark.parametrize( + "overrides, message", + [ + ({"rank": 2}, "invalid F rank"), + ({"attention_ranks": 0}, "rank counts must be positive"), + ({"port": 0}, "valid port"), + ({"afd_timeout_s": 0}, "--afd-timeout must be positive"), + ({"shutdown_timeout": float("inf")}, "finite and positive"), + ( + {"connector_extra_config": {"payload_format": "fp8_block128_fp32"}}, + "payload_format='bf16'", + ), + ({"connector_extra_config": {"num_lanes": 2}}, "disagrees with no --dbo"), + ( + {"dbo": True, "connector_extra_config": {"num_lanes": 1}}, + "disagrees with --dbo", + ), + ], + ids=[ + "rank", + "attention-ranks", + "port", + "afd-timeout", + "shutdown-timeout", + "fp8-payload", + "lanes-without-dbo", + "lanes-with-dbo", + ], +) +def test_invalid_launch_settings_are_rejected(overrides, message): + with pytest.raises(ValueError, match=message): + _settings(**overrides) + + +def test_matching_explicit_lane_count_is_accepted(): + assert _settings(dbo=True, connector_extra_config={"num_lanes": 2}).num_lanes == 2 + + +@pytest.mark.parametrize( + "text, message", + [("{", "not valid JSON"), ("[1]", "must be a JSON object")], + ids=["malformed", "not-object"], +) +def test_connector_extra_config_must_be_a_json_object(text, message): + with pytest.raises(ValueError, match=message): + parse_connector_extra_config(text) diff --git a/tests/unit/ffn_faas/test_runner.py b/tests/unit/ffn_faas/test_runner.py index a3e475f8d..7a6c99f93 100644 --- a/tests/unit/ffn_faas/test_runner.py +++ b/tests/unit/ffn_faas/test_runner.py @@ -74,6 +74,41 @@ def run(): assert backend.finished.wait(timeout=5) +def test_stop_cancels_idle_receive_and_releases(make_runner): + backend = HostBackend(block=True, cancellable=True) + runner, service = make_runner(backend) + done = threading.Event() + + def run(): + try: + runner.run() + finally: + done.set() + + thread = threading.Thread(target=run) + thread.start() + try: + assert backend.entered.wait(timeout=5) + service.request_stop() + assert done.wait(timeout=5) + assert service.snapshot()["phase"] == "stopped" + assert service.snapshot()["resources_released"] is True + assert backend.released + finally: + backend.release.set() + thread.join() + + +def test_backend_ended_receive_exits_loop_and_releases(make_runner): + backend = HostBackend(end_receive=True) + runner, service = make_runner(backend) + runner.run() + assert service.snapshot()["phase"] == "stopped" + assert service.snapshot()["resources_released"] is True + assert service.snapshot()["completed_batches"] == 0 + assert backend.released + + def test_stop_before_backend_construction_needs_no_platform(tmp_path): def unavailable(): raise RuntimeError("no device backend is available") diff --git a/tests/unit/v1/worker/test_ffn_model_runner.py b/tests/unit/v1/worker/test_ffn_model_runner.py index 93a299f8a..b7a78d4b8 100644 --- a/tests/unit/v1/worker/test_ffn_model_runner.py +++ b/tests/unit/v1/worker/test_ffn_model_runner.py @@ -50,11 +50,14 @@ def make(control_plane): worker._ffn_shutdown_event = threading.Event() worker._ffn_loop_error = None worker.device = SimpleNamespace(type="cpu") - worker.model_runner = SimpleNamespace( - connector=SimpleNamespace(control_plane=control_plane, is_initialized=True), - use_cuda_graph=False, - model="owned weights", + runner = object.__new__(GPUFFNModelRunner) + runner.connector = SimpleNamespace( + control_plane=control_plane, is_initialized=True ) + runner.use_cuda_graph = False + runner.model = "owned weights" + runner.device = SimpleNamespace(type="cpu") + worker.model_runner = runner return worker return make @@ -603,6 +606,49 @@ def compute_experts_output( ] +def test_connector_driven_step_executes_the_received_payload(): + runner = _runner_with_connector_and_model(_FakeModel()) + runner.device = SimpleNamespace(type="cpu") + metadata = _metadata() + runner.connector.attn_outputs.append(_payload("hidden", metadata)) + runner.connector.recv_dp_metadata_list = lambda: AFDControlPayload( + dp_metadata_list={0: _FakeDPMetadata([1])} + ) + + assert runner.execute_connector_driven_step(cancel_requested=lambda: False) + assert runner.connector.ffn_outputs == [("ffn(hidden, layer=0)", metadata)] + + +@pytest.mark.parametrize( + "error, cancel_requested, returns_false", + [ + (AFDControlPlaneClosedError("closed"), True, True), + (AFDControlPlaneClosedError("closed"), False, False), + (RuntimeError("NCCL failure"), True, False), + ], +) +def test_connector_driven_step_reports_only_requested_cancellation( + error, cancel_requested, returns_false +): + def receive(): + raise error + + runner = _runner_with_connector_and_model(_FakeModel()) + runner.device = SimpleNamespace(type="cpu") + runner.connector.recv_dp_metadata_list = receive + + if returns_false: + assert not runner.execute_connector_driven_step( + cancel_requested=lambda: cancel_requested + ) + else: + with pytest.raises(type(error)) as caught: + runner.execute_connector_driven_step( + cancel_requested=lambda: cancel_requested + ) + assert caught.value is error + + def test_ffn_runner_requires_dp_metadata_list(): runner = object.__new__(GPUFFNModelRunner) runner.prof = None @@ -751,9 +797,8 @@ def test_ffn_worker_loop_rejects_connector_without_control_plane(): worker._ffn_shutdown_event = event worker.device = SimpleNamespace(type="cpu") - worker.model_runner = SimpleNamespace( - connector=_ConnectorDrivenFakeConnector(), - ) + worker.model_runner = object.__new__(GPUFFNModelRunner) + worker.model_runner.connector = _ConnectorDrivenFakeConnector() with pytest.raises(NotImplementedError, match="control-plane-driven"): worker._run_ffn_server_loop() diff --git a/tests/unit/v1/worker/test_ffn_routed_experts_model_runner.py b/tests/unit/v1/worker/test_ffn_routed_experts_model_runner.py index a28a99820..d29c3ac83 100644 --- a/tests/unit/v1/worker/test_ffn_routed_experts_model_runner.py +++ b/tests/unit/v1/worker/test_ffn_routed_experts_model_runner.py @@ -216,6 +216,73 @@ def test_local_experts_reject_backends_that_reorder_weights(fp8, backend): local_experts_from_vllm(_routed_experts(fp8=fp8, backend=backend)) +class _AttachableModel: + """An externally loaded model exposing two MoE layers' local experts.""" + + def __init__(self, expert_ids=(4, 5)): + self.experts = { + layer: _routed_experts(fp8=False, expert_ids=expert_ids) + for layer in MOE_LAYERS + } + + def get_experts_layer_indices(self): + return MOE_LAYERS + + def get_routed_experts(self, layer_idx): + return self.experts[layer_idx] + + +def _unattached_runner(): + runner = object.__new__(GPUFFNRoutedExpertsModelRunner) + runner.connector = FakeRoutedConnector() + runner.model = None + runner.model_memory_usage = 0 + runner.moe_layer_indices = () + runner.local_experts = {} + return runner + + +def test_attached_model_experts_reach_connector_startup(): + """A model attached from outside feeds the connector's startup collective + the same layers and expert views ``load_model`` would. + + Why: the standalone FFN service loads its own FFN-only model; the + connector's post-load placement check must see exactly those experts. + """ + runner = _unattached_runner() + model = _AttachableModel() + + runner.attach_model(model, model_memory_usage=1024) + runner.initialize_afd_connector() + + sent = runner.connector.init_kwargs + assert sent["moe_layer_indices"] == MOE_LAYERS + assert { + layer: experts.expert_ids for layer, experts in sent["local_experts"].items() + } == {layer: (4, 5) for layer in MOE_LAYERS} + assert ( + sent["local_experts"][MOE_LAYERS[0]].weights.w13.data_ptr() + == model.experts[MOE_LAYERS[0]].w13_weight.data_ptr() + ) + + +@pytest.mark.parametrize("when", ["second_attach", "after_connector_init"]) +def test_attach_model_rejects_late_or_repeated_attach(when): + runner = _unattached_runner() + if when == "second_attach": + runner.attach_model(_AttachableModel(), model_memory_usage=0) + else: + runner.initialize_afd_connector() + + with pytest.raises(RuntimeError, match="attached once, before connector startup"): + runner.attach_model(_AttachableModel(), model_memory_usage=0) + + +def test_attach_model_rejects_negative_memory_usage(): + with pytest.raises(ValueError, match="non-negative"): + _unattached_runner().attach_model(_AttachableModel(), model_memory_usage=-1) + + def _kernel_config(moe_backend: str) -> SimpleNamespace: return SimpleNamespace(kernel_config=SimpleNamespace(moe_backend=moe_backend)) From 626e9091b82dcfdc9f216450e0dfd89fc3279206 Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:44:08 +0200 Subject: [PATCH 04/42] [GPU] FFN service: host the M2N rendezvous under torchrun torchrun sets TORCHELASTIC_USE_AGENT_STORE=True in its workers, so every tcp:// rendezvous joins the agent's store as a client. The F-only world needs that, but the connector's M2N rendezvous must be hosted by F rank 0; with the flag set nobody listened and both roles timed out. Drop the flag once vLLM's F-only groups exist. Also fix the new runner unit test (AFDControlPayload needs explicit flags) and the GPU smoke harness (interleaved per-rank announcements, and kill torchrun's workers, which are outside the launcher's process group). Co-Authored-By: Claude Opus 5.5 Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- .../ffn_faas/platforms/gpu/environment.py | 16 ++++++-- tests/gpu/test_ffn_faas_gpu.py | 40 +++++++++++++++---- tests/unit/v1/worker/test_ffn_model_runner.py | 4 +- 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/afd_plugin/ffn_faas/platforms/gpu/environment.py b/afd_plugin/ffn_faas/platforms/gpu/environment.py index f22b5ef43..65302d55b 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/environment.py +++ b/afd_plugin/ffn_faas/platforms/gpu/environment.py @@ -27,6 +27,8 @@ AFDEP_TRANSPORT_MODULE = "afdep.deepep" DEEPGEMM_MODULE = "deep_gemm" FUSED_EXPERTS_BACKEND = "fused_experts" +# Set by torchrun: rendezvous in its workers join the agent's TCP store. +TORCHELASTIC_AGENT_STORE_ENV = "TORCHELASTIC_USE_AGENT_STORE" # vLLM's GPU Worker gives DBO two workspace slots. DBO_WORKSPACE_UBATCHES = 2 @@ -159,9 +161,10 @@ def initialize_compute(self, config: VllmConfig) -> None: size and rank, so EP spans exactly the F ranks; this is re-checked because the connector's handshake assumes contiguous expert ranges over that world. Ownership is recorded before the first collective so - a partial initialization is still torn down by ``close``. Finally - initializes vLLM's workspace manager with the Worker's micro-batch - count (two under DBO). + a partial initialization is still torn down by ``close``. Then stops + later rendezvous from joining the torchrun agent's store, so the M2N + world can host its own, and initializes vLLM's workspace manager with + the Worker's micro-batch count (two under DBO). """ if self._device is None or self._groups_owned: raise RuntimeError("GPU FFN compute initialization requires its device") @@ -187,6 +190,13 @@ def initialize_compute(self, config: VllmConfig) -> None: or self._ep_group.rank_in_group != settings.rank ): raise ValueError("vLLM expert group differs from the F-only world") + # torchrun asks every ``tcp://``/``env://`` rendezvous in its workers + # to join the agent's store. That is right for the F-only world above, + # but the connector's M2N rendezvous at ``--host:--port`` must be + # hosted by M2N world rank 0 (F rank 0), as in a ``vllm serve`` FFN; + # with the flag set, every rank would wait as a client and nobody + # would listen. vLLM's groups are already created, so drop it. + os.environ.pop(TORCHELASTIC_AGENT_STORE_ENV, None) # Same slot count as Worker._init_workspace_and_model_runner. init_workspace_manager( self._device, diff --git a/tests/gpu/test_ffn_faas_gpu.py b/tests/gpu/test_ffn_faas_gpu.py index 4dd7dafb2..7e1795e75 100644 --- a/tests/gpu/test_ffn_faas_gpu.py +++ b/tests/gpu/test_ffn_faas_gpu.py @@ -378,12 +378,18 @@ def _start_ffn_service( def _management_ports(log_path: Path) -> dict[int, int]: - """Management port per FFN rank, from the line each rank prints at start.""" + """Management port per FFN rank, from the object each rank prints at start. + + Both ranks share one log, so their announcements can land on one line. + """ + text = log_path.read_text(errors="replace") + decoder = json.JSONDecoder() ports = {} - for line in log_path.read_text(errors="replace").splitlines(): - if line.startswith(MANAGEMENT_PREFIX): - announced = json.loads(line) - ports[int(announced["rank"])] = int(announced["management"][1]) + start = text.find(MANAGEMENT_PREFIX) + while start >= 0: + announced, end = decoder.raw_decode(text, start) + ports[int(announced["rank"])] = int(announced["management"][1]) + start = text.find(MANAGEMENT_PREFIX, end) return ports @@ -415,11 +421,31 @@ def _status_files(status_file: Path) -> dict[int, dict[str, Any]]: return files +def _descendants(pid: int) -> list[int]: + """Every live descendant of ``pid``, read from ``/proc`` (Linux only). + + torchrun's workers are not in the launcher's process group, so killing + the group alone leaves them holding GPUs. + """ + children: dict[int, list[int]] = {} + for stat in Path("/proc").glob("[0-9]*/stat"): + with contextlib.suppress(OSError, ValueError, IndexError): + fields = stat.read_text().rsplit(")", 1)[1].split() + children.setdefault(int(fields[1]), []).append(int(stat.parent.name)) + found, pending = [], [pid] + while pending: + for child in children.get(pending.pop(), []): + found.append(child) + pending.append(child) + return found + + def _kill(service: subprocess.Popen[bytes], attention: list[Any]) -> None: """Best-effort teardown so a failed test does not leak GPU processes.""" if service.poll() is None: - with contextlib.suppress(ProcessLookupError): - os.killpg(service.pid, signal.SIGKILL) + for pid in [*_descendants(service.pid), service.pid]: + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGKILL) service.wait(timeout=30) for proc in attention: if proc.is_alive(): diff --git a/tests/unit/v1/worker/test_ffn_model_runner.py b/tests/unit/v1/worker/test_ffn_model_runner.py index b7a78d4b8..f44a93c81 100644 --- a/tests/unit/v1/worker/test_ffn_model_runner.py +++ b/tests/unit/v1/worker/test_ffn_model_runner.py @@ -612,7 +612,9 @@ def test_connector_driven_step_executes_the_received_payload(): metadata = _metadata() runner.connector.attn_outputs.append(_payload("hidden", metadata)) runner.connector.recv_dp_metadata_list = lambda: AFDControlPayload( - dp_metadata_list={0: _FakeDPMetadata([1])} + dp_metadata_list={0: _FakeDPMetadata([1])}, + is_graph_capturing=False, + is_warmup=False, ) assert runner.execute_connector_driven_step(cancel_requested=lambda: False) From 8c114ae4204f54ebb3c05fa56fd688a457ca9549 Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:53:40 +0200 Subject: [PATCH 05/42] [GPU] FFN service guide: Attention-first failure behavior Measured by the GPU smoke test: the NCCL watchdog aborts the F ranks after afd_process_group_timeout_s, so no status file is written. Co-Authored-By: Claude Opus 5.5 Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- docs/gpu/STANDALONE_FFN_SERVICE.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/gpu/STANDALONE_FFN_SERVICE.md b/docs/gpu/STANDALONE_FFN_SERVICE.md index d22058c29..1ea134901 100644 --- a/docs/gpu/STANDALONE_FFN_SERVICE.md +++ b/docs/gpu/STANDALONE_FFN_SERVICE.md @@ -145,7 +145,10 @@ release only when Attention also shuts down (within the DeepEP GPU timeout, A step that is in flight when the stop arrives completes first; it needs Attention to finish the same step. If Attention dies or stops without F, -F's pending receive or release fails, F reports `failed` with -`resources_released=false` and exits non-zero. A restarted Attention needs a +F's pending control receive cannot complete: once +`afd_process_group_timeout_s` passes, PyTorch's NCCL watchdog aborts the F +ranks (SIGABRT, so no status file is written) and `torchrun` exits non-zero. +A failure F detects itself (for example a DeepEP timeout in a step) reports +`failed` with `resources_released=false` and exits non-zero. A restarted Attention needs a new M2N world, so restart F with it. Hot restart, reconnect and graceful in-flight drain are not supported. From 022d8a7e7a521994f0bbda5d9c278c71e8ceb1dd Mon Sep 17 00:00:00 2001 From: Li Ruixin <616517220@qq.com> Date: Thu, 5 Nov 2026 17:55:50 -0800 Subject: [PATCH 06/42] [NPU]:Add Pull-based MegaMoE execution for Async CAM (#517) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Purpose Add opt-in Pull-based MegaMoE execution for A3 Async CAM, including per-channel W4A8 and W8A8 expert weights. Attention publishes routed inputs; FFN pulls and executes the expert computation, and Attention combines the results. The existing Push/Layered path remains the default. ## Issue - Related issue(s): Not specified. - Closing keyword, only if fully resolved: Not specified. ## Scope - In scope: the Pull protocol and native MegaMoE executor, W4A8/W8A8 execution plans, eager and FULL graph integration, lifecycle negotiation, tests and documentation. - Out of scope: per-group MegaMoE weights, MegaMoE with FFN two-stream, a general-purpose Pull transport, and other hardware generations. ## Implementation Notes - Enable `connector_extra_config.ffn_mega_moe=true` consistently on Attention and FFN ranks; use dynamic quantization and per-channel weights. With Mega disabled, Async CAM retains Push/Layered execution. - The native executor selects the W4A8 or W8A8 plan from the Layered weight metadata. Mega follows the Layered behavior: apply the configured SwiGLU limit for W4A8 and do not apply it for W8A8. - The connector owns the Pull lifecycle through the generic FFN execution-session contract. Attention publishes `pull_quiet`; FFN drains and publishes `pull_stopped` before resources are released. The current runtime helpers are grouped under `afd_plugin/connectors/npu/async_cam_runtime/`. - Receive capacity is `int(262144 * BATCH_SIZE_FACTOR)`. The Pull layout requires `(max_num_batched_tokens / attn_ranks_per_dp) * topk <= 65535`; the matched performance runs use 16384 maximum batched tokens. - Native operators and the PyTorch binding must be built for the selected Ascend/CANN stack. Both CANN 9.1 and 9.2 header layouts are supported. This branch uses plugin-owned integration. ## Test Plan - Exercise CPU-side connector, execution-session, native registration, weight-selection and graph lifecycle contracts. - Build the native operators for A3 and check the Pull layout and representative W4A8/W8A8 native execution. - Run real-model GSM8K inference and matched Mega ON/OFF end-to-end performance tests. Keep request correctness, actual graph/DBO coverage, service cleanup and timing as separate results. - Validate Mega ON lifecycle and numerical behavior on the target A3 stack. Record baseline behavior as measurement context. ## Test Result **Revision and environment.** The E2E timing runs used production sources at `f47c992234b1a11c7d78f1a250d2f10911f1ce5d` with working-tree E2E shutdown-budget changes; the current Pull worker timeout change was tested separately and was not included in those timings. The CANN 9.2 native build was reused, with source and binary hashes and per-rank loaded libraries checked. Runtime: vLLM 0.32, torch-npu 2.10.0.post7, 16 A3 devices per precision. **CPU checks.** On the current working tree, the branch-related unit-test selection completed with 183 passed and 44 skipped; skips require PyTorch, vLLM, NumPy or the compiled Ascend 910C extension. The build and CANN activation-generation tests contributed 32 of the passed tests. The package and Async CAM startup module import successfully without PyTorch or vLLM. Ruff passed for the branch's changed Python files. GPU E2E tests were not run. **GSM8K inference.** With Mega ON, both models used Attention DP2×TP4 / FFN EP8×TP1, 8-shot GSM8K, 300 samples, concurrency 12 and maximum generation 512: W4A8 DSpark 287/300 strict matches (95.67%); W8A8 MTP 292/300 (97.33%). Inference completed, but the runs reported shutdown failure due to `EngineDeadError`; these are not clean complete-run passes. The W8A8 performance topology below is different and has not received a separate GSM8K run. **End-to-end performance.** Each precision used one ON→OFF pair on a fresh service, with the same frozen inputs, model and settings within the pair. W4A8 DSpark used Attention DP6×TP2 / FFN EP4×TP1; W8A8 MTP used Attention DP4×TP2 / FFN EP8×TP1. W8A8 EP4 expert tensors alone require about 66.17 GiB per FFN rank, exceeding 64 GiB before other allocations. Attention used eager+DBO; FFN used FULL graph; max model length 65537, max batched tokens 16384, memory utilization 0.92 and receive capacity 262144. All 2256 formal requests and 127 warmups per mode completed successfully; actual DP counters, FFN graph replay and Attention decode DBO execution were checked. Six fixed-length prefill cases and the imbalanced case used 96 requests at an offered 6 RPS; the mixed case used 1536 frozen requests at 6 RPS; long and short decode used 24 requests in two 12-concurrent waves. The long/short input/output lengths were 32768/512 and 128/256. W4 and W8 use different checkpoints and topologies, so their numbers do not isolate quantization effects. TTFT includes queueing; offered-rate throughput is not a capacity measurement. Each pair has only one ordering and does not establish stable gains. **Important:** All four timed performance runs logged FFN receive shutdown timeouts, so they are timing results, not clean E2E passes. ON/OFF outputs were not fully identical. Fixed-length cases compare returned text; other cases compare returned token IDs. A matching output count is not an accuracy result.
W4A8 measured performance TTFT: mean / P50 / P99 in ms; throughput: input tokens/s. Δ is ON relative to OFF. | Scenario | OFF TTFT | ON TTFT | Δ mean | OFF / ON throughput | Matching outputs | |---|---:|---:|---:|---:|---:| | tokens-32768 | 19,898.29 / 19,862.05 / 36,696.68 | 18,204.61 / 18,157.37 / 33,493.12 | -8.51% | 59,937.35 / 63,733.30 | 96/96 | | tokens-16384 | 6,044.64 / 6,040.31 / 10,458.61 | 4,949.81 / 4,959.88 / 8,427.10 | -18.11% | 59,832.09 / 64,590.81 | 96/96 | | tokens-8192 | 777.76 / 779.34 / 796.59 | 720.68 / 720.33 / 728.29 | -7.34% | 47,431.37 / 47,496.66 | 95/96 | | tokens-4096 | 408.36 / 406.32 / 448.01 | 399.38 / 396.99 / 435.00 | -2.20% | 24,210.59 / 24,225.80 | 93/96 | | tokens-2048 | 267.10 / 263.30 / 307.51 | 267.63 / 267.24 / 282.17 | +0.20% | 12,213.55 / 12,200.95 | 96/96 | | tokens-128 | 116.77 / 116.02 / 124.87 | 118.54 / 117.77 / 125.29 | +1.52% | 770.24 / 770.01 | 96/96 | | imbalanced-prefill | 1,812.49 / 261.49 / 8,268.94 | 1,737.14 / 273.46 / 7,835.66 | -4.16% | 23,441.98 / 23,875.72 | 96/96 | | mixed1536 | 5,791.89 / 5,338.73 / 14,933.90 | 3,563.77 / 3,020.74 / 10,996.32 | -38.47% | 56,543.49 / 57,358.96 | 1455/1536 | Decode TPOT: mean / P50 / P99 in ms/token, measured in each wave’s common active window after excluding one second at each edge; throughput: output tokens/s. | Scenario | OFF TPOT | ON TPOT | Δ mean | OFF / ON throughput | Matching outputs | |---|---:|---:|---:|---:|---:| | long-context-decode | 263.65 / 259.21 / 290.51 | 268.34 / 266.29 / 279.74 | +1.78% | 45.62 / 44.73 | 10/24 | | short-context-decode | 264.75 / 260.95 / 292.34 | 270.54 / 268.23 / 280.32 | +2.19% | 45.45 / 44.43 | 21/24 |
W8A8 measured performance TTFT: mean / P50 / P99 in ms; throughput: input tokens/s. Δ is ON relative to OFF. | Scenario | OFF TTFT | ON TTFT | Δ mean | OFF / ON throughput | Matching outputs | |---|---:|---:|---:|---:|---:| | tokens-32768 | 30,589.92 / 30,560.19 / 58,149.17 | 29,556.24 / 29,581.49 / 55,993.40 | -3.38% | 42,506.98 / 43,709.37 | 96/96 | | tokens-16384 | 10,757.98 / 10,748.58 / 20,025.23 | 10,194.38 / 10,203.84 / 18,919.10 | -5.24% | 43,859.78 / 45,267.12 | 96/96 | | tokens-8192 | 1,656.57 / 1,580.03 / 2,786.48 | 1,267.61 / 1,177.10 / 2,254.43 | -23.48% | 43,642.46 / 45,098.99 | 96/96 | | tokens-4096 | 412.64 / 410.34 / 445.70 | 398.14 / 396.35 / 424.27 | -3.51% | 24,207.19 / 24,226.81 | 95/96 | | tokens-2048 | 268.11 / 264.99 / 287.66 | 271.28 / 269.27 / 297.56 | +1.18% | 12,215.54 / 12,203.56 | 92/96 | | tokens-128 | 117.49 / 116.44 / 126.13 | 120.10 / 120.32 / 127.11 | +2.22% | 770.39 / 770.01 | 96/96 | | imbalanced-prefill | 5,709.20 / 265.18 / 30,758.92 | 5,450.46 / 274.47 / 29,287.33 | -4.53% | 11,806.21 / 12,199.14 | 96/96 | | mixed1536 | 48,893.54 / 48,856.68 / 111,240.40 | 43,534.13 / 40,755.28 / 86,161.97 | -10.96% | 41,786.15 / 44,340.77 | 1452/1536 | Decode TPOT: mean / P50 / P99 in ms/token, measured in each wave’s common active window after excluding one second at each edge; throughput: output tokens/s. | Scenario | OFF TPOT | ON TPOT | Δ mean | OFF / ON throughput | Matching outputs | |---|---:|---:|---:|---:|---:| | long-context-decode | 261.90 / 261.01 / 270.54 | 267.10 / 268.36 / 273.72 | +1.99% | 45.84 / 44.96 | 14/24 | | short-context-decode | 262.94 / 262.84 / 271.58 | 267.92 / 268.83 / 274.27 | +1.89% | 45.70 / 44.87 | 18/24 |
**Current shutdown validation.** With the working-tree Pull-session timeout change, 32K prefill and short decode completed and all worker groups exited normally for W4A8 ON (10 groups) and W8A8 ON (12 groups). W8A8 logged `EngineDeadError` after an explicit shutdown signal, worker exit and MPClient shutdown. The worker test file passed 75 cases, included in the CPU result above. ## Docs Impact - Files updated: `csrc/npu/README.md`, `docs/design/module/execution_platforms.md`, `docs/design/module/ffn_runtime.md`, and `docs/npu/ASYNC_FFN_RUNTIME.md`. - If none, reason: Not applicable. ---
Essential PR Checklist - [x] Purpose is clear and linked to public context when possible. - [x] Scope is bounded. - [x] Compatibility with vLLM v0.32.0 and, for NPU changes, the pinned vLLM-Ascend source is considered, as applicable. - [x] No changes are made to the vLLM source checkout. - [x] Plugin-owned classes or explicit dotted class paths are preferred over monkey patches. - [x] Any compat shim or monkey patch is isolated, idempotent, version-guarded, documented, and tested. - [x] Imports remain CPU-safe; CUDA-heavy work is delayed or GPU-gated. - [x] Validation evidence is included, including skipped GPU tests when applicable. - [x] Documentation impact is stated.
--------- Signed-off-by: lirx-pd <616517220@qq.com> --- afd_plugin/compat/npu/feature_validation.py | 7 +- afd_plugin/connectors/__init__.py | 64 +- afd_plugin/connectors/base.py | 87 ++- afd_plugin/connectors/npu/__init__.py | 24 +- afd_plugin/connectors/npu/async_cam.py | 171 ++++- .../npu/async_cam_runtime/__init__.py | 3 + .../npu/async_cam_runtime/async_cam_pull.py | 77 ++ .../async_cam_runtime}/async_cam_startup.py | 26 +- .../model_executor/npu/async_cam_layered.py | 14 + .../model_executor/npu/async_cam_mega.py | 92 +++ .../v1/worker/npu/async_ffn_model_runner.py | 88 ++- .../worker/npu/attention_model_runner_v2.py | 3 + afd_plugin/v1/worker/npu/ffn_worker.py | 25 +- afd_plugin/validation.py | 6 +- csrc/npu/README.md | 36 + .../op_api/aclnn_afd_mega_moe_comm.cpp | 18 + .../op_api/aclnn_afd_mega_moe_comm.h | 13 + .../op_host/afd_mega_moe_comm.cpp | 73 ++ .../op_kernel/afd_mega_moe_comm.cpp | 306 ++++++++ .../op_kernel/afd_mega_moe_comm_tiling.h | 15 + .../op_api/aclnn_afd_mega_moe_pull_native.cpp | 22 + .../op_api/aclnn_afd_mega_moe_pull_native.h | 21 + .../op_host/afd_mega_moe_pull_native.cpp | 106 +++ .../afd_mega_moe_patch_activation.py | 49 ++ .../op_kernel/afd_mega_moe_plan_w4a8.h | 55 ++ .../op_kernel/afd_mega_moe_plan_w8a8.h | 25 + .../op_kernel/afd_mega_moe_pull_native.cpp | 257 +++++++ .../afd_mega_moe_pull_native_tiling.h | 21 + .../op_kernel/afd_mega_moe_pull_pipeline.h | 699 ++++++++++++++++++ .../cmake_files/op_kernel/CMakeLists.txt | 37 + ...ped_matmul_swiglu_quant_v2_layered_utils.h | 17 +- .../npu/ascend_kernels/operator_registry.json | 10 +- .../op_kernel/afd_native_mega_workspace.h | 46 ++ .../utils/op_kernel/afd_pull_layout.h | 135 ++++ .../utils/op_kernel/afd_pull_protocol.h | 352 +++++++++ .../utils/op_kernel/afd_swiglu_limit.h | 32 + csrc/npu/pybind/CMakeLists.txt | 4 +- .../pybind/torch_binding_mega_moe_comm.cpp | 249 +++++++ .../pybind/torch_binding_mega_moe_pull.cpp | 115 +++ docs/design/module/execution_platforms.md | 15 + docs/design/module/ffn_runtime.md | 13 +- docs/npu/ASYNC_FFN_RUNTIME.md | 51 +- tests/e2e/README.md | 11 +- .../e2e/models/deepseek_v4_flash/ffn_graph.py | 2 + .../deepseek_v4_flash/ffn_graph_evidence.py | 30 +- tests/e2e/runner.py | 16 +- tests/npu/async_cam_runtime.py | 389 ++++++++-- tests/npu/check_pull_layout.cpp | 85 +++ tests/npu/test_async_cam_layered_w8a8.py | 9 +- tests/npu/test_cam_async_ops.py | 133 ++++ .../compat/npu/test_feature_validation.py | 28 +- .../compat/patches/npu/test_mrv2_ubatching.py | 32 + .../connectors/npu/test_async_cam_pull.py | 155 ++++ .../{ => npu}/test_async_cam_startup.py | 166 ++++- .../npu}/test_async_cam_startup_wiring.py | 270 ++++++- .../connectors/test_async_cam_connector.py | 218 ++++++ tests/unit/fakes/async_cam_startup.py | 44 ++ tests/unit/fakes/async_ffn.py | 5 +- tests/unit/fakes/connector.py | 41 + .../model_executor/test_async_cam_layered.py | 132 ++++ tests/unit/package/test_ascend_build_files.py | 2 + .../package/test_mega_activation_patch.py | 43 ++ tests/unit/v1/worker/test_async_ffn_graph.py | 5 + .../unit/v1/worker/test_async_ffn_runtime.py | 51 ++ .../v1/worker/test_async_ffn_two_stream.py | 4 +- .../v1/worker/test_npu_model_runner_v2.py | 4 +- tests/unit/v1/worker/test_npu_runtime.py | 7 +- 67 files changed, 5122 insertions(+), 239 deletions(-) create mode 100644 afd_plugin/connectors/npu/async_cam_runtime/__init__.py create mode 100644 afd_plugin/connectors/npu/async_cam_runtime/async_cam_pull.py rename afd_plugin/{ => connectors/npu/async_cam_runtime}/async_cam_startup.py (84%) create mode 100644 afd_plugin/model_executor/npu/async_cam_mega.py create mode 100644 csrc/npu/ascend_kernels/afd_mega_moe_comm/op_api/aclnn_afd_mega_moe_comm.cpp create mode 100644 csrc/npu/ascend_kernels/afd_mega_moe_comm/op_api/aclnn_afd_mega_moe_comm.h create mode 100644 csrc/npu/ascend_kernels/afd_mega_moe_comm/op_host/afd_mega_moe_comm.cpp create mode 100644 csrc/npu/ascend_kernels/afd_mega_moe_comm/op_kernel/afd_mega_moe_comm.cpp create mode 100644 csrc/npu/ascend_kernels/afd_mega_moe_comm/op_kernel/afd_mega_moe_comm_tiling.h create mode 100644 csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_api/aclnn_afd_mega_moe_pull_native.cpp create mode 100644 csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_api/aclnn_afd_mega_moe_pull_native.h create mode 100644 csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_host/afd_mega_moe_pull_native.cpp create mode 100644 csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_patch_activation.py create mode 100644 csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_plan_w4a8.h create mode 100644 csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_plan_w8a8.h create mode 100644 csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_pull_native.cpp create mode 100644 csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_pull_native_tiling.h create mode 100644 csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_pull_pipeline.h create mode 100644 csrc/npu/ascend_kernels/utils/op_kernel/afd_native_mega_workspace.h create mode 100644 csrc/npu/ascend_kernels/utils/op_kernel/afd_pull_layout.h create mode 100644 csrc/npu/ascend_kernels/utils/op_kernel/afd_pull_protocol.h create mode 100644 csrc/npu/ascend_kernels/utils/op_kernel/afd_swiglu_limit.h create mode 100644 csrc/npu/pybind/torch_binding_mega_moe_comm.cpp create mode 100644 csrc/npu/pybind/torch_binding_mega_moe_pull.cpp create mode 100644 tests/npu/check_pull_layout.cpp create mode 100644 tests/unit/connectors/npu/test_async_cam_pull.py rename tests/unit/connectors/{ => npu}/test_async_cam_startup.py (67%) rename tests/unit/{ => connectors/npu}/test_async_cam_startup_wiring.py (71%) create mode 100644 tests/unit/fakes/async_cam_startup.py create mode 100644 tests/unit/package/test_mega_activation_patch.py diff --git a/afd_plugin/compat/npu/feature_validation.py b/afd_plugin/compat/npu/feature_validation.py index cf151fac1..4e9e756df 100644 --- a/afd_plugin/compat/npu/feature_validation.py +++ b/afd_plugin/compat/npu/feature_validation.py @@ -273,9 +273,10 @@ def _fail_if_unsupported_npu_afd_async_features( "AFD DSpark draft query capacity exceeds max_num_batched_tokens: " f"{draft_rows} > {max_num_batched_tokens}" ) - # whole-expert chunks need a full TP group's batch; smaller - # buffers require native sub-expert chunking. - if receive_capacity < max_num_batched_tokens: + if receive_capacity < 1: + raise RuntimeError("BATCH_SIZE_FACTOR must allocate at least one receive row") + # Push chunks whole experts. Pull can split one expert across receive rounds. + if not extra_info.ffn_mega_moe and receive_capacity < max_num_batched_tokens: raise RuntimeError( "CAMAsyncAFDConnector requires BATCH_SIZE_FACTOR to allocate at least " f"max_num_batched_tokens={max_num_batched_tokens} " diff --git a/afd_plugin/connectors/__init__.py b/afd_plugin/connectors/__init__.py index 2b270cc6e..00384908f 100644 --- a/afd_plugin/connectors/__init__.py +++ b/afd_plugin/connectors/__init__.py @@ -2,27 +2,49 @@ # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project """AFD connector namespace.""" -from afd_plugin.connectors.base import ( - AFDConnectorBase, - AFDControlPlane, - AFDRoutedExpertsConnector, - ConnectorExtraInfo, -) -from afd_plugin.connectors.factory import AFDConnectorFactory -from afd_plugin.connectors.metadata import ( - AFDA2FTransferPayload, - AFDControlPayload, - AFDDPMetadata, - AFDExpertRoutingSpec, - AFDF2ATransferPayload, - AFDForwardContextMetadata, - AFDSingleDPMetadata, - AFDTransferContext, - AFDTransferMetadata, - AFDTransferState, - FFNLocalExperts, - LocalExpertWeights, -) +from importlib import import_module +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from afd_plugin.connectors.base import ( + AFDConnectorBase, + AFDControlPlane, + AFDRoutedExpertsConnector, + ConnectorExtraInfo, + ) + from afd_plugin.connectors.factory import AFDConnectorFactory + from afd_plugin.connectors.metadata import ( + AFDA2FTransferPayload, + AFDControlPayload, + AFDDPMetadata, + AFDExpertRoutingSpec, + AFDF2ATransferPayload, + AFDForwardContextMetadata, + AFDSingleDPMetadata, + AFDTransferContext, + AFDTransferMetadata, + AFDTransferState, + FFNLocalExperts, + LocalExpertWeights, + ) + + +def __getattr__(name: str): + if name in { + "AFDConnectorBase", + "AFDControlPlane", + "AFDRoutedExpertsConnector", + "ConnectorExtraInfo", + }: + module = "base" + elif name == "AFDConnectorFactory": + module = "factory" + elif name in __all__: + module = "metadata" + else: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + return getattr(import_module(f"{__name__}.{module}"), name) + __all__ = [ "AFDRoutedExpertsConnector", diff --git a/afd_plugin/connectors/base.py b/afd_plugin/connectors/base.py index e0fd3528d..071beaa38 100644 --- a/afd_plugin/connectors/base.py +++ b/afd_plugin/connectors/base.py @@ -5,9 +5,10 @@ from __future__ import annotations from abc import ABC, abstractmethod -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence +from contextlib import AbstractContextManager, nullcontext from dataclasses import dataclass -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Protocol import torch @@ -36,6 +37,75 @@ def to_mapping(self) -> dict[str, Any]: return {} +class FFNExecutionSession(Protocol): + """Backend lifecycle used by FFN runners and workers. + + Workers stop serving before draining and finishing the receiver, then + validate shutdown before releasing communication and model resources. + ``requires_drain`` marks backends that must retain resources until their + pending work and receiver handshake have completed. + """ + + requires_drain: bool + + def keep_serving(self, shutdown_requested: bool) -> bool: + """Whether to serve another step given the local shutdown_requested flag. + + A backend may keep serving after local shutdown until remote producers + are quiet. + """ + ... + + def before_first_step(self) -> None: + """Prepare backend state once before the first FFN step.""" + ... + + def after_step(self) -> None: + """Check backend status after a completed FFN step.""" + ... + + def drain(self, run_one: Callable[[], None]) -> None: + """Drain pending work using run_one to execute and wait for a transaction.""" + ... + + def finish_receiver(self) -> None: + """Finish the receiver handshake after pending work has drained.""" + ... + + def validate_shutdown(self) -> None: + """Raise RuntimeError if resources cannot yet be released safely.""" + ... + + +class DefaultFFNSession: + """Lifecycle for connectors that need no drain or receiver handshake. + + Service stops when ``shutdown_requested`` is true. Other hooks are no-ops; + in particular, ``drain`` never invokes its ``run_one`` callback. Workers + and runners still own device synchronization and resource release. + """ + + requires_drain = False + + def keep_serving(self, shutdown_requested: bool) -> bool: + return not shutdown_requested + + def before_first_step(self) -> None: + pass + + def after_step(self) -> None: + pass + + def drain(self, run_one: Callable[[], None]) -> None: + pass + + def finish_receiver(self) -> None: + pass + + def validate_shutdown(self) -> None: + pass + + class AFDConnectorBase(ABC): """Base class for plugin-owned AFD Attention/FFN connectors. @@ -112,6 +182,17 @@ def __init__( # Lifecycle methods # ============================== + def attention_step(self) -> AbstractContextManager[None]: + """Return a context for ``with connector.attention_step():`` around forward. + + Backends may defer transport status checks until the Attention model + step completes. The default context performs no additional work. + """ + return nullcontext() + + def ffn_session(self) -> FFNExecutionSession: + return DefaultFFNSession() + @abstractmethod def close(self) -> None: """Release connector-owned communication resources. @@ -378,4 +459,6 @@ def recv_dp_metadata_list(self) -> AFDControlPayload: "AFDControlPlaneClosedError", "AFDRoutedExpertsConnector", "ConnectorExtraInfo", + "DefaultFFNSession", + "FFNExecutionSession", ] diff --git a/afd_plugin/connectors/npu/__init__.py b/afd_plugin/connectors/npu/__init__.py index 14e61344e..8abb26e20 100644 --- a/afd_plugin/connectors/npu/__init__.py +++ b/afd_plugin/connectors/npu/__init__.py @@ -2,12 +2,24 @@ # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project """NPU-specific AFD connector implementations.""" -from afd_plugin.connectors.npu.camp2p import ( - CAMP2pAFDConnector, - CAMP2PExtraInfo, - CAMP2PTransferState, - build_camp2p_topology, -) +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from afd_plugin.connectors.npu.camp2p import ( + CAMP2pAFDConnector, + CAMP2PExtraInfo, + CAMP2PTransferState, + build_camp2p_topology, + ) + + +def __getattr__(name: str): + if name in __all__: + from afd_plugin.connectors.npu import camp2p + + return getattr(camp2p, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + __all__ = [ "CAMP2pAFDConnector", diff --git a/afd_plugin/connectors/npu/async_cam.py b/afd_plugin/connectors/npu/async_cam.py index f981fe153..a229286a3 100644 --- a/afd_plugin/connectors/npu/async_cam.py +++ b/afd_plugin/connectors/npu/async_cam.py @@ -29,7 +29,8 @@ import os import time -from collections.abc import Mapping +from collections.abc import Iterator, Mapping +from contextlib import AbstractContextManager, contextmanager, nullcontext from dataclasses import dataclass from datetime import timedelta from typing import TYPE_CHECKING, Any, Final, cast @@ -38,11 +39,6 @@ from torch import Tensor from vllm.logger import init_logger -from afd_plugin.async_cam_startup import ( - GRAPH_STARTUP_TIMEOUT_SECONDS, - POLL_SECONDS, - AsyncCAMGraphStartup, -) from afd_plugin.compat.npu.ops import ensure_cam_async_ops_available from afd_plugin.config import AFDConfig from afd_plugin.config_utils import ( @@ -54,6 +50,8 @@ from afd_plugin.connectors.base import ( AFDConnectorBase, ConnectorExtraInfo, + DefaultFFNSession, + FFNExecutionSession, ) from afd_plugin.connectors.metadata import ( AFDA2FTransferPayload, @@ -61,6 +59,12 @@ AFDTransferMetadata, AFDTransferState, ) +from afd_plugin.connectors.npu.async_cam_runtime.async_cam_pull import PullFFNSession +from afd_plugin.connectors.npu.async_cam_runtime.async_cam_startup import ( + GRAPH_STARTUP_TIMEOUT_SECONDS, + POLL_SECONDS, + AsyncCAMGraphStartup, +) from afd_plugin.distributed import ( create_hccl_process_group_options, init_afd_process_group, @@ -75,11 +79,17 @@ AFD_ASYNC_CAM_GROUP_NAME = "afd_async_cam" CAM_COMM_ID = 0 +PULL_COMM_ID = 0x41464401 +PULL_FAILURE_INDEX = 24 +PULL_SOURCE_INDEX = 32 +PULL_IDLE_BUDGET_INDEX = 64 +PULL_IDLE_SCANS = 8 ATTN_RANKS_PER_DP_CONFIG_KEY = "attn_ranks_per_dp" _AFD_ASYNC_EXTRA_CONFIG_FIELDS: Final[frozenset[str]] = frozenset( { "dynamicQuant", + "ffn_mega_moe", "attn_ranks_per_dp", "hccl_buffer_size", "ffn_two_stream", @@ -149,8 +159,10 @@ class AFDAsyncExtraInfo(ConnectorExtraInfo): dynamic_quant: Dynamic quantization mode accepted by CAM operators. attn_ranks_per_dp: Number of Attention ranks in each data-parallel group. hccl_buffer_size: Optional buffer size in MB for the CAM HCCL domain. + ffn_mega_moe: Enable Mega FFN and its pull protocol on every A/F rank. """ + ffn_mega_moe: bool = False dynamic_quant: int = 0 attn_ranks_per_dp: int = 1 hccl_buffer_size: int | None = None @@ -177,7 +189,10 @@ def from_mapping(cls, raw: Mapping[str, Any] | None) -> AFDAsyncExtraInfo: + ", ".join(unknown), ) - return cls( + result = cls( + ffn_mega_moe=coerce_extra_bool( + raw.get("ffn_mega_moe", False), field_name="ffn_mega_moe" + ), dynamic_quant=coerce_extra_int( raw.get("dynamicQuant", 0), field_name="dynamicQuant", @@ -204,11 +219,19 @@ def from_mapping(cls, raw: Mapping[str, Any] | None) -> AFDAsyncExtraInfo: ), ) + if result.ffn_mega_moe and (result.dynamic_quant != 1 or result.ffn_two_stream): + raise ValueError( + "ffn_mega_moe requires dynamicQuant=1 and ffn_two_stream=false" + ) + return result + def to_mapping(self) -> dict[str, Any]: result: dict[str, Any] = { "dynamicQuant": self.dynamic_quant, "attn_ranks_per_dp": self.attn_ranks_per_dp, } + if self.ffn_mega_moe: + result["ffn_mega_moe"] = True if self.hccl_buffer_size is not None: result["hccl_buffer_size"] = self.hccl_buffer_size if self.ffn_two_stream or ( @@ -319,6 +342,7 @@ def __init__( self.expert_per_rank = self.topology.expert_per_rank self.comm_args: Tensor | None = None self._placeholder: Tensor | None = None + self._defer_pull_status_check = False self._pending_attention_payloads: dict[ int, list[tuple[AFDTransferContext, Tensor, Tensor]], @@ -326,6 +350,10 @@ def __init__( self.graph_startup = AsyncCAMGraphStartup( self.world_rank, self.attn_size, self.topology.world_size ) + if extra_info.ffn_mega_moe: + self._session: FFNExecutionSession = PullFFNSession(self) + else: + self._session = DefaultFFNSession() @property def is_initialized(self) -> bool: @@ -358,7 +386,33 @@ def init_afd_connector(self) -> None: backend = self.cam_pg._get_backend(torch.device("npu")) self.group_name = str(backend.get_hccl_comm_name(self.world_rank)) device = f"npu:{self.local_rank}" - self.comm_args = torch.empty((1,), dtype=torch.float16, device=device) + if self.extra_info.ffn_mega_moe: + self.comm_id = PULL_COMM_ID + self.comm_args = torch.ops.afd_ascend.afd_mega_pull_init( + torch.empty(1, dtype=torch.bfloat16, device=device), + [ + self.max_num_batched_tokens // self.tp_size, + self.hidden_size, + self.topk, + self.ffn_size, + self.attn_size, + self.expert_per_rank, + self.world_rank, + self.tp_size, + ], + list(self.graph_startup.plan.layer_ids), + self.graph_startup.session_epoch, + self.group_name, + ) + try: + torch.npu.synchronize() + self._check_pull_status() + self.graph_startup.phase("pull-initialized") + except Exception as exc: + self.graph_startup.fail(exc) + raise + else: + self.comm_args = torch.empty((1,), dtype=torch.float16, device=device) self._placeholder = torch.empty( (1,), dtype=self.activation_dtype, @@ -385,13 +439,18 @@ def _initialize_graph_startup(self, store: Store) -> None: self.max_num_batched_tokens, ) ) + if self.extra_info.ffn_mega_moe: + signature = "pull_v1:" + signature self.graph_startup.initialize(store, signature) if self.graph_startup.mode == "full": try: if not self.vllm_config.use_v2_model_runner: raise ValueError("Async FFN FULL requires MRV2 on every rank") validate_graph_capacity( - self.max_num_batched_tokens, self.tp_size, cam_receive_capacity() + self.max_num_batched_tokens, + self.tp_size, + cam_receive_capacity(), + allow_row_slicing=self.extra_info.ffn_mega_moe, ) self.graph_startup.phase("validated") except Exception as exc: @@ -426,6 +485,9 @@ def wait_graph_device(self, event: torch.npu.Event) -> None: def graph_startup_chunks(self) -> int: """Every A TP rank sends one row to experts [0, topk).""" + if self.extra_info.ffn_mega_moe: + # A fused launch drains every receive round of the selected source. + return 1 local_experts = max( 0, min( @@ -462,8 +524,26 @@ def _run_attention_graph_startup(self) -> None: self.graph_startup.fail(exc) raise + def ffn_session(self) -> FFNExecutionSession: + return self._session + + def attention_step(self) -> AbstractContextManager[None]: + if self.extra_info.ffn_mega_moe: + return self._defer_pull_status() + return nullcontext() + def close(self) -> None: """Destroy the HCCL process group and clear pending transfer states.""" + if isinstance(self._session, PullFFNSession): + if self._pending_attention_payloads: + raise RuntimeError( + "Pull dispatch is pending; retain communication resources" + ) + if self._initialized: + if self.world_rank < self.attn_size: + self._session.close_attention() + else: + self._session.validate_shutdown() if self.cam_pg is not None: import torch.distributed as dist @@ -523,6 +603,16 @@ def send_attn_output( "Async CAM batch exceeds max_num_batched_tokens / TP capacity" ) hidden_states = hidden_states.contiguous() + pull = self.extra_info.ffn_mega_moe + if pull: + if self._pending_attention_payloads: + raise RuntimeError( + "Pull window is pending; finish Combine before Dispatch" + ) + # A launch can enqueue work before raising; retain ownership on failure. + self._pending_attention_payloads[metadata.stage_idx] = [ + (context, topk_ids, topk_weights) + ] _log_cam_op_values( "async_dispatch_send", @@ -545,7 +635,11 @@ def send_attn_output( dynamic_quant=self.dynamic_quant, group_name=self.group_name, ) - torch.ops.afd_ascend.afd_async_dispatch_send( + ( + torch.ops.afd_ascend.afd_mega_dispatch_send + if self.extra_info.ffn_mega_moe + else torch.ops.afd_ascend.afd_async_dispatch_send + )( hidden_states, topk_ids, self.comm_args, @@ -564,11 +658,10 @@ def send_attn_output( self.dynamic_quant, self.group_name, ) - self._pending_attention_payloads.setdefault( - context.metadata.stage_idx, [] - ).append( - (context, topk_ids, topk_weights), - ) + if not pull: + self._pending_attention_payloads.setdefault(metadata.stage_idx, []).append( + (context, topk_ids, topk_weights), + ) return None @@ -605,9 +698,7 @@ def recv_ffn_output( "CAMAsyncAFDConnector recv_ffn_output context does not match " "the next pending Attention dispatch", ) - context, topk_ids, topk_weights = payloads.pop(0) - if not payloads: - self._pending_attention_payloads.pop(ubatch_idx, None) + context, topk_ids, topk_weights = payloads[0] states = _require_async_transfer_state(context) _validate_topk_payload( @@ -635,7 +726,11 @@ def recv_ffn_output( world_size=self.topology.world_size, group_name=self.group_name, ) - output = torch.ops.afd_ascend.afd_async_combine_recv( + output = ( + torch.ops.afd_ascend.afd_mega_combine_recv + if self.extra_info.ffn_mega_moe + else torch.ops.afd_ascend.afd_async_combine_recv + )( placeholder, topk_ids, topk_weights, @@ -650,10 +745,48 @@ def recv_ffn_output( self.world_rank, self.topology.world_size, self.group_name, + *( + (self.max_num_batched_tokens, self.tp_size) + if self.extra_info.ffn_mega_moe + else () + ), ) _log_cam_op_values("async_combine_recv", "outputs", output=output) + if not self._defer_pull_status_check: + self._check_pull_status() + payloads.pop(0) + if not payloads: + self._pending_attention_payloads.pop(ubatch_idx, None) return output + @contextmanager + def _defer_pull_status(self) -> Iterator[None]: + """Check sticky device errors before the model step returns, not per MoE.""" + previous = self._defer_pull_status_check + self._defer_pull_status_check = True + try: + yield + if not previous: + self._check_pull_status() + finally: + self._defer_pull_status_check = previous + + def _check_pull_status(self) -> None: + if self.extra_info.ffn_mega_moe: + assert self.comm_args is not None + # The model runner batches this fence outside the DBO receive hook. + status = int(self.comm_args.view(torch.int64)[PULL_FAILURE_INDEX].item()) + if status: + raise RuntimeError(f"AFD pull transaction failed with status {status}") + + def _enable_pull_idle_returns(self) -> None: + assert self.comm_args is not None + self.comm_args.view(torch.int64)[PULL_IDLE_BUDGET_INDEX].fill_(PULL_IDLE_SCANS) + + def _pull_has_active_source(self) -> bool: + assert self.comm_args is not None + return int(self.comm_args.view(torch.int64)[PULL_SOURCE_INDEX].item()) >= 0 + def recv_attn_output( self, ubatch_idx: int = 0, diff --git a/afd_plugin/connectors/npu/async_cam_runtime/__init__.py b/afd_plugin/connectors/npu/async_cam_runtime/__init__.py new file mode 100644 index 000000000..721549de4 --- /dev/null +++ b/afd_plugin/connectors/npu/async_cam_runtime/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Async CAM startup and FFN lifecycle helpers.""" diff --git a/afd_plugin/connectors/npu/async_cam_runtime/async_cam_pull.py b/afd_plugin/connectors/npu/async_cam_runtime/async_cam_pull.py new file mode 100644 index 000000000..ae552a04f --- /dev/null +++ b/afd_plugin/connectors/npu/async_cam_runtime/async_cam_pull.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Pull service and shutdown handshake for the async CAM connector.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING + +from afd_plugin.validation import cam_receive_capacity + +if TYPE_CHECKING: + from afd_plugin.connectors.npu.async_cam import CAMAsyncAFDConnector + + +class PullFFNSession: + requires_drain = True + + def __init__(self, connector: CAMAsyncAFDConnector) -> None: + self.connector = connector + self.receiver_finished = False + self._idle_enabled = False + self._drained = False + + def keep_serving(self, shutdown_requested: bool) -> bool: + startup = self.connector.graph_startup + assert startup._store is not None + startup.check_failure() + return not startup._store.check( + startup._keys("pull_quiet", range(startup.attn_size)) + ) + + def before_first_step(self) -> None: + if not self._idle_enabled: + self.connector._enable_pull_idle_returns() + self._idle_enabled = True + + def after_step(self) -> None: + self.connector._check_pull_status() + + def drain(self, run_one: Callable[[], None]) -> None: + connector = self.connector + rounds = ( + connector.max_num_batched_tokens * connector.topk + + cam_receive_capacity() + - 1 + ) // cam_receive_capacity() + for _ in range(rounds): + if not connector._pull_has_active_source(): + self._drained = True + return + run_one() + if connector._pull_has_active_source(): + raise RuntimeError("Pull source did not drain; retain resources") + self._drained = True + + def finish_receiver(self) -> None: + if self.receiver_finished: + return + if not self._drained: + raise RuntimeError("Pull receiver has not drained; retain resources") + startup = self.connector.graph_startup + startup._wait(startup._keys("pull_quiet", range(startup.attn_size))) + startup._publish("pull_stopped") + self.receiver_finished = True + + def close_attention(self) -> None: + startup = self.connector.graph_startup + startup._publish("pull_quiet") + startup._wait( + startup._keys("pull_stopped", range(startup.attn_size, startup.world_size)) + ) + + def validate_shutdown(self) -> None: + if not self.receiver_finished or self.connector._pull_has_active_source(): + raise RuntimeError("Pull receiver has not drained; retain resources") + self.connector._check_pull_status() diff --git a/afd_plugin/async_cam_startup.py b/afd_plugin/connectors/npu/async_cam_runtime/async_cam_startup.py similarity index 84% rename from afd_plugin/async_cam_startup.py rename to afd_plugin/connectors/npu/async_cam_runtime/async_cam_startup.py index f2c48f14f..77bd8e7da 100644 --- a/afd_plugin/async_cam_startup.py +++ b/afd_plugin/connectors/npu/async_cam_runtime/async_cam_startup.py @@ -70,15 +70,19 @@ def __init__(self, world_rank: int, attn_size: int, world_size: int) -> None: self.plan = FFNGraphPlan() self.mode = "eager" self.warmup_layer_id: int | None = None + self.session_epoch = 0 self._store: Store | None = None self._prefix = "" + self._shared_errors = False self.validate_attention_streams: Callable[[], None] | None = None def initialize(self, store: Store, transport_signature: str) -> None: self._store = store if self.world_rank == 0: store.set("ffn_graph/nonce", uuid.uuid4().hex) - self._prefix = f"ffn_graph/{store.get('ffn_graph/nonce').decode()}/" + nonce = store.get("ffn_graph/nonce").decode() + self.session_epoch = int(nonce[:15], 16) or 1 + self._prefix = f"ffn_graph/{nonce}/" self._negotiate( self.plan if self.world_rank >= self.attn_size else None, transport_signature, @@ -89,10 +93,19 @@ def _keys(self, stage: str, ranks: range) -> list[str]: def _publish(self, stage: str, value: str = "ready") -> None: assert self._store is not None + if stage == "error": + # Publish the shared signal first so new peers still fail if the + # compatibility publication below loses its Store connection. + self._store.set(f"{self._prefix}error", value) self._store.set(f"{self._prefix}{stage}/{self.world_rank}", value) def check_failure(self) -> None: assert self._store is not None + signal = f"{self._prefix}error" + if self._store.check([signal]): + raise RuntimeError(self._store.get(signal).decode()) + if self._shared_errors: + return for key in self._keys("error", range(self.world_size)): if self._store.check([key]): raise RuntimeError(self._store.get(key).decode()) @@ -126,6 +139,7 @@ def _negotiate(self, plan: FFNGraphPlan | None, transport_signature: str) -> Non json.dumps( { "transport": transport_signature, + "shared_errors": True, "ffn": asdict(plan) if plan else None, } ), @@ -133,6 +147,10 @@ def _negotiate(self, plan: FFNGraphPlan | None, transport_signature: str) -> Non keys = self._keys("plan", range(self.world_size)) self._wait(keys) entries = [json.loads(self._store.get(key)) for key in keys] + # Old peers publish only per-rank errors; keep their original check path. + self._shared_errors = all( + entry.get("shared_errors", False) for entry in entries + ) plans = [] for entry in entries[self.attn_size :]: fields = entry["ffn"] @@ -141,7 +159,11 @@ def _negotiate(self, plan: FFNGraphPlan | None, transport_signature: str) -> Non self.mode = resolve_ffn_mode(plans) ffn_plan = plans[0] two_stream = bool(ffn_plan.stream_signature) - validate_transport = self.mode == "full" or two_stream + validate_transport = ( + self.mode == "full" + or two_stream + or any(entry["transport"].startswith("pull_v1:") for entry in entries) + ) if ( validate_transport and len({entry["transport"] for entry in entries}) != 1 diff --git a/afd_plugin/model_executor/npu/async_cam_layered.py b/afd_plugin/model_executor/npu/async_cam_layered.py index 6079ed274..06962e7a4 100644 --- a/afd_plugin/model_executor/npu/async_cam_layered.py +++ b/afd_plugin/model_executor/npu/async_cam_layered.py @@ -108,6 +108,20 @@ def __init__(self, layers: list[LayeredFFNWeights]) -> None: self.w13_op = torch.ops.afd_ascend.gmm_swiglu_quant_v2_layered self.w2_op = torch.ops.afd_ascend.grouped_matmul_layered + def geometry(self) -> tuple[int, int, int]: + """Return (experts, hidden, intermediate) from the loaded weight layout. + + INT4 weights pack eight nibbles per INT32, so the stored last dimension + is a quarter of the gate/up width; INT8 stores that width directly. + Callers that need only the logical geometry must not re-derive it. + """ + experts, hidden, last = self.w13[0].shape + if self.quantization == "W8A8": + if last % 2: + raise ValueError("layered GMM requires an even W13 gate/up width") + return experts, hidden, last // 2 + return experts, hidden, last * INT4_PER_INT32 // 2 + def graph_signature(self, capacity: int) -> str: """Compare static layouts across ranks without local device addresses.""" tensors = ( diff --git a/afd_plugin/model_executor/npu/async_cam_mega.py b/afd_plugin/model_executor/npu/async_cam_mega.py new file mode 100644 index 000000000..1370346de --- /dev/null +++ b/afd_plugin/model_executor/npu/async_cam_mega.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""One native launch drains an exclusive quantized transaction per layer.""" + +from __future__ import annotations + +import torch + +from afd_plugin.model_executor.npu.async_cam_layered import AsyncCAMLayeredExecutor +from afd_plugin.validation import cam_receive_capacity + +# Mirror Pull::MetadataWords in afd_pull_layout.h (all fields are int64_t): +# [total, source, layer, round_begin, round_end], tp source prefixes, +# tp * experts original counts, then [session, sequence]. +METADATA_WORD_BYTES = torch.int64.itemsize +METADATA_FIXED_WORDS = 7 +METADATA_PREFIX_WORDS_PER_SOURCE = 1 +# NativeMegaWorkspace in afd_native_mega_workspace.h aligns scratch regions to +# 512 bytes. Match its counts offset, including padding after batch metadata. +WORKSPACE_ALIGNMENT_BYTES = 512 + + +class AsyncCAMMegaExecutor: + """Pull transport with a compiled W4A8 or W8A8 execution plan.""" + + def __init__(self, weights: AsyncCAMLayeredExecutor, connector) -> None: + if not weights.per_channel: + raise ValueError("native Mega requires per-channel W4A8/W8A8 weights") + self.weights = weights + self.connector = connector + self.capacity = cam_receive_capacity() + experts, hidden, intermediate = weights.geometry() + self.config = [ + connector.max_num_batched_tokens, + hidden, + intermediate, + connector.topk, + connector.ffn_size, + connector.attn_size, + experts, + connector.world_rank, + connector.topology.world_size, + connector.tp_size, + self.capacity, + 1 if weights.quantization == "W4A8" else 2, + 1, + len(weights.layer_ids), + ] + device = weights.w13[0].device + size = torch.ops.afd_ascend.afd_mega_moe_pull_workspace_size( + self.capacity, hidden, intermediate, experts, connector.tp_size + ) + self.scratch = torch.empty(size, dtype=torch.uint8, device=device) + metadata_words = METADATA_FIXED_WORDS + connector.tp_size * ( + METADATA_PREFIX_WORDS_PER_SOURCE + experts + ) + metadata_bytes = METADATA_WORD_BYTES * metadata_words + count_offset = ( + (metadata_bytes + WORKSPACE_ALIGNMENT_BYTES - 1) + // WORKSPACE_ALIGNMENT_BYTES + * WORKSPACE_ALIGNMENT_BYTES + ) + count_bytes = METADATA_WORD_BYTES * experts + self.batch_info = self.scratch[:metadata_bytes].view(torch.int64) + self.group_list = self.scratch[count_offset : count_offset + count_bytes].view( + torch.int64 + ) + self.anchor = torch.empty(1, dtype=torch.bfloat16, device=device) + mapping = [-1] * (weights.layer_ids[-1] + 1) + for slot, layer_id in enumerate(weights.layer_ids): + mapping[layer_id] = slot + self.layer_map = torch.tensor(mapping, dtype=torch.int64, device=device) + self.op = torch.ops.afd_ascend.afd_mega_moe_pull_native + + def __call__(self) -> torch.Tensor: + c, w = self.connector, self.weights + return self.op( + self.anchor, + c.comm_args, + self.layer_map, + self.scratch, + w.w13, + w.w2, + w.w13_scale, + w.w2_scale, + w.w13_bias if w.quantization == "W4A8" else [], + w.w2_bias if w.quantization == "W4A8" else [], + self.config, + c.group_name, + w.routed_scaling_factor, + w.applied_swiglu_limit, + ) diff --git a/afd_plugin/v1/worker/npu/async_ffn_model_runner.py b/afd_plugin/v1/worker/npu/async_ffn_model_runner.py index ebb7c0245..9d13ed394 100644 --- a/afd_plugin/v1/worker/npu/async_ffn_model_runner.py +++ b/afd_plugin/v1/worker/npu/async_ffn_model_runner.py @@ -12,11 +12,6 @@ from vllm.logger import init_logger from vllm_ascend.utils import AscendDeviceType, get_ascend_device_type -from afd_plugin.async_cam_startup import ( - GRAPH_STARTUP_TIMEOUT_SECONDS, - POLL_SECONDS, - FFNGraphPlan, -) from afd_plugin.compat.npu import fail_if_unsupported_npu_afd_features from afd_plugin.compat.npu.ops import ensure_cam_async_ops_available from afd_plugin.compat.npu.profiler import ( @@ -25,14 +20,21 @@ stop_afd_npu_profiler, ) from afd_plugin.config import parse_afd_config +from afd_plugin.connectors.metadata import AFDA2FTransferPayload from afd_plugin.connectors.npu.async_cam import ( AFDAsyncTransferState, CAMAsyncAFDConnector, ) +from afd_plugin.connectors.npu.async_cam_runtime.async_cam_startup import ( + GRAPH_STARTUP_TIMEOUT_SECONDS, + POLL_SECONDS, + FFNGraphPlan, +) from afd_plugin.model_executor.npu.async_cam_layered import ( AsyncCAMLayeredExecutor, LayeredFFNWeights, ) +from afd_plugin.model_executor.npu.async_cam_mega import AsyncCAMMegaExecutor from afd_plugin.v1.worker.npu.async_ffn_two_stream import AsyncFFNTwoStream from afd_plugin.validation import ( cam_receive_capacity, @@ -76,6 +78,9 @@ def __init__( self.model_memory_usage = 0 self.layer_ids: tuple[int, ...] = () self._layered_executor: AsyncCAMLayeredExecutor | None = None + self._mega_executor: AsyncCAMMegaExecutor | None = None + self.session = self.connector.ffn_session() + self._first_step = True self.requested_mode = requested_ffn_mode(vllm_config) self.effective_mode = "eager" self._graph: torch.npu.NPUGraph | None = None @@ -165,6 +170,11 @@ def initialize_afd_connector(self) -> None: self._initialize_layered_executor() assert self._layered_executor is not None signature = self._layered_executor.graph_signature(cam_receive_capacity()) + if config.ffn_mega_moe: + self._mega_executor = AsyncCAMMegaExecutor( + self._layered_executor, self.connector + ) + signature = "native_mc2:" + signature except Exception as exc: executor_error = str(exc) or type(exc).__name__ executor = self._layered_executor @@ -243,6 +253,7 @@ def _initialize_graph(self, stream: torch.npu.Stream) -> None: self._graph_stream = stream self._graph_pool = torch.npu.graph_pool_handle() self._graph_tensors: list[torch.Tensor] = [] + self._graph_payload: AFDA2FTransferPayload | None = None def _check_graph_live(self) -> None: if self._graph_failure is not None: @@ -267,21 +278,27 @@ def _capture_graph(self) -> None: with torch.npu.graph( self._graph, pool=self._graph_pool, stream=self._graph_stream ): - self._graph_payload = self.connector.recv_attn_output() - state = cast(AFDAsyncTransferState, self._graph_payload.context.states) - assert state.token_nums_rankid_layeridx is not None - assert state.group_list is not None - assert state.dynamic_scales is not None - self._graph_output = self._layered_executor( - self._graph_payload.hidden_states, - state.dynamic_scales, - state.group_list, - state.token_nums_rankid_layeridx, - retained_tensors=self._graph_tensors, - ) - self.connector.send_ffn_output( - self._graph_output, self._graph_payload.context - ) + if self._mega_executor is not None: + self._graph_payload = None + self._graph_output = self._mega_executor() + else: + self._graph_payload = self.connector.recv_attn_output() + state = cast( + AFDAsyncTransferState, self._graph_payload.context.states + ) + assert state.token_nums_rankid_layeridx is not None + assert state.group_list is not None + assert state.dynamic_scales is not None + self._graph_output = self._layered_executor( + self._graph_payload.hidden_states, + state.dynamic_scales, + state.group_list, + state.token_nums_rankid_layeridx, + retained_tensors=self._graph_tensors, + ) + self.connector.send_ffn_output( + self._graph_output, self._graph_payload.context + ) self._graph_capture_count = 1 except Exception as exc: self._graph_failure = exc @@ -327,6 +344,7 @@ def _complete_graph(self) -> bool: # One fence per bounded batch. The original DR cannot be cancelled; # a blocked batch retains its graph, model and communication buffers. torch.npu.synchronize() + self.session.after_step() except Exception as exc: self._graph_failure = exc raise @@ -385,11 +403,19 @@ def _initialize_layered_executor(self) -> None: ) @torch.inference_mode() - def execute_connector_driven_step(self) -> bool: + def execute_connector_driven_step(self, shutdown_requested: bool = False) -> bool: step_afd_npu_profiler(self.prof) + if not self.session.keep_serving(shutdown_requested): + return False + if self._first_step: + self.session.before_first_step() + if self._graph is not None: + self._graph_stream.wait_stream(torch.npu.current_stream(self.device)) + self._first_step = False if self._graph is None and self._two_stream is None: # Eager and graph modes use the same layered executor. self._ffn_forward_connector_driven() + self.session.after_step() return True if not self._service_announced: self.connector.graph_startup.service_ready() @@ -399,6 +425,18 @@ def execute_connector_driven_step(self) -> bool: self._submit_graph(len(self.layer_ids)) return self._complete_graph() + def drain(self) -> None: + self.session.drain(self._run_one_transaction) + + def _run_one_transaction(self) -> None: + if self._graph is not None: + self._submit_graph(1) + self._complete_graph() + else: + self._ffn_forward_connector_driven(single=True) + torch.npu.synchronize() + self.session.after_step() + def _ffn_forward_connector_driven( self, *, @@ -409,6 +447,10 @@ def _ffn_forward_connector_driven( self._two_stream.submit(1 if single else len(self.layer_ids)) self._two_stream.complete(wait=self._wait_graph_device if single else None) return None + if self._mega_executor is not None: + for _ in self.layer_ids[:1] if single else self.layer_ids: + rank_ffn_output = self._mega_executor() + return rank_ffn_output rank_ffn_output = None assert self._layered_executor is not None for _ in self.layer_ids[:1] if single else self.layer_ids: @@ -445,6 +487,9 @@ def shutdown(self) -> None: raise RuntimeError( "Failed graph startup retains resources" ) from self._failure + if self.session.requires_drain and not self._graph_receiver_exited: + raise RuntimeError("Receiver has not exited; retain resources") + self.session.validate_shutdown() from vllm.v1.worker.gpu.shutdown import free_before_shutdown if self._two_stream is not None: @@ -454,6 +499,7 @@ def shutdown(self) -> None: self._close_graph() stop_afd_npu_profiler(self.prof) self.connector.close() + self._mega_executor = None self._layered_executor = None free_before_shutdown(self.vllm_config) self.model = None diff --git a/afd_plugin/v1/worker/npu/attention_model_runner_v2.py b/afd_plugin/v1/worker/npu/attention_model_runner_v2.py index 246079a65..1c2b8214d 100644 --- a/afd_plugin/v1/worker/npu/attention_model_runner_v2.py +++ b/afd_plugin/v1/worker/npu/attention_model_runner_v2.py @@ -488,6 +488,9 @@ def execute_model( ) try: with ( + self.connector.attention_step() + if self.connector is not None + else nullcontext(), skip_dp_coordination() if self.afd_config.async_dp else nullcontext(), _use_camp2p_dp_padding( self.afd_config.num_attention_ranks, self.afd_config.num_ffn_ranks diff --git a/afd_plugin/v1/worker/npu/ffn_worker.py b/afd_plugin/v1/worker/npu/ffn_worker.py index e56a7f930..6bec2e38c 100644 --- a/afd_plugin/v1/worker/npu/ffn_worker.py +++ b/afd_plugin/v1/worker/npu/ffn_worker.py @@ -48,6 +48,8 @@ logger = logging.getLogger(__name__) FFN_SHUTDOWN_TIMEOUT_SECONDS = 5 +# Pull receivers must wait for all Attention ranks to stop publishing. +ASYNC_FFN_DRAIN_TIMEOUT_SECONDS = 60 def create_npu_ffn_model_runner( @@ -201,16 +203,20 @@ def _run_ffn_server_loop(self) -> None: return torch.npu.set_device(self.device) - while not event.is_set(): - if isinstance(self.model_runner, AsyncNPUFFNModelRunner): - if not self.model_runner.execute_connector_driven_step(): - return + if isinstance(self.model_runner, AsyncNPUFFNModelRunner): + # The runner consults the session once per step; a second + # predicate here would add Store round trips to the Pull hot path. + while self.model_runner.execute_connector_driven_step(event.is_set()): if ( self.model_runner.effective_mode == "eager" and self.model_runner._two_stream is None ): torch.npu.synchronize() - continue + self.model_runner.drain() + self.model_runner.session.finish_receiver() + return + + while not event.is_set(): control_plane = self.model_runner.connector.control_plane assert control_plane is not None payload = control_plane.recv_dp_metadata_list() @@ -233,6 +239,7 @@ def _requires_async_drain(self) -> bool: return isinstance(self.model_runner, AsyncNPUFFNModelRunner) and ( self.model_runner.effective_mode == "full" or self.model_runner._two_stream is not None + or self.model_runner.session.requires_drain ) def raise_ffn_loop_error_if_any(self) -> None: @@ -249,7 +256,13 @@ def stop_ffn_server_loop(self) -> None: event.set() thread = self._ffn_thread if thread is not None: - thread.join(timeout=FFN_SHUTDOWN_TIMEOUT_SECONDS) + timeout = ( + ASYNC_FFN_DRAIN_TIMEOUT_SECONDS + if isinstance(self.model_runner, AsyncNPUFFNModelRunner) + and self.model_runner.session.requires_drain + else FFN_SHUTDOWN_TIMEOUT_SECONDS + ) + thread.join(timeout=timeout) if thread.is_alive(): raise RuntimeError( "Async graph receive is pending; retain resources" diff --git a/afd_plugin/validation.py b/afd_plugin/validation.py index 87dcb3255..bfef19f77 100644 --- a/afd_plugin/validation.py +++ b/afd_plugin/validation.py @@ -76,10 +76,12 @@ def requested_ffn_mode(config: VllmConfig) -> Literal["eager", "full"]: return "full" -def validate_graph_capacity(max_tokens: int, tp_size: int, capacity: int) -> None: +def validate_graph_capacity( + max_tokens: int, tp_size: int, capacity: int, *, allow_row_slicing: bool = False +) -> None: if min(max_tokens, tp_size, capacity) <= 0 or max_tokens % tp_size: raise ValueError("Graph dispatch envelope must be positive and divisible by TP") - if capacity < max_tokens: + if capacity < max_tokens and not allow_row_slicing: raise ValueError( "Async FFN graph capacity is smaller than the dispatch envelope" ) diff --git a/csrc/npu/README.md b/csrc/npu/README.md index 0bb34bdc6..5be6f9bfe 100644 --- a/csrc/npu/README.md +++ b/csrc/npu/README.md @@ -42,6 +42,42 @@ The supported SOC generations and the operators compiled for each are declared in `ascend_kernels/operator_registry.json`; `scripts/select_ops.py` resolves the operator list from that registry for a given SOC. +## Pull Mega MC2 dependency + +Set `connector_extra_config.ffn_mega_moe=true` on every Attention and FFN +rank to enable MegaMoE and its dedicated pull protocol. With the switch off, +CAM retains its original push Dispatch/Combine and FFN execution path. +Pull is not a standalone transport option: there is no `dispatch_mode` +setting or separate pull DispatchRecv/CombineSend operator. MegaMoE requires +`dynamicQuant=1`, per-channel W4A8 or W8A8 weights, and +`ffn_two_stream=false`. Per-group weights are not supported by MegaMoE. + +The A3 `AfdMegaMoePullNative` kernel directly includes the installed CANN MC2 +W4A8 `BlockMmad`. The build supports the CANN 9.1 MegaMoE header layout and +the CANN 9.2 layout with Catlass under `ascendc/common/catlass/include`. Set +`ASCEND_CANN_PACKAGE_PATH` or `ASCEND_HOME_PATH` to the CANN installation. +When this operator is selected, configuration fails if its MC2 headers are +missing; builds that exclude it do not need this dependency. The dependency +tree is not copied into this repository. + +One native launch selects a layer and drains one source transaction. Receive +capacity follows ordinary CAM: `int(262144 * BATCH_SIZE_FACTOR)`, default 262144 +rows. Only transactions exceeding this capacity use multiple device rounds; +capacity is independent of the MC2 matrix tile size. Callers must not launch +once per receive round. The workspace allocation ABI and Attention Combine +interface are unchanged. Both quantization plans use SwiGLU. W4A8 applies the +model `swiglu_limit` in both ACT paths as +`SiLU(min(gate, L)) * clamp(up, -L, L)`; zero, the default, disables clipping. +W8A8 does not apply `swiglu_limit`, matching the Layered W8A8 path. + +For W4A8 intermediate widths up to 3072, ACT directly uses the installed MC2 +W4A8 two-stage UB epilogue and expert-group synchronization. Wider supported +W4A8 rows and W8A8 use bounded-column activation. Return-address caching is +bounded independently of receive capacity. Large capacity reserves HBM scratch +without computing padded rows; for H4096/I2048 the default capacity needs about +12.5 GiB per FFN executor, excluding weights and communication windows. This +includes a 2 GiB buffer for BF16 output staging. + ## Default Behavior Ascend custom ops build by default. Set `AFD_BUILD_ASCEND_OPS=0` only when diff --git a/csrc/npu/ascend_kernels/afd_mega_moe_comm/op_api/aclnn_afd_mega_moe_comm.cpp b/csrc/npu/ascend_kernels/afd_mega_moe_comm/op_api/aclnn_afd_mega_moe_comm.cpp new file mode 100644 index 000000000..c552024f7 --- /dev/null +++ b/csrc/npu/ascend_kernels/afd_mega_moe_comm/op_api/aclnn_afd_mega_moe_comm.cpp @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +#include "aclnn_afd_mega_moe_comm.h" +#include "aclnnInner_afd_mega_moe_comm.h" +extern "C" void __attribute__((weak)) NnopbaseSetHcclServerType(void *, int32_t); +extern "C" aclnnStatus aclnnAfdMegaMoeCommGetWorkspaceSize( + const aclTensor *x, const aclTensor *expertIds, const aclTensor *expertScales, + const aclTensor *state, const aclIntArray *config, char *hcclGroupName, + const aclTensor *combined, const aclTensor *completion, + uint64_t *workspaceSize, aclOpExecutor **executor) { + return aclnnInnerAfdMegaMoeCommGetWorkspaceSize(x, expertIds, expertScales, state, + config, hcclGroupName, combined, completion, workspaceSize, executor); +} +extern "C" aclnnStatus aclnnAfdMegaMoeComm(void *workspace, uint64_t workspaceSize, + aclOpExecutor *executor, aclrtStream stream) { + if (NnopbaseSetHcclServerType) NnopbaseSetHcclServerType(executor, 1); + return aclnnInnerAfdMegaMoeComm(workspace, workspaceSize, executor, stream); +} diff --git a/csrc/npu/ascend_kernels/afd_mega_moe_comm/op_api/aclnn_afd_mega_moe_comm.h b/csrc/npu/ascend_kernels/afd_mega_moe_comm/op_api/aclnn_afd_mega_moe_comm.h new file mode 100644 index 000000000..0a036ee8d --- /dev/null +++ b/csrc/npu/ascend_kernels/afd_mega_moe_comm/op_api/aclnn_afd_mega_moe_comm.h @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +#pragma once +#include "aclnn/aclnn_base.h" +extern "C" { +__attribute__((visibility("default"))) aclnnStatus aclnnAfdMegaMoeCommGetWorkspaceSize( + const aclTensor *x, const aclTensor *expertIds, const aclTensor *expertScales, + const aclTensor *state, const aclIntArray *config, char *hcclGroupName, + const aclTensor *combined, const aclTensor *completion, + uint64_t *workspaceSize, aclOpExecutor **executor); +__attribute__((visibility("default"))) aclnnStatus aclnnAfdMegaMoeComm( + void *workspace, uint64_t workspaceSize, aclOpExecutor *executor, aclrtStream stream); +} diff --git a/csrc/npu/ascend_kernels/afd_mega_moe_comm/op_host/afd_mega_moe_comm.cpp b/csrc/npu/ascend_kernels/afd_mega_moe_comm/op_host/afd_mega_moe_comm.cpp new file mode 100644 index 000000000..f529558c4 --- /dev/null +++ b/csrc/npu/ascend_kernels/afd_mega_moe_comm/op_host/afd_mega_moe_comm.cpp @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +#include +#include +#include "register/op_def_registry.h" +#include "tiling/platform/platform_ascendc.h" +#include "tiling/hccl/hccl_tiling.h" +#include "../op_kernel/afd_mega_moe_comm_tiling.h" +#include "../op_kernel/afd_pull_layout.h" +#include "ops_log.h" +namespace optiling { +static ge::graphStatus MegaCommTiling(gert::TilingContext *ctx) { + const auto attrs = ctx->GetAttrs(); + const auto config = attrs->GetListInt(0); + if (!config || config->GetSize() != 11) return ge::GRAPH_FAILED; + const auto c = config->GetData(); + if ((c[0] != 0 && c[0] != 1 && c[0] != 4) || c[8] <= 0 || c[1] % c[8] || c[9] < 0 || + c[7] < 0 || c[7] >= c[4] + c[5] || c[10] < 0 || c[10] >= 65536 || + ((c[0] == 1 || c[0] == 4) && (c[7] >= c[5] || c[9] > c[1] / c[8]))) return ge::GRAPH_FAILED; + Cam::Pull::Shape shape{uint64_t(c[1] / c[8]), uint64_t(c[2]), uint64_t(c[3]), + uint64_t(c[4]), uint64_t(c[5]), uint64_t(c[6]), uint64_t(c[8])}; + Cam::Pull::Layout layout{}; + auto platform = platform_ascendc::PlatformAscendC(ctx->GetPlatformInfo()); + const auto aiv = platform.GetCoreNumAiv(); + uint64_t ub = 0; + platform.GetCoreMemSize(platform_ascendc::CoreMemType::UB, ub); + if (!aiv || aiv > Cam::Pull::MAX_AIVS || !Cam::Pull::MakeLayout(shape, layout) || + layout.bytes > GetMaxWindowSize() || Cam::Pull::TransportUbBytes(shape) > ub || + (c[0] == 1 && Cam::Pull::PublisherUbBytes(shape, aiv) + Cam::Pull::TransportUbBytes(shape) > ub) || + (c[0] == 4 && Cam::Pull::CombineUbBytes(shape, aiv) > ub)) + return ge::GRAPH_FAILED; + auto data = ctx->GetTilingData(); + std::memset(data, 0, sizeof(*data)); + std::copy(c, c + 11, data->config); + data->aivNum = aiv; data->ubSize = ub; data->windowSize = GetMaxWindowSize(); + AscendC::Mc2CcTilingConfig comm(attrs->GetAttrPointer(1), 8U, + "AlltoAll=level0:fullmesh;level1:pairwise"); + comm.GetTiling(data->mc2InitTiling); comm.GetTiling(data->mc2CcTiling1); + ctx->SetBlockDim(aiv); ctx->SetTilingKey(1); ctx->SetScheduleMode(1); + ctx->GetWorkspaceSizes(1)[0] = 16 * 1024 * 1024; + return ge::GRAPH_SUCCESS; +} +} +namespace ops { +class AfdMegaMoeComm : public OpDef { +public: + explicit AfdMegaMoeComm(const char *name) : OpDef(name) { + Input("x").ParamType(REQUIRED).DataType({ge::DT_BF16}).Format({ge::FORMAT_ND}); + Input("expertIds").ParamType(REQUIRED).DataType({ge::DT_INT32}).Format({ge::FORMAT_ND}); + Input("expertScales").ParamType(REQUIRED).DataType({ge::DT_FLOAT}).Format({ge::FORMAT_ND}); + Input("state").ParamType(REQUIRED).DataType({ge::DT_FLOAT16}).Format({ge::FORMAT_ND}); + Output("combined").ParamType(REQUIRED).DataType({ge::DT_BF16}).Format({ge::FORMAT_ND}); + Output("completion").ParamType(REQUIRED).DataType({ge::DT_INT64}).Format({ge::FORMAT_ND}); + Attr("config").ListInt(); Attr("hccl_group_name").String(); + SetInferShape([](gert::InferShapeContext *ctx) { + const auto c = ctx->GetAttrs()->GetListInt(0)->GetData(); + const int64_t sizes[] = {c[0] == 4 ? std::max(1, c[9]) * c[2] : 1, 1}; + for (uint32_t j = 0; j < 2; ++j) { + auto shape = ctx->GetOutputShape(j); shape->SetDimNum(1); shape->SetDim(0, sizes[j]); + } + return ge::GRAPH_SUCCESS; + }); + SetInferDataType([](gert::InferDataTypeContext *ctx) { + const ge::DataType types[] = {ge::DT_BF16,ge::DT_INT64}; + for (uint32_t j = 0; j < 2; ++j) ctx->SetOutputDataType(j, types[j]); + return ge::GRAPH_SUCCESS; + }); + AICore().AddConfig("ascend910_93"); AICore().SetTiling(optiling::MegaCommTiling); + MC2().HcclGroup({"hccl_group_name"}); + } +}; +OP_ADD(AfdMegaMoeComm); +} diff --git a/csrc/npu/ascend_kernels/afd_mega_moe_comm/op_kernel/afd_mega_moe_comm.cpp b/csrc/npu/ascend_kernels/afd_mega_moe_comm/op_kernel/afd_mega_moe_comm.cpp new file mode 100644 index 000000000..3ef0b4bc3 --- /dev/null +++ b/csrc/npu/ascend_kernels/afd_mega_moe_comm/op_kernel/afd_mega_moe_comm.cpp @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +#include "kernel_operator.h" +#include "afd_mega_moe_comm_tiling.h" +#include "afd_pull_protocol.h" + +namespace AfdMegaComm { +using namespace AscendC; +using Cam::Pull::Fence; + +// Publish only to the local A window. Statistics retain the existing Combine prefix layout. +__aicore__ inline void Publish(GM_ADDR x, GM_ADDR ids, GM_ADDR state, + const Cam::AfdMegaMoeCommTilingData &t, TPipe &pipe, + Cam::Pull::Transport &control, const Cam::Pull::Layout &layout) { + const auto c = t.config; + const uint32_t id = GetBlockIdx(), cores = t.aivNum, h = c[2], k = c[3], experts = c[4] * c[6]; + const uint32_t columns = c[4] + experts, rows = c[9]; + const uint32_t begin = rows / cores * id + (id < rows % cores ? id : rows % cores); + const uint32_t end = begin + rows / cores + (id < rows % cores ? 1 : 0); + const auto local = Cam::Pull::Window(c[7], c[7]); + TBuf statsBuf, offsetsBuf, idsBuf, mathBuf; + pipe.InitBuffer(statsBuf, Cam::Pull::PublisherStatsBytes( + {uint64_t(c[1] / c[8]), h, k, uint64_t(c[4]), uint64_t(c[5]), uint64_t(c[6]), uint64_t(c[8])}, cores)); + pipe.InitBuffer(offsetsBuf, 2 * Cam::Pull::Align(4ULL * experts)); + pipe.InitBuffer(idsBuf, Cam::Pull::Align(4ULL * k)); + pipe.InitBuffer(mathBuf, 11ULL * h + 128); + auto stats = statsBuf.Get(); + auto starts = offsetsBuf.Get(); + auto prefixes = offsetsBuf.GetWithOffset(experts, Cam::Pull::Align(4ULL * experts)); + auto routes = idsBuf.Get(); + auto input = mathBuf.Get(); + auto value = mathBuf.GetWithOffset(h, 2 * h); + auto temp = mathBuf.GetWithOffset(h, 6 * h); + auto quant = mathBuf.GetWithOffset(h + 32, 10 * h); + auto maximum = mathBuf.GetWithOffset(8, 11 * h + 32); + GlobalTensor globalStats; + GlobalTensor globalIds; + GlobalTensor globalX; + GlobalTensor payload; + globalStats.SetGlobalBuffer(reinterpret_cast<__gm__ uint32_t *>(local)); + globalIds.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(ids)); + globalX.SetGlobalBuffer(reinterpret_cast<__gm__ bfloat16_t *>(x)); + payload.SetGlobalBuffer(reinterpret_cast<__gm__ int8_t *>(local + layout.payload)); + Duplicate(stats, uint32_t(0), columns); + Fence(); + int64_t error = 0; + for (uint32_t token = begin; token < end; ++token) { + DataCopyPad(routes, globalIds[token * k], {1U, 4U * k, 0U, 0U, 0U}, {false, 0U, 0U, 0U}); + Fence(); + for (uint32_t j = 0; j < k; ++j) { + const auto expert = routes.GetValue(j); + if (expert < 0 || uint32_t(expert) >= experts) error = -1; + else stats.SetValue(c[4] + expert, stats.GetValue(c[4] + expert) + 1); + } + Fence(); + } + const auto lanes = Cam::Pull::LaneErrorsOffset(c[5] / c[8], control.ReadState(Cam::Pull::MAP_SIZE)); + control.Write(state + (lanes + id * Cam::Pull::WORDS_PER_LINE) * 8, error); + Fence(); + DataCopyPad(globalStats[id * columns], stats, {1U, 4U * columns, 0U, 0U, 0U}); + Fence(); + SyncAll(); + // Each lane owns adjacent expert columns; UB rows are padded independently of GM rows. + const uint32_t firstExpert = id * (experts / cores) + (id < experts % cores ? id : experts % cores); + const uint32_t expertCount = experts / cores + (id < experts % cores); + if (expertCount) { + const uint32_t stride = Cam::Pull::Align32(4U * expertCount) / 4; + DataCopyPad(stats, globalStats[c[4] + firstExpert], + {uint16_t(cores), 4U * expertCount, 4U * (columns - expertCount), 0U, 0U}, + {false, 0U, 0U, 0U}); + Fence(); + for (uint32_t lane = 1; lane < cores; ++lane) + for (uint32_t e = 0; e < expertCount; ++e) { + const auto pos = lane * stride + e; + stats.SetValue(pos, stats.GetValue(pos) + stats.GetValue(pos - stride)); + } + Fence(); + DataCopyPad(globalStats[c[4] + firstExpert], stats, + {uint16_t(cores), 4U * expertCount, 0U, 4U * (columns - expertCount), 0U}); + Fence(); + } + if (id == 0) { + GlobalTensor errors; + errors.SetGlobalBuffer(reinterpret_cast<__gm__ int64_t *>(state) + lanes); + DataCopy(stats.ReinterpretCast(), errors, cores * Cam::Pull::WORDS_PER_LINE); + Fence(); + for (uint32_t lane = 0; lane < cores; ++lane) { + const auto failure = stats.ReinterpretCast().GetValue(lane * Cam::Pull::WORDS_PER_LINE); + if (!error && failure) error = failure; + } + if (error) control.WriteState(Cam::Pull::FAILURE, error); + } + SyncAll(); + error = control.ReadState(Cam::Pull::FAILURE); + DataCopyPad(stats, globalStats[(cores - 1) * columns + c[4]], + {1U, 4U * experts, 0U, 0U, 0U}, {false, 0U, 0U, 0U}); + if (id) DataCopyPad(starts, globalStats[(id - 1) * columns + c[4]], + {1U, 4U * experts, 0U, 0U, 0U}, {false, 0U, 0U, 0U}); + Fence(); + uint32_t prefix = 0; + for (uint32_t e = 0; e < experts; ++e) { + prefixes.SetValue(e, prefix); + if (!id) starts.SetValue(e, 0); + prefix += stats.GetValue(e); + } + for (uint32_t token = begin; !error && token < end; ++token) { + DataCopyPad(routes, globalIds[token * k], {1U, 4U * k, 0U, 0U, 0U}, {false, 0U, 0U, 0U}); + DataCopy(input, globalX[token * h], h); + Fence(); + // Preserve the original Dispatch's FP32 -> INT32 -> FP16 -> INT8 rounding. + Cast(value, input, RoundMode::CAST_NONE, h); + PipeBarrier(); + Abs(temp, value, h); + PipeBarrier(); + ReduceMax(maximum, temp, temp, h, false); + PipeBarrier(); + Duplicate(temp, 127.0f, 1); + PipeBarrier(); + Div(temp, temp, maximum, 1); + Fence(); + const float multiplier = temp.GetValue(0); + Fence(); + Muls(value, value, multiplier, h); + PipeBarrier(); + Cast(value.ReinterpretCast(), value, RoundMode::CAST_RINT, h); + PipeBarrier(); + SetDeqScale(static_cast(1.0f)); + PipeBarrier(); + Cast(value.ReinterpretCast(), value.ReinterpretCast(), RoundMode::CAST_ROUND, h); + PipeBarrier(); + Cast(quant, value.ReinterpretCast(), RoundMode::CAST_TRUNC, h); + Fence(); + quant.ReinterpretCast().SetValue(h / sizeof(float), 1.0f / multiplier); + Fence(); + Fence(); + for (uint32_t j = 0; j < k; ++j) { + const uint32_t expert = routes.GetValue(j); + const uint32_t row = prefixes.GetValue(expert) + starts.GetValue(expert); + DataCopy(payload[row * layout.rowStride], quant, h + 32); + Fence(); + starts.SetValue(expert, starts.GetValue(expert) + 1); + } + Fence(); + Fence(); + } + SyncAll(); + if (id == 0) { + const auto seq = control.ReadState(Cam::Pull::ACTIVE_SEQUENCE) + 1; + control.WriteState(Cam::Pull::ACTIVE_SEQUENCE, seq); + auto counts = offsetsBuf.Get(); + for (uint32_t e = 0; e < experts; ++e) counts.SetValue(e, stats.GetValue(e)); + GlobalTensor publishedCounts; + publishedCounts.SetGlobalBuffer(reinterpret_cast<__gm__ int64_t *>(local + layout.counts)); + Fence(); + DataCopyPad(publishedCounts, counts, {1U, 8U * experts, 0U, 0U, 0U}); + Fence(); + const int64_t values[] = {seq, c[10], rows, prefix, seq, error}; + for (uint32_t field = Cam::Pull::SEQUENCE; field <= Cam::Pull::ERROR; ++field) + if (field != Cam::Pull::COMMIT) + control.Write(local + layout.header + field * Cam::Pull::ALIGNMENT, values[field - 1]); + control.Write(local + layout.header + Cam::Pull::COMMIT * Cam::Pull::ALIGNMENT, seq); + } +} + +__aicore__ inline bool WaitCombine(GM_ADDR state, const Cam::AfdMegaMoeCommTilingData &t, + const Cam::Pull::Layout &layout, Cam::Pull::Transport &control) { + const auto c = t.config; + const auto local = Cam::Pull::Window(c[7], c[7]); + if (GetBlockIdx() == 0) while (true) { + auto error = control.ReadState(Cam::Pull::FAILURE); + if (!error) error = control.Read(local + layout.header + Cam::Pull::ERROR * Cam::Pull::ALIGNMENT); + bool ready = true; + for (uint32_t f = 0; f < c[4]; ++f) { + if (!error) error = control.Read(local + layout.errors + f * Cam::Pull::ALIGNMENT); + if (!uint32_t(control.Read(local + layout.flags + 4ULL * f))) ready = false; + } + if (error) { + control.WriteState(Cam::Pull::FAILURE, error); + break; + } + if (ready) break; + } + SyncAll(); + return !control.ReadState(Cam::Pull::FAILURE); +} + +// Publish and Combine partition tokens identically; only the last and preceding +// statistics rows are needed to recover each route's returned-row position. +__aicore__ inline void Combine(GM_ADDR ids, GM_ADDR probabilities, GM_ADDR combined, + const Cam::AfdMegaMoeCommTilingData &t, + const Cam::Pull::Layout &layout, TPipe &pipe) { + const auto c = t.config; + const uint32_t id = GetBlockIdx(), cores = t.aivNum, h = c[2], k = c[3], rows = c[9]; + const uint32_t experts = c[4] * c[6], columns = c[4] + experts; + const uint32_t begin = rows / cores * id + (id < rows % cores ? id : rows % cores); + const uint32_t end = begin + rows / cores + (id < rows % cores); + const auto local = Cam::Pull::Window(c[7], c[7]); + pipe.Reset(); + TBuf offsetsBuf, routesBuf, mathBuf; + pipe.InitBuffer(offsetsBuf, 2 * Cam::Pull::Align(4ULL * experts)); + pipe.InitBuffer(routesBuf, 2 * Cam::Pull::Align(4ULL * k)); + pipe.InitBuffer(mathBuf, 14ULL * h); + auto offsets = offsetsBuf.Get(); + auto counts = offsetsBuf.GetWithOffset(experts, Cam::Pull::Align(4ULL * experts)); + auto routes = routesBuf.Get(); + auto weights = routesBuf.GetWithOffset(k, Cam::Pull::Align(4ULL * k)); + auto input = mathBuf.Get(); + auto value = mathBuf.GetWithOffset(h, 2 * h); + auto product = mathBuf.GetWithOffset(h, 6 * h); + auto sum = mathBuf.GetWithOffset(h, 10 * h); + GlobalTensor stats; + GlobalTensor globalIds; + GlobalTensor globalWeights; + GlobalTensor returned, output; + stats.SetGlobalBuffer(reinterpret_cast<__gm__ uint32_t *>(local)); + globalIds.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(ids)); + globalWeights.SetGlobalBuffer(reinterpret_cast<__gm__ float *>(probabilities)); + returned.SetGlobalBuffer(reinterpret_cast<__gm__ bfloat16_t *>(local + layout.returned)); + output.SetGlobalBuffer(reinterpret_cast<__gm__ bfloat16_t *>(combined)); + if (begin < end) { + DataCopyPad(counts, stats[(cores - 1) * columns + c[4]], + {1U, 4U * experts, 0U, 0U, 0U}, {false, 0U, 0U, 0U}); + if (id) DataCopyPad(offsets, stats[(id - 1) * columns + c[4]], + {1U, 4U * experts, 0U, 0U, 0U}, {false, 0U, 0U, 0U}); + Fence(); + uint32_t prefix = 0; + for (uint32_t e = 0; e < experts; ++e) { + offsets.SetValue(e, prefix + (id ? offsets.GetValue(e) : 0)); + prefix += counts.GetValue(e); + } + for (uint32_t token = begin; token < end; ++token) { + DataCopyPad(routes, globalIds[token * k], + {1U, 4U * k, 0U, 0U, 0U}, {false, 0U, 0U, 0U}); + DataCopyPad(weights, globalWeights[token * k], + {1U, 4U * k, 0U, 0U, 0U}, {false, 0U, 0U, 0U}); + Duplicate(sum, 0.0f, h); + Fence(); + for (uint32_t j = 0; j < k; ++j) { + const uint32_t expert = routes.GetValue(j), row = offsets.GetValue(expert); + const float weight = weights.GetValue(j); + DataCopy(input, returned[uint64_t(row) * h], h); + Fence(); + // Keep the existing FP32 multiply/add and top-k accumulation order. + Cast(value, input, RoundMode::CAST_NONE, h); + PipeBarrier(); + Muls(product, value, weight, h); + PipeBarrier(); + Add(sum, sum, product, h); + Fence(); + offsets.SetValue(expert, row + 1); + } + PipeBarrier(); + Cast(input, sum, RoundMode::CAST_RINT, h); + Fence(); + DataCopy(output[uint64_t(token) * h], input, h); + Fence(); + Fence(); + Fence(); + } + } + // All returned rows have been consumed before the single source window is reusable. + SyncAll(); + if (id == 0) { + GlobalTensor flags; + flags.SetGlobalBuffer(reinterpret_cast<__gm__ uint32_t *>(local + layout.flags)); + Duplicate(offsets, uint32_t(0), uint32_t(c[4])); + Fence(); + DataCopyPad(flags, offsets, {1U, uint32_t(4 * c[4]), 0U, 0U, 0U}); + Fence(); + } +} +} // namespace AfdMegaComm + +extern "C" __global__ __aicore__ void afd_mega_moe_comm( + GM_ADDR x, GM_ADDR ids, GM_ADDR probabilities, GM_ADDR state, GM_ADDR combined, + GM_ADDR completion, GM_ADDR workspace, GM_ADDR tiling) { + using namespace AfdMegaComm; + REGISTER_TILING_DEFAULT(Cam::AfdMegaMoeCommTilingData); + REGISTER_TILING_FOR_TILINGKEY("TILING_KEY_VAR == 1", Cam::AfdMegaMoeCommTilingData); + if (TILING_KEY_IS(1)) { + GET_TILING_DATA_WITH_STRUCT(Cam::AfdMegaMoeCommTilingData, t, tiling); + const auto c = t.config; + const Cam::Pull::Shape shape{uint64_t(c[1] / c[8]), uint64_t(c[2]), uint64_t(c[3]), + uint64_t(c[4]), uint64_t(c[5]), uint64_t(c[6]), uint64_t(c[8])}; + Cam::Pull::Layout layout{}; + Cam::Pull::MakeLayout(shape, layout); + TPipe pipe; + { + Cam::Pull::Transport transport(state, shape, c[7], t.aivNum, pipe); + SyncAll(); + if (transport.ReadState(Cam::Pull::FAILURE)) {} + else if (c[0] == 0) transport.Initialize(); + else if (c[0] == 1) Publish(x, ids, state, t, pipe, transport, layout); + else if (c[0] == 4 && WaitCombine(state, t, layout, transport)) { + Combine(ids, probabilities, combined, t, layout, pipe); + } + } + SyncAll(); + // Release phase buffers before allocating the final status transport. + pipe.Reset(); + if (GetBlockIdx() == 0) { + Cam::Pull::Transport final(state, shape, c[7], t.aivNum, pipe); + final.Write(completion, final.ReadState(Cam::Pull::FAILURE)); + } + } +} diff --git a/csrc/npu/ascend_kernels/afd_mega_moe_comm/op_kernel/afd_mega_moe_comm_tiling.h b/csrc/npu/ascend_kernels/afd_mega_moe_comm/op_kernel/afd_mega_moe_comm_tiling.h new file mode 100644 index 000000000..ec0304778 --- /dev/null +++ b/csrc/npu/ascend_kernels/afd_mega_moe_comm/op_kernel/afd_mega_moe_comm_tiling.h @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +#pragma once +#include "kernel_tiling/kernel_tiling.h" +namespace Cam { +// mode,maxSeqLen,H,K,F,A,E,rank,TP,rows,layer. +struct AfdMegaMoeCommTilingData { + Mc2InitTiling mc2InitTiling; + Mc2CcTiling mc2CcTiling1; + Mc2CcTiling mc2CcTiling2; + int64_t config[11]; + uint32_t aivNum; + uint64_t ubSize, windowSize; +}; +} diff --git a/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_api/aclnn_afd_mega_moe_pull_native.cpp b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_api/aclnn_afd_mega_moe_pull_native.cpp new file mode 100644 index 000000000..216c798e2 --- /dev/null +++ b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_api/aclnn_afd_mega_moe_pull_native.cpp @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +#include "aclnn_afd_mega_moe_pull_native.h" +#include "aclnnInner_afd_mega_moe_pull_native.h" +extern "C" void __attribute__((weak)) NnopbaseSetHcclServerType(void *, int32_t); +extern "C" aclnnStatus aclnnAfdMegaMoePullNativeGetWorkspaceSize( + const aclTensor *anchor, const aclTensor *commArgs, const aclTensor *layerMap, + const aclTensor *scratch, const aclTensorList *w13, const aclTensorList *w2, + const aclTensorList *s13, const aclTensorList *s2, const aclTensorList *b13, + const aclTensorList *b2, const aclIntArray *config, char *hcclGroupName, double routedScale, + double swigluLimit, const aclTensor *completion, uint64_t *workspaceSize, + aclOpExecutor **executor) { + return aclnnInnerAfdMegaMoePullNativeGetWorkspaceSize( + anchor, commArgs, layerMap, scratch, w13, w2, s13, s2, b13, b2, config, hcclGroupName, + routedScale, swigluLimit, completion, workspaceSize, executor); +} +extern "C" aclnnStatus aclnnAfdMegaMoePullNative(void *workspace, uint64_t workspaceSize, + aclOpExecutor *executor, aclrtStream stream) { + if (NnopbaseSetHcclServerType) + NnopbaseSetHcclServerType(executor, 1); + return aclnnInnerAfdMegaMoePullNative(workspace, workspaceSize, executor, stream); +} diff --git a/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_api/aclnn_afd_mega_moe_pull_native.h b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_api/aclnn_afd_mega_moe_pull_native.h new file mode 100644 index 000000000..7174ab8ce --- /dev/null +++ b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_api/aclnn_afd_mega_moe_pull_native.h @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +#pragma once +#include "aclnn/aclnn_base.h" +#ifdef __cplusplus +extern "C" { +#endif +__attribute__((visibility("default"))) aclnnStatus aclnnAfdMegaMoePullNativeGetWorkspaceSize( + const aclTensor *anchor, const aclTensor *commArgs, const aclTensor *layerMap, + const aclTensor *scratch, const aclTensorList *w13, const aclTensorList *w2, + const aclTensorList *s13, const aclTensorList *s2, const aclTensorList *b13, + const aclTensorList *b2, const aclIntArray *config, char *hcclGroupName, double routedScale, + double swigluLimit, const aclTensor *completion, uint64_t *workspaceSize, + aclOpExecutor **executor); +__attribute__((visibility("default"))) aclnnStatus aclnnAfdMegaMoePullNative(void *workspace, + uint64_t workspaceSize, + aclOpExecutor *executor, + aclrtStream stream); +#ifdef __cplusplus +} +#endif diff --git a/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_host/afd_mega_moe_pull_native.cpp b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_host/afd_mega_moe_pull_native.cpp new file mode 100644 index 000000000..669ae6c7d --- /dev/null +++ b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_host/afd_mega_moe_pull_native.cpp @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +#include +#include +#include +#include "register/op_def_registry.h" +#include "tiling/platform/platform_ascendc.h" +#include "tiling/hccl/hccl_tiling.h" +#include "../op_kernel/afd_mega_moe_pull_native_tiling.h" +#include "../op_kernel/afd_pull_layout.h" +#include "ops_log.h" + +namespace optiling { +static ge::graphStatus NativeMegaTiling(gert::TilingContext *ctx) { + const auto attrs = ctx->GetAttrs(); + const auto config = attrs->GetListInt(0); + if (config == nullptr || config->GetSize() != 14) return ge::GRAPH_FAILED; + const auto c = config->GetData(); + for (uint32_t j = 0; j < 14; ++j) + if (c[j] <= 0 || c[j] > 262144) return ge::GRAPH_FAILED; + if (c[8] != c[4] + c[5] || c[7] < c[5] || c[7] >= c[8] || + c[9] == 0 || c[0] % c[9] || c[5] % c[9] || + c[2] % 256 || c[2] > 8192 || c[6] > 128 || + c[10] > Cam::NATIVE_MAX_ROWS || (c[11] != 1 && c[11] != 2) || c[12] != 1 || + !std::isfinite(*attrs->GetFloat(2))) return ge::GRAPH_FAILED; + // Zero keeps clipping disabled, matching the layered W4A8 operator. + const float *swigluLimitPtr = attrs->GetFloat(3); + const float swigluLimit = swigluLimitPtr != nullptr ? *swigluLimitPtr : 0.0f; + if (!std::isfinite(swigluLimit) || swigluLimit < 0.0f) return ge::GRAPH_FAILED; + Cam::Pull::Shape shape{uint64_t(c[0] / c[9]), uint64_t(c[1]), uint64_t(c[3]), + uint64_t(c[4]), uint64_t(c[5]), uint64_t(c[6]), uint64_t(c[9])}; + Cam::Pull::Layout layout{}; + if (!Cam::Pull::MakeLayout(shape, layout) || layout.bytes > GetMaxWindowSize()) + return ge::GRAPH_FAILED; + auto platform = platform_ascendc::PlatformAscendC(ctx->GetPlatformInfo()); + const auto aic = platform.GetCoreNumAic(), aiv = platform.GetCoreNumAiv(); + uint64_t ub = 0; + platform.GetCoreMemSize(platform_ascendc::CoreMemType::UB, ub); + if (aic == 0 || aiv != 2 * aic || aiv > Cam::NATIVE_MAX_AIVS || + Cam::Pull::TransportUbBytes(shape) > ub || + Cam::Pull::Align(8 * Cam::Pull::MetadataWords(shape.tp, shape.experts)) + + 10ULL * c[1] + 1280 > ub || + 184ULL * 1024 + 8ULL * Cam::NATIVE_METADATA_ROWS > ub) return ge::GRAPH_FAILED; + // MC2 full-row ACT uses 54 bytes per intermediate column plus per-core scales. + if (c[11] == 1 && c[2] <= 3072 && 54ULL * c[2] + 256 + 4ULL * ((c[10] + aiv - 1) / aiv) > ub) + return ge::GRAPH_FAILED; + const Cam::NativeMegaWorkspace w(c[10], c[1], c[2], c[6], c[9]); + if (ctx->GetInputShape(3)->GetStorageShape().GetDim(0) != int64_t(w.bytes)) + return ge::GRAPH_FAILED; + auto data = ctx->GetTilingData(); + std::memset(data, 0, sizeof(*data)); + std::copy(c, c + 14, data->config); + data->aicNum = aic; + data->aivNum = aiv; + // All AIVs take part in MC2 dispatch/MSD, activation and paired output. + data->commAivNum = 0; + data->layerMapSize = ctx->GetInputShape(2)->GetStorageShape().GetDim(0); + if (data->layerMapSize == 0 || data->layerMapSize > 65536) return ge::GRAPH_FAILED; + data->routedScale = *attrs->GetFloat(2); + data->swigluLimit = swigluLimit; + AscendC::Mc2CcTilingConfig comm(attrs->GetAttrPointer(1), 8U, + "AlltoAll=level0:fullmesh;level1:pairwise"); + comm.GetTiling(data->mc2InitTiling); + comm.GetTiling(data->mc2CcTiling1); + ctx->SetBlockDim(aic); + ctx->SetTilingKey(c[11]); + ctx->SetScheduleMode(1); + ctx->GetWorkspaceSizes(1)[0] = 16 * 1024 * 1024; + return ge::GRAPH_SUCCESS; +} +} + +namespace ops { +class AfdMegaMoePullNative : public OpDef { +public: + explicit AfdMegaMoePullNative(const char *name) : OpDef(name) { + Input("anchor").ParamType(REQUIRED).DataType({ge::DT_BF16, ge::DT_BF16}).Format({ge::FORMAT_ND, ge::FORMAT_ND}); + Input("commArgs").ParamType(REQUIRED).DataType({ge::DT_FLOAT16, ge::DT_FLOAT16}).Format({ge::FORMAT_ND, ge::FORMAT_ND}); + Input("layerMap").ParamType(REQUIRED).DataType({ge::DT_INT64, ge::DT_INT64}).Format({ge::FORMAT_ND, ge::FORMAT_ND}); + Input("scratch").ParamType(REQUIRED).DataType({ge::DT_UINT8, ge::DT_UINT8}).Format({ge::FORMAT_ND, ge::FORMAT_ND}); + for (const char *w : {"w13", "w2"}) + Input(w).ParamType(DYNAMIC).DataType({ge::DT_INT32, ge::DT_INT8}).Format({ge::FORMAT_FRACTAL_NZ, ge::FORMAT_FRACTAL_NZ}); + Input("s13").ParamType(DYNAMIC).DataType({ge::DT_INT64, ge::DT_FLOAT}).Format({ge::FORMAT_ND, ge::FORMAT_ND}); + Input("s2").ParamType(DYNAMIC).DataType({ge::DT_INT64, ge::DT_BF16}).Format({ge::FORMAT_ND, ge::FORMAT_ND}); + for (const char *b : {"b13", "b2"}) + Input(b).ParamType(DYNAMIC).DataType({ge::DT_FLOAT, ge::DT_FLOAT}).Format({ge::FORMAT_ND, ge::FORMAT_ND}); + Output("completion").ParamType(REQUIRED).DataType({ge::DT_INT64, ge::DT_INT64}).Format({ge::FORMAT_ND, ge::FORMAT_ND}); + Attr("config").ListInt(); + Attr("hccl_group_name").String(); + Attr("routed_scale").Float(); + Attr("swiglu_limit").Float(); + SetInferShape([](gert::InferShapeContext *ctx) { + auto shape = ctx->GetOutputShape(0); + shape->SetDimNum(1); shape->SetDim(0, 1); + return ge::GRAPH_SUCCESS; + }); + SetInferDataType([](gert::InferDataTypeContext *ctx) { + ctx->SetOutputDataType(0, ge::DT_INT64); return ge::GRAPH_SUCCESS; + }); + AICore().AddConfig("ascend910_93"); + AICore().SetTiling(optiling::NativeMegaTiling); + MC2().HcclGroup({"hccl_group_name"}); + } +}; +OP_ADD(AfdMegaMoePullNative); +} diff --git a/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_patch_activation.py b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_patch_activation.py new file mode 100644 index 000000000..b6eec2a72 --- /dev/null +++ b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_patch_activation.py @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Generate a Layered-compatible MC2 activation from the installed CANN header.""" + +import re +import sys +from pathlib import Path + + +def patched_activation(source: str) -> str: + source = source.replace( + '"../template_linear_algebra_v2/', '"template_linear_algebra_v2/' + ) + for header in ("get_tensor_addr.hpp", "gated_activation.hpp"): + source = source.replace(f'"{header}"', f'"utils/{header}"') + + source, multipliers = re.subn( + r"127\.0f / rowAbsMax", + "1.0f / (rowAbsMax / 127.0f)", + source, + ) + source, legacy_multipliers = re.subn( + r"127\.f / GMubDequantScale", + "(GMubDequantScale > 0.0f ? 1.0f / (GMubDequantScale / 127.f) : 0.0f)", + source, + ) + if multipliers + legacy_multipliers != 1: + raise ValueError("unsupported CANN activation scale calculation") + + pattern = ( + r"AscendC::Cast\(ubQuantS32, (\w+), AscendC::RoundMode::CAST_RINT, (\w+)\);" + r"\s*AscendC::PipeBarrier\(\);" + r"\s*AscendC::SetDeqScale\(static_cast\(1(?:\.0)?\)\);" + r"\s*AscendC::Cast\(ubQuantF16, ubQuantS32, " + r"AscendC::RoundMode::CAST_RINT, \2\);" + ) + source, casts = re.subn( + pattern, + r"AscendC::Cast(ubQuantF16, \1, AscendC::RoundMode::CAST_RINT, \2);", + source, + ) + if casts != 1: + raise ValueError("unsupported CANN activation quantization sequence") + source = re.sub(r"^\s*auto ubQuantS32 = .*?;\n", "\n", source, flags=re.MULTILINE) + return source + + +if __name__ == "__main__": + Path(sys.argv[2]).write_text(patched_activation(Path(sys.argv[1]).read_text())) diff --git a/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_plan_w4a8.h b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_plan_w4a8.h new file mode 100644 index 000000000..f6efc8554 --- /dev/null +++ b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_plan_w4a8.h @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +/** + * W4A8 execution plan for the native MegaMoE pull operator. + * + * Everything the pipeline needs that depends on the weight/activation dtype + * lives here, so afd_mega_moe_pull_pipeline.h stays dtype-agnostic and a new + * quantization only adds a sibling plan header. + * + * Include order: this header assumes kernel_operator.h and the installed MC2 + * Catlass headers are already included, plus the shared AfdNativeMega aliases + * (Arch, RowMajor, Nz, VectorLayout) in afd_mega_moe_pull_native.cpp. + */ +#ifndef AFD_MEGA_MOE_PLAN_W4A8_H +#define AFD_MEGA_MOE_PLAN_W4A8_H + +namespace AfdNativeMega { + +struct W4A8Plan { + static constexpr bool kPackedInt4 = true; + static constexpr uint32_t kRows = 2; + using OutputElement = half; + using OutputScale = float; + // Packed INT4 weights. An INT8 activation row of h elements is MSD-split + // into 2*h INT4 halves, so GMM1 rows are twice the logical row count. + using Element = AscendC::int4b_t; + // Per-channel weight scales arrive as INT64-encoded floats. + using WeightScale = uint64_t; + + using A = Catlass::Gemm::GemmType; + using B = Catlass::Gemm::GemmType; + using C = Catlass::Gemm::GemmType; + + using Copy = Catlass::Gemm::Tile::QuantTileCopy; + using Policy = Catlass::Gemm::MmadAtlasA2W4A4MatmulPerChannelDequant< + 1, 2, 2, 2, 1, false, true>; + using Mmad = Catlass::Gemm::Block::BlockMmad, + Catlass::GemmShape<128, 256, 256>, A, B, C, void, Copy>; + using Scheduler = Catlass::Gemm::Block::GemmIdentityBlockSwizzle<9, 1>; + + // Installed MC2 W4A8 epilogue, patched to match Layered's rounding. + using ActOutput = Catlass::Gemm::GemmType; + using ActScale = Catlass::Gemm::GemmType; + using ActCopy = Catlass::Epilogue::Tile::TileCopy, ActScale, ActOutput>; + using ActMuls = Catlass::Epilogue::Tile::TileElemWiseMuls, 0>; + using NativeActivation = Catlass::Epilogue::Block::BlockEpilogue< + Catlass::Epilogue::EpilogueAtlasA2W4A8PostPerTokenDequantSwigluQuant<2>, + C, ActScale, ActOutput, ActMuls, ActCopy>; +}; + +} // namespace AfdNativeMega +#endif // AFD_MEGA_MOE_PLAN_W4A8_H diff --git a/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_plan_w8a8.h b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_plan_w8a8.h new file mode 100644 index 000000000..4a9370498 --- /dev/null +++ b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_plan_w8a8.h @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +#ifndef AFD_MEGA_MOE_PLAN_W8A8_H +#define AFD_MEGA_MOE_PLAN_W8A8_H + +namespace AfdNativeMega { +struct W8A8Plan { + static constexpr bool kPackedInt4 = false; + static constexpr uint32_t kRows = 1; + using Element = int8_t; + using OutputElement = int32_t; + using WeightScale = float; + using OutputScale = bfloat16_t; + using A = Catlass::Gemm::GemmType; + using B = Catlass::Gemm::GemmType; + using C = Catlass::Gemm::GemmType; + // Keep the integer accumulator until Vector applies Layered's FP32 scales. + using Policy = Catlass::Gemm::MmadAtlasA2PreloadAsyncWithCallback< + 1, 2, 2, 2, 1, false, true>; + using Mmad = Catlass::Gemm::Block::BlockMmad, + Catlass::GemmShape<128, 256, 128>, A, B, C>; + using Scheduler = Catlass::Gemm::Block::GemmIdentityBlockSwizzle<9, 1>; +}; +} // namespace AfdNativeMega +#endif diff --git a/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_pull_native.cpp b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_pull_native.cpp new file mode 100644 index 000000000..c27d93502 --- /dev/null +++ b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_pull_native.cpp @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +#include "kernel_operator.h" +#include "lib/matmul_intf.h" // CANN generated MIX wrapper calls clearWorkspace. +#include "afd_mega_moe_pull_native_tiling.h" +#include "afd_pull_protocol.h" +#include "layered/layered_tensor_addr.h" + +// Use the installed MC2 implementation; do not vendor its template dependency tree. +#if defined(AFD_CANN_CATLASS_92) +#include "catlass/arch/resource.hpp" +#include "catlass/gemm/block/block_mmad.hpp" +#include "catlass/gemm/block/block_swizzle.hpp" +#include "catlass/epilogue/tile/tile_copy.hpp" +#include "catlass/epilogue/tile/tile_elemwise_muls.hpp" +#include "catlass/epilogue/block/block_epilogue.hpp" +#else +#include "template_linear_algebra_v2/arch/resource.hpp" +#include "template_linear_algebra_v2/gemm/block/block_mmad.hpp" +#include "template_linear_algebra_v2/gemm/block/block_swizzle.hpp" +#include "template_linear_algebra_v2/epilogue/tile/tile_copy.hpp" +#include "template_linear_algebra_v2/epilogue/tile/tile_elemwise_muls.hpp" +#include "template_linear_algebra_v2/epilogue/block/block_epilogue.hpp" +#endif +#include "utils/block_mmad_w4a4.hpp" +// The installed MC2 epilogue uses unqualified AscendC names inside this namespace. +namespace Catlass::Epilogue::Block { using namespace AscendC; } +#include "afd_mega_moe_activation.h" +#include "afd_swiglu_limit.h" + +namespace AfdNativeMega { +using namespace AscendC; +// Shared Catlass aliases. The per-quantization plan header below consumes these. +using Arch = Catlass::Arch::AtlasA2; +using Resource = Catlass::Arch::Resource; +using RowMajor = Catlass::layout::RowMajor; +using Nz = Catlass::layout::zN; +using VectorLayout = Catlass::layout::VectorLayout; +using Coord = Catlass::GemmCoord; +} // namespace AfdNativeMega + +#include "afd_mega_moe_plan_w4a8.h" +#include "afd_mega_moe_plan_w8a8.h" + +namespace AfdNativeMega { +constexpr uint32_t CUBE_M = 128, CUBE_N = 256; +constexpr uint32_t FLAG_SETS = 15, GMM1_READY = 10, ACT_READY = 11; +constexpr uint32_t MAX_EXPERTS = 128; +constexpr uint32_t ROUTE_UB = 184 * 1024; + +template __aicore__ inline void Fence() { + constexpr auto id = EVENT_ID7; + SetFlag(id); + WaitFlag(id); +} +__aicore__ inline int64_t Load(GM_ADDR address, uint32_t index = 0) { + GlobalTensor gm; + gm.SetGlobalBuffer(reinterpret_cast<__gm__ int64_t *>(address) + index); + __asm__ __volatile__("" ::: "memory"); + DataCacheCleanAndInvalid(gm); + __asm__ __volatile__("" ::: "memory"); + return *(reinterpret_cast<__gm__ volatile int64_t *>(address) + index); +} +__aicore__ inline Cam::Pull::Shape PullShape(const Cam::AfdMegaMoePullNativeTilingData &t) { + const auto c = t.config; + return {uint64_t(c[0] / c[9]), uint64_t(c[1]), uint64_t(c[3]), uint64_t(c[4]), + uint64_t(c[5]), uint64_t(c[6]), uint64_t(c[9])}; +} +// MC2 W4A8 emits activation groups at the exponentially shrinking expert tail. +__aicore__ inline bool SyncExpert(uint32_t expert, uint32_t experts) { + const uint32_t tail = experts - expert; + return (tail & (tail - 1)) == 0; +} +struct ReceiveRound { + uint32_t begin[MAX_EXPERTS + 1]; + uint32_t active[MAX_EXPERTS]; + uint32_t activeCount = 0, expertCount; + uint32_t source, offset, total; + __aicore__ ReceiveRound(GM_ADDR scratch, const Cam::NativeMegaWorkspace &w, uint32_t experts) { + expertCount = experts; + source = Load(scratch + w.batch, 1); + offset = Load(scratch + w.batch, 3); + total = Load(scratch + w.batch, 4) - offset; + begin[0] = 0; + auto counts = reinterpret_cast<__gm__ volatile int64_t *>(scratch + w.counts); + for (uint32_t e = 0; e < experts; ++e) { + // Invalidate once per cache line after the preceding receive round reused it. + if (e % 8 == 0) Load(scratch + w.counts, e); + begin[e + 1] = begin[e] + counts[e]; + activeCount += begin[e + 1] != begin[e]; + } + // Keep dense rounds on the original expert order without writing an index list. + if (activeCount != experts) { + uint32_t index = 0; + for (uint32_t e = 0; e < experts; ++e) { + if (Rows(e)) active[index++] = e; + } + } + } + __aicore__ uint32_t Rows(uint32_t e) const { return begin[e + 1] - begin[e]; } + __aicore__ uint32_t Expert(uint32_t index) const { + return activeCount == expertCount ? index : active[index]; + } +}; + +// Same Catlass block implementation, preload depth, K shuffle and expert swizzle as MC2. +// The plan supplies the operand element type, GEMM tiling and scheduler; the +// receive/round/flag structure below is identical for every quantization. +template +__aicore__ inline void Compute(GM_ADDR scratch, const Cam::NativeMegaWorkspace &w, + const Cam::AfdMegaMoePullNativeTilingData &t, const ReceiveRound &round, GM_ADDR weights, + GM_ADDR scales, int64_t layer, bool second, Resource &resource) { + using Element = typename Plan::Element; + const uint32_t k = second ? t.config[2] : t.config[1]; + const uint32_t n = second ? t.config[1] : 2 * t.config[2]; + GlobalTensor a, b; + GlobalTensor scale; + GlobalTensor out; + a.SetGlobalBuffer(reinterpret_cast<__gm__ Element *>(scratch + (second ? w.a2 : w.a1))); + b.SetGlobalBuffer(GetLayerTensorAddr(layer, t.config[13], weights)); + scale.SetGlobalBuffer( + GetLayerTensorAddr(layer, t.config[13], scales)); + out.SetGlobalBuffer(reinterpret_cast<__gm__ typename Plan::OutputElement *>(scratch + (second ? w.c2 : w.c1))); + const auto layoutB = Nz::MakeLayout(k, n); + typename Plan::Mmad mm(resource); + typename Plan::Scheduler scheduler; + uint32_t startCore = 0, syncGroup = 0; + for (uint32_t index = 0; index < round.activeCount; ++index) { + const uint32_t e = round.Expert(index); + const uint32_t rows = Plan::kRows * round.Rows(e); + if (second) { + if (index == 0 || SyncExpert(index - 1, round.activeCount)) CrossCoreWaitFlag<2>(ACT_READY); + } else { + CrossCoreWaitFlag<2>(index / FLAG_SETS); + } + scheduler.Update(Coord{rows, n, k}, Catlass::MatrixCoord{CUBE_M, CUBE_N}); + const auto jobs = scheduler.GetCoreLoops(); + const auto first = (get_block_idx() + t.aicNum - startCore) % t.aicNum; + const RowMajor layoutA{rows, k}, layoutC{rows, n}; + auto expertB = b[uint64_t(e) * k * n]; + expertB.SetL2CacheHint(rows <= CUBE_M ? CacheMode::CACHE_MODE_DISABLE : CacheMode::CACHE_MODE_NORMAL); + for (uint32_t job = first; job < jobs; job += t.aicNum) { + const auto coord = scheduler.GetBlockCoord(job); + const auto actual = scheduler.GetActualBlockShape(coord); + const uint32_t m = coord.m() * CUBE_M, col = coord.n() * CUBE_N; + const int32_t done = second && job + t.aicNum >= jobs ? int32_t(index) : -1; + const uint64_t row = uint64_t(Plan::kRows) * round.begin[e] + m; + if constexpr (Plan::kPackedInt4) { + mm(scale[e * n + col], VectorLayout{n}, a[row * k], layoutA, + expertB[layoutB.GetOffset(Catlass::MatrixCoord{0, col})], layoutB, + out[row * n + col], layoutC, actual, ::Callback{}, ::Callback{}, done, 0); + } else { + mm(a[row * k], layoutA, + expertB[layoutB.GetOffset(Catlass::MatrixCoord{0, col})], layoutB, + out[row * n + col], layoutC, actual, ::Callback{}, ::Callback{}); + } + } + if (!second && SyncExpert(index, round.activeCount)) { + mm.SynchronizeBlock(); + if constexpr (Plan::kPackedInt4) mm.Finalize(syncGroup++, GMM1_READY); + else CrossCoreSetFlag<2, PIPE_FIX>(GMM1_READY); + } + if constexpr (!Plan::kPackedInt4) { + if (second) { + mm.SynchronizeBlock(); + CrossCoreSetFlag<2, PIPE_FIX>(index / FLAG_SETS); + } + } + startCore = (startCore + jobs) % t.aicNum; + } + mm.SynchronizeBlock(); + if constexpr (Plan::kPackedInt4) { + if (second) mm.Finalize(int32_t(round.activeCount) - 1, 0); + } +} +} // namespace AfdNativeMega + +#include "afd_mega_moe_pull_pipeline.h" + +namespace AfdNativeMega { +template +__aicore__ inline void Process(GM_ADDR state, GM_ADDR layerMap, GM_ADDR scratch, + GM_ADDR w13, GM_ADDR w2, GM_ADDR s13, GM_ADDR s2, GM_ADDR b13, GM_ADDR b2, + GM_ADDR completion, const Cam::AfdMegaMoePullNativeTilingData &t) { + const auto c = t.config; + const Cam::NativeMegaWorkspace w(c[10], c[1], c[2], c[6], c[9]); + Resource resource; + TPipe pipe; + // TPipe seeds M_MTE1 0..2; MC2 must own these events exclusively on AIC. + if ASCEND_IS_AIC { pipe.Destroy(); } + const uint32_t id = GetBlockIdx(); + // A fixed workspace drains one selected A transaction, including all receive rounds. + do { + if ASCEND_IS_AIV { + if (id == 0) { + Cam::Pull::Transport rx(state, PullShape(t), c[7], t.aivNum, pipe, 0); + rx.PrepareRound(scratch + w.batch, scratch + w.counts, c[10]); + const auto modelLayer = rx.Read(scratch + w.batch + 16); + int64_t slot = -1; + if (rx.ReadState(Cam::Pull::SOURCE) >= 0 && !rx.ReadState(Cam::Pull::FAILURE)) { + if (modelLayer >= 0 && modelLayer < t.layerMapSize) + slot = rx.Read(layerMap + modelLayer * 8); + if (slot < 0 || slot >= c[13]) { + const auto source = rx.ReadState(Cam::Pull::SOURCE); + rx.Fail(source >= 0 && source + c[9] <= c[5] ? source : 0, -4); + slot = -1; + } + } + rx.Write(scratch + w.layer, slot); + } + pipe.Reset(); + } + SyncAll(); + const auto layer = Load(state, Cam::Pull::FAILURE) ? -1 : Load(scratch + w.layer); + if (layer >= 0 && layer < c[13]) { + const ReceiveRound round(scratch, w, c[6]); + if (round.total) { + if ASCEND_IS_AIC { + Compute(scratch, w, t, round, w13, s13, layer, false, resource); + Compute(scratch, w, t, round, w2, s2, layer, true, resource); + } + if ASCEND_IS_AIV { + PullAndPrepare(scratch, w, t, round, resource); + Activate(scratch, w, t, round, Plan::kPackedInt4 ? b13 : s13, layer, resource); + Combine(scratch, w, t, round, Plan::kPackedInt4 ? b2 : s2, layer, resource); + } + } + } + SyncAll(); + if ASCEND_IS_AIV { pipe.Reset(); } + if ASCEND_IS_AIV { + if (id == 0) { + Cam::Pull::Transport tx(state, PullShape(t), c[7], t.aivNum, pipe, 0); + if (layer >= 0 && layer < c[13]) tx.CompleteRound(scratch + w.batch, c[10]); + tx.Write(completion, tx.ReadState(Cam::Pull::FAILURE)); + } + } + // All GMM/epilogue events have drained before scratch and flag slots are reused. + SyncAll(); + if ASCEND_IS_AIV { pipe.Reset(); } + } while (!Load(state, Cam::Pull::FAILURE) && Load(state, Cam::Pull::SOURCE) >= 0); +} +} // namespace AfdNativeMega + +extern "C" __global__ __aicore__ void afd_mega_moe_pull_native( + GM_ADDR anchor, GM_ADDR state, GM_ADDR layerMap, GM_ADDR scratch, GM_ADDR w13, GM_ADDR w2, + GM_ADDR s13, GM_ADDR s2, GM_ADDR b13, GM_ADDR b2, GM_ADDR completion, + GM_ADDR workspace, GM_ADDR tiling) { + using namespace AfdNativeMega; + KERNEL_TASK_TYPE_DEFAULT(KERNEL_TYPE_MIX_AIC_1_2); + REGISTER_TILING_DEFAULT(Cam::AfdMegaMoePullNativeTilingData); + REGISTER_TILING_FOR_TILINGKEY("TILING_KEY_VAR == 1", Cam::AfdMegaMoePullNativeTilingData); + REGISTER_TILING_FOR_TILINGKEY("TILING_KEY_VAR == 2", Cam::AfdMegaMoePullNativeTilingData); + GET_TILING_DATA_WITH_STRUCT(Cam::AfdMegaMoePullNativeTilingData, t, tiling); + if (TILING_KEY_IS(1)) Process(state, layerMap, scratch, w13, w2, s13, s2, b13, b2, completion, t); + if (TILING_KEY_IS(2)) Process(state, layerMap, scratch, w13, w2, s13, s2, b13, b2, completion, t); +} diff --git a/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_pull_native_tiling.h b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_pull_native_tiling.h new file mode 100644 index 000000000..a737f8e14 --- /dev/null +++ b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_pull_native_tiling.h @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +#ifndef AFD_MEGA_MOE_PULL_NATIVE_TILING_H +#define AFD_MEGA_MOE_PULL_NATIVE_TILING_H +#include +#include "kernel_tiling/kernel_tiling.h" +#include "afd_native_mega_workspace.h" +namespace Cam { +struct AfdMegaMoePullNativeTilingData { + Mc2InitTiling mc2InitTiling; + Mc2CcTiling mc2CcTiling1; + Mc2CcTiling mc2CcTiling2; + int64_t config[14]; + uint32_t aicNum, aivNum, commAivNum, layerMapSize; + alignas(8) uint8_t cube[1024]; + float routedScale; + // Model swiglu_limit; zero disables clipping, matching the layered W4A8 op. + float swigluLimit; +}; +} +#endif diff --git a/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_pull_pipeline.h b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_pull_pipeline.h new file mode 100644 index 000000000..ac9842065 --- /dev/null +++ b/csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel/afd_mega_moe_pull_pipeline.h @@ -0,0 +1,699 @@ +/** + * Copyright (c) 2026 Huawei Technologies Co., Ltd. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * Modifications Copyright contributors to the AFD plugin project. + */ +#ifndef AFD_MEGA_MOE_PULL_PIPELINE_H +#define AFD_MEGA_MOE_PULL_PIPELINE_H + +// Port of MC2 arch22/a2_a3_copy_helpers.hpp and W4A8 block epilogues. +// AFD changes: frozen source routes, layer descriptors, direct A return windows, +// and the existing BF16 rounding before routed_scale. No all-to-all prologue. +namespace AfdNativeMega { +__aicore__ inline uint32_t Min(uint32_t a, uint32_t b) { return a < b ? a : b; } + +// MC2's signed high nibble and low nibble minus eight; bias compensates the latter. +__aicore__ inline void Msd(LocalTensor q, LocalTensor work, + LocalTensor packed, uint32_t n) { + auto high = work, lowBits = work[n], low = work[2 * n]; + auto mask = work[3 * n].ReinterpretCast(); + Duplicate(mask, static_cast(0x0f0f), 128); + Cast(high, q, RoundMode::CAST_NONE, n); + PipeBarrier(); + Muls(high, high, static_cast(0.0625f), n); + PipeBarrier(); + Cast(packed, high, RoundMode::CAST_FLOOR, n); + And(lowBits.ReinterpretCast(), q.ReinterpretCast(), mask, + 128, n / 256, {1, 1, 1, 8, 8, 0}); + PipeBarrier(); + Cast(low, lowBits.ReinterpretCast(), RoundMode::CAST_NONE, n); + PipeBarrier(); + Adds(low, low, static_cast(-8), n); + PipeBarrier(); + Cast(packed[n], low, RoundMode::CAST_NONE, n); +} + +__aicore__ inline void FetchRow(LocalTensor input, GlobalTensor remote, + uint32_t hidden, uint32_t stage) { + WaitFlag(stage); + WaitFlag(stage); + DataCopyPad(input, remote, {1U, hidden + uint32_t(sizeof(float)), 0U, 0U, 0U}, {false, 0U, 0U, 0U}); + SetFlag(stage); +} + +template +__aicore__ inline void PullAndPrepare(GM_ADDR scratch, const Cam::NativeMegaWorkspace &w, + const Cam::AfdMegaMoePullNativeTilingData &t, const ReceiveRound &round, Resource &resource) { + using Element = typename Plan::Element; + const uint32_t h = t.config[1], experts = t.config[6], tp = t.config[9]; + const auto shape = PullShape(t); + Cam::Pull::Layout layout{}; + Cam::Pull::MakeLayout(shape, layout); + const uint32_t metadataBytes = 8 * Cam::Pull::MetadataWords(tp, experts); + auto meta = resource.ubBuf.GetBufferByByte(0); + GlobalTensor batch; + batch.SetGlobalBuffer(reinterpret_cast<__gm__ int64_t *>(scratch + w.batch)); + DataCopyPad(meta, batch, {1U, metadataBytes, 0U, 0U, 0U}, {false, 0U, 0U, 0U}); + Fence(); + uint32_t ub = Cam::Pull::Align(metadataBytes); + LocalTensor input[2]; + LocalTensor packed[2]; + for (uint32_t b = 0; b < 2; ++b) { + input[b] = resource.ubBuf.GetBufferByByte(ub); + ub += h + Cam::Pull::ROW_PADDING; + packed[b] = resource.ubBuf.GetBufferByByte(ub); + ub += h; + SetFlag(b); + SetFlag(b); + SetFlag(b); + SetFlag(b); + } + auto work = resource.ubBuf.GetBufferByByte(ub); + GlobalTensor out; + GlobalTensor scales; + GlobalTensor routes; + out.SetGlobalBuffer(reinterpret_cast<__gm__ int8_t *>(scratch + w.a1)); + scales.SetGlobalBuffer(reinterpret_cast<__gm__ float *>(scratch + w.scale1)); + // The old INT8 staging allocation now holds the per-row return address cache. + routes.SetGlobalBuffer(reinterpret_cast<__gm__ uint64_t *>(scratch + w.x)); + uint64_t block = 0; + uint32_t active = 0; + for (uint32_t e = 0; e < experts; ++e) { + const uint32_t rows = round.Rows(e), id = GetBlockIdx(); + const uint32_t perCore = rows / t.aivNum, tail = rows % t.aivNum; + const uint32_t first = round.begin[e] + id * perCore + Min(id, tail); + const uint32_t count = perCore + (id < tail); + for (uint32_t s = 0; s < tp; ++s) { + const uint32_t index = 5 + tp + s * experts + e; + const uint64_t sourceRows = meta.GetValue(index); + const uint64_t prefix = meta.GetValue(5 + s); + // Even an empty slice may precede this round's offset in the source window. + meta.SetValue(5 + s, prefix + sourceRows); + block += sourceRows; + if (!rows) continue; + const auto slice = Cam::Pull::Intersect(block - sourceRows, sourceRows, round.offset + first, count); + if (slice.count) { + GlobalTensor remote; + const auto window = Cam::Pull::Window(round.source + s, t.config[7]); + remote.SetGlobalBuffer(reinterpret_cast<__gm__ int8_t *>(window + layout.payload)); + FetchRow(input[0], remote[(prefix + slice.skip) * layout.rowStride], h, 0); + for (uint32_t r = 0; r < slice.count; ++r) { + const uint32_t stage = r % 2; + if (r + 1 < slice.count) + FetchRow(input[1 - stage], remote[(prefix + slice.skip + r + 1) * layout.rowStride], h, 1 - stage); + WaitFlag(stage); + WaitFlag(stage); + if constexpr (Plan::kPackedInt4) Msd(input[stage], work, packed[stage], h); + SetFlag(stage); + SetFlag(stage); + WaitFlag(stage); + const uint32_t row = first + slice.local + r; + if constexpr (Plan::kPackedInt4) + DataCopy(out[uint64_t(row) * h], packed[stage].template ReinterpretCast(), h); + else DataCopy(out[uint64_t(row) * h], input[stage], h); + SetFlag(stage); + DataCopyPad(scales[row], input[stage][h].ReinterpretCast(), {1U, 4U, 0U, 0U, 0U}); + auto address = input[stage][h + 32].ReinterpretCast(); + // The next source may reuse this scalar address before its DMA drains. + WaitFlag(stage); + address.SetValue(0, reinterpret_cast(window + layout.returned) + + (prefix + slice.skip + r) * h * sizeof(bfloat16_t)); + Fence(); + DataCopyPad(routes[row], address, {1U, 8U, 0U, 0U, 0U}); + SetFlag(stage); + SetFlag(stage); + } + } + } + if (!rows) continue; + SyncAll(); + CrossCoreSetFlag<2, PIPE_MTE3>(active++ / FLAG_SETS); + } + for (uint32_t b = 0; b < 2; ++b) { + WaitFlag(b); + WaitFlag(b); + WaitFlag(b); + WaitFlag(b); + } +} + +// The current MC2 epilogue's bounded column tiles also support I > UB capacity. +// For a multi-tile row, pass one reduces a global maximum and pass two quantizes. +template +class Activation { +public: + using Element = typename Plan::Element; + using Raw = typename Plan::OutputElement; + static constexpr uint32_t TILE = 3072, SCALE_BATCH = 256; + __aicore__ Activation(Resource &resource, uint32_t intermediate, float limit) + : i_(intermediate), limit_(limit) { + uint32_t ub = 0; + for (uint32_t b = 0; b < 2; ++b) { + raw_[b] = resource.ubBuf.GetBufferByByte(ub); ub += 8 * TILE; + bias_[b] = resource.ubBuf.GetBufferByByte(ub); ub += 8 * TILE; + packed_[b] = resource.ubBuf.GetBufferByByte(ub); ub += TILE; + SetFlag(b); + SetFlag(b); + } + gate_ = resource.ubBuf.GetBufferByByte(ub); ub += 4 * TILE; + up_ = resource.ubBuf.GetBufferByByte(ub); ub += 4 * TILE; + value_ = resource.ubBuf.GetBufferByByte(ub); ub += 4 * TILE; + quant_ = resource.ubBuf.GetBufferByByte(ub); ub += TILE; + // Msd reuses gate/up/value after the activation has been quantized. + msdWork_ = gate_.ReinterpretCast(); + maximum_ = resource.ubBuf.GetBufferByByte(ub); ub += 32; + scales_ = resource.ubBuf.GetBufferByByte(ROUTE_UB); + for (uint32_t b = 0; b < 2; ++b) { + scaleOut_[b] = resource.ubBuf.GetBufferByByte(ub); ub += SCALE_BATCH * 4; + SetFlag(b); + } + } + __aicore__ void LoadScales(GlobalTensor source, uint32_t rows) { + if (rows) DataCopyPad(scales_, source, {1U, 4 * rows, 0U, 0U, 0U}, {false, 0U, 0U, 0U}); + Fence(); + } + __aicore__ void Row(GlobalTensor raw, GlobalTensor bias, + GlobalTensor out, LocalTensor scaleOut, + uint32_t scaleIndex) { + const float scale = scales_.GetValue(scaleIndex); + Duplicate(maximum_, 0.0f, 8); + PipeBarrier(); + Fetch(raw, bias, 0, 0); + for (uint32_t col = 0, stage = 0; col < i_; col += TILE, stage ^= 1) { + const auto length = Min(TILE, i_ - col); + if (col + TILE < i_) Fetch(raw, bias, col + TILE, stage ^ 1); + Evaluate(stage, length, scale); + Abs(gate_, value_, length); + PipeBarrier(); + ReduceMax(up_, gate_, up_, length, false); + PipeBarrier(); + Max(maximum_, maximum_, up_, 1); + PipeBarrier(); + } + Fence(); + const float absMax = maximum_.GetValue(0); + const float quantScale = Plan::kPackedInt4 ? absMax / 127.0f : absMax * (1.0f / 127.0f); + scaleOut.SetValue(scaleIndex, quantScale); + const float multiplier = Plan::kPackedInt4 ? (absMax > 0 ? 1.0f / quantScale : 0.0f) : quantScale; + Fence(); + if (i_ <= TILE) { + Quantize(out, 0, i_, 0, multiplier); + } else { + Fetch(raw, bias, 0, 0); + for (uint32_t col = 0, stage = 0; col < i_; col += TILE, stage ^= 1) { + const auto length = Min(TILE, i_ - col); + if (col + TILE < i_) Fetch(raw, bias, col + TILE, stage ^ 1); + Evaluate(stage, length, scale); + Quantize(out, col, length, stage, multiplier); + } + } + } + __aicore__ LocalTensor BeginScale(uint32_t stage) { + WaitFlag(stage); + return scaleOut_[stage]; + } + __aicore__ void EndScale(GlobalTensor output, uint32_t stage, uint32_t count) { + Fence(); + if (count) DataCopyPad(output, scaleOut_[stage], {1U, 4 * count, 0U, 0U, 0U}); + SetFlag(stage); + } + __aicore__ void Finalize() { + for (uint32_t b = 0; b < 2; ++b) { + WaitFlag(b); + WaitFlag(b); + WaitFlag(b); + } + } +private: + __aicore__ void Fetch(GlobalTensor raw, GlobalTensor bias, uint32_t col, uint32_t b) { + WaitFlag(b); + const uint32_t n = Min(TILE, i_ - col); + for (uint32_t part = 0; part < 2 * Plan::kRows; ++part) + DataCopy(raw_[b][part * TILE], raw[part * i_ + col], n); + DataCopy(bias_[b], bias[col], n); + DataCopy(bias_[b][TILE], bias[i_ + col], n); + SetFlag(b); + } + __aicore__ void Evaluate(uint32_t b, uint32_t n, float scale) { + WaitFlag(b); + Cast(gate_, raw_[b], RoundMode::CAST_NONE, n); + if constexpr (Plan::kPackedInt4) { + Cast(value_, raw_[b][2 * TILE], RoundMode::CAST_NONE, n); + PipeBarrier(); + Muls(gate_, gate_, 16.0f, n); + PipeBarrier(); + Add(gate_, gate_, value_, n); + PipeBarrier(); + Cast(up_, raw_[b][TILE], RoundMode::CAST_NONE, n); + Cast(value_, raw_[b][3 * TILE], RoundMode::CAST_NONE, n); + PipeBarrier(); + Muls(up_, up_, 16.0f, n); + PipeBarrier(); + Add(up_, up_, value_, n); + PipeBarrier(); + Add(gate_, gate_, bias_[b], n); + Add(up_, up_, bias_[b][TILE], n); + } else { + Cast(up_, raw_[b][TILE], RoundMode::CAST_NONE, n); + PipeBarrier(); + Mul(gate_, gate_, bias_[b], n); + Mul(up_, up_, bias_[b][TILE], n); + } + SetFlag(b); + PipeBarrier(); + Muls(gate_, gate_, scale, n); + Muls(up_, up_, scale, n); + PipeBarrier(); + // Match Layered: only W4A8 applies the model limit. + if constexpr (Plan::kPackedInt4) ClampSwiGLUInputs(gate_, up_, limit_, n); + Muls(value_, gate_, -1.0f, n); + PipeBarrier(); + Exp(value_, value_, n); + PipeBarrier(); + Adds(value_, value_, 1.0f, n); + PipeBarrier(); + Div(value_, gate_, value_, n); + PipeBarrier(); + Mul(value_, value_, up_, n); + PipeBarrier(); + } + __aicore__ void Quantize(GlobalTensor out, uint32_t col, uint32_t n, + uint32_t b, float multiplier) { + if constexpr (Plan::kPackedInt4) { + Muls(gate_, value_, multiplier, n); + PipeBarrier(); + auto fp16 = gate_.ReinterpretCast(); + Cast(fp16, gate_, RoundMode::CAST_RINT, n); + PipeBarrier(); + Cast(quant_, fp16, RoundMode::CAST_RINT, n); + PipeBarrier(); + WaitFlag(b); + Msd(quant_, msdWork_, packed_[b], n); + SetFlag(b); + WaitFlag(b); + DataCopy(out[col / 2], packed_[b].template ReinterpretCast(), n / 2); + DataCopy(out[i_ / 2 + col / 2], packed_[b][n].template ReinterpretCast(), n / 2); + } else { + Duplicate(gate_, multiplier, n); + PipeBarrier(); + Div(gate_, value_, gate_, n); + PipeBarrier(); + auto integer = gate_.template ReinterpretCast(); + Cast(integer, gate_, RoundMode::CAST_RINT, n); + PipeBarrier(); + SetDeqScale(half(1.0f)); + auto fp16 = up_.template ReinterpretCast(); + Cast(fp16, integer, RoundMode::CAST_ROUND, n); + PipeBarrier(); + WaitFlag(b); + Cast(packed_[b], fp16, RoundMode::CAST_TRUNC, n); + PipeBarrier(); + SetFlag(b); + WaitFlag(b); + DataCopy(out[col], packed_[b], n); + } + SetFlag(b); + } + uint32_t i_; + float limit_; + LocalTensor raw_[2]; + LocalTensor msdWork_; + LocalTensor packed_[2]; + LocalTensor quant_; + LocalTensor bias_[2], gate_, up_, value_, maximum_, scales_, scaleOut_[2]; +}; + +template +__aicore__ inline void ActivateNative(GM_ADDR scratch, const Cam::NativeMegaWorkspace &w, + const Cam::AfdMegaMoePullNativeTilingData &t, const ReceiveRound &round, GM_ADDR biases, + int64_t layer, Resource &resource) { + const uint32_t i = t.config[2], experts = t.config[6], id = GetBlockIdx(); + // Adapt the selected layer to MC2's one-tensor list and int32 expert counts. + auto metadata = resource.ubBuf.GetBufferByByte(0); + metadata.SetValue(0, 8); + metadata.SetValue(1, reinterpret_cast(GetLayerTensorAddr(layer, t.config[13], biases))); + GM_ADDR descriptor = scratch + w.flags + id * 64; + GlobalTensor desc; + desc.SetGlobalBuffer(reinterpret_cast<__gm__ uint64_t *>(descriptor)); + Fence(); + DataCopy(desc, metadata, 4); + GlobalTensor counts; + counts.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(scratch + w.tileCounts)); + if (id == 0) { + auto localCounts = resource.ubBuf.GetBufferByByte(64); + for (uint32_t e = 0; e < experts; ++e) localCounts.SetValue(e, round.Rows(e)); + Fence(); + DataCopyPad(counts, localCounts, {1U, 4 * experts, 0U, 0U, 0U}); + } + Fence(); + Load(descriptor); + GlobalTensor raw; + GlobalTensor s1, s2; + GlobalTensor out; + raw.SetGlobalBuffer(reinterpret_cast<__gm__ half *>(scratch + w.c1)); + s1.SetGlobalBuffer(reinterpret_cast<__gm__ float *>(scratch + w.scale1)); + s2.SetGlobalBuffer(reinterpret_cast<__gm__ float *>(scratch + w.scale2)); + out.SetGlobalBuffer(reinterpret_cast<__gm__ int8_t *>(scratch + w.a2)); + typename Plan::NativeActivation activation(resource, 2 * i); + uint32_t begin = 0; + for (uint32_t index = 0; index < round.activeCount; ++index) { + if (!SyncExpert(index, round.activeCount)) continue; + const uint32_t e = round.Expert(index); + CrossCoreWaitFlag<2>(GMM1_READY); + SyncAll(); + if (begin == 0) { + for (uint32_t j = 0; j < experts; j += 16) Load(scratch + w.tileCounts + 4 * j); + } + const uint32_t rows = round.begin[e + 1] - begin; + if (rows) activation(raw[4ULL * begin * i], Catlass::MatrixCoord{rows, 2 * i}, + s1[begin], reinterpret_cast<__gm__ float *>(descriptor), out[uint64_t(begin) * i], + counts, begin, s2[begin], experts, 1, 0, 1, resource, t.aivNum, t.swigluLimit, +#if defined(AFD_CANN_CATLASS_92) + static_cast(Catlass::Epilogue::GatedActivationMode::SWIGLU), + Catlass::Epilogue::SwigluOaiActivation::DEFAULT_ALPHA, + Catlass::Epilogue::SituActivation::DEFAULT_BETA, +#endif + 2 * i); + SyncAll(); + CrossCoreSetFlag<2, PIPE_MTE3>(ACT_READY); + begin = round.begin[e + 1]; + } + activation.Finalize(); +} + +template +__aicore__ inline void Activate(GM_ADDR scratch, const Cam::NativeMegaWorkspace &w, + const Cam::AfdMegaMoePullNativeTilingData &t, const ReceiveRound &round, GM_ADDR biases, + int64_t layer, Resource &resource) { + if constexpr (Plan::kPackedInt4) { + if (t.config[2] <= Activation::TILE) { + ActivateNative(scratch, w, t, round, biases, layer, resource); + return; + } + } + const uint32_t i = t.config[2], experts = t.config[6], id = GetBlockIdx(); + GlobalTensor raw; + GlobalTensor bias, s1, s2; + GlobalTensor out; + raw.SetGlobalBuffer(reinterpret_cast<__gm__ typename Plan::OutputElement *>(scratch + w.c1)); + bias.SetGlobalBuffer(GetLayerTensorAddr(layer, t.config[13], biases)); + s1.SetGlobalBuffer(reinterpret_cast<__gm__ float *>(scratch + w.scale1)); + s2.SetGlobalBuffer(reinterpret_cast<__gm__ float *>(scratch + w.scale2)); + out.SetGlobalBuffer(reinterpret_cast<__gm__ int8_t *>(scratch + w.a2)); + Activation activation(resource, i, t.swigluLimit); + uint32_t groupBegin = 0, expert = 0, group = 0; + for (uint32_t index = 0; index < round.activeCount; ++index) { + if (!SyncExpert(index, round.activeCount)) continue; + const uint32_t e = round.Expert(index); + CrossCoreWaitFlag<2>(GMM1_READY); + SyncAll(); + const uint32_t rows = round.begin[e + 1] - groupBegin; + const uint32_t each = rows / t.aivNum, tail = rows % t.aivNum; + const uint32_t first = groupBegin + id * each + Min(id, tail); + const uint32_t count = each + (id < tail); + // Bound UB usage even when one expert consumes the entire receive capacity. + for (uint32_t offset = 0; offset < count; offset += Activation::SCALE_BATCH) { + const uint32_t length = Min(Activation::SCALE_BATCH, count - offset); + const uint32_t stage = group++ % 2; + auto scaleOut = activation.BeginScale(stage); + activation.LoadScales(s1[first + offset], length); + for (uint32_t r = 0; r < length; ++r) { + const uint32_t row = first + offset + r; + while (expert < e && row >= round.begin[expert + 1]) ++expert; + activation.Row(raw[2ULL * Plan::kRows * row * i], bias[2ULL * expert * i], + out[uint64_t(row) * i], scaleOut, r); + } + activation.EndScale(s2[first + offset], stage, length); + } + SyncAll(); + CrossCoreSetFlag<2, PIPE_MTE3>(ACT_READY); + groupBegin = round.begin[e + 1]; + } + activation.Finalize(); +} + +template +class Output { +public: + using Raw = typename Plan::OutputElement; + using Scale = typename Plan::OutputScale; + static constexpr uint32_t ROWS = 16, COLS = CUBE_N, ELEMENTS = ROWS * COLS; + __aicore__ Output(Resource &resource, GM_ADDR routeBase, uint32_t rows, GM_ADDR staged) : totalRows_(rows) { + uint32_t ub = 0; + for (uint32_t b = 0; b < 2; ++b) { + hi_[b] = resource.ubBuf.GetBufferByByte(ub); ub += sizeof(Raw) * ELEMENTS; + if constexpr (Plan::kPackedInt4) { + lo_[b] = resource.ubBuf.GetBufferByByte(ub); ub += 2 * ELEMENTS; + } + bias_[b] = resource.ubBuf.GetBufferByByte(ub); ub += 4 * COLS; + y_[b] = resource.ubBuf.GetBufferByByte(ub); ub += 2 * ELEMENTS; + scale_[b] = resource.ubBuf.GetBufferByByte(ub); ub += 4 * ROWS; + scaleRow_[b] = UINT32_MAX; + SetFlag(b); + SetFlag(b + 2); + SetFlag(b); + } + value_ = resource.ubBuf.GetBufferByByte(ub); ub += 4 * ELEMENTS; + temp_ = resource.ubBuf.GetBufferByByte(ub); + routes_ = resource.ubBuf.GetBufferByByte(ROUTE_UB); + routeGm_.SetGlobalBuffer(reinterpret_cast<__gm__ uint64_t *>(routeBase)); + staged_ = reinterpret_cast(staged); + } + __aicore__ void Tile(GlobalTensor raw, GlobalTensor bias, GlobalTensor scale, + uint32_t row, uint32_t rows, uint32_t col, uint32_t hidden, float routedScale) { + const uint32_t b = stage_; + WaitFlag(b); + DataCopyPad(hi_[b], raw[uint64_t(Plan::kRows) * row * hidden + col], + {uint16_t(rows), COLS * uint32_t(sizeof(Raw)), + (Plan::kRows * hidden - COLS) * uint32_t(sizeof(Raw)), 0U, 0U}, + {false, 0U, 0U, 0U}); + if constexpr (Plan::kPackedInt4) { + DataCopyPad(lo_[b], raw[(2ULL * row + 1) * hidden + col], + {uint16_t(rows), COLS * 2, (2 * hidden - COLS) * 2, 0U, 0U}, {false, 0U, 0U, 0U}); + DataCopy(bias_[b], bias[col], COLS); + } else { + DataCopy(bias_[b].template ReinterpretCast(), bias[col], COLS); + } + SetFlag(b); + WaitFlag(b + 2); + if (scaleRow_[b] != row || scaleRows_[b] != rows) { + DataCopyPad(scale_[b], scale[row], {1U, 4 * rows, 0U, 0U, 0U}, {false, 0U, 0U, 0U}); + scaleRow_[b] = row; + scaleRows_[b] = rows; + } + if (row < routeBegin_ || row + rows > routeBegin_ + routeCount_) { + routeBegin_ = row; + routeCount_ = Min(Cam::NATIVE_METADATA_ROWS, totalRows_ - row); + DataCopyPad(routes_, routeGm_[row], {1U, 8 * routeCount_, 0U, 0U, 0U}, + {false, 0U, 0U, 0U}); + } + Fence(); + WaitFlag(b); + const uint32_t length = rows * COLS; + Cast(value_, hi_[b], RoundMode::CAST_NONE, length); + if constexpr (Plan::kPackedInt4) { + Cast(temp_, lo_[b], RoundMode::CAST_NONE, length); + PipeBarrier(); + Muls(value_, value_, 16.0f, length); + PipeBarrier(); + Add(value_, value_, temp_, length); + PipeBarrier(); + for (uint32_t r = 0; r < rows; ++r) Add(value_[r * COLS], value_[r * COLS], bias_[b], COLS); + } else { + Cast(temp_, bias_[b].template ReinterpretCast(), RoundMode::CAST_NONE, COLS); + PipeBarrier(); + for (uint32_t r = 0; r < rows; ++r) Mul(value_[r * COLS], value_[r * COLS], temp_, COLS); + } + SetFlag(b); + PipeBarrier(); + for (uint32_t r = 0; r < rows; ++r) Muls(value_[r * COLS], value_[r * COLS], scale_[b].GetValue(r), COLS); + SetFlag(b + 2); + PipeBarrier(); + WaitFlag(b); + Cast(y_[b], value_, RoundMode::CAST_RINT, length); + if (routedScale != 1.0f) { + PipeBarrier(); + Cast(value_, y_[b], RoundMode::CAST_NONE, length); + PipeBarrier(); + Muls(value_, value_, routedScale, length); + PipeBarrier(); + Cast(y_[b], value_, RoundMode::CAST_RINT, length); + } + SetFlag(b); + WaitFlag(b); + for (uint32_t r = 0; r < rows;) { + const uint64_t address = staged_ + uint64_t(row + r) * hidden * sizeof(bfloat16_t); + uint32_t contiguous = 1; + while (r + contiguous < rows) ++contiguous; + GlobalTensor remote; + remote.SetGlobalBuffer(reinterpret_cast<__gm__ bfloat16_t *>(address)); + DataCopyPad(remote[col], y_[b][r * COLS], + {uint16_t(contiguous), COLS * 2, 0U, (hidden - COLS) * 2, 0U}); + r += contiguous; + } + SetFlag(b); + stage_ ^= 1; + } + __aicore__ void DrainLocal() { + // Observe every local MTE3 write; keep tokens seeded for next expert/finalize. + for (uint32_t b=0;b<2;++b) { + WaitFlag(b); + SetFlag(b); + } + } + __aicore__ void Finalize() { + for (uint32_t b = 0; b < 2; ++b) { + WaitFlag(b); + WaitFlag(b + 2); + WaitFlag(b); + } + } +private: + uint32_t stage_ = 0, scaleRow_[2], scaleRows_[2]; + uint64_t staged_; + uint32_t totalRows_, routeBegin_ = 0, routeCount_ = 0; + LocalTensor hi_[2]; + LocalTensor lo_[2]; + LocalTensor bias_[2], scale_[2], value_, temp_; + LocalTensor y_[2]; + LocalTensor routes_; + GlobalTensor routeGm_; +}; + + +// Independent UB and events keep TX in flight while subsequent output is produced. +class ExpertRowTx { +public: + static constexpr uint32_t BATCH=1, META_ROWS=256, PUMP_ROWS=1, READY_ROWS=512; + static constexpr uint32_t QUEUE_UB=84*1024, TX_UB=96*1024, EVENT_BASE=4; + __aicore__ ExpertRowTx(Resource &resource, GM_ADDR routeBase, GM_ADDR staged, + uint32_t hidden, uint32_t cores) : h_(hidden),cores_(cores) { + local_.SetGlobalBuffer(reinterpret_cast<__gm__ bfloat16_t *>(staged)); + routeGm_.SetGlobalBuffer(reinterpret_cast<__gm__ uint64_t *>(routeBase)); + queue_=resource.ubBuf.GetBufferByByte(QUEUE_UB); + for (uint32_t b=0;b<2;++b) { + value_[b]=resource.ubBuf.GetBufferByByte(TX_UB+b*BATCH*h_*2); + SetFlag(b+EVENT_BASE); + } + routes_=resource.ubBuf.GetBufferByByte(TX_UB+2*BATCH*h_*2); + } + __aicore__ void Enqueue(uint32_t base,uint32_t rows,uint32_t expert) { + const uint32_t id=GetBlockIdx(),per=rows/cores_,rem=rows%cores_; + const uint32_t count=per+(id=cacheEnd_) { + cacheBegin_=cursor_;cacheEnd_=cursor_+Min(META_ROWS,count_-cursor_); + DataCopyPad(routes_,routeGm_[first_+cursor_],{1U,8*(cacheEnd_-cursor_),0U,0U,0U},{false,0U,0U,0U}); + Fence(); + } + uint32_t n=1; + const uint32_t limit=Min(BATCH,Min(quota,Min(count_-cursor_,cacheEnd_-cursor_))); + const uint64_t address=routes_.GetValue(cursor_-cacheBegin_); + // Never merge across a destination discontinuity or metadata-window boundary. + while (n(b+EVENT_BASE); + WaitFlag(b+EVENT_BASE); + GlobalTensor remote; + remote.SetGlobalBuffer(reinterpret_cast<__gm__ bfloat16_t *>(address)); + DataCopyPad(remote,value_[b],{1U,2*h_*n,0U,0U,0U}); + SetFlag(b+EVENT_BASE); + cursor_+=n;quota-=n; + } + } + __aicore__ void Drain() { + Pump(0x7fffffff); + for (uint32_t b=0;b<2;++b) {WaitSlot(b);SetFlag(b+EVENT_BASE);} + } + __aicore__ void Finalize() { + Pump(0x7fffffff); + for (uint32_t b=0;b<2;++b) WaitSlot(b); + } +private: + __aicore__ void WaitSlot(uint32_t b) {WaitFlag(b+EVENT_BASE);} + uint32_t h_,cores_,stage_=0,head_=0,tail_=0,loaded_=UINT32_MAX; + uint32_t first_=0,count_=0,cursor_=0,cacheBegin_=0,cacheEnd_=0; + LocalTensor value_[2]; + LocalTensor routes_; + LocalTensor queue_; + GlobalTensor local_; + GlobalTensor routeGm_; +}; + +template +__aicore__ inline void Combine(GM_ADDR scratch, const Cam::NativeMegaWorkspace &w, + const Cam::AfdMegaMoePullNativeTilingData &t, const ReceiveRound &round, GM_ADDR biases, + int64_t layer, Resource &resource) { + const uint32_t h = t.config[1]; + GlobalTensor raw; + GlobalTensor bias; + GlobalTensor scales; + raw.SetGlobalBuffer(reinterpret_cast<__gm__ typename Plan::OutputElement *>(scratch + w.c2)); + bias.SetGlobalBuffer(GetLayerTensorAddr(layer, t.config[13], biases)); + scales.SetGlobalBuffer(reinterpret_cast<__gm__ float *>(scratch + w.scale2)); + Output output(resource, scratch + w.x, round.total, scratch + w.output); + ExpertRowTx rowTx(resource, scratch + w.x, scratch + w.output, h, t.aivNum); + typename Plan::Scheduler scheduler; + uint32_t startCore = 0, sync = 0, groupBegin = 0; + for (uint32_t index = 0; index < round.activeCount; ++index) { + const uint32_t e = round.Expert(index); + scheduler.Update(Coord{Plan::kRows * round.Rows(e), h, uint32_t(t.config[2])}, + Catlass::MatrixCoord{CUBE_M, CUBE_N}); + const auto jobs = scheduler.GetCoreLoops(); + const auto first = (get_block_idx() + t.aicNum - startCore) % t.aicNum; + for (uint32_t job = first; job < jobs; job += t.aicNum) { + const auto coord = scheduler.GetBlockCoord(job); + const auto actual = scheduler.GetActualBlockShape(coord); + for (; sync <= index; ++sync) CrossCoreWaitFlag<2>(sync / FLAG_SETS); + // Each paired AIV handles bounded groups of logical output rows. + constexpr uint32_t physicalRows = Plan::kRows * Output::ROWS; + const uint32_t blocks = (actual.m() + physicalRows - 1) / physicalRows; + const uint32_t half = blocks / 2; + const uint32_t begin = get_subblockid() == 0 ? 0 : half; + const uint32_t end = get_subblockid() == 0 ? half : blocks; + for (uint32_t b = begin; b < end; ++b) { + const uint32_t rows = Min(physicalRows, actual.m() - b * physicalRows) / Plan::kRows; + const uint32_t row = round.begin[e] + coord.m() * (CUBE_M / Plan::kRows) + b * Output::ROWS; + output.Tile(raw, bias[e * h], scales, row, rows, coord.n() * CUBE_N, h, t.routedScale); + rowTx.Pump(ExpertRowTx::PUMP_ROWS); + } + } + const uint32_t groupRows=round.begin[e+1]-groupBegin; + if (groupRows>=ExpertRowTx::READY_ROWS || index+1==round.activeCount) { + output.DrainLocal(); + SyncAll(); + rowTx.Enqueue(groupBegin,groupRows,e); + rowTx.Pump(ExpertRowTx::PUMP_ROWS); + groupBegin=round.begin[e+1]; + } + startCore = (startCore + jobs) % t.aicNum; + } + for (; sync < round.activeCount; ++sync) CrossCoreWaitFlag<2>(sync / FLAG_SETS); + output.Finalize(); + rowTx.Finalize(); + +} +} // namespace AfdNativeMega +#endif diff --git a/csrc/npu/ascend_kernels/cmake_files/op_kernel/CMakeLists.txt b/csrc/npu/ascend_kernels/cmake_files/op_kernel/CMakeLists.txt index 1aaadef07..994be3ba2 100644 --- a/csrc/npu/ascend_kernels/cmake_files/op_kernel/CMakeLists.txt +++ b/csrc/npu/ascend_kernels/cmake_files/op_kernel/CMakeLists.txt @@ -14,6 +14,43 @@ set(KERNEL_COMPILE_OPTS) # (resolves the shared utils/op_kernel headers copied next to the kernels). list(APPEND KERNEL_COMPILE_OPTS -I${CMAKE_CURRENT_SOURCE_DIR} -DASCENDC_DUMP=0 --cce-auto-sync=off -Wno-deprecated-declarations -Werror) +# Mega uses CANN's W4A8 BlockMmad directly; ordinary operator builds do not need it. +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/afd_mega_moe_pull_native.cpp") + set(AFD_TRANSFORMER_DIR "opp/built-in/op_impl/ai_core/tbe/impl/ops_transformer/ascendc") + find_path(AFD_MC2_INCLUDE_DIR + NAMES template_linear_algebra_v2/catlass.hpp + HINTS "${ASCEND_CANN_PACKAGE_PATH}" "$ENV{ASCEND_HOME_PATH}" + PATH_SUFFIXES ${AFD_TRANSFORMER_DIR}/mega_moe/arch22 + NO_DEFAULT_PATH) + if(AFD_MC2_INCLUDE_DIR AND EXISTS "${AFD_MC2_INCLUDE_DIR}/utils/block_mmad_w4a4.hpp") + list(APPEND KERNEL_COMPILE_OPTS -I${AFD_MC2_INCLUDE_DIR}) + set(AFD_MC2_ACTIVATION_DIR "${AFD_MC2_INCLUDE_DIR}") + else() + find_path(AFD_MC2_92_DIR NAMES utils/block_mmad_w4a4.hpp + HINTS "${ASCEND_CANN_PACKAGE_PATH}" "$ENV{ASCEND_HOME_PATH}" + PATH_SUFFIXES ${AFD_TRANSFORMER_DIR}/mega_moe/op_kernel/arch22 NO_DEFAULT_PATH) + find_path(AFD_CATLASS_92_DIR NAMES catlass/catlass.hpp + HINTS "${ASCEND_CANN_PACKAGE_PATH}" "$ENV{ASCEND_HOME_PATH}" + PATH_SUFFIXES ${AFD_TRANSFORMER_DIR}/common/catlass/include NO_DEFAULT_PATH) + if(NOT AFD_MC2_92_DIR OR NOT AFD_CATLASS_92_DIR) + message(FATAL_ERROR "AfdMegaMoePullNative requires CANN MC2 W4A8 and Catlass headers") + endif() + list(APPEND KERNEL_COMPILE_OPTS -DAFD_CANN_CATLASS_92=1 -DCATLASS_ARCH=2201 + -Wno-ignored-attributes -I${AFD_MC2_92_DIR} -I${AFD_CATLASS_92_DIR}) + set(AFD_MC2_ACTIVATION_DIR "${AFD_MC2_92_DIR}") + endif() + # Keep the SDK pipeline while matching Layered's activation rounding. + execute_process( + COMMAND python3 "${CMAKE_CURRENT_SOURCE_DIR}/afd_mega_moe_patch_activation.py" + "${AFD_MC2_ACTIVATION_DIR}/utils/block_epilogue_w4a8post_pertoken_swiglu.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/afd_mega_moe_activation.h" + RESULT_VARIABLE AFD_ACTIVATION_PATCH_RESULT + ERROR_VARIABLE AFD_ACTIVATION_PATCH_ERROR) + if(NOT AFD_ACTIVATION_PATCH_RESULT EQUAL 0) + message(FATAL_ERROR "Unable to adapt CANN activation: ${AFD_ACTIVATION_PATCH_ERROR}") + endif() +endif() + # set custom compile options (single call to avoid overwriting) if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") npu_op_kernel_options("" ALL OPTIONS -g -O0 ${KERNEL_COMPILE_OPTS}) diff --git a/csrc/npu/ascend_kernels/grouped_matmul_swiglu_quant_v2_layered/op_kernel/grouped_matmul_swiglu_quant_v2_layered_utils.h b/csrc/npu/ascend_kernels/grouped_matmul_swiglu_quant_v2_layered/op_kernel/grouped_matmul_swiglu_quant_v2_layered_utils.h index defff5118..ce3ca6248 100644 --- a/csrc/npu/ascend_kernels/grouped_matmul_swiglu_quant_v2_layered/op_kernel/grouped_matmul_swiglu_quant_v2_layered_utils.h +++ b/csrc/npu/ascend_kernels/grouped_matmul_swiglu_quant_v2_layered/op_kernel/grouped_matmul_swiglu_quant_v2_layered_utils.h @@ -26,6 +26,7 @@ #define OP_KERNEL_GROUPED_MATMUL_DEQUANT_SWIGLU_QUANT_V2_LAYERED_UTILS_H #include "layered/layered_tensor_addr.h" +#include "afd_swiglu_limit.h" // A8W4 MSD scenario #if defined(ORIG_DTYPE_X) && defined(DT_INT8) && ORIG_DTYPE_X == DT_INT8 && defined(ORIG_DTYPE_ALL_WEIGHT) && \ @@ -71,22 +72,6 @@ constexpr int64_t SIZE_OF_HALF_2 = 2; constexpr uint8_t NUM_8 = 8; constexpr float QUANT_SCALE_INT8 = 127.0f; -// ### PATCH START: DSV4 clipped SwiGLU -// Patch reason: the model limit applies to the dequantized branches before SiLU. -// Patch functionality: clamp in UB without another kernel or GM round trip. -__aicore__ inline void ClampSwiGLUInputs(LocalTensor gate, LocalTensor up, - float limit, uint32_t count) -{ - if (limit > 0.0f) { - Mins(gate, gate, limit, count); - Mins(up, up, limit, count); - PipeBarrier(); - Maxs(up, up, -limit, count); - PipeBarrier(); - } -} -// ### PATCH END: DSV4 clipped SwiGLU - constexpr MatmulConfig matmulCFGUnitFlag{false, false, true, 0, 0, 0, false, false, false, false, false, 0, 0, 0, 0, 0, 0, 0, false}; constexpr MatmulConfig NZ_CFG_MDL = GetMDLConfig(false, false, 0, true, false, false, false); diff --git a/csrc/npu/ascend_kernels/operator_registry.json b/csrc/npu/ascend_kernels/operator_registry.json index 2e152dd13..03375e3a2 100644 --- a/csrc/npu/ascend_kernels/operator_registry.json +++ b/csrc/npu/ascend_kernels/operator_registry.json @@ -9,7 +9,9 @@ "afd_async_combine_send", "afd_async_combine_recv", "grouped_matmul_swiglu_quant_v2_layered", - "grouped_matmul_layered" + "grouped_matmul_layered", + "afd_mega_moe_pull_native", + "afd_mega_moe_comm" ], "ascend950": [ "a2e", @@ -38,8 +40,14 @@ "grouped_matmul_swiglu_quant_v2_layered": { "requires_shmem": false }, + "afd_mega_moe_pull_native": { + "requires_shmem": false + }, "grouped_matmul_layered": { "requires_shmem": false + }, + "afd_mega_moe_comm": { + "requires_shmem": false } } } diff --git a/csrc/npu/ascend_kernels/utils/op_kernel/afd_native_mega_workspace.h b/csrc/npu/ascend_kernels/utils/op_kernel/afd_native_mega_workspace.h new file mode 100644 index 000000000..abdc5c2cd --- /dev/null +++ b/csrc/npu/ascend_kernels/utils/op_kernel/afd_native_mega_workspace.h @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +#ifndef AFD_NATIVE_MEGA_WORKSPACE_H +#define AFD_NATIVE_MEGA_WORKSPACE_H +#include +#include "afd_pull_layout.h" + +#ifdef __CCE_AICORE__ +#define AFD_NATIVE_DEVICE __aicore__ +#else +#define AFD_NATIVE_DEVICE +#endif + +namespace Cam { +constexpr uint32_t NATIVE_MAX_AIVS = 48; +constexpr uint32_t NATIVE_STAGES = 5; +constexpr uint32_t NATIVE_MAX_ROWS = Pull::MAX_CAPACITY; +constexpr uint32_t NATIVE_METADATA_ROWS = 512; +constexpr uint32_t NATIVE_TILE_ROWS = 64; +AFD_NATIVE_DEVICE constexpr uint64_t NativeAlign(uint64_t n) { return (n + 511) / 512 * 512; } + +struct NativeMegaWorkspace { + uint64_t batch, counts, tileCounts, layer, flags, scale1, scale2; + uint64_t x, a1, a2, c1, c2, output, bytes; + AFD_NATIVE_DEVICE NativeMegaWorkspace(uint64_t rows, uint64_t h, uint64_t i, + uint64_t experts, uint64_t tp) { + batch = 0; + counts = NativeAlign(8 * Pull::MetadataWords(tp, experts)); + tileCounts = counts + NativeAlign(8 * experts); + layer = tileCounts + NativeAlign(8 * experts * ((rows + NATIVE_TILE_ROWS - 1) / NATIVE_TILE_ROWS)); + flags = layer + 512; + scale1 = flags + NativeAlign(NATIVE_STAGES * (NATIVE_MAX_AIVS + 1) * 64); + scale2 = scale1 + NativeAlign(4 * rows); + x = scale2 + NativeAlign(4 * rows); + a1 = x + NativeAlign(rows * h); + a2 = a1 + NativeAlign(rows * h); + c1 = a2 + NativeAlign(rows * i); + c2 = c1 + NativeAlign(8 * rows * i); + output = c2 + NativeAlign(4 * rows * h); + // Full BF16 rows are published locally before asynchronous remote TX. + bytes = output + NativeAlign(2 * rows * h); + } +}; +} // namespace Cam +#undef AFD_NATIVE_DEVICE +#endif diff --git a/csrc/npu/ascend_kernels/utils/op_kernel/afd_pull_layout.h b/csrc/npu/ascend_kernels/utils/op_kernel/afd_pull_layout.h new file mode 100644 index 000000000..8c51b1685 --- /dev/null +++ b/csrc/npu/ascend_kernels/utils/op_kernel/afd_pull_layout.h @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +#ifndef AFD_PULL_LAYOUT_H +#define AFD_PULL_LAYOUT_H + +#include + +#ifdef __CCE_AICORE__ +#define AFD_PULL_DEVICE __aicore__ +#else +#define AFD_PULL_DEVICE +#endif + +namespace Cam { namespace Pull { +constexpr uint32_t VERSION = 1; +constexpr uint32_t ALIGNMENT = 64; +constexpr uint32_t MAX_AIVS = 48; +constexpr uint32_t ROW_PADDING = 512; +constexpr uint64_t MAX_SOURCE_ASSIGNMENTS = 65535; +constexpr uint64_t MAX_CAPACITY = 262144; +constexpr int64_t MAGIC = 0x41464401; +constexpr int64_t INIT_MAGIC = 0x41464402; +constexpr uint32_t WORDS_PER_LINE = ALIGNMENT / sizeof(int64_t); +enum State : uint32_t { + SESSION = 0, SOURCE_CAPACITY = 8, TP_SIZE = 9, MAP_SIZE = 16, FAILURE = 24, + SOURCE = 32, ACTIVE_SEQUENCE = 40, ROUND_BEGIN = 48, ROUND_END = 56, + IDLE_BUDGET = 64, ACTIVE_LAYER = 72, CURSOR = 80, NEXT_SOURCE = 88, + COMPLETED = 96, STOP = 104, LAST_SEQUENCE = 112 +}; +AFD_PULL_DEVICE constexpr uint64_t LayerMapOffset(uint64_t sources) { + return LAST_SEQUENCE + sources * WORDS_PER_LINE; +} +AFD_PULL_DEVICE constexpr uint64_t LaneErrorsOffset(uint64_t sources, uint64_t mapSize) { + return LayerMapOffset(sources) + (mapSize + WORDS_PER_LINE - 1) / WORDS_PER_LINE * WORDS_PER_LINE; +} +AFD_PULL_DEVICE constexpr uint64_t StateWords(uint64_t sources, uint64_t mapSize) { + return LaneErrorsOffset(sources, mapSize) + MAX_AIVS * WORDS_PER_LINE; +} + +// One control line has one writer; payload/counts precede the commit line. +enum Header : uint32_t { EPOCH, SEQUENCE, LAYER, TOKENS, ROWS, COMMIT, ERROR, HEADER_LINES }; + +struct Shape { + uint64_t sourceCapacity, hidden, topK, ffnRanks, attnRanks, experts, tp; +}; + +AFD_PULL_DEVICE constexpr uint64_t Align(uint64_t n) { + return (n + ALIGNMENT - 1) / ALIGNMENT * ALIGNMENT; +} + +struct Layout { + uint64_t stats, flags, returned, header, errors, counts, payload, rowStride, bytes; +}; + +// The old Combine reads 32B-aligned statistics and flags at these exact offsets. +AFD_PULL_DEVICE constexpr uint64_t Align32(uint64_t n) { return (n + 31) / 32 * 32; } + +AFD_PULL_DEVICE inline bool MakeLayout(const Shape &s, Layout &out) { + if (s.sourceCapacity == 0 || s.hidden == 0 || s.hidden > 8192 || s.hidden % 256 || + s.topK == 0 || s.ffnRanks == 0 || s.attnRanks == 0 || s.experts == 0 || + s.ffnRanks > 384 || s.attnRanks > 384 || s.experts > 256 || + s.ffnRanks + s.attnRanks > 384 || s.tp == 0 || s.attnRanks % s.tp || + s.topK > s.ffnRanks * s.experts || + s.sourceCapacity > MAX_SOURCE_ASSIGNMENTS / s.topK || + s.sourceCapacity > MAX_CAPACITY / s.tp) return false; + out.stats = 0; + out.flags = Align32(4 * (s.ffnRanks + s.ffnRanks * s.experts) * MAX_AIVS); + out.returned = out.flags + Align32(4 * s.ffnRanks); + out.header = Align(out.returned + 2 * s.sourceCapacity * s.topK * s.hidden); + out.errors = out.header + HEADER_LINES * ALIGNMENT; + out.counts = Align(out.errors + s.ffnRanks * ALIGNMENT); + out.payload = Align(out.counts + 8 * s.ffnRanks * s.experts); + out.rowStride = s.hidden + ROW_PADDING; + out.bytes = out.payload + s.sourceCapacity * s.topK * out.rowStride; + return true; +} + +AFD_PULL_DEVICE constexpr uint64_t MetadataWords(uint64_t tp, uint64_t experts) { + return 7 + tp + tp * experts; +} +AFD_PULL_DEVICE constexpr uint64_t TransportUbBytes(const Shape &s) { + return ALIGNMENT + Align(8 * MetadataWords(s.tp, s.experts)) + + Align(8 * s.ffnRanks * s.experts) + Align(8 * s.experts) + + Align(2 * s.hidden + ROW_PADDING); +} + +// Publish retains these buffers alongside Transport for the entire phase. +AFD_PULL_DEVICE inline uint64_t PublisherStatsBytes(const Shape &s, uint64_t cores) { + const auto stats = Align(4 * (s.ffnRanks + s.ffnRanks * s.experts) * cores); + // Reuse this buffer for the per-lane error lines before loading token offsets. + return stats > cores * ALIGNMENT ? stats : cores * ALIGNMENT; +} +AFD_PULL_DEVICE inline uint64_t PublisherUbBytes(const Shape &s, uint64_t cores) { + const auto e = s.ffnRanks * s.experts; + return PublisherStatsBytes(s, cores) + 2 * Align(4 * e) + + Align(4 * s.topK) + Align(11 * s.hidden + 128); +} + +// Retain the former admission bound while accounting for Mega's per-token routes. +AFD_PULL_DEVICE inline uint64_t CombineUbBytes(const Shape &s, uint64_t cores) { + const auto e = s.ffnRanks * s.experts; + const auto base = 65536 + Align32(2 * e); + const auto preprocess = base + Align32(4 * (s.ffnRanks + e) * cores); + const auto retained = base + Align32(4 * e) + Align32(2 * e) + + 3 * Align32(4 * s.hidden) + Align32(2 * s.hidden) + Align32(4 * s.ffnRanks); + const auto legacy = preprocess > retained ? preprocess : retained; + const auto mega = 2 * Align(4 * e) + 2 * Align(4 * s.topK) + Align(14 * s.hidden); + return legacy > mega ? legacy : mega; +} + +struct Slice { + uint64_t count, skip, local; +}; + +// Empty intersections carry no usable source offset. +AFD_PULL_DEVICE inline Slice Intersect(uint64_t block, uint64_t count, + uint64_t begin, uint64_t capacity) { + const uint64_t lo = block > begin ? block : begin; + const uint64_t end = begin + capacity; + const uint64_t hi = block + count < end ? block + count : end; + return hi > lo ? Slice{hi - lo, lo - block, lo - begin} : Slice{0, 0, 0}; +} + +AFD_PULL_DEVICE constexpr uint64_t Rounds(uint64_t rows, uint64_t capacity) { + return rows == 0 ? 1 : 1 + (rows - 1) / capacity; +} + +AFD_PULL_DEVICE constexpr uint64_t ReturnRow(uint64_t rankPrefix, uint64_t expertPrefix, + uint64_t skip, uint64_t row) { + return rankPrefix + expertPrefix + skip + row; +} +} } // namespace Cam::Pull + +#undef AFD_PULL_DEVICE +#endif diff --git a/csrc/npu/ascend_kernels/utils/op_kernel/afd_pull_protocol.h b/csrc/npu/ascend_kernels/utils/op_kernel/afd_pull_protocol.h new file mode 100644 index 000000000..dd1033e3f --- /dev/null +++ b/csrc/npu/ascend_kernels/utils/op_kernel/afd_pull_protocol.h @@ -0,0 +1,352 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +#ifndef AFD_PULL_PROTOCOL_H +#define AFD_PULL_PROTOCOL_H + +#include "kernel_operator.h" +#include "cam_async_comm_group.h" +#include "afd_pull_layout.h" + +namespace Cam::Pull { +template __aicore__ inline void Fence() { + const auto id = GetTPipePtr()->FetchEventID(event); + AscendC::SetFlag(id); + AscendC::WaitFlag(id); +} + +// Uses the SDK's complete context; ordinary tensor addresses are never peer addresses. +__aicore__ inline GM_ADDR Window(uint32_t rank, uint32_t self) { + auto ctx = reinterpret_cast<__gm__ HcclOpResParam *>(AscendC::GetHcclContext()); + if (rank == self) return reinterpret_cast(ctx->localWindowsIn); + return reinterpret_cast(reinterpret_cast<__gm__ HcclRankRelationResV2 *>( + ctx->remoteRes[rank].nextDevicePtr)->windowsIn); +} + +class Transport { +public: + __aicore__ Transport(GM_ADDR state, Shape shape, uint32_t rank, uint32_t cores, + AscendC::TPipe &pipe, uint32_t id = AscendC::GetBlockIdx()) + : state_(state), shape_(shape), rank_(rank), cores_(cores), id_(id), pipe_(pipe) { + pipe_.Reset(); + pipe_.InitBuffer(control_, ALIGNMENT); + MakeLayout(shape_, layout_); + const auto ctx = reinterpret_cast<__gm__ HcclOpResParam *>( + AscendC::GetHcclContext()); + if (ctx->winSize < layout_.bytes || ctx->rankSize != shape_.attnRanks + shape_.ffnRanks || + ctx->localUsrRankId != rank_) { + if (id_ == 0) WriteState(FAILURE, -7); + } + if (!ReadState(FAILURE) && + (ReadState(SOURCE_CAPACITY) != int64_t(shape_.sourceCapacity) || + ReadState(TP_SIZE) != int64_t(shape_.tp))) { + if (id_ == 0) Fail(0, -6); + } + const uint64_t metaBytes = Align(8 * MetadataWords(shape_.tp, shape_.experts)); + pipe_.InitBuffer(metadata_, metaBytes); + pipe_.InitBuffer(counts_, Align(8 * shape_.ffnRanks * shape_.experts)); + pipe_.InitBuffer(groups_, Align(8 * shape_.experts)); + pipe_.InitBuffer(row_, Align(2 * shape_.hidden + ROW_PADDING)); + } + + __aicore__ int64_t Read(GM_ADDR address) { + AscendC::GlobalTensor gm; + gm.SetGlobalBuffer(reinterpret_cast<__gm__ int64_t *>(address)); + auto value = control_.Get(); + AscendC::DataCopyPad(value, gm, {1U, 8U, 0U, 0U, 0U}, {false, 0U, 0U, 0U}); + Fence(); + const int64_t result = value.GetValue(0); + Fence(); + return result; + } + + __aicore__ void Write(GM_ADDR address, int64_t value) { + auto local = control_.Get(); + local.SetValue(0, value); + AscendC::GlobalTensor gm; + gm.SetGlobalBuffer(reinterpret_cast<__gm__ int64_t *>(address)); + Fence(); + AscendC::DataCopyPad(gm, local, {1U, 8U, 0U, 0U, 0U}); + Fence(); + } + + __aicore__ int64_t ReadState(uint32_t field) { return Read(state_ + field * 8ULL); } + __aicore__ void WriteState(uint32_t field, int64_t value) { Write(state_ + field * 8ULL, value); } + + __aicore__ void Initialize() { + if (rank_ >= shape_.attnRanks) return; + AscendC::SyncAll(); + if (ReadState(FAILURE)) return; + auto zero = row_.Get(); + AscendC::Duplicate(row_.Get(), uint32_t(0), ALIGNMENT / sizeof(uint32_t)); + Fence(); + // Keep the payload untouched; only protocol state requires initialization. + for (uint64_t offset = id_ * ALIGNMENT; + offset < layout_.counts; offset += cores_ * ALIGNMENT) { + if (offset >= layout_.returned && offset < layout_.header) continue; + AscendC::GlobalTensor dst; + dst.SetGlobalBuffer(reinterpret_cast<__gm__ uint8_t *>(Peer(rank_) + offset)); + AscendC::DataCopy(dst, zero, ALIGNMENT); + } + Fence(); + AscendC::SyncAll(); + if (id_ == 0) + Write(Peer(rank_) + layout_.header + EPOCH * ALIGNMENT, ReadState(SESSION)); + } + + __aicore__ void Fail(uint32_t source, int64_t error) { + const auto first = ReadState(FAILURE); + if (!first) WriteState(FAILURE, error); + else error = first; + if (rank_ >= shape_.attnRanks) { + for (uint32_t s = 0; s < shape_.tp; ++s) + Write(Peer(source + s) + layout_.errors + + (rank_ - shape_.attnRanks) * ALIGNMENT, error); + } + } + + __aicore__ bool Select() { + if (ReadState(FAILURE)) return false; + if (ReadState(SOURCE) >= 0) return true; + if (ReadState(STOP)) return false; + const auto sources = shape_.attnRanks / shape_.tp; + const auto first = uint32_t(ReadState(NEXT_SOURCE)); + const auto budget = ReadState(IDLE_BUDGET); + const auto session = ReadState(SESSION); + uint64_t scans = 0; + while (!ReadState(STOP)) { + for (uint32_t n = 0; n < sources; ++n) { + const uint32_t dp = (first + n) % sources; + const uint32_t source = dp * shape_.tp; + const auto expected = ReadState(LAST_SEQUENCE + dp * WORDS_PER_LINE) + 1; + bool ready = true; + int64_t layer = -1; + for (uint32_t s = 0; s < shape_.tp; ++s) { + const auto header = Peer(source + s) + layout_.header; + const auto commit = Read(header + COMMIT * ALIGNMENT); + if (commit < expected) { ready = false; break; } + AscendC::GlobalTensor gm; + gm.SetGlobalBuffer(reinterpret_cast<__gm__ int64_t *>(header)); + auto values = row_.Get(); + AscendC::DataCopy(values, gm, HEADER_LINES * WORDS_PER_LINE); + Fence(); + if (commit != expected || values.GetValue(EPOCH * WORDS_PER_LINE) != session) { + Fail(source, -2); return false; + } + const auto currentLayer = values.GetValue(LAYER * WORDS_PER_LINE); + if (values.GetValue(ERROR * WORDS_PER_LINE) || + values.GetValue(SEQUENCE * WORDS_PER_LINE) != commit || + values.GetValue(COMMIT * WORDS_PER_LINE) != commit || + (layer >= 0 && layer != currentLayer)) { + Fail(source, -3); return false; + } + layer = currentLayer; + if (Read(header + COMMIT * ALIGNMENT) != commit) { ready = false; break; } + } + if (!ready) continue; + const auto mapSize = ReadState(MAP_SIZE); + if (layer < 0 || layer >= mapSize || + Read(state_ + 8 * (LayerMapOffset(sources) + layer)) < 0) { + Fail(source, -4); return false; + } + WriteState(SOURCE, source); + WriteState(ACTIVE_SEQUENCE, expected); + WriteState(ACTIVE_LAYER, layer); + WriteState(CURSOR, 0); + return true; + } + if (budget > 0 && ++scans >= uint64_t(budget)) return false; + } + return false; + } + + // Load immutable original counts; slice counts are separate from return coordinates. + __aicore__ uint64_t LoadMetadata() { + auto meta = metadata_.Get(); + const auto source = uint32_t(ReadState(SOURCE)); + const uint32_t ffn = rank_ - shape_.attnRanks; + uint64_t total = 0; + for (uint32_t s = 0; s < shape_.tp; ++s) { + AscendC::GlobalTensor gm; + gm.SetGlobalBuffer(reinterpret_cast<__gm__ int64_t *>(Peer(source + s) + layout_.counts)); + auto counts = counts_.Get(); + AscendC::DataCopyPad(counts, gm, + {1U, uint32_t(8 * shape_.ffnRanks * shape_.experts), 0U, 0U, 0U}, + {false, 0U, 0U, 0U}); + Fence(); + uint64_t prefix = 0, sourceTotal = 0; + for (uint32_t i = 0; i < shape_.ffnRanks * shape_.experts; ++i) { + const auto count = counts.GetValue(i); + if (count < 0 || uint64_t(count) > shape_.sourceCapacity * shape_.topK) { + if (id_ == 0) Fail(source, -5); + return 0; + } + if (i < ffn * shape_.experts) prefix += count; + sourceTotal += count; + if (i / shape_.experts == ffn) { + const uint32_t index = 5 + shape_.tp + s * shape_.experts + i % shape_.experts; + meta.SetValue(index, count); + total += count; + } + } + if (sourceTotal > shape_.sourceCapacity * shape_.topK || + uint64_t(Read(Peer(source + s) + layout_.header + ROWS * ALIGNMENT)) != sourceTotal) { + if (id_ == 0) Fail(source, -5); + return 0; + } + meta.SetValue(5 + s, prefix); + Fence(); + } + meta.SetValue(0, total); + meta.SetValue(1, source); + meta.SetValue(2, ReadState(ACTIVE_LAYER)); + meta.SetValue(MetadataWords(shape_.tp, shape_.experts) - 2, ReadState(SESSION)); + meta.SetValue(MetadataWords(shape_.tp, shape_.experts) - 1, ReadState(ACTIVE_SEQUENCE)); + return total; + } + + // Mega's fused pipeline owns payload DMA; this prepares only control metadata. + __aicore__ void PrepareRound(GM_ADDR batch, GM_ADDR groupCounts, uint64_t capacity) { + const bool continuing = ReadState(SOURCE) >= 0; + Select(); + auto meta = metadata_.Get(); + auto groups = groups_.Get(); + if (ReadState(FAILURE) || ReadState(SOURCE) < 0) { + ClearMetadata(); + SaveMetadata(batch, groupCounts); + return; + } + // A retains the original counts until the last round has returned its output. + if (continuing) LoadSnapshot(batch); + else LoadMetadata(); + const uint64_t total = meta.GetValue(0); + const auto source = ReadState(SOURCE); + const auto sequence = ReadState(ACTIVE_SEQUENCE); + if (!ReadState(FAILURE)) { + for (uint32_t s = 0; s < shape_.tp; ++s) { + if (Read(Peer(source + s) + layout_.header + COMMIT * ALIGNMENT) != sequence) { + Fail(source, -9); break; + } + } + } + if (ReadState(FAILURE)) { + ClearMetadata(); + SaveMetadata(batch, groupCounts); + return; + } + const uint64_t begin = ReadState(CURSOR); + if (begin > total) { + Fail(source, -9); + ClearMetadata(); + SaveMetadata(batch, groupCounts); + return; + } + const uint64_t end = begin + capacity < total ? begin + capacity : total; + meta.SetValue(3, begin); + meta.SetValue(4, end); + if (id_ == 0) { + WriteState(ROUND_BEGIN, begin); + WriteState(ROUND_END, end); + } + uint64_t block = 0; + for (uint32_t e = 0; e < shape_.experts; ++e) { + uint64_t group = 0; + for (uint32_t s = 0; s < shape_.tp; ++s) { + const uint64_t count = meta.GetValue(5 + shape_.tp + s * shape_.experts + e); + const auto slice = Intersect(block, count, begin, capacity); + group += slice.count; + block += count; + } + groups.SetValue(e, group); + } + SaveMetadata(batch, groupCounts); + Fence(); + } + + // Called only by the control owner after the mixed-core output drain. + __aicore__ bool ValidateReturn(GM_ADDR batch, uint64_t inputRows) { + if (ReadState(FAILURE) || ReadState(SOURCE) < 0) return false; + LoadSnapshot(batch); + auto meta = metadata_.Get(); + const auto source = ReadState(SOURCE); + if (source < 0 || uint64_t(source) + shape_.tp > shape_.attnRanks || + source % shape_.tp || meta.GetValue(1) != source || + meta.GetValue(2) != ReadState(ACTIVE_LAYER) || + meta.GetValue(MetadataWords(shape_.tp, shape_.experts) - 2) != ReadState(SESSION) || + meta.GetValue(MetadataWords(shape_.tp, shape_.experts) - 1) != ReadState(ACTIVE_SEQUENCE) || + meta.GetValue(3) != ReadState(CURSOR) || + meta.GetValue(3) != ReadState(ROUND_BEGIN) || + meta.GetValue(4) != ReadState(ROUND_END) || + meta.GetValue(3) < 0 || meta.GetValue(4) < meta.GetValue(3) || + meta.GetValue(4) > meta.GetValue(0) || + uint64_t(meta.GetValue(4) - meta.GetValue(3)) > inputRows) { + Fail(source >= 0 && uint64_t(source) + shape_.tp <= shape_.attnRanks ? source : 0, -9); + } + if (ReadState(FAILURE)) return false; + return true; + } + + __aicore__ void CompleteRound(GM_ADDR batch, uint64_t inputRows) { + if (!ValidateReturn(batch, inputRows)) return; + auto meta = metadata_.Get(); + const uint64_t end = meta.GetValue(4); + const uint32_t source = meta.GetValue(1); + if (end < uint64_t(meta.GetValue(0))) { WriteState(CURSOR, end); return; } + const auto sequence = ReadState(ACTIVE_SEQUENCE); + WriteState(LAST_SEQUENCE + source / shape_.tp * WORDS_PER_LINE, sequence); + WriteState(SOURCE, -1); + WriteState(CURSOR, 0); + WriteState(NEXT_SOURCE, (source / shape_.tp + 1) % (shape_.attnRanks / shape_.tp)); + WriteState(COMPLETED, ReadState(COMPLETED) + 1); + auto flag = control_.Get(); + for (uint32_t s = 0; s < shape_.tp; ++s) { + flag.SetValue(0, meta.GetValue(5 + s) + 1); + AscendC::GlobalTensor dst; + dst.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(Peer(source + s) + layout_.flags) + + rank_ - shape_.attnRanks); + Fence(); + AscendC::DataCopyPad(dst, flag, {1U, 4U, 0U, 0U, 0U}); + Fence(); + } + } + +private: + __aicore__ GM_ADDR Peer(uint32_t rank) { return Window(rank, rank_); } + __aicore__ void LoadSnapshot(GM_ADDR batch) { + AscendC::GlobalTensor gm; + gm.SetGlobalBuffer(reinterpret_cast<__gm__ int64_t *>(batch)); + AscendC::DataCopyPad(metadata_.Get(), gm, + {1U, uint32_t(8 * MetadataWords(shape_.tp, shape_.experts)), 0U, 0U, 0U}, + {false, 0U, 0U, 0U}); + Fence(); + } + __aicore__ void ClearMetadata() { + auto meta = metadata_.Get(); + auto groups = groups_.Get(); + for (uint32_t i = 0; i < MetadataWords(shape_.tp, shape_.experts); ++i) meta.SetValue(i, 0); + for (uint32_t e = 0; e < shape_.experts; ++e) groups.SetValue(e, 0); + const auto offset = LayerMapOffset(shape_.attnRanks / shape_.tp); + for (int64_t layer = 0; layer < ReadState(MAP_SIZE); ++layer) { + if (Read(state_ + 8 * (offset + layer)) >= 0) { meta.SetValue(2, layer); break; } + } + } + __aicore__ void SaveMetadata(GM_ADDR batch, GM_ADDR counts) { + if (id_ != 0) return; + AscendC::GlobalTensor dstBatch, dstCounts; + dstBatch.SetGlobalBuffer(reinterpret_cast<__gm__ int64_t *>(batch)); + dstCounts.SetGlobalBuffer(reinterpret_cast<__gm__ int64_t *>(counts)); + Fence(); + AscendC::DataCopyPad(dstBatch, metadata_.Get(), + {1U, uint32_t(8 * MetadataWords(shape_.tp, shape_.experts)), 0U, 0U, 0U}); + AscendC::DataCopyPad(dstCounts, groups_.Get(), + {1U, uint32_t(8 * shape_.experts), 0U, 0U, 0U}); + Fence(); + } + GM_ADDR state_; + Shape shape_; + Layout layout_{}; + uint32_t rank_, cores_, id_; + AscendC::TPipe &pipe_; + AscendC::TBuf control_, metadata_, counts_, groups_, row_; +}; +} // namespace Cam::Pull +#endif diff --git a/csrc/npu/ascend_kernels/utils/op_kernel/afd_swiglu_limit.h b/csrc/npu/ascend_kernels/utils/op_kernel/afd_swiglu_limit.h new file mode 100644 index 000000000..c0db806f2 --- /dev/null +++ b/csrc/npu/ascend_kernels/utils/op_kernel/afd_swiglu_limit.h @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +/** + * Shared DSV4 clipped-SwiGLU clamp used by every AFD GMM/epilogue path. + * + * The model's swiglu_limit applies to the dequantized gate/up branches before + * SiLU: gate is upper-clamped at L, up is clamped to [-L, L]. Zero disables + * clipping, matching the ACLNN and PyTorch surfaces that default to 0. + * + * This lives in utils/op_kernel so the layered W4A8/A4W4 postprocessors and + * the native MegaMoE activation share one definition instead of each carrying + * its own Mins/Maxs sequence. + */ +#ifndef AFD_SWIGLU_LIMIT_H +#define AFD_SWIGLU_LIMIT_H + +#include "kernel_operator.h" + +__aicore__ inline void ClampSwiGLUInputs(AscendC::LocalTensor gate, + AscendC::LocalTensor up, + float limit, uint32_t count) +{ + if (limit > 0.0f) { + AscendC::Mins(gate, gate, limit, count); + AscendC::Mins(up, up, limit, count); + AscendC::PipeBarrier(); + AscendC::Maxs(up, up, -limit, count); + AscendC::PipeBarrier(); + } +} + +#endif // AFD_SWIGLU_LIMIT_H diff --git a/csrc/npu/pybind/CMakeLists.txt b/csrc/npu/pybind/CMakeLists.txt index f9877545a..6b6f16291 100644 --- a/csrc/npu/pybind/CMakeLists.txt +++ b/csrc/npu/pybind/CMakeLists.txt @@ -61,7 +61,9 @@ if(AFD_SOC_VERSION STREQUAL "") endif() if(AFD_SOC_VERSION STREQUAL "910c" OR AFD_SOC_VERSION MATCHES "^ascend910_93") target_sources(afd_plugin_ascend_C PRIVATE - "${CMAKE_CURRENT_LIST_DIR}/torch_binding_cam_async.cpp") + "${CMAKE_CURRENT_LIST_DIR}/torch_binding_cam_async.cpp" + "${CMAKE_CURRENT_LIST_DIR}/torch_binding_mega_moe_pull.cpp" + "${CMAKE_CURRENT_LIST_DIR}/torch_binding_mega_moe_comm.cpp") elseif(AFD_SOC_VERSION STREQUAL "950" OR AFD_SOC_VERSION MATCHES "^[Aa]scend950") message(STATUS "CAM asynchronous bindings disabled for Ascend 950") else() diff --git a/csrc/npu/pybind/torch_binding_mega_moe_comm.cpp b/csrc/npu/pybind/torch_binding_mega_moe_comm.cpp new file mode 100644 index 000000000..b1dfab444 --- /dev/null +++ b/csrc/npu/pybind/torch_binding_mega_moe_comm.cpp @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. +// Copyright contributors to the AFD plugin project +// +// Adapted from cam_async/src/comm_operator/pybind/moe_*_async.cpp. +// Dedicated Mega pull adapters; the original CAM entry points remain unchanged. + +#include +#include +#include + +#include +#include + +#include "pytorch_extension/op_api_common.h" +#include "utils/op_kernel/afd_pull_layout.h" +#include "afd_mega_moe_comm/op_api/aclnn_afd_mega_moe_comm.h" + +namespace afd_plugin::mega_comm { +namespace { + +constexpr int64_t MAX_SEQUENCE_LENGTH = 1024 * 256; + +void check_tensor(const at::Tensor &tensor, const at::Tensor &anchor, + const char *name) { + TORCH_CHECK(!tensor.requires_grad(), name, + ": CAM asynchronous operators support inference only"); + TORCH_CHECK(tensor.device() == anchor.device(), name, + " must be on the same device as the input"); + TORCH_CHECK(tensor.is_contiguous(), name, " must be contiguous"); +} + +void check_topology(int64_t hidden_size, int64_t top_k, int64_t moe_rank_num, + int64_t attn_rank_num, int64_t route_expert_num_per_moe, + int64_t world_size) { + TORCH_CHECK(hidden_size > 0, "hidden_size must be positive"); + TORCH_CHECK(moe_rank_num > 0 && attn_rank_num > 0 && + route_expert_num_per_moe > 0, + "rank counts and route_expert_num_per_moe must be positive"); + TORCH_CHECK(world_size == moe_rank_num + attn_rank_num, + "world_size must equal attn_rank_num + moe_rank_num"); + TORCH_CHECK(top_k > 0 && top_k <= moe_rank_num * route_expert_num_per_moe, + "top_k must be in [1, moe_rank_num * route_expert_num_per_moe]"); +} + +void check_tp(int64_t max_seq_len, int64_t tp_size, int64_t attn_rank_num) { + TORCH_CHECK(tp_size > 0 && attn_rank_num % tp_size == 0, + "tp_size must be positive and divide attn_rank_num"); + TORCH_CHECK(max_seq_len >= tp_size && max_seq_len <= MAX_SEQUENCE_LENGTH && max_seq_len % tp_size == 0, + "max_seq_len must be in [tp_size, 262144] and divisible by tp_size"); +} + +void check_pull_state(const at::Tensor &state, int64_t attn_ranks, + int64_t tp_size, const at::Tensor &anchor) { + TORCH_CHECK(tp_size > 0 && attn_ranks % tp_size == 0, "pull TP topology is invalid"); + TORCH_CHECK(anchor.scalar_type() == at::kBFloat16, "pull requires BF16"); + TORCH_CHECK(state.scalar_type() == at::kHalf && state.dim() == 1 && + state.numel() >= int64_t(4 * Cam::Pull::StateWords(attn_ranks / tp_size, 0)), + "pull state is missing; initialize with afd_mega_pull_init"); +} + +void check_float_input(const at::Tensor &tensor) { + check_tensor(tensor, tensor, "input"); + TORCH_CHECK(tensor.scalar_type() == at::kHalf || + tensor.scalar_type() == at::kBFloat16, + "input must have dtype float16 or bfloat16"); +} + +} // namespace + +template +std::vector run_comm(const at::Tensor &x, const at::Tensor &ids, + const at::Tensor &probabilities, const at::Tensor &state, + at::IntArrayRef c, c10::string_view group_name) { + Cam::Pull::Layout layout{}; + TORCH_CHECK(Cam::Pull::MakeLayout({uint64_t(c[1] / c[8]), uint64_t(c[2]), uint64_t(c[3]), + uint64_t(c[4]), uint64_t(c[5]), uint64_t(c[6]), uint64_t(c[8])}, layout), "invalid Mega communication layout"); + std::vector out{ + at::empty({c[0] == 4 ? std::max(1, c[9]) * c[2] : 1}, x.options()), + at::empty({1}, x.options().dtype(at::kLong))}; + if constexpr (EXECUTE_NPU) { + std::string group(group_name.data(), group_name.size()); + auto group_ptr = group.data(); + auto &combined = out[0], &completion = out[1]; + EXEC_NPU_CMD(aclnnAfdMegaMoeComm, x, ids, probabilities, state, c, group_ptr, + combined, completion); + } + return out; +} + +// Source: moe_dispatch_send_async.cpp::cam_dispatch_send_async_impl_npu/meta. +// AFD adapter: shared NPU/Meta validation and allocation; no autograd wrapper. +// Signature retains the source arguments; the template chooses execution only. +template +at::Tensor dispatch_send( + const at::Tensor &x, const at::Tensor &expert_ids, + const at::Tensor &comm_args, const int64_t comm_id, + const int64_t max_seq_len, const int64_t batch_size, + const int64_t hidden_size, const int64_t top_k, + const int64_t moe_rank_num, const int64_t attn_rank_num, + const int64_t route_expert_num_per_moe, const int64_t attn_rank_id, + const int64_t world_size, const int64_t layer_index, + const int64_t tp_size, const int64_t dynamic_quant, + c10::string_view group_name) { + TORCH_CHECK(comm_id == Cam::Pull::MAGIC, "Mega communication requires pull_v1"); + check_float_input(x); + check_pull_state(comm_args, attn_rank_num, tp_size, x); + check_tensor(expert_ids, x, "expert_ids"); + check_tensor(comm_args, x, "comm_args"); + check_topology(hidden_size, top_k, moe_rank_num, attn_rank_num, + route_expert_num_per_moe, world_size); + check_tp(max_seq_len, tp_size, attn_rank_num); + TORCH_CHECK(batch_size >= 0 && + batch_size <= max_seq_len / tp_size, + "batch_size is outside the dispatch envelope"); + TORCH_CHECK(attn_rank_id >= 0 && attn_rank_id < attn_rank_num, + "attn_rank_id must identify an Attention rank"); + TORCH_CHECK(layer_index >= 0, "layer_index must be nonnegative"); + TORCH_CHECK(dynamic_quant == 1, "pull requires dynamic_quant=1"); + TORCH_CHECK(x.dim() == 2 && x.size(0) == batch_size && x.size(1) == hidden_size, + "x must have shape [batch_size, hidden_size]"); + TORCH_CHECK(expert_ids.scalar_type() == at::kInt && expert_ids.dim() == 2 && + expert_ids.size(0) == batch_size && expert_ids.size(1) == top_k, + "expert_ids must be int32[batch_size, top_k]"); + TORCH_CHECK(comm_args.scalar_type() == at::kHalf, + "comm_args must have dtype float16"); + auto probs = at::empty({1}, x.options().dtype(at::kFloat)); + return run_comm(x, expert_ids, probs, comm_args, + {1,max_seq_len,hidden_size,top_k,moe_rank_num,attn_rank_num,route_expert_num_per_moe, + attn_rank_id,tp_size,batch_size,layer_index}, group_name)[1]; +} + +// Source: moe_combine_recv_async.cpp::cam_combine_recv_async_impl_npu/meta. +// Empty transactions still launch a physical output row to consume completion flags. +// Signature retains the source arguments; the template chooses execution only. +template +at::Tensor combine_recv( + const at::Tensor &expand_x, const at::Tensor &expert_ids, + const at::Tensor &expert_scales, const at::Tensor &comm_args, + const int64_t comm_id, const int64_t batch_size, const int64_t hidden_size, + const int64_t top_k, const int64_t moe_rank_num, const int64_t attn_rank_num, + const int64_t route_expert_num_per_moe, const int64_t attn_rank_id, + const int64_t world_size, c10::string_view group_name, + int64_t max_seq_len, int64_t tp_size) { + TORCH_CHECK(comm_id == Cam::Pull::MAGIC, "Mega communication requires pull_v1"); + check_float_input(expand_x); + check_tp(max_seq_len, tp_size, attn_rank_num); + check_pull_state(comm_args, attn_rank_num, tp_size, expand_x); + check_tensor(expert_ids, expand_x, "expert_ids"); + check_tensor(expert_scales, expand_x, "expert_scales"); + check_tensor(comm_args, expand_x, "comm_args"); + check_topology(hidden_size, top_k, moe_rank_num, attn_rank_num, + route_expert_num_per_moe, world_size); + TORCH_CHECK(batch_size >= 0 && + batch_size <= MAX_SEQUENCE_LENGTH, + "batch_size is outside the combine envelope"); + TORCH_CHECK(attn_rank_id >= 0 && attn_rank_id < attn_rank_num, + "attn_rank_id must identify an Attention rank"); + TORCH_CHECK(expand_x.dim() == 1 && expand_x.size(0) == 1, + "expand_x must be a one-element anchor"); + TORCH_CHECK(expert_ids.scalar_type() == at::kInt && expert_ids.dim() == 2 && + expert_ids.size(0) == batch_size && expert_ids.size(1) == top_k, + "expert_ids must be int32[batch_size, top_k]"); + TORCH_CHECK(expert_scales.scalar_type() == at::kFloat && + expert_scales.dim() == 2 && expert_scales.size(0) == batch_size && + expert_scales.size(1) == top_k, + "expert_scales must be float32[batch_size, top_k]"); + TORCH_CHECK(comm_args.scalar_type() == at::kHalf, + "comm_args must have dtype float16"); + TORCH_CHECK(batch_size <= max_seq_len / tp_size, "batch_size exceeds Mega source capacity"); + auto out = run_comm(expand_x, expert_ids, expert_scales, comm_args, + {4,max_seq_len,hidden_size,top_k,moe_rank_num,attn_rank_num,route_expert_num_per_moe, + attn_rank_id,tp_size,batch_size,0}, group_name)[0]; + return out.view({std::max(1,batch_size),hidden_size}).narrow(0,0,batch_size); +} + +at::Tensor pull_init(const at::Tensor &anchor, at::IntArrayRef config, + at::IntArrayRef layer_ids, int64_t epoch, c10::string_view group_name) { + TORCH_CHECK(config.size() == 8, "pull config must be [Ncap,H,K,F,A,E,rank,TP]"); + const auto c = config.data(); + Cam::Pull::Layout layout{}; + TORCH_CHECK(Cam::Pull::MakeLayout({uint64_t(c[0]), uint64_t(c[1]), uint64_t(c[2]), + uint64_t(c[3]), uint64_t(c[4]), uint64_t(c[5]), uint64_t(c[7])}, layout), + "invalid pull layout or uint16 Combine bound"); + TORCH_CHECK(c[6] >= 0 && c[6] < c[3] + c[4] && epoch > 0, + "pull rank/epoch is invalid"); + TORCH_CHECK(anchor.scalar_type() == at::kBFloat16, "pull requires BF16"); + int64_t map_size = 0; + for (auto layer : layer_ids) { + TORCH_CHECK(layer >= 0 && layer < 65536, "pull layer map is out of bounds"); + map_size = std::max(map_size, layer + 1); + } + TORCH_CHECK(c[6] < c[4] || !layer_ids.empty(), "FFN pull requires a layer table"); + std::vector values(Cam::Pull::StateWords(c[4] / c[7], map_size), 0); + values[Cam::Pull::SESSION] = epoch; + values[Cam::Pull::SOURCE_CAPACITY] = c[0]; + values[Cam::Pull::TP_SIZE] = c[7]; + values[Cam::Pull::MAP_SIZE] = map_size; + values[Cam::Pull::SOURCE] = -1; + const auto offset = Cam::Pull::LayerMapOffset(c[4] / c[7]); + std::fill(values.begin() + offset, values.begin() + offset + map_size, -1); + for (int64_t slot = 0; slot < int64_t(layer_ids.size()); ++slot) { + const auto layer = layer_ids[slot]; + TORCH_CHECK(layer >= 0 && values[offset + layer] == -1, "pull layer IDs must be unique and nonnegative"); + values[offset + layer] = slot; + } + auto host = at::from_blob(values.data(), {int64_t(values.size())}, at::TensorOptions().dtype(at::kLong)); + auto state = host.to(anchor.device(), at::kLong, false, true).view(at::kHalf); + if (anchor.is_meta()) return state; + auto ids = at::empty({1}, anchor.options().dtype(at::kInt)); + auto probs = at::empty({1}, anchor.options().dtype(at::kFloat)); + run_comm(anchor, ids, probs, state, + {0,c[0]*c[7],c[1],c[2],c[3],c[4],c[5],c[6],c[7],0,0}, group_name); + return state; +} + +} // namespace afd_plugin::mega_comm + +TORCH_LIBRARY_FRAGMENT(afd_ascend, ops) { + ops.def("afd_mega_pull_init(Tensor anchor, int[] config, int[] layer_ids, int epoch, str group_name) -> Tensor"); + ops.def("afd_mega_dispatch_send(Tensor x, Tensor expert_ids, Tensor(a!) comm_args, " + "int comm_id, int max_seq_len, int batch_size, int hidden_size, int top_k, " + "int moe_rank_num, int attn_rank_num, int route_expert_num_per_moe, " + "int attn_rank_id, int world_size, int layer_index, int tp_size, " + "int dynamic_quant, str group_name) -> Tensor"); + ops.def("afd_mega_combine_recv(Tensor expand_x, Tensor expert_ids, " + "Tensor expert_scales, Tensor(a!) comm_args, int comm_id, int batch_size, " + "int hidden_size, int top_k, int moe_rank_num, int attn_rank_num, " + "int route_expert_num_per_moe, int attn_rank_id, int world_size, " + "str group_name, int max_seq_len, int tp_size) -> Tensor"); +} + +TORCH_LIBRARY_IMPL(afd_ascend, PrivateUse1, ops) { + ops.impl("afd_mega_pull_init", &afd_plugin::mega_comm::pull_init); + ops.impl("afd_mega_dispatch_send", &afd_plugin::mega_comm::dispatch_send); + ops.impl("afd_mega_combine_recv", &afd_plugin::mega_comm::combine_recv); +} + +// Explicitly enforce the inference-only contract, including when grad mode is on. +TORCH_LIBRARY_IMPL(afd_ascend, AutogradPrivateUse1, ops) { + ops.impl("afd_mega_dispatch_send", &afd_plugin::mega_comm::dispatch_send); + ops.impl("afd_mega_combine_recv", &afd_plugin::mega_comm::combine_recv); +} + +TORCH_LIBRARY_IMPL(afd_ascend, Meta, ops) { + ops.impl("afd_mega_pull_init", &afd_plugin::mega_comm::pull_init); + ops.impl("afd_mega_dispatch_send", &afd_plugin::mega_comm::dispatch_send); + ops.impl("afd_mega_combine_recv", &afd_plugin::mega_comm::combine_recv); +} diff --git a/csrc/npu/pybind/torch_binding_mega_moe_pull.cpp b/csrc/npu/pybind/torch_binding_mega_moe_pull.cpp new file mode 100644 index 000000000..ef0a17d15 --- /dev/null +++ b/csrc/npu/pybind/torch_binding_mega_moe_pull.cpp @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +#include +#include +#include +#include +#include "pytorch_extension/op_api_common.h" +#include "pytorch_extension/NPUBridge.h" +#include "utils/op_kernel/afd_native_mega_workspace.h" +#include "utils/op_kernel/afd_pull_layout.h" +#include "afd_mega_moe_pull_native/op_api/aclnn_afd_mega_moe_pull_native.h" + +namespace { +int64_t workspace_size(int64_t rows, int64_t h, int64_t i, int64_t experts, int64_t tp) { + TORCH_CHECK(rows > 0 && rows <= Cam::NATIVE_MAX_ROWS && h > 0 && h <= 8192 && + i > 0 && i <= 8192 && h % 256 == 0 && i % 256 == 0 && + experts > 0 && experts <= 128 && tp > 0 && tp <= 384, + "native Mega workspace dimensions are unsupported"); + return Cam::NativeMegaWorkspace(rows, h, i, experts, tp).bytes; +} + +template +at::Tensor native_mega(const at::Tensor &anchor, const at::Tensor &state, + const at::Tensor &layer_map, const at::Tensor &scratch, + at::TensorList w13, at::TensorList w2, at::TensorList s13, at::TensorList s2, + at::TensorList b13, at::TensorList b2, at::IntArrayRef c, + c10::string_view group_name, double routed_scale, double swiglu_limit) { + TORCH_CHECK(c.size() == 14, "native Mega config requires 14 fields"); + for (auto n : c) TORCH_CHECK(n > 0 && n <= 262144, "native Mega config out of range"); + TORCH_CHECK(c[8] == c[4] + c[5] && c[7] >= c[5] && c[7] < c[8] && + c[0] % c[9] == 0 && c[5] % c[9] == 0 && (c[11] == 1 || c[11] == 2) && c[12] == 1 && + std::isfinite(routed_scale), "native Mega requires a valid F rank and per-channel weights"); + TORCH_CHECK(std::isfinite(swiglu_limit) && swiglu_limit >= 0.0 && + swiglu_limit <= std::numeric_limits::max(), + "native Mega swiglu_limit must be finite, nonnegative and representable as float32"); + Cam::Pull::Shape shape{uint64_t(c[0] / c[9]), uint64_t(c[1]), uint64_t(c[3]), + uint64_t(c[4]), uint64_t(c[5]), uint64_t(c[6]), uint64_t(c[9])}; + Cam::Pull::Layout layout{}; + TORCH_CHECK(Cam::Pull::MakeLayout(shape, layout), "native Mega pull shape unsupported"); + auto check = [&](const at::Tensor &x, at::ScalarType dtype) { + TORCH_CHECK(x.device() == anchor.device() && x.scalar_type() == dtype && + x.is_contiguous() && !x.requires_grad(), "native Mega tensor has wrong device, dtype or layout"); + }; + check(anchor, at::kBFloat16); + TORCH_CHECK(anchor.numel() == 1, "native Mega requires a one-element anchor"); + check(state, at::kHalf); + check(layer_map, at::kLong); + TORCH_CHECK(layer_map.dim() == 1 && layer_map.numel() > 0 && layer_map.numel() <= 65536, + "native Mega layer map must cover model layer IDs"); + TORCH_CHECK(state.dim() == 1 && state.numel() >= int64_t(4 * + Cam::Pull::StateWords(c[5] / c[9], layer_map.numel())), "native Mega pull state is missing"); + check(scratch, at::kByte); + TORCH_CHECK(scratch.dim() == 1 && scratch.numel() == workspace_size(c[10], c[1], c[2], c[6], c[9]), + "native Mega scratch has the wrong size"); + const bool packed = c[11] == 1; + TORCH_CHECK(packed || (b13.empty() && b2.empty()), "native Mega W8A8 requires empty compensation lists"); + const std::vector> lists{ + {w13, packed ? at::kInt : at::kChar}, {w2, packed ? at::kInt : at::kChar}, + {s13, packed ? at::kLong : at::kFloat}, {s2, packed ? at::kLong : at::kBFloat16}}; + for (const auto &list : lists) { + TORCH_CHECK(int64_t(list.first.size()) == c[13], "native Mega layer list length mismatch"); + for (const auto &tensor : list.first) check(tensor, list.second); + } + if (packed) { + for (const auto list : {b13, b2}) { + TORCH_CHECK(int64_t(list.size()) == c[13], "native Mega layer list length mismatch"); + for (const auto &tensor : list) check(tensor, at::kFloat); + } + } + for (int64_t l = 0; l < c[13]; ++l) { + if constexpr (EXECUTE) { + TORCH_CHECK(vllm_ascend::NPUBridge::GetNpuStorageImplDesc(w13[l]).npu_format_ == ACL_FORMAT_FRACTAL_NZ && + vllm_ascend::NPUBridge::GetNpuStorageImplDesc(w2[l]).npu_format_ == ACL_FORMAT_FRACTAL_NZ, + "native Mega requires physically NZ packed weights"); + } + TORCH_CHECK(w13[l].sizes() == at::IntArrayRef({c[6], c[1], packed ? c[2] / 4 : 2 * c[2]}) && + w2[l].sizes() == at::IntArrayRef({c[6], c[2], packed ? c[1] / 8 : c[1]}), + "native Mega weights mismatch"); + TORCH_CHECK(s13[l].numel() == c[6] * 2 * c[2] && s2[l].numel() == c[6] * c[1], + "native Mega scale dimensions mismatch"); + if (packed) TORCH_CHECK(b13[l].numel() == c[6] * 2 * c[2] && b2[l].numel() == c[6] * c[1], + "native Mega compensation dimensions mismatch"); + } + auto completion = at::empty({1}, anchor.options().dtype(at::kLong)); + if constexpr (EXECUTE) { + std::string group(group_name.data(), group_name.size()); + auto group_ptr = group.data(); + // CANN matches dynamic inputs by dtype even when the W8A8 plan never reads them. + std::vector empty; + if (!packed) empty.push_back(s13[0].slice(0, 0, 0)); + const at::TensorList compensation13 = packed ? b13 : at::TensorList(empty); + const at::TensorList compensation2 = packed ? b2 : at::TensorList(empty); + EXEC_NPU_CMD(aclnnAfdMegaMoePullNative, anchor, state, layer_map, scratch, + w13, w2, s13, s2, compensation13, compensation2, c, group_ptr, routed_scale, swiglu_limit, completion); + } + return completion; +} +} // namespace + +TORCH_LIBRARY_FRAGMENT(afd_ascend, m) { + m.def("afd_mega_moe_pull_workspace_size(int rows, int hidden, int intermediate, int experts, int tp) -> int", + &workspace_size); + m.def("afd_mega_moe_pull_native(Tensor anchor, Tensor(a!) state, Tensor layer_map, Tensor(b!) scratch, " + "Tensor[] w13, Tensor[] w2, Tensor[] s13, Tensor[] s2, Tensor[] b13, Tensor[] b2, " + "int[] config, str group_name, float routed_scale, float swiglu_limit=0.0) -> Tensor"); +} +TORCH_LIBRARY_IMPL(afd_ascend, PrivateUse1, m) { + m.impl("afd_mega_moe_pull_native", &native_mega); +} +TORCH_LIBRARY_IMPL(afd_ascend, AutogradPrivateUse1, m) { + m.impl("afd_mega_moe_pull_native", &native_mega); +} +TORCH_LIBRARY_IMPL(afd_ascend, Meta, m) { + m.impl("afd_mega_moe_pull_native", &native_mega); +} diff --git a/docs/design/module/execution_platforms.md b/docs/design/module/execution_platforms.md index 9ae4ed2e5..00813fcac 100644 --- a/docs/design/module/execution_platforms.md +++ b/docs/design/module/execution_platforms.md @@ -24,6 +24,7 @@ related_code_paths: depends_on: - "plugin_boundary.md" validation_paths: + - "tests/npu/check_pull_layout.cpp" - "tests/unit/compat/test_profiler.py" - "tests/unit/compat/test_ascend_ops.py" - "tests/unit/compat/npu/test_profiler.py" @@ -94,6 +95,20 @@ the top-level package, common configuration/validation, version checks, and the graph policy helper; importing a CUDA or Ascend runtime module requires the corresponding runtime stack. +## Backend execution session + +Connectors expose `attention_step()` and a backend-owned `FFNExecutionSession`. +The Async FFN runner and worker use its service predicate, first-step hook, +completion hook, drain callback, receiver completion and pre-release validation; +they do not inspect Pull state. Push uses the no-op default session. Pull owns +idle enablement, sticky status checks and the `quiet → drain → stopped` handshake +in `connectors/npu/async_cam_runtime/async_cam_pull.py`. Shutdown validates the +session before releasing graph, profiler, model or communication resources; +failures retain those resources. Attention batches the same device-status fence +at step exit. +Generic graph startup negotiation remains backend-owned and is imported without +loading torch, vLLM or NPU modules through the connector package namespaces. + ## Platform mechanism map | Mechanism | CUDA owner | NPU owner | diff --git a/docs/design/module/ffn_runtime.md b/docs/design/module/ffn_runtime.md index 3ed7e65b2..b12f8a5a2 100644 --- a/docs/design/module/ffn_runtime.md +++ b/docs/design/module/ffn_runtime.md @@ -15,6 +15,8 @@ primary_code_paths: - "afd_plugin/v1/worker/npu/async_ffn_two_stream.py" - "afd_plugin/v1/worker/npu/ffn_worker.py" related_code_paths: + - "afd_plugin/connectors/npu/async_cam_runtime/async_cam_startup.py" + - "afd_plugin/connectors/npu/async_cam_runtime/async_cam_pull.py" - "afd_plugin/connectors/**" - "afd_plugin/model_executor/**" - "afd_plugin/compat/patches/engine_core.py" @@ -33,8 +35,9 @@ validation_paths: - "tests/unit/v1/worker/test_model_runner_v2.py" - "tests/unit/v1/worker/test_npu_runtime.py" - "tests/unit/v1/worker/test_async_ffn_two_stream.py" - - "tests/unit/test_async_cam_startup_wiring.py" - - "tests/unit/connectors/test_async_cam_startup.py" + - "tests/unit/connectors/npu/test_async_cam_startup_wiring.py" + - "tests/unit/connectors/npu/test_async_cam_startup.py" + - "tests/unit/connectors/npu/test_async_cam_pull.py" - "tests/npu/async_cam_runtime.py" - "tests/e2e/models/deepseek_v4_flash/ffn_graph.py" - "tests/unit/compat/patches/test_engine_core.py" @@ -240,8 +243,10 @@ daemon thread: ### Connector-driven loop The Ascend worker checks `connector.control_plane`. When it is `None`, as for -CAM async, the worker calls `execute_connector_driven_step()` instead of a -control-plane receive. For each layer, the runner: +CAM async, the worker calls `execute_connector_driven_step(shutdown_requested)` +instead of a control-plane receive. The runner asks the execution session once +per step whether to keep serving, so Pull adds no extra Store round trip to the +loop. For each layer, the runner: 1. receives a capacity-sized payload and its original CAM metadata through `connector.recv_attn_output()`; diff --git a/docs/npu/ASYNC_FFN_RUNTIME.md b/docs/npu/ASYNC_FFN_RUNTIME.md index 136100682..f4e66e43f 100644 --- a/docs/npu/ASYNC_FFN_RUNTIME.md +++ b/docs/npu/ASYNC_FFN_RUNTIME.md @@ -145,7 +145,7 @@ the target stack passes those checks. ## Resource lifetime -A host event stops further batch submissions. Graph reset and model/communicator +With default Push transport, a host event stops further batch submissions. Graph reset and model/communicator release are permitted only after the receive thread exits normally and device completion is confirmed. Partial capture, partial launch, execution failure or a still-live receive thread retains resources for process cleanup. A failure @@ -159,6 +159,17 @@ zero. Test reports distinguish successful inference from cleanup, and forced termination must not be described as normal shutdown. Single-stream eager and synchronous stopping policies retain their connector/process-level cleanup boundaries. +### Opt-in Pull execution session + +`connector_extra_config.ffn_mega_moe=true` selects Pull; Push remains the default. +The connector owns its execution session and all Pull status/Store handshakes. +Attention fences pending dispatch at `attention_step()` exit. FFN keeps serving +until every Attention source is quiet, drains active work, then publishes stopped; +only then may Attention and FFN release their communication resources. Failed +handshakes retain those resources. Standalone `close()` also validates shutdown +status before release: this is an additional defensive shutdown check, not a +relocated hot-path fence. Pull cannot be combined with `ffn_two_stream`. + ## Computation semantics For W4A8, FULL and layered eager forward the same model `swiglu_limit` to W13. @@ -166,8 +177,20 @@ A positive `L` clamps gate above at `L` and up to `[-L, L]` before SiLU gating a quantization. Zero disables clipping. DeepSeek-V4's configured `10.0` therefore applies on both execution paths. The limit is a host scalar fixed at graph capture; layer selection remains controlled by the device `layer_index` tensor. -Layered W8A8 retains its unclipped computation and records the model limit as -metadata only. +Layered and Mega W8A8 retain unclipped computation regardless of the model +`swiglu_limit`; the executors pass zero and the native W8A8 Plan skips clamp. +Native Mega W4A8 applies the limit in its MC2 ACT epilogue and bounded-column +fallback. Native W8A8 uses an INT8 GEMM with INT32 intermediate results and a +bounded-column activation; weight scales stay FP32 for W13 and BF16 for W2. +Both plans share Pull transport and the BF16 return path. W8A8 applies weight +and token scales in Vector before activation, preserving the Layered rounding +sequence. The mode is selected once through `config[11]` (1=W4A8, 2=W8A8). + +The synthetic transport fixture accepts `--quantization W8A8 --swiglu-limit 0` +or `--swiglu-limit 10`; W8A8 ignores either model limit. Save matching Layered +eager outputs with `--save-outputs`, +then compare Mega eager and `--full-graph` using `--reference-outputs`; add +`--boundary` to exercise receive rounds, sparse experts and poisoned scratch. ## Validation @@ -178,28 +201,36 @@ A3/910C, 4A/4F, A TP4, F TP1, receive capacity 128 and DBO disabled. Select this topology still needs target-device validation. Synthetic per-group coverage does not qualify loading real per-group checkpoints. -- Host contracts: `tests/unit/connectors/test_async_cam_startup.py`, +- Host contracts: `tests/unit/connectors/npu/test_async_cam_pull.py`, + `tests/unit/connectors/npu/test_async_cam_startup.py`, `tests/unit/v1/worker/test_async_ffn_graph.py`, and `tests/unit/v1/worker/test_async_ffn_runtime.py`. CPU-safe two-stream coverage is in `tests/unit/v1/worker/test_async_ffn_two_stream.py` and - `tests/unit/test_async_cam_startup_wiring.py`; fakes verify host ordering and + `tests/unit/connectors/npu/test_async_cam_startup_wiring.py`; fakes verify host ordering and ownership, not device execution. - Finite native comparison: `tests/npu/async_cam_runtime.py`. Run eager with `--save-outputs PREFIX`, then FULL with `--full-graph --reference-outputs PREFIX`, keeping other arguments identical. `--boundary` covers chunking and poisoned unused capacity; `--attn-ranks 2` covers different Attention sources. The test supplies finite work counts, compares outputs and checks captured addresses. + For Pull, add `--mega --idle-shutdown` to both eager and FULL runs so the + fixture exercises the real worker drain/finish handshake. Compare FULL to its + matching ON eager reference separately from the OFF-to-ON numerical check. - Real MRV2 generation: the bounded 4A/4F fixture below. Its test observers add synchronization for functional evidence, so it is not a performance test. ```bash python -m tests.e2e.models.deepseek_v4_flash.ffn_graph \ - --mode eager --model /path/to/checkpoint --afd-host "$AFD_HOST" \ - --devices 0,1,2,3,4,5,6,7 --output-dir /path/to/eager-evidence + --mode eager --mega --model /path/to/checkpoint --afd-host "$AFD_HOST" \ + --devices 0,1,2,3,4,5,6,7 --output-dir /path/to/on-eager-evidence python -m tests.e2e.models.deepseek_v4_flash.ffn_graph \ - --mode full --model /path/to/checkpoint --afd-host "$AFD_HOST" \ - --devices 0,1,2,3,4,5,6,7 --output-dir /path/to/full-evidence \ - --reference-dir /path/to/eager-evidence + --mode full --mega --model /path/to/checkpoint --afd-host "$AFD_HOST" \ + --devices 0,1,2,3,4,5,6,7 --output-dir /path/to/on-full-evidence \ + --reference-dir /path/to/on-eager-evidence +python -m tests.e2e.models.deepseek_v4_flash.ffn_graph \ + --mode eager --model /path/to/checkpoint --afd-host "$AFD_HOST" \ + --devices 0,1,2,3,4,5,6,7 --output-dir /path/to/off-eager-evidence \ + --reference-dir /path/to/on-eager-evidence ``` The fixture saves inputs, generated tokens, actual runner identities and per-rank diff --git a/tests/e2e/README.md b/tests/e2e/README.md index de0892bcc..96095fdfb 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -400,10 +400,19 @@ SIGTERM/SIGINT cancels pending HTTP operations before service cleanup, without waiting for the request timeout. It does **not** run GSM8K or establish general model accuracy. Service liveness and owned-process cleanup are also required to pass. -This 16-NPU case allows 60 seconds for service shutdown before escalation; +NPU async CAM cases allow 60 seconds for each vLLM worker, 90 seconds for the +service, and 120 seconds for the test runner to shut down. Equivalent external +launchers should set `VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS=60` and pass +`--shutdown-timeout 90` to both services; a service timeout alone does not +change the worker deadline. Explicit worker environment overrides are retained. Attention SIGKILL escalation remains a failure. FFN uses the existing scoped async CAM cleanup exception. +vLLM 0.32 can log `AsyncLLM`'s `EngineDeadError` after an API client starts +its own shutdown. Record this as an upstream advisory only after inference +succeeds, every worker group exits gracefully, and process exit codes and +cleanup checks pass. An inference-time error or forced worker exit still fails. + ```bash export AFD_E2E_BACKEND=npu export AFD_E2E_DEVICES=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 diff --git a/tests/e2e/models/deepseek_v4_flash/ffn_graph.py b/tests/e2e/models/deepseek_v4_flash/ffn_graph.py index 4e83be3c3..71176767f 100644 --- a/tests/e2e/models/deepseek_v4_flash/ffn_graph.py +++ b/tests/e2e/models/deepseek_v4_flash/ffn_graph.py @@ -71,6 +71,7 @@ def profile(args) -> SimpleNamespace: json.dumps( { "dynamicQuant": 1, + "ffn_mega_moe": args.mega, "attn_ranks_per_dp": 4, "hccl_buffer_size": 256, } @@ -332,6 +333,7 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--mode", required=True, choices=("eager", "full")) parser.add_argument("--model", required=True) + parser.add_argument("--mega", action="store_true") parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--reference-dir", type=Path) parser.add_argument("--vllm-bin", default="vllm") diff --git a/tests/e2e/models/deepseek_v4_flash/ffn_graph_evidence.py b/tests/e2e/models/deepseek_v4_flash/ffn_graph_evidence.py index 2c7f52f29..ac1c31e15 100644 --- a/tests/e2e/models/deepseek_v4_flash/ffn_graph_evidence.py +++ b/tests/e2e/models/deepseek_v4_flash/ffn_graph_evidence.py @@ -9,10 +9,12 @@ import time from functools import wraps from pathlib import Path +from types import SimpleNamespace import torch from afd_plugin.connectors.npu.async_cam import CAMAsyncAFDConnector +from afd_plugin.model_executor.npu.async_cam_mega import AsyncCAMMegaExecutor from afd_plugin.v1.worker.npu.async_ffn_model_runner import AsyncNPUFFNModelRunner from afd_plugin.v1.worker.npu.attention_model_runner_v2 import ( AFDNPUAttentionModelRunnerV2, @@ -72,6 +74,7 @@ def afd_ffn_graph_snapshot(self) -> dict: def install_observers() -> None: complete = AsyncNPUFFNModelRunner._complete_graph send_eager = CAMAsyncAFDConnector.send_ffn_output + execute_mega = AsyncCAMMegaExecutor.__call__ execute = AFDNPUAttentionModelRunnerV2.execute_model shutdown_attention = AFDNPUAttentionModelRunnerV2.shutdown shutdown_ffn = AsyncNPUFFNModelRunner.shutdown @@ -83,6 +86,8 @@ def observe(connector, context): # RPC snapshots read this host copy without touching device buffers. state = context.states header = state.token_nums_rankid_layeridx.cpu().tolist() + if header[2] < 0: + return # Pull idle returns are not layer transactions. rows = int(state.group_list.sum().item()) previous = getattr(connector, "_e2e_completion", {}) layers = dict(previous.get("layers", {})) @@ -96,11 +101,33 @@ def observe(connector, context): "zero_rows": previous.get("zero_rows", 0) + int(rows == 0), } + def mega_context(executor): + return SimpleNamespace( + states=SimpleNamespace( + token_nums_rankid_layeridx=executor.batch_info, + group_list=executor.group_list, + ) + ) + @wraps(complete) def completed(self): result = complete(self) if result: - observe(self.connector, self._graph_payload.context) + context = ( + mega_context(self._mega_executor) + if self._mega_executor is not None + else self._graph_payload.context + ) + observe(self.connector, context) + return result + + @wraps(execute_mega) + def mega_executed(self): + result = execute_mega(self) + if self.connector.graph_startup.mode == "eager": + # Never read device headers while FULL is warming up or capturing. + torch.npu.current_stream().synchronize() + observe(self.connector, mega_context(self)) return result @wraps(send_eager) @@ -168,6 +195,7 @@ def ffn_stop_requested(self): AsyncNPUFFNModelRunner._complete_graph = completed CAMAsyncAFDConnector.send_ffn_output = sent + AsyncCAMMegaExecutor.__call__ = mega_executed AFDNPUAttentionModelRunnerV2.execute_model = executed AFDNPUAttentionModelRunnerV2.shutdown = attention_closed AsyncNPUFFNModelRunner.shutdown = ffn_closed diff --git a/tests/e2e/runner.py b/tests/e2e/runner.py index 84c4966ba..5ee36469f 100644 --- a/tests/e2e/runner.py +++ b/tests/e2e/runner.py @@ -107,6 +107,9 @@ NPU_ASYNC_PROCESS_KILL_TIMEOUT_S = 120 LOG_THREAD_JOIN_TIMEOUT_S = 2 VLLM_SHUTDOWN_TIMEOUT_S = 10 +NPU_ASYNC_VLLM_SHUTDOWN_TIMEOUT_S = 90 +NPU_ASYNC_WORKER_SHUTDOWN_TIMEOUT_S = 60 +NPU_ASYNC_TERMINATION_TIMEOUT_S = 120 DEFAULT_GSM8K_SAMPLE_LIMIT = 7 GSM8K_NUM_FEWSHOT = 8 GSM8K_FULL_SAMPLE_COUNT = 1319 @@ -688,7 +691,11 @@ def build_vllm_command( "serve", args.model, "--shutdown-timeout", - str(VLLM_SHUTDOWN_TIMEOUT_S), + str( + NPU_ASYNC_VLLM_SHUTDOWN_TIMEOUT_S + if is_npu and args.afd_async + else VLLM_SHUTDOWN_TIMEOUT_S + ), "--served-model-name", served_model_name(args, role), "--data-parallel-size", @@ -865,6 +872,8 @@ def uses_npu_async_process_cleanup(args: argparse.Namespace) -> bool: def process_termination_timeout(args: argparse.Namespace) -> float: """Return the scenario's teardown budget, shared by every runner.""" + if args.device_backend == "npu" and args.afd_async and not args.baseline: + return NPU_ASYNC_TERMINATION_TIMEOUT_S if args.scenario in DSV4_SCENARIOS: return DSV4_PROCESS_TERMINATION_TIMEOUT_S return PROCESS_TERMINATION_TIMEOUT_S @@ -1327,6 +1336,11 @@ def build_env( env[E2E_PROCESS_ROLE_ENV] = role if args.device_backend == "npu": env.pop("VLLM_ASCEND_ENABLE_FLASHCOMM1", None) + if args.afd_async and not args.baseline: + env.setdefault( + "VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS", + str(NPU_ASYNC_WORKER_SHUTDOWN_TIMEOUT_S), + ) env.pop("AFD_PLUGIN_EARLY_ENGINE_PATCH", None) current_pythonpath = env.get("PYTHONPATH") env["PYTHONPATH"] = ( diff --git a/tests/npu/async_cam_runtime.py b/tests/npu/async_cam_runtime.py index c539c89e2..cb4f1845d 100644 --- a/tests/npu/async_cam_runtime.py +++ b/tests/npu/async_cam_runtime.py @@ -31,6 +31,7 @@ from afd_plugin.connectors.npu.async_cam import CAMAsyncAFDConnector from afd_plugin.model_executor.npu.async_cam_layered import LayeredFFNWeights from afd_plugin.v1.worker.npu.async_ffn_model_runner import AsyncNPUFFNModelRunner +from afd_plugin.v1.worker.npu.ffn_worker import AFDNPUFFNWorker HIDDEN = 256 EXPERTS = 4 @@ -41,35 +42,106 @@ class SyntheticModel(torch.nn.Module): """Diagonal experts shared by eager/FULL runs; no checkpoint is loaded.""" - def __init__(self, device, per_channel): + def __init__( + self, + device, + per_channel, + hidden=HIDDEN, + intermediate=HIDDEN, + weight_seed=None, + routed_scale=1.0, + quantization="W4A8", + swiglu_limit=LIMIT, + experts=EXPERTS, + ): super().__init__() self.layers = [] for slot, layer_id in enumerate(LAYERS): - w13, s13, b13 = self.make_weight(device, per_channel, slot + 1, True) - w2, s2, b2 = self.make_weight(device, per_channel, 1, False) + seed = None if weight_seed is None else weight_seed + slot * 2 + w13, s13, b13 = self.make_weight( + device, + per_channel, + slot + 1, + True, + hidden, + intermediate, + seed, + quantization, + experts, + ) + w2, s2, b2 = self.make_weight( + device, + per_channel, + 1, + False, + hidden, + intermediate, + None if seed is None else seed + 1, + quantization, + experts, + ) self.layers.append( LayeredFFNWeights( - layer_id, w13, w2, s13, s2, b13, b2, per_channel, LIMIT, 1.0 + layer_id, + w13, + w2, + s13, + s2, + b13, + b2, + per_channel, + swiglu_limit, + routed_scale, + quantization=quantization, ) ) @staticmethod - def make_weight(device, per_channel, factor, gate_up): - width = HIDDEN * (2 if gate_up else 1) - values = torch.zeros(EXPERTS, HIDDEN, width, dtype=torch.int8) - diagonal = torch.arange(HIDDEN) - values[:, diagonal, diagonal] = 1 - if gate_up: - values[:, diagonal, diagonal + HIDDEN] = 1 + def make_weight( + device, + per_channel, + factor, + gate_up, + hidden=HIDDEN, + intermediate=HIDDEN, + seed=None, + quantization="W4A8", + experts=EXPERTS, + ): + k = hidden if gate_up else intermediate + width = 2 * intermediate if gate_up else hidden + if seed is None: + values = torch.zeros(experts, k, width, dtype=torch.int8) + diagonal = torch.arange(min(k, intermediate if gate_up else width)) + values[:, diagonal, diagonal] = 1 + if gate_up: + values[:, diagonal, diagonal + intermediate] = 1 + else: + generator = torch.Generator().manual_seed(seed) + values = torch.randint( + -3 if quantization == "W4A8" else -127, + 4 if quantization == "W4A8" else 128, + (experts, k, width), + dtype=torch.int8, + generator=generator, + ) groups = 1 if per_channel else 2 - scale = torch.full((EXPERTS, groups, width), float(factor)) + scale = torch.full((experts, groups, width), float(factor)) + if seed is not None: + scale *= 0.01 + 0.02 * torch.rand(scale.shape, generator=generator) if not per_channel: scale[:, 1] *= 2 - dequant = values.float() * scale.repeat_interleave(HIDDEN // groups, 1) + if quantization == "W8A8": + weight = torch_npu.npu_format_cast(values.to(device), 29) + weight_scale = scale.squeeze(1) + if not gate_up: + weight_scale = weight_scale.bfloat16() + return weight, weight_scale.to(device), None + dequant = values.float() * scale.repeat_interleave(k // groups, 1) bias = (8 * dequant.sum(1)).to(device) packed = values.reshape(-1, 2) packed = ((packed[:, 1] << 4) | (packed[:, 0] & 15)).reshape( - EXPERTS, HIDDEN, width // 2 + experts, k, width // 2 ) weight = torch_npu.npu_format_cast(packed.to(device), 29).view(torch.int32) encoded = scale.view(torch.int32).to(torch.int64) @@ -87,16 +159,46 @@ def get_async_cam_layered_weights(self): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--per-group", action="store_true") + parser.add_argument("--mega", action="store_true") + parser.add_argument("--quantization", choices=("W4A8", "W8A8"), default="W4A8") + parser.add_argument("--swiglu-limit", type=float, default=LIMIT) + parser.add_argument( + "--experts-per-rank", type=int, choices=(4, 32, 64), default=EXPERTS + ) + parser.add_argument("--idle-probe", action="store_true") + parser.add_argument( + "--hidden-size", type=int, choices=(256, 512, 1024, 2048), default=HIDDEN + ) + parser.add_argument( + "--intermediate-size", + type=int, + choices=(256, 512, 1024, 2048, 3072, 3328, 4096, 8192), + default=HIDDEN, + ) + parser.add_argument("--weight-seed", type=int) + parser.add_argument("--routed-scale", type=float, default=1.0) parser.add_argument("--full-graph", action="store_true") - parser.add_argument("--attn-ranks", type=int, choices=(1, 2), default=1) + parser.add_argument("--attn-ranks", type=int, choices=(1, 2, 4), default=1) + parser.add_argument("--attention-tp", type=int, choices=(1, 2, 4), default=1) parser.add_argument("--boundary", action="store_true") + parser.add_argument("--receive-capacity", type=int, default=16) + parser.add_argument("--boundary-tokens", type=int, default=16) + parser.add_argument("--idle-shutdown", action="store_true") parser.add_argument("--save-outputs") parser.add_argument("--reference-outputs") args = parser.parse_args() + if not 1 <= args.receive_capacity <= 512 or not 7 <= args.boundary_tokens <= 512: + parser.error("capacity must be in [1,512], boundary tokens in [7,512]") + if args.attn_ranks % args.attention_tp: + parser.error("Attention rank count must be divisible by TP") + if (args.idle_shutdown or args.idle_probe) and not args.mega: + parser.error("idle shutdown/probe requires --mega") + if args.per_group and (args.mega or args.quantization == "W8A8"): + parser.error("native Mega requires per-channel weights") if args.full_graph and not args.reference_outputs: parser.error("FULL replay requires saved matching eager outputs") faulthandler.dump_traceback_later(60) - os.environ["BATCH_SIZE_FACTOR"] = str(16 / 262144) + os.environ["BATCH_SIZE_FACTOR"] = str(args.receive_capacity / 262144) os.environ["HCCL_BUFFSIZE"] = "64" ensure_cam_async_ops_available() rank = int(os.environ["RANK"]) @@ -129,7 +231,8 @@ def main(): "port": port[0], "connector_extra_config": { "dynamicQuant": 1, - "attn_ranks_per_dp": 1, + "ffn_mega_moe": args.mega, + "attn_ranks_per_dp": args.attention_tp, "hccl_buffer_size": 64, }, } @@ -138,21 +241,23 @@ def main(): dtype=torch.bfloat16, enforce_eager=not (args.full_graph and is_ffn), hf_config=SimpleNamespace( - hidden_size=HIDDEN, + hidden_size=args.hidden_size, num_experts_per_tok=2, - n_routed_experts=8, + n_routed_experts=2 * args.experts_per_rank, num_hidden_layers=max(LAYERS) + 1, architectures=["DeepseekV4ForCausalLM"], model_type="deepseek_v4", ), ), - scheduler_config=SimpleNamespace(max_num_batched_tokens=16), + scheduler_config=SimpleNamespace( + max_num_batched_tokens=args.boundary_tokens * args.attention_tp + ), parallel_config=SimpleNamespace( enable_dbo=False, use_ubatching=False, num_ubatches=2, - data_parallel_size=2 if is_ffn else args.attn_ranks, - tensor_parallel_size=1, + data_parallel_size=2 if is_ffn else args.attn_ranks // args.attention_tp, + tensor_parallel_size=1 if is_ffn else args.attention_tp, pipeline_parallel_size=1, prefill_context_parallel_size=1, decode_context_parallel_size=1, @@ -174,21 +279,65 @@ def main(): parse_afd_config(config), rank - args.attn_ranks if is_ffn else rank, ) - report = {"rank": rank, "per_group": args.per_group, "work": []} + report = { + "rank": rank, + "per_group": args.per_group, + "mega": args.mega, + "quantization": args.quantization, + "swiglu_limit": args.swiglu_limit, + "experts_per_rank": args.experts_per_rank, + "hidden": args.hidden_size, + "intermediate": args.intermediate_size, + "weight_seed": args.weight_seed, + "routed_scale": args.routed_scale, + "attention_tp": args.attention_tp, + "work": [], + } print(json.dumps({"rank": rank, "stage": "connector_created"}), flush=True) if is_ffn: runner = AsyncNPUFFNModelRunner(config, device, connector) - runner.model = SyntheticModel(device, not args.per_group) + runner.model = SyntheticModel( + device, + not args.per_group, + args.hidden_size, + args.intermediate_size, + None if args.weight_seed is None else args.weight_seed + rank * 10000, + args.routed_scale, + args.quantization, + args.swiglu_limit, + args.experts_per_rank, + ) print(json.dumps({"rank": rank, "stage": "weights_ready"}), flush=True) runner.initialize_afd_connector() assert runner._layered_executor is not None + + def work_state(): + if args.mega: + mega = runner._mega_executor + return SimpleNamespace( + token_nums_rankid_layeridx=mega.batch_info, + group_list=mega.group_list, + ) + return runner._graph_payload.context.states + if args.full_graph: assert runner.effective_mode == "full" assert runner._graph_capture_count == 1 def capture_addresses(): session = runner - state = session._graph_payload.context.states + if args.mega: + mega = session._mega_executor + return tuple( + t.data_ptr() + for t in ( + mega.scratch, + mega.layer_map, + mega.anchor, + session._graph_output, + ) + ) + state = work_state() return tuple( tensor.data_ptr() for tensor in ( @@ -216,7 +365,7 @@ def forbid_python_transaction(*args, **kwargs): # Test-only per-replay observations, fenced once at the batch end. # Production batching performs no host metadata copy. session = runner - state = session._graph_payload.context.states + state = work_state() headers = torch.empty( (len(LAYERS), state.token_nums_rankid_layeridx.numel()), dtype=torch.int64, @@ -247,10 +396,12 @@ def observed_replay(): # warms the original operators concurrently before ordered DP cases. with torch.inference_mode(): if is_ffn: - for _ in range(args.attn_ranks): + for _ in range(args.attn_ranks // args.attention_tp): runner._ffn_forward_connector_driven(single=True) else: - hidden = torch.ones((1, HIDDEN), dtype=torch.bfloat16, device=device) + hidden = torch.ones( + (1, args.hidden_size), dtype=torch.bfloat16, device=device + ) context = AFDTransferContext(AFDTransferMetadata(LAYERS[0], 0, [1])) connector.send_attn_output( hidden, @@ -262,21 +413,72 @@ def observed_replay(): torch.npu.synchronize() dist.barrier() print(json.dumps({"rank": rank, "stage": "transport_ready"}), flush=True) + if args.idle_probe: + assert args.mega + if is_ffn: + pull_state = connector.comm_args.view(torch.int64) + before = int(pull_state[96].item()) + pull_state[64].fill_(8) + if args.full_graph: + runner._graph_stream.wait_stream(torch.npu.current_stream(device)) + connector.graph_startup.service_ready() + with torch.inference_mode(): + for _ in range(3): + if args.full_graph: + batch_index = 0 + runner._submit_graph(1) + runner._complete_graph() + else: + runner._ffn_forward_connector_driven(single=True) + torch.npu.synchronize() + assert int(pull_state[96].item()) == before + assert int(pull_state[32].item()) == -1 + connector.ffn_session().after_step() + pull_state[64].zero_() + torch.npu.synchronize() + report["idle_probe"] = "PASS" + dist.barrier() errors = [] + accuracy_failures: list[str] = [] + data_replays = 0 if not is_ffn: runner = thread = None cases = [(2, False, 7), (5, True, 7), (2, False, 7)] if args.boundary: - cases.extend(((5, False, 16), (2, True, 16))) + cases.extend( + ((5, False, args.boundary_tokens), (2, True, args.boundary_tokens)) + ) expected: list[tuple[int, int, int | None]] = [] for layer_id, sparse, tokens in cases: - for source in range(args.attn_ranks): - chunks = 2 if sparse and tokens == 16 and rank == args.attn_ranks else 1 - rows = 0 if sparse and rank == args.attn_ranks + 1 else None - expected.extend( - (source, layer_id, 16 if chunks == 2 else rows) for _ in range(chunks) - ) + ids = torch.arange(tokens * 2).remainder( + 2 if sparse else 2 * args.experts_per_rank + ) + local_ffn = rank - args.attn_ranks + rows = int( + ( + (ids >= local_ffn * args.experts_per_rank) + & (ids < (local_ffn + 1) * args.experts_per_rank) + ).sum() + ) + rows *= args.attention_tp + # Layered DR packs whole experts; Mega Pull slices individual rows. + chunks = [0] + for expert in range( + local_ffn * args.experts_per_rank, (local_ffn + 1) * args.experts_per_rank + ): + count = int((ids == expert).sum()) * args.attention_tp + assert count <= args.receive_capacity + if chunks[-1] + count > args.receive_capacity: + chunks.append(0) + chunks[-1] += count + for source in range(0, args.attn_ranks, args.attention_tp): + if args.mega: + # The exposed metadata is the final slice of one fused transaction. + last_rows = (rows - 1) % args.receive_capacity + 1 if rows else 0 + expected.append((source, layer_id, last_rows)) + continue + expected.extend((source, layer_id, count) for count in chunks) def observe(context): # Test-only host inspection after the matching DR/FFN/CS fence. @@ -290,7 +492,17 @@ def observe(context): } ) - if is_ffn and not args.full_graph: + if is_ffn and not args.full_graph and args.mega: + original_mega = runner._mega_executor.op + + def observe_mega(*op_args): + result = original_mega(*op_args) + torch.npu.current_stream().synchronize() + observe(SimpleNamespace(states=work_state())) + return result + + runner._mega_executor.op = observe_mega + elif is_ffn and not args.full_graph: original_send = connector.send_ffn_output def observe_eager(output, context, **kwargs): @@ -301,8 +513,26 @@ def observe_eager(output, context, **kwargs): connector.send_ffn_output = observe_eager + def finish_receiver(): + if args.idle_shutdown: + if args.full_graph: + runner._graph = graph + elif args.mega: + runner._mega_executor.op = original_mega + else: + connector.send_ffn_output = original_send + event = threading.Event() + # A local shutdown must still serve until all A ranks stop publishing. + event.set() + worker = SimpleNamespace( + _ffn_shutdown_event=event, device=device, model_runner=runner + ) + AFDNPUFFNWorker._run_ffn_server_loop(worker) + report["idle_shutdown"] = "PASS" + runner._graph_receiver_exited = True + def serve(): - nonlocal batch_index + nonlocal batch_index, data_replays try: torch.npu.set_device(local_rank) if not args.full_graph: @@ -311,12 +541,14 @@ def serve(): with torch.inference_mode(): for _ in expected: runner._ffn_forward_connector_driven(single=True) + finish_receiver() return connector.graph_startup.service_ready() + replay_before = runner._graph_replay_count for start in range(0, len(expected), len(LAYERS)): batch_index = 0 # Test-only finite work count, including the final short batch. - # The production loop has no EOS or cross-role stop protocol. + # The data check uses a fixed work count before the idle-stop probe. count = min(len(LAYERS), len(expected) - start) runner._submit_graph(count) runner._complete_graph() @@ -340,10 +572,14 @@ def serve(): torch.inference_mode(), torch.npu.stream(session._graph_stream), ): - session._graph_payload.hidden_states.fill_(113) - session._graph_payload.context.states.dynamic_scales.fill_(3) + if args.mega: + session._mega_executor.scratch.fill_(113) + else: + session._graph_payload.hidden_states.fill_(113) + work_state().dynamic_scales.fill_(3) session._graph_output.fill_(123) - runner._graph_receiver_exited = True + data_replays = runner._graph_replay_count - replay_before + finish_receiver() except Exception as exc: errors.append(exc) print(json.dumps({"rank": rank, "service_error": repr(exc)}), flush=True) @@ -361,19 +597,25 @@ def serve(): f"{args.reference_outputs}.rank{rank}.pt", weights_only=True ) for layer_id, sparse, tokens in cases: - for source in range(args.attn_ranks): - if rank == source: - values = torch.tensor([-12, -9, -0.5, 0.5, 9, 12, 1]).repeat(3)[:tokens] + for source in range(0, args.attn_ranks, args.attention_tp): + if source <= rank < source + args.attention_tp: + values = torch.tensor([-12, -9, -0.5, 0.5, 9, 12, 1]) + values = values.repeat((tokens + values.numel() - 1) // values.numel())[ + :tokens + ] inputs = ( (values * (rank + 1)) .bfloat16()[:, None] - .expand(-1, HIDDEN) + .expand(-1, args.hidden_size) .contiguous() ) + if args.weight_seed is not None: + feature = torch.sin(torch.arange(args.hidden_size).float() * 0.17) + inputs = (inputs.float() * feature[None, :] + 0.125).bfloat16() ids = ( torch.arange(tokens * 2) .reshape(tokens, 2) - .remainder(2 if sparse else 8) + .remainder(2 if sparse else 2 * args.experts_per_rank) ) hidden = inputs.to(device) context = AFDTransferContext(AFDTransferMetadata(layer_id, 0, [tokens])) @@ -396,9 +638,25 @@ def serve(): time.sleep(0.01) actual = output.cpu() if references is not None: - torch.testing.assert_close( - actual, references[len(outputs)], rtol=0, atol=0 - ) + try: + torch.testing.assert_close( + actual, references[len(outputs)], rtol=0, atol=0 + ) + except AssertionError as exc: + if args.save_outputs and not accuracy_failures: + torch.save( + { + "actual": actual, + "reference": references[len(outputs)], + "layer": layer_id, + "sparse": sparse, + "tokens": tokens, + }, + f"{args.save_outputs}.mismatch.rank{rank}.pt", + ) + # Finish all peer transactions before reporting numerical + # failure, so teardown cannot invalidate a pending pull. + accuracy_failures.append(str(exc)) outputs.append(actual) report["work"].append( {"layer": layer_id, "sparse": sparse, "tokens": tokens} @@ -411,8 +669,11 @@ def serve(): torch.save(outputs, f"{args.save_outputs}.rank{rank}.pt") if references is not None: assert len(references) == len(outputs) - report["eager_exact_match"] = True + report["eager_exact_match"] = not accuracy_failures + report["accuracy_failures"] = len(accuracy_failures) assert not connector._pending_attention_payloads + if args.idle_shutdown: + connector.close() else: assert thread is not None thread.join(timeout=30) @@ -420,22 +681,26 @@ def serve(): if errors: raise errors[0] assert len(report["work"]) == len(expected) - for observed, (source, layer, rows) in zip( + for observed, (source, layer, expected_rows) in zip( report["work"], expected, strict=True ): assert (observed["source"], observed["layer"]) == (source, layer) - if rows is not None: - assert observed["rows"] == rows, "zero work or chunk mismatch" + if expected_rows is not None: + assert observed["rows"] == expected_rows, ( + f"zero work or chunk mismatch: {report['work']=}, {expected=}" + ) if args.full_graph: report["graph"] = { "capture_count": runner._graph_capture_count, "replay_count": runner._graph_replay_count, + "data_replay_count": data_replays, "python_transaction_disabled": True, "stable_exposed_addresses": len(captured_addresses), } - assert runner._graph_replay_count == len(expected) + assert data_replays == len(expected) # Retain the TCPStore owner until finite eager peers have consumed all work. - dist.barrier() + if not args.idle_shutdown: + dist.barrier() if is_ffn: runner.shutdown() runner.shutdown() @@ -444,12 +709,22 @@ def serve(): connector.close() assert not connector.is_initialized and connector.cam_pg is None report["cleanup_complete"] = True - reports = [None] * world_size + reports: list[dict] = [{} for _ in range(world_size)] dist.all_gather_object(reports, report) dist.destroy_process_group() faulthandler.cancel_dump_traceback_later() + failed = any(item.get("accuracy_failures", 0) for item in reports) if rank == 0: - print(json.dumps({"result": "PASS", "reports": reports}), flush=True) + print( + json.dumps({"result": "FAIL" if failed else "PASS", "reports": reports}), + flush=True, + ) + if failed: + raise AssertionError( + accuracy_failures[0] + if accuracy_failures + else "peer accuracy comparison failed" + ) if __name__ == "__main__": diff --git a/tests/npu/check_pull_layout.cpp b/tests/npu/check_pull_layout.cpp new file mode 100644 index 000000000..4ab0129cb --- /dev/null +++ b/tests/npu/check_pull_layout.cpp @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +#include +#include +#include +#include +#include +#include "utils/op_kernel/afd_pull_layout.h" +#include "utils/op_kernel/afd_native_mega_workspace.h" + +int main() { + using namespace Cam::Pull; + Layout layout{}; + assert(MakeLayout({2048, 4096, 8, 8, 8, 32, 4}, layout)); + assert(layout.payload >= layout.returned + 128ULL * 1024 * 1024); + assert(layout.bytes - layout.payload == 72ULL * 1024 * 1024); + assert(!MakeLayout({8192, 4096, 8, 8, 8, 32, 4}, layout)); + assert(!MakeLayout({2048, 4096, 8, 8, 7, 32, 4}, layout)); + assert(!MakeLayout({UINT64_MAX, 4096, 8, 8, 8, 32, 4}, layout)); + assert(TransportUbBytes({8, 256, 2, 383, 1, 256, 1}) > 192 * 1024); + assert(PublisherUbBytes({8, 256, 2, 383, 1, 256, 1}, 48) > 192 * 1024); + const Shape largePublisher{8, 7168, 8, 4, 1, 128, 1}; + assert(PublisherUbBytes(largePublisher, 48) + TransportUbBytes(largePublisher) > 192 * 1024); + assert(CombineUbBytes({8, 256, 2, 4, 1, 256, 1}, 48) > 192 * 1024); + assert(CombineUbBytes({128, 4096, 6, 4, 4, 64, 4}, 48) <= 192 * 1024); + const Cam::NativeMegaWorkspace native(512, 4096, 2048, 128, 4); + assert(native.a2 - native.a1 >= 512ULL * 4096); + assert(native.c1 - native.a2 >= 512ULL * 2048); + assert(native.c2 - native.c1 >= 8ULL * 512 * 2048); + assert(native.output - native.c2 >= 4ULL * 512 * 4096); + assert(native.bytes - native.output == 2ULL * 512 * 4096); + assert(native.output % 512 == 0); + assert(native.layer - native.tileCounts >= 8ULL * 128 * (512 / Cam::NATIVE_TILE_ROWS)); + assert(native.a1 - native.x >= 512ULL * 4096); + static_assert(Cam::NATIVE_MAX_ROWS == MAX_CAPACITY); + for (uint64_t capacity : {uint64_t(4096), uint64_t(65536), uint64_t(MAX_CAPACITY)}) { + const Cam::NativeMegaWorkspace large(capacity, 8192, 8192, 128, 4); + assert(large.a2 - large.a1 >= capacity * 8192); + assert(large.c1 - large.a2 >= capacity * 8192); + assert(large.c2 - large.c1 >= 8 * capacity * 8192); + assert(large.output - large.c2 >= 4 * capacity * 8192); + assert(large.bytes - large.output == 2 * capacity * 8192); + assert(large.output % 512 == 0); + assert(Rounds(capacity, capacity) == 1); + assert(Rounds(capacity + 1, capacity) == 2); + } + const Cam::NativeMegaWorkspace model(262144, 4096, 2048, 64, 2); + assert(model.bytes - model.output == 2ULL * 1024 * 1024 * 1024); + std::mt19937 random(20261027); + for (uint32_t trial = 0; trial < 1000; ++trial) { + const uint32_t sources = 1 + random() % 4, experts = 1 + random() % 8; + const uint64_t capacity = 1 + random() % 23; + std::vector counts(sources * experts), prefixes(counts.size()); + uint64_t rows = 0; + for (uint32_t e = 0; e < experts; ++e) { + for (uint32_t s = 0; s < sources; ++s) { + const auto i = s * experts + e; + prefixes[i] = rows; + counts[i] = random() % 29; + rows += counts[i]; + } + } + std::vector visits(rows); + for (uint64_t q = 0; q < Rounds(rows, capacity); ++q) { + uint64_t roundRows = 0; + for (uint32_t s = 0; s < sources; ++s) { + uint64_t expertPrefix = 0; + for (uint32_t e = 0; e < experts; ++e) { + const auto i = s * experts + e; + const auto slice = Intersect(prefixes[i], counts[i], q * capacity, capacity); + for (uint64_t r = 0; r < slice.count; ++r) { + ++visits[prefixes[i] + slice.skip + r]; + assert(slice.local + r < capacity); + assert(ReturnRow(0, expertPrefix, slice.skip, r) < + expertPrefix + counts[i]); + } + expertPrefix += counts[i]; + roundRows += slice.count; + } + } + assert(roundRows == std::min(capacity, rows - q * capacity)); + } + assert(std::all_of(visits.begin(), visits.end(), [](unsigned n) { return n == 1; })); + } +} diff --git a/tests/npu/test_async_cam_layered_w8a8.py b/tests/npu/test_async_cam_layered_w8a8.py index 6534fe988..a5cc30805 100644 --- a/tests/npu/test_async_cam_layered_w8a8.py +++ b/tests/npu/test_async_cam_layered_w8a8.py @@ -20,8 +20,8 @@ from torch import Tensor -@pytest.fixture -def w8a8_runtime(): +@pytest.fixture(params=[0.0, 10.0]) +def w8a8_runtime(request): if os.environ.get("AFD_RUN_ASCEND_OP_RUNTIME") != "1": pytest.skip("requires opt-in 910C runtime") torch = pytest.importorskip("torch") @@ -70,16 +70,17 @@ def w8a8_runtime(): w13_bias=None, w2_bias=None, per_channel=True, - swiglu_limit=0.0, + swiglu_limit=request.param, routed_scaling_factor=1.0, quantization="W8A8", ) ) executor = AsyncCAMLayeredExecutor(layers) + assert executor.applied_swiglu_limit == 0.0 x = torch.randint( -8, 8, (capacity, hidden), dtype=torch.int8, generator=generator ).npu() - scales = torch.full((capacity,), 0.01, device="npu") + scales = torch.full((capacity,), 0.01 if request.param == 0 else 2.0, device="npu") return torch, torch_npu, layers, executor, x, scales diff --git a/tests/npu/test_cam_async_ops.py b/tests/npu/test_cam_async_ops.py index e4ef456f5..cb150892c 100644 --- a/tests/npu/test_cam_async_ops.py +++ b/tests/npu/test_cam_async_ops.py @@ -43,6 +43,8 @@ def _dispatch_recv(torch: ModuleType, *, dynamic_quant: int = 0, moe_rank_id: in "afd_async_dispatch_recv", "afd_async_combine_send", "afd_async_combine_recv", + "afd_mega_dispatch_send", + "afd_mega_combine_recv", ), ) def test_cam_dispatcher_registers_inference_and_meta_kernels( @@ -56,6 +58,137 @@ def test_cam_dispatcher_registers_inference_and_meta_kernels( assert all("shared" not in argument.name for argument in schema.arguments) +def _native_mega_inputs(torch, mode): + packed = mode == 1 + + def make(shape, dtype): + return torch.empty(shape, dtype=dtype, device="meta") + + anchor = make((1,), torch.bfloat16) + size = torch.ops.afd_ascend.afd_mega_moe_pull_workspace_size(16, 256, 256, 2, 1) + return [ + anchor, + make((4096,), torch.float16), + make((6,), torch.int64), + make((size,), torch.uint8), + [make((2, 256, 64 if packed else 512), torch.int32 if packed else torch.int8)], + [make((2, 256, 32 if packed else 256), torch.int32 if packed else torch.int8)], + [make((2, 512), torch.int64 if packed else torch.float32)], + [make((2, 256), torch.int64 if packed else torch.bfloat16)], + [make((2, 512), torch.float32)] if packed else [], + [make((2, 256), torch.float32)] if packed else [], + [8, 256, 256, 2, 2, 1, 2, 1, 3, 1, 16, mode, 1, 1], + "meta_only", + 1.0, + ] + + +@pytest.mark.parametrize("mode", [1, 2]) +def test_native_mega_meta_quantization_contract(cam_runtime, mode): + torch = cam_runtime + completion = torch.ops.afd_ascend.afd_mega_moe_pull_native( + *_native_mega_inputs(torch, mode), swiglu_limit=10.0 + ) + assert completion.shape == (1,) and completion.dtype == torch.int64 + assert completion.device.type == "meta" + + +@pytest.mark.parametrize( + "invalid", ["mode", "weight", "scale", "bias", "shape", "layers"] +) +def test_native_mega_meta_rejects_invalid_w8a8(cam_runtime, invalid): + torch = cam_runtime + inputs = _native_mega_inputs(torch, 2) + if invalid == "mode": + inputs[10][11] = 3 + elif invalid == "weight": + inputs[4][0] = inputs[4][0].to(torch.int32) + elif invalid == "scale": + inputs[7][0] = inputs[7][0].to(torch.float32) + elif invalid == "bias": + inputs[9] = [torch.empty((2, 256), dtype=torch.int32, device="meta")] + elif invalid == "shape": + inputs[4][0] = inputs[4][0].reshape(2, 512, 256) + else: + inputs[10][13] = 2 + with pytest.raises(RuntimeError, match="native Mega"): + torch.ops.afd_ascend.afd_mega_moe_pull_native(*inputs) + + +def test_mega_dispatch_meta_publishes_completion(cam_runtime): + torch = cam_runtime + state = torch.ops.afd_ascend.afd_mega_pull_init( + torch.empty(1, dtype=torch.bfloat16, device="meta"), + [8, 256, 2, 2, 1, 4, 0, 1], + [], + 1, + "meta_only", + ) + sent = torch.ops.afd_ascend.afd_mega_dispatch_send( + torch.empty((3, 256), dtype=torch.bfloat16, device="meta"), + torch.empty((3, 2), dtype=torch.int32, device="meta"), + state, + 0x41464401, + 8, + 3, + 256, + 2, + 2, + 1, + 4, + 0, + 3, + 2, + 1, + 1, + "meta_only", + ) + assert sent.shape == (1,) and sent.dtype == torch.int64 + schema = torch.ops.afd_ascend.afd_mega_dispatch_send.default._schema + comm_arg = next(arg for arg in schema.arguments if arg.name == "comm_args") + assert comm_arg.alias_info.is_write + + +def test_pull_empty_combine_preserves_logical_shape(cam_runtime): + torch = cam_runtime + anchor = torch.empty(1, dtype=torch.bfloat16, device="meta") + state = torch.ops.afd_ascend.afd_mega_pull_init( + anchor, [8, 256, 2, 2, 1, 4, 0, 1], [], 1, "meta_only" + ) + output = torch.ops.afd_ascend.afd_mega_combine_recv( + anchor, + torch.empty((0, 2), dtype=torch.int32, device="meta"), + torch.empty((0, 2), dtype=torch.float32, device="meta"), + state, + 0x41464401, + 0, + 256, + 2, + 2, + 1, + 4, + 0, + 3, + "meta_only", + 8, + 1, + ) + assert output.shape == (0, 256) and output.dtype == torch.bfloat16 + + +@pytest.mark.parametrize("layers", [(2, 2), (-1,), (65536,), (2**63 - 1,)]) +def test_pull_init_rejects_invalid_layer_map(cam_runtime, layers): + torch = cam_runtime + with pytest.raises(RuntimeError, match="layer"): + torch.ops.afd_ascend.afd_mega_pull_init( + torch.empty(1, dtype=torch.bfloat16, device="meta"), + [8, 256, 2, 2, 1, 4, 1, 1], + list(layers), + 1, + "meta_only", + ) + + @pytest.mark.parametrize("dtype_name", ["float16", "bfloat16"]) @pytest.mark.parametrize("dynamic_quant", [0, 1]) def test_cam_four_phase_meta_contracts( diff --git a/tests/unit/compat/npu/test_feature_validation.py b/tests/unit/compat/npu/test_feature_validation.py index 8b4f702f2..99ca5f04b 100644 --- a/tests/unit/compat/npu/test_feature_validation.py +++ b/tests/unit/compat/npu/test_feature_validation.py @@ -292,21 +292,25 @@ def test_npu_afd_num_ubatches_reads_two_ubatch_dbo(): @pytest.mark.parametrize("role", ["attention", "ffn"]) @pytest.mark.parametrize( - ("factor", "max_num_batched_tokens", "rejected"), + ("factor", "max_num_batched_tokens", "mega", "rejected"), [ - ("0.015625", 8192, True), - ("0.015625", 4096, False), - ("0.5", 8192, False), - (None, 262144, False), - ("0", 16, True), - ("nan", 16, True), - ("invalid", 16, True), - ("0.01_5625", 4096, True), + ("0.015625", 8192, False, True), + ("0.015625", 8192, True, False), + ("0.015625", 4096, False, False), + ("0.5", 8192, False, False), + (None, 262144, False, False), + ("0", 16, False, True), + ("nan", 16, False, True), + ("invalid", 16, False, True), + ("0.01_5625", 4096, False, True), + ("0.00000001", 16, False, True), + ("0.00000001", 16, True, True), ], ) def test_npu_async_feature_validation_checks_receive_capacity( monkeypatch, role, + mega, factor, max_num_batched_tokens, rejected, @@ -324,7 +328,11 @@ def test_npu_async_feature_validation_checks_receive_capacity( use_v2_model_runner=True, num_attention_ranks=2, tensor_parallel_size=2 if role == "attention" else 1, - extra_config={"attn_ranks_per_dp": 2, "dynamicQuant": 1}, + extra_config={ + "attn_ranks_per_dp": 2, + "ffn_mega_moe": mega, + "dynamicQuant": 1, + }, ) config.model_config.hf_text_config.model_type = "deepseek_v4" config.scheduler_config.max_num_batched_tokens = max_num_batched_tokens diff --git a/tests/unit/compat/patches/npu/test_mrv2_ubatching.py b/tests/unit/compat/patches/npu/test_mrv2_ubatching.py index ba812ef26..5fa429137 100644 --- a/tests/unit/compat/patches/npu/test_mrv2_ubatching.py +++ b/tests/unit/compat/patches/npu/test_mrv2_ubatching.py @@ -205,6 +205,7 @@ def dp_scope(): skip_dp.reset(token) class NativeRunner: + connector: object vllm_config: SimpleNamespace afd_config: SimpleNamespace prof: None @@ -256,6 +257,7 @@ def execute_model( future = ast.parse("from __future__ import annotations").body namespace = { "NativeRunner": NativeRunner, + "CAMAsyncAFDConnector": type("CAMAsyncAFDConnector", (), {}), "CUDAGraphMode": _GraphMode, "nullcontext": nullcontext, "step_afd_npu_profiler": lambda profiler: None, @@ -279,6 +281,7 @@ def make_runner(async_dp): Callable[[], NativeRunner], namespace["AFDNPUAttentionModelRunnerV2"] ) runner = runner_factory() + runner.connector = None runner.vllm_config = SimpleNamespace( compilation_config=SimpleNamespace(cudagraph_mode=_GraphMode.NONE) ) @@ -326,6 +329,35 @@ def test_execute_preserves_native_input_policy_and_restores_scopes( assert execute_contract.skip_dp.get() is False +@pytest.mark.parametrize("fail", [False, True]) +def test_execute_scopes_connector_step_around_native_execution(execute_contract, fail): + runner = execute_contract.make_runner(False) + events = [] + + @contextmanager + def attention_step(): + events.append("enter") + try: + yield + finally: + events.append("exit") + + runner.connector = SimpleNamespace(attention_step=attention_step) + + def install_metadata(context): + assert events == ["enter"] + events.append("native") + + runner.install_afd_metadata_on_forward_context = install_metadata + scheduler = SimpleNamespace(total_num_scheduled_tokens=7, fail=fail) + if fail: + with pytest.raises(RuntimeError, match="native execution failed"): + runner.execute_model(scheduler) + else: + runner.execute_model(scheduler) + assert events == ["enter", "native", "exit"] + + def test_execute_keeps_native_default_input_policy(execute_contract): runner = execute_contract.make_runner(False) result = runner.execute_model( diff --git a/tests/unit/connectors/npu/test_async_cam_pull.py b/tests/unit/connectors/npu/test_async_cam_pull.py new file mode 100644 index 000000000..0ba8298d0 --- /dev/null +++ b/tests/unit/connectors/npu/test_async_cam_pull.py @@ -0,0 +1,155 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Host Pull shutdown protocol; these tests do not qualify device execution.""" + +from types import SimpleNamespace + +import pytest + +from afd_plugin.connectors.npu.async_cam_runtime import ( + async_cam_startup as startup_module, +) +from afd_plugin.connectors.npu.async_cam_runtime.async_cam_pull import PullFFNSession +from tests.unit.fakes.async_cam_startup import MemoryStore, startup_world + + +@pytest.fixture +def pull_world(monkeypatch): + monkeypatch.setattr(startup_module, "GRAPH_STARTUP_TIMEOUT_SECONDS", 0) + store = MemoryStore() + connectors = [ + SimpleNamespace( + graph_startup=startup, + max_num_batched_tokens=8, + topk=2, + _pull_has_active_source=lambda: False, + _check_pull_status=lambda: None, + ) + for startup in startup_world(store=store) + ] + return SimpleNamespace( + sessions=[PullFFNSession(connector) for connector in connectors], + connectors=connectors, + store=store, + ) + + +@pytest.mark.parametrize("shutdown_requested", [False, True]) +@pytest.mark.parametrize("quiet_ranks", [(), (0,), (1,), (0, 1)]) +def test_keep_serving_requires_every_attention_quiet( + pull_world, quiet_ranks, shutdown_requested +): + for rank in quiet_ranks: + pull_world.store.set(f"ffn_graph/test/pull_quiet/{rank}", "ready") + for receiver in pull_world.sessions[2:]: + assert receiver.keep_serving(shutdown_requested) is (len(quiet_ranks) != 2) + + +def test_peer_failure_wins_over_quiet_shutdown(pull_world): + for rank in range(2): + pull_world.store.set(f"ffn_graph/test/pull_quiet/{rank}", "ready") + pull_world.connectors[0].graph_startup.fail(RuntimeError("attention failed")) + with pytest.raises(RuntimeError, match="attention failed"): + pull_world.sessions[2].keep_serving(True) + + +@pytest.mark.parametrize("transactions_to_drain", [0, 1, 4, 5, None]) +def test_drain_requires_source_completion_within_capacity_bound( + pull_world, monkeypatch, transactions_to_drain +): + monkeypatch.setenv("BATCH_SIZE_FACTOR", str(4 / 262144)) + session, connector = pull_world.sessions[2], pull_world.connectors[2] + remaining = [transactions_to_drain] + connector._pull_has_active_source = lambda: remaining[0] != 0 + + def run_one(): + if remaining[0] is not None: + remaining[0] -= 1 + + if transactions_to_drain in (5, None): + with pytest.raises(RuntimeError, match="retain resources"): + session.drain(run_one) + if transactions_to_drain == 5: + assert remaining[0] == 1 + with pytest.raises(RuntimeError, match="not drained.*retain resources"): + session.finish_receiver() + assert not pull_world.store.check(["ffn_graph/test/pull_stopped/2"]) + else: + session.drain(run_one) + assert remaining[0] == 0 + for rank in range(2): + pull_world.store.set(f"ffn_graph/test/pull_quiet/{rank}", "ready") + session.finish_receiver() + assert pull_world.store.check(["ffn_graph/test/pull_stopped/2"]) + + +def test_failed_drain_does_not_publish_receiver_stopped(pull_world): + session, connector = pull_world.sessions[2], pull_world.connectors[2] + connector._pull_has_active_source = lambda: True + + def fail_transaction(): + raise RuntimeError("transaction failed") + + with pytest.raises(RuntimeError, match="transaction failed"): + session.drain(fail_transaction) + with pytest.raises(RuntimeError, match="retain resources"): + session.finish_receiver() + assert not pull_world.store.check(["ffn_graph/test/pull_stopped/2"]) + + +def test_close_waits_for_all_attention_then_all_drained_receivers(pull_world): + first_a, last_a, first_f, last_f = pull_world.sessions + with pytest.raises(TimeoutError): + first_a.close_attention() + first_f.drain(lambda: pytest.fail("an idle receiver needs no transaction")) + with pytest.raises(TimeoutError): + first_f.finish_receiver() + assert not pull_world.store.check(["ffn_graph/test/pull_stopped/2"]) + with pytest.raises(TimeoutError): + last_a.close_attention() + first_f.finish_receiver() + with pytest.raises(TimeoutError): + first_a.close_attention() + last_f.drain(lambda: pytest.fail("an idle receiver needs no transaction")) + last_f.finish_receiver() + first_a.close_attention() + last_a.close_attention() + assert pull_world.store.check( + [f"ffn_graph/test/pull_stopped/{rank}" for rank in (2, 3)] + ) + pull_world.connectors[3].graph_startup.fail(RuntimeError("drain failed")) + with pytest.raises(RuntimeError, match="drain failed"): + first_a.close_attention() + + +def test_finish_receiver_is_idempotent_after_store_release(pull_world): + session = pull_world.sessions[2] + for rank in range(2): + pull_world.store.set(f"ffn_graph/test/pull_quiet/{rank}", "ready") + session.drain(lambda: pytest.fail("an idle receiver needs no transaction")) + session.finish_receiver() + pull_world.store.closed = True + session.finish_receiver() + + +@pytest.mark.parametrize("state", ["unfinished", "active", "failed", "drained"]) +def test_shutdown_requires_finished_idle_error_free_receiver(pull_world, state): + session, connector = pull_world.sessions[2], pull_world.connectors[2] + if state != "unfinished": + for rank in range(2): + pull_world.store.set(f"ffn_graph/test/pull_quiet/{rank}", "ready") + session.drain(lambda: pytest.fail("an idle receiver needs no transaction")) + session.finish_receiver() + if state == "active": + connector._pull_has_active_source = lambda: True + if state == "failed": + + def status_failure(): + raise RuntimeError("status -5") + + connector._check_pull_status = status_failure + if state == "drained": + session.validate_shutdown() + else: + with pytest.raises(RuntimeError, match="retain resources|status -5"): + session.validate_shutdown() diff --git a/tests/unit/connectors/test_async_cam_startup.py b/tests/unit/connectors/npu/test_async_cam_startup.py similarity index 67% rename from tests/unit/connectors/test_async_cam_startup.py rename to tests/unit/connectors/npu/test_async_cam_startup.py index 186019a31..c530d33bb 100644 --- a/tests/unit/connectors/test_async_cam_startup.py +++ b/tests/unit/connectors/npu/test_async_cam_startup.py @@ -2,15 +2,19 @@ # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project """Pure host negotiation tests; no device or model runner initialization.""" +import json +import subprocess +import sys from concurrent.futures import ThreadPoolExecutor from dataclasses import replace -from threading import Condition from types import SimpleNamespace import pytest -from afd_plugin import async_cam_startup as startup_module -from afd_plugin.async_cam_startup import ( +from afd_plugin.connectors.npu.async_cam_runtime import ( + async_cam_startup as startup_module, +) +from afd_plugin.connectors.npu.async_cam_runtime.async_cam_startup import ( AsyncCAMGraphStartup, FFNGraphPlan, resolve_ffn_mode, @@ -20,41 +24,29 @@ requested_ffn_mode, validate_graph_capacity, ) - - -class MemoryStore: - def __init__(self): - self.values = {} - self.condition = Condition() - self.closed = False - - def set(self, key, value): - assert not self.closed, "Store used after release" - with self.condition: - self.values[key] = value.encode() if isinstance(value, str) else value - self.condition.notify_all() - - def get(self, key): - assert not self.closed, "Store used after release" - with self.condition: - assert self.condition.wait_for(lambda: key in self.values, timeout=3) - return self.values[key] - - def check(self, keys): - assert not self.closed, "Store used after release" - with self.condition: - return all(key in self.values for key in keys) - - -def startup_world(attn=2, ffn=2, store=None): - store = store or MemoryStore() - owners = [] - for rank in range(attn + ffn): - owner = AsyncCAMGraphStartup(rank, attn, attn + ffn) - owner._store = store - owner._prefix = "ffn_graph/test/" - owners.append(owner) - return owners +from tests.unit.fakes.async_cam_startup import MemoryStore, startup_world + + +def test_startup_import_needs_no_torch_or_vllm_in_clean_process(): + result = subprocess.run( + [ + sys.executable, + "-c", + "import sys; " + "sys.modules.update(torch=None, vllm=None, " + "torch_npu=None, vllm_ascend=None); " + "from afd_plugin.connectors.npu.async_cam_runtime.async_cam_startup " + "import AsyncCAMGraphStartup; " + "from afd_plugin.connectors.npu.async_cam_runtime.async_cam_pull " + "import PullFFNSession; " + "assert AsyncCAMGraphStartup(0, 1, 2).mode == 'eager'; " + "assert PullFFNSession.requires_drain", + ], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr def full_plan(**changes): @@ -88,6 +80,71 @@ def negotiate(owners, plans, signatures): return [future.exception(timeout=3) for future in futures] +@pytest.mark.parametrize("world", [4, 16]) +def test_shared_failure_check_uses_one_store_request(world, monkeypatch): + owners = startup_world(attn=world - 2, ffn=2) + assert not any( + negotiate( + owners, + [None] * (world - 2) + [full_plan(requested="eager")] * 2, + ["wire"] * world, + ) + ) + calls = [] + original = owners[0]._store.check + + def check(keys): + calls.append(keys) + return original(keys) + + monkeypatch.setattr(owners[0]._store, "check", check) + owners[-1].check_failure() + assert len(calls) == 1 + owners[0].fail(RuntimeError("source failed")) + for owner in owners: + with pytest.raises(RuntimeError, match="source failed"): + owner.check_failure() + + +def test_legacy_peer_keeps_per_rank_failure_detection(monkeypatch): + owners = startup_world() + original = owners[0]._publish + + def legacy_publish(stage, value="ready"): + if stage == "plan": + fields = json.loads(value) + fields.pop("shared_errors") + value = json.dumps(fields) + original(stage, value) + + monkeypatch.setattr(owners[0], "_publish", legacy_publish) + assert not any( + negotiate( + owners, [None, None] + [full_plan(requested="eager")] * 2, ["wire"] * 4 + ) + ) + owners[0]._store.set(owners[0]._keys("error", range(1))[0], "legacy failure") + for owner in owners: + with pytest.raises(RuntimeError, match="legacy failure"): + owner.check_failure() + + +def test_shared_failure_survives_compatibility_write_failure(monkeypatch): + owners = startup_world() + original = owners[0]._store.set + + def fail_rank_error(key, value): + if key.endswith("error/0"): + raise ConnectionError("connection lost") + original(key, value) + + monkeypatch.setattr(owners[0]._store, "set", fail_rank_error) + with pytest.raises(ConnectionError): + owners[0].fail(RuntimeError("original failure")) + with pytest.raises(RuntimeError, match="original failure"): + owners[-1].check_failure() + + def test_mixed_requested_modes_are_not_silently_downgraded(): with pytest.raises(ValueError, match="same execution mode"): resolve_ffn_mode([full_plan(), FFNGraphPlan()]) @@ -211,6 +268,39 @@ def test_graph_accepts_conservative_uint16_boundary(): validate_graph_capacity(65536, 2, 65536) +def test_pull_row_slicing_accepts_capacity_below_dispatch_envelope(): + validate_graph_capacity(32, 2, 3, allow_row_slicing=True) + with pytest.raises(ValueError, match="smaller"): + validate_graph_capacity(32, 2, 3) + with pytest.raises(ValueError, match="uint16"): + validate_graph_capacity(65536, 1, 3, allow_row_slicing=True) + + +def test_pull_requires_matching_transport_even_in_eager_mode(): + owners = startup_world(1, 1) + errors = negotiate( + owners, [None, full_plan(requested="eager")], ["pull_v1:layout", "push"] + ) + assert all("transport signatures" in str(error) for error in errors) + + +@pytest.mark.parametrize( + ("bad", "message"), + [ + (FFNGraphPlan(), "signatures differ"), + (full_plan(requested="eager", layer_ids=(3,)), "layer tables"), + ], +) +def test_pull_eager_validates_executor_and_layer_tables(bad, message): + owners = startup_world(1, 2) + errors = negotiate( + owners, + [None, full_plan(requested="eager"), bad], + ["pull_v1:layout"] * 3, + ) + assert all(message in str(error) for error in errors), errors + + @pytest.mark.parametrize("stage", ["validated", "warmup", "captured"]) def test_phase_requires_all_ranks_and_failure_wins_over_ready(monkeypatch, stage): owners = startup_world() diff --git a/tests/unit/test_async_cam_startup_wiring.py b/tests/unit/connectors/npu/test_async_cam_startup_wiring.py similarity index 71% rename from tests/unit/test_async_cam_startup_wiring.py rename to tests/unit/connectors/npu/test_async_cam_startup_wiring.py index dbae8fcd4..ca2e76fb6 100644 --- a/tests/unit/test_async_cam_startup_wiring.py +++ b/tests/unit/connectors/npu/test_async_cam_startup_wiring.py @@ -11,12 +11,16 @@ import pytest -from afd_plugin.async_cam_startup import AsyncCAMGraphStartup, resolve_ffn_mode from afd_plugin.compat.npu.feature_validation import ( _fail_if_unsupported_npu_afd_async_features, fail_if_unsupported_npu_afd_features, ) from afd_plugin.config import parse_afd_config +from afd_plugin.connectors.npu.async_cam_runtime.async_cam_startup import ( + AsyncCAMGraphStartup, + resolve_ffn_mode, +) +from tests.unit.fakes.async_cam_startup import startup_world from tests.unit.fakes.async_ffn import ( load_async_modules, load_host_module, @@ -132,7 +136,7 @@ def executor(layers): connector = SimpleNamespace( is_initialized=False, dynamic_quant=1, - extra_info=SimpleNamespace(ffn_two_stream=False), + extra_info=SimpleNamespace(ffn_two_stream=False, ffn_mega_moe=False), graph_startup=AsyncCAMGraphStartup(1, 1, 2), ) connector.init_afd_connector = lambda: resolve_ffn_mode( @@ -173,6 +177,38 @@ def test_ffn_startup_validates_target_and_draft_weights( assert connector.graph_startup.plan.signature == "(2, 5)" +@pytest.mark.parametrize("mode", ["eager", "full"]) +def test_mega_executor_failure_is_published_during_startup(modules, monkeypatch, mode): + _, _, runtime = modules + runner = runtime.AsyncNPUFFNModelRunner.__new__(runtime.AsyncNPUFFNModelRunner) + runner.model = SimpleNamespace(get_experts_layer_indices=lambda: (2,)) + runner.draft_model = None + runner._failure = runner._layered_executor = runner._mega_executor = None + runner.requested_mode = mode + connector = SimpleNamespace( + is_initialized=False, + extra_info=SimpleNamespace(ffn_two_stream=False, ffn_mega_moe=True), + graph_startup=AsyncCAMGraphStartup(1, 1, 2), + ) + connector.init_afd_connector = lambda: resolve_ffn_mode( + [connector.graph_startup.plan] + ) + runner.connector = connector + runner._initialize_layered_executor = lambda: setattr( + runner, + "_layered_executor", + SimpleNamespace(graph_signature=lambda _: "layered"), + ) + + def fail_executor(*_args): + raise ValueError("unsupported Mega weights") + + monkeypatch.setattr(runtime, "AsyncCAMMegaExecutor", fail_executor) + with pytest.raises(ValueError, match="unsupported Mega weights"): + runner.initialize_afd_connector() + assert connector.graph_startup.plan.executor_error == "unsupported Mega weights" + + @pytest.mark.parametrize("value", [True, 0, -1, 1.5]) def test_two_stream_rejects_invalid_core_settings(modules, value): _, connector, _ = modules @@ -291,6 +327,7 @@ def create(*args, **kwargs): ) connector.__dict__.update( _initialized=False, + extra_info=connector_module.AFDAsyncExtraInfo(), afd_config=SimpleNamespace(host="localhost", port=1), topology=SimpleNamespace(world_size=2), world_rank=0, @@ -366,16 +403,21 @@ def test_graph_dbo_keeps_existing_role_and_count_validation( def test_ffn_announces_service_only_on_receiver_execution(modules): - _, _, runtime = modules + _, connector_module, runtime = modules runner = runtime.AsyncNPUFFNModelRunner.__new__(runtime.AsyncNPUFFNModelRunner) calls: list[str | tuple[str, str | int]] = [] runner.__dict__.update( prof=None, _graph=object(), _service_announced=False, + _first_step=True, + _graph_stream=MagicMock(), + device="npu:0", + session=sys.modules["tests.unit.fakes.connector"].FakeFFNSession(), layer_ids=(2, 5), connector=SimpleNamespace( - graph_startup=SimpleNamespace(service_ready=lambda: calls.append("ready")) + extra_info=connector_module.AFDAsyncExtraInfo(), + graph_startup=SimpleNamespace(service_ready=lambda: calls.append("ready")), ), _submit_graph=lambda count: calls.append(("submit", count)), _complete_graph=lambda: True, @@ -386,6 +428,30 @@ def test_ffn_announces_service_only_on_receiver_execution(modules): assert calls == ["ready", ("submit", 2), ("submit", 2)] +@pytest.mark.parametrize("session_serves", [False, True]) +def test_ffn_step_lets_session_decide_service_after_shutdown(modules, session_serves): + _, connector_module, runtime = modules + runner = runtime.AsyncNPUFFNModelRunner.__new__(runtime.AsyncNPUFFNModelRunner) + submitted: list[int] = [] + serving = [True] if session_serves else [] + runner.__dict__.update( + prof=None, + _graph=object(), + _service_announced=True, + _first_step=False, + session=sys.modules["tests.unit.fakes.connector"].FakeFFNSession( + serving=serving + ), + layer_ids=(2, 5), + connector=SimpleNamespace(extra_info=connector_module.AFDAsyncExtraInfo()), + _submit_graph=submitted.append, + _complete_graph=lambda: True, + ) + # Push stops on the shutdown request; Pull keeps serving until A is quiet. + assert runner.execute_connector_driven_step(True) is session_serves + assert submitted == ([2] if session_serves else []) + + @pytest.mark.parametrize("capture_fails", [False, True]) @pytest.mark.parametrize("two_stream", [False, True]) def test_ffn_warmup_capture_order_and_failure_propagation( @@ -419,6 +485,7 @@ def capture(): draft_model=None, _initialize_layered_executor=lambda: None, _layered_executor=SimpleNamespace(graph_signature=lambda capacity: "weights"), + _mega_executor=None, connector=SimpleNamespace( is_initialized=False, graph_startup=startup, @@ -527,6 +594,7 @@ def receiver_graph_worker(ffn_owners, monkeypatch): ) connector = SimpleNamespace( is_initialized=True, + ffn_session=sys.modules["afd_plugin.connectors.base"].DefaultFFNSession, close=lambda: None, recv_attn_output=lambda: SimpleNamespace( hidden_states=object(), context=SimpleNamespace(states=state) @@ -586,19 +654,30 @@ def serve(): @pytest.mark.parametrize("state", ["live", "failed", "drained"]) @pytest.mark.parametrize("two_stream", [False, True]) -def test_worker_retains_resources_until_clean_exit(ffn_owners, state, two_stream): +@pytest.mark.parametrize("requires_drain", [False, True]) +def test_worker_retains_resources_until_clean_exit( + ffn_owners, state, two_stream, requires_drain +): _, runtime, module = ffn_owners calls = [] + timeouts = [] runner = runtime.AsyncNPUFFNModelRunner.__new__(runtime.AsyncNPUFFNModelRunner) runner.effective_mode = "eager" if two_stream else "full" runner._two_stream = object() if two_stream else None + runner.session = SimpleNamespace(requires_drain=requires_drain) runner.connector = SimpleNamespace(close=lambda: calls.append("connector-close")) runner.shutdown = lambda: calls.append("release") worker = module.AFDNPUFFNWorker.__new__(module.AFDNPUFFNWorker) worker.model_runner = runner worker._ffn_shutdown_event = SimpleNamespace(set=lambda: calls.append("stop")) + + def join(**kwargs): + calls.append("join") + timeouts.append(kwargs["timeout"]) + worker._ffn_thread = SimpleNamespace( - join=lambda **kw: calls.append("join"), is_alive=lambda: state == "live" + join=join, + is_alive=lambda: state == "live", ) worker._ffn_loop_error = ( RuntimeError("native failure") if state == "failed" else None @@ -616,6 +695,7 @@ def test_worker_retains_resources_until_clean_exit(ffn_owners, state, two_stream worker.raise_ffn_loop_error_if_any() with pytest.raises(RuntimeError): worker.start_ffn_server_loop() + assert timeouts == [60 if requires_drain else 5] @pytest.mark.parametrize("async_connector", [False, True]) @@ -665,7 +745,10 @@ def test_ffn_runner_loads_model_and_rejects_second_load( monkeypatch.setattr(runtime, "validate_npu_model_runner_v2_config", calls.validate) monkeypatch.setattr(runtime, "fail_if_unsupported_npu_afd_features", calls.validate) monkeypatch.setattr(runtime, "create_afd_npu_profiler", lambda role: None) - connector, device, model = object(), SimpleNamespace(type="npu"), object() + connector = SimpleNamespace( + ffn_session=sys.modules["afd_plugin.connectors.base"].DefaultFFNSession + ) + device, model = SimpleNamespace(type="npu"), object() connector_name = "CAMAsyncAFDConnector" if async_connector else "CAMP2pAFDConnector" config = npu_vllm_config(role="ffn", connector=connector_name) config.model_config.enforce_eager = False @@ -715,3 +798,176 @@ def get_loader(load_config): assert runner.get_model() is model and runner.model_memory_usage == 1234 with pytest.raises(RuntimeError, match="already loaded"): runner.load_model() + + +@pytest.fixture +def connector_world(monkeypatch): + _, module, runtime = load_async_modules(monkeypatch) + owners = startup_world() + connectors = [] + for rank, startup in enumerate(owners): + role = "attention" if rank < 2 else "ffn" + config = npu_vllm_config( + role=role, + connector="CAMAsyncAFDConnector", + num_attention_ranks=2, + num_ffn_ranks=2, + extra_config={"ffn_mega_moe": True, "dynamicQuant": 1}, + ) + config.model_config.hf_config = SimpleNamespace( + hidden_size=32, num_experts_per_tok=2, n_routed_experts=4 + ) + config.model_config.dtype = "bfloat16" + connector = module.CAMAsyncAFDConnector( + rank, 0, config, parse_afd_config(config), rank % 2 + ) + connector.graph_startup = startup + connector._initialized = True + connector.comm_args = object() + connectors.append(connector) + return SimpleNamespace(connectors=connectors, runtime=runtime) + + +def test_default_session_obeys_shutdown_without_drain(connector_world): + session = sys.modules["afd_plugin.connectors.base"].DefaultFFNSession() + assert session.keep_serving(False) + assert not session.keep_serving(True) + assert not session.requires_drain + session.before_first_step() + session.after_step() + session.drain(lambda: pytest.fail("Push must not execute a drain transaction")) + session.finish_receiver() + session.finish_receiver() + session.validate_shutdown() + + +@pytest.mark.parametrize("failure", ["unfinished", "active", "status"]) +def test_close_preserves_receiver_communication_on_unsafe_session( + connector_world, monkeypatch, failure +): + connector = connector_world.connectors[2] + resource = connector.comm_args + connector.cam_pg = group = object() + session = connector.ffn_session() + if failure != "unfinished": + monkeypatch.setattr(connector, "_pull_has_active_source", lambda: False) + session.drain(lambda: pytest.fail("idle source must not launch")) + for attention in connector_world.connectors[:2]: + attention.graph_startup._publish("pull_quiet") + session.finish_receiver() + if failure == "active": + monkeypatch.setattr(connector, "_pull_has_active_source", lambda: True) + elif failure == "status": + + def fail_status(): + raise RuntimeError("sticky status -5") + + monkeypatch.setattr(connector, "_check_pull_status", fail_status) + expected = "sticky status -5" if failure == "status" else "has not drained" + with pytest.raises(RuntimeError, match=expected): + connector.close() + assert connector.is_initialized and connector.comm_args is resource + assert connector.cam_pg is group + + +@pytest.mark.parametrize("failure", ["body", "status"]) +def test_attention_step_recovers_after_error(connector_world, monkeypatch, failure): + connector = connector_world.connectors[0] + error: list[RuntimeError | None] = [None] + + def check_status(): + if error[0] is not None: + raise error[0] + + monkeypatch.setattr(connector, "_check_pull_status", check_status) + expected = ValueError if failure == "body" else RuntimeError + with pytest.raises(expected, match="failed"), connector.attention_step(): + if failure == "body": + raise ValueError("model failed") + error[0] = RuntimeError("status failed") + error[0] = RuntimeError("status failed after recovery") + with ( + pytest.raises(RuntimeError, match="after recovery"), + connector.attention_step(), + ): + pass + error[0] = None + with connector.attention_step(): + pass + + +def test_nested_attention_step_defers_status_until_outer_boundary( + connector_world, monkeypatch +): + connector = connector_world.connectors[0] + + def check_status(): + raise RuntimeError("sticky status") + + monkeypatch.setattr(connector, "_check_pull_status", check_status) + with pytest.raises(RuntimeError, match="sticky status"), connector.attention_step(): + with connector.attention_step(): + pass + # Reaching this line proves the nested scope did not check sticky status. + completed_inner_scope = True + assert completed_inner_scope + + +@pytest.mark.parametrize("fails", [False, True]) +def test_runner_drain_returns_completed_response_or_session_error( + connector_world, fails +): + runtime = connector_world.runtime + runner = runtime.AsyncNPUFFNModelRunner.__new__(runtime.AsyncNPUFFNModelRunner) + runner.model = object() + runner.connector = connector_world.connectors[2] + runner.session = sys.modules["tests.unit.fakes.connector"].FakeFFNSession( + transactions=1, + drain_error=RuntimeError("source did not drain") if fails else None, + ) + runner._graph = runner._two_stream = None + runner.layer_ids = (2, 5) + responses = [] + runner.connector.send_ffn_output = lambda output, context: responses.append(output) + # The host executor emits a peer response; it does not simulate device compute. + runner._mega_executor = lambda: runner.connector.send_ffn_output("response", None) + if fails: + with pytest.raises(RuntimeError, match="source did not drain"): + runner.drain() + else: + runner.drain() + assert responses == ["response"] + + +@pytest.mark.parametrize("failure", ["unfinished", "active", "status"]) +def test_runner_shutdown_retains_model_and_communication_on_unsafe_session( + connector_world, monkeypatch, failure +): + runtime = connector_world.runtime + runner = runtime.AsyncNPUFFNModelRunner.__new__(runtime.AsyncNPUFFNModelRunner) + connector = connector_world.connectors[2] + runner.connector = connector + runner.session = connector.ffn_session() + runner.model = model = object() + runner._failure = None + runner._is_shutdown = False + runner._graph_receiver_exited = True + resource = connector.comm_args + if failure != "unfinished": + monkeypatch.setattr(connector, "_pull_has_active_source", lambda: False) + runner.session.drain(lambda: pytest.fail("idle source must not launch")) + for attention in connector_world.connectors[:2]: + attention.graph_startup._publish("pull_quiet") + runner.session.finish_receiver() + if failure == "active": + monkeypatch.setattr(connector, "_pull_has_active_source", lambda: True) + elif failure == "status": + + def fail_status(): + raise RuntimeError("status failed; retain resources") + + monkeypatch.setattr(connector, "_check_pull_status", fail_status) + with pytest.raises(RuntimeError, match="retain resources"): + runner.shutdown() + assert runner.get_model() is model + assert connector.is_initialized and connector.comm_args is resource diff --git a/tests/unit/connectors/test_async_cam_connector.py b/tests/unit/connectors/test_async_cam_connector.py index 46c72c541..f92dd85b1 100644 --- a/tests/unit/connectors/test_async_cam_connector.py +++ b/tests/unit/connectors/test_async_cam_connector.py @@ -29,6 +29,7 @@ CAMAsyncAFDConnector, build_async_topology, ) +from tests.unit.fakes.async_cam_startup import MemoryStore # noqa: E402 class _FakeTensorLike: @@ -94,6 +95,9 @@ def afd_async_combine_recv(self, *args): hidden_size = args[6] return _FakeTensor((batch_size, hidden_size)) + afd_mega_dispatch_send = afd_async_dispatch_send + afd_mega_combine_recv = afd_async_combine_recv + class _FakeTorch: def __init__(self): @@ -173,6 +177,58 @@ def test_async_extra_info_rejects_unknown_and_invalid_fields(): AFDAsyncExtraInfo.from_mapping({"attn_ranks_per_dp": 0}) +def test_mega_config_round_trips_protocol_and_normalized_quantization(): + info = AFDAsyncExtraInfo.from_mapping( + {"ffn_mega_moe": "true", "dynamicQuant": "1", "ffn_two_stream": "false"} + ) + assert info.ffn_mega_moe and info.dynamic_quant == 1 + assert AFDAsyncExtraInfo.from_mapping(info.to_mapping()) == info + + +@pytest.mark.parametrize("capacity", [1, 16]) +def test_mega_warmup_launches_once_per_source(monkeypatch, capacity): + monkeypatch.setenv("BATCH_SIZE_FACTOR", str(capacity / 262144)) + connector = CAMAsyncAFDConnector( + 0, + 0, + _vllm_config( + extra_config={ + "ffn_mega_moe": True, + "dynamicQuant": 1, + "attn_ranks_per_dp": 4, + } + ), + _afd_config(role="ffn"), + 0, + ) + assert connector.graph_startup_chunks() == 1 + + +@pytest.mark.parametrize( + "raw", + [ + {"ffn_mega_moe": True}, + {"ffn_mega_moe": True, "dynamicQuant": 1, "ffn_two_stream": True}, + ], +) +def test_mega_requires_quantized_single_stream(raw): + with pytest.raises(ValueError, match="ffn_mega_moe"): + AFDAsyncExtraInfo.from_mapping(raw) + + +@pytest.mark.parametrize( + "field", + [ + "async_moe_ubatching", + "async_moe_num_ubatches", + "async_moe_split", + ], +) +def test_async_extra_info_rejects_removed_fields(field): + with pytest.raises(ValueError, match="unknown AFD async connector_extra_config"): + AFDAsyncExtraInfo.from_mapping({field: True}) + + def test_async_connector_factory_creates_import_safe_connector(): connector = AFDConnectorFactory.create_connector( 0, @@ -455,6 +511,168 @@ def test_async_rejects_nondivisible_expert_placement(): build_async_topology(_afd_config(role="ffn"), 0, num_routed_experts=7) +def test_async_rejects_fused_shared_expert_ids(): + from afd_plugin.compat.npu.feature_validation import ( + _fail_if_unsupported_npu_afd_async_features, + ) + + config = _vllm_config() + config.additional_config["mix_placement"] = True + with pytest.raises(RuntimeError, match="routed-only.*mix_placement"): + _fail_if_unsupported_npu_afd_async_features( + config, + _afd_config(role="attention"), + AFDAsyncExtraInfo(), + ) + + +@pytest.mark.parametrize("mega", [False, True]) +def test_dispatch_failure_preserves_protocol_window_ownership(monkeypatch, mega): + fake_torch = _FakeTorch() + monkeypatch.setattr(async_cam_module, "torch", fake_torch) + connector = CAMAsyncAFDConnector( + 0, + 0, + _vllm_config( + extra_config={ + "attn_ranks_per_dp": 4, + "ffn_mega_moe": mega, + "dynamicQuant": 1, + } + ), + _afd_config(role="attention"), + 0, + ) + connector._initialized = True + connector.comm_args = _FakeTensor((1,), dtype="fp16") + context = AFDTransferContext( + metadata=AFDTransferMetadata.create_attention_metadata( + layer_idx=2, + stage_idx=0, + seq_len=3, + ) + ) + + def fail_dispatch(*_args): + raise RuntimeError("dispatch failed") + + monkeypatch.setattr( + fake_torch.ops.afd_ascend, + "afd_mega_dispatch_send" if mega else "afd_async_dispatch_send", + fail_dispatch, + ) + with pytest.raises(RuntimeError, match="dispatch failed"): + connector.send_attn_output(_FakeTensor((3, 16)), context, **_topk_payload(3)) + if mega: + with pytest.raises(RuntimeError, match="pending"): + connector.close() + else: + connector.close() + assert not connector.is_initialized + + +@pytest.fixture +def pull_attention(monkeypatch): + # CPU status words and host transfer recording, not NPU validation. + fake_torch = _FakeTorch() + fake_torch.int64 = torch.int64 + monkeypatch.setattr(async_cam_module, "torch", fake_torch) + connector = CAMAsyncAFDConnector( + 0, + 0, + _vllm_config( + extra_config={ + "ffn_mega_moe": True, + "attn_ranks_per_dp": 4, + "dynamicQuant": 1, + } + ), + _afd_config(role="attention"), + 0, + ) + connector._initialized = True + state = torch.zeros(128, dtype=torch.int64) + connector.comm_args = state.view(torch.float16) + store = MemoryStore() + connector.graph_startup._store = store + connector.graph_startup._prefix = "ffn_graph/test/" + for rank in (4, 5): + store.set(f"ffn_graph/test/pull_stopped/{rank}", "ready") + hidden = _FakeTensor((3, 16)) + context = AFDTransferContext( + metadata=AFDTransferMetadata.create_attention_metadata( + layer_idx=2, stage_idx=0, seq_len=3 + ) + ) + return SimpleNamespace( + connector=connector, + state=state, + hidden=hidden, + context=context, + torch=fake_torch, + ) + + +@pytest.mark.parametrize("phase", ["launch", "status"]) +def test_pull_combine_failure_retains_window_until_completion( + pull_attention, monkeypatch, phase +): + h = pull_attention + connector = h.connector + connector.send_attn_output(h.hidden, h.context, **_topk_payload(3)) + combine = h.torch.ops.afd_ascend.afd_mega_combine_recv + + def fail(*_args): + raise RuntimeError("combine failed") + + if phase == "launch": + monkeypatch.setattr(h.torch.ops.afd_ascend, "afd_mega_combine_recv", fail) + else: + h.state[async_cam_module.PULL_FAILURE_INDEX] = -5 + with pytest.raises(RuntimeError, match="combine failed|status -5"): + connector.recv_ffn_output(h.hidden, context=h.context) + with pytest.raises(RuntimeError, match="pending"): + connector.close() + monkeypatch.setattr(h.torch.ops.afd_ascend, "afd_mega_combine_recv", combine) + h.state[async_cam_module.PULL_FAILURE_INDEX] = 0 + connector.recv_ffn_output(h.hidden, context=h.context) + connector.close() + assert not connector.is_initialized and connector.comm_args is None + + +@pytest.mark.parametrize("nested", [False, True]) +def test_attention_step_reports_status_after_queued_combine(pull_attention, nested): + h = pull_attention + connector = h.connector + with pytest.raises(RuntimeError, match="status -5"), connector.attention_step(): + connector.send_attn_output(h.hidden, h.context, **_topk_payload(3)) + h.state[async_cam_module.PULL_FAILURE_INDEX] = -5 + if nested: + with connector.attention_step(): + output = connector.recv_ffn_output(h.hidden, context=h.context) + else: + output = connector.recv_ffn_output(h.hidden, context=h.context) + assert output.shape == h.hidden.shape + # An exception at the step boundary must restore immediate status checks. + connector.send_attn_output(h.hidden, h.context, **_topk_payload(3)) + with pytest.raises(RuntimeError, match="status -5"): + connector.recv_ffn_output(h.hidden, context=h.context) + h.state[async_cam_module.PULL_FAILURE_INDEX] = 0 + connector.recv_ffn_output(h.hidden, context=h.context) + connector.close() + assert not connector.is_initialized + + +def test_attention_step_restores_status_checks_after_model_error(pull_attention): + h = pull_attention + with pytest.raises(ValueError, match="model failed"), h.connector.attention_step(): + raise ValueError("model failed") + h.connector.send_attn_output(h.hidden, h.context, **_topk_payload(3)) + h.state[async_cam_module.PULL_FAILURE_INDEX] = -5 + with pytest.raises(RuntimeError, match="status -5"): + h.connector.recv_ffn_output(h.hidden, context=h.context) + + def test_close_releases_every_stage_routing(monkeypatch): fake_torch = _FakeTorch() monkeypatch.setattr(async_cam_module, "torch", fake_torch) diff --git a/tests/unit/fakes/async_cam_startup.py b/tests/unit/fakes/async_cam_startup.py new file mode 100644 index 000000000..89642b8e7 --- /dev/null +++ b/tests/unit/fakes/async_cam_startup.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""In-memory Store and rank world for the host startup protocol.""" + +from threading import Condition + +from afd_plugin.connectors.npu.async_cam_runtime.async_cam_startup import ( + AsyncCAMGraphStartup, +) + + +class MemoryStore: + def __init__(self): + self.values = {} + self.condition = Condition() + self.closed = False + + def set(self, key, value): + assert not self.closed, "Store used after release" + with self.condition: + self.values[key] = value.encode() if isinstance(value, str) else value + self.condition.notify_all() + + def get(self, key): + assert not self.closed, "Store used after release" + with self.condition: + assert self.condition.wait_for(lambda: key in self.values, timeout=3) + return self.values[key] + + def check(self, keys): + assert not self.closed, "Store used after release" + with self.condition: + return all(key in self.values for key in keys) + + +def startup_world(attn=2, ffn=2, store=None): + store = store or MemoryStore() + owners = [] + for rank in range(attn + ffn): + owner = AsyncCAMGraphStartup(rank, attn, attn + ffn) + owner._store = store + owner._prefix = "ffn_graph/test/" + owners.append(owner) + return owners diff --git a/tests/unit/fakes/async_ffn.py b/tests/unit/fakes/async_ffn.py index 838084860..912e169ba 100644 --- a/tests/unit/fakes/async_ffn.py +++ b/tests/unit/fakes/async_ffn.py @@ -46,10 +46,9 @@ def stub(name, **attributes): "afd_plugin.v1.worker.ffn_metadata", ): stub(name) - stub( - "afd_plugin.connectors.base", AFDConnectorBase=object, ConnectorExtraInfo=object - ) stub("afd_plugin.connectors.metadata", AFDTransferState=object) + load_host_module(monkeypatch, "afd_plugin.connectors.base") + load_host_module(monkeypatch, "tests.unit.fakes.connector") group = load_host_module(monkeypatch, "afd_plugin.distributed.afd_process_group") stub("afd_plugin.distributed", init_afd_process_group=group.init_afd_process_group) connector = load_host_module(monkeypatch, "afd_plugin.connectors.npu.async_cam") diff --git a/tests/unit/fakes/connector.py b/tests/unit/fakes/connector.py index 7fc57c220..fa158f5e0 100644 --- a/tests/unit/fakes/connector.py +++ b/tests/unit/fakes/connector.py @@ -8,6 +8,7 @@ from __future__ import annotations +from collections.abc import Callable, Iterable from types import SimpleNamespace from typing import Any @@ -19,6 +20,46 @@ ) +class FakeFFNSession: + """Configurable host lifecycle seam, with no device behavior.""" + + def __init__( + self, + *, + serving: Iterable[bool] = (), + requires_drain: bool = False, + drain_error: Exception | None = None, + transactions: int = 0, + ) -> None: + self.requires_drain = requires_drain + self.serving = iter(serving) + self.drain_error = drain_error + self.transactions = transactions + self.receiver_finished = False + + def keep_serving(self, shutdown_requested: bool) -> bool: + return next(self.serving, not shutdown_requested) + + def before_first_step(self) -> None: + pass + + def after_step(self) -> None: + pass + + def drain(self, run_one: Callable[[], None]) -> None: + for _ in range(self.transactions): + run_one() + if self.drain_error is not None: + raise self.drain_error + + def finish_receiver(self) -> None: + self.receiver_finished = True + + def validate_shutdown(self) -> None: + if self.requires_drain and not self.receiver_finished: + raise RuntimeError("receiver has not finished; retain resources") + + class FakeRoutedConnector(AFDRoutedExpertsConnector): def __init__(self) -> None: self.events: list[tuple] = [] diff --git a/tests/unit/model_executor/test_async_cam_layered.py b/tests/unit/model_executor/test_async_cam_layered.py index 32578cf72..7ea83f76b 100644 --- a/tests/unit/model_executor/test_async_cam_layered.py +++ b/tests/unit/model_executor/test_async_cam_layered.py @@ -19,6 +19,9 @@ AsyncCAMLayeredExecutor, LayeredFFNWeights, ) +from afd_plugin.model_executor.npu.async_cam_mega import ( # noqa: E402 + AsyncCAMMegaExecutor, +) def weights(layer_idx=0, per_channel=True, quantization="W4A8"): @@ -58,6 +61,56 @@ def weights(layer_idx=0, per_channel=True, quantization="W4A8"): ) +def test_native_rejects_grouped_scale_before_receiving(operators): + executor = AsyncCAMLayeredExecutor([weights(per_channel=False)]) + with pytest.raises(ValueError, match="per-channel"): + AsyncCAMMegaExecutor(executor, None) + + +@pytest.mark.parametrize("quantization", ["W4A8", "W8A8"]) +def test_native_forwards_the_applied_swiglu_limit(monkeypatch, operators, quantization): + calls = [] + + def execute(*args): + calls.append(args) + return torch.empty(1, dtype=torch.int64) + + monkeypatch.setenv("BATCH_SIZE_FACTOR", str(16 / 262144)) + monkeypatch.setattr( + torch.ops.afd_ascend, + "afd_mega_moe_pull_workspace_size", + lambda *args: 65536, + raising=False, + ) + monkeypatch.setattr( + torch.ops.afd_ascend, "afd_mega_moe_pull_native", execute, raising=False + ) + executor = AsyncCAMMegaExecutor( + AsyncCAMLayeredExecutor( + [replace(weights(quantization=quantization), swiglu_limit=10.0)] + ), + SimpleNamespace( + max_num_batched_tokens=8, + topk=2, + ffn_size=2, + attn_size=1, + world_rank=1, + topology=SimpleNamespace(world_size=3), + tp_size=1, + comm_args=torch.empty(512, dtype=torch.float16), + group_name="test-pull", + ), + ) + executor() + assert calls[0][-1] == (10.0 if quantization == "W4A8" else 0.0) + assert calls[0][10][11] == (1 if quantization == "W4A8" else 2) + assert calls[0][4][0].dtype == ( + torch.int32 if quantization == "W4A8" else torch.int8 + ) + if quantization == "W8A8": + assert calls[0][8] == calls[0][9] == [] + + @pytest.fixture def operators(monkeypatch): calls = [] @@ -203,6 +256,85 @@ def test_repeated_interleaved_layer_ids(operators): assert [call[1][6].item() for call in operators[::2]] == [1, 0, 1, 1] +def runner_method(name, namespace): + tree = ast.parse( + Path("afd_plugin/v1/worker/npu/async_ffn_model_runner.py").read_text() + ) + cls = next( + node + for node in tree.body + if isinstance(node, ast.ClassDef) and node.name == "AsyncNPUFFNModelRunner" + ) + method = next( + node + for node in cls.body + if isinstance(node, ast.FunctionDef) and node.name == name + ) + module = ast.Module( + body=[ + ast.ImportFrom( + module="__future__", names=[ast.alias(name="annotations")], level=0 + ), + method, + ], + type_ignores=[], + ) + exec( + compile(ast.fix_missing_locations(module), "", "exec"), + namespace, + ) + return namespace[name] + + +def test_runner_preserves_raw_context_and_chunk_capacity(): + payloads = [ + SimpleNamespace( + hidden_states=torch.empty(8, 32), + context=SimpleNamespace( + states=SimpleNamespace( + dynamic_scales=torch.ones(8), + group_list=torch.tensor([0, 0]), + token_nums_rankid_layeridx=torch.tensor([99, 1, 5, 0]), + ) + ), + ) + for _ in range(2) + ] + received = iter(payloads) + sent = [] + computed = [] + connector = SimpleNamespace( + recv_attn_output=lambda: next(received), + send_ffn_output=lambda result, ctx: sent.append((result, ctx)), + ) + + def compute(*args): + computed.append(args) + return torch.empty(8, 32, dtype=torch.bfloat16) + + runner = SimpleNamespace( + model=object(), + connector=connector, + _two_stream=None, + _mega_executor=None, + _layered_executor=compute, + layer_ids=(2, 5), + ) + method = runner_method( + "_ffn_forward_connector_driven", + { + "cast": cast, + "CAMAsyncAFDConnector": SimpleNamespace, + "AFDAsyncTransferState": SimpleNamespace, + }, + ) + method(runner) + assert [ctx for _, ctx in sent] == [p.context for p in payloads] + for args, payload in zip(computed, payloads, strict=True): + assert args[0] is payload.hidden_states + assert args[3] is payload.context.states.token_nums_rankid_layeridx + + def model_weights_method(monkeypatch): source = Path("afd_plugin/model_executor/models/npu/deepseek_v4.py").read_text() function = next( diff --git a/tests/unit/package/test_ascend_build_files.py b/tests/unit/package/test_ascend_build_files.py index 713ec23d8..3ab0b90cc 100644 --- a/tests/unit/package/test_ascend_build_files.py +++ b/tests/unit/package/test_ascend_build_files.py @@ -183,6 +183,8 @@ def _run_select_ops(*args: str) -> subprocess.CompletedProcess[str]: "afd_async_combine_recv", "grouped_matmul_swiglu_quant_v2_layered", "grouped_matmul_layered", + "afd_mega_moe_pull_native", + "afd_mega_moe_comm", ], ), ("ascend950", ["a2e", "e2a"]), diff --git a/tests/unit/package/test_mega_activation_patch.py b/tests/unit/package/test_mega_activation_patch.py new file mode 100644 index 000000000..efaff843a --- /dev/null +++ b/tests/unit/package/test_mega_activation_patch.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Check that unsupported SDK quantization is rejected instead of left unpatched.""" + +import runpy +from pathlib import Path + +import pytest + +PATCH = ( + Path(__file__).resolve().parents[3] + / "csrc/npu/ascend_kernels/afd_mega_moe_pull_native/op_kernel" + / "afd_mega_moe_patch_activation.py" +) +patched_activation = runpy.run_path(str(PATCH))["patched_activation"] + + +@pytest.mark.parametrize( + ("scale", "tensor", "length"), + [ + ("127.0f / rowAbsMax", "ubGateFp32", "tileLength"), + ("127.f / GMubDequantScale", "ubOutputTmp", "ChunkTileLen"), + ], +) +def test_layered_rounding_is_applied(scale, tensor, length): + source = f""" + auto ubQuantS32 = ubGateFp32.template ReinterpretCast(); + auto scale = {scale}; + AscendC::Cast(ubQuantS32, {tensor}, AscendC::RoundMode::CAST_RINT, {length}); + AscendC::PipeBarrier(); + AscendC::SetDeqScale(static_cast(1.0)); + AscendC::Cast(ubQuantF16, ubQuantS32, AscendC::RoundMode::CAST_RINT, {length}); + """ + result = patched_activation(source) + assert "1.0f / (" in result + cast = f"Cast(ubQuantF16, {tensor}, AscendC::RoundMode::CAST_RINT, {length})" + assert cast in result + assert "ubQuantS32" not in result + + +def test_unknown_sdk_header_is_rejected(): + with pytest.raises(ValueError, match="unsupported CANN activation"): + patched_activation("unrecognized SDK implementation") diff --git a/tests/unit/v1/worker/test_async_ffn_graph.py b/tests/unit/v1/worker/test_async_ffn_graph.py index a403788e5..578edb996 100644 --- a/tests/unit/v1/worker/test_async_ffn_graph.py +++ b/tests/unit/v1/worker/test_async_ffn_graph.py @@ -12,6 +12,7 @@ pytest.importorskip("torch_npu") import torch # noqa: E402 +from afd_plugin.connectors.base import DefaultFFNSession # noqa: E402 from afd_plugin.v1.worker.npu.async_ffn_model_runner import ( # noqa: E402 AsyncNPUFFNModelRunner, ) @@ -78,10 +79,14 @@ def send(output, context): connector.recv_attn_output = receive connector.send_ffn_output = send + connector.ffn_session = DefaultFFNSession session = AsyncNPUFFNModelRunner.__new__(AsyncNPUFFNModelRunner) session.model = torch.nn.Linear(1, 1) session._layered_executor = execute + session._mega_executor = None + session._two_stream = None session.connector = connector + session.session = connector.ffn_session() session._graph = None session._graph_capture_count = session._graph_replay_count = 0 session._graph_pending = session._graph_closed = False diff --git a/tests/unit/v1/worker/test_async_ffn_runtime.py b/tests/unit/v1/worker/test_async_ffn_runtime.py index 726002ae0..cd042b88d 100644 --- a/tests/unit/v1/worker/test_async_ffn_runtime.py +++ b/tests/unit/v1/worker/test_async_ffn_runtime.py @@ -4,6 +4,7 @@ from __future__ import annotations +import threading from collections import deque from types import SimpleNamespace @@ -24,6 +25,8 @@ CAMAsyncAFDConnector, ) from afd_plugin.v1.worker.npu import async_ffn_model_runner as runtime # noqa: E402 +from afd_plugin.v1.worker.npu.ffn_worker import AFDNPUFFNWorker # noqa: E402 +from tests.unit.fakes.connector import FakeFFNSession # noqa: E402 from tests.unit.fakes.vllm_config import npu_vllm_config # noqa: E402 @@ -135,3 +138,51 @@ def execute(hidden, scales, groups, header): ): torch.testing.assert_close(sent[1], hidden.to(torch.bfloat16), rtol=0, atol=0) assert sent[2] is received_payload.context.states.token_nums_rankid_layeridx + + +@pytest.mark.parametrize("fail", [False, True]) +def test_full_receiver_exit_requires_success_even_during_shutdown(harness, fail): + worker = AFDNPUFFNWorker.__new__(AFDNPUFFNWorker) + worker.model_runner = harness.runner + harness.runner.effective_mode = "full" + harness.runner.session = FakeFFNSession(requires_drain=True) + worker._ffn_thread = None + worker._ffn_loop_error = None + worker._bind_cpus_once = lambda: None + + def serve(): + worker._ffn_shutdown_event.set() + if fail: + raise RuntimeError("partial device completion") + + worker._run_ffn_server_loop = serve + worker.start_ffn_server_loop() + assert isinstance(worker._ffn_thread, threading.Thread) + worker._ffn_thread.join(3) + assert not worker._ffn_thread.is_alive() + assert harness.runner._graph_receiver_exited is not fail + assert (worker._ffn_loop_error is not None) is fail + + +@pytest.mark.parametrize("drains", [True, False]) +def test_drain_propagates_session_result(harness, monkeypatch, drains): + h = harness + h.runner.session = FakeFFNSession( + transactions=1, + drain_error=None if drains else RuntimeError("retain resources"), + ) + h.runner._layered_executor = lambda hidden, *_args: hidden.to(torch.bfloat16) + request = payload(5, 3) + h.payloads.append(request) + monkeypatch.setattr(torch.npu, "synchronize", lambda: None) + if drains: + h.runner.drain() + else: + with pytest.raises(RuntimeError, match="retain resources"): + h.runner.drain() + returned = [ + call for call in h.calls if isinstance(call, tuple) and call[0] == "send" + ] + assert len(returned) == 1 + torch.testing.assert_close(returned[0][1], request.hidden_states.to(torch.bfloat16)) + assert returned[0][2] is request.context.states.token_nums_rankid_layeridx diff --git a/tests/unit/v1/worker/test_async_ffn_two_stream.py b/tests/unit/v1/worker/test_async_ffn_two_stream.py index acd2fbbfb..a715afb44 100644 --- a/tests/unit/v1/worker/test_async_ffn_two_stream.py +++ b/tests/unit/v1/worker/test_async_ffn_two_stream.py @@ -9,7 +9,9 @@ import pytest -from afd_plugin.async_cam_startup import AsyncCAMGraphStartup +from afd_plugin.connectors.npu.async_cam_runtime.async_cam_startup import ( + AsyncCAMGraphStartup, +) from tests.unit.fakes.async_ffn import load_async_modules diff --git a/tests/unit/v1/worker/test_npu_model_runner_v2.py b/tests/unit/v1/worker/test_npu_model_runner_v2.py index c35dbcafa..62b47d734 100644 --- a/tests/unit/v1/worker/test_npu_model_runner_v2.py +++ b/tests/unit/v1/worker/test_npu_model_runner_v2.py @@ -56,7 +56,9 @@ def _runner(*, async_dp=False): runner.afd_config = SimpleNamespace( async_dp=async_dp, num_attention_ranks=2, num_ffn_ranks=2 ) - runner.connector = SimpleNamespace(extra_info=SimpleNamespace()) + runner.connector = SimpleNamespace( + extra_info=SimpleNamespace(), attention_step=nullcontext + ) runner.cudagraph_manager = None runner.ubatch_runner = None runner.prof = object() diff --git a/tests/unit/v1/worker/test_npu_runtime.py b/tests/unit/v1/worker/test_npu_runtime.py index 2eae24c4e..b62ddfd43 100644 --- a/tests/unit/v1/worker/test_npu_runtime.py +++ b/tests/unit/v1/worker/test_npu_runtime.py @@ -23,6 +23,7 @@ AFDTransferMetadata, AFDTransferState, ) +from afd_plugin.connectors.base import DefaultFFNSession from tests.unit.fakes.dp_metadata import ( # noqa: E402 DPMetadataValues as _FakeDPMetadata, ) @@ -827,7 +828,9 @@ def test_npu_ffn_worker_uses_connector_driven_loop_for_async_connector(monkeypat event = threading.Event() calls: list[str | tuple[str, SimpleNamespace]] = [] - def execute_connector_driven_step(): + def execute_connector_driven_step(shutdown_requested): + if shutdown_requested: + return False calls.append("step") event.set() return True @@ -836,7 +839,9 @@ def execute_connector_driven_step(): worker.device = SimpleNamespace(type="cpu") worker.model_runner = object.__new__(ffn_worker.AsyncNPUFFNModelRunner) worker.model_runner.connector = SimpleNamespace(control_plane=None) + worker.model_runner.session = DefaultFFNSession() worker.model_runner.effective_mode = "eager" + worker.model_runner._two_stream = None worker.model_runner.execute_connector_driven_step = execute_connector_driven_step monkeypatch.setattr( From 127d8ecc5be0f86fe43c44a30a343966b1c9d090 Mon Sep 17 00:00:00 2001 From: Li Ruixin <616517220@qq.com> Date: Thu, 5 Nov 2026 19:28:25 -0800 Subject: [PATCH 07/42] [NPU] Remove FFN two-stream execution (#550) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Purpose Remove the NPU Async CAM FFN two-stream execution path, simplifying FFN startup and execution around the remaining eager and FULL paths. ## Issue - Related issue(s): Not linked. - Closing keyword, only if fully resolved: None. ## Scope - In scope: Remove the two-stream executor, its connector options and startup negotiation, associated tests, and documentation; update the remaining FFN runner and worker paths. - Out of scope: Changes to the native CAM operators, Attention DBO, or the behavior and configuration of the existing layered and MegaMoE executors. ## Implementation Notes - Remove `AsyncFFNTwoStream` and its eager/FULL capture, submission, completion, and shutdown branches. FFN continues through its existing eager or single-graph FULL execution path; Pull retains its session-driven drain. - Remove `ffn_two_stream`, `ffn_compute_aic_num`, `ffn_compute_aiv_num`, and `ffn_combine_aiv_num` from `connector_extra_config`. **This is a configuration compatibility break**: any of these fields, including `ffn_two_stream=false`, is now rejected as unknown. Remove them from existing configurations before upgrading. - Remove two-stream-specific startup signatures, Attention validation, and resource-budget checks. MegaMoE still requires `dynamicQuant=1`. - The FFN startup plan changes; deploy Attention and FFN ranks with the same plugin version. No new compatibility shim or monkey patch is introduced. - Delete two-stream-specific tests and update configuration, startup, graph, worker, and fake coverage for the remaining paths. Add coverage that the removed options are rejected. ## Test Plan - Run CPU unit tests, including connector configuration and startup, graph negotiation, and rejection of the retired options. - Run matched real-model GSM8K evaluations for Layered and Mega ON using Async CAM FULL on A3. - Check repository deployment examples for the four retired options. ## Test Result - Branch: `chore/remove-ffn-two-stream`, commit `28fcfb706cb505cac9550e9e76fbfc909214891d`, based on `upstream/main` at `022d8a7e7a521994f0bbda5d9c278c71e8ceb1dd`. The tested patch has SHA256 `e43d9afb1f1773eea9a6560a247156ea46af243716e36a15c543b2209ab7cad3`; the commit adds one Markdown blank line after testing. - `git diff --check` and applicable pre-commit checks, including Ruff, Markdown lint and mypy: passed. The affected CPU suites reported **127 passed, 5 skipped**; the skips require local `torch` or `vllm`. - CPU unit suite, `pytest -o addopts='' -q tests/unit -m 'not gpu and not vllm_runtime' -ra`, with isolated `setuptools 80.10.2`: **621 passed, 91 skipped, 8 deselected, 10 subtests passed**. Skips include unavailable `torch`, `vllm`, `vllm-ascend`, `numpy`, and `cmake`. CUDA tests were not run. - A3 real-model evaluation used DeepSeek-V4 Flash W4A8, Attention DP2×TP4 eager, FFN EP8×TP1 FULL, DBO off, 8-shot GSM8K, 300 samples, concurrency 12, seed 1024, and up to 512 generated tokens. Both modes used identical question IDs, prompts and inputs. | FFN mode | GSM8K strict match | GSM8K flexible extract | | --- | ---: | ---: | | Layered | 286/300 (95.33%) | 286/300 (95.33%) | | Mega ON | 286/300 (95.33%) | 286/300 (95.33%) | All eight FFN ranks captured and replayed the expected FULL graph in each mode. The 14 unmatched questions in each run include 10 shared cases and four cases unique to each mode. - The A3 image used `vllm 0.32.0+empty`, `vllm-ascend 0.19.1rc2.dev3100+gbd42eea01`, `torch 2.10.0+cpu`, and `torch-npu 2.10.0.post7`; the installed vLLM package did not record its source commit. Existing native binaries were verified against their recorded hashes and reused; the native build inputs match the recorded build except for a removed comment in one header. A fresh native build and FFN eager-mode device validation were not run. - Local raw results and per-sample outputs: `results/remove-ffn-two-stream-022d8a7e/`. ## Docs Impact - Files updated: `csrc/npu/README.md`, the connector, execution platform, FFN runtime and model integration design documents, the NPU Async FFN runtime and Async CAM connector guides, and the DeepSeek-V4 Flash recipe. - If none, reason: Not applicable. ---
Essential PR Checklist - [x] Purpose is clear and linked to public context when possible. - [x] Scope is bounded. - [x] Compatibility with vLLM v0.32.0 and, for NPU changes, the pinned vLLM-Ascend source is considered, as applicable. - [x] No changes are made to the vLLM source checkout. - [x] Plugin-owned classes or explicit dotted class paths are preferred over monkey patches. - [x] Any compat shim or monkey patch is isolated, idempotent, version-guarded, documented, and tested. - [x] Imports remain CPU-safe; CUDA-heavy work is delayed or GPU-gated. - [x] Validation evidence is included, including skipped GPU tests when applicable. - [x] Documentation impact is stated.
Signed-off-by: lirx-pd <616517220@qq.com> --- afd_plugin/compat/npu/feature_validation.py | 18 -- afd_plugin/connectors/npu/async_cam.py | 58 +---- .../async_cam_runtime/async_cam_startup.py | 25 +- .../v1/worker/npu/async_ffn_model_runner.py | 68 +---- .../v1/worker/npu/async_ffn_two_stream.py | 203 --------------- afd_plugin/v1/worker/npu/ffn_worker.py | 6 +- csrc/npu/README.md | 4 +- docs/design/module/connector_contracts.md | 2 +- docs/design/module/execution_platforms.md | 2 +- docs/design/module/ffn_runtime.md | 14 -- docs/design/module/model_integration.md | 2 +- docs/npu/ASYNC_FFN_RUNTIME.md | 72 +----- docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md | 10 - .../deepseek_v4_flash/README.md | 2 +- .../compat/npu/test_feature_validation.py | 9 - tests/unit/config/test_config.py | 6 +- .../connectors/npu/test_async_cam_startup.py | 39 +-- .../npu/test_async_cam_startup_wiring.py | 115 ++------- .../connectors/test_async_cam_connector.py | 15 +- tests/unit/fakes/async_ffn.py | 1 - .../model_executor/test_async_cam_layered.py | 1 - tests/unit/v1/worker/test_async_ffn_graph.py | 1 - .../v1/worker/test_async_ffn_two_stream.py | 234 ------------------ tests/unit/v1/worker/test_npu_runtime.py | 1 - 24 files changed, 49 insertions(+), 859 deletions(-) delete mode 100644 afd_plugin/v1/worker/npu/async_ffn_two_stream.py delete mode 100644 tests/unit/v1/worker/test_async_ffn_two_stream.py diff --git a/afd_plugin/compat/npu/feature_validation.py b/afd_plugin/compat/npu/feature_validation.py index 4e9e756df..0efeeeda6 100644 --- a/afd_plugin/compat/npu/feature_validation.py +++ b/afd_plugin/compat/npu/feature_validation.py @@ -170,22 +170,6 @@ def _fail_if_unsupported_npu_afd_async_features( ) parallel_config = vllm_config.parallel_config - if extra_info.ffn_two_stream: - if not afd_config.is_ffn_server: - raise RuntimeError("ffn_two_stream may only be enabled on the FFN role") - if ( - not vllm_config.use_v2_model_runner - or not afd_config.compute_gate_on_attention - or extra_info.dynamic_quant != 1 - or parallel_config.tensor_parallel_size != 1 - or parallel_config.use_sequence_parallel_moe - or os.environ.get("VLLM_ASCEND_ENABLE_FLASHCOMM1", "0").strip().lower() - in ("1", "true") - ): - raise RuntimeError( - "FFN two-stream requires MRV2 layered GMM, dynamicQuant=1, " - "Attention-side gate, F TP=1 and SP disabled" - ) if not is_afd_async_dp(vllm_config): raise RuntimeError( "CAMAsyncAFDConnector requires additional_config['afd'] " @@ -222,8 +206,6 @@ def _fail_if_unsupported_npu_afd_async_features( if speculative_config.method == "dspark": if vllm_config.model_config.hf_text_config.model_type != "deepseek_v4": raise RuntimeError("AFD DSpark supports only DeepSeek-V4 on NPU") - if extra_info.ffn_two_stream: - raise RuntimeError("AFD DSpark does not support FFN two-stream") # The shared MRV2 validator owns PP/CP and static EP constraints. # Keep only the additional target/draft restrictions here. # F may replay its expert transaction in FULL; Attention and native diff --git a/afd_plugin/connectors/npu/async_cam.py b/afd_plugin/connectors/npu/async_cam.py index a229286a3..3a430ef6f 100644 --- a/afd_plugin/connectors/npu/async_cam.py +++ b/afd_plugin/connectors/npu/async_cam.py @@ -92,10 +92,6 @@ "ffn_mega_moe", "attn_ranks_per_dp", "hccl_buffer_size", - "ffn_two_stream", - "ffn_compute_aic_num", - "ffn_compute_aiv_num", - "ffn_combine_aiv_num", }, ) @@ -166,10 +162,6 @@ class AFDAsyncExtraInfo(ConnectorExtraInfo): dynamic_quant: int = 0 attn_ranks_per_dp: int = 1 hccl_buffer_size: int | None = None - ffn_two_stream: bool = False - ffn_compute_aic_num: int = 12 - ffn_compute_aiv_num: int = 24 - ffn_combine_aiv_num: int = 24 @classmethod def from_mapping(cls, raw: Mapping[str, Any] | None) -> AFDAsyncExtraInfo: @@ -205,24 +197,10 @@ def from_mapping(cls, raw: Mapping[str, Any] | None) -> AFDAsyncExtraInfo: raw.get("hccl_buffer_size"), field_name="hccl_buffer_size", ), - ffn_two_stream=coerce_extra_bool( - raw.get("ffn_two_stream", False), field_name="ffn_two_stream" - ), - ffn_compute_aic_num=coerce_extra_positive_int( - raw.get("ffn_compute_aic_num", 12), field_name="ffn_compute_aic_num" - ), - ffn_compute_aiv_num=coerce_extra_positive_int( - raw.get("ffn_compute_aiv_num", 24), field_name="ffn_compute_aiv_num" - ), - ffn_combine_aiv_num=coerce_extra_positive_int( - raw.get("ffn_combine_aiv_num", 24), field_name="ffn_combine_aiv_num" - ), ) - if result.ffn_mega_moe and (result.dynamic_quant != 1 or result.ffn_two_stream): - raise ValueError( - "ffn_mega_moe requires dynamicQuant=1 and ffn_two_stream=false" - ) + if result.ffn_mega_moe and result.dynamic_quant != 1: + raise ValueError("ffn_mega_moe requires dynamicQuant=1") return result def to_mapping(self) -> dict[str, Any]: @@ -234,17 +212,6 @@ def to_mapping(self) -> dict[str, Any]: result["ffn_mega_moe"] = True if self.hccl_buffer_size is not None: result["hccl_buffer_size"] = self.hccl_buffer_size - if self.ffn_two_stream or ( - self.ffn_compute_aic_num, - self.ffn_compute_aiv_num, - self.ffn_combine_aiv_num, - ) != (12, 24, 24): - result.update( - ffn_two_stream=self.ffn_two_stream, - ffn_compute_aic_num=self.ffn_compute_aic_num, - ffn_compute_aiv_num=self.ffn_compute_aiv_num, - ffn_combine_aiv_num=self.ffn_combine_aiv_num, - ) return result @@ -423,9 +390,6 @@ def init_afd_connector(self) -> None: self._run_attention_graph_startup() def _initialize_graph_startup(self, store: Store) -> None: - self.graph_startup.validate_attention_streams = ( - self._validate_two_stream_attention - ) signature = repr( ( self.attn_size, @@ -457,24 +421,6 @@ def _initialize_graph_startup(self, store: Store) -> None: self.graph_startup.fail(exc) raise - def _validate_two_stream_attention(self) -> None: - config = self.vllm_config - parallel = config.parallel_config - if ( - not config.use_v2_model_runner - or not config.model_config.enforce_eager - or parallel.pipeline_parallel_size != 1 - or parallel.prefill_context_parallel_size != 1 - or parallel.decode_context_parallel_size != 1 - or parallel.use_sequence_parallel_moe - or os.environ.get("VLLM_ASCEND_ENABLE_FLASHCOMM1", "0").strip().lower() - in ("1", "true") - or config.speculative_config is not None - ): - raise ValueError( - "FFN two-stream requires eager MRV2 Attention without PP/CP/SP" - ) - def wait_graph_device(self, event: torch.npu.Event) -> None: deadline = time.monotonic() + GRAPH_STARTUP_TIMEOUT_SECONDS while not event.query(): diff --git a/afd_plugin/connectors/npu/async_cam_runtime/async_cam_startup.py b/afd_plugin/connectors/npu/async_cam_runtime/async_cam_startup.py index 77bd8e7da..97b12f7cd 100644 --- a/afd_plugin/connectors/npu/async_cam_runtime/async_cam_startup.py +++ b/afd_plugin/connectors/npu/async_cam_runtime/async_cam_startup.py @@ -7,7 +7,6 @@ import json import time import uuid -from collections.abc import Callable from dataclasses import asdict, dataclass from typing import TYPE_CHECKING, Literal @@ -24,18 +23,11 @@ class FFNGraphPlan: executor_error: str = "" signature: str = "" layer_ids: tuple[int, ...] = () - stream_signature: str = "" - stream_error: str = "" def resolve_ffn_mode(plans: list[FFNGraphPlan]) -> str: if not plans or len({plan.requested for plan in plans}) != 1: raise ValueError("All FFN ranks must request the same execution mode") - if len({plan.stream_signature for plan in plans}) != 1: - raise ValueError("FFN two-stream execution signatures differ across ranks") - errors = [plan.stream_error for plan in plans if plan.stream_error] - if errors: - raise ValueError("Async FFN two-stream is unsupported: " + "; ".join(errors)) errors = [plan.executor_error for plan in plans if plan.executor_error] if errors: raise ValueError( @@ -46,8 +38,6 @@ def resolve_ffn_mode(plans: list[FFNGraphPlan]) -> str: if mode not in ("eager", "full"): raise ValueError("Unknown Async FFN execution mode") execution = "FULL" if mode == "full" else "eager" - if mode == "eager" and plans[0].stream_signature: - execution = "two-stream" if len({plan.signature for plan in plans}) > 1: raise ValueError( f"Async FFN {execution} is unsupported: " @@ -74,7 +64,6 @@ def __init__(self, world_rank: int, attn_size: int, world_size: int) -> None: self._store: Store | None = None self._prefix = "" self._shared_errors = False - self.validate_attention_streams: Callable[[], None] | None = None def initialize(self, store: Store, transport_signature: str) -> None: self._store = store @@ -158,24 +147,14 @@ def _negotiate(self, plan: FFNGraphPlan | None, transport_signature: str) -> Non plans.append(FFNGraphPlan(**fields)) self.mode = resolve_ffn_mode(plans) ffn_plan = plans[0] - two_stream = bool(ffn_plan.stream_signature) - validate_transport = ( - self.mode == "full" - or two_stream - or any(entry["transport"].startswith("pull_v1:") for entry in entries) + validate_transport = self.mode == "full" or any( + entry["transport"].startswith("pull_v1:") for entry in entries ) if ( validate_transport and len({entry["transport"] for entry in entries}) != 1 ): raise ValueError("Async CAM graph transport signatures differ") - if two_stream: - if ( - self.world_rank < self.attn_size - and self.validate_attention_streams is not None - ): - self.validate_attention_streams() - self.phase("streams-validated") if self.mode == "full": self.warmup_layer_id = ffn_plan.layer_ids[0] except Exception as exc: diff --git a/afd_plugin/v1/worker/npu/async_ffn_model_runner.py b/afd_plugin/v1/worker/npu/async_ffn_model_runner.py index 9d13ed394..4ef9070e6 100644 --- a/afd_plugin/v1/worker/npu/async_ffn_model_runner.py +++ b/afd_plugin/v1/worker/npu/async_ffn_model_runner.py @@ -35,7 +35,6 @@ LayeredFFNWeights, ) from afd_plugin.model_executor.npu.async_cam_mega import AsyncCAMMegaExecutor -from afd_plugin.v1.worker.npu.async_ffn_two_stream import AsyncFFNTwoStream from afd_plugin.validation import ( cam_receive_capacity, requested_ffn_mode, @@ -58,7 +57,6 @@ def get_async_cam_layered_weights(self) -> tuple[list[LayeredFFNWeights], str]: class AsyncNPUFFNModelRunner: afd_expected_role = "ffn" - _two_stream: AsyncFFNTwoStream | None = None def __init__( self, @@ -155,7 +153,7 @@ def initialize_afd_connector(self) -> None: if self.connector.is_initialized: return assert self.model is not None - stream_signature = stream_error = executor_error = signature = "" + executor_error = signature = "" config = self.connector.extra_info try: models = (self.model,) + ( @@ -178,42 +176,15 @@ def initialize_afd_connector(self) -> None: except Exception as exc: executor_error = str(exc) or type(exc).__name__ executor = self._layered_executor - if config.ffn_two_stream: - stream_signature = repr( - ( - 1, - 2, - config.ffn_compute_aic_num, - config.ffn_compute_aiv_num, - config.ffn_combine_aiv_num, - ) - ) - try: - if not executor_error: - assert executor is not None - self._two_stream = AsyncFFNTwoStream( - self.device, - config, - self.connector, - executor, - ) - except Exception as exc: - stream_error = str(exc) or type(exc).__name__ self.connector.graph_startup.plan = FFNGraphPlan( requested=self.requested_mode, executor_error=executor_error, signature=signature, layer_ids=self.layer_ids, - stream_signature=stream_signature, - stream_error=stream_error, ) try: self.connector.init_afd_connector() self.effective_mode = self.connector.graph_startup.mode - if self._two_stream is not None: - self._two_stream.compute.wait_stream( - torch.npu.current_stream(self.device) - ) if self.effective_mode == "full": assert executor is not None chunks = self.connector.graph_startup_chunks() * ( @@ -225,10 +196,9 @@ def initialize_afd_connector(self) -> None: done.record() self._wait_graph_device(done) self.connector.graph_startup.phase("warmup") - if self._two_stream is None: - stream = torch.npu.Stream(device=self.device) - stream.wait_stream(torch.npu.current_stream(self.device)) - self._initialize_graph(stream) + stream = torch.npu.Stream(device=self.device) + stream.wait_stream(torch.npu.current_stream(self.device)) + self._initialize_graph(stream) self._capture_graph() self.connector.graph_startup.phase("captured") except Exception as exc: @@ -266,10 +236,6 @@ def _check_graph_live(self) -> None: @torch.inference_mode() def _capture_graph(self) -> None: self._check_graph_live() - if self._two_stream is not None: - self._two_stream.capture() - self._graph_capture_count = self._two_stream.capture_count - return if self._graph is not None: raise RuntimeError("Async FFN may only capture once") assert self._layered_executor is not None @@ -307,13 +273,6 @@ def _capture_graph(self) -> None: @torch.inference_mode() def _submit_graph(self, count: int = 1) -> None: self._check_graph_live() - if self._two_stream is not None: - if self._graph_pending or self._graph_receiver_exited: - raise RuntimeError("Async FFN graph is not ready for replay") - self._graph_pending = True - self._two_stream.submit(count) - self._graph_replay_count = self._two_stream.sequence - return if ( self._graph is None or self._graph_capture_count != 1 @@ -334,10 +293,6 @@ def _submit_graph(self, count: int = 1) -> None: def _complete_graph(self) -> bool: self._check_graph_live() - if self._two_stream is not None: - result = self._two_stream.complete() - self._graph_pending = False - return result if not self._graph_pending: raise RuntimeError("Async FFN has no pending replay") try: @@ -355,10 +310,6 @@ def _close_graph(self) -> None: if self._graph_closed: return self._check_graph_live() - if self._two_stream is not None: - self._two_stream.close(receiver_exited=self._graph_receiver_exited) - self._graph_closed = True - return if self._graph_pending or not self._graph_receiver_exited: raise RuntimeError( "Async graph resources require receiver exit and device completion" @@ -412,7 +363,7 @@ def execute_connector_driven_step(self, shutdown_requested: bool = False) -> boo if self._graph is not None: self._graph_stream.wait_stream(torch.npu.current_stream(self.device)) self._first_step = False - if self._graph is None and self._two_stream is None: + if self._graph is None: # Eager and graph modes use the same layered executor. self._ffn_forward_connector_driven() self.session.after_step() @@ -443,10 +394,6 @@ def _ffn_forward_connector_driven( single: bool = False, ) -> torch.Tensor | None: assert self.model is not None - if self._two_stream is not None: - self._two_stream.submit(1 if single else len(self.layer_ids)) - self._two_stream.complete(wait=self._wait_graph_device if single else None) - return None if self._mega_executor is not None: for _ in self.layer_ids[:1] if single else self.layer_ids: rank_ffn_output = self._mega_executor() @@ -492,10 +439,7 @@ def shutdown(self) -> None: self.session.validate_shutdown() from vllm.v1.worker.gpu.shutdown import free_before_shutdown - if self._two_stream is not None: - self._two_stream.close(receiver_exited=self._graph_receiver_exited) - self._two_stream = None - elif self._graph is not None: + if self._graph is not None: self._close_graph() stop_afd_npu_profiler(self.prof) self.connector.close() diff --git a/afd_plugin/v1/worker/npu/async_ffn_two_stream.py b/afd_plugin/v1/worker/npu/async_ffn_two_stream.py deleted file mode 100644 index f7b1310fe..000000000 --- a/afd_plugin/v1/worker/npu/async_ffn_two_stream.py +++ /dev/null @@ -1,203 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project -"""Two-slot Async CAM receive/compute and FIFO-send execution.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, cast - -import torch - -from afd_plugin.connectors.npu.async_cam import AFDAsyncTransferState - -if TYPE_CHECKING: - from afd_plugin.connectors.metadata import AFDA2FTransferPayload - from afd_plugin.connectors.npu.async_cam import ( - AFDAsyncExtraInfo, - CAMAsyncAFDConnector, - ) - from afd_plugin.model_executor.npu.async_cam_layered import AsyncCAMLayeredExecutor - - -@dataclass -class _Slot: - ready: torch.npu.Event - done: torch.npu.Event - submitted: bool = False - payload: AFDA2FTransferPayload | None = None - output: torch.Tensor | None = None - tensors: list[torch.Tensor] = field(default_factory=list) - compute_graph: torch.npu.NPUGraph | None = None - send_graph: torch.npu.NPUGraph | None = None - pools: tuple | None = None - - def release_transaction(self) -> None: - self.payload = self.output = None - self.tensors.clear() - - -class AsyncFFNTwoStream: - def __init__( - self, - device: torch.device, - config: AFDAsyncExtraInfo, - connector: CAMAsyncAFDConnector, - executor: AsyncCAMLayeredExecutor, - ) -> None: - self.connector, self.executor = connector, executor - self.sequence = self.capture_count = self.replay_count = 0 - self.pending = self.closed = False - self.failure: Exception | None = None - self.compute = torch.npu.Stream(device=device) - self.send = torch.npu.Stream(device=device) - limits = torch.npu.get_stream_limit(self.compute) - if ( - config.ffn_compute_aic_num > limits["cube_core_num"] - or 2 * config.ffn_compute_aic_num > config.ffn_compute_aiv_num - or config.ffn_compute_aiv_num + config.ffn_combine_aiv_num - > limits["vector_core_num"] - ): - raise ValueError( - "FFN two-stream core limits exceed the available 1:2 budget" - ) - torch.npu.set_stream_limit( - self.compute, - cube_num=config.ffn_compute_aic_num, - vector_num=config.ffn_compute_aiv_num, - ) - torch.npu.set_stream_limit(self.send, vector_num=config.ffn_combine_aiv_num) - self.compute.wait_stream(torch.npu.current_stream(device)) - self.slots = [_Slot(torch.npu.Event(), torch.npu.Event()) for _ in range(2)] - - def _check_live(self) -> None: - if self.failure is not None: - raise RuntimeError( - "Async FFN two-stream failed; retain resources" - ) from self.failure - if self.closed: - raise RuntimeError("Async FFN two-stream is closed") - - def _compute(self, slot: _Slot) -> None: - slot.payload = self.connector.recv_attn_output() - state = cast(AFDAsyncTransferState, slot.payload.context.states) - slot.output = self.executor( - slot.payload.hidden_states, - state.dynamic_scales, - state.group_list, - state.token_nums_rankid_layeridx, - retained_tensors=slot.tensors, - ) - - def _send(self, slot: _Slot) -> None: - assert slot.payload is not None and slot.output is not None - self.connector.send_ffn_output(slot.output, slot.payload.context) - - def capture(self) -> None: - self._check_live() - if self.pending or self.capture_count: - raise RuntimeError("Async FFN two-stream is not ready for capture") - try: - for slot in self.slots: - slot.compute_graph, slot.send_graph = ( - torch.npu.NPUGraph(), - torch.npu.NPUGraph(), - ) - slot.pools = ( - torch.npu.graph_pool_handle(), - torch.npu.graph_pool_handle(), - ) - with torch.npu.graph( - slot.compute_graph, pool=slot.pools[0], stream=self.compute - ): - self._compute(slot) - with torch.npu.graph( - slot.send_graph, pool=slot.pools[1], stream=self.send - ): - self._send(slot) - self.capture_count += 2 - except Exception as exc: - self.failure = exc - raise - self.sequence = self.replay_count = 0 - - def submit(self, count: int) -> None: - self._check_live() - if self.pending or count < 1: - raise RuntimeError("Async FFN two-stream is not ready for submission") - self.pending = True # Even a failed call can leave queued native work. - try: - for _ in range(count): - slot = self.slots[self.sequence % 2] - captured = slot.compute_graph is not None - if slot.submitted and not captured: - # An idle DR has no startup deadline; retain until CS completes. - slot.done.synchronize() - slot.release_transaction() - with torch.npu.stream(self.compute): - if slot.submitted: - self.compute.wait_event(slot.done) - if captured: - assert slot.compute_graph is not None - slot.compute_graph.replay() - self.replay_count += 1 - else: - self._compute(slot) - slot.ready.record(self.compute) - with torch.npu.stream(self.send): - self.send.wait_event(slot.ready) - if captured: - assert slot.send_graph is not None - slot.send_graph.replay() - self.replay_count += 1 - else: - self._send(slot) - slot.done.record(self.send) - slot.submitted = True - self.sequence += 1 - except Exception as exc: - self.failure = exc - raise - - def complete( - self, *, wait: Callable[[torch.npu.Event], None] | None = None - ) -> bool: - self._check_live() - if not self.pending: - raise RuntimeError("Async FFN two-stream has no pending submission") - try: - if wait is None: - # Match the existing bounded-batch completion policy. - torch.npu.synchronize() - else: - # FIFO send completion also covers all preceding compute work. - wait(self.slots[(self.sequence - 1) % len(self.slots)].done) - except Exception as exc: - self.failure = exc - raise - self.pending = False - if not self.capture_count: - for slot in self.slots: - slot.release_transaction() - return True - - def close(self, *, receiver_exited: bool) -> None: - if self.closed: - return - self._check_live() - if self.pending or not receiver_exited: - raise RuntimeError( - "Two-stream resources require receiver exit and device completion" - ) - for slot in self.slots: - if slot.compute_graph is not None: - slot.compute_graph.reset() - if slot.send_graph is not None: - slot.send_graph.reset() - slot.release_transaction() - slot.compute_graph = slot.send_graph = None - slot.pools = None - self.slots.clear() - del self.compute, self.send, self.connector, self.executor - self.closed = True diff --git a/afd_plugin/v1/worker/npu/ffn_worker.py b/afd_plugin/v1/worker/npu/ffn_worker.py index 6bec2e38c..ae7b84dee 100644 --- a/afd_plugin/v1/worker/npu/ffn_worker.py +++ b/afd_plugin/v1/worker/npu/ffn_worker.py @@ -207,10 +207,7 @@ def _run_ffn_server_loop(self) -> None: # The runner consults the session once per step; a second # predicate here would add Store round trips to the Pull hot path. while self.model_runner.execute_connector_driven_step(event.is_set()): - if ( - self.model_runner.effective_mode == "eager" - and self.model_runner._two_stream is None - ): + if self.model_runner.effective_mode == "eager": torch.npu.synchronize() self.model_runner.drain() self.model_runner.session.finish_receiver() @@ -238,7 +235,6 @@ def _run_ffn_server_loop(self) -> None: def _requires_async_drain(self) -> bool: return isinstance(self.model_runner, AsyncNPUFFNModelRunner) and ( self.model_runner.effective_mode == "full" - or self.model_runner._two_stream is not None or self.model_runner.session.requires_drain ) diff --git a/csrc/npu/README.md b/csrc/npu/README.md index 5be6f9bfe..d3ce61c4e 100644 --- a/csrc/npu/README.md +++ b/csrc/npu/README.md @@ -49,8 +49,8 @@ rank to enable MegaMoE and its dedicated pull protocol. With the switch off, CAM retains its original push Dispatch/Combine and FFN execution path. Pull is not a standalone transport option: there is no `dispatch_mode` setting or separate pull DispatchRecv/CombineSend operator. MegaMoE requires -`dynamicQuant=1`, per-channel W4A8 or W8A8 weights, and -`ffn_two_stream=false`. Per-group weights are not supported by MegaMoE. +`dynamicQuant=1` and per-channel W4A8 or W8A8 weights. +Per-group weights are not supported by MegaMoE. The A3 `AfdMegaMoePullNative` kernel directly includes the installed CANN MC2 W4A8 `BlockMmad`. The build supports the CANN 9.1 MegaMoE header layout and diff --git a/docs/design/module/connector_contracts.md b/docs/design/module/connector_contracts.md index 407bf4673..8e20603c7 100644 --- a/docs/design/module/connector_contracts.md +++ b/docs/design/module/connector_contracts.md @@ -103,7 +103,7 @@ than an `AFDConfig` field. Unknown fields fail in the selected connector parser. | `P2pNcclAFDConnector` | None; the mapping must be empty. | | `DeepEPM2NAFDConnector` | `payload_format`, `expert_backend`, `capacity_tokens_per_rank`, `num_lanes`, `bucket_granularity`, `cpu_sync_min_tokens`, `num_sms`, `num_qps`, `expert_num_sms`, `cpu_timeout_s`, `gpu_timeout_s`, and a nested `network` mapping (`gin`, `hybrid`, `rdma_traffic_class`). All are optional; see the [DeepEP M2N guide](../../gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md#connector_extra_config) for defaults. | | `CAMP2pAFDConnector` | `core_num`, optional `attn_core_num` / `ffn_core_num`, `compute_gate_on_attention`, `quant_mode`, and optional positive `hccl_buffer_size`. Core counts must be positive; the current runtime rejects gate-on-Attention and any nonzero quantization mode. | -| `CAMAsyncAFDConnector` | `dynamicQuant`, `attn_ranks_per_dp`, optional positive `hccl_buffer_size`, and FFN two-stream options. Attention microbatching uses native DBO configuration. | +| `CAMAsyncAFDConnector` | `dynamicQuant`, `ffn_mega_moe`, `attn_ranks_per_dp`, and optional positive `hccl_buffer_size`. Attention microbatching uses native DBO configuration. | The common `afd_process_group_timeout_s` defaults to 120 seconds and controls AFD rendezvous. `hccl_buffer_size` sizes connector-owned HCCL domains in MB; diff --git a/docs/design/module/execution_platforms.md b/docs/design/module/execution_platforms.md index 00813fcac..86912bdec 100644 --- a/docs/design/module/execution_platforms.md +++ b/docs/design/module/execution_platforms.md @@ -396,7 +396,7 @@ Async CAM; standard DeepSeek-V3.2 was already rejected by model admission. Finegrained TP, explicit shared-expert DP configuration, and native MTP and draft-model paths other than DeepSeek-V4 DSpark are excluded. DSpark keeps Attention/native proposal eager and permits independent layered FFN FULL; -it does not support DBO or FFN two-stream. +it does not support DBO. FFN remains connector-driven. The row's upgrade evidence retains its original scope; MRV2 deployment settings are documented in the [CAM async guide](../../npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md#modelrunnerv2-deployment). diff --git a/docs/design/module/ffn_runtime.md b/docs/design/module/ffn_runtime.md index b12f8a5a2..593847b07 100644 --- a/docs/design/module/ffn_runtime.md +++ b/docs/design/module/ffn_runtime.md @@ -12,7 +12,6 @@ primary_code_paths: - "afd_plugin/v1/worker/ffn_worker.py" - "afd_plugin/v1/worker/npu/ffn_model_runner.py" - "afd_plugin/v1/worker/npu/async_ffn_model_runner.py" - - "afd_plugin/v1/worker/npu/async_ffn_two_stream.py" - "afd_plugin/v1/worker/npu/ffn_worker.py" related_code_paths: - "afd_plugin/connectors/npu/async_cam_runtime/async_cam_startup.py" @@ -34,7 +33,6 @@ validation_paths: - "tests/unit/v1/worker/test_ffn_routed_experts_model_runner.py" - "tests/unit/v1/worker/test_model_runner_v2.py" - "tests/unit/v1/worker/test_npu_runtime.py" - - "tests/unit/v1/worker/test_async_ffn_two_stream.py" - "tests/unit/connectors/npu/test_async_cam_startup_wiring.py" - "tests/unit/connectors/npu/test_async_cam_startup.py" - "tests/unit/connectors/npu/test_async_cam_pull.py" @@ -181,18 +179,6 @@ FFN. Async FFN FULL accepts eager Attention DBO and continues to receive work items independently of Attention's threads. These limits describe the pair even though only Attention inherits a native V2 runner. -Async CAM also has an experimental, default-off `ffn_two_stream` mode for -layered W4A8/W8A8 eager/FULL. `AsyncFFNTwoStream`, owned by the Async runner, retains -two slots, compute/send streams and ready/done events. FULL owns four private -graph pools and four graphs; eager retires each slot only after its send is -observed complete. Both modes retain the bounded batch fence. Enabled eager -and FULL agree on executor, layer, capacity and core signatures before HCCL -creation and reject Attention DBO/SP combinations. Shutdown requires receiver -exit and device completion before release. Partial native failures retain -resources and remain visible to repeated health checks. Host tests do not -qualify native window reuse, graph capture or stream overlap; see the -[Async runtime guide](../../npu/ASYNC_FFN_RUNTIME.md#experimental-ffn-two-stream-execution). - ## Daemon step selection The worker selects one of two FFN step paths from the optional diff --git a/docs/design/module/model_integration.md b/docs/design/module/model_integration.md index f343b9c51..7c9128b74 100644 --- a/docs/design/module/model_integration.md +++ b/docs/design/module/model_integration.md @@ -174,7 +174,7 @@ describes A/F serving. In PD deployments, Prefill uses the upstream complete draft model and KV connector. Attention and native draft proposal execution stay eager. F may use the existing FULL transaction graph with one homogeneous W4A8 or W8A8 table covering target and draft; selection remains on device. DSpark excludes -DBO and FFN two-stream. +DBO. Shared-expert layout is model-specific. DeepSeek-V4 uses `AFDDeepseekV4SharedExperts` to preserve the native policy: diff --git a/docs/npu/ASYNC_FFN_RUNTIME.md b/docs/npu/ASYNC_FFN_RUNTIME.md index f4e66e43f..2167b5cae 100644 --- a/docs/npu/ASYNC_FFN_RUNTIME.md +++ b/docs/npu/ASYNC_FFN_RUNTIME.md @@ -19,7 +19,6 @@ F TP=1 and Attention-side gating. Both roles select Attention may enable native DBO under the existing two-microbatch constraints. Enable native DBO only on Attention; FFN consumes CAM work items without joining Attention's DBO thread schedule. -The experimental two-stream option below uses the same Attention DBO contract. The [0.32 validation record](https://github.com/vllm-project/afd-plugin/pull/507) identifies the tested W4A8 eager/FULL fixture and its limits, including graceful-exit failures. @@ -33,7 +32,7 @@ DeepSeek-V4 DSpark uses this same F graph for target and draft routed experts. The combined layer table and startup agreement cover target0–42/draft43–45; all layers must share the same W4A8 or W8A8 layout. Native P loads the complete draft model. Target/draft Attention and the native speculative configuration -remain eager. DSpark excludes DBO and FFN two-stream; see the +remain eager. DSpark excludes DBO; see the [DSpark recipe](../../recipe/npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md). W8A8 reuses the loaded INT8 NZ weights, FP32 W13 scales and BF16 W2 scales. @@ -87,62 +86,6 @@ host metadata read or cancellation protocol. Changing weights, layout, capacity or topology requires fresh workers. There is no graph cache, online weight replacement or partial-rank restart. -## Experimental FFN two-stream execution - -Set these fields in the **FFN** `afd.connector_extra_config` to opt in: - -```json -{ - "ffn_two_stream": true, - "ffn_compute_aic_num": 12, - "ffn_compute_aiv_num": 24, - "ffn_combine_aiv_num": 24 -} -``` - -`ffn_two_stream` defaults to `false`. Both eager and FULL use exactly two -streams and two slots: receive plus the complete layered executor on the -compute stream, and FIFO combine-send on the send stream. Each send waits for -its matching compute-ready event; slot reuse waits for its previous send-done -event. The original batch metadata and CS return tensor remain owned by the -slot. The CS return tensor is an allocation anchor, not a completion flag. - -Core counts must be positive integers. Compute Cube must fit the available -AIC budget, compute Vector must be at least twice compute Cube, and the sum -of compute/send Vector limits must fit available AIV. The send stream sets -only its Vector limit. Invalid budgets, inconsistent FFN layouts and unsupported -Attention capabilities fail through pre-HCCL Store negotiation. Configuration -hashing includes connector extras. - -This option requires the layered MRV2 prerequisites above. Keep PP/PCP/DCP -at 1, SP and FlashComm1 off on both roles, and speculative decoding off. - -Attention DBO, when enabled, uses exactly two native ubatches and the existing -receive hook: the preceding raw receive is queued before the next ubatch sends -into CAM's single physical window. FFN slots remain independent work-item -buffers; they do not correspond to Attention ubatch IDs. Enable DBO only on -Attention, which remains eager, including when FFN uses FULL graph. -Streams are limited before warmup; FULL preserves -the existing matched warmup receive count. Enablement adds no startup receives. - -Eager retains each payload, intermediate and output until send completion is -observed before reallocating that slot. FULL captures **four graphs**, one -compute and one send graph per slot, each with a private pool and retained -mutable addresses. Events stay outside graph capture. Both modes preserve -the existing bounded batch size of `len(layer_ids)` and one device fence at -batch completion. Four graph objects do not mean four business transactions: -FULL submits two graph replays per transaction. - -Two-stream eager uses the same conservative shutdown rules as FULL: join the -receiver before releasing graph/tensor/model/communicator resources. Failed -capture, submission or device completion is terminal and retains resources. - -Host protocol and lifecycle tests cover this implementation. **NPU validation -is pending**: stream limits reaching custom tiling, native same-group DR/CS -window safety, split-graph capture and actual overlap/performance have not been -qualified. Keep this experimental option off in validated deployments until -the target stack passes those checks. - ## Resource lifetime With default Push transport, a host event stops further batch submissions. Graph reset and model/communicator @@ -168,7 +111,7 @@ until every Attention source is quiet, drains active work, then publishes stoppe only then may Attention and FFN release their communication resources. Failed handshakes retain those resources. Standalone `close()` also validates shutdown status before release: this is an additional defensive shutdown check, not a -relocated hot-path fence. Pull cannot be combined with `ffn_two_stream`. +relocated hot-path fence. ## Computation semantics @@ -203,9 +146,8 @@ coverage does not qualify loading real per-group checkpoints. - Host contracts: `tests/unit/connectors/npu/test_async_cam_pull.py`, `tests/unit/connectors/npu/test_async_cam_startup.py`, - `tests/unit/v1/worker/test_async_ffn_graph.py`, and - `tests/unit/v1/worker/test_async_ffn_runtime.py`. CPU-safe two-stream coverage - is in `tests/unit/v1/worker/test_async_ffn_two_stream.py` and + `tests/unit/v1/worker/test_async_ffn_graph.py`, + `tests/unit/v1/worker/test_async_ffn_runtime.py`, and `tests/unit/connectors/npu/test_async_cam_startup_wiring.py`; fakes verify host ordering and ownership, not device execution. - Finite native comparison: `tests/npu/async_cam_runtime.py`. Run eager with @@ -235,11 +177,7 @@ python -m tests.e2e.models.deepseek_v4_flash.ffn_graph \ The fixture saves inputs, generated tokens, actual runner identities and per-rank execution evidence. FULL must capture once per F, replay during requests and -match eager tokens. These repository fixtures exercise the single-stream path. -Two-stream native and real-model qualification uses independent drivers outside -`tests/e2e` and `tests/npu`, setting the FFN connector options above and recording -four captures per F in FULL mode. Preserve driver sources, launch configuration, -source hashes, inputs and raw outputs with the evidence. +match eager tokens. Batch-end metadata samples do not prove per-layer coverage. Inference results and cleanup are recorded separately; remaining owned processes or failed requests block acceptance. Validation evidence must identify the diff --git a/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md b/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md index dd75f8b05..b2016d28d 100644 --- a/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md +++ b/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md @@ -195,16 +195,6 @@ including `async_dp`; enable DBO through the Attention CLI flags below. | `dynamicQuant` | `int` | `0` | Set to `1`: FFN receives dynamically quantized routed activations and scale tensors. The default `0` is not supported by the layered FFN path. | | `attn_ranks_per_dp` | `int` | `1` | Positive Attention TP rank count per DP replica. It is the CAM Attention grouping width and is independent of the FFN process's local TP size. | | `hccl_buffer_size` | `int` | unset | Positive CAM HCCL buffer size in MB. The override applies only to the `afd_async_cam` communication domain. When unset, HCCL uses `HCCL_BUFFSIZE`, then its built-in default. Configure the same value on every Attention and FFN rank in the domain. | -| `ffn_two_stream` | `bool` | `false` | Experimental FFN-only receive/compute and send streams with two slots. Requires an eligible layered MRV2 executor; eager and FULL are supported in code, with NPU qualification pending. | -| `ffn_compute_aic_num` | `int` | `12` | Positive compute Cube limit, bounded by available AIC. | -| `ffn_compute_aiv_num` | `int` | `24` | Positive compute Vector limit, at least twice the compute Cube count. | -| `ffn_combine_aiv_num` | `int` | `24` | Positive send Vector limit; compute plus send Vector must fit available AIV. | - -For `ffn_two_stream=true`, disable SP and FlashComm1 on both roles. Attention -may use native DBO with two ubatches under the constraints below; it remains -eager, and FFN must keep DBO disabled even when using FULL graph. -See [two-stream execution](ASYNC_FFN_RUNTIME.md#experimental-ffn-two-stream-execution) -for graph ownership, shutdown and the remaining hardware qualification gates. For a DP+SP deployment such as DP3TP2 Attention + DP2TP1/EP2 FFN, set `VLLM_ASCEND_ENABLE_FLASHCOMM1=1` only on Attention and explicitly set it to diff --git a/recipe/npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md b/recipe/npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md index f295f6452..48d51b85a 100644 --- a/recipe/npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md +++ b/recipe/npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md @@ -80,7 +80,7 @@ F captures one DR → layered FFN → CS transaction and repeatedly replays it f both target and draft layers. The bounded replay group is independent of request and target/draft phase boundaries. A and the native draft configuration retain eager execution. FULL requires matching W4A8 or W8A8 layouts on all layers; unsupported -graph modes fail startup. DSpark does not support DBO or FFN two-stream. +graph modes fail startup. DSpark does not support DBO. Layered W4A8 forwards the model's `swiglu_limit` for both target and draft experts in eager and FULL. A positive limit clamps gate above and up on both diff --git a/tests/unit/compat/npu/test_feature_validation.py b/tests/unit/compat/npu/test_feature_validation.py index 99ca5f04b..93ed73baf 100644 --- a/tests/unit/compat/npu/test_feature_validation.py +++ b/tests/unit/compat/npu/test_feature_validation.py @@ -69,15 +69,6 @@ def test_dspark_ffn_graph_requires_full_mode(mode): fail_if_unsupported_npu_afd_features(config) -@pytest.mark.parametrize("eager", [True, False]) -def test_dspark_rejects_ffn_two_stream(eager): - config = npu_dspark_config(role="ffn") - config.model_config.enforce_eager = eager - config.additional_config["afd"]["connector_extra_config"]["ffn_two_stream"] = True - with pytest.raises(RuntimeError, match="DSpark.*two-stream"): - fail_if_unsupported_npu_afd_features(config) - - @pytest.mark.parametrize( "anchor,num_requests,capacity,allowed", [ diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 263e3bca7..0a303745c 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -27,17 +27,17 @@ def test_parse_empty_additional_config_is_inactive(): def test_connector_extra_fields_participate_in_the_graph_hash(): - extra = {"ffn_two_stream": True, "ffn_compute_aic_num": 12} + extra = {"ffn_mega_moe": True, "dynamicQuant": 1} raw = { "connector": "CAMAsyncAFDConnector", "role": "ffn", "connector_extra_config": extra, } first = afd_config_from_mapping(raw) - extra["ffn_compute_aic_num"] = 10 + extra["ffn_mega_moe"] = False second = afd_config_from_mapping(raw) assert first.compute_hash() != second.compute_hash() - raw["connector_extra_config"] = {"ffn_compute_aic_num": 10, "ffn_two_stream": True} + raw["connector_extra_config"] = {"dynamicQuant": 1, "ffn_mega_moe": False} assert second.compute_hash() == afd_config_from_mapping(raw).compute_hash() diff --git a/tests/unit/connectors/npu/test_async_cam_startup.py b/tests/unit/connectors/npu/test_async_cam_startup.py index c530d33bb..94c7d9fc8 100644 --- a/tests/unit/connectors/npu/test_async_cam_startup.py +++ b/tests/unit/connectors/npu/test_async_cam_startup.py @@ -163,38 +163,6 @@ def test_executor_error_reaches_all_attention_and_ffn_ranks(mode): ), errors -@pytest.mark.parametrize("mode", ["eager", "full"]) -def test_two_stream_signature_and_failure_are_checked_in_both_modes(mode): - plan = full_plan(requested=mode, stream_signature="2/12C/24V/24V") - assert resolve_ffn_mode([plan, plan]) == mode - with pytest.raises(ValueError, match="two-stream execution signatures"): - resolve_ffn_mode([plan, replace(plan, stream_signature="")]) - with pytest.raises(ValueError, match="core budget"): - resolve_ffn_mode([plan, replace(plan, stream_error="core budget")]) - with pytest.raises(ValueError, match="signatures differ"): - resolve_ffn_mode([plan, replace(plan, signature="other-capacity")]) - - -@pytest.mark.parametrize("peer_failure", [False, True]) -def test_two_stream_waits_for_attention_capability_check(peer_failure): - owners = startup_world(1, 1) - - def validate(): - if peer_failure: - raise ValueError("DBO is not qualified") - - owners[0].validate_attention_streams = validate - plan = full_plan(requested="eager", stream_signature="2/12C/24V/24V") - errors = negotiate(owners, [None, plan], ["wire"] * 2) - if peer_failure: - assert all( - isinstance(error, (ValueError, RuntimeError)) and "DBO" in str(error) - for error in errors - ), errors - else: - assert errors == [None, None] - - def test_whole_world_agrees_before_service_readiness(): owners = startup_world() assert ( @@ -208,13 +176,12 @@ def test_whole_world_agrees_before_service_readiness(): owner.wait_service_ready() -@pytest.mark.parametrize("two_stream", [False, True]) @pytest.mark.parametrize("mode", ["eager", "full"]) -def test_transport_check_preserves_single_stream_eager_scope(two_stream, mode): +def test_transport_check_preserves_eager_scope(mode): owners = startup_world(1, 1) - plan = full_plan(requested=mode, stream_signature="two" if two_stream else "") + plan = full_plan(requested=mode) errors = negotiate(owners, [None, plan], ["a", "b"]) - if two_stream or mode == "full": + if mode == "full": assert all( isinstance(error, (ValueError, RuntimeError)) and "transport signatures" in str(error) diff --git a/tests/unit/connectors/npu/test_async_cam_startup_wiring.py b/tests/unit/connectors/npu/test_async_cam_startup_wiring.py index ca2e76fb6..a7a7fc60c 100644 --- a/tests/unit/connectors/npu/test_async_cam_startup_wiring.py +++ b/tests/unit/connectors/npu/test_async_cam_startup_wiring.py @@ -136,14 +136,14 @@ def executor(layers): connector = SimpleNamespace( is_initialized=False, dynamic_quant=1, - extra_info=SimpleNamespace(ffn_two_stream=False, ffn_mega_moe=False), + extra_info=SimpleNamespace(ffn_mega_moe=False), graph_startup=AsyncCAMGraphStartup(1, 1, 2), ) connector.init_afd_connector = lambda: resolve_ffn_mode( [connector.graph_startup.plan] ) runner.connector = connector - runner._failure = runner._layered_executor = runner._two_stream = None + runner._failure = runner._layered_executor = None runner.requested_mode = mode monkeypatch.setattr( runtime, "get_ascend_device_type", lambda: runtime.AscendDeviceType.A3 @@ -187,7 +187,7 @@ def test_mega_executor_failure_is_published_during_startup(modules, monkeypatch, runner.requested_mode = mode connector = SimpleNamespace( is_initialized=False, - extra_info=SimpleNamespace(ffn_two_stream=False, ffn_mega_moe=True), + extra_info=SimpleNamespace(ffn_mega_moe=True), graph_startup=AsyncCAMGraphStartup(1, 1, 2), ) connector.init_afd_connector = lambda: resolve_ffn_mode( @@ -209,91 +209,19 @@ def fail_executor(*_args): assert connector.graph_startup.plan.executor_error == "unsupported Mega weights" -@pytest.mark.parametrize("value", [True, 0, -1, 1.5]) -def test_two_stream_rejects_invalid_core_settings(modules, value): - _, connector, _ = modules - with pytest.raises((TypeError, ValueError), match="ffn_compute_aic_num"): - connector.AFDAsyncExtraInfo.from_mapping({"ffn_compute_aic_num": value}) - - -def test_two_stream_config_round_trip_and_default_off(modules): - _, connector, _ = modules - default = connector.AFDAsyncExtraInfo.from_mapping({}) - assert not default.ffn_two_stream - assert "ffn_two_stream" not in default.to_mapping() - enabled = connector.AFDAsyncExtraInfo.from_mapping({"ffn_two_stream": True}) - assert enabled.ffn_compute_aic_num == 12 - assert enabled.ffn_compute_aiv_num == enabled.ffn_combine_aiv_num == 24 - assert connector.AFDAsyncExtraInfo.from_mapping(enabled.to_mapping()) == enabled - disabled = connector.AFDAsyncExtraInfo.from_mapping({"ffn_compute_aic_num": 8}) - assert connector.AFDAsyncExtraInfo.from_mapping(disabled.to_mapping()) == disabled - - -@pytest.mark.parametrize("feature", ["use_sequence_parallel_moe"]) -def test_two_stream_attention_preflight_rejects_unqualified_modes( - modules, monkeypatch, feature -): - _, module, _ = modules - connector = module.CAMAsyncAFDConnector.__new__(module.CAMAsyncAFDConnector) - connector.vllm_config = npu_vllm_config( - enable_dbo=feature == "enable_dbo", - num_ubatches=2, - use_sequence_parallel_moe=feature == "use_sequence_parallel_moe", - ) - monkeypatch.setenv( - "VLLM_ASCEND_ENABLE_FLASHCOMM1", "1" if feature == "flashcomm1" else "0" - ) - if feature in (None, "enable_dbo"): - connector._validate_two_stream_attention() - else: - with pytest.raises(ValueError, match="two-stream"): - connector._validate_two_stream_attention() - - -@pytest.mark.parametrize("role", ["attention", "ffn"]) @pytest.mark.parametrize( - "value,enabled", + "field", [ - (None, False), - ("0", False), - ("1", True), - (" TRUE ", True), - ("yes", False), + "ffn_two_stream", + "ffn_compute_aic_num", + "ffn_compute_aiv_num", + "ffn_combine_aiv_num", ], ) -def test_two_stream_flashcomm1_matches_ascend_boolean_parsing( - modules, monkeypatch, role, value, enabled -): - _, module, _ = modules - if value is None: - monkeypatch.delenv("VLLM_ASCEND_ENABLE_FLASHCOMM1", raising=False) - else: - monkeypatch.setenv("VLLM_ASCEND_ENABLE_FLASHCOMM1", value) - config = npu_vllm_config( - role=role, - connector="CAMAsyncAFDConnector", - async_dp=True, - compute_gate_on_attention=True, - use_mla=True, - ) - if role == "ffn": - - def validate(): - _fail_if_unsupported_npu_afd_async_features( - config, - parse_afd_config(config), - module.AFDAsyncExtraInfo(ffn_two_stream=True, dynamic_quant=1), - ) - - else: - connector = module.CAMAsyncAFDConnector.__new__(module.CAMAsyncAFDConnector) - connector.vllm_config = config - validate = connector._validate_two_stream_attention - if enabled: - with pytest.raises((ValueError, RuntimeError), match="two-stream"): - validate() - else: - validate() +def test_removed_two_stream_options_are_rejected(modules, field): + _, connector, _ = modules + with pytest.raises(ValueError, match="unknown AFD async connector_extra_config"): + connector.AFDAsyncExtraInfo.from_mapping({field: True}) @pytest.mark.parametrize( @@ -453,9 +381,8 @@ def test_ffn_step_lets_session_decide_service_after_shutdown(modules, session_se @pytest.mark.parametrize("capture_fails", [False, True]) -@pytest.mark.parametrize("two_stream", [False, True]) def test_ffn_warmup_capture_order_and_failure_propagation( - modules, monkeypatch, capture_fails, two_stream + modules, monkeypatch, capture_fails ): # long-test: warmup/capture/readiness are one ordered startup protocol _, connector_module, runtime = modules calls: list[str | tuple[str, str | int]] = [] @@ -467,8 +394,6 @@ def initialize_connector(): calls.append("connector") assert startup.plan.signature == "weights" assert startup.plan.layer_ids == (2, 5) - assert not startup.plan.stream_error - assert bool(startup.plan.stream_signature) == two_stream startup.mode = "full" def capture(): @@ -489,7 +414,7 @@ def capture(): connector=SimpleNamespace( is_initialized=False, graph_startup=startup, - extra_info=connector_module.AFDAsyncExtraInfo(ffn_two_stream=two_stream), + extra_info=connector_module.AFDAsyncExtraInfo(), init_afd_connector=initialize_connector, attn_size=2, tp_size=1, @@ -510,7 +435,7 @@ def capture(): *(["transaction"] * 4), "device-done", "warmup", - *([] if two_stream else ["graph-resources"]), + "graph-resources", "capture", ("failure", "capture failed") if capture_fails else "captured", ] @@ -653,17 +578,13 @@ def serve(): @pytest.mark.parametrize("state", ["live", "failed", "drained"]) -@pytest.mark.parametrize("two_stream", [False, True]) @pytest.mark.parametrize("requires_drain", [False, True]) -def test_worker_retains_resources_until_clean_exit( - ffn_owners, state, two_stream, requires_drain -): +def test_worker_retains_resources_until_clean_exit(ffn_owners, state, requires_drain): _, runtime, module = ffn_owners calls = [] timeouts = [] runner = runtime.AsyncNPUFFNModelRunner.__new__(runtime.AsyncNPUFFNModelRunner) - runner.effective_mode = "eager" if two_stream else "full" - runner._two_stream = object() if two_stream else None + runner.effective_mode = "full" runner.session = SimpleNamespace(requires_drain=requires_drain) runner.connector = SimpleNamespace(close=lambda: calls.append("connector-close")) runner.shutdown = lambda: calls.append("release") @@ -925,7 +846,7 @@ def test_runner_drain_returns_completed_response_or_session_error( transactions=1, drain_error=RuntimeError("source did not drain") if fails else None, ) - runner._graph = runner._two_stream = None + runner._graph = None runner.layer_ids = (2, 5) responses = [] runner.connector.send_ffn_output = lambda output, context: responses.append(output) diff --git a/tests/unit/connectors/test_async_cam_connector.py b/tests/unit/connectors/test_async_cam_connector.py index f92dd85b1..1f67c07e2 100644 --- a/tests/unit/connectors/test_async_cam_connector.py +++ b/tests/unit/connectors/test_async_cam_connector.py @@ -178,9 +178,7 @@ def test_async_extra_info_rejects_unknown_and_invalid_fields(): def test_mega_config_round_trips_protocol_and_normalized_quantization(): - info = AFDAsyncExtraInfo.from_mapping( - {"ffn_mega_moe": "true", "dynamicQuant": "1", "ffn_two_stream": "false"} - ) + info = AFDAsyncExtraInfo.from_mapping({"ffn_mega_moe": "true", "dynamicQuant": "1"}) assert info.ffn_mega_moe and info.dynamic_quant == 1 assert AFDAsyncExtraInfo.from_mapping(info.to_mapping()) == info @@ -204,16 +202,9 @@ def test_mega_warmup_launches_once_per_source(monkeypatch, capacity): assert connector.graph_startup_chunks() == 1 -@pytest.mark.parametrize( - "raw", - [ - {"ffn_mega_moe": True}, - {"ffn_mega_moe": True, "dynamicQuant": 1, "ffn_two_stream": True}, - ], -) -def test_mega_requires_quantized_single_stream(raw): +def test_mega_requires_dynamic_quantization(): with pytest.raises(ValueError, match="ffn_mega_moe"): - AFDAsyncExtraInfo.from_mapping(raw) + AFDAsyncExtraInfo.from_mapping({"ffn_mega_moe": True}) @pytest.mark.parametrize( diff --git a/tests/unit/fakes/async_ffn.py b/tests/unit/fakes/async_ffn.py index 912e169ba..a11af183f 100644 --- a/tests/unit/fakes/async_ffn.py +++ b/tests/unit/fakes/async_ffn.py @@ -52,7 +52,6 @@ def stub(name, **attributes): group = load_host_module(monkeypatch, "afd_plugin.distributed.afd_process_group") stub("afd_plugin.distributed", init_afd_process_group=group.init_afd_process_group) connector = load_host_module(monkeypatch, "afd_plugin.connectors.npu.async_cam") - load_host_module(monkeypatch, "afd_plugin.v1.worker.npu.async_ffn_two_stream") runtime = load_host_module( monkeypatch, "afd_plugin.v1.worker.npu.async_ffn_model_runner" ) diff --git a/tests/unit/model_executor/test_async_cam_layered.py b/tests/unit/model_executor/test_async_cam_layered.py index 7ea83f76b..6b6e90b1a 100644 --- a/tests/unit/model_executor/test_async_cam_layered.py +++ b/tests/unit/model_executor/test_async_cam_layered.py @@ -315,7 +315,6 @@ def compute(*args): runner = SimpleNamespace( model=object(), connector=connector, - _two_stream=None, _mega_executor=None, _layered_executor=compute, layer_ids=(2, 5), diff --git a/tests/unit/v1/worker/test_async_ffn_graph.py b/tests/unit/v1/worker/test_async_ffn_graph.py index 578edb996..d4ee66169 100644 --- a/tests/unit/v1/worker/test_async_ffn_graph.py +++ b/tests/unit/v1/worker/test_async_ffn_graph.py @@ -84,7 +84,6 @@ def send(output, context): session.model = torch.nn.Linear(1, 1) session._layered_executor = execute session._mega_executor = None - session._two_stream = None session.connector = connector session.session = connector.ffn_session() session._graph = None diff --git a/tests/unit/v1/worker/test_async_ffn_two_stream.py b/tests/unit/v1/worker/test_async_ffn_two_stream.py deleted file mode 100644 index a715afb44..000000000 --- a/tests/unit/v1/worker/test_async_ffn_two_stream.py +++ /dev/null @@ -1,234 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project -"""CPU stream protocol checks; device qualification is recorded separately.""" - -import sys -import weakref -from types import SimpleNamespace -from unittest.mock import MagicMock, call - -import pytest - -from afd_plugin.connectors.npu.async_cam_runtime.async_cam_startup import ( - AsyncCAMGraphStartup, -) -from tests.unit.fakes.async_ffn import load_async_modules - - -@pytest.fixture -def pipeline(monkeypatch): - _, connector, runtime = load_async_modules(monkeypatch) - module = sys.modules["afd_plugin.v1.worker.npu.async_ffn_two_stream"] - npu, history, produced = module.torch.npu, MagicMock(), [] - for name in ("compute_stream", "send_stream", "ready0", "done0", "ready1", "done1"): - history.attach_mock(MagicMock(), name) - npu.NPUGraph.side_effect = lambda: MagicMock() - npu.graph_pool_handle.side_effect = object - npu.Stream.side_effect = [history.compute_stream, history.send_stream] - npu.Event.side_effect = [ - history.ready0, - history.done0, - history.ready1, - history.done1, - ] - - class Tensor: - pass - - def receive(): - hidden = Tensor() - produced.append(weakref.ref(hidden)) - state = SimpleNamespace( - dynamic_scales=1, group_list=2, token_nums_rankid_layeridx=hidden - ) - return SimpleNamespace( - hidden_states=hidden, context=SimpleNamespace(states=state) - ) - - def send(output, context): - assert output is context.states.token_nums_rankid_layeridx - - history.receive.side_effect = receive - history.compute.side_effect = lambda hidden, *a, **kw: hidden - history.send.side_effect = send - config = connector.AFDAsyncExtraInfo(ffn_two_stream=True) - owner = module.AsyncFFNTwoStream( - "npu:0", - config, - SimpleNamespace(recv_attn_output=history.receive, send_ffn_output=history.send), - history.compute, - ) - history.reset_mock() - return SimpleNamespace( - owner=owner, npu=npu, history=history, produced=produced, runtime=runtime - ) - - -@pytest.fixture -def warmup_runner(pipeline): - p = pipeline - runtime = p.runtime - runner = runtime.AsyncNPUFFNModelRunner.__new__(runtime.AsyncNPUFFNModelRunner) - runner.model = object() - runner._two_stream = p.owner - startup = AsyncCAMGraphStartup(1, 1, 2) - store = MagicMock() - startup._store = store - runner.connector = SimpleNamespace(graph_startup=startup) - p.history.done0.query.return_value = True - runner._ffn_forward_connector_driven(single=True) - p.history.reset_mock() - p.produced.clear() - return runner, store - - -@pytest.mark.parametrize("outcome", ["ready", "peer_failure", "timeout"]) -def test_warmup_observes_completion_or_failure_without_blocking_fence( - pipeline, warmup_runner, monkeypatch, outcome -): - p = pipeline - runner, store = warmup_runner - p.history.done1.query.return_value = outcome == "ready" - store.check.return_value = outcome == "peer_failure" - store.get.return_value = b"Attention rank failed" - clock = iter([0, p.runtime.GRAPH_STARTUP_TIMEOUT_SECONDS]) - monkeypatch.setattr(p.runtime.time, "monotonic", lambda: next(clock)) - p.npu.synchronize.side_effect = AssertionError("warmup must poll completion") - if outcome == "ready": - runner._ffn_forward_connector_driven(single=True) - p.owner.close(receiver_exited=True) - p.history.reset_mock() - assert all(ref() is None for ref in p.produced) - else: - error, message = ( - (RuntimeError, "Attention rank failed") - if outcome == "peer_failure" - else (TimeoutError, "warmup timed out; retain resources") - ) - with pytest.raises(error, match=message): - runner._ffn_forward_connector_driven(single=True) - with pytest.raises(RuntimeError, match="failed; retain resources"): - p.owner.close(receiver_exited=True) - p.history.reset_mock() - assert all(ref() is not None for ref in p.produced) - - -def test_eager_orders_events_and_retires_before_allocating_third_item(pipeline): - p = pipeline - p.owner.submit(3) - expected = [] - for index, slot in enumerate((0, 1, 0)): - if index == 2: - expected += ["done0.synchronize", "compute_stream.wait_event"] - expected += [ - "receive", - "compute", - f"ready{slot}.record", - "send_stream.wait_event", - "send", - f"done{slot}.record", - ] - assert [c[0] for c in p.history.mock_calls] == expected - p.history.send_stream.wait_event.assert_has_calls( - [call(p.history.ready0), call(p.history.ready1), call(p.history.ready0)] - ) - p.history.compute_stream.wait_event.assert_called_once_with(p.history.done0) - p.history.ready0.record.assert_has_calls([call(p.owner.compute)] * 2) - p.history.done0.record.assert_has_calls([call(p.owner.send)] * 2) - p.history.reset_mock() # Drop mock call arguments before checking tensor lifetime. - assert [ref() is not None for ref in p.produced] == [False, True, True] - p.owner.complete() - assert all(ref() is None for ref in p.produced) - - -def test_full_keeps_slot_addresses_and_reuses_with_device_edges(pipeline): - p = pipeline - p.owner.capture() - graphs = [ - graph - for slot in p.owner.slots - for graph in (slot.compute_graph, slot.send_graph) - ] - assert len({id(graph) for graph in graphs}) == 4 - assert len({pool for slot in p.owner.slots for pool in slot.pools}) == 4 - assert len(p.produced) == 2 and p.produced[0]() is not p.produced[1]() - p.history.reset_mock() # Call recording must not keep captured tensors alive. - p.owner.submit(5) - p.history.done0.synchronize.assert_not_called() - p.history.compute_stream.wait_event.assert_any_call(p.history.done0) - assert len(p.produced) == 2 - p.owner.complete() - assert all(ref() is not None for ref in p.produced) - with pytest.raises(RuntimeError, match="receiver exit"): - p.owner.close(receiver_exited=False) - p.owner.close(receiver_exited=True) - p.history.reset_mock() - assert all(ref() is None for ref in p.produced) - assert all(graph.reset.call_count == 1 for graph in graphs) - - -@pytest.mark.parametrize( - "failure", ["capture", "receive", "compute", "send", "retire", "replay", "fence"] -) -def test_partial_failure_retains_resources_and_disallows_retry(pipeline, failure): - p = pipeline - if failure == "replay": - p.owner.capture() - failing_call = { - "capture": p.history.send, - "receive": p.history.receive, - "compute": p.history.compute, - "send": p.history.send, - "retire": p.history.done0.synchronize, - "replay": p.owner.slots[0].compute_graph.replay - if failure == "replay" - else None, - "fence": p.npu.synchronize, - }[failure] - failing_call.side_effect = RuntimeError("native failure") - with pytest.raises(RuntimeError, match="native failure"): - if failure == "capture": - p.owner.capture() - else: - p.owner.submit(3 if failure == "retire" else 1) - p.owner.complete() - for operation in ( - p.owner.capture, - lambda: p.owner.submit(1), - lambda: p.owner.close(receiver_exited=True), - ): - with pytest.raises(RuntimeError, match="failed"): - operation() - assert all( - slot.compute_graph is None or not slot.compute_graph.reset.called - for slot in p.owner.slots - ) - p.history.reset_mock() - assert all(ref() is not None for ref in p.produced) - - -@pytest.mark.parametrize("budgets", [(25, 24, 24), (12, 23, 24), (12, 24, 25)]) -def test_invalid_budget_rejected_before_communication(monkeypatch, budgets): - _, connector, runtime = load_async_modules(monkeypatch) - config = connector.AFDAsyncExtraInfo( - ffn_two_stream=True, - ffn_compute_aic_num=budgets[0], - ffn_compute_aiv_num=budgets[1], - ffn_combine_aiv_num=budgets[2], - ) - communication = MagicMock() - with pytest.raises(ValueError, match="core limits"): - runtime.AsyncFFNTwoStream("npu:0", config, communication, MagicMock()) - assert not communication.mock_calls - - -def test_pending_work_cannot_be_resubmitted_or_closed(pipeline): - owner = pipeline.owner - owner.submit(1) - with pytest.raises(RuntimeError, match="not ready"): - owner.submit(1) - with pytest.raises(RuntimeError, match="device completion"): - owner.close(receiver_exited=True) - owner.complete() - owner.close(receiver_exited=True) - owner.close(receiver_exited=True) diff --git a/tests/unit/v1/worker/test_npu_runtime.py b/tests/unit/v1/worker/test_npu_runtime.py index b62ddfd43..e4f98d5fb 100644 --- a/tests/unit/v1/worker/test_npu_runtime.py +++ b/tests/unit/v1/worker/test_npu_runtime.py @@ -841,7 +841,6 @@ def execute_connector_driven_step(shutdown_requested): worker.model_runner.connector = SimpleNamespace(control_plane=None) worker.model_runner.session = DefaultFFNSession() worker.model_runner.effective_mode = "eager" - worker.model_runner._two_stream = None worker.model_runner.execute_connector_driven_step = execute_connector_driven_step monkeypatch.setattr( From 6fa7148283aecd693f9ebdb8fa1ba80d45150b65 Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Fri, 6 Nov 2026 05:57:56 +0200 Subject: [PATCH 08/42] refactor qwen3_moe Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- afd_plugin/ffn_faas/model_utils.py | 30 +++++++ afd_plugin/ffn_faas/platforms/gpu/backend.py | 2 +- .../platforms/gpu/qwen3_routed_model.py | 86 ++++++------------- afd_plugin/ffn_faas/platforms/npu/model.py | 8 +- afd_plugin/model_executor/checkpoint_paths.py | 19 ++++ afd_plugin/model_executor/models/qwen3_moe.py | 17 +--- 6 files changed, 78 insertions(+), 84 deletions(-) create mode 100644 afd_plugin/ffn_faas/model_utils.py create mode 100644 afd_plugin/model_executor/checkpoint_paths.py diff --git a/afd_plugin/ffn_faas/model_utils.py b/afd_plugin/ffn_faas/model_utils.py new file mode 100644 index 000000000..47a0fa295 --- /dev/null +++ b/afd_plugin/ffn_faas/model_utils.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Shared utilities for standalone FFN model owners.""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import TYPE_CHECKING, Protocol + +if TYPE_CHECKING: + from torch import nn + + +class ParameterStorageOwner(Protocol): + """A module-like owner whose distinct parameter storages are counted.""" + + def parameters(self) -> Iterator[nn.Parameter]: ... + + +def parameter_storage_bytes(model: ParameterStorageOwner) -> int: + """Bytes of the distinct storages behind ``model``'s parameters. + + Views that share storage (for example the connector's expert views) are + counted once. + """ + storage_sizes: dict[tuple[str, int], int] = {} + for parameter in model.parameters(): + storage = parameter.untyped_storage() + storage_sizes[(str(parameter.device), storage.data_ptr())] = storage.nbytes() + return sum(storage_sizes.values()) diff --git a/afd_plugin/ffn_faas/platforms/gpu/backend.py b/afd_plugin/ffn_faas/platforms/gpu/backend.py index 7e250655f..3bd8ad3bd 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/backend.py +++ b/afd_plugin/ffn_faas/platforms/gpu/backend.py @@ -16,12 +16,12 @@ from vllm.config import set_current_vllm_config from afd_plugin.connectors.gpu.deepep_m2n.connector import DeepEPM2NAFDConnector +from afd_plugin.ffn_faas.model_utils import parameter_storage_bytes from afd_plugin.ffn_faas.platforms.gpu.environment import GPUFFNEnvironment from afd_plugin.ffn_faas.platforms.gpu.qwen3_model import Qwen3CheckpointDescription from afd_plugin.ffn_faas.platforms.gpu.qwen3_routed_model import ( Qwen3RoutedExpertsModel, load_qwen3_routed_experts, - parameter_storage_bytes, ) from afd_plugin.ffn_faas.platforms.gpu.settings import GPUFFNConfig from afd_plugin.ffn_faas.service import FFNService, ServicePhase diff --git a/afd_plugin/ffn_faas/platforms/gpu/qwen3_routed_model.py b/afd_plugin/ffn_faas/platforms/gpu/qwen3_routed_model.py index 7561bf74a..3d9dbc775 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/qwen3_routed_model.py +++ b/afd_plugin/ffn_faas/platforms/gpu/qwen3_routed_model.py @@ -31,7 +31,7 @@ from vllm.model_executor.models.utils import AutoWeightsLoader from vllm.utils.torch_utils import set_default_torch_dtype -from afd_plugin.model_executor.models.qwen3_moe import _weight_layer_path +from afd_plugin.model_executor.checkpoint_paths import weight_layer_path if TYPE_CHECKING: from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts @@ -45,10 +45,10 @@ def _is_routed_mlp_path(name: str) -> bool: Keeps ``model.layers..mlp.*`` (routed experts and the unused gate) and drops embeddings, norms, Attention and the LM head, which this shell does - not construct. Reuses the AFD Qwen3 wrapper's path parser so both models - classify checkpoint names identically. + not construct. Reuses the shared checkpoint path parser so the Worker-path + wrapper and this shell classify checkpoint names identically. """ - layer_path = _weight_layer_path(name) + layer_path = weight_layer_path(name) return layer_path is not None and layer_path[1] == ROUTED_STAGE @@ -75,10 +75,6 @@ def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: class _Qwen3RoutedExpertsStack(nn.Module): """The ``model`` submodule: only ``layers``, each holding its MoE block.""" - # Upstream: vLLM v0.32.0, vllm/model_executor/models/qwen3_moe.py, - # Qwen3MoeModel.hf_to_vllm_mapper; reused, not copied. - hf_to_vllm_mapper = native.Qwen3MoeModel.hf_to_vllm_mapper - def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: """Build one ``_Qwen3RoutedExpertsLayer`` per decoder layer.""" super().__init__() @@ -88,33 +84,6 @@ def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: for layer_idx in range(num_layers) ) - # Upstream source: vLLM v0.32.0 (5df63a7203cdb9328d824a27b682e77c7b9928be), - # vllm/model_executor/models/qwen3_moe.py, Qwen3MoeModel.load_weights. - # Copied unchanged: the same AutoWeightsLoader, ignored suffixes and - # checkpoint mapper, so expert stacking and EP sharding stay in FusedMoE's - # own loader. Re-copy on a vLLM upgrade. - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - """Load ``layers.*`` tensors through the native Qwen3 model loader path. - - ``AutoWeightsLoader`` walks the module tree and hands each MoE block's - ``experts.*`` tensors to FusedMoE's ``load_weights``, which stacks - per-expert gate/up/down projections into ``w13``/``w2`` and keeps - only this rank's experts. Returns the loaded parameter names for the - strict loaded-weights check. - """ - loader = AutoWeightsLoader( - self, - ignore_unexpected_suffixes=[ - ".bias", - "_bias", - ".weight_scale", - "_weight_scale", - ".input_scale", - "_input_scale", - ], - ) - return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) - class Qwen3RoutedExpertsModel(nn.Module): """FFN-only Qwen3 MoE model consumed by ``GPUFFNRoutedExpertsModelRunner``. @@ -138,26 +107,34 @@ def __init__(self, *, vllm_config: VllmConfig) -> None: super().__init__() self.model = _Qwen3RoutedExpertsStack(vllm_config, MODEL_PREFIX) + def is_unused_checkpoint_weight(self, name: str) -> bool: + """Whether vLLM can skip a shard containing only this checkpoint name.""" + return not _is_routed_mlp_path(name) + # Upstream source: vLLM v0.32.0 (5df63a7203cdb9328d824a27b682e77c7b9928be), - # vllm/model_executor/models/qwen3_moe.py, Qwen3MoeForCausalLM.load_weights. - # Copied with one marked difference, the routed-path filter. + # vllm/model_executor/models/qwen3_moe.py, Qwen3MoeModel.hf_to_vllm_mapper. + # AFD difference: filter to routed MoE paths and apply the native mapper at + # this shell's outer AutoWeightsLoader. def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: """Drop non-MoE checkpoint tensors, then load like the native CausalLM. - The native CausalLM delegates to ``AutoWeightsLoader(self)``, which - recurses into ``model.load_weights``. Tensors this shell does not - construct (embeddings, norms, Attention, LM head) are filtered first, - as ``AFDQwen3MoeForCausalLM.load_weights`` filters by role, but - stricter: that wrapper still keeps non-layer tensors for its - embedding and LM head. + ``AutoWeightsLoader`` walks this shell's module tree and hands each MoE + block's expert tensors to FusedMoE. Tensors this shell does not construct + (embeddings, norms, Attention, LM head) are filtered first; the native + Qwen3 mapper still applies unchanged. """ # ### PATCH START: keep only routed MoE checkpoint paths. weights = ( - (name, tensor) for name, tensor in weights if _is_routed_mlp_path(name) + (name, tensor) + for name, tensor in weights + if not self.is_unused_checkpoint_weight(name) ) # ### PATCH END: keep only routed MoE checkpoint paths. loader = AutoWeightsLoader(self) - return loader.load_weights(weights) + return loader.load_weights( + weights, + mapper=native.Qwen3MoeModel.hf_to_vllm_mapper, + ) def get_experts_layer_indices(self) -> tuple[int, ...]: """Every decoder layer is an MoE layer for a supported checkpoint.""" @@ -171,8 +148,8 @@ def get_routed_experts(self, layer_idx: int) -> RoutedExperts: # NPU parallel: afd_plugin/ffn_faas/platforms/npu/model.py::CAMRoutedModel.load # (layer-by-layer safetensors load with per-layer progress, then storage # accounting). -# Reuse: PARTIAL - storage accounting is the same computation -# (REFACTOR-CANDIDATE(R7)); loading differs: vLLM's loader exposes no +# Reuse: PARTIAL - storage accounting is shared through +# ``parameter_storage_bytes``; loading differs: vLLM's loader exposes no # per-layer callback, so progress is reported only before and after. # Upstream source: vLLM v0.32.0 (5df63a7203cdb9328d824a27b682e77c7b9928be), # vllm/model_executor/model_loader/base_loader.py, BaseModelLoader.load_model. @@ -201,18 +178,3 @@ def load_qwen3_routed_experts( gc.collect() torch.accelerator.empty_cache() return model.eval() - - -# NPU parallel: CAMRoutedModel._load_native storage accounting. -# Reuse: YES - same computation; REFACTOR-CANDIDATE(R7). -def parameter_storage_bytes(model: nn.Module) -> int: - """Bytes of the distinct storages behind ``model``'s parameters. - - Views that share storage (for example the connector's expert views) are - counted once. - """ - storage_sizes: dict[tuple[str, int], int] = {} - for parameter in model.parameters(): - storage = parameter.untyped_storage() - storage_sizes[(str(parameter.device), storage.data_ptr())] = storage.nbytes() - return sum(storage_sizes.values()) diff --git a/afd_plugin/ffn_faas/platforms/npu/model.py b/afd_plugin/ffn_faas/platforms/npu/model.py index 8209002d6..efbb91bb2 100644 --- a/afd_plugin/ffn_faas/platforms/npu/model.py +++ b/afd_plugin/ffn_faas/platforms/npu/model.py @@ -14,6 +14,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Protocol +from afd_plugin.ffn_faas.model_utils import parameter_storage_bytes from afd_plugin.ffn_faas.platforms.npu.settings import validate_native_expert_map if TYPE_CHECKING: @@ -166,12 +167,7 @@ def _load_native( if tuple(view.layer_idx for view in views) != adapter.layer_indices: raise ValueError("native views do not match the routed model layers") self._views = views - storage_sizes: dict[tuple[str, int], int] = {} - for parameter in owner.parameters(): - storage = parameter.untyped_storage() - key = (str(parameter.device), storage.data_ptr()) - storage_sizes[key] = storage.nbytes() - self._parameter_bytes = sum(storage_sizes.values()) + self._parameter_bytes = parameter_storage_bytes(owner) self._loaded = True def get_experts_layer_indices(self) -> tuple[int, ...]: diff --git a/afd_plugin/model_executor/checkpoint_paths.py b/afd_plugin/model_executor/checkpoint_paths.py new file mode 100644 index 000000000..f433b526e --- /dev/null +++ b/afd_plugin/model_executor/checkpoint_paths.py @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""CPU-safe checkpoint path parsing shared by AFD model wrappers.""" + +from __future__ import annotations + + +def weight_layer_path(name: str) -> tuple[int, str, tuple[str, ...]] | None: + """Return ``(layer index, stage, remainder)`` for a decoder path.""" + parts = name.split(".") + for marker_idx, part in enumerate(parts[:-2]): + if part != "layers": + continue + try: + layer_idx = int(parts[marker_idx + 1]) + except ValueError: + continue + return layer_idx, parts[marker_idx + 2], tuple(parts[marker_idx + 3 :]) + return None diff --git a/afd_plugin/model_executor/models/qwen3_moe.py b/afd_plugin/model_executor/models/qwen3_moe.py index 54dc0f406..10eb5c311 100644 --- a/afd_plugin/model_executor/models/qwen3_moe.py +++ b/afd_plugin/model_executor/models/qwen3_moe.py @@ -25,6 +25,7 @@ AFDConfig, parse_afd_config, ) +from afd_plugin.model_executor.checkpoint_paths import weight_layer_path from afd_plugin.model_executor.models.deepseek_v2 import RemoteFFNProxy if TYPE_CHECKING: @@ -49,20 +50,6 @@ def _routes_on_attention(afd_config: AFDConfig) -> bool: return afd_config.connector in DEEPEP_M2N_CONNECTORS -def _weight_layer_path(name: str) -> tuple[int, str, tuple[str, ...]] | None: - """Return ``(layer index, stage, remainder)`` for a native decoder path.""" - parts = name.split(".") - for marker_idx, part in enumerate(parts[:-2]): - if part != "layers": - continue - try: - layer_idx = int(parts[marker_idx + 1]) - except ValueError: - continue - return layer_idx, parts[marker_idx + 2], tuple(parts[marker_idx + 3 :]) - return None - - def _checkpoint_weight_roles( name: str, *, @@ -74,7 +61,7 @@ def _checkpoint_weight_roles( Attention routes with it, and FFN keeps the native gate parameter that vLLM's strict loader requires but never runs it. """ - layer_path = _weight_layer_path(name) + layer_path = weight_layer_path(name) if layer_path is None: return _BOTH_ROLES From b26bdc34e66d2c30dbf7a9eefb336c5084842364 Mon Sep 17 00:00:00 2001 From: jiaran-king Date: Fri, 6 Nov 2026 14:27:35 +0800 Subject: [PATCH 09/42] refactor(ffn): reuse native engine CLI and AFD configuration Signed-off-by: jiaran-king --- afd_plugin/ffn_faas/__main__.py | 29 ++++-- afd_plugin/ffn_faas/interfaces.py | 4 +- afd_plugin/ffn_faas/platforms/npu/adapter.py | 85 ++++++++-------- afd_plugin/ffn_faas/platforms/npu/backend.py | 47 ++++----- .../ffn_faas/platforms/npu/environment.py | 21 +++- afd_plugin/ffn_faas/platforms/npu/settings.py | 65 ++---------- docs/design/module/ffn_runtime.md | 6 ++ docs/npu/STANDALONE_FFN_SERVICE.md | 74 ++++++++++---- tests/npu/standalone_cam_attention.py | 42 ++++---- tests/unit/fakes/ffn_faas.py | 60 ++++++++---- tests/unit/ffn_faas/test_main.py | 98 +++++++++++++++++++ tests/unit/ffn_faas/test_settings.py | 74 +++++++++----- tests/unit/ffn_faas/test_v4_model.py | 7 +- 13 files changed, 381 insertions(+), 231 deletions(-) diff --git a/afd_plugin/ffn_faas/__main__.py b/afd_plugin/ffn_faas/__main__.py index 6108dc201..9aedb9936 100644 --- a/afd_plugin/ffn_faas/__main__.py +++ b/afd_plugin/ffn_faas/__main__.py @@ -13,6 +13,7 @@ from contextlib import suppress from pathlib import Path +from afd_plugin.ffn_faas.interfaces import FFNPlatformAdapter from afd_plugin.ffn_faas.platforms import SUPPORTED_BACKENDS, get_platform_adapter from afd_plugin.ffn_faas.runner import FFNModelRunner from afd_plugin.ffn_faas.service import FFNService @@ -24,6 +25,24 @@ def _report_exception() -> None: traceback.print_exc() +def create_parser(adapter: FFNPlatformAdapter) -> argparse.ArgumentParser: + parser = adapter.create_parser() + parser.description = __doc__ + parser.add_argument("command", choices=["serve"]) + parser.add_argument("model_tag", nargs="?", help="local model checkpoint") + parser.add_argument("--backend", choices=SUPPORTED_BACKENDS, default="ascend-cam") + parser.add_argument( + "--receiver-shutdown-timeout", + type=float, + default=5.0, + help="seconds to join the standalone receiver before process isolation", + ) + parser.add_argument("--status-host", default="127.0.0.1") + parser.add_argument("--status-port", type=int, default=29700) + parser.add_argument("--status-file", type=Path) + return parser + + def main() -> int: selection = argparse.ArgumentParser(add_help=False) selection.add_argument( @@ -31,15 +50,7 @@ def main() -> int: ) selected, _ = selection.parse_known_args() adapter = get_platform_adapter(selected.backend) - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("command", choices=["serve"]) - parser.add_argument("model", type=Path) - parser.add_argument("--backend", choices=SUPPORTED_BACKENDS, default="ascend-cam") - parser.add_argument("--shutdown-timeout", type=float, default=5.0) - parser.add_argument("--status-host", default="127.0.0.1") - parser.add_argument("--status-port", type=int, default=29700) - parser.add_argument("--status-file", type=Path) - adapter.add_arguments(parser) + parser = create_parser(adapter) args = parser.parse_args() try: launch = adapter.configure(args) diff --git a/afd_plugin/ffn_faas/interfaces.py b/afd_plugin/ffn_faas/interfaces.py index 91f419d7b..9211cd407 100644 --- a/afd_plugin/ffn_faas/interfaces.py +++ b/afd_plugin/ffn_faas/interfaces.py @@ -42,8 +42,8 @@ class FFNLaunch: class FFNPlatformAdapter(Protocol): - """Register platform options and validate a CPU-only launch description.""" + """Build the selected CLI and describe a launch without constructing a backend.""" - def add_arguments(self, parser: ArgumentParser) -> None: ... + def create_parser(self) -> ArgumentParser: ... def configure(self, args: Namespace) -> FFNLaunch: ... diff --git a/afd_plugin/ffn_faas/platforms/npu/adapter.py b/afd_plugin/ffn_faas/platforms/npu/adapter.py index 50d1bf8ed..3a5879210 100644 --- a/afd_plugin/ffn_faas/platforms/npu/adapter.py +++ b/afd_plugin/ffn_faas/platforms/npu/adapter.py @@ -1,14 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project -"""CPU-only Ascend/V4 launch adaptation; native imports happen on preparation.""" +"""Ascend/V4 CLI adaptation; importing this module remains CPU-only.""" from __future__ import annotations import os from argparse import ArgumentParser, Namespace from functools import partial +from pathlib import Path from typing import TYPE_CHECKING +from afd_plugin.config import AFD_ASYNC_CONNECTOR, parse_afd_config from afd_plugin.ffn_faas.interfaces import FFNLaunch from afd_plugin.ffn_faas.platforms.npu import get_model from afd_plugin.ffn_faas.platforms.npu.settings import FFNConfig @@ -18,31 +20,39 @@ from afd_plugin.ffn_faas.platforms.npu.model import CAMRoutedModel -def create_backend(model: CAMRoutedModel, settings: FFNConfig) -> FFNBackend: +def create_backend( + model: CAMRoutedModel, settings: FFNConfig, args: Namespace +) -> FFNBackend: from vllm_ascend.utils import bootstrap_custom_op_env bootstrap_custom_op_env() from afd_plugin.ffn_faas.platforms.npu.backend import CAMFFNBackend - return CAMFFNBackend(model, settings) + return CAMFFNBackend(model, settings, args) class AscendCAMAdapter: """Keep model, quantization, topology and device options out of the host loop.""" - def add_arguments(self, parser: ArgumentParser) -> None: - parser.add_argument("--host", default="127.0.0.1") - parser.add_argument("--port", type=int, default=29600) - parser.add_argument("--attention-ranks", type=int, required=True) - parser.add_argument("--attention-tp", type=int, required=True) - parser.add_argument("--ffn-ranks", type=int, required=True) - parser.add_argument("--devices", help="allocated logical NPU IDs on this node") - parser.add_argument("--max-num-batched-tokens", type=int, default=2048) - parser.add_argument("--max-num-seqs", type=int, default=256) - parser.add_argument("--full-graph", action="store_true") - parser.add_argument("--hccl-buffer-size", type=int) + def create_parser(self) -> ArgumentParser: + # Selected-platform imports: the Service/host entry stays CPU-only. + from vllm.engine.arg_utils import EngineArgs + from vllm.plugins import load_general_plugins + from vllm.utils.argparse_utils import FlexibleArgumentParser + + load_general_plugins() + parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) + # This process is already launched by torchrun, not a vLLM executor. + parser.set_defaults(distributed_executor_backend="external_launcher") + return parser def configure(self, args: Namespace) -> FFNLaunch: + if args.model_tag is not None: + # Match vLLM serve's positional-model precedence. + args.model = args.model_tag + afd = parse_afd_config(args.additional_config, expected_role="ffn") + if afd.connector != AFD_ASYNC_CONNECTOR: + raise ValueError("standalone Ascend FFN requires CAMAsyncAFDConnector") rank, local_rank, world = ( int(os.environ.get(key, default)) for key, default in ( @@ -51,47 +61,30 @@ def configure(self, args: Namespace) -> FFNLaunch: ("WORLD_SIZE", "1"), ) ) - if world != args.ffn_ranks: - raise ValueError("F-only launcher WORLD_SIZE must equal --ffn-ranks") - local_world = int(os.environ.get("LOCAL_WORLD_SIZE", str(world))) - devices = ( - tuple(map(int, args.devices.split(","))) - if args.devices - else tuple(range(local_world)) - ) - if ( - not 0 <= local_rank < len(devices) - or len(set(devices)) != len(devices) - or min(devices) < 0 - ): - raise ValueError("--devices must supply distinct local NPU IDs") - if os.environ.get("VLLM_USE_V2_MODEL_RUNNER") != "1": - raise ValueError("set VLLM_USE_V2_MODEL_RUNNER=1 on both A and F") - if os.environ.get("AFD_ASYNC_CAM_LAYERED_GMM") != "1": - raise ValueError("standalone V4 requires AFD_ASYNC_CAM_LAYERED_GMM=1") + if world != afd.num_ffn_ranks or world != args.data_parallel_size: + raise ValueError( + "F-only WORLD_SIZE, --data-parallel-size and " + "additional_config.afd.num_ffn_ranks must agree" + ) + if args.data_parallel_rank is not None and args.data_parallel_rank != rank: + raise ValueError("--data-parallel-rank must match the F-only launcher RANK") + args.data_parallel_rank = rank + if args.distributed_executor_backend != "external_launcher": + raise ValueError("standalone FFN requires the external_launcher backend") settings = FFNConfig( - model=args.model, - attention_ranks=args.attention_ranks, - attention_tp=args.attention_tp, - ffn_ranks=args.ffn_ranks, + model=Path(args.model), + ffn_ranks=world, rank=rank, local_rank=local_rank, - device=devices[local_rank], - host=args.host, - port=args.port, - max_num_batched_tokens=args.max_num_batched_tokens, - max_num_seqs=args.max_num_seqs, - full_graph=args.full_graph, - hccl_buffer_size=args.hccl_buffer_size, - shutdown_timeout=args.shutdown_timeout, + shutdown_timeout=args.receiver_shutdown_timeout, ) - model = get_model(args.model, settings.ffn_ranks, settings.rank) + model = get_model(settings.model, settings.ffn_ranks, settings.rank) return FFNLaunch( rank=rank, local_rank=local_rank, model_path=str(model.root), backend_name="CAMAsyncAFDConnector", shutdown_timeout=settings.shutdown_timeout, - backend_factory=partial(create_backend, model, settings), + backend_factory=partial(create_backend, model, settings, args), initial_status={"cam_paired": False}, ) diff --git a/afd_plugin/ffn_faas/platforms/npu/backend.py b/afd_plugin/ffn_faas/platforms/npu/backend.py index e75bd7268..402cc2d74 100644 --- a/afd_plugin/ffn_faas/platforms/npu/backend.py +++ b/afd_plugin/ffn_faas/platforms/npu/backend.py @@ -8,12 +8,17 @@ from __future__ import annotations +from argparse import Namespace + import torch import torch_npu # noqa: F401 from vllm.config import set_current_vllm_config from vllm.engine.arg_utils import EngineArgs -from afd_plugin.compat.npu import fix_all2all_backend_for_afd +from afd_plugin.compat.npu import ( + fail_if_unsupported_npu_afd_features, + fix_all2all_backend_for_afd, +) from afd_plugin.config import parse_afd_config from afd_plugin.connectors.npu.async_cam import CAMAsyncAFDConnector from afd_plugin.ffn_faas.platforms.npu.environment import AscendFFNEnvironment @@ -21,45 +26,35 @@ from afd_plugin.ffn_faas.platforms.npu.settings import FFNConfig from afd_plugin.ffn_faas.service import FFNService, ServicePhase from afd_plugin.v1.worker.npu.async_ffn_model_runner import AsyncNPUFFNModelRunner +from afd_plugin.validation import validate_npu_model_runner_v2_config class CAMFFNBackend: """Keep weights and communication owners until receiver exit is proven.""" - def __init__(self, model: CAMRoutedModel, settings: FFNConfig) -> None: + def __init__( + self, model: CAMRoutedModel, settings: FFNConfig, args: Namespace + ) -> None: self.settings = settings + self.args = args self.environment = AscendFFNEnvironment(settings) self.model = model self.runtime: AsyncNPUFFNModelRunner | None = None self.connector: CAMAsyncAFDConnector | None = None def prepare(self, service: FFNService) -> None: - self.environment.initialize_device() - args = EngineArgs( - model=str(self.settings.model), - trust_remote_code=True, - dtype="bfloat16", - quantization="ascend", - enforce_eager=not self.settings.full_graph, - compilation_config={"cudagraph_mode": "FULL"} - if self.settings.full_graph - else {}, - tensor_parallel_size=1, - data_parallel_size=self.settings.ffn_ranks, - data_parallel_rank=self.settings.rank, - enable_expert_parallel=True, - distributed_executor_backend="external_launcher", - # F owns a dispatch envelope, not a request-context/KV capacity. - max_model_len=self.settings.max_num_batched_tokens, - max_num_batched_tokens=self.settings.max_num_batched_tokens, - max_num_seqs=self.settings.max_num_seqs, - additional_config=self.settings.additional_config(), - ) + args = EngineArgs.from_cli_args(self.args) config = args.create_engine_config() - # Platform configuration can choose an executor for ordinary Workers; - # this loader belongs to the already launched F-only torchrun world. + # Keep the externally launched F-only ownership after platform defaults. config.parallel_config.distributed_executor_backend = "external_launcher" fix_all2all_backend_for_afd(config) + validate_npu_model_runner_v2_config( + config, expected_role="ffn", device_type="npu" + ) + fail_if_unsupported_npu_afd_features(config) + if config.speculative_config is not None: + raise ValueError("standalone V4 loads main-model routed layers only") + self.environment.initialize_device(config) self.environment.initialize_compute(config) with set_current_vllm_config(config): self.model.load( @@ -72,7 +67,7 @@ def prepare(self, service: FFNService) -> None: ) self.connector = CAMAsyncAFDConnector( rank=self.settings.rank, - local_rank=self.settings.device, + local_rank=self.environment.device.index, vllm_config=config, afd_config=parse_afd_config(config, expected_role="ffn"), role_rank=self.settings.rank, diff --git a/afd_plugin/ffn_faas/platforms/npu/environment.py b/afd_plugin/ffn_faas/platforms/npu/environment.py index 952e59acf..5a9f2065e 100644 --- a/afd_plugin/ffn_faas/platforms/npu/environment.py +++ b/afd_plugin/ffn_faas/platforms/npu/environment.py @@ -47,7 +47,7 @@ def ep_group(self) -> GroupCoordinator: raise RuntimeError("Ascend FFN expert group has not been initialized") return self._ep_group - def initialize_device(self) -> None: + def initialize_device(self, config: VllmConfig) -> None: if self._groups_owned or self._device is not None: raise RuntimeError("Ascend FFN environment is already initialized") settings = self.settings @@ -65,6 +65,8 @@ def initialize_device(self) -> None: import torch import torch_npu # noqa: F401 from vllm.distributed.parallel_state import model_parallel_is_initialized + from vllm.platforms import current_platform + from vllm.platforms.interface import set_assigned_physical_gpu_ids from vllm_ascend.utils import enable_custom_op from afd_plugin.compat.npu.ops import ensure_cam_async_ops_available @@ -72,8 +74,17 @@ def initialize_device(self) -> None: if torch.distributed.is_initialized() or model_parallel_is_initialized(): raise RuntimeError("Ascend FFN cannot borrow existing native groups") ensure_cam_async_ops_available() - torch.npu.set_device(settings.device) - self._device = torch.device(f"npu:{settings.device}") + # Reuse the native Worker's mapping for --device-ids and visibility. + assigned = config.parallel_config.assigned_physical_gpu_ids + if assigned is not None: + if settings.local_rank >= len(assigned): + raise ValueError("LOCAL_RANK is outside the assigned device list") + set_assigned_physical_gpu_ids(assigned) + index = current_platform.logical_device_id_to_visible_device_id( + settings.local_rank + ) + self._device = torch.device(f"npu:{index}") + torch.npu.set_device(self._device) # Native NPUModelRunner v1 installs this packing policy at import time. # This service does not import that runner; native W4A8 still needs NZ. torch.npu.config.allow_internal_format = True @@ -110,7 +121,7 @@ def initialize_compute(self, config: VllmConfig) -> None: init_distributed_environment( world_size=settings.ffn_ranks, rank=settings.rank, - local_rank=settings.device, + local_rank=self.device.index, distributed_init_method="env://", backend="hccl", timeout=timedelta(seconds=COMPUTE_GROUP_TIMEOUT_SECONDS), @@ -151,7 +162,7 @@ def bind_host_threads(self, config: VllmConfig) -> None: bind_cpus( self.settings.local_rank, npu_id=current_platform.device_id_to_physical_device_id( - self.settings.device + self.settings.local_rank ), migrate_memory=not (engram is not None and engram.cpu_offload), ) diff --git a/afd_plugin/ffn_faas/platforms/npu/settings.py b/afd_plugin/ffn_faas/platforms/npu/settings.py index 891ed5c6e..034623ebf 100644 --- a/afd_plugin/ffn_faas/platforms/npu/settings.py +++ b/afd_plugin/ffn_faas/platforms/npu/settings.py @@ -8,77 +8,24 @@ from dataclasses import dataclass from pathlib import Path -from afd_plugin.config import AFD_ASYNC_CONNECTOR - @dataclass(frozen=True) class FFNConfig: model: Path - attention_ranks: int - attention_tp: int ffn_ranks: int rank: int = 0 local_rank: int = 0 - device: int = 0 - host: str = "127.0.0.1" - port: int = 29600 - max_num_batched_tokens: int = 2048 - max_num_seqs: int = 256 - full_graph: bool = False - hccl_buffer_size: int | None = None shutdown_timeout: float = 5.0 def __post_init__(self) -> None: - positive = ( - self.attention_ranks, - self.attention_tp, - self.ffn_ranks, - self.max_num_batched_tokens, - self.max_num_seqs, - ) - if any(value <= 0 for value in positive): - raise ValueError( - "rank counts, TP and scheduler capacities must be positive" - ) - if self.attention_ranks % self.attention_tp: - raise ValueError("Attention ranks must be divisible by Attention TP") - if self.max_num_batched_tokens % self.attention_tp: - raise ValueError( - "the CAM dispatch envelope must be divisible by Attention TP" - ) - if not 0 <= self.rank < self.ffn_ranks or min(self.local_rank, self.device) < 0: - raise ValueError("invalid F rank or local device") - if not self.host or not 0 < self.port < 65536: - raise ValueError("CAM rendezvous requires a host and a valid port") + # Engine/connector definitions and validation belong to vLLM/AFD. + # Only external-launcher and host-shutdown facts live here. + if self.ffn_ranks <= 0 or not 0 <= self.rank < self.ffn_ranks: + raise ValueError("invalid F-only rank or world size") + if self.local_rank < 0: + raise ValueError("local rank must be nonnegative") if not math.isfinite(self.shutdown_timeout) or self.shutdown_timeout <= 0: raise ValueError("shutdown timeout must be finite and positive") - if self.hccl_buffer_size is not None and self.hccl_buffer_size <= 0: - raise ValueError("explicit CAM HCCL buffer size must be positive") - - def additional_config(self) -> dict: - extra = { - "dynamicQuant": 1, - "attn_ranks_per_dp": self.attention_tp, - } - if self.hccl_buffer_size is not None: - extra["hccl_buffer_size"] = self.hccl_buffer_size - return { - "afd": { - "connector": AFD_ASYNC_CONNECTOR, - "role": "ffn", - "async_dp": True, - "host": self.host, - "port": self.port, - "num_attention_ranks": self.attention_ranks, - "num_ffn_ranks": self.ffn_ranks, - "compute_gate_on_attention": True, - "connector_extra_config": extra, - }, - "enable_prefill_mc2": False, - "enable_force_eplb": False, - "enable_cpu_binding": True, - "ascend_fusion_config": {"fusion_ops_gmmswigluquant": True}, - } def validate_static_experts( diff --git a/docs/design/module/ffn_runtime.md b/docs/design/module/ffn_runtime.md index ce4798fe2..457e87fab 100644 --- a/docs/design/module/ffn_runtime.md +++ b/docs/design/module/ffn_runtime.md @@ -97,6 +97,12 @@ a vLLM Worker, upstream ModelRunner, EngineCore, scheduler or KV cache. Attention retains its existing model, connector and DBO path. Configuration, parameter loading and quantization remain vLLM/vLLM-Ascend dependencies. +The standalone Ascend CLI reuses vLLM engine argument definitions, parsing and +configuration construction. Existing AFD options remain in `additional_config`; +only host management and shutdown options are standalone-specific. The host +entry delegates parser creation to its selected platform, so CPU-only service +and host-loop imports remain independent of vLLM. + The CPU-only host `FFNModelRunner` accepts an `FFNBackend` factory, retains the backend before preparation, and owns the receiving thread and failure boundary. `FFNService` manages host status and stop requests. Platform options diff --git a/docs/npu/STANDALONE_FFN_SERVICE.md b/docs/npu/STANDALONE_FFN_SERVICE.md index 25d8411bb..fb1a6f612 100644 --- a/docs/npu/STANDALONE_FFN_SERVICE.md +++ b/docs/npu/STANDALONE_FFN_SERVICE.md @@ -54,33 +54,71 @@ afd_cam_vendor=/path/to/afd_plugin/_cann_ops_custom/vendors/afd-plugin ascend_native_vendor=/path/to/vllm_ascend/_cann_ops_custom/vendors/custom_transformer export ASCEND_CUSTOM_OPP_PATH="$ascend_native_vendor:$afd_cam_vendor${ASCEND_CUSTOM_OPP_PATH:+:$ASCEND_CUSTOM_OPP_PATH}" export VLLM_USE_V2_MODEL_RUNNER=1 -# F-only setting: keep this unset or 0 in the Attention environment. -export AFD_ASYNC_CAM_LAYERED_GMM=1 export SOC_VERSION=910c export VLLM_PLUGINS=ascend,afd,ascend_model,ascend_model_loader,ascend_kv_connector # Keep BATCH_SIZE_FACTOR, HCCL/network settings and receive capacity consistent # with the existing A launch and CAM runtime guide. +export ASCEND_RT_VISIBLE_DEVICES=12,13,14,15 torchrun --nproc-per-node=4 --master-addr=127.0.0.1 --master-port=29500 \ -m afd_plugin.ffn_faas serve /path/to/model --backend ascend-cam \ - --host 127.0.0.1 --port 29600 \ - --attention-ranks 12 --attention-tp 4 --ffn-ranks 4 \ - --devices 12,13,14,15 --max-num-batched-tokens 2048 \ - --full-graph --status-port 29700 + --trust-remote-code --dtype bfloat16 --quantization ascend \ + --enable-expert-parallel --data-parallel-size 4 \ + --max-num-batched-tokens 2048 --max-num-seqs 256 \ + --compilation-config '{"cudagraph_mode":"FULL"}' \ + --additional-config '{ + "afd": { + "connector":"CAMAsyncAFDConnector","role":"ffn","async_dp":true, + "host":"127.0.0.1","port":29600, + "num_attention_ranks":12,"num_ffn_ranks":4, + "compute_gate_on_attention":true, + "connector_extra_config":{ + "dynamicQuant":1,"attn_ranks_per_dp":4 + } + }, + "enable_prefill_mc2":false,"enable_force_eplb":false,"enable_cpu_binding":true, + "ascend_fusion_config":{"fusion_ops_gmmswigluquant":true} + }' --status-port 29700 ``` -`--devices` indexes the process's visible logical devices, not physical IDs. -When F has a restricted visibility list, use `--devices 0,1,2,3` instead. The -external launcher's `WORLD_SIZE` is F-only. CAM creates the separate -`[A0 ... A11, F0 ... F3]` world using the same host, port, group name, topology, -quantization, layer table and startup signatures as traditional F. Use a -different rendezvous port for the F-only compute world and CAM. +Engine arguments use vLLM's `FlexibleArgumentParser`, `EngineArgs.add_cli_args` +and `EngineArgs.from_cli_args`; definitions, JSON/dotted options, defaults and +configuration validation come from the pinned upstream stack. The selected +Ascend CLI imports that layer, while importing the Service, host runner or +platform adapter module remains CPU-only. No serving engine is constructed. + +Use `--additional-config` for the existing AFD topology, connector and HCCL +options. Engine capacities and compilation are passed through +without rebuilding a second configuration. Only backend selection, management +endpoints, status files and shutdown control add standalone options. The old +`--full-graph`, `--attention-ranks`, `--attention-tp`, `--ffn-ranks`, `--devices` +and standalone stream/budget flags have been removed; update launch scripts to +the standard options above. Device selection uses the native `--device-ids` +mapping or `ASCEND_RT_VISIBLE_DEVICES` plus the launcher's `LOCAL_RANK`. + +The external launcher's `WORLD_SIZE`, `--data-parallel-size` and +`additional_config.afd.num_ffn_ranks` must agree. An omitted +`--data-parallel-rank` is taken from `RANK`; an explicit value must match it. +The executor defaults to `external_launcher` because torchrun already owns +these processes. Set `VLLM_USE_V2_MODEL_RUNNER=1` as on A. All other engine +options retain upstream defaults; the example explicitly selects the qualified +W4A8 profile. Reusing the parser does not qualify additional models, speculative +loading or serving/KV features for this service. + +The host receive-loop join uses `--receiver-shutdown-timeout` (default: five +seconds). It is separate from vLLM's `--shutdown-timeout`, whose EngineCore +shutdown semantics do not govern this standalone receiver. + +CAM creates the separate `[A0 ... A11, F0 ... F3]` world using the same host, +port, group name, topology, quantization, layer table and startup signatures as +traditional F. Use a different rendezvous port for the F-only compute world +and CAM. `--max-num-batched-tokens` must match A's dispatch envelope; it is not the -maximum request context length. F creates configuration objects for native -loading, but allocates no request KV cache. Omit `--full-graph` to use the -mature layered eager path. FULL preserves paired warmup, private graph pools -and device layer selection. FFN two-stream execution and its core-budget -options were removed in #550; older launch commands must omit those options. +maximum request context length. F allocates no request KV cache. To use eager, +replace `--compilation-config` with `--enforce-eager`. FULL preserves paired +warmup, private graph pools and device layer selection. FFN two-stream +execution and its core-budget options were removed in #550; older launch +commands must omit those options. ## Management and shutdown @@ -93,7 +131,7 @@ Each local F rank exposes a management endpoint on `--status-port + LOCAL_RANK`. weights and paired CAM startup are prepared. Neither proves progress of an outstanding device receive. - `POST /stop`, SIGINT and SIGTERM request a stop between completed batches. - The host joins its receive loop for `--shutdown-timeout` seconds. + The host joins its receive loop for `--receiver-shutdown-timeout` seconds. The shutdown timeout does not cancel a CAM operator or prove device quiescence. It applies after startup; native loading and rendezvous retain their existing diff --git a/tests/npu/standalone_cam_attention.py b/tests/npu/standalone_cam_attention.py index b7a54c4a2..713434cff 100644 --- a/tests/npu/standalone_cam_attention.py +++ b/tests/npu/standalone_cam_attention.py @@ -31,15 +31,33 @@ AFDTransferMetadata, ) from afd_plugin.connectors.npu.async_cam import CAMAsyncAFDConnector # noqa: E402 -from afd_plugin.ffn_faas.platforms.npu.settings import FFNConfig # noqa: E402 from afd_plugin.ffn_faas.platforms.npu.v4_model import ( # noqa: E402 V4CheckpointDescription, ) -def attention_config(checkpoint, settings): - additional = settings.additional_config() - additional["afd"]["role"] = "attention" +def attention_config(checkpoint, args): + additional = { + "afd": { + "connector": "CAMAsyncAFDConnector", + "role": "attention", + "async_dp": True, + "host": "127.0.0.1", + "port": args.port, + "num_attention_ranks": 1, + "num_ffn_ranks": args.ffn_ranks, + "compute_gate_on_attention": True, + "connector_extra_config": { + "dynamicQuant": 1, + "attn_ranks_per_dp": 1, + "hccl_buffer_size": 64, + }, + }, + "enable_prefill_mc2": False, + "enable_force_eplb": False, + "enable_cpu_binding": True, + "ascend_fusion_config": {"fusion_ops_gmmswigluquant": True}, + } return SimpleNamespace( additional_config=additional, model_config=SimpleNamespace( @@ -54,9 +72,7 @@ def attention_config(checkpoint, settings): model_type="deepseek_v4", ), ), - scheduler_config=SimpleNamespace( - max_num_batched_tokens=settings.max_num_batched_tokens - ), + scheduler_config=SimpleNamespace(max_num_batched_tokens=32), parallel_config=SimpleNamespace( enable_dbo=False, use_ubatching=False, @@ -89,17 +105,7 @@ def main(): ) args = parser.parse_args() checkpoint = V4CheckpointDescription.read(args.model) - settings = FFNConfig( - args.model, - attention_ranks=1, - attention_tp=1, - ffn_ranks=args.ffn_ranks, - port=args.port, - max_num_batched_tokens=32, - max_num_seqs=8, - hccl_buffer_size=64, - ) - config = attention_config(checkpoint, settings) + config = attention_config(checkpoint, args) args.output.parent.mkdir(parents=True, exist_ok=True) ensure_cam_async_ops_available() torch.npu.set_device(args.device) diff --git a/tests/unit/fakes/ffn_faas.py b/tests/unit/fakes/ffn_faas.py index 861f6e8e3..9c68a9232 100644 --- a/tests/unit/fakes/ffn_faas.py +++ b/tests/unit/fakes/ffn_faas.py @@ -6,9 +6,9 @@ import atexit import json -import os import sys import threading +from argparse import ArgumentParser, Namespace from contextlib import ExitStack from dataclasses import dataclass from pathlib import Path @@ -123,16 +123,16 @@ def close(self) -> None: class HostPlatformAdapter: """An independent CPU host contract; it has no model/device requirements.""" - def add_arguments(self, parser) -> None: - pass + def create_parser(self) -> ArgumentParser: + return ArgumentParser() def configure(self, args) -> FFNLaunch: return FFNLaunch( rank=3, local_rank=0, - model_path=str(args.model), + model_path=str(args.model_tag), backend_name="host-contract", - shutdown_timeout=args.shutdown_timeout, + shutdown_timeout=args.receiver_shutdown_timeout, backend_factory=HostBackend, ) @@ -151,7 +151,7 @@ def run_cpu_platform(model: str, status: str) -> int: "0", "--status-file", status, - "--shutdown-timeout", + "--receiver-shutdown-timeout", "0.1", ] with ( @@ -171,24 +171,12 @@ def run_cli_host_case(model: str, marker: str, fault: str, released: bool) -> in from afd_plugin.ffn_faas import __main__ as cli atexit.register(lambda: Path(marker).write_text("ordinary shutdown")) - os.environ.update( - RANK="0", - LOCAL_RANK="0", - WORLD_SIZE="1", - LOCAL_WORLD_SIZE="1", - VLLM_USE_V2_MODEL_RUNNER="1", - AFD_ASYNC_CAM_LAYERED_GMM="1", - ) args = [ "ffn_faas", "serve", model, - "--attention-ranks", - "1", - "--attention-tp", - "1", - "--ffn-ranks", - "1", + "--backend", + "host-contract", "--status-port", "0", ] @@ -244,7 +232,39 @@ def run(self): if not released and fault != "returned_unreleased": raise RuntimeError("host-only unreleased session") + patches.enter_context( + patch.object(cli, "SUPPORTED_BACKENDS", ("host-contract",)) + ) + patches.enter_context( + patch.object(cli, "get_platform_adapter", lambda _: HostPlatformAdapter()) + ) patches.enter_context(patch.object(cli, "FFNService", HostService)) patches.enter_context(patch.object(cli, "FFNModelRunner", HostRunner)) patches.enter_context(patch.object(sys, "argv", args)) return cli.main() + + +def cam_cli_args(model: Path, **overrides) -> Namespace: + """Parsed engine arguments for CPU-only standalone launch admission tests.""" + values = { + "model": str(model), + "model_tag": None, + "data_parallel_size": 2, + "data_parallel_rank": None, + "distributed_executor_backend": "external_launcher", + "receiver_shutdown_timeout": 5.0, + "additional_config": { + "enable_cpu_binding": False, + "afd": { + "connector": "CAMAsyncAFDConnector", + "role": "ffn", + "async_dp": True, + "compute_gate_on_attention": True, + "num_attention_ranks": 12, + "num_ffn_ranks": 2, + "connector_extra_config": {"dynamicQuant": 1, "attn_ranks_per_dp": 4}, + }, + }, + } + values.update(overrides) + return Namespace(**values) diff --git a/tests/unit/ffn_faas/test_main.py b/tests/unit/ffn_faas/test_main.py index f6219716b..07d483b4f 100644 --- a/tests/unit/ffn_faas/test_main.py +++ b/tests/unit/ffn_faas/test_main.py @@ -175,3 +175,101 @@ def test_cpu_entry_rejects_platform_imports( assert result.returncode == 1 assert f"CPU host contract imported {package}" in result.stderr assert "platform stub ran" not in result.stderr + + +@pytest.fixture +def engine_parser(): + pytest.importorskip("vllm") + from afd_plugin.ffn_faas.__main__ import create_parser + from afd_plugin.ffn_faas.platforms.npu.adapter import AscendCAMAdapter + + return create_parser(AscendCAMAdapter()) + + +@pytest.fixture(params=[False, True], ids=["eager", "full"]) +def native_engine_case(engine_parser, request): + from vllm.engine.arg_utils import EngineArgs + + graph = request.param + config = { + "enable_cpu_binding": False, + "afd": { + "connector": "CAMAsyncAFDConnector", + "role": "ffn", + "connector_extra_config": {"hccl_buffer_size": 256}, + }, + } + mode = ( + ["--compilation-config", '{"cudagraph_mode":"FULL"}'] + if graph + else ["--enforce-eager"] + ) + parsed = engine_parser.parse_args( + [ + "serve", + "/model", + "--dtype", + "bfloat16", + "--max-num-batched-tokens", + "2K", + "--max-num-seqs", + "8", + "--data-parallel-size", + "4", + "--device-ids", + "4,5,6,7", + "--additional-config", + json.dumps(config), + "--shutdown-timeout", + "0", + "--receiver-shutdown-timeout", + "0.5", + *mode, + ] + ) + return ( + graph, + config, + EngineArgs.from_cli_args(parsed), + parsed.receiver_shutdown_timeout, + ) + + +def test_native_engine_options_preserve_configuration(native_engine_case): + graph, config, args, receiver_timeout = native_engine_case + assert args.max_num_batched_tokens == 2048 + assert args.max_num_seqs == 8 + assert args.dtype == "bfloat16" and args.data_parallel_size == 4 + assert args.device_ids == [4, 5, 6, 7] + assert args.additional_config == config + assert args.shutdown_timeout == 0 and receiver_timeout == 0.5 + assert args.enforce_eager is (not graph) + if graph: + assert args.compilation_config.cudagraph_mode.name == "FULL" + + +def test_native_dotted_json_options(engine_parser): + from vllm.engine.arg_utils import EngineArgs + + parsed = engine_parser.parse_args( + [ + "serve", + "/model", + "--additional-config.afd.role", + "ffn", + "--additional-config.afd.port", + "29600", + "--compilation-config.cudagraph_mode", + "FULL", + ] + ) + args = EngineArgs.from_cli_args(parsed) + assert args.additional_config == {"afd": {"role": "ffn", "port": 29600}} + assert args.compilation_config.cudagraph_mode.name == "FULL" + + +@pytest.mark.parametrize("flag", ["--full-graph", "--attention-ranks", "--devices"]) +def test_retired_standalone_engine_flags_are_rejected(engine_parser, flag): + with pytest.raises(SystemExit) as error: + engine_parser.parse_args(["serve", "/model", flag]) + assert error.value.code == 2 diff --git a/tests/unit/ffn_faas/test_settings.py b/tests/unit/ffn_faas/test_settings.py index 2ffc8cc4e..942f7705d 100644 --- a/tests/unit/ffn_faas/test_settings.py +++ b/tests/unit/ffn_faas/test_settings.py @@ -9,60 +9,84 @@ import pytest -from afd_plugin.config import connector_extra_config_from_source, parse_afd_config +from afd_plugin.ffn_faas.platforms.npu.adapter import AscendCAMAdapter from afd_plugin.ffn_faas.platforms.npu.settings import ( FFNConfig, validate_native_expert_map, validate_static_experts, ) +from tests.unit.fakes.ffn_faas import cam_cli_args @pytest.fixture def settings(): - return FFNConfig(Path("/model"), attention_ranks=12, attention_tp=4, ffn_ranks=4) + return FFNConfig(Path("/model"), ffn_ranks=4) -def test_existing_cam_configuration_roundtrip(settings): - config = settings.additional_config() - afd = parse_afd_config(config, expected_role="ffn") - assert afd.connector == "CAMAsyncAFDConnector" - assert afd.async_dp and afd.compute_gate_on_attention - assert (afd.num_attention_ranks, afd.num_ffn_ranks) == (12, 4) - assert connector_extra_config_from_source(config) == { +@pytest.fixture +def launcher(monkeypatch): + monkeypatch.setenv("WORLD_SIZE", "2") + monkeypatch.setenv("RANK", "1") + monkeypatch.setenv("LOCAL_RANK", "1") + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + + +@pytest.mark.parametrize("positional", [False, True]) +def test_engine_configuration_keeps_user_values(checkpoint_dir, launcher, positional): + args = cam_cli_args(checkpoint_dir) + if positional: + args.model = "/unused-option-model" + args.model_tag = str(checkpoint_dir) + launch = AscendCAMAdapter().configure(args) + assert (launch.rank, launch.local_rank) == (1, 1) + assert launch.model_path == str(checkpoint_dir) + assert args.model == str(checkpoint_dir) + assert args.data_parallel_rank == 1 + assert args.additional_config["enable_cpu_binding"] is False + assert args.additional_config["afd"]["connector_extra_config"] == { "dynamicQuant": 1, "attn_ranks_per_dp": 4, } -def test_explicit_native_hccl_budget(settings): - settings = replace(settings, hccl_buffer_size=256) - assert connector_extra_config_from_source(settings.additional_config()) == { - "dynamicQuant": 1, - "attn_ranks_per_dp": 4, - "hccl_buffer_size": 256, - } +@pytest.mark.parametrize( + "change", + [ + {"data_parallel_size": 1}, + {"data_parallel_rank": 0}, + {"distributed_executor_backend": "mp"}, + {"receiver_shutdown_timeout": 0}, + ], +) +def test_launcher_rejects_inconsistent_engine_arguments( + checkpoint_dir, launcher, change +): + with pytest.raises(ValueError): + AscendCAMAdapter().configure(cam_cli_args(checkpoint_dir, **change)) + + +@pytest.mark.parametrize( + "field,value", [("port", 0), ("role", "attention"), ("num_ffn_ranks", 0)] +) +def test_cli_uses_existing_afd_validation(checkpoint_dir, launcher, field, value): + args = cam_cli_args(checkpoint_dir) + args.additional_config["afd"][field] = value + with pytest.raises(ValueError): + AscendCAMAdapter().configure(args) @pytest.mark.parametrize( "change", [ - {"attention_ranks": 0}, - {"attention_ranks": 11}, {"ffn_ranks": 0}, - {"attention_tp": 0}, - {"max_num_batched_tokens": 2047}, {"rank": 4}, {"local_rank": -1}, - {"device": -1}, - {"port": 0}, - {"port": 65536}, {"shutdown_timeout": 0}, {"shutdown_timeout": float("nan")}, {"shutdown_timeout": float("inf")}, - {"hccl_buffer_size": 0}, ], ) -def test_invalid_launch_rejected_before_native_runtime(settings, change): +def test_invalid_host_launch_rejected(settings, change): with pytest.raises(ValueError): replace(settings, **change) diff --git a/tests/unit/ffn_faas/test_v4_model.py b/tests/unit/ffn_faas/test_v4_model.py index dd62da738..b40fef922 100644 --- a/tests/unit/ffn_faas/test_v4_model.py +++ b/tests/unit/ffn_faas/test_v4_model.py @@ -17,6 +17,7 @@ V4CheckpointDescription, V4ModelAdapter, ) +from tests.unit.fakes.vllm_config import npu_vllm_config def test_main_routed_metadata_without_parameter_allocation(checkpoint_dir): @@ -91,16 +92,16 @@ def test_model_selection_rejects_other_quantization_before_loading(checkpoint_di def test_wrong_compute_world_rejected_before_native_imports( checkpoint_dir, monkeypatch ): - settings = FFNConfig(checkpoint_dir, attention_ranks=1, attention_tp=1, ffn_ranks=2) + settings = FFNConfig(checkpoint_dir, ffn_ranks=2) environment = AscendFFNEnvironment(settings) monkeypatch.setenv("RANK", "0") monkeypatch.setenv("WORLD_SIZE", "1") with pytest.raises(ValueError, match="differs from the F-only"): - environment.initialize_device() + environment.initialize_device(npu_vllm_config(role="ffn")) def test_unused_compute_owner_can_close_without_importing_runtime(checkpoint_dir): - settings = FFNConfig(checkpoint_dir, attention_ranks=1, attention_tp=1, ffn_ranks=1) + settings = FFNConfig(checkpoint_dir, ffn_ranks=1) environment = AscendFFNEnvironment(settings) environment.close() with pytest.raises(RuntimeError, match="device has not been initialized"): From 21f8ae7e1b2eb4b31bdff7d49dd98199a29f5dca Mon Sep 17 00:00:00 2001 From: Li Ruixin <616517220@qq.com> Date: Fri, 6 Nov 2026 01:35:52 -0800 Subject: [PATCH 10/42] Refactor: clarify Async CAM session ownership (#551) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Purpose Separate Async CAM Push and Pull session ownership so each protocol keeps its transaction state and lifecycle rules together. The connector retains shared payload validation, HCCL ownership and startup coordination. ## Issue - Related issue(s): Not linked. - Closing keyword, only if fully resolved: None. ## Scope - In scope: The common FFN session base, Async CAM connector and sessions, associated tests and fakes, the device harness, and connector/runtime documentation. - Out of scope: Model and expert-weight interfaces, Layered/Mega compute implementations, runner execution orchestration, static validation consolidation, native kernels and ABI, and changes to supported configurations or defaults. ## Implementation Notes - Rename the Pull module to `async_cam_session.py` and keep `PushCAMSession` and `PullCAMSession` together. The connector selects one session at construction. - Push owns a FIFO per Attention stage, routing references and its placeholder. It records routing after a successful dispatch and removes it after a successful combine. FFN retains separate receive, compute and send operations. - Pull owns one in-flight window across stages, native publish/combine, status/idle/source state and the quiet/drain/stopped handshake. It receives Startup, topology and required dimensions explicitly, without retaining the connector. Ownership is recorded before dispatch; failures retain the window and routing references. DBO checks sticky status at the Attention step boundary. - Merge the lifecycle interface and default implementation into one concrete `FFNExecutionSession`. Push and Pull inherit it; remove `DefaultFFNSession` and duplicate default hooks. Custom sessions should inherit `FFNExecutionSession` and override the required hooks. Keep torch and metadata imports in the base module under type checking so session imports remain safe before device bootstrap. - Initialize sessions after negotiation and HCCL setup. Broadcast initialization failures before marking the connector ready. Check session shutdown requirements before destroying HCCL and releasing references. Read-only `comm_args` and `comm_id` expose the live session resources; `AFDAsyncTransferState` retains its existing definition path. - Mega FFN continues to use its fused transaction. The connector rejects separate FFN receive/send calls for Pull. Eager/FULL execution, graph references, service threads and device completion remain with the runner and worker. - Share native argument preparation across transfer methods and use registered schema keyword names. Argument values and ABI remain unchanged. Diagnostic input fields now follow the native schema and continue to report tensor shape, dtype and device without metadata reads. - Move host message-contract coverage into CPU-safe tests and shared recording fakes. Cover stage isolation, Pull exclusivity, combine failure retention, unsupported Pull entry points, closed-connector rejection and startup failure propagation. - Use the public pending-state observation in the device harness. Apply the whole-expert receive-capacity assertion only to Push; baseline and candidate use the same harness correction. ## Test Plan - [x] Full CPU unit suite, recording missing-dependency skips separately. - [x] Changed-file pre-commit checks, including Ruff, mypy 3.11, Markdown lint, SPDX and whitespace checks. - [x] Related units with actual torch, vLLM and vLLM-Ascend dependencies on A3. - [x] Matched W4A8/W8A8, Layered/Mega and eager/FULL device transactions, including TP2, capacity boundaries, empty work, graph replay, Pull idle/drain and resource cleanup. - [x] Matched real-model concurrent requests with active Attention DBO. ## Test Result Results cover base `28fcfb706cb505cac9550e9e76fbfc909214891d` and the current patch. - CPU suite with isolated setuptools 80.10.2: **642 passed, 91 skipped, 8 deselected, 10 subtests passed**. Skips require unavailable torch, vLLM, vLLM-Ascend, numpy or cmake. One environment warning reports Babel's deprecated `cgi` import. - Changed-file pre-commit, Ruff, mypy 3.11, Markdown lint and `git diff --check`: passed. - A3 related units: **209 passed, 17 warnings** from dependency deprecations. - Finite device transactions: **16 matched baseline/candidate runs passed** across W4A8/W8A8, Layered/Mega and eager/FULL. All **80 candidate output tensors exactly matched baseline eager outputs**. Graph capture/replay passed, including eight candidate captures; all ranks completed cleanup, and Mega idle/shutdown passed. Configuration: 2A/2F, Attention TP2, synthetic hidden/intermediate dimensions 256, layers (2, 5), seed 1024, Push receive capacity 32 and Pull capacity 16. Eight baseline runs were reused after verifying unchanged baseline source, native artifacts and fixed inputs; all eight final candidate runs were rerun. - Real model: DeepSeek-V4 Flash W4A8, Attention DP2×TP4, FFN EP8×TP1 Layered eager, token budget/receive capacity 8192 and seed 1024. Baseline and candidate each passed the same **10 concurrent arithmetic requests**, with identical prompts and answers and **E2E exit code 0**. All eight Attention ranks recorded **four nonempty two-stage executions**. DBO prefill/decode thresholds were 2, temperature was 0 and maximum generation length was 256 tokens. The test-only observer enables existing execution evidence without changing production execution or answer assertions. - Cleanup: no NPU processes remained after the model run. The model harness uses process termination; both baseline and candidate emitted two `KeyboardInterrupt` lines and two resource-tracker warnings. Exit code 0 does not establish graceful shutdown of every FFN process. - Environment: CANN 9.2.0, vLLM `0.32.0+empty` at `5df63a7203cdb9328d824a27b682e77c7b9928be`, vLLM-Ascend `0.19.1rc2.dev3100+gbd42eea01` at `bd42eea01d8312c4df469fabaed45f77a12df4d3`, torch `2.10.0+cpu` and torch-npu `2.10.0.post7`. All 47 reused native artifacts matched their recorded hashes. No upstream checkout was modified. CUDA validation, full GSM8K evaluation, performance qualification and a fresh native build were not run. Synthetic transactions and concurrent arithmetic checks do not establish full model accuracy or performance. ## Docs Impact - Files updated: Connector contracts, execution platforms, FFN runtime, Async CAM user/runtime guides. - If none, reason: Not applicable. - Document session ownership, lifecycle defaults, live resource access, shutdown requirements and the validation scope. Register the new host tests in the owning modules' validation paths. ---
Essential PR Checklist - [x] Purpose is clear and linked to public context when possible. - [x] Scope is bounded. - [x] Compatibility with vLLM v0.32.0 and, for NPU changes, the pinned vLLM-Ascend source is considered, as applicable. - [x] No changes are made to the vLLM source checkout. - [x] Plugin-owned classes or explicit dotted class paths are preferred over monkey patches. - [x] Any compat shim or monkey patch is isolated, idempotent, version-guarded, documented, and tested. No production shim or monkey patch is added. - [x] Imports remain CPU-safe; CUDA-heavy work is delayed or GPU-gated. - [x] Validation evidence is included, including skipped GPU tests when applicable. - [x] Documentation impact is stated.
Signed-off-by: lirx-pd <616517220@qq.com> --- afd_plugin/connectors/base.py | 81 +-- afd_plugin/connectors/npu/async_cam.py | 555 ++++++------------ .../npu/async_cam_runtime/__init__.py | 2 +- .../npu/async_cam_runtime/async_cam_pull.py | 77 --- .../async_cam_runtime/async_cam_session.py | 277 +++++++++ docs/design/module/connector_contracts.md | 34 +- docs/design/module/execution_platforms.md | 2 +- docs/design/module/ffn_runtime.md | 19 +- docs/npu/ASYNC_FFN_RUNTIME.md | 11 +- docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md | 9 + tests/npu/async_cam_runtime.py | 5 +- tests/unit/connectors/npu/test_async_cam.py | 317 ++++++++++ ..._cam_pull.py => test_async_cam_session.py} | 65 +- .../connectors/npu/test_async_cam_startup.py | 7 +- .../npu/test_async_cam_startup_wiring.py | 34 +- .../connectors/test_async_cam_connector.py | 379 +----------- tests/unit/connectors/test_base.py | 19 + tests/unit/fakes/async_cam.py | 234 ++++++++ tests/unit/fakes/async_ffn.py | 3 +- tests/unit/fakes/connector.py | 9 +- tests/unit/v1/worker/test_async_ffn_graph.py | 4 +- tests/unit/v1/worker/test_npu_runtime.py | 4 +- 22 files changed, 1200 insertions(+), 947 deletions(-) delete mode 100644 afd_plugin/connectors/npu/async_cam_runtime/async_cam_pull.py create mode 100644 afd_plugin/connectors/npu/async_cam_runtime/async_cam_session.py create mode 100644 tests/unit/connectors/npu/test_async_cam.py rename tests/unit/connectors/npu/{test_async_cam_pull.py => test_async_cam_session.py} (76%) create mode 100644 tests/unit/connectors/test_base.py create mode 100644 tests/unit/fakes/async_cam.py diff --git a/afd_plugin/connectors/base.py b/afd_plugin/connectors/base.py index 071beaa38..4bf86a7e2 100644 --- a/afd_plugin/connectors/base.py +++ b/afd_plugin/connectors/base.py @@ -8,22 +8,22 @@ from collections.abc import Callable, Mapping, Sequence from contextlib import AbstractContextManager, nullcontext from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Protocol - -import torch +from typing import TYPE_CHECKING, Any from afd_plugin.config import AFDConfig, connector_extra_config_from_source -from afd_plugin.connectors.metadata import ( - AFDA2FTransferPayload, - AFDControlPayload, - AFDTransferContext, - FFNLocalExperts, - LocalExpertWeights, -) if TYPE_CHECKING: + import torch from vllm.config import VllmConfig + from afd_plugin.connectors.metadata import ( + AFDA2FTransferPayload, + AFDControlPayload, + AFDTransferContext, + FFNLocalExperts, + LocalExpertWeights, + ) + @dataclass(frozen=True) class ConnectorExtraInfo: @@ -37,73 +37,35 @@ def to_mapping(self) -> dict[str, Any]: return {} -class FFNExecutionSession(Protocol): +class FFNExecutionSession: """Backend lifecycle used by FFN runners and workers. - Workers stop serving before draining and finishing the receiver, then - validate shutdown before releasing communication and model resources. - ``requires_drain`` marks backends that must retain resources until their - pending work and receiver handshake have completed. + By default, serving stops on local shutdown and the remaining hooks do + nothing. Backends with remote producers override the hooks and set + ``requires_drain``; pending work and the receiver handshake must finish + before communication or model resources are released. """ - requires_drain: bool + requires_drain = False def keep_serving(self, shutdown_requested: bool) -> bool: - """Whether to serve another step given the local shutdown_requested flag. - - A backend may keep serving after local shutdown until remote producers - are quiet. - """ - ... + """Continue until local shutdown; remote producers may extend serving.""" + return not shutdown_requested def before_first_step(self) -> None: """Prepare backend state once before the first FFN step.""" - ... def after_step(self) -> None: """Check backend status after a completed FFN step.""" - ... def drain(self, run_one: Callable[[], None]) -> None: - """Drain pending work using run_one to execute and wait for a transaction.""" - ... + """Drain remote work when overridden; by default do not invoke run_one.""" def finish_receiver(self) -> None: """Finish the receiver handshake after pending work has drained.""" - ... - - def validate_shutdown(self) -> None: - """Raise RuntimeError if resources cannot yet be released safely.""" - ... - - -class DefaultFFNSession: - """Lifecycle for connectors that need no drain or receiver handshake. - - Service stops when ``shutdown_requested`` is true. Other hooks are no-ops; - in particular, ``drain`` never invokes its ``run_one`` callback. Workers - and runners still own device synchronization and resource release. - """ - - requires_drain = False - - def keep_serving(self, shutdown_requested: bool) -> bool: - return not shutdown_requested - - def before_first_step(self) -> None: - pass - - def after_step(self) -> None: - pass - - def drain(self, run_one: Callable[[], None]) -> None: - pass - - def finish_receiver(self) -> None: - pass def validate_shutdown(self) -> None: - pass + """Reject release when backend resources are still in use.""" class AFDConnectorBase(ABC): @@ -191,7 +153,7 @@ def attention_step(self) -> AbstractContextManager[None]: return nullcontext() def ffn_session(self) -> FFNExecutionSession: - return DefaultFFNSession() + return FFNExecutionSession() @abstractmethod def close(self) -> None: @@ -459,6 +421,5 @@ def recv_dp_metadata_list(self) -> AFDControlPayload: "AFDControlPlaneClosedError", "AFDRoutedExpertsConnector", "ConnectorExtraInfo", - "DefaultFFNSession", "FFNExecutionSession", ] diff --git a/afd_plugin/connectors/npu/async_cam.py b/afd_plugin/connectors/npu/async_cam.py index 3a430ef6f..60a504189 100644 --- a/afd_plugin/connectors/npu/async_cam.py +++ b/afd_plugin/connectors/npu/async_cam.py @@ -1,36 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project -"""Ascend CAM asynchronous connector for Attention/FFN disaggregation. - -``CAMAsyncAFDConnector`` supports eager Attention and eager or FULL FFN. -Attention ranks run MoE routing, submit activations with CAM async -dispatch-send, and receive combined expert output with combine-recv. FFN ranks -receive routed-expert activations with dispatch-recv, execute their -local experts, and return the results with combine-send. - -The connector creates one HCCL world ordered as -``[A0, A1, ..., F0, F1, ...]``. Attention world ranks therefore equal their -role ranks, while FFN world ranks start at ``num_attention_ranks``. Unlike the -synchronous connectors, CAM async carries routing and token-count metadata in -the CAM operator payload and does not create a separate Gloo DP-metadata -control plane. - -The supported deployment requires ``async=true``, Ascend CAM -operator packages, and matching topology/configuration on every rank. Regular -prefill and autoregressive decode steps use the same connector. Attention -microbatching uses the native DBO runtime with Ascend adaptations. FFN FULL -requires MRV2 and accepts Attention-side DBO. Native DBO owns microbatch -splitting and scheduling. See ``docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md`` for -configuration, rank -derivation, launch guidance, and the full limitations. +"""Ascend CAM connector for Attention/FFN disaggregation. + +The connector validates payloads and manages the Attention-first HCCL world +``[A0, A1, ..., F0, F1, ...]``. Push/Pull sessions own transfer state and +protocol rules; the FFN ModelRunner owns eager/FULL execution. Attention +microbatching uses native DBO. Native operators carry routing/count metadata +without a separate Gloo control plane. + +See ``docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md`` for configuration, rank +derivation and supported paths. """ from __future__ import annotations import os import time -from collections.abc import Iterator, Mapping -from contextlib import AbstractContextManager, contextmanager, nullcontext +from collections.abc import Mapping +from contextlib import AbstractContextManager from dataclasses import dataclass from datetime import timedelta from typing import TYPE_CHECKING, Any, Final, cast @@ -50,7 +37,6 @@ from afd_plugin.connectors.base import ( AFDConnectorBase, ConnectorExtraInfo, - DefaultFFNSession, FFNExecutionSession, ) from afd_plugin.connectors.metadata import ( @@ -59,7 +45,11 @@ AFDTransferMetadata, AFDTransferState, ) -from afd_plugin.connectors.npu.async_cam_runtime.async_cam_pull import PullFFNSession +from afd_plugin.connectors.npu.async_cam_runtime.async_cam_session import ( + CAM_COMM_ID, + PullCAMSession, + PushCAMSession, +) from afd_plugin.connectors.npu.async_cam_runtime.async_cam_startup import ( GRAPH_STARTUP_TIMEOUT_SECONDS, POLL_SECONDS, @@ -77,13 +67,9 @@ from vllm.config import VllmConfig from vllm.model_executor.layers.fused_moe import FusedMoERouter +logger = init_logger(__name__) + AFD_ASYNC_CAM_GROUP_NAME = "afd_async_cam" -CAM_COMM_ID = 0 -PULL_COMM_ID = 0x41464401 -PULL_FAILURE_INDEX = 24 -PULL_SOURCE_INDEX = 32 -PULL_IDLE_BUDGET_INDEX = 64 -PULL_IDLE_SCANS = 8 ATTN_RANKS_PER_DP_CONFIG_KEY = "attn_ranks_per_dp" _AFD_ASYNC_EXTRA_CONFIG_FIELDS: Final[frozenset[str]] = frozenset( @@ -95,8 +81,6 @@ }, ) -logger = init_logger(__name__) - def select_cam_experts( *, @@ -215,20 +199,6 @@ def to_mapping(self) -> dict[str, Any]: return result -@dataclass(slots=True) -class AFDAsyncTransferState(AFDTransferState): - """Routed activations, counts and untouched compact combine metadata.""" - - batch_size: int = 1 - hidden_size: int = 1 - topk: int = 1 - layer_idx: int = 0 - token_nums_rankid_layeridx: Tensor | None = None - group_list: Tensor | None = None - dynamic_scales: Tensor | None = None - send_completion: Tensor | None = None - - @dataclass(frozen=True, slots=True) class AFDAsyncTopology: """Role-local and HCCL-world rank information for one CAM participant.""" @@ -246,6 +216,20 @@ def world_size(self) -> int: return self.attn_size + self.ffn_size +@dataclass(slots=True) +class AFDAsyncTransferState(AFDTransferState): + """Routed activations, counts and untouched compact combine metadata.""" + + batch_size: int = 1 + hidden_size: int = 1 + topk: int = 1 + layer_idx: int = 0 + token_nums_rankid_layeridx: Tensor | None = None + group_list: Tensor | None = None + dynamic_scales: Tensor | None = None + send_completion: Tensor | None = None + + class CAMAsyncAFDConnector(AFDConnectorBase): """CAM-backed asynchronous connector for Ascend NPU AFD. @@ -295,7 +279,6 @@ def __init__( self.max_num_batched_tokens = ( vllm_config.scheduler_config.max_num_batched_tokens ) - self.comm_id = CAM_COMM_ID self.tp_size = extra_info.attn_ranks_per_dp self.cam_pg: ProcessGroup | None = None self.topology = build_async_topology( @@ -307,20 +290,35 @@ def __init__( self.attn_size = self.topology.attn_size self.ffn_size = self.topology.ffn_size self.expert_per_rank = self.topology.expert_per_rank - self.comm_args: Tensor | None = None - self._placeholder: Tensor | None = None - self._defer_pull_status_check = False - self._pending_attention_payloads: dict[ - int, - list[tuple[AFDTransferContext, Tensor, Tensor]], - ] = {} self.graph_startup = AsyncCAMGraphStartup( self.world_rank, self.attn_size, self.topology.world_size ) - if extra_info.ffn_mega_moe: - self._session: FFNExecutionSession = PullFFNSession(self) - else: - self._session = DefaultFFNSession() + self._session: PushCAMSession | PullCAMSession = ( + PullCAMSession( + self.graph_startup, + self.topology, + hidden_size=self.hidden_size, + topk=self.topk, + max_num_batched_tokens=self.max_num_batched_tokens, + tp_size=self.tp_size, + receive_capacity=cam_receive_capacity(), + ) + if extra_info.ffn_mega_moe + else PushCAMSession(self.activation_dtype) + ) + + @property + def comm_args(self) -> Tensor | None: + """Expose the selected protocol's live window to Mega compute.""" + return self._session.comm_args + + @property + def comm_id(self) -> int: + return self._session.comm_id + + @property + def has_pending_attention(self) -> bool: + return self._session.has_pending_attention @property def is_initialized(self) -> bool: @@ -352,39 +350,11 @@ def init_afd_connector(self) -> None: ) backend = self.cam_pg._get_backend(torch.device("npu")) self.group_name = str(backend.get_hccl_comm_name(self.world_rank)) - device = f"npu:{self.local_rank}" - if self.extra_info.ffn_mega_moe: - self.comm_id = PULL_COMM_ID - self.comm_args = torch.ops.afd_ascend.afd_mega_pull_init( - torch.empty(1, dtype=torch.bfloat16, device=device), - [ - self.max_num_batched_tokens // self.tp_size, - self.hidden_size, - self.topk, - self.ffn_size, - self.attn_size, - self.expert_per_rank, - self.world_rank, - self.tp_size, - ], - list(self.graph_startup.plan.layer_ids), - self.graph_startup.session_epoch, - self.group_name, - ) - try: - torch.npu.synchronize() - self._check_pull_status() - self.graph_startup.phase("pull-initialized") - except Exception as exc: - self.graph_startup.fail(exc) - raise - else: - self.comm_args = torch.empty((1,), dtype=torch.float16, device=device) - self._placeholder = torch.empty( - (1,), - dtype=self.activation_dtype, - device=device, - ) + try: + self._session.initialize(f"npu:{self.local_rank}", self.group_name) + except Exception as exc: + self.graph_startup.fail(exc) + raise self._initialized = True if self.graph_startup.mode == "full" and self.world_rank < self.attn_size: self._run_attention_graph_startup() @@ -403,8 +373,7 @@ def _initialize_graph_startup(self, store: Store) -> None: self.max_num_batched_tokens, ) ) - if self.extra_info.ffn_mega_moe: - signature = "pull_v1:" + signature + signature = self._session.signature_prefix + signature self.graph_startup.initialize(store, signature) if self.graph_startup.mode == "full": try: @@ -414,7 +383,7 @@ def _initialize_graph_startup(self, store: Store) -> None: self.max_num_batched_tokens, self.tp_size, cam_receive_capacity(), - allow_row_slicing=self.extra_info.ffn_mega_moe, + allow_row_slicing=self._session.allow_row_slicing, ) self.graph_startup.phase("validated") except Exception as exc: @@ -431,7 +400,7 @@ def wait_graph_device(self, event: torch.npu.Event) -> None: def graph_startup_chunks(self) -> int: """Every A TP rank sends one row to experts [0, topk).""" - if self.extra_info.ffn_mega_moe: + if isinstance(self._session, PullCAMSession): # A fused launch drains every receive round of the selected source. return 1 local_experts = max( @@ -474,30 +443,17 @@ def ffn_session(self) -> FFNExecutionSession: return self._session def attention_step(self) -> AbstractContextManager[None]: - if self.extra_info.ffn_mega_moe: - return self._defer_pull_status() - return nullcontext() + return self._session.attention_step() def close(self) -> None: """Destroy the HCCL process group and clear pending transfer states.""" - if isinstance(self._session, PullFFNSession): - if self._pending_attention_payloads: - raise RuntimeError( - "Pull dispatch is pending; retain communication resources" - ) - if self._initialized: - if self.world_rank < self.attn_size: - self._session.close_attention() - else: - self._session.validate_shutdown() + self._session.prepare_close(self._initialized) if self.cam_pg is not None: import torch.distributed as dist dist.destroy_process_group(self.cam_pg) self.cam_pg = None - self.comm_args = None - self._placeholder = None - self._pending_attention_payloads.clear() + self._session.release() self._initialized = False def select_experts(self, **kwargs: Any) -> tuple[Tensor, Tensor]: @@ -510,12 +466,7 @@ def send_attn_output( context: AFDTransferContext, **kwargs: Any, ) -> None: - """Dispatch Attention activations and top-k routing to FFN ranks. - - The matching context and routing tensors are queued per async stage - for ``recv_ffn_output``. The input token count and top-k tensor shapes - must match the transfer metadata and model configuration. - """ + """Validate routed activations, then submit through the selected protocol.""" self._require_initialized() metadata = context.metadata if not metadata.validate_tensor_shape(tuple(hidden_states.shape)): @@ -549,67 +500,18 @@ def send_attn_output( "Async CAM batch exceeds max_num_batched_tokens / TP capacity" ) hidden_states = hidden_states.contiguous() - pull = self.extra_info.ffn_mega_moe - if pull: - if self._pending_attention_payloads: - raise RuntimeError( - "Pull window is pending; finish Combine before Dispatch" - ) - # A launch can enqueue work before raising; retain ownership on failure. - self._pending_attention_payloads[metadata.stage_idx] = [ - (context, topk_ids, topk_weights) - ] - - _log_cam_op_values( - "async_dispatch_send", - "inputs", - hidden_states=hidden_states, - topk_ids=topk_ids, - comm_args=self.comm_args, - comm_id=self.comm_id, + args = self._cam_args(states) | dict( + x=hidden_states, + expert_ids=topk_ids, max_seq_len=self.max_num_batched_tokens, batch_size=states.batch_size, - hidden_size=states.hidden_size, - topk=states.topk, - ffn_size=self.ffn_size, - attn_size=self.attn_size, - expert_per_rank=self.expert_per_rank, - rank=self.world_rank, - world_size=self.topology.world_size, - layer_idx=states.layer_idx, + attn_rank_id=self.world_rank, + layer_index=states.layer_idx, tp_size=self.tp_size, dynamic_quant=self.dynamic_quant, - group_name=self.group_name, - ) - ( - torch.ops.afd_ascend.afd_mega_dispatch_send - if self.extra_info.ffn_mega_moe - else torch.ops.afd_ascend.afd_async_dispatch_send - )( - hidden_states, - topk_ids, - self.comm_args, - self.comm_id, - self.max_num_batched_tokens, - states.batch_size, - states.hidden_size, - states.topk, - self.ffn_size, - self.attn_size, - self.expert_per_rank, - self.world_rank, - self.topology.world_size, - states.layer_idx, - self.tp_size, - self.dynamic_quant, - self.group_name, ) - if not pull: - self._pending_attention_payloads.setdefault(metadata.stage_idx, []).append( - (context, topk_ids, topk_weights), - ) - - return None + _log_cam_op_values("async_dispatch_send", "inputs", **args) + self._session.dispatch((context, topk_ids, topk_weights), **args) def recv_ffn_output( self, @@ -617,34 +519,21 @@ def recv_ffn_output( ubatch_idx: int = 0, **kwargs: Any, ) -> Tensor: - """Receive and combine expert output on an Attention rank. - - The optional context must match the oldest pending dispatch for the - stage. Routing tensors always come from that dispatch. ``ref_tensor`` - selects the output device and dtype for the CAM receive placeholder. - - Args: - ref_tensor: Tensor from which device/dtype are taken for the CAM - combine-recv placeholder. - ubatch_idx: Stage/microbatch index of the pending Attention - dispatch payload. Defaults to ``0``. - **kwargs: Optional ``context`` from the matching dispatch. Uses - FIFO order when omitted. - """ + """Combine the oldest matching Push dispatch or the exclusive Pull window.""" self._require_initialized() context = kwargs.get("context") - payloads = self._pending_attention_payloads.get(ubatch_idx) - if not payloads: + payload = self._session.pending_for(ubatch_idx) + if payload is None: raise RuntimeError( "CAMAsyncAFDConnector recv_ffn_output is missing pending " "Attention metadata", ) - if context is not None and context is not payloads[0][0]: + if context is not None and context is not payload[0]: raise RuntimeError( "CAMAsyncAFDConnector recv_ffn_output context does not match " "the next pending Attention dispatch", ) - context, topk_ids, topk_weights = payloads[0] + context, topk_ids, topk_weights = payload states = _require_async_transfer_state(context) _validate_topk_payload( @@ -654,92 +543,25 @@ def recv_ffn_output( topk=states.topk, ) placeholder = ref_tensor.new_empty((1,)) - _log_cam_op_values( - "async_combine_recv", - "inputs", - placeholder=placeholder, - topk_ids=topk_ids, - topk_weights=topk_weights, - comm_args=self.comm_args, - comm_id=self.comm_id, + args = self._cam_args(states) | dict( + expand_x=placeholder, + expert_ids=topk_ids, + expert_scales=topk_weights, batch_size=states.batch_size, - hidden_size=states.hidden_size, - topk=states.topk, - ffn_size=self.ffn_size, - attn_size=self.attn_size, - expert_per_rank=self.expert_per_rank, - rank=self.world_rank, - world_size=self.topology.world_size, - group_name=self.group_name, - ) - output = ( - torch.ops.afd_ascend.afd_mega_combine_recv - if self.extra_info.ffn_mega_moe - else torch.ops.afd_ascend.afd_async_combine_recv - )( - placeholder, - topk_ids, - topk_weights, - self.comm_args, - self.comm_id, - states.batch_size, - states.hidden_size, - states.topk, - self.ffn_size, - self.attn_size, - self.expert_per_rank, - self.world_rank, - self.topology.world_size, - self.group_name, - *( - (self.max_num_batched_tokens, self.tp_size) - if self.extra_info.ffn_mega_moe - else () - ), + attn_rank_id=self.world_rank, ) + _log_cam_op_values("async_combine_recv", "inputs", **args) + output = self._session.combine(ubatch_idx, **args) _log_cam_op_values("async_combine_recv", "outputs", output=output) - if not self._defer_pull_status_check: - self._check_pull_status() - payloads.pop(0) - if not payloads: - self._pending_attention_payloads.pop(ubatch_idx, None) return output - @contextmanager - def _defer_pull_status(self) -> Iterator[None]: - """Check sticky device errors before the model step returns, not per MoE.""" - previous = self._defer_pull_status_check - self._defer_pull_status_check = True - try: - yield - if not previous: - self._check_pull_status() - finally: - self._defer_pull_status_check = previous - - def _check_pull_status(self) -> None: - if self.extra_info.ffn_mega_moe: - assert self.comm_args is not None - # The model runner batches this fence outside the DBO receive hook. - status = int(self.comm_args.view(torch.int64)[PULL_FAILURE_INDEX].item()) - if status: - raise RuntimeError(f"AFD pull transaction failed with status {status}") - - def _enable_pull_idle_returns(self) -> None: - assert self.comm_args is not None - self.comm_args.view(torch.int64)[PULL_IDLE_BUDGET_INDEX].fill_(PULL_IDLE_SCANS) - - def _pull_has_active_source(self) -> bool: - assert self.comm_args is not None - return int(self.comm_args.view(torch.int64)[PULL_SOURCE_INDEX].item()) >= 0 - def recv_attn_output( self, ubatch_idx: int = 0, **kwargs: Any, ) -> AFDA2FTransferPayload: - """Receive routed activations and preserve their compact metadata.""" - self._require_initialized() + """Receive a Push work item and preserve its compact return metadata.""" + session = self._require_push_session() batch_size = int(kwargs.get("batch_size", self.max_num_batched_tokens) or 1) layer_idx = int(kwargs.get("layer_idx", 0) or 0) metadata = AFDTransferMetadata.create_ffn_metadata( @@ -757,42 +579,16 @@ def recv_attn_output( metadata=metadata, states=states, ) - placeholder = kwargs.get("placeholder", self._placeholder) - _log_cam_op_values( - "async_dispatch_recv", - "inputs", - placeholder=placeholder, - comm_args=self.comm_args, - comm_id=self.comm_id, + placeholder = kwargs.get("placeholder", session.placeholder) + args = self._cam_args(states) | dict( + x=placeholder, max_seq_len=self.max_num_batched_tokens, - batch_size=states.batch_size, - hidden_size=states.hidden_size, - topk=states.topk, - ffn_size=self.ffn_size, - attn_size=self.attn_size, - expert_per_rank=self.expert_per_rank, - rank=self.world_rank, - world_size=self.topology.world_size, + moe_rank_id=self.world_rank, tp_size=self.tp_size, dynamic_quant=self.dynamic_quant, - group_name=self.group_name, - ) - outputs = torch.ops.afd_ascend.afd_async_dispatch_recv( - placeholder, - self.comm_args, - self.comm_id, - self.max_num_batched_tokens, - states.hidden_size, - states.topk, - self.ffn_size, - self.attn_size, - self.expert_per_rank, - self.world_rank, - self.topology.world_size, - self.tp_size, - self.dynamic_quant, - self.group_name, ) + _log_cam_op_values("async_dispatch_recv", "inputs", **args) + outputs = torch.ops.afd_ascend.afd_async_dispatch_recv(**args) hidden_states, dynamic_scales, batch_info, expert_token_nums = outputs _log_cam_op_values( "async_dispatch_recv", @@ -816,12 +612,8 @@ def send_ffn_output( context: AFDTransferContext, **kwargs: Any, ) -> None: - """Send routed expert outputs for CAM combination. - - ``TokenNums_Rankid_Layeridx`` from the matching dispatch-recv is - mandatory because CAM uses it to return results to Attention ranks. - """ - self._require_initialized() + """Return a Push work item with its original dispatch metadata.""" + self._require_push_session() states = _require_async_transfer_state(context) token_nums_rankid_layeridx = states.token_nums_rankid_layeridx if token_nums_rankid_layeridx is None: @@ -831,95 +623,45 @@ def send_ffn_output( "AFD async CAM combine send requires " "TokenNums_Rankid_Layeridx from async_dispatch_recv", ) - _log_cam_op_values( - "async_combine_send", - "inputs", - ffn_output=ffn_output, + args = self._cam_args(states) | dict( + expand_x=ffn_output, + batch_info=token_nums_rankid_layeridx, + max_seq_len=self.max_num_batched_tokens, + moe_rank_id=self.world_rank, + tp_size=self.tp_size, + ) + _log_cam_op_values("async_combine_send", "inputs", **args) + states.send_completion = torch.ops.afd_ascend.afd_async_combine_send(**args) + + def _require_push_session(self) -> PushCAMSession: + self._require_initialized() + if not isinstance(self._session, PushCAMSession): + raise RuntimeError( + "Pull FFN receive/compute/send belongs to the Mega executor" + ) + return self._session + + def _cam_args( + self, states: AFDAsyncTransferState + ) -> dict[str, Tensor | int | str | None]: + """Fields shared by the four registered CAM operator schemas.""" + return dict( comm_args=self.comm_args, - token_nums_rankid_layeridx=token_nums_rankid_layeridx, - metadata_values=False, comm_id=self.comm_id, - max_seq_len=self.max_num_batched_tokens, - batch_size=states.batch_size, hidden_size=states.hidden_size, - topk=states.topk, - ffn_size=self.ffn_size, - attn_size=self.attn_size, - expert_per_rank=self.expert_per_rank, - rank=self.world_rank, + top_k=states.topk, + moe_rank_num=self.ffn_size, + attn_rank_num=self.attn_size, + route_expert_num_per_moe=self.expert_per_rank, world_size=self.topology.world_size, - tp_size=self.tp_size, group_name=self.group_name, ) - states.send_completion = torch.ops.afd_ascend.afd_async_combine_send( - ffn_output, - self.comm_args, - token_nums_rankid_layeridx, - self.comm_id, - self.max_num_batched_tokens, - states.hidden_size, - states.topk, - self.ffn_size, - self.attn_size, - self.expert_per_rank, - self.world_rank, - self.topology.world_size, - self.tp_size, - self.group_name, - ) def _require_initialized(self) -> None: if not self._initialized: raise RuntimeError("CAMAsyncAFDConnector is not initialized") -def _require_async_transfer_state( - context: AFDTransferContext, -) -> AFDAsyncTransferState: - states = context.states - if not isinstance(states, AFDAsyncTransferState): - raise RuntimeError( - "CAMAsyncAFDConnector requires AFDAsyncTransferState in the " - "transfer context", - ) - return states - - -_CAM_LOG_SKIPPED_ARGS = frozenset({"comm_args", "comm_id", "group_name"}) -_CAM_OP_IO_LOG_ENV = "AFD_CAM_OP_IO_LOG" - - -def _log_cam_op_values( - op_name: str, label: str, *, metadata_values: bool = True, **kwargs: object -) -> None: - if os.environ.get(_CAM_OP_IO_LOG_ENV, "").lower() not in { - "1", - "true", - "yes", - "on", - }: - return - lines: list[str] = [] - for name, value in kwargs.items(): - if name in _CAM_LOG_SKIPPED_ARGS: - continue - if isinstance(value, Tensor): - description = ( - f"Tensor(dtype={value.dtype}, shape={tuple(value.shape)}, " - f"device={value.device})" - ) - if metadata_values and name == "token_nums_rankid_layeridx": - try: - first5: object = value.detach().flatten()[:5].cpu().tolist() - except Exception as exc: # pragma: no cover - defensive logging helper - first5 = f"" - description = f"{description}, first5={first5!r}" - else: - description = repr(value) - lines.append(f" {name}={description}") - logger.warning("AFD CAM %s %s:\n%s", op_name, label, "\n".join(lines)) - - def build_async_topology( afd_config: AFDConfig, role_rank: int, @@ -971,20 +713,57 @@ def build_async_topology( ) +def _require_async_transfer_state( + context: AFDTransferContext, +) -> AFDAsyncTransferState: + states = context.states + if not isinstance(states, AFDAsyncTransferState): + raise RuntimeError( + "CAMAsyncAFDConnector requires AFDAsyncTransferState in the " + "transfer context", + ) + return states + + +_CAM_LOG_SKIPPED_ARGS = frozenset({"comm_args", "comm_id", "group_name"}) +_CAM_OP_IO_LOG_ENV = "AFD_CAM_OP_IO_LOG" + + +def _log_cam_op_values(op_name: str, label: str, **kwargs: object) -> None: + if os.environ.get(_CAM_OP_IO_LOG_ENV, "").lower() not in { + "1", + "true", + "yes", + "on", + }: + return + lines: list[str] = [] + for name, value in kwargs.items(): + if name in _CAM_LOG_SKIPPED_ARGS: + continue + if isinstance(value, Tensor): + description = ( + f"Tensor(dtype={value.dtype}, shape={tuple(value.shape)}, " + f"device={value.device})" + ) + + else: + description = repr(value) + lines.append(f" {name}={description}") + logger.warning("AFD CAM %s %s:\n%s", op_name, label, "\n".join(lines)) + + def _validate_topk_payload( topk_ids: Tensor, topk_weights: Tensor | None, *, batch_size: int, topk: int, - require_weights: bool = True, ) -> None: if tuple(topk_ids.shape) != (batch_size, topk): raise ValueError( f"topk_ids shape must match ({batch_size}, {topk}), got {topk_ids.shape!r}", ) - if not require_weights and topk_weights is None: - return if topk_weights is None: raise ValueError("topk_weights is required") if tuple(topk_weights.shape) != (batch_size, topk): diff --git a/afd_plugin/connectors/npu/async_cam_runtime/__init__.py b/afd_plugin/connectors/npu/async_cam_runtime/__init__.py index 721549de4..281bcb14b 100644 --- a/afd_plugin/connectors/npu/async_cam_runtime/__init__.py +++ b/afd_plugin/connectors/npu/async_cam_runtime/__init__.py @@ -1,3 +1,3 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project -"""Async CAM startup and FFN lifecycle helpers.""" +"""Async CAM startup and Push/Pull communication sessions.""" diff --git a/afd_plugin/connectors/npu/async_cam_runtime/async_cam_pull.py b/afd_plugin/connectors/npu/async_cam_runtime/async_cam_pull.py deleted file mode 100644 index ae552a04f..000000000 --- a/afd_plugin/connectors/npu/async_cam_runtime/async_cam_pull.py +++ /dev/null @@ -1,77 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project -"""Pull service and shutdown handshake for the async CAM connector.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import TYPE_CHECKING - -from afd_plugin.validation import cam_receive_capacity - -if TYPE_CHECKING: - from afd_plugin.connectors.npu.async_cam import CAMAsyncAFDConnector - - -class PullFFNSession: - requires_drain = True - - def __init__(self, connector: CAMAsyncAFDConnector) -> None: - self.connector = connector - self.receiver_finished = False - self._idle_enabled = False - self._drained = False - - def keep_serving(self, shutdown_requested: bool) -> bool: - startup = self.connector.graph_startup - assert startup._store is not None - startup.check_failure() - return not startup._store.check( - startup._keys("pull_quiet", range(startup.attn_size)) - ) - - def before_first_step(self) -> None: - if not self._idle_enabled: - self.connector._enable_pull_idle_returns() - self._idle_enabled = True - - def after_step(self) -> None: - self.connector._check_pull_status() - - def drain(self, run_one: Callable[[], None]) -> None: - connector = self.connector - rounds = ( - connector.max_num_batched_tokens * connector.topk - + cam_receive_capacity() - - 1 - ) // cam_receive_capacity() - for _ in range(rounds): - if not connector._pull_has_active_source(): - self._drained = True - return - run_one() - if connector._pull_has_active_source(): - raise RuntimeError("Pull source did not drain; retain resources") - self._drained = True - - def finish_receiver(self) -> None: - if self.receiver_finished: - return - if not self._drained: - raise RuntimeError("Pull receiver has not drained; retain resources") - startup = self.connector.graph_startup - startup._wait(startup._keys("pull_quiet", range(startup.attn_size))) - startup._publish("pull_stopped") - self.receiver_finished = True - - def close_attention(self) -> None: - startup = self.connector.graph_startup - startup._publish("pull_quiet") - startup._wait( - startup._keys("pull_stopped", range(startup.attn_size, startup.world_size)) - ) - - def validate_shutdown(self) -> None: - if not self.receiver_finished or self.connector._pull_has_active_source(): - raise RuntimeError("Pull receiver has not drained; retain resources") - self.connector._check_pull_status() diff --git a/afd_plugin/connectors/npu/async_cam_runtime/async_cam_session.py b/afd_plugin/connectors/npu/async_cam_runtime/async_cam_session.py new file mode 100644 index 000000000..7405077af --- /dev/null +++ b/afd_plugin/connectors/npu/async_cam_runtime/async_cam_session.py @@ -0,0 +1,277 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Push/Pull CAM transactions, communication state and FFN lifecycle.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from contextlib import AbstractContextManager, contextmanager, nullcontext +from typing import TYPE_CHECKING + +from afd_plugin.connectors.base import FFNExecutionSession + +if TYPE_CHECKING: + import torch + from torch import Tensor + + from afd_plugin.connectors.metadata import AFDTransferContext + from afd_plugin.connectors.npu.async_cam import AFDAsyncTopology + from afd_plugin.connectors.npu.async_cam_runtime.async_cam_startup import ( + AsyncCAMGraphStartup, + ) + +CAM_COMM_ID = 0 +PULL_COMM_ID = 0x41464401 +PULL_FAILURE_INDEX = 24 +PULL_SOURCE_INDEX = 32 +PULL_IDLE_BUDGET_INDEX = 64 +PULL_IDLE_SCANS = 8 + + +class PushCAMSession(FFNExecutionSession): + """Push owns per-stage routing queues; FFN uses separate receive/send calls.""" + + comm_id = CAM_COMM_ID + signature_prefix = "" + allow_row_slicing = False + + def __init__(self, dtype: torch.dtype) -> None: + self.dtype = dtype + self.comm_args: Tensor | None = None + self.placeholder: Tensor | None = None + self.pending: dict[int, list[tuple[AFDTransferContext, Tensor, Tensor]]] = {} + + @property + def has_pending_attention(self) -> bool: + return bool(self.pending) + + def initialize(self, device: str, group_name: str) -> None: + import torch + + self.comm_args = torch.empty((1,), dtype=torch.float16, device=device) + self.placeholder = torch.empty((1,), dtype=self.dtype, device=device) + + def pending_for( + self, stage: int + ) -> tuple[AFDTransferContext, Tensor, Tensor] | None: + queue = self.pending.get(stage) + return queue[0] if queue else None + + def dispatch( + self, + payload: tuple[AFDTransferContext, Tensor, Tensor], + **args: Tensor | int | str | None, + ) -> None: + import torch + + torch.ops.afd_ascend.afd_async_dispatch_send(**args) + self.pending.setdefault(payload[0].metadata.stage_idx, []).append(payload) + + def combine(self, stage: int, **args: Tensor | int | str | None) -> Tensor: + import torch + + output = torch.ops.afd_ascend.afd_async_combine_recv(**args) + queue = self.pending[stage] + queue.pop(0) + if not queue: + del self.pending[stage] + return output + + def attention_step(self) -> AbstractContextManager[None]: + return nullcontext() + + def prepare_close(self, initialized: bool) -> None: + pass + + def release(self) -> None: + self.comm_args = self.placeholder = None + self.pending.clear() + + +class PullCAMSession(FFNExecutionSession): + """One in-flight Attention window; FFN runs the native fused transaction.""" + + requires_drain = True + comm_id = PULL_COMM_ID + signature_prefix = "pull_v1:" + allow_row_slicing = True + + def __init__( + self, + startup: AsyncCAMGraphStartup, + topology: AFDAsyncTopology, + *, + hidden_size: int, + topk: int, + max_num_batched_tokens: int, + tp_size: int, + receive_capacity: int, + ) -> None: + self.startup = startup + self.max_num_batched_tokens = max_num_batched_tokens + self.topk = topk + self.tp_size = tp_size + self.receive_capacity = receive_capacity + self._init_config = [ + max_num_batched_tokens // tp_size, + hidden_size, + topk, + topology.ffn_size, + topology.attn_size, + topology.expert_per_rank, + topology.world_rank, + tp_size, + ] + self.comm_args: Tensor | None = None + self._pending: tuple[AFDTransferContext, Tensor, Tensor] | None = None + self._defer_status = False + self.receiver_finished = False + self._idle_enabled = False + self._drained = False + + @property + def has_pending_attention(self) -> bool: + return self._pending is not None + + def initialize(self, device: str, group_name: str) -> None: + import torch + + self.comm_args = torch.ops.afd_ascend.afd_mega_pull_init( + torch.empty(1, dtype=torch.bfloat16, device=device), + self._init_config, + list(self.startup.plan.layer_ids), + self.startup.session_epoch, + group_name, + ) + torch.npu.synchronize() + self.check_pull_status() + self.startup.phase("pull-initialized") + + def pending_for( + self, stage: int + ) -> tuple[AFDTransferContext, Tensor, Tensor] | None: + if self._pending is not None and self._pending[0].metadata.stage_idx == stage: + return self._pending + return None + + def dispatch( + self, + payload: tuple[AFDTransferContext, Tensor, Tensor], + **args: Tensor | int | str | None, + ) -> None: + import torch + + if self._pending is not None: + raise RuntimeError("Pull window is pending; finish Combine before Dispatch") + # A launch can enqueue work before raising; retain ownership on failure. + self._pending = payload + torch.ops.afd_ascend.afd_mega_dispatch_send(**args) + + def combine(self, stage: int, **args: Tensor | int | str | None) -> Tensor: + import torch + + output = torch.ops.afd_ascend.afd_mega_combine_recv( + **args, max_seq_len=self.max_num_batched_tokens, tp_size=self.tp_size + ) + if not self._defer_status: + self.check_pull_status() + self._pending = None + return output + + @contextmanager + def attention_step(self) -> Iterator[None]: + previous = self._defer_status + self._defer_status = True + try: + yield + if not previous: + self.check_pull_status() + finally: + self._defer_status = previous + + def check_pull_status(self) -> None: + import torch + + assert self.comm_args is not None + # The model runner batches this fence outside the DBO receive hook. + status = int(self.comm_args.view(torch.int64)[PULL_FAILURE_INDEX].item()) + if status: + raise RuntimeError(f"AFD pull transaction failed with status {status}") + + def enable_pull_idle_returns(self) -> None: + import torch + + assert self.comm_args is not None + self.comm_args.view(torch.int64)[PULL_IDLE_BUDGET_INDEX].fill_(PULL_IDLE_SCANS) + + def pull_has_active_source(self) -> bool: + import torch + + assert self.comm_args is not None + return int(self.comm_args.view(torch.int64)[PULL_SOURCE_INDEX].item()) >= 0 + + def prepare_close(self, initialized: bool) -> None: + if self.has_pending_attention: + raise RuntimeError( + "Pull dispatch is pending; retain communication resources" + ) + if initialized: + if self.startup.world_rank < self.startup.attn_size: + self.close_attention() + else: + self.validate_shutdown() + + def release(self) -> None: + self.comm_args = None + self._pending = None + + def keep_serving(self, shutdown_requested: bool) -> bool: + startup = self.startup + assert startup._store is not None + startup.check_failure() + return not startup._store.check( + startup._keys("pull_quiet", range(startup.attn_size)) + ) + + def before_first_step(self) -> None: + if not self._idle_enabled: + self.enable_pull_idle_returns() + self._idle_enabled = True + + def after_step(self) -> None: + self.check_pull_status() + + def drain(self, run_one: Callable[[], None]) -> None: + rounds = ( + self.max_num_batched_tokens * self.topk + self.receive_capacity - 1 + ) // self.receive_capacity + for _ in range(rounds): + if not self.pull_has_active_source(): + self._drained = True + return + run_one() + if self.pull_has_active_source(): + raise RuntimeError("Pull source did not drain; retain resources") + self._drained = True + + def finish_receiver(self) -> None: + if self.receiver_finished: + return + if not self._drained: + raise RuntimeError("Pull receiver has not drained; retain resources") + startup = self.startup + startup._wait(startup._keys("pull_quiet", range(startup.attn_size))) + startup._publish("pull_stopped") + self.receiver_finished = True + + def close_attention(self) -> None: + startup = self.startup + startup._publish("pull_quiet") + startup._wait( + startup._keys("pull_stopped", range(startup.attn_size, startup.world_size)) + ) + + def validate_shutdown(self) -> None: + if not self.receiver_finished or self.pull_has_active_source(): + raise RuntimeError("Pull receiver has not drained; retain resources") + self.check_pull_status() diff --git a/docs/design/module/connector_contracts.md b/docs/design/module/connector_contracts.md index 8e20603c7..cc8504a95 100644 --- a/docs/design/module/connector_contracts.md +++ b/docs/design/module/connector_contracts.md @@ -42,7 +42,7 @@ related_issues: - "#105" - "#107" - "#129" -last_reviewed: 2026-10-30 +last_reviewed: 2026-11-05 --- # Connector contracts @@ -68,7 +68,7 @@ depend on role worker implementations. | CUDA P2P | [`gpu/p2p.py`](../../../afd_plugin/connectors/gpu/p2p.py), [`topology.py`](../../../afd_plugin/distributed/topology.py) | [`test_p2p_connector.py`](../../../tests/unit/connectors/test_p2p_connector.py), [DeepSeek-V2-Lite E2E](../../../tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py) | | CUDA DeepEP M2N | [`gpu/deepep_m2n/`](../../../afd_plugin/connectors/gpu/deepep_m2n/), `AFDRoutedExpertsConnector` in [`base.py`](../../../afd_plugin/connectors/base.py) | [`test_deepep_m2n_connector.py`](../../../tests/unit/connectors/test_deepep_m2n_connector.py), [GPU device tests](../../../tests/gpu/README.md), [Qwen3 MoE E2E](../../../tests/e2e/models/qwen3_moe/test_qwen3_moe.py), [DeepSeek-V2-Lite E2E](../../../tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py) (`test_deepseek_v2_lite_deepep_m2n`) | | Ascend CAMP2P | [`npu/camp2p.py`](../../../afd_plugin/connectors/npu/camp2p.py) | [`test_camp2p_connector.py`](../../../tests/unit/connectors/test_camp2p_connector.py), [DeepSeek-V2-Lite E2E](../../../tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py) | -| Ascend CAM async | [`npu/async_cam.py`](../../../afd_plugin/connectors/npu/async_cam.py) | [Connector unit tests](../../../tests/unit/connectors/test_async_cam_connector.py), [fused routing device test](../../../tests/npu/test_async_cam_connector.py), [`test_async_cam_npu.py`](../../../tests/e2e/models/deepseek_v4_flash/test_async_cam_npu.py) | +| Ascend CAM async | [`npu/async_cam.py`](../../../afd_plugin/connectors/npu/async_cam.py), [`Push/Pull sessions`](../../../afd_plugin/connectors/npu/async_cam_runtime/async_cam_session.py) | [Connector unit tests](../../../tests/unit/connectors/test_async_cam_connector.py), [CAM host contracts](../../../tests/unit/connectors/npu/test_async_cam.py), [fused routing device test](../../../tests/npu/test_async_cam_connector.py), [`test_async_cam_npu.py`](../../../tests/e2e/models/deepseek_v4_flash/test_async_cam_npu.py) | | Process-group construction | [`afd_process_group.py`](../../../afd_plugin/distributed/afd_process_group.py) | Connector initialization tests plus platform E2E paths | ## Factory and construction @@ -316,6 +316,36 @@ metadata is present. | Ascend CAMP2P | AFD process group, one HCCL communication group per ubatch, FFN HCCL state, Gloo metadata group, custom-op state and transfer handles. | FFN-first ordering and `A >= F`; group construction derives each FFN/Attention mapping. | | Ascend CAM async | Attention-first HCCL group, plugin-owned routed-only CAM operator state, per-stage pending Attention payload queues, and connector work-item state. | Role ranks map into a combined Attention-first world; Compact metadata determines the actual layer and expert interval; counts.sum() gives this chunk’s routed token count. | +`CAMAsyncAFDConnector` owns configuration, topology, the HCCL group, startup, +payload validation and final cleanup decisions. It selects one session at +construction: `PushCAMSession` or `PullCAMSession`, both in +`async_cam_runtime/async_cam_session.py`. Push keeps a FIFO per Attention +stage and separate FFN receive/send calls. Pull owns one in-flight window across +stages, native publish/combine, status/idle/source state and quiet/drain/stopped. +Mega's fused FFN operation owns its receive/compute/send transaction; the public +FFN receive/send methods reject Pull use. + +The four public transfer methods still enter through the connector. The existing +`AFDAsyncTransferState` definition stays at its original import path; +`comm_args` and `comm_id` expose the selected session's live resources through read-only +properties for the current Mega compute path. `has_pending_attention` provides +the diagnostic pending-state read without exposing the queue. + +Pull Session receives Startup, topology and required dimensions/capacity +explicitly, without retaining the connector. Sessions allocate their buffers +after Store negotiation and HCCL setup; initialization failures reach Startup +before the connector is marked ready. Push records routing after a successful +launch; Pull retains window ownership before launch. Combine failures retain +routing/window references. DBO defers the sticky status check to the Attention +step boundary. Graph capture, device completion and service threads remain with +the runner/worker. + +`FFNExecutionSession` in `base.py` defines the lifecycle hooks and provides +local-stop/no-drain defaults in one concrete class. Push/Pull sessions inherit +those defaults and override their protocol-specific behavior. The base module +keeps torch and transfer metadata imports under type checking so session imports +remain safe before worker device bootstrap. + The common process-group helper creates a plugin-owned group while temporarily switching PyTorch's default group so vLLM parallel-state helpers initialize against it. It currently calls private PyTorch/vLLM symbols and is therefore an diff --git a/docs/design/module/execution_platforms.md b/docs/design/module/execution_platforms.md index 86912bdec..c4644554a 100644 --- a/docs/design/module/execution_platforms.md +++ b/docs/design/module/execution_platforms.md @@ -102,7 +102,7 @@ The Async FFN runner and worker use its service predicate, first-step hook, completion hook, drain callback, receiver completion and pre-release validation; they do not inspect Pull state. Push uses the no-op default session. Pull owns idle enablement, sticky status checks and the `quiet → drain → stopped` handshake -in `connectors/npu/async_cam_runtime/async_cam_pull.py`. Shutdown validates the +in `connectors/npu/async_cam_runtime/async_cam_session.py`. Shutdown validates the session before releasing graph, profiler, model or communication resources; failures retain those resources. Attention batches the same device-status fence at step exit. diff --git a/docs/design/module/ffn_runtime.md b/docs/design/module/ffn_runtime.md index 593847b07..07d3576a4 100644 --- a/docs/design/module/ffn_runtime.md +++ b/docs/design/module/ffn_runtime.md @@ -15,7 +15,7 @@ primary_code_paths: - "afd_plugin/v1/worker/npu/ffn_worker.py" related_code_paths: - "afd_plugin/connectors/npu/async_cam_runtime/async_cam_startup.py" - - "afd_plugin/connectors/npu/async_cam_runtime/async_cam_pull.py" + - "afd_plugin/connectors/npu/async_cam_runtime/async_cam_session.py" - "afd_plugin/connectors/**" - "afd_plugin/model_executor/**" - "afd_plugin/compat/patches/engine_core.py" @@ -35,7 +35,9 @@ validation_paths: - "tests/unit/v1/worker/test_npu_runtime.py" - "tests/unit/connectors/npu/test_async_cam_startup_wiring.py" - "tests/unit/connectors/npu/test_async_cam_startup.py" - - "tests/unit/connectors/npu/test_async_cam_pull.py" + - "tests/unit/connectors/test_base.py" + - "tests/unit/connectors/npu/test_async_cam_session.py" + - "tests/unit/connectors/npu/test_async_cam.py" - "tests/npu/async_cam_runtime.py" - "tests/e2e/models/deepseek_v4_flash/ffn_graph.py" - "tests/unit/compat/patches/test_engine_core.py" @@ -58,7 +60,7 @@ related_issues: - "#105" - "#107" - "#129" -last_reviewed: 2026-10-20 +last_reviewed: 2026-11-05 --- # FFN runtime @@ -78,6 +80,17 @@ FFN consumes plugin configuration, connector transport, model-side compute, platform mechanisms, and EngineCore compatibility. No shared module may depend on the FFN worker or runner implementation. +Async CAM's connector owns HCCL, payload validation and initialization/cleanup +coordination. It selects a Push or Pull session at construction. Both inherit +the concrete `FFNExecutionSession` lifecycle defaults from `connectors/base.py`. Push owns +per-stage routing queues; Pull owns its single window, publish/combine, device +status and quiet/drain/stopped lifecycle. Pull takes Startup, topology and the +dimensions/capacity used for initialization and bounded drain; it does not +retain the connector. The FFN runner continues to own eager/FULL execution, +graph references and completion; the worker still owns the receiver thread. +The current Mega executor reads the same live window through the connector's +read-only `comm_args` property. + ## Runtime selection FFN is launched as a `vllm serve` process, and AFD config normalization selects diff --git a/docs/npu/ASYNC_FFN_RUNTIME.md b/docs/npu/ASYNC_FFN_RUNTIME.md index 2167b5cae..a56e05531 100644 --- a/docs/npu/ASYNC_FFN_RUNTIME.md +++ b/docs/npu/ASYNC_FFN_RUNTIME.md @@ -61,7 +61,10 @@ F HTTP API initialization is not a readiness requirement. ## Capture boundary and ownership By default, the runner owns capture/replay, weights, stable buffers, a private graph pool -and one execution stream. The connector owns CAM communication. The host-only `AsyncCAMGraphStartup` +and one execution stream. The connector owns HCCL, payload validation and +communication lifecycle coordination. Its Push session owns per-stage queues; +its Pull session owns a single window, native publish/combine, device status +and quiet/drain/stopped. The host-only `AsyncCAMGraphStartup` coordinator owns Store negotiation, stage waits and failure propagation; it owns no device resources or threads. A captured transaction receives activations, scales, expert counts and the original return metadata from DR. Device-side layer selection uses @@ -75,7 +78,8 @@ metadata goes to CS; host code does not select the layer or copy per-item inputs | CS completion tensor | Retained transfer state | | Weights and scale/compensation lists | Runner's model and executor | | Captured native workspace | Graph's private allocator pool | -| HCCL group, communication arguments and windows | Connector | +| HCCL group and final communication cleanup decision | Connector facade | +| Communication arguments/windows and pending routing references | Selected Push/Pull session | Each business batch submits as many replays as there are loaded FFN layers, then waits once for device completion. This is a bounded submission count, @@ -144,7 +148,8 @@ A3/910C, 4A/4F, A TP4, F TP1, receive capacity 128 and DBO disabled. Select this topology still needs target-device validation. Synthetic per-group coverage does not qualify loading real per-group checkpoints. -- Host contracts: `tests/unit/connectors/npu/test_async_cam_pull.py`, +- Host contracts: `tests/unit/connectors/npu/test_async_cam.py`, + `tests/unit/connectors/npu/test_async_cam_session.py`, `tests/unit/connectors/npu/test_async_cam_startup.py`, `tests/unit/v1/worker/test_async_ffn_graph.py`, `tests/unit/v1/worker/test_async_ffn_runtime.py`, and diff --git a/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md b/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md index b2016d28d..9f1cbb97a 100644 --- a/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md +++ b/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md @@ -221,6 +221,14 @@ prefill/decode batches and must be at least the microbatch count. Native slicing including parent padding and requests crossing a microbatch boundary. The model owns SP padding and each owner's layout restoration/TP collectives. +The connector keeps configuration, topology, HCCL, payload validation and +lifecycle coordination in `async_cam.py`. Its Push session keeps a FIFO per +stage. Both sessions live in `async_cam_runtime/async_cam_session.py`. Pull owns one window +across stages, native publish/combine, device status and shutdown coordination. +The session is selected once at construction. Attention uses the existing +connector entry points; FFN uses separate receive/send for Push, or Mega's fused +transaction for Pull. Graph execution remains in the FFN ModelRunner. + CAM has one physical window per Attention rank. Before another ubatch sends, a native peer receive hook completes the preceding raw receive with its owner's explicit transfer context. The resumed owner restores layout and adds shared @@ -351,6 +359,7 @@ slice of the original metadata, or a device mapping, selects the layer; the original metadata goes to combine-send. Empty ranks, chunks, and ubatches use the existing communication protocol. With `AFD_CAM_OP_IO_LOG` enabled, the path logs only Tensor shape, dtype, and device without reading metadata values. +Input field names follow the registered native operator schema. The Async eager worker synchronizes after each complete FFN layer loop; FULL submits a bounded batch on one stream and waits once for that batch. These submission loops do not define request boundaries. Measure peak diff --git a/tests/npu/async_cam_runtime.py b/tests/npu/async_cam_runtime.py index cb4f1845d..98bf25799 100644 --- a/tests/npu/async_cam_runtime.py +++ b/tests/npu/async_cam_runtime.py @@ -468,7 +468,8 @@ def observed_replay(): local_ffn * args.experts_per_rank, (local_ffn + 1) * args.experts_per_rank ): count = int((ids == expert).sum()) * args.attention_tp - assert count <= args.receive_capacity + if not args.mega: + assert count <= args.receive_capacity if chunks[-1] + count > args.receive_capacity: chunks.append(0) chunks[-1] += count @@ -671,7 +672,7 @@ def serve(): assert len(references) == len(outputs) report["eager_exact_match"] = not accuracy_failures report["accuracy_failures"] = len(accuracy_failures) - assert not connector._pending_attention_payloads + assert not connector.has_pending_attention if args.idle_shutdown: connector.close() else: diff --git a/tests/unit/connectors/npu/test_async_cam.py b/tests/unit/connectors/npu/test_async_cam.py new file mode 100644 index 000000000..ec7f1dbd7 --- /dev/null +++ b/tests/unit/connectors/npu/test_async_cam.py @@ -0,0 +1,317 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Native CAM message contracts and ownership through the public connector.""" + +import sys +import weakref +from types import SimpleNamespace + +import pytest + +from tests.unit.fakes.async_cam import ( + _FakeTensor, + _topk_payload, + recording_cam_connector, +) +from tests.unit.fakes.async_cam_startup import MemoryStore, startup_world +from tests.unit.fakes.async_ffn import load_async_modules + + +@pytest.fixture +def cam(monkeypatch): + _, connector, _ = load_async_modules(monkeypatch) + return SimpleNamespace( + create=lambda **kwargs: recording_cam_connector( + monkeypatch, connector, **kwargs + ), + connector_module=connector, + metadata=sys.modules["afd_plugin.connectors.metadata"], + ) + + +def test_async_connector_calls_cam_shaped_ops(cam): + h = cam.create(pcp_size=3) + connector, fake_torch = h.connector, h.torch + hidden_states = _FakeTensor((3, 16)) + metadata = cam.metadata.AFDTransferMetadata.create_attention_metadata( + layer_idx=2, + stage_idx=0, + seq_len=3, + ) + context = cam.metadata.AFDTransferContext(metadata=metadata) + + output = connector.send_attn_output( + hidden_states, + context, + **_topk_payload(3), + ) + combined = connector.recv_ffn_output( + ref_tensor=hidden_states, + ubatch_idx=0, + ) + + assert output is None + assert combined.shape == (3, 16) + assert fake_torch.ops.afd_ascend.calls[0][0] == "dispatch_send" + assert fake_torch.ops.afd_ascend.calls[1][0] == "combine_recv" + assert fake_torch.ops.afd_ascend.calls[0][1][3] == cam.connector_module.CAM_COMM_ID + assert fake_torch.ops.afd_ascend.calls[1][1][4] == cam.connector_module.CAM_COMM_ID + assert fake_torch.ops.afd_ascend.calls[0][1][5:11] == (3, 16, 2, 2, 4, 4) + assert fake_torch.ops.afd_ascend.calls[1][1][5:11] == (3, 16, 2, 2, 4, 4) + assert fake_torch.ops.afd_ascend.calls[0][1][14] == 4 + assert isinstance(context.states, cam.connector_module.AFDAsyncTransferState) + assert isinstance(context.states, cam.metadata.AFDTransferState) + + +def test_async_explicit_receive_consumes_matching_pending_dispatch(cam): + h = cam.create() + connector, fake_torch = h.connector, h.torch + hidden_states = _FakeTensor((3, 16)) + contexts = [] + payloads = [] + for layer_idx in (2, 3): + context = cam.metadata.AFDTransferContext( + metadata=cam.metadata.AFDTransferMetadata.create_attention_metadata( + layer_idx=layer_idx, stage_idx=0, seq_len=3 + ) + ) + payload = _topk_payload(3) + connector.send_attn_output(hidden_states, context, **payload) + contexts.append(context) + payloads.append(payload) + + with pytest.raises(RuntimeError, match="does not match the next pending"): + connector.recv_ffn_output(hidden_states, ubatch_idx=0, context=contexts[1]) + assert len(fake_torch.ops.afd_ascend.calls) == 2 + + connector.recv_ffn_output( + hidden_states, + ubatch_idx=0, + context=contexts[0], + **payloads[1], + ) + assert fake_torch.ops.afd_ascend.calls[-1][1][1] is payloads[0]["topk_ids"] + + connector.recv_ffn_output(hidden_states, ubatch_idx=0) + assert fake_torch.ops.afd_ascend.calls[-1][1][1] is payloads[1]["topk_ids"] + + +@pytest.mark.parametrize("mega", [False, True]) +def test_protocol_owns_stage_queue_or_exclusive_window(cam, mega): + h = cam.create(mega=mega) + hidden = _FakeTensor((3, 16)) + contexts = [ + cam.metadata.AFDTransferContext( + cam.metadata.AFDTransferMetadata.create_attention_metadata( + layer_idx=2, stage_idx=stage, seq_len=3 + ) + ) + for stage in (0, 1) + ] + h.connector.send_attn_output(hidden, contexts[0], **_topk_payload(3)) + if mega: + with pytest.raises(RuntimeError, match="Pull window is pending"): + h.connector.send_attn_output(hidden, contexts[1], **_topk_payload(3)) + assert len(h.torch.ops.afd_ascend.calls) == 1 + h.connector.recv_ffn_output(hidden, ubatch_idx=0, context=contexts[0]) + h.connector.send_attn_output(hidden, contexts[1], **_topk_payload(3)) + else: + h.connector.send_attn_output(hidden, contexts[1], **_topk_payload(3)) + h.connector.recv_ffn_output(hidden, ubatch_idx=1, context=contexts[1]) + if mega: + assert h.torch.ops.afd_ascend.calls[-1][0] == "mega_combine_recv" + assert h.torch.ops.afd_ascend.calls[-1][1][4] == 0x41464401 + assert h.torch.ops.afd_ascend.calls[-1][1][-2:] == (16, 4) + else: + assert h.connector.has_pending_attention + h.connector.recv_ffn_output(hidden, ubatch_idx=0, context=contexts[0]) + assert not h.connector.has_pending_attention + + +@pytest.mark.parametrize("mega", [False, True]) +def test_failed_combine_retains_original_routing_for_retry(cam, monkeypatch, mega): + h = cam.create(mega=mega) + hidden = _FakeTensor((3, 16)) + context = cam.metadata.AFDTransferContext( + cam.metadata.AFDTransferMetadata.create_attention_metadata( + layer_idx=2, stage_idx=0, seq_len=3 + ) + ) + payload = _topk_payload(3) + h.connector.send_attn_output(hidden, context, **payload) + operation = "afd_mega_combine_recv" if mega else "afd_async_combine_recv" + with monkeypatch.context() as patch: + + def fail(**kwargs): + raise RuntimeError("combine launch failed") + + patch.setattr(h.torch.ops.afd_ascend, operation, fail) + with pytest.raises(RuntimeError, match="combine launch failed"): + h.connector.recv_ffn_output(hidden, context=context) + assert h.connector.has_pending_attention + h.connector.recv_ffn_output(hidden, context=context) + assert h.torch.ops.afd_ascend.calls[-1][1][1] is payload["topk_ids"] + assert not h.connector.has_pending_attention + + +def test_pull_ffn_rejects_separate_receive_and_send(cam): + h = cam.create(mega=True, role="ffn") + with pytest.raises(RuntimeError, match="Mega executor"): + h.connector.recv_attn_output() + with pytest.raises(RuntimeError, match="Mega executor"): + h.connector.send_ffn_output(_FakeTensor((3, 16)), None) + assert not h.torch.ops.afd_ascend.calls + + +def test_async_ffn_side_dispatch_recv_and_combine_send(cam): + h = cam.create(role="ffn", tp_size=2, pcp_size=2) + connector, fake_torch = h.connector, h.torch + recv_output = connector.recv_attn_output(batch_size=4, layer_idx=1) + connector.send_ffn_output(recv_output.hidden_states, recv_output.context) + + states = recv_output.context.states + assert recv_output.hidden_states.shape == (connector.max_num_batched_tokens, 16) + assert states.dynamic_scales.shape == (connector.max_num_batched_tokens,) + assert states.group_list.shape == (connector.topology.expert_per_rank,) + assert fake_torch.ops.afd_ascend.calls[0][0] == "dispatch_recv" + assert fake_torch.ops.afd_ascend.calls[1][0] == "combine_send" + assert fake_torch.ops.afd_ascend.calls[0][1][11] == 2 + assert fake_torch.ops.afd_ascend.calls[1][1][12] == 2 + assert fake_torch.ops.afd_ascend.calls[1][1][2] is states.token_nums_rankid_layeridx + + +def test_async_combine_send_requires_dispatch_recv_token_metadata(cam): + h = cam.create(role="ffn") + connector = h.connector + metadata = cam.metadata.AFDTransferMetadata.create_ffn_metadata( + layer_idx=1, + stage_idx=0, + seq_lens=[4], + ) + # token_nums_rankid_layeridx is left unset (None) so combine send must fail. + states = cam.connector_module.AFDAsyncTransferState( + batch_size=4, + hidden_size=16, + topk=2, + layer_idx=1, + group_list=_FakeTensor((4,), dtype="int64"), + ) + context = cam.metadata.AFDTransferContext(metadata=metadata, states=states) + + with pytest.raises(RuntimeError, match="TokenNums_Rankid_Layeridx"): + connector.send_ffn_output(_FakeTensor((4, 16)), context) + + +@pytest.mark.parametrize("mega", [False, True]) +def test_dispatch_failure_preserves_protocol_window_ownership(cam, monkeypatch, mega): + h = cam.create(mega=mega) + connector, fake_torch = h.connector, h.torch + context = cam.metadata.AFDTransferContext( + metadata=cam.metadata.AFDTransferMetadata.create_attention_metadata( + layer_idx=2, + stage_idx=0, + seq_len=3, + ) + ) + + def fail_dispatch(*_args, **_kwargs): + raise RuntimeError("dispatch failed") + + monkeypatch.setattr( + fake_torch.ops.afd_ascend, + "afd_mega_dispatch_send" if mega else "afd_async_dispatch_send", + fail_dispatch, + ) + with pytest.raises(RuntimeError, match="dispatch failed"): + connector.send_attn_output(_FakeTensor((3, 16)), context, **_topk_payload(3)) + if mega: + with pytest.raises(RuntimeError, match="pending"): + connector.close() + else: + connector.close() + assert not connector.is_initialized + + +def test_close_releases_every_stage_routing(cam): + h = cam.create() + connector, fake_torch = h.connector, h.torch + references: list[weakref.ReferenceType] = [] + for stage in (0, 1): + context = cam.metadata.AFDTransferContext( + metadata=cam.metadata.AFDTransferMetadata.create_attention_metadata( + layer_idx=2, stage_idx=stage, seq_len=3 + ) + ) + payload = _topk_payload(3) + references.extend(weakref.ref(tensor) for tensor in payload.values()) + connector.send_attn_output(_FakeTensor((3, 16)), context, **payload) + del payload, context + fake_torch.ops.afd_ascend.calls.clear() + assert all(reference() is not None for reference in references) + connector.close() + assert all(reference() is None for reference in references) + + +@pytest.mark.parametrize( + "operation", + ["send_attn_output", "recv_ffn_output", "recv_attn_output", "send_ffn_output"], +) +def test_closed_connector_rejects_transfers_before_native_launch(cam, operation): + h = cam.create() + h.connector.close() + hidden = _FakeTensor((1, 16)) + context = cam.metadata.AFDTransferContext( + metadata=cam.metadata.AFDTransferMetadata.create_attention_metadata( + layer_idx=2, stage_idx=0, seq_len=1 + ) + ) + arguments = { + "send_attn_output": (hidden, context), + "recv_ffn_output": (hidden,), + "recv_attn_output": (), + "send_ffn_output": (hidden, context), + } + with pytest.raises(RuntimeError, match="not initialized"): + getattr(h.connector, operation)(*arguments[operation]) + assert h.torch.ops.afd_ascend.calls == [] + + +def test_reinitialization_updates_legacy_window_reader(cam): + h = cam.create() + first_window = h.connector.comm_args + h.connector.close() + assert h.connector.comm_args is None + h.connector.ffn_session().initialize("npu:1", "replacement") + replacement = h.connector.comm_args + assert replacement is not None and replacement is not first_window + assert replacement.device == "npu:1" + with pytest.raises(AttributeError): + h.connector.comm_args = first_window + + +@pytest.mark.parametrize("mega", [False, True]) +def test_window_allocation_failure_reaches_peer_before_ready(cam, monkeypatch, mega): + h = cam.create(mega=mega) + connector = h.connector + connector._initialized = False + connector.ffn_session().release() + backend = SimpleNamespace(get_hccl_comm_name=lambda rank: "window-failure") + group = SimpleNamespace(_get_backend=lambda device: backend) + monkeypatch.setattr( + cam.connector_module, "init_afd_process_group", lambda **kwargs: group + ) + store = MemoryStore() + peer = startup_world(attn=4, ffn=2, store=store)[1] + connector.graph_startup._store = store + connector.graph_startup._prefix = peer._prefix + + def fail_allocation(*args, **kwargs): + raise RuntimeError("window allocation failed") + + monkeypatch.setattr(h.torch, "empty", fail_allocation) + with pytest.raises(RuntimeError, match="window allocation failed"): + connector.init_afd_connector() + with pytest.raises(RuntimeError, match="window allocation failed"): + peer.check_failure() + assert not connector.is_initialized and connector.cam_pg is group + assert connector.comm_args is None diff --git a/tests/unit/connectors/npu/test_async_cam_pull.py b/tests/unit/connectors/npu/test_async_cam_session.py similarity index 76% rename from tests/unit/connectors/npu/test_async_cam_pull.py rename to tests/unit/connectors/npu/test_async_cam_session.py index 0ba8298d0..9be1f7330 100644 --- a/tests/unit/connectors/npu/test_async_cam_pull.py +++ b/tests/unit/connectors/npu/test_async_cam_session.py @@ -1,7 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project -"""Host Pull shutdown protocol; these tests do not qualify device execution.""" +"""Host session lifecycle contracts; not device execution qualification.""" +import sys from types import SimpleNamespace import pytest @@ -9,7 +10,10 @@ from afd_plugin.connectors.npu.async_cam_runtime import ( async_cam_startup as startup_module, ) -from afd_plugin.connectors.npu.async_cam_runtime.async_cam_pull import PullFFNSession +from afd_plugin.connectors.npu.async_cam_runtime.async_cam_session import ( + PullCAMSession, +) +from tests.unit.fakes.async_cam import FakePullState, _FakeTorch from tests.unit.fakes.async_cam_startup import MemoryStore, startup_world @@ -17,20 +21,29 @@ def pull_world(monkeypatch): monkeypatch.setattr(startup_module, "GRAPH_STARTUP_TIMEOUT_SECONDS", 0) store = MemoryStore() - connectors = [ - SimpleNamespace( - graph_startup=startup, - max_num_batched_tokens=8, + startups = startup_world(store=store) + monkeypatch.setitem(sys.modules, "torch", _FakeTorch()) + states = [FakePullState() for _ in startups] + sessions = [] + for startup, state in zip(startups, states, strict=True): + session = PullCAMSession( + startup, + SimpleNamespace( + ffn_size=2, + attn_size=2, + expert_per_rank=2, + world_rank=startup.world_rank, + ), + hidden_size=16, topk=2, - _pull_has_active_source=lambda: False, - _check_pull_status=lambda: None, + max_num_batched_tokens=8, + tp_size=1, + receive_capacity=4, ) - for startup in startup_world(store=store) - ] + session.comm_args = state + sessions.append(session) return SimpleNamespace( - sessions=[PullFFNSession(connector) for connector in connectors], - connectors=connectors, - store=store, + sessions=sessions, states=states, startups=startups, store=store ) @@ -48,23 +61,23 @@ def test_keep_serving_requires_every_attention_quiet( def test_peer_failure_wins_over_quiet_shutdown(pull_world): for rank in range(2): pull_world.store.set(f"ffn_graph/test/pull_quiet/{rank}", "ready") - pull_world.connectors[0].graph_startup.fail(RuntimeError("attention failed")) + pull_world.startups[0].fail(RuntimeError("attention failed")) with pytest.raises(RuntimeError, match="attention failed"): pull_world.sessions[2].keep_serving(True) @pytest.mark.parametrize("transactions_to_drain", [0, 1, 4, 5, None]) def test_drain_requires_source_completion_within_capacity_bound( - pull_world, monkeypatch, transactions_to_drain + pull_world, transactions_to_drain ): - monkeypatch.setenv("BATCH_SIZE_FACTOR", str(4 / 262144)) - session, connector = pull_world.sessions[2], pull_world.connectors[2] + session, words = pull_world.sessions[2], pull_world.states[2].words remaining = [transactions_to_drain] - connector._pull_has_active_source = lambda: remaining[0] != 0 + words[32] = -1 if remaining[0] == 0 else 0 def run_one(): if remaining[0] is not None: remaining[0] -= 1 + words[32] = -1 if remaining[0] == 0 else 0 if transactions_to_drain in (5, None): with pytest.raises(RuntimeError, match="retain resources"): @@ -84,8 +97,8 @@ def run_one(): def test_failed_drain_does_not_publish_receiver_stopped(pull_world): - session, connector = pull_world.sessions[2], pull_world.connectors[2] - connector._pull_has_active_source = lambda: True + session, words = pull_world.sessions[2], pull_world.states[2].words + words[32] = 0 def fail_transaction(): raise RuntimeError("transaction failed") @@ -117,7 +130,7 @@ def test_close_waits_for_all_attention_then_all_drained_receivers(pull_world): assert pull_world.store.check( [f"ffn_graph/test/pull_stopped/{rank}" for rank in (2, 3)] ) - pull_world.connectors[3].graph_startup.fail(RuntimeError("drain failed")) + pull_world.startups[3].fail(RuntimeError("drain failed")) with pytest.raises(RuntimeError, match="drain failed"): first_a.close_attention() @@ -134,20 +147,16 @@ def test_finish_receiver_is_idempotent_after_store_release(pull_world): @pytest.mark.parametrize("state", ["unfinished", "active", "failed", "drained"]) def test_shutdown_requires_finished_idle_error_free_receiver(pull_world, state): - session, connector = pull_world.sessions[2], pull_world.connectors[2] + session, words = pull_world.sessions[2], pull_world.states[2].words if state != "unfinished": for rank in range(2): pull_world.store.set(f"ffn_graph/test/pull_quiet/{rank}", "ready") session.drain(lambda: pytest.fail("an idle receiver needs no transaction")) session.finish_receiver() if state == "active": - connector._pull_has_active_source = lambda: True + words[32] = 0 if state == "failed": - - def status_failure(): - raise RuntimeError("status -5") - - connector._check_pull_status = status_failure + words[24] = -5 if state == "drained": session.validate_shutdown() else: diff --git a/tests/unit/connectors/npu/test_async_cam_startup.py b/tests/unit/connectors/npu/test_async_cam_startup.py index 94c7d9fc8..0ca545f3f 100644 --- a/tests/unit/connectors/npu/test_async_cam_startup.py +++ b/tests/unit/connectors/npu/test_async_cam_startup.py @@ -37,10 +37,11 @@ def test_startup_import_needs_no_torch_or_vllm_in_clean_process(): "torch_npu=None, vllm_ascend=None); " "from afd_plugin.connectors.npu.async_cam_runtime.async_cam_startup " "import AsyncCAMGraphStartup; " - "from afd_plugin.connectors.npu.async_cam_runtime.async_cam_pull " - "import PullFFNSession; " + "from afd_plugin.connectors.npu.async_cam_runtime.async_cam_session " + "import PullCAMSession, PushCAMSession; " "assert AsyncCAMGraphStartup(0, 1, 2).mode == 'eager'; " - "assert PullFFNSession.requires_drain", + "assert PullCAMSession.requires_drain; " + "assert not PushCAMSession.requires_drain", ], capture_output=True, text=True, diff --git a/tests/unit/connectors/npu/test_async_cam_startup_wiring.py b/tests/unit/connectors/npu/test_async_cam_startup_wiring.py index a7a7fc60c..452364201 100644 --- a/tests/unit/connectors/npu/test_async_cam_startup_wiring.py +++ b/tests/unit/connectors/npu/test_async_cam_startup_wiring.py @@ -276,6 +276,7 @@ def create(*args, **kwargs): ), graph_startup=AsyncCAMGraphStartup(0, 1, 2), ) + connector._session = connector_module.PushCAMSession(connector.activation_dtype) if dbo_flag is not None: setattr(connector.vllm_config.parallel_config, dbo_flag, True) @@ -519,7 +520,7 @@ def receiver_graph_worker(ffn_owners, monkeypatch): ) connector = SimpleNamespace( is_initialized=True, - ffn_session=sys.modules["afd_plugin.connectors.base"].DefaultFFNSession, + ffn_session=sys.modules["afd_plugin.connectors.base"].FFNExecutionSession, close=lambda: None, recv_attn_output=lambda: SimpleNamespace( hidden_states=object(), context=SimpleNamespace(states=state) @@ -667,7 +668,7 @@ def test_ffn_runner_loads_model_and_rejects_second_load( monkeypatch.setattr(runtime, "fail_if_unsupported_npu_afd_features", calls.validate) monkeypatch.setattr(runtime, "create_afd_npu_profiler", lambda role: None) connector = SimpleNamespace( - ffn_session=sys.modules["afd_plugin.connectors.base"].DefaultFFNSession + ffn_session=sys.modules["afd_plugin.connectors.base"].FFNExecutionSession ) device, model = SimpleNamespace(type="npu"), object() connector_name = "CAMAsyncAFDConnector" if async_connector else "CAMP2pAFDConnector" @@ -743,14 +744,15 @@ def connector_world(monkeypatch): rank, 0, config, parse_afd_config(config), rank % 2 ) connector.graph_startup = startup + connector.ffn_session().startup = startup connector._initialized = True - connector.comm_args = object() + connector.ffn_session().comm_args = object() connectors.append(connector) return SimpleNamespace(connectors=connectors, runtime=runtime) def test_default_session_obeys_shutdown_without_drain(connector_world): - session = sys.modules["afd_plugin.connectors.base"].DefaultFFNSession() + session = sys.modules["afd_plugin.connectors.base"].FFNExecutionSession() assert session.keep_serving(False) assert not session.keep_serving(True) assert not session.requires_drain @@ -771,19 +773,23 @@ def test_close_preserves_receiver_communication_on_unsafe_session( connector.cam_pg = group = object() session = connector.ffn_session() if failure != "unfinished": - monkeypatch.setattr(connector, "_pull_has_active_source", lambda: False) + monkeypatch.setattr( + connector.ffn_session(), "pull_has_active_source", lambda: False + ) session.drain(lambda: pytest.fail("idle source must not launch")) for attention in connector_world.connectors[:2]: attention.graph_startup._publish("pull_quiet") session.finish_receiver() if failure == "active": - monkeypatch.setattr(connector, "_pull_has_active_source", lambda: True) + monkeypatch.setattr( + connector.ffn_session(), "pull_has_active_source", lambda: True + ) elif failure == "status": def fail_status(): raise RuntimeError("sticky status -5") - monkeypatch.setattr(connector, "_check_pull_status", fail_status) + monkeypatch.setattr(connector.ffn_session(), "check_pull_status", fail_status) expected = "sticky status -5" if failure == "status" else "has not drained" with pytest.raises(RuntimeError, match=expected): connector.close() @@ -800,7 +806,7 @@ def check_status(): if error[0] is not None: raise error[0] - monkeypatch.setattr(connector, "_check_pull_status", check_status) + monkeypatch.setattr(connector.ffn_session(), "check_pull_status", check_status) expected = ValueError if failure == "body" else RuntimeError with pytest.raises(expected, match="failed"), connector.attention_step(): if failure == "body": @@ -825,7 +831,7 @@ def test_nested_attention_step_defers_status_until_outer_boundary( def check_status(): raise RuntimeError("sticky status") - monkeypatch.setattr(connector, "_check_pull_status", check_status) + monkeypatch.setattr(connector.ffn_session(), "check_pull_status", check_status) with pytest.raises(RuntimeError, match="sticky status"), connector.attention_step(): with connector.attention_step(): pass @@ -875,19 +881,23 @@ def test_runner_shutdown_retains_model_and_communication_on_unsafe_session( runner._graph_receiver_exited = True resource = connector.comm_args if failure != "unfinished": - monkeypatch.setattr(connector, "_pull_has_active_source", lambda: False) + monkeypatch.setattr( + connector.ffn_session(), "pull_has_active_source", lambda: False + ) runner.session.drain(lambda: pytest.fail("idle source must not launch")) for attention in connector_world.connectors[:2]: attention.graph_startup._publish("pull_quiet") runner.session.finish_receiver() if failure == "active": - monkeypatch.setattr(connector, "_pull_has_active_source", lambda: True) + monkeypatch.setattr( + connector.ffn_session(), "pull_has_active_source", lambda: True + ) elif failure == "status": def fail_status(): raise RuntimeError("status failed; retain resources") - monkeypatch.setattr(connector, "_check_pull_status", fail_status) + monkeypatch.setattr(connector.ffn_session(), "check_pull_status", fail_status) with pytest.raises(RuntimeError, match="retain resources"): runner.shutdown() assert runner.get_model() is model diff --git a/tests/unit/connectors/test_async_cam_connector.py b/tests/unit/connectors/test_async_cam_connector.py index 1f67c07e2..ea8efe7f9 100644 --- a/tests/unit/connectors/test_async_cam_connector.py +++ b/tests/unit/connectors/test_async_cam_connector.py @@ -3,13 +3,11 @@ from __future__ import annotations -import weakref +import sys from types import SimpleNamespace import pytest -from afd_plugin.config import AFDConfig - pytest.importorskip("torch") pytest.importorskip("torch_npu") @@ -19,19 +17,31 @@ AFDConnectorFactory, AFDTransferContext, AFDTransferMetadata, - AFDTransferState, ) from afd_plugin.connectors.npu import async_cam as async_cam_module # noqa: E402 from afd_plugin.connectors.npu.async_cam import ( # noqa: E402 - CAM_COMM_ID, AFDAsyncExtraInfo, - AFDAsyncTransferState, CAMAsyncAFDConnector, build_async_topology, ) +from afd_plugin.connectors.npu.async_cam_runtime import ( + async_cam_session as session_module, # noqa: E402 +) +from tests.unit.fakes.async_cam import ( # noqa: E402 + _afd_config, + _dp2_afd_config, + _FakeTensor, + _FakeTorch, + _topk_payload, + cam_vllm_config, +) from tests.unit.fakes.async_cam_startup import MemoryStore # noqa: E402 +def _vllm_config(**kwargs): + return cam_vllm_config(dtype=torch.bfloat16, **kwargs) + + class _FakeTensorLike: def __init__(self, name, *, shape=None, device="npu:0"): self.name = name @@ -44,132 +54,6 @@ def __getitem__(self, item): return f"{self.name}[{start}:{stop}]" -class _FakeTensor: - def __init__(self, shape, *, dtype="bf16", device="npu:0"): - self.shape = tuple(shape) - self.dtype = dtype - self.device = device - - def to(self, *, dtype): - self.dtype = dtype - return self - - def contiguous(self): - return self - - def new_zeros(self, shape): - return _FakeTensor(shape, dtype=self.dtype, device=self.device) - - def new_empty(self, shape): - return _FakeTensor(shape, dtype=self.dtype, device=self.device) - - -class _FakeCamOps: - def __init__(self): - self.calls = [] - - def afd_async_dispatch_send(self, *args): - self.calls.append(("dispatch_send", args)) - return args[0] - - def afd_async_dispatch_recv(self, *args): - self.calls.append(("dispatch_recv", args)) - batch_size = args[3] - hidden_size = args[4] - expert_per_rank = args[8] - tp_size = args[11] - return ( - _FakeTensor((batch_size, hidden_size)), - _FakeTensor((batch_size,), dtype="fp32"), - _FakeTensor((5 + tp_size * (1 + expert_per_rank),), dtype="int64"), - _FakeTensor((expert_per_rank,), dtype="int64"), - ) - - def afd_async_combine_send(self, *args): - self.calls.append(("combine_send", args)) - return args[0] - - def afd_async_combine_recv(self, *args): - self.calls.append(("combine_recv", args)) - batch_size = args[5] - hidden_size = args[6] - return _FakeTensor((batch_size, hidden_size)) - - afd_mega_dispatch_send = afd_async_dispatch_send - afd_mega_combine_recv = afd_async_combine_recv - - -class _FakeTorch: - def __init__(self): - self.bfloat16 = "bf16" - self.float16 = "fp16" - self.float32 = "fp32" - self.int32 = "int32" - self.int64 = "int64" - self.ops = SimpleNamespace(afd_ascend=_FakeCamOps()) - - def device(self, name): - return name - - def empty(self, shape, *, dtype, device): - return _FakeTensor(shape, dtype=dtype, device=device) - - def zeros(self, shape, *, dtype, device): - return _FakeTensor(shape, dtype=dtype, device=device) - - -def _vllm_config(*, tp_size: int = 1, pcp_size: int = 1, extra_config=None): - return SimpleNamespace( - additional_config={ - "afd": { - "connector_extra_config": extra_config - if extra_config is not None - else {"attn_ranks_per_dp": 4}, - }, - }, - parallel_config=SimpleNamespace( - data_parallel_size=1, - data_parallel_rank=0, - prefill_context_parallel_size=pcp_size, - tensor_parallel_size=tp_size, - ), - scheduler_config=SimpleNamespace(max_num_batched_tokens=16), - model_config=SimpleNamespace( - dtype=torch.bfloat16, - hf_config=SimpleNamespace( - hidden_size=16, - num_experts_per_tok=2, - n_routed_experts=8, - ), - ), - ) - - -def _afd_config(*, role: str): - return AFDConfig( - connector="CAMAsyncAFDConnector", - role=role, - num_attention_ranks=4, - num_ffn_ranks=2, - ) - - -def _dp2_afd_config(*, role: str): - return AFDConfig( - connector="CAMAsyncAFDConnector", - role=role, - num_attention_ranks=8, - num_ffn_ranks=8, - ) - - -def _topk_payload(batch_size: int, topk: int = 2): - return { - "topk_ids": _FakeTensor((batch_size, topk), dtype="int32"), - "topk_weights": _FakeTensor((batch_size, topk), dtype="fp32"), - } - - def test_async_extra_info_rejects_unknown_and_invalid_fields(): with pytest.raises(ValueError, match="unknown AFD async connector_extra_config"): AFDAsyncExtraInfo.from_mapping({"core_num": 8}) @@ -305,152 +189,6 @@ def test_async_topology_uses_one_world_for_all_dp_replicas( assert topology.expert_per_rank == 4 -def test_async_connector_calls_cam_shaped_ops(monkeypatch): - fake_torch = _FakeTorch() - monkeypatch.setattr(async_cam_module, "torch", fake_torch) - connector = CAMAsyncAFDConnector( - 0, - 0, - _vllm_config( - pcp_size=3, - extra_config={"attn_ranks_per_dp": 4}, - ), - _afd_config(role="attention"), - 0, - ) - connector._initialized = True - connector.comm_args = _FakeTensor((1,), dtype="fp16") - connector._placeholder = _FakeTensor((8, 16)) - hidden_states = _FakeTensor((3, 16)) - metadata = AFDTransferMetadata.create_attention_metadata( - layer_idx=2, - stage_idx=0, - seq_len=3, - ) - context = AFDTransferContext(metadata=metadata) - - output = connector.send_attn_output( - hidden_states, - context, - **_topk_payload(3), - ) - combined = connector.recv_ffn_output( - ref_tensor=hidden_states, - ubatch_idx=0, - ) - - assert output is None - assert combined.shape == (3, 16) - assert fake_torch.ops.afd_ascend.calls[0][0] == "dispatch_send" - assert fake_torch.ops.afd_ascend.calls[1][0] == "combine_recv" - assert fake_torch.ops.afd_ascend.calls[0][1][3] == CAM_COMM_ID - assert fake_torch.ops.afd_ascend.calls[1][1][4] == CAM_COMM_ID - assert fake_torch.ops.afd_ascend.calls[0][1][5:11] == (3, 16, 2, 2, 4, 4) - assert fake_torch.ops.afd_ascend.calls[1][1][5:11] == (3, 16, 2, 2, 4, 4) - assert fake_torch.ops.afd_ascend.calls[0][1][14] == 4 - assert isinstance(context.states, AFDAsyncTransferState) - assert isinstance(context.states, AFDTransferState) - - -def test_async_explicit_receive_consumes_matching_pending_dispatch(monkeypatch): - fake_torch = _FakeTorch() - monkeypatch.setattr(async_cam_module, "torch", fake_torch) - connector = CAMAsyncAFDConnector( - 0, 0, _vllm_config(), _afd_config(role="attention"), 0 - ) - connector._initialized = True - connector.comm_args = _FakeTensor((1,), dtype="fp16") - hidden_states = _FakeTensor((3, 16)) - contexts = [] - payloads = [] - for layer_idx in (2, 3): - context = AFDTransferContext( - metadata=AFDTransferMetadata.create_attention_metadata( - layer_idx=layer_idx, stage_idx=0, seq_len=3 - ) - ) - payload = _topk_payload(3) - connector.send_attn_output(hidden_states, context, **payload) - contexts.append(context) - payloads.append(payload) - - with pytest.raises(RuntimeError, match="does not match the next pending"): - connector.recv_ffn_output(hidden_states, ubatch_idx=0, context=contexts[1]) - assert len(fake_torch.ops.afd_ascend.calls) == 2 - - connector.recv_ffn_output( - hidden_states, - ubatch_idx=0, - context=contexts[0], - **payloads[1], - ) - assert fake_torch.ops.afd_ascend.calls[-1][1][1] is payloads[0]["topk_ids"] - - connector.recv_ffn_output(hidden_states, ubatch_idx=0) - assert fake_torch.ops.afd_ascend.calls[-1][1][1] is payloads[1]["topk_ids"] - - -def test_async_ffn_side_dispatch_recv_and_combine_send(monkeypatch): - fake_torch = _FakeTorch() - monkeypatch.setattr(async_cam_module, "torch", fake_torch) - connector = CAMAsyncAFDConnector( - 0, - 0, - _vllm_config( - pcp_size=2, - extra_config={"attn_ranks_per_dp": 2}, - ), - _afd_config(role="ffn"), - 0, - ) - connector._initialized = True - connector.comm_args = _FakeTensor((1,), dtype="fp16") - connector._placeholder = _FakeTensor((8, 16)) - - recv_output = connector.recv_attn_output(batch_size=4, layer_idx=1) - connector.send_ffn_output(recv_output.hidden_states, recv_output.context) - - states = recv_output.context.states - assert recv_output.hidden_states.shape == (connector.max_num_batched_tokens, 16) - assert states.dynamic_scales.shape == (connector.max_num_batched_tokens,) - assert states.group_list.shape == (connector.topology.expert_per_rank,) - assert fake_torch.ops.afd_ascend.calls[0][0] == "dispatch_recv" - assert fake_torch.ops.afd_ascend.calls[1][0] == "combine_send" - assert fake_torch.ops.afd_ascend.calls[0][1][11] == 2 - assert fake_torch.ops.afd_ascend.calls[1][1][12] == 2 - assert fake_torch.ops.afd_ascend.calls[1][1][2] is states.token_nums_rankid_layeridx - - -def test_async_combine_send_requires_dispatch_recv_token_metadata(monkeypatch): - fake_torch = _FakeTorch() - monkeypatch.setattr(async_cam_module, "torch", fake_torch) - connector = CAMAsyncAFDConnector( - 0, - 0, - _vllm_config(), - _afd_config(role="ffn"), - 0, - ) - connector._initialized = True - metadata = AFDTransferMetadata.create_ffn_metadata( - layer_idx=1, - stage_idx=0, - seq_lens=[4], - ) - # token_nums_rankid_layeridx is left unset (None) so combine send must fail. - states = AFDAsyncTransferState( - batch_size=4, - hidden_size=16, - topk=2, - layer_idx=1, - group_list=_FakeTensor((4,), dtype="int64"), - ) - context = AFDTransferContext(metadata=metadata, states=states) - - with pytest.raises(RuntimeError, match="TokenNums_Rankid_Layeridx"): - connector.send_ffn_output(_FakeTensor((4, 16)), context) - - def test_async_select_experts_rejects_mix_placement(): with pytest.raises(RuntimeError, match="routed-only.*mix_placement"): async_cam_module.select_cam_experts( @@ -517,51 +255,6 @@ def test_async_rejects_fused_shared_expert_ids(): ) -@pytest.mark.parametrize("mega", [False, True]) -def test_dispatch_failure_preserves_protocol_window_ownership(monkeypatch, mega): - fake_torch = _FakeTorch() - monkeypatch.setattr(async_cam_module, "torch", fake_torch) - connector = CAMAsyncAFDConnector( - 0, - 0, - _vllm_config( - extra_config={ - "attn_ranks_per_dp": 4, - "ffn_mega_moe": mega, - "dynamicQuant": 1, - } - ), - _afd_config(role="attention"), - 0, - ) - connector._initialized = True - connector.comm_args = _FakeTensor((1,), dtype="fp16") - context = AFDTransferContext( - metadata=AFDTransferMetadata.create_attention_metadata( - layer_idx=2, - stage_idx=0, - seq_len=3, - ) - ) - - def fail_dispatch(*_args): - raise RuntimeError("dispatch failed") - - monkeypatch.setattr( - fake_torch.ops.afd_ascend, - "afd_mega_dispatch_send" if mega else "afd_async_dispatch_send", - fail_dispatch, - ) - with pytest.raises(RuntimeError, match="dispatch failed"): - connector.send_attn_output(_FakeTensor((3, 16)), context, **_topk_payload(3)) - if mega: - with pytest.raises(RuntimeError, match="pending"): - connector.close() - else: - connector.close() - assert not connector.is_initialized - - @pytest.fixture def pull_attention(monkeypatch): # CPU status words and host transfer recording, not NPU validation. @@ -583,7 +276,8 @@ def pull_attention(monkeypatch): ) connector._initialized = True state = torch.zeros(128, dtype=torch.int64) - connector.comm_args = state.view(torch.float16) + monkeypatch.setitem(sys.modules, "torch", fake_torch) + connector.ffn_session().comm_args = state.view(torch.float16) store = MemoryStore() connector.graph_startup._store = store connector.graph_startup._prefix = "ffn_graph/test/" @@ -613,19 +307,19 @@ def test_pull_combine_failure_retains_window_until_completion( connector.send_attn_output(h.hidden, h.context, **_topk_payload(3)) combine = h.torch.ops.afd_ascend.afd_mega_combine_recv - def fail(*_args): + def fail(*_args, **_kwargs): raise RuntimeError("combine failed") if phase == "launch": monkeypatch.setattr(h.torch.ops.afd_ascend, "afd_mega_combine_recv", fail) else: - h.state[async_cam_module.PULL_FAILURE_INDEX] = -5 + h.state[session_module.PULL_FAILURE_INDEX] = -5 with pytest.raises(RuntimeError, match="combine failed|status -5"): connector.recv_ffn_output(h.hidden, context=h.context) with pytest.raises(RuntimeError, match="pending"): connector.close() monkeypatch.setattr(h.torch.ops.afd_ascend, "afd_mega_combine_recv", combine) - h.state[async_cam_module.PULL_FAILURE_INDEX] = 0 + h.state[session_module.PULL_FAILURE_INDEX] = 0 connector.recv_ffn_output(h.hidden, context=h.context) connector.close() assert not connector.is_initialized and connector.comm_args is None @@ -637,7 +331,7 @@ def test_attention_step_reports_status_after_queued_combine(pull_attention, nest connector = h.connector with pytest.raises(RuntimeError, match="status -5"), connector.attention_step(): connector.send_attn_output(h.hidden, h.context, **_topk_payload(3)) - h.state[async_cam_module.PULL_FAILURE_INDEX] = -5 + h.state[session_module.PULL_FAILURE_INDEX] = -5 if nested: with connector.attention_step(): output = connector.recv_ffn_output(h.hidden, context=h.context) @@ -648,7 +342,7 @@ def test_attention_step_reports_status_after_queued_combine(pull_attention, nest connector.send_attn_output(h.hidden, h.context, **_topk_payload(3)) with pytest.raises(RuntimeError, match="status -5"): connector.recv_ffn_output(h.hidden, context=h.context) - h.state[async_cam_module.PULL_FAILURE_INDEX] = 0 + h.state[session_module.PULL_FAILURE_INDEX] = 0 connector.recv_ffn_output(h.hidden, context=h.context) connector.close() assert not connector.is_initialized @@ -659,36 +353,11 @@ def test_attention_step_restores_status_checks_after_model_error(pull_attention) with pytest.raises(ValueError, match="model failed"), h.connector.attention_step(): raise ValueError("model failed") h.connector.send_attn_output(h.hidden, h.context, **_topk_payload(3)) - h.state[async_cam_module.PULL_FAILURE_INDEX] = -5 + h.state[session_module.PULL_FAILURE_INDEX] = -5 with pytest.raises(RuntimeError, match="status -5"): h.connector.recv_ffn_output(h.hidden, context=h.context) -def test_close_releases_every_stage_routing(monkeypatch): - fake_torch = _FakeTorch() - monkeypatch.setattr(async_cam_module, "torch", fake_torch) - connector = CAMAsyncAFDConnector( - 0, 0, _vllm_config(), _afd_config(role="attention"), 0 - ) - connector._initialized = True - connector.comm_args = _FakeTensor((1,), dtype="fp16") - references: list[weakref.ReferenceType] = [] - for stage in (0, 1): - context = AFDTransferContext( - metadata=AFDTransferMetadata.create_attention_metadata( - layer_idx=2, stage_idx=stage, seq_len=3 - ) - ) - payload = _topk_payload(3) - references.extend(weakref.ref(tensor) for tensor in payload.values()) - connector.send_attn_output(_FakeTensor((3, 16)), context, **payload) - del payload, context - fake_torch.ops.afd_ascend.calls.clear() - assert all(reference() is not None for reference in references) - connector.close() - assert all(reference() is None for reference in references) - - def test_async_routing_preserves_supplied_router_results(): expected_weights = torch.tensor([[1.5, 0.5]]) expected_ids = torch.tensor([[2, 6]], dtype=torch.int32) diff --git a/tests/unit/connectors/test_base.py b/tests/unit/connectors/test_base.py new file mode 100644 index 000000000..d34924d8e --- /dev/null +++ b/tests/unit/connectors/test_base.py @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Host FFN lifecycle defaults shared by connector implementations.""" + +import pytest + +from afd_plugin.connectors.base import FFNExecutionSession + + +@pytest.mark.parametrize("shutdown_requested", [False, True]) +def test_default_session_stops_locally_without_drain(shutdown_requested): + session = FFNExecutionSession() + assert session.keep_serving(shutdown_requested) is not shutdown_requested + assert not session.requires_drain + session.before_first_step() + session.after_step() + session.drain(lambda: pytest.fail("Default session does not drain remote work")) + session.finish_receiver() + session.validate_shutdown() diff --git a/tests/unit/fakes/async_cam.py b/tests/unit/fakes/async_cam.py new file mode 100644 index 000000000..9996be0a6 --- /dev/null +++ b/tests/unit/fakes/async_cam.py @@ -0,0 +1,234 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Host recording boundary for CAM native messages; no device simulation.""" + +import sys +from inspect import Parameter, Signature +from types import SimpleNamespace + +from afd_plugin.config import AFDConfig + + +class _FakeTensor: + def __init__(self, shape, *, dtype="bf16", device="npu:0"): + self.shape = (shape,) if isinstance(shape, int) else tuple(shape) + self.dtype = dtype + self.device = device + + def to(self, *, dtype): + self.dtype = dtype + return self + + def contiguous(self): + return self + + def new_zeros(self, shape): + return _FakeTensor(shape, dtype=self.dtype, device=self.device) + + def new_empty(self, shape): + return _FakeTensor(shape, dtype=self.dtype, device=self.device) + + +class FakePullState(_FakeTensor): + """Host status words at the native tensor boundary; no device execution.""" + + def __init__(self): + super().__init__((128,), dtype="fp16") + self.words = {32: -1} + + def view(self, dtype): + return self + + def __getitem__(self, index): + return SimpleNamespace( + item=lambda: self.words.get(index, 0), + fill_=lambda value: self.words.__setitem__(index, value), + ) + + +class _FakeCamOps: + def __init__(self): + self.calls = [] + + def _record(self, name, fields, args, kwargs): + # Native schema binding accepts either positional or named arguments. + signature = Signature( + [ + Parameter(field, Parameter.POSITIONAL_OR_KEYWORD) + for field in fields.split() + ] + ) + values = tuple(signature.bind(*args, **kwargs).arguments.values()) + self.calls.append((name, values)) + return values + + def afd_mega_pull_init(self, *args): + self.calls.append(("pull_init", args)) + return FakePullState() + + def afd_async_dispatch_send(self, *args, **kwargs): + values = self._record( + "dispatch_send", + "x expert_ids comm_args comm_id max_seq_len batch_size hidden_size " + "top_k moe_rank_num attn_rank_num route_expert_num_per_moe " + "attn_rank_id world_size layer_index tp_size dynamic_quant group_name", + args, + kwargs, + ) + return values[0] + + def afd_async_dispatch_recv(self, *args, **kwargs): + values = self._record( + "dispatch_recv", + "x comm_args comm_id max_seq_len hidden_size top_k moe_rank_num " + "attn_rank_num route_expert_num_per_moe moe_rank_id world_size " + "tp_size dynamic_quant group_name", + args, + kwargs, + ) + batch_size, hidden_size, expert_per_rank, tp_size = ( + values[3], + values[4], + values[8], + values[11], + ) + return ( + _FakeTensor((batch_size, hidden_size)), + _FakeTensor((batch_size,), dtype="fp32"), + _FakeTensor((5 + tp_size * (1 + expert_per_rank),), dtype="int64"), + _FakeTensor((expert_per_rank,), dtype="int64"), + ) + + def afd_async_combine_send(self, *args, **kwargs): + values = self._record( + "combine_send", + "expand_x comm_args batch_info comm_id max_seq_len hidden_size " + "top_k moe_rank_num attn_rank_num route_expert_num_per_moe " + "moe_rank_id world_size tp_size group_name", + args, + kwargs, + ) + return values[0] + + def afd_async_combine_recv(self, *args, **kwargs): + values = self._record( + "combine_recv", + "expand_x expert_ids expert_scales comm_args comm_id batch_size " + "hidden_size top_k moe_rank_num attn_rank_num route_expert_num_per_moe " + "attn_rank_id world_size group_name", + args, + kwargs, + ) + return _FakeTensor((values[5], values[6])) + + def afd_mega_dispatch_send(self, *args, **kwargs): + result = self.afd_async_dispatch_send(*args, **kwargs) + self.calls[-1] = ("mega_dispatch_send", self.calls[-1][1]) + return result + + def afd_mega_combine_recv(self, *args, max_seq_len, tp_size, **kwargs): + result = self.afd_async_combine_recv(*args, **kwargs) + self.calls[-1] = ( + "mega_combine_recv", + self.calls[-1][1] + (max_seq_len, tp_size), + ) + return result + + +class _FakeTorch: + def __init__(self): + self.bfloat16 = "bf16" + self.float16 = "fp16" + self.float32 = "fp32" + self.int32 = "int32" + self.int64 = "int64" + self.ops = SimpleNamespace(afd_ascend=_FakeCamOps()) + self.npu = SimpleNamespace(synchronize=lambda: None) + + def device(self, name): + return name + + def empty(self, shape, *, dtype, device): + return _FakeTensor(shape, dtype=dtype, device=device) + + def zeros(self, shape, *, dtype, device): + return _FakeTensor(shape, dtype=dtype, device=device) + + +def cam_vllm_config(*, tp_size=1, pcp_size=1, extra_config=None, dtype="bf16"): + return SimpleNamespace( + additional_config={ + "afd": { + "connector_extra_config": extra_config + if extra_config is not None + else {"attn_ranks_per_dp": 4}, + }, + }, + parallel_config=SimpleNamespace( + data_parallel_size=1, + data_parallel_rank=0, + prefill_context_parallel_size=pcp_size, + tensor_parallel_size=tp_size, + ), + scheduler_config=SimpleNamespace(max_num_batched_tokens=16), + model_config=SimpleNamespace( + dtype=dtype, + hf_config=SimpleNamespace( + hidden_size=16, + num_experts_per_tok=2, + n_routed_experts=8, + ), + ), + ) + + +def _afd_config(*, role: str): + return AFDConfig( + connector="CAMAsyncAFDConnector", + role=role, + num_attention_ranks=4, + num_ffn_ranks=2, + ) + + +def _dp2_afd_config(*, role: str): + return AFDConfig( + connector="CAMAsyncAFDConnector", + role=role, + num_attention_ranks=8, + num_ffn_ranks=8, + ) + + +def _topk_payload(batch_size: int, topk: int = 2): + return { + "topk_ids": _FakeTensor((batch_size, topk), dtype="int32"), + "topk_weights": _FakeTensor((batch_size, topk), dtype="fp32"), + } + + +def recording_cam_connector( + monkeypatch, module, *, role="attention", mega=False, tp_size=4, pcp_size=1 +): + fake_torch = _FakeTorch() + monkeypatch.setattr(module, "torch", fake_torch) + monkeypatch.setitem(sys.modules, "torch", fake_torch) + connector = module.CAMAsyncAFDConnector( + 0, + 0, + cam_vllm_config( + pcp_size=pcp_size, + extra_config={ + "attn_ranks_per_dp": tp_size, + "ffn_mega_moe": mega, + "dynamicQuant": 1, + }, + ), + _afd_config(role=role), + 0, + ) + monkeypatch.setattr(connector.graph_startup, "phase", lambda phase: None) + connector.ffn_session().initialize("npu:0", "recorded") + connector._initialized = True + fake_torch.ops.afd_ascend.calls.clear() + return SimpleNamespace(connector=connector, torch=fake_torch) diff --git a/tests/unit/fakes/async_ffn.py b/tests/unit/fakes/async_ffn.py index a11af183f..b87b880e5 100644 --- a/tests/unit/fakes/async_ffn.py +++ b/tests/unit/fakes/async_ffn.py @@ -46,7 +46,8 @@ def stub(name, **attributes): "afd_plugin.v1.worker.ffn_metadata", ): stub(name) - stub("afd_plugin.connectors.metadata", AFDTransferState=object) + stub("vllm.forward_context") + load_host_module(monkeypatch, "afd_plugin.connectors.metadata") load_host_module(monkeypatch, "afd_plugin.connectors.base") load_host_module(monkeypatch, "tests.unit.fakes.connector") group = load_host_module(monkeypatch, "afd_plugin.distributed.afd_process_group") diff --git a/tests/unit/fakes/connector.py b/tests/unit/fakes/connector.py index fa158f5e0..7222e2270 100644 --- a/tests/unit/fakes/connector.py +++ b/tests/unit/fakes/connector.py @@ -18,9 +18,10 @@ AFDTransferContext, AFDTransferMetadata, ) +from afd_plugin.connectors.base import FFNExecutionSession -class FakeFFNSession: +class FakeFFNSession(FFNExecutionSession): """Configurable host lifecycle seam, with no device behavior.""" def __init__( @@ -40,12 +41,6 @@ def __init__( def keep_serving(self, shutdown_requested: bool) -> bool: return next(self.serving, not shutdown_requested) - def before_first_step(self) -> None: - pass - - def after_step(self) -> None: - pass - def drain(self, run_one: Callable[[], None]) -> None: for _ in range(self.transactions): run_one() diff --git a/tests/unit/v1/worker/test_async_ffn_graph.py b/tests/unit/v1/worker/test_async_ffn_graph.py index d4ee66169..888119ac9 100644 --- a/tests/unit/v1/worker/test_async_ffn_graph.py +++ b/tests/unit/v1/worker/test_async_ffn_graph.py @@ -12,7 +12,7 @@ pytest.importorskip("torch_npu") import torch # noqa: E402 -from afd_plugin.connectors.base import DefaultFFNSession # noqa: E402 +from afd_plugin.connectors.base import FFNExecutionSession # noqa: E402 from afd_plugin.v1.worker.npu.async_ffn_model_runner import ( # noqa: E402 AsyncNPUFFNModelRunner, ) @@ -79,7 +79,7 @@ def send(output, context): connector.recv_attn_output = receive connector.send_ffn_output = send - connector.ffn_session = DefaultFFNSession + connector.ffn_session = FFNExecutionSession session = AsyncNPUFFNModelRunner.__new__(AsyncNPUFFNModelRunner) session.model = torch.nn.Linear(1, 1) session._layered_executor = execute diff --git a/tests/unit/v1/worker/test_npu_runtime.py b/tests/unit/v1/worker/test_npu_runtime.py index e4f98d5fb..552bed742 100644 --- a/tests/unit/v1/worker/test_npu_runtime.py +++ b/tests/unit/v1/worker/test_npu_runtime.py @@ -23,7 +23,7 @@ AFDTransferMetadata, AFDTransferState, ) -from afd_plugin.connectors.base import DefaultFFNSession +from afd_plugin.connectors.base import FFNExecutionSession from tests.unit.fakes.dp_metadata import ( # noqa: E402 DPMetadataValues as _FakeDPMetadata, ) @@ -839,7 +839,7 @@ def execute_connector_driven_step(shutdown_requested): worker.device = SimpleNamespace(type="cpu") worker.model_runner = object.__new__(ffn_worker.AsyncNPUFFNModelRunner) worker.model_runner.connector = SimpleNamespace(control_plane=None) - worker.model_runner.session = DefaultFFNSession() + worker.model_runner.session = FFNExecutionSession() worker.model_runner.effective_mode = "eager" worker.model_runner.execute_connector_driven_step = execute_connector_driven_step From 717ca64f1722eae7ae65cdb3f81bec403846b042 Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:56:14 +0200 Subject: [PATCH 11/42] Share FFN launcher environment checks Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- afd_plugin/ffn_faas/platforms/gpu/adapter.py | 37 ++++++----------- afd_plugin/ffn_faas/platforms/launch.py | 42 ++++++++++++++++++++ afd_plugin/ffn_faas/platforms/npu/adapter.py | 32 ++++++--------- 3 files changed, 68 insertions(+), 43 deletions(-) create mode 100644 afd_plugin/ffn_faas/platforms/launch.py diff --git a/afd_plugin/ffn_faas/platforms/gpu/adapter.py b/afd_plugin/ffn_faas/platforms/gpu/adapter.py index 3f6f5197d..319800051 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/adapter.py +++ b/afd_plugin/ffn_faas/platforms/gpu/adapter.py @@ -9,7 +9,6 @@ from __future__ import annotations -import os from argparse import ArgumentParser, Namespace from functools import partial from typing import TYPE_CHECKING @@ -24,12 +23,14 @@ GPUFFNConfig, parse_connector_extra_config, ) +from afd_plugin.ffn_faas.platforms.launch import ( + read_ffn_launcher_environment, + require_v2_model_runner, +) if TYPE_CHECKING: from afd_plugin.ffn_faas.interfaces import FFNBackend -V2_MODEL_RUNNER_ENV = "VLLM_USE_V2_MODEL_RUNNER" - # NPU parallel: afd_plugin/ffn_faas/platforms/npu/adapter.py::create_backend # (bootstraps the Ascend OPP environment, then imports the backend). @@ -50,11 +51,9 @@ def create_backend( # NPU parallel: afd_plugin/ffn_faas/platforms/npu/adapter.py::AscendCAMAdapter. -# Reuse: PARTIAL - the RANK/LOCAL_RANK/WORLD_SIZE parsing, the WORLD_SIZE == -# --ffn-ranks check and the ModelRunnerV2 environment check are the same; -# the option set differs because GPU adopts Attention TP, capacity and the -# bucket ladder from the M2N handshake. -# REFACTOR-CANDIDATE(R2): a shared F-only launcher-environment reader. +# Reuse: YES - the shared launch module owns the F-only torchrun and +# ModelRunnerV2 checks. The option set still differs because GPU adopts +# Attention TP, capacity and the bucket ladder from the M2N handshake. class GPUDeepEPM2NAdapter: """Keep GPU, topology and connector options out of the host loop.""" @@ -106,24 +105,14 @@ def configure(self, args: Namespace) -> FFNLaunch: experts divide across the F ranks. Raises ``ValueError`` or ``OSError``, which the entry point reports as a usage error. """ - rank, local_rank, world = ( - int(os.environ.get(key, default)) - for key, default in ( - ("RANK", "0"), - ("LOCAL_RANK", "0"), - ("WORLD_SIZE", "1"), - ) - ) - if world != args.ffn_ranks: - raise ValueError("F-only launcher WORLD_SIZE must equal --ffn-ranks") - if os.environ.get(V2_MODEL_RUNNER_ENV) != "1": - raise ValueError(f"set {V2_MODEL_RUNNER_ENV}=1 on both A and F") + launcher = read_ffn_launcher_environment(args.ffn_ranks) + require_v2_model_runner() settings = GPUFFNConfig( model=args.model, attention_ranks=args.attention_ranks, ffn_ranks=args.ffn_ranks, - rank=rank, - local_rank=local_rank, + rank=launcher.rank, + local_rank=launcher.local_rank, host=args.host, port=args.port, graph=args.graph, @@ -137,8 +126,8 @@ def configure(self, args: Namespace) -> FFNLaunch: checkpoint = Qwen3CheckpointDescription.read(args.model) checkpoint.local_experts(settings.ffn_ranks) return FFNLaunch( - rank=rank, - local_rank=local_rank, + rank=launcher.rank, + local_rank=launcher.local_rank, model_path=str(checkpoint.root), backend_name=DEEPEP_M2N_CONNECTOR, shutdown_timeout=settings.shutdown_timeout, diff --git a/afd_plugin/ffn_faas/platforms/launch.py b/afd_plugin/ffn_faas/platforms/launch.py new file mode 100644 index 000000000..713957999 --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/launch.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Shared CPU-only launch-environment checks for standalone FFN backends.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass + +V2_MODEL_RUNNER_ENV = "VLLM_USE_V2_MODEL_RUNNER" + + +@dataclass(frozen=True) +class FFNLauncherEnvironment: + """One rank's externally launched F-only world coordinates.""" + + rank: int + local_rank: int + world_size: int + local_world_size: int + + +def read_ffn_launcher_environment(ffn_ranks: int) -> FFNLauncherEnvironment: + """Read ``torchrun`` coordinates and require an F-only world.""" + rank = int(os.environ.get("RANK", "0")) + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + world_size = int(os.environ.get("WORLD_SIZE", "1")) + local_world_size = int(os.environ.get("LOCAL_WORLD_SIZE", str(world_size))) + if world_size != ffn_ranks: + raise ValueError("F-only launcher WORLD_SIZE must equal --ffn-ranks") + return FFNLauncherEnvironment( + rank=rank, + local_rank=local_rank, + world_size=world_size, + local_world_size=local_world_size, + ) + + +def require_v2_model_runner() -> None: + """Require the model-runner generation used by both A and F ranks.""" + if os.environ.get(V2_MODEL_RUNNER_ENV) != "1": + raise ValueError(f"set {V2_MODEL_RUNNER_ENV}=1 on both A and F") diff --git a/afd_plugin/ffn_faas/platforms/npu/adapter.py b/afd_plugin/ffn_faas/platforms/npu/adapter.py index c37346ff1..6f63ed441 100644 --- a/afd_plugin/ffn_faas/platforms/npu/adapter.py +++ b/afd_plugin/ffn_faas/platforms/npu/adapter.py @@ -10,6 +10,10 @@ from typing import TYPE_CHECKING from afd_plugin.ffn_faas.interfaces import FFNLaunch +from afd_plugin.ffn_faas.platforms.launch import ( + read_ffn_launcher_environment, + require_v2_model_runner, +) from afd_plugin.ffn_faas.platforms.npu import get_model from afd_plugin.ffn_faas.platforms.npu.settings import FFNConfig @@ -47,30 +51,20 @@ def add_arguments(self, parser: ArgumentParser) -> None: parser.add_argument("--hccl-buffer-size", type=int) def configure(self, args: Namespace) -> FFNLaunch: - rank, local_rank, world = ( - int(os.environ.get(key, default)) - for key, default in ( - ("RANK", "0"), - ("LOCAL_RANK", "0"), - ("WORLD_SIZE", "1"), - ) - ) - if world != args.ffn_ranks: - raise ValueError("F-only launcher WORLD_SIZE must equal --ffn-ranks") - local_world = int(os.environ.get("LOCAL_WORLD_SIZE", str(world))) + launcher = read_ffn_launcher_environment(args.ffn_ranks) + require_v2_model_runner() + local_world = launcher.local_world_size devices = ( tuple(map(int, args.devices.split(","))) if args.devices else tuple(range(local_world)) ) if ( - not 0 <= local_rank < len(devices) + not 0 <= launcher.local_rank < len(devices) or len(set(devices)) != len(devices) or min(devices) < 0 ): raise ValueError("--devices must supply distinct local NPU IDs") - if os.environ.get("VLLM_USE_V2_MODEL_RUNNER") != "1": - raise ValueError("set VLLM_USE_V2_MODEL_RUNNER=1 on both A and F") if os.environ.get("AFD_ASYNC_CAM_LAYERED_GMM") != "1": raise ValueError("standalone V4 requires AFD_ASYNC_CAM_LAYERED_GMM=1") settings = FFNConfig( @@ -78,9 +72,9 @@ def configure(self, args: Namespace) -> FFNLaunch: attention_ranks=args.attention_ranks, attention_tp=args.attention_tp, ffn_ranks=args.ffn_ranks, - rank=rank, - local_rank=local_rank, - device=devices[local_rank], + rank=launcher.rank, + local_rank=launcher.local_rank, + device=devices[launcher.local_rank], host=args.host, port=args.port, max_num_batched_tokens=args.max_num_batched_tokens, @@ -95,8 +89,8 @@ def configure(self, args: Namespace) -> FFNLaunch: ) model = get_model(args.model, settings.ffn_ranks, settings.rank) return FFNLaunch( - rank=rank, - local_rank=local_rank, + rank=launcher.rank, + local_rank=launcher.local_rank, model_path=str(model.root), backend_name="CAMAsyncAFDConnector", shutdown_timeout=settings.shutdown_timeout, From 288f323ca9ab82a679e75519d631bde5d37140c0 Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:57:31 +0200 Subject: [PATCH 12/42] Share standalone FFN launch validation Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- afd_plugin/ffn_faas/platforms/gpu/settings.py | 26 +++++++++---------- afd_plugin/ffn_faas/platforms/launch.py | 23 ++++++++++++++++ afd_plugin/ffn_faas/platforms/npu/settings.py | 25 +++++++++--------- 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/afd_plugin/ffn_faas/platforms/gpu/settings.py b/afd_plugin/ffn_faas/platforms/gpu/settings.py index e97f3877e..94c686a80 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/settings.py +++ b/afd_plugin/ffn_faas/platforms/gpu/settings.py @@ -12,7 +12,6 @@ from __future__ import annotations import json -import math from collections.abc import Mapping from dataclasses import dataclass, field from pathlib import Path @@ -20,6 +19,7 @@ from typing import Any from afd_plugin.config import DEEPEP_M2N_CONNECTOR, AFDConfig +from afd_plugin.ffn_faas.platforms.launch import validate_common_ffn_launch # The only qualified payload: BF16 rows against BF16 expert weights. BF16_PAYLOAD = "bf16" @@ -29,17 +29,14 @@ FFN_GRAPH_MODE = "FULL_DECODE_ONLY" EAGER_MODE = "eager" EXTERNAL_LAUNCHER = "external_launcher" -MAX_PORT = 65535 DEFAULT_AFD_PORT = AFDConfig.port DEFAULT_AFD_TIMEOUT_S = AFDConfig.afd_process_group_timeout_s # NPU parallel: afd_plugin/ffn_faas/platforms/npu/settings.py::FFNConfig. -# Reuse: PARTIAL - rank, host/port and shutdown-timeout validation are the same -# checks; NPU also owns the CAM dispatch envelope, Attention TP and core +# Reuse: PARTIAL - the launch module owns rank, rendezvous and shutdown +# validation; NPU also owns the CAM dispatch envelope, Attention TP and core # budgets, which GPU adopts from the M2N handshake instead. -# REFACTOR-CANDIDATE(R3): a shared base for the F-only launch fields and -# their validation. @dataclass(frozen=True) class GPUFFNConfig: """Frozen settings of one F rank in a standalone DeepEP M2N deployment.""" @@ -68,16 +65,17 @@ def __post_init__(self) -> None: ``--dbo`` implies (1 without, 2 with). Every other connector option is validated by ``DeepEPM2NConfig.from_mapping`` once the backend runs. """ - if self.attention_ranks <= 0 or self.ffn_ranks <= 0: - raise ValueError("Attention and FFN rank counts must be positive") - if not 0 <= self.rank < self.ffn_ranks or self.local_rank < 0: - raise ValueError("invalid F rank or local rank") - if not self.host or not 0 < self.port <= MAX_PORT: - raise ValueError("DeepEP M2N rendezvous requires a host and a valid port") + validate_common_ffn_launch( + attention_ranks=self.attention_ranks, + ffn_ranks=self.ffn_ranks, + rank=self.rank, + local_rank=self.local_rank, + host=self.host, + port=self.port, + shutdown_timeout=self.shutdown_timeout, + ) if self.afd_timeout_s <= 0: raise ValueError("--afd-timeout must be positive") - if not math.isfinite(self.shutdown_timeout) or self.shutdown_timeout <= 0: - raise ValueError("shutdown timeout must be finite and positive") payload_format = self.connector_extra_config.get("payload_format", BF16_PAYLOAD) if payload_format != BF16_PAYLOAD: raise ValueError( diff --git a/afd_plugin/ffn_faas/platforms/launch.py b/afd_plugin/ffn_faas/platforms/launch.py index 713957999..5673f9e79 100644 --- a/afd_plugin/ffn_faas/platforms/launch.py +++ b/afd_plugin/ffn_faas/platforms/launch.py @@ -4,10 +4,12 @@ from __future__ import annotations +import math import os from dataclasses import dataclass V2_MODEL_RUNNER_ENV = "VLLM_USE_V2_MODEL_RUNNER" +MAX_PORT = 65535 @dataclass(frozen=True) @@ -40,3 +42,24 @@ def require_v2_model_runner() -> None: """Require the model-runner generation used by both A and F ranks.""" if os.environ.get(V2_MODEL_RUNNER_ENV) != "1": raise ValueError(f"set {V2_MODEL_RUNNER_ENV}=1 on both A and F") + + +def validate_common_ffn_launch( + *, + attention_ranks: int, + ffn_ranks: int, + rank: int, + local_rank: int, + host: str, + port: int, + shutdown_timeout: float, +) -> None: + """Validate launch fields shared by every standalone FFN backend.""" + if attention_ranks <= 0 or ffn_ranks <= 0: + raise ValueError("Attention and FFN rank counts must be positive") + if not 0 <= rank < ffn_ranks or local_rank < 0: + raise ValueError("invalid F rank or local rank") + if not host or not 0 < port <= MAX_PORT: + raise ValueError("FFN rendezvous requires a host and a valid port") + if not math.isfinite(shutdown_timeout) or shutdown_timeout <= 0: + raise ValueError("shutdown timeout must be finite and positive") diff --git a/afd_plugin/ffn_faas/platforms/npu/settings.py b/afd_plugin/ffn_faas/platforms/npu/settings.py index cb78d996b..9c84fcbbb 100644 --- a/afd_plugin/ffn_faas/platforms/npu/settings.py +++ b/afd_plugin/ffn_faas/platforms/npu/settings.py @@ -4,11 +4,11 @@ from __future__ import annotations -import math from dataclasses import dataclass from pathlib import Path from afd_plugin.config import AFD_ASYNC_CONNECTOR +from afd_plugin.ffn_faas.platforms.launch import validate_common_ffn_launch @dataclass(frozen=True) @@ -34,28 +34,29 @@ class FFNConfig: def __post_init__(self) -> None: positive = ( - self.attention_ranks, self.attention_tp, - self.ffn_ranks, self.max_num_batched_tokens, self.max_num_seqs, ) if any(value <= 0 for value in positive): - raise ValueError( - "rank counts, TP and scheduler capacities must be positive" - ) + raise ValueError("Attention TP and scheduler capacities must be positive") if self.attention_ranks % self.attention_tp: raise ValueError("Attention ranks must be divisible by Attention TP") if self.max_num_batched_tokens % self.attention_tp: raise ValueError( "the CAM dispatch envelope must be divisible by Attention TP" ) - if not 0 <= self.rank < self.ffn_ranks or min(self.local_rank, self.device) < 0: - raise ValueError("invalid F rank or local device") - if not self.host or not 0 < self.port < 65536: - raise ValueError("CAM rendezvous requires a host and a valid port") - if not math.isfinite(self.shutdown_timeout) or self.shutdown_timeout <= 0: - raise ValueError("shutdown timeout must be finite and positive") + validate_common_ffn_launch( + attention_ranks=self.attention_ranks, + ffn_ranks=self.ffn_ranks, + rank=self.rank, + local_rank=self.local_rank, + host=self.host, + port=self.port, + shutdown_timeout=self.shutdown_timeout, + ) + if self.device < 0: + raise ValueError("invalid local NPU device") optional_positive = ( self.ffn_compute_aic_num, self.ffn_compute_aiv_num, From 4b20b9e118c112394d959574b224c88f9ca087cf Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:59:40 +0200 Subject: [PATCH 13/42] Share F-only FFN engine arguments Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- .../ffn_faas/platforms/gpu/environment.py | 7 +-- afd_plugin/ffn_faas/platforms/gpu/settings.py | 47 +++++++++---------- afd_plugin/ffn_faas/platforms/launch.py | 35 ++++++++++++++ afd_plugin/ffn_faas/platforms/npu/backend.py | 42 +++++++++-------- 4 files changed, 84 insertions(+), 47 deletions(-) diff --git a/afd_plugin/ffn_faas/platforms/gpu/environment.py b/afd_plugin/ffn_faas/platforms/gpu/environment.py index 65302d55b..9261ea50b 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/environment.py +++ b/afd_plugin/ffn_faas/platforms/gpu/environment.py @@ -14,7 +14,8 @@ import os from typing import TYPE_CHECKING -from afd_plugin.ffn_faas.platforms.gpu.settings import EXTERNAL_LAUNCHER, GPUFFNConfig +from afd_plugin.ffn_faas.platforms.gpu.settings import GPUFFNConfig +from afd_plugin.ffn_faas.platforms.launch import reassert_external_launcher if TYPE_CHECKING: import torch @@ -116,7 +117,7 @@ def initialize_device(self) -> None: # NPU parallel: CAMFFNBackend.prepare (EngineArgs, create_engine_config, # re-asserted external_launcher). - # Reuse: PARTIAL - see REFACTOR-CANDIDATE(R4) in settings.engine_kwargs. + # Reuse: YES - both use the shared F-only EngineArgs and launcher helpers. def build_config(self) -> VllmConfig: """Build the F-only ``VllmConfig`` exactly as the GPU FFN Worker sees it. @@ -140,7 +141,7 @@ def build_config(self) -> VllmConfig: config = EngineArgs(**self.settings.engine_kwargs()).create_engine_config() # The F world is the already launched torchrun world, whatever executor # platform defaults would pick for an ordinary engine. - config.parallel_config.distributed_executor_backend = EXTERNAL_LAUNCHER + reassert_external_launcher(config) config.model_config = get_afd_model_config( config.model_config, device_type="cuda" ) diff --git a/afd_plugin/ffn_faas/platforms/gpu/settings.py b/afd_plugin/ffn_faas/platforms/gpu/settings.py index 94c686a80..ec40874f2 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/settings.py +++ b/afd_plugin/ffn_faas/platforms/gpu/settings.py @@ -19,7 +19,10 @@ from typing import Any from afd_plugin.config import DEEPEP_M2N_CONNECTOR, AFDConfig -from afd_plugin.ffn_faas.platforms.launch import validate_common_ffn_launch +from afd_plugin.ffn_faas.platforms.launch import ( + ffn_engine_kwargs, + validate_common_ffn_launch, +) # The only qualified payload: BF16 rows against BF16 expert weights. BF16_PAYLOAD = "bf16" @@ -28,7 +31,6 @@ DBO_LANES = 2 FFN_GRAPH_MODE = "FULL_DECODE_ONLY" EAGER_MODE = "eager" -EXTERNAL_LAUNCHER = "external_launcher" DEFAULT_AFD_PORT = AFDConfig.port DEFAULT_AFD_TIMEOUT_S = AFDConfig.afd_process_group_timeout_s @@ -121,12 +123,9 @@ def afd_additional_config(self) -> dict[str, Any]: } # NPU parallel: afd_plugin/ffn_faas/platforms/npu/backend.py:: - # CAMFFNBackend.prepare builds its EngineArgs inline. - # Reuse: PARTIAL - external_launcher, DP = F ranks, TP = 1 and expert - # parallelism are identical; NPU adds Ascend quantization and the CAM - # dispatch envelope, GPU adds graph mode, DBO and the EP weight filter. - # REFACTOR-CANDIDATE(R4): one F-only EngineArgs builder with - # platform-specific extras. + # CAMFFNBackend.prepare builds its EngineArgs through the shared builder. + # Reuse: YES for the F-only topology; NPU adds Ascend quantization and the + # CAM dispatch envelope, GPU adds graph mode, DBO and the EP weight filter. def engine_kwargs(self) -> dict[str, Any]: """Keyword arguments for ``EngineArgs`` of this F-only rank. @@ -144,23 +143,21 @@ def engine_kwargs(self) -> dict[str, Any]: pin ``auto`` to Triton. The EP weight filter skips non-local expert tensors before they are read, as the Worker-path recipes do. """ - return { - "model": str(self.model), - "dtype": "bfloat16", - "skip_tokenizer_init": True, - "tensor_parallel_size": 1, - "data_parallel_size": self.ffn_ranks, - "data_parallel_rank": self.rank, - "enable_expert_parallel": True, - "enable_ep_weight_filter": True, - "distributed_executor_backend": EXTERNAL_LAUNCHER, - "enforce_eager": not self.graph, - "compilation_config": ( - {"cudagraph_mode": FFN_GRAPH_MODE} if self.graph else {} - ), - "enable_dbo": self.dbo, - "additional_config": self.afd_additional_config(), - } + return ffn_engine_kwargs( + model=self.model, + ffn_ranks=self.ffn_ranks, + ffn_rank=self.rank, + additional_config=self.afd_additional_config(), + platform_extras={ + "skip_tokenizer_init": True, + "enable_ep_weight_filter": True, + "enforce_eager": not self.graph, + "compilation_config": ( + {"cudagraph_mode": FFN_GRAPH_MODE} if self.graph else {} + ), + "enable_dbo": self.dbo, + }, + ) def parse_connector_extra_config(text: str) -> Mapping[str, Any]: diff --git a/afd_plugin/ffn_faas/platforms/launch.py b/afd_plugin/ffn_faas/platforms/launch.py index 5673f9e79..676b0a0bd 100644 --- a/afd_plugin/ffn_faas/platforms/launch.py +++ b/afd_plugin/ffn_faas/platforms/launch.py @@ -6,10 +6,17 @@ import math import os +from collections.abc import Mapping from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any V2_MODEL_RUNNER_ENV = "VLLM_USE_V2_MODEL_RUNNER" MAX_PORT = 65535 +EXTERNAL_LAUNCHER = "external_launcher" + +if TYPE_CHECKING: + from vllm.config import VllmConfig @dataclass(frozen=True) @@ -63,3 +70,31 @@ def validate_common_ffn_launch( raise ValueError("FFN rendezvous requires a host and a valid port") if not math.isfinite(shutdown_timeout) or shutdown_timeout <= 0: raise ValueError("shutdown timeout must be finite and positive") + + +def ffn_engine_kwargs( + *, + model: Path | str, + ffn_ranks: int, + ffn_rank: int, + additional_config: Mapping[str, Any], + platform_extras: Mapping[str, Any], +) -> dict[str, Any]: + """Build common ``EngineArgs`` keywords and merge backend-specific extras.""" + kwargs: dict[str, Any] = { + "model": str(model), + "dtype": "bfloat16", + "tensor_parallel_size": 1, + "data_parallel_size": ffn_ranks, + "data_parallel_rank": ffn_rank, + "enable_expert_parallel": True, + "distributed_executor_backend": EXTERNAL_LAUNCHER, + "additional_config": dict(additional_config), + } + kwargs.update(platform_extras) + return kwargs + + +def reassert_external_launcher(config: VllmConfig) -> None: + """Keep an already launched F-only world from borrowing an executor.""" + config.parallel_config.distributed_executor_backend = EXTERNAL_LAUNCHER diff --git a/afd_plugin/ffn_faas/platforms/npu/backend.py b/afd_plugin/ffn_faas/platforms/npu/backend.py index a18e40de2..fa5e146df 100644 --- a/afd_plugin/ffn_faas/platforms/npu/backend.py +++ b/afd_plugin/ffn_faas/platforms/npu/backend.py @@ -16,6 +16,10 @@ from afd_plugin.compat.npu import fix_all2all_backend_for_afd from afd_plugin.config import parse_afd_config from afd_plugin.connectors.npu.async_cam import CAMAsyncAFDConnector +from afd_plugin.ffn_faas.platforms.launch import ( + ffn_engine_kwargs, + reassert_external_launcher, +) from afd_plugin.ffn_faas.platforms.npu.environment import AscendFFNEnvironment from afd_plugin.ffn_faas.platforms.npu.model import CAMRoutedModel from afd_plugin.ffn_faas.platforms.npu.settings import FFNConfig @@ -36,29 +40,29 @@ def __init__(self, model: CAMRoutedModel, settings: FFNConfig) -> None: def prepare(self, service: FFNService) -> None: self.environment.initialize_device() args = EngineArgs( - model=str(self.settings.model), - trust_remote_code=True, - dtype="bfloat16", - quantization="ascend", - enforce_eager=not self.settings.full_graph, - compilation_config={"cudagraph_mode": "FULL"} - if self.settings.full_graph - else {}, - tensor_parallel_size=1, - data_parallel_size=self.settings.ffn_ranks, - data_parallel_rank=self.settings.rank, - enable_expert_parallel=True, - distributed_executor_backend="external_launcher", - # F owns a dispatch envelope, not a request-context/KV capacity. - max_model_len=self.settings.max_num_batched_tokens, - max_num_batched_tokens=self.settings.max_num_batched_tokens, - max_num_seqs=self.settings.max_num_seqs, - additional_config=self.settings.additional_config(), + **ffn_engine_kwargs( + model=self.settings.model, + ffn_ranks=self.settings.ffn_ranks, + ffn_rank=self.settings.rank, + additional_config=self.settings.additional_config(), + platform_extras={ + "trust_remote_code": True, + "quantization": "ascend", + "enforce_eager": not self.settings.full_graph, + "compilation_config": {"cudagraph_mode": "FULL"} + if self.settings.full_graph + else {}, + # F owns a dispatch envelope, not a request-context/KV capacity. + "max_model_len": self.settings.max_num_batched_tokens, + "max_num_batched_tokens": self.settings.max_num_batched_tokens, + "max_num_seqs": self.settings.max_num_seqs, + }, + ) ) config = args.create_engine_config() # Platform configuration can choose an executor for ordinary Workers; # this loader belongs to the already launched F-only torchrun world. - config.parallel_config.distributed_executor_backend = "external_launcher" + reassert_external_launcher(config) fix_all2all_backend_for_afd(config) self.environment.initialize_compute(config) with set_current_vllm_config(config): From ad89f42135a5dbc596ef00109e4637343e7210b5 Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Fri, 6 Nov 2026 10:46:36 +0200 Subject: [PATCH 14/42] Share FFN model attachment validation Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- .../worker/ffn_routed_experts_model_runner.py | 22 +++++++++---------- afd_plugin/v1/worker/model_attachment.py | 18 +++++++++++++++ .../v1/worker/npu/async_ffn_model_runner.py | 12 ++++++---- 3 files changed, 37 insertions(+), 15 deletions(-) create mode 100644 afd_plugin/v1/worker/model_attachment.py diff --git a/afd_plugin/v1/worker/ffn_routed_experts_model_runner.py b/afd_plugin/v1/worker/ffn_routed_experts_model_runner.py index 7178163fe..266bbdbc6 100644 --- a/afd_plugin/v1/worker/ffn_routed_experts_model_runner.py +++ b/afd_plugin/v1/worker/ffn_routed_experts_model_runner.py @@ -20,6 +20,9 @@ ) from afd_plugin.connectors.metadata import FP8_DTYPE from afd_plugin.v1.worker.ffn_model_runner import GPUFFNModelRunner +from afd_plugin.v1.worker.model_attachment import ( + validate_external_model_attachment, +) if TYPE_CHECKING: from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts @@ -154,11 +157,9 @@ def load_model(self, *, load_dummy_weights: bool = False, **kwargs: Any) -> None self._take_local_experts(self.get_model()) # NPU parallel: afd_plugin/v1/worker/npu/async_ffn_model_runner.py:: - # AsyncNPUFFNModelRunner.attach_model (same guard: before connector - # startup and at most once; NPU borrows layered W4A8 weight views). - # Reuse: PARTIAL - the attach-once/before-init guard is identical; the - # expert extraction is GPU-specific (FusedMoE Triton layout views). - # REFACTOR-CANDIDATE(R5): a shared attach guard for both runners. + # AsyncNPUFFNModelRunner.attach_model. + # Reuse: YES for the attachment guard; expert extraction is GPU-specific + # (FusedMoE Triton layout views). def attach_model( self, model: RoutedExpertsModel, *, model_memory_usage: int ) -> None: @@ -172,12 +173,11 @@ def attach_model( all-gathers these views in its startup collective, and only once, because the connector keeps no way to swap them afterwards. """ - if self.model is not None or self.connector.is_initialized: - raise RuntimeError( - "a routed FFN model must be attached once, before connector startup" - ) - if model_memory_usage < 0: - raise ValueError("model memory usage must be non-negative") + validate_external_model_attachment( + model_attached=self.model is not None, + connector_initialized=self.connector.is_initialized, + model_size_bytes=model_memory_usage, + ) self.model = model self.model_memory_usage = model_memory_usage self._take_local_experts(model) diff --git a/afd_plugin/v1/worker/model_attachment.py b/afd_plugin/v1/worker/model_attachment.py new file mode 100644 index 000000000..526610746 --- /dev/null +++ b/afd_plugin/v1/worker/model_attachment.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Shared validation for models attached outside a Worker model runner.""" + +from __future__ import annotations + + +def validate_external_model_attachment( + *, + model_attached: bool, + connector_initialized: bool, + model_size_bytes: int, +) -> None: + """Require an external model attachment before connector startup.""" + if model_attached or connector_initialized: + raise RuntimeError("FFN model must be attached once, before connector startup") + if model_size_bytes < 0: + raise ValueError("model memory usage must be non-negative") diff --git a/afd_plugin/v1/worker/npu/async_ffn_model_runner.py b/afd_plugin/v1/worker/npu/async_ffn_model_runner.py index 89b8d2345..76851bd10 100644 --- a/afd_plugin/v1/worker/npu/async_ffn_model_runner.py +++ b/afd_plugin/v1/worker/npu/async_ffn_model_runner.py @@ -33,6 +33,9 @@ AsyncCAMLayeredExecutor, LayeredFFNWeights, ) +from afd_plugin.v1.worker.model_attachment import ( + validate_external_model_attachment, +) from afd_plugin.v1.worker.npu.async_ffn_two_stream import AsyncFFNTwoStream from afd_plugin.validation import ( cam_receive_capacity, @@ -89,10 +92,11 @@ def __init__( def attach_model(self, model: FFNModel, *, parameter_bytes: int) -> None: """Borrow an independently loaded model without constructing a Worker.""" - if self.model is not None or self.connector.is_initialized: - raise RuntimeError("Async FFN model must be attached before CAM startup") - if parameter_bytes < 0: - raise ValueError("model parameter size must be nonnegative") + validate_external_model_attachment( + model_attached=self.model is not None, + connector_initialized=self.connector.is_initialized, + model_size_bytes=parameter_bytes, + ) self.model = model self.model_memory_usage = parameter_bytes From b57c2ab3ab3358ebafb1372d37567ddcb7e80bd5 Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Fri, 6 Nov 2026 10:46:38 +0200 Subject: [PATCH 15/42] Share standalone FFN environment invariants Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- afd_plugin/ffn_faas/platforms/environment.py | 124 ++++++++++++++++++ .../ffn_faas/platforms/gpu/environment.py | 70 +++------- .../ffn_faas/platforms/npu/environment.py | 64 +++------ 3 files changed, 158 insertions(+), 100 deletions(-) create mode 100644 afd_plugin/ffn_faas/platforms/environment.py diff --git a/afd_plugin/ffn_faas/platforms/environment.py b/afd_plugin/ffn_faas/platforms/environment.py new file mode 100644 index 000000000..cb5a8ea3c --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/environment.py @@ -0,0 +1,124 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Shared invariants for standalone FFN device and compute environments.""" + +from __future__ import annotations + +import os +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Generic, Protocol, TypeVar + +if TYPE_CHECKING: + import torch + from vllm.distributed.parallel_state import GroupCoordinator + + +class FFNEnvironmentSettings(Protocol): + """The standalone-environment subset shared by GPU and NPU settings.""" + + @property + def ffn_ranks(self) -> int: ... + + @property + def rank(self) -> int: ... + + +FFNEnvironmentSettingsT = TypeVar( + "FFNEnvironmentSettingsT", bound=FFNEnvironmentSettings +) + + +class FFNEnvironmentBase(ABC, Generic[FFNEnvironmentSettingsT]): + """Own common F-only environment state without owning native setup.""" + + def __init__( + self, settings: FFNEnvironmentSettingsT, *, platform_name: str + ) -> None: + self.settings = settings + self._platform_name = platform_name + self._device: torch.device | None = None + self._ep_group: GroupCoordinator | None = None + self._groups_owned = False + + @property + def device(self) -> torch.device: + """This rank's device, available after ``initialize_device``.""" + if self._device is None: + raise RuntimeError( + f"{self._platform_name} FFN device has not been initialized" + ) + return self._device + + @property + def ep_group(self) -> GroupCoordinator: + """The F-only expert-parallel group, after ``initialize_compute``.""" + if self._ep_group is None: + raise RuntimeError( + f"{self._platform_name} FFN expert group has not been initialized" + ) + return self._ep_group + + def require_uninitialized(self) -> None: + """Reject a second device initialization.""" + if self._groups_owned or self._device is not None: + raise RuntimeError( + f"{self._platform_name} FFN environment is already initialized" + ) + + def require_launcher_environment(self) -> None: + """Require the launcher world and rank to still match the settings.""" + try: + world_size = int(os.environ["WORLD_SIZE"]) + rank = int(os.environ["RANK"]) + except (KeyError, ValueError) as error: + raise ValueError( + f"{self._platform_name} FFN requires an external F-only launcher" + ) from error + if world_size != self.settings.ffn_ranks or rank != self.settings.rank: + raise ValueError( + f"{self._platform_name} FFN environment differs from the " + "F-only launcher" + ) + + def require_no_borrowed_native_groups(self) -> None: + """Refuse process groups this standalone F process does not own.""" + import torch + from vllm.distributed.parallel_state import model_parallel_is_initialized + + if torch.distributed.is_initialized() or model_parallel_is_initialized(): + raise RuntimeError( + f"{self._platform_name} FFN cannot borrow existing native groups" + ) + + def require_compute_initializable(self) -> None: + """Require device selection and reject repeated compute initialization.""" + if self._device is None or self._groups_owned: + raise RuntimeError( + f"{self._platform_name} FFN compute initialization requires its device" + ) + + def mark_groups_owned(self) -> None: + """Record ownership before the first collective can partially fail.""" + self._groups_owned = True + + def validate_ep_group(self, *, error_message: str) -> None: + """Require the expert group to span exactly the F-only world.""" + ep_group = self.ep_group + if ( + ep_group.world_size != self.settings.ffn_ranks + or ep_group.rank_in_group != self.settings.rank + ): + raise ValueError(error_message) + + def close(self) -> None: + """Destroy owned groups, then clear environment ownership state.""" + if not self._groups_owned: + return + self._destroy_groups() + self._groups_owned = False + self._ep_group = None + self._device = None + + @abstractmethod + def _destroy_groups(self) -> None: + """Destroy this platform's model-parallel and process groups.""" diff --git a/afd_plugin/ffn_faas/platforms/gpu/environment.py b/afd_plugin/ffn_faas/platforms/gpu/environment.py index 9261ea50b..7f2d63ca4 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/environment.py +++ b/afd_plugin/ffn_faas/platforms/gpu/environment.py @@ -14,13 +14,12 @@ import os from typing import TYPE_CHECKING +from afd_plugin.ffn_faas.platforms.environment import FFNEnvironmentBase from afd_plugin.ffn_faas.platforms.gpu.settings import GPUFFNConfig from afd_plugin.ffn_faas.platforms.launch import reassert_external_launcher if TYPE_CHECKING: - import torch from vllm.config import VllmConfig - from vllm.distributed.parallel_state import GroupCoordinator # afdEP-comm transport and the DeepGEMM grouped GEMMs the connector runs; both # are imported lazily by the connector, so a missing package would otherwise @@ -36,36 +35,17 @@ # NPU parallel: afd_plugin/ffn_faas/platforms/npu/environment.py:: # AscendFFNEnvironment. -# Reuse: PARTIAL - the lifecycle (launcher check, refuse borrowed groups, -# device, F-only world + EP group == F world, per-thread device, ordered -# teardown) is the same; NPU initializes HCCL explicitly plus Ascend groups -# and CPU binding, GPU reuses vLLM's GPU Worker distributed initializer, -# pins the routed MoE backend and selects the AFD model architecture. -# REFACTOR-CANDIDATE(R1): a platform-neutral F-only environment base -# (launcher check, ownership flags, EP-group check, teardown order). -class GPUFFNEnvironment: +# Reuse: PARTIAL - the shared base owns state, launcher checks, ownership +# flags, the EP-group check and close-state handling; NPU initializes HCCL +# explicitly plus Ascend groups and CPU binding, GPU reuses vLLM's GPU Worker +# distributed initializer, pins the routed MoE backend and selects the AFD +# model architecture. +class GPUFFNEnvironment(FFNEnvironmentBase[GPUFFNConfig]): """Never borrow unrelated groups or tear them down on a failed native seam.""" def __init__(self, settings: GPUFFNConfig) -> None: """Record the settings; nothing native happens until ``initialize_device``.""" - self.settings = settings - self._device: torch.device | None = None - self._ep_group: GroupCoordinator | None = None - self._groups_owned = False - - @property - def device(self) -> torch.device: - """This rank's GPU, available after ``initialize_device``.""" - if self._device is None: - raise RuntimeError("GPU FFN device has not been initialized") - return self._device - - @property - def ep_group(self) -> GroupCoordinator: - """vLLM's EP group over the F-only world, after ``initialize_compute``.""" - if self._ep_group is None: - raise RuntimeError("GPU FFN expert group has not been initialized") - return self._ep_group + super().__init__(settings, platform_name="GPU") # NPU parallel: AscendFFNEnvironment.initialize_device. # Reuse: PARTIAL - launcher and borrowed-group checks are identical; @@ -81,21 +61,13 @@ def initialize_device(self) -> None: to the visible GPU the way vLLM's GPU Worker does (honoring ``CUDA_VISIBLE_DEVICES``) and requires SM90 or newer. """ - if self._groups_owned or self._device is not None: - raise RuntimeError("GPU FFN environment is already initialized") + self.require_uninitialized() + self.require_launcher_environment() + self.require_no_borrowed_native_groups() settings = self.settings - try: - world, rank = int(os.environ["WORLD_SIZE"]), int(os.environ["RANK"]) - except (KeyError, ValueError) as error: - raise ValueError("GPU FFN requires an external F-only launcher") from error - if world != settings.ffn_ranks or rank != settings.rank: - raise ValueError("GPU FFN environment differs from the F-only launcher") import torch - from vllm.distributed.parallel_state import model_parallel_is_initialized from vllm.platforms import current_platform - if torch.distributed.is_initialized() or model_parallel_is_initialized(): - raise RuntimeError("GPU FFN cannot borrow existing native groups") importlib.import_module(AFDEP_TRANSPORT_MODULE) expert_backend = settings.connector_extra_config.get("expert_backend") if expert_backend != FUSED_EXPERTS_BACKEND: @@ -167,8 +139,7 @@ def initialize_compute(self, config: VllmConfig) -> None: world can host its own, and initializes vLLM's workspace manager with the Worker's micro-batch count (two under DBO). """ - if self._device is None or self._groups_owned: - raise RuntimeError("GPU FFN compute initialization requires its device") + self.require_compute_initializable() from vllm.config import set_current_vllm_config from vllm.distributed.parallel_state import get_ep_group from vllm.platforms import current_platform @@ -177,7 +148,7 @@ def initialize_compute(self, config: VllmConfig) -> None: settings = self.settings with set_current_vllm_config(config): - self._groups_owned = True + self.mark_groups_owned() init_worker_distributed_environment( config, settings.rank, @@ -186,11 +157,9 @@ def initialize_compute(self, config: VllmConfig) -> None: current_platform.dist_backend, ) self._ep_group = get_ep_group() - if ( - self._ep_group.world_size != settings.ffn_ranks - or self._ep_group.rank_in_group != settings.rank - ): - raise ValueError("vLLM expert group differs from the F-only world") + self.validate_ep_group( + error_message="vLLM expert group differs from the F-only world" + ) # torchrun asks every ``tcp://``/``env://`` rendezvous in its workers # to join the agent's store. That is right for the F-only world above, # but the connector's M2N rendezvous at ``--host:--port`` must be @@ -219,15 +188,13 @@ def activate_thread(self) -> None: # NPU parallel: AscendFFNEnvironment.close. # Reuse: PARTIAL - same order and ownership guard; NPU also destroys the # Ascend model-parallel groups. - def close(self) -> None: + def _destroy_groups(self) -> None: """Destroy the vLLM model-parallel groups and the F-only world. Only after the runner released CUDA graphs and DeepEP buffers; a failed native seam never reaches here, so groups stay owned until process isolation. """ - if not self._groups_owned: - return from vllm.distributed.parallel_state import ( destroy_distributed_environment, destroy_model_parallel, @@ -237,6 +204,3 @@ def close(self) -> None: reset_workspace_manager() destroy_model_parallel() destroy_distributed_environment() - self._groups_owned = False - self._ep_group = None - self._device = None diff --git a/afd_plugin/ffn_faas/platforms/npu/environment.py b/afd_plugin/ffn_faas/platforms/npu/environment.py index 952e59acf..575d0ef7d 100644 --- a/afd_plugin/ffn_faas/platforms/npu/environment.py +++ b/afd_plugin/ffn_faas/platforms/npu/environment.py @@ -9,68 +9,46 @@ from __future__ import annotations import logging -import os from datetime import timedelta from typing import TYPE_CHECKING +from afd_plugin.ffn_faas.platforms.environment import FFNEnvironmentBase from afd_plugin.ffn_faas.platforms.npu.settings import ( FFNConfig, validate_static_experts, ) if TYPE_CHECKING: - import torch from vllm.config import VllmConfig - from vllm.distributed.parallel_state import GroupCoordinator COMPUTE_GROUP_TIMEOUT_SECONDS = 180 -class AscendFFNEnvironment: +# GPU parallel: afd_plugin/ffn_faas/platforms/gpu/environment.py:: +# GPUFFNEnvironment. +# Reuse: PARTIAL - the shared base owns state, launcher checks, ownership +# flags, the EP-group check and close-state handling; this class owns Ascend +# device selection, custom operators, HCCL groups and CPU binding. +class AscendFFNEnvironment(FFNEnvironmentBase[FFNConfig]): """Never borrow unrelated groups or tear them down on a failed native seam.""" def __init__(self, settings: FFNConfig) -> None: - self.settings = settings - self._device: torch.device | None = None - self._ep_group: GroupCoordinator | None = None - self._groups_owned = False - - @property - def device(self) -> torch.device: - if self._device is None: - raise RuntimeError("Ascend FFN device has not been initialized") - return self._device - - @property - def ep_group(self) -> GroupCoordinator: - if self._ep_group is None: - raise RuntimeError("Ascend FFN expert group has not been initialized") - return self._ep_group + super().__init__(settings, platform_name="Ascend") def initialize_device(self) -> None: - if self._groups_owned or self._device is not None: - raise RuntimeError("Ascend FFN environment is already initialized") + self.require_uninitialized() + self.require_launcher_environment() settings = self.settings - try: - world, rank = int(os.environ["WORLD_SIZE"]), int(os.environ["RANK"]) - except (KeyError, ValueError) as error: - raise ValueError( - "Ascend FFN requires an external F-only launcher" - ) from error - if world != settings.ffn_ranks or rank != settings.rank: - raise ValueError("Ascend FFN environment differs from the F-only launcher") from vllm_ascend.utils import bootstrap_custom_op_env bootstrap_custom_op_env() import torch import torch_npu # noqa: F401 - from vllm.distributed.parallel_state import model_parallel_is_initialized from vllm_ascend.utils import enable_custom_op from afd_plugin.compat.npu.ops import ensure_cam_async_ops_available - if torch.distributed.is_initialized() or model_parallel_is_initialized(): - raise RuntimeError("Ascend FFN cannot borrow existing native groups") + self.require_no_borrowed_native_groups() ensure_cam_async_ops_available() torch.npu.set_device(settings.device) self._device = torch.device(f"npu:{settings.device}") @@ -81,8 +59,7 @@ def initialize_device(self) -> None: raise RuntimeError("native V4 W4A8 custom operators are unavailable") def initialize_compute(self, config: VllmConfig) -> None: - if self._device is None or self._groups_owned: - raise RuntimeError("Ascend FFN compute initialization requires its device") + self.require_compute_initializable() settings = self.settings parallel = config.parallel_config validate_static_experts( @@ -106,7 +83,7 @@ def initialize_compute(self, config: VllmConfig) -> None: register_ascend_customop(config) init_ascend_config(config) # Ownership starts before a collective can partially initialize. - self._groups_owned = True + self.mark_groups_owned() init_distributed_environment( world_size=settings.ffn_ranks, rank=settings.rank, @@ -122,11 +99,9 @@ def initialize_compute(self, config: VllmConfig) -> None: ) init_ascend_model_parallel(parallel) self._ep_group = get_ep_group() - if ( - self._ep_group.world_size != settings.ffn_ranks - or self._ep_group.rank_in_group != settings.rank - ): - raise ValueError("native V4 expert group differs from the F world") + self.validate_ep_group( + error_message="native V4 expert group differs from the F world" + ) def activate_thread(self) -> None: import torch @@ -162,10 +137,8 @@ def bind_host_threads(self, config: VllmConfig) -> None: error, ) - def close(self) -> None: + def _destroy_groups(self) -> None: """Only after CAM/executor completion; failures retain parameter owners.""" - if not self._groups_owned: - return from vllm.distributed.parallel_state import ( destroy_distributed_environment, destroy_model_parallel, @@ -175,6 +148,3 @@ def close(self) -> None: destroy_ascend_model_parallel() destroy_model_parallel() destroy_distributed_environment() - self._groups_owned = False - self._ep_group = None - self._device = None From 46772c2e6be8f254955b6d125391a7195f54a9f1 Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:36:32 +0200 Subject: [PATCH 16/42] Load Qwen3 FFN-only routed experts through vLLM Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- afd_plugin/__init__.py | 3 + afd_plugin/ffn_faas/platforms/gpu/backend.py | 13 +- .../ffn_faas/platforms/gpu/environment.py | 8 +- .../platforms/gpu/qwen3_routed_model.py | 180 ------------------ .../model_executor/models/model_utils.py | 17 +- afd_plugin/model_executor/models/qwen3_moe.py | 84 +++++++- .../worker/ffn_routed_experts_model_runner.py | 29 --- afd_plugin/v1/worker/ffn_worker.py | 13 +- afd_plugin/validation.py | 1 + docs/design/module/ffn_runtime.md | 10 +- docs/design/module/model_integration.md | 4 + docs/gpu/STANDALONE_FFN_SERVICE.md | 2 +- .../model_executor/models/test_model_utils.py | 24 +++ .../models/test_qwen3_moe_construction.py | 34 ++++ tests/unit/package/test_package.py | 3 + .../test_ffn_routed_experts_model_runner.py | 67 ------- 16 files changed, 192 insertions(+), 300 deletions(-) delete mode 100644 afd_plugin/ffn_faas/platforms/gpu/qwen3_routed_model.py diff --git a/afd_plugin/__init__.py b/afd_plugin/__init__.py index eeff9c467..a586088c5 100644 --- a/afd_plugin/__init__.py +++ b/afd_plugin/__init__.py @@ -125,6 +125,9 @@ def get_spawn_context(method: str | None = None): "Qwen3MoeForCausalLM": ( "afd_plugin.model_executor.models.qwen3_moe:AFDQwen3MoeForCausalLM" ), + "Qwen3MoeRoutedExperts": ( + "afd_plugin.model_executor.models.qwen3_moe:AFDQwen3MoeRoutedExperts" + ), } _QWEN3_5_MODEL_REGISTRATIONS = MappingProxyType( diff --git a/afd_plugin/ffn_faas/platforms/gpu/backend.py b/afd_plugin/ffn_faas/platforms/gpu/backend.py index 3bd8ad3bd..20996a804 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/backend.py +++ b/afd_plugin/ffn_faas/platforms/gpu/backend.py @@ -19,12 +19,9 @@ from afd_plugin.ffn_faas.model_utils import parameter_storage_bytes from afd_plugin.ffn_faas.platforms.gpu.environment import GPUFFNEnvironment from afd_plugin.ffn_faas.platforms.gpu.qwen3_model import Qwen3CheckpointDescription -from afd_plugin.ffn_faas.platforms.gpu.qwen3_routed_model import ( - Qwen3RoutedExpertsModel, - load_qwen3_routed_experts, -) from afd_plugin.ffn_faas.platforms.gpu.settings import GPUFFNConfig from afd_plugin.ffn_faas.service import FFNService, ServicePhase +from afd_plugin.model_executor.models.qwen3_moe import AFDQwen3MoeRoutedExperts from afd_plugin.v1.worker.ffn_routed_experts_model_runner import ( GPUFFNRoutedExpertsModelRunner, ) @@ -49,7 +46,7 @@ def __init__( self.settings = settings self.environment = GPUFFNEnvironment(settings) self.runner: GPUFFNRoutedExpertsModelRunner | None = None - self.model: Qwen3RoutedExpertsModel | None = None + self.model: AFDQwen3MoeRoutedExperts | None = None self._cancel_requested = threading.Event() def prepare(self, service: FFNService) -> None: @@ -64,7 +61,7 @@ def prepare(self, service: FFNService) -> None: 4. construct the routed-experts runner, which runs the ModelRunnerV2 and CUDA-graph validation and creates the connector (no collective yet); - 5. load the FFN-only Qwen3 shell and attach it to the runner; + 5. load the FFN-only Qwen3 shell through the runner's native loader; 6. start the connector: the collective M2N handshake with Attention, the model-info check, the NCCL control group, the lanes' DeepEP buffers and the dispatch-kernel precompile. @@ -93,12 +90,12 @@ def prepare(self, service: FFNService) -> None: f"{type(connector).__name__}" ) service.loading_progress(0, total_experts) - self.model = load_qwen3_routed_experts(config, environment.device) + self.runner.load_model() + self.model = self.runner.get_model() # vLLM's loader reports no per-expert progress; this rank's experts # are all loaded once it returns. service.loading_progress(total_experts, total_experts) parameter_bytes = parameter_storage_bytes(self.model) - self.runner.attach_model(self.model, model_memory_usage=parameter_bytes) local_expert_ids = self.runner.local_experts[ self.runner.moe_layer_indices[0] ].expert_ids diff --git a/afd_plugin/ffn_faas/platforms/gpu/environment.py b/afd_plugin/ffn_faas/platforms/gpu/environment.py index 7f2d63ca4..779b7d8d2 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/environment.py +++ b/afd_plugin/ffn_faas/platforms/gpu/environment.py @@ -97,8 +97,8 @@ def build_config(self) -> VllmConfig: ``EngineArgs.create_engine_config`` (with the AFD plugin's config patches, which accept DBO without a DeepEP all2all backend), then re-asserts the ``external_launcher`` executor, swaps in the - AFD-prefixed model architecture that ``AFDFFNWorker.init_device`` - selects, and pins vLLM's MoE backend to Triton as + FFN-only routed-experts architecture that ``AFDFFNWorker.init_device`` + selects for this connector, and pins vLLM's MoE backend to Triton as ``AFDFFNWorker._make_model_runner`` does. The pin must happen before any FusedMoE is constructed: FusedMoE picks its weight layout at construction, and a reordered layout would be read silently wrong. @@ -115,7 +115,9 @@ def build_config(self) -> VllmConfig: # platform defaults would pick for an ordinary engine. reassert_external_launcher(config) config.model_config = get_afd_model_config( - config.model_config, device_type="cuda" + config.model_config, + device_type="cuda", + routed_experts_connector=True, ) select_routed_moe_backend(config) return config diff --git a/afd_plugin/ffn_faas/platforms/gpu/qwen3_routed_model.py b/afd_plugin/ffn_faas/platforms/gpu/qwen3_routed_model.py deleted file mode 100644 index 3d9dbc775..000000000 --- a/afd_plugin/ffn_faas/platforms/gpu/qwen3_routed_model.py +++ /dev/null @@ -1,180 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project -"""FFN-only Qwen3 MoE model: the routed experts and nothing else. - -The Worker-path FFN loads ``AFDQwen3MoeForCausalLM`` with ``role="ffn"``. Its -decoder layers already skip Attention, but the CausalLM still builds the -embedding, final norm, LM head and logits processor (about 1.16 GiB per F -rank for Qwen3-30B-A3B, never used). This shell keeps the native module tree -only where the routed experts live, ``model.layers..mlp``, built from the -native ``Qwen3MoeSparseMoeBlock``, so vLLM's own loader, EP weight filter, -FusedMoE expert loading and ``process_weights_after_loading`` run unchanged. - -The MoE forward never runs here: the M2N connector's executor reads the -experts' parameters as views (``local_experts_from_vllm``). The native gate is -built because the block constructs it and the strict loader expects its -weight; it is never executed (Attention routes). -""" - -from __future__ import annotations - -import gc -from collections.abc import Iterable -from typing import TYPE_CHECKING - -import torch -from torch import nn -from vllm.config import VllmConfig -from vllm.model_executor.model_loader import get_model_loader -from vllm.model_executor.model_loader.utils import process_weights_after_loading -from vllm.model_executor.models import qwen3_moe as native -from vllm.model_executor.models.utils import AutoWeightsLoader -from vllm.utils.torch_utils import set_default_torch_dtype - -from afd_plugin.model_executor.checkpoint_paths import weight_layer_path - -if TYPE_CHECKING: - from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts - -ROUTED_STAGE = "mlp" -MODEL_PREFIX = "model" - - -def _is_routed_mlp_path(name: str) -> bool: - """Whether a checkpoint tensor belongs to a decoder layer's MoE block. - - Keeps ``model.layers..mlp.*`` (routed experts and the unused gate) and - drops embeddings, norms, Attention and the LM head, which this shell does - not construct. Reuses the shared checkpoint path parser so the Worker-path - wrapper and this shell classify checkpoint names identically. - """ - layer_path = weight_layer_path(name) - return layer_path is not None and layer_path[1] == ROUTED_STAGE - - -# NPU parallel: afd_plugin/ffn_faas/platforms/npu/model.py::RoutedWeightOwner -# via V4ModelAdapter.create_owner/create_layer (native routed modules built -# layer by layer from the pinned Ascend loader). -# Reuse: NO - NPU constructs and loads W4A8 layered weights through its own -# safetensors reader; GPU keeps vLLM's module tree and native loader. -class _Qwen3RoutedExpertsLayer(nn.Module): - """One decoder layer reduced to its native MoE block at ``.mlp``.""" - - def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: - """Construct the native ``Qwen3MoeSparseMoeBlock`` with its native prefix. - - The prefix keeps FusedMoE's ``layer_name`` and its - ``static_forward_context`` key identical to the Worker-path model. - """ - super().__init__() - self.mlp = native.Qwen3MoeSparseMoeBlock( - vllm_config=vllm_config, prefix=f"{prefix}.{ROUTED_STAGE}" - ) - - -class _Qwen3RoutedExpertsStack(nn.Module): - """The ``model`` submodule: only ``layers``, each holding its MoE block.""" - - def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: - """Build one ``_Qwen3RoutedExpertsLayer`` per decoder layer.""" - super().__init__() - num_layers = vllm_config.model_config.hf_text_config.num_hidden_layers - self.layers = nn.ModuleList( - _Qwen3RoutedExpertsLayer(vllm_config, f"{prefix}.layers.{layer_idx}") - for layer_idx in range(num_layers) - ) - - -class Qwen3RoutedExpertsModel(nn.Module): - """FFN-only Qwen3 MoE model consumed by ``GPUFFNRoutedExpertsModelRunner``. - - Implements ``RoutedExpertsModel``: ``get_experts_layer_indices`` and - ``get_routed_experts``, the same accessors ``AFDQwen3MoeForCausalLM`` - exposes on the Worker path. - """ - - # Same as native Qwen3MoeForCausalLM: load safetensors only, never fall - # back to PyTorch checkpoint files. - fall_back_to_pt_during_load = False - - def __init__(self, *, vllm_config: VllmConfig) -> None: - """Build ``model.layers..mlp`` per layer, named as in the native model. - - Must run under ``set_current_vllm_config`` and after - ``select_routed_moe_backend``: FusedMoE reads the current config and - picks its weight layout at construction. - """ - super().__init__() - self.model = _Qwen3RoutedExpertsStack(vllm_config, MODEL_PREFIX) - - def is_unused_checkpoint_weight(self, name: str) -> bool: - """Whether vLLM can skip a shard containing only this checkpoint name.""" - return not _is_routed_mlp_path(name) - - # Upstream source: vLLM v0.32.0 (5df63a7203cdb9328d824a27b682e77c7b9928be), - # vllm/model_executor/models/qwen3_moe.py, Qwen3MoeModel.hf_to_vllm_mapper. - # AFD difference: filter to routed MoE paths and apply the native mapper at - # this shell's outer AutoWeightsLoader. - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - """Drop non-MoE checkpoint tensors, then load like the native CausalLM. - - ``AutoWeightsLoader`` walks this shell's module tree and hands each MoE - block's expert tensors to FusedMoE. Tensors this shell does not construct - (embeddings, norms, Attention, LM head) are filtered first; the native - Qwen3 mapper still applies unchanged. - """ - # ### PATCH START: keep only routed MoE checkpoint paths. - weights = ( - (name, tensor) - for name, tensor in weights - if not self.is_unused_checkpoint_weight(name) - ) - # ### PATCH END: keep only routed MoE checkpoint paths. - loader = AutoWeightsLoader(self) - return loader.load_weights( - weights, - mapper=native.Qwen3MoeModel.hf_to_vllm_mapper, - ) - - def get_experts_layer_indices(self) -> tuple[int, ...]: - """Every decoder layer is an MoE layer for a supported checkpoint.""" - return tuple(range(len(self.model.layers))) - - def get_routed_experts(self, layer_idx: int) -> RoutedExperts: - """The vLLM module holding one MoE layer's local experts.""" - return self.model.layers[layer_idx].mlp.experts.routed_experts - - -# NPU parallel: afd_plugin/ffn_faas/platforms/npu/model.py::CAMRoutedModel.load -# (layer-by-layer safetensors load with per-layer progress, then storage -# accounting). -# Reuse: PARTIAL - storage accounting is shared through -# ``parameter_storage_bytes``; loading differs: vLLM's loader exposes no -# per-layer callback, so progress is reported only before and after. -# Upstream source: vLLM v0.32.0 (5df63a7203cdb9328d824a27b682e77c7b9928be), -# vllm/model_executor/model_loader/base_loader.py, BaseModelLoader.load_model. -# Same steps; the model is constructed here instead of by initialize_model, -# and online quantization is not handled (BF16 only). -def load_qwen3_routed_experts( - vllm_config: VllmConfig, device: torch.device -) -> Qwen3RoutedExpertsModel: - """Construct the FFN-only shell on ``device`` and load its weights. - - Follows ``BaseModelLoader.load_model``: construct and load under the model - dtype and target device, run vLLM's ``DefaultModelLoader.load_weights`` - (EP weight filter, then the strict loaded-weights check), release loading - temporaries, run ``process_weights_after_loading`` (the Triton backend - keeps the FusedMoE layout), and return the model in eval mode. The caller - holds ``set_current_vllm_config``. - """ - model_config = vllm_config.model_config - loader = get_model_loader(vllm_config.load_config) - with set_default_torch_dtype(model_config.dtype), torch.device(device): - model = Qwen3RoutedExpertsModel(vllm_config=vllm_config) - loader.load_weights(model, model_config) - gc.collect() - torch.accelerator.empty_cache() - process_weights_after_loading(model, model_config, device) - gc.collect() - torch.accelerator.empty_cache() - return model.eval() diff --git a/afd_plugin/model_executor/models/model_utils.py b/afd_plugin/model_executor/models/model_utils.py index 5a0518475..226c262b9 100644 --- a/afd_plugin/model_executor/models/model_utils.py +++ b/afd_plugin/model_executor/models/model_utils.py @@ -5,7 +5,7 @@ from __future__ import annotations from copy import deepcopy -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Final, Literal from afd_plugin import _MODEL_REGISTRATIONS, _QWEN3_5_MODEL_REGISTRATIONS from afd_plugin.config import AFD_ASYNC_CONNECTOR, parse_afd_config @@ -14,6 +14,11 @@ from vllm.config import ModelConfig, VllmConfig +_ROUTED_EXPERTS_MODEL_ARCHITECTURES: Final[dict[str, str]] = { + "Qwen3MoeForCausalLM": "AFDQwen3MoeRoutedExperts", +} + + def has_afd_model_registration(model_config: ModelConfig) -> bool: """Return whether the model resolves to a registered AFD implementation.""" @@ -27,6 +32,7 @@ def get_afd_model_config( model_config: ModelConfig, *, device_type: Literal["cuda", "npu"], + routed_experts_connector: bool = False, ) -> ModelConfig: """Return a model config that resolves to an AFD model implementation.""" @@ -44,7 +50,14 @@ def get_afd_model_config( # automatically. vLLM Ascend uses that identity to distinguish # text models from multimodal models. afd_model_config = deepcopy(model_config) - afd_model_config.hf_config.architectures = [f"AFD{model_arch}"] + routed_architecture = ( + _ROUTED_EXPERTS_MODEL_ARCHITECTURES.get(model_arch) + if routed_experts_connector and device_type == "cuda" + else None + ) + afd_model_config.hf_config.architectures = [ + routed_architecture or f"AFD{model_arch}" + ] return afd_model_config return model_config diff --git a/afd_plugin/model_executor/models/qwen3_moe.py b/afd_plugin/model_executor/models/qwen3_moe.py index 10eb5c311..2e8717073 100644 --- a/afd_plugin/model_executor/models/qwen3_moe.py +++ b/afd_plugin/model_executor/models/qwen3_moe.py @@ -18,6 +18,7 @@ from transformers import Qwen3MoeConfig from vllm.config import VllmConfig from vllm.model_executor.models import qwen3_moe as native +from vllm.model_executor.models.utils import AutoWeightsLoader, maybe_prefix from vllm.platforms import current_platform from afd_plugin.config import ( @@ -34,6 +35,7 @@ _ATTENTION_ROLE = frozenset(("attention",)) _FFN_ROLE = frozenset(("ffn",)) _BOTH_ROLES = frozenset(("attention", "ffn")) +ROUTED_STAGE = "mlp" def _is_moe_layer(config: Qwen3MoeConfig, layer_idx: int) -> bool: @@ -50,6 +52,12 @@ def _routes_on_attention(afd_config: AFDConfig) -> bool: return afd_config.connector in DEEPEP_M2N_CONNECTORS +def _is_routed_mlp_path(name: str) -> bool: + """Whether a checkpoint tensor belongs to a decoder layer's MoE block.""" + layer_path = weight_layer_path(name) + return layer_path is not None and layer_path[1] == ROUTED_STAGE + + def _checkpoint_weight_roles( name: str, *, @@ -279,6 +287,80 @@ def get_routed_experts(self, layer_idx: int) -> "RoutedExperts": return self.layers[layer_idx].mlp.experts.routed_experts +class _Qwen3RoutedExpertsLayer(nn.Module): + """One decoder layer reduced to its native MoE block at ``.mlp``.""" + + def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: + super().__init__() + self.mlp = native.Qwen3MoeSparseMoeBlock( + vllm_config=vllm_config, prefix=f"{prefix}.{ROUTED_STAGE}" + ) + + +class _Qwen3RoutedExpertsStack(nn.Module): + """The ``model`` submodule: only ``layers``, each holding its MoE block.""" + + def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: + super().__init__() + num_layers = vllm_config.model_config.hf_text_config.num_hidden_layers + self.layers = nn.ModuleList( + _Qwen3RoutedExpertsLayer(vllm_config, f"{prefix}.layers.{layer_idx}") + for layer_idx in range(num_layers) + ) + + +class AFDQwen3MoeRoutedExperts(nn.Module): + """FFN-only Qwen3 MoE model for routed-experts connectors. + + Builds only the native ``Qwen3MoeSparseMoeBlock`` tree at + ``model.layers..mlp``. The MoE forward never runs here; the connector + reads the local expert parameter views directly. + """ + + fall_back_to_pt_during_load = False + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + afd_config = parse_afd_config(vllm_config, validate=False) + _validate_supported_config(vllm_config, afd_config) + super().__init__() + self.afd_config = afd_config + self.model = _Qwen3RoutedExpertsStack( + vllm_config, maybe_prefix(prefix, "model") + ) + + def is_unused_checkpoint_weight(self, name: str) -> bool: + """Whether vLLM can skip a shard containing only this checkpoint name.""" + return not _is_routed_mlp_path(name) + + # Upstream source: vLLM v0.32.0 + # (5df63a7203cdb9328d824a27b682e77c7b9928be), + # vllm/model_executor/models/qwen3_moe.py, Qwen3MoeModel.hf_to_vllm_mapper. + # AFD difference: filter to routed MoE paths and apply the native mapper at + # this shell's outer AutoWeightsLoader. + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + """Drop non-MoE checkpoint tensors, then load like the native model.""" + # ### PATCH START: keep only routed MoE checkpoint paths. + weights = ( + (name, tensor) + for name, tensor in weights + if not self.is_unused_checkpoint_weight(name) + ) + # ### PATCH END: keep only routed MoE checkpoint paths. + loader = AutoWeightsLoader(self) + return loader.load_weights( + weights, + mapper=native.Qwen3MoeModel.hf_to_vllm_mapper, + ) + + def get_experts_layer_indices(self) -> tuple[int, ...]: + """Every decoder layer is an MoE layer for a supported checkpoint.""" + return tuple(range(len(self.model.layers))) + + def get_routed_experts(self, layer_idx: int) -> "RoutedExperts": + """The vLLM module holding one MoE layer's local experts.""" + return self.model.layers[layer_idx].mlp.experts.routed_experts + + class AFDQwen3MoeForCausalLM(native.Qwen3MoeForCausalLM): """Qwen3 MoE causal LM wrapper for AFD CUDA execution.""" @@ -398,4 +480,4 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: return super().load_weights(role_weights) -__all__ = ["AFDQwen3MoeForCausalLM"] +__all__ = ["AFDQwen3MoeForCausalLM", "AFDQwen3MoeRoutedExperts"] diff --git a/afd_plugin/v1/worker/ffn_routed_experts_model_runner.py b/afd_plugin/v1/worker/ffn_routed_experts_model_runner.py index 266bbdbc6..a5b8c09ab 100644 --- a/afd_plugin/v1/worker/ffn_routed_experts_model_runner.py +++ b/afd_plugin/v1/worker/ffn_routed_experts_model_runner.py @@ -20,9 +20,6 @@ ) from afd_plugin.connectors.metadata import FP8_DTYPE from afd_plugin.v1.worker.ffn_model_runner import GPUFFNModelRunner -from afd_plugin.v1.worker.model_attachment import ( - validate_external_model_attachment, -) if TYPE_CHECKING: from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts @@ -156,32 +153,6 @@ def load_model(self, *, load_dummy_weights: bool = False, **kwargs: Any) -> None super().load_model(load_dummy_weights=load_dummy_weights, **kwargs) self._take_local_experts(self.get_model()) - # NPU parallel: afd_plugin/v1/worker/npu/async_ffn_model_runner.py:: - # AsyncNPUFFNModelRunner.attach_model. - # Reuse: YES for the attachment guard; expert extraction is GPU-specific - # (FusedMoE Triton layout views). - def attach_model( - self, model: RoutedExpertsModel, *, model_memory_usage: int - ) -> None: - """Adopt a model loaded outside this runner, as ``load_model`` would. - - The standalone FFN service builds and loads its own FFN-only model. - This hands it to the runner and builds the per-layer local-expert - views the connector reads, through the same - ``local_experts_from_vllm`` check ``load_model`` uses (Triton layout). - Must run before ``initialize_afd_connector`` because the connector - all-gathers these views in its startup collective, and only once, - because the connector keeps no way to swap them afterwards. - """ - validate_external_model_attachment( - model_attached=self.model is not None, - connector_initialized=self.connector.is_initialized, - model_size_bytes=model_memory_usage, - ) - self.model = model - self.model_memory_usage = model_memory_usage - self._take_local_experts(model) - def _take_local_experts(self, model: RoutedExpertsModel) -> None: """Record the model's MoE layers and this rank's expert views per layer.""" self.moe_layer_indices = tuple(model.get_experts_layer_indices()) diff --git a/afd_plugin/v1/worker/ffn_worker.py b/afd_plugin/v1/worker/ffn_worker.py index 6686da273..99702d130 100644 --- a/afd_plugin/v1/worker/ffn_worker.py +++ b/afd_plugin/v1/worker/ffn_worker.py @@ -13,7 +13,7 @@ from vllm.v1.worker.gpu_worker import Worker from vllm.v1.worker.worker_base import CompilationTimes -from afd_plugin.config import parse_afd_config +from afd_plugin.config import AFDConfig, parse_afd_config from afd_plugin.connectors import AFDConnectorFactory, AFDRoutedExpertsConnector from afd_plugin.model_executor.models.model_utils import get_afd_model_config from afd_plugin.v1.worker.ffn_model_runner import GPUFFNModelRunner @@ -32,6 +32,11 @@ FFN_LOOP_JOIN_TIMEOUT_S = 5 +def _uses_routed_experts_connector(afd_config: AFDConfig) -> bool: + connector_cls = AFDConnectorFactory.get_connector_class(afd_config.connector) + return issubclass(connector_cls, AFDRoutedExpertsConnector) + + class AFDFFNWorker(Worker): """FFN worker that owns the AFD daemon loop. @@ -75,7 +80,7 @@ def init_device(self): """Initialize the native GPU worker with the connector-driven runner.""" # ### PATCH START: validate and select the AFD model. - assert_compatible_afd_stack( + afd_config = assert_compatible_afd_stack( self.vllm_config, caller="AFDFFNWorker.init_device", expected_role="ffn", @@ -88,6 +93,7 @@ def init_device(self): afd_model_config = get_afd_model_config( self.vllm_config.model_config, device_type="cuda", + routed_experts_connector=_uses_routed_experts_connector(afd_config), ) self.vllm_config.model_config = afd_model_config self.model_config = afd_model_config @@ -103,8 +109,7 @@ def init_device(self): def _make_model_runner(self) -> GPUModelRunner: """Construct the connector-driven runner via the native factory hook.""" afd_config = parse_afd_config(self.vllm_config, expected_role="ffn") - connector_cls = AFDConnectorFactory.get_connector_class(afd_config.connector) - if issubclass(connector_cls, AFDRoutedExpertsConnector): + if _uses_routed_experts_connector(afd_config): from afd_plugin.v1.worker.ffn_routed_experts_model_runner import ( GPUFFNRoutedExpertsModelRunner, select_routed_moe_backend, diff --git a/afd_plugin/validation.py b/afd_plugin/validation.py index 87dcb3255..9dcf42801 100644 --- a/afd_plugin/validation.py +++ b/afd_plugin/validation.py @@ -52,6 +52,7 @@ # with the hidden states, as a routed-experts connector (DeepEP M2N) needs. ATTENTION_ROUTING_MODEL_ARCHITECTURES: Final[tuple[str, ...]] = ( "Qwen3MoeForCausalLM", + "Qwen3MoeRoutedExperts", "DeepseekV2ForCausalLM", ) diff --git a/docs/design/module/ffn_runtime.md b/docs/design/module/ffn_runtime.md index 5f30107eb..4c8636fc0 100644 --- a/docs/design/module/ffn_runtime.md +++ b/docs/design/module/ffn_runtime.md @@ -143,15 +143,15 @@ Qwen3-30B-A3B (BF16) over `DeepEPM2NAFDConnector` has a GPU standalone F, with `--dbo` are checked before any device work. - `GPUFFNEnvironment` selects the GPU, preflights afdEP and DeepGEMM, builds the F-only `VllmConfig` (`external_launcher`, DP = F ranks, TP 1, EP), selects - the AFD architecture, pins vLLM's MoE backend to Triton before any FusedMoE - is built, and creates the F-only world through vLLM's + the FFN-only routed-experts architecture, pins vLLM's MoE backend to Triton + before any FusedMoE is built, and creates the F-only world through vLLM's `init_worker_distributed_environment`. -- `Qwen3RoutedExpertsModel` keeps only `model.layers..mlp` (native +- `AFDQwen3MoeRoutedExperts` keeps only `model.layers..mlp` (native `Qwen3MoeSparseMoeBlock`) and loads through vLLM's `DefaultModelLoader`, so the EP weight filter, FusedMoE expert loading and strict loaded-weights check run unchanged; the embedding, final norm and LM head are not allocated. -- `DeepEPM2NFFNBackend` constructs `GPUFFNRoutedExpertsModelRunner`, attaches - the model (`attach_model`), starts the connector, and delegates each step to +- `DeepEPM2NFFNBackend` constructs `GPUFFNRoutedExpertsModelRunner`, loads the + shell through that runner, starts the connector, and delegates each step to `execute_connector_driven_step()`, the method the Worker loop also calls. CUDA graphs are captured lazily from Attention's payloads, as on the Worker path, so `/ready` means paired and receiving. diff --git a/docs/design/module/model_integration.md b/docs/design/module/model_integration.md index f343b9c51..fb5068f2d 100644 --- a/docs/design/module/model_integration.md +++ b/docs/design/module/model_integration.md @@ -64,6 +64,7 @@ make a backend-specific worker class the shared model API. | DeepSeek V4 CUDA role boundary | [`deepseek_v4.py`](../../../afd_plugin/model_executor/models/deepseek_v4.py) | [`test_deepseek_v4_construction.py`](../../../tests/unit/model_executor/models/test_deepseek_v4_construction.py), [`test_deepseek_v4_proxy.py`](../../../tests/unit/model_executor/models/test_deepseek_v4_proxy.py), [`test_deepseek_v4_weight_policy.py`](../../../tests/unit/model_executor/models/test_deepseek_v4_weight_policy.py) | | DeepSeek V4 Ascend Hash-id boundary | [`npu/deepseek_v4.py`](../../../afd_plugin/model_executor/models/npu/deepseek_v4.py), [`npu/deepseek_v4_attention_gate.py`](../../../afd_plugin/model_executor/models/npu/deepseek_v4_attention_gate.py) | [`test_deepseek_v4_hash_ids.py`](../../../tests/unit/model_executor/test_deepseek_v4_hash_ids.py), [NPU model role tests](../../../tests/unit/model_executor/models/npu/test_deepseek_v4.py), ids-mode cases in [`test_camp2p_token_ids.py`](../../../tests/unit/connectors/test_camp2p_token_ids.py) | | Qwen3 MoE role-aware model and weight loading | [`qwen3_moe.py`](../../../afd_plugin/model_executor/models/qwen3_moe.py) | [`test_qwen3_moe_construction.py`](../../../tests/unit/model_executor/models/test_qwen3_moe_construction.py), [`test_qwen3_moe_weight_policy.py`](../../../tests/unit/model_executor/models/test_qwen3_moe_weight_policy.py) | +| Qwen3 MoE FFN-only routed-experts shell | [`qwen3_moe.py`](../../../afd_plugin/model_executor/models/qwen3_moe.py) | [`test_qwen3_moe_construction.py`](../../../tests/unit/model_executor/models/test_qwen3_moe_construction.py) | | CUDA remote-experts boundary | [`deepseek_v2.py`](../../../afd_plugin/model_executor/models/deepseek_v2.py), [`gpu/p2p.py`](../../../afd_plugin/connectors/gpu/p2p.py) | [`test_p2p_experts_contract.py`](../../../tests/unit/connectors/test_p2p_experts_contract.py), [`test_deepseek_v2_proxy.py`](../../../tests/unit/model_executor/models/test_deepseek_v2_proxy.py) | | Forward-context adapter | [`forward_context.py`](../../../afd_plugin/model_executor/models/forward_context.py) | [`test_forward_context.py`](../../../tests/unit/model_executor/models/test_forward_context.py) | | NPU Async CAM native DBO handoff | [`npu/async_cam_dbo.py`](../../../afd_plugin/model_executor/models/npu/async_cam_dbo.py), [`npu/ubatch_runner.py`](../../../afd_plugin/v1/worker/npu/ubatch_runner.py) | [`test_async_cam_dbo.py`](../../../tests/unit/model_executor/test_async_cam_dbo.py), [`test_npu_model_runner_v2.py`](../../../tests/unit/v1/worker/test_npu_model_runner_v2.py) | @@ -87,11 +88,14 @@ registers lazy AFD wrapper paths under `AFD`-prefixed aliases. | `DSparkDraftModel` | Unsupported | `AFDDSparkDeepseekV4ForCausalLM` (Async CAM: eager A, eager/FULL F) | | `GlmMoeDsaForCausalLM` | `AFDGlmMoeDsaForCausalLM` | `AFDGlmMoeDsaForCausalLM` | | `Qwen3MoeForCausalLM` | `AFDQwen3MoeForCausalLM` | `AFDQwen3MoeForCausalLM` | +| `Qwen3MoeRoutedExperts` | `AFDQwen3MoeRoutedExperts` | `AFDQwen3MoeRoutedExperts` (internal FFN-only alias) | | `Qwen3_5MoeForConditionalGeneration` | `AFDQwen3_5MoeForConditionalGeneration` | `AFDQwen3_5MoeForConditionalGeneration` | Only AFD workers switch their worker-local model configuration to the matching alias before constructing the AFD model runner. Non-AFD workers keep the checkpoint architecture and resolve to vLLM's native model class. +For CUDA FFN workers using a routed-experts connector, Qwen3 MoE selects the +internal FFN-only shell alias instead of the CausalLM wrapper. The DeepSeek, DeepSeek V2/V3/V3.2, and GLM aliases share the DeepSeek V2-derived implementation. DeepSeek V4, Qwen3 MoE, and Qwen3.5/3.6 each have a diff --git a/docs/gpu/STANDALONE_FFN_SERVICE.md b/docs/gpu/STANDALONE_FFN_SERVICE.md index 1ea134901..d274b05e8 100644 --- a/docs/gpu/STANDALONE_FFN_SERVICE.md +++ b/docs/gpu/STANDALONE_FFN_SERVICE.md @@ -25,7 +25,7 @@ The routed experts must divide evenly across the F ranks. CPU service / host runner -> FFNBackend DeepEPM2NFFNBackend (ffn_faas/platforms/gpu/backend.py) GPUFFNEnvironment: GPU, F-only NCCL world and vLLM EP group - Qwen3RoutedExpertsModel: model.layers..mlp only, native vLLM loader + AFDQwen3MoeRoutedExperts: model.layers..mlp only, native vLLM loader GPUFFNRoutedExpertsModelRunner: the Worker path's FFN runner DeepEPM2NAFDConnector: handshake, control plane, lanes, experts ``` diff --git a/tests/unit/model_executor/models/test_model_utils.py b/tests/unit/model_executor/models/test_model_utils.py index ef4f2bebb..1413b1e03 100644 --- a/tests/unit/model_executor/models/test_model_utils.py +++ b/tests/unit/model_executor/models/test_model_utils.py @@ -83,6 +83,30 @@ def test_model_config_rewrites_architecture_in_private_copy( assert hf_config.architectures == [architecture] +@pytest.mark.parametrize( + "routed_experts_connector,expected_architecture", + [ + (False, "AFDQwen3MoeForCausalLM"), + (True, "AFDQwen3MoeRoutedExperts"), + ], + ids=["full-model", "routed-experts-shell"], +) +def test_qwen3_routed_connector_selects_the_ffn_only_shell( + model_utils, routed_experts_connector, expected_architecture +): + hf_config = SimpleNamespace(architectures=["Qwen3MoeForCausalLM"]) + original = SimpleNamespace(hf_config=hf_config, hf_text_config=hf_config) + + mapped = model_utils.get_afd_model_config( + original, + device_type="cuda", + routed_experts_connector=routed_experts_connector, + ) + + assert mapped.hf_config.architectures == [expected_architecture] + assert hf_config.architectures == ["Qwen3MoeForCausalLM"] + + def test_model_config_preserves_distinct_nested_text_config(model_utils): text = SimpleNamespace(model_type="qwen3_5_moe_text") original = SimpleNamespace( diff --git a/tests/unit/model_executor/models/test_qwen3_moe_construction.py b/tests/unit/model_executor/models/test_qwen3_moe_construction.py index bdae43e44..11ac52cbd 100644 --- a/tests/unit/model_executor/models/test_qwen3_moe_construction.py +++ b/tests/unit/model_executor/models/test_qwen3_moe_construction.py @@ -235,6 +235,40 @@ def fake_native_init( assert model.afd_role == "attention" +def test_routed_experts_shell_builds_only_native_moe_blocks( + monkeypatch: pytest.MonkeyPatch, + construction_env, +) -> None: + vllm_config = _model_config(layer_count=2) + monkeypatch.setattr(adapter, "current_platform", _platform(is_cuda=True)) + monkeypatch.setattr( + adapter, + "parse_afd_config", + lambda *_args, **_kwargs: AFDConfig( + role="ffn", + connector=DEEPEP_M2N_CONNECTOR, + compute_gate_on_attention=True, + ), + ) + + model = adapter.AFDQwen3MoeRoutedExperts( + vllm_config=vllm_config, + prefix="draft", + ) + + assert construction_env["moe"] == [ + "draft.model.layers.0.mlp", + "draft.model.layers.1.mlp", + ] + assert set(dict(model.named_parameters())) == { + "model.layers.0.mlp.weight", + "model.layers.1.mlp.weight", + } + assert model.get_experts_layer_indices() == (0, 1) + assert model.is_unused_checkpoint_weight("lm_head.weight") + assert not model.is_unused_checkpoint_weight("model.layers.0.mlp.gate.weight") + + def test_attention_causal_lm_reports_no_local_moe_modules( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/package/test_package.py b/tests/unit/package/test_package.py index 8447094d6..5f682e2f6 100644 --- a/tests/unit/package/test_package.py +++ b/tests/unit/package/test_package.py @@ -109,6 +109,9 @@ def test_register_afd_preserves_native_models_and_is_idempotent(registration): "AFDQwen3MoeForCausalLM": ( "afd_plugin.model_executor.models.qwen3_moe:AFDQwen3MoeForCausalLM" ), + "AFDQwen3MoeRoutedExperts": ( + "afd_plugin.model_executor.models.qwen3_moe:AFDQwen3MoeRoutedExperts" + ), "AFDQwen3_5MoeForConditionalGeneration": ( "afd_plugin.model_executor.models.qwen3_5:" "AFDQwen3_5MoeForConditionalGeneration" diff --git a/tests/unit/v1/worker/test_ffn_routed_experts_model_runner.py b/tests/unit/v1/worker/test_ffn_routed_experts_model_runner.py index d29c3ac83..a28a99820 100644 --- a/tests/unit/v1/worker/test_ffn_routed_experts_model_runner.py +++ b/tests/unit/v1/worker/test_ffn_routed_experts_model_runner.py @@ -216,73 +216,6 @@ def test_local_experts_reject_backends_that_reorder_weights(fp8, backend): local_experts_from_vllm(_routed_experts(fp8=fp8, backend=backend)) -class _AttachableModel: - """An externally loaded model exposing two MoE layers' local experts.""" - - def __init__(self, expert_ids=(4, 5)): - self.experts = { - layer: _routed_experts(fp8=False, expert_ids=expert_ids) - for layer in MOE_LAYERS - } - - def get_experts_layer_indices(self): - return MOE_LAYERS - - def get_routed_experts(self, layer_idx): - return self.experts[layer_idx] - - -def _unattached_runner(): - runner = object.__new__(GPUFFNRoutedExpertsModelRunner) - runner.connector = FakeRoutedConnector() - runner.model = None - runner.model_memory_usage = 0 - runner.moe_layer_indices = () - runner.local_experts = {} - return runner - - -def test_attached_model_experts_reach_connector_startup(): - """A model attached from outside feeds the connector's startup collective - the same layers and expert views ``load_model`` would. - - Why: the standalone FFN service loads its own FFN-only model; the - connector's post-load placement check must see exactly those experts. - """ - runner = _unattached_runner() - model = _AttachableModel() - - runner.attach_model(model, model_memory_usage=1024) - runner.initialize_afd_connector() - - sent = runner.connector.init_kwargs - assert sent["moe_layer_indices"] == MOE_LAYERS - assert { - layer: experts.expert_ids for layer, experts in sent["local_experts"].items() - } == {layer: (4, 5) for layer in MOE_LAYERS} - assert ( - sent["local_experts"][MOE_LAYERS[0]].weights.w13.data_ptr() - == model.experts[MOE_LAYERS[0]].w13_weight.data_ptr() - ) - - -@pytest.mark.parametrize("when", ["second_attach", "after_connector_init"]) -def test_attach_model_rejects_late_or_repeated_attach(when): - runner = _unattached_runner() - if when == "second_attach": - runner.attach_model(_AttachableModel(), model_memory_usage=0) - else: - runner.initialize_afd_connector() - - with pytest.raises(RuntimeError, match="attached once, before connector startup"): - runner.attach_model(_AttachableModel(), model_memory_usage=0) - - -def test_attach_model_rejects_negative_memory_usage(): - with pytest.raises(ValueError, match="non-negative"): - _unattached_runner().attach_model(_AttachableModel(), model_memory_usage=-1) - - def _kernel_config(moe_backend: str) -> SimpleNamespace: return SimpleNamespace(kernel_config=SimpleNamespace(moe_backend=moe_backend)) From a607c3dbf7ebb64bbd537dbedd555c1e537964b9 Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:49:27 +0200 Subject: [PATCH 17/42] Fix GPU worker unit-test configuration stubs Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- .../models/test_deepseek_v2_construction.py | 1 - tests/unit/v1/worker/test_model_runner_v2.py | 9 +++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/unit/model_executor/models/test_deepseek_v2_construction.py b/tests/unit/model_executor/models/test_deepseek_v2_construction.py index 03756f275..ab846f7d9 100644 --- a/tests/unit/model_executor/models/test_deepseek_v2_construction.py +++ b/tests/unit/model_executor/models/test_deepseek_v2_construction.py @@ -764,7 +764,6 @@ def test_model_constructor_rejects_sequence_parallel_moe_before_allocation( def test_npu_async_attention_allows_model_owned_sp( monkeypatch, construction_env, - pp_size, ): vllm_config = _vllm_config() vllm_config.parallel_config.use_sequence_parallel_moe = True diff --git a/tests/unit/v1/worker/test_model_runner_v2.py b/tests/unit/v1/worker/test_model_runner_v2.py index 8c7e35c58..2e7a55562 100644 --- a/tests/unit/v1/worker/test_model_runner_v2.py +++ b/tests/unit/v1/worker/test_model_runner_v2.py @@ -25,6 +25,7 @@ from vllm.v1.worker.gpu.ubatch_utils import UBatchState # noqa: E402 from vllm.v1.worker.ubatch_utils import UBatchSlice # noqa: E402 +from afd_plugin.config import AFDConfig # noqa: E402 from afd_plugin.model_executor.models import ( # noqa: E402 forward_context as afd_forward_context, ) @@ -1443,7 +1444,9 @@ def _prepare_gpu_worker_runtime(monkeypatch, worker_case, runner_cls): monkeypatch.setattr(native_v2, "GPUModelRunner", _NativeRunnerSentinel) monkeypatch.setattr(worker_module.Worker, "init_device", _fake_worker_init) monkeypatch.setattr( - worker_module, "assert_compatible_afd_stack", lambda *args, **kwargs: None + worker_module, + "assert_compatible_afd_stack", + lambda *args, **kwargs: AFDConfig(role=role), ) monkeypatch.setattr(worker_module.torch.accelerator, "empty_cache", lambda: None) return worker @@ -1476,7 +1479,9 @@ def unexpected_construction(*_args, **_kwargs): monkeypatch.setattr(worker_module, runner_name, unexpected_construction) monkeypatch.setattr(worker_module.Worker, "init_device", unexpected_construction) monkeypatch.setattr( - worker_module, "assert_compatible_afd_stack", lambda *args, **kwargs: None + worker_module, + "assert_compatible_afd_stack", + lambda *args, **kwargs: AFDConfig(role=role), ) with pytest.raises(RuntimeError, match="requires ModelRunnerV2"): From 3003fc9448bf4f5425ab9eb79ea013994f124c7b Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:32:30 +0200 Subject: [PATCH 18/42] [GPU] FFN service: type the loaded FFN-only model GPUFFNRoutedExpertsModelRunner.get_model() returns Any, so assigning it to the Optional self.model attribute left the value typed as possibly None when it was passed to parameter_storage_bytes, and the mypy-3.11 pre-commit hook failed on backend.py. Bind the loaded model to a typed local first, then store it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- afd_plugin/ffn_faas/platforms/gpu/backend.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/afd_plugin/ffn_faas/platforms/gpu/backend.py b/afd_plugin/ffn_faas/platforms/gpu/backend.py index 20996a804..33825103c 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/backend.py +++ b/afd_plugin/ffn_faas/platforms/gpu/backend.py @@ -91,11 +91,12 @@ def prepare(self, service: FFNService) -> None: ) service.loading_progress(0, total_experts) self.runner.load_model() - self.model = self.runner.get_model() + model: AFDQwen3MoeRoutedExperts = self.runner.get_model() + self.model = model # vLLM's loader reports no per-expert progress; this rank's experts # are all loaded once it returns. service.loading_progress(total_experts, total_experts) - parameter_bytes = parameter_storage_bytes(self.model) + parameter_bytes = parameter_storage_bytes(model) local_expert_ids = self.runner.local_experts[ self.runner.moe_layer_indices[0] ].expert_ids From f86a41cda25246af88c9e2844317d5bcb25b032c Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:01:25 +0200 Subject: [PATCH 19/42] test(e2e): sweep FFN service torchrun workers on teardown torchrun starts its workers in their own sessions, outside the launcher process group that the E2E harness owns and signals. A SIGTERM to torchrun lets it stop the workers itself (a cancelled FFN service case on 4xH100 left no workers behind), but when torchrun does not exit within the harness grace period, the group SIGKILL reaches only torchrun and its workers survive holding GPUs into the next case (E2E-INV-003/E2E-INV-006). Give the FFN service scenarios an E2E run id so the existing environment-matched FFN sweep also reaps the torchrun workers, as the NPU async path already does. Co-Authored-By: Claude Opus 5.5 Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- tests/e2e/runner.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/e2e/runner.py b/tests/e2e/runner.py index e6891e35f..814cd1b21 100644 --- a/tests/e2e/runner.py +++ b/tests/e2e/runner.py @@ -154,9 +154,12 @@ def main() -> int: ffn_devices = parse_csv(args.ffn_devices) validate_topology(args, attention_devices, ffn_devices) use_npu_async_process_cleanup = uses_npu_async_process_cleanup(args) + # torchrun starts the FFN service workers in their own sessions, outside + # the launcher's process group, so the FFN service also needs the + # environment-matched sweep to reap every worker on teardown. e2e_run_id = ( f"{os.getpid()}-{time.monotonic_ns()}" - if use_npu_async_process_cleanup + if use_npu_async_process_cleanup or uses_ffn_faas(args) else None ) From d1b4c80e9d21b68c4781ab122115924c663c1e75 Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:32:32 +0200 Subject: [PATCH 20/42] Inline the NPU FFN attach guard validate_external_model_attachment was extracted for the GPU and NPU attach_model methods. The GPU attach_model was removed when the standalone GPU service moved to loading through the routed runner's native load_model, so the helper had a single NPU caller and a new v1/worker module with no owning design page. Restore the guard inline in AsyncNPUFFNModelRunner.attach_model and delete the module. Co-Authored-By: Claude Opus 5.5 Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- afd_plugin/v1/worker/model_attachment.py | 18 ------------------ .../v1/worker/npu/async_ffn_model_runner.py | 12 ++++-------- 2 files changed, 4 insertions(+), 26 deletions(-) delete mode 100644 afd_plugin/v1/worker/model_attachment.py diff --git a/afd_plugin/v1/worker/model_attachment.py b/afd_plugin/v1/worker/model_attachment.py deleted file mode 100644 index 526610746..000000000 --- a/afd_plugin/v1/worker/model_attachment.py +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project -"""Shared validation for models attached outside a Worker model runner.""" - -from __future__ import annotations - - -def validate_external_model_attachment( - *, - model_attached: bool, - connector_initialized: bool, - model_size_bytes: int, -) -> None: - """Require an external model attachment before connector startup.""" - if model_attached or connector_initialized: - raise RuntimeError("FFN model must be attached once, before connector startup") - if model_size_bytes < 0: - raise ValueError("model memory usage must be non-negative") diff --git a/afd_plugin/v1/worker/npu/async_ffn_model_runner.py b/afd_plugin/v1/worker/npu/async_ffn_model_runner.py index 69ace4131..8ce1e175a 100644 --- a/afd_plugin/v1/worker/npu/async_ffn_model_runner.py +++ b/afd_plugin/v1/worker/npu/async_ffn_model_runner.py @@ -35,9 +35,6 @@ LayeredFFNWeights, ) from afd_plugin.model_executor.npu.async_cam_mega import AsyncCAMMegaExecutor -from afd_plugin.v1.worker.model_attachment import ( - validate_external_model_attachment, -) from afd_plugin.validation import ( cam_receive_capacity, requested_ffn_mode, @@ -95,11 +92,10 @@ def __init__( def attach_model(self, model: FFNModel, *, parameter_bytes: int) -> None: """Borrow an independently loaded model without constructing a Worker.""" - validate_external_model_attachment( - model_attached=self.model is not None, - connector_initialized=self.connector.is_initialized, - model_size_bytes=parameter_bytes, - ) + if self.model is not None or self.connector.is_initialized: + raise RuntimeError("Async FFN model must be attached before CAM startup") + if parameter_bytes < 0: + raise ValueError("model parameter size must be nonnegative") self.model = model self.model_memory_usage = parameter_bytes From b7728171bcd7ee61633ba9d795d8ab35f663e9ec Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:37:45 +0200 Subject: [PATCH 21/42] Make GPU FFN service model agnostic Resolve the model path in the CPU adapter and let vLLM model config plus the AFD registry select the FFN-only shell. Remove the duplicate Qwen3 checkpoint parser and derive expert progress from the loaded runner. Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- afd_plugin/ffn_faas/platforms/gpu/__init__.py | 12 +- afd_plugin/ffn_faas/platforms/gpu/adapter.py | 21 ++- afd_plugin/ffn_faas/platforms/gpu/backend.py | 31 ++-- .../ffn_faas/platforms/gpu/qwen3_model.py | 132 ------------------ afd_plugin/ffn_faas/platforms/launch.py | 1 - docs/design/module/ffn_runtime.md | 22 +-- docs/gpu/STANDALONE_FFN_SERVICE.md | 20 +-- tests/unit/ffn_faas/test_gpu_adapter.py | 23 ++- tests/unit/ffn_faas/test_gpu_qwen3_model.py | 74 ---------- tests/unit/ffn_faas/test_gpu_settings.py | 2 + 10 files changed, 62 insertions(+), 276 deletions(-) delete mode 100644 afd_plugin/ffn_faas/platforms/gpu/qwen3_model.py delete mode 100644 tests/unit/ffn_faas/test_gpu_qwen3_model.py diff --git a/afd_plugin/ffn_faas/platforms/gpu/__init__.py b/afd_plugin/ffn_faas/platforms/gpu/__init__.py index ec845d1e7..262c46548 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/__init__.py +++ b/afd_plugin/ffn_faas/platforms/gpu/__init__.py @@ -1,15 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project -"""GPU standalone FFN backend: Qwen3 MoE routed experts over DeepEP M2N. +"""GPU standalone FFN backend for routed experts over DeepEP M2N. -The adapter, settings and checkpoint description are CPU-only. The -environment, FFN-only model and backend import torch, vLLM and afdEP lazily, -after the adapter has validated the launch. +The adapter and settings are CPU-only. The environment, FFN-only model and +backend import torch, vLLM and afdEP lazily, after the adapter has validated +the launch. """ # NPU parallel: afd_plugin/ffn_faas/platforms/npu/__init__.py (get_model # selects the one qualified NPU model). -# Reuse: NO - the GPU backend reads one checkpoint description directly and -# needs no model-adapter selection step. +# Reuse: NO - the GPU platform passes the model path to vLLM and lets the AFD +# model registry select the FFN-only architecture. GPU_DEEPEP_M2N_BACKEND = "gpu-deepep-m2n" diff --git a/afd_plugin/ffn_faas/platforms/gpu/adapter.py b/afd_plugin/ffn_faas/platforms/gpu/adapter.py index 319800051..219560866 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/adapter.py +++ b/afd_plugin/ffn_faas/platforms/gpu/adapter.py @@ -15,7 +15,6 @@ from afd_plugin.config import DEEPEP_M2N_CONNECTOR from afd_plugin.ffn_faas.interfaces import FFNLaunch -from afd_plugin.ffn_faas.platforms.gpu.qwen3_model import Qwen3CheckpointDescription from afd_plugin.ffn_faas.platforms.gpu.settings import ( BF16_PAYLOAD, DEFAULT_AFD_PORT, @@ -37,7 +36,7 @@ # Reuse: PARTIAL - the deferred-import factory shape is the same; GPU needs # no environment bootstrap before importing torch. def create_backend( - checkpoint: Qwen3CheckpointDescription, settings: GPUFFNConfig + settings: GPUFFNConfig, ) -> FFNBackend: """Import the GPU backend only when the host loop constructs it. @@ -47,7 +46,7 @@ def create_backend( """ from afd_plugin.ffn_faas.platforms.gpu.backend import DeepEPM2NFFNBackend - return DeepEPM2NFFNBackend(checkpoint, settings) + return DeepEPM2NFFNBackend(settings) # NPU parallel: afd_plugin/ffn_faas/platforms/npu/adapter.py::AscendCAMAdapter. @@ -101,14 +100,16 @@ def configure(self, args: Namespace) -> FFNLaunch: F-only world (``WORLD_SIZE == --ffn-ranks``) and ``VLLM_USE_V2_MODEL_RUNNER=1`` (Attention runs ModelRunnerV2 and the FFN runner validates the same deployment). Parses the connector JSON - structurally, reads the checkpoint description, and checks that the - experts divide across the F ranks. Raises ``ValueError`` or - ``OSError``, which the entry point reports as a usage error. + structurally and resolves the model path without interpreting its + architecture; vLLM and the AFD model registry own model selection. + Raises ``ValueError`` or ``OSError``, which the entry point reports as + a usage error. """ launcher = read_ffn_launcher_environment(args.ffn_ranks) require_v2_model_runner() + model_path = args.model.resolve(strict=True) settings = GPUFFNConfig( - model=args.model, + model=model_path, attention_ranks=args.attention_ranks, ffn_ranks=args.ffn_ranks, rank=launcher.rank, @@ -123,15 +124,13 @@ def configure(self, args: Namespace) -> FFNLaunch: afd_timeout_s=args.afd_timeout, shutdown_timeout=args.shutdown_timeout, ) - checkpoint = Qwen3CheckpointDescription.read(args.model) - checkpoint.local_experts(settings.ffn_ranks) return FFNLaunch( rank=launcher.rank, local_rank=launcher.local_rank, - model_path=str(checkpoint.root), + model_path=str(model_path), backend_name=DEEPEP_M2N_CONNECTOR, shutdown_timeout=settings.shutdown_timeout, - backend_factory=partial(create_backend, checkpoint, settings), + backend_factory=partial(create_backend, settings), initial_status={ "payload_format": BF16_PAYLOAD, "num_lanes": settings.num_lanes, diff --git a/afd_plugin/ffn_faas/platforms/gpu/backend.py b/afd_plugin/ffn_faas/platforms/gpu/backend.py index 33825103c..55d6f1aff 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/backend.py +++ b/afd_plugin/ffn_faas/platforms/gpu/backend.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project -"""FFN-only Qwen3 routed experts composed with the existing GPU M2N runtime. +"""FFN-only routed experts composed with the existing GPU M2N runtime. Imported inside the externally launched F process, after the adapter has validated the launch. No vLLM Worker, upstream ModelRunner, EngineCore, @@ -18,12 +18,11 @@ from afd_plugin.connectors.gpu.deepep_m2n.connector import DeepEPM2NAFDConnector from afd_plugin.ffn_faas.model_utils import parameter_storage_bytes from afd_plugin.ffn_faas.platforms.gpu.environment import GPUFFNEnvironment -from afd_plugin.ffn_faas.platforms.gpu.qwen3_model import Qwen3CheckpointDescription from afd_plugin.ffn_faas.platforms.gpu.settings import GPUFFNConfig from afd_plugin.ffn_faas.service import FFNService, ServicePhase -from afd_plugin.model_executor.models.qwen3_moe import AFDQwen3MoeRoutedExperts from afd_plugin.v1.worker.ffn_routed_experts_model_runner import ( GPUFFNRoutedExpertsModelRunner, + RoutedExpertsModel, ) @@ -38,15 +37,12 @@ class DeepEPM2NFFNBackend: """Keep weights and communication owners until receiver exit is proven.""" - def __init__( - self, checkpoint: Qwen3CheckpointDescription, settings: GPUFFNConfig - ) -> None: + def __init__(self, settings: GPUFFNConfig) -> None: """Hold the CPU-side launch facts; native work starts in ``prepare``.""" - self.checkpoint = checkpoint self.settings = settings self.environment = GPUFFNEnvironment(settings) self.runner: GPUFFNRoutedExpertsModelRunner | None = None - self.model: AFDQwen3MoeRoutedExperts | None = None + self.model: RoutedExpertsModel | None = None self._cancel_requested = threading.Event() def prepare(self, service: FFNService) -> None: @@ -61,14 +57,14 @@ def prepare(self, service: FFNService) -> None: 4. construct the routed-experts runner, which runs the ModelRunnerV2 and CUDA-graph validation and creates the connector (no collective yet); - 5. load the FFN-only Qwen3 shell through the runner's native loader; + 5. load the FFN-only routed-experts shell through vLLM's native loader; 6. start the connector: the collective M2N handshake with Attention, the model-info check, the NCCL control group, the lanes' DeepEP buffers and the dispatch-kernel precompile. - Phases: ``loading`` through step 5 (with ``total_experts`` and, after - the load, ``loaded_experts``), ``connecting`` from step 5 on; the host - marks ``ready`` afterwards. CUDA graphs are not captured here: they + Phases: ``loading`` through step 5 (with ``loaded_experts`` and + ``total_experts`` after the load), ``connecting`` from step 5 on; the + host marks ``ready`` afterwards. CUDA graphs are not captured here: they are captured lazily, driven by Attention's warmup and capture payloads, exactly as on the Worker path. Every owner is assigned before the next native step, so a failure leaves it reachable. @@ -77,10 +73,6 @@ def prepare(self, service: FFNService) -> None: environment.initialize_device() config = environment.build_config() environment.initialize_compute(config) - # Counted like the NPU backend: this rank's experts over every layer. - total_experts = self.checkpoint.num_layers * self.checkpoint.local_experts( - self.settings.ffn_ranks - ) with set_current_vllm_config(config): self.runner = GPUFFNRoutedExpertsModelRunner(config, environment.device) connector = self.runner.connector @@ -89,12 +81,15 @@ def prepare(self, service: FFNService) -> None: f"standalone GPU FFN needs DeepEPM2NAFDConnector, got " f"{type(connector).__name__}" ) - service.loading_progress(0, total_experts) self.runner.load_model() - model: AFDQwen3MoeRoutedExperts = self.runner.get_model() + model: RoutedExpertsModel = self.runner.get_model() self.model = model # vLLM's loader reports no per-expert progress; this rank's experts # are all loaded once it returns. + total_experts = sum( + len(self.runner.local_experts[layer_idx].expert_ids) + for layer_idx in self.runner.moe_layer_indices + ) service.loading_progress(total_experts, total_experts) parameter_bytes = parameter_storage_bytes(model) local_expert_ids = self.runner.local_experts[ diff --git a/afd_plugin/ffn_faas/platforms/gpu/qwen3_model.py b/afd_plugin/ffn_faas/platforms/gpu/qwen3_model.py deleted file mode 100644 index 7f2d9710e..000000000 --- a/afd_plugin/ffn_faas/platforms/gpu/qwen3_model.py +++ /dev/null @@ -1,132 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project -"""CPU-only description of the Qwen3 MoE checkpoints the GPU FFN service serves. - -Reads ``config.json`` only. Rejects every Qwen3 shape the routed-experts M2N -path cannot serve before any device, process group or weight is touched. -""" - -from __future__ import annotations - -import json -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -# NPU parallel: afd_plugin/ffn_faas/platforms/npu/v4_model.py:: -# V4CheckpointDescription (CPU-only checkpoint facts for /status and -# startup validation). -# Reuse: PARTIAL - the read-validate-describe pattern and the positive-int -# helper are the same; the V4 description also maps W4A8 safetensors names -# for its own loader, while Qwen3 loads through vLLM's native loader. -# REFACTOR-CANDIDATE(R6): a shared CPU checkpoint-config reader (config -# JSON parse, positive-int fields, dtype check). - -QWEN3_MOE_ARCHITECTURE = "Qwen3MoeForCausalLM" -BF16_DTYPE = "bfloat16" -BF16_BYTES = 2 -# gate_proj, up_proj and down_proj per routed expert. -PROJECTIONS_PER_EXPERT = 3 -# Qwen3 MoE places an MoE block on every layer whose index + 1 is a multiple of -# decoder_sparse_step; the routed-experts path requires every layer to be MoE. -DENSE_FREE_SPARSE_STEP = 1 - - -def _positive_int(config: dict[str, Any], name: str) -> int: - """Read a required positive integer field, rejecting bools and absent keys.""" - value = config.get(name) - if not isinstance(value, int) or isinstance(value, bool) or value <= 0: - raise ValueError(f"Qwen3 MoE checkpoint {name} must be a positive integer") - return value - - -@dataclass(frozen=True) -class Qwen3CheckpointDescription: - """Facts about one supported Qwen3 MoE checkpoint, read on the CPU.""" - - root: Path - num_layers: int - num_experts: int - hidden_size: int - moe_intermediate_size: int - top_k: int - - @property - def layer_indices(self) -> tuple[int, ...]: - """MoE layer indices; every layer is MoE for a supported checkpoint.""" - return tuple(range(self.num_layers)) - - @classmethod - def read(cls, model_path: str | Path) -> Qwen3CheckpointDescription: - """Parse ``config.json`` and accept only the routed-experts Qwen3 shape. - - Accepts ``Qwen3MoeForCausalLM`` in BF16 without quantization, with an - MoE block on every layer (no ``mlp_only_layers``, sparse step 1) and - no shared expert. ``shared_expert_intermediate_size`` that is absent or - ``null`` counts as zero, as vLLM reads it; the stock Qwen3-30B-A3B - config omits it. Newer configs name the dtype ``dtype`` instead of - ``torch_dtype``; either is read. Raises ``ValueError`` for anything - else, and ``OSError`` when the path or file is missing. - """ - root = Path(model_path).resolve(strict=True) - try: - config = json.loads((root / "config.json").read_text()) - except json.JSONDecodeError as error: - raise ValueError(f"invalid Qwen3 config.json: {error}") from error - if not isinstance(config, dict): - raise ValueError("Qwen3 config.json must be a JSON object") - if config.get("architectures") != [QWEN3_MOE_ARCHITECTURE]: - raise ValueError( - f"GPU standalone FFN supports only {QWEN3_MOE_ARCHITECTURE}, got " - f"{config.get('architectures')!r}" - ) - dtype = config.get("torch_dtype", config.get("dtype")) - if dtype != BF16_DTYPE: - raise ValueError( - f"GPU standalone FFN requires a BF16 checkpoint, got {dtype!r}" - ) - if config.get("quantization_config") is not None: - raise ValueError( - "GPU standalone FFN does not support quantized checkpoints" - ) - if config.get("mlp_only_layers"): - raise ValueError("GPU standalone FFN requires every layer to be MoE") - if config.get("decoder_sparse_step", DENSE_FREE_SPARSE_STEP) != ( - DENSE_FREE_SPARSE_STEP - ): - raise ValueError("GPU standalone FFN requires decoder_sparse_step=1") - if (config.get("shared_expert_intermediate_size") or 0) > 0: - raise ValueError("GPU standalone FFN does not support shared experts") - return cls( - root=root, - num_layers=_positive_int(config, "num_hidden_layers"), - num_experts=_positive_int(config, "num_experts"), - hidden_size=_positive_int(config, "hidden_size"), - moe_intermediate_size=_positive_int(config, "moe_intermediate_size"), - top_k=_positive_int(config, "num_experts_per_tok"), - ) - - def local_experts(self, ffn_ranks: int) -> int: - """Experts per F rank under linear placement; rejects an uneven split.""" - if ffn_ranks <= 0 or self.num_experts % ffn_ranks: - raise ValueError( - f"{self.num_experts} routed experts do not divide across " - f"{ffn_ranks} FFN ranks" - ) - return self.num_experts // ffn_ranks - - def estimated_local_expert_bytes(self, ffn_ranks: int) -> int: - """BF16 bytes of this rank's routed experts over every layer. - - ``layers * local experts * 3 projections * hidden * intermediate * 2``; - 48 x 64 x 3 x 2048 x 768 x 2 B is about 27 GiB for Qwen3-30B-A3B at 2F. - The MoE gate (``num_experts x hidden`` per layer) is not counted. - """ - return ( - self.num_layers - * self.local_experts(ffn_ranks) - * PROJECTIONS_PER_EXPERT - * self.hidden_size - * self.moe_intermediate_size - * BF16_BYTES - ) diff --git a/afd_plugin/ffn_faas/platforms/launch.py b/afd_plugin/ffn_faas/platforms/launch.py index 676b0a0bd..11f3dcce3 100644 --- a/afd_plugin/ffn_faas/platforms/launch.py +++ b/afd_plugin/ffn_faas/platforms/launch.py @@ -83,7 +83,6 @@ def ffn_engine_kwargs( """Build common ``EngineArgs`` keywords and merge backend-specific extras.""" kwargs: dict[str, Any] = { "model": str(model), - "dtype": "bfloat16", "tensor_parallel_size": 1, "data_parallel_size": ffn_ranks, "data_parallel_rank": ffn_rank, diff --git a/docs/design/module/ffn_runtime.md b/docs/design/module/ffn_runtime.md index c276411d3..d5d0ef78f 100644 --- a/docs/design/module/ffn_runtime.md +++ b/docs/design/module/ffn_runtime.md @@ -135,22 +135,24 @@ native work retains all owners until whole-pair process isolation. A host See the [standalone service guide](../../npu/STANDALONE_FFN_SERVICE.md) for supported configuration, launch and shutdown behavior. -Qwen3-30B-A3B (BF16) over `DeepEPM2NAFDConnector` has a GPU standalone F, -`--backend gpu-deepep-m2n`, under `ffn_faas/platforms/gpu/`: +CUDA routed-experts models over `DeepEPM2NAFDConnector` have a GPU standalone +F, `--backend gpu-deepep-m2n`, under `ffn_faas/platforms/gpu/`: -- `GPUDeepEPM2NAdapter`, `GPUFFNConfig` and `Qwen3CheckpointDescription` are - CPU-only: launcher world, ModelRunnerV2 environment, connector JSON - structure, BF16/no-shared-expert/all-MoE checkpoint shape and lane parity - with `--dbo` are checked before any device work. +- `GPUDeepEPM2NAdapter` and `GPUFFNConfig` are CPU-only: launcher world, + ModelRunnerV2 environment, connector JSON structure and lane parity with + `--dbo` are checked before any device work. The adapter resolves the model + path but does not parse or name a model family. - `GPUFFNEnvironment` selects the GPU, preflights afdEP and DeepGEMM, builds the F-only `VllmConfig` (`external_launcher`, DP = F ranks, TP 1, EP), selects the FFN-only routed-experts architecture, pins vLLM's MoE backend to Triton before any FusedMoE is built, and creates the F-only world through vLLM's `init_worker_distributed_environment`. -- `AFDQwen3MoeRoutedExperts` keeps only `model.layers..mlp` (native - `Qwen3MoeSparseMoeBlock`) and loads through vLLM's `DefaultModelLoader`, so - the EP weight filter, FusedMoE expert loading and strict loaded-weights check - run unchanged; the embedding, final norm and LM head are not allocated. +- The AFD model registry selects the FFN-only shell for the configured model. + For example, `AFDQwen3MoeRoutedExperts` keeps only `model.layers..mlp` + (native `Qwen3MoeSparseMoeBlock`) and loads through vLLM's + `DefaultModelLoader`, so the EP weight filter, FusedMoE expert loading and + strict loaded-weights check run unchanged; the embedding, final norm and LM + head are not allocated. - `DeepEPM2NFFNBackend` constructs `GPUFFNRoutedExpertsModelRunner`, loads the shell through that runner, starts the connector, and delegates each step to `execute_connector_driven_step()`, the method the Worker loop also calls. diff --git a/docs/gpu/STANDALONE_FFN_SERVICE.md b/docs/gpu/STANDALONE_FFN_SERVICE.md index d274b05e8..69a7e0bec 100644 --- a/docs/gpu/STANDALONE_FFN_SERVICE.md +++ b/docs/gpu/STANDALONE_FFN_SERVICE.md @@ -1,17 +1,17 @@ # Standalone GPU FFN service -Run the Qwen3-30B-A3B routed experts in independent F-only processes with -`torchrun -m afd_plugin.ffn_faas serve --backend gpu-deepep-m2n`. The service -loads only the routed experts, reuses vLLM's native loader and the existing -DeepEP M2N FFN runtime, and constructs no vLLM Worker, upstream ModelRunner, -EngineCore, scheduler, KV cache or complete CausalLM. Attention keeps its -existing `vllm serve` launch, model, connector and DBO path. +Run a registered CUDA routed-experts model in independent F-only processes +with `torchrun -m afd_plugin.ffn_faas serve --backend gpu-deepep-m2n`. The +service loads only the routed experts, reuses vLLM's native loader and the +existing DeepEP M2N FFN runtime, and constructs no vLLM Worker, upstream +ModelRunner, EngineCore, scheduler, KV cache or complete CausalLM. Attention +keeps its existing `vllm serve` launch, model, connector and DBO path. ## Supported configuration | Supported | Rejected at startup | | --- | --- | -| `Qwen3MoeForCausalLM`, BF16, every layer MoE, no shared experts (`Qwen/Qwen3-30B-A3B`) | FP8 or any `quantization_config`, dense or partial-MoE Qwen3, shared experts, other models | +| A registered CUDA routed-experts model; currently `Qwen3MoeForCausalLM`, BF16, every layer MoE, no shared experts | Unregistered models, FP8 or any `quantization_config`, dense or partial-MoE Qwen3, shared experts | | `DeepEPM2NAFDConnector` with `payload_format: bf16` | P2P, afdEP-direct, flexible placement, FP8 payload | | F TP=1, EP over all F ranks, linear expert placement | EPLB, elastic EP, redundant experts, PP/CP, LoRA, speculative decoding | | Eager, FULL_DECODE_ONLY CUDA graphs, DBO with two lanes (eager and graph) | Other graph modes, more than two micro-batches | @@ -25,7 +25,7 @@ The routed experts must divide evenly across the F ranks. CPU service / host runner -> FFNBackend DeepEPM2NFFNBackend (ffn_faas/platforms/gpu/backend.py) GPUFFNEnvironment: GPU, F-only NCCL world and vLLM EP group - AFDQwen3MoeRoutedExperts: model.layers..mlp only, native vLLM loader + AFD routed-experts model selected by the model registry GPUFFNRoutedExpertsModelRunner: the Worker path's FFN runner DeepEPM2NAFDConnector: handshake, control plane, lanes, experts ``` @@ -115,8 +115,8 @@ Each local F rank serves `--status-port + LOCAL_RANK` (`0` chooses a free port; each rank prints its address at start). - `/status`: phase, `loaded_experts`/`total_experts` (this rank's experts over - all layers; vLLM's loader reports no per-expert progress, so the count - jumps from 0 to the total), `parameter_bytes`, `local_expert_first`/ + all layers; vLLM's loader reports no per-expert progress, so both values are + reported together after loading), `parameter_bytes`, `local_expert_first`/ `local_expert_last`, `m2n_paired`, `attn_tp_size`, `capacity_tokens_per_rank`, `num_buckets`, `network_hybrid`, `compute_ep_world_size`, `num_lanes`, `graph_mode`, `dbo` and diff --git a/tests/unit/ffn_faas/test_gpu_adapter.py b/tests/unit/ffn_faas/test_gpu_adapter.py index 0503916aa..33d20238e 100644 --- a/tests/unit/ffn_faas/test_gpu_adapter.py +++ b/tests/unit/ffn_faas/test_gpu_adapter.py @@ -14,7 +14,6 @@ import pytest from afd_plugin.ffn_faas.platforms import get_platform_adapter -from tests.unit.fakes.ffn_faas import write_qwen3_checkpoint REPO_ROOT = Path(__file__).resolve().parents[3] NATIVE_MODULES = ("torch", "vllm", "afdep", "deep_gemm") @@ -35,9 +34,10 @@ def launch_env(monkeypatch): @pytest.fixture def configure(tmp_path): """Parse a ``serve`` command line for the GPU adapter and configure it.""" - checkpoint = write_qwen3_checkpoint(tmp_path / "model") + model = tmp_path / "model" + model.mkdir() - def run(*options, model=checkpoint): + def run(*options, model=model): adapter = get_platform_adapter("gpu-deepep-m2n") parser = ArgumentParser() parser.add_argument("model", type=Path) @@ -69,7 +69,9 @@ def test_backend_selection_imports_no_native_stack(): assert json.loads(result.stdout) == [] -def test_valid_launch_describes_the_rank_and_its_lanes(launch_env, configure, tmp_path): +def test_valid_launch_preserves_the_model_path_and_describes_the_rank( + launch_env, configure, tmp_path +): launch = configure("--graph", "--dbo", "--port", "6239") assert (launch.rank, launch.local_rank) == (1, 1) @@ -103,16 +105,9 @@ def test_invalid_launch_is_rejected( configure(*options) -def test_unsupported_checkpoint_is_rejected(launch_env, configure, tmp_path): - fp8 = write_qwen3_checkpoint( - tmp_path / "fp8", quantization_config={"quant_method": "fp8"} - ) - with pytest.raises(ValueError, match="quantized"): - configure(model=fp8) - - def test_cli_reports_a_bad_launch_as_a_usage_error(tmp_path): - checkpoint = write_qwen3_checkpoint(tmp_path / "model") + model = tmp_path / "model" + model.mkdir() env = {**os.environ, "WORLD_SIZE": "1", "VLLM_USE_V2_MODEL_RUNNER": "1"} result = subprocess.run( [ @@ -120,7 +115,7 @@ def test_cli_reports_a_bad_launch_as_a_usage_error(tmp_path): "-m", "afd_plugin.ffn_faas", "serve", - str(checkpoint), + str(model), "--backend", "gpu-deepep-m2n", "--attention-ranks", diff --git a/tests/unit/ffn_faas/test_gpu_qwen3_model.py b/tests/unit/ffn_faas/test_gpu_qwen3_model.py deleted file mode 100644 index d234742b0..000000000 --- a/tests/unit/ffn_faas/test_gpu_qwen3_model.py +++ /dev/null @@ -1,74 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project -"""The GPU FFN service accepts only the routed-experts Qwen3 MoE shape.""" - -from __future__ import annotations - -import pytest - -from afd_plugin.ffn_faas.platforms.gpu.qwen3_model import Qwen3CheckpointDescription -from tests.unit.fakes.ffn_faas import write_qwen3_checkpoint - -# Qwen3-30B-A3B at 2F: 48 layers x 64 experts x 3 x 2048 x 768 x 2 B. -QWEN3_30B_2F_EXPERT_BYTES = 48 * 64 * 3 * 2048 * 768 * 2 - - -def test_stock_qwen3_30b_checkpoint_is_described(tmp_path): - checkpoint = Qwen3CheckpointDescription.read(write_qwen3_checkpoint(tmp_path)) - - assert checkpoint.root == tmp_path.resolve() - assert checkpoint.layer_indices == tuple(range(48)) - assert (checkpoint.num_experts, checkpoint.top_k) == (128, 8) - assert checkpoint.local_experts(2) == 64 - assert checkpoint.estimated_local_expert_bytes(2) == QWEN3_30B_2F_EXPERT_BYTES - - -@pytest.mark.parametrize( - "overrides", - [ - {"shared_expert_intermediate_size": 0}, - {"shared_expert_intermediate_size": None}, - {"torch_dtype": None, "dtype": "bfloat16"}, - ], - ids=["shared-zero", "shared-absent", "dtype-key"], -) -def test_equivalent_config_spellings_are_accepted(tmp_path, overrides): - checkpoint = Qwen3CheckpointDescription.read( - write_qwen3_checkpoint(tmp_path, **overrides) - ) - assert checkpoint.num_layers == 48 - - -@pytest.mark.parametrize( - "overrides, message", - [ - ({"architectures": ["DeepseekV2ForCausalLM"]}, "Qwen3MoeForCausalLM"), - ({"torch_dtype": "float16"}, "BF16"), - ( - {"quantization_config": {"quant_method": "fp8"}}, - "quantized", - ), - ({"mlp_only_layers": [0]}, "every layer to be MoE"), - ({"decoder_sparse_step": 2}, "decoder_sparse_step=1"), - ({"shared_expert_intermediate_size": 512}, "shared experts"), - ({"num_experts": 0}, "num_experts must be a positive integer"), - ], - ids=[ - "architecture", - "dtype", - "fp8", - "mlp-only-layers", - "sparse-step", - "shared-experts", - "no-experts", - ], -) -def test_unsupported_qwen3_shapes_fail_on_cpu(tmp_path, overrides, message): - with pytest.raises(ValueError, match=message): - Qwen3CheckpointDescription.read(write_qwen3_checkpoint(tmp_path, **overrides)) - - -def test_experts_must_divide_across_ffn_ranks(tmp_path): - checkpoint = Qwen3CheckpointDescription.read(write_qwen3_checkpoint(tmp_path)) - with pytest.raises(ValueError, match="do not divide across 3 FFN ranks"): - checkpoint.local_experts(3) diff --git a/tests/unit/ffn_faas/test_gpu_settings.py b/tests/unit/ffn_faas/test_gpu_settings.py index 06e31ccfb..1815acc0a 100644 --- a/tests/unit/ffn_faas/test_gpu_settings.py +++ b/tests/unit/ffn_faas/test_gpu_settings.py @@ -56,6 +56,8 @@ def test_afd_config_matches_a_worker_path_m2n_ffn(): def test_engine_args_describe_an_f_only_ep_rank(graph, dbo, eager, compilation_config): kwargs = _settings(graph=graph, dbo=dbo).engine_kwargs() + assert kwargs["model"] == "/model" + assert "dtype" not in kwargs assert kwargs["distributed_executor_backend"] == "external_launcher" assert (kwargs["data_parallel_size"], kwargs["data_parallel_rank"]) == (2, 1) assert kwargs["tensor_parallel_size"] == 1 From 22a1eccb142e4456ef577391524a1d977a7ac646 Mon Sep 17 00:00:00 2001 From: Ronen Kat <16743404+ronenkat@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:44:14 +0200 Subject: [PATCH 22/42] [Skill] deploy-afd-k8s: simplify readiness, timeouts, and recipe parsing (#533) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Goal Simplify the `deploy-afd-k8s` skill: one readiness check, consistent timeouts, and recipe parsing in a single place. ## Changes 1. **Single readiness signal** (`resources/deploy.md` step 4f): wait for each pod's `Ready` condition, which the existing `readinessProbe` already drives, and stop early when a pod goes `Failed`. This replaces the second pass that grepped pod logs for the `READY`/`ERROR:` lines. 2. **Consistent timeouts** (`templates/pod.yaml`): the per-log 40-minute loops and the separate `/health` loop become one 55-minute startup budget. It stays below the 60-minute `Ready` wait in step 4f, so a slow pod always fails with its own `ERROR:` reason first. The `/health` wait also notices if the recipe process has exited. 3. **One recipe parser** (`resources/resolve-recipe.md`): step 1 records `GPU_COUNT` for each pod (and for a plain recipe), and `deploy.md` step 2 just uses it. This removes the `awk` role-block parser. ## Shutdown instructions `resources/report-and-teardown.md` step 5 now ends by asking the caller (the user, or the task that invoked the skill, e.g. `run-vllm-bench-k8s`) when to delete the running pods. The caller owns that decision and triggers teardown (step 6). The skill never tears down on its own, and the model PVC is still kept. Also renames `references/` to `resources/` and updates the links. ## Validation - The rendered `pod.yaml` parses as YAML, and its startup script passes `bash -n`. - The startup script, run locally against a fake recipe: it reaches `READY`, fails with `ERROR:` when its time budget runs out, and fails when the recipe exits early. - The step 4f wait loop, run against a stubbed `kubectl`: it finishes when every pod is `Ready` and exits non-zero with logs when a pod is `Failed`. - Every relative link resolves, and pre-commit passes. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- .agents/skills/deploy-afd-k8s/SKILL.md | 16 ++--- .../{references => resources}/deploy.md | 67 +++++++++---------- .../report-and-teardown.md | 7 ++ .../resolve-recipe.md | 17 +++++ .../{references => resources}/services.md | 0 .../skills/deploy-afd-k8s/templates/pod.yaml | 32 +++++---- 6 files changed, 78 insertions(+), 61 deletions(-) rename .agents/skills/deploy-afd-k8s/{references => resources}/deploy.md (78%) rename .agents/skills/deploy-afd-k8s/{references => resources}/report-and-teardown.md (69%) rename .agents/skills/deploy-afd-k8s/{references => resources}/resolve-recipe.md (85%) rename .agents/skills/deploy-afd-k8s/{references => resources}/services.md (100%) diff --git a/.agents/skills/deploy-afd-k8s/SKILL.md b/.agents/skills/deploy-afd-k8s/SKILL.md index f51ae4190..7600ffd06 100644 --- a/.agents/skills/deploy-afd-k8s/SKILL.md +++ b/.agents/skills/deploy-afd-k8s/SKILL.md @@ -19,13 +19,13 @@ On success this skill guarantees, in the target namespace: | Serving | the model named by `MODEL_ID` | | Pod(s) / Service | `vllm-pod`, or one pod per pod the recipe's header comment describes (named `vllm-pod-`); Service `vllm-service`, labels `app=afd-recipe,role=serve` | | Model cache | `PVC_NAME`, mounted at `/models`, `HF_HOME=/models/.hf_home` | -| Left running | yes, deliberately -- so weights stay warm for follow-up runs | +| Left running | yes, deliberately -- so weights stay warm for follow-up runs; step 5 asks the caller when to delete the pods | Whenever a recipe's header spans more than one pod, two more internal-only Services may also exist (`vllm-ffn-p2p-service` always; `vllm-attn-dp-service` and/or `vllm-ffn-dp-service` only if that role is itself split across more than one pod) -- callers never talk to these directly. See -[services.md](references/services.md). +[services.md](resources/services.md). ## Scope @@ -43,7 +43,7 @@ than one pod) -- callers never talk to these directly. See block, don't infer shape from flags. Today's recipes spell it as one `# For pod N: POD=...` line per pod, but the comment isn't guaranteed to use that exact grammar going forward. See - [resolve-recipe.md](references/resolve-recipe.md). + [resolve-recipe.md](resources/resolve-recipe.md). - Not for `tools/benchmarks/decode_bench_server.sh` (local, non-k8s). ## Requirements @@ -75,13 +75,13 @@ than one pod) -- callers never talk to these directly. See Load these in order; each covers one phase and nothing else. Services must exist **before** any Pod is deployed -- see -[services.md](references/services.md) for why: +[services.md](resources/services.md) for why: -| Reference | Covers | +| Resource | Covers | |---|---| -| [resolve-recipe.md](references/resolve-recipe.md) | Step 1: read the recipe's serve-block fields and model id; read its placement (or lack of one) from its header comment -- never ask. | -| [deploy.md](references/deploy.md) | Steps 2-4f: GPU_COUNT, confirm, PVC, ConfigMap, fsGroup, Services (step 4d -- which of the four, see [services.md](references/services.md)), pod(s) via `templates/pod.yaml`, wait for Running and for each pod's own readiness line. | -| [report-and-teardown.md](references/report-and-teardown.md) | Step 5: report the endpoint/model/nodes. Step 6: teardown (pods/Services only -- never the PVC unless asked). | +| [resolve-recipe.md](resources/resolve-recipe.md) | Step 1 (the only step that parses the recipe): read the recipe's serve-block fields and model id; read its placement (or lack of one) from its header comment -- never ask; record `GPU_COUNT` per pod. | +| [deploy.md](resources/deploy.md) | Steps 2-4f: take step 1's `GPU_COUNT`, confirm, PVC, ConfigMap, fsGroup, Services (step 4d -- which of the four, see [services.md](resources/services.md)), pod(s) via `templates/pod.yaml`, wait for Running and then for each pod's `Ready` condition. | +| [report-and-teardown.md](resources/report-and-teardown.md) | Step 5: report the endpoint/model/nodes and ask when to delete the running pods. Step 6: teardown when the caller says so (pods/Services only -- never the PVC unless asked). | All k8s manifests live in `templates/*.yaml`, applied via `envsubst | kubectl apply -f -`; no manifest is inlined in these docs. diff --git a/.agents/skills/deploy-afd-k8s/references/deploy.md b/.agents/skills/deploy-afd-k8s/resources/deploy.md similarity index 78% rename from .agents/skills/deploy-afd-k8s/references/deploy.md rename to .agents/skills/deploy-afd-k8s/resources/deploy.md index eccd5423a..a6842857a 100644 --- a/.agents/skills/deploy-afd-k8s/references/deploy.md +++ b/.agents/skills/deploy-afd-k8s/resources/deploy.md @@ -1,32 +1,10 @@ -# 2-4. Compute GPU_COUNT, confirm, and deploy +# 2-4. Confirm and deploy ## 2. GPU_COUNT -Plain single-pod recipe (no plan from [resolve-recipe.md](resolve-recipe.md)): -union of every literal `CUDA_VISIBLE_DEVICES=` in the script. - -```bash -GPU_COUNT="$(grep -oE 'CUDA_VISIBLE_DEVICES=[0-9,]+' "$RECIPE_SCRIPT_PATH" \ - | cut -d= -f2 | tr ',' '\n' | sort -u | wc -l)" -``` - -Per plan entry: count the devices in the literal `CUDA_VISIBLE_DEVICES=` inside -the recipe's `if` block that tests this pod's role (`ATTENTION` or `FFN` -- -whichever prefix `POD` starts with). Every pod sharing a role shares that -literal, so this is a plain text extraction, not a per-pod computation: - -```bash -role_block() { # $1=script $2=ATTENTION|FFN - awk -v role="$2" ' - /if.*\$POD/ && $0 ~ role { inblock=1; next } - inblock && /^fi/ { inblock=0 } - inblock { print } - ' "$1" -} -devices="$(role_block "$RECIPE_SCRIPT_PATH" "$ROLE" \ - | grep -oE 'CUDA_VISIBLE_DEVICES=[0-9,]+' | head -1 | cut -d= -f2)" -GPU_COUNT=$(( $(grep -o ',' <<<"$devices" | wc -l) + 1 )) -``` +Use the `GPU_COUNT` recorded by [resolve-recipe.md](resolve-recipe.md) -- +one value for a plain recipe, one per plan entry. Don't re-read the script +here; all recipe parsing lives in step 1. ## 3. Confirm before touching the cluster @@ -168,12 +146,25 @@ deploy_pod vllm-pod-attention-1 2 worker none "POD=ATTENTION_1 ATTENTION_HEADLES deploy_pod vllm-pod-ffn-0 2 none head "POD=FFN_0" ``` -## 4f. Wait for Running, then for each pod's own readiness line +## 4f. Wait for Running, then for each pod's Ready condition Bounded 15-minute wait per pod for `phase=Running` (unschedulable pods never -reach `Failed`, so an unbounded loop would spin forever), then poll logs for -`=== pod READY ===` (budget 60 minutes -- model download and graph -capture dominate): +reach `Failed`, so an unbounded wait would spin forever), then a 60-minute +wait per pod for its `Ready` condition (model download and graph capture +dominate). + +`Ready` is the single readiness signal. The pod's `readinessProbe` only +turns it on after the in-pod startup script has seen every log marker, got +`/health` from the attention head, and printed `=== pod READY ===`; +the same condition is what puts the attention head behind `vllm-service`. +If the in-pod script gives up, it prints `ERROR:` with log tails and exits, +the pod goes `Failed`, and the loop below stops at once. + +Timeouts are nested on purpose: the in-pod startup deadline +(`STARTUP_TIMEOUT_S` in `templates/pod.yaml`, 55 minutes, one budget shared +by every log and `/health`) is shorter than the 60-minute wait here, so a +slow pod always fails with its own `ERROR:` reason instead of an outer +timeout. Change the two together. ```bash for pod_name in ; do @@ -184,17 +175,21 @@ done for pod_name in ; do deadline=$(( $(date +%s) + 3600 )) while true; do - logs="$(kubectl logs "pod/$pod_name" 2>/dev/null || true)" - echo "$logs" | grep -q "pod ${pod_name} READY" && break - echo "$logs" | grep -q "ERROR:" && { kubectl logs "$pod_name" --tail=200; exit 1; } - [ "$(date +%s)" -ge "$deadline" ] && { kubectl logs "$pod_name" --tail=200; exit 1; } + read -r phase ready < <(kubectl get pod "$pod_name" \ + -o jsonpath='{.status.phase} {.status.conditions[?(@.type=="Ready")].status}' || true) + [ "$ready" = "True" ] && break + if [ "$phase" = "Failed" ] || [ "$(date +%s)" -ge "$deadline" ]; then + echo "pod $pod_name not Ready (phase=${phase:-unknown})" + kubectl logs "$pod_name" --tail=200 + exit 1 + fi sleep 10 done done ``` -Only report the endpoint ready once every pod in the plan has printed its -own `READY` line -- one pod becoming ready doesn't imply the others did. +Only report the endpoint ready once every pod in the plan is `Ready` -- one +pod becoming ready doesn't imply the others did. Startup markers are keyed on the fixed log names every colocation recipe uses today -- see `templates/pod.yaml`'s `ready_marker()` for the exact diff --git a/.agents/skills/deploy-afd-k8s/references/report-and-teardown.md b/.agents/skills/deploy-afd-k8s/resources/report-and-teardown.md similarity index 69% rename from .agents/skills/deploy-afd-k8s/references/report-and-teardown.md rename to .agents/skills/deploy-afd-k8s/resources/report-and-teardown.md index 70fbb7e03..7949c2fbb 100644 --- a/.agents/skills/deploy-afd-k8s/references/report-and-teardown.md +++ b/.agents/skills/deploy-afd-k8s/resources/report-and-teardown.md @@ -12,6 +12,13 @@ kubectl get pods -l app=afd-recipe -o wide Every node matters if a follow-up pod needs to attach the same `ReadWriteOnce` PVC -- it can only attach from that node. +**Ask when to delete the running pods.** End the report by asking the +caller -- the user, or the task that invoked this skill (e.g. +`run-vllm-bench-k8s`) -- when the pods should be deleted: right after its +follow-up work, at a later point it names, or never (keep them warm). The +caller owns that decision and triggers step 6 when the time comes; this +skill never tears down on its own. + # 6. Teardown Left running by design, so weights stay warm -- report what's up and how to diff --git a/.agents/skills/deploy-afd-k8s/references/resolve-recipe.md b/.agents/skills/deploy-afd-k8s/resources/resolve-recipe.md similarity index 85% rename from .agents/skills/deploy-afd-k8s/references/resolve-recipe.md rename to .agents/skills/deploy-afd-k8s/resources/resolve-recipe.md index 84e4ac2a9..c6c1f20f6 100644 --- a/.agents/skills/deploy-afd-k8s/references/resolve-recipe.md +++ b/.agents/skills/deploy-afd-k8s/resources/resolve-recipe.md @@ -88,3 +88,20 @@ For each entry, derive (used by [deploy.md](deploy.md) and which case `worker` if that line's `ATTENTION_HEADLESS` override is `1`, else `head`. - **`afd-ffn-node-role`**: symmetric, `none` unless `POD` starts with `FFN_`. +- **`GPU_COUNT`**: the number of devices in the literal + `CUDA_VISIBLE_DEVICES=` of the `vllm serve` block this pod runs -- the one + inside the recipe's branch that tests `$POD` for this pod's role + (`ATTENTION` or `FFN`). Every pod sharing a role shares that literal, so + read it off the script; don't derive it from ranks or DP sizes. + +## GPU_COUNT for a plain single-pod recipe + +The union of every literal `CUDA_VISIBLE_DEVICES=` in the script: + +```bash +GPU_COUNT="$(grep -oE 'CUDA_VISIBLE_DEVICES=[0-9,]+' "$RECIPE_SCRIPT_PATH" \ + | cut -d= -f2 | tr ',' '\n' | sort -u | wc -l)" +``` + +This step is the only place that parses the recipe: +[deploy.md](deploy.md) uses the `GPU_COUNT` (and plan) recorded here as-is. diff --git a/.agents/skills/deploy-afd-k8s/references/services.md b/.agents/skills/deploy-afd-k8s/resources/services.md similarity index 100% rename from .agents/skills/deploy-afd-k8s/references/services.md rename to .agents/skills/deploy-afd-k8s/resources/services.md diff --git a/.agents/skills/deploy-afd-k8s/templates/pod.yaml b/.agents/skills/deploy-afd-k8s/templates/pod.yaml index 225524486..561f16f18 100644 --- a/.agents/skills/deploy-afd-k8s/templates/pod.yaml +++ b/.agents/skills/deploy-afd-k8s/templates/pod.yaml @@ -92,35 +92,33 @@ spec: esac } - echo "=== wait for every worker to report startup ===" + # One startup budget shared by every log marker and /health. Keep + # it below the 60-minute Ready wait in resources/deploy.md (step + # 4f), so a slow pod fails here with its own ERROR: reason first. + STARTUP_TIMEOUT_S=3300 + STARTUP_DEADLINE=$(( $(date +%s) + STARTUP_TIMEOUT_S )) + past_deadline() { [ "$(date +%s)" -ge "$STARTUP_DEADLINE" ]; } + + echo "=== wait for every worker to report startup (budget ${STARTUP_TIMEOUT_S}s) ===" for log in $LOGS; do marker="$(ready_marker "$log")" echo "--- waiting on $log (marker: $marker) ---" - ready=0 - for i in $(seq 1 240); do - if [ -f "/work/$log" ] && grep -q "$marker" "/work/$log"; then - echo "$log: ready" - ready=1 - break - fi + until [ -f "/work/$log" ] && grep -q "$marker" "/work/$log"; do kill -0 "$RECIPE_PID" 2>/dev/null || fail "recipe process exited before $log came up" + past_deadline && fail "timed out after ${STARTUP_TIMEOUT_S}s waiting for $log" sleep 10 done - [ "$ready" = "1" ] || fail "timed out waiting for $log" + echo "$log: ready" done if [ "$AFD_ATTN_NODE_ROLE" = "head" ]; then echo "--- waiting on attention server /health (127.0.0.1:${TEMPLATE_CLIENT_PORT}) ---" - endpoint_ready=0 - for i in $(seq 1 60); do - if curl -fsS "http://127.0.0.1:${TEMPLATE_CLIENT_PORT}/health" >/dev/null 2>&1; then - echo "attention server: ready" - endpoint_ready=1 - break - fi + until curl -fsS "http://127.0.0.1:${TEMPLATE_CLIENT_PORT}/health" >/dev/null 2>&1; do + kill -0 "$RECIPE_PID" 2>/dev/null || fail "recipe process exited before /health answered" + past_deadline && fail "attention server did not answer /health within ${STARTUP_TIMEOUT_S}s" sleep 5 done - [ "$endpoint_ready" = "1" ] || fail "attention server did not answer /health" + echo "attention server: ready" fi echo "=== pod ${TEMPLATE_POD} READY ===" From 0a7ecfeb5e2a105ffce8a1e603ec2af2fc23bea7 Mon Sep 17 00:00:00 2001 From: WeiQing Chen <40507679+david6666666@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:45:30 +0800 Subject: [PATCH 23/42] docs: update WeChat community QR code (#556) ## Summary - Replace the WeChat QR image currently referenced by the README. ## Note - The QR image says it is valid for seven days, until 2026-11-16; refresh it after expiration if needed. Signed-off-by: David Chen <530634352@qq.com> --- docs/assets/WeChat.png | Bin 225716 -> 223809 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/docs/assets/WeChat.png b/docs/assets/WeChat.png index 8fd55cdb4e386d39ac669727cb2db0ee38502916..9805c873d95d4bde7193f126f3b7708c64e1feed 100644 GIT binary patch delta 131046 zcmc$GWn5Nom-P*bC@P48f{3VqbVxUdNJ^Jl>Fy5cP79?|Kw7#%N$J^# z|1J*)tdn2A5Bb?fLDp+gK_#2!UcdHY zCQ_RBqD!>#l%yr`r%CKPSoh`nh<%7I&DSD-%NsJHOWB*LLZuoKms-*vvAL|du2V%< z%|)BV#mv^ycQ+u2d3TEM1yVLKPE(|L=9$}CwRvfs`%cRnLQ48NtI$JT%uz}d4(3eM zJ)>6~RzDTgK4?<=`ug_dsc$+BKT?SZ?*YRAn!( z`swNEN?BIRieW)#>3zZJx)pyoJ)z(jh;7s1&#$sl_Ru zUgEa;oVy}0RuEGD2>x(W&@JocV>2_e)}C~kQ_fx1ubwWE6%`e+Im|xllZu_1no38G zkJ#L|t@Fc?nv zU?_b#{@xcw1A}BnjpDSPmxn%x&*^9Z*J3&vNBWwy%4M^LWm$GMc6QGWm!34K^`(M_ zmZo6JboZTA!-`=U7XHm`awYq>JH7J4A6zeQhqU5L1dvRql{)j>NT+O@oSLHM;!-R7 zWfE4*y>7=Be{(o8KA!3PRYun7by>{sSB6(HB*uS#Pejy*%9oh78*eWUBI}d>ED@`x zE$eEOwYB5YGBR?m-ml-CB}-h@&Q#UaO;|kOICt*c`ZqFL6}~%3(>5wI1o(N9OCu~& zQc@w&(b;^R{lZ6o2o?rQ%pPlt654ov(Zb5F{WJBw_2_V4RYK5exV-(hVSn<2*DAvr zER=>}KWkhH@)aY#yt>Ffy_S#Y)bzjSNzO=066w#=P&OxIfIAQmyiP6Xwt05X? zC*KYHXm+!AVhes_jZjN}7(-W;gY%S%#%fMe_&!EvDUb_(4O1(4ck|=SX;r8wOz1tY?0$Nk7v(>x~v#iNxXN!kddLV?QYATDGUt>p}KwhlhNyKf0p{Q^Miw(Li>-LsuE~Z)niYcc;Gek}`PP%3SlvdHLtW)-D7Yx8{FOGGVV;4)Y!{v9GD^{Lag?ckT@6xWtQzi$`>*tJ%|~1xwFvE)G`|92{(P#`v#d zezEvzh`f=Mm3880&*OTnU}2HtxH8l?*FZ>4&QxO7xxIG)7b|#VDlP3tb@#5Dj)Wr7 z+jsB!!zEZ}!n$WJUQgRk5?fiey2VmuSEao^@tVSoQC>=_ z+p|g%{mgaFmg2~Ve?xX4TNpR`5#M|mLoE~WxIT*5#*-3Wm^vZ7kdSU!rN{Emd#TD_ zjw3@ul^32A(kW+D26}f>a&xPfiD-1&mKNCC+fy+iOtKq0E~(;y(?P;Vtjjiq_1gFl zF4Tg8qek!2Le9;%dU zXl7?;dwCAH?aN6@OVbC&hK5oW)NRKMZ&<9LKfflRaMn^g%J-McRPO2N5wM3(g+wFl zW-TGqG8}=kSF<(u>oC|a4X<9h)3T=WD2AhXic_b?dCT&1b5>Rs-rA6|g2JyqsR>b0 zPnw6TzJL8175`!;+R0fbXkl6i30R zkrvgI_y_6=3b)+1mxDuTR-i$!`KPf3GUnic< z$53QfR{s1wDbgCj}b%ck75se}&?Ci>%^+RWG3F5)h$}Ol5+1%VL zjIKO7+Fy%Q%~flTcNwGG8Alohwu^>W&;k-vb+b&{X5PL^U^bino-49Gjkn?N?;pCg zG$5m>s7S3f8Q;4=l-Wy%zP9v+Oqs&*^B=Dfm3UtY3l5$rvTAyN`>ye25|gzT%AX@o zF*8++y+Wk#9x~C=b~(CcjcMucFa7%TGN<)$u&QAuLQB-J*j7+_R`yS$Nu*9j^#P;| zq3;)PHUodX_4DiL%SRGD*57_F{`>b?B7t__>MP9x-MS?5vo+exL9U@f6l_;N-{UVg{?P{Fifcm{5@`$s)m7D7TMiwHsz z5_Bih>&{H(PJuzt+3w8Q=Ns(HNeR=CggT-)a|%@%F^$mE3LfJj|b&nr19gJHvtx zflQP24moh$?3OV3Guaq3>AwFV*Xp!_>yfrN?WETZKP8UgW>SJ^_jGe8H497Ul>HZp zo33KawZBD!WaiZJwX@yKT}4GjyO`yNYh4`{j?GL=q}0?Rw7qs`A{b3oeF4Wjzkl!i z+1t#KG6^2*Qhjq7vaz$uZBQy-&jj(M7GaxpBPkE7S7(VVDwXfnCfoX)DHD}`dFJAy zrmqNbV9R26>T5{leQ&>&6c-zJr#uYrNaQr@xX~ASS~=rUw_wXts9s*}Yuw0@N8-ST zH-o*W(JBcmb|zTj=AUOaAHA`Pp*Hs1QM`Zu`Qj41PX+^4qN`VPw$JIYBy(NauR|~g zCzohjw-u3mCF_+xe6 zb=M=fM0!$k4uj99cUNoH1em$wggmREbbMV(8-&7~(@69B%WX9JSGCJVzJ8o|+Qd{b zVz_YSdj0%ZUCr`PnamclQvg^hJ@Qj{-M+2{1j_|YEEBk^lNU)n*J0Rj&084C@9mVN zq+9y@K^7HCY7Pf~?Sf3!>i6@fXJ%ZXoP_lpufe0ztB6vYvd({BY}%Qy^5?qVU77sF z_4W1ItHp*!MkZZJ_W@E}sah^>rGf&1u2+Co$elaufTo)Bb!yU*l0I(o`nLVNS8o5O zdB#3EUbO3JmvKFsU1!v41zHvybB=_<16t?}08bnp?AkAVsBo)wMjx`{f>O|UM_k;E zf}%{h_{ClinK2u~OJrmWb#7bq@lo{vZ(Nq2vO)+CHXse(n@a%hM(7GBasJ zxqI4BQvtK>d9uRNoFCYmF#Hxkn$wC)aFg-M-ABK9p3(C!I5d+k=wB;|;j&&VY8#hP z_`stcCLCJkPNsU*&jBG(b@W!e=}F1qxwnw{&+-X1i2XEU-wfzmC6R%jK35D11EY3j z*HXHb|5c`tnzec@u_Jop@IYh>AcYMP(5xQnSYY1w0;gMUw{ZK6|CdEfd-C1 zk|;rUXWvl&t3*Vn4{hw9+i%V*0-y;Ut#Ofo=R497_U+rZ7Kn41PM;j|Uus+#!x_6f&CJNSxf^nx;b!5=P+99-$(Jz% z&)|9O^3vhX_>TEe{Q4CRb)R}dw_5Ty;|`sIca5hKea~OZx%vAp6;&k!doIArki^7- zp)+4z6GTFBe*5NG{D1}EXfFx@n!9)5B9znA3v{c6agWACKH>d;;HS2*BWq=q$N0GL zmSptd!xes7fGRG}dy!?C%Y$n;`ltOgyqf`$sMrRstX#0+Q6ke&-Oh7VQ&U?wT}?+v z$L6|jTtQxB+i@RbUbNI;Z{9?MY9sOSXP~egTpa?6#rewnL@OnXN*# zvUy~SY6ZGF3t~NES%G1M_m0~5OTaZP=jvQDI=AMRa|fcYr?;1ZDI_g36W3F0!Yt8q zwkXeYf15bJ1c13n`cxm}%$dJ`rtBMu)g>Q23ZSB)iCnEHf>K7?*CvO_TAZAmq~hZ0 zw?;Ie6;qhgJ6kU3zTE`)-WYw60KeS@=NT2fb0i2R%)4CPu-Nchej3a`J(@_+I)^U&4ebWZo2Ev1PEX<2gQn5*mN z*y$9frT)#pOG;43+y;d8#KVy&Sn16_i;Pqgd+o24h3u9nDJydxw};5o5#E3r7b7=) zfsE@>1e4ZPpMayFp&j+t_~i83W>9Nvj@9;cIfjaizE4B=L{>S>wZI*lKxCzNsy3p@ z|5Fp=M`LJEq3*Z!1WQ5-`D%QOmzi0v_xVqG;lpV=HAaM!Q?=T8C1+tRD{xw&#z$&R zW8#BWg)|&r)x{A&*7x+AZ{NNR|7O*b_I6;-ZEG=|q2Z~Tib~p2+eVIRE_PHPP0c7X zq?z@oN+AXY8HgcA9k)40K2?J94~Cy!;Pm8au&B;Z7-XvXzKd+p*4S&@IBN| zA%r-2o-BE)@tU4WUEVz4K=#{YU@Z0^u+pAD{P? zMw&_DG%32PLQ!6EsI7$yfoct} zc%knwiftWG$n*90#22&+3X`Ey^E-FgjGHcza$1Bfgyvbyd}#mn zj+|j|a4>xJNxpVfPigll2(So%uLTv#2(s0u&fuXJFP9lW%KmMQf7JKIHQi_Tx7Ws5 z*v&ffehcn*B1cnBKGTp6&E}u`j@qbuB4S5-ahOcuVwSY8l7c^V}dJ$KX~V?w_tEtd9qylSR3vK7ddbU@YU{Z#?Dy)cSP zE1dqKNpD|(*ma|wTQH)`YOtp;uC}|U;fpyGQC>9QzDk>AF{I2fKcRjw-AXaPz0jNa zncNJT{Ls6y>&WB$xe#JX0=xvzZHG; z?>^@?{U;b)RYoZWAe(wleVU(tqDb`6iYxX8ue}%mLvy!PaR%Nbx8ZW2B zC@5$S91C!-$Md^vYd-re5p+{f1@YI|-Q7hA1yWx7G;N1=&dbEa)DIs%w4ZJw94s_= z;LB~^W%1ePTu6Mp>^ST5-i6J*nuw5)%a>CFOPiXTgW+y+g)&X@Dd2{d1_}y(m#9Y< z`-VVPk~TC3@BY1PaTanIA7SF`j7fA0^j5^1T zd+JzA;{^Ljbk9ODure}m$` zU9AT~T+K?~z_DqiRWpn~TicFbblZEyf;8mVS821EwvqI;vKtv0F|pG#psRgH8W6g# z!y$lPX*go-W{*s`ZiXh?9t|6tl4*MkGEbY*I(Euz1yc9`KTe6 zMlfomEQCz>{KL^nOwd5;p6yIzId$g3ds22&6*qHFAO)ZZ%Af>7g8ORl_R8>5!mxta z$Xx}NU}%Fl-&K$iM{&|*C4|Odl8DhK$1qp~blqhN3O%=lL_)+RkDksrn^@_tT-=kE zXbVV~TU8;gBp+@G3DrB|pSNvEuv5DNKoWYxjJ9~_I;VF&)WHp!bj0!Yq&z%-iv7F* zWjVi~pkR_qcr5T_j!XURbA}54kS%+Typ}*TnJo0Adj;~LNx4wp?^v1bPODb8K0w$A zr1GV~L*G2$#R?n_0Rc3Br$F*tjgyDw6VW?-7LsX2SZTZ|pCn-!NXmX!zvz^~{`Sf` zpsW_CK8}HJV%bdmMv&t};&Kx>5!wU>x3xWT{nL1j;ZS!NxW3_YucoJsQMwFr!LxYv zLdG(g3;a%tX?sHnaQ5>*@vu1|N^lVBSDRWQ2xG9v0NV9KlTilCM_u z+fzZp*Xxya6mPdqJdJYJn)55~9gZInRc#iKCMGVeeH$1UUNP>aHQNzyDSb~yEDcs? zPtyHUq=Igxt0`~4OuPsB?y-H?mp*kpCJhY@Hpob$)@x4@9%EqduvnPU#%b8LfeF@Y z-{BZwYLtyup<(Z*hkC_jGcw;h&SIWwJzTzfu#@Z8f`|4hrC)FG61Q)uuIs38vrjql znV{UsCtQjr^%^2jJK!L00b_xFe)QzBQPI&F3r#a2-GEVc#@q`W7k@PW-i;AMzFv+T zg~ir9mx$yqdC~^Q#N;ew5K#ki+*}*i3keNH4_f@}k74^bdfn;!Z0fqM>A+x`!}3|& zpIy29pe%M&#rZSKr3TxX9l?gyUbP!51fBxU`Ku&* zJDYF({gva50k9*FCzC$ZPJDSS6UnSYAAc<^?_X@Vs3Cr2{|BvB2?|dt|7$D}#0+XG zZci2M%d3}t=GAI{utEEaLL}4E@3&OeNGY%Z?3^Xq3$47fh z1%kR$OU;)VR(7z@3i)YZM0J6V5ZIj#)t%Fg=5xwfco+r?LlPKTlfE2P0FtzTy+Q~n z#iL?Eg{5b4q-15oAxfI}ww6!;oQ`u7df=_azEGg&qyWu~IOn);&QIK=4N3g=#pg~0W+dQobIe-u+2f%#^@L#DwS8(BHQR-rx=_y?NFj!EEM2&@U zFFx5+)R3!~n40=E={fX-KRR$n0XM{+yhByv*RM?t$A`8-Gr%nk+@J*1ZrT=gn?P{$ zeV$H@n%Ccn*FDY1e(UVd!~Zsh*9zL4=Y)(=Awl4&W(2G1w;n_PrFd;K(-vaC*Q| zG+hk1D3$XT7Lg=l&=}hs=g)WKiYNTu=yzeD=Rqs|rY8qx;;rFT@Aq@opVi7_1lu?~ z{ywef7Mw^j&Sw`ibM&MO`TG4kuJNY#s7>VRXiem-Fi<73OByCUUbZbaRo*^9##dW@ z6zJ(yJ1!VX=0WU^CPqRb`m*XZBQvg0la^k*mI(jX*{~w53Vgn&oox=laey#Tjlnpa zM98Buj>k?Eat_0tNjoWhD7YE1*s$!S2l90ey;xeVmI*=R^wica^yNk-uOSf821PAB z4yQ;soojSJ=1@hG*&xFz%wVbi z``IWuLD77o&8DLD1(4dscx!h|M^-i0tN+qyoZU1D|Z+f`LnUUTO#;_!09_?#9Mny?k9pGM!tV9v^b%9U3OiPNSP zIDvFX(d)mz5Lo&FO*rCbU=j#&&DssDkF)`NxC-Ga?EF|5jJF60ROQ8W^WKbR1iCLO zAXk^Fp#g*56=?cr+OO`c4`eZ>()qt3oLYv`JtH5VY)PWS)VOv7Q6ag}GGjwmE+r>LIM6r zN3OkUGad&`T$M1A6dU7RXad|GED>PkEF^28os)*X}BFqkB1l;3n*8`D8GCXfim03b;ZkP192++66D0G;;b zdw6!3(n31KFC{AeH2e(zl5o&VK>vMYgU03G@)Pp!J>CB}8}e_u&;N1af1MZk_dV+W z^%BtM|9^Q2$iFAd{?~g)Zyou!Mf(4K3Gn>?FCRY^a$8c72wIZ>Na3V_Cw}DXXjyTQ z5%>$-xpOB?ufb~)8fr)yDYSlrI(Qh&>EIMv`@KzbNQWU_hkH_hEM>=qCz1~pYn-iJ z0n(*sWxes_y(KnCBor+o$?J7cxja&(d{;Jx3d$8XKl$Hmlt8f^6R29PUYedn{8_m8 z$ss}NL*r0$03bU2p3yf+@X-t|e3Xypa|!|P1**|9z?sq2^D zn;gy=GFE+Wft?WtwB;&MhM2;#|NGkjg)>lCek`l#7Y0*Gv?$99Qb+2#We;X!E(I@| z(tvdU0Tv`2PNCFT$>xi@4-?!KuwBlOmN`48(g);(~aMN$!sO%m2 zzZtRnPM&_OHwLY`$=VoFr#rwxmMIl_x2Mw1sE-G}91IoM)jG2H4<)<-%YpnOPLcPX z+k^mSjDG)qVrfB`{|H4uApTaQ+&;@{v&%0{@Yhg5(pIk5D(MGv*K7O52=%s$xAYVh)`+UnP;`=;68 zRrh^HYfPvCh~!YPjGTFr8{qKkBUUt<37&5r>(+WJi;S3<5BLM5m}{2VN_w%9FswRY zDMEqaL$PN*yGB4Awt)Mlebqigqx4CzVaXszS$Mmtas;!Eg>y@ciJ2LLx}Im*thJ0R zBCG0kxHaHCy9quEdbsnx5n*@+*`Va{CGj$86pN`-JXlyY`WVeTH00~T2wq>mT1jeE=E4SHyaP2RsCAi8l6sG#)1 z!medzToqTM5=t=h+$wA!p(~hKw+S^Q=e*eg)Ok=SS%LEo+Ug}>COZ} zCtFk!m$+{^R_j_lg&tR<@@cYG98h=ZaG}*kV88J6>s`2Dy0NegEDdd{L{NX7H9WPt zVz(`eipRh;C(L474t@`n7s@8Ob}jY?HZi~B{QGr4@rz@3Z}xlqwBj#85L?hMVdz(`%s@YE>hdWGoK8Yzs#D(r+vVz%>JtJh2CZ7a zvVk3<3^(Vwn)tjYOOY~h9bE|{leOzD3_=w<0KP^IcduuHa*t(6le)P&Iy|?s4-H+r zT!ScsT3(lcXx4!dNZB?;%=Lc_6OqZU-@a8hMuR_~bEOUzl0H|W&Hm2XPE{R++x+c{ z$MD>1I#-DN8e-u&DTAK1(#!Rya2tF)+{WkgL{t@IF;T!Au(O%>P?hw0pEK=OZjf!El%maUvQ4c0GK{)uWa&{sCjc&yzZ5Kb=-mY8M| zu8aP8edTy9Ox%-mr;`2O|-$M0_g0>Tmm-4}IPenRqLym+0Xt4=pj z$TQ6pX^+8bYT^Nrt|woIe{_kOib@hFs?mt>H9bu+F)>MCzegj=fTYnZeUi+NdWBAE z)9TiXW@rQicScg(>wrB)E6s2*xCmOnpP0kn?~QfJvEUTutPvij_s!WL`XgF;dbS#} zGkvI8veC4}+=48&J>yX#A~d=j#_0|uS{M-H7G7&>HETjB8HLpj%8#qxsLjC07>LaT z&k`Z_a`HQyz^Mw!7{vM(S?x?}fAiTdtdopb%GuoBuFB^)pX54>ppOewjI3`lJa#GhUb{b(<+Hzm5iqa^JT2{Z zAOeEq6(PbEY5=yDYv~~C$5b%gy(^B&393eQ2kWiUqt#BF->>sIX01b7I_>Jupbqk6 z?)dMsowyi>pG3Yz{oWeboZ4QXO({X0JE`Sl$6Ff>W}OL0IQUa=rt9bd&kU7W;!-)& z!W)VMom@mLt;hKqAxhvzFD!14xvw>YiIih27kHd~ju@GO0bO@lAl1^d>(&~Mz0fR# zozM$-ffG4GsrI#^X^6BKDj3NSi^Qq6x zqcEtbb^xx(m?3^@>W7AghHY{hCAGBzgSD!@PF_QP?=X`)bZf*%@;4JG!tq7ibn}4n^!Qb9rp^mm`Lu z3oe}6Y4OK7L~H;&xF+v2*Ior%4!ONaNm*P9d?!MpT>pSYxJoL5jh z$oBNtM|#y;=Sn9h4K<35-!f{IKLRqs?8$PD8uX{Pv`?o*^??@M95>oM;F>~@c5 z6-onj{k$P}YPL(*4#)fbk9tpc02F?uh~)tD^c-JNj748gGpwpxsMizf1k_nBermvl zrVs(3rOLw1%HD3(FS8toczx4N@;bK-Bdinl+9aG?fB#0D{wxLyh0Yj90iNlIW3Mcm z;IwuwsjZZp?M6^F@k5c?OHlE!qugQj@Q52+>k%o*#$8Q{WI(roe>#0GwwW z?Nx+Vu27D9Y{-7BU}9j%<{@|!+gI7EQ>;sHyC4X1jy`;r{qBz^>}3IJ>?L$XzW^#aDXC^^$eK zXkqf}BZ7Kcy1m0c2RVr0VDMhJ z6X83rjWK~6p?VL6H#6Xy+Ip6+oGC|qwYsHdcQzLNEefOCm*&xD0uoH$!5(_`4zm|< zcZC+gzGFV2=g+@d`Jnz_T?UnK5$rh9mTI`snHT4oS@{F$lVL5PY?+##)_h93ry2b080w5SsMV)y%o+64wC&c^XqC2;E zWusW{k3E~ZIC@wN_#e%GJtp6SC?cVYnqQ3schP&OQZ-CVy_rFfI_LesInM~$=)((hntqdfIK(p}GUK3Dd5^gRL3 zV!Gb4SI^>AY!g(hV0kNi-(g07?alg;fbEiYTsLQ=@Ct?Bc;qGx<}7o)3)WI<6pjy; z*Ly4#>La4GzxAgvK9g@RV)l>|QY%9AJ|5;5ftLgZ{T#+y4i2DRS%y?v6JUJ!{>#^& z|C$WQuYw8>pZ;&Z#%p{bsroq%C`IHqKmKEA5-}LZlv{?(Llpnzzy`E&ojgNUTwlsN z2Y8-ajX(85)&ef)SR1P#Y&8gONN}Q_%De3|KmJr<@`At!edh0daaXMhoA6ea2JHbp zu2|WQc>ZhX%LlZFR)%~p+mgZQx*hy$W*ivIKVE{dQ1F9PGNI6SwNoCfz9z6Js{6Ya z!6^{|()Yqxxo(|X^Jm}lr$UqwT2!SFfDox3Hb;3skk;SFp=JPA=n$OdN}>6z`tyxU zCK6VFJj7;tjh^#-*Zl|lAuySsZGEGm+)Bsm*l%?8`n?|#dSE31ZeIs+X-@bgo+YI~YUZ2MjK>pG}5xEz@Z zo8b0PMr+f;!a{g`ziCM`7-y!zOnUB#TtUN8^}+63B1)kyY_ZgBhBz+tG#c%44ro&X zKa85TEEgleI~078j4MkBeD{>(e2(wI?Vxu3I~)lzW;*oQ;WTTnVYN!5^zG&X(i>yH z2T4E8&B`fE zkQ55VifUYE4OA3PgDwm;06Z{!`v%yQ!BX>36h^=_%|(AQEAS5iIHCk;E*uDB(yK0m z#U`lj0(oFaOiU~V&Gre)`(o}#wlbOo9~hb#8%s8L9bR}#ea9ZEGWzDy($meE3ty)| zAKL`-@N|ThO>k3_nvm zAJ;;jXj^s!LJE5o0efsDiU3eEy3Dm=F)_GaYB2HZy3P=z(O_Ul8jXXlr&||DwqANb zKZ4#^02z13`)A3h+q2ktMRQggC>MhjXe>g)!>QR#+gSODAAoQmSHHK&z>6H!-u^=( z3rD$f31jdvi^3SZBIr~Qm0A{}QTwb2?YdzZJaMMKzttndV!lcN4QDtY!)&1nC>*;$ z_HZ_$s~{a7Urr~yQMm!J8F+!~dm{kaOhE05nH+H*wzmIrkvmIh2l6gDs>9KU#xDx( zdo2$qK-YsNsDE<^3IfPs>fY>^!D)l6p)FI_<`4?q0r}nd6%e4TqC$lRTgHI6m>Bw; zvEvk<3{^Q~0_$(?W)0^}gPefX(Zti(*jTLp3A=qf5=a&xf#i;$+*elARK#rH#CO4M1;v8Bz$laBJS0+12A1*?C}~Uz_)aJ1zh?D zkL3VjzM4*(9W+Z+1sB7t_D$r#u8#*|S!t-;I+wSjoqIgb<7js-2PsN;HhHd{7s#ra zh0H{?z!Hl-dTP+t9puo?cX1yY5PM05Q~l2qeK`3q(k}9RIx6!5@iqVBTW(5fSL` z!$I7SiZ3?>pwqK8D(JMR(#K=jpQnhnJiPC|Oelaau*TbF7~bm#ZP%Ib7zZfj6~(sW zLP0zY2kZW58`f?I5tLO692VASImle;P&s29P@CQQpYIf8+dKY#evVtQem6nYdNmE& zm;Ph9N-y02DqX#hmb&}!CUM&_XDs#>f*#|ZboJP-a6~x;NTYa zP5~q5#rq}5vDXZk9e<3eGppYL$$(Ym`V5m^y*Anbxz_=b>022GaB))L-j%?r9fSol z%=RJDE_@5}OA7@5RJj#Y!*w8ImfnQ}Q3_60dqit{Hr96W-3F|`6Gu-$W+PCV6TN}g zz}1|(ds(Co?fXy;WOb|DW|SUQ-bE4dZ*IT|Iu9LiDMCRZ)0xf^U%f;ymXYkI4OSxNa!`t~&a)e_7nHgAqdwjh}%xh4E==+8b|j zwro4v&RknjDZ6JD!IiwndvAZAO=TEImF)2H$vG~)LW>*3Y4w2nY_!?P(I`5R(*GfL zl?P2DRA66ciiP2fRUYV~LZIcYVp;+_NvS!~EAEnq3JKcU+6xWVT(Kyx>=Fk#+I2qp zwIf6Wb)@#4Ac6Hat6QL3qDcd*Niq!`Dyea?9bH0snxP6?3h#gbP5|ViOFsZ6LGwoJ zZ6&G<)<{7-1{>;^qCd2W|UsKTMpMKud?!4$0k134o4J`-&s5J@kN> zu-cnx!T;FUa-O9Ys2J#Ll500Q_)Rm(9bus;z&KsbZV5VCG_%nu%*}7zj*`=av{$>TC}a0FaH3>1XSebL^1RH#fVj&V zY`ZOx1`1IA2ZkFxzy$p2@GvFiPCHUlECG+{HZL&|5lMtHtFv=FhkR`fih7` zkGDp^^+^E`kWlyA71WaeQ)Uc*1L#fL6NSg$ zdzujfX{>?vXz)+LM8O;TB;&zYl zvcB;G0rv=6VC5Uo9Du|Y4rLU_aTt0T2A~4EYpntH*aHv^;_*vCl|q$VHCg4nhQk$G z``I?qzBq8EQbJD=ynD9YMqdJq78HLvhQYyU~kz zC<*J}9JH{>Y!0PyUx|c4S%)|XpjPlWt4^VQyzuI5MhXrt^0I2^$84-f81uX#X+< zHoG2FJGkx6M25p9)ik!i>z1sp`G#yZ$Oo0yD%NjE;352@H zGl750Z-Ejw8_^57Dja1P-j#t2=Ur_cF?tYj+Fxq zRyo7*?6-cR!ydd$nW^QmI%Ie*4pRMUgtC%SNGGpvHMmxm)f|_Gy2ehJ5^UsOR0*5C@zu@q z72EhInoo@+98_lcEkT>`6O%-Mk}d~g zY(*kZhe|B_?eXsA1pMF1aMoF7X8#~U01cOZWKZ?(8><3- zxjn$*8Awh_y0E==p1Ja4rk4gZ`-%0!In}^}z^Zyq$(sS-gaAwz2xcp5L5XTm7Z7kkhZ*r;eYAszX(oA$<`MtzkjM%~)Ss9wPFJsZ($ zs0Zb;;Ivf@D7xT}O$IL2q62lCX@HFVPx(amD1gtxE%kiR;_iz>EAtPycZWQfA*LNi zj_V}i(8)Y>AAngDVJV3RkR-<_Lth*Ym)%|j^QYmENOyN;Pow)(q@j+}jdC52KaGeq zR9lHSptu=8XzsZILOagATzF;Rck=oW!f+ zxz$%~ZvnN_A`T^fvb8GmXL(SA1#sh}UDoH#&_S;g@u!ZGk}@39=s$fJ$MH(bO^b2) zdRkh9mX$G6!Q+OI$Iti9F;H*}c(GVi+V;%5O3oQ0zx(>)|F0PPJRTH(~HH^5<#jis% zVClC7y_X~i>Kt|bRw%WKR)u1ZU3=LA3gD|Zmj*hGT;T;!2@C6icOrvM4yD=QB62zf z_m@l1s)h168INIp8XeZKMBJ=-c3~rnba*}J&B+!VqXSR(a(bQs(H{ncpSjl|AUu^w z;Xhxm*sqT8YLr=oXYhdd9sxp>yUh#b+uOy<-lzop;Skx!< z;ME1izzwvWu0A!2inCT@wY=}!)6l(Ns$lSla|USQFgQ6q>{tPWs=UA-A0(fj(Rry~ z7N|aB5eE=2-p|&2L&+eZT+Tlc0!XZWq9j=p5fUb&vv8dK3fS8V^V4VI{=@l1t=&{mn*a z!N#@_3CJyBfTnPbhef-gSq%r7NyCL7=sr3yIe2CB}c6%Zh$@bN*(>SO3pL!tQS?6_$jK<9}LHM`e=U5FP5Jg`eNcXmd? ztuE@Wflzl*gbUe)l!NM<)dR!e&cYP$`uDdYX03dld!-PmgVxcSi35|s@rA=@Jwg2d>8H+1W+ig2F-tm^Q(!Fqzq}Rf8N;HGI?oO%XG2NeH|J=m#R8o1WJtnD7OX z^pdyz~{+u3ujZ1FBIZlUJLT9cnu8yqIx(OmY>>t z6F)93sQPSeeD#Gt5KM&sVq;MI47nWs&r=K|!`weASVqK6ecjw&lMRvmH)_X+onaHP z70*AdMtfB(fC%ou=8*r#F#iaIL%>(i(xg$U%Ue;n38L=u>n{e9c?!KC+@p0K3`w|Z zKPN6!1T!SRI`!D0yMd5?UOqe*{#0r^KDNw(jm}U4n!E6}aSiW8F%0tqHXh8KQNUv_Nlk5+LsF$ToW2(wxGj5YMiv0RCD31xAL^3x<`k2VXob<`b~(yh=m0TbvsYhJ zSn*EqYC398g}6pJR}iOuz~nb<$iu7zoO6kCQ5?(-sG|Egx!1u`X$8@Iw%iKx1i0F= zl_j8bj6|TOp4G}J{86L`JvgXxeZUJs7K?;o%h4sMhce7{n>`p~D%eZt*l4Z(OKR3w z#dG0M*|l?2r_XZwC}7`rTQj^AOfN}zHi_t=AI>M1q@L5k@BC7{InpWXq(XH?2{^;v zW3WWA7fd?$ac%z=HQ(hmiyn*~9Z6go^7kK&rrF_jcSfAImzUd~(W~Z!{dQ{r3oN#| zZWL@cAXV@M$bW|Sv%cXB>UagpX_AC{ppji%T^cN*LG3`5{jfhqmdEZo%kkRs@*`ON zg!#%<_9oZCV))xlpe2Gy2D)@Y-2&H*SxCw(IF||gK4?XPPl>ix7xtyICx33$5>wN-!91={{t2f3~?NEjK zp+MR#gSKLroSa;2qB7Nwl$tApaglBN_rwE@eKT~{&;&N%iQtUA42bRV+E`uH@DBjJ z&T}2)sb!D~zEMd{=1@vq=707(shbZFt`P*;WIj6#UJZV~F9!w7Of^SkVl@*MICSOP zkGda$bZ!RIVk3v8uuV!taB%XI4B#2g20n)kG!VL&*Z02l=j*h_H;;`akOK~V5)awD zAB@DS_gF#B+lGm2Z6_Fn1Sif$C3`rX@~e1W2k~wpxAp0BWc<#IE*UUuR|4O8$o^15 zqNK2pb@gcVNA{H(r=^H)O<=~ZUXKJvp?ayc<|b4KKoA_OM=%M`#t&ow(yd7&pI}^H zUq5nK2h%zc@Gn=+%8HTfp5bYi3@93?1)#Z~2CC}_Qj;|7N6cW{+DKwVKp|udCBq0p zH=HT;dPP^4!Zz>{ML_ z+M7p_(ReuG-F-q02KD@~l_xJjz=5HWko1TUXd1w2E1}(bl}^d$;NT!Y=o?9DYb1-q z@hFVm&hGvMqy)Q&JkOPb-HV!shhz8GVqjAkt+2SbwlXVl_ZaV2iFxJg)=9%8lXmNw zGiPFE&VGYsYR96LE=}CnaKE#y4M_vV#GZ*kN~vDr?`WNSWP&i#q1rk=o+uz75Fiv0 z7xxH)buwq_*Dn!bdX-lhPXE%!blL2k+qiPRl!^p@xXJ<|`LPA;^kC&gn zRSDgG><7%eX;_%Z$jfH}`OpJf$QggIl!FMGo|<~#?Cf0J5B*QR)qclco{LHNT4kqT zNC@^fP=pkdZQLAUA%S)A_tqA*p5FLCKBZKM6$SR;;i1W)!W~>?FTT`L@DP&B4N^az&*ySlpi;9j<2 zqhIllKkMsvSXfvzEHubq0fu_n=#uDT<9%G8Ks^xXKZPg1;Kv982~(`cShcrbrt1fE z!+6rgdctt3=J{i4sgRjB{x1+?Q&TXCYCm^4_uVTk2kV#W{?3l} z_A38*1S6oEP`R-;BtyjT@$siuR};)Ws3TX^6a;zSo#n`ctajNWq;2Z8*jrBKCl znj$Lbplr08N6|o6zj}2fm!xGfm;?o1`BYE|zL=*o8!EZ2TFnP{@4f{HSw2_s)T!n9 z_P-+TA1)z!g-*3Jr-^K%x*fN~fQOkw;$3Q6T3V_O;NVCSsQeYJX*oGYCm%4MO$-yZ z{KOj^2bpmm2L*pEW%|B(5cs}Y?58k!k4oMukA$s@)Flh1X{zk4L0lXe)vJ6pqOoi6 z$30xMcrjyTWa~iQL&pjSB3Wgqyw+2;#`mDC&l~!6I}%StW2Falo&g4WDE3^oxrXl# zdj|*Z_rjw0nDhL`uH)AE+hW*PNG2I8j81Orer}+SmX-ZZz|AtUmYPKuNuPC}f%~#@ z2)2E$mNU`WN!UK=Z87ngT2uYDFSh(;l{eJb#~iR!_m9;gdm4`$hS_Jg1MF*8tF)@b zXZtwxCZ@cd^Yhgj%6Kvvc(d>~Tl0$v`(c47zs==Q!6~AAS5#q?e&oTf?IU!+TQenvobh1Y92KQ9)e(JEH_XA>(^&8pcoB_};`ReYs0 zE8V`nCd;j&x|+REC*``=(IY9XgwSf|U&+WfZ1rVvVhC{HxEtk00M~5fx|{ zn0WH^Y1H-AG#3@tNiU-542!B1)+K864f6M4h?yd{WUfg z3CLz!CuQ{Rw-5Whc-1(zf*t!+4vLD2^|cE><<;|CtU4Q6I$_tsmjW0;H@ z(1{&*9`y(b7Xas%Bx&U-4{_0UE~BT#?_$%igv5#iaFx($1%-3@Cg%|$;%U*Id9M(p zUb>VKItbH4b%~nU=G;wEaHFUSp3%~xn9b_L>Fjo(WC;TqKQ5qM)qnJ3*Z$Q=3q**rVop(+Yax3Gu{XLIXuA-yk!sB+z<18NH%Tj!3 zIypEwIggu~oz>LbhPQLOdzVR%T~;}Ce&YrrboeEYfCNDxw2Q_Ll*MAB6om2UtpIw# z+wgqI$}F8iKI(cRdio`g5*!Y`$>y*n>2+@k-gk${@6HOnWv5s&^8G{;|te5){G zQ>VJSyR~KsD`ARUG<38n7h?c-`6Yz?ot*_QY0_ujX9h2IB;e#^*FY)CHT2{bi?-Ao zs9ELbVsJN`V$QHe+5(M=g*d*m=RI>rH0JhtOtp$KOD&gmc6NTlf7fP!3-?oa*BpGzLu1m&}>PA@!ua^e$=!7`Jc_N&ky0h!-F9Xd90JtFSe{2*kVOZjY5M2 zxVt7;tJuDX8?`_IO)=ISlevcGew(DTNNb+;{mp!GWDNQefO-E2P|{ryH)rDHp6=c6 zf9^f@iSK~P`hUbAk-Gt$RJ?ukanH>S_<3I06!6(IX!?u31c1eo0dDzaTi5OrUg7v# zo=}sv)JxZPL@XLECK+(zM- zTeyiIKYoYgNmW9{-dS$SF?RC)jP7k--H~Nhj?tx78GORqbbT@?E6rzKp+0IhNFd4g(ym>*Ivr3aGyrQd z){xTmoM+j~{QL||%v^yc5O)&KT4#W>vvV7sUWS@XWU}=2_m_vnp{AKbJbvF0(lMX< zb3J=iV@n9cXdI1w{2(6cVfk0DUJU{+zHj)WEp2y3T3UB4NK?K@H39003JR-7wgN_8 z6~Q{RQ0~d5rbb5PL@Hz)jiDeFkBp6xwXLlIK~5q9=4PiZLc}dPERS1nfRL^yLqB=0 z3LV4er2nrgF)jY{C3;RFt}+qGD1V3|XD}xh*Qv)6Ui|ppgt{eA2Y@KZJ(h4Rq;$R= zQ<|(n`2~H=?eiT>Wp!8AMG@bG1RkKG!E6DTdZnJrMh*tW#Bc)X?qa9}PP?~nXoxSk zvb_9`1q=|6<+0@(4G#~O2FN|(dbLldAof%=$bl6K`UYLd^Lvf(1%}F|G`*9CMn;^$ z+&gzlFI)_60o5(x!=46!eXm~}s(5Jt`3k__GgjK51w0?G%H1uGDmDl~s4_btc;!C{ zgXX^IxB6LisrPc$7`mpL1kYA~jzLcu$p9mchpu!2QC|hXqI3X?!3L<#8BdLsqO`fU z%AP(QiA0m7@7F`!Nv0Uez4f%b{0ksi+>gG!Wjw?xVZXCL$Jtrpu*EAauAKA!TJ@R+#(zb$L+sQmzlHaQD4v8cS4o^cHkRP8rYC7!nW!G& z5nZk`3Pa1oBhY6*E-Wl;=NCNV$?Bf@`$pOUhmg6}FZPQbtt!iL_|VbOh=8`H2XuQE zw)YLRm3(M`Vo$|1p2gPDjGr%Gv0~SJ$3WHN;9G#1>V zX*rD%W@l=U|^8Jw|ZWD<+HQcpJE|3(l;|pxX$4De{KXRnq`-q zgTfSJA|E}X*8f2GO#9gqFUP521zKN;oUhZ7cE|LZZ)v zDMraXRTjVhHg$5{(Vc7is}~(m^z6w$;M}$!yC_i+ifpytwPEw-6Hz54W(e-LQIm7Bmn)%N9V;D99;21Zw`9$b7zI`$|MXD-9s< z5oFVhl@O%wK~3fi{*HflzxR??8tUWnelq=_fUEE}Vr%0G28w1qh^mUI>@;;+4=B#2 z+9v4FHgDY;cPbi7I}O{I&caltff0%=Ae{gsphHPxc1(ozN_#$Ves|!X3Tabz+RRaF_tHt>$(mEE{% z-8wZ16&aZ&zU=qJec$`Kp}#iA^>zlJ!>z^LXQ+9C`vx`no2Rlq?B{c?Xm8%WJ;68{ zZZ=^gMh(0EfVJnc?}-l>t~AuunWof%Lb5x}{l#{%L&~)pr2N~DbBXL+tkPv7Uy zC0I_cuXGjF?2vShZCfHmdqyc1SO*%R{)lB2F$p#n8|BUtLQAgE)K`t!g|@H zp~1m6^fultE7z?vf~T>2_$T=7y{@Bk!1FTz5ZV-<#|B>p=_J4yZ*K!=FuoAm7)yqh zDz%8W5l$3!c-itU@&M5`z$pr*Jh~aquH9b}Yq0tiXJ@ARb#zi%i8w6|BO`jeDE}!a_wGZj zq==%{lWdCvCNS*2fI^MOx~USvrm+87>W?F&mvq>&-qUNwIv;Gjp*+_ahRL4+ZM;VU$J|I#lS`g3qX zlHC*Ne~Y$-38kX&%yADVKu9A?0gQG1GoMm4eIMTuht^3ScH}!TG~+B5`;Rw~CFu>K zR`31GGhvc@&YU^J#zsu)mA;)s!Yw}Is~fN3px~sFZp@rJ!ur|P#HDoENy8iG zb^otdmljTPsrTbq_n>-FQi(l6qRh#qrXB_H!G<#5Jt1~|FcvKyIhdf^c5rYo7E>Yp z6duESixw?%HzT|c=Bm8~p;4yqU#!T>&tH`S;zWGudt)^UbCP@ORcq@Aly|1iEzHdF zVA3`xH-n)a9Uo`Vd3X{ENBM1I1X}>eUHG^7&Yfi-cuwUiBGi*xL`2>GaO#!l2Y-k% z<4P$Ljn))7gB`r+tT=9T1&^TM6LjC-upeqwo%WG|Y$pm`4!Ug=aRx?4>Nu$o1H?~G z=4pT|s&PkMU0n%63e{dud?PkS=z?~5xVtYm$kxZU4g1|{G1tE$NFB6JYQ!w5S>&Ae zO6GQ!6A~!CVZD&hgygTbVl1Gl?xSP6VnqS|Aq)keTyjv5Ru(`IS*h9aO-rezOIP&w zHQ>{}|9%ajnA~!*vfNe!K1HHV6I2GpX-jSayKECcM!e5fKstfDb|LF*(Y@8uly_+TZ=DHjVmGRnXtt+XbCpw2D8hKLs+>v zKC(3(KX{M8n!K{8q$EBVM|ahg`|rP~hOwWAoP>;m>iH1^LW1~dloYwN0B;<@$gy-Er|d?TzrV^=6V>?zUZ7f^zEMd;Jqv$Q znM91p#uJC3{RPk4+ueO>JPl}T1JIAMb1}HF;}D%~b>T(XL>fx@xjT194|BG_ zX>uIoV^6#Q=t&~e^p?C15ExywfLaKckp^r9W6lLYTRJ&6&f_k{!YgoswWFAlA}0)J z*^;V*WZiDf%`5TjQ(_HJd*u1yPH`8)VTRv)Y0HDC+1m+<=r@gF$#js^!CnD+qbDA@ z(K@5Qe(_JY!M!;)E1rVP(PegXt^8LyMF7=oYE8fB{A;NToSQ2-+0Yk9VlK7e6NfBT zOKqSj9o=6LI%=?E_yh#oage0;jfOhFIxbNs0GG1-@X#m(;dwbznpSAaCA!YofS;ZnoPJbvMA^=DPCn#dm`3_7D9Rp490iBy+qGr~8V3+)CUtS_V zJ`Ht0o@G0e9tI;l%-u72JjySY!waP>sA6Sxz?UK|-@}Iw8^jQF!((&xYIicosA^y) zMwxyoDf{hM*e>6=apT0mU&z??&gdXH2fkesYXKY>89mQxb~wk@Xbjmp|>ZUgI(zP6ej@PNL%|Um1KMhlc^HBuWzE5#(r3wVm|iS|Ph@Ko`8?7{?pAa(*Ou4fYa09uTs*8=cVs1>~YX1R^sE?UKEOs4hg3w8Dvtaak& zPLkgCIjVI)!Evp;Ly<#(7>BjtL0jjo7wV{{NkapOxGLMa{0D@l!~%N1ahfkHVG_+z zh<;ENJP`1b<&jOt3kK1g#nug9W_-20y!U^a$ef(h=kBn~%U+x^rumqiyQ{q*fH$g+ zZl@)7#`?Rh4k|7a-Yg~U##f!U2UQ<@y18T7$X>JUUBB0_!mIqwzHUFeXkq_0rrrz; zuJ86{NetNIWm(|*D)5r?Vx~L4*3iOtpspD&xqg@CmEmq#^lN&lnV|)^)QY7&+iB8= z$@?rHTu;Xo^Y&guaEcoFry3oPHS7y-YJpoxV$mu7*9UC7VOd1y1zM&|G$Z^#TJ|sR z70VxaZZ6NsdOfg(mQGzF&rjk|kR>e-S6k(v@3(NVUY4|;14VSSrdz1H-X&`{xLwu! z+jpTBmhH^7J=;q}_==?HXbhYG*LAFWkFw0b(IQ-wlyo?J2Yx|Ayug)i=XNq4zG*sS zT#%Mf)jo4Ed9qTd=Uc)515>(*hr1&OzL4)!kFygkdm35PSq4^dao2iNJbG>$L6db1w&A3=BX` zv5Lk-&eTLEHUPOBk_E8u`61U*X#oC;(uF$Gi2iTQ52RLQ03=`0It^2h2JrYJNsv=? zi(#YC0~RL`iwKGe;(_Ycgm1zC^VWWXtfhg(Sl(*VRdg5-+7rERZj26de?`rjVjqv# zlLn!2U;@8qXqb-8ba^YZ49kFU9jstrWvv6Y?qPP=ex!s7R|MQ2gcV^Gf+lotYqsn@ zG_~e{i%SED^Ux1TEvLw&JBjUnr4i`RpjX+lo}jnezP$DH=$M47A`PO6TbgK54N$*I zv#g7ZqNAhJ(J`IKt;JeW*VGIY&97s}NWJOG1r%bNLL*6PE||!8s0Fgs+9C*MT-I(e zl+4;J0B=FvA)5^gVl)$n@=PYX1aQ`p_DTbB6>dT*&x3mbiD*j2p)K&g5 z8zC!fbz+d3SGTnIiL1B*QL1Sfs(AZ9T7nSb0ehEGi$a$>@MNF1SeN`7YP)c6au&KW zi2s&>d*|{^Js&)Xh9w5CG%#=)ow>5e;{uG}Zwt+_hg5=OwzI^j6AdxX)#f;i!(hv{ zyYLE8LQjb{;oM1Jm*^}0KG%huec)5A=C#d8D!{vn3g~s9?B26w%NCDK zbkd=!l15u-#+^{>Xh0Plz%Hvrsofa2p)S3>F>uCO&Y6 zIhQ`gV0b=VUpCuy))6|Z6DQI!H(O1qK#0(Rkbms;<4BvZx@d9AhkX(c^7bT>S$G}@ z|F9;;f*t6W0s}6X1;SW52{*fS4&Tn5ZctrN=Z4xkai&7u;i5gVmx=JSL5%3dFhPAy zZp^m0a2+}FTG)mU10rOzqdMB#(;-UzXdI1=I0IX_bxsO;O@cJ#{Xo`t29nsV2)(O^ z=a)aw1gkyK{@0hzStNsxXsN9?@Ja)$mAX#yw_CpA)4k`YHh2|Ax33H&XY)P3N))@1 zw#W~8Lt0+#7<3Q>DKxy2WCA(X2Rs{jGekV$5GwBqzPq}r$}>Og#KRpH{Ayw1F4KZ$ zAJX^ONkNxQG-_qWNie)F_pX9HvDr8TAX_?+(_t|@@RIosW6?ezU&vBug)fmVYF5zo z?{6(NS^5RcpVmtd%tW-#ovfqjmShNl!~8(3kmI;Gh1r7GXURa ze3PRjVZqSMqC^7W=KZo#p8(hy85x_dbdiSS2cphzqLM>>Vr*=jTsesp!IiC7?d*(5 zltgTXO~2ysI?t3(Ag*3Pt8=jhrc(n9ag51iSjz@ybW1+re9~o{4Y?s%2&wPbyhhV0 zMJT}r5GZLCf;*mp9i>9ZnSsQ+bZ39Sca4~nSK+nb(bVC2!}WfvzoA|1svXrbcJJXs zBOJuB{>lYi-JNL9LyjcD0(U@4N~&8Ew4<(Lp-$9}&(LLVC9QFHb|HdKh!%%F3)Px) zK_LJrnQ9G(3(!d&)y2cC5Wz@I5b{?rN z8bU}VE`FhbwAe0TPo=j1T1Co?Z#zzbFifgl3 zXfT=!IH>qY653eb8E)iJI(5>?xv_!D7e88*8G7bPR#yMHwzKZRJss_3RDMgx*aE1N z^w82wZwP9afAGq2TAuZ{d5a#4~^^Sz^3(@b)4_o+sF0}-o2F-w)n)rP=ljny|5Q66fr}4{vl3o zY?+dDmn`}p)jLwfYxyP*J@a4q)&H(3SfHxl|8+&o8~ql~t`S-0^|HQeM65S>{VQBq z>wcO#5AKTebIP4XuJ<{>gC`FFJ&ye`K1G-pa4o*J-i(e z!B#97=g85JkQqZc7>iK2(g7li{p31h0pScIXeP{e>;*bNUE;?h*4iowCCK;It=Cl5{1%?`UVtz6YQ>=GA@aEHiRvLB7@Kv_ zUinG>9XWPu2Bx87fR;BTPe2hvuwVC90ZueS(@1x5Z{ilT*dO@N zW;_#l8tp5BK@3X=kcScsS&+F^7>6VLW2QSfRe?;s@c0y9PV>es!1!H3_>>XMQ`y?Txza;m3`&L0rv&J0+MJ8S# z4%v-r(RjI3jPs`(iqivdhHE%AxQi5 z#dgD?#!{$LN8sOKvFz+@b3i921cPQHH*MWo`O@xZ?QAE(G4V1&#m~dBoym6(LsQ#^)55%3K7bT} zGgR0v$?lWG9qj`9hUz4t8gyamIKD2ggRuA6=tCm2kYHL$dKoi_)NcYY3(R{PEqZ;B z$*i5qG_wtMnreQ~oZm|0lz zW=tRzMH8ew)JF60d&Bf#cB$K67Ex<=$erJ1nPb~m7>ha!jzXF`>T`LLfw`#C0I%5m_Z4^pL-bXg*{oV-u=H?1nu8=#vAI^?9)XQvO$ zX!azY!TxE8;aj3eyVS3G;o3Dp+#pXg4K%KlBP{i!&dw+mPo6w^b1nfL4~~I)o0YgZ zSi->vQjUr&^HZlzg`V6V6BX5fSJ2<;52Pp(bf%2~_J^~{k?N~F0Zk}5Iy&B{lJ$|n z^S%d5lfagG3+Vm7Ibrj$4k?Lc{|yZ^`=ywQ@#o+91DEe!R9;Twy&V*^eac}oglAW7 z%ZwsYb@32z(UPyYnSh^^F~2d^sTMlg4%>*ZNZ8lE;2FJVZD+?>sB`hNwGVp(hBt`H zI==OExlFMrgh_@Z)Z)wzV+c@Au%3a~8R%r?!F}LZ&R|9FOI^F6AyI?Mu-10q?ErQH zd#|HbfSez1rOTX15*L;o6w|Q_Zw?Y!Nmn=gOMpza0#*-3Kcn$qnD%bE&iye;7Q-!z zf=Xz)5hKYjyBc=K5^UBubTd4`r-F!}3Y69YqKnQY1VaAmM{+x`NTTtkTPOl1gSKx1 zKv_+IR8sVFb$Bqu)l4~`!bsHFfB7%Mc}68#o6)Gmf;ZC&AmtEWLeF$i3dvF?pii;^ zT5LIVla*svA6qoy^(&x(6pv{qx&|nDc&$WbeT>meH{m1@^4-|4>67Z0plDot+}!tY z>dtUNTK#w_3Iz;nb{`=Ymb0z*nbN=k}V zbg-HyCJkM{Ut5v|5cmy;$0a2m5L3Ai^7F+{D=G@d`U19qlxwdR`eOrL0fENw^ZNQP ztU`$Xf?yccn>ZtJjK*`G0fR2|Uoq;@nCNj@AbT)(Kfxbw5!+vGE#_NmZ@N50yQHCk zmlkP6C&$Z)1);a)lgTbNh!0|}k1q-sJ#ypQTm2Aaf2p5%il@H* zurYimdUm%zdfcF^Y?Dt~|@;(YCPN{1fK^6em)oKeUd<+wr{bLO8^MJPYvGg0YBy z(O2#OHp>VdI@&|@nn0W5Sy*X1&k**Z;0^ERFVWG4s_7PkZ?`PwNQz;|o0hGe^GhhS zp3K=??%TeyBln4KT+jQ!aur3*PMvp4J5JAOTA$6g@RRne+fH+1Ms3jbpi&nvYLW5m z(ePo<+Rd<66s_L$MbDlU()*aY`V3agdZX`JXq5=lclFD1@!x%WS`oU`q6mqznc@Ha z3nx1_N|vZ^5`9?xp3Tf`T`V-+WA86h{rWYt>u4SXi6d-Z?(QoqSalQc1)nseF8wEp z;+wQnY{@QMLaVqs1oULG`mIM;lBC#vt?Og2&w5O@(@r^?sp3%4NXvs35 ztf^Wpe;u>jr#Q`qv6?#n5th+!S+1V`P192#uTW)B({sV|=NVBAoB74k-Q*2;+*!7y za%eJBb$%msWr0MLB&A}tIyJgW@c3%g;B$<18t^`3uImS7zmMy)q=(lv-CMZ6%@OACM!F;Lz={# zX$Zr|KUM$>(5nR{cgs7nnj&p6`PP8%%pDK`(}9+%H+~F?kiH*3-f5SYmEDF)=FK?g z+iJ+m>%cu<2}6B%njVjf2tXBdFLS$d=MMTG6b>VB6L<#}mJbkR19iXy>QVj`<&L4| z#c_Zhj63{+O_-MV9R$(Sef19!p~i0ljtJ@@ci@!I8;BXYme?cOEukX0%)p%##6 z0L&P6EFp^JNCMyhV_^#aY;|QN2$zr$mKlgfy<7HD+AlNfv$<2Ih4 ztIYke{;Sl!liu5oLzUefFMtEqQ8@e|PEOX2L?7*|ld?ZGjL6?1DC!+>_J)6(Zb4KM zc73>07Qlpx1QA24QDA%_gJTmH7mu_hs$6RB59bC(iBQSeag}Mf=NTx=4x}d=L-eXO zI|~Fy388QdbuW&icaF@=6ijv%L!0^M_wU%?XI(Hn8bh-tqdJnyEMT}Ag3+x>HZzkJ z#C@p70tpy+aP`mtbXmtq3;E+`nCtXC2B<2SDZ4>f*-}i94HNrVA`xC-ny#~%RU1Htl@Hwbx|xblr}5#`?yiZ9dE0B(Uf{SZuarie*Jw73 z@o32C|7G5o(l<6S5rvImZzztkv&qDB0GCA>6X~3Bu$+%k5-`?2rz;U^0}q)+eSoep zz|e$4B~STN`!Ci`b+Udqdh8fbNzJ{{=jwn_aEprOmGp<*HAVta@6Zr#Ae67hn9a0O zy6_~ZVDEow@yrWhwZ;%Y%E&CuJ`>Z}0K{l4k>onOgvO|KAP3-!p{U&_0KIZJYId*H z3}%jr?~Qi=zI(Ui+5cjLIf1%2Fbh4d8}|BS`;Oj-)`9QParbZQT3TqNnE@>eLJo>^ z%k02F#bMBNXn_J=5+V6kg;F^5{Sih8?-mm83>k2UnX_7)vIialu489+KaTbQO?zM3U5zdnnRX)!}vCH;; z|1L`aq#xoC$H~;A0nXJa#?BmrVXIeT0GVPhnwpve$$2x zuLwUgEQYnOM+9BC66A3**>&8#OU|w_Jt>@z>7oS^>|sfqEU8D1RcX+a`b&-6s&C%r zgcJFo62WCTP4b#GL@c!nfhYlSk7CDXOz=e#nPn&GI$z^pXv<8Pb8lfx1M7_^6KHG= zQM_x%;ISVBFAp3R&NeK#u>1$mssxGJM3b9|0qX-u-I|a%s5H%?T{&T<5G+ES{q~~< z2fy+n$GBmVPWltJ&+rW(>@VT>x3rt=a^!ywdgS^oO{8mV2z;{>|4+cfTVn8d=t5Rs z8h`~+t{5h1p_Bk}s?-*()Y#wp-!Jib)X@^V3lKbnNX8#8Oyj6Bn`%{`A6=S_g>p-i zLRVu5TD>$2eR9&Nj^4~muj!xPOB}p_&Ef+dw&n+x!Vn@{Cb+;Xk(AsLBOYgLb_@%C zZ-N6p)vQZ_g1pqi=K6I*Oanr(B*k^}`t>`t640{^0!z7{X@Qkm-P)=qLRRk@4k1kU ze1ey#M=k`txPYWIHVfazJFKo;ox<3%i2vjIwG0P%oXtA2ldPnOB?*{heRvD*Y#pS@ z9%i<{ZVe0!3KPlK58pSh7G(Z+;Y~U6*j3ExGtCL3gu4KwT)xCs9o~W@-kn>mSEH?-a1 z)xDKyCqtyDPoImRI@`+3QD=s2s&wF42d2YscnnGhz+X}@A}2_rw&mOHe_LL@DkTRH z#I1QL`w=&iRj9?u#&1ptr^sgyZW7#$Es`1*4ZNS61IqIj4QwJVNja&PUq^y67q zfr9|L{^GxsmKHPq=1i`lzCL>hlI83V+Yr=wL=2-*CVWp&bE)9Ph#$hf5QhK&WWuO< z$fP>uV(};4E)Ix)h?SyS4ALMWldjt*@j4!Ns5`50RC@K+mo78euUw+dgS>`1v~sO( zBwsAvr!@dA$~cZs@}ECsTP@}Kx9x??_8pK1Wy0#&JpqgLF;H5*{E-c2OD-*QA`2 z1Ak;RE;!LvQk(EDSM`?_tR?ubZ$@#$JWV%g;VrCP$C5l)wj@z*Y1zBiWPa+RJv#J1 zm*in%7_g-lAnCs=KU!;9lr{J2)GDhN&_c1A??1wGAao^1|CV6#K@R>?5-?T6P)S>` zc~;%nFY*ufrQA+bIB?}`HZ}Pb^DyPTW~jOKL{1iu`OvxT(-$=4CQ`I)fBjv&;ZvA> z-0aK7={f(-reRB((mHa#*hEC0p=$6Id_v&GaVX;s`aASkZ3_IGN2CTzqJj8p^=U+zbDW8pRe&a|_8 z0xMe{)0{xAC3H#%RN>GfM?~SVCx9f0dbTeF$g1SAl9A!m3k6WdF(AfQr5tjZzIJO~ z4cH%!ujkPs)wv1KEx^^9C;UILwW9I~=Ta%yVVe-P(5;Dmm;cYq%&L^~NJw88n3%W< zKH=emWc>Trc{u35BWwv77l^pWM&LW**yjhdH5K_4=uP%MIuYcB@Cp=U@;PtKNFr(H zAJ7{-BaGmHa56Tx;z%1&0JSUe7WkCJ=uLG`phG%}t{OK6Pc&o+ot?(`p!8Gt*#kJL z*tFn~Jfd3U{KtPj2k!=$c||KmU|wPnus^&5qjyaAYb+x*kikR%aBgs)O>Jhh3X4% zDYWbaN(Ctz67m5eOZ}m)VC*ezfGTrG*Kgp#m55t?c8drgHVK?IlsiL^q(RW?C(LHS zgjQn^a$5ZtYyW<>f+EzG3#{I(ASN+@hg8&c^NbEaons>-pLZ-q5i*2KJr8m^n>#3) z5A%z$-PfW8jydZ2)-Kivo$kL=4!%z~PNFrl&Z()bePHn+{jG5llwRpTJ+E4-VD~5+ z?kY|(##Yl&4)@;M!}LTi0AvlL5wIc1D_-YzoX;?%6t2UwOMIkg$5#{@B_l+AsKRh+ z{JRR#YuQX;7TigWQdmgRFvPa-b@(SNr>Q^-wQOO^Xd`;L0YM~=c-*ijphCaOH}KBTekivp?>?Z%DozP+L zJsaU(dN1=kto%E4w|(By4`Pjz~3YBU0OGD1866cfp}#5`$npf5WQMe?QLzn zPESbY%y)=$w6d^vWH4UzAa8@4F`+^!#?9-=Ds<8A8dztEa zH9)13=U?~paAP2%1&CGoUqlCF;C+97;|6j8`${;EQc{V(|E9;!&wp)}nzl_AL;8s- zqHEwOAe$1A67XFBXS)~SGY#d?UVMidu#-AeypWjS`1d!z8VN1wbI|_@kI#3g+O^2+ zz5t+B6+J*F)PTk*&u?OMGy?an|Icq+FK9>Kg!71ygK6Wj8JJYo1ps&f|D}~&{(o6n z&70TKXzwF86j<^QYLJS?C08nhMwvw#Wlh79B{eTeV;acwr{m3M25Uoank%NJO@Dqp>^M z<1-bk-og%$lAOaa>}uMUx(j^={1Fi-F*ovyLxO`xJB8s;@WPOnS3ZS9xV^3fmHb7$ z5X_=1E_;z6u{<(kARHwd2qsYV4R#$kHRzp7=;17cgf$R>@eEOHlWl&%aMQ3L${-*M zq9z~riw=SK*eke?^{}IUW-Y#zI7o^*0}zZ_<02`Lg$||>$O=jm1inumJ<`o=K??>X z!k5Y~#wnkB1ph3h9TaH1*GE_4g~$1sVoXm_`|1t++qZAuykWy7rG5Yx<|xBQZ81;4 z1&SHfh(3#Cd}zhM8bJ~y=xyP<0K5GIYzsER&bafoHP@V&B~h=<|sb z^Y}f;A2BDpu*Cq?oUp~|_sDa$0JMBS<nCxffx*)_+C~e9KEv64sc4wyfxNT?8k((QVrKy8-qU8F2%@njR-)$63$0 z$ol9HaAwVPD2IprX2_n5R~Wi_uYbGOtC>x#Nnf7`JZ!IHaiXr)-{Q@B!e(?LYi2v` z4FNqt(>trL4c29Zkz25>i|$|JsN~+lxid3#-Q2qCZ(g-ua#}*OuFO09YH}$pk&)cw zN1Lx}ftmj$_9wHx*ki@?cKAGU{B=cB3zy`%n##GucGCV2cwKj5sZ$7x;CPC|^I|%R z_FLC!h4qdA;!Dvazrgh*^_HRNNX>zNjU-Wg(lS>=`w_l8>8Fy}Vb;H_L>};Za(VB~ zp*^H-{Zkh(JfC1O4L(+C9bWvZ-2|cpzr`J=iB9gHzJNha(V*I zhCl*9(WyTNUHmLl2M@`bxu4i|h>zM9=spT%!{;K%uvWIFWo7L_Lvw2eBx>b{4;s1W z!TP|}HvDzo!rVL-)MF`8ZG-(1SF?b1u3Hlp4v!xf&Zv~Rd3&$<^XCsY8Z-#LD=~oD zABK?*-V)WFk0-_`&?Y(%zkPKKMKuIvbUgQbt;MEiK&2#i_^@-ov^exZlMMRXKK6h`mv9;ZU<2XrXM{e#TfVv}&&|&6scE02q$Ddom8m>BY5LW)L}h6Am^NJPf=$3A zvlg|ISi^`?&6Zfpq1AhoponOR1kJ3yb~`XALl zCZ-DQav;8IwjO+A91Vxo!DY*q4QoPvht+!G^VY1j6spnDo^&*o1R}xCp9ZehX6iBQ z?=ui&Hb1aPLnwhdr22LMQ{SM1d$$B0xU`70YoB!dpt>Nlbu}No9;)^(L3-?WF~*_9 zLy8^T$|yZ6%K+t#UvmI)lDb&Eomubv_t!yklwds5n-2f^6~dAaXTvC*2InkKAjyFM zOyDo>8y3cT0nVBRO$@m`pF-Df0C^VGAcjw82r#0vC!gRbfR5#^`|qPnqPXd&L?8WBOqBJLL`Y+*Y73pb3w zj(G>MHP!>qe$w9qFb64`ub-Nrn^#S{{C1IlmM}Uk;{MktixL9b#zl4zty$8z1XAFq zwkm6+AZzLelJ(Rm5V(!lO~#y^fqz}YAg~Ddz6`N-kjMfq9z^zJAes^m%5nYj=r@q&fT0C;MBN z@&fx0;yZp0`~!c@E=!_ud>@RWGyUJfSb8$8{`)ikuu%M=XZ-a1vbiG4JbfKoEZ55A z!})o;zHS~qx~;wZFr!ebpirZhXOoxmxjy14R5U7b_P*Hj9@_Nak^%@3YqQz3v z@9J(GsV%GyaWb+$%=&Of$*`}e&!v`mxX|WV-j)*c|Ml+Gw|}1&rnw~&K{e@L_{&bM zcW0a5!)8iB*n#6vPj%lvQbU}LUi`_wi0#Vy`gP%F8L%j{!?>eH43FPK>^|{719hT+ zTtjZ*{^KHHre#cwFZ>#D=bs7`Pxs3TlM*jvNg-Xwi{wu!<-& z%(QObzO8I*96Kn4C>Ba^zEkFHGvkgOSA1+X0i5%MHMpYr3~25sH#gb`GYBRazV`N7 zc~@0bRIsJAupVc3uNE9xtLstE&5N zLk=}57bwGFj(TqOmKOD_VpG{-H^Thz{`hem*>HmZ%hg_*E5%{FwZ6DicKwnTocnVN zFzP=|&kp~&xjA#(J^BntoZf{TlSzb3-TM0Dx7!5_yUR=fkBp!l$geO{ zAgHeC=#;L;X8P(~Uv z5CnNSV5-(AauyvM9sRQ73=mL#;Ixl3f>cd#94FDd=NFS> zH%S1{wGXtY&DfSzzrZb|i+d39>D#6a2?$-BL2vkUAFFCKFow=$`qcjYDERNI{$SWV zu01;aRvFH;Y}mXCNp{J-+O+I!Ln1FTXw~q-B@$bGgUcs`Ru%?Bm63`^q(Ky*iH}9P z3nsiojJgqK1m8xp@-&JlECe8yYY;*S^(4Aczd!;^fS}y_G8(kR)?lhh@HRRq&p=(5 zv`qK`_V=R_7-9E}uz_s7)OGqhr$)P>JVA{=3ggX%(OyNJ-cQ`aP8^qrI2F~B3 zjFNbW;!c9ZRx@q@x#tN0_IQdNaunWGRrx+=i@>G7tF7H!sPoFA5$FUL_RVV~9}^>v zr%qiHITXY(1fw|KJbraEEd^I5i@e~eVa9f0M&>aISkuRf`~Duq_ymSh=2{D=4h*6S ztk?FUvmL86pWMOFBQ8yoj+*V0?|r6QnbjR< zq&4IorrfNvH0d~XtlU;N4nwh{1hmkJ}}tE+wm(yF@y$7ua9y zlC?~<)vqt!e!kOuKi%rrlN-JMpwXiVA(ex2llx3#C^^7<_t+7??83~<$=HxRQ-^?# zR-b{dGIlF@h!^!E;N7!beihh%xA!E}ZSM`6Z8WCazt$|h zECty|GoP{yGip~x$tie4brIF-O|kDb)%xPP7c*~=~faplZy`@OP9_ne37-X zqcivH->8tNiQ;V?DuEp4&Gqwr^Ap{FL9a}5KsqS?6NM0uc?&?CcYZX6%`_}44KrezQHSC+*K8W+Az?D-c|w#A zIO4kUF#X(A@q8LGgWk0Z>)~`fJ>sdJnoI|FObnB9{%gF_Pzlq>?)0$gKW!&dwHSnL zu`Yf1ylbdFa>!aDK?{VS8^47tkID}U@x%cGZE^056G&(Igyy~4!qRbj4j9shWzlmT zM~whO9kArc&ayV1d0T61Hak0vnDmVJuzc5<3xDTWe|-mo*zd*pz+0a9_>?~F`SL}% zH>Q+HkHo}2ea#dJCE>ddhoh)cplb*Yx;kM8*Fi;DS(z>|gh6v<^HoG#JV?I-KqE=+ z(CVpdZhk7b&3jM?^t00W^P8cJw6}i*ZP%a>ap)j5H+Pz3NU284Z~b6)-`jf++m|1l zmX<%&t_0zV^Q?(j8h0i=r$!yF@^1j8v{I-beB%(vYJB(Uo`(-k1#jM(+hiKLd9{`P8yNsnkshWmxy=0dzuIsZ=ODy*(L4$2eQEEevAAF|VAi!Mi>xSXwk$DU{O zKJmguK~R^>Y-cycR(v_S+4MW~Zd#%@cPgQc&fRfgBUr2P !$>r@x6&M{jj^xlM zzP_)Yo&yq?o|Dt>m>8AT>(U1djX-w;X^?pGjxM4>hXd*D?&M%3bUs30s-3s#PQ%r| z|NLS6Gohx3gh%pR&;C~Pu7ZuH5lB>?H5po)o0~topwB>0@9w6&9v^p9cF((ip7*SS&c6D{%8=%WXv`mQ` z0Sg}lyyany?;EOCRyze`3=bk;Hhq$GfFw*Im-G}oqrF`OWwKrL8%wm(b&Q3i`|%j4zPy%k4J;x6eTB2L3Nl0hO`pOX!$Gv(_Vy7h8`jfP51Y&gncA7pKC4P@`)MLd`Kk-Pl3(Fx5Hn*0_}Gc z#ek>>ZN!!k0qfq1qiPxd_70>uAmjcKk!hy2mh$z4-G5#t*_0fAIT=cdH{`=Mpm z$9I*UdBz*M<()4d4h~X-ZTJr3uYG?D8fYW<+^63rU4FgY;^;d>+-R8nBWy^H$!4{v z!`8jaZLF;~!f7`%S$Vz@@-Ug!$Ohv!z^4<|w}8Q6=;*dnQyMYMrnqQCx6!L*rk_8>bfz zL?L|~!qoKMKnS9u{NK)03LTrp+GamJ(^T>CqqedX5MM4z+c6C)Iz9&pJRv+%I3nV_ zHPL4>4$?^iU3=6-obw+{fg?OsRRMZFZjJMp_Be=-jIv$^`hy;i^&|(0!6=0VYZ&Ey7x1c>P_`sW$%1^(@aZqpZJT8`#$Ht=)45ZO|-AG5IBqX#$VvEEshqIhq%X$D-J==eJ&gJdg3YxU&g0SkGO3#g4eyNX< zWs`Ahrni?k$6YNgS8vJ%5Vw&gw+hV`%`HCQ_4emJ%p*a^W}X;UpV<~(Ooxt%Q>OZo z69BV+P0_#K+QmFVRaG(9+AO8r))@WCGl}q)cSyXW(P9I)t$TA-abx4r|N9zJrA4gh zPDhme^0M)Hw)y|}64ZB|mb7wzf6}{WYroY&zVSn}*vH#)INNj zlpj`|6X;Ch0mb^1QO@whjk``FO&pO34WdA-$*^hHvE8y@mskw(BA}`?4E#c9&vShB z_jJxgHYj0vPh26+3lqU@4H|F^>;X{UFc6Rc2C88|T|(K!EtTBc?U> zWy|jT4MSQ}ES}8>8#FC(C%ma`5v>A3ae76cvn!vyFeez{sv!XvXXlEls?*obgS{Gm zVDaLC0|fo?IDPBe1dVq8Evd7fTpa0e(TTnk)bMk{q;vr%R>$ODvxU#{cy zhX%n&rYHlfpD!S=!dV0OVN`xA`o3Ik=ipbBgvZ<uuvFMlTQ+XI=K6*2`10TQ^^nrfAly?ELFXs*j5SyJ& z0D>gO&d6X<0{v`P zs;CW85D2qdzKX%~9_O*X_ux)Y@(>oTzv1RM4{E7M=ewXbO*GvA#by<>0#wnQ-BtI(SFbLjPxV-62D5~4hX4G@JrO@_w(`IKVx-5< zl7Ighep$jLXMTL%(Yu9f>eNrJ*6)eeVXts7 z^(GI;c?4w9&U>B=)25kNr2`-*t6ak#2J6D##y}wMgdo^R9y+)vwER8p?NtTv8-u53 zJD`hQa2!3XY^B}s!fiyFZO7jo|#gj5PYQ8&^o&lGDWmJ{7>h+Fh!Xgom zNB7BzllTd~e|Ybrrjj@)%eV=HTPkkBSVqQU3+pZ2`ua`w*Jxm z7s3AlMzYdLl7<%AjMl%vg9y!OkcU@GH;Bn=4QQO+?TidL2Yu`t0~HmOB4a`37IkJ; z*r3~|Jw-s854((beK95LH&T@cHYCr-CZu!Zf~^*KC<*)xSxHIBiXLECI#2AOKk_A$ zn-1%kB~vdie+bLXGGsh)1qtOFH0rbJpF9k{zLvDl-P zG4scveNg11>f74d@&Sm1g!}sY=b@dV-2MR)W}FvatJdO!lliRzFfdyw92$!4Xx}BB zU~AK&?0^Z?LQvOl`L!|oX}m|$e08jxR+kAAB6bn8P7S4$wx#90jsfsDG{Hxa)ibMo zxPK*tbO3b3#CBG8nOVYi!sw8+6eMECS1e%yCb@L!)_w&~8B{F^jn>#ypd&0*kOaXT zg%XNuHE0ezazEKKyH`S3S7~zuY_?YX89Q;Tv=8RdG0@~> z*Q1gW_4)&lywWn`k79dkt2UaU9ii~vf0BJj;>0>Y<=cN{Wx;&`#TFB{NlG znlV4qTSk)Qch582KXzchGoz$Hudl2+%l(1r{Mlf3F-sKfg1&nf3*ML_)pNGMr+|lX zCmRp{T=|*1iTcGbqK9V{Oxn;mfl((ent$$ui{tSh?sFpVe;37EIQ?wWf3TUuv!*Bq zaQyqK>Wxjouav!kz{F%sVeXQbtC~Tb{LJ^W{{P-peGW&F%Z*V^=7C_o)2o#H^`rSB z87=`0(zpR8rC-!dpR)m$!`bOAHq%WK#y>77*^xNKJ&aG`kRfno!HTpKSbpRIU(toy zjW{mm@r93(RBuO@Evn|JZR<4vZ%q6?Jw3mUe?UOnj*`FXsWp%ojEP%aVMu--l%N=w z`>oJV+J!n}UbZngV4)agv6hTbL5Lv0S9C$bseQ%L)2iXI4Y`^FurWxz00iD|EIm@D zJQ|djhqkS1+4ax={!~Z7;NOvin5>m$+xnB8HFatPpuKBlC^R<`;^Xv)98+PJt$dVB z3;Puq6A>;(kJ>jFMRb*0JN5dKsogB>Da^4t)dH}wmq zmkvUaK6sWml9i&97;&0LVs+^I5$8Xc=W|gWzn+Auut7fD<=29eva)qJln4w#G1uN3 zhm6b%06U7vN3bB#HvI(LQ!TKx2jIEX+O@3TQ){wdKv{Cx7JG`?Qx1xMhKBeY8%P6( z{!y+A7dcv_k0a@*)=%J^Y>*>+4$vwVCJ_>XR2*uf##Vcr7GQgxs_9;1mr0F}C;)ef{5AKOh3ffU}Id|6C-VqF_C`E%-OuxdYzaj(3S6fufF% zj^C;&r9lwhK<{uFOh9#zebfTU$;E4Ou;)(FI)TWIEvSF&$m z&v89@=NyiEXw(w7-{gjVVho^d+p1Um&aICM&WTBh29U z-jN=N%8}K|sU8I4K{G~zVg7+!v{w~E{-iGNI z1|Q*MC;7T)F$ZJ1=BNB)MoE;Wt$bCE1$SQuIr5o~w@dXwJ?|a80W*)Y6DuD-W;d+E z|=nPJHdblQsMl-|EI!X*C$jSAc`VP)Uh#ur_$$Zip5S)Zil5 z+3VMRT{g|9Dd!$_>mc*qTkVvVo=)F$&_KWX1W1#d>6iuZ0>S*v$4oL#0rWq=7_=%g zm;-pn*YqLd*od*I?w12j#XR>kjemxB|n{Q<=&mdO2 zGYEfK1v)Hh+7Do!MxtY6!ZD6)fW+K(=k?h?OFGEK637P8o!h#)B!|4}AwV-g{^3g2 zcgh-O*#^i`^3eQ_+4qiU@j&>LkFGRKQ} zR-&rjJjsm7mj8kpBH13Gf<*m4h{j-G8vrU*dF9tM+jt)h8xU>V2sCth+Vv!-34r%=k zNZ~}PV4uK>t8ey{fh18fAr^;Ak-l#ow$YBC1_Fw%&njCdDNSp3DrETFRX6e8c|V}G zQLg{dlBp^sIw2QEF4%o!^*6MWO+QY;P>*&$R8{;MXb$>j69uppZOAdj+_umD}?osKD+XmepE zqcn@)mIsXGrigWXfjVQKld0<@Gx$+ z%vF!K})1X|e66E=gr&TYg8&Uh?2XHz7~ePS1>NzhKj%dMkOcuk72Vb7}{i zCwv;Cv}!WB5s36OG+~y^f_tgwAh-|%tXR>7%GWz1t zfEs1r=@_jTKbVdNH*wj{EKH?K=@2(2ZqQOK>uL$C2Yf(i;dT zS*=c=H+48q-jEP>*!ffLd17W>wj~=3^7!P~`l%QeFr4(LUIcIX)56m$xOL74C^@;> zh*zA!;@UUHVZ2y3&d>Eha;R^#zhDsdZB9F~JR3jspX1ukpd(ziPuGy3U*vo(z008Y z%(VHU5?#HdP44A|JiY6oAQ6gNI@9!7!>%GY0+6$?%eX~~l!3|WF7gWn@42~*Ok1ye zcu4s1%%9H|VAsQ(o-D0^KX6)MI{5t{u&*5091xhKW4yzw04RM41rzX( z^xr=Q0pLcrTmnp-91LJXhO7-LJ)GCb^cep0xj~az??n-Q2OgMA7&L{I)pqK*0AksW zl){L-JeGhYUcr8H0j_QjC3H$C`UScWN}{lavBt2~9OPZ(Lw|8F40j4t_s6zCK2*>S zl{(7b}gPgV;f zRBVOPzS=A7e|&oB;1`Y5^4)5zS`^v>wwtTLiUv-l+cH9_8@xYKl8_4X4Vm@K08hK2 zT^a?eQ>$1YBKi*Mh8d!5wj9)kl!AauL0@8VXViz*2J%x^hiLnX&mRhvP%{x_!m9xT z^l{JeOvv5#FD0lSf4!f2H3>Q3cXs0c@{$;fNwmBF{bRE3gx@NL>Bm;NHaxclaU&XG zuE>Zx2rT@1GSx7;2?zN#1oXFE8{Ifq4OBGic^eG^4c_ANYs}qd}BRS~UA`8J;bNe%dp&G{#S&eRz z%gyDhS4)h%&4=M4cMR*)sQx>g8}@^)?mamC=a0>|aCBrbOiGjUTR5bu8@=11Xm;rymNy!@e09d(|&RyeE>RgkH({#1fEhSF#bmemf`B7)H*51 zC5`54_3kYYB;Mn+1LPp)6iyVXaP&^f4Jhf6V1v}8XI6EweXgDfLHnV2muhRqjB zz&!O^wb*mXiWO>={BtjHDz{+{rIo7kv zK?fInFj6pP@OFMw{)lhg*9naIIG5tVMb6c`=WxWRi@!GDIl(X{&;O@SptsFyin7TT zDaXI@Ka`%Cum^0%FoyK-2gTcW{ud4&2^p#JaKi1AZn8YFm|^Da@v1+i z-lS;w^T#aqKNy90(S%75v=)^fKKEsCJ`>R|bG2tZ`XT>;1vZ#4t9N^`3sz0d$zhn7 z+5hd7xTyubn!p_&D0*|ToRIs0A$e6q=d=IsRS;B>`|$l)C2J{5!KMIL&AIT^za#Ld zDd0Jq#5nKOkzv~dbcZ2k}H(wuXNsV>M1Hp!jLNoD^_af!;$aXWuJ!G?=Jae0Eg zt*iMqOTJm(C*;nNJNatmf?sf z&J?tbxtlp5P+DHk|JD-m2Lz2ME3j6KBZCQT+hnl)z(RGXp zt+Ci0X?<2xbC#W*9Z%Ta(_?q>!z`+m9qr00Dgx%=P<|AlwwqB!emWI~*rI;cMsVdz zg&(AQN+&pi0Wm_%>OB)pfA`6{1xp#GU80DZvd5LGujfNwC3*BF$C3}`TU27@a-1kV zC{^>#jwOCAPPOaFU$~;wfWE) zQlGJZWouiTjgP zZbIt8rD>U&6eNjIi75b;=&5Z*;kX<@pOT4^0x z!iZS>MmZ*~h>Otc=sv$P5{?s>hubFO6c8X-p-Ve!SoR_7`)hZKw@Oj^qyuJ2AmuGF zreR0wpb5zSev)kzzGN!M*gK}roGJIUFQm^b0D0y>@5c}p03F;zwvw&Zj;_O=h<}5} z@We3rW99fT1t4epeY0c(w^u{hKJL};3eEVW1W2kl?Wh)k4TIE~;WzxtP`Cmu)AE)B zp(_v;4a&-S--uqm3y2e^9pp6~L>{mfzZVAjI%YhV3&Q|`JA0^6JFUMjD!MwVG=pd2O>{svPhF#@3#XFtZvec$pZ zm=C>TdT)f+O&CFL@qTnQC)5o@7b4sSN?%YPbe@%$TFXqy8Tqx)3WG%UIdg0^0rUJY zGT6(u*aWqtNgd9I-~m}#u?fK~CN$aH5KH|i-ubdE9?NSKK(Dj2ldvanFtUj+f^lW9 zaGx9$iN=R(4(!g8@kVR(l2P3E8=wJ6ksi%T_3Clk4=`W`oo5sYN3hm@%OnwpXfgSR?t`dD1O?RJ| zJ20Hl=HZkM05>cRFZk6Qs8KMobZz@;G+50xP9^gZJA3c8fjQkeRlvtXeh5 z#J%h98N6o;omvXHf1$nRadICDFXHR;r;+7M#d!M{yTVDHywK32k4h(VZ}_bD*#Vw5 zKm%F1oj6cK%iiLMHbnbo$59;LYoL^AnsJPS{48*sMb%g1`~$;UK3hu0_qH_Olqu{K zdg9rkctRwLy?pI6Cm^stGVM|uyId%EdqQVx|)+^OFTi$)wDUx#% zSgKD1$I9pC7_7+%?`{j!PMcT)f}Z z?F`>B@2dXF8g|{@8yBEm@>O^bhyFeOp3aOhn|h$jG6&I?IsV6(SE^x%R5;Fx}r|Ueyp)cy!!pxkqiYdl` z_b4*^f#u{byqau@M^a$c)2HOCvFn?U^T@FYxjM~OR~W4ow_ok!RmxuN zy1N{U7lt9*(fw2l6kdpvipH7X6KMmU^j|e4SNuWt+T;r(UNBZrzYMCfQi7gOj^5|o zIo|_MzQfRxI6Mf$e&y;Jp!~s_h6JxIU%%eNC0mSN@-H2)9^HkeY}lIXQ_;ryIxc zTUAZ|2C3k5#6D4vIVpU-BWHaC#1|1{-H1jkF1l3t_}_yQT=M^d;`5(hWJLwPZ^v&I z-bXv(p>_Q_fX}*5=H(HG1=c$qASZ|sxwK@$nuvUl87HJQ{+D#EwPHb^;M@&bQZ6m$ zUb+8JZozDNaCC%$OvI>bho+~mAzdcHMaTp6p9pHn%k_qXY1m+n_ zF@CiN6z%TDWbaOi>2|rX?5%LP@&v&Lwg&)ZtDhzV0@A~zwF*O!t0y4Cx@LT{0LmxP zFezv~YGw7JjcLY0wkDX?#{t90>h(1R0lJ!>ZH&DZk5)MvZB7@Kyal%_0l_J4#5E`c z>IIhcEpo@+Ds}37;su3T9D`c>oyZUgRt#Mqi84|lk9bJ~inDQEcc}nZ;thjdCS~xr z5}J18TGkPYSJA>T+>-fB z**JR3l>7spvSZJ+>rEx#R8$P=0I!RSpHb-x5HJkjTHp{u(9+3<$J*jF#E_bf&Ko~{ z{`?ZtVvanZVF77htJp-lntq(|M3d$%CQossIjv6%GyJE)I^=Iv$7!Jhh3ax4Sx7r;7d zt<;d2xH}8Q!mFUP%pFRnv&l1|St2|th$T9KAZOBr*79O|lnkl1v9OqGqsI~s_ zpnMHw^_Q<=U=@p@cn(XB{0YP;z?>Nu)yrg3O8+ zjUOIvZ%id3`IOHV4Z~0v8dPFYLKU){eO4MQm-sgBz1&U0w#xG_^zMT7%WJ3fe#}a1e}D2 zW?DVY48%bajfNqwAx#YwM*)=>jsi$Ey5jqDj%zdnf-Dxs**Dk!_8Xztqh!=sQHYOE z-%JDi-*Nu&J9Ozl=v@=!8_I_6M!N>7Wd58Y-CLN@K0i8U_(Y zDS;>hFoukNZ&r;a;c?JRA*s;+ES0RU&i?%|h3<)cUz-Nnw8Bwi&yN^^i%abRtXlr$ zWMnB3;LN-Qa3TSBQg?R{uo9p#W8Xh@IHh{%xLga$L&|}lI1i?1|16#MpW0zz%=olP z*uKs}I#MRiW2mf*gqJ2QVLaQrr(-h9|Vv~l}0BjTPL1XAji@7 zp?HTm_ihl=B!1XYg$1=}$XMsN|E^o*R53Sl=FR)3v+^2ekdK>%k0pNrch%v=) zQFB@QwQqF%DT2eK^ivD^+E~>c#LFqR1L#QJ6J%^C?ueIR+NJ_e_^t->2}VB@78hUR z*eZVJwj%559nRwN z+&gd7v-@t~`yl<#Gkixz2zFqb>9z3h!a%T-VDInswl?+D&T$=ioQ_m;)o`2vlETTH zoVJgwfz9p6v<76|U~TpBNk%QZtpH(_=)?~b=VR?QE(Gc?y>XyW&#T>o4rFViB98Kq7K4+Bq78DGVDLgjZi3si@1deXj2asU zH?i@?fPeXb;l9z4DMw#zy`X>wq9WMf% zg4F=~wMuH^#sdL0t3YEQV_bU;utCfaLjK6(KPM0vqU=bzh@-=^S}3*rP^CFYmrf&E zI$#_nfp4F;<(t+0YzruVvpyRDs=2Khx2CvHCjSG{9u!3WMTaCs=Vz$z*>e=7H{Zt; z@DzFLj_yyxtw|jkV7spO#$DL=7qkrvOzhwe;}H@vJ{EqwsIs#10~ijcz|Cos?0>iMrUb-+(38jyZGVVjMf#(hm~Y-Cbxp6QUZ<%wXmHGb!^?; zsS^{BiBnAdS=^8|?%>D?`CbNT3sHKYx4ni%_XNPi6bkJefE_W$wVgR*#iABII)B%n=>$$~L=KgDu zbAsU`>$QzPhx#NH_*x}pF7QnqQni*BEt2(HHjv?ay>dk4Xg%8kGrviw+iz*F2@4ya zCxo&%PP2 zuP&h`yUNRE_Yw#8%iyet{+Gz{3b)Piw99hNiqGDXDwVuhdd<~InxJ-N0qW5~?(Z2a zF_oQ&Ase)a6Tnow+&ddABIIJE>e0UQ4Zc}XXlFqQe<*U8h)qajqi|?0Xw5Rtw0I*B zeRQIm`3tbqrTo1rp(iod8Rk${{R)nHWZm#nk@p{(CL7kBmE{FFy7JiCwEpv8_o5FP zg07YZj2t77m9gGnA);|aax%)&$lKem9Re_-${=0C=$0MpPc`1fLc0gPb=H$7n8b5Q z`%YX0$caThGa~e39GxY2|BKWzF*>TV-by)s-sM#qnIm)OE>1f18w8314ngcDufF%Jz6dR_I+&V}} z%vqtRv3x7U3z6iI;9yn_K@mhz#nwX#TM$dY8%8FyISQc}vd`s+Z3xUiuaSedV!Xt3;k4k>D#=;~`uc7MIX4!7>m6i& zyXpSJ=yThbLU{NiBToPijfggQAek@hJjoyjoKcE*_Mq`Tv1^|N5F;c_m07MGj3fDf z-d1Zxv*6seSAs(ezo-UR{!)>EPK9;-`(x)dclQWDcWP>CQO}-j$}WJ!u>upB>Ftt| zk_v&-ffQPVe*wq9U4NpLg4yvT{s%F657=kWO>IYojr z+i&`?7*QRe71fzGbJrIJFcKfc0ut{)38Pm3Xlt_ zoXQ^H16HgH*%B%w9qE*{^5okF7GHxrqMJF65&vINr(qvI72}h>q!fXy4wx=Ms|=jO+oUkir0yPg zcV2ZYpUi!2)c_j)n{iUk5xRw8*fRgDUomwe0KX&9pqLQkq!6ZAF+}kX$uZnzj|g~O znRdY>t>)m@BeQ58AVjr~>+uLkzMo)UySxT7GXVI!`M3w(dnQ0YB>d9ZF;{(51-*jE z{nugI)#Uw2NypiDu`+lf+QX&f8BpvC|%xtBqC zNNMpi$MjNO?w-c%5SCc{L*!QV(B#Dh;cus^lNNN^KRIFtKH;*RH*Tp!%bXnp^gzbb zM)IWaMVLeD1O**E%9Kn9bx_$#{v1VAd>6U>>f&Tons5==8x|M~$?(Sa5Vr`;jfl0^%ya>xHj1E_TK6ngGxj*|1T5fZ!mlI5b z+kNhg-)S&J0`(TDCB+C>I;aKe96>2U$6X~r&rUQQl>3{+#NbP_4Wvl@$4@@v6)PcM z)&vMmL&8zoosE=SiPvM||DFG##B#&WlpMO#!%N{oEd=u3rRQU#&LGDi=>dd_TFo~e z72ouIVr8Is(p-Bo&V~>{?};E48yCOJZD;)Du`$rjmo~Zy8kRiaK06u}0iN`>Q`;$P zv6X6XW41a`3!Rc+W#qM3@p<9!e9WI!BI3LuihfaG9-w+`bhrT(;|qUXXx#VRd3prs zcSCy|x?Rh0@CrN|11A12jkQ02;VCWjAW2M1oDIlVY&T~jcoKc+;vh+q`J|0Y{Xr8t zMlT{4EUD}A<^d_Xa#%K{FEybG@Fja3a2c^LwP8ZX&5aoj!CQvA*Q~&ZhnH2V6bXsF zO$lnJ&d(yp!h7#J$>yY?Uj2^UTEi%+zX2O(31Ma%FB*%U*j8wXR=}sXbl?NPEFC29 z<|&0~BxI)1Kq&=Jo<4mC(ap7IEWI-eR&PpD^Q)`#|6IL^G9OpUvriK=Q4ko>k8Fon z1?Aniln855rmJU@TkV=Z677&CfWB*nMp)}}VRf*Ru0NZu16^c%qAq$R?_VeHHHPCu zS7po$hs8Km$w~a8A^6$1<0VHiO8fy%6~5xE)6MKe=aGS$EIYKJKA_t>vwAHtkW3@M z=jOCScc&CC)`eks7@N;vo?xdm*HtWMA~V?303IlW+TwsXx#kaaR9xIhR!_+94pe6* zOgP2zn__8cKgsez3+(}{%~}re?giA1*XLWj_`dyd74MLc5*9OvBZ4It6(2`c6J@yo zt_Vp3@zez~AhFLd)N{TFu?E$_c&ic!!?LVU{Rz}RJnow$XP5>CG5BZ2ADf^vlM)tw zX+ome9S1KzKrohCwJJrc3@M! zvlj=nW{@e%4x-Zu1E|OH{SSFB`z%86GHr z#6S43qUvS|G#{xuqsEx$AemkvXeUbH-U5L61?}~<-fv{;g&?CRZAIr2%P}V%hRauA zRnXc_PJmR_*=r(3)gnzEuq3+BZ6wVdNvjS%3A#HrA!4=UL}u?LM zJ3nG-1DoS9s=#SFZs{}h=gnT&9s%7q>UW}JsrlvP?0S@9JAs}=rPI{W{mJXb+VKp? z6<@iXtP|`pnJT-Urh6i3k4lF>`u)K^1Pp4`N7+#_kH%Y6%@-RCN+4{fV*qf>p$@AE za$xTWWx)rqLi}fbbsHj_+^#ftKLZ?z?Armo%;BzpE8#Kp638y&vgDdo|Mpqu zxcn@_?G_F}-1Z;`w+iAc$E$?FuB-!`@2DDQeBi;F7kO$hz(n}HPTnue$1J0B_as)S z3H0GJ+5hTh4YTWkX5JdhI3Y7#s2(U~1(v6BJHJBaABY}0k+DRjpWQQaB9l+5gc(|= zKdW^s>mS$iv6~o~;~#TnbX}EvjM_z1xxk}zpKxj80n{**^2sTXmlP^?n@rH*Fw7@pc^>0p|bdC4V#~R$Rpe zDtXEPb&e*29e81tF-7;#6Jo!BTvIYFbi$XSZf5=OJIJ)ktDx-d3blr2D zs$`B+X+F;iYxK|9DFi5LrTU~pyvIs_Yeu z$5C2|JDY63UcYv2gTA#ngT_+`nhipWD?kE)E#* zt+_A>n)YQHW1LHBFaAD3%?}mxXSbJL@KIt*s1_hmLY2$M?_~4~Pc&s^q~wt09*Wuo zv;k$8YwHLK0-rPpVh-K5<^tRig&!MJxjw&!H2nvjLlEH90r}CVPQqZ z#or?h=<+d9z$XoIAS{!&pIpXSHNJNn$j9p@_Mg^g!PSdD^q!ZI$X*;MIfdChPWobT zx-1jGRLPD3cpGBtYdSrGE!)Gw^}BYUn?WfPFGCjlb?0S?YlARoHz`RIqEXQNG3lA~(vT-9`QkiB8-DJ_w zz#(z5V<7*=B6XbP#bbtFg9d;6C{moZq3{c$DT#q`gy}u;JBQ&!qNp~jtE;gIzs+r@ zi8?4M;*GPNF897Wws!5`u~dh48|}SgW4TR%E=|gFY_@8j-uk|wW|!_--XcD2NsSS| zU<-YV{g$f?UEbR!yjNpO8tfAEidPPp3iKZHSbky2)NA4!FXonpHZ2}$sV`j8&SiJB zm+OA}7n{XfLbC=pf9t#RD%+u~L4HtiED~A1z&tXI?1$rrj){p$OgmJc$B@Hw=8ddZ z^@+#c(*L3RifxMY2?XJWCUMA`Mel}j9Opm?lG=*jEO-os?&47L#_ql?{0y?Ukz*Zi zBW2CZK63T+;lNr8U42-=^4&oI3HF8IUU!Dpb8vE2f)RcB3zZZ^1nB#KNObBgGlJ}N z<+GU#|FlU=-ebqU<-mvqmql65$HYbgijN$LYBWG!>!^>24LN6qd$0$rTtTkyhPQij ziZ(Rn<7Q(9l-nV5X}^(iByW1GPQr>jFxX~6F(WYOT!UJK=-}D%R5Ifk0R5&s!^<}~ zxELn3FBY}mK6>Xc#tyFk>(zX^wV3zUYo0x)Ta}ehV73{lLi|5NT9<}(X;;J6g9j6~ z!Dd@^ro@C(2V0F%4;nZ+W@VL(Tjii5%Fj&&Dr7Wf4?hr(0wqT&iWa=@1A@aLK~KkG zs21-<>iz+VtRcu4K>%Q>d`4_E2c>SEPMarrG2>u;X*RM|6A^h^K`A(Dg~9C2xvlN( z>0q?j*3f*lZgAq#+Uyam*8u{FbU_r~XZEHj*x%3Z0lu^yJddU=-~9P;>#Ipagm&{| zQ##C(9z1xUgDPC`7iue;0>|#uf)9An{x(=x%8)KNd+~2rummB2u=JYX7XF&wHJl?t z#E>;K9QO%NL01iuhWO%k$8XayL^K1ofK~^sbPWu4K`8WPe@$)eA8)(YP{`^1VTGx zj$LreQVH{=76VmOIQ4-1Nr18;ZUed@usChd4Y^0`O2H~SuYH7P#BOIk_DLnmb1%L= z?UqgE=AMx2Y#JQ8x%pSn2{udIaO>6ygKs9&%dB0i*XHR?NEo0l5@=q!^$eAc{bCIr^uzV{1R6i%)dY);=_75Sd=_N=ixX zZlCVVtX|1A&J3I-I@1Enhale$7SG$F|BXr9s}zepU1w_7q! z7f(leG23K(%#;@O`#`d>$aY$bahyk(8BH%qO&JtE+T_c))na8=qOMc*5eYjLIWW=$ z&;*C{8}Ha!Tas6x%u5pi_jWPsTSzQ%?YL_x|3V#I0>(Ba={NnKBCLiB4;| z!q)kwo|G}s2LP@@Qb*PMLKw)n6j!dS*ZQ18>JVDP%3#J>i^7(@HPVX(*EkM6LU)w2cY@L%vpc>s7>*X~89Y(DJx4Hx_PfBI zxhFhe#`y+Sd(r0RE~N7VnwiG?_ZJgS?%+2Na?((#21n+#`J?7S`{wn*Z-9$ZLql0X zLpwrj7XmCB6iFubL1>sTiMhwg1dSf1RhAiVUzYG9(aW&`kEbI$9+kf|E8`&QG?!Lm#nG}gp)0`6d@F=+h+ zrVF&aSNSZQG(HB>Yy@N@1Nr2{q?MEOwM`PMU5aENr+RR&Q)FU1DY(z1XczfH5r3=J|GdwaQyB+_BCbD*I*0lfXC4XCjSMDR*aoEONdOgIrsA-RE1a;h z;!Kd!v{|z(06EARZXVqngyZXIT1^HAY8#o_;Z=d)#m{K}3=EU1$pe}uw^0y$K!LL5M5&aEW)5*Jurp?KIH^NLDDS4OQ%~v@G&*S#! zzpk*%z?T33!a`BuA3d!JSnXDfrrhBU=aa>(Iaq;dhbH{a&6$!85McMu;DDC`ob)AE zaU2q|v{sIQPLQbA{5Alsa(S2m_6=tq68Lu>FaQ<2IMxPc>2AX<46`(d5mMS3bl$;I z*8XW-OS8b{O*@q}j+{TfpwDH^p!4zGnnz}MR`$^N`92G{d2==}>bWepH;&{YtBG!{ zsj**tb!-7mP3KH!=0zEeiazlkc=%AuFU$u z5ivGVeCCZkjMf2@)iacqTH3yJJ5}~Nve9IB$DY`Bz~W@u?g;9k)81zR`iXhYzs8QQ z4Zikt(?mDfX7)D+Q$78nAv66v59hmW*)M|LhVPa{o=^VDX*FFaUh@25efk7Fz9QoT zMfk9=*Q#ka&g3rp^uNEWo~1Ek;7ah>$72zSxAg(KccP-de zPQ_0vRXt*ijTnu`xbM#A0AUu>aSb@A4TKz(R^>8yvc=jw5 z>w86}Zh{{5X%HauPXzD4wu;2sj%4xhxc>LyoqlPuKYsoDOENisIAxIMMnSLJ>2eYb zFLYxTqkYkQEk-|4LQJgP8@AMXog;6Gi=*I1`4-^kAD_%^;IT#l@ebKIlQ}!}|Gcck zqvMfSM?t=?&%J#0suMf@{%`j1;iX|L@J0%**b}RVFne!Fe8n*$-uZ=ur~~9xVKrg^ z>%`(Yp4-K#wL~zSiO9_5_|8mk#==5|Hoq|i##{`Qeggdt~|P%DZXD(!>%<@`URKD~>wFL>k3 zXc;rbC8~2V3$x_bOaoaE@4>s_X)FT`1DUeDkQMw-VonQUJ|${Vk1|uDiqCwq9=SuM z3=gBE4L1c9n9b6pJWOC0U9!Z@1$A(nCVCy^;WPy;%Th#jE~HbU8`;rwF&k*!I@Vij z$|D*z+LipM^zD(pI=DdpJ1L#q98e#MSZQL(o51C_ft80^G!$;1cgM50YK$bJ5$@wX z63iOSmM0r^$~QO0p`tkh(y(EdnRw&6AX8FtQGN+n^e_l|7H`0FTqPwVQ_*k=%__76 z0Nw<}G};1_nm_2O&B&ffCr8qd6W%Gma8Wfj5jzr2jXBQK*Dcw9$JAZv{DDhirzyYkvK zfagJDdp~Rh6U^*^>H|hLIEIe_+}*oH5g;IfsK7o@5G=HDz+_d+$^DhOd=vdMcy5J0 z|NVIeM0~Oi3TY*;59uOl*}a=_)a_?x!5e%v`w0<`umYSG7^cf-ORo%oUwP;=Yda5R8`g%iRR*A$KPy3($E z^Uihasy8}c1P>0$4s}9c$WbdL|H%*YBN(9vjp1eyqfKY&1*Rd4!lGPrt1?lt=0 zT`h-vL_mNJnI1M9QPL0Ec=mUPxvsCSgdM!vC$=SCsTjKfXXw{WMA z2*40;)n4`zFZ%j85hde*$Lc%oe2&5xuQ7m%a(e5uCkW!|^yt0@Quz$vWx>FJOps_9 zdeYE4@4~lY(K`&s?zlD&5F0 zXOS8;ZV+}@<0s+)X?7Y|F@<%RU?qhhVYhj7ALmUsdN65iU^h%gB-igi>|Nh+aT@^s zH(^d0SP&~-e=Rl`HN<&s zh&^g(VG&!=INJ_7yCj(PbN1|P+22x$)!A%>n}O|8<-w+7$S0X=2qNjw`gMMi75Ru( zqd07K8it-4MS94_5lb}UW{gi7Lvu$@tPE8#99kcpP6GIlidI}7)+l2h*G0lvgNl;-N+&Ctmn)aq8MDf@G%)sdiWa!xFxktGN{Wgi;fl3DRyGB85>rn_>nBQv)uW=;J$W+abQ11+?XSaEemQP0 zw`Ot`4zHfUSc+zGdSg(`{*|yN4M8p22}U4aMh=1K5a6pI?=kx&pT=0O**>^%_Nk+) z!w`aU^Or(hxqlju(IhtsT$Vps0$^?E{^gyF^Qs?Gcmey28I1U=SMBSo&pbD^?wP>6 z!rSFeZ&o^#9ehsLV`w5HozG7Lm&kP+L3sYhXO6n5`_zn{*cU~F+e~t^2x-O4~4qJRFhwTp{c1{*xs3Y3O% zGVk4F?w7v&Gm8}AO#bBeJ0sp_R) zW!gY|shfz6p(QEtqtcOyU-aHm4#XfG1oq1;s5VfA6qc9Y#KK8n$3`)|>)DKUb#-+K z*smaSgAwI*$d81wT+ue7VPl*hBYBz9*>=!8SW~tb@1JBnYZ~)lH}nEgh{4_nN!X`4 zm=1y}{}zUT6PSWcv>hX(mWqkR?aV*yscmSu;A6IPAWJ6t7)mP4%gd{M{I~=`TBzhO zNhbuYfCTIsP-_y_mr34#whj4Wp2&BGZ~Fgv>(y5|7;z5-AE>C`A&h#s7orIXC{+S? zZxf7<8yk&}=7_5Lls(4`amX-<4TE(}dGHEvyFV;mp##xx{*5)TTXFiD5O?WOVh_?%_BiZgUe<%H z`W8fnmD^P!^yEgELt0u&bmeH@=<4asL#&GUoP~qhH#^%!L-!#+=2=|-z7Ev6hbURX z3IIB8CAP#3IaGw8Y^Q%elL7Rjhwkgk`pv`L>14l0yBPs+`YEC|A_aNtc%lLP4_KYA z6zl?tVO>YVl>!JUXMXL2DbD(i22Lhv-Xtc`FDR%G$EC7fkFe_8r|9Hl>9(DLJd&aV z**!ytzW`Q2Df8kUKJ*)(WB89H5>hZ;YeL9Qd_0h4J7BH`sNrVGy0FztPfcwu!~N5B zV!t5CP6Jiew#FI5Zn{zYr)9nwhZ~We*bJ5+2}$+zu*dBam=;m5aNLD9iD_=?=(<>M=^%LHhXM)p#VtPweJ?CDj9GP!BH zJPkWDvKS?_&KkO$936{-Jbu7nO4rd*zUB#v-Mz-<5&DSPn>wJ+V>QuVw~GIjvAiTTrf@5ng0EJgOH z*(OgJ1f6!{G3^nZmS>oJc$xGThibHddN&LUu!vb{>94rtVTA#}11X>a{!+W7tB)_w zGT7<(oBp`vq%~P zC9-8)ER!oWIy!s`q=%J%&zt`J{bO(A!2MVvd!W0zD)b$FKFEG|g!=f*{e>_)qhQ*M z8K&^I4~(%!$G;oxxp`#?P>0deQ%L>Q`>_Pa{$*1zyj78-fit~<41k-Rq`^!?kh#5a zL`}K={N(Erd@<^~cCjYlG|xR*gO0sSgt-M=>(*exDYxr4aL}W8#6ZeN89BAUvH9|R z15Ahj*&DHn2#cpRf%lu@3V3Z0PH9ls=}@%9Q*jO!R<2{>Kl-s23}caw{!Qyo1d>Eyed^OAnrksov3P z`SW>m*?Fe=MZ@qZEzX$i@U|^mB;Y`0vYB>ihK`2j+tWulXKx4TIo?I;@R-*Z9?a7N zzOR&giD&KV<^5ueTV282l%%{L(=Z_rh@KLkn&JyxC)cQ?k2ctD?@X}u$;F(*-OA}(icZ!`=E8pXnkx@_?F*^QxT=Sa$3*0T4!N!Dt zGOQPzt9F0Ao`%QBG={l1_xrbzcm<6M7lQjPnp-ndr zyw`nM@5VN}JV(zTL}QWdDvXIG2wgF7xWntF9NhSGV8DOrvQNC++*<*`Axf*LeEq8k zSPwK62|=0|tOuLU=AN@atF?f2Uw+i}A$1wT!rs44C{R&5E)y(3`lfKETm0D~Mkxaf z-DT*o`Qi|i1w$7{2%AgF7AsBkl`W{@SVls)gITsQI;1U2L%;ZR0V>GDNNoZ#tnn3r z;SKUo3kA7gT$ht+HWIkqJ|evan9z1B!@Bw;#Cl0U?HUGpYwjSURd= zyLO^so06$WOOQ&C`HKj-QDkb@ywZ{bn4lUE8W&;#d1+b zmDmWFlUmIB3%Y>dq6J>EHxB(+pX=AJclV@Q9Kvus0-O>XL1Rp|-w~$sDNZy(Il#6U zt-mv9X-IMJWsOKLiORR6biu1hYb;8aM15^qFC4{Uk}_F{Hf}&ulzNpZ8!)FP!Txff zAbK}SNU1+tHmxb&v>n~m>V;wu-03G7yx=V;D2R(fl*qsGKbVY`aOgNwbW+g&4n2|L z#Kwd3-PNV*U71@vLn8Rc-w-Mx-0HrdVw=ay^6u>sobfNYhq?s|tk5I2|JkCNlH~zl zLm7F8Z?+4dm$j&Ce64PHLKN`4c%u+;he6%T-5a;XW#}!2ryr`V&qN0gy}?D?dlh%) zI=dkGc!9?%!N{5|P-D8_&`$A#lej7{8?DV4Z=<#n~3lcV)^5raUatFpk zu68J9Ot|*G;hN2u>_)`e=CI^pYU*JyM=BR~A$LBCFx!v{nX7ru0X-Bq#4RtQq02OV z*Z(sWV-T{OEjO{EtfWDeWyCNwl>(-^k*M|S zK&CRZ)470jF>-L)48E*Dd}TLg7^LUw`uzLz1h7lmG(+lj_iWBXiI7!~5WD#dXj5u7 zM@&t}=d}nzj=T;Tc(%Mp!}(k828e9^`Gy$H^j!ux{&N{i7b{M2TuWKUI%6e zYZo_@Q6Xe+^)yItnrBNRqPV+H4ckC=UbI;qsSvx&90zML&$)zuw_%6`)Dkj-XBqH+XOP z&yG;w`)kl@Xx>0$Vxn(usVpKt7@R@P252rQ*kIq7e3!99wYoeZM-C#0Yg^kjfW>!1 zNWRWth|$_kJ7hP^VwBG=Wb^s`0|LHo(ns1W;H9!b|M>BQ6Amv|tXzq0F8S;uF5#ci z`&E{b8G%>#C&!{PN0{_X{^sKJID1q7zywp-yN3jZwiwREjX~G@PS!hSiEtEFC#vA& ztaqy=w*LwQx=58rVg6(M#id}e2yWt zrksG+iFOUfM1i8>gNd`-8Xu_G&fHdgNv;Md;)|p}VVA@5S=Hv)91Z5^@HXTSwZgSyT zl0QODS*5UpuTZR$H3?eUM|<@m0>1b4MH9CP6DqTGC!7@#6+#|hIfJMJ1{18dMCyOe z_5T0pdhfU#`|yAG%Fev0h-9XLC`pT~LP=UmgV3U(B_(l&LP;o6DJ5Fk(LzJgP$DgD z3GGCSQvKe?<-Wh)-}5}L$6xnNU0t2mc^;qRc&|gUdgLh1R#@u?wTKBGKok8-`?BHs_SkxxO$nsXV@3Q^C9ybBL&e{l%n9HyU$E;kIP z(LFi-#0|jRBGIRPNVjbfyLn(DzFSYCf{&HsLYG$XXe^ifs{7K)KE{KCMF$ob_@WRXR{+}M)+*85^)99E*OI+pX(Uc{5YG*mR|p&cLIo}o8eY~;OID&jvQ({djnHtXC4HAel}QS<;bSY>v)a2 zfNkW;VBScA@aK)6Z@5~t7Ric8#B>xzGS?wbErzH>yNd)#TaYxyt zrzcjLlmPK3_Aq;{Dt6tY(;NWN%K9vTnQeI@qM{}+MBK3n!nu%~o4X?zaTK~Gh+mS% zk4ZcAIZA8w;MmCb83h4S)=S1_&r~~nxD>8|L z-eTujtQFL4BspLYHF|esXogGM=ksXwh^sjKtmsVO;`IGutOV23n)U0=Aj+mV%~%wI z+@)(1E4*nn2yKOe#2^G!t~kpu^%QA-m_{kk71d#b`^-}U$rm1Dg#rw@Yv=Wch(`mX zRLXDmtGD^Ul8_1rF@C@^r(?Yssv7da^cBa&8@IlRiv6&4Us_->C|0dQgg zPXRPE)N2K=Y#Tz$X4lkU4zWILTFdNr<=X?UR^GqARWA*bY6UF@eFp^tPS-2xpQYvG z$Xc3VzY)%zG-2S07KaQC4Lcg1rc|o8_hLM!4?gH@UvquQ?-ffY-!nvp6-BrHu4lTqO}Z?$TNzE#-Ot*4_evu z>g%4KE9B%xmxi-JMqXhS=1&^i43w>t8`Vq7Q{41h@~k426v2joAwDeG8;IWII% zbGUv8mOmGWe(&1Zi%qyGPeXdFYzuU}20cn4e*x+%j50Md8PU`wK)YJ+|8wW+M2#3%67!TYX$`8uA~sl zVhH<}r|la9M{C%5oeQDkczE+O_n(3A5#qyR;nEO@A$a;sy$qQ`@S!ko^d16fC$U{m zD(0Ld*j*H@W1&0~1AT)VEPMQXxZIaT6n)}0oC7C7e?C$=TCfO8 z?DVf>qiMfxk0zGAawq7pQ}bj$A0G`7RWR-MkXQHqIw$xSlVA68m!pv#ZQVNF%$(E* zXa5nmOMiati!%~LOL?at$-k!O_AFNAD;zIc1k}OYad9Mc?1+cAcl;?kd2~6=(m3tu zAu~14G||UK8HyE*{|yv?NFNfgKbUiEjvo|>Phej4l*TEH1&ceS$y`JBk+oJ}8Rjyw zB1|HrXPyA|-uQ62rs!jJac>jAbAFoXRfD#U{kjBmJ1WnUck=ot5`%|kCyyTnf&URj z#3WiBh4!AbZJ3QJ#yd-J2eL_^%Sy2K$;~LUuT(C%fuj$-=I}} z+*@WYmcBX4;FFR~LdFrACdrtUqTGy%x*(z)Gu=2kpBV%$CM2H5 z#hx8KW$Qt{MMChu_)*RpUr;$z2eDlAAWSJBxA)Wa1W33*xqRxg z3r58$Y2D51mAbmRW_ohXpC7?haC1G^(zOh_Wgc5Ne`m_eV@te(-;n6?%8>)Qp+PaLeAFtJjZoV9mS(&n(fJj@%tBA~SETKLn!EkQTt{gg1Bt&D8X zoQc%>&>!7{eXOhEQ#-yKdpj`IwRRRR$FdoFrC&}I;~th`v6fmi8m_htZDX39_3cMF zLQm73R=C4M$cdGIacmY#`BC158Ad(p?^MZ$f8A+ZA2MY%MRLwzSI%w!sP=Mf^N2kc zdgw&Ark=b)e&&X~9^HsIXW_@NSQl+B&S5FLT{vB?aV%5(gdD44KD{kn;UAOcUb}#E_@{d98Q+w50pV`49Ij{RZ9XO}5)U$W zV?5w7GHeGI96mR2pjjF?1_84 zb2Ix+lLpcq4tWI72Hx6MQn}-!zwE_|ZDPjymNr}AgFTMLl(IKKqkAto+a=z75VIqB z@7p&IzFCtnjbMZaaE^0Y7ra8a(0yj?Omp`|=HkUDygS(&fN9n7FuNNO5oT%m<#*f9 z+!^aBh@kg0%6-Lerr1w38Wd2jEnYuRHA zN3X}NtgL9&e6B4#>nD)0m|ORw7w-Xa|9Vc(5xxs$$X%w?=-uYNnP|T_3pRZDGoR(< z#=+X0i8kz_hZuTz$I`q6pqp=Y~wpnwmB6QfG7`;HHH9 z&r(&yPeZWk7z`*YpJpGcr7W#|$4Vx|FP+SKLX-KSESCU_hINOSXKEnI2jq1iv_>cW z`L720fJCST8i4w(T)YTth1|0!=+=)vu#17DO);)g5MFmOCKth_JzD&dk7SoglQIn($6G zwn7|r0&}Cmh>e+9tFn$~(Q@dZF4fOS${c`6RbU%;6!dAo|e79w2F&>pGXh{B% zG4jmX$EK>MCXbj`6XMEn{(_a|lMus5Bq*^@*))Qs) zp1vP3fHD(6iUGl3DLwes!h(X`-@Ib= zv9ct1%UM5*1_(0BFbas+?>b@*IQ$W2X#2S50EaI1{4+H_Qagr{A81NN~!G|C6 z{7#4hUdKAu>Y9f~(p7simeWR@AHsQHFEgyy{OL^wBbGi#C0MqyzJ200X4X;2`X?Wq(r8fezFS4xr_TE2BN2H) zu?t69_pY*!P8P{{{M2>-c*9G4}uyKg%!mU7Hayxrov>dQ3NPn?ggJ1V{$@tU*a zvzj%r4^f;hyb}{ulo|Ix)v@5MRFX+B|eYR(1`+t5OP9iS7! z*mB9@o?yS|zwefOkT)lfrM6{NPqw(({J%|r+O(1rByr{C8iu~I+lb*Dv4*HLM-NZ( zw&j22umAlUr_Q?nVJfB{WIthTfftY&z=eEA6u*B^Vxn9YoR!cQS$=2 zE&;G^mx94Hu*LU~BoFCTm`1ceW-iA%+$m6z`A9|r1N;V6d$2E@5`4%jYCL#j>zFi3 zv78^i@VRZv<=Aev%2R%h-GoE`W}Y7Sein;4*5>fS zAvBMvH3Wm*ew||Fv4y6XpN37EjPl}AmOw%a8<4xbPLQ0=JPI)HdUuryM!c1oC?wmbv;sX%;9y+_o~_0XcI;nep?C zjg8gNS7uZ>z|ZQ&xJ?R>SOM$oLw|qXcnAFyb43_~E$+NS-`{c@hAJKl%y{UeN>fhm zgKxkMk8AZaa<q2eom^q3Wk=8%- zJQW4DRqi(Y6`IMC;N&8hdg=bPt;PyBW=a>DvAdFyzkWGYLCf7EclxrsyKYHvB5t&^ z-6_@SomOUM1@hRRVjpQ$21+Im#X8+W9u^k%(UqzMg~BE%hkF)fq^DEPLSbMzAb}c4 za|<$;PMbFE2s)Z_K4`Q}NJl;LOU|@}7t4K@S&YQilrq6`y0p9{b1E842$omBE_(ac zWPN7;Jjd+*pFfX*<g}NPS@x@{ASP`N0z>!_OyP>Xs`(r7`!uH_ z87vu`T%er8wT(!0{stehK_3R`dvXrGl+kK)9gz7)3QONBk6XhCv0J$^lF&cjj63vP zTc`?ZXOz>hv7lsG5O6 z(9DIS<*l&eIW~7AaX|qvvd{9BEyJ)ox}`zH7Xj@@;mzv~2K zLOrB)YW`rZ@7jrxXMs0VPy0~y?v)y4kB`YA=6hPfFn*ILXpcXSW59aKjGu2Nhd?A* zq^zg1-dudSmSMU7!|&o;ar$!p<9+5DD=QY5^Yxz*o8P~x*rcS7+delyvX%+;uV<;v zWA`@LWsWG=T~5ytG^`GE(_zWTr%Q7R^LKT<)Atb|eyX=qe>EC?+7Q%x8bdVGk14zG z=B^biTCWt9Tfo&I9P$4@WV+%a(TiIq~5r zq(j=IFn<_K+;!MO+|vk0n^_}V4Lr_0?FaI(K?4=toCP)Peg&V0SY=%5Hr z$xWImlg1GSJduuO_cUzSTMIA}G6r-5rtj{HE~a|C5keS<4Ke-zYs*bSAh0UtAfE9d zygKcDmsZ7nEFvqb=Oo8cSL>H_3Ud?_CjPjgG-98fYB;U*aOm5c;% z`((jkrwekCV1bk1LoW*SO*)>UH;D7ZerPW-ppZXN-KI)Hy0@W;&@&r@^Q%8`PYZHa zeFj;5sIOf~d1cAe#R@18!;zlX(|~&3Ln8M=$g%GBYjfjF)VVb18*Y-~coOQTtX{BK<_f+j2up;UFC zZr^#gRPMAFM7ZyGun^&-tj-tLbm&zy5QA6!3?M;7PR_Q!tOVGd3}PiT5ia%EMIM9% zgJPO-48g0nFyQpd-@S835zXBfPdJbHDLke687tKCKTHvQ)3pmr?~~n_{y>sQ?C0M< z)J$>YtZYCY4z+*C`x8*oNGXQma~qtf&PD{u*V_V+i4G0T$%~UDY6G?`#y;gO+GP5m z9kX~-^!DMvm2nJ);CfHEM*CY4&_#6K+dn%$`mEk040iIKRPs(k6VN3iavhX3RNpq0 zMR*Oe=0UmXY=ljZsW6_JyuEUjK8kuEDl!K$tU;PElw{m9 zbP{L_h>Ud>bpWcO5TaK<&2lJ6gFUN(S%Wnl8tOs zA>vN{dJ54B%`Ui}1qeBRz$JuxpT-`H_wk2{aXSJ=M@lTMzIC^+#kLM`h-hIDeK`N0 ze?Y@Rt!`4pTuEYj1l9)Lpcx9Ev845yyNdL zZ14~PB#G#Qb$=37i3cfHhV~)j%xxpOc7A8glSrFbSO7Pq=pC9%x@g3rZ-4t*i2ib+v4K&C0xO6%Bi5mgioKT)BbHzt#6 z>#OlvfYD;5>t4P2R(#dTCr6CZq6u#7ZmYjr~5B6WZ;xL_)xC9@A;X`8|9dZ_N6Dp95dLk~OcZcd)EOrn@0PQ`XlwocJ_y{cQU-Gf8&iBfdE2T#0D2Ul!+}Ui;2lv-QeiWh!;w;zQ z2NCw#2g|-fUdpfEzmxD7X?tQZZ!Ws3QvL$eU#K&Yuy7d>c9ZOFY(AYd#0J!ZyLRmg zmqtZ8O5#dyDmJvdD^rG;VY%6JlPG^Dqc{&{#O1d&Q7KH`DV?DCiF|j5R)UzJ4HTJ4 zI1l_mNjklEC z5Cdj$Z59f!1Tv?3g3u6(TQR`To z-V$I-dJf`|Hbrl7=W`I4aj-P5a^5j9%$l=^nm36$Mehe!0L*+yO{KSbb1SejmkQQQ-iHlyPy*7N{^tmJEj)bGLT zYYXgOx5NGmK}+_VAn2c9^h~XFu0zW~adz!>nnny(Sjf);6MOY)44oXeB$tNMs)M%7 zp~jVA9i7CX!CI8Cp$bxveVAel25=63r&BID?M9lRvX27t+HZHi37-m6KpTR0mgBpu zV1Gd06t#g8!%6oL(}<*+Fm&ZbwVPIikq!}OPI^bC9rINghobA^iUa4&WydUo;TH(&7y;)hG>R*g+-j_Xf5u?CHvW7DfD-fC zr_mXjn=&Aq0oE%#H1>hJpaeOI&RAXK5jA{%At(_@d*e4$k*A8Gbeq~aE|}CP{w&iL zMpb2Kom)H+<~mg`+1TS&%!L1}zwAyYk@x>AFlGNk&apx{&dIE4atM8W{bE7o=wLRq z!UszCo2UTSc|fGXTSc)`ZE8r`&Tf=O3xXqsv5c2d0^3e8oY04gizgk+deHHc7R*B@ zTy)=e9zpx(M+bD`W#mZT0nnzLkH&DITe}oo>`iHVsG!9|pFe1@=<{FkI7=1Y&VED} zLd^=_m6n!P-huBj_TO%1)6Yd7Q3M`dyU|gYh~k)I@k?cZ!v&)Q%dK2Lb=D$<{aXfZ zcgZX)#uH2L%3xqNLeUX}uCtKh4-r;ZbRd#U72qHCebjIb{0*xWu|^77qhrY!9w_b+ zsWI>>Q1qPvlTAb_+gJ|w%)Su$;5FU?N;fu};+LxCW`G^?m|+KC1s0!XyfHe3$Te~z z+f_!R3)^V&=JskM;tU>XM5lLbgj++;@$NpB;!##8FMbhIxEz|ZR_;Zm`3P#qSdH22#U(5{$-%)!7|E4cfN&>eg$En*-KXUnI9ZVQvcb)6fA z!Y;rh6?|(jNwt-#01NW*KuQ$|KojWOTH|Q}2~Bs62Qd(FJ=50$z=nvQSngH?^T(pA z+u;~IjAz6oj_zf*oqT>e7=xP988o061f3L0Igh7ae zzrhDFgPPC@oJ7>4qZ-#XC2UtL#2~k-J{vI;HNe_TirRrn@(fSteK~LOVx|O(YofSf zz`^I*g2Jr9YU)#;WA)R#D-#S2rNJBXB2U<7t-}W3U)*SH<(k1@6!IY-qb%b2r-J?MNftV61?Qt?iI@{nOZkcKOuz#x21Gq|j-WW+1c% ze!l(%|BdA24HW;=dm2qeKS1!}o3}PlvI$91($E{rLi(-WnuIzsqd+~Tl6=z5-D~eW zI46(B8_-xp-$zvZ==Ai1(p6AKof->4h6l+Ey|HgY70X4z|EHrl2K+kR+7TvX1nm1U zK+5RFpk!|D2K`G>sIoVRokp61n-Ph)a4`0sW<|g_suTULuMlYjP^ZHJ@(j6NhqBKf za(WK;05X)kOVPpdb9rmx18DaTz$A@(IX8(QIba!RiV(CC27Ti+>&Wl*g2)L2lkdhD zEwnp7gOyKwt-F($+4beTM!QP#q|HAp(M(Kr)+hNW@SqRxuy)pq-0joVedly3Z?;t* zX1S$Q)&)Lj>=pYnnmcavJ?mjudgJ(otrI7FxHesQ!o`b=W_a?Sy(W6hW7X;F;)`yK z8o`&k(R`lo@jQixg5rXHQR+rg%wXfOmzQVBg>IaDd-d0`vBLe?@m+5+i)YHoy)bO; z`jIHt_bbcsJNLTh60Fds&XL_+>p8Z?AUH8}-mgb$WjmOAbNZ!T%&uF{ay_5$eQ8qm zBvu?&Pq-?7a(i(6LH66JV;g2Byc(Rr3JnPg0Y9xKp@NuDx2p3;bc7 z?C|kC2dm$=4{K{!=Cldz&ih))Srzge4^qaeHvG>INH?9&;$Gkn;EvOvJe2+N!+eLY zbT44J&OxUedVu{;L1NsZ{b|2*3@`tE-Csu-PL>{R54xk)xxOp^o5qMFABOWog74wx z`5Y0MYNcp(=8K@t$kWFCEOX?*ireMZzBhdHoI|It>CJ9J2k!i?o-O-wXlkgV2yf$$VE4TkKh|v~Z zBy}>#CKxxp2F`Ey>Xsj1gEbK~?Ox3XFF699g>UYWhiB-i(~o1#1GMNO#O0?)sXc(* zM|c8fRHNaXOAR`cFjeh#|MfFV^6%I69*3z3y^xwyCUH=}wIllKAvkmE1Z2`Gmj>PN z99X#zr&l4rItuFqayxk_wHsgP*XvBqB@B%nn}@*ise^5Bw^(R@aw2O3&Z1{ChzBTM8VE{12S zC|sAcOtL-z(z=6K0TLEa(T2(15=zwimSmK#wK%S<<49{TgxRwTG?rq$UisPKei2Zb z>l~MoxDkL|XB>97l_DS@!-B%f7|5mN41+N;lFF%PC?3yn7Z^p)K{7sLjuvvPyF8o! z4GQz4xBZCoY=zIN6T#*u(ai51!ycD|N3DRZxS37RM^mP zs98Y)%B%kVcFlN`y?0hCVqS7Kq<{qs`nHu!QexsVSlW8L@@Z`tk)S44mX`3}>o+TO z&!}(GZSbj1-!I6{1`xf!VsgEDh94e7`fq$g)U}2{0^GZw_MP}1HKKk&_%EW zyl;}fhAGGyCUk?8H+R?Xrwnnu`P*ipbrxQ}{DOt<)TvXkcT`JyF8s4A@H6a1JD~x> zp5_lDfk6c&cv#ip|JLCL;Em?|^BA*F6ww!?=kZHPNm&Kh`Vy1Sgd+yEKHVIEH?3{1 zPG&9JiX!zduIE~lBl*9|Y0KMU4v4^kAXK6!;Vp(nzCX%K=h`xg?pGUsNh0W6pyx&e z?kHctIYZ+X=Aazw4fiJ|Cp)+3MB)b{x#~)c-}C3sUnba}SZL%m!dL_=STDJ2N9%Rr z&51Xm8~xd~*eF8!68d%Z^%oRXU;iqQY)$DtPIsem)GXSqt`+L6G=Zs$jkZ& zWd2yQ+8q1UcEeU=Q0yn7c^L_X+^Pz2NxV82`N8h%n_KSgzyFLLxL#0#`yFW4XT4Mg zl-iVR^VO|c?HCa24KED#_KN*-h{qAvlkl>xwssMI!CNjPf|6KaC62q>hOV)mfYb%G zmaKN%!}W#|{r&xq=B?`O+~`{5|MTJ@R_GeAsI<$rGuKxk4-`@H~H$HsQUD9;M>C0!~#W+ZX0||1Y#o2Pc7vMv%jI#@h--Xaa7=&@FGZbK9Nzemy3c$!i)-TTR((yZz(&&ZkR|BoQjpI4y0 zAfeufLd)xiD2|kCAe(|#uW;?!wQe1#!RUZPHCr2*%(Jm@mgNQ@JK4JT?yeNb>yIJC zgXg3^!f0W34pWVT8{r9#km=RKm~^gHD!s>xxsMcrnjtfJ68%a%rL^6e64Vuc(xye! zv%}Jw7)we%tKP$)^6h?}(?Ajkz+hPrua;QQnd<-|=k+nh$jHGQt+r-5dnXI}HqYcv zFFPc^YhzJK^WZ4rFPw#Df3ZOW{;j8HUC;^iSe{>Qm7^if_@4N?qXkp)QCQABjcIzo zA;eRPG06oxoClLtckP-&(faP46$yr0o4$O<6sL>w;KnZfvc7@)?=zc^0_x|9FUOtd z;%iAr4bGh06xg*N7LEwgSYAaA-c)GEFQHcz@u7H|dal?6dF)va`?z!hR{y<*-ccy% z1jc61FC4BH*}DWV@T_xTo%=BjNr}-1u=nL*hm*5&M}=;>%>#7@sA@0?iN5J!xDaMr zFuLGl^-_S}<P%udlO|q79E>K(#m}m@V!V9>lziesR2rJ}B8|63|Jb5tB$yHPfU&Z(%GQg>w<%PD|D1hQZryTV>V)I_1dia9^}vHI`)TKG-Y(c4A!R3fwY!s4c|GS3*SSRfSpQ1xsm-zh{&%wv}`1PJv zjq6#=Ve!&xnbpf%(I4A{>WpySuZmy(;J>&05@%J(g7^RvJ-s@^&%b8;8%O`;MFhv} z{>iT-1xl{?j9PU`tF>@u$=r;|PGUwUaBq?98mdSgeJpwdknvCTSHGsP#MzxR;74pS zUT`?UAj1yXhS7ZnHa3asQz>h(^G{cE^L83F9pHs1MCyQ3W|&r(0_S7hClS5Q*n~j^ zA-EM0Ges|7Zcj&wNlIaE*sre3xDnU9@D2}Og@#PzCi87;!i_UaaA#qiNYu7PD^P2( z;3Kft8GVY@gB_D6&ntp4tuRV_>Egw)efC(~6on=u*8z#@q!5yQM{{brH_mhQ@0#i} zrcGN0u(5f77CoZ|Ocz*zRj0BzMrS0DVRrI|!WcyG(RSA6fz{fvd$9knbh^NYt>4bP z04t5`qrRWmK>H%g{vMub#NA}t#L;gXR0kWZ2IAIj_)*1I>%<3@+QVzN>-rgc-n+O8 zSbeNd zA&LY>w9dQ!sz7Pm6_8uD_Iw90k(i#IzT#w3I~HATKW;GCQMtElGkQX{?Y^ZBu>d4b zEmOhVV1$km{MSk}V1tPMTaA;F0G=Lez{sewcnjoTMr#|-g;g)`*WF4`J$VuwVXCMj zqkr&VB816n8uldzY(BFATj9T)G=!1}id6NDEs-7->TdA~z13-jWHlB@X?PK|PKPzP*B9BpjIK{CJA5;M%8y`z)0uFZf-jDjVo=#7q3bAFixSN@hNiTGB^oJJnhV3C%JU))D`_gW|(~gk2YYM5GtUvkRRM z8=#kogjivr%Q?8m4N$|wAmpk1)CuHhDp04W99abgtb+&zAY~^SGli5cl;!`RWh|h2hiI1<0 z=?1Dk0uTV-ez`yHaE6^bt%-`aqC7m>m~<%tt1@rh-+U(b&IyE$Oxb!JyUezegz2LiB-V#~=V34ff)w*51s;YNLfH|Nar$Ia*c-~ z;u;pzmZn%NpYj3wdteW9)RfT1J9kMQ+bhew*B zF`&rkcvVWdoOt4`WRDZYdtXZS%2 zd*+sbQ7;i1_z>(w>!5#IwAm>!*W)VY!hJ}>!RsTsiVfc-%@!2QB#(R&d?15m7N)~ zh>udHjeZPzvmK7tV{kr%S(QJ7o`=S9bo+D;Np-D0$v0Wg9bE?}XS8^^IKq zk$BRRr}M0~xv>fg1X9)9lihT@9;!=!#77UMpK@?8X>#ei9u`1g;`NxP zG8Kr_Bb=dv(RS&eCY%7oz}p1F;hBsvWX*?EG}SZqEzV+`q@|@3Q6{t_ zwV^1@*k5?~9Cn3lzm44 zVD6H=zAzRzRE2XfF5Q*RdT7O)X*lFJrS@Cd&@{mPcvK1 z9D249DN*&svH&Y%an9#DU^8|N>Nod^I(*;(m{n~A<>a2b_;zEG!5Z*)*LI_l^T1=s zJ+m56oChY=czvzx9>7EWn2T}d0b%_Bjh{Pv1rlPTR^PlT3{h~WUr_=gmqGz{-QC*y z^{cw2Wh`S8jPIxQ$i2>+?{8&P!4T$lwx125VHsZMbZsrI+r6^eD6bDta&9LSF6yeP z-V;2oLTBWVzc~Sr#pDmJGL+JcskV6lk%j5mpQkSan^qiU^bPl(muy<|(Jue}%E}u$ z5HW{$CL$?$z$p`?*c;}}8|U8=MqW$gU4#t^KHBpw$?v&`49dRD{rO97*p^HZ<7anp zEidA4$@>Tt`711OKXC&%-KDW9GAFDybG(?|!k6Q6;@pVj!0#K;7Z~eb zkTb_H>)QNbzAxBDq<8tptRq>8qZ3@U*+YpQh+DQg-n6A3>`x>RVeog|(QM6_tw%UV~W0msj_+iCcaYqZQbD=(8F` z&;c{u4EpIQ^uP<0N6a2l0@Z+9aF;+| zb^%XwTLlSq`jG!WH{Jp4gCefhJs(9z!fHI?HNwKl&I|&$QbjhB|=kVGc*KrnCD)hGg!b2ch(&Wlq*NM(nacOSU}EZ zJj%&14c*?_oEgZcC4EB;is zx^;ean^7AWrN=GafwWa5l;mCk;$-|=N@c|}A10xMkI-;|HzycbDRup?) zqR@zdbFgb_7lX7z`L^0%;!W^bble}#0wFLx;#L!hu~BsminW0=K@UNl!zmnS>^z24 zj9d`LJQ_-YwE*%#Ssi{f`ZNtW?$R{!D#PcRahGArv)Ung1#DPwAoAF2&o4MxhEhOL z%7TQ{;l~UEhsCXz)$0$x*n{_sy9!(a8)4Pc(3nR&Sqx<}RK_2=XHk>ug#EcVC9YZM z;^=#<5$^f2`UZIQ(Q&K{{rE}e$@ltC~qJg^$t zJiAIx>@)+X*)v{UfuobQZyOXp!~Sd(0sM|E#T{+a^;w}&xYWrcOf_^$Msv}HL4b{h zO{WQE$X#+|pfmF0A&bLE)Y^!py`Os`p-)T1)lih6Pzew%c7}|B1HgG%M^24glK!n@ zjic!m3A`enkIVOZiyr|ssg~-SeGMSpFGUNUQAdiL!7oKzngvs?%bjZb2cjU{`#5LN zj%_K{K~OY}SeKux{=3bIQ_nI8@+i)3_*-iqrIq4gJ&0Z_qjTn>jeD5y{B|K*nNpY^ z3xg40>49J^ed`Ene9{`7uZ{eo#xrw4x2m%8F#uIxOMwSbWSe%nR0Io`((+^C(RwS2 zIDKx@e6djvZ+7mYiO_+6&~L@(x)u}VbF$Ib;4B}g!oq5IwFpRJK{oW%%OGZ&+1Vqz z^j!a~ntG3oc;wmU-VjtR54nFP9G^so716o%jXihctFH77 z+W$;4&0_&y>@Y8AKioB|d2rwT&AQ+7I9wN#6=%!K;SM8APqL01&^#O=9xlEd`PhU$ zaSgIUFoPXPx55%?_6!}3?2uB#!LKo}05jBu2~{=)r-P6|B*uPAAat@0r1UVaju$ns zK-f+GyKYHz1eh>N{St1CZMTLHy|L@~iq6*$X(ts7TCyo<;a{P1akId~trE%J zD+&R$TPIBKE1TS^(Aj|}Lsk|#?T(7yAkcI%b-7!xq2~Pj)ju1HzCwO%38Po86%?x!Hc@Sm0&&fwh{bW&g%eI2yIX{|YRkulT!$SKHKf^+94|D6IE%kb%pg^Vn zmP)=#>_2nYsE4_&IKyj%z^%q871*IXO?-->Ik~>%?y&qFj%8Os%P-ur&M?#A9PG_y z4)VXhaN&|IMICrp+uGR~quI*PYL{mrhsU+azt<`v%u zF~^#pS#W%OY#X}|vQJgiNbsD;ReZ4R6~8gNc=be9kK16qw~1CTp4HBZ03d{Aqo;t` zDmKbxC@3iKHZ)cgpJY^A;b*rL8Z+u9(7U);2al1l# z9mkhNcX^wJqmx4VO?+aq7S#$(4eyJJ%TD>3weZ+F8IEn|+3=K-j|%^GZD3;l-ZDW^ z$6j3W<1ux)2eSFiho58W*IPyg#;WYbCvN+}W|W0XIBzZ5uT_DFg>OpuM+dz+v@VSM z4=eeKk4<{=OU15?<^SJJ6i2_rl^@r;7d)NWQs)wyea&yth<@pQ(`d?p_`BuNCKt4} z%pVipy7k8w=BND@yD}Q?7nPL#^^fsE;9o4XF@d;f-g{T}os zJbNZ>t4OysvU79j2%@(>Q#`SFyVYG-5P^!o{l3m!++bnhg< za>za=CmE&-P}zu|Bv7!LaUM;_{I?sIVP3o@0CtzwP5Fy&fC7!e4GVM;6#<)r1q@;! z=-Cxq$4!%{ITD0=ov|wgi3hRURw8^IUxI{aGfZ=u8_V%z>jK)! zkf5c@sOqZ&+ff?K0QuDi;Iic~6uf^t0&s%_UO0X;nmN?lCphLYYQv{N(VFKW2K5ia zb53b@^WOBs>?wR-X&qP^6NqZ-t-wW@%vrKo%)#<~I$DAaxRyP(HPjq0Sa3I?6j$IQ z4z;(3B5~qcBJNx8c#S4aOHjafV$v0{k-=t2urK{p`qm=M{S#$j6cC-4(uK+yjZ*@+P@sg_>}nlstO!HL za1%*>VQ0`;pFpQjU(7%POU;)j&l+J_*=IRl@l}+S1~mELr0CFe{b#2{VAbVj16VM- zUJChw48rNaaRNHWF{R7{1$lOd%sym^++gFMJ)>W7>&(i^LcOOXe4SwQ3A}&4e8BB; z99Y@ps^u&2(*JTtyaVQk0HkBx}cVF&z0U~ja{05cu$Zj+){3m|=i5NL7#0Ou@yHJ@ZpQhY1E`j2~tMs*H4iezqA9Ua&XKM8Jns z5VhYw;0^8|3Z^p{MMmFl&{S(KYOz8${{M31plSYVj7xTLYEH%fd_`rsgKnLTAz}{T zhF7)i+_{$*naMbKx7v<|fH9PZby?}V1?CB@=clcDIvIoP^9`RmDTCrTq<{ti^5cK( zJCMEHSdFJf2_`Uu02pf85k6ut9$EDV$5eeaH^e%8B*0O;sr(#n1&W3?7(E0sqBg@m z@@2K0Lx%#+?bV48VLml|%PhL&_#6EK-@YK}Bm!`;*)bFy8UeVN8mJ)Z4(@}pLj+Pw zkC;+Qt3YG1L-%?ahBIoCtCQh1+SD=IHsUAbBp(3&r2BQ_`Jk%_2?e=RYoYDDP^{#m zKq6oOU90co);~!axp>BejGMIasj+(Ock5CJdlpv@^39vKgRGP%3XR#Km*p!_??BWEJ>>ZBV?M|Pr(l>+rBG$Obr``dq|Hq4yXY^MAdYk*^3k3LS z^tJ;i?};Blqp7fLxHtek2|(k{IPzl>#G@h6(!?#10%s_>@Ei7n5x0bEIl|j26^jK$ zH?D&&foAi8D(BWe)E3NZ&sXk8qzbo(7zqx4S1^d({NTitLVd2WLg73nw$BzZKS zqbs|VRA08=1cpv(LWve-CKpvt9Z{mi_ zRKQCtsFea_m3Z~)RTl>|!1*{#GCFSyt6x0V`*Nx?(fu|G`p)ZRw}cV7HZ0x*7?Nieak9skTl zEeW=>zW)a8UqRdXVF-#<_?x=06q2u@2?KdWG|K$?heW1BqIkH9bIu$o=X)dAGUfSf zhC3?Qo2c_pu~bI)As207LlbeXkE{@gvT-=`nwOWBfUm*qrHs7gDJ%yFH0k)IXbR+i z*NtL2C48cp@v&Xmj?hi5@1*`IGkdYz=~&eW^z?iU_~jOn7S45w_)2T7kzwbq?5)m) zKgap*DH-x>n)$?tehW)asjm1^fBx&^mDQUfcbRuu%QR>Dn(^C5do+?N4pD#)?bp1T(Kg5c)t+GPe){V?zKF3MnJldOGRj^)kbabvmP*4y#(o};X$REVcI?igc6Ck4B zTQv?4)QswA-6w+BO`-Gcy5+81%E^z@Oho zV@1wO9HA==)@jNqd6p4qE&E`vombU0i7QNn6o0_3vJ)jjyGVi776?F{9mWv-V47ZA80I_tt4bjvJJow zh5AG^h*s(R9^40*m^;4wiP?S<#`w~P*aNW#)zh`%Kwoig5^J(Auf(OT0UX>EijNg_ zMjgD%3O{`C0i3*#pSF?!&b=$3v?AaCC4(ua4Wk)WZ!i&QrN9QaL%6(Z1-26?`4BbJ z4+&v=9LaWpZZ|*Psc;*aG>KLS4SoPszJ29yl=^n;1+CCW0Nq6*m(Z3>9@GRd-HvI` zB{)GG{6!e8(~QXm46qyhjfCy7zSDrl(Y#6OFgnp1?l>jQm_%&=7VY486|{O!r$MDo zct@iwo_$g?+Et!t%GzR8pRWGxFkolZ**Ldsx~BqI7nYQ)JQ7E%2LU^HM149x45i8m z41=C&2^fFb5gI?9o@|h%dddAx0Q}8wcXEa*OdfK?VSEDX(FIj4nMF8HA7VJ?Qhzy0 z+!9ecp96O6*g+d)3|j0!PF1`(W1s1Ora%Esf=`djA*ZK>XD?NtlX?glSp4WV1%*jq zTi!H@vvMXAnoH{oAxw$@kOQ@bEGB+$tRYq5pCPS#@Sqwd$u^jRgg}EZt2HBL>f(l# z{p)0P6(o^`38C+})v|^=)(~78-Lh>t8pmrWREp@7!vJL7lby{z3JVDFytZQ<>p zYaRfe4UqLN0&4Xh-Hpf9Q$0TYpBHf_jHxJW1YL6}1*7{8j2M*}m)##Y0}KkoweO7c z1vfxy8nHP>UN{w^fyKUfE2#(lW;dKBIq0cxK^{hc#Saxw%;6~YHUo|Tv7(fG89`68 zVRYn#F-`u4wL9p76~XL#nrS@5ps%K(@u4`RY2$qwabOMeG($sGS=_ljt9_JG$dE(_ zUK*5aF>FrQ^7-@YJL`_``0Fx#>&$Sc2&`$m0w+UyY7ckS>F=wf~ zF`kFHS(m0S`8Gt`qMx4|NsYb|R8)daPst~*QU|jHg)+rGLEoSa(QwcC@3kpwprm)s zl0~hu1WR1cIKX^UgI<=qn-Xv~!MoF=0%DJ#5>Hc-RKY9rtyBwkXv%|7l)$d=iqQ^>|*Dsbmb(PXto@b+! zl;D@;e*2U;(`U*~7HL9S`7r=m(AO1%=RAlWEq&xUx_3)-xSpo6!0;>@U&uJpte1pO zP&n`e?lD?`Rv$nac8JvEd5irI2LTO;B@v!Y$l&RIv$^p+ixxeWxaE&D0Ayc_7|?2gQACM(_LII8G+KxaS;Ma=2xDE>ef+xFGa4RiPK=t8@PjTqf2D1I(%t=fmxzstf zgZi9z_{MHKyPw4gM?KTA>Ao8&OaKw^nLLC5V@84_@N9apc=jlq`tey25TSpsGx*VP z5Gt#0?g=K%Io3!q*N!>9KZGElE$4={oH8*7ulb7?Ah_~9Zhr1P3Qtj#K^I@{b+)LA zE&FZ$coDK9%kVt8SIa?CT3S7i?ZhUvTCX!kqerOSA~)Nz6EY9OBkq(*6$bQP6b5zH z8C16%Hm^C!zTsjSH&#C-gnc|DhGcTQ~01C)(`|GPbk&*x12JgM51s%hkHj0b<4c*MMkePk{=%e ziGi^p7~`oj(3jj!%Ibl*Dtg4*BR}z_Ok0QAi~0ch=DZ{{|FR@M8$AN zNZTw!QV?!HATaXq+%b;8yI}7?$_{fpnL0eCiMhP zzBM( z#rXR9DUafu5$rP;&W0#y*sF1TvhppwP=y>|~t#g<}k} z~=Q|pmWKf=X#dV~B#%zPX#Jm5u;^Sn3FW3jHU#gWp z)~}pVl)%9n0{x!t5NWPz@WB7Az4MHIt&nt$79J@34n*eyQBu)|7Mqxvl_Pn8Cps7n z9UYey+7Chg>1ChaGQ|8uPN(FrS_psJ0DKuZ(1(Gs*4c0`HpG%ovoa|qB^>@5TTy*p zoDZiE+Zk`t04xub51<-UhAH?mZ#KV1y&I4ofkxpPI67m*AH7WgIV z{^i`zo-wj--oE{+mx}(uG$mv+kO{J!S=4;`^l8QKr=L)Ig(0Q3sKKEeP$^_bOS~!n zqlk0BSmH>F0;e`zXaY-kG2jyaoAtiB zFi3{OJ*7O*yHBR`XMMt!l|yd-#*p;Bn*51lV;BV8TG{8%sThmo<>p?*McK|SM%;fJ zwqvuQN#_rcbPk~?^?GRQxi8<8R%4#1tmC5or&x{jQVWV*5 zx3K7xGF=2r&@mWjZ(+l#(B+`M>svmQmHoJ$f`{_WmUZZ_tP)EpXGNkiR#&rUoycie zFO2J?XiBel07LHi=p5S7Z>3Lhs$6n_{-)+t!~V&K?!AGKOpZb^W>x6|LUWA3`OVjG zCOAe(F&NnpW*DICt(gx;It@pu9@of>jz>DhcNi28&!QnpEYd+gX-Kfw4 zU?3_Dw~v_seofAFG|#?w6a&r(lm+f}1CSCuf$k(&kAcl*Zais!Fc3L(kOu*#2Kb}0 zTu4%~?oH1`iJLcXF5B$)e~5b%Xe`^dZTM1YAe9DbGKI=mA(E-mV91mT88akB5*fl- zN>Pc(6f#elWk{wpA*p1JQ07^gLio4CeLwHN*1!JsukZcV`>(IHp69;1xvuNH&ht3- zW8b%J+qWHV1f*?VB@7!LPByYXbg0?!ji4_bwUdiiYyQWWfLP=b3y&8n5JO@hnqc2h z^=7Dmr?@1&YXcupW51onDEKL98xfck-V9Pto9P zC<}+0?ibH#5gT+twAw`XEm^Xprv^?a4;KXNkkRpx+haKJC-r3fLzOT%Jb`Wc9Clzb z6Yg-CRU|J{ulXLo0y^EJLc$yj@jLQuABBW5x)kb*jtbHl0Qdtr zgTaoIh=X3A#`=~t;c0k%5`{bDk?FHuuza$zrJ$nxZ5oEsCVJ)oj*CY*dWBB4Jwzm0 zn^+5OA$n#7O?k0_$dWI#J0ZSqZXKHq-q$(0Nayv@)&#ps^(NiQuqb-qIQ-bTxRcu2 z8pjq0{N#1_^;M5K&r|&4GQAaAK{`kB*l6qjg)z7l_S!AM)YY}FjpBl1vj277T63+^ zxx3;lQOZ;wnqT~n$Y!y1)FPt0`R6D5OT`~9`^iu}-*fq&-bZtqEAt|-HcM$VuC1#d zQ0hS|c8!VC5;pyVn2ZD(VVAZW9%A^H$BD3zU|yE;=C&YVm@Y)l|J!{@C!tP9-sPX4 z5WzPAgcjsQqO{`WH$gKk2B@-shrG4an!6WaE>09cG+N8?R0H`3DWlU)f0Xu=DR@!s z3SPlCyuY;^*gwj8R;hdibM|RQH zGMTs3_0NxGh1tlhx7WzRcE08;VF~_0BvJe~;&&ep@a{edP&FAnN6p3xzT7BED8a&P zRK*oQ3Gyz&5+DQapdCp-1Gd4z?}J{vyysey@b;#D6@BF5$E`q8>C0hQgL{Ew4gtVh zHGo`HBg+y8t|1G*sZ?_bAl7@~EH?b_FkfhZI(=*mc$<(f^+VGqXnHD~5u~0mW#aW}LxUN(Sbj ziw_-ps2Z-;BJB_|D7rK|?;VoPg=>BsT+OaS(u3StAv{nzA2bUVK zi&?B+kStSZ9IFC0p@1{AZe|+vkvSavb%TjE9>6TflHJH50#Oz7R220>mqf4%05E99 zynrqq;NW{uGLji*XX4Q7ir@2ZsR#0)0$=&IzIe2Te#QSFjs+)o&&#XxLl&whxI5I$ zOal@&pP&XBb%RkH9EXnc#LbEeRnS%ea&e)bz{3_be!tQQqXWi^T!EGMea{y^L%~$7 z8d~QEANbD!MHwrepY=|%gvDwM^aPDy-%ULFY(l?5?Au`lbv>mQ|9siVdqj;#+`TxTpnQU<12T?zg7ELLOG#o#Dg z_!)jdkWxi;vajD$fGY}+?w1>Mi+f-$9tCyD$Cra*R$88qHxCJ{)@eRk`g6M{?9repb@ z_s7xfH@|jPoaOMLLkY;{?w$XJ8O{~`bp_L2jxJN+SL6Yc)t}vg)CH7&O!d;kgBYw4 zHxJ*sr6Q!H#n4?#>)DOto|FgXJ&@s~?)^(d$g#MNdDB4n#?QrCb*~T2x3<8~T>)+i zLL;*~xX4u9Z1bu^iM)lxXyYZSKIRS0Md+42six-BLgM!r{OAjkqc6{HoCg{Bw;Xxcjt7p8pgYiA)RgS4mccgdRg@dx4S)r&Ib;vJm#%s_$rqw43Z} z4~7<3NOlxK{8XLY0edmP1s7A>u`Vxf4FvBI4Hbut*RYV@OEr`SB|0|RnINF*(r+>Xn6=84VxK&QkxBA}E_OaYuK3%YhAF9exg1jp^x32#TRZTyQw*t3`E! z_e{ro_oR$N-O1op0hw8U%5!Y`)3J&hh9rr*2!WIPNS7ew@RaprE^(b+X!AgPGe()# zo?1(!Ml%zZ?X9eja!zInEWK+j-W(mtUfVZoWG2{D!@2N-K%FyV~!?s_zU!yT_f zmA;w?zSvX&lU#yE$>}1QEn=<+z7oMda%43tg=l(0vp@q8YzmIw$6 zy3dffDrraZVicgfy7ABhlGzORvMh0LlELA>4kiQ2MXvauoCI(UZ4(aE0idm?Vk|*$ z8-Ye;{?yUmUjYm2)kpTCE%hGsd1oc6fl2yb&U7K8FzoPAWNHIt*9Xemos2TA$9G^O zDl(N6l8;IP?!JUdK+=vJ^T>X0U&Q8jlXm{2_Gr=yjNwN#eMtbW3In4sR3QnYwq(>b z=*&9x>@6uw!<3=bGQ*ktIvZ~jeX<*!4$Xc8X)>@wXuyuNwZO8alwLd}hT`xuPzggJ zWut0RIK6zu3M15SK2zW4RbfCf$xi9JWWnDKE#CaTL=~~ne+d_XsN-z3ovf*(*<&dI zh0syrUVq=iBLYCvE+3B`9FDeFMnA??!evfCVJi6pv2uAmT!pRY)>$L=EyxXIo8AVhcz*t~JD zV4ivAh!YUinyC+GfK%deE57)IAlHj;FZr1a>V$g0{UX4nBi-n2`-_{ji;qvSI}g$I zEcHSKRS|w$wL6bo7t(;+7YW7-<*)!g07z@cmQg8;RaQcq5(D@2<&W|3Pf0ufF^2)Z z`w$Z?Ewe*IV_v~Dk+gOX~0Vv37J#o^n) z)z-5EO#VXK+CZnwSkJA_l1gX1(0V#Pclzv1O~5hs|C_Vi$sIbv&scwBk04mRT*W7MH#2WEvwU=9Z2ct!h7ev7OSpI7mlpCjzj=)`rV?5-pt* zIlV)RUnN9JTg2mo+J)cR9Hh3CHn{Rzt=kl574GZ!AjO?r95F&67k6+M84?2UgfWdM zqBKDL0XQR}-vWc4Mx2nA#uJ`IN*sOBKO}*BiMsxE2EE)kv)cgo z5#q&!T-?3>Ec%o`eFowbjckjdzPGhlmQd_i{QwSP3Cf*@Eh+(m+hsv#MaG-Hwj(`i z5cyySj7&^?67O<^XKzr8%cLHw{X-HV0+U>I!@3d7 zj$tWoOdT&-#;yjUg&`~vEh(f8EUX)=`Pc$VPb{1k=-53aSGCYH zPzOVoGR*jJkY+x{Wc|tSPv<;_R!;SqKwMAM zTW~_@d(3z09@6vUm*bGXJjcVkdVU&Nn>I#}M9sz=MKZL|rP%x> zxc`Fct%=N)n}CT3Imm$*oI^?0^=Ul|%X_p#jMgF|Q%0mv7$UwnnE)<&E3+Ugi`ppxOD>WW0^@A~(} zE2#&2&^Ap5Jr&yFMd+VX(j}^#mOtfBVgLLEqUaaXiO3O<4^mN@lL_B&uSsJ9H%S>v zTsl!703|x6Ye!s9fQX$buq4f-gzB9@`lp6>9ItZ(Ds)6%-ntg{j3Rx0H=}U+fPwT) zD#oA$;?8sL{Jn<{<8Ym+u6$>61Wuu=?DDqvtYNUUx~>c7m@kItiJ>l^M22@}8&sb7 zrQ{aU24)MJyTGXyO;UVoLR5Jx7+hTsk}BOnw9hF%ay~Z(t_gYsUnt~5snD&&^h==q zH+cI6-x|_ndCz6<@F}p=QNY?#C(xFAB16@*un+qb!h$!osz?-gL0-QTp}Rq(z`Hr~ zc%K7WB+>!*)LZ2OGViYZt3i_!B<=Q0uR=u^3N1v<1rnNeFA(tGkf{;{_P}uM6tK8Z z$kSc~Was4U`?Fz#7^&JI>R9k&fu2GUN-plbOQA{vT&gjA5p@dg!Bt|r@*_!Tug}HS z$EfFlZRpyyh%RL{CO79)e8e8Zdsi-?=C5I5m%rfBdfOQDK5aj)-?aFMm=K+uW7ky{ ziit)u8^%6HtU3m4oL&Y(D(DKo)jZ0Bick#($1fHJYY!PPoDh_j4g(PVZnk9{)|~(> zPi)IClXm=l8wGUPcU6omiEc5_#8?$`EcIqPQsMw&fZX$1bK&(f=B+@bV{^&i26#4v z=P&3kC_p(=edzkALPqw2w0UCTd5@V9w--Fy2M4Ob4UefrksE|AlgcpSeEHbNM~&z_ z@mdnNhjPYys@^0|oWM*2md$~PW^oTcVvOHS06=eQ;m&pdP=8^|iX;?5z#%r-%Mx}4 z3C1Ni7IJo3uM%5xrH|^B{E$dp0KUS_vIOM;@e=n?pCooG zzU%#ZN9qItbokZ$Y9*+Cc)u9FW~0&eFR+vHtBdZiE!YXJ=6(LhaBqb~0kOYh zUbDuquZNz@Z&J4erbX-v@$JnK8k=+PP_w__S05zl`Y0F@PxV=t-Xfm=t&l`d$g{uD+w;vY6c%SM*d~KOWqNgUTh_;|@9m9T zZ;IZXxS3faK}QFX|IE4v!%?y50&pS@$9OYQ+Olh3_FiReb)TaDy7k*KS{HQ?za^%U zPDi^j1tAMzKKK9Sig|CqK}V~h%^aiA)QHyyv}YfGY~;KeWW9L&^&f(~3h5v>O<8b6 z{x^3MB<%th%?=s`0j&*MCN`wHyXbxZvfzH=+Wenntpd9J_9xo3h@}nTH#YB}MCoXc zZ;(cIHL=j3#S4V*?#LCicl6{_BaZ)>np%-&Y-Dth90Q;=QT!Z+S;p&r>{N9~Dc2w8 zX2%ADsn#0(&loBT^mD`U7}zvep@(c|!BgVH+|W?)w-Jyenyzm5KNisbnlQP-1O6Et z1rNC?g{hVus7#4gG_`YRi12gnp9uQ{8no6i6(g329kMQz2sR5!{cbdAvt|qDJb%6& zJzhV?9U9;fRd{)CmkXZ-p%wa}e)EOUWzmHWyq#6ep69UZkStY(4;Zn{u-*nH0Ttro z5hq&2-jy^8EUJkk84mA@#;Mae$`;P}q$2^Y0?!+~a&sXUbmPWyztt%bz$ zML#Y(6Y4E0DB|5W_2N1BCw_|FzN>%s7v-d2KVkc|akAXxv&r6>rEgvxp)U})$LMra zn&b5;`M3DXt>r%%^aI`;S-p{Yed@*y-tBbWZo4y8&9;jKD+iQ6{q#*ivT*qGu(xx* ztDb_rMs083P#@eB$AT4J_Vvc|)t*vy>2s9%ETrBYN8J~Cxzl7n8BYP5lQp_3r%r8{ zX=Kezsv`u@i4-cno_!lH&uY)D?Vw$?Tv|;n>%NP`{2I z6az7&nAVxVjoGK?{OcWOYyZ$t+_1~=b+-xoBS+|^r#k$i`c^p0gnOqJPJBxU4Pm4H zUU7m7oZf%5Q!@QAwZscwjsuG{!LKvNmJrAE%{^I&A)09zqoOf3$?xUT5H;tyvQ7eo&~{O8mP+yO&#_ z!LDV>Sr5Fts!)>&_FRWV^IzTniK zlDkCkI+-K%_jd(Ps`?nQ{$_LiV|1{z6FE=s+zd6mdS(vry9z**q=0BFhRC~Y_sRua z4F3MyNH07iJ3lI05PbaB?C|`YaN%@clG}$o{C}@V_>^+lTUhsnv(w~Wd$@dZq%G%Q7O6=dbArEQv@H92I})!Q1i~e z{lXt-$zXakQJT5+`(a8%(L{Edf=+yDs!$GC>JmF-gQlycMsCE|Das2Mj4VPXv%VnQ z;^M_dgZb$*3jJec+r2%Wl&5BA`=heDq`iOXt%EfKPB`U_&U>vpP|<}Nx}l*VC}yUY z(csUblZl#c{kCnd=(@%!C!Um(SyCvVkXtH`^BsY1Sc#kmK;~9~WXK5S@cK0qg0vg!*LAoIx85&bb}E z8K@Y0u3q=57x(zDnc3v*>{GU35D7Hk61Wt?jvhOv?c!3{BiU;!Iq~K2Y6XpW`LTG} zSx^7Rk3CvcG2X+Meid=2kE`!LQlFTU6F}q{{H5nI(gOlmsya!Xdtm6rtuyflh6_$o znjHKi7Pu(0=i0ur*hU92)l9yg_-Xsr?0E7NQ-am$ZoNu+`BL)7;$;x{Bxhu(kL7eC zor8ik{7hYa@thr0bwV$bQJZy*PH{IDsXG5$7quw)W<4+w$Ih)X&RUwgMMf|(^n6;; zG+$aWuIHZ6v6R!#KRb6(A_X>2o%(iRaF_7QWT=2LA3j{AFdAWjk)Bh0@RGAW4D8gFL`0bwFUS_F+-9cX>Lw<>hi5)q4@?=y!HaFM(cU%X(qDtF; zsoC4-_Kc(P-Q2+;Xfl6p&b4KauqJo+#Ypr{b4hju>eri{KTprpt7mcP(gSq8ITqii zI)Wyh3M*2B*R5M;BD(oV;D+AT6*P|1h)Ks6t=#{d!}G_VJF#_ZUzPq^Z_qH`uPVy^ zg6G>WKl5?V9FtZVnveJrjE?pp+CQ^&i8leVU+mKV&^8M+)M<7wCGaP52B zX}#F*qp#)sj;0CP5~Z&8v#aXkO8#|%blGo zeoR09IKxh>BVWo_E^9L%w#%5Osxud#DbEvZt591>y@Dh<=HqYgI4oRgdBEbOrR%sw zIzHYLMDV^bz6epBdriunbLgk~Zc`zszKg%A+4>EWCUERw3%MmR1y!nV?>KSQ?W*Fo zSQTVW%?_?j2{l)ID|tj|@8vD(9Zef6XMN3YwyKRQWDQ@AdWtVy#{hHawggSSZ~6AZ z(tk(Z{7O~S1U%#+GhOAa0e{wlq|DU9-yis+>dV0hBNftj`zEk&WPaqXtcQLYcA)EY zb-1G1_9)b@PM|=Jq~(H3d zcP+SQ+!Y~GO;2yzRLXbSJ7TvRJ=T0gj{;8AVm=m-rk;AT& z`30?l(p~P;Z`kML6k|nmg&whul1)3@T_KeJ!0Kf$Z&hbAZmTT<-f!2*$>MoE+x?PZ z51%@AZE6{Y$CJ~`zGq4Df#Vo^rg8P^^PX`U?c5zYx#l(Mn)=={)8?A}rWslblelGm zd8*2%P!E4OxQs=!o2C`~W+!AI#eCw#i40On$^=;Db+V9%eIe}q z`}YkfGndQ{y5ckU+<%uzC2tL^c;||gk3r}>u)5^pVp)^0rAwD;Sz4yF&5i-}6A7%~ z>2>WJ8Zy&KIc*)*;?U02H913$4P|BJV%7SQ~V*)$0Lg2g|x;|8_o zDCV}f1q$==`KIr?U!X5s2}3@e)3F-r7;*JhjLRlZ*JN+-62YV@^PLbg{@l#Nd;i^` zYx#O#GI80DA3fr{;^m>tyDm>x=wL-xM~50tz_Ljxw$+uDl`HrV<^A;2kQfXM2sjq% zA2dbt%)6M4{bOTUZJf%`>ruYrWslq%FR?feqP8b^ljp_t(pI@42#@rFA-=wcP-q#o zif{Jn7@b<;A6ws#0nLm$*97C@;+6=*1dlam+xcvraZgj+WC?sj}!FN6mC{IElQ-2_JDAGgWhT_U>oiJ;=uNK)W z&cw?DJURlhy7@$zO;=Z!xrN0&R%8CTSx=VhJ1}}%F&d$w!4$iMcfdczC<)aV) zlB*#lw}tNy>x+TRir2i{-T{%3oD+$vE{fJR9b2qk+}yPCj*sbVuPU`x_uqo_<)}YeF*eCG3Gv0uI0sjI+O{s;UHu$P)I&#ed!j z2i02m+sbZ}c6tnR9){Fv3?{tZHIAfm9qsmALl(KMN4B#bT0h&k(#W)bSCH7=y{}1- ztZK4DrnOH(f|VNXOS?$edjzwn`D^eU5=aXzFITvR-RvpUjd`qQ(AxT5I!jTq`9m-J z9Y*B+fEr+Fzyv0sb~EnO04_s`=4 z^dilSCCLpsswK&&O85Mx9D&g;Dr{@10dt3SnC8~%*&LE1z zQ*Z62Q6ZJ_E19pp6`O{#R4LQavj-j8%)=yO6=|srj6FU%2_}E*@Uyf&=t;f*G~d?W z{S!5q3v;;et?QAJTOWFOS0tsrymuyE*!c$|Gy7s>uD%2Dpdyaw?x1uhehkH$JQwEliHFms?W zSC!$<+#t@Cq} zy{PJT~boE z3Sh*wu7AAd>sPLqq0gT2J$v?SZQ;{3h+jmvR@E=IZ=VJRHxApgPn8{$4~(w0xKN*} z7gg_<1o-9KOS`0=apYqOsi`Wc0*bK*w8jY5o8>ZdZH2`;JpjP50Ku#Jp`T~J`CxfX z4OOG&=$O|tj;MA#zWRzRzrW&3T>X(iESSSs>jkYqR($l+HtN-(aU4+hkuj%35dQc9 z89S0@U(OSFTx|<&&Fz0u9D|w3;q#bZwElV1;IEdfn4hCl4QbTu#feAW&fI;J$o{`6 z1jER{;d`(Nu~N&JGTsjDk99#3(}N7GZa*}TGRUwY8B#RM)(P|<1)-b?#rhuiL!1er zPo8X^z_{4sFD;P(?Y)TmSYQcr1hV_jo__eb+Zcet4*I}ojl_l6*_*=?h0=+P{wbMB z#1O*S-+{#s?6XZPEV45(d`e?vu8zX|!iNWjJnXaS^8u7-A3b`ft^dP@GrA?pyeA8i z1kZk!8eN%`kg%qp7_AHM5I2jGw?vD_UiEj34o3+L8E0Or>T^GV=98&bgJ@ZU!gf*m%{+3XyUFENiIPV8LXyI(&{#G0fj{lb#k4|R^F|caw}B;X_Pn^{5P?5TvUTVM zuS24E5-d;IJQrcK z&%b}<_F;qm=P!+FrTO>^M;In=__Yoj*zHFSKvH&eeE`t&G&MB5Q8X4S{K!Xr(%s#i zzsu|8R98qZN?$jb$w$X1wiO|M+{}Rbl*Ss2y@B!F<4=viofRLL!D!vch}jfiLNF?i zIephERcg8?dhD+3--BvlMZf*izA%Y5bC_!{AH=D5K>APHEw?$S#s;GaUgulii(6JFm9{GeZXC)<-789}&HksCDks2~5<|X%MADuQ7MwpP%>L zWJZ+?rq|AvQh$$?O$x9u>RoxaL4Ci_a9%eX)SDg=lC#~^+zvgv+EYL=d#T>YEtd<_ z)xyX`Tp&_w$AW3JD21oz*l4f09*MO6@OKv8bzTITMDMo7Ze(3ZlO81td49kwQP7el zGe}3DLwG7_qK%nk&)b3*Z(yM37Zfyw($Vu5%dswKyaZ%r4MwiOk+hN&a<>JNfar|O z@p7F$dv*(z8M-6f*%w$&$;<&f{ok8d{2GZ{3=~N8!^05SPZSQjsD70s-%UBKo>Ath zt>m`r81Z>1;MCc({C|Qpcdk#4jXm5V4|wjHV#bSYNxgwl=Cledqd1)l}RG-%DGJ9{|vjx3wab zxMQbt_kmW=o@O#>ZhB^hr!fa~!Aa#<4bRk**RJgwopScAxqxNP92SDJ#-Wp(cFfxX zi7h4;&`*D!F1kZ)v6_61I4~SKrN+J-R*tUn5aK5Ta&vR9c1j>SNed32Y3}Bp%BSdF zP2T~ALQ^O^m$7{FqVyUNMN6_3bb-f>Ng@XW#5!!h>h3M%=slSNEcYXfB7SxJawxOP zy|4ed)8u zlzQs%;hJO^r?*lDt1)xR4o!-?`y7sFmiz9L2lBtORppgiWGh~?%51!ZdP_cSWAD^q zkK<{}fB?v8c-Vx7QZ0S|h6TOy=OQ{7N}_C9a`ibH%JdrL5^SKb%+4`j;WFUcpA8 zdcG7-cp*`8yl65p197dqlpDAQ7VW9PwJ20F;U40;;r);!tz@N%dy9fvj`wl&8X+}$>V z3qW-9KXPk%ol2w{;wRNChopoD1^RHG+-)B}Zlr$g{Ebf6+Wy=$fAr>;SNpbqFtaNG zNJtiPG7v}i?N4zmE&AIjT)lX;lpHL1k&z|S+4#O?nXq@}X_`}cOOwyy{yFzBPQIx~ zK4~>G(zmiUB+s}jbAnOPe0R0Zn$7yUGZH9a&_*}_lzBsOS4(&CG^x2hmN z-DdD_8d;SiT-A5@_9zc6nY|~N-iHIlQ!iMJSnQ{MeE9J;G(}*gujcgeH{`PHNXuR7 zn)US4v8xouW?aI9I2|*yhN9cM?dTdsT=_9l+$Cw<~(u`DHdg% zH{-?%7V;0ckdn{31=g`LAjgW zvl=7PMD=T)IaA8Z%j?j|>UDJDmp7$rt4;lFXXiN}enLfsxMCt`T)&>rJr2i3l+`5Y zy;M1T_;7&@j;8$nR4QLS5K%SKv^x&wbEB^;X`!Y4sJPx-OX^U4Px*mn)UD|4N5&ZH0^wsjBy)vY;{rkVN&p$ZEXgYZZd8 zLY&CjB!O+*l@{M0rgH6v`$mSA9J26z!$StH9pD!h-l!YDTIP@3?~92ISF7=C%WME_ z?zXfLI-#R%bYJQUU8@wUUoO-Q$2Jp~<7w}AEe_%QyrzHd z|NpraNYVah_R)XyYvb4d*>}``HvRvh6}WK!{|~M}>VGz*tu=h`2uHEK*trkgqbv05 z+JE7A3QXSz%1^o3DmS73!FucOimPy*HCx$($=gFA2#?q=JHEHi;S7x4Gswk3tcbO2(PIE<);6hQCF6lkB#Y;3%R`l{pO<1y8}9_Cl(H+dc1 z8?2mVTEU{RbA3jF47Z1;=L)dE99&VpzH9kl_Lb$!A| zW7mLV?C0)iXj~qt#7F=A>kNDHbp=)euy?rw3v`9F`f9jno^x^y3Jr}x;*qH(uuzir ztEj40cXV)L7Whe5R~h@lTV=?%JswuU8vqb7S9H~JsCjB24>~v)W3(@%+;I0AW_5ji zHh{hEx>g{81t=CqX6D+O8lTfiUfSD?6M@EqgF#XBEG^Bz&Jmy+G(j98l}9mPNmE-} z0UqFc5_Zv!6ZOhGQi5ub)*IHZSA!-^-c(}is=ogI?Oaih9x*>lO4^<+IaO>1V!6Xw z!ra>0Csbv}AjE<1I^lg&-ar1ere=LMm6Kyr%+AIZhapj1T+dQd^~H8BEr0jJ+S+Se z>-z|j2aC&>cf?mKKx>+X#?Bavw!QDmI&Bm*ax@aTy^d~IXykaPkr;FHCf$>e5PbBn zUsu5NZ_FAhD_c&0laC))f(fdE6o5XZdvq$HurLhJ3nn#wfhJ?82qhx&(^AW`x;F+d z~;-fV`J1rq~$x07v;xg(t~SuiC@a>jtM2x2?!5V4&fZ=e zmTw+9;t@a!GPg)@B9YQgsS9$Aqg3LR?s42HZ71{L{TeUv`YRp*q^Pi=j7vuB_=9ZdpEqY?*ib%iDe99-FV3B6~C@Cv9a-;p(yMdQhU#eoCdI4LjN4ew7g!( zqW)d}A7WhOjQ52kV#P2jqQX>=z3l4xl+`%u`sJiri)V?6kr47j^u&9sB;6u}XKQZL z38cGNTx7jriwEi-N~+bs6D%7JFE@#TFO6@dSv%WY%7nDn#^P30cIT0jXH><8jT=wG zNOA8W@AB&u^#o&!mC#Ki0ClqK>e<~6-PA34JT}fFi$Awb$2FsRUED#TtTZ^$b zJreHT!TRs(v*;Tb;EsA1-(Jr4aqng?T@K*MDNcu3diKj@HP9FWU>GR9ACqsQr>C7t zyb93gc%{(kHqjq%L^ybaOS6)A`wdoe^8H6DK$p7w`Ehs8>_@4;2yNN|0Y3HX-?P@Ygg$*5mrA7y>94li|0Fb&8$o8HuJys! zOzJ~}gPrsP{Cs?6!;_4zH-f{$ydH0)tyl>e1{c|KAcd*{Q69sm;HIFV!bv+?_qfJn z+nuj(-!9P>U^%Fle#WBX)5ni*1tOgl6&VmAm+QL%37w%VEeD@%oNc>N{WSy>)IkdO zzUKS)6L_Ft4&`fk*iPr~-;X}0CtIe_6uI(142?@fMn(nHvV0b+yjRE4#=&c&cdpge z)gA3rNYw1K5Pc#RxUT2Q<`Wv0!d2&lJs?^e>elMkxySN8D56Ek-`~GEJp#Be-_D)O zx7Y@Uha0eP>kLW){SWXzqfV(GM1U5RmOcR-O;b%R-C|AUrG3>9t37KvEML>w`gx|& zT=Zy4JoM~g5i~!Yl(>8M82W~XLCc)LU3Gg{g~&KG@98RdmeK3zVIW&EntTe8YCal? zE2pp*o99ssy@-s)_XOqpS#X_}kOmxQ$37hxUam7SPSqQ%o|-|^#k1t(7{o*;U;t+X zcfhccacBSdbb4z(XjusDqOpC}GFN^1vIdy{SDc1RYQ^ggZRWsp7#my@pqMGnRoBr{pwD6U@tyC&y>h6szf5U!i?#Q4FQGU$e3aIIY;k76r_}bk&FkUpu3r}|s$up#Slro)MC0<=4GJ@i3L{Ob&ni*Tc*y8!?oa32Zcu zum>4F6-(JCLDn(_uLC?i`fX)&baeB)pW|_#qtVRY^UJzNuaqRiN#pjmD2nCY>7>qY zyNPEgu!(1LIHFAUh2(A30$xqJXs)w9S+2=fd(W{2N9yoa;|#5+%^xvJSjy|c10SqL z2yu^(IO(Sa5>=jrq0nhSxDir%k?1eKBJFZkM<*dJj`zu_#ai|tEtYF&YRcnA>F(}C z?>u^m{HPjL&3djsal|lHcQ;ASFc3ir7on-I|A}695!0EdcOgH)l%(+o$Vy3l?A*UR z2cr6J>r^rcFXwsDeJFWzEC2XFb+2vI z-BQGb)JM7Biacxn{E*?uL zd|QUSudcKXUFk|J5eua$DTucxO??<}&!M95vgoL@z) zTxmU{mU4PWbGibp4eYI<=eKViJP<=@Rcxa;w*R56WU1B0`gf7 z+nKG79?d98Mj}Anarw&r2}$fmwMyuJV7;r}f7k)b$^InASYiSqNlQzMsq|c(;xn6d zr;>B(du?r6Iiucfk=RdpYx14a4R2fa$%vn+yeZvias9drNOK`Tdfg(5`DRr-fx4ML z&pS!6wWwu4z*-_A%D`j#YUiIHOjYMxc_Yzqww96!GR@W@UE|DY>|4#@h>+D28dw5` z&arhEp2JoRPZeVci>i@Si~M`j?}N2SEl#MaLV8njPudEcLiB{aI-frzct^qD3E_AE z1aFIpQt&(9vCCf(X>q%gm3yY`3!R-rz9S~~ZU)qgw&qp+rQz$Sw-0r-PADj>u-8>q zQAuU}cuHSKZ>b$RYy*H;S2rf;ej8yB6EM%_@I@lcbbkHX2u7>hDK+c(9QJd1>0~@# z57MTk2qi**%Ww5aRq?=mx7|fwEIygXD7I%$iI|kTvR2#Zl)1UN2h_NZOeQI(m6({A z$ULN~=jMvqR#ErHCopO13UwYbqhCHg@{v{M2^6N3w$y6D+jrFl7M8ELx5)GmL5)Uf ziM4&F&fjEObrhMz`&2Z??!{WdB$ypcs{IP) zOd+W@b@Otg2fywdMlN}m$popk51x&vsS_7CxwfaT&YPMF_WN&e_)Snp42Rm0R-Zqx zqMcA`)IL6l5{bM;KC2;Vm+$+|=H{*SFE(t}Tsd?lTf;~3(rzUI@a==Lu^PL|*Syj~ zZPDKOXhnQ}en@{|E{a`*9WA>se1(=1!xnCf-II6{d38d+i%Q&-s5&=X2}!jtR9i@u z8RiDG9)~2H%PU0QP{8?x2=rR13Lb5=EnIyQcl78{E)@%Daq-gr@-0c6_s{VzGkoA4 zrvBM*_lR2#q!64HJqR9i2B%KFqu1%MF0jb{mfpl84;`6!bI8uWGHcJpcC?ioJo&t_ zKwu%%abC~pqhkkYdD7T;J3`VgC$`h4PW^gViqMj}jhxNb%BT)PcFt00m|Qq*49afH zAEM_yynMt#s>4fVIBykM-e#MoGOK0ZEkEO}%M z<()PpMS;Q+IYYb4dCPrU&{^{P4&8S1B^>Mka&L2ru6|Tv*e2x0=g)U(@1PvD_Z+h3 z!oornw`|%xQcQ*qez`5fy48L23ZceR@D$$$`|az@J>Du>4+}+>q_=U@DoVS#<@+gJ zdBgnoacWc)*E6Qi@*B6RJ&xipL>B+_)jH2`z2umf+X&?W-?t|FYj*PT?aHkmo*Z3d zj2!93F3%fpezbk+#J_V=eqndlIv0tBRN~Y<+{LEjS=#=C$$)4(L^X98a6*boNht&5 zD3H60BKgxN6>w@5Ys0S>wwOMTb!QCS;&s%D_0-9eei6YN*14Zf+At=^u@whk=c(ob z$I&MrbOx`EIZ5NT{kBn~U&3pyo~U`{z^6kl#i%hIE#;})rb_zfAe-Ohs3$#p_9D^- zB95eq8@l(^{F)#alDCU1gl1>xM=v@!@6``Nf&w6>%Pj<=Uv&5%9vmD5xx(VDQ0Zuc=ygn6RFW@uD-vlW(sPHKZuopGhVAjyo2qrSOGLQg zT({gRbm_9h+e@EE1>3us?+HA_ab^p1N`A;ln+yND6MEKf&c;`-m|I2UnJCh0&Tsd7 z)@+UPJaGKsT7Jsm+P;i)8HLF23u2nEB{JFL@=?;S-)O$p7$k3jpNr`7bE?n*gb!6e45Q^34#9SPa!b1mEQl5no$MJu60J*;;5Gg0C~Q>g1Z1X+`H z)7S6cE5njTfB$yq?gQ1M@4Y?hU7I15u;uTo3;-QMNpb3pn2BlWV$ffAw>3tffCx)NzIy>_$`~mMXN3m}^@Z){tjx`|?50c<#c3S$q1i^1-c^aMqEH-9%;GpqZy()c zOd@tmrQ8e74MKqzwJ-DTCpa9_MldwC@K7?XKm}+$6P25LM$_hF_QL6=eM(bvdC?=U znDeKzZfTdSutp2CNm#m$zDp&d z>+cWRNo*|`y@VX5*-7WP=N=aIsLt5D2vC1GFHm!o%1mS=jEh!QRJ6MhC?@L8W*j4e zAwTx$6t0DAL!SOQdSdd}*5k_;MCc+(#_{ZjzBhO`oW!|;f(2FR@~M&cR=L6l;&FIb z#pO7xynKJ(d)8SB6xaF!XY0#@`0e{nJs8X~mtG2dL62pCDRf4Vg>e?Y(NV`##QVV8 zd$bS56|qMvuvxln*^yX}FQ{;JE7sm@9qwk^YLTr2FsftR=>pE5XH&CocQ+&V0Hn2H zRt{4u?&9z`id>G&KV#enfh%$c^xy2TTy&%1;BLk`F|?&rXlG~w#c2ph!x9-fl&Z;> zh_N9i+wblF#P(>R8w0EGMWd6gq{6rDQ!QUk(C;V zZXJGyfkxPss2aI1O$=3ukLY}ht1Hm9g{JQGC9NsvqV%SmCsbR3Ix0`*?#uVH_+57b zg0jCQ6esSXRwBL3ed+#~50*Q+LmeM^yqlO98?!!qN@VX|bEN9Nlvx3i^c08ZmvWmN zpPzqsXvF@anb}e6Lp@+Q4~k^HJ7jIBnQ~e_UEG9n1KhPbz8!Fm$<;%1-N zJ9nt1R@X1>YsxI=U361`OYt7(`v)0fvFp}6A5v7nX-t*&~2w2E+USyMvi1)mN|wqq~*Ohf6RmaUM`>qnTy-Z38WVdir6v55V_ zR>Cx7zKSt8G_+%p1)sk~t<^L>9!BQ3#HiE~Lk|bVF{oTED#O{+MR$usPJHEej~e9Q z!*Md&;Q?t~+aZ&DQWqW;*2!yroEuX=3pjmHR*;B&F2e_jZ5EZ4l`t8jyacraf0udh z@UX$?7mWR!_Y{G8&Lkp1r551zLy&yajDKG-KF<}PJ`Q=5Qo zDC!-Iznler--cqRnQAJj+k1D#FsCh)QjBlSizj@hNRZie_D7l=Ig ztS!E`s$l=BA^%hA>gdi)j&k4gzR8s<>eDAm1(|1slf1meh3IS2Jawz#S9bR58r6Gj zS1)#sPC1BtSgT&vvJMD`rkgp=vtgWbiKwjXXL?CiW8Kw}vCr#x7UzF2>Ps(ayB1Js zcGhl6t-Q&YQ^JFMc|^s6$-!#}HY%qjC$DUJqnM~k+KOMze1H9VQQD;Ijd$m?0|Gez zP)2R&xoP0^EU`=g(K2+uVOV5(=W5Ur8F7n z!{nDgJ;lR;K?i%m{}6Tt-K{%uOhCpVOn1R>sXxuHTpq9%@-XgxHi-v&^u0Y0(oLD} z;JEh@_aeo}mW@yC@Lv zuEAf-&3Xn%1Qq{21xZn**`;)u71CrXL4K;L>sr2JsTvf&P+!|fixKJvnH~4|e;4$}H-doL8N*A+Tb)`ljSA8tJ;4egs@G358%DjxglLs9$yTyak~-_csVhg(PL z)1t#}9*0_O_r6a+u$cV*86gRa`Nav1HXMQ|_jHH*uq@Ik>5JfE2+@dznwXH_f&E!- zF8UR;8M2tG^=@r9ZdI9MBO!6<7u;MK+`6_L1B-+l8e#VEV)nc7kw`Ux{G7Y}3N4L| zdC5Dj?~Z&^{kqa=8d@Z&pI5`N3E8$jtF_R#PAez~4S8i`Xvl%k*(Mk8e--!T@mRK9 zyLYKX$xs@kNJ*I$NdrwN9zdLDvzN10>aednLV2P)13s2$R^i?>+x>OzLK8vb$AW5d(7JQ3)_)V&iJwHL27G z@6~0mT9MgmIH@P%`dwedGj|ZLLs~DjX#Ep1DdzAyaLI(74hdOf>Suqr0-4IunYo0Y zLAqsbEEY>`vsxIWqgP^kxqZR=(@<7k0j}G=d1Ls_bmZas3#E4NK8mj2fO8g2(5;e> z9h#e)Q}4aNF?3VvktD^%GQ8-pctGh<;H$jZo3K5v%t}|a{&5omO8Smhwu*0zX2Vu& zX?ue8BgP(;(;wWpSBtFjUpYPwRgEXz-4pPaZ|e>CBlecz2(`GS)RT1S`dM z@H?%YezWXPGrt=u96Yn`^fsP7kJn@?ETKpFQLIafn8ziz>2q)9nmv>W$d0%S=|RV< zU(PLi<+GOlNob-wkAQfRvut30@h&r%17YQ@Fu&LS$xsLL}aH%`qD9<#Hc`u-d0($XCI?)+_3}Ohv*f8fbMeO1cukp)4Lz%Xs4&w6yb0YqVT(;{H}TL z5t)005;Hq@_#GYsrsCbq8S(|)e}OG(;7 z>rBuu_rc8@t%CqJUvH`+{7^^dUG23i01nvj><-_zRh#bL64YdH-7%WWUxDZ`+frIO zIxBKj_dfi{>0Zk|Vi^o0mC&u_J7M#R=5QUwt84z`+C|~yDUd#sjqlJeo@}33XXScT zRAvM<51Y0CnR7_CB2^aFBX8v=fmYV;iQG687_8kvB_}3!cKzbo zuwhHddd$ew8&@f=p;x#uSuB;FE-fux%5vuP3A@VsF1xw>^9HTYpVyN(?1_vTnZ_u6 zpwvYCSnYlxbudU`%Evb)0v95GYpVN1zA(S*o>wL5#lg>zAwq0c4TZ6{vdeY*2k1be z*BDF%LvLqN$iCq4OLDRFPPh%Zd17QSsy8$>G>(8FFuX%Y$8@r#Qns1cufv64iR0_6 zE@~H`K>TT>`R;=UJ?p&p7rre)<+=80SyJaNE);RiyR4)yg1s+WM=;dT>iP=NY~F#K z7Nk6`=0U(lVeslsGe5M|X1BK@i*KEt@&MKm5HtE}M>zayYA47_sAKCZE53kjw2Omxur`^Vyg z)q7wZ5!Fh9QkxSE>TUaS@$$QyXMWfOiiQbw+T2v$xKLb76(h%|hKaQqeJO2 zraL$~>g4GGMEkpnTlY$2jk`b7PKbZ@>?YE+uwdSvkJCv$ylshr@LJMB(cXm~hjnPr zdchY^`u21dMmU@x(@wt#uP8GJzkByEzz5_OR1xenIGUZ`HFEar*wkODnH+49j-z1O z6CPE*tyr^32}h`rdOA&N%O5V6esHGUozvq$Y5HaBAKP}0g!^A6DhlGlhL2^y03u_N zV^oI%lvpU?`UuF_?d2Ya!0~8-G_tf$5p$}SE}?CE-f3ILa2#w@6n~)7AkRwXR}JMOLsB^9iqIyEWn|U1JT(Q;Fy%|4 zS?9XFgy3Rl9)Yd8rbmi}3=CeI>RG#*aEc&JZhaZ=q}b!<2Y~%nh7aJ5o1znbaD+w>Y$4UqI8o)_-kydet_(AejtSB>r^*zbEdj%l16 z)uwEY3-fb%lR8$-BrS}dOktM?t*$+rL=SyN3!h-|^+*A~#evK?E9}cVeVM&-pCe96 zO1hvIXm#=89dwgXADy%aLVeJGXw|)ChOhKRi1RKCFE1|-XMKp?7Kr#*z#^OpF1We+ zgzRE4n_nI;z1ULZ+;thWk-)6Rvb#w2+G<>T<9GF#uUc#yIDbBUk82U3p&8xL=c4~W zszy0Mjsa!dcb@$W!MJp@5un!qFzkFORwRAp3uk~x&k3z13M#W&AlW7W;g z`O#E%3iWIFL-EzYt9lRbF>k(o|Iwq5D@Khtz1|PoBQMe{p{%m9=jZEU)4Lajk4s#0SJ&3j5zwO9_G%6u5n-8sCnMoy zYb$2^%Z_h(=XE~)*^lyE*{IsfD=NS1qcGa}=50`HVPYMJv>25!-n~a#hMC~#DsYoK zT&9r4+-Kjhgk9uk+})0=A3++Sb>I_{au5SLXd`ZoL<9sKj>7~vIZ26yLlj1AbD=Oo z!Ryzrw^XKbNAv;VgY9aW+yT8vjE&rO?lep;0>zfM0RUF&nD*FyiMg=H96zPVXqXn! zR9)fP)>c(`LUcGN0A&Zxy2$~zq8nl>z8Q5*kXg?i*nbgiH|f$_1DC#Z1|(19V+$l= z1_)^0BfbDY7VS~RY5{nNt$68o=gw$u1p=@dWX{fdep^#jU!U2%3rFBcYbQ$296pI= zWl$OXL+#VU8@-h`|DbS4m1rR$(ECr9es+3Y9ew zRg|Cb(64}uRL+ud==9(IUV=3|3HS|*M+%AZE8$pjOzLZ*)5^%9%jXGEinXy+Wg#Q( z5FemsbNx!n?2nTsp2*(J&=G)ZiEWWtK4ic3(t4vEfb2b8TDPCKZx~z*a^nQX_pV*j z4h#C>qYN1DC(tl?H~m#RX2r@u>~o=2J*e=E2JsDpNph@94dK1Q8ptsAzQL^aZ-TWF z3a`Tb4%mG}fbOIB07Vehy`P>_uICq_0)+3yI-B-fp76b$nCi0)nD5-i#e;+63n}^g zJ*@=4{G{bAU$m0<@wL5QtJ<%a?RVmt@ydQXWtT9t;4vhlwK?T;(&p!x&CXWd+Q$yv z6hD64K~A?Rl=sP%+`*ps80L3xRF6Bi&NUC~+d4W%?sKxzikiec=TvM4$^|eF>yizX z`J0UT=qX7Bkaoei4fJ1SU+UH1mh&R&-w*Oi6BtHfmCjrZU2Rdwfy#moP*KP`n_jvk ziC8@*AwdYI{2N4vtJ~TZ3h&1#*+xwzL= z_3;^{74|97;>skjS;T0H|G8P&J$u%orCNDj6BqARRU8B=cu@?yCf+vwE7>WgJFb6yi;ui5!OMCs=k8rgmQ69dPx zp#T;s<7iPfsonLnuFHyjxO5$cZyGvCy?=$5D3$K76XT}v6%`zE-!n21*$nYpW^pp# zcq-O~>d$*E4G=>VnA;i!}<0;~i$M)rL5wptY}5 zFjUDNS)Yi7UX0Mv*`x|dslAZ9#=F3!u-Y9^!M2BjsQDxdRuR>joA4eJyvjp&l{(~b zBj8nDoP^>#+eVjHAZ!qBtcK`r02&atSAK;_?SPR++o6@V*{uFL>WbQ6CFOXQg(rH~ z(H$LgCUK)*UqfGYDFX_ZZ{e})8S{Dh_`IfjpYO#~jogpUQ$g4%pr4LM++tm|sstEi zppCxZ%aV>WU0+L2bo-GXQ4(2~o(9K|Idr`fN2nC#vcWZf!C>u$#7_`IHE6&#=z-5- zfQ*Ec6eoo;Q{N9^Vu~0P%e~jmscRwGwjxv~h~e{W?YkE};{ zIUAA8di3bg*#{Vo*>Dc}m~+DZm`s+G)8dqo)rM4O&>3y>l^pCs`kd(y-P}NjET24N zr&YvXKQfOdPl>UVbs?RqDk{Z@13D1KlTxbs5Iu9@GWzS0+ z&LKe)DM!g-whuQ#7K#1)H)6ip?^EHv>x{^^E~WHerm5?x*%3vR#~BgN`V;0JfLy?) zWyd7rJE*C`L9SyIdJnfYnIJgjU3nsW>6m7Op>{JJ%MGQvYAXFG68o4jz(vP&6BV%} z^o0`83zNU;>Mr7Kf7y$NUs(4ukkI$mhizv z*KgQx3w?6@8`&xU7nV9nl214A@T6?uO|gs$1k3(wqQTf&`vior{a|>KzWepz&{N+a z1v$HdllID)m6!r{w%Cf*Mf^s@PwU8l}Mcurk3EZ9qpzA*3pDl@YLdQGy~a|m{< zEb2Aa-ok^NXpLH!ZlLo$s7UUkxkG}$Yi2OZWpe2BKy%h`I-L%}TYFAF!uUw;=Vq}w z+c-l}fw?t8>B;v_&2blaOzlEa)NpL`>6FoerY5QhmRG;VR|t9-FnBn4E@Ry)hIFrk zXn0WwIHEgwRWv-cW+vZFxmSae&+9>Y?Y8Y8yeQ3TJcwm(%1`z+R$>*_*LoXG7Z=ZiHp9Yxi>U< z-e5MO=a za4wGSid`sWI~jH{b^f3-w!tk#gZy$SXZMsV%!HS!kHC0D=v_d8xtfI~8MF)nB|A(7=8}1CdeaaRW;_-%p?K zPai5pLDdsFozluWZ)@B3a@h|&=81r`&}1%1tC|9zL2jEJ`ve|2Xz6ldcwZER`^HOZ zrq~$ZS0f=ngg4%S83M%IVzb}E>`(<*+X39-=Vq$;NkM^E@s@H-x(sS)vii(}GB98& zX{(Yn6c6-go>Rvz@jzAX2(&e~e+$9(Q;>}^zI6EU1gV$6Y^XaJtX2xu@O6>wTKUXB z-44{l6-{|~RZEP@ElgFyw76HYo3!~W7mCT1m2AGB6P53|~=v6G4cDP!{UJ)26MAfs)* z(5fNiM06E;(ipBlh5TaUfOlW({`-Ge$TM<;hW-*nnN(J{tQC2DNTg6~e+@NfiQ#C; zQGzZnLY5xDd{ibcSp60{CHF{<+U)7+nF?2^R?m>ED=fOV3gm3k{jSWQLOUc@0g(6? zyz%yvrg;aHQJJNY{{GvM-3qb_Ifluu1@w@UV|eIPOlXSW_zQM1;(D4DDtsVq`&CZ< z+hE+-yjryo7sSkv6R5j*8R zlYwViC-SucnDnwuZn!D^`LirNb3;4mpoc$tbO(e=HfD0+bzhWEPkgZ;$3Fb_L5f7k z!ul=;nau}W*w}|HZ7ZUfj5isU>okQOJ7$>YA~7}RMwLH}Ph4v#b4CX*6R=j<;;t9u zOvnd(=Ka#FBS}dAPyyAO%(Dg#fZ?YRbybS zq=_|777!7%~SiP$027gX82q) zXAf2QhF@0t>oJ^UF@_}owRK-VOo887_!Y3A+oPD}T*u%~NH~}y4kOQxM#nc&(KvpX z!SRGV-o}kvA5?5-bb{NJ;^%R@BO>!mm925)s75S3_u}nBF}v? zx!chy`efe-^{o^zB#@g|(IcBOD4XfSL6pkU8y+q0%EJ_55H&%i>ew2^rl}S;JCrsn zk`g4$8k)7m$n#ew!^tp|dofN1G>0N)NZ@G7$7|k(^ATY>#n(z_1UENTs2hXe9|7As zon*sXa2$BnWTSj8lHs8C9B6`ps9^Hz+mI7g1H@(ZlPN)cjvFRo=84*YE{B|7P;v^6 zLbg7*Sd55E11eypW5Btg7(<2W%WP$`Qf~)oujwAf&gs{Q3zkEg{Ye>(`~>i5(iWqdJY(-CO4SfKdi5VBnwwq*QV*<9lWE$j4-?DfFF z_q$x7vEbzDb$OuiZeq9#rgtt?+lw5=P722&tb9mMI6T0`I4xTAPZ>Ns1=i4w!#gOy_wZoNR{)aV#s+1QB)-gQIL@$uF=Uj&yyBF2EjS@If?!x4jkad<{QBhL{tvmeoqMot{_dHB(^qkHu!2SP<{=Q zWtvulP@it+%?Ad@2uQuv7J!6uD4`6qIkwMOO-&7iOc785z>5MHX)B8xl6&?fHz)py z1AI}JEv?=nmJ?jUhnsrP7Mv2UxQvLI8(A+B!r+^4!){(|Ty)mclk!B=$BRpBz;q_1 zkEUKP_`AX)7Ll9Ai>On+M?q9V0+mHz9Y^3K8Ud9oZDaDZQ9ov}a0xCoKwsi~e1kf6 z2}*cigW7KLxOgOjl8{P@34?f$N_Bnx&%(p%=J%&>ta=Bud~ap?oy`Yx2OMQEErJ@i-380(|3%4P-RB*?6$9U2n9htuv$dv zfGB*Rr^m+b)D?TX^eR{A?l$=6RHOr?Sa>m86NhIbVJ0f~te>p~;0PSwt{XSdp^Plq zCd95~Z&G)e(`#%2+UC+XSJB~H;BMunsH-#Z)PUg5W%n6EduQ2)StWILy*AqkcrnGJ z`BAf>u8u!;D-T%6QVfqWg&#v*L`U2FV4H@qD9rgLxrJFrL@mFdR4429(qN}byjOvc zxrp)fjB`E~hTl9B{};MglOHsb6cTHL&Yj{yGubG$>7h%PF4a-V7?$K7S{VBeYod z1&?A^DhyQ-tflsNt|wv^;*t?PPgI+~LOiZ|&9`pgfuLs)F+4FncDUyJJT*yIM2>D2 z!f|t^ZjGL|NF)O)zRVh|zPzPnPt42=jDI%j4I>Y_J2#DBGQEk0Xv3#Z=`VsG7zY)* zo$_Mta&WDS5+VzM=#M|)bFwD9Gy)HKU`b$c>?*AF6HK}(4xmuFHa|%%NF&Q^3Z{u7 zJG&91cyqnqW6AG_8Zqs{vzCASsDN?r5jyTSQm%@xK>k@Uso5_ES}Qc>C^U%Jk&s&{ zR%4(sbt|!8(=VSHiYi8*JYLtq@$?grQt-0x@IuC6tw*Ao^;!EG9A-lAh!KDU)=)xH z^3{;gqERFM9apIvh&M=kW?$)wKm|%DnflVY-(H>={o!|CbnT}U#kC%!Q(OA$Gz<1(5jibMX zE)H2O|1D#8oaM}K@sxOK%illn=WoT8r#FAgV_vCcBU+fWKrVdxiWVBRZ-Gpy%-=wx zQA+p||8`yiL?tkVZ*+7tad+h4Iofj2-bCHGkFOGfgd_*vBRl&DVrTfWe&;s89s*l| z%toMAc7(~+qkU( z1c@*ia*_>>V~c17%;YXzahhY&laY%J9v*JVSq-0bx6EU2(?PVqBAmey!IW8F#H}P; zz#>PDqilp3XjS6W*^e|H^jI=#p->^6jsVTxFXy=LG=&i#BF5il^jteR~ zOHR74k?h{Rx?^Le4M_$lS0T|gfC>{CueDCk~MS&M* z+hx3E?gK6_yDwt`omxTL>&v#l_;rRnv@81Lfmwh|f`@0Or%k~INrfRzSPX4Y4n&`c zO6jgN+FA>#qrcDJ+j6o;VPh)%UQC5my}9O1qb+}af`T%E7J|qfp-L-^#304g5MQN; z_dH0}d<;#R0^lZd2PViN$5?TrA8E)56d}a4{wq=G%~IN%(%mtOX^n?)wx*15#9umF zkO%*S@DVS1h0z|+I-skRyo_w5-r_-hd;s#5y%6ov?-(%Aeo#HcVuV}5^ZOdz1Kt!* zz!)Lyt$TSXIP#q{HuY7wog}|FJPU}yZ?JZRUxm$v^xfLLR(LNU{9`$W^Smu6xl}E% zwXtcb8eB^IwYw^N5zU0193ikIC4}?^3*l=tJT=y14?@@lAXSCZJ5w~1R>X|A`sOne zw&Gf{4}4tO`XQiUUEQzso&&()8^(E;4Wk~!6!wmf4+*0fOMGA-u~Ncl#3%yizm<;+ zL`N|9^1D-WbF&$?@8671=VT~vg2N^ntzMNZ&=t6or)TZ(yC00`&#C7Ee0-1=GqbS~ z&I!Q4z&k9BO0+9-JmiKDeaU!%`tgZ-=#&wHn-^%KgTI8`*Lb#`9S1*Wej_vOhrr*K z5Qc&DaPf5zLxxjmZi^-e!cc*`Qhs5v?U@Js%&$S=;H@dBe+WcESz>!wx)t4>yRUfa0+t#1~IA1jXm}1{XOqNE% zzesL=Bw%UE>yy3h$W~!ujaXSF8G>X}ABZiGzg~}QEX>%Fw6XmS^bse2`SPj@|a1Q6G!hVuU0UP@} zBT!wP+UZGt0KNp~1Bry4T1#jqXMP_wacSx2d2BQjhQIqT|MF2C;-#*LUB{B(&wun< z`1}9K=l!G0gZ|II<@5fXpZ?E(nEH?2_<#P=zpqUF2iEMr=+eKhjP?HwGX7}`{=cj@ z^?yZ2|F~2BE1&vbADa62Y|{UpOaHzy*8hLQg#Z0d{jU#A{m*dppLyxuSEl|468@)O w`uCNw{E>CN`*+U&{GT_^yXW{~f7~+6%zfS0wbr-R{kfAo{}=gt%EN&N|H0vKXFpoIJ-O#>71@oYFhd}v2o7N{qd(Lk1!(#6B8#P8b7z7JOP@a z#om@?l2A;GNo4p~IWq+n70XP2QHWx^WNNS`Cw5U`PMP_dufV0f zHav?v_ncGe8>e&uu8jK}5uV23osHqY6 zQ;5&S;{31PZ3tpg9;nO?D18e**t7Sa6g>?SBcu8>)pS>fHY?8u7E~WTeBgAP)#;Lo zsIIQ2=iyNc2nbLKte)!541D!!U$x7yk%>uqGX<%>K9E5*L?QB4TY6PC&$zapUi8?) zX$gP$G_lFV>A)K=6>r~uo}p8c+x}+T+xuV`ChS(q#B0Y)rKGb`q`&gvJp9qC6|m~# zT3;WJF&Zd-(q4sOANyw#tB(oIh)H`s6@iNhXL!hQ-r*b#jXt~C%;Q_qE}3OmMmVGbdsDwzak8KKgoC zGvCy|F_KSt@sNXxN|3m?cyUzg?Dvn=$&Mz&*sfa_$=S2;`;3ET&V%OzPiij-S47On z9P<)W7rHAcDgEeCocZzS$VlvB%(ctg^xylwb*3tFGsOM;`BNoO6E-rs?eSvOyQ`Dw z+2NO7e8Ah<(zCJ2DaSn-t@G!6_+?+!aVFNb;wJ{{@B+cgn9xvkI35-C)ARE7WUh8U z-+N$mA~`0wH_yakCu0Bp{R?n2$_RPR%aiTHVT0Boe70lvB6u(>mDRSzv8F}lKfF#0 zh9zU+7UJRqljV2)sA*~2ljNvU)wA^`T#e|(uDny>Ozv@89c1u2cBwtm?!dD&+OYlm zylZM|@Q$8cV<)6#WrKv>HlA+thI5(GN=Zpk5AXh|r5W5YmX&>LH`!VLi~n`py})a7 zc3(&QoJ_RxO-B~1Fp7*jckUQ2jMj6yu00;4q&IgoapN2PM#9%e*i=pw3SPS=x%${2 z?*Dla_i`gIiKnL=U0usF^s9#0r1J12X(B`3i%ct6{om4e;1Hx=L#OYdo$1U!`DC%ecSYoU9Nt8_f0k-8({(TtslhU6va+ zK1y$DDQvz|SHiq5QQ;?skpD7w<@6pF9##-)* zi$5)}$?2a-t5(d-&4o((>7>=CKv7dF$}|{R>d7x<DXG&3X88_Q1p6m%25>;0j#|^qxK~ zhVl}h??|W~c^`JuB<;A9b zydhUOX7{`GwmN6J&2XjOZYH&mb4oLoiJ$&{Lxv%b?1R<0`Ur*~d)TNPo8Nx_Hr4%; zy8NkCK(_n$LK;6^+H|?p8Fb`fzyIEGfij&&N?*pc$3tfl-uQi0N=}YE+-C~T2J>9` zqVLj7zq;qKOOqenrya@~qXc@avhK1~lk2jF^b4xeCn?0}J#>Be@L?`y)z8Dk%9>#) zBWTMX_0x`@v@w-kwB%m84>m>lZO$VHMa|62tcjBEE*RaRt5jTJxx0Q-q zeL~gx`g*vh*QDD%2i5JhUw&!CbUc0-cGt7eX>Gp%C*^WfBDzSmel)A4jiRFc^D%Ig zbeA_~%jQbijvPOZt!{i#oiR%xr{Y=a_sAiV@DaRHqjMuxSFT*KuK4xHz`$OPOYi>u z*znLr<2p+pD5|}cZefg8RIW$^@b5!ISyN{6&pjE8zMSgABp=+DN#L^T>wI-DEh|n>bMHS#Pm~R2@wYuS zHcn#IuPkq0&pXSi5$hcI;M?2tr7q`Z`wG&f7dP4T%D+$9vpibi+5@+W(|%IrynsVS zzl)gXQU53b$KY!|3yX`YA`ALNpyy~5Cd#dG z`t<2g_grmVI(qt4dD9~;lbxwo<&!&FIXOyy(~1%Il6eqGN)_i72}2&nX({SzQgg9P z%1L!5F>Yf#1CLjxd+|-&Jx?RK1fV~0ua7>yW%#@GL#aX6!il}mzTwLhE|HVlEX@t; zSFVpyb^X$}w6vVKUYx5i%wmD{<{G)n#7_4W5Us+hM{-67H2>=xKC5eKS?#Tl?~4!W ziWGEex9aUHut41n^^u^RT>uGbOWtPO(PM|1SeTgVXKoJ32I~xx#>Sjw4Lv+P8(?dY z!Y#7=zrCZ9f$9X8SqJx0TzqtMt}4W)a$OQu2$PFZu6ujRfQ2DOhdreC${IPP+!zsh zg|7A25k)B3%WEp_Ea|VHVSar>DHFo3J0L7zSa(9Rz?{(k@uq-4zd`R!ljbG!N_vIk zHx2@RfyNswFB=mRlU+9b;5zhFKG(IJ{-(UW2aW|DqfzPiqPup@IZ~TQ*%|OTN&s_Q z=^y4j{2$PJ0ta8Bw#-eEknK1de-dCoxYv@C0ELwVG|7b9FaJrK`&v_T@6n@21deY< z4j)d-ch3pEY2rJ%@LwpFJ-R`DLkEAiZ@4}K08X=&rm9!0)|0H1G}-RiRCUeE%@0-D z_J3P9rFuS5u{vanX^%a=mBw|x@H~97Y~(G)<%-;jT*>>sxc%WS<0J3>c-)T#J83+Rg9igF&o4&oFrDq-*-&d zQITC;aQxD$9KdqoY54mxXm?XrDAmdzY>q|PlvfH&Gf|8YAL`qb|M65`oQuRnZre)0 zV$^nctm!%q8r|Vi&9m&~>^d^!?d^@P$ZeP$uBsX(ky5)qhu$z41x!Lg@#j{ys*;it zfg@VLevf=Im~Qbl3$t9O05Mx~q_$_A(0B6hFVcI_#+3U0~63xCbFn z*E7C%JCYSb;P<89oMlVU%ricD>5{JV&^Nf$_|EGBxY*JoKr&ieTk%BLzgzsfFREsS zt1$icjt(QJZ0*_lB7T%nMslIoZwIq!UwcEz;ZPP|3>Smn?Os@$e=WZQh*5E3qswEc ztYV1nYYVD#PuJU#Z>nB6# z>p&I<+N1Bmmh}`F;$vl%?eO@%I*iRZsI%UHlIHFqhsGk`aA2WcVu4Ol;3$(UCoiCgTt~FdPo2yJOIbaCnQ(cO&Px8mp*^+S; zuQP*~%Us1Z@n03N&Z*X3)g!H4w&noF7obF@UkfAxreB(hlE)zbk#Saut zHk?bUJ45s8p+kp+V-5<6hzw`WK*d$NwbTh9E^0SSs6{y^23k7}>NE)Op1)~aH{F|O z^ylYSPe<*d{u#aF5fLeBVW@Nsf0=Wx_eyv*6anV@|(6c%@wo z>3Vu0Yut{Xd$mt zG}@Qe0k8NeWOoLf=uU%GNh4CSY0&{W&CYi#(P0l0bi#7ghvi~~-At+WeCo)F1CT4` z*}P7CvHysQp9Q_bxz4LXLb`=k{c02AL#~T*KJWr>hNSg0$GBcwYIqCHLQ74LYC)EO zkPrscdSRWlCH<&p^2`nkZ_SGp84(6um0g@u`l@eiv3+lL*%SH~=);D;x;)jxyN&sV zQOL&#>ypH~>-?`K8r1r>XXy&~QAPr=v&QaDy(i@4Tu04j3(5d3tx~q0(0Z-N&dBni zw?tBaYPwo|hGw3)xM63CBB%S-(+&pBt_I~OdOEs$YHDhU4B=F~)?AsZqK1YEfLous zMsB>}sMHu)owVaO`dQW;PRUe+jn{NbXotHxM!l=g_@O3dz7t-q|!qZuT#bh(gx{ zlH{Y(oKg(`!?diG4}f+f7X%PsxHCa5NB770_;{ZUg#`|WUB*d?-&P@xHAS$ac05hOzk51?o4flO9Bo-AQnlDH%E~R9k_)_@At22abjM8qEpH?)%xM(!y8U< ziEalvpwQStO(pXt%-&ZQZ&7ZQF0G5fKDe|9t*SizISu5YFCUQf@=?tj3vmy2w$@4P zp~{%DGAXY`IIt^SUHhm1A{<@nQ+HMe9b;;*oZgxCP)9}hFAnTTM!Et}|3u0ca^YMQ z5{fm(dG&xN41axlmsD$=A8;hii^g4})M19%p{&9#qH1@?1xUg2(+^LkGK-vL1y#NR zV)>WcBNYLDEigR`E0spAJtybB?5fUB8OT?4vB?PmeddQV&b zy*IF%!-QN4CtbA0HlCE|8gYups0B4< zhnvE=mA7?r0E}HfP|u3SS6B`UrA~1zL?(e9)tjUTs$CTM7ww2-Jgz?%$ z*E*=>idgsNCcdTOmA`ll^D=VAMbuB&Ndw#lT~4Xv{RqEQcN|I@lnW zsPN|tb91Xg15AXPkMGfSlLBVpn-!T*l8O&0Ksq{*4K+GGW%tJ6D=}$$#!?$WDsHye%y*wC<=5D1U z)BrS1d8`?f%Cv>}3NbM;1S;Aq-CXuwT@<=nD8S9#m8?L}{*Sh#$M6z*Y63JwJ#v$G zZ{G$%VNgx&(KIzR^*E%wr6A?Usvz|ekel|;L!M{ob747!nwJA3B3Lk+s_nwDXb%+1q6%>?n=6fw8W9ep(`5WuM0am z0td)c>jkCH(B})HC4w_cjFWgHD`o8Cjo3vvj}`N&YqG6TTu*LY%zf~gOlAeY2F3D9 zSXdY$NQ3rsz)j#Xg%#Ley?RwQ*_BSh0AO}(4+18a23cv+@SB!|ghYLFlt9JCCcXU< zXs|BZtLbkVPHIAz&#Sq}ccy7sH8-1OGvdkr5kkhi_JaTguslwCZV=WBpY4K1_WI&@KkI9x2XT34a1F%+j?Z;l-VKJW?tVUe!cp~FN(Cq*K04+8Gs)?v+ z(`ImEV`IB*7)y4!5@;B9)87@BT^6}&?$12rB@E6le*H7EW!u70x%MR|H`8wD$fsS9 zD$mB5+NVzI;((2y?h~F}T=NU>JaG0;qa~<|9ho7<3&Y z(7*TG@REThQNG-*{a4$pa(7Flq@;IDC(3(UfBoP&GBSdgJ|c}=^jBp(xyjF6|70_) z4dVOX-+B7yQP+o(ae>`ORcT_X(mR#iM_Ly9zX2{yTUvdoDKacPcUH3Xt5kY_AwEKO(zP`TAO>F$f=a-3x>MLAUODnQ@ zYVKP-P1WSxIt%^(uJ1*r?Nz`I(<|pO8=GK_iE_$kyZ()2C+4AqkU+Csy|vl^N}8NP z9Q^lgIIj&m0$zwfetN!_URZdjGc+yEQ13lR=IbD<8iAmW2h`NQ&kqQZCpncg+-_1* zgf+9HlF)-sxFRVtmhiiB;t6h>D*-`4h8kOlvI8~r@x%N=pwUT#PXH7*cj?h}&^s^i zSTYJZFWryo0Fw0(U^Lt7xN~O%MZV9h;3-vMXVEUKM%vMz+ATC~y94E}Je-&bczdCN zCA*ldYVGG6?hpjmZO0n@4^Tf9Ewt>V)puVvni$5~;_^ z^a{*XCv3(6O|%LQN7M~lv9PntJK~uH1$CfeX0W8v(*_U-MydR(YS8HHTSm@NP+UQ6 zN86^ND`)XQ*`QwdGdun$Y~dS~jcMrzrAL5g(++VLT=4|KfPHoDb#%G;D7lD&RZ-k4 zWl+^*MegoB5eZxT;M3@9ma0^f50h>fd<&WyJatrhjK9P7ZchZA6tH{+=i?x{PXMzp z+X4>Jogb}wZDuuLT7uxcGLzN#l~SLz!g-l9Ap>yLU{wePb+$U%og~+PpQ>G{cZUHK z!yZ9xN$dD*cXji=LaV8dcjLQGYSJUb1~fMSK);W7G5MkC&eXspFEUg(06wN|=?2M} zS8UFfB|dTDeEdUHtZEm!Wpx9yOv~N}o$!r7zIIKt%1EFLrWlqmf&=+i^LD}(4C|Yk z2*3@h+H1O_9ETxK;p>&z=?8 zO(=nB#^LC1NLqh~)93&QF%Ud4K{(q8q9dzrsS1z*E%y3HUE*p!Yim!D&oV#%xM(JY z(dQ==2iRP~SXfX6os?$#qFuEod76}X!RS5JbA;0(`l^ANwAkvwmCj~K--{C;DYHbx z!L~3oG%Ts;OhqGf)qksZRA49A0-d6k<5h7<4gQ@Ij+Z*h2VPOPBcI_ zkal?>AuhhQb;!?|AZN)JCI9z;g4#?SV|orMOPS-`+2jr;Aix2Qjj~0Uo)Od>))s!(x;^2LQ0W6HK=W{pX9{&@$PVl(-@ktu z8fGMVV+=q_d14B1)4?Ot~1WZp)n+MAF7C(va1X}!{kXJxJ`vR}^^s?}0 z5VCh|vLs=xgKwGld6AOPnnXaefb-G`3EKBCLR z;c^ZJXl@h(VF4}y2tM7eH$^d?O9-3!$O_A$TWa@2dniug2-MD0d?fC{s~0cGQnicN zeOPf_WYtIa;6(3-i(r-ITMI-|aAH!DG$yGR9GW;tVRLY20$}t020W|>M~UBbd;KB%0DlGluYB--J{LY8{`~KK1?+!!`El6);{jctRxc>jYuKl3s z>$p0wMQgy>MgSbh`lGzwrPd;!S~zDvE$ujb;Z*+Dq*% z%)aM*=pDhWAwscdbUX$N26PETTBh;X{(q=A4ZQ^y8*jGfn}mSu+0VNI4mU~E{v9|j_tl%Aem;&MFWYaNGPqxKS86F(#PQxEGe zn5Dl`HCSpN2ykNEt{;0b2th`=e00zZ@>2D zvL4W|F8?E$cZa(R)LIhICW8AeEiH%uQJ`t02LA$m379frb{OJ5Z`e!kL4&IeMb7X# zDMX)npo$PQ{L$d9S$l$%YWfEyU<|fYm5FDu81Fi^q5)d+3tacv5M>Df1LR4Sn#1r= zhUU$Z5l<@DLOCET{ve>fuRE;NBC=cFnW3o+(SvL6sCanGwfFqt zg&y?PK;n!T9F@&;UB`bl&NEA6iD*iK`y*WtIaub%vb)h!#TCL0ROu@Qf-^x`MJ1sz zT1eZLFGSBd6~NF)2S2~VbSR3MG}+sMyT=H+c$!hg`dEbY%YVqZidulbF=JbQycCK! zLTA2z6f+yHtQgw>OFbVyT1Syr-3=*~tI|3;kz-aCJy}t?2DQiXs#nTjKWmi0yKwKi z&sD7Ik;<^SN?Vq3U|9|2PK#VUj=+8i;69EX8h1QG$=<1}4z7lpp6j2lcoO)M zGQgqV%;tgA8VW;Jj>}C>40E zl<54^hru=1>a#WM4e46nOZGjh=>Ejt^$Wk`?%_*~2~BoMID@0SFyt~9cByh0lsiV) zo*lXhe}hQ5ZlG6`63qsRKY}l=mlX-16-5Mkbk#twSeVML+DM4j!?x`cofgMR4V<_B z)Dj>=)Q&%Pi9g}!c}{UhKEQU0;HDe>=0QQM2Dn_i+AqX#4%iZJsW4u7z9&CleV94; zOc-F|sWX(6EPQ-?Ni;!D>$}U45?}`DhP&+024T>8H7zTva$^!t5=0R+7p17(K9G3K z_`ZJqsu$9cuFetD+2%OxHm)kj6CnrtRkkG8dFl6v{pKroKcSV-p5<)_B%#ZOBmHZ+ zP-x?KyfNlA#|zG%-mZI#f~1S~Fl|t zhJ~3q0E%xYdNE3Pu(R_B9F6URz*OjO|GU&T1onlR0;62$!Su1POOl_C)YK?m*1Nx_ z@{UW9ujqjOKY3L<8!{lZJ#;vCT@e^b?-g9vS@bnn)U&wIZ_o4<;8&(K^QsR%eg5=H z)BdY(Pye1_YW-EcyYJ@?XK0j=i&}S zruz@zT74l+csX|e�%d{QshMu76*8Vy?#a`LV0}on7Mog86^{d1mX#glO^nIN2r1 z!%HjTEv!=qYqk^pUs$;S{usInXnXz*#Z}4QddX^;TAZe>v3TOndJERD+@qgemJu!F zA`P((A{!OIUCJz;y=7&WAME z_ChP|rsaV$M;j7|Dvw#l!2|qTKk<`2+1ZeAtVy;oXPHs%AmCe1P3Q+sYy$} zhR`2(&nke5G~EcU_KoE!0^D(}MMsD>ijZJcZU|V)Rc^0nC&E&N7tN%lXDJOm{Y8%I zaT6B7%K}@eLq`FkiBMGiwD~{)ZGZ#F9im)htxM0xr^%hPv}8xm#ie4a3CFt#KmX%) zOn1j(i%2$?X{)HMEEF>}l%gxbL!g2H$JK&88BR(ANIgd19ZsJ;YrfN8{$h3ir;9ui zhvqA1L{BWLI<6y3@s$$OXMG*#-DLc$N_A^aAKRt}Won_`QB^~O5xo9CQoYZALsv&k zjly4gN>|+Q%d11w#J+N;JlDl0-tdMANa3LLZ$TtTToT9&EbxLAAnrXb_~3D7F=VP1 zy0vY5?va+EYu@zn@%f|CEyGbk2NxMWY4*Dn`v7-0WX*N+MK89qNW%x_X3~bQuWvAq zRbS}Ej}Gva3M_kb!I82FfxK#v`}RiI#gBZ2U46uHke zspZjGC1V#R2Fr>b_TN`gp|hXr=FTY3jdoknOoinVjXbcw8dHcCWKt0P^GczTMPdYh z;rd=w73dj!CDAuAod5Be6I4TU!sR~&KYu>bEWsxxBqtL=4x;x zsEOlvZgcGV)whb^GDx?O9HzfNH0{-q-#b?PX&Q$+7Y~h_)8N}t^u7sjFpa%VshD@B zTtQkD%3WCUL>L+wghP$}bBop@;~e6CLChgHj0O7_QC^!b?Harv({u!3mRPoCB#K5M9%L%EwM)G6m*c z0g$Pw2fXaY<8>A%qKFHXJnH;f8cup|gi>ZgV&eIgLCul+CrIAh+IPplD(3ZSe zt|UlC>)z-}Q54fGvW`+PK-rbq(1nEs*9(D)kh36M;5PqlzSmE9BSpc$8CJr+2qS{@ zvCfqQaB0$dZv?N+ikU13Vx=om+~%D*ji=7;Z8Pu_N-85q=sx+zAmo2b4IqaVqF`|N ztfX*LYlpJg2o)@ur<)_L@$+@D%Lf@{1rO1{U4Mt-MCw$^rzNOerf)cbX775kW1z$q z-79rj01Z&-FJn%-+pBITJ3x#=Mrx@+DJUqU)(>LBKJ|>?2qLExUK-w`cv>RbY3#g| zwDf|MEb;{ech-KD_8a{t49su?d8R0;K4rZZ6?(|atkESw!u>92`u3U1Iy$T%v;KXW(=;a@ddo=VP$GGk7+IPTktpT!S zEU+s|`$C(H1CwLiwItu38wm_Wv;|pompTBsvIEY=7U6DIECry5Sr0$7MA)rJZDY7` z{_Du&I$(-+J*y~W1NMrr58x>IXx9a)!5eI#Zi5~NJ8Elv(Wpl7I^GzE`!`GAO#wR; z8pOckYI0#8Gw5#Z@pq5wPJOrJSqJ1y`uT=qMN0PbG~h7*yBN$bI8tdqxe>p6cats& z*sh*qL{YD0uI;#d$z)o(>&a6m4*m|?uQ9ye_y8_;()EcbJ3D(YWSwLb75ztyHf7)g z67?#am4L?QiG9om#q%EggjFj)rM;!kk_UkBTR~|0!ta=%m+ovWDmX#s;x5=-UmPb6 zVvfTz9icqd1B~De5fp^i6aM}ZL^0j;PZt`i&*6KBC8b6-&98^A{Zr$%>tx&KVb;foMBO` zgP6`}N9Ye41dCxYh<|YmB(fIy)_5l(a}L(w#LZH8pLluc}@v*;rW=jI9gDo zv#uB=CMMPcQzmDU5g&!Ef$o%)ynJ`$gX7|mo_!Ba$HYKMUT3;`OAcOHNr~>tmCx^U zizEL*~m39{li%I9B13cK~?;3r^Hz!|@M@SMG$TWedmlQ_9|42=+i`@Pd+B ztmel@9s>Y$&=^#n*q?Nme=)VD0lXCqC3y1|?JopPP^eZe2E~zeu3dhruj4BZ<3rWxzt=cH=pLdR;(RSn5O`fk@er-}=JeKci+}Ca^ zYoh%RC?mDf5%49T|DE{MNcchngZ01*nD(E zVBF}_vpoRDG5D!kceZ}EmYR}M0Di=S(wv>;%9Yzl4pS(yfy7ej${-3Dn+XDBMHz*$ zr<2{8>8yI?8U-1UfM7vt*jVmAiNYW$-)2}3yt_A2+S$4Qie@n@n97S@=u}o9u$X`ai3}dQ5c__s zA|^0>InAaDP<$Hbm;PXXH)xYZuet$35MXCdu!H5?Y|_69$ZX0t8H!c|P$svt9e)K5 zJ3#e}EyA0YC3pmUU}H_NTfmZ|T(Jys9hfBwYEQCbt~}la$Q>lVMq!=0Du&jpm64!`48>Xe%IqME%j#@bMKo4b0kO)`9zM5? zd>Cq~9LSA~yods|b|D2IZ!j@3wpK)ofWG&%YYZd-3)tjLMV7EodU8LfiEe?sV!04j zVSpJCVec`U?DB=zv-U`ka)M88VB3VXF#=z!l-N*L_Z~d6;GKR5XcjoJLCA{{(q=ta z1@#Y~N3ER~Bnf*F-~S6ux(>|@R=udGeCf9N_CA5?+ISDg@{IO#jp2dEVwdA|$+|&C z1Jq8fN`K|c{;OnHKKv8u=Wys>z+?CGUzTrZ*uttJ@_eZBA)90YLm&NrUySuI2rXF3x(ll7 zug|c!UT6WT76i(+`OXZKWPMGTh%?s&rCwR9$U3tm12khMkZ?h-29&6F^3*Br9_4hk z%pj0tW4Z%-VTP8w$HFK(Kc7qNQA0}$FKd#hYifa<4b1-httH=2d*pLHI2&JjD>)&d z?$x1FX*$gqBuGc0Ve?*&`xFGyf7hkS#l+-f8B5EIJrq8mouCSQlu4!ubgj(wSp%4% zKwMn-%~w;Ncez5zZLax3q(`EO)+Dh@}#nU z&yrPqr^96LXW)v4U>Ai8_rolQ&)c_4Yde7JM^Rl4HpuH81;K>?_{74ug9<#azh8XO zao~y9@%Z%!z&4<3D|85N#l2w<`28C*^_!am;N%W6tu*mtuVp)!B!1&$kk$t=e^#1E zK~XXqATtWX8{JEq>CB+30)ho>q@@*plv=R3fi&G;r0}J>n%tZ5`XwP}6_`C$BH>{? z1Hry$-mmoZo=LI8_yJC}_V(#$G{|Y}_2VYwM&5%$u2O|2g>7SS0F{=npg%2F&h0m@Ra$dryPQHIXify}N)6 zlP-L7v=9pnV6>_D0+I(Fj|6UWO_ANLJcxFS*~&uqugstoi{i7D1W26V)Q>m=Kn`Os zD@}lXA!wv6yuSkyI6{Ro@mmmFL`}2F>IVqgYF2yUMnKX(*{MIPVnxeuF%G>?R@p2+ z)^r0#*E zxC4ffb3p9^Acfade^Np*0kTne@M7>pkYh=pvjidHw>T^>C52+th{X`uMC^XQhw!DT zb@tz3U`%cl{I|}nq9!;5Rdr}=wWOpZTsWKY_0e7WPhV=}P-`q%o8m&fUtnWYyC-Jm(QXKM4J zv3ergF8ntU_SXUidYrngEOhfA(6wU3(Ip$vZgH5iWK?03WAUkfT&Vp7({0>fV&KE+C4fMR z;Fg|>As9uW7ZIsi8b%%{cy7Ph85kJue)GMgZQTnRM#6f%tr6^(dN53~Tq-7KA@mkR zsrM8FX%Ik>6-Og}Iuz10YFSN~#hLTxHQ8J~e&$7l4QUpl9*wi@G>r_AujBSDudFNO zva&z?LAnil`yMN@KdVP} zIR$!;Uw#aMe6&&a-DJt1OVfFv`EZkre3@KDVaxsD>67u#mNC@X_i?dAKE^*M>38zt zVwrFG;{F1R6RjDKznFQv(!PGEaN}6v%pRQn!)8_-{eRSr7hz}m9CMmRPXB(9S(*7h zXFu6(a4<&n8^iX;?nAWfv8d$rygQr5#~45{c|Peg%<=ZlqtkyylV{J2Jx?60NxE1KZQ!q3Rx%6$L$$hFUE!=DR2d2O6M069-xyX~zwqx-|3` z`w-hU3Cem8N<9s0P4FY^LZAdDfF6JvKU6^?A_Fc$Fj~%S{+O|Q3i;m19$s|@_t3!t z3z1Z`Sk&r-J)lE1U)ay_z~Dl@D(x7 zAnhI4Thi&6S|(S>Ra%tQM(Tf-ePfPc{mJt;9=*z{(T8JY1bQVhL)Nb6{@{94bI@s} z*90jj-Q~?_$ep8VD;G_5Fx-G3xD2)X+CxOq$L!G)0MNb#xL;|9mW2*NjfjtwM1r(i zAA9@dXqUP^TahdF_4OH~1E#8`2Pfe{Qs%N5(g7PG5hT#?UEv)b7-|WF_*hnr8>B}K z;kuD(TU|#+gu0yw?_PTHDVtXQd!Sc=TV4>ah`&{(89m1UojN;Sd@r>stdtW^9{?(Pyc+Slu$g~TVxZH)WV@t zd>^`t+eOuRXBb8r1^+z=WO-0kkYWjHyYM?RAeIPFG|f1mIko}|^&%1?W#y0tU4Z&y z&YTPyD`MmudeF7Jy}`1UGh)-P)X^%m#DGXYUj1(1R}afig8XOO(mE(eYpkCy)S(KG z1P#!+EjG3+vMa7ZG)~y}@W0h2k7+AlPscfXbw-etKt0hQ;eonb3(c2pPkL3m9c1gfl|&@)mpn9NPOkxE#{*h)n-2@gi|69Q4Q z{OQLlXZk`>I6@*AnifaH5 z(81$Jl=%R0AiX}dauqNWdd>>hS3`ivBVRnicZ_52$z$@tFAp4tWd|~-<1$kq?8|t4 z!Yl-y`7u9PZ({scxv3*FQwjLVpBpoTLL1x z$}mvF;JegL;>vu4nFzIPJvP)-8ZKOa@XT(iTW+}m(v4^sBxx8!h~5zP1$-Skb|16$Yuf} zf;7aJ$E-@B?+}WHoMj3!kb14Yjg%6TWrKWp2ni#g?RK4E6b=VuZR7CWofn>(p*w@Z z1Ka|JO?pp9_M;#`c+niB&y|2?PN_A+*Z3<|kPx>(<3oqEEwiBBFhlj~z;qPguWI0Q z#%guLp-4{GJqAh5f=ICZNHn`^USn=LK*WRK_O@nT3?U|>N122ZQ+3M<7k=niKLMb^Z}xE{r|>`t$zm0vG^Wo6KJ@9Agu!K0w$Mn;VE<}1wk^X4 zq2=C@&8GKPFtH%;q9(JFfw6Z?x3Unz!_&Adx@A`j?crGw>rg~xh=`lxL9?A=N8}|{ z?NBCsLB<|Em&{q`QjmxT0%jZs?L%hq;EsK$`SvZC#RQaX6kD|qu?J|*0(0D?hRL>g zuUR~F5Kgmpx|q&Y022gk8=KNEH8_0ekRn99?`LHI4PyfBsa01EhE9WQ$D0YkivU&7 z6O$lw0;canpQ{0yI~X9{PFNv%c^iTYp zWs%@8t04%7Cn7f_#1@hRy(PBEv&-1l4B)OHI1htUkfK1!H8R6NMEU^Ar2MKghy<&* zxxOhulX3+Ff(AoFcKZ8yb`kc1Go9rkl+F*!slJ&61z#RC4lg{h+)N*!DIg7BJQ0n} z!T4XRZa2s!SE{xaWJ-)6`x8BjM{{Zzi8P_7@CFpFd$2!dmm#P#3SWq|XnKn$y9}>0`6RQl8JMMd*Y(9{?sE$K+ z4(_0-+O3gMGGbAR{-@HlZQ2uU?EjJj^aE@p|IU}Dh8Gp?AhrNBngI;-wkA>&C;dN= zlg(J;0vih*S32$YB97L5wr&T0?!4m5dqDT!s>xga%P@+DLe+@6z|;+jX*C|(tC#cT z5QDAp`dP}HXFQ)PMGmdZ+`NNQ6X5L{dySle`8P51o-C_-n*aH>u_~w@!e}V^DH57N zs1Cg(Sui%^bCn_Tb9Mh*z!41<-f51JO!!G1Jg%eZST)1RlP3}6@eKOU6pVhH^mA|F4&FD8I{15pu3gmHpzjwd2x3x-D_@CCRq zLGs>@MCbne zax4mroLfNEk1GqD8dYSawnHwY`VrT!E~A{OKQ{!=u+p^x?Vw}&J@@7uwwsO~++s@=%D=q((Q$m_hVCxSClTIF z_Oi=0Cr@fW@=5$ax9{Vb#NgNTru(hh%n9A?ai@pVQnGHVwkehM$+V}j(XCv*O-D!Z zYI-wrn8xj=iD5t_CJ-iIYqH=j)FK$&IIy|GXVN?%U^~`aVf!6L{7LZWg^$3NAFl>_ zc^!D;zU+l+ z`weaYSDy%qk9;%VhZ{U`uKGqWt~4&y#sh_P-|Y(K6c>lKcVHdp=@u87x~>Tb2&6$* z{pMHKmuqwc1tP9$LgO$rGfVYnRn5|`YIb9+C@OOF7=D_RoZJCtvlt&48CloVBu6rM zd!y8Sn_vAHg(7sFk`Tg67;{n5)1&UmrKYC7uvzk|vczHL!fGx+<<@kY*1&)OMgf5c z3r$DNarLlYY$K5v55IvYg+@dKCMPpEzVjRf*bo#Q-CU4SSSWX#TJXn^V`)i=e_&u> zLoMV*7=?wSS!%z34=t~#C<)H=1=YILaSlEY#?A@U&*9jyfde$wU=G;{k8x?|jU^_K z)E|>TgL^YDI9L))@{fs8P*cOyF4TirdlN-VFVr)GI3j+WMmPcP&$}`6z?U!gmJ}Di zhX+qc+QNhPxL1bG(2-q(sk)#C~s?Zkf^%TSLRb6d{&suj|SBknFJN?g&)pzklu<{O+Kr!o^ikR|=D`DY?|QPt<+;mh>gYHaIBgzNDliVfn=Ns&e#e_mi-K9v(qqVVXul;=68( z);2aAPI3S$AtKw?S;hO-~YDFS>j%16)H^c*<0J zIHUx>8Mj0?4JyEmegr#)IS<<4HS?nY$fWyt5OCZC4XFX4Al<_5!E($g zP*X$WvEOqDhFtf!qsYzxL(O^KRR0wwcv%6q_rNwFZc|WYvT~DvKfk^Z0xdx~EHF6u z+ds#t8>1H6?$UgOoGgLJ#KiOv#3u$vImnQ)LMYfs4$`fSEiHa75htmsn_&V${L%62+JA+!SIFM=bbfP<8w-y58Rz;poSDt zi-1Q%A{ya=8YiO+tgM=lsv8s-8d?XB8e`_SfD!R&5ZR6VZKkIO?(mY7WohQ!>ds9z%^iq;FO{|{a79gk%j{|{fKU0PC#qLM8l2^E#b zrHsr74Wp2C*}|zkQG~3>C@Zpex*K-(%1nrmWESD~K6KyD_xpQZ&(mLbH(ldA&*S)f zKJRtlLzwD)w4jKA-0jl^KGA@vd@G5=wCUSgobD@4OnL7W3w^J?T`a`O$(e>Iv0bN1 z2ydt(b&cy4WOaf@+Gozh`PRXO5ETqs+=}z%(6F$W*szo5p=T;Gq~dz-v%~~vTF1R) zK9Pq;K1ZTWbgOcyzlc_eDJ%{;QY%-kq{tV2&yh7gEDYyuL$?|8k2Qy43x3^4Rexqr zK|(sAppDVRwA<0qk*}L11ia)pxVn)@%#&-)&&bKStCr)-)(!r+=Q(<979V)}=JpiY zD5nx_)j66~xsc;VwGrLbeF?l8N=H6rC#dy}n;2}?>ncskmng1mrd}-Vd+nz2G9qTi z?~IaNvU+!sbGR+^XlxpO6m!aG@7f?J?D|%p&R+$KkA6QyF%3QH9A7Kv`Tu$o zmvgl{MTQWhffx|U3_;?4m)WKvO8J+$oRp$4PcCChGJL+qZN-cgTjg_6O)-bCx& zYHPpMb|7n4*Nh)t^4Gcz8>r5_U_ZWf%dp4P*=?k$D9*h<#P#7It}YY`?YepB;MByA z7uYuXvmb9d;>o%1=hag=EAsN0^IS!|-}QfbI5pKh<9nE53dQf#%uG-1dbT5}x>^-x zf$gqC|13snpv5jFm+}H?BYGGq0e=1}JN`xVXDXGxl%N(Ly2^$qtDnQe+h9Vk>i<1C zDKP#0^Jh7u1k13sRnM*{EIelGnUu84>B<$sR;?U+bB49`nvm{}j)Qyl>`|CN zYpY=E85Old!lYV$)OjD4QVhnq@P?F(3| zvf8!pfylwt(I|cj6It2We(WRr@TaK0KD|i-TifI2Jx~dF4SfH8raXNA`(O+`z&`h8n=Fb-O?~|R zQ_5{P|K&O>v|(8{v_q{9UV80~-U+OOl3;9G$>_|jd$3ks5dn>nCV&C?u?lpa8ed%? z=f_3ykST0#4gx&wTUuOHM4dzPzRbb%=FNpzq-wMD4y)-&yLtqV4hC{?a5N%hj>kk1 z1jmMv;o&a{F=(>}Kp3b8>=?JJg`$Ao{G1L+ydesDi@8}{EbDzw&w9I0Iujs_#0(4! z2Hl{RU`IS0zkdoSuX-?6gM{))LC+Qw&(BBk9k0zCfKIX`7<}5@x|e7CWnQ4;9>5OQ zO>hNmM80L(Ac~<8Td_txNrVtzym;}T(Al$R;i`~WQ*r5|FT#IJTZURdvAzM`{|*eP zUF*Kq*M;1l(;X9dmn@VPQ~ue3!ErD9gjo`qa2E8+qr=ZXf^dpcEAM8r!- zSa?S<&$dnC#w+;OEnB z;Mf*Udd;`#FZFW#5GeG~R~*PGrfJmxpWxt=$O4EFyS?iucCvLSLk+*{RQVH;k)EaWt`yPKMw z;`~rhz!+0IeVWY<1a={7|ItgA_+THg8LNSgORb~o2t3x|KuWB-M+aMjz)r-8y}%;` zQlxPIPQm6qB(HLm0_(zZsxGk|?TH#J)EZwg(^Nt5o5{FrZvKtLR=s+T;F20_2bq&5 zC^xl;nOm0fdA7T!=Y+jsHvrj_&!0c{Hf)fGLTeQ2cm;0{uNqqRRqRHa-)0HP_2EBg zw1>AI`)G&B4q0!Klfvua2oijExe6Od5{y{_kql%BT!=R99b8X%g2YP>TNy%C} z4K@7PV_M5cH_Nu}Tx8akE9IbwUHLL-e|(TXzpPJbHQU+S8{>)hEtsMB6O8W!&9u_w zB&nf3rpm2)KU}P9rWIq0<554vv29e1IXXGfRaY52NP~66c7H)ZK|JoCcWs0UpSz3VIz`6#|xf(QOy<_M;h4BGS zU6gsl7J;CZrLn)gq?=2j0tGbkC~Q->xA$5ApzHJWEi88Kclo74K8bf&OS&Q!@O~`n z%B|bC=Poc4+c)d-WQB;6p8ZbFO6J4;)M2w#y(`oGn{K;0eKe*(E%dLYnF?KYoo5?8 zAm4~=`%Sx-SYF#(^z-^d4q4H9?z(T(176aDH5pyLd=k?|wcWD3ggkW6Nf7tnc;Q`S;zh`6l&(ap# zE^a>Y-NxFQ%*o%fS5#ibGuKJor{g8i9Ijk?O%A)my0vS=`&H%cW1vttaNvL;tY`hI zqt`fOmlXeGGElq*#xeTV&zG)P5MB73iTN|x2`A7;9D0BrS^$$8wu71|h$&d3!oJ}Gy2#kHFbxa>s;?i1u<#N#6>DE`7L7X4(B zp+#$|=_g`OdXBl^+@aDpiotsy9hm)NgW;>jj-Mv61-H!bZEI`D`klHI$!J82$Sn#t ztWLHqKXxM1?pg83rCj{7OJbUpRmT%KLS?e*xHFtfip=*FCxtFJ_)t%e5qG=jm8rME znaD5SzR|Hie^6e@td7quteg1|0EeC4GTQ1jyJ!yy`bbv%IO?g`wzrBEmakvGzKju< zjID8O;cG#?ngAndGB%Em zg#5q{12>r{#5@tC9`NYV8m*7m2tjTZo8f;OT;sB8;2z^XXJW!HJ3R zw=(DPf#OzWtnCXxO}y)&DlR}dD`-_F4(P7tcnm1w=w1f{=cJhXA52UB@R=y$nqNjRW=lmgA$fnN$DE|o zm!7~@jyrt0=;79uDT9_SYjH`*i(pj9{~W;h>8&`#$#hkZk@--#*4Q1n(|1{Ey&)vJeQ+5nv1zNHR*xpxQqFlj2x zv~jIp>Tb%#t-$rn@~f-=wIRiI_pV)AQC{x~ZW9m^;=Xz4pS1XZfQ=j3*-z-M!D$xr zCMxVce*9QmbbK;Jr=r7tbX9R3Ll@e zbfDnvJJ=SKSP%F;d?*KesiOqh1zYRDgmfF=R@ru!uqJR|KQ?Vn7j8(V)jbLcQG_e9 zN2k8J+S|p&MZksn1MIg&_L*5#_(Gsv4B_Rmtywm=A*y+w77xf7w}!6!XdmfyXAP48 zYm%AF`UpylmaXIKo#-noaW9U9(A+>v2-qOt;N7iNoaL_7wGdgnf{fl*8w}b8JMV4) zB(Y^#7`f(9C`FJ30?MAc6rN0yqze2CwyQo_ei{c-6C3fNvI>bA{!{+!4B%}XC}D)U z^tpTzGAZB@o_=R8?5Qc}bxUUTi+7d$t)UB-u8(}`Hc zo$JJZd|6PRT?6{T#Z_Cs#rMg-SaObHi`x4WruV zpBa3uO3;jo2-EE-M<4;)r^ZGVXr59T4Clj)u|5r~!u6m+A72TV2DOrR74dP-ky44BZB&?nJwJ{ zpQH(-qb;%;E>)Nk`FHOg{oRhQf>Q_trC(s2hkW2=4@E&DG75p)o${^Qz$Qf^OOvIF z^3m?oPkV_Di1>8V>K8_z1P=SDxj8Ov@YVTum+&}O%x(e8Ro#AJpX;RhcWKjDfsKX?^C)%Y;*_@KBP}fEkbOMHE(R1_SE|0ge)pB}%MeU$abTyt z7nP8R`uzDE#>n-U74DmD1^^L=lsuxj`sPMa6rNr(JkG)P07#)NzkUg$R@djxVVctt z0T~&R1OwWsmn}~=Nu3_3Tb_YcQ$hXzz9T{D0EQSP^!#(+n1Dbm!wJ(q47YIk@+2c< z{LoE2aH|&F>a?3jTTu_gU6z~Q)+O^@At=Ch7&ZJ8cjlD3`nrbMPatxatz4=3`}C<( zYe;vN^BeH;r=gQ4v~nEn_h{yOz?1ICAT^E3!7Gk&1*;wYeK`p$HQ~W>=SNWedG51S_!&z z+$xQoxRjXMJwR%|ycvsPa(3>aoO_gMql?P_ckKZt_&5Iu-%j$RtsafWcP8I$zbhmg zx}_`G_dNJo5h`MRL>`lTIOcU5VHwv++rYin;U$HjDPpkD=kAp6(_9zAXz)s&O}cR8 z=B%Zyt?kXR*WlFRs@;0q^JVco*ch3&X8S_#Rh$;j0d(Wm)Svwr-6B>|2Y$(Q8-UEg z+{-&Sn~`jDfm~Kw*sco&u?u=CeRz;%4nAXY*S&-EHb(wNVZJIp|Ds9=OFO}J?Z1jZ667NYy!!wZYdBe|JM+FpmhN6kSNf)Gyf9!Kvs(%DapwK zEy~fVFx~gy*I(cPvhEhr&3yHC`U^q#TS8x#wYKI}Pmcod_=aXvS9{HxHLOpkFskt_ zq9vmeYZ{xGmEO72Z){>xOuU39ihy?w&?2@-ppYjc#K*rBiH8@<%gbA&z^BEdQ4lZ- z6f;_eMyjVA&eHWVPQ#R8RxmQMMUNdjHpl_^cMtSbz7k_U+C$Oc*jxk@n}kG`zWq`< z!)TMm%*@ON(=o*oPCB}bVQ1Hk3Q!Lc!dD9LA%DuNSD(w1Qc?^64hj2@^j2;E+STz zwH!Xz+QA=K1wB$=AinQf&A}0`j`JpbY7;O8d~;HQJFiKC_{3wL1FMLwYd?CFIvTj? zfb+ptb03`8k#HiBz@?N#YYUgxZeao$<@OY>pJ-35+O+$440#Ho z^@t#lyMo$G>*Wj-9O>rgf%2XPa%$?|y__0+ znx}X)`inYTeV&gi2Rcs+5M=q_s59-{6LjJf7fYR!!H&YsIW4lHY&sOXnACVi?5gE` z-Ud^&6#*?1%Cx%ky;HooZu7!ckU1?}+LG;+!G=}SFJ(ucR%=0CjLk40WTVf?E|5-H!2{kFW*JpVPCBKMxaXz4UY13C5 z1lhz6x65wSxk8h3MVIh!X>g5P&We}$%mExz&6O4O`R9tdUQ_d{ zEF{DVcGtHv%z~bq#JVyDEbUTuOZ}r+Q{dCW#-K`*%o)*iPnIs#M~A5sbc^rrX6Vim z-|%5I&e6D*gYOVRL%!xwbTbrm6;L?uh#{>qMJTFpGo3tV2~0|=CE3*M>)v2!qmluk zMFk_39Z$JvmG+hKf%6DXsIXbLY$3L-$lhRgcbXqEvee1MO=9qSZ4@StSJ}s4+G- z1?+lK@hETMDZ~^4tIgy-g51M2bKJHZ3CLssQB_2_Eo*K*W2=c}#dFKE?YOe?hSp;s zjSMhG*Vuj2C@C*DM)=g4*4F?c#BLtiN>?NU_Tb=PxEV%HwGs%DDMPR~>q15ls^#Cj zXCrYxq7h2GJNN=pUq17ou4u?)%o|@Fv~5O&iaD5UUb-SdL7*VUWy^KkB`74MUZQ$2 z^xMFIu0``>P~Gs)iAeXub}5`uk!$CG-GVhc9`L76ciyV?nw9s5n{Y(4;u0oLKFMe*S=8Z9E=AYrt!oEF7d%&R9Ouj zHFT7jL^&k*!!`hl;w=u%IQHS)G8+tWPww7bs%0@7^yNP-f#==3^eBm&H*Q!pjKb)m z`NXEXgkl{GhIdz82feJULp?S+!Lm=N*A7Z@a&l1;buntUbMh^5?v#n&GfZRMTl^ug z$=<1zib=)B>jn5}EkOy4MOI#=Kx~)UU?_XvnocZMg62LEZ{Elax-}79e<^D>xRg$d zX5<0I_y!W4=wKk>|9zI(5UEmng-fT0WTqKtNWGg@ zi?&`+U=`2vx0|BTdKY8o`tL8Ji?BWX`$x|5NpCidrh@4IW7r{?rWC-baw@5@n~EBHY0ane>egoLZ?nW zloTZ99bFlOeo=_+vk_)ry;_dA-7>d@1HdAPL_;#{0K5=X`FgeHI{6?cB9noMQXc%| z*Mw7g8+Pi3;bT|}HzR+-41zr0l0q)H28$H0_rn)ioO481wd=1Hz(yGVUat=(p7*R9)A3w2xN(#{eLcOqmcbb&nNX*m@tTj2{y#KD=F8QoINLQq{0CwZEE!kppl<>fUe zwz>q@Vau$g(T{GWr~Bv_^mU~$0}12O$-d)5scF0Us&|P6M*BwM8%NEFT_1SV^U&%mlz1NOR9@2H*{_KiX`lJX6s#5~(g z;UJ_}*mdrggC`Q7LT%b%(u}#|G?v?uhIJeqx)#Z4DDiGK9t9W~NQ4{LM)Q zw^i#aWU1LQsN>`>etfd)I{&(RJyKXKMi|~%LjZj*w#QCOpXA4&H#YD|%~la=zYEw; z)JrfKKkel}H@<$$mW0PPFs@*hl&R82de5R05En;=rdzg`gBlRZeh&3`5Il~u6>!4z zJT;_#)&prshl-ruw1`GE{XacwZ^*7keG##6UqxtYVj1}|-gC5$4dW?uZ5lZ6un1SK zZi|0^L!y5qVWi_USk}-r{X0BMW;6PbaD6l3Is>&6q+=B} z*iN6@m*X+oVZu(kv8qKiZ=uEM% z<)6}R=#)EK;FHiC7c+4~bs{5SYAJ+GqzgQ>8mvy(say41+)K{onr)aolM`Fh{wr*$ zW1eY4>9qN!Y0FD6?2XO!eX%3A&U#wv4$WI+x>oVE+cVQ%%fC(Hcm@yEzN#q6l@U=K zG1lGK?no{Dx3>%Y^Oar3nfCqdBSF%{?HtXR5gnb`GdgZ^jVJ%JJjLW5Jonl|x8})F z9b<8N|GnlGI#bh4k-hW6p1%RBb&{=#T?K@noRMf-+Vbg___gV+g?E=f;Doy_hIe-{ zRoTQu>m%x@lb?eZ(I!hP_466G|Gwcv1FADvyXhu4FF8dbCkS{W#Y`LBpr8NWN5XHI zF%HRle)o|JxlM>BwDW3oMK>lqr# zN$f39DN#Vdqd{MVH{dwIaKsGow@%DV{kUJAl$I8S7pm(2DJm)X0?`Ne^eaG_5f%w= z4IJP(wt2{uU~L=4Y_Fj5sN>3SyP|i+#dpE{Y&|F;57>0KaT!}xKScCT@D%$FBEXT4 zfb+KXr9aGTH;>=~P-^{o%w8yVs$|1S+<;n+$z3a3ImFz@*O%3UQ&#`m0U%t%V`Ho# z5>Nuf!e7`s=13GU(fNJHQB&3F^W5#1Jm_dVOOqlj2)W2++9H&Q$neBO;?Pe}aA;;gzOf##he0~q zX@p0MJg3lK1WwH^oF;lDkdR+=((&%zn;(sYM@REz+w`n#NG1w0i1I*3RpCq0L4N{z z>Z?1Ey#0edQC>h_{aFX&{4>@q{Ps|nNnaTk((-)q;)SMX5haDc=;gbGm?4>^SrrVW z`T;*E)AOgt*|mt;&ttQUU(QiGForbE$-hxmFzs(Ah+JA{QC9FD)ikqJh7Vn_gpSOP5$W6vuK9Y!PUJed0o)7 zI4K5_oM`M?>5(IgGZ|14pMaQcjb7Yo zQtLPGb9`v_3D5i{DMn;TA$rIY2M-?9o$mVn{iAEbkt{2F`>)deL0n4^2dNDFU4Zj! z`sIznkVwRZ!s2nMn=0`Y;DkJIDwu6Dy>GK4=${zf=)E43Bd>hDa3+ijth8CC z414>8QJ3Smy0!p(1#IrzL7YT}LWSK9NuZOvkvo}Sbeoy|C>Ey8O|Sc(_5f4o*V9&B zhJ85-Q{E**-T5>&4!vNzEwnceah%XAPinIa<1GEf08e+_7)Z4zP=)pes_O|WSioe_ z2vGP)hB+GO_{d0k{(kUt;Wp?sHS~Wfvk)Kvon>zzUidu(I=;yO6|})Z9=X_xV>P|g zca67a!U3~RPYqnUDuSAJ1tEr@HyG>3mz^n)q{kOLpf+yLeh$Df1;J1RHoJy_oo=gy zAwe6NTioU~k-541z77uV%yz}RuONj!c78Pw z29H!V0r5r03Hr6_x9u@e1 z4*-f&Dldu7^f6MoXiuF)K3gmt@S5Ir3us)tdl8*1Cy_M!noY%Qe@t|cu#sY5c=3#% zf$TSDq2ib3vGqQkBhM{f$Nm&*KtIHBHlI-%qvRLMp=YTU8QGu2y@N=(X_w~D4dn7b z`xmln-W&20x<-W;-T*u)e!&tp2*EBfe}9?-4J zoPT>LFn7lpe`p3pyPXXEIDpZ0l|P(fXW7`aqi|+V*H32J&^E#@((XX*Bzo(l&oSfT zpVzk(_}Fg*C}!<42Pkg!0`b2A4I8!>L*|~c`Jb=zep?EM5q}CcOkU% zw7+~M?aqx=qNMw}%JVG$pjy}7Bu~7X#KOPd%DZzh+u>W}hyS1N6BY=WPAj`kR$T-g z0I8i#XHwPcqz0CEW$xcq+rv%^ zt6!5N(pc9P^^rtQ6DXFQ3P`nEG zj|_DjHpbK$VgS*izU!Cu<)&T zu#P&)aU#(Lj3}%>AM}-DxSmsEHscZfbd1wAQY;P~Pk4d*p78)&fN)dzyY6;L%%c_E ze8c8>WFf8c2IQfLf!kKVw)C1xN4~r~sw7D)qU-wr&++y1)6{{g5`Rdpv!NwLyi9?^ zXgP^=KWp*|JS@9mw(*sM0XiHN*Bj*~ITST1UOx$wymziE#B93L!b|28Q4>DB##cnv^84-N?^uV4MZ6yP|O8v zaA8kCzyMkknr$?$%0^01A4yoY1;^#%9L8Q~;#kwoA%`VR0YtgF93@GKkA$F6so_z0no}OR(UJy?2+O=x}W!^!6RjNTF+==dUL!7THRI#yd=->aQ^)j>Q zZNT|2B>;RBfuvDxD{qL!mOrY5_#T<#XK!i)MgB1I;9hx>%6KD$p@L;njze5Pf;2N{ z?*E4PGmOMsWVfnBplIAhvT21tC{@6QR;&g-ZNW0g=}e+}njCkLqa}%tQH)nW;2t0> z?d}ngCG&yV-F2owi;2Ae#;X~B`k=hJ(bS#jYHNs2E#PN#j_O&>&jJn}++?EY35zeC zohC#A{Mrqp64V(r#1foH^sNYoNSb?ZLd26J`V#4CQ2UvnQb8`CUI9fEykUCh!JIYo zZQXjce@q4`^UEqi9H`kDSCaXkS&(u6{_?!(@#ebkw*>U^oPCLqKaIlky%heaL&Jex zmmu1MRX3`K(i)#+#bjX(d8vHSi)Nz4Om9Nrkbj*z66<%?i4Q=!9buYUFJY|BJj{!e-8zIaT1vd zv7-G%Zx1A29T$EUU6n>+xx}(_R&2DQ0uQ-km-0&Dnc<4E$=cC3-04&%yx z!_1`((Mnbdz@@ji)xf7VrUG<-g@t-bi`uDbw?n9ajhr%mL~j?d<_XvkU=JWprVa3( z7sJcUA51%>N$!&Ph$BLb%JHqA`1@~gP&#*x>%NDF#t(W`GBokYSgVYTm+1At&)GqA z4RZPKAOWnV-2O+KunDRd06p+=U>Yx6c#Fh?FfqyutZh6Lg)Q1pdl*Cg&`kmxKaoDs z1}j4wnHZ7NFbWkSko%n;iI`}_qT2D2?n!PCWEKCkD&q|amQ3VNc#O98nR;LbF-E`A z&42-}2$Rel?AC~nZd-YwbrSPR0CBm>E?h`0Ov1HzOspv9;1BC6M;Tr}g@G1}%Ee_L z&>l_)MJTV%fh1NBz}?&iWny^<>!wX5=(%>7bJEmTkjWm0zrKUIs4S03I0sIkduU8m z_{|V_63(;Y&P-4xc;|bnyt@mte>}jn@MauV;9AU+y7TPoGQhe=8fp4dBj0owL9c6LU@VZ!bD_{kGK)3;#*^#C){5pqj6^@gQ- zA_ceQ{va=Ab^G7LMtc5<5`>QjH61?n4Y5@h(F<4D;AoQ5_>f0>Ag?DV={M@t;#4zy zH)qf zgdn2RkBC&A^8yC}SfhL0;hYBpdA7^%xNU|DaX!42vz~k|Nu2mmxS60CK z#hZX|gT~T_3fuP4*{N&k%)CQ`T+90W}NGu4A_45ct+PTfW~7Q{0cj*as?VbHg|vCg!f8@^k)OX$HF$>YH4S%RnSzur}{Ib#|TGDqp8jzaOyqG z`H+uyFg^IUBV@Yd-EubnRys9*`sPoHdy9TP?OgO6q5JDAyPU)WOIqvOnWbIxJL&uW z1J&es2+k+rBEBN*%-&=sAb!(3Xlw!mK)>-UuR}6*yeMSrgk;;D>(6|q`ZpZ2r}c_t zS)QfQYUfz#$}Y)vhGXY7w39|f)55oZuHf2{mG5S4xW0&5Sm5L6I6dF}84_*>063)N92Fp7K7zW}%31W#^o2ReX08>pM;cv&F`%%ENri6#|NDk$DR&3gRhln0++mQpaL%I;MSBT!DZO>X+i7s5Q;7&OOJ<~R5-tXxv z2f;Of)KWM!%xa!9dvHjp!nJ|Tn*b6ldHO0UtHHa{RUR4V?fdEk^<51bIAD4h$ntRX zYaA3RsS zGG^6OBwYykU3t7mN{P~Zj+Z@P3jY#G@{E+g&gBIR^l45$)BO33$kXP2Nms+U}HGs6mhf3veW{z}4JEmMDMHqb zQ5?RMqX}A$(qBiiI1u$558GSIG_pzyu$8Lp^OLMdQ&>YJma_t zHkKl;E-sB|TGWj<@3nH*24-n9QG-518xDlX?@aFZmOkjXT}xwlW6LDEU!8Q^NH}0q zmT`LByZ2>=!hO^4Fd)-GZLNAjF#M*cAPLW$JR4R8}Y?(*}*G<=x2gejspc2M;7D8+`1ZM&z$1nd;CflkfVrGZ1OgmMxA z7w8I*$9xoG`N8&l1ar54)kz1N1Dr|N`bZ#&t$@nst4@7UQFL_#2Ut^uIXU_;V0(Ny zkiCL}_mORr#X!Fvjg6jgy|55{qlmVxR9aeE6+POHgyT=~1?V7cbW8WZM*OWPH?Rw{ zfjY37wD{2XJ+0CIi3vhEi2>e4XvYq+n`bd}_4LeYPg#jzOV|X)@rl_d3<;+Y9t%7Z zs&^PUbfv&g&&dqIZ9|$y5d$p37qihs_$WN_y=-l1a@hz>bI&v7Lsi*um+EmD9@^x1lJlLt5`9@C-d!oR;jKdg!(6TTZhD{dpR zg0$}ribPrse2YQcvZ)X%CUC&$K*&@h;4gSM0dWON+3-x*NSZh8Mqr0;>T_`VpcmL2 zD3LSiC%dz>vF0G*v7T#i5uLErq#4!w+uvLn@aT|}!KRG;kt7lgT1z3*6&V|0p=e`z zp1C`Xz1%bJE1jEMza(nuj>Y%%brVx(TLLe(TG+qrx{V&NI@3rEZ*z*S{q-ItH zJi$@uW*PaKmint$}vTPn-yqh2sJTL~ z26Fr()V!s%O_49fKQhU6W3q2VWLLTC%_qst_*fse$Wm_ck73#^eBj}6q?>Ky#w09* zs9;QkDQJi8BgntvfB}&NX`_7!x4~Fg0gC(OV+6>&ot*d~B5j7@MIiA;kIvMN=RHp{uf7Lf-#@ zDI_4tdtC)U5tFME(W0C0rf1s>XPjWJsi68!K!ljfb!|gWxPDMNJO+@4Xg#T_>CJz0 zIYN67Ltu|ar%DROH*$ba%yZDNkR{?X(xexbNmy=dbq$+^TtxP3d$cl(n0%{dN3x3NxEZ&U2I-h2a*=8^^x~0uFxK6Abq0r5kKZ1YGb=H^+Go-+L@@(P1jZqXvQG zf0-9K=B_yQCeoZ_UNm4c=AkR%+GxZ9s)CiwKa!_1fpE=*-2}$zAC3cdAFbI;;iHH3 zSC6S$S-om%9>R`@&fVcJjf`{6C!Jv1c-_=|71NzSU1a$NRU-)$k^L1}ZqwP&QidMT zi+K(zD_W0**1O2fc*!|h$>zO8V?r%nT z|4c%;b9TAr?Up`jUhBh7vxTPXVcIH-k{{IO0SimHoSt;Y};$8(3 z8G6BIJpVk>=Gwrf{AFWP{|0L6ERAnY0zjI?#ww=cCp;~O=0#rHo`PHX^!>a34YU=k zgz}zm$wnJpte>djb1s*uSLA`4zom*q!Q}pE{mCefnWUL`3fxP|{lGr*du} zccD{LkJA)hDo`zCm(FyJ>tRq((1JR)1B&0puX;g>i1Kw`zk-4*?wb=J>X!}oVkh=s zTuzPDnNwBR6W@dTWbdms5#k4x{~yLW*Yu?8)H*Q7iCyg+YG zcN#Hxj4?bc#PW*s;I_bhf8!M~N2{G6v1|SFNX2AaZmSkm3H3Qcr&2@D;LCq*a zj^~*Fk5?VSf8>?!9}El*4sL`mS1~A z;=ud8$C?mGi>RuN4ir9F3U=`R?8|v+*dvTl3tHO=hJlS0(dcSw0l;r2%s#+O>;e8s z(?}6xqY+WgYXg9QEbX-}adK%xp^{~T6oyaOgw*RtA^t#SZ<14Q3KpR2Bp9r^)*}YXmrz zm6czR{Ch^7p}9GcD#Z`|B$fw^^_ix`-vX_}%MznFL1eulqa@RG*{W6KoR1inKY)jj z?19;V% zX^5;eJgs4|fozR}@F0HpCyav56Ri$e`hsY_o`J>AdRzDcgAiUv<0~#9pp*!x*rHwj z;AhWB-cj01pbx3QBbWJagscKAlvQ0SjHR(aReVe5yYK$LdEWm&nwR<_@`*@nQg7vaQxQ3F^w6bwhYtNXzUEr0f7{k;qY0`DP2Qf^uzG>~-6hA>9`8I7 zRlNMTPaAXfR=GnbjweP^1JlzTK7URpZ=J9(w!g^RUNKU}^eIH@%HXFQafcs+ggTG<8*%AJyzcu{Gax$iWF(*9;qg?Uv+5A7qSBwUy zx2QAM@#i&6wFPPUwsv+=mal&w{h1EQ!TuiW;Zmv;-6MYZ+);_KcB3Q`cpcEYC0;s! zK9M*KSD%_0;|b5$BCZ8|oEWI?k%`h7&Zu?fLBT*bpV%lB{%O+6Vi=JN79hTerfn6> zy@htN3mAO7g0Db})+wk+R!kwdhPi+&7YB}Q7l0b=u{QhX;Rw}JkpdSbL)H?Kwu|J* zyR4e8_1Hk4xdk-%GNGP)RPYljc4d7FpE@~|R!}P#Z3JKf@xi^=tuF(4SMaCr!SlT~ zyTlk)snd|#ChWz{G{BedPJr)$6RxVfaB?f-Dm6Hfzg_CSLMRQ8Mej0BMWEndoX2PgT%^1`w(A~Vr;~W|tH6&{)T&g$*km#Yp7qGktK2^h}MP>C1 z)7Jv#4gF^?zWruF{oK6{%}C=c2HF0?ajI)IkX|AK7?cLL@Cp|xNXDG@a2`4W^)Ikw zT>#5_h>3sO1*}O!yk5=yI_MS$uV=?BHh!3a*Ik~st5hGH{U?ycmV=`sbwFP`(oy_U zAS&(sK>fx8b2{n5hZ_bO(JEKnu}X~l8z;>G(fA4o#OsNVyQD`wMs==m3lslGt=3y>?{A^T6V{N+$^9fqI#tu>P3 z`NhOk9c8w_=nw%!V40-@2&Q{pUJW_lft0+pM)f6jwzthcVVA(GP%0hYfeVmym1Tz9 ziTDDl#>UTw>W)I6(r_^`}hg3{D6?Ragcqxf&8V@FietR*DdhJR*W?jV`LgHdEoN`_qW7^Bk-(X-kQ@Z zg(o;AR=!WuwLB4)Hfi>9zv+u^X9LPa$4`Z_6mH<8Et4c~e51a?gjS;3>8`WWbf8@) zA!A2n)6ClG;?z`K%TddkF8v-eU0TTdzus=giaT;tUTOc@FaFRz;PG2z#?%xb!L znPW%){b?f$oy3ey`iV4}6J!nU3bl!6Ly1tP<2xRfBiMB7bIgIJ8XD~wANJ$E3sD~> z^ye2>C{7=$2AX;^12XT;Bwy|7gRV4mw;kpLlf~+E&LBP^avV~}Y1G^(Y{or*uyK(z z-@UMk&`1yDD5G{J(`u_PzuHm99B`)n(|TyY2~uno4EwM%;+}41be_K)$|+&G4p;^M z!!=bD{|ly!wdi;V z4?f@ZZ5BXhiPS8wxy@zsFrQxbZUmsHU;(tXwINGpTW&~_0lOM-0A`p-msnmbv-W5R za(znhU?4XNKd5wc$Rz3ErUAB(`vMxV+_@qAzB}~-iE*^!J8IKW=ihuw-<=8M z=!0^AbgUog9y2pjmZTxqBmO4Q5GBfAPzi`XJHl)nIDesKXoS6l)%*+`8R+_19F#~* zDzKNVFEDK;6I%F35I}Ll^5M4_46!Lhf@ayoYQEf*gQzA{xNktQQQ^QI@jb0U)#HC$ z0|=A3z-hF14N19h$j9-O(uBFqb!C8mIDQVSR+q&YU|$#?D33t!2h z{{;7nO1s8j;N`5ejEs-Jzt+|R_@1^|T3Xt?DD&8L z6h|v2kCsBlh`^?fd!60gzCAAn0s4&o`E8T17bA~)xIDEqswCnLaMp)PFFH-n-d(!` zDYymaf|2$DfUWah=a(4oGo3uhORlyQN{~27% z2Z!CAJ8?=;3ce*H%>_`W5_b`6VO^ast{JQ4nUg11K)_}_b^t41zya&%?rP&%+fU5|3j%9+cZSl3G5c^yNy(Rc z8c=y4*s@zEgG+u~5yv`|j=b0>a`ts3W}1Y(P~kYuyeH^4L3CMh*#xu_I2y9Ua`D0N z`KS(^hSpY?NlX3upTroEu6N**c!J&c@84t4yezvo`s3S1=n*pO?X0bx`jt74Ou8fW zVd#T>>SsLG1o=!$fP_lPU>>cccs5mscJ40ZHrs8hD*-XGJs5$Bs~(~0gQvmn8=_|@ zV&%obB)x{|(C;Akn<^#UD`dyad9uQf$BH5a_MJu8zq2e#GOtGx031OV?h5b8BI3y z5b*@`b#JW^=JT}Q)E=bMde2UXim~HAKOMqFL04)-4DW7(Aa8xl#0($~dTUKQQp9I3 zp4K)B#G(;55gTo?zNv>glx$o629nOH{AR;sePtvy!U#7>u1f2*FGb*cfu35fnm1K4s zITn)p1(4Twad;$o;~`a`sz~lIQnjYmX(IKIdlY+G^s9!EQrZ zw*g&6hNi-T_R@XZu=IIxzQd(snw_72e{?p>nlW{ErmhE9QYX>TX?FfxN(*~M^l^Oi znfBFGGxF+w9X(!dIT4-YYI@!IdT?rPzWR3I2KFfy8f|1wwHh6`BX^BJ5&NbSmqN3O ze}5FJWq*^Jy07E<^irDp@44U+^^a@YmUJ+CBwIZdq?JZ>H^iZ@r2Xws!5czBhTQcb`qoQ#H%*DRy?HYC zqsW3z7Mk4otHqQMjTxCLSBGdPyt&DLw3+WvcpQcYce$(^RH!j18_pR3DNpYuATv4E z18rs;NH-MXrIrM5PEMYjZGDB46Et4xXw0qhXu`cV|K!t2<%7sziDl1^AIjvh%|O$+ zv@j_t>6t^D)3G1vco#|yNg+24G-B27!oxIXaRmU#%}zvDs9^LQvxrl9b|~RM1u@P6 z76~26KvAv3F`j!pF+76s>uc4FILZuod%b5JR1qEKjwE0(v0?;A@jPm4M~Rb+tysKh zk#|N$Mw>Y~cE*YW)e;~XCP*$$OXyDpuoi~wjY=LrL+jYzYJ4&@BRM%7^1|ho7h|xX zqw!e3ZAO|HS(EFE?i7JOMhz3n)g#SHMTPQ@bTp#g2-&cTmz?-NTzz*SmhIdB-BM{t z(U6viLdhtjC{dAQmQ@KEMOKT{r6(FP3MskGh6rUuibx?NBO(#W9#J7H_4^!pzVG|} zz0W_-^OSMj*L7a!c^sd0>^8Er4L=3fEh<@&Yuqdz*O@Ho4hYVyR9-*X>Hiu&s9AY6 z7}WBb%C#)Do(c?9wKf%d<6boSJ?z4p>q;TDvy<3MlE9`p+<$=$Y$@hQ^sjS-8aNjp=Z2^z z@`+$G6*|T6Bl%iw)bnbD|dEk_b&fE;`NSiN3*25mZP4(zx zHw++t6RQ`)oj9ahTs5BD0yRwB zZ@dD2+8W*a-Y-W%V6aY2=$m;w? zQ_q1hbA9H}IB_2vdJ=7TI%d@m_IEJ4=uejJ%iO$qa|g!JzUljoHCuri?T61wwYg9- zuPL?vLO~<;cB5Tr&ukEB=2EZZ^1}jF^ky#^ ztpEa>{sn1d=W)E5e)|n!8zJ03e75!Xu9foChL(U~si|z$OPd0}lX1d{tu7!gqj4BH z4MaoEr;GM)R(*6uhc#uQjX43j%}GCkNoO0au{D^SfO9JvG_hz{1t4~7z>}c4lb=0Oi)_ zVS;Mp$N`iW{BjgWaN+(}6o=4ue>o;nZP^th)+jGraE9s4+yOsPLEmpYG=I(LqK#-G zQ=sPA^r-MDo)}W(&0jcPzKr-DHF{4`<${0N1DkoH(J4GyYRNsN{7b>g+n^G_4S1k^ zfwFczr6!i8BGHb>gh9FF(}SC*z$3JE635KXM%CbQu%q2`*q+UFfenZTla6!V?WoCvACUQf%TQB+i^YNUcNKy4P6vhZ zV6=nmMx&gK;=m)JtQ8^w_zuvBqLQEg%qapuEC4Z+1X;3JgBNM>ZTkF~$wozV-@4X! z>E=SJ?Lq@WLKcT$NZd<<-M6K=pULk-gE2FYrltn_`Ei}%x3}-z^k}}&O$fXs$l}iS zTClDJFY^OyO{a7yE>TuVpQHB6>T)YD_MOt`Dpdp*3Ql~{Q0@L zp4dDLuglZ=D@NHOOZoofT2yuN-(_HMtE}1a=a(bGRcOb0KH=Rs$G!PWAS zLO&)b|3o+Y3PPP&15M^y7H$CFiDFO#`rQ|@F;LTAPF|j~tr;p;C|s`16BZZWZSBAI z3=Fx4N$Yplj-~aKX3(s;9u!0bQBExd&Y^6YxQH7rR2?U$q;Dtj=lwWQrbv!QR$wIc zILx>jS(t6TSe58d=lcePg&kum`(8r~a{lv-b#QyrSn*Guvt#Gf3@%qpqe$; z-e!3h;MZ0>FrW5DSVrdkgR_Z1gqEWD+3qv1=?n9vKW}&5PZ#t24J_91IB2%h-=aU& zyNQ2U1lIseIPovPPp|j*KAUAtG(16$70Q_~G;HaTsGq=!ABQ}7nggU{bk3Zq3=0>D z2kp*can5d<#400&DDX^0Io+l$s$at=N7W-DNCw5#c^4bIj7fkJ5!yJ8C61RxETaoe z^D?G4E&2C5Cv(ia@98>;(PH=5%*{ti_XJkKKF<0g;W)uQ zXXxLuZcXG9XTAq#xA=^I=Y#J-}1y}A=bbP=8=Gg@GUjR%{BJc3dkE5{y5Xjj= z{%>+&lA8Yq5JUqUeCm*WAA%6j%wN#4MFufq;Ir>rWAwCI9436#oj|b7k|Jb-@zNa` zgA~!k4v?Wf^XNb*&=!$#h2nf<>WzbXt#VuEEnZv%t=`kF!0N@xP#RK=)FX?VNVA31 zgRbC3c??P?{JADho2rmvy=2JEQW2QglS%>tz)Zm)imn0*EEv0u)+v$edQ!)6c@q8i zuyLz-LuO7`9}~mG*cRVfUsw7G*&w`1maBw?lR-Q=5B6e%PVDdRH*qA<7GS)BPpL_V zi`FO}o9Ldi%WP)D4!jzyCW$-|Kr3i9$y}0x;L2wJ0n;lc@}jEUv>TNk>P>)SSMW*R z6mnW+5D4Y?*Bo>RpCj-Zb8QPc?FdY~0GM_=U3cOPY+PG#B1TS_#BmKMEjFXu3PSj{ z$IJhI-wNQGrQYQ;=65fkI$BumH7xt0ujIVw0(F?M6c>_vfadT5qhh z{qIlks06g$SA{j0m=u9RCaWHN)bfoRH|9FaBeZ#a-jBBci6|ljDSmw3&Y({)y6L5H zTIo49`UPDo!mWKq_CIwvk3H*YCqdj$S#H>Idpj9taEMmmR#Jd9u0mQFKMIM!&{78i z6WfVlXsb%kuis9`cRifIHSd^axg&3J>4<6ec~}qLY%Or=h2HaURD)h|>-t$!`Vi)L zee^k{H0|~E4Y=1R%o7ovW6cKuB~wD*jyIpl6!j!r?6TRAt;Cps5$mArJ!!L zaSDn#Nem0=aV-s1*!Ml)Bo1=j_Q~Ji1CdHyzPqUZ2Qg{0RXUHSp zN|!WBz4=R)*a5!AJMb7Z8C@42Q8c)Kp_GYou%W4ai+Q%`4j5Q7h}aRD?2 zZ+d;z)yxl8DL9)lUpt7S4PV%1Ra3U>~$n*WeQGh;t6DDRI6kGl}R3a^)a4pXb z4ppYYtt6-PRl)JgWEe~+QDsm3M24ixYIwroZx%BHpO^x>U?H9kD zL@zAWImE>>wQ<;N!I~v`(wEPp0%3Ic50*8BKnhucs%ot}UR}g2i*KswDWv zjT@>!aPiYwDk`Mk!C!I5*fTMyZkH?NUNRi4h{^x;)&Bm(Tw(mV64nF*L;4GRgli$v>CH!uR^pD zYPU|9C1`ouS7$LyX4cr)7zxU99LWv~!$?l8VJsFcCMkx0JBjG0<(MuqMRLt8H1D?+ zrm!}v|13ZeFx?(~_ilt1a9Z3siw@zgfU4It`)cGy0l8uRq^$)sgCka|Q5Nr86vPJY zpRg%yM1;&dU!W;U097(wHnle5T%aC%IYOX$%~%m?Xe%sLUT13DmLA{c$g>K%!lC*( zI&v`#Q!8gvNUWgR7gB1_aFz>>@O~E5pB>!aBMZv$FbSj>wCpM9v-YCU?~$J}b!syB zH0Nl$Yj0j)f6x_Ir&h1*w-f9ZUafadz=0+MpENi%iXlX!lOPK_nK>YVFTT2fKz>3r z%~xy#VU_ZjK40Fb$E`x2ncHzPWbo~flgV>?POBbiX9Yyi#f6xiPL$~&{ibm-a zG&CG+^!C0G9^=lpTF&Bq$MJs5xOaTyXKwit7wAP<9aT$UmjNrZFrya$H9B)$9 zrt58sL$N}af~JZD`QSXJ_Zs{cF#FxtZj9JAya-*jC#?9%F=$n6uzN6^R+I0a#H5}P zguFt223|25$U(?*QPx+W(NSob6RO#QYLLr8qJ*Qr5>lrGwinN8Wk5_in}}y5$8EY3 z1?tl1D3|(%2ihx1PVvG~P%R1wz7|?Y4=0&uw2WmqMM|IaK&tp-Q?)+IOyFPoyMoTi zp$C(9_{`d;3H(TixNeiK`6{SsIsn8CCkT%--B++SExv&%8?3+&c-jf?k2(7DI}Qn~ z2<2AUu1zc+l1&xAe0hF!%~pTGL|oEs1-k&$1j~T3Hzm+eoW+}6Q_?Fq7LiB|aX5_g z-~{w|RhX}9==}Mtt(bvEx(G$ZD{!g#-vY`VHH-n0oXe!Z#@?~kM9lOd(1LaQs*#Rs z27UU*g%8QRiqjkO8XqG!QS_r?Iz@+2CS%&$s;VG*iX7%w`gaMK5nZ~_3iVzwQgsz5 zkFyqbmPh`F&EV58gJ8B!4wsY+(5}skLM9s>D{yOh?H03cQMh8ZpbAI)qjdn}q)&CO zq8Z+IItA6{h5+M-gk}*UcS_r<3xU`lE_!t zi*O*-PKX+m?l`-i)#=0X+CEqW1t+)*mc$aWhiLrMxJJ*!J7(VK;DzZ^k|sE3J5M)V z6zCSux7fHy&M+sjD@=dwtgPbmRrmQnmCMjn%KXRHBEn3ZC(!CSGRb`}rBiH~R5_$em%<;e zF#|Cbe?=@TKKDHb0J-?s_dAw|RghBYFHwLmbmy=h^LbHKD?C*$D;T|^`)&F;8p=N5 zJu%#!liMe;OuNr*DHxmjbN6sS^7gzuN$CmqTG}*=>?XOpNg)4X9=U(f*9D61^F`v* zErk;Svq+&!!EYM|qWqVAvX`IcMFlR1-ynHgDc9y8wW! zs^1>SdvoOU6HE*f*`Ni1^{^c@@h+D>u`etxwm@Xpd|%46RQ8-q1~sH$ zwPW3_a~(}h0S~#5=bq@NII>f=3T;TRX0+~IO;je7_>?0Dahl;e5~so9G2gcj`GId- zizl=Ct}T2uxMM=LG}KQqz)VaI8WRAH-931@=s*pPVMOku352aN^JpB_syCb(9az_u zY4!&b7{xN|78n>29_M%PmSRV0V&aO$ix<8}FihF6RKS8NL)+BPRpZ`Fa~#jEW&yd0EuzE2Zky4lDZk}VyDzDg4? z+;HA?!7H?#>oX7c@)o{lTb8iZz~q;vY5EJ61A|frXe4o(F)`tX_gEAA6GWHDQRUi4 zqQyJ-U2UVcrhp8Q0*Yhwq6;-7Nh2}&@%861ijJCvSf__}c#G~~k63(T9qYjiyubi0 z{Pqeb3#thKn?_RN3S(o&SU@1wFnJZFiXidffwCtj^2P)0DgBxZT1X$e2y9OM&#yi% zw2yG8MFW&UQI(r+$%CL{{78Zoun?Du@3w@jFaRhWvzIRuzhymnFUH~G#)VaQ4fEfPUn9!RSj=2PQQ!sTv^ok1 zxZUiM_@QL6fmVm6B6#`u+ykbd>+1TfVCliD5*B!m$KN}xVBG0*06=j zIo4Mx!vDax>9j+b9G0{n)b#%Lqc~m&YDb*d%sdxNmZ1g#WtwP zQB3}+8nO6Oi<$=aeEq^UknA)PE^|Zk`w|9CHCgZaiq+OqE-Yx7=FwmCxZeZDrAj5& z*?$*+7Scvv{?Xe2kP*waVUszXbUf(o2S)55*}@H97a>!OR^2%}a8tMNA-nQs;4bB(U2h zA$O`Wo zf~jvC8a&-i!g7cGLXtA$&!Q168BYHZLMd4YQ`yL^{i2jP2n0C>>!EOi_ht0ADmdaz z#-1Z}kffs*90AK%V=ppk+69OcCCkkuIG}Sx|2V&%a7a&z)M-6J#L)vIVXL)+Rxg0Wxa~GJ(VSW9FHS zb3^Ar#yRHG{4aF;MZEB4Da4`otUD14 z$_a;k_)B)P6B2jEMar&FuO(uQirx+|p*d6-o{--46^>Z^t+_-M_20oj#!rRP@J=X! zZi{M?_6i}rrw*<$P8AzPIEY33)i*D;iQ~7;2$486arDd8bx=7+#SL7$fZY(|@T@>5 z971tyiS=RmDEOYZ`fVwz*@Ne{_zTiDd%+R0G{xBeUM-1+gbTClrT4K3Q#w}1$o71c zMZ^I%*wsdNP+cYhcX6o3-~vBPjLbd4(v!-~O~?F8S!hZvQFXTZ(&S^(=z{)3 zmIDc)d(I&TV&6%$_9l*aM{9`dI!dF%svaGDWDO|s+c;JVTU;o-FFL0k4r0G#U|?9E zX@DEFfbZI2t^*`RU(f~wMR}fK%`R+q&a+B@maOI{*DPQZy>%r>G&CSU+x$dtJ*udm zV#1oZ8D$<@f9gOALTtNDe2e_;LZ zPW^aU@kRPgnzn>(B^jVn&5?+^Am<)q=T5?!fVAj@MYs8 zwHe>{fG$sB@5wYEouLCi$VT9RQ9FI`#fIaEBxtVEzHQ58DBl~X=7M7l2xjg7!p4y9 zaY7po_36(~e~*ID0+r^&1DVgp6&@CNmE^`XK#<6IHt~q=^(Lg{I)W-1FQT@&@d33T zQ$FbzP?e5c+N8A|Arv3@;^E=3Fth?)VZYg;(TnlO*>rVOQH(iv7Kax31~9Pe z*65=Hnpkxko}GP^HLM7YP`O;oz;ws0xNSA-n`uk{_Zl(cD$1=|}u@4nq zamHk_J54~6O;VYU3v0P{XJT+f2#ss;ymUz_R8!b^boUCtq;&Y;4D%rwTV_zuV-eM6 z<0p^Sp^UA%nv7+6=L;BZ;(itsec;1H42|o2_yLPEv-l+~E|WAIr`h%=Z^4kPE8$Z5 zWqLd`@OFQQob&ziu^$8WhSB%$^D#@Jf&O@<%z_4Z@kC_hONPx*DKT(rG3{Xomh7n!RN!!{0f}Ntuxh1^HirfUI^X zhN6(KYS^q&)rF{L5o8qjE>qG3C%WO3ylG?QQ>Db_8S2eDdrktWQ;u31n?|4V>)u-l}|eq?K}mAy2X$AdlZ zP5U1EeK;3@G9QaPqx(K$>m^78lV6tT#9S;YzC<@6RI2~!F2%Eg=$ z=e@bFopx+L6-$vJZv;jLXITTA62aNWfONF!Ii9V|lUMB}XQ-U%=r-|RQs@8NlIXnf zpGS6s5vM^~@WBC{D$axOjBH5ZZa=2uP@vDjI>LTh_T#rUo=-qf7Z=NugzD0$-Q(UX zX1|gvj9A_rm&P|5kdCh}FAip;NM_ioncIz^!3W@dslQE1O~u2Ga5_@Q&@CAlORK7# z$_g6Sooh!bPwD*l4;p*fDox}Ym@O~XNd6-GM` z^Rpuz(TMj5={p5g1*QA~0;-Cq$ox6=zBFwJ;3F98N%j401GZ@@R-tNn zP}(%OkSoush5z-+uHz!2~pX$|J3*v9)zOw)_S4 zB_9%fVtf`!T|}CbXBtv4Zr3;9LZfi-&d9ex|0xb|eLocAa5zWPer;ckW;~ONXXb#* zQXmIWpnGT7Pi0+O$H9tq#70eW&h08DyC9^mf&Oh)o>97!(7*C;TYO@UCfb>)_^p$p zlnokcw2uvT26FpNJ+f4A;9EJI$Kh$lBwg}U9CDeGE?>TE;t1js4NKG0F1=X2)jt7! zFD|x0eNmjBe@P*_RB_ZBTv4;vJg{ZJCGwX!a1D%i||Md0IFcvE_5}|kskyt zHdr7}c%(z0+ktto#k%D%r{WApnsYwD44$jr3~72$C)*BvYpffSM7BP~3MW;0rp;0p zBRb1(nNrY`%|%@1(;$lQ38~B+^SjI4^}G~M^Kwr047u- z@a%190AyLD{)>{Kba+%}t;dWZ;H$yZES|S&nn$ad1m4a`lE!%hhVd8>#zKXS5jvjFHS6o^ZE3Gq@mo;-?p9Lx44@ z`3stPXAT^Mj}JYBW>_MW2x=;U_83$C`Sa&ZO*G^IirF7N40vK3?rp=f5Iw;&4gv9F za07r(=P)@k{*8;pvrKNzr=5s2(FBxPEENsF;sWI7>7C@kLs*~mLK1HR zV;4%0zaeJmj~@D#m8KK)3k;?4nG7HYD?|0{1wmlPsLGgPeS0)IkXW1zOQc2%T6>cd zm`Dh)!-q~Y26z3S?mz&L5DD|owILAQb!m4X$b=H5_) z3wr+sbmBV^SSdT`tcQbJj?}0;GPs& zuM)_F!fq0Ss>&-3&s6x|zS4QOB~8#pWw;%yc#1C}>NFLMy87d7B}*UQRtpX=5vChs-)@Q$5N68)_39 z9ux=R;s8uWL0yjbu*En_t>_^tzUGdO;J&ZKO1kGv`#W+JdX9u82>5%i9%Mkg1tAyn z3?E^U-vJU(^y4&433t)GEMl9OdE){M5@w8}FYA=brb{y`4 zYW_d=gKBMuBCN14zy)Rj0`(G)%kK}9qp@yvi4=M_lr{yWE(KdZw`0(_=u0n?1Mz|P ztZo3k;F++}P|fI-KtX!svEW;RD-iqkv`OQZ#EMeIYY1O6&g!m+<~kKeY-K5%0>+oX z#Mc`?a12!EsamwRH?3-sOuVs&jhq{^?+s;i^rHzYI6065DF|g|zUW%yd_#X*5|fmK z>_Vy1(G&9fOZITJeE#}1v`ua>J7b|a%^vR=%LeJ%=MA`Cyk~3+xS77XjWO3XqwnC4 zovSaHqKYaW8n!c=0mxma85yG@C|M;rd!wy2{>^-rlN3k`lbPGNDL-&i`0i`5Fz zO5Tg#BS#w_Dp0gG10_LMF6??!0I1IOg68cEta}I85SgGml5J#Tv|AyF7@inipv7I7 zxZVT%hXBk!e~q3RjS5cDZgdc?iDc1flm+ZbnS!qLc95{&)0{SDG2AO* zO@z%*fQJJcw>lX%hdZP7Au7wVB8qVI^>=D%;rrsiV+||zDD-BmENUW*tsy`5V3s0y z%%n|A&ApM`t$bR^zT+zM5obgqgy5WMN6`kS`v&8Mnd_??sqASox=IF&;UQeqf@9(S z`+h+HYJq^Q969<7>ol>KP*3OpcYbTLyk(&fcILkd4Pr&%<`DGb$vg&5mnAOQ?NU6Laihv`HI*R%8ElbkgHE z^KKPekSeZ;C|VBb@ndaf%EFGj{#iZ1>gua4m?5PXhnKM{Hy9byDDQD7%E6ovT0U^B z;=a!T&Hvkd)FOQ4+r)t^rg~AueDeh)LPZr-<7DNX2$kZvh^5*m5f6W)s_iYRJm6z` zLHVws_(v^V)Op>Je5sh=>aIbwR<4QOpSZy66>G$|RYO60r%SNy?qJs%y$7ax*n2fy z9nbyxzT=LcC^ih8yLPaJNI$E$Dq(923Il(Y`)qZQwdUpTP?Mm;3g%y$pM;vo6{wb^ z##}e~s%y;t=d~nH{)ck~&LQg)i}wrf>S(w~+cLm{zv+3pug+!r4^A0Q^hk~QE+?`xi_tSSX0Z+s zoPFb(pvzjWNAr!2`h6q`IKM`gA>5G%ZJ>+Oi#L)nG*|amp@Ik%UBIJ9_6ehtP^O`- z1Yjsw)I_^J_AphrfYW=>rbYuuD1!sT!X>VnM(+K*#me5}lOMgYacDBj=&*UuWY*_{ zQw z8p0pMpFL8<#{4RP4CUQ>4j|Yw=#>L0Sz^*|vx@d3!W)XNn_o=pb@08KGZUAvn0$xD5A%Zg> z23*L|0-&mSgO*W{QCm}ac3Qtp=DP9A`R}V)D8X8IKX1>)_2K$&XcgyCiQp;RID z_!qxz{q{_}&OI;f+Es#rWhg{#z(n4y=s@l{QpSK^?WFPnM;|s`-ou9vW78D&J(JnBU`$Mt82rcV;!{-4Kw`Qi8I%%C@zsoj2MS^Q;)T1at)qH48_%ne}d z9S}rlAI_ROuq|u7_^t;V|F;W`53R_a%W>K$s>Rh<`A^lyu4+A;Mm)m>;U8&!0T^0} zrk5fc7aqqpXav)lk?WB4qtVpD6OKV)p$BpFg|>wNJ?~NpgI>QKoBVOeaFFUOc{pH; zP|6D>ZR63(Aiov=+vvfm!+Iqa$utcdW#|0dz8Z=`m@BuFh=F`wfQbRwXP!X&Y6IOC z0&TY7U_?i=i@^&@|BI>|vgJ_G1RO1>-2Fy=e%%Ew58+UnV5UgQh&18VC{28@i)*~s zkXRYtyQi{B5e|=O|9b;wE@aJ2_L$C>tODO= zSLw4qYpgI#ScAbj!tY;R*&7du6qHg>t|E(5wC!Zx!gzzke@Mz}=3S1+2kYOzhCT21 zu+(4S!DYKSv}&<#JWfqr1zfuEE6OdWvpR4-+E=o=-WFBj4+DIBhS+J5>}E4?`Fgha zllNTmLTZHr{NE({Hz8?e(fb2u&M>2H=6#P76AMl_b#658yC%50x+@3WR<~T{@c0*< zknGtQE-W|dE~GVjG${h>mIj}1f}syJ5P35pSD+iI>ctaD+IvYUDH~MyB%F0RbxIf; zYfYzs+FW4J2;fvQG)3!mGAanZMBVzy7IIk+bGgZceP z#{g{M_tv&+U2~kbR&oL>HV>inc&K^U6BcnhiUaDTDJs$n9|G4V0u)l1kEU#ZE>hSR zdt`A^+2Y)URF0ZI{Ca zT<4P~Eh#kz9hH@QXi3~K$@jn;P$)d^L9n3Osx$RI`^8?uh&P^;7=owr0bsOqF2MN( zA2(v*Cc-pvD>63^d&83C)YR5v;SiHig%c%5ZkY#43`7R+OxeIHC6OM~Ba6>$i#O3L z)edlRG4asCh{-wm?ekVu=ZP#4kt^%n26?vZhQ4#liWMuynQ@A0L3tC@CXfGW8F5V) zRKu9*+)cfIfK!hTwM)d&y$#J|;i=4jsOTK&I0x3IguUG;Avl-j?;}_nC3Fj)wIDM6CYIjG@8455Ca(Ni78Q=m^Ew z59LUS7lr{fy{@1GK`}jW8~)+dCMbCfX^|FT3mzL_hzLo{OcF27T%=6dCnfro;IKN- zMvXrcYWhjweOePTr@wPm%FJU6;wZHo{W|ZcVNwjv*LUxkhxk%} zlKY;k;l-{{g_M+fAqzH3k5{(wp## zUSu#^^`+@}#P2g@ifoygCr7ePP|~N^D7+G(wt+D#i(z_X!SSI{;rnyffZ)l*Kfl_P z>Q0c=+!*4^IJs|%srzdf4aOOT?5-bMP0j!eow!Of8x%BQ>7&rKnXS_BdPP2*JBTr0$UB_A=?>7{oNP-=P?|SEw9s0bj^QwFzbZ_Tk>$>V=Rfz78>q= z;D>em4^b4kI)YPJ>z3{$z?k82Si-~d?&<*GyBYbgdFcPzedqQjL}e1YR2>83Z#Oo; z0CBGJ{B>`Bg#Y<<39vvx(DRsf97sqTM~+W;MG$u4M7&bB6l4rZn~vFS0Y9%~%)dwG zJlO>(dp4k-&viP2cFaLDAtAvQ2aH!ULxdVo0s%H!3S^x^#|tT(B~7nvB&XfKx*#6_ z*#agjah#HnFH;t+lo>3O+`c4?27C6FEsL>R%F9wL)*(W!8zScFq7G0gg~M~USRfi~ zHS#9jUg;w+DO zB@k}5QXqgsh2nAZ-DKCbQq1Y7xOldrXH|Kqd}gx9c#-{Qa_0- zzo^T#hI8P3Dw=A*eiEnnl)>l|gFwigfB^YVdC&Hj??BMS(bnR?M&PQTbnl`O2v*>I zbEyiwihVeL^QN{H>Jl;~s3n9X#OH(OV5zUuP zPbH5&5J4JR0$C_{uhTQS*EPPKE~2sXFFgdcRmv}hgJY88*XKx}M%hp%5Y`10m;m(k zdDNa!zkr=_dlm<97ixcK8K9~dpRFru^ZhhhD?&kWxvF@E_wex{b&_5aB&b%2jgUxq z=XZ=;0M%j_o((}0$OH@>Cd;|8^~ zWZNIoLc-0TwLKrLPZXA{7O^kby@?~DeGjO#JF3sU5t|Tx8H1aumt>)Kr0(f@a)6+4 zrDJe==z>+o5HiL9Zk$p$EAUR<81i+6Wm5OpHUF{{LM)c`gL?@$B)eZEj=K5fTfa<>QPq`!q>jyJRgQW(;I2Jo>9?&rGP-3^ zUBYRXP^pxtTLkd(%4N{1o<)tY^T08ypi`XH!_%K6x%x5E2|?%gi!QI z(xXPRp?_xf<0YwX=iUhvL$io!Vz^(n?&4fS5lu#^anRV)V2Zb6bq*Nq6Bq&+uL=a1 z$S8cRlZ!miDfAK9OaaDlWtZk~{>R)8XDG82Hh1>8x0cQDo9rh`BtIMdK>jZ1&PW zOZfQIu|oZqMB~nps_#NR#`GP1H~B3#;|)X4VT$O_nX>FGUv~f#CF|`Tdj$Xm^Yr zn=o%mCVyoit1FBusp7=-*hoV$_?PRb$@FE?~M}r!Lo|kXa zPuNaEq_^0sH*m0cau^oNGNPj6GDu{q+#QoI>-2vF9J~dYu{Q z%mwSIE8^UWbKDUh@@>`9N}7dbm`_s&AKqs6U`0Pd*s82Bgg3v@3hBM2jb3T!^Q1;L z&XEEr<0SaH;3Dg8YZEqUfF!Aj&F0rHidF^ncPH+oVcn<#<|%+j5$Cr;G2sSIQos@J zg<Bz=mhlL#w#87T#&;>4q97nXpukoHp-mbu%t9kWm z+tV=8FrfmIJfO|e@_%h|F`;@2>ncXL-@!fSv^$D1pg17*$*nw$Mo#hfA{{kctzoyD zEZjsiT_#c%xPu-|tggmP*J*z=B5-m5YBD|k2~2kfr~_p%{mew~1Ki3gcdNU|G^Gp} z1y>@N5ihO=FD|HsNIPUVA=k>Jysi3OsW>uPwjoV5#dhc`5! z6HD_r#62*?V24lm&=rH93+N*dwiXo?>~kdQQ#G5nIb)?cO6!gfiHx~dt{b1s^SELS z2x)U_Lz+I&zJ*dtxmF&!MO?pL$z_xyF1H$ zg7_>;U}-c0q8&rr<`*5pk593!kwhIJ2psC|ZcTeN0yI!h-EK)QJ4_1I9fc)arTFgb z&UR+N6&y<>nnxqCONh13pJXsJpNPHeAKrh$CbMjnj~wZ2`m@X5;#>U5#;o1JVkT=d z5A300mkhi?S3FP1FM)1*T8p`r)ei8Zv2B4jZ_*3{q1IQ$NRcq;F6Fh`zjyC^jL?{; zUGNoT!F#a_b0DhL)6!aA$%FwM_Zl;_La;nU&Wb1g>|hjRWksA$$%~s?;{DLi$}db8lu(Uc^#gw|R4DziVdM_jjK@Wqt_%+1k>w$V7ZN<+M%HA63-OKfZnY zj0#iV8N(A=u-9_7(V6xF8S$}lxH;c|ka_-CA<*;g-qmt7BfC#{_0v(t<$N7wun9@Y zJI48A?nMF6wmBso$nah-AL)n-#pY%5bU$>Tf%f$oG1i7Ewc$cRfFOUMu$GmQ(k+)% z+Kv5SnNk)UP&HxIe9+P4*go3)!577rqn6gSZ{MAuW!NtKQ4^CrfJDqli);%!mF2gbA{+CtUS@5`lmNt?tf%anv8^CsRs%ev)*7UTB(hw3HuAJ2boXV|Sx$WEg z6rkz=G)^Zv)WG>z0LD&|PZYcabdH`~XOILHSJ6>yV#Yc>dGjRJV$vEnKL4=UMMv>u z_pKwOU*7X`>P_qfiqKzquNj6$^E)6;VFe1e{to0#zkuwh*xt-*mB zG`G;$UwCRV_^G!)GpZusi?(FWe0sTo^fD_21T3L_lAE&&>8opoar^U|*$5YjRo8?hb^YpP0$fL`kK$0T&Dv8AU})7tN#RUy5CO zh$CdxvUTAZ@?Fp+Y#cr7Y}1p^cjt8!6BNne(N zr!b(w*>h@F%Bl?;s+u=QfXKXHaNljkK>L$PtfFi`w+Xg$LQdxk>kcL#x7Fndo3wRy z{0+~Y#;3w{Lm2VQV|h;R%)*RF|4#00!?2?K{I>6In{oEw(pVCATfghX^m+NW35&C5 zq+HvkmQuw*nNPP+fa=)DFEiqoRkkWjvW?$tNK~0ND8k|&CaBX~HXZtQCuDZ^HtQ@? zO{1gsMn#u)rcc-)U>C&2`pk{}b4#!ub9*))hi`;Lif|uH+!D3>`D;^b`i9c6#~euh zvnRS@QQg3dEp|p{dTqp)nLC`D$TIePDaH~BDBC(oC~|Y~J`9oq2Jp$Yv>NYD{yxj~ zQlUyry=`gyVVb&vv&ukz7VB!_MbVr8_04gSlh#~{zmfkjg8BXSA^H*MPoK|CJM-=H zgtNmfH!Gp=g2qmCtWD`pBp~^sutA)=SCJlmxgE(DXrni*wl_|OyA*8S{d*Ox#|y=0 z^YAJzYy35tm0I}n9pblslsT>rj(@u%GI#a~L$jk_)w-~rVkiJ9!OD^n=s=ab_tZ0) zyKr8s^nX*2g3|RmDrr=U0HUPN{In&)9_~xUfhW_TpFn8AyRLy0EYsW~3!k}j=R(0# z^v-6>qNpcL_o{xZ$U!8B)=5)|8QC0%Oz&9S+BPxJW(i1kIUmVI+eps&Ak9f}dI-8K$jw##c@8RXj!9DBZK$@Yar6$A zLMsqHQql6IGS0B#GjW0tkl$QFf^~F|vz9*8e;CwFGDITvnG5;QeDCHv{zxyz|45t0 zJy`_>UEt#uy=@&GY6utS@I#ls>32ao|NVaZ30qdoWNlatRP)Nt)8LdsW07m!?0^Sm z=r0)Qh8G0k+Am{Usc%mp{Ba4Omv1;wV~%J7WBh_!+sMZ6b7{=+jBUNEdUXM_&XQr6 zMpuAbO@MkI-mYMN_bvO`-_KKmBx`@DmKy{dVHhcbT#VZn8LqjMOdfUkx^uV0AZ6U3 zK>R|<{FxaUVz+nUN`|4N_NvOPBA*e69*gTE9JTAwQk`^k^ma%=^({k%LkTIP3(7{8i zVk2DwO~3~vusyX!)a)-n?VK-e2k~MOC-kLr6QZ#y`jD#Zlg9@S35u#|)u+(J;s)0i z3vsyb5VSP8Ih`?mNOO!bzJfJ)V77`2YMgSoA3Re}AgZ6hpEXu0JcxXz=(w^}G=wSe zbKH;;fMabLr7c<4qU5dsFsN6uGei+{-y7x5(5OD>arNa^MuEs63f&_OFneVA33A;K z+NSJ?jYt)r?*-%FD_}cjwOSpNe7%3#;of~6yj`x=_L6>R`;QyaifrH}%4x$*b!3QNJ^{(is zr@-dQV7zF@b56Bh+MWcNVD?IY6lJ%7>%4>3(j)aNEQVm4x3_o)lE=Yy;Qc;N#cPi$2lES8~m^awkJR-J}@vYF_Z1I*ZoXJv@pp$cytE>>^CGrbR=^R&!;^n}qT z?d>04e@loa@oerZmcr`!N8l9rYiJwDn)i{Rp$Lg8yN{eSL?c0i5!>ZJ%n~{MVE_#O zJ@}`mZSIAsmu91Gucplu4_Yvg68O*dz1^r>Iz4x>tV4vuVgn4@Qag1xB@lQ^!ELFB z*ul0}!?PKDe^2;Yy*=1V`Vpe{rVzo8w}OS)Q{AJ78vZc9^GRR!;;K4A{}}u{;0`-A zHD{+;mn#ox%43(>WbtOsokX(oLx+@;2z~imbc(S-_P>U{cOwsrtqU?~80~CqYd}f=E;RC=!P6!KYR&Bm+WBeiQLGP7C?>9; z6}FWw!P_n;A)zI^7o^!AU@WQ>=k1Ia-lIuYUj7H}pnm*Hkf}1$`li#ogk)0%~T-vAU;;4olLXJ^O)zMoi0$B{Bm?Y6Lg!Rv`8Yd+IP<^E+NH z@z8Tu@Q{U`+u}@7Wn^g9h&Z%8D^YfedwK|O1!E#Cg3GGW0wTWYdyyb-`(BS zC91I3!NKAB`3tvYju!ehb$1sdzgCNh53{Yokhdb7tw~)OAd!MJGgS_{eV>B#t^+4) z5Yd4lE`1JGrCSz0m{m)Z zJ~#UCL$=sGrqNM*%NV-r&8NTNqQE#x$zEKatkzbH0e$aFf>~Orha_ffSy{2+~qqz{pN%? zW@`PT<*|5UDS#5`E>*wuHPOWN7m7a8Lfdsucfw=NDhVx*KHk58wzZ0lyh{56)~%Z| zZtfH8Of9kfXFULTO1UdA3_r81Ek&cddt}i@>!~A?S`W_}@cAwDzu!C4Xv}o7>BF*@ zQ`1qxS|k7O{P;cuZ!=BWz-<%{4LXk<5H0=V<9u42?Oga(l^aKHvP>WHiy>KJlgI2y z?gQDQycz=-Ay|(7kf@EyWUD=t00HJhYu<&O*eaSkJCAfoNAO6L9Xn8^)gXItf_r?9 z>3sL?aH36PEzKF_RlgGv!MMYKxRulh03ngJ!v$}IF5X~LGj?|eK6g;WG*PHBH1Gsg zqLe~rN}F!lBlwb+D4najtB~?f%c0LX8F7&>9AEs~oA_k2IapP%_!P6?Xk%yeXj5@8UINbPy;7CZHw8@m4S5Q8HOy99$4LFDiq{C@Qnl1pn9HoxX+!2F?f#2^ySK7chL>1!&I~+~! z;e1Bx`==Hc@pUd9+r@gUPv_vZoMoD}SK@EJ4XPPA8vP{6X1%aP7O+zJvm?@>if2Ez zyXw|nfqVw_Kn3>XcJkmeVD|ZtBKMLNl$E6fquKQXk2cCNB1`zE#$AXH|1)FT^U18; zhgfAEXc&;eSP^{LLVVqeA%TXcj<4Qz+_VOkl2kMwQg0x18!kJe@PxE5x!V9uS}Qbv z`h~!*0_GV~AJC$t$7qmNZlw6^xrvU6{+=lC^&c*oSz;L3?HK--%A_9Ht0)T z5LP3m=C3vN(C6&)?7%>ajug-K0oVF0Cd;7)6wMHhs_m4tchwmF1EIR>6w0c%_>F5b zP8Ac_G$P%Qngfv785nD0T@;a1_jgq8f0+e|5HQ4P8L|~$SaB~|%R99@(^>{d<1Kx8$ zAtV->^ccsm1Kv{4+7<%JHgC?{$p()=2b;rYm^oAjoL8{|_s^9|;GI(DUsq?+_%Tx5_2$3m2p&ll+ z)mz7I%hl%o5MZ~^j25)Zi-$=~h_J=>_O~#eZ_WLT5*5B(4!t)I|Ni^W9M9*ACT_Wc z93VP@{Qj(U+Ze*^VBi|WXgTB~M!0Iy$cnJA-$O1wEq(2O&~EGE)&mV1RJfRn`Z&7H%=c%UzK#zU zW~Aho;2L`%Q~un!39MWk#P|Bzn9`f4HyyHcf>H~+p^?Du0I(T4nE6@uT~a_GM@;FU z5(ZIgoyA20iG%>8D}C)}JbjSUYxEkhi`=L1cAo&i6F8uGI=OKBDHQp}T6J#H?_RsM zMMckSnDnYhs)}u!47#h+8$j~F+WpU7e|pZGS`gOljAPqvY~%!LLFd+)8xb~s+P?~| z(>Jsg`ft@%gOqK-5ZP@{UjU+e3vL1(E*O?Zh2v`euc1OrzJ28~QlVU+Ow1Yh+SazO z`_n^;n)=%Yaw!gFx4$kmTCT$(W>B0ND1^N>1Hsvi<``ze?7RNF{74O2oEpS8Js^sV z3dxNd_q``~hw#yfL+CDIZ~OT})|L*4DCcmrVu z^a|8g;hpt5tp2Fx*@ zFMty*aLG{&zhGa#ekk(Yf<4FN9-FSu*9nRlHGC5Vu)0*uvaA zm~`OLKA~uQ$|Ssr-Ru|8Nb3L4euo1v9rmZBq7BeIUQn=HfIWuPYzmqn(M|*8jtvxt zy%Dm^)e6xM9*8KO`}K5<+ywXG!1%BUtT##Uhd0_k<6MU-T~~-=qC*1x1l8Q&Cee4& zDFn5tk2qUQty|Bpk@LKCDI|~rf7c9)&=C7o)q?^1C}t&uKN|Zcu9dLa z!9s9E`UTTj#rdx1+GnxmzhqX`Z!WAHtMq!fS5ZCr#t$3u@Gm0z4fn2GP?(C`ttzLP zGQ#}pcAIzf1$?wy)XVFhGy}C8izUh1^ZDleSAM>-ERozQKVc)$@Two{IogE(p2Y}L zQHv3UI3UnN^yc^}{Xc)BZ?J7@X;8$Ax4R1P1q>_jpYt>Nk}i(q4Nqs;@Ob93E_$A!)kW*uAnPEG2I_e3o`@hgxmb;Gf zrDd#$?*9*V)$)`%{>^rJXKjXr74|8ES6jGC_An^x2r-O(d~tS|_|~%_H~j~ExcFER zJjP04*70seMLTR#4=A!8A47%VZbpg|1RQsd5$cDIa*c(G9)d~$w*_S$L}5hxs;74# zCS~l#yK*D*AfPNLM3gd9M> zDYs$8w&Ah)<#Bd>UfMg!T!K43XyCHBO z@;TOwB8tZYNkGf~*ny{oZ1ma^Ot}dmq1zcHZqRhp{GZA4;$=i{SEfy1`5a=oM}I&p zKgu)-?&;zbu>p!a_I3q!gLH84IdT{Y+XtgA?MLW;>kwqxm;?>?4U`A*M>Ii|x5DT# zy^V%p)uXeEm0K+Y9vcj}kjVy^M#5x{29+CN{Kq1+kJIyEaFsfUK~zk!(EOD~v@xXz z3V{?HKXjkrC2z$3XCQBPh)jN%cQoVXBm5$^zkk#b8VPmt{Zi}?u}2iujWAm4Z<|eW z0E+s)62N8`Z+7jUd6i2U~^Fy?yUI0(hexh~A#JZRJ_$k?-L{#)JjvP$mLIfOBJeQMZ%F zwo?wf{#9(5JFwp$fVT1;Hk*g|N&;3Px3;;`VrGe;*8BKeJd2M4Od2Q2nUk26zVm&; z15xo`DU(wk#GNqg{q%gtjQL{Aqx}7zgK?(J#u-*(r3hz08l!6aX*om(Lwu~WIT|`q zZta4B-73;xYH;}LY!<#p^L{DdMn&l6Bh-?r30uxJ197Ajl}K5%C4OCcyRm0QLMuN# zH2)Rz>Xi-BIO$;OB!GJy*ZQC1dm5xT{~q3~hv10t-x8*UKFe5@9vi0PfA%;%K0c8Z zaTfhiR7ptQ6xw!h>|1~Ne~GI5=5dTd2?a^=bU@rz2lAN7(IqI zp{n*?d#&%A-wa?|tggKJxwG>q(1U%V@vAm+*I?(n+1v66tXu`B)Ie$g%H(*6=vW%+ zsD)WQ^iV-x#UYx!i@EMu=^9ndzwn zT_i`ZIQ} zVQ#sr;PNi0iWH(dfMQ`K=L!U0^g+Xy)Ek`%|3k3M?$%5%f^ndevzV-rmq#sD4=4U8 zM_z>uXvK6U!Z6?;aZalW7+v+-Z*h2W&51B*ef~hVu{kN8prn8{eSAqcGbSIKlfL1^ zQF}}IY|;?eL1VkYM2pZzM8F^{*sld|L8sS_JP&^{l=2zFa;7L-0ofdST!*(_g+8hz z9VqrR@;BA8BGA-T0;ehwhwyk5IFm*tIeRmPD*(vrFlF8N50_wXv<|T?Qj=4&2JMoK24|N_KuFdxE%Ym z1RyNUX!(tu&y4)dC;NK|8AKe21e&cerdo?DXT%vw-Xie8Z2Okt3haXc4F_RwlVB!; z0_QV)6*~!DPvzpVMmYnn{p7^NrYg)7ym;W2D|{a3W;3$A#m=Lu5y!B*VU$VkJK3t= z5+#(Rgv9%wHJ#nvZ_qmndiMZ`lSFa>v^19MOhAFK0M%GB=*0`6veMEdwKFKBf*Yn? zGXB62Hdavu!@(;G0fsP{r#i2!GXa3doy7#H&?P{Kzfh9hNwvB=9BSXesMhi{#bQb-Rddi=;PDZR-3|>F&C7>PZE1#@yCEeFPj&>8|OG`yD z1YGHIMg_oMrFgwKl#MEr1KbK5j{HfrU3_>AkFK(W066P5Xtg!U`!T2DjueVl4$)ti z3`!aIL@3CV8oXO+57yXWYMG9we;-o0#4CM&tK1B<{C%`;0C1&{fU&+E5sQzoJpl8B3saROaYkjg5Hv-n9<*liHu z04k1bdkNZkg#o?$09uwUjt$<$bU)N=ShpT1$fS`5|neF*r&6$lJKi z=c%@jg|VAP<62`(G0Ck`p1*Y_RopA{hi~*Ag~uO|M5nEwW!|A!ruG=`o(KcbYn?=! z|F*jLNjK#hA?c1BtCk*uTl$0fmxv-6#sAkRC)6N- zE=CN3$O0cfzc-$M0hw0=2g9&o>vo5QKzZMgL}>(a9zejr!we?5adHE8{WgW zgy_|e4`|2N*?4%!z@lmtN3s|npYr72OngQLtAI#h`?AGtV=x%W9y&qjiM4--?NkWR zU>(Qa0b||NkwZpU8IN5p-)EpMG)vRP$mpC{JZk36ZdK<9#*bcYB?ISY85JK<=jQlBSLEIS3{9-#^Knm-`X@RRg+EKlZ6(t>+Oe zh(he*;0&omWUatYy_qAT0p{5d4ChU?vAx3$^Y8b+Ur)eN{^-TB`DQjN3&j$RzCS5! z{YTs_6t=3St#~#`_?B-Pf)Jx223YwPkbkU6gv$xBlj;PuA|koWRLDiLEL+d_+@)D; z6@v6;e$T4{@v0Dv7zyt@$`X(j8ChX|N~8ub+qqwpjrxm?lT&wa2B4@K*nekJC{!&0 z!NI{b*|EFgI|zzwYyCXpo*JrALdIWd*^2c8TlYgR52D#59XT6uqJy?>e^_)PW;I9g zots)3k)jU5d8+LRkfzxqUa-(2tdX?f@pGEt#Zw0-J&_&KuJxuV;2?Z>ER6AZPf`%S ztZXEMstP}#BML4uk{=6_#IHZXaX%tdEwy{D1=!wg1AYB-mHohjy8&LVW{{o7@7_Iu z4aktZ(}7g*PBHvO1|D72h74kTQ|7ysXYMB}773;liH|pWrfO#x-cGUWMNoXInG9wO z(Dhh!h51_T;&uh52gJ6!cCuc#mEN1u&^k>e%IkMn9va z*d)?D4kbS^nZ)SlY;K{RbLX-`cq;l6^^FusP;@0F3l>0vRyHo*2n4!(I@y@`jG`1h zM{54X(SyzRFUL`y=Zut7KR(u?*XCl@A|6_;&CTV+gF^s9xU5mob(I{O0rxrM<&%1r z?v(_ZE;~E;B1119=uLv?lZmn&lr_j%zn}@|$pMW{xJJO^X;2K9)+1-gc$?xmLp(9Z9im1caVJ{Ob0znS>$_M?;KZ*d#xDVouWGN`Hu%GoRqrQQfa zarD=j1Lz@jnq;H4RY7cvE_*QSxuIjquzL*43AFvhQ#PY;&LwjVoUAr(+_<9*E{cpJ z9oN(y$@NYatsxr^P}7*fFt}&k5x7pXpiwyFg!Mt{$T(AaFQ|mmOkLu({2O$(Ne`>* zGeDh54BGg>0x!*7k!|30te*@!J*-7*{0=jAq#MLkt)M$vP#==;G}cR3}`KMbQ; zgcC0Lw>(6^B?JW~mc^ly(il}xRu&eI55DADzVkeK17$yKbmw>}J!j`!g=vha69euA zC{G{${?QgzP-TR`d|E~p>WM!dW&o0Vtdk9b!{w$?2)EWUCp$X;UM%RgoHU|WLv9e%nj?SUm37pyezTgOBd z|KVJ;gyx;z(-kYShOOClo>{JW@dx*v{yV|?vS_GEBz z`UG>uvsi>i-{>(48yIiT7E( zSFc?g58mnf_Gj>FB-GYThq(|@a@Ai{;3*){6C{w;VhLAn7g*DIK7R;*r`_?Tr zd;84QzcV!zl2PuY47wCyC}%fixR~OE!z+CMPDRPM&m`PuCCVvc-ZJTdU7$Ydj}G*lor$$kaAgw?qH8X8?Fcl{E;6o zf?vAcXUr5-3BM$x59BFocfPS#VZ(cptx@#j3yX%`l0(>ddjROH%85m5`W@PabCtv^ zO$Um=&aeo#VR1iSU;Yg|7^3O9PiS;HVv8z2a6aX1mIY`NgjGQB6H4nL+Mxt96nJ2(%3&HpSTRK|4+|Q^9ld1#H1pba>Z2{6UgQI2I#bAsxUx zMAkzJI?QCPLZSdZU(^r5#(9w)dx>{1=5_o!9fYd{3J#E4R6pB63bp2>;9fU^-9dw2 z``EE1;E$fo)xCA=)+qYbe<70WzXnb+y}u9r!bf1YUI&M;T~4Y$d#stR?{#*YLV)s1 zTg5cc`H0@BCL5(r(k0&IqoADL1q98TX^FjqcmICHp1a)t6?zfRt2uJ|_K%gQj zdf*$}Ot4q8xvD^PNuZPzsnh7(`S6U8y`?gSj;BDdTu;lx9@7DPc0WYjE|02h1ixn3a zkX1I|K}lhhzbyhIPh;56-XaQK8By-p>tgOr63+G^)S8EIQgHASA_=~eKhqMIzAujv zp)S6gCb(jeg}+#mcs3OOk6>#NQp^4O=lnm5jLayMDs?DTys}BsWIsv08ds7t*VTZx9LgIXwP znm{)j1>feCZ;3-U2GBKqC}|o$0xCGj#=tv`2Asl{-wsP%f~ln79{6K}uE~x=zyb9X zyY$NHp26x48U}qBcg#deIy5u?Fo4#*hb-aUuBt( ztRwwz>yQI@$Q8iB%`Z3q{<{vNggd$&u-KXb@4!oco~YumnpiB&l}O*ZbElm1EJ%C$ z^wW<7uNOknv5gUs`9X5-iMOXH;7ZxRy*f4Pqs1bAf+VLVPE=ia7G z0V;?PLWqqyWF7}I`DwI-Vl|;^X3t#(m#T@IUy5Y^y7dqwhC4`vA5B5!ot={tys50J z$`36D>lf6ikjcdX?57}yMvJjL9UfbTWiX8E`RlJG4PWQ*f*E$`Jk|Zf@UC;wy<`|* zI{f>aj{_56`N)3o8^h#d^HpXPFo7i!S}e|X+_KvrS#uWE1SjffJUDpfzu!S$c&iFQVzv8Iu)ZbGL+o! z5Tk(?$r!+tKK5O@0vZd@?F`EReEKF0WdP&LutP=k1Qh`J8hmDV=38v!Ru*)v8lRjD z;#8`{+>s{5`JaQnBxlpXSJzM&E>DM-C8g}MamyBvWZ;;={T7rN3IaZ`w}bfc%s!kc zMDR|zI4h?_{Z7b%3Y62}oS#x=gH}V|me|Vs9aX_n6X8WR?v4Eul@)-muBefijHI3z z5umu-IoCm{(Ufq1LADD8cy4U4YROc17>3VjJZkg#4Rw3ty{?#;m`82P(ao6LuLJi~ zxvC#S-lY0Kc4EAH?hL^4L*6gII}99f0|Jxuz!KCeZU*7AkWmUCrawpBj4=3Fo(`3$ zvQJlUuW?i@a_N9x)~pl#-uu*RN`1dy;v-m=sO`;G+oMox=&t=TqW+qP5|yK5e%*D0 zfuhTFXRz~rLjwtsd%>nkjZ9^_b)5xEz-V5ixhU4{+CO(RVWcDI$;59F0TZdqw4Qyd zt?5E;lmoaoMo^;1fY@^KaSb zw?h$$5en_YYeFHp{M(kXWA55(`5TX1WN$B`Jz1tD6#*z@(_&py!DMhL!D`_8|Hn%& zzB2Xy^QCV+nz+ASg9Lm{_(d~;!L)KQR=I>jM&MT#_wu( zjItcGlI>D^qJ@=;E?5*`=ql{XBqxOF)IT79D9azaQ&2!j7tIdmpD+O5o26{mqRxQB z1h6T=@3xh}{{`@%?j99%af^T?7ATIgRMUBOcI4CHuqhD-LTZDS-?tRZJWg0}ZTo;( zHrA@vo+|WKhB?;>S<6%fi^#uWIs6?rrRo`w0ZY%~#F}!3x(ycc=wEsl0kty+QeJ5AhoJp#3LwcnVPLfQgI>lSGfUG&aJq-w&t~g`WcGn}|yeq#)D2Uk!)yh89Nz zU?adF+V(_CAY8m4^4i$ah+LmVj`Ix0NwT0P-;fOi?!bz&UZ5YkzDo z$_dneY6wF;n}QFcjyz@*Ki>g;sbXMUlZ_9#ckSA>)I%Y)D$uijVJKj8MZm}?1y>a! z+9znkS6OalW0S|=i{Eh`qqBWPH`fR*DOyXc9#RydvA#*D{(+?Ajk0{5i@L1_C$m@f z1Y!w4gjp-jz=vrc7zL)SXahokAD%<}v5!%DB7^A(<&E`ghh=$l_$k&m{h3}Q)EHNV z8{zqB`01eunhfKEsr{IcSEDf~nIP>25QrT6mScF=WmvNt3x)K-b_5R;I$x9=dSaLZ z5Q5L|zGMHk=hoUJPZ4#dUZq)W3<>H&9llSe1End|g{ubEKVi~WR-XQmg?uDogsvn$ z4s1j#??D*U1MhIA!IO8lyvE$=_!n4o-GXNXmkx1~{z4!~T@BFz+~zr4 zY7mJ5+u|G#CK_r0nR~98s4E2gczEGb=c`vzO7W~`jTa9!xJJCC@WbrGJEgR+Y%g>- z4nxPjVc0^(MUAu`UWx*L;u<05!l8Mh9{C{+D}GZv024vXT=wN#VEMlS>+;4$Tv5^d zmoMM@iC~s_Lkvs36BQq-87y}qX|B+8XU7}e4>oMZpexCwA-t`CI=naP4OGGzc2FpI z@1I`GCpV<~gYiU8h2=J|Kz)Z|BEx{tA>rz?|CeoPxva2<5r_qyhloN@omAN7`e>Q?!9f(FrB~vJ{M(c#y~47YQRB4 z80kC($wtoEbE3S*YxAY&_Frj6?4kdibMH=xV=M=+9hv6YdGvWO!7vzU0u0X!wAC>M1^t#ecl<B&z_5+;2EO60Z`M>GCl$yV(b~7_?-#Kcbg69%%xg zvX27x?3e<`dh6U$K@gTP3FnfmLg`P?q~-VKp~SbJowRIDCpm)K&n(`=C8XF}VZp6aKJ7#JB0N8qVwUzy`SB$cjuA!;vPe5bn7bbRNsLT>o*``K}^_ zfc}AX9!zmEFzq59F3qzK@nHB7uhQmOB0~X3Oyk`tO@iOUf~dUR3J?2!LuMM+Vi9G# zA~GU7GBG{ndcZ`?@oY62)ET1ME}x#*KX8@i=R8@7lgjeaor%;vO2O4yUst6B#kxKoV{pW8@fj?tAWzJ08(sMkUG%YWVt`TEG~mBovs70(&#S4)3s z+%Df3soYER+_Fuel4XMRa=~JexmX;MEDLh(yA@%*5ifJr?eGqpdRL{=-l`q2=P|;~ z2N_3laCz0EPf)&w-;N-vVfC6d=XUMx0m1emMY?gS&o z>ecsKu)fHiCKGjZ(c2e>kbixQluDfOXz~d*bffIn+8rvO2xk}&<|XEyM&Ec!>P3rC z{%^`qJrATZ5hKxzsjU+B`+$um-1oP|iPBejq~k$BWP0K0PEmm*>ke*V;mlK^n?C~V zS**|I-%9}hcV&b-8GgF}f(&6+t^S>NV1lCS@nEZNo+YLTWOW_eLKEJ7@Sq>0TW=6c zhDR_0Ao$(Vnc@6z7a=l(3m6eAoAlN`9wxXrT5U+9q09LJaGr=q(6JZIjeaOGGDj@R z4sqKzi*R$8Ig2gkM%7j{cQ zYD{Wi!^{M)+A}#hCKRWsM>qr&ajf$ zetf|RpN#YA05%?&5bnPAYJ6&nfiAtF_9_e2`qJS*eqP?H9ib#&p{lwXM?1|Z=*`_7 zk<6wFz@)(5#83trX;rd0>K0+R(#aTtl71vx%Nn!*g0{XXmrH;}Bh;@+M)mXH%digTrAC%CK~k>g^cIoP8IsYeS+7AK)ce5SXzdxxd+4q@VkqT;)jiik~h2nE# z3;SI%xI*a-80_m;Mes&cVbXyu{T`5vo7t9_@0ndlU-E0*2>+GEKj)djEco~DK(ZE@ zK#Tvp$VzswQ*bG%jF3id%6=Q-z>j;&CfOkm3vRSutI9TnGfuLAVoOw`B}TJQA9*ok z1vQ|H@w#`gCZl(03+Ef6G!G77!Ne0?SDImuayw7P!xRKqe)hH0`TaP zXP-!|Dl1!bY8%WdFoR^TBFTJjt8PC;U^Nc&4SzDDOg%T>L_G)!I)Nz)(*1I+Uz>)Z zKza;h$Xc2-eI<6nnjAuPI45s@=w&cY0>Q=8aM9IOf}{-QvTj4Pag#wEPvkYOZSgQN zI`3fd(zeGZrBKJrEGcOk^&HX`QH5n3+ilgA>MH z)lb;>z09<|&)eF@Xd^Ljn1cUi;RvV8c7BY)yPzZA@&-d-6Li6vh^zd9DLTu;OKo_# ziIGIcKtBd&&&-^z=G$~FKLWteQ3YEHeU9@ba4lzu*Fa;1qhG@}Z(~B|{sAWceccWy zUZ_r7hb?aum6f}}-Dyr=-O*2;Q$)e-ldOU-=?~~zdQ)Tq!(=I_p&%ijMxnSUS;!^G zlL5(9hMIMI8KKAH(QLLMW>?)+-k67Xs81*Fe0Mr zg$Y$1#AyQY17Y?-_ArhD{=-3SJcVM14t=am4DJIeq(e@Y9!X44kB0qgJTWNt&n;H< zd6f*sT+sCgGYC@x&-1y(xjQU53>oK&JlqRyx^5jM(P%c{LKu~is2(r@64VNTA=u!i zfMGy=t>@JkW%3MItcX!H9h_aNIBvb}D=@wz-bU|NEPML*j+)Frp_k;Q*FjmNQJ#p+ ztt2P$U|3+fwy$|P-tjN;?)%z)?Qj4Cfvtr9#Hx>~zP+11tbpYw%FDy`1$banScAz1 z^#Rapn6gXKY!QYsZ z`odmYZS;{oE1a=nzm{L*i~Xv7=bPT%d%;?Kk!mTsPTCn52kc3FXAt)$eYuuJhhk$+ zK-s0ers;)DpZgc9G%rb0+t2cZPLtifu_BT%{k7Rpw7sAmdY*XDOEmV+ zFNysO-TZZ{MaaK%J@5Z~F{OmZF|3b?ma-zPLPS72%}QH#q1AQz!Q~G_q3pK{T4PjPvj5)i9e@96Y!|Ma||IF1^4nGLH{+hLY?xiz5SBc-R-50gtQAk+Iv*5 zWH0!9ut?bDChO!h#Wl1+5jL4;NLMrk2j+vL5^!t|1iU(Ukyr`+XoW{NSWt^P#J!l? zH_wKI!u@5?{ceFuT(%1)K{x1NaRmCk#WFa4iHCoCD9KQ%O>d&p>7T#Vh6$xc>9QYu zwsOUa71#+V;%T<&PXxqCiDEim-d^3NEG~*oT@`1)#=G9m^u!VKoh41_i7>(eY`;hH zJibs~r4oMcnIdN>t@tkLB?BCEgOIS37*_$k2+|Wpr*rMz?bWqzJv~vqtRhzA6Y84g zPoH$5hTX661!hE`7ULvqhX163SUgU<@1Pu?L;ZnA(}?uWi;r{>+{cyTEwf0ciW-zX z&5AII(V2d{i0;i0AEEOiYDH`wYI~}%tCChw%n#kKz{TfsLrH2kRpPsd=05{E3F4j0 z0guNJ9k7BsFT&$u4)6d4ZX=`=Y3EZPTW`aS^gt!#+e<1@=;YY_Hf`8Y0=r37H1GpW z>AS%XSjB*oW)%SW<6kVAo?o9wJXO~Z%+Nqz(clkN!*-hsk`>M*YwORTy$}jBT4)1Q zpPi#XwZieF`@BKA&}QT^t}6RZjgEeS1BTXlvgg?E1323`D)g&d~Vx1rmKH>Z(4e;{^Ik?-wKv1ozqvSPe|V?!ZG&PG{uT9 zotyjdmWeF$dec3sgZxjQ>R%Z2@%pXSKch%?w7=Sa*PwJogt$-}U5k35Q&r)TJM|AP zOr`lq=}gwPS@++qdPi4M`|BG7Kc5q28Vgx3{?PJ zfS8((E6-#&4r#T(DR_NP$2au&wK#t6hBal96h7iw%k{o!F*u7bPBfEYcL{M#v4{Ye zAabBdYYhyxEf*r= zPE(;LP#Mv+7(jM}A;r8lAXxei9(MqoB2D6us#pVGpRv^FeXB`=9YW)Lm)HlM!3XQd zX{O`el4oZU5xKJY`4wgWDSpW+nDJc2bfUH_P5|SlSYrRg&;NT^$^FNi8W(M610b#Tuqm#hq6Q3Xco8onTdf7#!Oo;V8**mX*Tov&rR!lWGMkaai@Sfy?@b&ErM5EJj7g9(4ce;#*z;jB~aVMVissKk%%MH zs%nDyr3*ZANK#{R2mE)Z>0>nF*^XMN%Hg+1kO8ZLEzBxb`&V-jB?~^Z?L;ERhs2XH z)d37wQy5X}IIta`rqe61->6wzQ-T-5(WZI=XWX4EZAFsHammZFc5K0gCWyCd9D$mN zwe^g!Ws5M`=*N}0%1$Hkw0!LAdz`$Ih*o}NE<@;2aF&X_Hw2V33XHrokF~JaA(9K} zKn~IuLmHVkLyUF8(?(X(TuYEA;71{klDPc2Z`y>do3v!9S%@_GNB5*ETO$OBi3T5F z8I3}%vkI)ow?O`Pi`1cHxCpUHt`I_O!cdlLn{f*gZFwWO<-_8a&CNf+8kFpUR*v0WQ|JVAHjvdDZecVOLmKwgoJ5*X+k zf@$9701|jJ3TVUZ+wN1JoQ_>^x4=8f2aB>^8QA_6;-=qKRgX8$zGH`gNF5S{9zEUH z76N!ZDhoOWx0`AAu~anJw`|$xU;$4nV=S$%DiU?=_;b!BF{GX_Vvk*D#?vpPbmq(v zM+?tmEO#_%`)ztFe(zKgEb^$!wPUFGYT>|Oli0a?#^mUMXUszG;IQV6G+9uVmd$V< zitR>ZfeF|qB_Zg>NCZNR;gWyemzS^C5_WY*noMx^lL;g$okuC!sZ89Pq3>->5P{mk z7{-%2$yn?h4v^7aM00U-AU8k%1@;5Q{y=dh7<+>Nylyy$SY{)y=tL?83Ab?PbaKz% zSWdBK9yCirt}D+?_BEF}aI!Z8m=Q6jVrboo0}hKBbtS+k`i)JUT6&y{@d%2yX)JnG?(~X64zl7srLpwLD+Yr{WH)aejuy}2IbHdvsh7OU3lI>%H)5g3)ww@Muk)3)sd9JNxRQrs9E3v_}W{a0_qpZd(_!aU6N1V=Id7I~kkD z-?CBcK7t)g#X46GvB>R3ip@XAilTNZce1Xbng0EMe0j0>>-hCJHR6t2J}WZ&5Y6i6 zMElRr7hk^EcLZN!yUBaacGZ8s=NYM;YF)1%wjAd5dHPcMvh&NT>m{O2t6QCYsndG% zqEp{O3q3q4^E#YU1C?5j`M1oZW=olQ$uK?IzWBb8PmSGL#W^33Mjbl-XEq%eFqM}a z8!XdSWz)itb%f3U8CSH^QZe}8X)KW7`?p|sn7W|IC>-Mw6h5wM3KY4d z6*YAoN|8N#s2P(gQY<>>qX4Kwe1*txC_D#Hv#+G%Ob9GpKtXyMbc%Ipuq0ng7{nuv zUf}j}&Ld8=q?MF(7&)@|&SJ||7BQ<|mzIU&5%9#u_Nx1U^yJ|(vaTIwR31bwGSOG|;bR!X)cl|zL+itkg*|PI2pVuMU`>8}rY8&bO5lhi7$wnQaFdnU;W3-4 zVs|OHOPf}+5n-b1L6)aTVd=fEBlPLM6o5^GU_on{lZuPir$l!EsDxTsPYUU?rRfKI)n38DV+^^ZUEXoh02%{(106)J0TaZ>JvV}dgkn*3(5K}^m>U@#M|=5h z>^pvU4UXAdvxn&95bKm10l3G;2Cf`s6FS%x_iJVtM%)`15D|hs8mTeA!Wlpf?4i7( zv^1R8C-A0qs`=I9?jDs1L43q+%(K6^4i^oDjw=}c=IrLim{v%K!?hwznv=Q)sX^tv ziVr^jeEE`OAObzLmZn!$`a;LGmq#rgu-92szjQmy1QL_(xf4=H2W^d%Sz-{A*?|g3K@#siJxD;@M^@!)WZ9 z6wV94jd^Byxw&W4_21@3Xs^R?vyD286yG35q%sIfXQ4H`z=;&A8*?tlfbnB#+e*{h z+hwX{L$BPmS29STFFaw;t&q6A7F;<96E{ty(Z+(+ILo%ID+bZnYfgzw^%A z*q`fo0)~X-0!A5CY;9*juzrAOGjOWcE2D!k03VOZDGrqc?$je4 z2kd5=moGo5S@jucbDGFvG6_1NT%q5;7-mByV~qZ*jfWMELc39|zSH2rS|gf%Hw%Ez z&~mz_t% z_i#FCy7*8MeEl)L7lTGpMZ5h5&m&)H_nC8Mad_OGW1Mo84t_ zK?d>wxYPmcGnldU)?X-w*AH|Q64DK#?%KK5 zyOb0a18U>J@k|G(vdUzpW8r7K&89F^NTOsUB!O1q3j8D<9?nL+^unJ)@}>!gguQ9n z+q(eoSJ8p)8(o`E^88B}!tr?{QPh9Z3gLF<%KsIgH z1}mx=Wl@YFsncBHnlFm`DYaRPWj`O^U8G4J6(jD!u>#qf&*9l2T!@h-SO`{^P$eb8 zX5_p9__T^X1Ju=5*wq7u<%aGvj#!NkcqGp;W%il1)dkW z6M4zk&8=NuzBCbPuoPI-`7dygU9Cq#!yXjwo6{T+9_A~7Zrli%@y0YE9IpKrg!vM{ zug<iAn`+>kq%NlTqhtI+jWG1c(7_r0lRzkNjvVdXU$7#15;2OSK*o`C_ zfTCqQNbl+H?vu&}JI$<~+A!VSl5e;3*_l-dUI*{oe`ZD8O9HN9k|SY)LnQuz@=LxS z1rf|mUOMI%EuUy9h1$>Ro8Q`5r96ikt1b_B1BQQP zTiFU)+%nRBU1wfO8yb~(opxwb_7Op|8y;KPXDlV^bBlycP$GJMzf20wiXqDLU0Z3i zV{e<4NTI(+PDgJWC2}~&BiUWbta`-)>Ln$+TOP~^D*i97&!Z+UfyvC~T zI(a9ow98T#E0&yHMFB}OHCbIWU0VGNb?q&$GnjV$J^f!_x0Bp?IOlspxcHQ|FUw{z zlOUTA?V<*m?bi;o$Vpa!Gtl*#ESX2GR)sd6CKvHJ@M67_W66%ajaFF;bC0!)Du17` zBxUixpBZ+fJ4>#oiymg=q|G-{44xMwV+Vel7scDNe}Xsg-=7x2N8Lk1XiVT^l!(oH z!-n+$Pkpe#^Sla&ewOr*U4_^rSm>A?MjZJrJ?>(T>>zP4;k~aw zqQ}8{>Qsz-)mQ?YOo1*doQjb}=N}FIWAbV%2F8@&EpkjXAX?UkoqxZMQZRkp9|zRq z)*xi{gmXT&Ao$h=mb#MxXPN=x9?=Ka<({F7FSNUTOZ4afS@TJqxiRXR1aQ(hrl7+N zKrd&V>clXD)q-863x5B05(YhedrqK#6)1GjkDb~mDHIRSs z?X)UeZ)fLgQml)RPzg=Aqyd-E$%Cc;djcXRtKi@@uZv^7z{;hM$^=;O6)hn`AtIsc z`R7rjHv=Gvog|?c72usqs>9*qBBGblBryImAcUe}r~XR^5StQ|}l+u*PR$&Nyr+^#}utzdBzqr zA_8#?dtfA=4&zp{R1r|vjIUlTmyX2hDZ_wujgX@K})|F z>8*m$pJqU}nZ`4i18X8tEZ4jNXex0?bW9%q0#FH)@R6$-{}3(#r67Ow+~YQb1KQbq z+Q^h|)Pip3*t0XJ&ug_z!1hjo`3O?hE~Hi9MBn}5wCiGE*Ac5K>1=!@9TbeVrgWxa zW!}4;W9c-r42+sd?M**P$L6a5w}*g;tJwF634Iy^3Y1X78l6Z7M}W{0bE#5b9ZFD7 zRE!bl`8xEOqXg}R=BRdOQNsdA{!p*@93UYk<#l!H16=?)o&ue*ajyiv@eu*TH*LS_ zgI)po(YuK@Oe$2t=gg7`!k`)GL+@i|pQkl|5cfY&-;B~K8ulO7Mo1xu!>ci-e^9XX z=);V4x>1$Q3wD`uVAQa^H&e!1ds zDOZ5A5F8dX2lvBHQIKX9yC2LUBqivr_V92$7PH|Z;?{vY=)yksufA8B0W4puz*U59 z*P3O15(K|In3u!#^(n~Cw$e^uLy>T_^4sHNjMFA<@8lGZZ_>2odWt1ci+&6p2IkX` zrQN9PZuzg7fb$>_UPre>W2;WkfNuF>l9Sg~{Rx#t#03QyQ&bjh|K}AyoX(c7-JUX$ zj?$CZMLNX+*JDgKp}OivRo0s{Mi{}PZ{%%w5r3&W?;gk-@kuf$AMtu zI8JhBEdeqUqFBC$D{RvxhdmDXl%U`=8hqD=n&b@#_PIq)XbT3xB2p8A)aPiQPY_P8 z!yyN5Ap|v&beC>25BE{3T-67f`+ghup2M)%1Iz1$=?U#HM>WB9o7rR`EXqe*=@YCg2}-4K8R!leMz;J8>eq zSQPWOPer^m+QO1WSQ#|_s3VMLRv51iqRJC-oV$h(bII8P3B*@0h@mG4+4U-)N2S%N zg%!@n$2WL79KF{0m-bgXCD5T;U?ry z_3L}sMa3FEhSl1OtgKB{=aSCxV*eB34H@r2&gwC%D!mSTYZH;3?rm>xiP?; zWJFSm<2Z5E#P3~7_- zN3<7rIg8Gc%lG2>^L0PlA;2-Mhf(i}14w~VnT-Q5s~a5zy}C80lF|nq7Of8VDNa<` z^nQ^sxFlJ9CF_VF%5+|>%zazI>V0LkoyMjmW`ZIKUsFel(ugD8+k*!W_U!Wakfz)w zm7PNY+xqzU)aDAHam19w%N%A?>c+;sp$m-e?z8gpH?6~=Fhq>@ruM2DXngho;C9Xj zEWGOKxDs8Q(6B&uMSTQJm$)d7IAX;1;Nio)yZnFly|hAnT|l;#BGMGmn60+jykp0+ zx}X2t$3Uo$!Zuveij0<1tw)eQLjeP%U0zxe-8n_3~=lVo&BPHho9>HueTCma87js9wjm$`<_x9u~VS$Tz-qeAQSvirs%ocqD0= zLSECp^Lzg{5cuEkpniRede@xC6l2q@5>4>JVqHAfON!+ho(djR)T}J?%WG8dPJp&} zmMlI7*E_C}GJ59bwBg0R9D)e9B-(fkj0I|%ZQDUBHedWD+XT1#@89!1P-$k)ke%lF zc(L29r<~;sQMUiDFT`>j>xIU~j$b0k9B2LHY4+W_hr)#tE&05B^Ct9$x>Vg{RO1A< zix|K88F1#xR;w#KcAcl_S~EJA|0&r*OWM%QcrlrA7ZK@a#?6Czp;NM$2eEgh6v z-VoR*Hj|=3$Q}DD-!5v`QAKWdu-f%uOIl(8R zdLC6DH^4&RZPru$*ndm?Duw{fqyW5VqYMaJP=sB`$otM61C;k&Rhi(hHV7CN1i@l6 zvk8K1z5dxaxI~De1d2Lx$ExU%`U`9r;A+VXyQ^ay82c?0=pCt}i^CIf9g_6K9_+$% zxNPZC>*@~(#Y@NMFeqC-S^aeFx4H9dd+7qLm&jZWhcN|Q2K?0OvCaHA>^Hn;`~COC ztSTTzny5YVgvg{BC@CWe2A^+%6y*vLrWqdQzMrR=Xg@ZuHrPLBVC8x2?x7Q4?Z)hT zA1dqhVL*?Ii-o0b#=H~Gs@KXUW;&>?p{F7*b&Ljthx>Xk>a0XCS=`0-g(K37^71WsOiFl;EvGF|W9*nGQWpe<{{SZo7f*98&v4A?I za~!Fm7KDT)Opu2IBSO&&oZ#nT8hIk(9^_&(m659%@%Y!dOjFg=d(tkWHwmDx5sxO1 z#Eqme*ySmfwWs`ceLt$92mb!AC)!Y=`(s$sScWHEg(!8BAPzl&Gp};Y z#@}7NE{-}qHmCj%aON6e}3i#a(7{Qi0=mpDHOGh69riVoo(s zo)8JL1D>q6pT1%4k%2k1StSvQL~Sg7rk$f=cCUB@4eC2AyM*r<+}Z` zE)+Zpl?KBQAywg?=gmqxQNubxIE#~2xhrHQjJ23>+O$7?e{4-=i?z;XSK-COr z!tIJCNst*@6;1Ovq6r)V{@D^kWvSo>j)4%KhIU!N5fqoyERb&m9`u7RS);{3Rgr=P z(B^=TY;^bnm#?3nF;?1_VRbNtn?Rk)Y-|D71FjHV!*%$PcfVRxnlme@*1lr^G5PW2 z4x1hK3knHC1F0+;igDOxGcev^G2x&P9g(>#6*vC{0a8pJ1Juh*b|)n&ir!Twk0qTB zeyFIprp^QVbkI^2b5($w*>OI?r92(~;qC&mJk(s5-pT5 zK7KdHil~n=m3#T%V}N36<&+zURvZ@&<2*73v3DbtRtG@^{KHfH`*BhjAQqw|%i~Aq z3SSAN@tEKN+L%@cV%626rHHsCEbh%!!U(dS>@MHP8&Z!1pZjjtCx%)fD-dB9iIjHZ z&b&V{(+I+`Bg9L-%4mYG0BG5mHi5R^ygm<)*wjPEj>3uIPm=V%8VUY#a{wcu?wMu9 z7?f+`!+Dt#!wo#=ebf2R^icR%fS-$_329K0?lYMuQc>gzfrr>v`|tw&LOu1VrMSKv!2+M?EC^4S7|7g`@nTK|I5l z?`g$wGInh45)RA<9u!C>5>xOaKBdSlLMEV9?yj@yf(X&9$pa2*Cw#a+qfbd5mcaTP z4tOale(Y5n=(Q*Kkq{pK6>eRBL|0)lY(HEF_mA!jp3cEyf$H+&wxfu2PB5 zUOwu$rrhID)g*58b1n%pZdTf@8*eF0>;-c+ZVPR7uK#U*x!tX4OPyu?MI0S{S7fdi zJ}Q|jy|&`t!T!E?O#U2e&(%A7BBIGk(Bx4^sTZB+EK61;Pgc*a+76x<-u2>0z@jhG z1bQb!flj)=-nzf5WME^0S4(eT%jmlPssF7&1OLnP@A+t(s0C8MMd?OH;xIWYv^aL> zSnIvIgrKv1M;-}PSaluF`H8n#>|CFF_9iGHa#yYszJXwl9Y?}B@@`CTMCwt$3Qi<-zxufRK8sbtT_uoNi+^sY zSk)`^AIKD%X(bSp*xJ*up96f{y=y%jEY@Q7uIro+snc0xS?$@wm86|>a1p`G$Ifye z4be?`dF;!(!IJG9YNr<5Gcq%;jV)~0tGGBNA2^qC(Vs%o8f&A6ZMVF%^hEA?IXTY} z(I+5rOarb^I5oBw^B`aG>VX?;W$xcEofE+phBozP4!kE0U|w4=U3BLT3&1?ZTtDEU zfE~@+=y0zg^Z&F4lrFvtXgMOj(w_cn+l}82Al_k}2wVF`_#29O37_LG| z_~ktiGnSPrF9U|o`>K=g=nFDNkedZz8lf7ldwdmC3T0KcVJ*}l!7X^mNwO4q2HPmR z&txKqj!R9*9Xi4`7jJKpU}U!44hQ;b_-U4W!bf&{fw`iN1Go=w0hAQ$k{Dvl78T5i zoi1THS$`9do$lTUu0nWn6=;c?i}Oj`j|?2TeFbG!t@%y@xWF7vai~CF;D964vCb08 zs{$BTJ~0S2LEL7A29L1t1Kdxad=$287jal+GU9Tr+y>*UMhS%a83X<6N$X-TfNn{~ zWi`S$Z+~V{7)-w!B|vd7Aq3~Nx@=2rM-YpvzQx%9!wDvlk8Y&#AYNKp=0r;a%-oJn zA(%=KLsP*KZ>G;8UWcwZ?vOi;-oGCMO`YmgJ#!Z~t>pji`bSlnlE0s@$7 z$uMF;KFiKU><|357%~EICHA6fm{!y603s@4O89Z63FS5~j*4U=MOm20`3u{8*UZLU z$5km)_%j*;%$aa_ZBN*bP@m>TU@*0Bzv|PQ-SYax=A@_|_}~Zyh95B>pO!hJ$~FUp z4>>Qo16`18$K$oF-77(3_y}RiBSsgYt)N6t(*kh@q2F$)_kid6sDz2V)`*8Fi!fnO zy1fC(m+HefWAQfvkq=jid@s5JgwQ8%-n_XM8NY8eEJCHK&-{+8QUA$r9n}*eIJs5u zYCvmo^xHP<;(9FMkW|XEr2~rj^)SWkO{znf{U99v+Yaa6$y^b5q<(7nycq zEa2 z*=06PV~uK}5ziZFYH7*9@<3u?%QXl(+b)>y<^nCU8iYi<265`A6sZ%_)MHMqCspCq z%6cl-#=!W>mDjo-Ko>*zqcjv)$Oreoz9e-5)#3P16T*IU3wynd2{IL#r3~;c`h3)tioLco-=CBqyHp>m^7TV%G9S7Xc z3U~+S4P4aHqB0PEmFV7*@ARx@lQzg(?}7p#3IAq`@=- zYuydW#cF1p(6mQU=+ssf9CCBJ(rpRm#HN?$rdzF2@J-38O5LYxT66GlVNi{6dsPUE z{)4d{m=M%mweWfRyzJhNqC~rIRJ(L|+2pX%7yNjt9d{O)!Bi3U5(dxM_Zr0OFa?S+ z{0bIom0j^Uc9DrF`P&;LCK8W%t*$-Xe>m9No1A{pRUpwiLqB&x)vvt@Lm@s(6$4?itG(E837?QKxQ`-V)C|%=*EnZguku8rf8qTFL#jP6K+HUkGz_i6$j7Cc`UTt;LT4e(nPGlCS;xlJ(9%SF`-t`v&vvKf9QmxAx|cVr`KO zy7-${d=cZheFanL(^p=zD>L^VdG(=fGMGktMm|9Cn*IUWxyTh6FB@H_&t6NoY*F4Mh)a(%KwmKHl}nwwq+)!l)gknZD3#{P44<3n7vElV6FfkYrTIB zKQFIBvdU1Nd5+(9fO-hj398hexLtMnaeI(5TIXL+1w{ zE55LP0M~pxJdD%f087bs-6`?&C-svLmLj{=;U|mJ`LT5>xF@Ij{bgWP=W{v|&SMs! zvQ&x)Ozee^9*K@D34x=(6V7@5y#ywMo;k!i4s3g0m#f3}%}20DJjZC{YDk2P`~EPJ zcgu$;wP+P+HV44nv`)oVWnS?iENq)2lz*KP6b>eRP}bC3N#hQ`iG5*L;l09#oNFvt zc*kAFL~PSAj`LRA0^|0V4obNyIKg8pnP~gk*h)v5^=KmK4`XoJ8I;kEVoxAYg+bX) z6j|@9s<@AM>9D)w&D#G<8vC6%hh-)OUlhJ>W3qiQujriv@51PQ>WEB*Q z{m@mp7>|yR$G~?pO$Hrz2*#%=jI?(`Ac@bwrQ!Cz1sMWALeM}7_lJ;FFd{(zIt6}I z0+_0`La;FZi1DL0zLBUK;0ePg9oPZ|f-0DPyg&@isgSTRC_b?Fy`$5Ey+;=cL0PK> zlm__^j&})UZqZ-0L2*f#?B?OPimDiYjty=6C_6R`UW)6m;-9#AR1_MkyWXSU)B*RH z;>y+vC4Phlm?^$l{s9#UL5~86MU7gdE+h0UjWPgMpws^O{Lm7)&cE=CZ^Cts!#r0J zs%KP#HJGa)x$K{&bk9KW=fNclCs(hrv5{a}XaR&B|2?r^c)sPMXb+*gdV&Q+rq%tB zpp^PCFC(5hBzmj?RtiKtHz@Y&i058QTHVp{esyeR%W;9KnpEYg(Uia+j-LzcdV`a{ zMAU*;7qPdW-^sU%+Zjhm2j9peM6yJ%Hc6-{as>m@yH>JLtgNw^KEIk?7_ifSHA|sj zAWllaBFs}ss1T@9XqaElw=UrD7B)Q_5dv351%OHQVz8^9z>QJHgzTdh4_sLwBf58O z4EiO|4WJIccBM4=qm8j3rFZkg&h|f|X*?_16I;4D_-lvfGjM`MZ8pD5pS#TQe z{OSZvM|+t1@i>%m#Be)r2B7ND5yp=F7R164G*+vh3sz%^7*N7xo?DvCpW4ILj&>AM zK2OhL`b{uDS8n%dP__F}S&;9Z$1Rh=Cv(#p*u%|Ti~pO~n*#zKMSK@rjG>dJ{XfLL zc{Eq=+cvCOMUzUTlBAR*b7g3t0Yy}X43(+KlquuZKuIV>=2B)6l6m-2WF}+A3>ibn zJiW)I-|xPkd#z`!_qm_>Js93wfb$P*`E#fUH)xM0MsK58fe9Aeas)l z+Itz)3wM=8Uah^*?Qfr1yJed&yB2(1lgxRGKC0Rol<`BN@Ri-LlIF|=#7cr-JhUDB z{6oh&83}3vcE)y_qSdLuEHQEO(8M;3nfwRJl<1^dz>K~ig+NerV=*P^M$#SwG@^;f zSNUGzm*Ct%>`Cd(bb*+1FOS->w@@aTuX$Dw3WWMP%WFbD-4)Ks29Ut!+ zoPZRk9=KTOcrUduvkMJCVas6R>jg*^o;-`_y*ZhNO6;)B?-1aUB#&jpV09}dObaMr zqAHnnV;M&3H>jYcNls&ZWiSVEO1+0rX32Tl+wVz!4CMO63>Ujyx}!ph%ofgvnE1r1hY4inqfHrc|34a=d3iv!x2j9N(^< zp8Wy6z}kE~&IS{l6`Ph-thJJ8=HembtlF#l^v2OD@}EO98^%D}KE zR2ETjqsnzVJK{tI)YVFXjJzv>6++5Iq@AIMZea)!?IEqqEJC*do%6g@*z4D3G_7mZ=4wrHv}PtsiHM?d%vs1QuYS8n?U^cKtvZ>T>avs9tYTuME-YORbrbGO2c z5PfQ4*sf#LFt|d9N9f+%ip7NOx&#<%W(XuW(f6d-PVu6xwe7gmdAW}J(PFJUQ6{!} zk4dgLINI1$uUXm?Y?9MOjmKa;m%R%Kwr8+w(nJX8*6$b$-*cwg0{u8k#Fr z7Ob&xLU+{gSo5Mf*ycCN#iiMSV3D6vLK8R zKt@D~2ne4_;ve+q)qZMBs1`tRe~?-FE>#19nC%8%;C- zb43vLViFnmf$Gp`e(2XP8Bo18#ljbP%U|7#2vOT()7(o(mqZ?D5CI6mbi+g-`FrU= zZsY-l>}99q!%^PKf!_Bhc{?PZ$RE!^02W}0rfSO|pbxL-XR;x+nV+A>(9%$oR&cYS z??bACI#^(^9|O0^Y{!&Zu!Cg?c;7foOH2D2YVfs2#8)Zd!(?_cH@Jt@#PkP~2kpr? zZ$N<6QCD2hIiW!B#Nir&Uz`P>C%iOeSj;}it7L3Wrr?&x5lR3pk z!*iaF=9$wI9+6ZAYKt_@t$jl%FI?red_H=7z2otVeuucIGZbY-8YXdo7ZYx-FXwYY=o zRfP|--UI~lyLF(g@3)zsO|2G#S+g&=sHPLQ$>>p#HWU$d21%#exGoZ?FoaUCs^f_U zoy1HV>O1tN7T`KY`ckm6&4orabp~rI2kd@mP?4M{gH4p${S!s4G8iG%GBnp!knBIM zrpagPfBOwA7*WD#ja1tP&Ip+@6Iy-o!iBxSU^$bu8^}< zZ1TrLkzL}plRTIVU{a%38RGa+wll*3$vVjIMPp4sDCJNgBrTH9NHQ5k{3ve+7g=9} zx)Q6;r9Tz060zJucNx$TB?(V<)pCRS54*3#C|v^32_#lw5xoR*ER9UmBY>7hW;Hpv z&b7KI1HTT(S?`&hEe`y&Kgr^Tx2NZ6EJz}!lY>Y_UEjZm6!!S#ptAXh1r1GwJ2I|H zpn}8?>he2U`;TLU0gN+#9BgyGY4wVKwa@K^mxws{_5gUpBBWWX?4bmO5Eklf3?#^R z`};+55h$y3=xotuBEv8i%Zbg2@=Eu2b-u=Pu)Q5Yh!8%`WU_E@%MW%(62Md=dgOl^B|1bkT#zQ zL+*4}x2S*CChjk;PQfED$eeU}139xz6(D9b)kXJSCCD6SZfRg|bF}DI4)AE3=G0Nh z1ep`uxYdTqR?>|jc#CC+}|_{Z`Qci?+&)OuW5 zkq_Z#??Lduje}rn!)dim6B*h?VRuNurO%yv?tNYZxw#Vh5045g(WAbtcv~8j8w6E` zvOj}cVvw?av4=fG+cD5ZwMm7YzcQ>JIP@t+9veHCr%ye+ImN`(DPeb4SJ!I0DinE- z_z+4RA*|A?hN;#KgxuUDpuTbOMyT*~PzR$G3Vx}o;%*r!shfbEHpPw&UfG2~x!+MQ zIriHkT5*`{yqV$6 z^rx8@Rn9my_iVEJr=1g_h)G=LIt@eYM2N#p6NH*8u6XTEXtA()p=Sk9f^n_k`S;`5yG<+B>*sO}{I* z-#%mW+CW3=mrNHW6FevAIOKWMo&MjD7r5`5XTLJ8Lx%7%Z@x06;@>?wnk#&c%S;rM zjF-{yto*g>KT$9~hhvdT z6lu(~*NmpKu9^+w5u@Rur@FGlk4y{~I@Im^S0|AClvcV92Mig^**xxZUFn|P(!U+h zzhGOz_H!_51P&KZ@pJ##MN_f`v7v-!-Q}DBU$jffOi=pj6!CwKCO$(e@-I+!eP#8e zzY>kz5isFQJ!l-5BTT4}@)aB5zaTsCdwTi5(6;Lv(ss&q2<4?!DzUsgLm5C8-~D`bd7-m z#1Lt)P=6;e4FWGS(o|LgB#UZX+Dv+Lpczq~j~_qwl7VIF8?04N#}y0Gc3@)e$JXe` zB5?~e1){r6cYFW-P-(zX&oCL^+)Vn->)?h<92egRxj|Y8YcbA43U}_rRAimUr9n3a zyenP2dDkvkqOP>h0uX`%<{1i-Sd#^uPq%w2-Zaf{u(44nO9`5U8l<<>4%;&vXaou2 zso9p0H4qxJO1j1Sd_?6TG6DpuioJ?E@L4ek0}ej|pKjx{A3*m2KzK^uhK&fyRj}?Q zBNn?_@~aU&U5GFmQBk#AakC@7YH{p(ib;piL}Puu90f1}d&8JAfSwmtjeNvKG&hxg z06BX#CLp%el_1A)j->g~x}Pq1*9snHySc5ma60oNxyr1zJ_NHeARPNK8X?Y{z7QZj zx=@mUV`}u9QI1vsO8O&z5Q9kX7#NJnOZ#KzT|&y3RSpUp>0A*0r;oSe6T)&rsg^V5 zZw&|rQV03bo<@9g55#^`ZlvJ_pOpx3$-eO+gCxU{N9}?__7ma_khbdSNzm@ed`MMI za9P5^bL#bCT(b&kEM`5HPgtRwM22TD=1QWlBJ-iIBpF%HP98C*pk+Mhi5_wwsJ`U| zIC*8@z4kTP4cA5((t}xFZNb9)j1lSFD2CzoD~aukEy)@;L;C#kHh5}8k25xmqmhX0 zQhAArdMmf;tD(`7N+4~$VC1OI{6?Yl4Nf?($A*C^$)7kuqnO>?+)Ukutohha0dL$t zl(X`S4~Cc@f>+0P;s8y>N+4cm-o<^1LFHXZ1~B=kNG!~rnFOz!c#so=yOus?72sdD za7>cPEs!=>T~kv~`4=mVM`)7oG)={Cq#nfM2Bf@Ke@<=PNNf!u2BpM8;2z?QSNLql zs=pxHAHpa3eFh?3Ql^s8rkZCfqsfm)@b5wP@b9yjX2oTq_5(F<`S)k?l2~hei)lAIo9>H_8xh$@fCHim;0T$e?D5twV&PS(?88zgF85>8Uak66 z7m_peOWTZ!E5Tf+2tn1veiWjz>!zDmT1PHza_UbP$xt zncDIl12})Jd&y}#=FJJb{jl#xUzk4OBOH7jvY*gaza?a#7s>NxUM4{Jl3oyw-&(LE z2E`~$vQNZzA>w(|7`kYCwGAqug-+m@cREB6s*E6#+MNs&2;vpx;^NZ%bGi>f)6loG~wrV!H{hXfy z2+V!_Zs(I)6_RmS?+8XQTYC%-TL%_yP3F*96`~9ZX%up-LecUnI{KhxHtGjE!sdsa z7M1f6%GOrkFVLsOitEnC+U}z`S`VHjL z3}u(0;t_txnFpYMnDV-%UoX+A$cGb2?UW@I7kg1hiH-TaW+f7#cb} zYX{=77C;yt3$ur*fL!&fBVHH>!?)?PJthP!LS33TLUNO^IKnS=qn2uN`DN5f41uus zaW4aniYb&EKwy86?iM|6OhEg!BbG&Ib$X6uUIR%PA~i4LFuZ_7NSHZ&X>$H7T36}x zkf`R9N!1~J+WT&*0L$Moxe%QgU}39o&l5UY#)VU-a{a}3anj*kqh~E~FCA53#+2!C zJMf%$0|PyMk4gvhZ$14dGT$z(WD0=jF}%xsJ3?xJ_tfe-(#L?Cz)0x=(oFI|bIswR zK=wTI1Zqq-XplaA+4PRHd>Eq{IZRjX5I zzI}E&^?82N-#m-vL``*y@uOw79j6vZD|7x^KxBNY-pspwG!B^IjT-5SLdX3S=GL6e z99^}P>Gt*8+?gkQaA~lb_a#dlt&Y-4P>zQlp~6=-gMF*Ral z78RC{RQ296&9ajB3+UWBh;bQxk|3sq5aG|2j}&A?Bi5M*8h3ZOL- zGX8PmFGmZt%2qx)l7jw$cRy1A|4wSK}SKcI`rD0Ie<& z_hIkR))B8Y0ur6n}TCof_G4 zV1!;vxrn>~6Qtxi#MH(wka2AQXIy8%+q{TmYICdrET$JtXxs|0_##PcY3#MWvy4XP z>Hp!ea1@OmNQ-_Hz1hE}5&k;VN3J+P(1EC<0?C|7FF%6#yjO~Ua9n{Xk^owT>d3!^ zxPlxXgI3MxE+D?sNBTm~poFt3g%4C}cbLY&$PM5qX|ccp4OVF=vp?z(&Mf}DJE#>* zr0Wm`rS`G!+m}c@Y1~Vjn*#&k0Q0Na%IsQA>g3$nPa@^<9l|>q8sj}*u|L}2L31PJ z1d}1tHK$nP+nipKOxIL;Xg+S1IvmKbKQ!}?d{~9|#pFMNnKJjCI3MkHHrdq3bf;kS zj`Sz?Mvw8Cv+Dibn$qJ(6SU~3>}$VCaRrXv3m$mSC!!mi*=paY@7rQOV{nC1RgF1b zyT@H5B<5=WuX*8DNtKnC#y&I1_89sA!q!8>;SKkAfWCpj6-;tW9JGeu@@-UX>^L); zQ3g(KvQW1TW#;2NdYuN>adWCWHe*D}988+Hcvx5KMaV2*w0UO!cMwdDja4g{`}BTl zG*o~9e17hdne_zKY^cCG5 z;faX~U=}Tv3;BInL7}5AFsD!*<2Szp&ZLoIznS=*9b5s{_>Ax1j_eQpti@Y8IRrx@ zE}g`qx3o8IG)Zqx00nE~3#Q^dYzaVOUVESItZ$`Wr=_VViej#fOz(Hoj9W%VK4D=T zYGYph{s*5BVNWeM-+-me*g2>qW)DvxL^c6;w3F-W4sZP}u*(+DBczAp3{D^b7ZbAvO z5HaUQ-&8ffQk|nyot~#v+bep0{qn+w>d|XOvXKCFW^g>HJDhCOZnEJ4cb|U#e2@AX z8X0-Mp|R22j>2IT8WHhTpC!DC6XSH=40cSK6_iX*TlWqO)NBd=`0+AQ;<5SFvscyB z4hi~Ss2j%}R32RGu(FVgVtusqWeJFUE=?u{I}+Va^GXMwlUlUmKJoEJ2^Z^{=bkG` zUd(d%!Hmdi8rF9#>mCnkW4x!GeV5s>FTG}cPc;SoAI_(Qe2qVwr^@XUtBugXE+|;r zuPeogB~-bkK#Lbf>JQ4xA?ExtVA20{%dTBRqZ_wKaPKjouBTZzr!q#&X}=rt4?F1D zIN*&1sAhsgjvX~Xum6^cFBf1nPl zo{bXpH;`gUZafs1Iqo+o49$SIo7=iL?Y_Z5Z#33=V#ASrt4~PoW;?B+5v*!fDVf$- zb=ZD*?P(@`jW{#^go9wjxeKL54wETLw6)^4w3&`e> zf{{^NfJr`6TJazBM{O>IKuS472Af zYO1OjybSb>j1=osOn8Cdcj~x|PA?YLg#eJ1&GVUbx4DfqnU+T) zu)VHk=(;K`?UL5%Ax)QjzhI}04E_4`kJBa#lEByBKA#S9o2>7axVLXlrUqb@bX{}U*XE!7kkcDBg>&T zYTY&edzAo8`1{(I(K4_nI&9#jtT#-Ip*k`>QM9=9&*>oc=f;h^D%H6V&^kA0Y5(Pj zuS{QLc+i@nPolHG^U}f(1kYNJmI{fQG#Mthnv{tz*k158Ole!0-H)NUocqTfrH;88 z+2+NKYw`L%us-?B7bW-1*c|vvm7k{yW$oK1?0>)=w)XQ#@}WS=yjIyyIJxPrOKUs; z)gI@n#Dl9^{!J!mYWg)XKjNY>dsA-S5KV;N=~=t1dv;%ldHQrs_D%RVtzHRn$sd|C6^ zx=l&;mCZagE4u7^P+PEZ)B4Fq?~F{wuS)Z@S7+8Gk1S_yWPD{#BXQURna(zRmasV{ zo(JO(PpmF*nY(4!T0PO^GCr{E^{zq$3FGUlx~{&q)_U@LPV%1mhCRHF9rNsdjY4Ch z6CrjpG(Bf|*FIUwQh&d~uXF`{s|L;eO>Q%u#xj%3DJkJvAq1Cm2su2`sueU2mX#UQ zvo47pG>?ui{^ZZElxf);)`uwbCl49RR7;xLxkxOGFR5_X+PYzd@P}olE=LkKoNK#x z^4ZtTxaj}(&RA`9hUZpn*#Aqm>#A_Ar7%Z54eK`YBd=3(!sIZf(LKyLy7blh@z-0_t8wH>bMd($beGCrfPr zLGZGG2cr(`Bf`oE0`3T3#s0@?%In`fJK5L;8w_?pXS91CsL%geCBs2^0;Jb( z0foy8Y_`03MC!eqoVRPWN0N-BE?f{`IVdi!0ryuCasN^sPA)Ef78qS9)!`$&L>$c+v6z1%(wh@zPffgbvj>eT%=g(oK~LfkKJ82}xq zd9!())uAd9XI^6Z$MIS;C~M%8|A11FeRXZU=%44XY5Uf^6927+sE@Jl=4npzBF^w_ z4VJ+Ond%)PV%S^P|BE?MhsWQ=OVk@O-KT>QpsQ(?a^N2b$l@8H2mRuT40jvO?)aG)9Pt*X(J6-*X&R8 z%5#Art5DlVjl(e6CSUskho{KG?5MhUdS6tumYLaQb=zS{H4|=RZBy*DZPo7lQ z)Rb<+ky>b(HM&C?;^Ak-Y8$o#Wi2>c|Yv_ZWLs>JiswQ8yq>V4* z@H{_1o}7&fAp>Lh*O5oX3Tw=^GyZ(AmiyHRJP-ZuKv!l60=Vzm!v)scMe#_~#b^Fm zZ_mocCLkb?cC#6;CjsCU=aAynt3^1ye%iFo1jSlM#l-yi#rW{&uf%e1^rBUR?zOL0 zD<&`RZk?JXekj}=UgA?-w%k6Mw`=}-M9loX4#a;gu(uKoecEmT(7DO$3-05dG;bo$ zI84$n?kx`IZpy2!ShkxXB7Uo&Q45!4i&b|_2dtc>K$@qK+8m^(r}r)@O0U-Bzh;AB zBI=7Bv)`^q{$)G8;Rts8B~}*Qszr_*>$YutYoAa{MGnzvh%HQ&_wVHE_sO0O)ksul zI&$pT0ih=2hl4-uT$k)&8!6V9mxs}hsG14+%3{p4jh>lsYQX2@0{&BOe(0LAGACMc z!R6&fnPXSumAKMw&h?xX_v78(kODjA@7*bSo|rv#$uO;#L#?(^yJ7 zE~R}_`2- zH8dR4WbrIyCqMY|<;x{0zwR^}=h5?GuCV@h9Ka=U%1clGM7H2X5)l<*#i?FUvS}l> z`}6dntZRw^Wd6t0&%;z?`m=eQz}~^Lo%_-nw3U^8MSkZj6BA!3Qp{pQ(}^!1hu{eE zg_(fRA<(pZpgFP57iMEa)6oZ0d2716&*a<8mKxc2m~wn0$7TZDP_f=qJ2#~wamch6 z6cORlm>=;mclRB`5}BTz1^>N9G>B@Y7iwbF4K(+T)n3A?b?Qi;h=bVeoLFlvg~O0wy0qAT$}U*$aVFAeIqDc`y>Miz@DB) zMp`rs6j906W*VU7(f7})U4NMSr8TBKFvX;9NB_sFiwav^w56y35Y(D0xOBic?aQ9; z&wijH3o7~*(Ad(7FRQD^&SNRAN|GTFg1%IXXoALYEg9#kh6cS`7MXA&4NlBw3e+C; zQn+~O(4j*o--qkRWnRVf$3E^VU)`X{fQ!0v$kU*@c=zVJ26B?jvX6R6DmB3KV~<~? zY;7~rhV!>SOfvxXei_`WO0_VMn9X54Xya;Ld?+~rhf8z(MbWX#HI0qO_mZKa;2jp5 z6%-a1D}Y&%Qpc-CHh9OAC&#)2Of&MWFhZaP9xrAEoQDowp{9y8GV+fhQq#Zejf##P zVeZD0zXCsbv66^A`+Wz3S|Ve=^;|wQ8>@s)d&5B({wN z*p3vQ(hx0Y>{Ws1zXRFyCh#Dl}b(R!&6!LIuuzEHRu@>7 z=c3PW8Em9B-j_Z;(NIy{Ygc-I6_ty&@I{QbNyBXmXs2}t+0gAx$k)K%t9$z0|A66` zr&rew*P(6lP(J8{HD(vm@1@Jvz~SuYA*_^oH?&O{_8)y%3G#BEz<++5CwIzb7Grtd z?7a75$%PoK<||07w!0KjFif ze>~o3bxlTp_#j6_xeWS1xt%(2+ZqMux2C2Tt04r4o5_vz1=zv0vv=Z^-o?jP8VAJ2a`*oFh0E?liSVtZ z#qeGx<`Oe)+eUlJS)!q#!Az7_;QJfu^yAMuiB|R&sj3*|ES6`2pYjerY&0m_z#i1R z_PeQc$TZZ{0tXA$?pxE9#VH55`JCP;y5={NR0M`D%Cwgoa8^ z&&&u6O;vRTUhMYL)UmNi|MTb1Hn*QctwgOrGR^(@C8svOwbM-a&i?zh>>J6sRf@v3 zMyy6Dj*a7v*Q}NvYDivxLTiuj3L0AdKO&#CRIe|2vBxhv?uD`K)2``N#h-zkqVZW5!}hUAxK zLv0G*v%-{|r5T#k(d69ttYzPUmF6;0gJRLh$bl#C@9A`DlHKCYF2aa#* zJMP!ND`(i1{D`IIAs(=-!O~McansQsNy!NyzNgjbChs$t$ZR#x%z4zY;s~3Ely(l7>!a9JWQV@S1fcydO`epr@j zaR2ER!?7K)xY9Md$BnHQ*hLVw+I~0F7mV-tYVDh1*H+=3QDd26N9CCV0!<8~dGzeb z^3EirD-JIU^^&5<>9OSCHQ|iG>;?Opx;n-W$NuJ$I>;rj#yS%T?8}!gwNpeuI9xAf zg%qol5^CjF4MG$hgYr$Uou8jS;$Un_u5rweZTImguxv;?0cIhJ{Ld-b0G*YnPoJ(b z^aJL;x|neHQ}f*v|6&!JoIPidXA+y%cc7-adb5|c9zZKZ_slllW~=VNsKi9R{n}7( zK2Q7cU@e_b?~AiLFKwkq0UlK(=yaRsI5XbGPlXi)pdJ_0{_ z>F=+fB7>t_O0J0E99DQAT3vz}28)R&3Ve>**7fm2#FzBOsrw63SNpp@GcBd{ICO0yKXJ>2JO{BL9uyjLBrbtgu@5f8{>IVb_ zYK&tr)A%$)`1|)4n$4;%ICV7LEj|>5O~jxNe9oyu3t<+`0KIl-L?zU0hri&i|5TjT zyMCQ@L+L7-2PkPaoH`@RWt+Lww?w{A=~O>X(<@{x%gwNwfmd|Q&ZSqi- z;!J{z-4+Ihx6v#}8Z*agubn(toPv%or)S~2bQ%22QD$;8Y6K4HpF(Ej`pRT>$%9Sgtky5@*m+?`zj?^z zfnwL&^68pksVqehK_@jECnn zmZ(^>2`PzcYHPcaWRPMAnb}(7sLssq9UQ!>F`3I~Rxtg|C+l5WdU|HQ^~^vr{ma{T z+l^HLP#wd(db0wnANN=BpSPf)J^9fSri&)tQ8l4%SkNZ1BO*chS+(6C*g? ze@}gfC#5jUIyDnW-PlnyP7Ki}y?C(^CqPBg9i$zgiA~K?8R_ZQ0A$1}0cpG`=$~ni z&H7m{sqd*L!VLz_rFjR^N7o!6uwer54LAyI>fGc^M@I)i^pjX^xNx&$Qd+;Rrl#h% zy1MX#Zr|^C7!N|NHlzzP@jBv{kCV{FfHqjPDV%xR{LD zWvQxg&v|?BqKwS55^(7)ndSh|9GIJ%OY?Shaq&cWnVN-%=UIX+Dpd{m|FTF&<)>x> z3nGZJ@{oGE`9iBMR^EZLi#$8U0K^kKwy|mD2sdARhLfT~OB_y+{6_AzhcbGj-p7YB z_*mn463;b6agpOX6#mdPkNC_fv7 zkIXM9*cUZKdXNVXjw+M2e{1JqzgW?iUS5;7zYL6xoW~8~xLjN{4HHp!P;rKkgdSnJ-u;*EcYbq+A@eUiZFqKsb6t_Czb&pSj%*s13 z-6388AuAg8uZ6VNxO`!Q3v627aRCAI20I1^+D$pb$u!&8kF9L49&JdVw5212ex_{V zT$c0r?ZQm<0w4dY0aZGwU4|=Ht*QYTAtpTtij7EAc^UcOJ@^bNA1YY2FfRD@F77C8 z;8UYnazC2%t=$ft_rKNGM~IF3mIJi~F+x4*(lhG_g3s_g^! zTD|xWuE)eoGcQX`D^#p<0C~2Ke!w5T(`^Do4nkk5XVw2c`J(Tab<^hN=8&_lPHA0!`ep^G;|=;(OifLBqR-dV)I{=?37@tUZN}u9 zsPJ%IDVE21Z|vv#HTq-2XJ=<$mwq@Ji&H}srCM6D0q_~IRYj{dD5J{V{ca-Xc-6s* zM5de8?1{6ZsB6C%&jIz0MuzH`)?=!3>()8Dg_%|%N>vujLI1n|{Kl9aB<*Z@@%A%~ zt$OX?fJ6ZD$TJR*=1!GByjvL>L~^;D0s+=nqc|W;IPvd zE>qSG428%mnzJf8E-MO<5?#SGd>?j2bn$&zJXwD_ySOMUOcsdqoAmYdRqypty(wIa z5vvzZ6|YC4+ZUTY$gA4Q2r$4)D<8$V>`RO`6t0Cmy6%Ml;PvJMNp;L_qmGAd|LKnT zm`DNhdTYax=NxA%HEXhRFb{#tZwyl zOLpUgyZxWHudA!#s5$-Y;y>Zy)ZBT7;?`;SYr1lj31tn_XZPFOWV;L0f5>3wm##P- z1m>pqTE;l9{sbI-MokuojC+_2W`5p1(V)!=d0crM?-pQ)?X!3&A z0t&x>!yx0SoI4eZU-jXZZ2qNI2g`qd0Xxu}8?MLmUOiij=WMsEMKX8TdFZgiEy=~K zT6klD)(hh$wEJmQ-_XO*4VAvwT*C52<^6AL++_?wB4Sm~bz0Tcg zwWXFw9Q05=v6*&)TD$wY;H}nI&yHO0+B!ymk8$;Y|G%#OWGSW|qOdxx_N*fphaJM7 zW7@WYErWELyxbqUwu&C78De{5mzcf$?@#-Y-Z~d6f;9pw#o zk%=S2aK+l>wV#X+mfyM15@g&cW^F-3wOHx5R{oiDU0#`GM?;De31?7xicp41o;r9A zXP)ive^!kEigZfbzpQ+{%o!#9ig008&N?Qx>2o^)dLRSEOix+zPw_2{n-*iO&z~q*MaV1CmOM`+H6cn_< z?r~ZKhP}CzL#)WWbdOiBZVA_B&oyQKRI-Eoi+&p@6(#=mUsGUR~W#7(llp#4t?ivhLr$ z$rDgKqXl#^c0RtUQB!0Q0noM~Db~{ZioYe*E8v0vvv22t)nkA>j6PYtXmbMOX-=Jy zSV~QkhPHLP^FFUqowZ|Ol3RR=LxZgQIL(o4uADWp$e4RiDvHx%rg`*2kJ7%H#%5)y z2a+~k|1IFreK=lH>y*f*#0=Z&9sO0o^BzV>9C=3@Y7~@F44`L4)G*a?~F+ zakwOeFp6lSrfI5KeAh1OHM>|xvFHr=>Bm;8p$ke8??AxfaEbz$BTv`nx=8IKPdwpK)ZW+Jhw*9)Pve-j<@Q>3O3HXUanT~$(J z*i0cdu8QYxiw+Ar1Gu!cN99vY%$?{FU7u)^h zcqzi4evKvKuz_gCB~}>O_;Vw>{%d05ROgGc1pH8pL2NaZihF57k%m2K6{NLuVcY}3 zhl7K`d28ks6bw693+~PS#Uy;p2c7Zs!89cINP!ojZX`po!UT&7Ko2pEIvhsE%Xhj{ zvu|UyWsmi;rw*p|+uF~+jI))T7@EULy?^!U)#~Cly`er@DB;rD1AwJdBSQ=GLupi8 zHaYf-z+y8$WK$|o_Z1D(O#9lVCN6?}ntv%QEAz%{BH1~Pz@1j0BLnrmbZIwQm(tIr zmtW@w)Ntxa_pC|A-bK9q`X|8ae-rAB57AOv{Ke)pd>d{X8#PhKw_Bv^s+Lhmc|q%b zAjW+`m{8wT$r21#3g!|LvTQJUOP)t_$KE-dlw(IZ&a;x@7G;J|$6YK}@)S6^xyPrc zTn=YN*wBTqJ)9v&YBXpb5CYQr8gX$DQhQCkW-RZSexndm!E#aAxUQAZbchUvQy$&60!;>ZCT!I4pY+j9%6q0EUQ z+*V}&g#9Sx-|B_CukY7Ij(8U#rsn`53}j5UyQJCgquI7~_DI)~Nw$?brEOaRc}Mzn1=M+2Sw(k|ktcP$Xndctv~ziK;PBxI=5Em76@7hu)B0^t zznu)evC*_2w=ah(F|85S$YJHmB}K)>=E>&)2_S$!NHf4`8TIa+ge9?mR?_e}1dth@jk!T17VZZ6 z0c6b~MD)d?R#6>m(2u*|Tvb~e_SD72+-D#3Kz8Ga!SogaON1}J!20|8^*lpdkL`+s zzUFAzGap;BlfsKXy#MFwvxdpnV5GV|_AQ(m%)e|=ymhnLqS4?0h2FVw3|hK1O-(Fb z1L=6~^aM{vN7YZ*C}Y?OeP2Nq3=}#EMYAMB@hvv}*((Tui+R*l%XZ`TgD3z|Nl@*EC|9q2n+Qi=xhr%5c2HD#;!LR#jj4sW)i^6S>ke8h~y>onO2ZZl8nuwN;Q8;>%HEtqc{B;yf+1AzuTGPCf8N{Dc9we?T`w zh~*$A6d`C8GZYvm&)Ah@pp0IfkVjCEL(sp}IDh~M;PhnZ z#w*S0`FJi|eEaroyRk4`L44x?@*cf=qK|^(>m*Jk8R3jv(~xx=`iuHFJJ0Ny^`f()I_v1@mcQYjFR(~&=H=&qXJw?N^*L^? z<@fIo!hsh{ZSEh_S$cw%1e*PL2-T7&Xu;chlt%lzs?#Oty4v-&ZGGD-&>D42qb+!8 z9AgB4){T+SJkk&fPpK_`Q~}&;!1TKQqvS$H9#94tE>ICh%jNl#cWq)cJ;>7UKL_s2 z3ZTD_?6IzJ8QM3HEXbk1dsrt(a2jkpbs?U}zV`Y&%RlXM=C}az{J)D4`;uIMqH*ejz zPvVs*x_|`fgz8NNyYI9b%cS#>^GIl2PlU{|%HJn-NP_2aJ=c$;ijTJx#T7u1}UF- z^%bNEG;m}MDWHpTj4Mu(#*GzngY#OBOqA@1dosi9iRQrdT6R^MG?gW-4 zl(_tqfD=1)u(PvAbai#THfY5uOYcLdf>D`NrWykbf6o#q6>xB`EQMVWi4=j*d7O#? zImf>CI?EXfNR;pz+IR&71y{b8LHEU}*fTKjJwP9U#qt?`6m$jFcmOZ++J1iq+p!P% z=MffNMcS^BG9eb(IGv|=ytElpcH zjfJa8OVy^)9=Jw%VOAviahATIO)~w_jD$BD3L&}Tx7G3N*H{~VqMS=Zs5UR4tUwxj zS=Ae`sbIh&I=RJK+R~SZHGsc*>8|4xFA(3!}uC)-F-OMY!QLZMDw{O$kP75{q|g#RiH+yBcw z{`ao<-@737Kl2#>uWu2b|9}5t{;#k2-@737|6%R?PuwE)UsmJ)#1;R07bNQszy8lR z%>VE$Qj4sh#e4YwV+a1AR)u;!b9Dupc>TxZ zrX65iKXY?)O565I(;|H?Wj?)X@P3Z79hylyY0Y;#oV$^v5{kBcRCF{_I2p+6cJ;B9 zt_A{y*cn!TM-Zr~Cm7PVed3~{J>A8o&jE4;A|y$nivoahE-5MLJecl)Rh$d8{9M}i z$bhq4j3yY`0%{TxXd#_aWn=vSJc>}S6cjvJj+vbUkYqO0jlcWVkR*+Mg@gopK}WbI zUJ&^gs3}U|7Js=TSl&C4f)fqEC_Bu1NC)Y3zlFJZAwcpklwJ}J)Mwo(%NJ6c4YM11 z(6qvAjBCLA;nQ^nHOd3Iv>zXoxbDKR50&sn@Yc|VQuooneftG$zVKHspqc^%Wvcou z)dZs!!I$uupSMTB8f0Qc81c5h?^oB=xfE)7Ak7COGVq~?lRA5{4XdhTcM!GZckY0N z^y$+NN$&}bsJW*%k{f@5&A`TX7q$YJHhaE0=(S9BOA9y7MjVl!+l;@Wz!w=9!HD{H zvtyyqB|se6W|fVc0RJu_p#=`4$IF*ns_*W}s2{MnZD6ov#~n3?w;2rz~edc?p|s{dw0){bG34=n%9H-K|La zNqP^~xH4v4xAB4o`4u_&6P6i&@0TxE+{ylhrX@O1XnLX#c?RfTyQwlhCQ8}_f(OkU z!yoYKH*I_Y?quPE6>BrN>+0&dZyQNcl9C?JpI(e7^1RVYEA8tHE3uL<+oVoX44WYtd2mj~ox0;&0fEG{gH<^@^vf3oa zR^TC51$=0PB?fM-0+}gj+=<2?J9dmt?Fn9Ip8Ka~!CKX|wMAH}Z_Eq8JX$f3E4SXi zw5vDrh$2`SZ`quWpk5-#4jPq&gA2@5;tHE|XMGkZ?=7@azh-A`q2<0HcaxgYymjlV z>Bh}mSQBa1jFaF%!h&({2r?g+C+E;p1ku(wKWG)YYkmFwqT+`^akF&y8U93EZHn<0 zh|g>StZ|gpL}tEKR<=!1NY4R&E}ylO35P7ivjly1Au~EAjl&$UwlI2!d%`?xpM9J^ z&5k{!re_cdBssNHEWH6M15f>)Y!r-8Bc-q3|Ezm!7Jge)QSlt#)Z5?h2~trx&j#Q~ z@+vBQV>3d~Q|zK<Y|0Kt~> z4$re^&OES~tF#P|b^}ie6nmF-sx36QzDZ~QwtT!I@W|;(uf41+ts6pC0HGm zL1Y5jTSV{#z8e1Y!t>%W&e!goZWD$(yN4v}?*MR=LjRpSWK+sC5!B6|zAZ;YM1G}m zJ|}jF=Lbv>2+(<0#$0trP#Ry_4U8w;3IQ^>z$CeGz|^+V*Fn%#%qtEr z4>b(a_U)ek{BzPom<8Xijb1qBo{I>TXP6ud1Y0X}aOS0a&}c5+BB7Us?XzO1`g(dc zV0VTGTKLta(S;ImM{ZU`#5r&R2DiK71KyabPZDZt8oYn&H1FC1M-0Nj%5}{^^#+KN z10*zgH8s(nS{miA2N~8&K-h&JWNtnXG}ws=jW%T~z1&?~`1otn2Ze7i5>NFpIzPEgUthmcZMfcVjDv;JNC>~@Df0l&0DsdhfSaa$7V12H0yNq z#BFx2B{s*1^R2qyLbJ0eDUVxWRL6ascbnqyi6N#61$zS2I4uDCd>oZrTU|bW{8*AA z%Eku5N$qSCv@suo?!wi(vSufEk@ykh4ATa^8;{v6X=-rLuzVqJ?iJs5^#2neUq(dQ zh|(NyQ(XhNd(cAi$B*}R?l^SdfCm&D<-8rm)RaB~Cz0Z*6=$LOtKO=CITE30L0nYg^71mOmknUPrKL4?ysfRRL!62zS~=R<2P&7kd}&X0C@v}*nWinm3GfayE(y22)Uq+}y?|B~ zs1E0+CN10z`8M|h?zKDLMMtJ@T9n&ITHSt18$167Pu1#we*gX*bGRpL`&Z#Zhc2L{ zth2ubA_9K#?P?lELrib^o?cM3w+-03=a2g3qk$JM?0}(swVb65f)Y3g@`{S#=P#Fi z+9&m)-3_Aq7qD_Drw-(Q9?H0cS`$4pK7+gD7z6Wwi0dwj3`=AWv7kDa?Vaj9?D8eV z%GU7My2p9lUkc2g#k}o)<$-6T8)OyMMer9e+k&z+z`8&Vb)m8NTbr@WdHG&KCaSY; zLHkF~LmAIE#A_t&*XQvmgm;@A+w<>Gg8FU);L8RrtYR}^iEZ!bpr7DI6FNu>()FwL zVj*1+kw55G-2<0mWGD15vtPeHfrY6{7lVihqoK9w_fv8A;XAs)V$vT!0-f9I!WhUC z3Sszx6ic-mF)xf!SnexZieg^5QLRJ@Y zu_(g1jg{~|*cD*K!3xCal3lokAcuMk!mO3YYP=P&@OOYE@;o>9l?5VlddXol zfF^Et2nh&S<;%Xh1y$GG+|~PTwfNhHyzua7Kd*$btriVyVY{MqNy@oYhavI%FRC|d zp!D<3ji%;zdDP0Yhz5pn_G)yE>4>mG^JFwgW`us5T_i^INhmq z*l}3P`^(K?5koFECyyl9*wYo&YZkZw@GXzGwY|4;qwihYT|y?lXz?Mx8$EWT9F!g% z(TM5Kt(v%t0Wz!gF4`wH)=fyIDZO!h2G4AATv;jTw5OJW6&ub{z<5Yy^M`^dPY(}v zaq(8ub+uqqRuniK+Un9B{|V=mS~ok=q6CnIOOM2bgq$Lo_*@xCLHGJ9?g&0^*7U4$ zeq^y;Vy%a=@SvBE4~Zt-8($$ABK-}rEZ`4iw~kkgnkw^n!C9vZaw)&=vTxS?oA5dw zokPbIc}K-4>~#FYAAbQ{Lb_7{7sKmZo?iF8_u!rGhITw8_}HaCVM#wREU@QorB*HS zQQL9W-zY@>@sztDEls#vOe(vwuop|eIJKm|M+S`~Cj8H&|NHA-==Vu+ey5OabUh&9 z7u^H_&wb=j#QW%8S}|Zzj3c@eQpxp&T&nX=TwUpVcgzmkm72MCVk}yH6+j`>uDek&@z=e!Lw!3!%m^0xH!Z^<8hUbP8YhYp; z*>TVU1*3`WJN@s5kYU4ax@e~d6^sKy)rJ8g=y0JaDfRvIp@d6~vH}B&WE(qlK6MOT z*o?FUvR)5@HUcL^&H~M;H>5+Pnn7-EWo>B1H)SGVMCe|q5JuHco;-P?AS@t7mS_6W zKQhpW_1tFMv4h%C#gXVG|E;>39c$o~$MdUF=cS~Io^_XzBZqw%8hP)SYzmR&p5~jm zXv<%vm&DA>&Bo1b*f1h-3i}mkHH#$>IZQJ-g}?=UZ83a4L{{z9cxWfPQ<1$QQ|*-K z*AyWJhNU>ThFKicxP7`m8`_?*eGyrh6+i{B#H|`A+-KI3|J0Auxzp^JfIDB``HBl7 zss4kxR_%AKFET$X+uC7~E?|xkwjocLs;e0@Eidc~zTSWI#japB^|&rPBY!tAM3S6e3|9cmY3H^~OVD38;JvxXF!$YG8#7aTdd+c4 zFlU|Tshpg@g;SVsr!xQD^k%v_F*J z{q)bDKc{50Kqq=yqgNxa#YT8L3&rxM>AS47wEwq*`m=7eILh~esu=+G`*9x-vqr5G zPb8N0FgBb0?%ik8f0$Mq@b2FIPq#%csng3fzq4iaQ|H`! zSFQ?v#h;|arNG@s0^Ezt)V@fuOOSpb{$54^=NDZM2Vwq!crX-Av*1epbF?xZP;K$( z0)wo`B|IR0Oc)UAo7AH1$j{G5f!;IdEba-{KuqZJf#?Cn>$k?ne$#tyIA4^rY*T>; z5K;^hII55U4uCI+tRg=C626}#SbjslP%%_jSKr#XOwfaWjbdBN;QezT@)sP+?ASnv zh3jIG(21?8x++zOkd4;jSwuE%^yM}lWqm6z4Puha|+L{U+`36=Ilh+3F&RB%$tz zQWzh21Xi~16B6X>_B8)fQCYbg%?SBRm)@lQKz~l2Qc&1~a!z5TnB}oR1s&X#P2pA_OW7n3nPwdrUhd-j3ho4s2U<{fR>gPPR)MjC%SBep8P!N|0(XP zqq5w$Zmn&C0Tv+^At50WlCn*Z5F`W%5ox5Q;n9u7MnppDB~%2Yr5jXA8bL}4k#1>_ zy7SrR+&k_aXWVhm`M&erZx6@Ve~1t7`~22#t-0n}Yc76^E^_p~CZ`<%m@ODXI|I|2 zsW^DQ!Me)Ikd!UGW)74G29}n*H!HY&^uCF>`K3ih5_(^IK;^^&G&%ZGP;e%B5i&jf z8*xG=SxEEu2Uf8FB!!gIAR?0IXdRi9t8XR!{ zC}#HCCM0<9^N_=y5(^5lN<(r6zgdgz@X^yd{nWdO_1GSw{PSCny9`e~OFh_T$Rc@> zJ1|lzBxdi`S?^ECE}WU4R~Vu;_)?;Yp zGvyKS=#f~L7vx#xyj26M`r0PsTSoL#(dJth`RG_Vcga&`L;WQVE0p-TgYYYqwG3g3<9pkeb2$y>cSOxS%=JtgU z{2E9?%>Fvjvmh6TtawwO0deW{v|!3*tPVKq?1mDFGm`+6>Rv#&WhviE7en8p$$*{u zWJ`?AfIN_Y;2(c%0HgJSbSHEC@X+KkZ*`U*P`xl5MR&c3_|gQXc;=Kc6w%DvZ}kCYPpT5caG^I~ zAl4Jf=s9=uT2-7+OrQ!n!p8ZIE)NCo8@%0Ia{n$`&!5HL?|PvWLYs58laEP=FE}cz zT-(|@m23c8fA}!Vbt}VDRtb4IypH#D3fOvj`2T#Sb7U|qhdcB{)H1!1-OT1wx?U^4+@`AY4e$1glLJQa2O)=I>hf*EOK0_Fg;@)r*Ju;Wa zDClQH=R;@{BYZ~>bzQoVsOpIJVet|9c6)3kFs>xw1=+pDDi3Dz)hkA@O77Ox+qPg* z#Ks+4s-+NXQ4&Y{Kh@xZ8|`5MMkM|){$e2$Fp0fy{N!oScujJ@tZkt|K_@U=yJN6d z=GED}E&cl|xXueuYNF|O=%p?b%PtWIC@3d}(>_3Jo;v#aTHK!~$$)aIn+Ya;gM(uo{f)p>fxbIJGBV~fv^=uz#y_f+;qA01F8|4xw?laH zqlSeqgd7GnYEPiCj_$_WH*ao#r4@v!+k7)PWz|l+MS5RUZZVzZStL|kTpY54I9?z- z6VhsP)xx{rrq`VV6?*-2e2#0eLL^@$u;h6;IT=~GF%Lapm%&^@4)3IBeMd*4^VvbD zWN7iH*Zc;%&rt+ch^d}IlOZz*y1(*e^on7A?>t79+4g&uHtxqLrua94AFB}bm7aJM zY>4buyczW+foVx7b==l{29PV+yT~i8&Fq>}trsObJZsY;(&`q5JB7$&nx|K6^Ew)z zsRun?HwQF9TcAtX*ElfPAu=y5&8Cdv+1PO~;)Lcm@xNM2wb6`?StGB30T77} z+*Ih9F738_0(uI^sLqY_84v5v&%aOdq#%kxmT{1|5B>*M5e9LL>|}wx09sBxjpa)* zbZgs;_~0 z$FmmNYz#fytBznU#EI3YUe8(_R9K<+fqTPekH5Sq$LMW= zQU>C(0b?X&L-Z=U2zdT9H|DP<877-cn&UGwoB5kNkzM7xCGS8_E65v^$+6}A5%{RTN;njdi{=zM zAD9Bkqp{OrFnZ~*lXrIp8)iHLO}Nq-@h8-lIkKxogirrpO_~9gKms3 z@NYvw;pOjv$n}xo^Gj%-NqHRgWOdt_GQgtEqoSuBpZ_%CTNM3nDIC3`jTE;MWAb8s zD?vd)>W5pF)r%4nYitz5!@|VJ-uG4!0VnC>43uAMwg*W=@kf72nvB6ineDlg`#u4eQ@1{$+)o)KV)&#nD?nSwr3lttcAC(lY6)9H z_9EBfBwoM>>S!h8m==3f@O|0acGnWO{Eq2a!+U)m7?UXJv#vmMxS^{e?-_O{v7J8{ z<95XV_;E{N7tsW|_>9ZK&-AqM*||BBvbiby=wu_g-+!y@op@i;cAgFBjPJ{Fo>t5t z#mHf-y>c5{DDT*Bj<;^TCh<9jH8?>#LMNMt%vYGDK~zQoNP{Z?3UcON@K*{Cr|~mP z^xO=orLywPY~i~g3{;n$pCy$r@R2BCR{C4Bxv4}}o0&D%2ktqOsW8=4Ojf|iiHjO% zWMd)Pb3G6!^Y5l*?MxpT9-dmf{G-Gs3=TEf`zCs!t4)eJz_jIj5R%x5{$yXkN$E?M z9wQvX4{jS|CWAnTt7ld>3vP#aMFJ+&f+dN<+iL_d)?s{&K|N${@`~uKF21E_ZF~nc z^;>3SYi2w>h890{@ORLvm{07bXk}kFdw+ATbnAMJhy&;QBlm7#-hJiA8qdc)eWx}^ z-;S2o9%!Fu=e%ZRHR(0$v)n?gwAie{f(7zrT7O^&v8a-&5D#UM)hBPzBCNRQq1iV>NwF$rsl_ zf%-fujY^MHLBF$aIf;1FFV_gQzDDye1^M|d91pPW`JMk97Y|PiRochycW0hIExi@9 zp(>4;(88P0@V*joee+dVpeoU0?_F*ad$PgL;(Wfg#YL~Fw3BU1!&mS`}Hcru8dm47x&w_qT zGvrQCXsDR|>}R>agEDO8?usm7l(f)Dq&sLcEt;$$Hc|OpK3}vlN6Fcq zi$(X<`!;{Lc@-5pRI;xjMFT68k&_b(@+UDlnbI;^@?ID6zEo+LP%%gvuI{h3u&=;e zP*!@8ow@mVv)hCaB4g+;Dhbold3dteeFGj1Cly3ZTN;a&`*;GiXB{9KD*qFulq;VU z)zl0L^Y6Tpk`Yb3oP3|QKm47&xDKT`>>`lGnW%SNW;=u^c0isxFffp!t9s!==|deI zRaHAq-Tn8b_ZY0-CdTBAK#!U!nvf!~^8DIHFKjZBla~)6l9Q9Oss|qw>!L(4f`R9+ zt*!N_i5q>BV8m-PXsp7$^xsIlrksd>dj42HS44Tw8T*G)9KmiY>fC8}tWWGsrMky$ zBIBe|k=Fm}>L=@IJHH>3*{p1AHM>vo@F?JeO*x4DS>9vtc5$eYt@KvP4mF+70cYwo z8lCw&FNA?#sHvz1!C4vHJS8d`uMi=D^u0QCT`Ppf?)G3J+zec-a)|0)(e5daI=e_y6Tq0wh@ZEcxltcpe#r1ma`3xKF^(dC2BErJFh9ls- zSl)?>iG2ZArRp^EhFlP9a0Jy-E5f{IUfD&W>*1B!x>MrfcLH8%vO+pnwSDYe055^i~RdC<9g_!9)^uo8|>7D+G$dZQW-f3Op?%gJu+o&BF zLx{Xj=1+VV{-dtZrIA{v0(+h^F4O!QeeA{PKxE zpb~*=&ZqXFwz~Q)xN=k7PgtT-R$gA$USOxI+pUzB#X85;(cbvSS+d&Ix(#PXJW`H22)ILk@Zh`lv$Zuf!S3b}b%xt=N116n zj8G^XJvZ~S`&{WQKG$C}((&=}{YSdV=4r17>SfcPpn!t1VVN#wXmf2P(@MH@qIIRF zlBC<(okU(>J1~+vX^}Ilu;{t@`HbA$I7|pWrI_iciZQj!%%tlJTp})ADvy1c4aT~m z$JEM7aneB(h>>pBS6?U-TnT#Nz@L(u>Z`hK#F0j=_jX)%_I1gX89%(ejCb!WyZ`)z z$zA6(`sk#j@pcJaI6|1t2(?5ctw%wxZ{1_??Q#kVBZGsN@fC+BCM5aa6`8OPLOk#5 z{UG4t$V-Ix-t7;2XPuBu8n`7DI?v04vOtoEDWN$3~^abBzaT5^Q1ehnM-P?XK# zM2%@I(f^!Q78Zf;VAG|kDidR>d#E!pztp%hEtNe*b}l+;EcTgmOrvq;kp=Wb*5>zL zj8x>WKgh^f{k6mZ-yImc`gvqd&Y3E$k&3R%3zHvTgnaw< zE#Tc?s;_&vt-XqXs%f+-!V&g@d>YK@@c$+}zFzcb^#*QB2Zy&671I37&*f!GeXqPm z*@{cI<^B8jUYo?*GSku`u`JNY+4ZyeV;Xd<66@<_}B-!=t z?V)&++!-Rn95fL+(UQ*C2yMO5zG-ZX$G5e$5cw4RvD3=0j5mJa;*wZ2n6c~8>(6ha zqocW^UJP^$3JD5o*zuln3fHGMVp%iywdcx9_BAZKCcsxHEfJxhrKN>BcRel@wrcP5 zF7KOkNG~ij@7$CN{Viy>{#1=H71a7tQuyXgZ&`6duio)R7t~H+>x#@Ya9HKRv^+o| zYK7Z6ySiE=F$a=f)Y)wr)UAx10P%Hk5otgiR)=0U2gc_K3Ji=vbuJO=>j+k<*2i;V zr(E}VU94;0Tl&0}!TQ#%A0J1?)kQ^Oln6ydD}!q*u+^$FR?SJ~WsWZ##Jc)9zLeC+ zn(MW+V{t;&nUO~ zr9NW@s$~bE8h7VA3*>#5icW4EMza>Iu|vjPyC*}(!1-4Rp22H}}`an{hb2&qiO`#jTBvrD5E-u*PenyCG*|rLr;NkB_K?1Vx z_Z$K})n9}$k?4+ym{+g5mDw{t|6G+2k&vj(4!j}gT!haQv1oDs?ECy!3sX~!e1FU7 z8sVV5(nhoMam}Pp*F@2zX)Cr`=-(nz^75A2_dP@?)6mrXQd|2B;0tDT`qU|3tI~UX zWdcr2B4P4i8OdrBG&f@;9;0FAWJFgPN0lem&JIlwYk6<{cn9E>{QA1;7~J~?21X!R zDZeDJ0)2jit6#7yUXk;Wr=VS~MY*_Tpq1)6_Jb?0X$%!yBq9hjvCBVH##M7#noG>t z3s2Nt&9vR_Ib-&Miv+M*u&%6|+p1Tfya?mY+4=b@pqn=3X|BMbZ{O|mY8WJY9`QEP z+{a>=H#R6ZmwYV&{J%6C;Hcyit-%FLO=g zc~O2Doi%Hz^$*`Bv# z*}l>Kkna9M=Zi;h9B&bP!E99YOa_VDg2;xLt~JUJoZ3-bN(QaR!)BGrqc zOE}U>N(`zEHlhRA<|V4t>2t95cEcK1%4EE4>Md5QGQJxCWSxbsraLWdtgV&IMScR8 zkS@>A@DdIXt!*#BRh`mUv?Y5J)7HCMSWHBjKS4hZP#ig(fZZ^Y&Z-!}exzOzl{$~e z!l$M}2#3{*=Gd43?Qd!Y4cj#vFeb!qRa&FcIo;N#enltho4*Oe5;}obK#UuA3_@`| zB3E!3Sd^ZLDRY+jiSAG`&>+3e`kG##(djr34`SPu#Z|nDQyh^(5)oe15-5Q6PA@h%QI};43kj(q z!K8EL9_Z_fg-$k3`_Ix_a8zR%TQuM_;_w==sbJsMO*7bA3;G}qI7+OXcH`u<%!cIQ zq{H*?UC#zRocM~n43N0bvot*~FTSQm4nsC?!Pa5bGXYoK5=(a?GD~5Kn{!}n)7`@t z06>=J^We;@P*cW~@=D zq`Qa-6>J_Ff!ae1P74C^Ge1gqfNn5M?5!WP0;+O~IytVzt#+{>m4;=ty{3kcdsy6U zx^2(IpCwAMh-{)~>cfJA`Kp+K84=UX7wNPjrs=;DF!Ie26&15*tOVNc`tR>bcMuD= zc6UeIMtWyzWHi`r^WMSbR@*b%Q#*seZP^RSqxR$RMfZc1g zI`+c_tok703~2ii*hE$>;f*)jZ)o`AB1fCHtCNF+(a<@Zq)i;NnVw#;l3UnNTu&9= zd`M8xyJh*Mr6vBIF>r&VH`%&rFaCIz&Wv}6dOdxr>2SwVB#fSs5z>X)Wyx(PY>;vz zu$|tR2z?||NGOXjA#ohS=7*55FoCXO0-5#f%*+YKoj2_C@{{)2*{@`88TXtRsAahP z;rdbLFqhVYhlhb)**88}S#c%X?S}b|N?Ua1e}-7Qs#;zg z*4WX;i#VkjOQ`>Nx%&E1ga?h9J}La=x*PlIKxwk4rPrg`9oqT>#7@f7fceF9gJy|4zjk&CGHhXBkkV8|p7O#yoJ3sIz4}CT zZEXlA2h?$!P(;ASXSvTt%1ASQVf{sp)WYa!LXOhMk=ZyX?zzYV--D+qYF42a>Rwn_ zsO(D$mMBDQZRf2z$az`DRE-o}w+T(j%8Eg9bcGi@Vtr4KUV-aUvyv|Z)n)++i6oe3 zRnO$$&!6?DR1*>ti2@&*V%)^ZNvo>6g^7t{4f-zuSkbcNH2LEpM4XWx%nUbi;?T6R zm4-QbG$PeqPSZC{O{KEbSH2l?ik3{nO~Pe_aa=vIw*jwFVeAAZ-YoAN{j4Q1et+o! z?~blRk&XthQNYJhhn^XMy`;Z;_i|{+_zJ<3n42qtAOOj41vo*c)#V&O5^l!bh=40~ zYs+;_+Mo6vEP#w<%qo=pW0phkX<-RtMKLtEEX~!NT32DP z=7TML$b4?tNzXVHz){JYMgxSLm0bSme#QZrzSG)jfuN93QzOynH!GnbRMTyVD6@d+TQg@%cP^w+}&X%s}B-Q5O$95Z=xmGqXXAh~pyqcTJ$h2ulT`!S)8) zCC9}}WTC0V=JBne+gzX9h&IQl2?_?a(M^T@(K4^DUPh=))+uU+Q$}Pbnqpj8V;W;Y zsXQ#!1qTM>X}`P{Lk2g9zL=O)TFaf*Y`Ms6zleLw@R7?!?PK@PSQQmi`3D5=2n7VP z3F^))x{xjbvKh!#;D~2uN7V2%*rh4~6o}MFn4i(JyeCHHkombO>RPOcNk^m|8@mJG zr}0PLZM^rmw{L}Qeir8CQKn{5mkC-cz$G*=Hjt9~@KESxw`J?8$Pi=4k_$B6FKpHP znKx5uCkt^F9KDqigoJRWyLii$AwAq-0JKxXFZ}U^0c;?vM^saFU^!blI6feJ0U@ET z*oOR)bpl~3oF2N(uyQ5|7o(-;+%HOQzJz|JX)@pBN}&v0O+$lkl#n9!UpH4-Yv()o zG)>qa2x}@D5W^vo{%$S2X7^+&_p}HVa@&BL7-nhFyr2cGpPRf!bNZB!5G6OaLYC*h zh^anLeW6V7Lj1ksBO}+2?5;;%fYcI^J{}3(8?Cb0$(~Eo1=?C#O~i6J(LOxO3R6Nx zNhcot_U+(T`u7$<$1V45?Y(A}R-Ar*zYpZnH@^hB!NmORRaMmh#kI<`gVar41O1Z@ zr=f%ET^34 zW5OM7Wq>A(6-dE81j}jB_PKr221E09KWcPjB0t4e-7w5xTuXW))1D~8kSxID&AEGP z=$Qc2mUU*of7obKK5jM}mLHEatJLBYJNI zdLK&MoSEh%*^tw4`(3ekQ9UR)^DvuEAD;LmEROD z=6nDNK(~DUFZw|da1sqn5HTC`n!V?FF_L{tJvh+R%9CMInYdBF(u>a5H8raj`p1*L zb#`WsP6e5;RO*8wuUc|#yE+a3nz}tYDymOL-+W#w`c=YFz66DNmt`C`9TU{7Iz1Zf zBYPEP2uWtIi;?j5?tRy>#aV1$W_o5Oz-17pDlMCU1hJX`Zew}^!8)MRplo0sD--X6 z@VLzU8r!u=&Mr|!)uZSzywIG)qk+IZ-sK3~$A@mus!shmTk`#wI(oC{)~!I&v45?u zOom=hO-JumO`kRXX)9Nc(-hI`k&{x{-Tey345O(z%gMO2YG4|u^%DY^qeO&C;ePLO z1%<|bZE(Q82UVZ#X{B9Fi4C4CfhQd6j_KoxVfS)-5n+?u= zN;AmK%*qmwlqBLZJ16Gj^YTRXa~^&+5`&m6m3yJvN}RjMdj$+{nf%SB$Dt10D&$Lt!XW=D4 z{^N&1W1Ng&i~T)zanaFHjA?qax`^u`nSljK35iUx{@l)`G=zC}8jqml;y!D3dM^!C zn&@ef4fpSX$v;^{P!Qw&3-ZV`!P%KKEx19z;}c*xP%YrK;E3|G!LY*G7enmsE<~|m z7EBMEH&8MeFM!Vvbe@-R`1$_S=jYkhbF8ilKR;NH8yuSy4-X2;p4kMlqXnU)x(M|M z3Pl-?kaNE$`$4M8ky!`;X%I4lzLI)R8%j|g2$p*XyGTR_SZsqs2{4t07)iK< zD|T$KPv?uHS0YWedCZQ|f{8BOMOv`nlE6zomrH0ycT!Uauc#x&A!{{R>;xx;0teJ1 z1%Ys9uxt+NZ)(LJWo2fPo012+R>!)M?yx*3io37E=QcM7oSE~LPoj9x)&?)T^n~PP zWJG~@@?TG`Pe2SHvpJB!P%aFneO)Ge{2-MED>F6fu>fR3OMlQugq1~eEa!WjcSS-z!b`A)k1&WrjsSopb zZ-ghH9~m!1g@lS~{#tigClewjEQK3PF1mc4z?>%5jj$(L*V!7DqUkTK+5VQke%}{4 zJ9L&{zoM&n=`lAfXghWuEJ}}1vmL`zfXKZ97Z|0f* zP+VeS6Otd*!WI}f7j<|t2a`9s=a_x1rf2dE)iy86yXiM+$Gb`I`W9Upn2YE2G7J?5 z*gJsz&kmnC3P8R5@JRYLs|qcorvvSTkmtGg&Ofjx-b}*i<|Q*{E3Z z^76?=v)+ME%BV}NPD`#eE-z_`13CYhwy`S=@EFuV zXs)!qFe4PUGask}TEXjxXMHeE1>VPFcH@ zlv!PLgUI?Xr=06ySTf!=NDVymZ9%~b@zq=w)hXmUDq32JVB~G=W}-9WBrSe>4wnc1 zuk6Mv3=NS)6gWb(wYehfu1*4 zZaV(ut}r8-PWg)yaP9u$&=@{~;9UWK{1`mdy zwKjyw8x2(A=xO!A0!{PYTpzS7C$~_nHvlWYMIw^O3zru~l?vrI5@q2N??{$L&jQ|8 zKhawQ`=rxG21G(74VG~9?T#v6c01th{Jqe+q>i)mw?h81r%s^xLPA24PV;uaY|z>V zuBVW4@~)wQL3Q|+*vd2GA>d%qWfZ(oDRS@oK|Mvhpm?DfFve3(1wx$`7;HWZ`g0)Z zGH>OkH+~GQwyIvCY)phVrD&tx=5`blu1xbB-@TPA*up;`(ltV$>h_X#8=hN0tZ8>! zURMYg%dt( zk_JrNzeG~?FhzsogGfnHPDKEj_O>Dp$rwhNflI{ZE5rsiq%Tmo)H1GG#A!`u(AD@f z>Y7#hEQma9LFWrte&6(G$(U^>t8s`uf!y|CK|#Fkp0Y_tFGjo}nZ3?pyz&)jb_N!A z*Mo=|5Y4z>rj?eN84b8F9rSZ)1(kov>D+u2iYqDzX>qsZg^yNWe*MZ-=T<}Y_A_&@ zmbL1x3d(Npa{j_qO#EsiBf@dv${I^@;|+l#K+u7I!e9U}1%iNa(JCIk0x7d(L~vLx z%4yLsw1mf%lMd~g;^v!z@%oTk;b#wI`hU;flC%rON9O&Wg#Miiio%QrH+QtZ#)fOG z+EjuUCQDM7H4CFQZ9ugMlLB;1kW9->Pfy1sC)a6IaIRZ-`LOQaWz@`ie$8wC zeOqxV()+*U3)W>yk&7qE|GpjhyQ;>$JG=ibFY;dSuczWKZ!$diyAH`a>W^H1*I#Je z{J);<{-1i@U%!9dKmR!5|MB_$)#v@Ik0bv7e%*igc*OsC9RJl{_aDA3&i`MqmVf`p zBkk{>x0Zi+egEmV_m}kE@%{Dt*Zs!=$G_<` Date: Mon, 9 Nov 2026 09:46:17 +0800 Subject: [PATCH 24/42] Add path-based pull request labels (#557) ## Purpose Add path-based labels for pull requests, and make the CI-failure issue form use the existing label name. This change is GitHub configuration only. It does not change Buildkite pipelines, CI trigger semantics, or runtime code. Pull requests currently get no automatic type or platform labels. Issue Forms already set `bug`, `feature request`, `RFC`, `documentation`, and (once this form matches the repo) `CI-failure`. `ready`, `merge-test`, and `weekly-test` stay manual because Buildkite reads them. ## Issue - Related issue(s): none - Closing keyword, only if fully resolved: ## Scope - In scope: - `.github/labeler.yml` maps changed paths to the existing `Ascend`, `NVIDIA`, and `documentation` labels. - `.github/workflows/labeler.yml` runs `actions/labeler@v5` with `contents: read` and `pull-requests: write`. - `.github/ISSUE_TEMPLATE/500-ci-failure.yml` now requests `CI-failure`, which is the label that already exists. The form previously requested `ci-failure`, so form-created issues did not receive the repo label. - Out of scope: - Creating labels, or auto-applying `ready`, `merge-test`, `weekly-test`, `high priority`, `bug`, `feature request`, `RFC`, or `CI-failure`. - Buildkite pipelines and any other CI trigger behavior. ## Implementation Notes `pull_request_target` is used so pull requests from forks can be labeled. A `pull_request` workflow from a fork only gets a read-only token. `pull_request_target` runs the workflow file from the base branch and can write labels. Untrusted pull request content is not checked out and is not executed. The workflow has no checkout step and no `run` step. `actions/labeler` reads the changed-file list and `.github/labeler.yml` from the base repository through the GitHub API. `sync-labels` is `false`, so the workflow only adds configured labels and does not strip labels a person added. Labels that are not in the config, including the Buildkite triggers, are never added or removed. The job does not request `issues: write`, so it cannot create new labels. `dot: true` is set explicitly so globs match `.buildkite` and `.agents`. Because `pull_request_target` uses the workflow on the default branch, this workflow does not label this pull request. It starts labeling after merge. ### Path to label rules Checked against the current tree with minimatch 9, the matcher `actions/labeler` v5 uses. No tracked file receives both `Ascend` and `NVIDIA`. **Ascend** - Any `npu` directory: `afd_plugin/**/npu/**`, `csrc/npu/**`, `docs/npu/**`, `recipe/npu/**`, `tests/npu/**`, and the unit-test mirrors. There is no `.buildkite/npu` directory. - Path segments or filenames containing `npu`, `ascend`, `cam`, or `dspark` (CAM/CAMP2P connectors and NPU DSpark). Uppercase `CAM` recipe and doc names sit under `npu/` and match the directory rule. - NPU-only skill trees: `.agents/skills/upgrade-npu/**` (matched by the `npu` segment), `.agents/skills/upgrade-layered-gmm-ops/**`, `.agents/skills/deploy-dsv4-flash-async/**`. - NPU tests whose filenames do not contain those tokens: async FFN worker tests, the mega-activation package test, the DSV4 Flash async deploy skill test, and the NPU-only DSV4 Flash E2E files `config.py`, `ffn_graph.py`, `ffn_graph_evidence.py`, and `test_deepseek_v4_flash_multi_pod.py`. **NVIDIA** - Any `gpu` directory: `afd_plugin/connectors/gpu/**`, `csrc/gpu/**`, `docs/gpu/**`, `recipe/gpu/**`, `tests/gpu/**`, plus `.agents/skills/upgrade-gpu-version/**`. - Filenames containing `gpu`, `cuda`, or `deepep`, including `cuda_graph.py`, `gpu_*.py`, and DeepEP unit tests. - `.buildkite/cuda/**` and `docker/Dockerfile.ci` (the CUDA `vllm-openai` CI image). `docker/` as a whole is not labeled. - NCCL P2P unit tests `tests/unit/connectors/test_p2p_*.py`. A broad `*p2p*` glob is not used, because it would also match Ascend CAMP2P and `docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md`. - GPU-only E2E suites `tests/e2e/models/qwen3_moe/**` and `tests/e2e/models/qwen3_6/**`. - GPU-only skill trees `.agents/skills/deploy-afd-k8s/**` and `.agents/skills/run-vllm-bench-k8s/**`. Shared trees stay unlabeled, including `tests/e2e/models/deepseek_v2_lite/**`, `tests/e2e/runner.py`, and connector factory/base modules. DSV4 Flash GPU files (`test_gpu.py`, `gpu_config.py`) get `NVIDIA` only. **documentation** - `docs/**` (including `docs/npu/**` and `docs/gpu/**`, which also get the platform label) - root `README.md` - `recipe/**/*.md` Module READMEs under code, tests, `csrc`, and `.agents` are not labeled, so a code change does not pick up `documentation`. Recipe shell scripts are not labeled `documentation`. ## Test Plan Local check: every tracked path was matched with minimatch 9 and `dot: true`. Assertions covered the NPU, CUDA, DeepEP, CAM, and docs examples above, and confirmed the forbidden label names are not keys. After merge, maintainers can verify on draft pull requests: 1. Open a draft PR that only touches `docs/npu/TESTING.md`. Expect `Ascend` and `documentation`. Expect `ready`, `merge-test`, `weekly-test`, `high priority`, `bug`, `feature request`, `RFC`, and `CI-failure` to stay absent. 2. Open a draft PR that only touches `tests/gpu/` or `.buildkite/cuda/`. Expect `NVIDIA` only. 3. On that draft, add `ready` by hand. This workflow should not add or remove it. Buildkite remains the reader of that label. 4. Open a CI failure issue from the form and confirm GitHub applies the existing `CI-failure` label. ## Test Result Config-only. No runtime, GPU, or NPU tests apply. `actionlint` 1.7.7 and the repo pre-commit checks for these files passed (YAML, trailing whitespace, typos, actionlint). ## Docs Impact - Files updated: `.github/labeler.yml`, `.github/workflows/labeler.yml`, `.github/ISSUE_TEMPLATE/500-ci-failure.yml` - If none, reason: user-facing docs under `docs/` were not changed. The new behavior is described in this pull request.
Open in Web Open in Cursor 
Signed-off-by: Cursor Agent Co-authored-by: Cursor Agent --- .github/ISSUE_TEMPLATE/500-ci-failure.yml | 3 +- .github/labeler.yml | 74 +++++++++++++++++++++++ .github/workflows/labeler.yml | 34 +++++++++++ 3 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 .github/labeler.yml create mode 100644 .github/workflows/labeler.yml diff --git a/.github/ISSUE_TEMPLATE/500-ci-failure.yml b/.github/ISSUE_TEMPLATE/500-ci-failure.yml index 5f6737c46..2364621d5 100644 --- a/.github/ISSUE_TEMPLATE/500-ci-failure.yml +++ b/.github/ISSUE_TEMPLATE/500-ci-failure.yml @@ -1,7 +1,7 @@ name: CI failure report description: Report a failing CI job, flaky test, or validation regression. title: "[CI Failure]: " -labels: ["ci-failure"] +labels: ["CI-failure"] body: - type: input @@ -51,4 +51,3 @@ body: attributes: label: CC list description: People who should be aware of this failure. - diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 000000000..b545cb044 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,74 @@ +# Path labels applied to pull requests by actions/labeler (v5). +# +# The workflow sets sync-labels: false, so this file only adds labels. +# A label that a person applied stays even when paths no longer match. +# Labels that are not keys in this file are never added or removed. +# +# Intentionally not auto-applied: +# ready, merge-test, weekly-test — Buildkite reads these CI triggers +# high priority — maintainer signal +# bug, feature request, RFC, CI-failure — GitHub Issue Forms +# +# There is no .buildkite/npu tree. Globs are case-sensitive, matching +# minimatch in actions/labeler@v5. dot: true in the workflow lets these +# patterns match .agents and .buildkite. + +Ascend: + - changed-files: + - any-glob-to-any-file: + # npu directories, plus names such as upgrade-npu and npu-compat.md. + - "**/npu/**" + - "**/*npu*/**" + - "**/*npu*" + - "**/*ascend*" + - "**/*ascend*/**" + # CAM and CAMP2P are Ascend connectors. Uppercase recipe and doc + # names live under npu/ and match the directory globs above. + - "**/*cam*" + - "**/*cam*/**" + # DSpark draft execution is implemented only on NPU. + - "**/*dspark*" + - "**/*dspark*/**" + - ".agents/skills/upgrade-layered-gmm-ops/**" + - ".agents/skills/deploy-dsv4-flash-async/**" + - "tests/e2e/test_deploy_dsv4_flash_async_skill.py" + - "tests/unit/v1/worker/test_async_ffn_*.py" + - "tests/unit/package/test_mega_activation_patch.py" + # DSV4 Flash E2E is mixed. These files are the NPU-only ones whose + # names do not contain npu, cam, ascend, or dspark. + - "tests/e2e/models/deepseek_v4_flash/config.py" + - "tests/e2e/models/deepseek_v4_flash/ffn_graph.py" + - "tests/e2e/models/deepseek_v4_flash/ffn_graph_evidence.py" + - "tests/e2e/models/deepseek_v4_flash/test_deepseek_v4_flash_multi_pod.py" + +NVIDIA: + - changed-files: + - any-glob-to-any-file: + # gpu directories, plus names such as upgrade-gpu-version and gpu_*.py. + - "**/gpu/**" + - "**/*gpu*/**" + - "**/*gpu*" + - "**/*cuda*" + - "**/*cuda*/**" + - "**/*deepep*" + - "**/*deepep*/**" + - ".buildkite/cuda/**" + # CUDA CI image (vllm-openai base). docker/ is not labeled as a whole. + - "docker/Dockerfile.ci" + # NCCL P2P unit tests. A broad *p2p* glob would also match CAMP2P + # and docs/npu/CAM_P2P, which are Ascend. + - "tests/unit/connectors/test_p2p_*.py" + # These E2E suites reject a non-GPU backend. + - "tests/e2e/models/qwen3_moe/**" + - "tests/e2e/models/qwen3_6/**" + - ".agents/skills/deploy-afd-k8s/**" + - ".agents/skills/run-vllm-bench-k8s/**" + +documentation: + - changed-files: + - any-glob-to-any-file: + # User-facing docs only. Module READMEs under code, tests, csrc, + # and .agents are left unlabeled so code PRs do not pick this up. + - "docs/**" + - "README.md" + - "recipe/**/*.md" diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 000000000..daaed09e6 --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 +name: labeler + +# Fork pull_request events only receive a read-only token, so they cannot +# apply labels. pull_request_target runs this file from the base branch +# with a token that can label. Do not check out the pull request head and +# do not run code from it: untrusted PR content must not execute with this +# token. actions/labeler reads the changed-file list and .github/labeler.yml +# from the base repository through the GitHub API. +on: + pull_request_target: + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + +concurrency: + group: labeler-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + labeler: + name: label pull request + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/labeler@v5 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + configuration-path: .github/labeler.yml + sync-labels: false + dot: true From 31652190515342b07941606470a19fbcddec830b Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Mon, 9 Nov 2026 11:22:42 +0800 Subject: [PATCH 25/42] docs: align module ownership, config fields, and guide links (#558) ## Purpose The module index omitted two owned runtime files, left `routing_ids.py` without a primary document, and missed five operational guides. Align the index with module front matter, complete the README configuration reference, and replace references to nonexistent multi-pod drivers with the actual deployment instructions. ## Issue Reported during a documentation audit; no linked issue. ## Scope - In scope: documentation ownership, guide links, configuration reference, multi-pod provisioning instructions, and review metadata. - Out of scope: runtime behavior, test implementation, and hardware qualification. ## Implementation Notes - Add `dspark_speculator.py` and `ffn_routed_experts_model_runner.py` to their existing Attention/FFN index rows. - Assign `compat/npu/routing_ids.py` to `compatibility_and_patches`, with its adapter behavior and existing validation references. - Link the DeepEP M2N, Async FFN runtime, layered W13/W2 validation, NPU custom-op build, and E2E guides. - Document `async_dp` and `afd_process_group_timeout_s`, including defaults and constraints. - Replace nonexistent Kubernetes/Docker driver references with the existing multi-pod deployment guide. - Refresh six `last_reviewed` dates to 2026-11-09 and state the current vLLM 0.32.0 (`5df63a72`) / vLLM-Ascend `bd42eea0` targets. The dates describe documentation consistency checks; historical hardware evidence retains its original scope. ## Test Plan - Run all applicable pre-commit hooks on the eight changed Markdown files and `git diff --check`. - Compare the production routing table with module front matter and verify unique ownership of tracked production files. - Resolve local Markdown links and heading targets in changed documents. - Parse the README JSON through `parse_afd_config` under Python 3.11 and compare its keys with `AFDConfig` fields. ## Test Result - All applicable pre-commit hooks passed, including Markdown lint, typos, whitespace, and merge-conflict checks; file-type-specific hooks had no applicable files. - `git diff --check` passed. - All seven production routing rows match front matter; 297 tracked production files each have exactly one primary owner. - All 156 checked local Markdown links and heading targets resolve. - README JSON includes all 10 configuration fields, passes parsing and validation, and matches the documented defaults. - GPU/NPU tests and native builds were not run for this documentation-only change. ## Docs Impact Eight files updated: root `README.md`, `tests/e2e/multi_pod/README.md`, and six documents under `docs/design/module/` (`index`, `attention_runtime`, `compatibility_and_patches`, `e2e_testing`, `execution_platforms`, and `plugin_boundary`). ---
Essential PR Checklist - [x] Purpose is clear and linked to public context when possible. - [x] Scope is bounded. - [x] Compatibility with vLLM v0.32.0 and, for NPU changes, the pinned vLLM-Ascend source is considered, as applicable. - [x] No changes are made to the vLLM source checkout. - [x] Plugin-owned classes or explicit dotted class paths are preferred over monkey patches. (No code changes.) - [x] Any compat shim or monkey patch is isolated, idempotent, version-guarded, documented, and tested. (No shim or patch changes.) - [x] Imports remain CPU-safe; CUDA-heavy work is delayed or GPU-gated. (No import changes.) - [x] Validation evidence is included, including skipped GPU tests when applicable. - [x] Documentation impact is stated.
Signed-off-by: jiangkuaixue123 --- README.md | 13 ++++++- docs/design/module/attention_runtime.md | 2 +- .../module/compatibility_and_patches.md | 5 ++- docs/design/module/e2e_testing.md | 2 +- docs/design/module/execution_platforms.md | 2 +- docs/design/module/index.md | 34 +++++++++++-------- docs/design/module/plugin_boundary.md | 2 +- tests/e2e/multi_pod/README.md | 30 ++++++++-------- 8 files changed, 54 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 2cace4383..dc36866e3 100644 --- a/README.md +++ b/README.md @@ -307,8 +307,10 @@ The canonical config shape is: "afd": { "role": "attention", "connector": "P2pNcclAFDConnector", + "async_dp": false, "host": "127.0.0.1", "port": 1239, + "afd_process_group_timeout_s": 120, "num_attention_ranks": 2, "num_ffn_ranks": 1, "compute_gate_on_attention": false, @@ -324,7 +326,16 @@ omit `additional_config["afd"]` to disable AFD. Connector-owned `connector_extra_config` is strictly validated by the selected connector parser when the connector/runtime is constructed. The plugin also accepts selected compatibility aliases such as `afd_role`, `afd_connector`, `afd_host`, and -`afd_port`. +`afd_port`, plus `async` for `async_dp`. + +`async_dp` defaults to `false` and enables AFD's asynchronous data-parallel +compatibility behavior. It is valid only with `CAMAsyncAFDConnector`, which +requires it to be `true`. + +`afd_process_group_timeout_s` is a positive integer timeout in seconds for the +AFD world process-group rendezvous (default: `120`). Increase it when rank +placement or model loading makes the group join slower; it also bounds the NPU +native DBO startup barrier, not device-kernel execution. ## Development diff --git a/docs/design/module/attention_runtime.md b/docs/design/module/attention_runtime.md index 31700b82d..db6b4e4f5 100644 --- a/docs/design/module/attention_runtime.md +++ b/docs/design/module/attention_runtime.md @@ -53,7 +53,7 @@ related_issues: - "#88" - "#105" - "#129" -last_reviewed: 2026-10-13 +last_reviewed: 2026-11-09 --- # Attention runtime diff --git a/docs/design/module/compatibility_and_patches.md b/docs/design/module/compatibility_and_patches.md index 815f23948..98d96abfd 100644 --- a/docs/design/module/compatibility_and_patches.md +++ b/docs/design/module/compatibility_and_patches.md @@ -10,6 +10,7 @@ primary_code_paths: - "afd_plugin/compat/vllm.py" - "afd_plugin/compat/npu/__init__.py" - "afd_plugin/compat/npu/feature_validation.py" + - "afd_plugin/compat/npu/routing_ids.py" - "afd_plugin/compat/npu/runtime.py" - "afd_plugin/compat/npu/runtime_config.py" - "afd_plugin/compat/patches/**/*.py" @@ -40,7 +41,7 @@ verified_platform_refs: related_issues: - "#86" - "#129" -last_reviewed: 2026-10-13 +last_reviewed: 2026-11-09 --- # Compatibility and patches @@ -85,6 +86,7 @@ compatibility evidence rather than a released package or container tag. | Core vLLM patches | [`compat/patches/`](../../../afd_plugin/compat/patches) | [`tests/unit/compat/patches/`](../../../tests/unit/compat/patches) | | CUDA benchmark routing | [`model_executor/routing_simulator.py`](../../../afd_plugin/model_executor/routing_simulator.py) | [`test_routing_simulator.py`](../../../tests/unit/model_executor/test_routing_simulator.py) | | NPU adapters | [`compat/npu/`](../../../afd_plugin/compat/npu) | [`test_runtime.py`](../../../tests/unit/compat/test_runtime.py), [`test_ascend_ops.py`](../../../tests/unit/compat/test_ascend_ops.py) | +| NPU routing-ID balancing | [`compat/npu/routing_ids.py`](../../../afd_plugin/compat/npu/routing_ids.py) | [`test_routing_ids.py`](../../../tests/unit/compat/npu/test_routing_ids.py), using the shared [`routing_ids_op.py` fixture](../../../tests/unit/fakes/routing_ids_op.py) | | NPU patch paths | [`compat/patches/npu/`](../../../afd_plugin/compat/patches/npu) | [`test_npu_runtime.py`](../../../tests/unit/v1/worker/test_npu_runtime.py) | | CUDA native V2 DBO preparation | [`gpu_mrv2_ubatching.py`](../../../afd_plugin/compat/patches/gpu_mrv2_ubatching.py) | [`test_gpu_mrv2_ubatching.py`](../../../tests/unit/compat/patches/test_gpu_mrv2_ubatching.py) | @@ -156,6 +158,7 @@ These modules adapt upstream behavior without replacing a global symbol: | [`model_executor/routing_simulator.py`](../../../afd_plugin/model_executor/routing_simulator.py) | Registers the opt-in `afd_balanced` strategy through vLLM's public `RoutingSimulator` interface. Select it with `VLLM_MOE_ROUTING_SIMULATION_STRATEGY=afd_balanced`; `AFD_BENCHMARK_FORCE_LB_TOPN_PER_RANK` optionally limits each rank's expert pool. It provides deterministic source-rank phases and normalized weights for controlled CUDA performance tests without replacing router internals. EPLB remains outside the validated scope. | | [`compat/npu/runtime_config.py`](../../../afd_plugin/compat/npu/runtime_config.py) | Preserves model SP while applying the non-SP backend fallback; normalizes Async CAM native ubatching to Attention DBO, checks thresholds, and derives model/PP tensor layout and active ubatch count. | | [`compat/npu/feature_validation.py`](../../../afd_plugin/compat/npu/feature_validation.py) | Parses connector-owned typed extra information through the factory and fails before execution for unsupported NPU connector, quantization, graph, DBO, gate, or async MoE combinations. | +| [`compat/npu/routing_ids.py`](../../../afd_plugin/compat/npu/routing_ids.py) | Registers the opaque `afd_balance_routing_ids_` custom op. It reads Ascend's live profile flag inside the op and balances logical expert IDs in place during profiling or when forced. NPU worker construction registers it before the first Attention-side gate; standalone scripts must import the module after device initialization and before using the op. | | [`compat/npu/forward_context.py`](../../../afd_plugin/compat/npu/forward_context.py) | Enters the pinned Ascend forward context for connector-driven FFN compute and installs AFD metadata in `additional_kwargs`. V1 uses `set_ascend_forward_context`; a V2-paired FFN uses native `set_forward_context` plus the Ascend MRV2 profile override so platform state follows the V2 proxy layout. | | [`compat/npu/ops.py`](../../../afd_plugin/compat/npu/ops.py) | Discovers plugin-owned CAMP2P and routed-only async CAM operators with explicit missing-runtime errors; operator build/runtime ownership is documented in [execution platforms](execution_platforms.md). | diff --git a/docs/design/module/e2e_testing.md b/docs/design/module/e2e_testing.md index e6ff87788..946e13bbd 100644 --- a/docs/design/module/e2e_testing.md +++ b/docs/design/module/e2e_testing.md @@ -51,7 +51,7 @@ verified_platform_refs: - "CUDA Qwen3 MoE" - "CUDA Qwen3.6 MoE" related_issues: [] -last_reviewed: 2026-10-13 +last_reviewed: 2026-11-09 --- # E2E testing diff --git a/docs/design/module/execution_platforms.md b/docs/design/module/execution_platforms.md index c4644554a..2e68f16ec 100644 --- a/docs/design/module/execution_platforms.md +++ b/docs/design/module/execution_platforms.md @@ -52,7 +52,7 @@ verified_platform_refs: related_issues: - "#86" - "#129" -last_reviewed: 2026-10-13 +last_reviewed: 2026-11-09 --- # Execution platforms diff --git a/docs/design/module/index.md b/docs/design/module/index.md index 21d5a848e..c37d4c1d7 100644 --- a/docs/design/module/index.md +++ b/docs/design/module/index.md @@ -24,7 +24,7 @@ verified_platform_refs: - "Ascend: current target in root installation instructions; V2-Lite and DSV4 Flash W4A8 evidence in the execution-platform matrix; older environments remain historical" related_issues: - "#129" -last_reviewed: 2026-10-13 +last_reviewed: 2026-11-09 --- # AFD module design @@ -35,12 +35,16 @@ verify boundaries, invariants, and validation evidence. The documents record plugin, role runtime, connector, model, platform, compatibility, and E2E testing contracts; open interfaces remain explicitly draft. -The 2026-10-13 source review used AFD `36dadb4`, vLLM `0.30.0` -(`ced6857a`), and Ascend `8d4409d6`; it is historical, not qualification of -the current pair. For the 0.32 upgrade, see the -[NPU validation record](https://github.com/vllm-project/afd-plugin/pull/507). Existing document -statuses and historical measurements retain their scope. Each module records -its remaining limitations and validation gaps. +The 2026-11-09 documentation review checked path ownership, configuration, +guide links, and upstream references against the current checkout. GPU and +NPU target vLLM `0.32.0` (`5df63a72`); the Ascend integration targets +vLLM-Ascend `bd42eea0`, as recorded in the +[installation instructions](../../../README.md#ascend-npu-installation). +The refreshed `last_reviewed` dates record documentation consistency checks; +hardware validation remains scoped to the +[runtime matrix](execution_platforms.md#tested-runtime-matrix) and the +[0.32 NPU validation record](https://github.com/vllm-project/afd-plugin/pull/507). +Existing document statuses and historical measurements retain their scope. ## Reading order @@ -100,12 +104,12 @@ contract. File-level entries deliberately resolve mixed directories. | Primary document | Production paths | | --- | --- | | [Plugin boundary](plugin_boundary.md) | `afd_plugin/__init__.py`, `afd_plugin/config.py`, `afd_plugin/config_utils.py`, `afd_plugin/envs.py`, `afd_plugin/validation.py`, `afd_plugin/py.typed`, `afd_plugin/v1/__init__.py`, `afd_plugin/v1/worker/__init__.py`, `afd_plugin/v1/worker/npu/__init__.py`, `pyproject.toml` | -| [Attention runtime](attention_runtime.md) | `afd_plugin/v1/worker/attention_metadata.py`, `afd_plugin/v1/worker/attention_model_runner_v2.py`, `afd_plugin/v1/worker/attention_worker.py`, `afd_plugin/v1/worker/ubatch_wrapper.py`, `afd_plugin/v1/worker/npu/attention_model_runner_v2.py`, `afd_plugin/v1/worker/npu/attention_worker.py` | -| [FFN runtime](ffn_runtime.md) | `afd_plugin/v1/worker/ffn_metadata.py`, `afd_plugin/v1/worker/ffn_model_runner.py`, `afd_plugin/v1/worker/ffn_worker.py`, `afd_plugin/v1/worker/npu/ffn_model_runner.py`, `afd_plugin/v1/worker/npu/async_ffn_model_runner.py`, `afd_plugin/v1/worker/npu/ffn_worker.py` | +| [Attention runtime](attention_runtime.md) | `afd_plugin/v1/worker/attention_metadata.py`, `afd_plugin/v1/worker/attention_model_runner_v2.py`, `afd_plugin/v1/worker/attention_worker.py`, `afd_plugin/v1/worker/ubatch_wrapper.py`, `afd_plugin/v1/worker/npu/attention_model_runner_v2.py`, `afd_plugin/v1/worker/npu/attention_worker.py`, `afd_plugin/v1/worker/npu/dspark_speculator.py` | +| [FFN runtime](ffn_runtime.md) | `afd_plugin/v1/worker/ffn_metadata.py`, `afd_plugin/v1/worker/ffn_model_runner.py`, `afd_plugin/v1/worker/ffn_routed_experts_model_runner.py`, `afd_plugin/v1/worker/ffn_worker.py`, `afd_plugin/v1/worker/npu/ffn_model_runner.py`, `afd_plugin/v1/worker/npu/async_ffn_model_runner.py`, `afd_plugin/v1/worker/npu/ffn_worker.py` | | [Connector contracts](connector_contracts.md) | `afd_plugin/connectors/**/*.py`, `afd_plugin/connectors/npu/bin/**`, `afd_plugin/distributed/**/*.py` | | [Model integration](model_integration.md) | `afd_plugin/model_executor/**/*.py` | | [Execution platforms](execution_platforms.md) | `afd_plugin/compat/profiler.py`, `afd_plugin/compat/npu/forward_context.py`, `afd_plugin/compat/npu/ops.py`, `afd_plugin/compat/npu/profiler.py`, `afd_plugin/v1/worker/cuda_graph.py`, `afd_plugin/v1/worker/dbo.py`, `afd_plugin/v1/worker/npu/mla_graph.py`, `afd_plugin/v1/worker/npu/ubatch_runner.py`, `csrc/**`, `setup.py`, `MANIFEST.in` | -| [Compatibility and patches](compatibility_and_patches.md) | `afd_plugin/compat/__init__.py`, `afd_plugin/compat/vllm.py`, `afd_plugin/compat/npu/__init__.py`, `afd_plugin/compat/npu/feature_validation.py`, `afd_plugin/compat/npu/runtime.py`, `afd_plugin/compat/npu/runtime_config.py`, `afd_plugin/compat/patches/**/*.py` | +| [Compatibility and patches](compatibility_and_patches.md) | `afd_plugin/compat/__init__.py`, `afd_plugin/compat/vllm.py`, `afd_plugin/compat/npu/__init__.py`, `afd_plugin/compat/npu/feature_validation.py`, `afd_plugin/compat/npu/routing_ids.py`, `afd_plugin/compat/npu/runtime.py`, `afd_plugin/compat/npu/runtime_config.py`, `afd_plugin/compat/patches/**/*.py` | The routing inventory covers runtime and package code under `afd_plugin/**`, native sources under `csrc/**`, and packaging files that affect shipped @@ -129,15 +133,15 @@ Only identified invariant blocks in a `normative` document may use `MUST`, Operational guides remain separate from normative module design: - [NCCL P2P connector guide](../../gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md) +- [DeepEP M2N connector guide](../../gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md) - [CAM P2P connector guide](../../npu/CAM_P2P_CONNECTOR_USER_GUIDE.md) - [CAM async connector guide](../../npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md) - [CAM routed operators](../../npu/CAM_ASYNC_ROUTED_OPS.md) +- [Async FFN runtime](../../npu/ASYNC_FFN_RUNTIME.md) +- [Layered W13/W2 graph validation](../../npu/LAYERED_W2_MDL_VALIDATION.md) - [NPU validation](../../npu/TESTING.md) and [troubleshooting](../../npu/TROUBLESHOOTING.md) - [Ascend NPU installation](../../../README.md#ascend-npu-installation) +- [Ascend NPU custom ops build](../../../csrc/npu/README.md) - [Connector overview](../../../afd_plugin/connectors/README.md) - [Deployment recipes](../../../recipe/README.md) - -- `afd_plugin/compat/npu/routing_ids.py` — opaque routing-id balancing boundary. - Registered during worker initialization; standalone scripts import the module - after device init. Tests: `tests/unit/compat/npu/test_routing_ids.py` and the - shared fixture in `tests/unit/fakes/routing_ids_op.py`. +- [E2E test guide](../../../tests/e2e/README.md) diff --git a/docs/design/module/plugin_boundary.md b/docs/design/module/plugin_boundary.md index 4ccd31f10..45b6b88e9 100644 --- a/docs/design/module/plugin_boundary.md +++ b/docs/design/module/plugin_boundary.md @@ -37,7 +37,7 @@ verified_platform_refs: related_issues: - "#89" - "#129" -last_reviewed: 2026-10-13 +last_reviewed: 2026-11-09 --- # Plugin boundary diff --git a/tests/e2e/multi_pod/README.md b/tests/e2e/multi_pod/README.md index 07b3bcb2d..904ec1bdb 100644 --- a/tests/e2e/multi_pod/README.md +++ b/tests/e2e/multi_pod/README.md @@ -44,10 +44,11 @@ it just needs the same launch inputs again. SIGTERM/SIGINT into that group, and raises if it exits non-zero. Critically, the test file's own docstring spells out the deployment -assumption: **the pods must already exist**. Something else — a human, or -`tests.e2e.multi_pod.driver.k8s` / `driver.docker` — has already created N -pods/containers and is about to (or already did) invoke this same pytest -node once inside each of them. `pytest tests/e2e/models/deepseek_v2_lite/ +assumption: **the pods must already exist**. A human or external deployment +tool creates N pods/containers and invokes this same pytest node once inside +each of them; see the +[multi-pod deployment instructions](../../../.agents/skills/run-e2e/resources/k8-multi-pod.md). +`pytest tests/e2e/models/deepseek_v2_lite/ test_deepseek_v2_lite_multi_pod.py::test_multi_pod[...]` running inside pod 0 and the identical invocation running inside pod 1 together *are* one multi-pod E2E run. `test_multi_pod()` only supplies the inputs the in-pod @@ -58,7 +59,7 @@ runner itself. So conceptually: ```text -driver (k8s Job / docker driver) +external provisioning (k8s Job / container launcher) └─ pod 0: pytest ... test_multi_pod[...] → runner.py --pod-layout 2A0F,0A2F ... └─ pod 1: pytest ... test_multi_pod[...] → runner.py --pod-layout 2A0F,0A2F ... ``` @@ -126,7 +127,7 @@ line 442). Reading it top to bottom: `identity.py`) checks, in order: an explicit `AFD_E2E_POD_INDEX` override, the k8s Indexed Job's `JOB_COMPLETION_INDEX`, then falls back to the trailing integer of `HOSTNAME` (for hand-applied manifests or the - Docker driver, which names containers `..-0`, `..-1`, …). + container launchers that name containers `..-0`, `..-1`, …). 3. **Wait for the rendezvous store's DNS to resolve, then connect.** `wait_for_address(args.store_host, args.store_dns_timeout)` exists @@ -362,12 +363,11 @@ output whenever that slot doesn't actually span pods. ## Provisioning: who creates the pods in the first place `runner.py` assumes N pods/containers already exist and that it is being -invoked once inside each. That provisioning step is a separate concern, -handled by `tests/e2e/multi_pod/driver/k8s.py` and `driver/docker.py`. Both -drivers describe themselves the same way: their whole job is to render and -apply the deployment (a k8s Indexed Job or a set of sibling Docker -containers), block on the platform's own completion primitive (`kubectl -wait` / a container-exit wait), collect exit codes and logs, and clean up — -they hold no test state and make no test decision. Launch order, readiness, -evaluation, and teardown are entirely owned by the pods running this module, -exactly as walked through above. +invoked once inside each. Provision them manually or through external +deployment tooling, following the +[multi-pod deployment instructions](../../../.agents/skills/run-e2e/resources/k8-multi-pod.md). +This repository does not provide a `tests/e2e/multi_pod/driver/` package. +The provisioner creates the pods/containers, invokes the same pytest node in +each, collects exit codes and logs, and removes the deployment afterward. +Service launch order, readiness, evaluation, and child-process teardown are +owned by the in-pod runner, as walked through above. From 49aff3bb6ae2c2c892cc6dbba3ea5760e423ccaa Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Mon, 9 Nov 2026 11:39:24 +0800 Subject: [PATCH 26/42] docs: remove stale runtime guidance and correct connector contracts (#559) ## Purpose Current documentation still describes removed V1 runners, obsolete Async CAM settings, and a Qwen-only DeepEP M2N restriction. Remove stale guidance and reconcile the remaining descriptions with the current implementation. This content-only pass updates 13 Markdown files and removes 108 lines overall. ## Issue Documentation audit follow-up to #558; no issue to close. ## Scope - In scope: runtime descriptions, connector/model constraints, configuration reference, native-operator guidance, and recipe discoverability. - Out of scope: code, tests, directory restructuring, CI changes, and new hardware qualification. ## Implementation Notes - Delete V1 execution descriptions and matrix rows, the unsupported DSV2/V3 Async CAM SP description, legacy connector-flag advice, and the outdated recipe tree. Replace repeated limits with links; retain historical evidence links and known correctness/shutdown limitations. - Describe CUDA runner construction through `_make_model_runner()` and NPU's required ModelRunnerV2 path. - Correct DeepEP M2N admission to Qwen3 MoE or DeepSeek-V2 and Qwen's connector-specific gate placement; link the existing DeepSeek-V2-Lite recipe. - Correct `connector_extra_config` storage/hash semantics. Complete the existing Async CAM parameter table with `ffn_mega_moe`, and distinguish default layered execution from Mega/Pull. - Correct native CAM operator selection and eager-Attention versus eager/FULL-FFN descriptions. Checked against `afd_plugin/config.py`, `validation.py`, CUDA/NPU workers, `compat/npu/{feature_validation,forward_context,runtime_config}.py`, the Qwen/DeepSeek wrappers, and Async CAM connector/executor code at base `31652190`. Runtime pins remain vLLM 0.32.0 (`5df63a72`) and vLLM-Ascend `bd42eea0`. Review dates indicate documentation checks, not new device results. ## Test Plan Run applicable pre-commit hooks, check local links/anchors, parse documented common AFD configurations, and compare the Async CAM parameter table with its source schema. ## Test Result - All applicable pre-commit hooks passed on the 13 changed files; `git diff --check` passed. - 335 local Markdown/HTML links and anchors checked across 41 non-agent documents. - 10 configuration examples passed the common AFD parser; README covers all 10 fields. Confirmed extra config is retained and affects the hash. - Async CAM table matches all four accepted source-schema keys; README E2E scenario names match runner definitions. - Checks ran on CPU-only macOS / Python 3.12.13 without torch or vLLM. No GPU/NPU execution, E2E, or native build was run for this documentation-only change. ## Docs Impact Root README, recipe index, NPU native-build README, seven module-design pages, NCCL P2P guide, Async CAM guide, and Async FFN runtime. No new documents or file moves. ---
Essential PR Checklist - [x] Purpose is clear and linked to public context when possible. - [x] Scope is bounded. - [x] Pinned vLLM / vLLM-Ascend compatibility is considered. - [x] No changes to the vLLM source checkout. - [x] Plugin-owned extension and patch requirements remain unchanged. - [x] Imports remain CPU-safe; no code changes. - [x] Validation evidence and hardware-validation limits are stated. - [x] Documentation impact is stated.
Signed-off-by: jiangkuaixue123 --- README.md | 31 +++---- csrc/npu/README.md | 10 +-- docs/design/module/attention_runtime.md | 50 +++--------- .../module/compatibility_and_patches.md | 2 +- docs/design/module/connector_contracts.md | 6 +- docs/design/module/execution_platforms.md | 81 +++++++------------ docs/design/module/ffn_runtime.md | 12 +-- docs/design/module/model_integration.md | 31 +++---- docs/design/module/plugin_boundary.md | 13 ++- docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md | 2 +- docs/npu/ASYNC_FFN_RUNTIME.md | 9 +-- docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md | 31 +++---- recipe/README.md | 18 +---- 13 files changed, 94 insertions(+), 202 deletions(-) diff --git a/README.md b/README.md index dc36866e3..ab052042e 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Core runtime support: recorded in the existing [NPU runtime matrix](docs/design/module/execution_platforms.md#tested-runtime-matrix). - DeepSeek MoE handoff at the remote-experts boundary on CUDA, with the gate placed on either Attention or FFN. -- Expert-routed M:N exchange on CUDA (`DeepEPM2NAFDConnector`) for Qwen3 MoE: +- Expert-routed M:N exchange on CUDA (`DeepEPM2NAFDConnector`) for Qwen3 MoE and DeepSeek-V2: Attention routes each token and dispatches it only to the FFN ranks that own its experts. @@ -52,7 +52,7 @@ Model support: | DeepSeekV2 / DeepSeekV3 / DeepSeekV3.2 | `DeepseekForCausalLM`, `DeepseekV2ForCausalLM`, `DeepseekV3ForCausalLM`, `DeepseekV32ForCausalLM` | `AFDDeepseekForCausalLM`, `AFDDeepseekV2ForCausalLM`, `AFDDeepseekV3ForCausalLM` | DeepSeekV3.2 uses `AFDDeepseekV3ForCausalLM`. Each AFD role constructs and loads only its role-required model components, while shared embedding, normalization, and output components remain available where required by the model lifecycle. | | DeepSeek V4 Flash | `DeepseekV4ForCausalLM` | `AFDDeepseekV4ForCausalLM` | CUDA synchronous and Ascend CAM paths; see the [runtime matrix](docs/design/module/execution_platforms.md#tested-runtime-matrix) for backend-specific validation limits. | | Qwen3 MoE | `Qwen3MoeForCausalLM` | `AFDQwen3MoeForCausalLM` | CUDA; DeepEP M2N routes on Attention, P2P routes on FFN. | -| Qwen3.5 / Qwen3.6 MoE | `Qwen3_5MoeForConditionalGeneration` | `AFDQwen3_5MoeForConditionalGeneration` | CUDA text-only Qwen3.6-35B-A3B with `--language-model-only`, native DP4/TP1/EP4, and synchronous AFD 2A1F. Eager and graph execution remain supported; GDN DBO is temporarily withdrawn; native hybrid/multimodal validation applies. Earlier DBO experiments are historical evidence only; see the [E2E limits](docs/design/module/e2e_testing.md). | +| Qwen3.5 / Qwen3.6 MoE | `Qwen3_5MoeForConditionalGeneration` | `AFDQwen3_5MoeForConditionalGeneration` | CUDA text-only Qwen3.6-35B-A3B with `--language-model-only`, native DP4/TP1/EP4, and synchronous AFD 2A1F. Eager and graph execution are supported; GDN DBO is unsupported; see the [E2E limits](docs/design/module/e2e_testing.md). | Connector support: @@ -61,9 +61,9 @@ See the [recipe index](recipe/README.md) for deployment and benchmark examples. | Connector | Platform | Recommend Stage | Sync or Async | Graph Support | Notes | | --- | --- | --- | --- | --- | --- | | `P2pNcclAFDConnector` | CUDA | Decode | Sync | `FULL_DECODE_ONLY` CUDA graph | FFN ranks are ordered before Attention ranks. `num_attention_ranks` must be greater than or equal to `num_ffn_ranks`; the Attention ranks are spread over the FFN ranks in blocks that differ in size by at most one. See the [DeepSeek V2 Lite recipe](recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md). | -| `DeepEPM2NAFDConnector` | CUDA (SM90+) | Decode | Sync | `FULL_DECODE_ONLY` CUDA graph | Expert-routed M2N transport: Attention computes the gate and top-k and dispatches each token only to the FFN ranks that own its experts; the FFN runs only routed experts. Requires `compute_gate_on_attention=true`, FFN TP 1, and the same DBO setting on both roles; supports two-microbatch DBO, Attention TP, BF16 or block-128 FP8 payloads, and one or several nodes. Qwen3 MoE only. Dispatch and combine use [afdEP-comm](https://github.com/afdEP/afdEP-comm) (the `afdep` extra); the default expert backend needs [DeepGEMM](https://github.com/deepseek-ai/DeepGEMM) installed separately. See the [DeepEP M2N guide](docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md) and the [Qwen3 recipes](recipe/gpu/DeepEPM2NAFDConnector/README.md). | +| `DeepEPM2NAFDConnector` | CUDA (SM90+) | Decode | Sync | `FULL_DECODE_ONLY` CUDA graph | Qwen3 MoE or DeepSeek-V2; Attention-side routing, FFN TP1, and matching DBO settings (two microbatches when enabled). Requires afdEP-comm and an expert backend; see the [DeepEP M2N guide](docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md) and [recipes](recipe/gpu/DeepEPM2NAFDConnector/README.md). | | `CAMP2pAFDConnector` | Ascend NPU | Decode | Sync | `FULL` or `FULL_DECODE_ONLY`; plain MLA graph DBO requires `FULL_DECODE_ONLY` | Uses HCCL/CAMP2P custom ops and native MRV2. DBO requires two ubatches and Attention DP > 1. See the [CAMP2P guide](docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md). | -| `CAMAsyncAFDConnector` | Ascend NPU | Prefill / decode | Async | FFN-only experimental `FULL` | Eager Attention supports DP+TP/SP and two native DBO microbatches, including DP1. FFN FULL requires the [W4A8 runtime contract](docs/npu/ASYNC_FFN_RUNTIME.md) and accepts eager Attention DBO; FFN does not join native DBO. PCP/DCP are unsupported. See the [Async CAM guide](docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md) for configuration and validation limits. | +| `CAMAsyncAFDConnector` | Ascend NPU | Prefill / decode | Async | FFN-only experimental `FULL` | DeepSeek-V4 W4A8/W8A8 on A3/910C. Eager Attention supports DP+TP/SP and two native DBO microbatches, including DP1; FFN stays outside native DBO. See the [Async CAM guide](docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md) and [FFN runtime contract](docs/npu/ASYNC_FFN_RUNTIME.md). | Connector implementations are grouped by backend package: `afd_plugin.connectors.gpu` for GPU-only connectors, @@ -71,30 +71,17 @@ Connector implementations are grouped by backend package: Known gaps: -- The GPU target is exactly vLLM `0.32.0`; other vLLM versions are not - claimed as supported on GPU. - NPU hardware evidence covers the specific single-Pod DeepSeek-V2-Lite and DSV4 configurations in the [0.32 validation record](https://github.com/vllm-project/afd-plugin/pull/507). Other models and topologies remain unverified. AFD post-request shutdown failures remain open; successful inference does not imply graceful shutdown. - Synchronous DSV4 E2E profiles are smoke cases; A5 concurrent answers remain incorrect. See the [profile limits](tests/e2e/README.md#dsv4-flash-sync-camp2p-concurrent-requests-local-4-or-16-npus). -- GPU Attention requires native ModelRunnerV2; there is no GPU V1 fallback. - FFN remains connector-driven. Synchronous `P2pNcclAFDConnector` supports - two-microbatch DBO and `FULL_DECODE_ONLY`, with model-specific gate placement. - See the [runtime matrix](docs/design/module/execution_platforms.md#tested-runtime-matrix) - for tested configurations and known DeepSeek-V2-Lite DBO shutdown failures. -- `DeepEPM2NAFDConnector` requires SM90 or newer GPUs, Attention-side - routing, and FFN TP 1, and supports only Qwen3 MoE models whose layers are - all MoE and have no shared expert. Its GPU device tests - ([`tests/gpu`](tests/gpu/README.md)) and E2E cases need SM90+ hardware and - run outside CI. -- GPU and NPU E2E tests are opt-in and require real hardware plus model weights. -- GPU CUDA graph support is limited to `FULL_DECODE_ONLY`. -- Native DBO is limited to exactly two ubatches. NPU CAMP2P supports eager - and graph DBO; Async CAM enables eager DBO only on Attention, including - when FFN uses FULL. FFN stays outside Attention's native DBO thread schedule. - Model evidence remains specific to its workload and topology. +- Both backends require native ModelRunnerV2. The + [runtime matrix](docs/design/module/execution_platforms.md#tested-runtime-matrix) + records graph/DBO constraints and known DeepSeek-V2-Lite shutdown failures. +- GPU and NPU device/E2E tests require real hardware; model E2E also requires + weights. DeepEP M2N needs SM90+ and its device/E2E cases run outside CI. - Qwen3 MoE requires Attention-side gate placement for DeepEP M2N and FFN-side gate placement for P2P. It rejects sequence-parallel MoE, EPLB, pipeline parallelism, speculative decoding, LoRA, and NPU. diff --git a/csrc/npu/README.md b/csrc/npu/README.md index d3ce61c4e..017b76f0e 100644 --- a/csrc/npu/README.md +++ b/csrc/npu/README.md @@ -169,17 +169,15 @@ ensure_afd_ascend_ops_loaded() ## Current Scope -A2E/E2A continue to serve the existing CAMP2P connector. The four 910C -routed-only CAM operators are an experimental native integration and are not -yet selected by `CAMAsyncAFDConnector`, which retains its external legacy CAM -interface. +A2E/E2A serve `CAMP2pAFDConnector`. `CAMAsyncAFDConnector` uses the plugin's +910C routed-only CAM operators; see the [operator contract](../../docs/npu/CAM_ASYNC_ROUTED_OPS.md). `grouped_matmul_swiglu_quant_v2_layered` is a fused grouped-matmul + SiLU + dynamic-quantization CAM operator (A8W8 / A4W4 / A8W4-MSD variants) built for `ascend910_93`. It is registered as `torch.ops.afd_ascend.gmm_swiglu_quant_v2_layered` (inference + Meta; no -autograd grad) and is excluded from `ascend950` builds. It is selected by the -opt-in layered W4A8 executor. +autograd grad) and is excluded from `ascend950` builds. The default Async CAM +layered executor uses it for W4A8 and W8A8. The trailing `swiglu_limit` argument defaults to `0.0` (no clipping). A finite positive `L` computes `SiLU(min(gate, L)) * clamp(up, -L, L)` before output diff --git a/docs/design/module/attention_runtime.md b/docs/design/module/attention_runtime.md index db6b4e4f5..9fe8a528d 100644 --- a/docs/design/module/attention_runtime.md +++ b/docs/design/module/attention_runtime.md @@ -160,15 +160,14 @@ Both platforms require V2. The paired deployment constraints are: | Constraint | CUDA V2 | Ascend V2 synchronous | Ascend V2 Async CAM | | --- | --- | --- | --- | -| Connector | Synchronous `P2pNcclAFDConnector` | Synchronous `CAMP2pAFDConnector` | `CAMAsyncAFDConnector`; `async=true` | -| Gate placement | Model-specific; Qwen requires `compute_gate_on_attention=false` | `compute_gate_on_attention=false` | `compute_gate_on_attention=true` | +| Connector | Synchronous `P2pNcclAFDConnector` or `DeepEPM2NAFDConnector` | Synchronous `CAMP2pAFDConnector` | `CAMAsyncAFDConnector`; `async_dp=true` | +| Gate placement | P2P: model-specific, Qwen requires `false`; DeepEP M2N: `true` | `compute_gate_on_attention=false` | `compute_gate_on_attention=true` | | Parallelism | PP=PCP=DCP=1; configured role ranks equal DP x TP; static EP enabled | Same | Same; `attn_ranks_per_dp` equals Attention TP | -| Excluded features | Elastic EP, EPLB, sequence-parallel MoE, and compile SP | Elastic EP and EPLB; model-specific SP restrictions | Elastic EP, EPLB, native draft models/MTP, finegrained TP, and shared-expert DP | +| Excluded features | Elastic EP, EPLB, sequence-parallel MoE, and compile SP | Elastic EP and EPLB; model-specific SP restrictions | Elastic EP, EPLB, native draft models/MTP except DSV4 DSpark, finegrained TP, and explicit shared-expert DP | | Sequence parallelism | Unsupported | Model-specific; synchronous DeepSeek-V2/V3 SP is rejected | Attention-local FlashComm1/SP supported; disabled on FFN; no PCP/DCP | | Native DBO | Exactly two microbatches, Attention DP1 or DP > 1 | Eager or graph; exactly two microbatches, Attention DP > 1 | Eager only; exactly two microbatches at Attention DP1 or DP > 1 | -| AFD async MoE ubatching | Not applicable | Not applicable | Legacy alias for native DBO; split selector is ignored | | Graph execution | Eager or `FULL_DECODE_ONLY` | Eager, `FULL`, or `FULL_DECODE_ONLY`; plain MLA graph DBO requires `FULL_DECODE_ONLY` | Eager Attention; independent FFN also supports the narrow W4A8/W8A8 FULL contract | -| Model | Must resolve to a registered AFD architecture | Same | DeepSeek-V4 with layered W4A8/W8A8 FFN experts and `dynamicQuant=1` | +| Model | Registered AFD architecture; DeepEP M2N is limited to Qwen3 MoE or DeepSeek-V2 | Registered AFD architecture | DeepSeek-V4 with layered W4A8/W8A8 FFN experts and `dynamicQuant=1` | Both roles validate the paired V2 topology, but only Attention uses the native V2 model runner. FFN remains connector-driven and uses its existing AFD runner @@ -247,12 +246,9 @@ plugin-owned model code. These fields describe the current implementation; the open metadata issues prevent the object shape and live connector reference from becoming a long-term API. -Ascend V1 creates pending AFD metadata while native Attention metadata is -built and installs it immediately before model forward. V2 temporarily installs a -provider at the native `ForwardContext` creation seam and attaches the same -sidecar to the context created by the upstream runner. The shared metadata -mixin supplies the ordinary single-stage and graph-control behavior. Ascend V1 -native ubatching still gives each child context stage-local metadata. +The runner temporarily installs a provider at the native `ForwardContext` +creation seam and attaches the sidecar to the upstream context. The shared +metadata mixin supplies the single-stage and graph-control behavior. Ascend V2 Async CAM uses `NPUUBatchRunner` to adapt native DBO child inputs to Ascend attention builders. Native slicing, execution threads and output merge @@ -325,25 +321,12 @@ hook. FlashComm1/SP is Attention-local and FFN topology validation is independent. See the CAM async user guide for the supported topology and environment matrix. -CAM async requires eager execution. Its -feature limits are recorded in the +CAM async requires eager Attention; FFN can use eager or FULL execution. +Feature limits are recorded in the [platform matrix](execution_platforms.md#tested-runtime-matrix). ## Ubatching and graph orchestration -On Ascend V1, when native ubatching is enabled, AFD accepts exactly two -ubatches. The Attention runner chooses or normalizes slices, creates -stage-local AFD/DP metadata, and installs the platform wrapper during model -load. The role-level contract is that control-plane side effects complete -before replayable model work enters formal graph capture. - -During graph warmup/capture, the Attention runner marks control payloads so the -FFN daemon can warm or capture the matching shape. A non-ubatched capture sends -the control payload explicitly before formal capture; an ubatched capture lets -the platform wrapper send the exact per-stage shape. CUDA Graph, ACL Graph, -stream, and wrapper implementation details are owned by -[execution platforms](execution_platforms.md). - CUDA V2 accepts exactly two native microbatches with Attention DP1 or DP > 1. The CUDA-only [V4 compatibility patch](../../../afd_plugin/compat/patches/gpu_deepseek_v4_dbo.py) owns aligned, persistent stage inputs, Indexer SM planning and pre-profile @@ -401,22 +384,9 @@ the matching platform and connector tests as well. ## Limitations and open issues -Current shared limits are the supported vLLM release and registered role-aware -model integrations. Historical Ascend V1 accepted exactly two ubatches. V2 -requires static EP and no PP/CP. CUDA V2 requires a synchronous control-plane -connector and supports the two-microbatch path above. Ascend V2 supports eager -native DBO with exactly two microbatches: synchronous CAMP2P requires Attention -DP > 1 and supports graph DBO; Async CAM supports eager Attention at DP1 or -DP > 1. Hardware evidence remains limited to the recorded model and -configuration cells; see the [0.32 record](https://github.com/vllm-project/afd-plugin/pull/507). -Platform/connector limits are intentionally centralized in +Platform/connector limits and hardware evidence are centralized in [execution platforms](execution_platforms.md#tested-runtime-matrix). -Async CAM DBO has a Lite two-stage hardware accuracy result; broader coverage needs -DP1/DP > 1, plain TP/model SP, uneven -prefill lengths, and requests crossing a child boundary, with live two-child -execution evidence and baseline output comparison. - This document does not decide the runtime refactor, connector metadata ownership, or transfer metadata/state shape. See [#86](https://github.com/JiusiServe/afd-plugin/issues/86), diff --git a/docs/design/module/compatibility_and_patches.md b/docs/design/module/compatibility_and_patches.md index 98d96abfd..380442bc0 100644 --- a/docs/design/module/compatibility_and_patches.md +++ b/docs/design/module/compatibility_and_patches.md @@ -159,7 +159,7 @@ These modules adapt upstream behavior without replacing a global symbol: | [`compat/npu/runtime_config.py`](../../../afd_plugin/compat/npu/runtime_config.py) | Preserves model SP while applying the non-SP backend fallback; normalizes Async CAM native ubatching to Attention DBO, checks thresholds, and derives model/PP tensor layout and active ubatch count. | | [`compat/npu/feature_validation.py`](../../../afd_plugin/compat/npu/feature_validation.py) | Parses connector-owned typed extra information through the factory and fails before execution for unsupported NPU connector, quantization, graph, DBO, gate, or async MoE combinations. | | [`compat/npu/routing_ids.py`](../../../afd_plugin/compat/npu/routing_ids.py) | Registers the opaque `afd_balance_routing_ids_` custom op. It reads Ascend's live profile flag inside the op and balances logical expert IDs in place during profiling or when forced. NPU worker construction registers it before the first Attention-side gate; standalone scripts must import the module after device initialization and before using the op. | -| [`compat/npu/forward_context.py`](../../../afd_plugin/compat/npu/forward_context.py) | Enters the pinned Ascend forward context for connector-driven FFN compute and installs AFD metadata in `additional_kwargs`. V1 uses `set_ascend_forward_context`; a V2-paired FFN uses native `set_forward_context` plus the Ascend MRV2 profile override so platform state follows the V2 proxy layout. | +| [`compat/npu/forward_context.py`](../../../afd_plugin/compat/npu/forward_context.py) | Uses native `set_forward_context` and the Ascend MRV2 profile override for connector-driven FFN compute, installing AFD metadata and the model instance in `additional_kwargs`. | | [`compat/npu/ops.py`](../../../afd_plugin/compat/npu/ops.py) | Discovers plugin-owned CAMP2P and routed-only async CAM operators with explicit missing-runtime errors; operator build/runtime ownership is documented in [execution platforms](execution_platforms.md). | Several scoped upstream adaptations live with their semantic owners rather diff --git a/docs/design/module/connector_contracts.md b/docs/design/module/connector_contracts.md index cc8504a95..dcaf453bc 100644 --- a/docs/design/module/connector_contracts.md +++ b/docs/design/module/connector_contracts.md @@ -42,7 +42,7 @@ related_issues: - "#105" - "#107" - "#129" -last_reviewed: 2026-11-05 +last_reviewed: 2026-11-09 --- # Connector contracts @@ -95,8 +95,8 @@ which lets feature validation use the same connector-owned schema. ## Connector-owned configuration -`additional_config["afd"]["connector_extra_config"]` is an envelope rather -than an `AFDConfig` field. Unknown fields fail in the selected connector parser. +`AFDConfig.connector_extra_config` stores the connector-specific mapping. +Unknown fields fail in the selected connector parser. | Connector | Accepted connector-specific fields | | --- | --- | diff --git a/docs/design/module/execution_platforms.md b/docs/design/module/execution_platforms.md index 2e68f16ec..f95152e80 100644 --- a/docs/design/module/execution_platforms.md +++ b/docs/design/module/execution_platforms.md @@ -44,7 +44,7 @@ upstream_refs: - "GPU: vLLM 0.32.0 5df63a7203cdb9328d824a27b682e77c7b9928be" - "NPU target: vLLM 0.32.0 5df63a7203cdb9328d824a27b682e77c7b9928be" - "vLLM vllm.compilation and vllm.v1.worker.gpu V2 graph/ubatching APIs" - - "vLLM-Ascend ACL graph, forward-context, and model-runner V1/V2 APIs in the tested environment" + - "vLLM-Ascend ACL graph, forward-context, and ModelRunnerV2 APIs in the tested environment" - "PyTorch CUDA, torch_npu, CMake, and Ascend CANN build interfaces used by the repository" verified_platform_refs: - "CUDA eager, graph, DBO, and ModelRunnerV2 E2E paths; see runtime matrix for tested scope" @@ -145,14 +145,13 @@ flowchart TB ### Worker and device setup -Both CUDA workers delegate device and distributed setup to native -`Worker.init_device()`. During that synchronous construction window, they -temporarily replace the native V2 runner symbol with the AFD runner -and restore it in `finally`. Both roles require ModelRunnerV2. Attention -selects `AFDAttentionModelRunnerV2`; FFN selects the connector-driven -`GPUFFNModelRunner`. `torch.accelerator.empty_cache()` releases -startup allocations. Attention retains the matching upstream request, -KV-cache, sampling, and output behavior; FFN remains connector-driven. +Both CUDA workers validate the AFD configuration and select the role's model +before delegating device and distributed setup to native `Worker.init_device()`. +Their `_make_model_runner()` overrides construct `AFDAttentionModelRunnerV2` +or the connector-driven FFN runner (`GPUFFNModelRunner` for P2P, +`GPUFFNRoutedExpertsModelRunner` for DeepEP M2N). Both roles require +ModelRunnerV2. Attention retains upstream request, KV-cache, sampling, and +output ownership. ### CUDA Graph policy @@ -198,9 +197,7 @@ Scoped factory and dispatch adapters enable the native path at Attention DP1 as well as DP > 1. AFD accepts exactly two native ubatches. Qwen3.5/3.6 GDN DBO is unsupported; native hybrid/multimodal DBO validation -applies. Qwen3 MoE DBO remains supported. Earlier Qwen GDN Graph + DBO and -six-cell recurrent-state/CUDA Graph parity results describe the removed adapter -and are historical evidence only. See the [runtime matrix](#tested-runtime-matrix) +applies. Qwen3 MoE DBO remains supported. See the [runtime matrix](#tested-runtime-matrix) for tested configurations and known DeepSeek-V2-Lite DBO shutdown failures. ### CUDA profiling @@ -221,12 +218,10 @@ upstream construction. During device initialization they: 1. validate Ascend-specific feature combinations; 2. apply the non-sequence-parallel all-to-all backend correction when needed, including legacy explicit-worker launches; -3. validate the synchronous CAMP2P or eager Async CAM ModelRunnerV2 subset - when V2 is selected; +3. require ModelRunnerV2 and validate the connector's role-specific execution mode; 4. call `NPUWorker._init_device()`; 5. initialize the vLLM workspace manager for one or two ubatches; -6. construct the matching V1 or V2 Attention runner, or the connector-driven - FFN runner. +6. construct `AFDNPUAttentionModelRunnerV2` or the connector-driven FFN runner. The all-to-all correction selects `flashinfer_all2allv` when sequence parallelism is disabled. Automatic worker selection receives the matching @@ -241,11 +236,10 @@ in `ForwardContext.additional_kwargs`. FFN uses connector-driven MoE compute, including token counts and ACL graph runtime mode when applicable. -When ModelRunnerV2 is selected, `ascend_forward_context()` uses native +`ascend_forward_context()` uses native `set_forward_context()` plus vLLM-Ascend's MRV2 profile override. This places Ascend-specific state, `model_instance`, and AFD metadata in -`additional_kwargs`, matching the proxy layout read by the V2 runtime. V1 -continues to use `set_ascend_forward_context()`. +`additional_kwargs`, matching the proxy layout read by the V2 runtime. For native ubatching, `create_ascend_forward_context()` creates one context per stage with stage attention/DP metadata, batch descriptor, and graph mode. @@ -257,9 +251,7 @@ reassembled to match the upstream Ascend layout. `NPUUBatchRunner` extends native MRV2 `UBatchRunner`. Native code owns child slicing, threads, handoff, capture/replay and output merge. AFD supplies Ascend child metadata builders, NPU stream/context scopes, model-specific transport -coordinates and nested auxiliary output handling. The old NPU MRV1 runner, -wrapper, custom registry and request-boundary splitting implementation are -removed; there is no execution fallback. +coordinates and nested auxiliary output handling. `v1/worker/dbo.py` registers the model-side yield operation and dispatches to the platform DBO implementation. Eager Async Attention uses native DBO at every @@ -312,12 +304,12 @@ register no FIA workspace, so those models, DeepSeek V4 among them, take the generic two-stage path with no MLA registries. With synchronous CAMP2P, the NPU V2 runner supports eager, `FULL`, and -`FULL_DECODE_ONLY`; Async CAM is eager-only. Like CUDA V2, +`FULL_DECODE_ONLY`; Async CAM Attention is eager-only. Like CUDA V2, it publishes descriptor-matched warmup/capture control outside formal graph capture and installs an instance-scoped pre-replay hook because native full -replay creates no `ForwardContext`. V2 does not use `AscendUBatchWrapper` and -uses native `NPUUBatchRunner` preparation, execution and merge for eager and -graph DBO. Plain MLA graph DBO requires `FULL_DECODE_ONLY`; SFA/DSA +replay creates no `ForwardContext`. Native `NPUUBatchRunner` owns preparation, +execution and merge for eager and graph DBO. Plain MLA graph DBO requires +`FULL_DECODE_ONLY`; SFA/DSA accepted modes still require model-specific qualification. Native two-stage capture shares the ordinary descriptor's FIA workspace; replay merges captured records with current metadata for Ascend updates. Each child carries its actual stage ID for the opaque CAMP2P HCCL @@ -329,8 +321,9 @@ The FFN runner owns a separate ACL graph cache keyed by stage token counts and A/F topology. Warmup runs the eager FFN path. Formal capture updates connector state through `connector.control_plane` before entering `torch.npu.graph(...)`, so replay contains only model and data-plane operations. Replay additionally requires the control payload's -`is_graph_replaying=true`; an unknown key or eager payload executes eagerly. CAM async does not enter this path because validation -requires eager execution and `connector.control_plane` is `None`. +`is_graph_replaying=true`; an unknown key or eager payload executes eagerly. +CAM async has no control plane and uses the independent +[Async FFN runtime](../../npu/ASYNC_FFN_RUNTIME.md). ### Ascend native operators and packaging @@ -373,18 +366,16 @@ during shutdown. ## Tested runtime matrix -This table records current validation gates and repository evidence. The -historical V1 rows are not current launch options or fallbacks. Evidence -does not expand the supported runtime contract. +This table separates accepted runtime configurations from validation evidence. +Both backends require ModelRunnerV2. A test case or a historical result does not +qualify every model/topology combination on the current target. | Platform/path | Execution | Ubatching | Routing/quantization limits | Evidence | | --- | --- | --- | --- | --- | -| Historical CUDA V1 (removed) + `P2pNcclAFDConnector` | Eager or `FULL_DECODE_ONLY` CUDA Graph | Native DBO, exactly two ubatches | Registered CUDA model boundaries; Attention-side or FFN-side gate where the model supports it | DeepSeek-V2-Lite eager/graph/DBO accuracy E2E; model, graph, connector, and profiler unit tests | | CUDA V2 + `P2pNcclAFDConnector` | Eager or `FULL_DECODE_ONLY` native V2 CUDA Graph | Exactly two native microbatches; Attention DP1 or DP > 1 | Model-specific gate placement; PP/CP, elastic EP, EPLB, SP MoE, and compile SP rejected; role ranks equal DP x TP | Source-specific GPU validation is linked below; coverage is limited to the recorded model/topology combinations. DeepSeek-V2-Lite has known DBO shutdown failures. Qwen3.5/3.6 GDN DBO is unsupported. | -| CUDA V2 + `DeepEPM2NAFDConnector` | Eager or `FULL_DECODE_ONLY` native V2 CUDA Graph; SM90 or newer | Exactly two native microbatches, enabled on both roles; Attention DP1 or DP > 1, TP1 or TP2 | Qwen3 MoE only (all layers MoE, no shared expert); `compute_gate_on_attention=true`; FFN TP1 with EP over all FFN ranks, linear placement, `allgather_reducescatter` all2all and the Triton MoE weight layout; BF16 or block-128 FP8 payload; EPLB, PP/CP and SP MoE rejected | Historical vLLM 0.30 Qwen3-30B-A3B 2A2F GSM8K E2E for eager, graph, graph + DBO, and Attention TP2 eager/graph on H100 and H200, and for eager, graph and graph + DBO on GB200; two-node 2A2F eager, graph and graph + DBO with Attention and FFN on separate nodes; [GPU connector device tests](../../../tests/gpu/README.md) | -| Historical Ascend V1 (removed) + `CAMP2pAFDConnector` | Eager or current ACL Graph path | Native DBO, exactly two ubatches | Common and connector-local `compute_gate_on_attention=false`; `connector_extra_config.quant_mode=0`; plugin CANN ops required | V2-Lite 2A2F graph 300-question comparison, 2A1F/2A2F functional checks, and Legacy DBO regressions; runtime, graph, ops, connector, and profiler unit tests | +| CUDA V2 + `DeepEPM2NAFDConnector` | Eager or `FULL_DECODE_ONLY` native V2 CUDA Graph; SM90 or newer | Exactly two native microbatches, enabled on both roles; Attention DP1 or DP > 1, TP1 or TP2 | Qwen3 MoE (all layers MoE, no shared expert) or DeepSeek-V2; `compute_gate_on_attention=true`; FFN TP1 with EP over all FFN ranks, linear placement, `allgather_reducescatter` all2all and the Triton MoE weight layout; BF16 or block-128 FP8 payload; EPLB, PP/CP and SP MoE rejected | Historical vLLM 0.30 Qwen3-30B-A3B 2A2F GSM8K E2E for eager, graph, graph + DBO, and Attention TP2 eager/graph on H100 and H200, and for eager, graph and graph + DBO on GB200; two-node 2A2F eager, graph and graph + DBO with Attention and FFN on separate nodes. DeepSeek-V2-Lite: [E2E definitions](../../../tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py). [GPU connector device tests](../../../tests/gpu/README.md) | | Ascend V2 + `CAMP2pAFDConnector` | Eager, `FULL`, or `FULL_DECODE_ONLY` native V2 ACL Graph | Eager or graph native DBO with two ubatches and Attention DP > 1; plain MLA graph DBO requires `FULL_DECODE_ONLY` | `compute_gate_on_attention=false`; PP/CP, elastic EP and EPLB rejected; model-specific SP policy; role ranks equal DP x TP | Historical 0.30 V2-Lite 2A2F FULL 300-question comparison and 2A1F functional checks; current 0.32 Lite evidence is in the [validation record](https://github.com/vllm-project/afd-plugin/pull/507) | -| Ascend V2 + `CAMAsyncAFDConnector` | Eager Attention; layered FFN eager or FULL | Two native DBO microbatches on Attention, including DP1 and with FFN FULL; FFN stays outside the native DBO schedule | Experimental DeepSeek-V4 W4A8/W8A8 on A3/910C; `async=true`, `compute_gate_on_attention=true`, and `dynamicQuant=1`; PP/CP unsupported; plugin-owned async CAM ops required | Historical DSV4 layered comparisons and limitations are in the [validation record](https://github.com/vllm-project/afd-plugin/pull/507); they predate removal of the legacy path | +| Ascend V2 + `CAMAsyncAFDConnector` | Eager Attention; layered FFN eager or FULL | Two native DBO microbatches on Attention, including DP1 and with FFN FULL; FFN stays outside the native DBO schedule | Experimental DeepSeek-V4 W4A8/W8A8 on A3/910C; `async_dp=true`, `compute_gate_on_attention=true`, and `dynamicQuant=1`; PP/CP unsupported; plugin-owned async CAM ops required | Historical DSV4 layered comparisons and limitations are in the [validation record](https://github.com/vllm-project/afd-plugin/pull/507); they predate removal of the legacy path | The 300-question Ascend evidence in this table is historical. The 0.32 [validation record](https://github.com/vllm-project/afd-plugin/pull/507) separately lists the @@ -401,12 +392,6 @@ FFN remains connector-driven. The row's upgrade evidence retains its original scope; MRV2 deployment settings are documented in the [CAM async guide](../../npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md#modelrunnerv2-deployment). -DSV2/V3 Async CAM Attention with model SP selects Ascend shared-expert DP before -constructing the shared MLP. Each rank holds complete shared weights and computes -its local token rows. Attention TP/SP and routed-expert parallelism are unchanged. -Without model SP, the user's shared-expert DP setting is preserved. MRV2's -existing rejection of explicit shared-expert DP configuration is unchanged. - Both backends target vLLM `0.32.0` (`5df63a72`); Ascend additionally targets vLLM-Ascend `bd42eea0`. Source-specific H20 model, topology, execution-mode and accuracy results are recorded in @@ -427,15 +412,11 @@ Validation limits: [validation record](https://github.com/vllm-project/afd-plugin/pull/507); the seven AFD Lite cases failed graceful exit. Historical NPU results do not qualify 0.32. -The historical [NPU validation record](https://github.com/vllm-project/afd-plugin/pull/425#issuecomment-6063910923) -covers five representative configurations with the same first 300 GSM8K -questions: V2-Lite MRV1 graph, MRV2 FULL, ordinary Async CAM, and DSV4 Flash -W4A8 layered off/on. The [Legacy DBO repair](https://github.com/vllm-project/afd-plugin/pull/430#issuecomment-6063840311) -adds 2A1F/2A2F content regressions and a same-question 128-sample comparison; -it does not add a new 300-question result. These records do not establish all -model/topology support, numerical equivalence, performance, or clean device -shutdown. Synchronous DSV4 profiles remain smoke-only, with known incorrect -A5 concurrent answers; see their [E2E limits](../../../tests/e2e/README.md#dsv4-flash-sync-camp2p-concurrent-requests-local-4-or-16-npus). +Historical [NPU validation](https://github.com/vllm-project/afd-plugin/pull/425#issuecomment-6063910923) +and [DBO repair](https://github.com/vllm-project/afd-plugin/pull/430#issuecomment-6063840311) +records retain their original versions and sample counts. Synchronous DSV4 +profiles remain smoke-only, with known incorrect A5 concurrent answers; see +their [E2E limits](../../../tests/e2e/README.md#dsv4-flash-sync-camp2p-concurrent-requests-local-4-or-16-npus). GPU/NPU rank topology and connector resource rules remain owned by [connector contracts](connector_contracts.md). diff --git a/docs/design/module/ffn_runtime.md b/docs/design/module/ffn_runtime.md index 07d3576a4..2f1af04f4 100644 --- a/docs/design/module/ffn_runtime.md +++ b/docs/design/module/ffn_runtime.md @@ -60,7 +60,7 @@ related_issues: - "#105" - "#107" - "#129" -last_reviewed: 2026-11-05 +last_reviewed: 2026-11-09 --- # FFN runtime @@ -397,15 +397,7 @@ async model E2E coverage. ## Limitations and open issues -Current shared limits are the supported vLLM release, connector-driven FFN, -and registered role-aware model integrations. Ascend V1 native DBO accepts -exactly two ubatches. A V2-paired FFN uses the same AFD FFN runner with a -V2-compatible construction and forward-context seam. CUDA FFN consumes the -two-stage control layout and replays the matching graph when requested; -Ascend V2 consumes two-stage synchronous control metadata in eager DBO mode. -CAM async instead uses eager connector work items; native DBO runs only on -Attention and does not create a separate FFN stage scheduler. -Platform-specific limits are centralized in +Platform-specific execution limits and validation evidence are centralized in [execution platforms](execution_platforms.md#tested-runtime-matrix). Issue [#107](https://github.com/JiusiServe/afd-plugin/issues/107) completed the diff --git a/docs/design/module/model_integration.md b/docs/design/module/model_integration.md index 7c9128b74..6bbb97883 100644 --- a/docs/design/module/model_integration.md +++ b/docs/design/module/model_integration.md @@ -38,7 +38,7 @@ related_issues: - "#88" - "#105" - "#129" -last_reviewed: 2026-10-30 +last_reviewed: 2026-11-09 --- # Model integration @@ -188,23 +188,21 @@ separate from the routed layered executor: W4A8 applies the model's The synchronous DSV4 E2E profiles check concurrent serving but disable answer checking; the A5 profile records incorrect concurrent answers. Async DSV4 has an exact-answer smoke case; V4 true native DBO still requires model accuracy -qualification. Lite two-stage DBO has the limited result recorded in the -[0.32 validation record](https://github.com/vllm-project/afd-plugin/pull/507). See +qualification. See [E2E testing](e2e_testing.md) and the [layered guide](../../npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md#w4a8-and-w8a8-layered-gmm). ### Qwen3 MoE CUDA boundary `AFDQwen3MoeModel` uses the native `decoder_layer_type` injection hook. The -Attention role constructs native Qwen Attention and normalization modules and -uses a parameter-free `RemoteFFNProxy`; the FFN role constructs the complete -native dense MLP or `Qwen3MoeSparseMoeBlock`. Native forward order, FusedMoE, -packed parameter mapping, quantization paths, and weight loading remain owned -by vLLM. Checkpoint weights are filtered once by layer-stage path before the -native loader consumes them. +Attention role constructs native Qwen Attention and normalization modules. +With P2P, a parameter-free `RemoteFFNProxy` sends hidden states to FFN, which +runs the complete native MLP or MoE block; `compute_gate_on_attention=false` +is required. With DeepEP M2N, Attention owns gate/top-k and FFN runs routed +experts only; this requires `compute_gate_on_attention=true`, every layer to +be MoE, and no shared expert. Native weight loading follows role-based filtering. -This path is CUDA-only and requires `compute_gate_on_attention=false`. It fails -during construction for sequence-parallel MoE, EPLB, pipeline parallelism, -speculative decoding, LoRA, or NPU. +Both paths are CUDA-only and reject sequence-parallel MoE, EPLB, pipeline +parallelism, speculative decoding, and LoRA during construction. ### Qwen3.5/3.6 MoE CUDA boundary @@ -218,9 +216,7 @@ This path is CUDA-only, text-only, and requires `--language-model-only`, multimodal execution, sequence-parallel MoE, EPLB, speculative decoding, and LoRA before language-model construction; NPU model-config resolution fails before model loading. GDN DBO is unsupported under native hybrid/multimodal -DBO validation; ordinary eager and graph execution remain supported. Earlier -H20 Graph + DBO and recurrent-state/CUDA Graph parity checks describe the -removed adapter and do not qualify the current implementation. The current +DBO validation; ordinary eager and graph execution remain supported. The 2A1F suite keeps static EP and PP1. Attention-side router transport, async communication, multi-node, quantization, and performance remain outside the validated scope. @@ -234,9 +230,8 @@ ad-hoc `ForwardContext.afd_metadata` attribute. The current metadata supplies stage/request/token slicing, stage count, optional transaction ID, and a live connector reference. -Native forward-context creation can bypass an AFD model-runner call site: -Ascend V1 dummy runs do so, while V2 creates its context inside native -`execute_model()`. During those scopes, `use_afd_metadata_provider()` +Native V2 creates its context inside `execute_model()`. +During that scope, `use_afd_metadata_provider()` temporarily wraps `vllm.forward_context.create_forward_context`, lets the runner install the same `additional_kwargs` entry, and restores the original function in `finally`. This is a scoped compatibility adapter, not a permanent diff --git a/docs/design/module/plugin_boundary.md b/docs/design/module/plugin_boundary.md index 45b6b88e9..a1dbd2880 100644 --- a/docs/design/module/plugin_boundary.md +++ b/docs/design/module/plugin_boundary.md @@ -138,7 +138,7 @@ must be placed under `connector_extra_config`. | `afd_process_group_timeout_s` | `120` | Positive AFD process-group rendezvous timeout in seconds; also bounds the NPU native DBO startup barrier. It is not a device-kernel cancellation deadline. | | `num_attention_ranks`, `num_ffn_ranks` | `1`, `1` | AFD role-group sizes used by topology construction. | | `compute_gate_on_attention` | `false` | Moves supported gate/MoE routing work to Attention. CUDA supports both gate placements at the remote-experts boundary; synchronous CAMP2P still requires `false`, while CAM async requires `true`. | -| `connector_extra_config` | `{}` | Envelope key parsed by the selected connector into a typed `ConnectorExtraInfo`; it is not stored on `AFDConfig`. | +| `connector_extra_config` | `{}` | Mapping stored on `AFDConfig` and parsed by the selected connector into a typed `ConnectorExtraInfo`. | The compatibility aliases `afd_connector`, `afd_role`, `afd_port`, `afd_host`, and `async` normalize to canonical fields. Supplying an alias and its canonical @@ -150,15 +150,14 @@ coordinates. It is resolved once by the connector factory and is not an `AFDConfig` field. Connector construction parses `connector_extra_config` through the selected -connector class. P2P accepts only an empty mapping; CAMP2P and CAM async each -define a closed, typed schema and reject unknown fields. The detailed schemas +connector class. P2P accepts only an empty mapping; the other connectors +define closed, typed schemas and reject unknown fields. The detailed schemas belong to [connector contracts](connector_contracts.md) and the operational connector guides. -`AFDConfig.compute_hash()` currently includes connector, async mode, role, and -both role counts. This is an implementation detail used by -graph-affecting configuration paths, not a complete serialization or public -configuration identity. +`AFDConfig.compute_hash()` includes connector, async mode, role, both role +counts, and connector-specific configuration. It is used by graph-affecting +configuration paths, not as a complete serialization or public identity. ## Validation and worker selection diff --git a/docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md b/docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md index 2b598994e..5c18a2476 100644 --- a/docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md +++ b/docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md @@ -203,7 +203,7 @@ alone does not authorize replay. These behaviors are covered by the [connector tests](../../tests/unit/connectors/test_p2p_connector.py) and [FFN graph-policy tests](../../tests/unit/v1/worker/test_cuda_graph.py). -## Requirements and limitations to document in code +## Requirements and limitations - CUDA-capable PyTorch/vLLM environment with NCCL and vLLM's `PyNcclCommunicator` available. - Hidden-state tensors must reside on CUDA devices; CPU tensors are rejected. diff --git a/docs/npu/ASYNC_FFN_RUNTIME.md b/docs/npu/ASYNC_FFN_RUNTIME.md index a56e05531..52b67eac1 100644 --- a/docs/npu/ASYNC_FFN_RUNTIME.md +++ b/docs/npu/ASYNC_FFN_RUNTIME.md @@ -1,14 +1,11 @@ # MRV2 Async CAM FFN eager/FULL runtime -AFD #449 captures one complete routed-expert transaction: original dispatch -receive (DR), layered W4A8/W8A8 FFN, and original combine send (CS). -Attention remains MRV2 eager. Issue #451 retires the separate MRV1 paths while -preserving this runtime and its original transport/lifecycle boundary. +FULL captures one routed-expert transaction: dispatch receive (DR), +W4A8/W8A8 FFN, and combine send (CS). Attention remains MRV2 eager. `AsyncNPUFFNModelRunner` owns Async CAM model loading, eager/FULL transactions and receive-safe resource release independently of the synchronous CAMP2P -`AFDNPUFFNModelRunner`. The FFN worker selects the owner from the connector; -neither runner inherits MRV1 or owns Attention request/KV state. +`AFDNPUFFNModelRunner`. The FFN worker selects the owner from the connector. ## Selection and startup diff --git a/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md b/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md index 9f1cbb97a..31eeb502a 100644 --- a/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md +++ b/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md @@ -139,9 +139,8 @@ from the model routed-expert count and `num_ffn_ranks`; use a model/topology in which routed experts divide evenly across FFN ranks. All roles must use the same model, routed-expert layout, quantization, HCCL address, rank counts, CAM settings. Enable native DBO only on Attention; FFN must not receive -`--enable-dbo` or native ubatching flags. Legacy connector flags may remain in -shared JSON because their normalization is Attention-only. In an async deployment, every Attention (A) -and FFN (F) process must also set `--max-num-batched-tokens` to the same value. +`--enable-dbo` or native ubatching flags. Every Attention and FFN process must +set `--max-num-batched-tokens` to the same value. ## AFD configuration @@ -193,6 +192,7 @@ including `async_dp`; enable DBO through the Attention CLI flags below. | Field | Type | Default | Meaning and constraint | | --- | --- | --- | --- | | `dynamicQuant` | `int` | `0` | Set to `1`: FFN receives dynamically quantized routed activations and scale tensors. The default `0` is not supported by the layered FFN path. | +| `ffn_mega_moe` | `bool` | `false` | Selects Mega FFN with Pull transport; requires `dynamicQuant=1` and the same setting on all A/F ranks. See the [Pull execution session](ASYNC_FFN_RUNTIME.md#opt-in-pull-execution-session). | | `attn_ranks_per_dp` | `int` | `1` | Positive Attention TP rank count per DP replica. It is the CAM Attention grouping width and is independent of the FFN process's local TP size. | | `hccl_buffer_size` | `int` | unset | Positive CAM HCCL buffer size in MB. The override applies only to the `afd_async_cam` communication domain. When unset, HCCL uses `HCCL_BUFFSIZE`, then its built-in default. Configure the same value on every Attention and FFN rank in the domain. | @@ -298,10 +298,8 @@ actual vendor library path. Missing source operators fail startup. parallel execution remains outside the supported Async CAM topology. - Native DBO requires eager Attention, two ubatches, and no speculative decoding. FFN keeps its independent connector-driven execution. -- Attention DP1 reuses native DBO through scoped Ascend compatibility adapters; - the legacy request/token split selector no longer controls boundaries. -- ModelRunnerV2 rejects PP, PCP, and DCP; V1 PCP and async MoE context - parallelism remain unsupported. +- Attention DP1 reuses native DBO through scoped Ascend compatibility adapters. +- ModelRunnerV2 rejects PP, PCP, and DCP. - Routed experts should divide evenly across FFN ranks. For startup, HCCL, CAM operator, and shutdown failures, see the @@ -309,18 +307,10 @@ For startup, HCCL, CAM operator, and shutdown failures, see the ## W4A8 and W8A8 layered GMM -Async CAM FFN always uses the two layered GMM operators. FFN startup logs the -quantization, layer count and layout as `Async CAM layered GMM`. - -### Migration from the legacy FFN path - -Remove `AFD_ASYNC_CAM_LAYERED_GMM` from all launch environments; the selector -has been removed and setting it to `0` no longer selects the old path. -DeepSeek-V2 and V2-Lite, including BF16 checkpoints, no longer support Async -CAM because their FFN weights do not implement the layered contract. The -standard DeepSeek-V3.2 architecture was already rejected by model validation and -remains unsupported. Other connectors keep their own model support boundaries. -Deployments needing the old Async CAM path must remain on an earlier revision. +By default, Async CAM FFN uses two layered GMM operators. +`ffn_mega_moe=true` selects the fused Mega executor and Pull transport using +the same weight table. Startup logs its quantization, layer count and layout +as `Async CAM layered GMM`. The intended configuration is Ascend 910C / `ascend910_93`, async CAM FFN, Attention-side gate, `dynamicQuant=1`, eager execution (or the separately @@ -334,8 +324,7 @@ loader rejects positive `group_size`; the current target is limited to per-channel W4A8, or dynamic W8A8 with FP32 W13/BF16 W2 per-channel scales. W8A8 uses loaded INT8 NZ tensors and a single shared zero INT32 W2 bias; it does not require W4A8 compensation or duplicate expert weights. -DSV4 Flash W4A8 has layered off/on 300-question comparisons and -real INT8-activation/INT4-weight operator traces in the +Historical DSV4 Flash W4A8 layered off/on comparisons and operator traces are in the [runtime validation record](../design/module/execution_platforms.md#tested-runtime-matrix). Per-group operator coverage does not extend the checkpoint loader to per-group weights. diff --git a/recipe/README.md b/recipe/README.md index ad6543d85..364b3be37 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -11,23 +11,6 @@ scope; the V3.2 recipes below retain their historical versions. Recipes are organized by hardware backend, connector, and model: -```text -recipe/ -├── gpu/ -│ ├── DeepEPM2NAFDConnector/ -│ │ ├── qwen3_235b_a22b_fp8/ -│ │ └── qwen3_30b_a3b/ -│ └── P2pNcclAFDConnector/ -│ └── deepseek_v2_lite/ -└── npu/ - ├── CAMAsyncAFDConnector/ - │ └── deepseek_v3_2/ - └── CAMP2pAFDConnector/ - └── deepseek_v3_2/ -``` - -Directory names follow these conventions: - - Hardware backend: `gpu` or `npu`. - Connector: the exact connector class name, such as `P2pNcclAFDConnector`, `DeepEPM2NAFDConnector`, `CAMP2pAFDConnector`, or `CAMAsyncAFDConnector`. @@ -39,6 +22,7 @@ Directory names follow these conventions: | Hardware | Connector | Model | Recommended stage | Status | Recipe | | --- | --- | --- | --- | --- | --- | | GPU | `P2pNcclAFDConnector` | DeepSeek-V2-Lite | Decode | Deployment examples; see the linked README for validation scope | [Launch examples](gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md) | +| GPU (SM90+) | `DeepEPM2NAFDConnector` | DeepSeek-V2-Lite | Decode | Deployment examples matching the checked-in E2E layouts; hardware qualification is separate | [Launch examples](gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/README.md) | | GPU (SM90+) | `DeepEPM2NAFDConnector` | Qwen3-30B-A3B | Decode | Same 2A2F layouts as the Qwen3 MoE DeepEP M2N E2E cases | [Launch examples](gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/README.md) | | GPU (SM90+) | `DeepEPM2NAFDConnector` | Qwen3-235B-A22B-FP8 | Decode | 4A2F FP8-payload configuration run on H200 | [Launch examples](gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/README.md) | | Ascend NPU | `CAMP2pAFDConnector` | DeepSeek-V3.2 | Decode | Historical v0.19.1rc1 recipe; not qualified for the 0.32 target | [Synchronous decode](npu/CAMP2pAFDConnector/deepseek_v3_2/README.md) | From 5aed7b9dc79f462076ccbf2088f5856035e0385e Mon Sep 17 00:00:00 2001 From: Ronen Kat <16743404+ronenkat@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:15:44 +0200 Subject: [PATCH 27/42] [GPU] DeepEP M2N: flexible Attention/FFN placement across pods (#555) ## Purpose Let the DeepEP M2N connector run pods that each hold Attention and FFN ranks (for example 2 x 4A1F), so a deployment grows by adding identical nodes. The handshake plans the transport order from NVLink domains and uses afdEP-comm v1.1.0's per-domain mode; FFN-prefix placements keep their current path. ## Issue - Related issue(s): #554 - Closing keyword, only if fully resolved: Closes # ## Scope - In scope: placements with equal pods and the same FFN count per pod (`4A1F,4A1F`, `1A1F,1A1F`, `6A2F,6A2F`), plus all FFN-prefix placements accepted today; afdep-comm pin moved to tag `v1.1.0`; a Qwen3-235B-A22B-FP8 2 x 4A1F recipe; multi-pod E2E cases. - Out of scope: unequal pod sizes and a varying FFN count per pod (needs afdEP-comm changes); checking the placement before weight load. ## Implementation Notes - `deepep_m2n/layout.py`: `RankPlacement`, `TransportLayout`, `plan_transport_layout` (pure). They order ranks domain by domain, FFN first, and pick FFN prefix (identity order) or per-domain mode; other placements are rejected with the pods' shapes. - `deepep_m2n/connector.py`: the handshake result carries the transport layout. Per-domain lanes use a separate gloo group (`afd_deepep_m2n_transport`) in transport order and pass `ffn_ranks_per_domain` to `afdep.deepep.ElasticBuffer`. The handshake, model checks, control plane and expert ownership keep the role order, and FFN-prefix layouts take the identical path (same group, same arguments). - `deepep_m2n/network.py`: network plan in transport order. - `pyproject.toml` / `uv.lock`: `afdep-comm @ ...@v1.1.0` (lock resolves to `741b9aa`). - No vLLM extension points, compat shims or monkey patches are added; pods use vLLM's existing `--data-parallel-start-rank` flags. ## Test Plan - CPU gate: `pytest -m "not gpu and not vllm_runtime" tests/unit`. - Full unit on the vLLM v0.32.0 image: `pytest tests/unit` (new `test_deepep_m2n_layout.py`, extended `test_deepep_m2n_connector.py`). - GPU-gated: `tests/e2e/models/qwen3_moe/test_qwen3_moe_multi_pod.py`, M2N `1A1F,1A1F` and `2A0F,0A2F`, eager/graph/DBO; single-host `2A2F`. - Manual: `recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_graph.sh` and a `4A2F` baseline, full GSM8K and `vllm bench serve`. ## Test Result Revisions: vLLM v0.32.0, torch 2.13.0+cu130, afdEP-comm `fdff967` for the GPU runs (same source tree as `v1.1.0` / `741b9aa`), `v1.1.0` pin for unit. - CPU gate: 1452 tests, 1405 passed, 47 skipped, 0 failed. - Full unit: 1466 tests, 1418 passed, 48 skipped (NPU and cmake only), 0 failed; the 13 layout and 121 M2N connector tests all ran. - Multi-pod E2E, Qwen3-30B-A3B, 2 pods x 2 H200, GSM8K-100: `1A1F,1A1F` and `2A0F,0A2F` x eager/graph/DBO all pass (graph 0.92/0.96). - Single host `2A2F`, Qwen3-30B-A3B: 0.92/0.96. - Qwen3-235B-A22B-FP8, H200, full GSM8K (flexible/strict): `4A2F` one pod 0.9371/0.9325; `4A1F,4A1F` two pods 0.9401/0.9340. - Bench, random 1024/256, 64 requests per Attention rank: `4A2F` (6 GPUs) 3394 tok/s, TPOT 68 ms; `4A1F,4A1F` (10 GPUs) 4624 tok/s, TPOT 98 ms. ## Docs Impact - Files updated: `docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md` (placement rules and table), `docs/design/module/connector_contracts.md`, `recipe/gpu/DeepEPM2NAFDConnector/README.md`, `recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/README.md`, `tests/e2e/README.md`, `.agents/skills/deploy-afd-k8s/` (`POD=AFD_`). - If none, reason: n/a ---
Essential PR Checklist - [x] Purpose is clear and linked to public context when possible. - [x] Scope is bounded. - [x] Compatibility with vLLM v0.32.0 and, for NPU changes, the pinned vLLM-Ascend source is considered, as applicable. - [x] No changes are made to the vLLM source checkout. - [x] Plugin-owned classes or explicit dotted class paths are preferred over monkey patches. - [x] Any compat shim or monkey patch is isolated, idempotent, version-guarded, documented, and tested. (none added) - [x] Imports remain CPU-safe; CUDA-heavy work is delayed or GPU-gated. - [x] Validation evidence is included, including skipped GPU tests when applicable. - [x] Documentation impact is stated.
--------- Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- .agents/skills/deploy-afd-k8s/SKILL.md | 4 +- .../skills/deploy-afd-k8s/resources/deploy.md | 7 + .../resources/resolve-recipe.md | 30 +++- .../deploy-afd-k8s/resources/services.md | 4 + .../connectors/gpu/deepep_m2n/connector.py | 81 ++++++++-- .../connectors/gpu/deepep_m2n/layout.py | 146 +++++++++++++++++- .../connectors/gpu/deepep_m2n/network.py | 7 +- docs/design/module/connector_contracts.md | 2 +- docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md | 80 +++++++++- pyproject.toml | 2 +- recipe/gpu/DeepEPM2NAFDConnector/README.md | 13 +- .../qwen3_235b_a22b_fp8/README.md | 40 ++++- .../multipod_4a1f_4a1f_graph.sh | 82 ++++++++++ tests/e2e/README.md | 6 +- .../qwen3_moe/test_qwen3_moe_multi_pod.py | 22 ++- .../connectors/test_deepep_m2n_connector.py | 95 ++++++++++-- .../unit/connectors/test_deepep_m2n_layout.py | 136 ++++++++++++++++ tests/unit/fakes/afdep.py | 10 +- uv.lock | 4 +- 19 files changed, 709 insertions(+), 62 deletions(-) create mode 100644 recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_graph.sh create mode 100644 tests/unit/connectors/test_deepep_m2n_layout.py diff --git a/.agents/skills/deploy-afd-k8s/SKILL.md b/.agents/skills/deploy-afd-k8s/SKILL.md index 7600ffd06..8a528f701 100644 --- a/.agents/skills/deploy-afd-k8s/SKILL.md +++ b/.agents/skills/deploy-afd-k8s/SKILL.md @@ -29,7 +29,9 @@ than one pod) -- callers never talk to these directly. See ## Scope -- Recipes under `recipe/gpu/P2pNcclAFDConnector/**` (GPU only, not NPU), +- Recipes under `recipe/gpu/P2pNcclAFDConnector/**` or + `recipe/gpu/DeepEPM2NAFDConnector/**` (GPU only, not NPU; the M2N image + must contain afdEP-comm and DeepGEMM), **`prefill_decode_colocation` only** (attention + FFN workers, no prefill split). `prefill_decode_disaggregation` recipes are **out of scope**: they need a diff --git a/.agents/skills/deploy-afd-k8s/resources/deploy.md b/.agents/skills/deploy-afd-k8s/resources/deploy.md index a6842857a..4bffc08c5 100644 --- a/.agents/skills/deploy-afd-k8s/resources/deploy.md +++ b/.agents/skills/deploy-afd-k8s/resources/deploy.md @@ -146,6 +146,13 @@ deploy_pod vllm-pod-attention-1 2 worker none "POD=ATTENTION_1 ATTENTION_HEADLES deploy_pod vllm-pod-ffn-0 2 none head "POD=FFN_0" ``` +`POD=AFD_` recipe, e.g. the DeepEP M2N 2 x 4A1F plan (5 GPUs per pod): + +```bash +deploy_pod vllm-pod-afd-0 5 head head "POD=AFD_0" +deploy_pod vllm-pod-afd-1 5 worker worker "POD=AFD_1" +``` + ## 4f. Wait for Running, then for each pod's Ready condition Bounded 15-minute wait per pod for `phase=Running` (unschedulable pods never diff --git a/.agents/skills/deploy-afd-k8s/resources/resolve-recipe.md b/.agents/skills/deploy-afd-k8s/resources/resolve-recipe.md index c6c1f20f6..1a7f5f0bb 100644 --- a/.agents/skills/deploy-afd-k8s/resources/resolve-recipe.md +++ b/.agents/skills/deploy-afd-k8s/resources/resolve-recipe.md @@ -1,7 +1,8 @@ # 1. Resolve the recipe, its model, and its placement Take the recipe path from the user, or ask. It must be a colocation recipe -under `recipe/gpu/P2pNcclAFDConnector/**/prefill_decode_colocation/` (GPU +under `recipe/gpu/P2pNcclAFDConnector/**/prefill_decode_colocation/` or +`recipe/gpu/DeepEPM2NAFDConnector/**/prefill_decode_colocation/` (GPU only; prefill/decode disaggregation recipes are out of scope -- see `SKILL.md` Scope). @@ -94,6 +95,33 @@ For each entry, derive (used by [deploy.md](deploy.md) and (`ATTENTION` or `FFN`). Every pod sharing a role shares that literal, so read it off the script; don't derive it from ranks or DP sizes. +### Pods that run both roles: `POD=AFD_` + +DeepEP M2N multi-pod recipes (e.g. +`DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_graph.sh`) +list `POD=AFD_0`, `POD=AFD_1`, ... with no other per-pod overrides. Every +such pod runs one FFN and one Attention server and derives its DP flags +(headless, start rank) from its own index, so: + +- **`afd-attn-node-role`** and **`afd-ffn-node-role`**: both `head` for + `AFD_0`, both `worker` for every other `AFD_`. +- The pod shape comes from the script's `NUM_PODS`, `ATTN_PER_POD` and + `FFN_PER_POD` defaults (`NUM_PODS` must equal the number of entries). If + the user overrides them, pass the same values to every pod as container + env. +- **`GPU_COUNT`**: `ATTN_PER_POD + FFN_PER_POD`, the same for every pod. + These recipes build `CUDA_VISIBLE_DEVICES` with `$(seq ...)`, so the + literal-device rules above and below don't apply: + + ```bash + default() { grep -oE "$1:-[0-9]+" "$RECIPE_SCRIPT_PATH" | grep -oE '[0-9]+$'; } + GPU_COUNT=$(( ${ATTN_PER_POD:-$(default ATTN_PER_POD)} + ${FFN_PER_POD:-$(default FFN_PER_POD)} )) + ``` +- Ports are literals in the script, not `*_PORT:-N` defaults: the AFD + rendezvous port is the `"port": N` of the `afd` JSON, and the FFN and + Attention DP RPC ports are the last argument of the `role_dp "$F" ...` and + `role_dp "$A" ...` lines. + ## GPU_COUNT for a plain single-pod recipe The union of every literal `CUDA_VISIBLE_DEVICES=` in the script: diff --git a/.agents/skills/deploy-afd-k8s/resources/services.md b/.agents/skills/deploy-afd-k8s/resources/services.md index 3aa423a8d..761d23e65 100644 --- a/.agents/skills/deploy-afd-k8s/resources/services.md +++ b/.agents/skills/deploy-afd-k8s/resources/services.md @@ -39,6 +39,10 @@ AFD_CONNECTOR_PORT="$AFD_CONNECTOR_PORT" envsubst '${AFD_CONNECTOR_PORT}' \ Only when **more than one** plan entry's `POD` carries the same role (i.e. that role is split across pods, not just placed in its own single pod). +Every `POD=AFD_` entry carries both roles, so a plan of two or more +such entries needs both Services; read their ports (and the rendezvous port) +as [resolve-recipe.md](resolve-recipe.md) describes for those recipes rather +than with the `*_PORT:-N` greps below. Carries that role's DP-RPC coordination traffic to its head pod. One per split role, generic over role via `templates/service-dp-coord.yaml`: diff --git a/afd_plugin/connectors/gpu/deepep_m2n/connector.py b/afd_plugin/connectors/gpu/deepep_m2n/connector.py index c0d2fa917..725479d1a 100644 --- a/afd_plugin/connectors/gpu/deepep_m2n/connector.py +++ b/afd_plugin/connectors/gpu/deepep_m2n/connector.py @@ -9,12 +9,22 @@ weighted results back, where combine sums each token's top-k contributions. Topology: - One AFD world ordered ``[F0, F1, ..., A0, A1, ...]`` (as the P2P - connector). DeepEP sees it as a symmetric EP group in which FFN rank ``g`` + One AFD role world ordered ``[F0, F1, ..., A0, A1, ...]`` (as the P2P + connector) for the handshake, the model checks and the control plane. + DeepEP sees the ranks as a symmetric EP group in which FFN rank ``g`` owns real experts ``[g * E/F, (g + 1) * E/F)`` and Attention ranks own dummy expert slots (see ``layout.py``). Any Attention:FFN ratio works; ``num_experts`` must be divisible by the FFN rank count. +Placement: + Every NVLink domain (on HGX, every pod) holds the same number of ranks, + and either the FFN ranks fill the first domains or every domain holds the + same number of FFN ranks, with FFN DP ranks rising in pod order (2 x 4A1F + is two pods of one FFN and four Attention ranks). The handshake derives + the transport's physical rank order from the ranks' NVLink domains + (``plan_transport_layout``) and picks afdEP-comm's FFN-prefix or + per-domain mode; nothing is configured. + Data path (per MoE layer and stage, identical order on every rank): Attention: ``send_attn_output(h, ctx, topk_ids=, topk_weights=)`` dispatches; ``recv_ffn_output(ref, ubatch_idx)`` combines and returns @@ -88,7 +98,12 @@ DeepEPM2NExperts, ExpertBackend, ) -from afd_plugin.connectors.gpu.deepep_m2n.layout import DeepEPM2NLayout +from afd_plugin.connectors.gpu.deepep_m2n.layout import ( + DeepEPM2NLayout, + RankPlacement, + TransportLayout, + plan_transport_layout, +) from afd_plugin.connectors.gpu.deepep_m2n.network import ( NetworkConfig, NetworkPlan, @@ -143,6 +158,9 @@ # CPU (gloo) group for the handshake and DeepEP's communicator bootstrap, # created before any NCCL communicator so the network plan can still set NCCL_*. BOOTSTRAP_GROUP_NAME = "afd_deepep_m2n_bootstrap" +# CPU (gloo) group in transport order for the lanes, created only when that +# order differs from the role world (per-domain FFN placement). +TRANSPORT_GROUP_NAME = "afd_deepep_m2n_transport" # FastAFD passes 120 s for both the CPU wait and the GPU spin timeouts. DEFAULT_TIMEOUT_S = 120 # Only Attention rank 0 sends the per-step control payload. @@ -433,6 +451,7 @@ class HandshakeResult: attn_tp_size: int capacity_tokens_per_rank: int bucket_ladder: tuple[int, ...] + transport: TransportLayout def expert_weight_format(quant_config: QuantizationConfig | None) -> str: @@ -626,9 +645,26 @@ def validate_handshake( raise RuntimeError( f"Attention ranks disagree on bucket_ladder: {sorted(map(str, ladders))}" ) - # Every rank evaluates the same plan on the same list. + # Every rank evaluates the same plans on the same list. The network plan + # sees the ranks in the transport order afdEP-comm will see. + topologies = [RankTopology.from_mapping(info["topology"]) for info in infos] + transport = plan_transport_layout( + [ + RankPlacement(info["role"], info["role_rank"], topology.nvlink_domain) + for info, topology in zip(infos, topologies, strict=True) + ], + layout.ffn_size, + ) plan = plan_network( - [RankTopology.from_mapping(info["topology"]) for info in infos], network + [ + RankTopology( + world_rank=transport.transport_rank(info["role"], info["role_rank"]), + host=topology.host, + nvlink_domain=topology.nvlink_domain, + ) + for info, topology in zip(infos, topologies, strict=True) + ], + network, ) return HandshakeResult( expert_alignment=int(alignments.pop()), @@ -636,6 +672,7 @@ def validate_handshake( attn_tp_size=attn_tp_size, capacity_tokens_per_rank=int(capacities.pop()), bucket_ladder=ladders.pop() or (), + transport=transport, ) @@ -962,6 +999,9 @@ def __init__( self._timeout = timedelta(seconds=afd_config.afd_process_group_timeout_s) self.group: dist.ProcessGroup | None = None self.bootstrap_group: dist.ProcessGroup | None = None + # The lanes' group: the bootstrap group, or one in transport order. + self.transport_group: dist.ProcessGroup | None = None + self.transport: TransportLayout | None = None self.network_plan: NetworkPlan | None = None self.lanes: list[ElasticBuffer] = [] self.lane_streams: list[torch.cuda.Stream] = [] @@ -1041,8 +1081,25 @@ def _handshake(self) -> None: self.attn_tp_size = result.attn_tp_size self.capacity_tokens_per_rank = result.capacity_tokens_per_rank self.bucket_ladder = result.bucket_ladder + self.transport = result.transport + if self.transport.is_identity: + self.transport_group = self.bootstrap_group + else: + # Transport rank 0 is FFN rank 0, the process that hosts the + # rendezvous store, so this group joins the same store. + self.transport_group = init_afd_process_group( + backend="gloo", + init_method=self._rendezvous, + world_size=self.layout.union_size, + rank=self.transport.transport_rank( + self.afd_config.role, self.role_rank + ), + group_name=TRANSPORT_GROUP_NAME, + timeout=self._timeout, + ) if self.world_rank == 0: logger.info("DeepEP M2N network plan: %s", self.network_plan.summary()) + logger.info("DeepEP M2N transport: %s", self.transport.summary()) # DeepEP reads both when each buffer is created: the GIN switch, and # with several lanes a communication stream per buffer. apply_gin_environment(self.network_plan) @@ -1117,10 +1174,11 @@ def init_afd_connector( def _new_lane(self) -> ElasticBuffer: """One lane's afdEP-comm transport.""" - assert self.bootstrap_group is not None and self.network_plan is not None + assert self.transport_group is not None and self.transport is not None + assert self.network_plan is not None assert self.capacity_tokens_per_rank is not None return ElasticBuffer( - group=self.bootstrap_group, + group=self.transport_group, capacity_tokens_per_rank=self.capacity_tokens_per_rank, hidden_size=self.hidden_size, num_experts=self.layout.deepep_num_experts, @@ -1132,8 +1190,10 @@ def _new_lane(self) -> ElasticBuffer: rdma_traffic_class=self.network_plan.rdma_traffic_class, cpu_timeout_s=self.config.cpu_timeout_s, gpu_timeout_s=self.config.gpu_timeout_s, - # The DeepEP M2N layout puts FFN ranks first, as afdEP-comm expects. + # FFN ranks are transport ranks [0, F) (FFN prefix), or the first + # ``ffn_ranks_per_domain`` of every NVLink domain (per-domain mode). ffn_ranks=self.ffn_size, + ffn_ranks_per_domain=self.transport.ffn_ranks_per_domain, ) def close(self) -> None: @@ -1157,8 +1217,9 @@ def is_initialized(self) -> bool: @property def domain(self) -> tuple[int, int]: - """DeepEP ``(scale-out ranks, scale-up ranks)``: nodes and GPUs per node - in hybrid mode, ``(1, world)`` when everything is in one NVLink domain.""" + """DeepEP ``(scale-out ranks, scale-up ranks)``: NVLink domains and + ranks per domain in hybrid mode (2 x 4A1F: ``(2, 5)``), ``(1, world)`` + when everything is in one NVLink domain.""" self._check_initialized() return self.lanes[0].num_scaleout_ranks, self.lanes[0].num_scaleup_ranks diff --git a/afd_plugin/connectors/gpu/deepep_m2n/layout.py b/afd_plugin/connectors/gpu/deepep_m2n/layout.py index 24d1bde89..d8bb225d2 100644 --- a/afd_plugin/connectors/gpu/deepep_m2n/layout.py +++ b/afd_plugin/connectors/gpu/deepep_m2n/layout.py @@ -7,16 +7,28 @@ DeepEP treats the union of Attention and FFN ranks as one symmetric EP group in which every rank owns ``experts_per_rank`` experts. FastAFD orders the union ``[Attention.., FFN..]`` and remaps real expert ids by an offset. The -plugin orders AFD worlds FFN first (``[F0.., A0..]``, as the P2P connector -does), so FFN rank ``g`` owns DeepEP experts ``[g * E_local, (g + 1) * E_local)``, -which are exactly the real expert ids: no remap. Attention ranks own dummy -expert slots after the real ones that routing never selects. +plugin orders the AFD role world FFN first (``[F0.., A0..]``, as the P2P +connector does), so FFN rank ``g`` owns real experts +``[g * E_local, (g + 1) * E_local)``. Attention ranks own dummy expert slots +that routing never selects. + +The transport world (``plan_transport_layout``) holds the same ranks in +physical order: NVLink domain by domain, as DeepEP numbers ranks, with each +domain's FFN ranks first. When the FFN ranks fill the first domains, it equals +the role world and DeepEP expert ids are the real ones (afdEP-comm's FFN +prefix). When every domain holds ``f`` FFN ranks, afdEP-comm's per-domain mode +places FFN rank ``j`` at transport rank ``(j // f) * U + j % f`` (``U`` ranks +per domain) and translates the real expert ids itself. """ from __future__ import annotations +from collections.abc import Sequence from dataclasses import dataclass +FFN_ROLE = "ffn" +ATTENTION_ROLE = "attention" + @dataclass(frozen=True) class DeepEPM2NLayout: @@ -50,10 +62,11 @@ def deepep_num_experts(self) -> int: return self.union_size * self.experts_per_rank def world_rank(self, role: str, role_rank: int) -> int: - if role == "ffn": + """Rank in the role world ``[F0.., A0..]``.""" + if role == FFN_ROLE: _check_rank("ffn role rank", role_rank, self.ffn_size) return role_rank - if role == "attention": + if role == ATTENTION_ROLE: _check_rank("attention role rank", role_rank, self.attn_size) return self.ffn_size + role_rank raise ValueError(f"unknown AFD role {role!r}") @@ -68,3 +81,124 @@ def expert_range(self, ffn_rank: int) -> range: def _check_rank(name: str, rank: int, size: int) -> None: if not 0 <= rank < size: raise ValueError(f"{name} must be in [0, {size}), got {rank}") + + +@dataclass(frozen=True) +class RankPlacement: + """Where one rank runs, as it reports in the handshake.""" + + role: str + role_rank: int + nvlink_domain: str + + +@dataclass(frozen=True) +class TransportLayout: + """The physical rank order afdEP-comm's transport sees. + + Attributes: + order: ``(role, role_rank)`` of each transport rank. + ffn_ranks_per_domain: afdEP-comm's ``ffn_ranks_per_domain``: 0 when the + FFN ranks are transport ranks ``[0, F)`` (FFN prefix), else the FFN + rank count of every NVLink domain (per-domain mode). + num_domains: NVLink domains. + domain_size: ranks per NVLink domain. + shape: ``AF`` per domain in transport order, for logs and errors. + """ + + order: tuple[tuple[str, int], ...] + ffn_ranks_per_domain: int + num_domains: int + domain_size: int + shape: str + + def transport_rank(self, role: str, role_rank: int) -> int: + return self.order.index((role, role_rank)) + + @property + def is_identity(self) -> bool: + """The transport order is the role world ``[F0.., A0..]``.""" + ffn_size = sum(role == FFN_ROLE for role, _ in self.order) + return all( + rank == (role_rank if role == FFN_ROLE else ffn_size + role_rank) + for rank, (role, role_rank) in enumerate(self.order) + ) + + @property + def mode(self) -> str: + return "per_domain" if self.ffn_ranks_per_domain else "ffn_prefix" + + def summary(self) -> str: + per_domain = ( + f" f={self.ffn_ranks_per_domain}" if self.ffn_ranks_per_domain else "" + ) + return ( + f"{self.mode}{per_domain} " + f"domains={self.num_domains}x{self.domain_size} placement={self.shape}" + ) + + +def plan_transport_layout( + placements: Sequence[RankPlacement], ffn_size: int +) -> TransportLayout: + """Order the ranks for afdEP-comm; every rank computes the same layout. + + Each NVLink domain is a contiguous block (DeepEP numbers ranks domain by + domain) with its FFN ranks first. Domains follow their lowest role-world + rank, so the domains holding FFN ranks come first. afdEP-comm takes two + shapes: the FFN ranks fill the first domains (FFN prefix: one domain, or + role-split pods), or every domain holds the same number of FFN ranks, in + pod order (per-domain mode). Every domain must hold the same number of + ranks. + """ + + def role_world_rank(placement: RankPlacement) -> int: + if placement.role == FFN_ROLE: + return placement.role_rank + return ffn_size + placement.role_rank + + domains: dict[str, list[RankPlacement]] = {} + for placement in placements: + domains.setdefault(placement.nvlink_domain, []).append(placement) + ordered = sorted( + domains.values(), key=lambda ranks: min(map(role_world_rank, ranks)) + ) + order: list[tuple[str, int]] = [] + shapes: list[str] = [] + ffn_counts: set[int] = set() + for ranks in ordered: + ffn = sorted(p.role_rank for p in ranks if p.role == FFN_ROLE) + attention = sorted(p.role_rank for p in ranks if p.role != FFN_ROLE) + order += [(FFN_ROLE, rank) for rank in ffn] + order += [(ATTENTION_ROLE, rank) for rank in attention] + shapes.append(f"{len(attention)}A{len(ffn)}F") + ffn_counts.add(len(ffn)) + shape = ",".join(shapes) + domain_sizes = {len(ranks) for ranks in ordered} + if len(domain_sizes) != 1: + raise RuntimeError( + f"NVLink domains hold {shape}: DeepEP needs the same number of ranks " + "in every NVLink domain (pod)" + ) + ffn_order = [rank for role, rank in order if role == FFN_ROLE] + if ffn_order != list(range(ffn_size)): + raise RuntimeError( + f"NVLink domains ({shape}) hold FFN ranks in the order {ffn_order}: " + "FFN DP ranks must rise in pod order (--data-parallel-start-rank)" + ) + if order[:ffn_size] == [(FFN_ROLE, rank) for rank in range(ffn_size)]: + ffn_ranks_per_domain = 0 + elif len(ffn_counts) == 1: + ffn_ranks_per_domain = ffn_counts.pop() + else: + raise RuntimeError( + f"NVLink domains hold {shape}: every domain (pod) must hold the same " + "number of FFN ranks, or the FFN ranks must fill the first domains" + ) + return TransportLayout( + order=tuple(order), + ffn_ranks_per_domain=ffn_ranks_per_domain, + num_domains=len(ordered), + domain_size=domain_sizes.pop(), + shape=shape, + ) diff --git a/afd_plugin/connectors/gpu/deepep_m2n/network.py b/afd_plugin/connectors/gpu/deepep_m2n/network.py index 715a49ed6..321f7d811 100644 --- a/afd_plugin/connectors/gpu/deepep_m2n/network.py +++ b/afd_plugin/connectors/gpu/deepep_m2n/network.py @@ -190,7 +190,12 @@ def plan_network(topologies: list[RankTopology], config: NetworkConfig) -> Netwo def _domain_blocks(topologies: list[RankTopology]) -> list[list[int]]: - """Ranks grouped by NVLink domain; DeepEP needs equal, contiguous blocks.""" + """Ranks grouped by NVLink domain; DeepEP needs equal, contiguous blocks. + + The connector passes the ranks in transport order + (``plan_transport_layout``), which makes the blocks contiguous; this + checks that order. + """ blocks: list[tuple[str, list[int]]] = [] for topology in topologies: if blocks and blocks[-1][0] == topology.nvlink_domain: diff --git a/docs/design/module/connector_contracts.md b/docs/design/module/connector_contracts.md index dcaf453bc..f9624ba8e 100644 --- a/docs/design/module/connector_contracts.md +++ b/docs/design/module/connector_contracts.md @@ -312,7 +312,7 @@ metadata is present. | Connector | Connector-owned resources | Topology constraints and mapping | | --- | --- | --- | | CUDA P2P | AFD process group, PyNccl data communicators, separate NCCL metadata group, compiled custom-op communicator registry, and graph-oriented receive buffers/state. | Requires `A >= F`. One FFN rank is grouped with a consecutive block of Attention ranks in FFN-first ordering; the blocks differ in size by at most one, and hold `A/F` ranks each when `F` divides `A`. | -| CUDA DeepEP M2N | AFD process group, a CPU bootstrap group for the handshake, the connector's NCCL control group, afdEP-comm transport buffers (one per lane), and compiled dispatch/combine kernels per bucket. | FFN-first ordering, any `A/F`, `num_experts % F == 0`, FFN TP 1. Across nodes every NVLink domain must hold an equal, contiguous block of world ranks. The startup handshake checks payload format, model shapes, lanes, network settings, SM90+, and the FFN expert deployment on every rank before buffers are allocated. | +| CUDA DeepEP M2N | AFD process group, a CPU bootstrap group for the handshake, a CPU transport group when the physical rank order differs from the role order, the connector's NCCL control group, afdEP-comm transport buffers (one per lane), and compiled dispatch/combine kernels per bucket. | FFN-first role ordering, any `A/F`, `num_experts % F == 0`, FFN TP 1. Every NVLink domain must hold the same number of ranks, and either the FFN ranks fill the first domains (afdEP-comm FFN prefix) or every domain holds the same number of FFN ranks (afdEP-comm per-domain mode); the transport sees the ranks domain by domain, FFN first in each. The startup handshake checks payload format, model shapes, lanes, network settings, SM90+, and the FFN expert deployment on every rank before buffers are allocated. | | Ascend CAMP2P | AFD process group, one HCCL communication group per ubatch, FFN HCCL state, Gloo metadata group, custom-op state and transfer handles. | FFN-first ordering and `A >= F`; group construction derives each FFN/Attention mapping. | | Ascend CAM async | Attention-first HCCL group, plugin-owned routed-only CAM operator state, per-stage pending Attention payload queues, and connector work-item state. | Role ranks map into a combined Attention-first world; Compact metadata determines the actual layer and expert interval; counts.sum() gives this chunk’s routed token count. | diff --git a/docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md b/docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md index f5cbd55fb..cc9aad6c3 100644 --- a/docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md +++ b/docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md @@ -49,6 +49,16 @@ owns real experts `[g * E/F, (g + 1) * E/F)`, so `num_experts` must be divisible by `F`; Attention ranks own dummy expert slots that routing never selects. Any Attention:FFN ratio works. +The handshake, the model checks and the control plane use this order. The +afdEP-comm transport sees the same ranks in physical order instead: NVLink +domain by domain (DeepEP numbers ranks that way), with each domain's FFN ranks +first. When the FFN ranks fill the first domains (one node, or FFN and +Attention on separate pods) the two orders are equal. When every domain holds +the same number of FFN ranks (`4A1F,4A1F`), the connector runs afdEP-comm's +per-domain mode, which finds the FFN ranks at the start of every domain. The +connector derives the order from where the ranks run; see +[Supported layouts](#supported-layouts). + ### Data path For every MoE layer and stage, in the same order on every rank: @@ -76,8 +86,9 @@ together if they disagree. The handshake checks the payload format, model shapes, lane count (DBO), `network`, bucket settings, GPU architecture (SM90 or newer), and the FFN deployment (expert parallel, EP size, FFN TP=1, linear expert placement, no EPLB, all2all backend and expert -weight format). The FFN adopts the Attention TP size, dispatch capacity and -bucket ladder reported by the Attention ranks. +weight format), and the placement of the ranks over NVLink domains. The FFN +adopts the Attention TP size, dispatch capacity and bucket ladder reported by +the Attention ranks. ## Installation @@ -90,7 +101,7 @@ The connector needs three components in addition to the plugin and vLLM: ```bash python -m pip uninstall -y afdep python -m pip install --no-deps \ - "afdep-comm @ git+https://github.com/afdEP/afdEP-comm.git@" + "afdep-comm @ git+https://github.com/afdEP/afdEP-comm.git@" python -m afdep doctor --build ``` @@ -226,7 +237,64 @@ that sets `capacity_tokens_per_rank` explicitly must agree with Attention. | Attention DP x TP | Supported, including TP > 1: each TP rank dispatches its slice of the DP rank's tokens and the TP group all-gathers the combined rows. | | FFN DP x TP1 | Required. FFN experts are not TP-sharded. | | One node | Supported; ranks reach each other over NVLink. | -| Several nodes | Supported when every node (NVLink domain) holds an equal, contiguous block of world ranks, with FFN ranks first. Writing each node's ranks as `AF`, the layouts `2A0F,0A2F` (Attention on one node, FFN on another) and `4A0F,0A4F` are valid; `4A1F,4A1F` (FFN ranks not contiguous) and `4A0F,0A2F` (unequal blocks) are not. The fabric must route between the NICs NCCL picks. | +| Several nodes or pods | Supported for the placements below. The fabric must route between the NICs NCCL picks. | + +Write each pod's ranks as `AF`, in pod order (on HGX every pod is one +NVLink domain; on NVL72, pods in one NVLink clique form one domain). A +placement is valid when: + +1. every pod holds the same number of GPUs; +2. either the FFN ranks fill the first pods, or every pod holds the same + number of FFN ranks; +3. FFN DP ranks rise in pod order, and so do Attention DP ranks; +4. `num_experts % F == 0`, and Attention TP stays inside a pod. + +Inside a pod, put the FFN ranks on the same local GPUs in every pod +(the recipes use the first ones): DeepEP's hybrid mode pairs ranks across pods +by their position in the pod. + +| Placement | Accepted | afdEP-comm mode | +| --- | --- | --- | +| `4A2F` (one pod) | yes | FFN prefix | +| `2A0F,0A2F`, `0A4F,4A0F` | yes | FFN prefix | +| `3A2F,5A0F` | yes | FFN prefix (one mixed pod at the boundary) | +| `0A1F,0A1F,1A0F,1A0F` | yes | FFN prefix, pods of one GPU | +| `4A1F,4A1F`, `1A1F,1A1F`, `6A2F,6A2F` | yes | per-domain, `f` = FFN ranks per pod | +| `4A1F,3A2F` | no | the FFN count varies from pod to pod | +| `0A1F,0A1F,2A0F,2A0F`, `0A2F,4A0F` | no | pods of different sizes | + +The rejected placements usually have an equivalent with equal pods: for +example `0A2F,2A0F,2A0F` instead of `0A2F,4A0F`. There is nothing to +configure; the handshake rejects other placements with the pods' shapes in +the message. + +## Multi-pod deployment + +Each pod starts one `vllm serve` per role it holds, with the AFD config +(`num_attention_ranks = A`, `num_ffn_ranks = F`, `host:port` = the pod that +holds FFN rank 0) the same on every rank. For each role, with `S` = the role's +DP size, `L` = this pod's local DP size and `start` = the sum of the local +sizes of the pods before it: + +| Flag | First pod holding the role | Other pods holding the role | +| --- | --- | --- | +| `--data-parallel-size` | `S` | `S` | +| `--data-parallel-size-local` | `L` | `L` | +| `--data-parallel-address` / `--data-parallel-rpc-port` | its own address | the first pod's address | +| `--data-parallel-start-rank` | omitted | `start` | +| `--headless` | no | yes | + +When one pod holds the whole role, pass only `--data-parallel-size`. Do not +pass `--data-parallel-start-rank 0` on the first pod: vLLM reads an explicit +start rank on a non-headless engine as hybrid load balancing and then rejects +the headless pods. The FFN DP rank is the FFN's EP rank, so it decides which +experts a pod loads; `--enable-ep-weight-filter` keeps each FFN from reading +the other experts' weights. + +The recipe +[`multipod_4a1f_4a1f_graph.sh`](../../recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/README.md#multi-pod-2-x-4a1f) +derives all of these flags for symmetric placements `N x AF` from the +pod index (`POD=AFD_`). ## Minimal launch shape @@ -273,7 +341,9 @@ and FP8 examples, see the - CPU unit tests: [`tests/unit/connectors/test_deepep_m2n_connector.py`](../../tests/unit/connectors/test_deepep_m2n_connector.py) - runs against a fake afdEP module when afdEP-comm is not installed. + runs against a fake afdEP module when afdEP-comm is not installed; + [`test_deepep_m2n_layout.py`](../../tests/unit/connectors/test_deepep_m2n_layout.py) + checks the placement table above. - GPU connector tests: [`tests/gpu`](../../tests/gpu/README.md) checks dispatch, DeepGEMM experts and combine against references on one node. - Model E2E: the Qwen3 MoE cases `afd-deepep-m2n-*-2a2f` in diff --git a/pyproject.toml b/pyproject.toml index 4f1d7518a..2687ed5de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [] [project.optional-dependencies] vllm = ["vllm==0.32.0"] afdep = [ - "afdep-comm @ git+https://github.com/afdEP/afdEP-comm.git@2b74d574e9d25cc794de8d2359f815fcfb48955a", + "afdep-comm @ git+https://github.com/afdEP/afdEP-comm.git@v1.1.0", ] [project.urls] diff --git a/recipe/gpu/DeepEPM2NAFDConnector/README.md b/recipe/gpu/DeepEPM2NAFDConnector/README.md index 5151e899a..bfbb024dd 100644 --- a/recipe/gpu/DeepEPM2NAFDConnector/README.md +++ b/recipe/gpu/DeepEPM2NAFDConnector/README.md @@ -19,7 +19,8 @@ supported layouts. │ └── prefill_decode_colocation/ # 2A2F: eager, graph, graph + DBO, Attention TP2 └── qwen3_235b_a22b_fp8/ ├── README.md - └── prefill_decode_colocation/ # 4A2F FP8 payload: graph, graph + DBO + └── prefill_decode_colocation/ # 4A2F FP8 payload: graph, graph + DBO; + # 2 pods x 4A1F (multipod_4a1f_4a1f_graph.sh) ``` Each model directory holds its own scripts; open its `README.md` before @@ -64,6 +65,14 @@ slower than later starts. ## Deploying on Kubernetes -The recipes have no placement header, so the +The single-node recipes have no placement header, so the **`deploy-afd-k8s`** skill (`.agents/skills/deploy-afd-k8s/SKILL.md`) deploys each one as a single pod. The image must contain afdEP-comm and DeepGEMM. + +`multipod_4a1f_4a1f_graph.sh` carries a `POD=AFD_` header, so the skill +deploys it as one pod per entry, each running both roles. Every pod gets the +same `NUM_PODS`, `ATTN_PER_POD` and `FFN_PER_POD` (2, 4 and 1 by default), +which select any symmetric layout `N x AF`; the FFN uses each pod's +first `f` GPUs. See +[Multi-pod deployment](../../../docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md#multi-pod-deployment) +for the placements the connector accepts. diff --git a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/README.md b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/README.md index 5b5ec1e74..c9bd1a761 100644 --- a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/README.md +++ b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/README.md @@ -5,8 +5,9 @@ Launch scripts for Qwen3-235B-A22B-FP8 on `DeepEPM2NAFDConnector` with vLLM ## Prerequisites -- 6 SM90 or newer GPUs on one node with enough memory for the model. This - configuration was run on H200. +- 6 SM90 or newer GPUs on one node with enough memory for the model, or two + pods of 5 such GPUs on an RDMA fabric for the multi-pod recipe. These + configurations were run on H200. - The environment described in the [connector recipes README](../README.md#prerequisites). - Qwen3-235B-A22B-FP8 weights on disk (block-128 FP8). The scripts default to @@ -21,10 +22,41 @@ owns half of the 128 experts. ```text prefill_decode_colocation/ -├── 4a2f_graph_dp4tp1.sh # 256 sequences per Attention rank, FULL_DECODE_ONLY graph -└── 4a2f_graph_dbo_dp4tp1.sh # 512 sequences per Attention rank, graph + DBO +├── 4a2f_graph_dp4tp1.sh # 256 sequences per Attention rank, FULL_DECODE_ONLY graph +├── 4a2f_graph_dbo_dp4tp1.sh # 512 sequences per Attention rank, graph + DBO +└── multipod_4a1f_4a1f_graph.sh # 2 pods x 4A1F, FULL_DECODE_ONLY graph (DBO optional) ``` +### Multi-pod: 2 x 4A1F + +`multipod_4a1f_4a1f_graph.sh` runs on two pods of 5 GPUs. Each pod runs one +FFN rank on GPU 0 and four Attention ranks on GPUs 1-4 (Attention DP8 x TP1, +FFN DP2/EP2 in total), so every FFN GPU holds half of the experts, as in 4A2F. +Pod 0 leads both DP groups and hosts the AFD rendezvous; pod 1 starts both +servers headless and serves no requests. Send traffic to pod 0's port `18000`. + +Only `POD` differs between pods: + +```bash +# Pod 0 +POD=AFD_0 bash recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_graph.sh +# Pod 1 +POD=AFD_1 bash recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_graph.sh +``` + +On Kubernetes the script reaches pod 0 through the Services that the +`deploy-afd-k8s` skill creates. Without Kubernetes, set `HEAD_HOST` to pod 0's +address on both pods; pod 0 must accept ports `6239`, `13345` (Attention DP) +and `13355` (FFN DP). Pass the fabric's NCCL settings (for example +`NCCL_IB_HCA`) through the environment. + +`NUM_PODS`, `ATTN_PER_POD` and `FFN_PER_POD` select another symmetric layout +(`2 x 6A2F`, ...); the tuning values (`MAX_NUM_SEQS`, `MAX_NUM_BATCHED_TOKENS`, +`MAX_MODEL_LEN`, `CAPTURE_SIZES`, `PAYLOAD_FORMAT`) and DBO (`ENABLE_DBO=1`, +`DBO_DECODE_TOKEN_THRESHOLD`, `DBO_PREFILL_TOKEN_THRESHOLD`) are environment +variables with the 4A2F values as defaults. A placement the connector cannot +carry fails at startup with the pods' shapes in the error. + ## Configuration notes - `connector_extra_config` sets `payload_format: "fp8_block128_fp32"`. The diff --git a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_graph.sh b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_graph.sh new file mode 100644 index 000000000..ed48f9b2a --- /dev/null +++ b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_graph.sh @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project + +# Qwen3-235B-A22B-FP8 with DeepEPM2NAFDConnector, 2 x 4A1F, FULL_DECODE_ONLY +# CUDA graphs: 2 pods of 5 GPUs, each with FFN on GPU 0 and Attention DP4 x TP1 +# on GPUs 1-4 (Attention DP8, FFN DP2/EP2 in total). Send traffic to port +# 18000 of the POD=AFD_0 pod. +# +# Use .agents/skills/deploy-afd-k8s/SKILL.md to deploy this recipe +# on 2 pods using the setup below. +# For pod 1: POD=AFD_0 +# For pod 2: POD=AFD_1 +# +# NUM_PODS x (ATTN_PER_POD Attention + FFN_PER_POD FFN) sets any symmetric +# layout (2 x 1A1F, 3 x 6A2F, ...); give every pod the same values. HEAD_HOST +# points the AFD rendezvous and both DP leaders at pod 0 (runs without k8s). + +MODEL_PATH=${MODEL_PATH:-/path/model_weights/Qwen3-235B-A22B-FP8} +NUM_PODS=${NUM_PODS:-2} +ATTN_PER_POD=${ATTN_PER_POD:-4} +FFN_PER_POD=${FFN_PER_POD:-1} +PAYLOAD_FORMAT=${PAYLOAD_FORMAT:-fp8_block128_fp32} +MAX_NUM_SEQS=${MAX_NUM_SEQS:-256} # per Attention DP rank +MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-3072} +MAX_MODEL_LEN=${MAX_MODEL_LEN:-4096} +CAPTURE_SIZES=${CAPTURE_SIZES:-8,16,32,64,96,128,192,256} # ascending +ENABLE_DBO=${ENABLE_DBO:-0} +DBO_DECODE_TOKEN_THRESHOLD=${DBO_DECODE_TOKEN_THRESHOLD:-2} +DBO_PREFILL_TOKEN_THRESHOLD=${DBO_PREFILL_TOKEN_THRESHOLD:-2} +export VLLM_USE_V2_MODEL_RUNNER=1 + +K=${POD#AFD_} A=$ATTN_PER_POD F=$FFN_PER_POD + +# role_dp PER_POD ADDRESS RPC_PORT -> DP: this pod's DP flags for one role. +# Pod 0 leads and omits its start rank: vLLM reads an explicit start rank on a +# non-headless engine as hybrid load balancing and rejects the headless peers. +role_dp() { + DP=(--data-parallel-size $((NUM_PODS * $1)) --data-parallel-size-local "$1" + --data-parallel-address "$2" --data-parallel-rpc-port "$3") + if ((K > 0)); then DP+=(--headless --data-parallel-start-rank $((K * $1))); fi +} + +afd() { + cat < ffn.log 2>&1 & + +role_dp "$A" "${HEAD_HOST:-vllm-attn-dp-service}" 13345 +CUDA_VISIBLE_DEVICES=$(seq "$F" $((F + A - 1)) | paste -sd, -) vllm serve "$MODEL_PATH" \ + --additional-config "$(afd attention)" "${COMMON[@]}" "${DP[@]}" \ + --port 18000 > attn.log 2>&1 & + +wait diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 96095fdfb..e3b779622 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -177,8 +177,10 @@ python -m pytest -q -s \ ``` `tests/e2e/models/qwen3_moe/test_qwen3_moe_multi_pod.py` runs the eager, -graph and graph + DBO cases with Attention and FFN in two pods, one role per -pod, so dispatch and combine cross nodes over NCCL GIN. The pods must be +graph and graph + DBO cases on two pods in two layouts: one role per pod +(`2A0F,0A2F`) and one Attention and one FFN rank per pod (`1A1F,1A1F`, which +runs afdEP-comm's per-domain mode). Dispatch and combine cross nodes over NCCL +GIN. The pods must be provisioned first; see [`.agents/skills/run-e2e/resources/k8-multi-pod.md`](../../.agents/skills/run-e2e/resources/k8-multi-pod.md) and the [multi-pod runner walkthrough](multi_pod/README.md). diff --git a/tests/e2e/models/qwen3_moe/test_qwen3_moe_multi_pod.py b/tests/e2e/models/qwen3_moe/test_qwen3_moe_multi_pod.py index 8b7511900..5918c0669 100644 --- a/tests/e2e/models/qwen3_moe/test_qwen3_moe_multi_pod.py +++ b/tests/e2e/models/qwen3_moe/test_qwen3_moe_multi_pod.py @@ -10,11 +10,12 @@ runner itself; this only supplies what the runner cannot infer from its own pod's environment. -DeepEP M2N needs FFN ranks first (afdEP-comm) and every NVLink domain to hold -an equal, contiguous block of world ranks (DeepEP), so the two pods split by -role with equal rank counts: Attention on one pod, FFN on the other. The -ranks then span two NVLink domains, so dispatch and combine cross pods over -NCCL GIN (RDMA). +DeepEP M2N needs every NVLink domain (pod) to hold the same number of ranks, +with the FFN ranks either filling the first pods or the same count in every +pod. The cases cover both afdEP-comm modes: the role split (Attention on one +pod, FFN on the other: FFN prefix) and the interleaved layout (one Attention +and one FFN rank per pod: per-domain mode). The ranks span two NVLink domains, +so dispatch and combine cross pods over NCCL GIN (RDMA). """ from __future__ import annotations @@ -43,11 +44,16 @@ # evidence stays comparable with the single-host rows. POD_LAYOUTS = { "2pod-role-split": "2A0F,0A2F", + "2pod-interleaved": "1A1F,1A1F", } MULTI_POD_CASES = [ - ("afd-deepep-m2n-eager-2a2f", "2pod-role-split"), - ("afd-deepep-m2n-graph-2a2f", "2pod-role-split"), - ("afd-deepep-m2n-graph-dbo-2a2f", "2pod-role-split"), + (scenario, layout) + for layout in POD_LAYOUTS + for scenario in ( + "afd-deepep-m2n-eager-2a2f", + "afd-deepep-m2n-graph-2a2f", + "afd-deepep-m2n-graph-dbo-2a2f", + ) ] diff --git a/tests/unit/connectors/test_deepep_m2n_connector.py b/tests/unit/connectors/test_deepep_m2n_connector.py index 7e8c3cc4a..8ef14c143 100644 --- a/tests/unit/connectors/test_deepep_m2n_connector.py +++ b/tests/unit/connectors/test_deepep_m2n_connector.py @@ -53,7 +53,11 @@ dequantize_fp8_block128, quantize_fp8_block128, ) -from afd_plugin.connectors.gpu.deepep_m2n.layout import DeepEPM2NLayout # noqa: E402 +from afd_plugin.connectors.gpu.deepep_m2n.layout import ( # noqa: E402 + DeepEPM2NLayout, + RankPlacement, + plan_transport_layout, +) from afd_plugin.connectors.gpu.deepep_m2n.network import ( # noqa: E402 NetworkConfig, RankTopology, @@ -577,9 +581,11 @@ def test_handshake_rejects_mismatches(overrides, message): def test_handshake_plans_gin_across_nvlink_domains(): - """Ranks on hosts ``aabb`` give a plan of 2 domains of 2 ranks with GIN. + """Ranks on hosts ``aabb`` (role-split pods) give a plan of 2 domains of 2 + ranks with GIN, and keep the role world as the transport order. - Why: the handshake must hand the network plan the right domains. + Why: the handshake must hand the network plan the right domains, and the + role-split layouts that run today must take the unchanged transport. """ layout = DeepEPM2NLayout(ffn_size=2, attn_size=2, num_experts=128) @@ -592,18 +598,54 @@ def test_handshake_plans_gin_across_nvlink_domains(): 2, 2, ) + assert result.transport.is_identity + + +def test_handshake_gives_interleaved_pods_the_per_domain_transport(): + """FFN 0 and FFN 1 on hosts ``a`` and ``b``, each with one Attention rank + (``1A1F,1A1F``), give a per-domain transport ``[F0, A0, F1, A1]`` over 2 + domains of 2 ranks. + + Why: afdEP-comm's lanes must see the ranks domain by domain with each + domain's FFN rank first; the network plan is checked in that order. + """ + layout = DeepEPM2NLayout(ffn_size=2, attn_size=2, num_experts=128) + + result = validate_handshake( + _handshake(layout, hosts="abab"), layout, NetworkConfig() + ) + + assert result.transport.order == ( + ("ffn", 0), + ("attention", 0), + ("ffn", 1), + ("attention", 1), + ) + assert result.transport.ffn_ranks_per_domain == 1 + assert (result.network_plan.num_domains, result.network_plan.ranks_per_domain) == ( + 2, + 2, + ) @pytest.mark.parametrize( - ("hosts", "message"), - [("aaab", "different rank counts"), ("abab", "not contiguous")], + ("ffn_size", "attn_size", "hosts", "message"), + [ + (2, 2, "aaab", "same number of ranks"), + # Host a holds F0, A0, A1 and host b F1, F2, A2. + (3, 3, "abbaab", "same number of FFN ranks"), + ], ) -def test_handshake_rejects_unsupported_domain_layouts(hosts, message): - """Hosts ``aaab`` (unequal counts) and ``abab`` (not contiguous) raise. +def test_handshake_rejects_unsupported_domain_layouts( + ffn_size, attn_size, hosts, message +): + """Hosts with unequal rank counts, or with FFN counts that differ while + the FFN ranks do not fill the first hosts, raise. - Why: DeepEP's hybrid mode needs equal, contiguous ranks per domain. + Why: DeepEP needs equal NVLink domains, and afdEP-comm finds FFN ranks + only as a prefix or as the same count in every domain. """ - layout = DeepEPM2NLayout(ffn_size=2, attn_size=2, num_experts=128) + layout = DeepEPM2NLayout(ffn_size=ffn_size, attn_size=attn_size, num_experts=96) with pytest.raises(RuntimeError, match=message): validate_handshake(_handshake(layout, hosts=hosts), layout, NetworkConfig()) @@ -1166,27 +1208,48 @@ def test_attention_init_rejects_local_experts_before_communicating(): assert connector.bootstrap_group is None -@pytest.mark.parametrize(("role", "role_rank"), [("attention", 2), ("ffn", 1)]) +@pytest.mark.parametrize( + ("role", "role_rank", "hosts", "ffn_ranks_per_domain"), + [ + ("attention", 2, "aaaaaa", 0), + ("ffn", 1, "aaaaaa", 0), + # Hosts of F0 F1 A0 A1 A2 A3: 2A1F,2A1F. + ("ffn", 1, "ababab", 1), + ("attention", 3, "ababab", 1), + ], +) def test_lane_transport_is_afdep_comm_with_the_ffn_rank_count( - monkeypatch, role, role_rank + monkeypatch, role, role_rank, hosts, ffn_ranks_per_domain ): """Each lane's transport is afdEP-comm's ``ElasticBuffer``, built with every - argument its constructor requires and the FFN rank count. + argument its constructor requires, the FFN rank count and the transport's + ``ffn_ranks_per_domain``, on the transport group. Why: ``ffn_ranks`` selects afdEP-comm's AFD transport (without it the - buffer is unspecialized DeepEP V2), and the stand-in fails on any argument - set the real constructor would refuse. + buffer is unspecialized DeepEP V2), ``ffn_ranks_per_domain`` tells it + where the FFN ranks sit, and the stand-in fails on any argument set the + real constructor would refuse. """ monkeypatch.setattr(connector_module, "ElasticBuffer", FakeElasticBuffer) - connector = _connector(role, role_rank, M2NTopology(ffn_size=2, attn_size=3), {}) - connector.bootstrap_group = SimpleNamespace() + topology = M2NTopology(ffn_size=2, attn_size=4) + connector = _connector(role, role_rank, topology, {}) + connector.transport_group = SimpleNamespace() + connector.transport = plan_transport_layout( + [ + RankPlacement(*topology.role(rank), f"host:{host}") + for rank, host in enumerate(hosts) + ], + ffn_size=topology.ffn_size, + ) connector.network_plan = SimpleNamespace(hybrid=True, rdma_traffic_class=0) connector.capacity_tokens_per_rank = 64 buffer = connector._new_lane() assert isinstance(buffer, FakeElasticBuffer) + assert buffer.kwargs["group"] is connector.transport_group assert buffer.kwargs["ffn_ranks"] == 2 + assert buffer.kwargs["ffn_ranks_per_domain"] == ffn_ranks_per_domain assert buffer.kwargs["capacity_tokens_per_rank"] == 64 assert buffer.kwargs["allow_hybrid_mode"] is True diff --git a/tests/unit/connectors/test_deepep_m2n_layout.py b/tests/unit/connectors/test_deepep_m2n_layout.py new file mode 100644 index 000000000..328f1177a --- /dev/null +++ b/tests/unit/connectors/test_deepep_m2n_layout.py @@ -0,0 +1,136 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""CPU tests for the DeepEP M2N transport planner (``plan_transport_layout``).""" + +from __future__ import annotations + +import pytest + +pytest.importorskip("torch") +pytest.importorskip("vllm") + +from afd_plugin.connectors.gpu.deepep_m2n.layout import ( # noqa: E402 + RankPlacement, + TransportLayout, + plan_transport_layout, +) + + +def _pods(placement: str) -> tuple[list[RankPlacement], int]: + """Ranks of ``AF,...`` with DP ranks rising in pod order. + + Each pod is one NVLink domain and lists its Attention ranks first, the way + a pod that starts Attention on its first GPUs reports them. + """ + ranks: list[RankPlacement] = [] + next_rank = {"attention": 0, "ffn": 0} + for pod, entry in enumerate(placement.split(",")): + attention, ffn = (int(n) for n in entry.removesuffix("F").split("A")) + for role, count in (("attention", attention), ("ffn", ffn)): + for _ in range(count): + ranks.append(RankPlacement(role, next_rank[role], f"host:pod{pod}")) + next_rank[role] += 1 + return ranks, next_rank["ffn"] + + +def _plan(placement: str) -> TransportLayout: + ranks, ffn_size = _pods(placement) + return plan_transport_layout(ranks, ffn_size) + + +def _order(layout: TransportLayout) -> str: + return " ".join(f"{role[0].upper()}{rank}" for role, rank in layout.order) + + +@pytest.mark.parametrize( + ("placement", "order", "ffn_ranks_per_domain", "is_identity"), + [ + ("4A2F", "F0 F1 A0 A1 A2 A3", 0, True), + ("2A0F,0A2F", "F0 F1 A0 A1", 0, True), + ("3A2F,5A0F", "F0 F1 A0 A1 A2 A3 A4 A5 A6 A7", 0, True), + ("0A1F,0A1F,1A0F,1A0F", "F0 F1 A0 A1", 0, True), + ("1A1F,1A1F", "F0 A0 F1 A1", 1, False), + ("4A1F,4A1F", "F0 A0 A1 A2 A3 F1 A4 A5 A6 A7", 1, False), + ("2A2F,2A2F,2A2F", "F0 F1 A0 A1 F2 F3 A2 A3 F4 F5 A4 A5", 2, False), + ], +) +def test_transport_orders_domains_ffn_first_and_picks_the_afdep_mode( + placement, order, ffn_ranks_per_domain, is_identity +): + """Each pod becomes one block with its FFN ranks first. FFN ranks that + fill the first pods keep the role world (FFN prefix); the same FFN count + in every pod gives afdEP-comm's per-domain mode. + + Why: DeepEP numbers ranks domain by domain, and afdEP-comm finds the FFN + ranks either at [0, F) or at the first f ranks of every domain; any other + order routes rows to the wrong ranks. Layouts that run today must keep the + role world so they take the unchanged path. + """ + layout = _plan(placement) + + assert _order(layout) == order + assert layout.ffn_ranks_per_domain == ffn_ranks_per_domain + assert layout.is_identity is is_identity + + +def test_per_domain_ffn_ranks_sit_where_afdep_expects_them(): + """In 3 x 6A2F, FFN rank ``j`` is transport rank ``(j // 2) * 8 + j % 2``. + + Why: an FFN rank loads the experts of its DP (EP) rank, and afdEP-comm + routes those experts to this transport rank; a mismatch silently computes + rows with the wrong experts. + """ + layout = _plan("6A2F,6A2F,6A2F") + + assert (layout.num_domains, layout.domain_size) == (3, 8) + assert [layout.transport_rank("ffn", j) for j in range(6)] == [ + (j // 2) * 8 + j % 2 for j in range(6) + ] + + +def test_role_split_pods_in_any_order_keep_the_ffn_prefix(): + """Attention pods listed before the FFN pod still give FFN-first order. + + Why: domains follow their lowest role-world rank, so the pod order does + not change the transport the role-split layouts run today. + """ + layout = _plan("4A0F,0A4F") + + assert _order(layout) == "F0 F1 F2 F3 A0 A1 A2 A3" + assert layout.is_identity + + +@pytest.mark.parametrize( + ("ranks", "ffn_size", "message"), + [ + # 4A1F,3A2F: the FFN count varies from pod to pod. + (_pods("4A1F,3A2F")[0], 3, "same number of FFN ranks"), + # Pods of 1, 1, 2 and 2 GPUs. + (_pods("0A1F,0A1F,2A0F,2A0F")[0], 2, "same number of ranks"), + (_pods("0A2F,4A0F")[0], 2, "same number of ranks"), + # 2A2F,2A2F with FFN DP ranks 0, 2 on pod 0 and 1, 3 on pod 1. + ( + [ + RankPlacement(role, rank, f"host:pod{pod}") + for pod, roles in enumerate( + [ + [("ffn", 0), ("ffn", 2), ("attention", 0), ("attention", 1)], + [("ffn", 1), ("ffn", 3), ("attention", 2), ("attention", 3)], + ] + ) + for role, rank in roles + ], + 4, + "FFN DP ranks must rise in pod order", + ), + ], +) +def test_transport_rejects_placements_afdep_cannot_carry(ranks, ffn_size, message): + """Unequal pods, an FFN count that varies per pod, and FFN DP ranks out + of pod order raise, naming the placement. + + Why: DeepEP needs equal NVLink domains and afdEP-comm has no mode for the + rest; failing in the handshake beats a hang in the first dispatch. + """ + with pytest.raises(RuntimeError, match=message): + plan_transport_layout(ranks, ffn_size) diff --git a/tests/unit/fakes/afdep.py b/tests/unit/fakes/afdep.py index aaf564877..1680d40c8 100644 --- a/tests/unit/fakes/afdep.py +++ b/tests/unit/fakes/afdep.py @@ -14,7 +14,7 @@ from types import ModuleType # Keyword-only parameters of afdep.deepep.ElasticBuffer at the pinned afdEP-comm -# (afdEP/afdEP-comm 2b74d57). The stand-in requires the required ones and +# (afdEP/afdEP-comm v1.1.0). The stand-in requires the required ones and # rejects any other keyword, so a lane built with arguments the real transport # would refuse fails here as it would on a GPU. ELASTIC_BUFFER_REQUIRED = ( @@ -28,7 +28,13 @@ "cpu_timeout_s", "gpu_timeout_s", ) -ELASTIC_BUFFER_OPTIONAL = ("use_fp8_dispatch", "num_sms", "num_qps", "ffn_ranks") +ELASTIC_BUFFER_OPTIONAL = ( + "use_fp8_dispatch", + "num_sms", + "num_qps", + "ffn_ranks", + "ffn_ranks_per_domain", +) class FakeElasticBuffer: diff --git a/uv.lock b/uv.lock index 3c3f6e353..1f13929e7 100644 --- a/uv.lock +++ b/uv.lock @@ -31,7 +31,7 @@ resolution-markers = [ [[package]] name = "afdep-comm" version = "1.1.0" -source = { git = "https://github.com/afdEP/afdEP-comm.git?rev=2b74d574e9d25cc794de8d2359f815fcfb48955a#2b74d574e9d25cc794de8d2359f815fcfb48955a" } +source = { git = "https://github.com/afdEP/afdEP-comm.git?rev=v1.1.0#741b9aa5364687ac84c5049f5c6239b8cae6a7eb" } dependencies = [ { name = "ninja" }, { name = "packaging" }, @@ -5584,7 +5584,7 @@ e2e-tests = [ [package.metadata] requires-dist = [ - { name = "afdep-comm", marker = "extra == 'afdep'", git = "https://github.com/afdEP/afdEP-comm.git?rev=2b74d574e9d25cc794de8d2359f815fcfb48955a" }, + { name = "afdep-comm", marker = "extra == 'afdep'", git = "https://github.com/afdEP/afdEP-comm.git?rev=v1.1.0" }, { name = "vllm", marker = "extra == 'vllm'", specifier = "==0.32.0" }, ] provides-extras = ["afdep", "vllm"] From e58fdec5876782b9c6a03683416611deea42af92 Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Mon, 9 Nov 2026 22:32:16 +0800 Subject: [PATCH 28/42] docs: consolidate documentation structure and entry points (#560) ## Purpose Installation, common configuration, support limits and NPU validation are scattered across the root README, connector guides and design pages. Give readers one documentation entry point and consolidate repeated content. The README shrinks from 372 to 78 lines; this PR removes 272 lines overall. ## Issue Second documentation cleanup PR, following merged #559. Based on main at `49aff3bb`; this PR contains documentation structure/navigation changes and reference fixes. ## Scope - In scope: Markdown organization, navigation, deduplication and references used by development skills. - Out of scope: runtime code, tests, CI changes and new hardware qualification. ## Implementation Notes - Add `docs/README.md` as the documentation index, with separate Support matrix and Documentation sections in the root README. Move installation, common AFD configuration and support/evidence into dedicated references; remove duplicate field tables and launch fragments from the landing page and guides. - Move Async FFN and routed CAM design into `docs/design/npu/`, and the historical Layered W13/W2 report into `docs/reports/`. - Merge `docs/npu/TESTING.md` into `tests/npu/README.md`, retaining runtime/DBO and layered precision requirements. Remove duplicate suite descriptions and obsolete eager-only wording. - Keep the module index focused on design ownership; remove its duplicate user-guide list and ASCII dependency diagram. Update incoming links and skill references, including upgrade/review instructions that locate the runtime matrix. - Preserve module status, ownership, validation paths, upstream refs and historical evidence. Runtime pins remain vLLM 0.32.0 (`5df63a72`) and vLLM-Ascend `bd42eea0`. ## Test Plan Check formatting, local navigation, documented configuration against current code, command syntax and preservation of relocated content. ## Test Result - All applicable pre-commit hooks and `git diff --check` passed. - 488 local Markdown/HTML links and anchors checked across 112 documents. - 10 AFD configuration examples parsed; all 10 common fields/defaults and four Async CAM extra keys match the source. - 16 installation/NPU validation shell blocks passed `bash -n`. - Ownership/validation metadata preserved across nine module pages. Three relocated documents and the support matrix retain their content/evidence apart from link rebasing and the validation-limits heading. - Checks ran on CPU-only macOS/Python 3.12.13. No unit suite, device E2E or native build was run for this documentation-only change. ## Docs Impact 44 Markdown files changed, +494/-766 lines. Four references/index pages added, three documents moved and one duplicate testing page merged away. ---
Essential PR Checklist - [x] Purpose is clear and linked to public context. - [x] Scope is bounded. - [x] Pinned vLLM / vLLM-Ascend compatibility is considered. - [x] No changes to the vLLM source checkout. - [x] Plugin-owned extension and patch requirements remain unchanged. - [x] Imports remain CPU-safe; no code changes. - [x] Validation evidence and hardware-validation limits are stated. - [x] Documentation impact is stated.
--------- Signed-off-by: jiangkuaixue123 --- .../references/afd-prefill.md | 2 +- .../references/environment.md | 4 +- .../references/validation.md | 4 +- .agents/skills/review-pr/SKILL.md | 2 +- .../checks/model-addition-checklist.md | 2 +- .../references/checks/perf-verification.md | 2 +- .../references/checks/verification.md | 2 +- .../features/async-cam-ubatching.md | 4 +- .../references/features/dspark-draft.md | 2 +- .../features/graphs-and-compilation.md | 2 +- .../references/modules/execution-platforms.md | 2 +- .../references/process/general-checks.md | 4 +- .../references/process/review-execution.md | 4 +- .agents/skills/upgrade-gpu-version/SKILL.md | 2 +- .../20261028-v0.30.0-to-v0.32.0-lessons.md | 2 +- .../references/documentation-refresh.md | 15 +- .../20261029-v0.30.0-to-v0.32.0-lessons.md | 4 +- .../references/documentation-refresh.md | 25 +- README.md | 338 ++---------------- afd_plugin/connectors/README.md | 2 +- csrc/npu/README.md | 6 +- docs/README.md | 31 ++ docs/configuration.md | 69 ++++ docs/design/module/attention_runtime.md | 6 +- .../module/compatibility_and_patches.md | 2 +- docs/design/module/execution_platforms.md | 71 +--- docs/design/module/ffn_runtime.md | 8 +- docs/design/module/index.md | 43 +-- docs/design/module/model_integration.md | 4 +- docs/design/module/plugin_boundary.md | 22 +- .../npu/async_ffn_runtime.md} | 2 +- .../npu/cam_async_routed_ops.md} | 6 +- docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md | 28 +- docs/installation.md | 92 +++++ docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md | 48 +-- docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md | 27 +- docs/npu/TESTING.md | 172 --------- docs/npu/TROUBLESHOOTING.md | 2 +- .../layered_w2_mdl_validation.md} | 0 docs/support.md | 78 ++++ .../deepseek_v2_lite/README.md | 2 +- tests/e2e/README.md | 6 +- tests/e2e/operators/README.md | 2 +- tests/npu/README.md | 109 +++++- 44 files changed, 494 insertions(+), 766 deletions(-) create mode 100644 docs/README.md create mode 100644 docs/configuration.md rename docs/{npu/ASYNC_FFN_RUNTIME.md => design/npu/async_ffn_runtime.md} (99%) rename docs/{npu/CAM_ASYNC_ROUTED_OPS.md => design/npu/cam_async_routed_ops.md} (98%) create mode 100644 docs/installation.md delete mode 100644 docs/npu/TESTING.md rename docs/{npu/LAYERED_W2_MDL_VALIDATION.md => reports/layered_w2_mdl_validation.md} (100%) create mode 100644 docs/support.md diff --git a/.agents/skills/deploy-dsv4-flash-async/references/afd-prefill.md b/.agents/skills/deploy-dsv4-flash-async/references/afd-prefill.md index cec0d2c01..5d6e98f29 100644 --- a/.agents/skills/deploy-dsv4-flash-async/references/afd-prefill.md +++ b/.agents/skills/deploy-dsv4-flash-async/references/afd-prefill.md @@ -68,4 +68,4 @@ bash "${DSV4_SCRIPT_DIR}/run_prefill.sh" `attn_ranks_per_dp=A TP`，跨节点 local DP 分段不重叠。 真实性能运行保持 `AFD_FORCE_BALANCED_TOPK_IDS=0`、`enable_force_eplb=false`，关闭 IO 调试日志。 验证路径、数学语义和停止边界见 [验证](validation.md) 与 -[主线 FFN runtime](../../../../docs/npu/ASYNC_FFN_RUNTIME.md)。 +[主线 FFN runtime](../../../../docs/design/npu/async_ffn_runtime.md)。 diff --git a/.agents/skills/deploy-dsv4-flash-async/references/environment.md b/.agents/skills/deploy-dsv4-flash-async/references/environment.md index 1d9dc86bf..dbea6c998 100644 --- a/.agents/skills/deploy-dsv4-flash-async/references/environment.md +++ b/.agents/skills/deploy-dsv4-flash-async/references/environment.md @@ -36,7 +36,7 @@ export PID_DIR="${LOG_DIR}/pids" | vLLM 0.30.0 | `ced6857afa0ea7b2e3f0846a62e1394e90f15607` | | vLLM-Ascend | `8d4409d6256d8a6729140ddcc0d1889e3f96cdd6` | -环境与历史实测见 [安装说明](../../../../README.md#ascend-npu-installation)。 +环境与历史实测见 [安装说明](../../../../docs/installation.md#ascend-npu-installation)。 记录实际 CANN、torch、torch-npu 和构建产物；已有 A3 记录不是任意版本组合的兼容承诺。 源码路径默认使用当前 AFD checkout 和同级 vLLM/Ascend，挂载或安装包不同则显式覆盖。 @@ -87,7 +87,7 @@ PYTHON 仅 `import afd_plugin` 或 loader 成功不证明旧安装包含这四个算子；注册检查也不代替实际通信验收。缺少算子时检查安装日志和实际导入路径，并重新安装匹配源码。P 的 Attention/FFN 节点均需匹配的算子版本。 -仓库内构建细节见 `csrc/npu/README.md`，通信算子说明见 `docs/npu/CAM_ASYNC_ROUTED_OPS.md`。 +仓库内构建细节见 `csrc/npu/README.md`，通信算子说明见 `docs/design/npu/cam_async_routed_ops.md`。 | 用途 | 变量 / 默认值 | | --- | --- | diff --git a/.agents/skills/deploy-dsv4-flash-async/references/validation.md b/.agents/skills/deploy-dsv4-flash-async/references/validation.md index 0bcdb2726..d6dff1bea 100644 --- a/.agents/skills/deploy-dsv4-flash-async/references/validation.md +++ b/.agents/skills/deploy-dsv4-flash-async/references/validation.md @@ -3,8 +3,8 @@ 此 skill 的命令和配置已迁移到固定 0.30 源码契约，硬件资格按具体运行记录认定。 AFD 原始模型、DBO 和 FFN FULL 的已有证据入口为 [PR #462](https://github.com/vllm-project/afd-plugin/pull/462)、 -[原生 DBO](../../../../docs/npu/TESTING.md#native-dbo-contracts-and-device-execution) 和 -[FFN FULL](../../../../docs/npu/ASYNC_FFN_RUNTIME.md#validation)。 +[原生 DBO](../../../../tests/npu/README.md#native-dbo-contracts-and-device-execution) 和 +[FFN FULL](../../../../docs/design/npu/async_ffn_runtime.md#validation)。 PR 中复用的旧运行、维护者确认和新运行各有范围，不能改写为本次 skill 的新硬件结果。 ## 按任务选择覆盖 diff --git a/.agents/skills/review-pr/SKILL.md b/.agents/skills/review-pr/SKILL.md index c1c0151f2..c15faf101 100644 --- a/.agents/skills/review-pr/SKILL.md +++ b/.agents/skills/review-pr/SKILL.md @@ -48,7 +48,7 @@ never from this skill and never from memory: - `docs/design/module/index.md` — `upstream_refs` (vLLM commit and vLLM-Ascend commit) and `last_reviewed`. - `pyproject.toml` optional `vllm` extra, `docker/Dockerfile.ci` - (`VLLM_BASE_TAG` plus its version assertion), and `README.md`'s supported + (`VLLM_BASE_TAG` plus its version assertion), and `docs/installation.md`'s runtime table. The values quoted in this skill are the `main` values as of its last refresh diff --git a/.agents/skills/review-pr/references/checks/model-addition-checklist.md b/.agents/skills/review-pr/references/checks/model-addition-checklist.md index 8d6aff4ca..4d595a839 100644 --- a/.agents/skills/review-pr/references/checks/model-addition-checklist.md +++ b/.agents/skills/review-pr/references/checks/model-addition-checklist.md @@ -45,7 +45,7 @@ skill; this checklist is the reviewer's evidence bar. - Docs: the relevant `docs/design/module/` page (model integration) and, for user-facing support, a user guide or guide section; the design-page registration table must gain the new row. -- For NPU support: manual validation evidence per `docs/npu/TESTING.md`; +- For NPU support: manual validation evidence per `tests/npu/README.md`; there is no NPU CI. A family missing any required piece is a P1 finding; state which piece and diff --git a/.agents/skills/review-pr/references/checks/perf-verification.md b/.agents/skills/review-pr/references/checks/perf-verification.md index 8fe3fbe23..3537351ef 100644 --- a/.agents/skills/review-pr/references/checks/perf-verification.md +++ b/.agents/skills/review-pr/references/checks/perf-verification.md @@ -9,7 +9,7 @@ changes, TP/SP scheduling changes, and routing changes. Accept a claim only with base and head runs that hold constant: - Hardware pool and placement (CI presets: `l4_1`–`l4_4`, `h100_4`; NPU: - documented manual environment per `docs/npu/TESTING.md`). + documented manual environment per `tests/npu/README.md`). - Container image and dependency versions (`docker/Dockerfile.ci` pins `VLLM_BASE_TAG` and asserts the installed vLLM version). - Workload and datasets (`tools/benchmarks/decode_bench.py`, diff --git a/.agents/skills/review-pr/references/checks/verification.md b/.agents/skills/review-pr/references/checks/verification.md index fe506ad5e..882b73892 100644 --- a/.agents/skills/review-pr/references/checks/verification.md +++ b/.agents/skills/review-pr/references/checks/verification.md @@ -9,7 +9,7 @@ and the review needs evidence beyond static reading and CI. | --- | --- | | GPU host with devices free | Narrowest changed unit tests, then the single most relevant E2E scenario via `run-e2e` | | GPU host for connector device tests | `tests/gpu/test_deepep_m2n_gpu.py` on SM90+ with afdEP-comm and DeepGEMM; most cases need 4 GPUs, the Attention-TP cases need 6 | -| NPU host | Manual per `docs/npu/TESTING.md`; only what the user explicitly asked to run | +| NPU host | Manual per `tests/npu/README.md`; only what the user explicitly asked to run | | CPU-only trusted checkout | Import preflight + `pytest tests/unit/ -m "not gpu and not vllm_runtime"` | | Static only (untrusted head, no sandbox) | SHA-addressed reads + existing CI evidence; report execution as a gap | diff --git a/.agents/skills/review-pr/references/features/async-cam-ubatching.md b/.agents/skills/review-pr/references/features/async-cam-ubatching.md index 57489f48b..a8f861270 100644 --- a/.agents/skills/review-pr/references/features/async-cam-ubatching.md +++ b/.agents/skills/review-pr/references/features/async-cam-ubatching.md @@ -7,7 +7,7 @@ interplay, the layered GMM executor, and the DSV3.2/DSV4 async CAM forward modules under `afd_plugin/model_executor/models/npu/`. Designs: `docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md`, -`docs/npu/ASYNC_FFN_RUNTIME.md`, `docs/npu/CAM_ASYNC_ROUTED_OPS.md`, +`docs/design/npu/async_ffn_runtime.md`, `docs/design/npu/cam_async_routed_ops.md`, `docs/design/module/connector_contracts.md`, `docs/design/module/ffn_runtime.md`. @@ -51,5 +51,5 @@ Designs: `docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md`, a regression. Name the NPU evidence (who ran what, on which environment per -`docs/npu/TESTING.md`) for any behavioral claim; CPU tests cover planning and +`tests/npu/README.md`) for any behavioral claim; CPU tests cover planning and state transitions only. diff --git a/.agents/skills/review-pr/references/features/dspark-draft.md b/.agents/skills/review-pr/references/features/dspark-draft.md index 902df0753..40cfd3729 100644 --- a/.agents/skills/review-pr/references/features/dspark-draft.md +++ b/.agents/skills/review-pr/references/features/dspark-draft.md @@ -17,7 +17,7 @@ guide: `recipe/npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md`. DSpark is an Ascend-only draft model paired with a DeepSeek-V4 target under `CAMAsyncAFDConnector`. The attention role keeps native eager proposal; the FFN role may independently use layered eager or `FULL` graph execution. Supported -combinations are enumerated in `execution_platforms.md`; everything else is +combinations are listed in the [support matrix](../../../../../docs/support.md#tested-runtime-matrix); everything else is excluded rather than best-effort. ## Feature checks diff --git a/.agents/skills/review-pr/references/features/graphs-and-compilation.md b/.agents/skills/review-pr/references/features/graphs-and-compilation.md index 84a48b4ba..f31d8d451 100644 --- a/.agents/skills/review-pr/references/features/graphs-and-compilation.md +++ b/.agents/skills/review-pr/references/features/graphs-and-compilation.md @@ -46,4 +46,4 @@ Test graph gating and metadata parity CPU-side where possible (`tests/unit/v1/worker/test_cuda_graph.py`, `test_npu_mla_graph.py`, `test_async_ffn_graph.py`, `tests/unit/connectors/test_async_cam_startup.py`); device behavior claims need the named GPU scenario (`afd-v2-graph-*`, -`afd-v2-graph-dbo-*`) or the NPU run recorded per `docs/npu/TESTING.md`. +`afd-v2-graph-dbo-*`) or the NPU run recorded per `tests/npu/README.md`. diff --git a/.agents/skills/review-pr/references/modules/execution-platforms.md b/.agents/skills/review-pr/references/modules/execution-platforms.md index 00b40ae8b..d3f42c909 100644 --- a/.agents/skills/review-pr/references/modules/execution-platforms.md +++ b/.agents/skills/review-pr/references/modules/execution-platforms.md @@ -29,7 +29,7 @@ application ([npu-compat.md](npu-compat.md)). `upgrade-layered-gmm-ops` workflow. - Keep version pins coherent: `TARGET_VLLM_VERSION` in `compat/vllm.py`, the `vllm` extra in `pyproject.toml`, `VLLM_BASE_TAG` and its version assertion - in `docker/Dockerfile.ci`, README, and design-page `upstream_refs` move + in `docker/Dockerfile.ci`, installation/support references, and design-page `upstream_refs` move together, or the PR states why not. Read the actual pin; do not trust the value quoted in a review note or in this skill. - Keep GPU path parity visible: `csrc/gpu/` contains only a README reserving diff --git a/.agents/skills/review-pr/references/process/general-checks.md b/.agents/skills/review-pr/references/process/general-checks.md index 3b4abe72e..161d30c40 100644 --- a/.agents/skills/review-pr/references/process/general-checks.md +++ b/.agents/skills/review-pr/references/process/general-checks.md @@ -77,7 +77,7 @@ enforce them as review rules, not style nits: | --- | --- | | Correctness | A reachable input reaches the changed code and produces wrong output, a hang, or a crash. | | Patch-contract drift | A patched function's signature, return type, or upstream copy diverges from the pinned ref, or missing/incorrect `# ### PATCH START/END` markers, `Patch reason:` / `Patch functionality:` / `# Upstream source:` comments break upgrade comparability. Patched methods now live in `afd_plugin/v1/worker/**` as well as `afd_plugin/compat/`, and `tests/unit/compat/patches/test_signatures.py` owns the signature table for both. | -| Version/compat | The change breaks the current `TARGET_VLLM_VERSION` / vLLM-Ascend pin contract, or moves a pin without updating the version gate, `docker/Dockerfile.ci` base tag and its assertion, `pyproject.toml`, README, and design-page `upstream_refs` together. Read the pin from `afd_plugin/compat/vllm.py`, not from memory. | +| Version/compat | The change breaks the current `TARGET_VLLM_VERSION` / vLLM-Ascend pin contract, or moves a pin without updating the version gate, `docker/Dockerfile.ci` base tag and its assertion, `pyproject.toml`, installation/support references, and design-page `upstream_refs` together. Read the pin from `afd_plugin/compat/vllm.py`, not from memory. | | Platform divergence | The GPU or NPU twin (worker/runner/model variant, sync vs async NPU FFN runner) is not updated and the change is not provably platform-neutral. | | CPU-safe imports | A new module-level import of torch/torch_npu/vLLM runtime breaks `pytest -m "not gpu and not vllm_runtime"` or plugin import on a CPU host. | | Concurrency/async | Async-DP busy loops, ubatching state machines, DBO yield, async CAM work items, or two-stream slots can deadlock, drop, reorder, or leak work items on abort/timeout. | @@ -90,7 +90,7 @@ enforce them as review rules, not style nits: | CI suppression | The diff is docs-only or skip-mark-only, so `skip_ci.py` suppresses all default CI. A green pipeline then proves nothing about the code; say so instead of treating it as validation. | | Vendored artifacts | New or changed files under `afd_plugin/connectors/npu/bin/` (`.run`/`.whl`) are pre-commit-excluded by design; they still need source, version, and target CANN justification. | | Agent/skill drift | A change to production paths, CI, or E2E gates that leaves `.agents/skills/**` stating stale behavior (pins, case counts, commands) is a P2 finding; those skills are executed instructions. | -| Evidence | Perf/accuracy claims lack comparable base/head runs; NPU behavior claims lack manual evidence per `docs/npu/TESTING.md`. | +| Evidence | Perf/accuracy claims lack comparable base/head runs; NPU behavior claims lack manual evidence per `tests/npu/README.md`. | | User contract | `AFDConfig` fields, defaults, `connector_extra_config` keys, or recipe launch scripts change meaning without a docs update and migration note. | ## Finding bar diff --git a/.agents/skills/review-pr/references/process/review-execution.md b/.agents/skills/review-pr/references/process/review-execution.md index 0138528d5..b96d58d7c 100644 --- a/.agents/skills/review-pr/references/process/review-execution.md +++ b/.agents/skills/review-pr/references/process/review-execution.md @@ -147,11 +147,11 @@ paths have supported findings or explicit no-issue conclusions: author evidence. Gate cases must never be turned into skips to make a run pass. 5. NPU paths have no in-repo CI. Require author-provided evidence per - `docs/npu/TESTING.md`, or a manually executed run the user explicitly asks + `tests/npu/README.md`, or a manually executed run the user explicitly asks for. Never simulate device evidence. 6. When a pin moves, verify the coherent set: `TARGET_VLLM_VERSION` in `afd_plugin/compat/vllm.py`, the `vllm` extra in `pyproject.toml`, - `VLLM_BASE_TAG` and its assertion in `docker/Dockerfile.ci`, `README.md`, + `VLLM_BASE_TAG` and its assertion in `docker/Dockerfile.ci`, installation/support references, and every design page's `upstream_refs`. ## Deliver maintainer-style findings diff --git a/.agents/skills/upgrade-gpu-version/SKILL.md b/.agents/skills/upgrade-gpu-version/SKILL.md index 129109f3b..853cccb49 100644 --- a/.agents/skills/upgrade-gpu-version/SKILL.md +++ b/.agents/skills/upgrade-gpu-version/SKILL.md @@ -175,7 +175,7 @@ and any detached read-only worktrees. Discover the current vLLM ref in this order: 1. `pyproject.toml`, lockfile, and package metadata; -2. compatibility constants, patch annotations, design matrices, README, GPU +2. compatibility constants, patch annotations, installation/support references, GPU guides, recipes, CI, and active upgrade RFC; 3. installed package versions and adjacent checkout state as corroboration only. diff --git a/.agents/skills/upgrade-gpu-version/references/20261028-v0.30.0-to-v0.32.0-lessons.md b/.agents/skills/upgrade-gpu-version/references/20261028-v0.30.0-to-v0.32.0-lessons.md index c9d9c2918..100971660 100644 --- a/.agents/skills/upgrade-gpu-version/references/20261028-v0.30.0-to-v0.32.0-lessons.md +++ b/.agents/skills/upgrade-gpu-version/references/20261028-v0.30.0-to-v0.32.0-lessons.md @@ -64,7 +64,7 @@ not a substitute for checking the next upstream version. ## Documentation and process lessons - Keep hardware, CPU-only, historical and excluded claims distinct. The - [runtime matrix](../../../../docs/design/module/execution_platforms.md#tested-runtime-matrix) + [runtime matrix](../../../../docs/support.md#tested-runtime-matrix) records the measured cells, score limits and shutdown caveats. Checked-in recipes were inspected but not all were executed as recipes. - Freeze source and runtime identity per test, preserve raw failed results, diff --git a/.agents/skills/upgrade-gpu-version/references/documentation-refresh.md b/.agents/skills/upgrade-gpu-version/references/documentation-refresh.md index 60c1c1744..33a29763f 100644 --- a/.agents/skills/upgrade-gpu-version/references/documentation-refresh.md +++ b/.agents/skills/upgrade-gpu-version/references/documentation-refresh.md @@ -8,7 +8,7 @@ must describe the exact target revision and the evidence actually collected. 1. Evidence ledger 2. Claim levels -3. Root documentation +3. Installation, support, and configuration 4. GPU connector and runtime guides 5. Recipes and launch scripts 6. Design and contributor documentation @@ -56,9 +56,9 @@ Do not turn CPU-only evidence into a GPU support claim. Do not describe an option as validated merely because it was accepted by the command line: confirm that logs, counters, traces, or execution behavior show the path was used. -## 3. Root Documentation +## 3. Installation, Support, and Configuration -Review the root `README.md` and any linked installation page for: +Review `docs/installation.md`, the [support matrix](../../../../docs/support.md#tested-runtime-matrix), and `docs/configuration.md` for: - the exact supported vLLM revision or range; - the target Python, PyTorch, CUDA, driver, compiler, and NCCL requirements; @@ -66,8 +66,7 @@ Review the root `README.md` and any linked installation page for: - installation commands and package extras; - supported models, connectors, parallel modes, and execution modes; - CUDA Graph, DBO or uBatch, TP, profiling, and isolation support; -- a minimal launch example with current arguments and environment variables; -- links to current recipes, tests, design notes, and troubleshooting pages; +- links from `README.md` and `docs/README.md` to current guides and recipes; - explicit unsupported or unvalidated combinations. Avoid broad phrases such as “GPU fully supported” when only selected models or @@ -77,7 +76,7 @@ record. ## 4. GPU Connector and Runtime Guides Review `afd_plugin/connectors/README.md` first. Keep its connector status matrix -consistent with the root README, GPU guide, recipes, design runtime matrix, and +consistent with `docs/support.md`, GPU guides, recipes, and E2E evidence. State eager, CUDA Graph, DBO/uBatch, TP, asymmetric topology, and profiling evidence separately where their support differs. @@ -126,7 +125,7 @@ appear related to the changed code: |---|---| | `docs/design/module/index.md` | target runtime, upstream refs, validation entry points, ownership, and links | | `docs/design/module/compatibility_and_patches.md` | exact patch symbols/signatures, reasons, tests, dispositions, and removal/upstream plans | -| `docs/design/module/execution_platforms.md` | GPU worker/runner initialization, device selection, CUDA Graph, DBO/uBatch, profiling, and runtime matrix | +| `docs/design/module/execution_platforms.md` | GPU worker/runner initialization, device selection, CUDA Graph, DBO/uBatch, and profiling | | `docs/design/module/connector_contracts.md` | NCCL/P2P configuration, payload/metadata ownership, ordering, topology, lifecycle, and cleanup | | `docs/design/module/attention_runtime.md` | Attention lifecycle, KV/scheduler ownership, forward context, connector handoff, graph, and ubatching | | `docs/design/module/ffn_runtime.md` | FFN daemon/EngineCore lifecycle, MoE execution, graph dispatch, failure propagation, and shutdown | @@ -213,7 +212,7 @@ Classify each hit as one of: - generated and must be regenerated; - unrelated to the GPU upgrade. -Then check that README, connector README, GPU guide, recipes, design modules, +Then check that installation/support references, connector README, GPU guides, recipes, design modules, tests, CI, package/build metadata, contributor templates, and release notes agree on the same target identity, native-build status, and support scope. diff --git a/.agents/skills/upgrade-npu/references/20261029-v0.30.0-to-v0.32.0-lessons.md b/.agents/skills/upgrade-npu/references/20261029-v0.30.0-to-v0.32.0-lessons.md index 39c04f9d3..6dd2643a3 100644 --- a/.agents/skills/upgrade-npu/references/20261029-v0.30.0-to-v0.32.0-lessons.md +++ b/.agents/skills/upgrade-npu/references/20261029-v0.30.0-to-v0.32.0-lessons.md @@ -9,7 +9,7 @@ ## Upstream changes that mattered -The architecture gate classified the release changes as local API/lifecycle adaptations rather than an authorized redesign. The shared engine startup/coordinator contract changed; native MRV2 execution, DP dispatch, Ascend configuration, graph metadata, SP ownership and model construction changed at their respective seams. AFD must preserve Attention request/KV ownership, connector-driven FFN lifecycle, control/transport ordering and platform separation. See [compatibility and patches](../../../../docs/design/module/compatibility_and_patches.md), [runtime matrix](../../../../docs/design/module/execution_platforms.md), and commits `d29db75`, `2e30083`, `49dc295`, `3eec8b4`, `d85159b` and `e6ced6c`. +The architecture gate classified the release changes as local API/lifecycle adaptations rather than an authorized redesign. The shared engine startup/coordinator contract changed; native MRV2 execution, DP dispatch, Ascend configuration, graph metadata, SP ownership and model construction changed at their respective seams. AFD must preserve Attention request/KV ownership, connector-driven FFN lifecycle, control/transport ordering and platform separation. See [compatibility and patches](../../../../docs/design/module/compatibility_and_patches.md), [runtime matrix](../../../../docs/support.md#tested-runtime-matrix), and commits `d29db75`, `2e30083`, `49dc295`, `3eec8b4`, `d85159b` and `e6ced6c`. ## Adaptations and patch lifecycle @@ -28,7 +28,7 @@ Check GPU fixes per backend before reusing them. NPU shares control-plane close/ ## Validation and environment lessons -The target `tests/unit` run recorded 1843 passed, six skipped, 180 subtests passed. Four device invocations had separate hardware coverage; two mutually exclusive absence cases skipped because the extension was present. The hardware and functional gates precede accuracy. The eight agreed Lite cases used identical first 100 GSM8K documents and prompts, eight-shot, max generation 512, strict match >= 0.27; all passed their request/path and accuracy gates. Seven AFD graceful exits failed, while native control's strict exit passed; physical cleanup and scheduler release were independently verified. See the [validation record](https://github.com/vllm-project/afd-plugin/pull/507) and [NPU tests](../../../../docs/npu/TESTING.md). +The target `tests/unit` run recorded 1843 passed, six skipped, 180 subtests passed. Four device invocations had separate hardware coverage; two mutually exclusive absence cases skipped because the extension was present. The hardware and functional gates precede accuracy. The eight agreed Lite cases used identical first 100 GSM8K documents and prompts, eight-shot, max generation 512, strict match >= 0.27; all passed their request/path and accuracy gates. Seven AFD graceful exits failed, while native control's strict exit passed; physical cleanup and scheduler release were independently verified. See the [validation record](https://github.com/vllm-project/afd-plugin/pull/507) and [NPU tests](../../../../tests/npu/README.md). The V4 W4A8 native/layered-off/layered-on/DBO first100 entries scored 96/95/94/95 for both strict and flexible filters, using the same documents, rendered prompts and targets, eight-shot, max generation 512 and concurrency 12. The eager Attention DBO entry used DP2/TP4 with FFN DP8/TP1 ordinary GMM; each DP0 Attention TP rank recorded 42 completed two-nonempty-microbatch steps and each DP1 TP rank recorded 44 during evaluation. This qualifies the agreed request-window cell, not per-question stage splitting, A/F transaction freshness, padding-only or other V4 boundary tests. V4 native DBO was outside the pinned Ascend runtime's support. These are the explicitly agreed 100-sample results, not 1,319-question qualification. diff --git a/.agents/skills/upgrade-npu/references/documentation-refresh.md b/.agents/skills/upgrade-npu/references/documentation-refresh.md index 4172e67a4..94faff293 100644 --- a/.agents/skills/upgrade-npu/references/documentation-refresh.md +++ b/.agents/skills/upgrade-npu/references/documentation-refresh.md @@ -51,21 +51,20 @@ backend supports the new vLLM revision. Distinguish these evidence levels: ## User-facing runtime documentation -### Root README +### Installation, support, and configuration -Review `README.md` as a complete public contract, including: +Review `docs/installation.md`, the [support matrix](../../../../docs/support.md#tested-runtime-matrix), and `docs/configuration.md` for: - target runtime wording and whether the version applies to NPU, GPU, or both; -- Current Status, supported models, connector matrix, and Known Gaps; +- supported models, connector matrix, and validation limits; - NPU hardware and full software-stack prerequisites; - vLLM and vLLM-Ascend installation pins and resolved source revisions; - build flags, `SOC_VERSION`, custom operator packages, and container guidance; - plugin installation and import or environment verification commands; -- NPU Attention and FFN launch commands, worker selection, and required - environment variables; +- worker selection and required environment variables; - AFD configuration keys, connector names and aliases, topology fields, graph, DBO, uBatch, PCP, quantization, and operator settings; -- links to the current connector guides, recipes, design docs, and E2E workflow. +- links from `README.md` and `docs/README.md` to current guides and recipes. Remove stale commands and unsupported examples. If a shortened launch example is not a complete runnable Attention/FFN deployment, label it as a fragment and @@ -74,7 +73,7 @@ link to the complete recipe. ### Connector package overview Review `afd_plugin/connectors/README.md`. Keep its connector status matrix -consistent with the root README, NPU guides, recipes, design runtime matrix, and +consistent with `docs/support.md`, NPU guides, recipes, and test evidence. State validation separately for CAMP2p and CAMAsync and for each relevant eager, graph, DBO, PCP, or topology cell. @@ -139,7 +138,7 @@ Refresh the internal source-of-truth documents when their contracts changed: | --- | --- | | `docs/design/module/index.md` | Runtime pair, upstream references, validation entry points, code ownership, and links | | `docs/design/module/compatibility_and_patches.md` | Exact target patch inventory, source symbols and signatures, patch reasons, tests, and upstream/removal plans | -| `docs/design/module/execution_platforms.md` | NPU Worker/ModelRunner initialization, ForwardContext, graph, DBO/uBatch, operator loading, and tested runtime matrix | +| `docs/design/module/execution_platforms.md` | NPU Worker/ModelRunner initialization, ForwardContext, graph, DBO/uBatch, and operator loading | | `docs/design/module/connector_contracts.md` | Configuration schema, payload/work-item ownership, control plane, topology, lifecycle, and execution order | | `docs/design/module/attention_runtime.md` | Attention worker/runner selection, metadata, forward context, connector handoff, graph, and ubatching | | `docs/design/module/ffn_runtime.md` | FFN daemon/EngineCore lifecycle, MoE execution, connector steps, graph dispatch, and shutdown | @@ -194,11 +193,11 @@ Compare every public support claim across these sources: | Claim | Sources that must agree | | --- | --- | -| Runtime versions | README, dependency pins, guides, recipes, design index, templates | -| Connector support | README, connector README, both NPU guides, recipe index, execution-platform matrix, E2E evidence | -| Models and quantization | README, recipes, model integration design, model E2E evidence | -| DBO/graph/PCP/topology | README, connector guides, recipes/scripts, execution and connector design docs, E2E evidence | -| Installation stack | README, guides, recipes, container/operator configuration, native-control evidence | +| Runtime versions | Installation and support references, dependency pins, guides, recipes, design index, templates | +| Connector support | Support reference, connector README, both NPU guides, recipe index, E2E evidence | +| Models and quantization | Support reference, recipes, model integration design, model E2E evidence | +| DBO/graph/PCP/topology | Support reference, connector guides, recipes/scripts, execution and connector design docs, E2E evidence | +| Installation stack | Installation reference, guides, recipes, container/operator configuration, native-control evidence | Search the whole documentation surface for stale versions and conflicting support language. Adapt these commands to the current and target refs: diff --git a/README.md b/README.md index ab052042e..c5644bab3 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@

-| Documentation | Recipe | Slides | DeepWiki | User Forum | Developer Slack | WeChat | +| Documentation | Recipe | Slides | DeepWiki | User Forum | Developer Slack | WeChat |

## Overview @@ -18,340 +18,46 @@ tests for GPU and Ascend NPU deployments. > This project is still experimental and needs more large-scale testing across > different hardware backends. -The GPU and Ascend target is **vLLM `v0.32.0`** at commit `5df63a72`. -The Ascend integration also targets vLLM-Ascend `bd42eea0`. The plugin does not modify the vLLM -source tree. AFD behavior is installed through the `vllm.general_plugins` entry -point, `--additional-config`, automatically selected role workers, plugin-owned -model wrappers, and narrow version-scoped compatibility shims. +The plugin extends vLLM through its general-plugin entry point without +modifying the vLLM source tree. ## Architecture ![afd-plugin architecture](docs/assets/vllm-afd-plugin-architecture.svg) -## Current Status +## Getting started -Core runtime support: +1. [Install AFD](docs/installation.md) for CUDA or Ascend NPU. +2. Choose a [connector and supported model](docs/support.md), then follow its + [deployment recipe](recipe/README.md). +3. Pass role/topology settings through + [`--additional-config`](docs/configuration.md). Send requests to Attention; + FFN serves connector work. -- vLLM plugin registration, AFD configuration, and runtime validation. -- Attention/FFN workers, model runners, model wrappers, and connector-driven - execution for CUDA and Ascend NPU. -- Eager and `FULL_DECODE_ONLY` graph execution, plus backend-specific profiling - support. -- Native DBO with exactly two ubatches on CUDA V2. Ascend constraints are - recorded in the existing [NPU runtime matrix](docs/design/module/execution_platforms.md#tested-runtime-matrix). -- DeepSeek MoE handoff at the remote-experts boundary on CUDA, with the gate - placed on either Attention or FFN. -- Expert-routed M:N exchange on CUDA (`DeepEPM2NAFDConnector`) for Qwen3 MoE and DeepSeek-V2: - Attention routes each token and dispatches it only to the FFN ranks that own - its experts. +## Support matrix -Model support: +See the [support matrix](docs/support.md) for models, connectors, tested +configurations, and known accuracy/shutdown limits. -| Model family | Registered architectures | Plugin model wrappers | Notes | -| --- | --- | --- | --- | -| DeepSeekV2 / DeepSeekV3 / DeepSeekV3.2 | `DeepseekForCausalLM`, `DeepseekV2ForCausalLM`, `DeepseekV3ForCausalLM`, `DeepseekV32ForCausalLM` | `AFDDeepseekForCausalLM`, `AFDDeepseekV2ForCausalLM`, `AFDDeepseekV3ForCausalLM` | DeepSeekV3.2 uses `AFDDeepseekV3ForCausalLM`. Each AFD role constructs and loads only its role-required model components, while shared embedding, normalization, and output components remain available where required by the model lifecycle. | -| DeepSeek V4 Flash | `DeepseekV4ForCausalLM` | `AFDDeepseekV4ForCausalLM` | CUDA synchronous and Ascend CAM paths; see the [runtime matrix](docs/design/module/execution_platforms.md#tested-runtime-matrix) for backend-specific validation limits. | -| Qwen3 MoE | `Qwen3MoeForCausalLM` | `AFDQwen3MoeForCausalLM` | CUDA; DeepEP M2N routes on Attention, P2P routes on FFN. | -| Qwen3.5 / Qwen3.6 MoE | `Qwen3_5MoeForConditionalGeneration` | `AFDQwen3_5MoeForConditionalGeneration` | CUDA text-only Qwen3.6-35B-A3B with `--language-model-only`, native DP4/TP1/EP4, and synchronous AFD 2A1F. Eager and graph execution are supported; GDN DBO is unsupported; see the [E2E limits](docs/design/module/e2e_testing.md). | +## Documentation -Connector support: - -See the [recipe index](recipe/README.md) for deployment and benchmark examples. - -| Connector | Platform | Recommend Stage | Sync or Async | Graph Support | Notes | -| --- | --- | --- | --- | --- | --- | -| `P2pNcclAFDConnector` | CUDA | Decode | Sync | `FULL_DECODE_ONLY` CUDA graph | FFN ranks are ordered before Attention ranks. `num_attention_ranks` must be greater than or equal to `num_ffn_ranks`; the Attention ranks are spread over the FFN ranks in blocks that differ in size by at most one. See the [DeepSeek V2 Lite recipe](recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md). | -| `DeepEPM2NAFDConnector` | CUDA (SM90+) | Decode | Sync | `FULL_DECODE_ONLY` CUDA graph | Qwen3 MoE or DeepSeek-V2; Attention-side routing, FFN TP1, and matching DBO settings (two microbatches when enabled). Requires afdEP-comm and an expert backend; see the [DeepEP M2N guide](docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md) and [recipes](recipe/gpu/DeepEPM2NAFDConnector/README.md). | -| `CAMP2pAFDConnector` | Ascend NPU | Decode | Sync | `FULL` or `FULL_DECODE_ONLY`; plain MLA graph DBO requires `FULL_DECODE_ONLY` | Uses HCCL/CAMP2P custom ops and native MRV2. DBO requires two ubatches and Attention DP > 1. See the [CAMP2P guide](docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md). | -| `CAMAsyncAFDConnector` | Ascend NPU | Prefill / decode | Async | FFN-only experimental `FULL` | DeepSeek-V4 W4A8/W8A8 on A3/910C. Eager Attention supports DP+TP/SP and two native DBO microbatches, including DP1; FFN stays outside native DBO. See the [Async CAM guide](docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md) and [FFN runtime contract](docs/npu/ASYNC_FFN_RUNTIME.md). | - -Connector implementations are grouped by backend package: -`afd_plugin.connectors.gpu` for GPU-only connectors, -`afd_plugin.connectors.npu` for NPU-only connectors. - -Known gaps: - -- NPU hardware evidence covers the specific single-Pod DeepSeek-V2-Lite and - DSV4 configurations in the [0.32 validation record](https://github.com/vllm-project/afd-plugin/pull/507). - Other models and topologies remain unverified. AFD post-request shutdown - failures remain open; successful inference does not imply graceful shutdown. -- Synchronous DSV4 E2E profiles are smoke cases; A5 concurrent answers remain - incorrect. See the [profile limits](tests/e2e/README.md#dsv4-flash-sync-camp2p-concurrent-requests-local-4-or-16-npus). -- Both backends require native ModelRunnerV2. The - [runtime matrix](docs/design/module/execution_platforms.md#tested-runtime-matrix) - records graph/DBO constraints and known DeepSeek-V2-Lite shutdown failures. -- GPU and NPU device/E2E tests require real hardware; model E2E also requires - weights. DeepEP M2N needs SM90+ and its device/E2E cases run outside CI. -- Qwen3 MoE requires Attention-side gate placement for DeepEP M2N and - FFN-side gate placement for P2P. It rejects sequence-parallel MoE, EPLB, - pipeline parallelism, speculative decoding, LoRA, and NPU. -- Qwen3.5/Qwen3.6 MoE adapter-family support is limited to the CUDA text-only - lane; repository hardware E2E evidence covers Qwen3.6-35B-A3B. - GDN DBO is unsupported; upstream hybrid/multimodal DBO validation applies. - NPU and multimodal execution are unsupported/unverified; - `compute_gate_on_attention=true`, pipeline parallelism, asynchronous and - multi-node execution are unsupported; quantization is unverified; no - performance claim is made. - -## Install - -Requires Python **3.11-3.13** and [uv](https://docs.astral.sh/uv/). - -```bash -git clone https://github.com/vllm-project/afd-plugin.git -cd afd-plugin -uv sync --group dev -``` - -`vllm` is an optional runtime extra so CPU-only or macOS development -environments can still run import/config tests without a CUDA wheel. - -### GPU installation - -For Linux / CUDA-capable environments only; Ascend users should skip this -command: - -```bash -uv sync --group dev --extra vllm -``` - -The optional GPU extra pins `vllm==0.32.0`. - -`DeepEPM2NAFDConnector` also needs the `afdep` extra (afdEP-comm) and -DeepGEMM, which is built from source; see -[Installation](docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md#installation) in -the connector guide. - -```bash -uv sync --group dev --extra vllm --extra afdep -``` - -### Ascend NPU installation - -The Ascend integration uses the source pair below. See the -[0.32 NPU validation record](https://github.com/vllm-project/afd-plugin/pull/507) for the tested -scope and retained shutdown limitations. Earlier -[0.30 results](https://github.com/vllm-project/afd-plugin/pull/425#issuecomment-6063910923) -retain their historical environment and sample counts. -Install a compatible driver and firmware, and confirm the devices with -`npu-smi info`. - -| Component | Version | -| --- | --- | -| Python | Package range `3.11–3.13`; tested `3.12.13` | -| vLLM | `0.32.0`, commit `5df63a7203cdb9328d824a27b682e77c7b9928be` | -| vLLM-Ascend | commit [`bd42eea0`](https://github.com/vllm-project/vllm-ascend/commit/bd42eea01d8312c4df469fabaed45f77a12df4d3) | -| Tested CANN / torch / torch-npu | `9.2` / `2.10.0+cpu` / `2.10.0.post7` | -| Tested Triton Ascend / CAM | `3.6` / `1.3` | - -> Hardware evidence is scoped to the recorded models, configurations, and -> sample counts; it does not cover every NPU deployment or guarantee clean -> device shutdown. - -#### Environment - -The recorded A3 environment uses the pinned pair and stack above. The -repository does not claim a released container tag for the Ascend target. Use the -[installation guide at that source snapshot](https://github.com/vllm-project/vllm-ascend/blob/bd42eea01d8312c4df469fabaed45f77a12df4d3/docs/source/getting_started/installation.md) -to prepare a matching A3/openEuler environment, then install AFD from the -repository root. - -#### Install AFD - -From the AFD repository root: - -```bash -AFD_BUILD_ASCEND_OPS=1 \ -SOC_VERSION=ascend910_9391 \ -python -m pip install -v --no-build-isolation --no-deps -e . -``` - -`--no-deps` preserves the matched NPU runtime, and `--no-build-isolation` uses -its CANN/torch-npu toolchain. The command above forces the Ascend op build for -reproducibility. Otherwise, leave `AFD_BUILD_ASCEND_OPS` unset to use the -auto-detection described in [Development](#development), set it to `1` if -detection misses an Ascend environment, or set it to `0` to skip the ops. - -#### Verify - -```bash -python - <<'PY' -import torch -import torch_npu -import vllm_ascend - -from afd_plugin.compat.npu import ensure_afd_ascend_ops_loaded - -assert torch.npu.is_available(), "torch-npu cannot see an Ascend device" -ensure_afd_ascend_ops_loaded() -print("AFD_OPS_OK") -PY -``` - -After `AFD_OPS_OK`, the environment is ready to run the NPU examples and E2E -tests. See the -[NPU testing guide](docs/npu/TESTING.md). -For implementation details, see the -[Attention runtime design](docs/design/module/attention_runtime.md) and -[FFN runtime design](docs/design/module/ffn_runtime.md). - -## Using the Plugin - -Install or sync the distribution as `vllm-afd-plugin`. Python imports use the -`afd_plugin` package name. - -AFD is configured through vLLM `--additional-config`. There is no separate -`--afd-config` flag. When AFD is configured and `--worker-cls` is omitted, the -plugin automatically selects the Attention or FFN worker for the active CUDA -or standard Ascend NPU platform. Explicit AFD worker paths remain accepted for -compatibility with existing commands, but are not required or stable launch -interfaces. - -GPU and NPU launches require `VLLM_USE_V2_MODEL_RUNNER=1` on both roles. -Attention uses native ModelRunnerV2; FFN keeps its connector-driven runner. -Neither backend has a V1 fallback. GPU DBO requires two microbatches and accepts Attention -DP1 or DP > 1. -See the [dedicated E2E scenarios](tests/e2e/README.md#gpu-mrv2-dbo-development-comparison). - -```bash -export VLLM_USE_V2_MODEL_RUNNER=1 -``` - -GPU Attention-side shape: - -```bash -vllm serve /path/to/DeepSeek-V2-Lite \ - --served-model-name deepseek-v2-lite-afd-attention \ - --data-parallel-size 1 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --enforce-eager \ - --host 127.0.0.1 \ - --port 18000 \ - --additional-config '{"afd":{"role":"attention","connector":"P2pNcclAFDConnector","host":"127.0.0.1","port":6239,"num_attention_ranks":1,"num_ffn_ranks":1}}' -``` - -GPU FFN-side shape: - -```bash -vllm serve /path/to/DeepSeek-V2-Lite \ - --served-model-name deepseek-v2-lite-afd-ffn \ - --data-parallel-size 1 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --enforce-eager \ - --host 127.0.0.1 \ - --port 18001 \ - --additional-config '{"afd":{"role":"ffn","connector":"P2pNcclAFDConnector","host":"127.0.0.1","port":6239,"num_attention_ranks":1,"num_ffn_ranks":1}}' -``` - -NPU uses the same config channel with `CAMP2pAFDConnector`; the plugin selects -the NPU worker automatically. Attention uses native Ascend ModelRunnerV2: - -```bash -VLLM_USE_V2_MODEL_RUNNER=1 vllm serve /path/to/DeepSeek-V2-Lite \ - --served-model-name deepseek-v2-lite-afd-attention \ - --data-parallel-size 1 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --enforce-eager \ - --host 127.0.0.1 \ - --port 18000 \ - --additional-config '{"afd":{"role":"attention","connector":"CAMP2pAFDConnector","host":"127.0.0.1","port":6239,"num_attention_ranks":1,"num_ffn_ranks":1}}' -``` - -Attention and FFN may be started in either order. Send requests only to the -Attention API server. FFN workers are connector-driven; scheduler-driven FFN -`execute_model()` calls fail fast. - -For repeatable local smoke testing, prefer the bundled runner: - -```bash -export HF_HOME=/path/to/huggingface -python -c 'from datasets import load_dataset; load_dataset("openai/gsm8k", "main")' - -uv run python tests/e2e/runner.py \ - --model /path/to/DeepSeek-V2-Lite \ - --device-backend gpu \ - --scenario afd-eager-2a1f \ - --attention-devices 0,1 \ - --ffn-devices 2 \ - --gsm8k-output-path /tmp/afd-e2e-gsm8k \ - --api-port-base 18000 \ - --afd-port 6239 \ - --common-vllm-arg=--trust-remote-code -``` - -For NPU, use `--device-backend npu`; the runner maps the same device arguments -to `ASCEND_RT_VISIBLE_DEVICES` and selects `CAMP2pAFDConnector`. - -## AFD Config - -The canonical config shape is: - -```json -{ - "afd": { - "role": "attention", - "connector": "P2pNcclAFDConnector", - "async_dp": false, - "host": "127.0.0.1", - "port": 1239, - "afd_process_group_timeout_s": 120, - "num_attention_ranks": 2, - "num_ffn_ranks": 1, - "compute_gate_on_attention": false, - "connector_extra_config": {} - } -} -``` - -`role` must be `attention` or `ffn`. `connector` must be `P2pNcclAFDConnector`, -`DeepEPM2NAFDConnector`, `CAMP2pAFDConnector`, or `CAMAsyncAFDConnector`. -AFD is active when `additional_config["afd"]` is present and passes common AFD config validation; -omit `additional_config["afd"]` to disable AFD. Connector-owned -`connector_extra_config` is strictly validated by the selected connector parser -when the connector/runtime is constructed. The plugin also accepts selected -compatibility aliases such as `afd_role`, `afd_connector`, `afd_host`, and -`afd_port`, plus `async` for `async_dp`. - -`async_dp` defaults to `false` and enables AFD's asynchronous data-parallel -compatibility behavior. It is valid only with `CAMAsyncAFDConnector`, which -requires it to be `true`. - -`afd_process_group_timeout_s` is a positive integer timeout in seconds for the -AFD world process-group rendezvous (default: `120`). Increase it when rank -placement or model loading makes the group join slower; it also bounds the NPU -native DBO startup barrier, not device-kernel execution. +Browse the [documentation index](docs/README.md) for user guides, module design, +validation and troubleshooting. ## Development -Before adding, changing, moving, or removing tests, or changing the behavior they -cover, read and follow the [test maintenance guide](docs/UT_MAINTENANCE.md). - -Run the default CPU-safe checks: +For CPU-only or macOS development, follow [source setup](docs/installation.md#source-setup), +then run: ```bash uv run pytest uv run ruff check . ``` -The NPU-dependent unit suite must run in an Ascend development environment; -see [Running NPU unit tests](docs/npu/TESTING.md). - -Native C/C++ sources are grouped by backend under `csrc/`: Ascend/CANN sources -live in `csrc/npu`, including the `a2e` and `e2a` ACLNN operators, and -`csrc/gpu` is reserved for GPU native sources. - -Ascend custom ops are built automatically only when the build environment looks -like an Ascend NPU platform, for example when `torch_npu`, CANN environment -variables, or the default Ascend toolkit path are present. GPU builds skip -Ascend ops by default. Set `AFD_BUILD_ASCEND_OPS=1` or -`AFD_BUILD_ASCEND_OPS=0` to override the auto-detection. - -## E2E Test - -To run E2E tests, use the [`run-e2e` skill](.agents/skills/run-e2e/SKILL.md). +Read the [contributor rules](AGENTS.md) and [test maintenance guide](docs/UT_MAINTENANCE.md) +before changing code or tests. See [module design](docs/design/module/index.md) +for ownership, [native builds](csrc/README.md) for backend operators, and the +[E2E guide](tests/e2e/README.md) for hardware validation. ## License diff --git a/afd_plugin/connectors/README.md b/afd_plugin/connectors/README.md index e77e20ede..2c5ed3bb8 100644 --- a/afd_plugin/connectors/README.md +++ b/afd_plugin/connectors/README.md @@ -23,7 +23,7 @@ AFD connector implementations are grouped by backend: Both backends target vLLM `0.32.0`; Ascend additionally targets vLLM-Ascend `bd42eea0`. See the -[runtime matrix](../../docs/design/module/execution_platforms.md#tested-runtime-matrix) +[runtime matrix](../../docs/support.md#tested-runtime-matrix) for tested configurations and backend-specific validation limits. Shared connector contracts, metadata containers, factory registration, and diff --git a/csrc/npu/README.md b/csrc/npu/README.md index 017b76f0e..60a96e65d 100644 --- a/csrc/npu/README.md +++ b/csrc/npu/README.md @@ -9,8 +9,8 @@ Attention-to-Expert and Expert-to-Attention transfers: The 910C build also contains four experimental routed-only CAM operators: `afd_async_dispatch_send`, `afd_async_dispatch_recv`, `afd_async_combine_send`, and `afd_async_combine_recv`, under the same `torch.ops.afd_ascend` namespace. -Their compact protocol and future connector migration are documented in -[the routed-only operator guide](../../docs/npu/CAM_ASYNC_ROUTED_OPS.md). +Their compact protocol is documented in +[the routed-only operator guide](../../docs/design/npu/cam_async_routed_ops.md). The NPU native sources live in this directory: @@ -170,7 +170,7 @@ ensure_afd_ascend_ops_loaded() ## Current Scope A2E/E2A serve `CAMP2pAFDConnector`. `CAMAsyncAFDConnector` uses the plugin's -910C routed-only CAM operators; see the [operator contract](../../docs/npu/CAM_ASYNC_ROUTED_OPS.md). +910C routed-only CAM operators; see the [operator contract](../../docs/design/npu/cam_async_routed_ops.md). `grouped_matmul_swiglu_quant_v2_layered` is a fused grouped-matmul + SiLU + dynamic-quantization CAM operator (A8W8 / A4W4 / diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..50d717def --- /dev/null +++ b/docs/README.md @@ -0,0 +1,31 @@ +# AFD documentation + +## Deploy and use + +1. [Install](installation.md) the pinned CUDA or Ascend environment. +2. Check [supported configurations and validation limits](support.md). +3. Choose a [deployment recipe](../recipe/README.md) and configure the + [Attention/FFN roles](configuration.md). + +| Backend | Connector guides | +| --- | --- | +| CUDA | [NCCL P2P](gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md), [DeepEP M2N](gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md) | +| Ascend NPU | [CAMP2P](npu/CAM_P2P_CONNECTOR_USER_GUIDE.md), [CAM async](npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md) | + +## Develop + +- [Module design and code ownership](design/module/index.md) +- [Contribution rules](../AGENTS.md) and [test maintenance](UT_MAINTENANCE.md) +- [Native operator builds](../csrc/README.md) +- NPU design: [Async FFN runtime](design/npu/async_ffn_runtime.md) and + [routed CAM operators](design/npu/cam_async_routed_ops.md) + +## Validate and troubleshoot + +- [Model serving and accuracy E2E](../tests/e2e/README.md), including + [multi-pod deployments](../tests/e2e/multi_pod/README.md) +- [GPU connector tests](../tests/gpu/README.md) +- [NPU runtime and operator tests](../tests/npu/README.md) +- [NPU troubleshooting](npu/TROUBLESHOOTING.md) +- Historical measurement: [Layered W13/W2 graph validation](reports/layered_w2_mdl_validation.md) + (environment, source revisions, and results retained with the report) diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 000000000..a073f0b0a --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,69 @@ +# AFD configuration + +Pass settings through vLLM `--additional-config` under the `afd` key. Omit +`afd` to disable the plugin. When `--worker-cls` is omitted, AFD selects the +Attention or FFN worker for CUDA or standard Ascend NPU. Explicit worker paths +are accepted for compatibility, but are not stable launch interfaces. + +Both roles require `VLLM_USE_V2_MODEL_RUNNER=1`. Attention uses native +ModelRunnerV2; FFN is connector-driven. Send requests only to the Attention API. + +## Common fields + +```json +{ + "afd": { + "role": "attention", + "connector": "P2pNcclAFDConnector", + "async_dp": false, + "host": "127.0.0.1", + "port": 1239, + "afd_process_group_timeout_s": 120, + "num_attention_ranks": 2, + "num_ffn_ranks": 1, + "compute_gate_on_attention": false, + "connector_extra_config": {} + } +} +``` + +| Field | Default | Meaning and constraint | +| --- | --- | --- | +| `role` | `attention` | `attention` or `ffn`. | +| `connector` | `P2pNcclAFDConnector` | One of the four connectors linked below. | +| `async_dp` | `false` | Must be `true` for CAM async and `false` for synchronous connectors. | +| `host` | `127.0.0.1` | Shared, reachable rendezvous host; placement depends on the connector. | +| `port` | `1239` | Rendezvous port in `1..65535`, separate from the HTTP service ports. | +| `afd_process_group_timeout_s` | `120` | Positive rendezvous timeout in seconds; also bounds the NPU native DBO startup barrier, not device-kernel execution. | +| `num_attention_ranks` | `1` | Positive total Attention worker count, including DP/TP ranks. | +| `num_ffn_ranks` | `1` | Positive total FFN worker count, including DP/TP ranks. | +| `compute_gate_on_attention` | `false` | Gate placement; allowed values depend on the connector and model. | +| `connector_extra_config` | `{}` | Connector-specific mapping; unknown keys fail in the selected connector parser. | + +The example describes Attention in a 2A1F deployment. Set `role` to `ffn` on +FFN; all ranks must agree on rendezvous and topology. Role rank is derived at +runtime and is not a configuration field. Increase the rendezvous timeout when +model loading or rank placement delays group initialization. + +## Compatibility aliases + +`afd_connector`, `afd_role`, `afd_host`, `afd_port`, and `async` map to +`connector`, `role`, `host`, `port`, and `async_dp`. Supplying an alias and its +canonical key together is rejected. Unknown top-level AFD keys are rejected; +put connector-specific settings under `connector_extra_config`. + +## Connector-specific settings + +Each guide owns its extra fields, topology rules and launch examples: + +- [NCCL P2P](gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md#configuration): synchronous CUDA; + extra config must be empty. +- [DeepEP M2N](gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md#configuration): synchronous CUDA; + Attention-side routing and FFN TP1. +- [CAMP2P](npu/CAM_P2P_CONNECTOR_USER_GUIDE.md#afd-configuration): synchronous NPU; + both common and connector-local gate settings must be `false`. +- [CAM async](npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md#afd-configuration): NPU; + requires `async_dp=true`, `compute_gate_on_attention=true`, and `dynamicQuant=1`. + +See [supported configurations](support.md) before combining graph, DBO, +parallelism, or quantization settings. diff --git a/docs/design/module/attention_runtime.md b/docs/design/module/attention_runtime.md index 9fe8a528d..ae6b48f28 100644 --- a/docs/design/module/attention_runtime.md +++ b/docs/design/module/attention_runtime.md @@ -173,7 +173,7 @@ Both roles validate the paired V2 topology, but only Attention uses the native V2 model runner. FFN remains connector-driven and uses its existing AFD runner surface. Qwen3.5/3.6 GDN DBO is unsupported; native hybrid/multimodal DBO validation applies. Qwen3 MoE DBO remains supported. See the -[runtime matrix](execution_platforms.md#tested-runtime-matrix) for tested +[runtime matrix](../../support.md#tested-runtime-matrix) for tested configurations and known shutdown limitations. The generic V2 E2E harness remains CUDA-only. @@ -323,7 +323,7 @@ supported topology and environment matrix. CAM async requires eager Attention; FFN can use eager or FULL execution. Feature limits are recorded in the -[platform matrix](execution_platforms.md#tested-runtime-matrix). +[platform matrix](../../support.md#tested-runtime-matrix). ## Ubatching and graph orchestration @@ -385,7 +385,7 @@ the matching platform and connector tests as well. ## Limitations and open issues Platform/connector limits and hardware evidence are centralized in -[execution platforms](execution_platforms.md#tested-runtime-matrix). +[support matrix](../../support.md#tested-runtime-matrix). This document does not decide the runtime refactor, connector metadata ownership, or transfer metadata/state shape. See diff --git a/docs/design/module/compatibility_and_patches.md b/docs/design/module/compatibility_and_patches.md index 380442bc0..86a936d58 100644 --- a/docs/design/module/compatibility_and_patches.md +++ b/docs/design/module/compatibility_and_patches.md @@ -68,7 +68,7 @@ Direct strict calls raise for a missing or different vLLM; plugin registration calls the check with `strict=False`, so it warns and continues. This warning policy does not make another vLLM release supported. Shared EngineCore and coordinator patches use the 0.32 contract. See the -[runtime matrix](execution_platforms.md#tested-runtime-matrix) for hardware +[runtime matrix](../../support.md#tested-runtime-matrix) for hardware validation scope. The NPU implementation targets vLLM `0.32.0` (`5df63a72`) and diff --git a/docs/design/module/execution_platforms.md b/docs/design/module/execution_platforms.md index f95152e80..b52d2ade5 100644 --- a/docs/design/module/execution_platforms.md +++ b/docs/design/module/execution_platforms.md @@ -61,8 +61,9 @@ last_reviewed: 2026-11-09 This document is the primary design for CUDA and NPU mechanisms: runtime class strategy, device graphs, native DBO/ubatching, streams, forward-context -adaptation, profilers, native operators, packaging, and the tested runtime -matrix. [Attention](attention_runtime.md) and [FFN](ffn_runtime.md) retain role +adaptation, profilers, native operators, and packaging. User-facing execution +limits and validation evidence live in the [support reference](../../support.md). +[Attention](attention_runtime.md) and [FFN](ffn_runtime.md) retain role lifecycle orchestration; [connector contracts](connector_contracts.md) retain transport and topology semantics. @@ -197,7 +198,7 @@ Scoped factory and dispatch adapters enable the native path at Attention DP1 as well as DP > 1. AFD accepts exactly two native ubatches. Qwen3.5/3.6 GDN DBO is unsupported; native hybrid/multimodal DBO validation -applies. Qwen3 MoE DBO remains supported. See the [runtime matrix](#tested-runtime-matrix) +applies. Qwen3 MoE DBO remains supported. See the [runtime matrix](../../support.md#tested-runtime-matrix) for tested configurations and known DeepSeek-V2-Lite DBO shutdown failures. ### CUDA profiling @@ -323,7 +324,7 @@ state through `connector.control_plane` before entering `torch.npu.graph(...)`, so replay contains only model and data-plane operations. Replay additionally requires the control payload's `is_graph_replaying=true`; an unknown key or eager payload executes eagerly. CAM async has no control plane and uses the independent -[Async FFN runtime](../../npu/ASYNC_FFN_RUNTIME.md). +[Async FFN runtime](../npu/async_ffn_runtime.md). ### Ascend native operators and packaging @@ -345,7 +346,7 @@ NPU extension, but the CAMP2P data path cannot run without it. all four `torch.ops.afd_ascend.afd_async_*` operators. The 910C package uses compact routed-only metadata; Attention owns shared experts and FFN owns routed experts. The 950 package does not register these async operators. -See [the routed-only guide](../../npu/CAM_ASYNC_ROUTED_OPS.md) for metadata +See [the routed-only guide](../npu/cam_async_routed_ops.md) for metadata and window lifetime contracts. Device validation remains necessary for each target topology. @@ -364,66 +365,6 @@ experimental output, role-specific defaults, optional stacks/modules, and a TensorBoard trace handler. Runners step the profiler on execution and stop it during shutdown. -## Tested runtime matrix - -This table separates accepted runtime configurations from validation evidence. -Both backends require ModelRunnerV2. A test case or a historical result does not -qualify every model/topology combination on the current target. - -| Platform/path | Execution | Ubatching | Routing/quantization limits | Evidence | -| --- | --- | --- | --- | --- | -| CUDA V2 + `P2pNcclAFDConnector` | Eager or `FULL_DECODE_ONLY` native V2 CUDA Graph | Exactly two native microbatches; Attention DP1 or DP > 1 | Model-specific gate placement; PP/CP, elastic EP, EPLB, SP MoE, and compile SP rejected; role ranks equal DP x TP | Source-specific GPU validation is linked below; coverage is limited to the recorded model/topology combinations. DeepSeek-V2-Lite has known DBO shutdown failures. Qwen3.5/3.6 GDN DBO is unsupported. | -| CUDA V2 + `DeepEPM2NAFDConnector` | Eager or `FULL_DECODE_ONLY` native V2 CUDA Graph; SM90 or newer | Exactly two native microbatches, enabled on both roles; Attention DP1 or DP > 1, TP1 or TP2 | Qwen3 MoE (all layers MoE, no shared expert) or DeepSeek-V2; `compute_gate_on_attention=true`; FFN TP1 with EP over all FFN ranks, linear placement, `allgather_reducescatter` all2all and the Triton MoE weight layout; BF16 or block-128 FP8 payload; EPLB, PP/CP and SP MoE rejected | Historical vLLM 0.30 Qwen3-30B-A3B 2A2F GSM8K E2E for eager, graph, graph + DBO, and Attention TP2 eager/graph on H100 and H200, and for eager, graph and graph + DBO on GB200; two-node 2A2F eager, graph and graph + DBO with Attention and FFN on separate nodes. DeepSeek-V2-Lite: [E2E definitions](../../../tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py). [GPU connector device tests](../../../tests/gpu/README.md) | -| Ascend V2 + `CAMP2pAFDConnector` | Eager, `FULL`, or `FULL_DECODE_ONLY` native V2 ACL Graph | Eager or graph native DBO with two ubatches and Attention DP > 1; plain MLA graph DBO requires `FULL_DECODE_ONLY` | `compute_gate_on_attention=false`; PP/CP, elastic EP and EPLB rejected; model-specific SP policy; role ranks equal DP x TP | Historical 0.30 V2-Lite 2A2F FULL 300-question comparison and 2A1F functional checks; current 0.32 Lite evidence is in the [validation record](https://github.com/vllm-project/afd-plugin/pull/507) | -| Ascend V2 + `CAMAsyncAFDConnector` | Eager Attention; layered FFN eager or FULL | Two native DBO microbatches on Attention, including DP1 and with FFN FULL; FFN stays outside the native DBO schedule | Experimental DeepSeek-V4 W4A8/W8A8 on A3/910C; `async_dp=true`, `compute_gate_on_attention=true`, and `dynamicQuant=1`; PP/CP unsupported; plugin-owned async CAM ops required | Historical DSV4 layered comparisons and limitations are in the [validation record](https://github.com/vllm-project/afd-plugin/pull/507); they predate removal of the legacy path | - -The 300-question Ascend evidence in this table is historical. The 0.32 -[validation record](https://github.com/vllm-project/afd-plugin/pull/507) separately lists the -eight Lite 100-question cells, W4A8 fixture, and observed shutdown failures. - -The Async CAM row supports DeepSeek-V4 with static EP, PP=PCP=DCP=1, -and Attention-local SP. DeepSeek-V2/V2-Lite are no longer supported by -Async CAM; standard DeepSeek-V3.2 was already rejected by model admission. -Finegrained TP, explicit shared-expert DP configuration, and native -MTP and draft-model paths other than DeepSeek-V4 DSpark are excluded. DSpark -keeps Attention/native proposal eager and permits independent layered FFN FULL; -it does not support DBO. -FFN remains connector-driven. The row's upgrade evidence retains its original -scope; MRV2 deployment settings are documented in the -[CAM async guide](../../npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md#modelrunnerv2-deployment). - -Both backends target vLLM `0.32.0` (`5df63a72`); Ascend additionally targets -vLLM-Ascend `bd42eea0`. Source-specific -H20 model, topology, execution-mode and accuracy results are recorded in -the [GPU upgrade validation matrix](https://github.com/vllm-project/afd-plugin/pull/501) -and [runtime-fix validation matrix](https://github.com/vllm-project/afd-plugin/pull/502). -Those records identify the exact tested revisions. Earlier GPU results on -vLLM 0.30 do not establish vLLM 0.32 hardware qualification. - -Validation limits: - -- GPU accuracy evidence covers selected first-300 GSM8K comparisons, not - full-dataset accuracy, numerical equivalence, or performance improvement. -- DeepSeek-V2-Lite TP2 Graph + DBO and DP2 eager DBO can require force - killing during shutdown. Inference success does not imply graceful exit. -- Native V4 DBO requires a compatible upstream communication backend. The - recorded native V4 accuracy reference uses Graph without DBO. -- NPU 0.32 evidence is limited to the configurations and checks in the - [validation record](https://github.com/vllm-project/afd-plugin/pull/507); the seven AFD Lite - cases failed graceful exit. Historical NPU results do not qualify 0.32. - -Historical [NPU validation](https://github.com/vllm-project/afd-plugin/pull/425#issuecomment-6063910923) -and [DBO repair](https://github.com/vllm-project/afd-plugin/pull/430#issuecomment-6063840311) -records retain their original versions and sample counts. Synchronous DSV4 -profiles remain smoke-only, with known incorrect A5 concurrent answers; see -their [E2E limits](../../../tests/e2e/README.md#dsv4-flash-sync-camp2p-concurrent-requests-local-4-or-16-npus). -GPU/NPU rank topology and connector resource rules remain owned by -[connector contracts](connector_contracts.md). - -The repository does not record a canonical CUDA container or assert an -official tag for the Ascend source pin `bd42eea0`; environment evidence is -not an authoritative package tag. - ## Failure and cleanup boundaries Unsupported graph, ubatch, async, gate, or quantization combinations fail in diff --git a/docs/design/module/ffn_runtime.md b/docs/design/module/ffn_runtime.md index 2f1af04f4..a8a9dc128 100644 --- a/docs/design/module/ffn_runtime.md +++ b/docs/design/module/ffn_runtime.md @@ -179,7 +179,7 @@ The V2 pair requires PP/PCP/DCP size 1, configured role ranks equal to DP x TP, static EP, and a registered AFD model. CUDA uses `P2pNcclAFDConnector` with model-specific gate placement and supports two native microbatches with Attention DP1 or DP > 1. See the -[runtime matrix](execution_platforms.md#tested-runtime-matrix) for tested +[runtime matrix](../../support.md#tested-runtime-matrix) for tested configurations and known DeepSeek-V2-Lite DBO shutdown failures. CUDA `DeepEPM2NAFDConnector` requires Attention-side gating, FFN TP 1, and the same DBO setting on both roles; see the @@ -257,7 +257,7 @@ loop. For each layer, the runner: Async eager does not use the synchronous DP-metadata graph cache. Its FULL mode captures a complete original DR/FFN/CS transaction with device-side layer selection. The independent Async runner owns one graph and stable resources; -see [Async FFN runtime](../../npu/ASYNC_FFN_RUNTIME.md) for startup, supported +see [Async FFN runtime](../npu/async_ffn_runtime.md) for startup, supported W4A8/W8A8 configurations and the original receiver's process-cleanup boundary. DeepSeek-V4 W4A8 and W8A8 use layered GMM for both eager and FULL. Incompatible @@ -373,7 +373,7 @@ thread does not exit, native resources are not released underneath it. The connector then releases its remaining communication resources during runner shutdown. An earlier daemon failure is still surfaced after the join. Ascend uses its own connector close or asynchronous drain path before joining the -daemon; see the [Async FFN runtime](../../npu/ASYNC_FFN_RUNTIME.md). +daemon; see the [Async FFN runtime](../npu/async_ffn_runtime.md). ## Candidate invariants @@ -398,7 +398,7 @@ async model E2E coverage. ## Limitations and open issues Platform-specific execution limits and validation evidence are centralized in -[execution platforms](execution_platforms.md#tested-runtime-matrix). +[support matrix](../../support.md#tested-runtime-matrix). Issue [#107](https://github.com/JiusiServe/afd-plugin/issues/107) completed the optional control-plane split. Connector metadata ownership, transfer state diff --git a/docs/design/module/index.md b/docs/design/module/index.md index c37d4c1d7..80a302433 100644 --- a/docs/design/module/index.md +++ b/docs/design/module/index.md @@ -21,7 +21,7 @@ upstream_refs: - "vLLM-Ascend target bd42eea01d8312c4df469fabaed45f77a12df4d3" verified_platform_refs: - "CUDA: tests/e2e marked e2e with AFD_E2E_BACKEND=gpu; no canonical image is recorded" - - "Ascend: current target in root installation instructions; V2-Lite and DSV4 Flash W4A8 evidence in the execution-platform matrix; older environments remain historical" + - "Ascend: current target in docs/installation.md; V2-Lite and DSV4 Flash W4A8 evidence in docs/support.md; older environments remain historical" related_issues: - "#129" last_reviewed: 2026-11-09 @@ -35,16 +35,8 @@ verify boundaries, invariants, and validation evidence. The documents record plugin, role runtime, connector, model, platform, compatibility, and E2E testing contracts; open interfaces remain explicitly draft. -The 2026-11-09 documentation review checked path ownership, configuration, -guide links, and upstream references against the current checkout. GPU and -NPU target vLLM `0.32.0` (`5df63a72`); the Ascend integration targets -vLLM-Ascend `bd42eea0`, as recorded in the -[installation instructions](../../../README.md#ascend-npu-installation). -The refreshed `last_reviewed` dates record documentation consistency checks; -hardware validation remains scoped to the -[runtime matrix](execution_platforms.md#tested-runtime-matrix) and the -[0.32 NPU validation record](https://github.com/vllm-project/afd-plugin/pull/507). -Existing document statuses and historical measurements retain their scope. +`last_reviewed` records documentation review, not hardware qualification. +For installation, deployment and validation, use the [documentation index](../../README.md). ## Reading order @@ -66,17 +58,6 @@ Dependencies flow from role and integration modules toward shared boundaries; platform and compatibility modules adapt those boundaries to upstream runtimes. A lower-level document must not depend on a role implementation. -```text -attention_runtime ----+----> connector_contracts ----> plugin_boundary - | | -ffn_runtime ----------+ +----> execution_platforms - | -model_integration ----+----> execution_platforms - -attention_runtime ----+ -ffn_runtime ----------+----> compatibility_and_patches ----> plugin_boundary -``` - ```mermaid flowchart LR ATT["Attention runtime"] --> CONN["Connector contracts"] @@ -127,21 +108,3 @@ The repository-level test contract is owned separately by Only identified invariant blocks in a `normative` document may use `MUST`, `MUST NOT`, or `SHOULD` as contract terms. - -## User and operational guides - -Operational guides remain separate from normative module design: - -- [NCCL P2P connector guide](../../gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md) -- [DeepEP M2N connector guide](../../gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md) -- [CAM P2P connector guide](../../npu/CAM_P2P_CONNECTOR_USER_GUIDE.md) -- [CAM async connector guide](../../npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md) -- [CAM routed operators](../../npu/CAM_ASYNC_ROUTED_OPS.md) -- [Async FFN runtime](../../npu/ASYNC_FFN_RUNTIME.md) -- [Layered W13/W2 graph validation](../../npu/LAYERED_W2_MDL_VALIDATION.md) -- [NPU validation](../../npu/TESTING.md) and [troubleshooting](../../npu/TROUBLESHOOTING.md) -- [Ascend NPU installation](../../../README.md#ascend-npu-installation) -- [Ascend NPU custom ops build](../../../csrc/npu/README.md) -- [Connector overview](../../../afd_plugin/connectors/README.md) -- [Deployment recipes](../../../recipe/README.md) -- [E2E test guide](../../../tests/e2e/README.md) diff --git a/docs/design/module/model_integration.md b/docs/design/module/model_integration.md index 6bbb97883..95893cdc2 100644 --- a/docs/design/module/model_integration.md +++ b/docs/design/module/model_integration.md @@ -31,7 +31,7 @@ verified_platform_refs: - "DeepSeek V2 Lite GPU and NPU model E2E paths" - "DeepSeek V2 Lite DeepEP M2N 2A2F E2E paths (eager, graph, graph + DBO, Attention TP2)" - "CAM async NPU model E2E path" - - "DeepSeek V4 CUDA contract tests and E2E profiles; see execution-platform matrix for hardware scope" + - "DeepSeek V4 CUDA contract tests and E2E profiles; see docs/support.md for hardware scope" - "DeepSeek V4 Ascend Hash-id boundary has focused unit coverage and a historical manual Ascend 950 run" related_issues: - "#86" @@ -150,7 +150,7 @@ CUDA-only and requires synchronous `P2pNcclAFDConnector`, sequence-parallel MoE, EPLB, and the `deep_gemm_mega_moe` backend. The P2P connector validates one-dimensional `torch.int32` input IDs and preallocates their receive buffers for graph execution. See the -[runtime matrix](execution_platforms.md#tested-runtime-matrix) for tested +[runtime matrix](../../support.md#tested-runtime-matrix) for tested CUDA configurations and validation limits. ### DeepSeek V4 Ascend boundary diff --git a/docs/design/module/plugin_boundary.md b/docs/design/module/plugin_boundary.md index a1dbd2880..ca4ea25cc 100644 --- a/docs/design/module/plugin_boundary.md +++ b/docs/design/module/plugin_boundary.md @@ -129,21 +129,9 @@ presence is the activation signal: `parse_optional_afd_config()` returns raise if the required namespace is missing. Unknown top-level AFD keys are rejected and connector-specific values must be placed under `connector_extra_config`. -| Canonical key | Default | Current meaning | -| --- | --- | --- | -| `connector` | `P2pNcclAFDConnector` | Connector factory name selected from the built-in supported-name allow-list. | -| `async_dp` | `false` | Enables AFD async-DP compatibility behavior; currently valid only with `CAMAsyncAFDConnector`. | -| `role` | `attention` | Process role: `attention` or `ffn`. | -| `host`, `port` | `127.0.0.1`, `1239` | Connector rendezvous/control endpoint inputs. | -| `afd_process_group_timeout_s` | `120` | Positive AFD process-group rendezvous timeout in seconds; also bounds the NPU native DBO startup barrier. It is not a device-kernel cancellation deadline. | -| `num_attention_ranks`, `num_ffn_ranks` | `1`, `1` | AFD role-group sizes used by topology construction. | -| `compute_gate_on_attention` | `false` | Moves supported gate/MoE routing work to Attention. CUDA supports both gate placements at the remote-experts boundary; synchronous CAMP2P still requires `false`, while CAM async requires `true`. | -| `connector_extra_config` | `{}` | Mapping stored on `AFDConfig` and parsed by the selected connector into a typed `ConnectorExtraInfo`. | - -The compatibility aliases `afd_connector`, `afd_role`, `afd_port`, `afd_host`, -and `async` normalize to canonical fields. Supplying an alias and its canonical -field together is rejected. Boolean-like strings and integer-like values are -normalized; invalid types fail during parsing. +Public fields, defaults and aliases are defined in the +[configuration reference](../../configuration.md). Parsing normalizes +boolean-like strings and integer-like values; invalid types fail. Role rank is runtime state derived from the global DP rank and local PCP/TP coordinates. It is resolved once by the connector factory and is not an @@ -177,7 +165,7 @@ communication resources are created. Both roles must use PP/PCP/DCP size 1, role ranks equal to DP x TP, static EP, and a registered AFD model; elastic EP and EPLB are rejected. CUDA V2 requires `P2pNcclAFDConnector`, model-specific gate placement, and exactly two microbatches with Attention DP1 or DP > 1. -See the [runtime matrix](execution_platforms.md#tested-runtime-matrix) for +See the [runtime matrix](../../support.md#tested-runtime-matrix) for tested configurations and known DeepSeek-V2-Lite DBO shutdown failures. Ascend V2 accepts synchronous `CAMP2pAFDConnector` with FFN-side gating or eager `CAMAsyncAFDConnector` with async DP and Attention-side gating. Ascend @@ -207,7 +195,7 @@ runtime integration and are not stable third-party extension interfaces. Profiling and connector diagnostics are read by their owning modules. Environment switches do not replace `additional_config["afd"]` as the activation or topology channel. See the -[runtime matrix](execution_platforms.md#tested-runtime-matrix) for hardware +[runtime matrix](../../support.md#tested-runtime-matrix) for hardware validation scope. `AFD_FORCE_SPAWN_MULTIPROCESSING=1` is an opt-in package-import adaptation for diff --git a/docs/npu/ASYNC_FFN_RUNTIME.md b/docs/design/npu/async_ffn_runtime.md similarity index 99% rename from docs/npu/ASYNC_FFN_RUNTIME.md rename to docs/design/npu/async_ffn_runtime.md index 52b67eac1..299dab9c9 100644 --- a/docs/npu/ASYNC_FFN_RUNTIME.md +++ b/docs/design/npu/async_ffn_runtime.md @@ -30,7 +30,7 @@ The combined layer table and startup agreement cover target0–42/draft43–45; all layers must share the same W4A8 or W8A8 layout. Native P loads the complete draft model. Target/draft Attention and the native speculative configuration remain eager. DSpark excludes DBO; see the -[DSpark recipe](../../recipe/npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md). +[DSpark recipe](../../../recipe/npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md). W8A8 reuses the loaded INT8 NZ weights, FP32 W13 scales and BF16 W2 scales. It needs no W4A8 compensation. The W2 binding requires per-layer bias entries, diff --git a/docs/npu/CAM_ASYNC_ROUTED_OPS.md b/docs/design/npu/cam_async_routed_ops.md similarity index 98% rename from docs/npu/CAM_ASYNC_ROUTED_OPS.md rename to docs/design/npu/cam_async_routed_ops.md index bd17ec6b3..d51399309 100644 --- a/docs/npu/CAM_ASYNC_ROUTED_OPS.md +++ b/docs/design/npu/cam_async_routed_ops.md @@ -36,7 +36,7 @@ snapshot, under `src/comm_operator/ascend_kernels/`. The integration renames operator symbols and shared headers for AFD isolation while preserving the compact routed-only communication layout. The complete upstream MIT copyright and permission notice is retained in -[the repository license](../../LICENSE) and included in package license data. +[the repository license](../../../LICENSE) and included in package license data. ## Communication window configuration @@ -70,7 +70,7 @@ the extension CMake configuration (`AFD_SOC_VERSION`). `AFD_SKIP_ACLNN_BUILD=1` requires an existing, matching AFD vendor package with all four async ACLNN symbols. Rebuild the complete operator package after changing its source or SOC. Use the dependency stack in the -[installation guide](../../README.md#ascend-npu-installation). +[installation guide](../../installation.md#ascend-npu-installation). Load the package using the lazy loader before accessing the operators: @@ -230,7 +230,7 @@ unused cells and reuses a destination base while its TP peer remains unchanged. These changes preserve the compact protocol and do not add concurrent windows or remove empty-expert completion notifications. -Use [the standalone precision and performance harness](../../tests/npu/README.md) +Use [the standalone precision and performance harness](../../../tests/npu/README.md) to compare identical routing, TP width, window size, dtype, quantization and chunk capacity before claiming a speedup. A source optimization alone is not performance evidence; include sparse/empty peers and repeated window reuse. diff --git a/docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md b/docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md index 5c18a2476..d87287845 100644 --- a/docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md +++ b/docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md @@ -80,26 +80,10 @@ AFD configuration is supplied through vLLM's `--additional-config` under the `af } ``` -### Fields - -| Field | Type | Default | Required / meaning | -| --- | --- | --- | --- | -| `role` | `"attention" \| "ffn"` | `"attention"` | Role owned by this process. Attention sends hidden states; FFN receives and returns FFN outputs. | -| `connector` | `str` | `"P2pNcclAFDConnector"` | Must be `P2pNcclAFDConnector` for this GPU path. | -| `host` | `str` | `"127.0.0.1"` | Non-empty rendezvous/control-plane host. All participating ranks must use a reachable, identical value. Host must be the first rank of FFN. | -| `port` | `int` | `1239` | Rendezvous port on `host`, valid range `1..65535`. It must be free and reachable. Subgroups share this rendezvous and need no ports of their own. | -| `num_attention_ranks` | `int` | `1` | Total number of AFD Attention ranks, including DP/TP-derived worker ranks. Must be positive. | -| `num_ffn_ranks` | `int` | `1` | Total number of AFD FFN ranks, including DP/TP-derived worker ranks. Must be positive. | -| `compute_gate_on_attention` | `bool` | `false` | When `false`, FFN owns the native gate and experts. When `true`, Attention owns the native gate and transfers router logits to the FFN external-router expert path. | -| `afd_process_group_timeout_s` | `int` | `120` | Rendezvous timeout, in seconds, for the AFD world process group join. Must be positive. The right value depends on rank count, placement, and model size; deployment tooling should compute it and pass it in. | -| `connector_extra_config` | `dict` | `{}` | Must remain empty; `P2pNcclAFDConnector` does not currently support connector-specific options. | -| `async` / `async_dp` | `bool` | `false` | Must remain `false` for `P2pNcclAFDConnector`; AFD async mode requires `CAMAsyncAFDConnector`. | - -Compatibility aliases currently accepted are `afd_role`, `afd_connector`, `afd_host`, and `afd_port`. Canonical field names should be used in new examples. - -The connector factory derives each process's role rank from its global DP rank -and local PCP/TP coordinates before connector construction. Do not configure a -role rank. +Use the [common configuration reference](../configuration.md) for defaults +and aliases. P2P requires `async_dp=false`, empty `connector_extra_config`, +and `host` on FFN rank 0. DeepSeek supports either gate placement; Qwen +requires `compute_gate_on_attention=false`. ## Topology rules @@ -131,7 +115,7 @@ export VLLM_USE_V2_MODEL_RUNNER=1 ``` The commands below illustrate a DP1/TP1 launch. See the -[runtime matrix](../design/module/execution_platforms.md#tested-runtime-matrix) +[runtime matrix](../support.md#tested-runtime-matrix) for tested configurations and known shutdown limitations. Attention side: @@ -183,7 +167,7 @@ exactly two microbatches and accepts Attention DP1 or DP > 1. The generic [GPU E2E runner](../../tests/e2e/runner.py) defines eager/graph DP2 and TP2 scenarios, including `afd-v2-eager-dbo-dp2` and `afd-v2-graph-dbo-dp2`. See the -[runtime matrix](../design/module/execution_platforms.md#tested-runtime-matrix) +[runtime matrix](../support.md#tested-runtime-matrix) for the tested scope and known DeepSeek-V2-Lite shutdown failures. A registered model does not qualify every topology. diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 000000000..94e0eba3b --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,92 @@ +# Installation + +Requires Python **3.11-3.13** and [uv](https://docs.astral.sh/uv/). + +## Source setup + +```bash +git clone https://github.com/vllm-project/afd-plugin.git +cd afd-plugin +uv sync --group dev +``` + +`vllm` is an optional runtime extra so CPU-only or macOS development +environments can still run import/config tests without a CUDA wheel. + +## GPU installation + +For Linux / CUDA-capable environments only; Ascend users should skip this +command: + +```bash +uv sync --group dev --extra vllm +``` + +The optional GPU extra pins `vllm==0.32.0`. + +`DeepEPM2NAFDConnector` also needs the `afdep` extra (afdEP-comm) and +DeepGEMM, which is built from source; see +[Installation](gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md#installation) in +the connector guide. + +```bash +uv sync --group dev --extra vllm --extra afdep +``` + +## Ascend NPU installation + +Use the pinned source pair and tested stack below. Install a compatible driver +and firmware, then confirm device visibility with `npu-smi info`. + +| Component | Version | +| --- | --- | +| Python | Package range `3.11–3.13`; tested `3.12.13` | +| vLLM | `0.32.0`, commit `5df63a7203cdb9328d824a27b682e77c7b9928be` | +| vLLM-Ascend | commit [`bd42eea0`](https://github.com/vllm-project/vllm-ascend/commit/bd42eea01d8312c4df469fabaed45f77a12df4d3) | +| Tested CANN / torch / torch-npu | `9.2` / `2.10.0+cpu` / `2.10.0.post7` | +| Tested Triton Ascend / CAM | `3.6` / `1.3` | + +See the [support reference](support.md#validation-limits) for hardware coverage +and known shutdown limitations. + +### Environment + +The recorded A3 environment uses the pinned pair and stack above. The +repository does not claim a released container tag for the Ascend target. Use the +[installation guide at that source snapshot](https://github.com/vllm-project/vllm-ascend/blob/bd42eea01d8312c4df469fabaed45f77a12df4d3/docs/source/getting_started/installation.md) +to prepare a matching A3/openEuler environment, then install AFD from the +repository root. + +### Install AFD + +From the AFD repository root: + +```bash +AFD_BUILD_ASCEND_OPS=1 \ +SOC_VERSION=ascend910_9391 \ +python -m pip install -v --no-build-isolation --no-deps -e . +``` + +`--no-deps` preserves the matched runtime; `--no-build-isolation` uses its +CANN/torch-npu toolchain. `AFD_BUILD_ASCEND_OPS=1` forces the native build; +`0` skips it. When unset, setup detects the Ascend environment. See the +[native build guide](../csrc/npu/README.md) for build controls and artifacts. + +### Verify + +```bash +python - <<'PY' +import torch +import torch_npu +import vllm_ascend + +from afd_plugin.compat.npu import ensure_afd_ascend_ops_loaded + +assert torch.npu.is_available(), "torch-npu cannot see an Ascend device" +ensure_afd_ascend_ops_loaded() +print("AFD_OPS_OK") +PY +``` + +After `AFD_OPS_OK`, choose a [deployment recipe](../recipe/README.md). +Use the [NPU test guide](../tests/npu/README.md) to validate the runtime. diff --git a/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md b/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md index 31eeb502a..df23e1314 100644 --- a/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md +++ b/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md @@ -6,9 +6,9 @@ exchange routed-expert activations with independent FFN expert ranks through CAM async dispatch/combine operators. Use the pinned dependency pair in the -[root installation instructions](../../README.md#ascend-npu-installation). +[installation instructions](../installation.md#ascend-npu-installation). Current runtime constraints and validation evidence are summarized in the -[runtime matrix](../design/module/execution_platforms.md#tested-runtime-matrix). +[runtime matrix](../support.md#tested-runtime-matrix). > [!WARNING] > CAM async is experimental. V4 true DBO boundary qualification and performance @@ -32,7 +32,7 @@ unsplit locally. Both token thresholds must be at least two. Attention DP replicas make that decision independently; FFN is connector-driven. FFN also supports the experimental complete-transaction FULL mode described in -[Async FFN runtime](ASYNC_FFN_RUNTIME.md#selection-and-startup). +[Async FFN runtime](../design/npu/async_ffn_runtime.md#selection-and-startup). FULL requires MRV2 on both roles and accepts Attention-side DBO under the existing constraints. Attention graph execution remains unsupported. @@ -168,31 +168,17 @@ There is no separate `--afd-config` option. } ``` -### Common fields - -| Field | Type | Default | Meaning and constraint | -| --- | --- | --- | --- | -| `role` | `"attention" \| "ffn"` | `"attention"` | Role owned by this process. | -| `connector` | `str` | `"P2pNcclAFDConnector"` | Must be `CAMAsyncAFDConnector`. | -| `async` / `async_dp` | `bool` | `false` | Must be `true`. `async` is the accepted compatibility alias for canonical `async_dp`. | -| `host` | `str` | `"127.0.0.1"` | **Must be the IP address of the node that owns Attention rank 0.** Every rank must use the same reachable value. | -| `port` | `int` | `1239` | HCCL rendezvous port in `1..65535`; it must be free and reachable. | -| `afd_process_group_timeout_s` | `int` | `120` | Positive AFD group rendezvous timeout in seconds; also bounds the NPU native DBO startup barrier, not device receive polling. | -| `num_attention_ranks` | `int` | `1` | Total Attention ranks across all DP and TP groups. | -| `num_ffn_ranks` | `int` | `1` | Total FFN expert ranks. | -| `compute_gate_on_attention` | `bool` | `false` | Must be `true`; CAM async runs MoE routing on Attention before dispatching to FFN ranks. | -| `connector_extra_config` | `dict` | `{}` | Connector-specific settings. Unknown top-level AFD fields are rejected. | - -Compatibility aliases `afd_role`, `afd_connector`, `afd_host`, `afd_port`, -and `async` are accepted. New configurations should use canonical names, -including `async_dp`; enable DBO through the Attention CLI flags below. +Use the [common configuration reference](../configuration.md) for defaults +and aliases. CAM async requires `async_dp=true`, +`compute_gate_on_attention=true`, and `host` on Attention rank 0. +Enable native DBO through Attention CLI flags, not connector extra config. ### CAM async `connector_extra_config` | Field | Type | Default | Meaning and constraint | | --- | --- | --- | --- | | `dynamicQuant` | `int` | `0` | Set to `1`: FFN receives dynamically quantized routed activations and scale tensors. The default `0` is not supported by the layered FFN path. | -| `ffn_mega_moe` | `bool` | `false` | Selects Mega FFN with Pull transport; requires `dynamicQuant=1` and the same setting on all A/F ranks. See the [Pull execution session](ASYNC_FFN_RUNTIME.md#opt-in-pull-execution-session). | +| `ffn_mega_moe` | `bool` | `false` | Selects Mega FFN with Pull transport; requires `dynamicQuant=1` and the same setting on all A/F ranks. See the [Pull execution session](../design/npu/async_ffn_runtime.md#opt-in-pull-execution-session). | | `attn_ranks_per_dp` | `int` | `1` | Positive Attention TP rank count per DP replica. It is the CAM Attention grouping width and is independent of the FFN process's local TP size. | | `hccl_buffer_size` | `int` | unset | Positive CAM HCCL buffer size in MB. The override applies only to the `afd_async_cam` communication domain. When unset, HCCL uses `HCCL_BUFFSIZE`, then its built-in default. Configure the same value on every Attention and FFN rank in the domain. | @@ -243,7 +229,7 @@ as DP > 1. The Ascend adapter extends native factory and threshold selection to DP1 because upstream otherwise returns early there. Async child contexts start with `dp_metadata=None`; FFN DP coordination belongs to the connector. DP1 model correctness still needs the hardware checks in the -[NPU validation guide](TESTING.md#native-dbo-contracts-and-device-execution). +[NPU validation guide](../../tests/npu/README.md#native-dbo-contracts-and-device-execution). For shape-only runtime diagnostics, set `AFD_ASYNC_MOE_LAYOUT_LOG=1` on Attention. The log reports the parent token @@ -253,7 +239,7 @@ It does not inspect tensor values or force a device synchronization. ## Requirements Use Ascend 910C, the pinned stack from the -[root installation instructions](../../README.md#ascend-npu-installation), +[installation instructions](../installation.md#ascend-npu-installation), and plugin-owned source-built operators on every node. Build the AFD operators from the repository root inside the container: @@ -278,7 +264,7 @@ rank: Async CAM host tiling reads the environment, whereas HCCL allocation can use the domain override. For the JSON example above, export `HCCL_BUFFSIZE=4096`. Changing only `hccl_buffer_size` can misalign the window capacity and tail-state address. `HCCL_BUFFSIZE` also affects unrelated groups -without explicit overrides; see the [operator window contract](CAM_ASYNC_ROUTED_OPS.md#communication-window-configuration). +without explicit overrides; see the [operator window contract](../design/npu/cam_async_routed_ops.md#communication-window-configuration). For the experimental FlashComm1/SP cases, set `VLLM_ASCEND_ENABLE_FLASHCOMM1=1` only on Attention. Set it to `0` for plain TP @@ -291,8 +277,8 @@ actual vendor library path. Missing source operators fail startup. ## Current limitations - Attention is eager. FFN FULL targets homogeneous W4A8 or W8A8 layers on A3/910C; - see [supported configuration](ASYNC_FFN_RUNTIME.md#selection-and-startup) - and [validation commands](ASYNC_FFN_RUNTIME.md#validation). + see [supported configuration](../design/npu/async_ffn_runtime.md#selection-and-startup) + and [validation commands](../design/npu/async_ffn_runtime.md#validation). Broader models and topologies require separate validation. - Regular prefill and autoregressive decode steps are supported; context parallel execution remains outside the supported Async CAM topology. @@ -325,7 +311,7 @@ per-channel W4A8, or dynamic W8A8 with FP32 W13/BF16 W2 per-channel scales. W8A8 uses loaded INT8 NZ tensors and a single shared zero INT32 W2 bias; it does not require W4A8 compensation or duplicate expert weights. Historical DSV4 Flash W4A8 layered off/on comparisons and operator traces are in the -[runtime validation record](../design/module/execution_platforms.md#tested-runtime-matrix). +[runtime validation record](../support.md#tested-runtime-matrix). Per-group operator coverage does not extend the checkpoint loader to per-group weights. Shared experts remain on Attention. DSV4 applies routed scaling in top-k, so @@ -336,7 +322,7 @@ For positive `L`, it computes `SiLU(min(gate, L)) * clamp(up, -L, L)` before quantization; zero keeps clipping disabled. This restores DeepSeek-V4's configured limit of `10.0`. Layered W8A8 retains its unclipped computation. FULL reuses the existing native DR and CS operators. See -[resource lifetime](ASYNC_FFN_RUNTIME.md#resource-lifetime) for completion and +[resource lifetime](../design/npu/async_ffn_runtime.md#resource-lifetime) for completion and process-cleanup requirements. Missing W4A8 `w13_scale_bias` or `w2_scale_bias`, dynamic EPLB, non-SiLU activation, incompatible parameters, other quantizations, mixed quantization, @@ -360,8 +346,8 @@ counts, parameter rejection, startup selection, and the absence of explicit metadata D2H in the Python path. They do not establish NPU numerical accuracy, empty-work completion, hidden synchronization, or a performance gain. Run the compiled Meta and NPU numerical tests described in the -[testing guide](TESTING.md#layered-w4a8-validation) and the -[FULL runtime checks](ASYNC_FFN_RUNTIME.md#validation). Deployments require +[testing guide](../../tests/npu/README.md#layered-operator-precision) and the +[FULL runtime checks](../design/npu/async_ffn_runtime.md#validation). Deployments require target-device validation of completion, checkpoint accuracy and performance. To switch from FULL to layered eager, pass `--enforce-eager` before restarting diff --git a/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md b/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md index c614b83b3..bc11c3064 100644 --- a/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md +++ b/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md @@ -64,7 +64,7 @@ decoding. The pair requires static EP, PP=PCP=DCP=1, and each role's ranks equal to DP x TP. Sequence parallelism follows the model contract; synchronous DeepSeek-V2/V3 rejects effective model SP. Native MRV2 owns request slicing, capture, graph selection, replay and output merge. -AFD supplies stage-specific Ascend metadata and CAMP2P control/communication. The [NPU testing guide](TESTING.md) separates executor +AFD supplies stage-specific Ascend metadata and CAMP2P control/communication. The [NPU testing guide](../../tests/npu/README.md) separates executor checks from model correctness evidence. DBO is configured with vLLM CLI flags, not inside the `afd` object: @@ -132,27 +132,10 @@ Pass AFD configuration through vLLM's `--additional-config` option under the } ``` -### Fields - -| Field | Type | Default | Meaning and constraints | -| --- | --- | --- | --- | -| `role` | `"attention" \| "ffn"` | `"attention"` | Role owned by this process. | -| `connector` | `str` | `"P2pNcclAFDConnector"` | Set to `CAMP2pAFDConnector` for this synchronous NPU path. | -| `host` | `str` | `"127.0.0.1"` | Non-empty rendezvous host shared by every rank. It must be reachable from all participating processes. If the Attention and FFN ranks run on different machines, set this field to the FFN machine's reachable IP address. | -| `port` | `int` | `1239` | AFD rendezvous port in `1..65535`. It is separate from the vLLM HTTP service ports. | -| `afd_process_group_timeout_s` | `int` | `120` | Positive AFD group rendezvous timeout in seconds; also used by the native NPU DBO startup barrier. It does not bound a polling device kernel. | -| `num_attention_ranks` | `int` | `1` | Total number of Attention worker ranks. Must be positive. | -| `num_ffn_ranks` | `int` | `1` | Total number of FFN worker ranks. Must be positive. | -| `compute_gate_on_attention` | `bool` | `false` | Controls whether the MoE gate is computed on the Attention side or the FFN side. Currently only `false` is supported. | -| `connector_extra_config` | `dict` | `{}` | CAMP2P-specific settings such as role-specific core counts and `quant_mode`. Unknown fields are rejected. | -| `async` / `async_dp` | `bool` | `false` | Must remain `false` for the current synchronous; Ascend async mode requires `CAMAsyncAFDConnector`. | - -Compatibility aliases are accepted for `afd_role`, `afd_connector`, -`afd_host`, `afd_port`. - -The connector factory derives each process's role rank from its global DP rank -and local PCP/TP coordinates before connector construction. Do not configure a -role rank. +Use the [common configuration reference](../configuration.md) for defaults +and aliases. CAMP2P requires `async_dp=false` and +`compute_gate_on_attention=false`. For multi-machine placement, use the FFN +machine's reachable IP as `host`. ### CAMP2P `connector_extra_config` diff --git a/docs/npu/TESTING.md b/docs/npu/TESTING.md deleted file mode 100644 index 3eabcc06f..000000000 --- a/docs/npu/TESTING.md +++ /dev/null @@ -1,172 +0,0 @@ -# NPU validation - -Configuration and source-contract tests can run on CPU; runtime contracts -depend on vLLM/vLLM-Ascend, and device tests also need the matching Ascend -runtime and hardware. Use vLLM 0.32.0 and the vLLM-Ascend source revision in -the [installation guide](../../README.md#ascend-npu-installation). -The [0.32 validation record](https://github.com/vllm-project/afd-plugin/pull/507) distinguishes measured -coverage from deferred qualification and records the graceful-exit failures. - -From the repository root, run: - -```bash -python3 -m pytest -q -ra tests/unit -m "not gpu and not vllm_runtime" -``` - -The suite covers the NPU worker and model-runner contracts, CAM connector -behavior, and module isolation. Tests that require unavailable NPU dependencies -are skipped in CPU-only CI, so a passing CPU run does not replace this check. - -`tests/conftest.py` establishes the vLLM-Ascend import order required by the -test suite. No manual module preloading is needed. - -When diagnosing a failure, run the failing pytest node by itself first. A test -that passes alone but fails in the full suite usually indicates leaked module -or monkeypatch state. Test fixtures and mocks must also follow the concrete -vLLM types and signatures used by the pinned runtime. - -## Native DBO contracts and device execution - -Run the focused contract checks in the matching runtime environment: - -```bash -python3 -m pytest -q -ra \ - tests/unit/compat/npu/test_runtime_config.py \ - tests/unit/config/test_validation.py \ - tests/unit/model_executor/test_async_cam_dbo.py \ - tests/unit/model_executor/test_async_cam_layout.py \ - tests/unit/v1/worker/test_npu_model_runner_v2.py \ - tests/unit/v1/worker/test_npu_ubatch_outputs.py \ - tests/unit/v1/worker/test_npu_mla_graph.py -``` - -The single-device executor test uses real NPU streams/events with isolated -subprocesses for failure injection: - -```bash -ASCEND_RT_VISIBLE_DEVICES=0 python3 -m pytest -q -ra tests/npu/test_mrv2_dbo.py -``` - -It checks native child handoff, stream/context restoration and peer failure -wakeup. It does not validate CAMP2P/CAM transport, checkpoint output, or model -accuracy. The recorded Lite two-stage result does not qualify every Async CAM -DBO configuration. For broader qualification, run model comparisons at -Attention DP1 and DP > 1, plain TP and model SP, with -uneven prefill lengths and requests crossing child boundaries. Enable DBO on -Attention only. Keep two ubatches and eager execution. - -For synchronous MRV2 CAMP2P, include eager two-stage execution at Attention -DP > 1, many-to-one padding, and graph+DBO. Plain MLA graph+DBO requires -`FULL_DECODE_ONLY`. Both existing Lite graph+DBO cases must report completed -joint two-stage replays during real requests on every participating A/F rank. -Dummy/profile/capture or padded-only stages do not qualify that gate. -`tests/npu/test_mrv2_graph_inputs.py` separately checks persistent input -addresses with real NPU graph reads and modeled builders; model correctness -still belongs to E2E. Record execution, response correctness, revisions and -teardown outcomes separately. - -The original DSV4 Async concurrent case keeps native DBO thresholds. Its ten -short prompts can remain unsplit while satisfying its exact-answer, `stop`, -overlap and physical-cleanup checks; graceful exit failed in the recorded run. -DSV4 DSA two-stage qualification remains deferred. Qualify it separately -with an above-threshold workload, retaining native defaults and requiring -completed nonempty stages on every Attention rank. - -## Operator and model gates - -Use [the operator harness](../../tests/npu/README.md) for isolated routed CAM -precision/performance and [model E2E cases](../design/module/e2e_testing.md) -for serving and accuracy. DSV4 synchronous A5/A3 profiles currently check -completion/liveness with `check_answer=False`; they are not correctness gates. -Any skipped native registration or numerical case means that capability was -not validated by that run. - -## Layered W4A8 validation - -The layered routed-expert operators are `gmm_swiglu_quant_v2_layered` (W13) and -`grouped_matmul_layered` (W2). Both select their layer on device, and their -`all_*` lists hold one element per layer rather than one per expert. - -Run the CPU contract and path-selection tests first: - -```bash -pytest -q tests/unit/test_envs.py \ - tests/unit/model_executor/test_async_cam_layered.py -``` - -In an environment with matching CANN and torch-npu versions, rebuild the AFD -extension and run the device suites. All four need one device and issue no -collective: - -```bash -SOC_VERSION=910c pytest -q tests/npu/test_gmm_layered.py -SOC_VERSION=910c pytest -q tests/npu/test_gmm_swiglu_quant_v2_layered.py -AFD_RUN_ASCEND_OP_RUNTIME=1 SOC_VERSION=910c LOCAL_RANK=0 \ - pytest -o addopts= -v -ra -s \ - tests/npu/test_async_cam_layered_w4a8.py \ - tests/npu/test_async_cam_layered_w8a8.py \ - tests/npu/test_gmm_swiglu_quant_v2_layered_precision.py \ - tests/npu/test_gmm_layered_device_group_list.py -``` - -The four device suites answer different questions and none is redundant: - -| Suite | Operator | Covers | -| --- | --- | --- | -| `test_async_cam_layered_w4a8.py` | W13 then W2 | The W4A8 `AsyncCAMLayeredExecutor` chain, limits 0 and 10 | -| `test_async_cam_layered_w8a8.py` | W13 then W2 | Native W8A8 reference, device layer selection, sparse/empty/capacity rows, exact eager/FULL replay | -| `test_gmm_swiglu_quant_v2_layered_precision.py` | W13 | Default A8W8/A8W4 precision and clipped A8W4/A4W4 | -| `test_gmm_layered_device_group_list.py` | W2 | A device-produced group list, both scenarios, 4 cases | - -`test_async_cam_layered_w4a8.py` uses two layers with different nonzero INT4 -weights, nonzero compensation, per-channel and per-group scales, interleaved -nonconsecutive layer IDs, and zero, single-row, uneven, and full-capacity expert -counts. It compares valid rows with an FP32 reference and perturbs the capacity -tail. The zero-limit cases retain the random FP32 reference. Positive-limit cases -use sparse, exactly representable matmuls that cross the gate upper bound and -both up bounds, avoiding FP16 rounding followed by cancellation in the W2 -reference. Both use the same output tolerance. The W13 precision suite also -checks quantized codes, scales and dequantized outputs against an independent CPU -golden, same-shape 0/10 switching, and graph replay with device-selected layers. -These tests do not check distributed CAM completion or compare actual checkpoint -results against the non-layered path. Recheck the tolerance and supported -configurations against the project's precision standard when the operator -changes. - -The two precision suites are described under "Layered operator precision" in -[`tests/npu/README.md`](../../tests/npu/README.md), including why A8W4 cannot use -the fused built-in as its reference and why each attribute combination needs its -own activation shape. Read that section before changing a tolerance or adding a -case: the tiling cache is keyed on shape and tiling key, and the two attributes -reach the tiling only as data. - -For end-to-end validation, compare the layered path with a recorded legacy-path -baseline from an earlier revision, using the same W4A8 or W8A8 checkpoint and -deployment configuration. Record the processed parameter -shapes, dtypes, and formats; software versions; actual-path startup logs; -accuracy; peak memory; and profiler traces. Cover completion on empty ranks, -multiple chunks of one layer, interleaved DP work, and async CAM ubatching. -Check that no layer-ID or count metadata D2H occurs between receive and send, -and measure the worker group synchronization separately. Performance gains -require a controlled A/B comparison. - -For operator-level performance, use -[`tests/npu/gmm_layered_performance.py`](../../tests/npu/gmm_layered_performance.py) -rather than an end-to-end comparison. It measures both launch paths - eager and -ACL graph - and reports the fusion contribution and the graph contribution -separately, which an end-to-end run cannot separate. Anchors measured on Ascend -910C at `M = 98304`, `C = 262144`: W13 reaches about 1.80x the concatenated -baseline on the steady-state per-layer figure, `chain` about 1.46x, and W2 sits -at parity once the graph removes its launch overhead. Re-measure rather than -reuse these numbers; they are a starting point for a regression check, not a -claim about a different shape, topology, or CANN version. - -The current ACLNN entry validates `layer_index` by its one-element int64 -contract, including a slice with nonzero storage offset. The GMM path accepts -the logical 3D A8W8 weight view backed by 5D NZ storage as well as the W4A8 -layout. Rebuild the extension and vendor library together after these native -changes; Meta-only success does not exercise the ACLNN entry or kernel. -Layered GMM is allowed with MRV2 pairing, but it still runs in the independent -eager FFN runner. The current shared-expert layout also has focused coverage -in `tests/unit/model_executor/models/test_deepseek_v4_shared_experts.py` and -`test_deepseek_v2_sequence_parallel.py`. diff --git a/docs/npu/TROUBLESHOOTING.md b/docs/npu/TROUBLESHOOTING.md index 5b9c7a090..740d10154 100644 --- a/docs/npu/TROUBLESHOOTING.md +++ b/docs/npu/TROUBLESHOOTING.md @@ -169,5 +169,5 @@ additionally requires Attention DP > 1, while Async CAM permits DP1. An `NPU DBO peer failed` or `Async CAM receive failed` error means the host-side handoff was aborted. Restart the A/F worker group; waking the peer does not recover a partially used device window. Use the -[executor tests](TESTING.md#native-dbo-contracts-and-device-execution) to +[executor tests](../../tests/npu/README.md#native-dbo-contracts-and-device-execution) to separate thread/stream failures from communication and model correctness. diff --git a/docs/npu/LAYERED_W2_MDL_VALIDATION.md b/docs/reports/layered_w2_mdl_validation.md similarity index 100% rename from docs/npu/LAYERED_W2_MDL_VALIDATION.md rename to docs/reports/layered_w2_mdl_validation.md diff --git a/docs/support.md b/docs/support.md new file mode 100644 index 000000000..175e99e7d --- /dev/null +++ b/docs/support.md @@ -0,0 +1,78 @@ +# Supported configurations + +AFD is experimental. Registration and accepted configuration do not establish +hardware qualification; evidence below is scoped to its model, topology and +source revision. See [installation](installation.md) for the pinned runtime stack. + +## Models + +| Model family | Execution scope | +| --- | --- | +| DeepSeek V2 / V3 / V3.2 | CUDA P2P and synchronous Ascend paths; DeepEP M2N admits V2 only. Each role loads its required components. | +| DeepSeek V4 Flash | CUDA P2P or Ascend CAMP2P / Async CAM; backend-specific limits are in the matrix below. | +| Qwen3 MoE | CUDA; P2P gates on FFN, DeepEP M2N gates on Attention. Rejects sequence-parallel MoE, EPLB, PP, speculative decoding, LoRA and NPU. | +| Qwen3.5 / Qwen3.6 MoE | CUDA text-only with `--language-model-only`; evidence covers Qwen3.6-35B-A3B native DP4/TP1/EP4 and synchronous AFD 2A1F eager/graph. GDN DBO, Attention-side routing, PP, async and multi-node execution are unsupported; NPU/multimodal are unsupported or unverified, quantization is unverified, and no performance claim is made. | + +Architecture aliases and wrapper ownership are recorded in +[model integration](design/module/model_integration.md#model-registration). +For deployment commands, use the [recipe index](../recipe/README.md). + +## Tested runtime matrix + +This table separates accepted runtime configurations from validation evidence. +Both backends require ModelRunnerV2. A test case or a historical result does not +qualify every model/topology combination on the current target. + +| Platform/path | Execution | Ubatching | Routing/quantization limits | Evidence | +| --- | --- | --- | --- | --- | +| CUDA V2 + `P2pNcclAFDConnector` | Eager or `FULL_DECODE_ONLY` native V2 CUDA Graph | Exactly two native microbatches; Attention DP1 or DP > 1 | Model-specific gate placement; PP/CP, elastic EP, EPLB, SP MoE, and compile SP rejected; role ranks equal DP x TP | Source-specific GPU validation is linked below; coverage is limited to the recorded model/topology combinations. DeepSeek-V2-Lite has known DBO shutdown failures. Qwen3.5/3.6 GDN DBO is unsupported. | +| CUDA V2 + `DeepEPM2NAFDConnector` | Eager or `FULL_DECODE_ONLY` native V2 CUDA Graph; SM90 or newer | Exactly two native microbatches, enabled on both roles; Attention DP1 or DP > 1, TP1 or TP2 | Qwen3 MoE (all layers MoE, no shared expert) or DeepSeek-V2; `compute_gate_on_attention=true`; FFN TP1 with EP over all FFN ranks, linear placement, `allgather_reducescatter` all2all and the Triton MoE weight layout; BF16 or block-128 FP8 payload; EPLB, PP/CP and SP MoE rejected | Historical vLLM 0.30 Qwen3-30B-A3B 2A2F GSM8K E2E for eager, graph, graph + DBO, and Attention TP2 eager/graph on H100 and H200, and for eager, graph and graph + DBO on GB200; two-node 2A2F eager, graph and graph + DBO with Attention and FFN on separate nodes. DeepSeek-V2-Lite: [E2E definitions](../tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py). [GPU connector device tests](../tests/gpu/README.md) | +| Ascend V2 + `CAMP2pAFDConnector` | Eager, `FULL`, or `FULL_DECODE_ONLY` native V2 ACL Graph | Eager or graph native DBO with two ubatches and Attention DP > 1; plain MLA graph DBO requires `FULL_DECODE_ONLY` | `compute_gate_on_attention=false`; PP/CP, elastic EP and EPLB rejected; model-specific SP policy; role ranks equal DP x TP | Historical 0.30 V2-Lite 2A2F FULL 300-question comparison and 2A1F functional checks; current 0.32 Lite evidence is in the [validation record](https://github.com/vllm-project/afd-plugin/pull/507) | +| Ascend V2 + `CAMAsyncAFDConnector` | Eager Attention; layered FFN eager or FULL | Two native DBO microbatches on Attention, including DP1 and with FFN FULL; FFN stays outside the native DBO schedule | Experimental DeepSeek-V4 W4A8/W8A8 on A3/910C; `async_dp=true`, `compute_gate_on_attention=true`, and `dynamicQuant=1`; PP/CP unsupported; plugin-owned async CAM ops required | Historical DSV4 layered comparisons and limitations are in the [validation record](https://github.com/vllm-project/afd-plugin/pull/507); they predate removal of the legacy path | + +The 300-question Ascend evidence in this table is historical. The 0.32 +[validation record](https://github.com/vllm-project/afd-plugin/pull/507) separately lists the +eight Lite 100-question cells, W4A8 fixture, and observed shutdown failures. + +The Async CAM row supports DeepSeek-V4 with static EP, PP=PCP=DCP=1, +and Attention-local SP. DeepSeek-V2/V2-Lite are no longer supported by +Async CAM; standard DeepSeek-V3.2 was already rejected by model admission. +Finegrained TP, explicit shared-expert DP configuration, and native +MTP and draft-model paths other than DeepSeek-V4 DSpark are excluded. DSpark +keeps Attention/native proposal eager and permits independent layered FFN FULL; +it does not support DBO. +FFN remains connector-driven. The row's upgrade evidence retains its original +scope; MRV2 deployment settings are documented in the +[CAM async guide](npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md#modelrunnerv2-deployment). + +Both backends target vLLM `0.32.0` (`5df63a72`); Ascend additionally targets +vLLM-Ascend `bd42eea0`. Source-specific +H20 model, topology, execution-mode and accuracy results are recorded in +the [GPU upgrade validation matrix](https://github.com/vllm-project/afd-plugin/pull/501) +and [runtime-fix validation matrix](https://github.com/vllm-project/afd-plugin/pull/502). +Those records identify the exact tested revisions. Earlier GPU results on +vLLM 0.30 do not establish vLLM 0.32 hardware qualification. + +## Validation limits + +- GPU accuracy evidence covers selected first-300 GSM8K comparisons, not + full-dataset accuracy, numerical equivalence, or performance improvement. +- DeepSeek-V2-Lite TP2 Graph + DBO and DP2 eager DBO can require force + killing during shutdown. Inference success does not imply graceful exit. +- Native V4 DBO requires a compatible upstream communication backend. The + recorded native V4 accuracy reference uses Graph without DBO. +- NPU 0.32 evidence is limited to the configurations and checks in the + [validation record](https://github.com/vllm-project/afd-plugin/pull/507); the seven AFD Lite + cases failed graceful exit. Historical NPU results do not qualify 0.32. + +Historical [NPU validation](https://github.com/vllm-project/afd-plugin/pull/425#issuecomment-6063910923) +and [DBO repair](https://github.com/vllm-project/afd-plugin/pull/430#issuecomment-6063840311) +records retain their original versions and sample counts. Synchronous DSV4 +profiles remain smoke-only, with known incorrect A5 concurrent answers; see +their [E2E limits](../tests/e2e/README.md#dsv4-flash-sync-camp2p-concurrent-requests-local-4-or-16-npus). +GPU/NPU rank topology and connector resource rules remain owned by +[connector contracts](design/module/connector_contracts.md). + +The repository does not record a canonical CUDA container or assert an +official tag for the Ascend source pin `bd42eea0`; environment evidence is +not an authoritative package tag. diff --git a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md index d4246f38d..2f18e18a9 100644 --- a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md +++ b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md @@ -3,7 +3,7 @@ End-to-end launch scripts for running DeepSeek-V2-Lite with the AFD (Attention-FFN Disaggregation) plugin targeting vLLM `v0.32.0`. These are deployment examples, not v0.32 recipe validation. See the -[runtime matrix](../../../../docs/design/module/execution_platforms.md#tested-runtime-matrix) +[runtime matrix](../../../../docs/support.md#tested-runtime-matrix) for tested E2E configurations. > [!NOTE] diff --git a/tests/e2e/README.md b/tests/e2e/README.md index e3b779622..eda0c0784 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -239,10 +239,10 @@ and graph mode change; the graph row uses `FULL_DECODE_ONLY` with capture size 8 The GPU validator accepts exactly two microbatches at Attention DP1 or DP > 1; FFN remains connector-driven. See the -[runtime matrix](../../docs/design/module/execution_platforms.md#tested-runtime-matrix) +[runtime matrix](../../docs/support.md#tested-runtime-matrix) for tested configurations and known DeepSeek-V2-Lite shutdown failures. NPU retains the separate eager DBO and graph -constraints documented in the [execution-platform matrix](../../docs/design/module/execution_platforms.md#tested-runtime-matrix). +constraints documented in the [support matrix](../../docs/support.md#tested-runtime-matrix). The GPU harness collects bounded per-rank summaries of actual two-stage and FULL-replay execution during evaluation. Historical hardware records retain the execution assertions of the test revision that produced them. On vLLM @@ -284,7 +284,7 @@ samples, with the existing 24-sample floor for DBO. `models/deepseek_v4_flash/test_gpu.py` provides separate CUDA qualification profiles for the synchronous `P2pNcclAFDConnector`. They use CUDA launch shapes independently of the Ascend W4A8 and HCCL profiles. See the -[runtime matrix](../../docs/design/module/execution_platforms.md#tested-runtime-matrix) +[runtime matrix](../../docs/support.md#tested-runtime-matrix) for source-specific hardware validation. The basic test has four cases: diff --git a/tests/e2e/operators/README.md b/tests/e2e/operators/README.md index d72cb6f80..d35a61e99 100644 --- a/tests/e2e/operators/README.md +++ b/tests/e2e/operators/README.md @@ -20,4 +20,4 @@ and repeated window use. Tolerances are fixed in the script before execution. The retired non-layered checkpoint probe remains available on earlier revisions for a before/after comparison. Validate the current layered implementation using the numerical and real-model fixtures in the -[NPU testing guide](../../../docs/npu/ASYNC_FFN_RUNTIME.md#validation). +[NPU testing guide](../../../docs/design/npu/async_ffn_runtime.md#validation). diff --git a/tests/npu/README.md b/tests/npu/README.md index 3b93a8f84..e68210f4e 100644 --- a/tests/npu/README.md +++ b/tests/npu/README.md @@ -1,7 +1,7 @@ -# Ascend 910C operator tests +# NPU validation -Standalone harnesses, dispatcher checks, and opt-in device tests for the Ascend -operators this plugin owns. Device tests run outside CI; CPU harness checks also +Runtime checks, standalone operator harnesses, and opt-in Ascend device tests. +Device tests run outside CI; CPU harness checks also run in the PR and merge unit-test jobs. Each entry states whether it needs a device, a multi-rank launch, or only a host Python environment. @@ -36,14 +36,12 @@ connector, model weights, or shared-expert tensors. The second group covers the layered routed-expert operators, `gmm_swiglu_quant_v2_layered` (W13) and `grouped_matmul_layered` (W2), which select their layer on device from a `layer_index` input. Those are the operators -`AsyncCAMLayeredExecutor` drives, and their `all_*` lists hold one element per layer -rather than one per expert. Their validation entry points are in -[docs/npu/TESTING.md](../../docs/npu/TESTING.md), and the two precision suites -are described under [Layered operator precision](#layered-operator-precision) -below. +`AsyncCAMLayeredExecutor` drives; their `all_*` lists hold one element per layer +rather than one per expert. See +[layered operator precision](#layered-operator-precision). The async CAM harness implements the AFD schemas documented in -[CAM_ASYNC_ROUTED_OPS.md](../../docs/npu/CAM_ASYNC_ROUTED_OPS.md). The input +[routed CAM operator contract](../../docs/design/npu/cam_async_routed_ops.md). The input values, route table, quantization result, and weighted sum are all produced inside this repository from the raw inputs; the harness depends on no external reference directory at runtime. @@ -54,6 +52,66 @@ unit-test suite. The three 910C dispatcher modules share the optional `ascend_ops_runtime` fixture in `conftest.py`; requesting it loads the compiled extension, while collection and host-only tests stay independent of torch-npu. +## Runtime contract checks + +Use the pinned [Ascend stack](../../docs/installation.md#ascend-npu-installation). +From the repository root: + +```bash +python3 -m pytest -q -ra tests/unit -m "not gpu and not vllm_runtime" +``` + +Run in the matching runtime environment to exercise NPU-dependent tests; +CPU-only skips do not establish runtime correctness. `tests/conftest.py` owns +the required vLLM-Ascend import order. A test that passes alone but fails with +the full suite may indicate leaked module or monkeypatch state. + +## Native DBO contracts and device execution + +Run the focused contract checks in the matching runtime environment: + +```bash +python3 -m pytest -q -ra \ + tests/unit/compat/npu/test_runtime_config.py \ + tests/unit/config/test_validation.py \ + tests/unit/model_executor/test_async_cam_dbo.py \ + tests/unit/model_executor/test_async_cam_layout.py \ + tests/unit/v1/worker/test_npu_model_runner_v2.py \ + tests/unit/v1/worker/test_npu_ubatch_outputs.py \ + tests/unit/v1/worker/test_npu_mla_graph.py +``` + +The single-device executor test uses real NPU streams/events with isolated +subprocesses for failure injection: + +```bash +ASCEND_RT_VISIBLE_DEVICES=0 python3 -m pytest -q -ra tests/npu/test_mrv2_dbo.py +``` + +It checks native child handoff, stream/context restoration and peer failure +wakeup. It does not validate CAMP2P/CAM transport, checkpoint output, or model +accuracy. For broader qualification, run model comparisons at +Attention DP1 and DP > 1, plain TP and model SP, with +uneven prefill lengths and requests crossing child boundaries. Enable DBO on +Attention only. Keep two ubatches and eager execution. + +For synchronous MRV2 CAMP2P, include eager two-stage execution at Attention +DP > 1, many-to-one padding, and graph+DBO. Plain MLA graph+DBO requires +`FULL_DECODE_ONLY`. Both existing Lite graph+DBO cases must report completed +joint two-stage replays during real requests on every participating A/F rank. +Dummy/profile/capture or padded-only stages do not qualify that gate. +`tests/npu/test_mrv2_graph_inputs.py` separately checks persistent input +addresses with real NPU graph reads and modeled builders; model correctness +still belongs to E2E. Record execution, response correctness, revisions and +teardown outcomes separately. + +The original DSV4 Async concurrent case keeps native DBO thresholds. Its ten +short prompts can remain unsplit while satisfying its exact-answer, `stop`, +overlap and physical-cleanup checks; graceful exit failed in the recorded run. +DSV4 DSA two-stage qualification remains deferred. Qualify it separately +with an above-threshold workload, retaining native defaults and requiring +completed nonempty stages on every Attention rank. + ## Python-only validation From the repository root, no torch or Ascend installation is needed for: @@ -215,17 +273,41 @@ execution, and concurrent layers require separate hardware qualification. ### Layered operator precision -Two suites cover the layered routed-expert operators. Both are pytest modules -gated on `AFD_RUN_ASCEND_OP_RUNTIME=1`, both need one device, and neither -performs any collective. +The W4A8/W8A8 executor chains and the W13/W2 precision suites each use one +device and no collective. Run the CPU contracts first, then rebuild the matching +extension/vendor library and run the device checks: + +```bash +pytest -q tests/unit/test_envs.py tests/unit/model_executor/test_async_cam_layered.py +SOC_VERSION=910c pytest -q tests/npu/test_gmm_layered.py \ + tests/npu/test_gmm_swiglu_quant_v2_layered.py +``` ```bash SOC_VERSION=910c AFD_RUN_ASCEND_OP_RUNTIME=1 LOCAL_RANK=0 \ pytest -o addopts= -v -ra -s \ + tests/npu/test_async_cam_layered_w4a8.py \ + tests/npu/test_async_cam_layered_w8a8.py \ tests/npu/test_gmm_swiglu_quant_v2_layered_precision.py \ tests/npu/test_gmm_layered_device_group_list.py ``` +The W4A8 chain compares valid rows with an FP32 reference across distinct +nonzero weights/compensation, per-channel/per-group scales, interleaved layer +IDs, empty/uneven/full expert counts and poisoned capacity tails. Positive +clipping limits use exactly representable matmuls to avoid FP16 cancellation; +zero-limit cases retain the random reference, with the same tolerance. +The W8A8 chain compares native operators and exact eager/FULL replay. +These operator checks do not establish distributed completion or checkpoint +accuracy; use the [model E2E guide](../e2e/README.md) for those gates. + +Checkpoint comparisons must hold the checkpoint and deployment settings fixed. +Record parameter shapes/dtypes/formats, versions, startup path, accuracy, peak +memory and traces. Cover empty ranks, multi-chunk layers, interleaved DP work +and async ubatching. Check for layer-ID/count metadata D2H between receive and +send; measure group synchronization separately. Performance claims require a +controlled A/B comparison. A skipped device check establishes no qualification. + `test_gmm_swiglu_quant_v2_layered_precision.py` collects 11 cases against W13. It covers A8W8 and A8W4, and it needs two reference strategies because only one scenario has a usable built-in counterpart: @@ -436,10 +518,9 @@ Device cleanup after failure still requires validation on the target stack. ## check_routing_boundary_reuse.py Device check for the opaque routing-id boundary. Requires an Ascend host with the pinned stack -(`vllm`, `vllm-ascend`). Run it directly: +(`vllm`, `vllm-ascend`). Run from the repository root: ```bash -cd python tests/npu/check_routing_boundary_reuse.py ``` From 7b13ed6f9f2bc3b1fbcbd55cdc31f1167a71701c Mon Sep 17 00:00:00 2001 From: Ronen Kat <16743404+ronenkat@users.noreply.github.com> Date: Tue, 10 Nov 2026 05:32:11 +0200 Subject: [PATCH 29/42] Fix DeepEP M2N transport group teardown (#564) Closes #563 ## Summary - Destroy the distinct per-domain `transport_group` during DeepEP M2N connector close. - Clear the transport-group reference so a second close remains a no-op. ## Validation - Added the regression test first and confirmed it failed without the teardown. - With the fix, the focused test passes. - The full DeepEP M2N connector and layout unit selection passes in a remote CUDA vLLM `0.32.0` environment. ## Documentation No impact; this is an internal lifecycle fix. Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- .../connectors/gpu/deepep_m2n/connector.py | 6 ++++ .../connectors/test_deepep_m2n_connector.py | 32 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/afd_plugin/connectors/gpu/deepep_m2n/connector.py b/afd_plugin/connectors/gpu/deepep_m2n/connector.py index 725479d1a..4eb11dca3 100644 --- a/afd_plugin/connectors/gpu/deepep_m2n/connector.py +++ b/afd_plugin/connectors/gpu/deepep_m2n/connector.py @@ -1203,6 +1203,12 @@ def close(self) -> None: _CONNECTORS.pop(self._op_id, None) for lane in self.lanes: lane.destroy() + if ( + self.transport_group is not None + and self.transport_group is not self.bootstrap_group + ): + dist.destroy_process_group(self.transport_group) + self.transport_group = None self.lanes = [] self.lane_streams = [] self._pending_combines.clear() diff --git a/tests/unit/connectors/test_deepep_m2n_connector.py b/tests/unit/connectors/test_deepep_m2n_connector.py index 8ef14c143..eb9e9fbea 100644 --- a/tests/unit/connectors/test_deepep_m2n_connector.py +++ b/tests/unit/connectors/test_deepep_m2n_connector.py @@ -869,6 +869,38 @@ def abort(group): connector.control_plane.recv_dp_metadata_list() +def test_close_destroys_the_distinct_transport_group_after_lanes(monkeypatch): + """A per-domain transport group is destroyed once, after its lanes. + + Why: afdEP-comm leaves the caller-owned Gloo group alive; keeping it + registered leaks process-group state across connector lifetimes. + """ + events = [] + + class Lane: + def destroy(self): + events.append("lane") + + monkeypatch.setattr( + connector_module, "_abort_process_group", lambda group: events.append("control") + ) + monkeypatch.setattr( + connector_module.dist, + "destroy_process_group", + lambda group: events.append("transport"), + ) + connector = _connector("ffn", 0, M2NTopology(ffn_size=2, attn_size=2)) + connector.group = SimpleNamespace() + connector.bootstrap_group = SimpleNamespace() + connector.transport_group = SimpleNamespace() + connector.lanes = [Lane()] + + connector.close() + connector.close() + + assert events == ["control", "lane", "transport"] + + @pytest.mark.parametrize( ("closes_first", "error", "expected"), [ From b33e20de9e0321ead5d498077f35bd63fc2f3b16 Mon Sep 17 00:00:00 2001 From: "Songym@Huawei" <109567708+Pingzii@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:36:47 +0800 Subject: [PATCH 30/42] feat(npu): transport BF16 Attention-side routing to the FFN role (#503) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Background The synchronous CAMP2P path normally runs attention on the Attention role and the router, shared experts, and routed experts on the FFN role. This PR makes router/shared placement an explicit deployment option. With `compute_gate_on_attention=true`, Attention computes top-k and shared experts, while FFN consumes the external routing and runs the routed experts. ## Motivation **Improve eager execution through a different A/F work split.** The measured Attention-side placement delivers **+17.6% throughput** versus the same-commit FFN-side placement in the evaluated eager workload. **Establish the routing contract for quantized activation transport.** Once FFN receives the routing alongside the activation, it can consume an activation quantized on Attention. The MXFP8 pass-through builds on this contract in #506. The option is **off by default** and belongs in the common `additional_config["afd"]` block. ## Implementation ```text Default Attention : attention FFN : router + shared experts + routed experts compute_gate_on_attention=true Attention : attention + router + shared experts FFN : transported top-k + routed experts ``` - **Reuse the A2E routing slots.** Send expert IDs as `int32` and top-k weights as `float32` through the existing ids channel. - **Owned prepare and comm subclasses.** `AFDPrepareAndFinalizeWithAllGather` and `AFDPrepareAndFinalizeWithMC2` provide `prepare_external`, which takes the routing for the local input rows and returns the complete `MoEPrepareOutput` together with the aligned routing. AFD comm subclasses override only `_get_prepare_finalize()`, so the native selector still decides which path runs and ordinary execution keeps the upstream classes. There are no global prepare patches left. - **Explicit routing contract.** The routing is an argument and a return value; nothing is stashed on the upstream class, and the module no longer reads or clears an `afd_external_routing` attribute. - **One MoE component.** `AFDExternalRoutingMoE` builds its comm candidates during model initialization, where the config context exists, and each forward only looks up the candidate matching the native selection. It runs prepare, expert execution and finalize, switches the forward-context comm for the duration of the call and restores it on every exit path, and rejects unsupported types such as fused-MC2. - **Consume the external top-k on FFN.** The component hands the aligned routing straight to expert execution. - **Place shared experts on Attention.** Run them between activation send and routed-output receive, then combine. - **Keep protocol selection explicit.** The model configuration selects token-ID or external-routing semantics; the model itself only builds the component for the synchronous FFN role and delegates. The supported opt-in configuration is synchronous CAMP2P, DeepSeek-V4, TP=1, without DBO or ubatching. Both roles must use matching routing settings. ## Performance and accuracy Measured at `828582a` on one A5 4A4F node during the same evening session, with BF16 A2E transport and automatic communication selection: | Configuration | Eager throughput (token/s) | Graph throughput (token/s) | | --- | ---: | ---: | | Attention-side gate (`828582a`) | **192,887** | **326,400** | | FFN-side gate (`828582a`) | 163,997 | 331,455.5 | | Clean baseline (`bcc0126`) | 164,543 | 333,407 | | Comparison | Eager | Graph | | --- | ---: | ---: | | Attention-side vs FFN-side placement | **+17.6%** | **-1.5%** | | FFN-side placement vs clean baseline | -0.3% | -0.6% | The result supports Attention-side placement as an eager optimization and a foundation for MXFP8 pass-through. Placement remains opt-in, allowing deployments to select the appropriate A/F split for their execution mode. Graph-mode Attention-side routing scored **90.60% on the full 1,319-question GSM8K evaluation**, with normal generated output. Measurement setup: concurrency **128**, reported input **8,275** / output **32** tokens per request, `max-num-batched-tokens=3072`, `max-num-seqs=128`, and zero failed requests. Graph uses `FULL_DECODE_ONLY` with capture sizes `[8,16,32,64,128]`; eager uses `--enforce-eager`. Each cell has two runs from the same firmware/session. These are workload-specific measurements with small run counts. ### Re-measured on the target stack at `e2c7075` The refactored implementation was re-measured on the stack this PR declares: vLLM `0.32.0` @ `5df63a7`, vLLM-Ascend `bd42eea0`, BF16 A2E transport, Attention-side gate, automatic communication selection, TP=1, eager (no CUDA/ACL graph), 4A4F on one A5 node. | Run | E2E (ms) | Prefill (token/s) | Output (token/s) | Total (token/s) | Failed | | --- | ---: | ---: | ---: | ---: | ---: | | 13:28 | 5,126.2 | 1,614.2 | 728.59 | 189,136 | 0 / 128 | | 13:34 | 5,146.5 | 1,607.9 | 726.69 | 188,643 | 0 / 128 | The two runs agree within 0.4%, so they are recorded as a repeat baseline rather than a single sample. GSM8K on the same configuration scored **89.92%** on the full 1,319-question evaluation. These numbers describe this configuration only: there is no same-stack reference implementation to compare against, so they are not evidence of a performance change from the refactor. The pre-refactor table above was measured on the previous stack and stays attributed to `828582a`. ### Placement re-measured on the target stack (`e2c7075`) Same node, same stack (vLLM `0.32.0` @ `5df63a7`, vLLM-Ascend `bd42eea0`), same workload (128 concurrency, 8,275 input / 32 output tokens per request), zero failed requests. The clean baseline is upstream main at `b093863`, which keeps the gate on the FFN side. | Mode | Clean baseline `b093863` (total token/s) | Attention-side gate `e2c7075` (total token/s) | Delta | | --- | ---: | ---: | ---: | | Eager | 165,345 | 189,136 / 188,643 | **+14.2%** (mean) | | Graph (`FULL_DECODE_ONLY`) | 364,627 / 362,563 | 354,731 / 353,972 | **-2.5%** (mean) | On this workload, Attention-side placement averages about **+14.2% higher total throughput in eager** and about **2.5% lower in graph**. That is a placement comparison across two commits on one stack; it is not a pure refactor cost, and it does not mean Attention-side placement is unsuitable for graph mode. Graph mode also raises this configuration's own throughput by **87.6%** over eager (189,136 -> 354,731 total token/s). GSM8K on the target stack scored **89.92%** in eager and **89.99%** in graph mode, both on the full 1,319-question evaluation. ## Baseline and scope Rebased onto main at `e58fdec`, which includes the merged #500 opaque routing-op fix. The retired `profile_run_eager` implementation and its wiring have been removed. #506 builds MXFP8 pass-through on this routing contract. [Incremental diff for this PR: `e58fdec...2c17cce`](https://github.com/Pingzii/afd-plugin/compare/e58fdec...2c17cce) ## Verification CPU, measured at `2c17cce`: **59 passed, 2 skipped**. The focused set covers the routing-id op, the runtime installer wiring, the CAMP2P token-ID path, the routing payload transforms, and the new MoE component. The maintained CPU suite also passes. Two mutation checks were run on the refactored component before the rebase, and both produced the expected failures: - disabling the per-token state cleanup makes the MC2 finalize case fail (it observes state that should already be cleared); - disabling the comm restore makes the prepare-failure case fail (it observes the AFD comm left in place). Device evidence: the head `7430a13` starts on one A5 4A4F node with BF16 transport, `compute_gate_on_attention=true`, the CAMP2P connector, TP=1 and eager mode, and completes a generation request (`POST /v1/completions` returned 200). Accuracy and throughput were measured at `e2c7075`. The pre-refactor table above stays attributed to `828582a`. --------- Signed-off-by: 宋元盟 Co-authored-by: 宋元盟 --- afd_plugin/compat/npu/feature_validation.py | 58 ++- afd_plugin/connectors/metadata.py | 8 + afd_plugin/connectors/npu/camp2p/__init__.py | 43 ++ .../npu/{camp2p.py => camp2p/connector.py} | 185 +++++++- .../connectors/npu/camp2p/external_routing.py | 159 +++++++ .../model_executor/models/npu/deepseek_v4.py | 202 +++++++- .../npu/deepseek_v4_external_routing_moe.py | 350 ++++++++++++++ afd_plugin/v1/worker/ffn_model_runner.py | 12 +- afd_plugin/v1/worker/npu/ffn_model_runner.py | 18 +- afd_plugin/validation.py | 10 +- docs/design/module/connector_contracts.md | 23 +- docs/design/module/model_integration.md | 13 +- docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md | 82 +++- docs/support.md | 2 +- .../test_sync_gate_feature_validation_cpu.py | 159 +++++++ .../unit/connectors/test_camp2p_connector.py | 2 +- .../unit/connectors/test_camp2p_token_ids.py | 187 +++++++- .../unit/connectors/test_external_routing.py | 137 ++++++ ...st_deepseek_v4_external_routing_moe_cpu.py | 431 ++++++++++++++++++ .../models/test_sync_attention_gate_proxy.py | 188 ++++++++ .../test_sync_attention_gate_validation.py | 96 ++++ tests/unit/v1/worker/test_ffn_model_runner.py | 6 + .../v1/worker/test_npu_model_runner_v2.py | 2 +- 23 files changed, 2318 insertions(+), 55 deletions(-) create mode 100644 afd_plugin/connectors/npu/camp2p/__init__.py rename afd_plugin/connectors/npu/{camp2p.py => camp2p/connector.py} (85%) create mode 100644 afd_plugin/connectors/npu/camp2p/external_routing.py create mode 100644 afd_plugin/model_executor/models/npu/deepseek_v4_external_routing_moe.py create mode 100644 tests/unit/compat/npu/test_sync_gate_feature_validation_cpu.py create mode 100644 tests/unit/connectors/test_external_routing.py create mode 100644 tests/unit/model_executor/models/test_deepseek_v4_external_routing_moe_cpu.py create mode 100644 tests/unit/model_executor/models/test_sync_attention_gate_proxy.py create mode 100644 tests/unit/test_sync_attention_gate_validation.py diff --git a/afd_plugin/compat/npu/feature_validation.py b/afd_plugin/compat/npu/feature_validation.py index 0efeeeda6..73edc9990 100644 --- a/afd_plugin/compat/npu/feature_validation.py +++ b/afd_plugin/compat/npu/feature_validation.py @@ -56,8 +56,15 @@ def fail_if_unsupported_npu_afd_features( return if afd_config.compute_gate_on_attention: - raise RuntimeError( - "AFD NPU runtime does not support compute_gate_on_attention=true yet", + # The sync Attention-side gate prototype is deliberately narrow: one + # connector, the target model and BF16 transport. Widening it without the + # corresponding transport work would produce wrong numerics instead of an + # error, so anything outside that shape stays rejected. + _fail_if_unsupported_sync_attention_gate( + vllm_config, + afd_config, + extra_info, + is_dsv4=is_dsv4, ) if afd_config.connector == CAMP2P_CONNECTOR: from afd_plugin.connectors.npu.camp2p import CAMP2PExtraInfo @@ -151,6 +158,53 @@ def _fail_if_unsupported_dsv4_connector(afd_config: AFDConfig) -> None: ) +def _fail_if_unsupported_sync_attention_gate( + vllm_config: VllmConfig, + afd_config: AFDConfig, + extra_info: object, + *, + is_dsv4: bool, +) -> None: + """Restrict the sync Attention-side gate prototype to what it supports today. + + DBO and ubatching are rejected rather than merely unused: the sync shell does not + call the DBO yield, so enabling DBO here would interleave microbatches against a + transport that never yields. Not calling the yield is not the same as forbidding + the feature. + """ + + parallel = vllm_config.parallel_config + # The routing travels on each rank's own activation, and only the TP=1 shape has + # been verified end to end; a wider TP would split the activation without a + # matching routing split on the Attention side. + if int(parallel.tensor_parallel_size) != 1: + raise RuntimeError( + "Attention-side gate is only supported at tensor-parallel size 1; got " + f"TP={int(parallel.tensor_parallel_size)}", + ) + if bool(parallel.enable_dbo) or bool(parallel.use_ubatching): + raise RuntimeError( + "Attention-side gate with the synchronous connector does not support " + "DBO or ubatching yet", + ) + if afd_config.connector != CAMP2P_CONNECTOR: + raise RuntimeError( + "Attention-side gate is only supported with " + f"{CAMP2P_CONNECTOR}, got {afd_config.connector!r}", + ) + if not is_dsv4: + raise RuntimeError( + "Attention-side gate is only supported for the DeepSeek V4 target", + ) + from afd_plugin.connectors.npu.camp2p import CAMP2PExtraInfo + + if not isinstance(extra_info, CAMP2PExtraInfo): + raise TypeError( + "CAMP2pAFDConnector requires CAMP2PExtraInfo, got " + f"{type(extra_info).__name__}", + ) + + def _fail_if_unsupported_npu_afd_async_features( vllm_config: VllmConfig, afd_config: AFDConfig, diff --git a/afd_plugin/connectors/metadata.py b/afd_plugin/connectors/metadata.py index 02351e7c8..4b5cf6bc9 100644 --- a/afd_plugin/connectors/metadata.py +++ b/afd_plugin/connectors/metadata.py @@ -288,12 +288,20 @@ class AFDA2FTransferPayload: router_logits: Optional Attention-side routing tensor. input_ids: Optional token-aligned identifiers required by the remote FFN implementation. + topk_ids: Optional expert indices selected by an Attention-side gate, + shaped ``(tokens, topk)``. Present instead of ``input_ids`` when the + Attention role transported its gate result over the ids channel. + topk_weights: Optional routing weights paired with ``topk_ids``, same + shape. The FFN side consumes them as the gate weights, so any + model-level routed scaling is already folded in. """ hidden_states: torch.Tensor context: AFDTransferContext router_logits: torch.Tensor | None = None input_ids: torch.Tensor | None = None + topk_ids: torch.Tensor | None = None + topk_weights: torch.Tensor | None = None @dataclass(slots=True) diff --git a/afd_plugin/connectors/npu/camp2p/__init__.py b/afd_plugin/connectors/npu/camp2p/__init__.py new file mode 100644 index 000000000..77752d2f9 --- /dev/null +++ b/afd_plugin/connectors/npu/camp2p/__init__.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Synchronous CAMP2P connector package. + +The connector implementation lives in :mod:`~.connector` and the Attention-side routing +payload in :mod:`~.external_routing`. Names are resolved lazily so importing the payload +does not pull in the connector's vLLM and Ascend dependencies. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from afd_plugin.connectors.npu.camp2p.connector import ( + CAMP2pAFDConnector, + CAMP2PAFDConnectorData, + CAMP2pAFDControlPlane, + CAMP2PExtraInfo, + CAMP2PTransferState, + build_camp2p_topology, + prepare_token_id_transfer, + received_token_ids, + ) + +__all__ = [ + "CAMP2pAFDConnector", + "CAMP2pAFDControlPlane", + "CAMP2PAFDConnectorData", + "CAMP2PExtraInfo", + "CAMP2PTransferState", + "build_camp2p_topology", + "prepare_token_id_transfer", + "received_token_ids", +] + + +def __getattr__(name: str) -> Any: + if name in __all__: + from afd_plugin.connectors.npu.camp2p import connector + + return getattr(connector, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/afd_plugin/connectors/npu/camp2p.py b/afd_plugin/connectors/npu/camp2p/connector.py similarity index 85% rename from afd_plugin/connectors/npu/camp2p.py rename to afd_plugin/connectors/npu/camp2p/connector.py index 99403a9ce..40d93aa35 100644 --- a/afd_plugin/connectors/npu/camp2p.py +++ b/afd_plugin/connectors/npu/camp2p/connector.py @@ -146,10 +146,10 @@ def aiv_num_for_role(self, role: str) -> int: return self.core_num def validate_supported(self) -> None: - if self.compute_gate_on_attention: - raise RuntimeError( - "AFD NPU runtime does not support compute_gate_on_attention=true yet", - ) + # The gate switch is read from ``additional_config["afd"]``; the copy in + # ``connector_extra_config`` is parsed for compatibility and deliberately has no + # say here. feature_validation owns the supported shapes, so a second opinion in + # this class could only disagree with it. if self.quant_mode != 0: raise RuntimeError("AFD NPU runtime currently supports only quant_mode=0") @@ -308,6 +308,112 @@ def received_token_ids( return sim_expert_ids[:expected_tokens, 0].contiguous() +def prepare_routing_transfer( + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + *, + expected_tokens: int, + top_k: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """Pack an Attention-side gate result for the A2E routing channel. + + The A2E ids channel transports an ``int32`` ids tensor and a ``float32`` + scales tensor of shape ``(batch, topk)``. It was opened to carry token + identity for a token-keyed router; a gate that ran on the Attention ranks + fills the same two slots with the routing it already computed, so the FFN + rank selects experts without running the gate a second time. + + Args: + topk_ids: Expert indices chosen by the Attention-side gate, shaped + ``(tokens, topk)``. + topk_weights: Routing weights for those indices, same shape. + expected_tokens: Token count the transfer metadata declares. + top_k: Experts per token the operator is configured with. The wire + declares this column count, so a routing tensor of another width + would be read with the wrong stride. + + Returns: + The ``(ids, scales)`` pair to hand to the operator, already in the + channel's ``int32`` and ``float32`` layouts. + + Raises: + ValueError: If the routing tensors are not two-dimensional, disagree on + shape, do not describe exactly ``expected_tokens`` tokens, or their + column count differs from ``top_k``. + """ + + if topk_ids.dim() != 2 or topk_weights.dim() != 2: + raise ValueError( + "A2E routing transfer needs two-dimensional (tokens, topk) tensors, " + f"got shapes {tuple(topk_ids.shape)!r} and {tuple(topk_weights.shape)!r}", + ) + if topk_ids.shape != topk_weights.shape: + raise ValueError( + "A2E routing transfer needs ids and weights of the same shape, got " + f"{tuple(topk_ids.shape)!r} and {tuple(topk_weights.shape)!r}", + ) + if int(topk_ids.shape[1]) != top_k: + raise ValueError( + f"A2E routing transfer carries {int(topk_ids.shape[1])} experts per " + f"token but the operator is configured with k={top_k}", + ) + num_tokens = int(topk_ids.shape[0]) + if num_tokens != expected_tokens: + raise ValueError( + f"A2E routing transfer covers {num_tokens} tokens but the AFD " + f"transfer declares {expected_tokens}", + ) + return ( + topk_ids.to(dtype=torch.int32).contiguous(), + topk_weights.to(dtype=torch.float32).contiguous(), + ) + + +def received_routing( + sim_expert_ids: torch.Tensor, + sim_expert_scales: torch.Tensor, + *, + expected_tokens: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """Trim the received routing channel to the tokens this FFN rank computes on. + + The operator sizes the ids and scales outputs to the capacity it was given, + so rows past the FFN rank's token count are padding the MoE must not route. + Both tensors are sliced together so ids and weights stay aligned. + + Args: + sim_expert_ids: The operator's ids output, shaped ``(N, topk)``. + sim_expert_scales: The operator's scales output, shaped ``(N, topk)``. + The channel reuses the scales slot for the routing weights. + expected_tokens: Token count derived from the FFN rank's DP metadata, + which is what the FFN compute will actually run on. + + Returns: + The ``(topk_ids, topk_weights)`` pair restricted to ``expected_tokens``. + + Raises: + ValueError: If either tensor arrives with fewer rows than the FFN rank + computes on, which would misroute tokens. + """ + + received_tokens = int(sim_expert_ids.shape[0]) + if received_tokens < expected_tokens: + raise ValueError( + f"received {received_tokens} routing rows but the FFN rank computes " + f"on {expected_tokens} tokens; the routing channel is not aligned " + "with the FFN token layout", + ) + if int(sim_expert_scales.shape[0]) < expected_tokens: + raise ValueError( + f"received {int(sim_expert_scales.shape[0])} routing scale rows but " + f"the FFN rank computes on {expected_tokens} tokens", + ) + return ( + sim_expert_ids[:expected_tokens].contiguous(), + sim_expert_scales[:expected_tokens].contiguous(), + ) + + class CAMP2pAFDConnector(AFDConnectorBase): """Move model data between Attention and FFN workers on Ascend NPU. @@ -513,17 +619,20 @@ def send_attn_output( hidden_states: Model data with shape ``(tokens, hidden_size)``. context: Transfer context whose ``metadata`` supplies the layer number, ubatch number, and token count for this transfer. - **kwargs: An optional token-aligned ``input_ids`` tensor. When it is - supplied, the transfer runs with ``compute_gate=1`` so the ids - reach the FFN rank through the operator's ids channel, and the - matching ``recv_attn_output(recv_input_ids=True)`` returns them - on the payload's ``input_ids`` field. + **kwargs: Either an optional token-aligned ``input_ids`` tensor or an + optional ``routing`` pair ``(topk_ids, topk_weights)`` produced + by an Attention-side gate; at most one may be supplied because + they share the operator's ids channel. Either one makes the + transfer run with ``compute_gate=1``, and the matching + ``recv_attn_output(recv_input_ids=True)`` or + ``recv_attn_output(recv_routing=True)`` reads the slot back. Raises: RuntimeError: If the communication groups are not ready. ValueError: If the number of tokens in ``hidden_states`` does not - match ``context.metadata`` outside a ``torch.compile`` trace, or - if a supplied ``input_ids`` tensor is malformed. + match ``context.metadata`` outside a ``torch.compile`` trace, if + both wire-slot payloads were supplied, or if a supplied + ``input_ids``/``routing`` value is malformed. """ if not self._initialized: raise RuntimeError("CAMP2P connector is not initialized") @@ -536,6 +645,15 @@ def send_attn_output( f"CAMP2P metadata token count {metadata.total_tokens}", ) input_ids = cast(torch.Tensor | None, kwargs.get("input_ids")) + routing = cast( + tuple[torch.Tensor, torch.Tensor] | None, + kwargs.get("routing"), + ) + if input_ids is not None and routing is not None: + raise ValueError( + "CAMP2P send cannot put both token ids and Attention-side " + "routing on the operator's ids channel", + ) expert_ids: torch.Tensor | None = None expert_scales: torch.Tensor | None = None compute_gate = 0 @@ -546,6 +664,14 @@ def send_attn_output( expected_tokens=metadata.total_tokens, ) compute_gate = 1 + elif routing is not None: + expert_ids, expert_scales = prepare_routing_transfer( + routing[0], + routing[1], + expected_tokens=metadata.total_tokens, + top_k=self.num_experts_per_tok, + ) + compute_gate = 1 transfer_state = CAMP2PTransferState( aiv_num=self.aiv_num, batch_size=metadata.total_tokens, @@ -631,27 +757,36 @@ def recv_attn_output( number needed to create it. ``recv_input_ids`` states that this FFN rank expects token ids on the transfer, which selects the operator's ids mode and makes the connector validate and expose - the operator's ids slot on the payload. The sending rank has to - select the same mode, so only request ids for a run whose - Attention role transports them. + the operator's ids slot on the payload. ``recv_routing`` states + that the same slot carries an Attention-side gate result + instead, exposed as ``topk_ids``/``topk_weights``. The two are + mutually exclusive because they share one wire slot, and the + sending rank has to select the matching mode. Returns: The received hidden states, the information FFN needs to process them - and send the result back, and the transported ``input_ids`` when the - ids mode was selected. + and send the result back, and whichever of the transported + ``input_ids`` or gate result the selected mode carries. Raises: RuntimeError: If communication is not ready, transfer information is missing, or the requested ubatch group does not exist. - ValueError: If ids were requested but do not align with the FFN - rank's token layout. + ValueError: If both wire-slot modes were requested, or if the + transported slot does not align with the FFN rank's token + layout. """ if not self._initialized: raise RuntimeError("CAMP2P connector is not initialized") layer_idx: int = kwargs.get("layer_idx", 0) max_num_tokens: int = kwargs.get("max_num_tokens", 0) recv_input_ids: bool = bool(kwargs.get("recv_input_ids", False)) - compute_gate_mode = 1 if recv_input_ids else 0 + recv_routing: bool = bool(kwargs.get("recv_routing", False)) + if recv_input_ids and recv_routing: + raise ValueError( + "CAMP2P receive cannot read the wire slot as both token ids and " + "Attention-side routing; request exactly one", + ) + compute_gate_mode = 1 if (recv_input_ids or recv_routing) else 0 batch_size = _num_tokens_for_ffn_rank( self.dp_metadata_list, ubatch_idx, @@ -707,15 +842,25 @@ def recv_attn_output( # the other mode would hand the model uninitialised device memory as token # ids, which a token-keyed router turns into an out-of-range table read. received_ids: torch.Tensor | None = None - if compute_gate_mode == 1: + received_topk_ids: torch.Tensor | None = None + received_topk_weights: torch.Tensor | None = None + if recv_input_ids: received_ids = received_token_ids( outputs[1], expected_tokens=batch_size, ) + elif recv_routing: + received_topk_ids, received_topk_weights = received_routing( + outputs[1], + outputs[2], + expected_tokens=batch_size, + ) return AFDA2FTransferPayload( hidden_states=outputs[0], context=context, input_ids=received_ids, + topk_ids=received_topk_ids, + topk_weights=received_topk_weights, ) def send_ffn_output( diff --git a/afd_plugin/connectors/npu/camp2p/external_routing.py b/afd_plugin/connectors/npu/camp2p/external_routing.py new file mode 100644 index 000000000..f519e8a00 --- /dev/null +++ b/afd_plugin/connectors/npu/camp2p/external_routing.py @@ -0,0 +1,159 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Routing inputs the Attention side computed and the FFN side must reuse. + +The FFN side runs the MoE with routing that arrived over the A2E transport +instead of computing it locally. That changes what ``prepare`` has to do: the +activation and the routing must be padded and gathered by exactly the same rule, +or a token row ends up weighted by another row's routing. + +Two shapes of work exist, and this module owns the part they share: + +- ALLGATHER pads to ``max_tokens_across_dp`` and then gathers across the DP + group, so every rank sees every token. +- MC2 pads to ``padded_num_tokens`` and does not gather; the dispatcher moves + tokens to their expert ranks. + +``padded``, ``gathered`` and ``split_rows`` are row-aligned transforms. They keep +the caller's row order and are not required to share storage with the input, so +callers compare contents, not buffers. + +``gathered`` takes the collective as an argument so the DP group, a fake, and a +unit test all use the same code path. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from typing import Final + +import torch + +# Expert indices reach the MoE kernels as integers, and gather/combine treat them as +# integers. Listing the accepted dtypes explicitly also rejects bool and complex, +# which ``is_floating_point()`` alone would let through. +SUPPORTED_INDEX_DTYPES: Final[tuple[torch.dtype, ...]] = ( + torch.int32, + torch.int64, +) + + +@dataclass(frozen=True, eq=False) +class AFDExternalRoutingInputs: + """Routing computed on the Attention side, in the FFN rank's local row order. + + Attributes: + topk_ids: Expert ids per token, ``[tokens, top_k]``. + topk_weights: Gate weights per token, ``[tokens, top_k]``. Any model-level + routed scaling is already folded into these, mirroring the native + router, so the FFN side must not apply it again. + """ + + topk_ids: torch.Tensor + topk_weights: torch.Tensor + + def __post_init__(self) -> None: + # Shapes and dtypes are checked here rather than at the padding site: the + # expert ids and the weights are consumed by kernels that do not validate + # them, so a wrong dtype is only noticed as wrong results. + _require_2d(self.topk_ids, "topk_ids") + _require_2d(self.topk_weights, "topk_weights") + if self.topk_ids.shape != self.topk_weights.shape: + raise ValueError( + "topk_ids and topk_weights must have the same shape, got " + f"{tuple(self.topk_ids.shape)} and {tuple(self.topk_weights.shape)}", + ) + if self.topk_ids.dtype not in SUPPORTED_INDEX_DTYPES: + raise ValueError( + "topk_ids must hold expert indices in one of " + f"{[str(dtype) for dtype in SUPPORTED_INDEX_DTYPES]}, got " + f"{self.topk_ids.dtype}", + ) + if not self.topk_weights.is_floating_point(): + raise ValueError( + "topk_weights must hold gate weights, got dtype " + f"{self.topk_weights.dtype}", + ) + + @property + def token_count(self) -> int: + """Rows before any padding.""" + + return int(self.topk_ids.shape[0]) + + def padded(self, pad_size: int) -> AFDExternalRoutingInputs: + """Append ``pad_size`` inert rows so later collectives stay aligned. + + Padding rows carry weight 0, so whichever expert they name contributes + nothing. That keeps the padding value irrelevant to the result and lets + the ids pad stay at 0. + """ + + if pad_size < 0: + raise ValueError(f"pad_size must be non-negative, got {pad_size}") + if pad_size == 0: + return self + + return replace( + self, + topk_ids=_pad_rows(self.topk_ids, pad_size, 0), + topk_weights=_pad_rows(self.topk_weights, pad_size, 0), + ) + + def gathered(self, all_gather) -> AFDExternalRoutingInputs: + """Concatenate the routing of every DP rank, in rank order. + + ``all_gather`` must be the same collective that gathers the activation, + invoked the same way, so the row order matches token for token. + + The gathered row count is ``dp_size * max(local_rows)``, not the sum of + the local row counts: every rank pads to the DP-wide maximum before the + gather, so each rank's padding rows survive into the result. + """ + + return replace( + self, + topk_ids=all_gather(self.topk_ids), + topk_weights=all_gather(self.topk_weights), + ) + + def split_rows(self, split) -> AFDExternalRoutingInputs: + """Keep one rank's slice, exactly as the activation is sliced. + + MC2 shards the token dimension across TP ranks instead of gathering it, so + the routing has to take the same slice. ``split`` must be the split the + caller applies to the activation. + """ + + return replace( + self, + topk_ids=split(self.topk_ids), + topk_weights=split(self.topk_weights), + ) + + +def _require_2d(tensor: torch.Tensor, name: str) -> None: + """Reject anything that is not a token-by-something matrix.""" + + if tensor.dim() != 2: + raise ValueError( + f"{name} must be 2-D (tokens by something), got shape " + f"{tuple(tensor.shape)}", + ) + + +def _pad_rows(tensor: torch.Tensor, pad_size: int, value: int) -> torch.Tensor: + """Append ``pad_size`` rows filled with ``value``.""" + + pad = torch.full( + (pad_size, *tensor.shape[1:]), + value, + dtype=tensor.dtype, + device=tensor.device, + ) + return torch.cat([tensor, pad], dim=0) + + +__all__ = [ + "AFDExternalRoutingInputs", +] diff --git a/afd_plugin/model_executor/models/npu/deepseek_v4.py b/afd_plugin/model_executor/models/npu/deepseek_v4.py index a025717bd..a7d45536d 100644 --- a/afd_plugin/model_executor/models/npu/deepseek_v4.py +++ b/afd_plugin/model_executor/models/npu/deepseek_v4.py @@ -24,7 +24,11 @@ from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.sequence import IntermediateTensors -from afd_plugin.config import AFD_ASYNC_CONNECTOR, parse_afd_config +from afd_plugin.config import ( + AFD_ASYNC_CONNECTOR, + CAMP2P_CONNECTOR, + parse_afd_config, +) from afd_plugin.connectors import ( AFDExpertRoutingSpec, AFDTransferContext, @@ -311,6 +315,72 @@ def forward( return output +class AFDDeepseekV4SyncAttentionGateRemoteMoE(AFDDeepseekV4AttentionGateRemoteMoE): + """DSV4 gate shell that keeps the router on Attention for the sync connector. + + Inheritance is deliberate: the gate construction, the FP32 precast weight, the + hash-layer table and the shared-expert module are identical to the async shell, + and only the transport differs. Duplicating that construction would let the two + shells drift on routing parameters. + + The forward differs in three ways, all forced by the sync contract: + + - routing travels on the A2E wire slot instead of a CAM dispatch payload, so the + send and the receive are separate calls; + - the shared experts run *between* those calls, which is the whole point of + moving them here: their work overlaps the FFN side's routed work; + - no DBO yield, because the first prototype keeps the sync path free of the + scheduling interaction the async shell needs. + + Reusing ``RemoteFFNProxy._send_and_receive`` is not possible here: it issues the + send and the receive back to back, leaving no window for the shared experts. + """ + + def forward( + self, + hidden_states: torch.Tensor, + input_ids: torch.Tensor | None = None, + hidden_states_fp32: torch.Tensor | None = None, + ) -> torch.Tensor: + from afd_plugin.model_executor.models.npu import deepseek_v4_attention_gate + + topk_weights, topk_ids = deepseek_v4_attention_gate.compute_attention_gate_topk( + self, + hidden_states, + input_ids=input_ids, + hidden_states_fp32=hidden_states_fp32, + ) + afd_metadata = get_afd_metadata_from_forward_context() + if afd_metadata is None: + raise RuntimeError("Sync Attention-side gate requires AFD forward metadata") + context = AFDTransferContext( + metadata=AFDTransferMetadata.create_attention_metadata( + layer_idx=self.layer_idx, + stage_idx=afd_metadata.stage_idx, + seq_len=int(hidden_states.shape[0]), + ) + ) + # Routing shares the wire slot with token ids, so this send carries routing + # only. The FFN side is configured with recv_routing for the same reason. + afd_metadata.connector.send_attn_output( + hidden_states, + context, + routing=(topk_ids, topk_weights), + ) + shared_output = ( + self.shared_experts(hidden_states) + if self.shared_experts is not None + else None + ) + output = afd_metadata.connector.recv_ffn_output( + ref_tensor=hidden_states, + ubatch_idx=afd_metadata.stage_idx, + ) + if shared_output is not None: + output = output + shared_output + return output + + class AFDDeepseekV4DecoderLayer(native.DeepseekV4DecoderLayer): """Role-local DSV4 decoder layer. @@ -347,6 +417,13 @@ def __init__( self.config = config self.afd_role = afd_config.role self.compute_gate_on_attention = bool(afd_config.compute_gate_on_attention) + # Which wire protocol this layer expects, decided from the configuration. + # The payload only validates that expectation; it must not pick the path, + # because a missing or mixed payload would otherwise silently reroute. + self.uses_external_routing = bool( + afd_config.compute_gate_on_attention + and afd_config.connector == CAMP2P_CONNECTOR + ) self.is_moe_layer = True # ### PATCH END: AFD decoder metadata cache_config = vllm_config.cache_config @@ -397,7 +474,14 @@ def __init__( need_gather_q_kv=self.use_sequence_parallel_moe and self.enable_dsa_cp, ) if self.compute_gate_on_attention: - self.mlp = AFDDeepseekV4AttentionGateRemoteMoE( + # Both shells keep the router on Attention; only the transport + # differs. Choosing by connector keeps the async CAM path unchanged. + gate_shell = ( + AFDDeepseekV4AttentionGateRemoteMoE + if afd_config.connector == AFD_ASYNC_CONNECTOR + else AFDDeepseekV4SyncAttentionGateRemoteMoE + ) + self.mlp = gate_shell( config=config, layer_idx=self.layer_idx, prefix=f"{prefix}.mlp", @@ -410,7 +494,10 @@ def __init__( self.self_attn = native.PPMissingLayer() _refresh_ascend_fused_moe() moe_config = config - if afd_config.connector == AFD_ASYNC_CONNECTOR: + if self.compute_gate_on_attention: + # The shared experts live wherever the router lives, because they + # run in the same window as the gate. That is true for either + # connector, not just for the async one. moe_config = copy(config) # HF validates this public field as int, but pinned native # MoE uses only None to omit shared construction (0 builds a @@ -427,6 +514,23 @@ def __init__( raise ValueError(f"Unsupported AFD role: {afd_config.role!r}") # ### PATCH END: role-local Attention and remote experts + # The external-routing component is built here, while a vLLM config context + # exists, so the first forward onto MC2 never constructs a comm. + self.afd_external_routing_moe = None + # Synchronous external routing only: the async CAM connector keeps its own + # FFN path, even though it also computes the gate on the Attention side. + if ( + self.uses_external_routing + and afd_config.role == _FFN_ROLE + and isinstance(self.mlp, native.DeepseekV4MoE) + ): + from afd_plugin.model_executor.models.npu import ( + deepseek_v4_external_routing_moe as external_routing_moe, + ) + + self.afd_external_routing_moe = external_routing_moe.AFDExternalRoutingMoE( + self.mlp.experts.routed_experts, + ) self.input_layernorm = native.RMSNorm(config.hidden_size, eps=self.norm_eps) self.post_attention_layernorm = native.RMSNorm( @@ -448,17 +552,90 @@ def __init__( def compute_ffn_output( self, hidden_states: torch.Tensor, + *, + group_list: torch.Tensor | None = None, + topk_ids: torch.Tensor | None = None, + topk_weights: torch.Tensor | None = None, **_: Any, ) -> torch.Tensor: if not isinstance(self.mlp, native.DeepseekV4MoE): raise RuntimeError( "DSV4 compute_ffn_output requires the FFN role to own the MoE" ) + # The configuration decides which protocol this layer speaks; the payload + # only has to satisfy it. Letting the payload choose would turn a missing or + # mixed payload into a silent reroute to a different computation. + routing_payload_present = topk_ids is not None or topk_weights is not None + + if self.uses_external_routing: + # CAM dispatch data belongs to the async connector. Keep rejecting it + # instead of letting ``**_`` absorb it silently: a layer speaks one + # protocol, and a mixed payload would otherwise run half-initialized. + if group_list is not None: + raise RuntimeError( + "DSV4 sync external routing received CAM dispatch data as well; " + "a layer speaks one protocol", + ) + if topk_ids is None or topk_weights is None: + raise RuntimeError( + "DSV4 sync external routing requires both topk_ids and " + f"topk_weights; got ids={topk_ids is not None}, " + f"weights={topk_weights is not None}", + ) + return self._compute_external_routing_ffn( + hidden_states, + topk_ids=topk_ids, + topk_weights=topk_weights, + ) + + # The async CAM FFN runs through the layered executor upstream; this method + # only serves the synchronous external-routing path and the target-hash MoE. if self.compute_gate_on_attention: raise RuntimeError("Async CAM FFN requires the layered executor") + if routing_payload_present: + raise RuntimeError( + "DSV4 gate-on-FFN received Attention-side routing; the FFN side owns " + "the gate for this configuration and must compute it itself", + ) # Target Hash MoE takes explicit token IDs from the CAM transfer. return self.mlp(hidden_states, input_ids=get_forward_context().input_ids) + def _compute_external_routing_ffn( + self, + hidden_states: torch.Tensor, + *, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + ) -> torch.Tensor: + """Run the FFN on routing the Attention side computed. + + Mirrors ``AscendRoutedExperts.forward`` from the pinned vLLM-Ascend tag, with + two differences: the top-k comes from the transport instead of a local gate, + and both the padding/gathering of the routing and the scale it carries are + supplied by the AFD prepare stage. + + Routed scaling needs no separate step here. The Attention-side selector folds + ``routed_scaling_factor`` into ``topk_weights``, and ``apply`` consumes those + weights directly in the token combine, so the factor is applied exactly once. + Applying it again here would scale the routed output a second time. + + Both ALLGATHER and MC2 are wired. They share this flow because both AFD prepare + implementations pad the routing, and MC2's additionally splits it for the TP + sharding before returning the aligned routing. What differs is only whether + finalize reduces across ranks. + + Any other communication method is rejected rather than silently taking one of + these routes. + """ + + component = self.afd_external_routing_moe + assert component is not None, "external routing component was not built at init" + return component.forward( + hidden_states, + topk_ids=topk_ids, + topk_weights=topk_weights, + ) + @native.support_torch_compile class AFDDeepseekV4Model(native.DeepseekV4Model): @@ -662,7 +839,8 @@ def compute_experts_output( router_logits: torch.Tensor, ) -> torch.Tensor: raise NotImplementedError( - "DSV4 AFD does not support compute_gate_on_attention yet" + "DSV4 AFD never returns gate outputs through this method; the " + "Attention-side gate keeps its top-k inside the gate shell" ) @@ -676,9 +854,20 @@ class AFDDeepseekV4ForCausalLM(native.AscendDeepseekV4ForCausalLM): # installs them in the forward context before the FFN compute. afd_requires_input_ids = True + # The wire slot carries either token ids or Attention-side routing, never both. + # When the router lives on Attention with the sync connector, the FFN side needs + # the routing instead of the ids, and the runner must ask for it by name. + afd_requires_attention_routing = False + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: self.afd_config = parse_afd_config(vllm_config, validate=False) self.afd_role = self.afd_config.role + # Per-instance, because the same model class serves both connectors and the + # async CAM shell carries its routing inside its own dispatch payload. + self.afd_requires_attention_routing = bool( + self.afd_config.compute_gate_on_attention + and self.afd_config.connector == CAMP2P_CONNECTOR + ) super().__init__(vllm_config=vllm_config, prefix=prefix) # vLLM 5df63a7 model_loader/utils.py invokes this model hook after quant @@ -733,11 +922,14 @@ def compute_experts_output( def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: # Only the gate-on-Attention configuration gives Attention a router; with # the gate on FFN its MoE slot is a parameter-free transfer shell. + # The shared experts follow the router, so their weights load on Attention + # for either connector; keying this on the async connector alone would + # construct the modules on the sync path without ever loading them. role_weights = _iter_role_weights( weights, role=self.afd_role, attn_owns_gate=bool(self.afd_config.compute_gate_on_attention), - attention_shared_experts=self.afd_config.connector == AFD_ASYNC_CONNECTOR, + attention_shared_experts=bool(self.afd_config.compute_gate_on_attention), ) loaded = super().load_weights(role_weights) return loaded diff --git a/afd_plugin/model_executor/models/npu/deepseek_v4_external_routing_moe.py b/afd_plugin/model_executor/models/npu/deepseek_v4_external_routing_moe.py new file mode 100644 index 000000000..c47a9eb28 --- /dev/null +++ b/afd_plugin/model_executor/models/npu/deepseek_v4_external_routing_moe.py @@ -0,0 +1,350 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""AFD-owned MoE execution for the external-routing FFN path. + +Attention computes the gate when ``compute_gate_on_attention`` is on, so the FFN +side receives routing for its own input rows instead of computing it. Upstream +prepares the routing inside the prepare stage because upstream owns the gate; +here the routing arrives as an explicit input, is aligned by the same padding, +gathering and sharding rules as the activation, and comes back as an explicit +result. + +The AFD prepare/finalize subclasses are selected through AFD comm implementations +that only override the factory method, so ordinary execution keeps the upstream +classes. The MoE component owns the per-token state that expert execution reads +back, and clears it on both success and failure. +""" + +from __future__ import annotations + +import torch +import torch.nn as nn +from vllm.distributed.parallel_state import get_pcp_group +from vllm_ascend.ops.fused_moe.dataclass.prepare_finalize import MoEPrepareOutput +from vllm_ascend.ops.fused_moe.moe_comm_method import ( + AllGatherCommImpl, + MC2CommImpl, +) +from vllm_ascend.ops.fused_moe.prepare_finalize import ( + PrepareAndFinalizeWithAllGather, + PrepareAndFinalizeWithMC2, +) +from vllm_ascend.quantization.quant_type import QuantType + +from afd_plugin.connectors.npu.camp2p.external_routing import ( + AFDExternalRoutingInputs, +) + + +class AFDPrepareAndFinalizeWithAllGather(PrepareAndFinalizeWithAllGather): + """ALLGATHER prepare that accepts routing from the Attention side.""" + + def prepare_external( + self, + hidden_states: torch.Tensor, + routing: AFDExternalRoutingInputs, + *, + router_logits: torch.Tensor | None = None, + replace_allreduce: bool = False, + quant_type: QuantType = QuantType.NONE, + ) -> tuple[MoEPrepareOutput, AFDExternalRoutingInputs]: + """Pad and gather the routing exactly like the activation. + + The native ``prepare`` stays untouched for callers that own the gate; this + entry returns the aligned routing instead of publishing it on ``self``. + """ + from vllm_ascend.ascend_forward_context import _EXTRA_CTX + + external = routing + if external.token_count != hidden_states.shape[0]: + raise ValueError( + "external routing covers a different token count than the activation: " + f"{external.token_count} routing rows for " + f"{hidden_states.shape[0]} activation rows", + ) + + if self.moe_config.dp_size > 1: + max_tokens_across_dp = _EXTRA_CTX.max_tokens_across_dp + + self.num_tokens = hidden_states.shape[0] + pad_size = max_tokens_across_dp - self.num_tokens + if pad_size > 0: + hidden_states = nn.functional.pad(hidden_states, (0, 0, 0, pad_size)) + if router_logits is not None: + router_logits = nn.functional.pad( + router_logits, (0, 0, 0, pad_size) + ) + external = external.padded(pad_size) + + hidden_states = self.moe_config.dp_group.all_gather(hidden_states, 0) + if router_logits is not None: + router_logits = self.moe_config.dp_group.all_gather(router_logits, 0) + external = external.gathered( + lambda tensor: self.moe_config.dp_group.all_gather(tensor, 0), + ) + + if self.moe_config.pcp_size > 1: + max_tokens_across_pcp = _EXTRA_CTX.max_tokens_across_pcp + + self.num_tokens_pcp = hidden_states.shape[0] + pad_size = max_tokens_across_pcp - self.num_tokens_pcp + if pad_size > 0: + hidden_states = nn.functional.pad(hidden_states, (0, 0, 0, pad_size)) + if router_logits is not None: + router_logits = nn.functional.pad( + router_logits, (0, 0, 0, pad_size) + ) + external = external.padded(pad_size) + + hidden_states = get_pcp_group().all_gather(hidden_states, dim=0) + if router_logits is not None: + router_logits = get_pcp_group().all_gather(router_logits, dim=0) + external = external.gathered( + lambda tensor: get_pcp_group().all_gather(tensor, dim=0), + ) + + return ( + MoEPrepareOutput( + hidden_states=hidden_states, + router_logits=router_logits, + mc2_mask=None, + padded_hidden_states_shape=None, + pertoken_scale=None, + ), + external, + ) + + +class AFDPrepareAndFinalizeWithMC2(PrepareAndFinalizeWithMC2): + """MC2 prepare that accepts routing from the Attention side.""" + + def prepare_external( + self, + hidden_states: torch.Tensor, + routing: AFDExternalRoutingInputs, + *, + router_logits: torch.Tensor | None = None, + replace_allreduce: bool = False, + quant_type: QuantType = QuantType.NONE, + ) -> tuple[MoEPrepareOutput, AFDExternalRoutingInputs]: + """Pad and TP-shard the routing exactly like the activation.""" + from vllm_ascend.ascend_forward_context import _EXTRA_CTX + from vllm_ascend.ops.fused_moe.moe_utils import _pad_tokens_with_cat + + external = routing + if external.token_count != hidden_states.shape[0]: + raise ValueError( + "external routing covers a different token count than the activation: " + f"{external.token_count} routing rows for " + f"{hidden_states.shape[0]} activation rows", + ) + + self.replace_allreduce = replace_allreduce + mc2_mask = _EXTRA_CTX.mc2_mask + if self.replace_allreduce: + self.num_tokens = hidden_states.shape[0] + start = self.tp_rank * self.num_tokens + mc2_mask = mc2_mask[start : start + self.num_tokens] + pad_size = _EXTRA_CTX.padded_num_tokens // self.tp_size - self.num_tokens + if pad_size > 0: + hidden_states = nn.functional.pad(hidden_states, (0, 0, 0, pad_size)) + if router_logits is not None: + router_logits = nn.functional.pad( + router_logits, (0, 0, 0, pad_size) + ) + external = external.padded(pad_size) + mc2_mask = nn.functional.pad(mc2_mask, (0, pad_size), value=False) + elif self.tp_size > 1: + split_mc2_mask = torch.tensor_split(mc2_mask, self.tp_size, dim=0) + mc2_mask = split_mc2_mask[self.tp_rank] + + padded_hidden_states_shape = hidden_states.shape + if not self.replace_allreduce: + self.num_tokens, _ = hidden_states.shape + target_pad_length = _EXTRA_CTX.padded_num_tokens + pad_size = target_pad_length - self.num_tokens + + if pad_size > 0: + hidden_states = _pad_tokens_with_cat(hidden_states, target_pad_length) + if router_logits is not None: + router_logits = _pad_tokens_with_cat( + router_logits, target_pad_length + ) + external = external.padded(pad_size) + padded_hidden_states_shape = hidden_states.shape + + if self.tp_size > 1: + split_hidden_states = torch.tensor_split( + hidden_states, self.tp_size, dim=0 + ) + if router_logits is not None: + split_router_logits = torch.tensor_split( + router_logits, self.tp_size, dim=0 + ) + router_logits = split_router_logits[self.tp_rank] + hidden_states = split_hidden_states[self.tp_rank] + external = external.split_rows( + lambda tensor: torch.tensor_split(tensor, self.tp_size, dim=0)[ + self.tp_rank + ], + ) + + return ( + MoEPrepareOutput( + hidden_states=hidden_states, + router_logits=router_logits, + mc2_mask=mc2_mask, + padded_hidden_states_shape=padded_hidden_states_shape, + pertoken_scale=None, + ), + external, + ) + + +class AFDAllGatherCommImpl(AllGatherCommImpl): + """ALLGATHER comm that prepares external routing instead of computing it.""" + + def _get_prepare_finalize(self) -> AFDPrepareAndFinalizeWithAllGather: + return AFDPrepareAndFinalizeWithAllGather(self.moe_config) + + +class AFDMC2CommImpl(MC2CommImpl): + """MC2 comm that prepares external routing instead of computing it.""" + + def _get_prepare_finalize(self) -> AFDPrepareAndFinalizeWithMC2: + return AFDPrepareAndFinalizeWithMC2(self.moe_config) + + +# Native comm types the external-routing FFN path supports, and the AFD +# counterparts that own the external-routing prepare/finalize. +_AFD_COMM_BY_NATIVE: dict[type, type] = { + AllGatherCommImpl: AFDAllGatherCommImpl, + MC2CommImpl: AFDMC2CommImpl, +} +_NATIVE_COMM_BY_AFD: dict[type, type] = { + afd: native for native, afd in _AFD_COMM_BY_NATIVE.items() +} + + +def _afd_types_by_comm_type() -> dict[object, type]: + """Map the runtime comm type to the AFD class that owns its prepare stage.""" + from vllm_ascend.ascend_forward_context import MoECommType + + return { + MoECommType.ALLGATHER: AFDAllGatherCommImpl, + MoECommType.MC2: AFDMC2CommImpl, + } + + +def _prebuilt_comms(moe_config) -> dict[type, object]: + """Build the AFD comm instances the native selector can pick from. + + The native selector is keyed by ``MoECommType`` and returns ``None`` when this + configuration was not initialised with that method, so a missing candidate means + the type is simply absent here. Construction happens in the caller's config + context, which is why it runs at component creation and not per forward. + """ + from vllm_ascend.ascend_forward_context import MoECommType + from vllm_ascend.ops.fused_moe.moe_comm_method import get_moe_comm_method + + candidates: dict[type, object] = {} + for comm_type, afd_type in ( + (MoECommType.ALLGATHER, AFDAllGatherCommImpl), + (MoECommType.MC2, AFDMC2CommImpl), + ): + native = get_moe_comm_method(comm_type, moe_config) + if native is None: + continue + candidates[afd_type] = afd_type(native.moe_config) + return candidates + + +class AFDExternalRoutingMoE: + """Run the routed experts on routing supplied by the Attention side. + + The AFD comm instances are built while the component is constructed, in a config + context. A forward only looks up the instance matching the native selection; it + never builds a missing one. + """ + + def __init__(self, routed_experts): + self.routed_experts = routed_experts + self._candidates = _prebuilt_comms(routed_experts.moe_config) + if not self._candidates: + raise RuntimeError( + "DSV4 external routing found no AFD comm candidate for this " + "configuration; the ALLGATHER and MC2 paths are the supported ones", + ) + + def _comm_for(self, comm_type, native_comm): + if native_comm is None: + # The MRV2 hook resolves the method without a config, which misses the + # (type, expert-count) registry key and leaves the instance unset. The + # comm type is always present, so fall back to it. + afd_type = _afd_types_by_comm_type().get(comm_type) + afd_comm = self._candidates.get(afd_type) if afd_type is not None else None + if afd_comm is not None: + return afd_comm + raise RuntimeError( + "DSV4 external routing could not resolve the MoE comm method for " + f"{comm_type} with this layer's config; no AFD candidate was built", + ) + for afd_type, afd_comm in self._candidates.items(): + if isinstance(native_comm, _NATIVE_COMM_BY_AFD[afd_type]): + return afd_comm + raise RuntimeError( + "DSV4 external routing is wired for the ALLGATHER and MC2 MoE paths; " + f"got {type(native_comm).__name__}. The all-to-all and fused-MC2 paths " + "shard differently and are not supported.", + ) + + def forward( + self, + hidden_states: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + ) -> torch.Tensor: + from vllm_ascend.ascend_forward_context import _EXTRA_CTX + + experts = self.routed_experts + previous_comm = _EXTRA_CTX.moe_comm_method + comm = self._comm_for(_EXTRA_CTX.moe_comm_type, previous_comm) + prepare_finalize = comm.prepare_finalize + + try: + # Scope switch, not a registry change: expert execution reads the comm + # from this context attribute, so both sides use the same object. + _EXTRA_CTX.moe_comm_method = comm + prepare_output, aligned = prepare_finalize.prepare_external( + hidden_states, + AFDExternalRoutingInputs( + topk_ids=topk_ids, + topk_weights=topk_weights, + ), + router_logits=None, + replace_allreduce=experts.moe_config.is_sequence_parallel, + quant_type=experts.quant_type, + ) + experts.ascend_pertoken_scale = prepare_output.pertoken_scale + experts.ascend_mc2_mask = prepare_output.mc2_mask + try: + fused = experts.quant_method.apply( + layer=experts, + x=prepare_output.hidden_states, + topk_weights=aligned.topk_weights, + topk_ids=aligned.topk_ids, + shared_experts=None, + shared_experts_input=None, + ) + finally: + experts.ascend_pertoken_scale = None + experts.ascend_mc2_mask = None + return comm.finalize( + hidden_states=fused.routed_out, + reduce_results=isinstance(comm, AllGatherCommImpl), + padded_hidden_states_shape=prepare_output.padded_hidden_states_shape, + ) + finally: + experts.ascend_pertoken_scale = None + experts.ascend_mc2_mask = None + _EXTRA_CTX.moe_comm_method = previous_comm diff --git a/afd_plugin/v1/worker/ffn_model_runner.py b/afd_plugin/v1/worker/ffn_model_runner.py index bab6ac42a..4bbc2c893 100644 --- a/afd_plugin/v1/worker/ffn_model_runner.py +++ b/afd_plugin/v1/worker/ffn_model_runner.py @@ -220,7 +220,13 @@ def _ffn_forward( stage_idx: self._make_ffn_dp_metadata(dp_metadata_list[stage_idx]) for stage_idx in stage_ids } - recv_input_ids = getattr(model, "afd_requires_input_ids", False) + # The wire slot carries one payload: either token ids or Attention-side + # routing. Routing wins when the model declares it, because a sync + # gate-on-Attention model does not need ids on the FFN side at all. + recv_routing = getattr(model, "afd_requires_attention_routing", False) + recv_input_ids = ( + False if recv_routing else getattr(model, "afd_requires_input_ids", False) + ) with _ffn_forward_context(self.vllm_config) as forward_context: for layer_idx in layer_indices: uses_remote_experts = layer_idx in experts_layer_indices @@ -233,7 +239,9 @@ def _ffn_forward( recv_kwargs: dict[str, Any] = {} if routing_spec is not None: recv_kwargs["routing_spec"] = routing_spec - if recv_input_ids: + if recv_routing: + recv_kwargs["recv_routing"] = True + elif recv_input_ids: recv_kwargs["recv_input_ids"] = True payload = self.connector.recv_attn_output( ubatch_idx=stage_idx, diff --git a/afd_plugin/v1/worker/npu/ffn_model_runner.py b/afd_plugin/v1/worker/npu/ffn_model_runner.py index 4cb11f326..72b7764f6 100644 --- a/afd_plugin/v1/worker/npu/ffn_model_runner.py +++ b/afd_plugin/v1/worker/npu/ffn_model_runner.py @@ -284,7 +284,13 @@ def _ffn_forward( # A model whose router is keyed by token identity declares that the FFN # role needs the tokens' ids. Resolve it once per forward, outside the # layer loop, so a per-layer re-read cannot drift within a step. - recv_input_ids = getattr(self.model, "afd_requires_input_ids", False) + # The wire slot carries one payload: either ids or Attention-side routing. + recv_routing = getattr(self.model, "afd_requires_attention_routing", False) + recv_input_ids = ( + False + if recv_routing + else getattr(self.model, "afd_requires_input_ids", False) + ) for layer_idx in _ffn_layer_indices(self): for stage_idx in stage_ids: @@ -315,6 +321,7 @@ def _ffn_forward( layer_idx=layer_idx, max_num_tokens=self.max_num_tokens, recv_input_ids=recv_input_ids, + recv_routing=recv_routing, ) context = payload.context metadata = context.metadata @@ -337,9 +344,18 @@ def _ffn_forward( assert states, "Context.states must not be None" _set_moe_layer_index(forward_context, layer_idx) + # Only a model that declared Attention-side routing gets the + # extra keywords. Passing them unconditionally changes the call + # contract for every other model wrapper, including ones that + # accept only (hidden_states, layer_idx). + ffn_kwargs: dict[str, Any] = {} + if recv_routing: + ffn_kwargs["topk_ids"] = payload.topk_ids + ffn_kwargs["topk_weights"] = payload.topk_weights rank_ffn_output = self.model.compute_ffn_output( hidden_states=hidden_states, layer_idx=layer_idx, + **ffn_kwargs, ) _send_ffn_output( self.connector, diff --git a/afd_plugin/validation.py b/afd_plugin/validation.py index bfef19f77..7d586b3a5 100644 --- a/afd_plugin/validation.py +++ b/afd_plugin/validation.py @@ -212,13 +212,13 @@ def validate_npu_model_runner_v2_config( "on Attention" ) requested_ffn_mode(vllm_config) - elif ( - afd_config.connector != CAMP2P_CONNECTOR or afd_config.compute_gate_on_attention - ): + elif afd_config.connector != CAMP2P_CONNECTOR: raise RuntimeError( - "AFD NPU ModelRunnerV2 requires synchronous CAMP2pAFDConnector " - "with compute_gate_on_attention=false, or Async CAM", + "AFD NPU ModelRunnerV2 requires either synchronous CAMP2pAFDConnector " + "or Async CAM", ) + # The sync Attention-side gate limits (connector, target, TP, DBO/ubatching) + # live in feature_validation and run from both workers before this point. parallel = vllm_config.parallel_config if ( diff --git a/docs/design/module/connector_contracts.md b/docs/design/module/connector_contracts.md index f9624ba8e..8403e52b5 100644 --- a/docs/design/module/connector_contracts.md +++ b/docs/design/module/connector_contracts.md @@ -67,7 +67,7 @@ depend on role worker implementations. | Payloads and control-plane codec | [`metadata.py`](../../../afd_plugin/connectors/metadata.py) | Connector unit tests and [`test_forward_context.py`](../../../tests/unit/model_executor/models/test_forward_context.py) | | CUDA P2P | [`gpu/p2p.py`](../../../afd_plugin/connectors/gpu/p2p.py), [`topology.py`](../../../afd_plugin/distributed/topology.py) | [`test_p2p_connector.py`](../../../tests/unit/connectors/test_p2p_connector.py), [DeepSeek-V2-Lite E2E](../../../tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py) | | CUDA DeepEP M2N | [`gpu/deepep_m2n/`](../../../afd_plugin/connectors/gpu/deepep_m2n/), `AFDRoutedExpertsConnector` in [`base.py`](../../../afd_plugin/connectors/base.py) | [`test_deepep_m2n_connector.py`](../../../tests/unit/connectors/test_deepep_m2n_connector.py), [GPU device tests](../../../tests/gpu/README.md), [Qwen3 MoE E2E](../../../tests/e2e/models/qwen3_moe/test_qwen3_moe.py), [DeepSeek-V2-Lite E2E](../../../tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py) (`test_deepseek_v2_lite_deepep_m2n`) | -| Ascend CAMP2P | [`npu/camp2p.py`](../../../afd_plugin/connectors/npu/camp2p.py) | [`test_camp2p_connector.py`](../../../tests/unit/connectors/test_camp2p_connector.py), [DeepSeek-V2-Lite E2E](../../../tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py) | +| Ascend CAMP2P | [`npu/camp2p/connector.py`](../../../afd_plugin/connectors/npu/camp2p/connector.py) | [`test_camp2p_connector.py`](../../../tests/unit/connectors/test_camp2p_connector.py), [`test_camp2p_token_ids.py`](../../../tests/unit/connectors/test_camp2p_token_ids.py), [`test_external_routing.py`](../../../tests/unit/connectors/test_external_routing.py), [DeepSeek-V2-Lite E2E](../../../tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py) | | Ascend CAM async | [`npu/async_cam.py`](../../../afd_plugin/connectors/npu/async_cam.py), [`Push/Pull sessions`](../../../afd_plugin/connectors/npu/async_cam_runtime/async_cam_session.py) | [Connector unit tests](../../../tests/unit/connectors/test_async_cam_connector.py), [CAM host contracts](../../../tests/unit/connectors/npu/test_async_cam.py), [fused routing device test](../../../tests/npu/test_async_cam_connector.py), [`test_async_cam_npu.py`](../../../tests/e2e/models/deepseek_v4_flash/test_async_cam_npu.py) | | Process-group construction | [`afd_process_group.py`](../../../afd_plugin/distributed/afd_process_group.py) | Connector initialization tests plus platform E2E paths | @@ -102,7 +102,7 @@ Unknown fields fail in the selected connector parser. | --- | --- | | `P2pNcclAFDConnector` | None; the mapping must be empty. | | `DeepEPM2NAFDConnector` | `payload_format`, `expert_backend`, `capacity_tokens_per_rank`, `num_lanes`, `bucket_granularity`, `cpu_sync_min_tokens`, `num_sms`, `num_qps`, `expert_num_sms`, `cpu_timeout_s`, `gpu_timeout_s`, and a nested `network` mapping (`gin`, `hybrid`, `rdma_traffic_class`). All are optional; see the [DeepEP M2N guide](../../gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md#connector_extra_config) for defaults. | -| `CAMP2pAFDConnector` | `core_num`, optional `attn_core_num` / `ffn_core_num`, `compute_gate_on_attention`, `quant_mode`, and optional positive `hccl_buffer_size`. Core counts must be positive; the current runtime rejects gate-on-Attention and any nonzero quantization mode. | +| `CAMP2pAFDConnector` | `core_num`, optional `attn_core_num` / `ffn_core_num`, `compute_gate_on_attention`, `quant_mode`, and optional positive `hccl_buffer_size`. Core counts must be positive. Gate placement follows the common `afd.compute_gate_on_attention` field; the connector-local copy is kept for compatibility and does not select the mode. `true` requires DeepSeek-V4 at TP=1 without DBO or ubatching. Any nonzero quantization mode is rejected today. | | `CAMAsyncAFDConnector` | `dynamicQuant`, `ffn_mega_moe`, `attn_ranks_per_dp`, and optional positive `hccl_buffer_size`. Attention microbatching uses native DBO configuration. | The common `afd_process_group_timeout_s` defaults to 120 seconds and controls @@ -113,10 +113,12 @@ environment rather than querying the allocated domain. The common `compute_gate_on_attention` field remains on `AFDConfig` and is the model-routing selector. CUDA P2P supports both values; DeepEP M2N requires -`true`. CAMP2P also parses a -connector-local field with that name for its operator contract, but the current -synchronous NPU runtime requires both common and connector-local values to be -`false`. CAM async requires the common field to be `true`. +`true`. CAMP2P also parses a connector-local field with that name, retained for +backwards compatibility with older configurations and ignored for routing: the +synchronous NPU runtime selects Attention-side routing from the common field, so +deployments leave the connector-local copy unset. The supported synchronous +shape is `true` on both roles for DeepSeek-V4 at TP=1 with no DBO or ubatching. +CAM async requires the common field to be `true`. ## Current connector modes @@ -207,7 +209,7 @@ The current plugin-owned objects are: | `AFDTransferMetadata` | Layer, stage, non-negative split lengths, and total-token validation; backend state is paired separately in `AFDTransferContext`. A split length may be `0`: with an expert-routed transport (DeepEP M2N) an FFN rank can receive no rows for a layer while still taking part in the collective. The list itself must not be empty. | **Draft** under [#88](https://github.com/JiusiServe/afd-plugin/issues/88). | | `AFDTransferContext` | Pairs common `metadata` with the backend `states` retained from receive through the matching send. | **Draft** under [#88](https://github.com/JiusiServe/afd-plugin/issues/88) and [#105](https://github.com/JiusiServe/afd-plugin/issues/105). | | `AFDTransferState` | Empty base for backend-specific state such as CAMP2P handles or async CAM state. | **Draft** under [#88](https://github.com/JiusiServe/afd-plugin/issues/88). | -| `AFDA2FTransferPayload` | Attention-to-FFN hidden states, common metadata, and optional router logits or token-aligned input IDs. | **Draft** and scheduled for state splitting in [#105](https://github.com/JiusiServe/afd-plugin/issues/105). | +| `AFDA2FTransferPayload` | Attention-to-FFN hidden states, common metadata, and optional router logits, token-aligned input IDs, or the routed top-k (`topk_ids` with `topk_weights`). | **Draft** and scheduled for state splitting in [#105](https://github.com/JiusiServe/afd-plugin/issues/105). | | `AFDF2ATransferPayload` | Structured routed/shared FFN outputs where the backend needs both. | **Draft** with the transfer-state work. | | `LocalExpertWeights` | One FFN rank's experts in vLLM `FusedMoE` layout (`w13` gate-first, `w2`), BF16 or block-128 FP8 with both block scales. | **Draft**; used only by routed-experts connectors. | | `FFNLocalExperts` | One MoE layer's global expert ids and their `LocalExpertWeights`, as vLLM loaded them. | **Draft**; used only by routed-experts connectors. | @@ -233,7 +235,12 @@ state. P2P can send optional router logits and one-dimensional, token-aligned the A2E operator's ids channel, which the receiving FFN rank selects with `recv_input_ids`; both roles derive that mode from the model's `afd_requires_input_ids` declaration, because the operator only writes the ids -slot in the mode the sender chose. FFN requests only the fields its +slot in the mode the sender chose. The same slot carries the one payload the +layer speaks: either token-aligned input IDs or the routed top-k, never both, and +the connector rejects a request for both. External routing keeps the expert IDs in +that ids slot and puts the gate weights in the paired scales slot; the receiving +rank selects it with `recv_routing`, derived from the model's +`afd_requires_attention_routing` declaration. FFN requests only the fields its model declares, concatenates them with the same per-peer token order as hidden states, and returns them in `AFDA2FTransferPayload`. A CUDA P2P FFN rank posts the receives for all of its Attention peers, each peer's tensors in wire order, diff --git a/docs/design/module/model_integration.md b/docs/design/module/model_integration.md index 95893cdc2..236f49611 100644 --- a/docs/design/module/model_integration.md +++ b/docs/design/module/model_integration.md @@ -68,6 +68,7 @@ make a backend-specific worker class the shared model API. | Forward-context adapter | [`forward_context.py`](../../../afd_plugin/model_executor/models/forward_context.py) | [`test_forward_context.py`](../../../tests/unit/model_executor/models/test_forward_context.py) | | NPU Async CAM native DBO handoff | [`npu/async_cam_dbo.py`](../../../afd_plugin/model_executor/models/npu/async_cam_dbo.py), [`npu/ubatch_runner.py`](../../../afd_plugin/v1/worker/npu/ubatch_runner.py) | [`test_async_cam_dbo.py`](../../../tests/unit/model_executor/test_async_cam_dbo.py), [`test_npu_model_runner_v2.py`](../../../tests/unit/v1/worker/test_npu_model_runner_v2.py) | | Ascend Attention-side gate | [`npu/deepseek_v2_attention_gate.py`](../../../afd_plugin/model_executor/models/npu/deepseek_v2_attention_gate.py) | Attention-gate unit cases in [`test_forward_context.py`](../../../tests/unit/model_executor/models/test_forward_context.py) | +| Ascend external-routing MoE execution | [`npu/deepseek_v4_external_routing_moe.py`](../../../afd_plugin/model_executor/models/npu/deepseek_v4_external_routing_moe.py) | [`test_deepseek_v4_external_routing_moe_cpu.py`](../../../tests/unit/model_executor/models/test_deepseek_v4_external_routing_moe_cpu.py) | | Ascend CAM dispatch layout | [`npu/async_cam_layout.py`](../../../afd_plugin/model_executor/models/npu/async_cam_layout.py) | [`test_async_cam_layout.py`](../../../tests/unit/model_executor/test_async_cam_layout.py) | | DeepSeek Ascend stage metadata ownership | [`npu/deepseek_attention_metadata.py`](../../../afd_plugin/model_executor/models/npu/deepseek_attention_metadata.py) | [`test_deepseek_attention_metadata.py`](../../../tests/unit/model_executor/test_deepseek_attention_metadata.py) | | Ascend CAM orchestration | [`npu/deepseek_v4.py`](../../../afd_plugin/model_executor/models/npu/deepseek_v4.py) | [`test_async_cam_npu.py`](../../../tests/e2e/models/deepseek_v4_flash/test_async_cam_npu.py) | @@ -157,10 +158,14 @@ CUDA configurations and validation limits. [`npu/deepseek_v4.py`](../../../afd_plugin/model_executor/models/npu/deepseek_v4.py) wraps the native Ascend model independently of the CUDA adapter. Attention -owns attention, mHC, normalization and finalization. Synchronous CAMP2P sends -normalized hidden states and Hash-layer token IDs to FFN, which owns the -native gate and MoE. Async CAM keeps the Hash/top-k router and shared experts -on Attention and sends only routed-expert work; `dynamicQuant=1` is required. +owns attention, mHC, normalization and finalization. Synchronous CAMP2P has two +shapes, selected by the common `compute_gate_on_attention` field: with `false` +the FFN role owns the native gate and MoE, and with `true` (DeepSeek-V4, TP=1, no +DBO or ubatching) the router and shared experts run on Attention and FFN executes +only the transported routed experts. The connector-local flag of the same name is +kept for compatibility and does not select the mode. Async CAM keeps the +Hash/top-k router and shared experts on Attention and sends only routed-expert +work; `dynamicQuant=1` is required. The DSpark adapter follows the same boundary. Native checkpoint names `mtp.0..2` and KV names are retained; its CAM layer IDs start at the target's diff --git a/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md b/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md index bc11c3064..5fc067e5e 100644 --- a/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md +++ b/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md @@ -133,9 +133,9 @@ Pass AFD configuration through vLLM's `--additional-config` option under the ``` Use the [common configuration reference](../configuration.md) for defaults -and aliases. CAMP2P requires `async_dp=false` and -`compute_gate_on_attention=false`. For multi-machine placement, use the FFN -machine's reachable IP as `host`. +and aliases. CAMP2P requires `async_dp=false`; `compute_gate_on_attention` may be `true` on +both roles for DeepSeek-V4 at TP=1 without DBO or ubatching. For multi-machine placement, use +the FFN machine's reachable IP as `host`. ### CAMP2P `connector_extra_config` @@ -144,7 +144,7 @@ machine's reachable IP as `host`. | `core_num` | `int` | `8` | Positive default AIV core count for both roles. | | `attn_core_num` / `ffn_core_num` | `int` | unset | Positive role-specific override for `core_num`. | | `quant_mode` | `int` | `0` | CAM quantization mode. The current runtime supports only `0`. | -| `compute_gate_on_attention` | `bool` | `false` | Connector-local operator option; must remain `false`, as must the common AFD field. | +| `compute_gate_on_attention` | `bool` | `false` | Kept for backwards compatibility with older configurations and ignored for routing placement: transport derives the gate from the payload it actually carries. Omit it when deploying and set the common `afd.compute_gate_on_attention` field instead. | | `hccl_buffer_size` | `int` | unset | Positive CAMP2P HCCL buffer size in MB. The override applies to every connector-owned `afd*` communication domain and the FFN-side `afd_moe` domain; the Gloo control group is unaffected. When unset, HCCL uses `HCCL_BUFFSIZE`, then its built-in default. Use the same value on all members of each HCCL domain. | The per-domain setting avoids increasing buffers for unrelated TP, DP, or EP @@ -234,6 +234,80 @@ vllm serve /path/to/model \ }' ``` +## Single-node `4A4F` synchronous gate-on-Attention example (DeepSeek-V4, TP=1) + +This is the supported Attention-side gate deployment: synchronous CAMP2P, DeepSeek-V4, TP=1, +BF16 transport (`quant_mode: 0`), no DBO and no ubatching, four Attention ranks on devices `0-3` +and four FFN ranks on devices `4-7`. + +Set `afd.compute_gate_on_attention` to `true` on **both** roles and leave the connector-local +`compute_gate_on_attention` key out of `connector_extra_config`; that key is kept for backwards +compatibility with older configurations and does not control routing. + +The FFN service uses devices `4,5,6,7` and starts first: + +```bash +ASCEND_RT_VISIBLE_DEVICES=4,5,6,7 VLLM_PLUGINS=ascend,afd VLLM_USE_V2_MODEL_RUNNER=1 \ +vllm serve /path/to/DeepSeek-V4 \ + --host 127.0.0.1 \ + --port 18010 \ + --data-parallel-size 4 \ + --tensor-parallel-size 1 \ + --enable-expert-parallel \ + --enforce-eager \ + --max-model-len 9216 \ + --max-num-seqs 128 \ + --max-num-batched-tokens 3072 \ + --additional-config '{ + "afd": { + "role": "ffn", + "connector": "CAMP2pAFDConnector", + "host": "127.0.0.1", + "port": 6239, + "num_attention_ranks": 4, + "num_ffn_ranks": 4, + "compute_gate_on_attention": true, + "connector_extra_config": { + "quant_mode": 0 + } + } + }' +``` + +The Attention service uses devices `0,1,2,3`: + +```bash +ASCEND_RT_VISIBLE_DEVICES=0,1,2,3 VLLM_PLUGINS=ascend,afd VLLM_USE_V2_MODEL_RUNNER=1 \ +vllm serve /path/to/DeepSeek-V4 \ + --host 127.0.0.1 \ + --port 18000 \ + --data-parallel-size 4 \ + --tensor-parallel-size 1 \ + --enable-expert-parallel \ + --enforce-eager \ + --max-model-len 9216 \ + --max-num-seqs 128 \ + --max-num-batched-tokens 3072 \ + --additional-config '{ + "afd": { + "role": "attention", + "connector": "CAMP2pAFDConnector", + "host": "127.0.0.1", + "port": 6239, + "num_attention_ranks": 4, + "num_ffn_ranks": 4, + "compute_gate_on_attention": true, + "connector_extra_config": { + "quant_mode": 0 + } + } + }' +``` + +Drop `--enforce-eager` and add `--compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64, 128]}'` to run the same pair with ACL graphs. +The configuration is rejected when `compute_gate_on_attention=true` is combined with DBO, +ubatching, TP > 1, a connector other than `CAMP2pAFDConnector`, or a target other than DeepSeek-V4. + For model-specific launch and validation cases, see the [E2E guide](../../tests/e2e/README.md). diff --git a/docs/support.md b/docs/support.md index 175e99e7d..d45f44791 100644 --- a/docs/support.md +++ b/docs/support.md @@ -27,7 +27,7 @@ qualify every model/topology combination on the current target. | --- | --- | --- | --- | --- | | CUDA V2 + `P2pNcclAFDConnector` | Eager or `FULL_DECODE_ONLY` native V2 CUDA Graph | Exactly two native microbatches; Attention DP1 or DP > 1 | Model-specific gate placement; PP/CP, elastic EP, EPLB, SP MoE, and compile SP rejected; role ranks equal DP x TP | Source-specific GPU validation is linked below; coverage is limited to the recorded model/topology combinations. DeepSeek-V2-Lite has known DBO shutdown failures. Qwen3.5/3.6 GDN DBO is unsupported. | | CUDA V2 + `DeepEPM2NAFDConnector` | Eager or `FULL_DECODE_ONLY` native V2 CUDA Graph; SM90 or newer | Exactly two native microbatches, enabled on both roles; Attention DP1 or DP > 1, TP1 or TP2 | Qwen3 MoE (all layers MoE, no shared expert) or DeepSeek-V2; `compute_gate_on_attention=true`; FFN TP1 with EP over all FFN ranks, linear placement, `allgather_reducescatter` all2all and the Triton MoE weight layout; BF16 or block-128 FP8 payload; EPLB, PP/CP and SP MoE rejected | Historical vLLM 0.30 Qwen3-30B-A3B 2A2F GSM8K E2E for eager, graph, graph + DBO, and Attention TP2 eager/graph on H100 and H200, and for eager, graph and graph + DBO on GB200; two-node 2A2F eager, graph and graph + DBO with Attention and FFN on separate nodes. DeepSeek-V2-Lite: [E2E definitions](../tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py). [GPU connector device tests](../tests/gpu/README.md) | -| Ascend V2 + `CAMP2pAFDConnector` | Eager, `FULL`, or `FULL_DECODE_ONLY` native V2 ACL Graph | Eager or graph native DBO with two ubatches and Attention DP > 1; plain MLA graph DBO requires `FULL_DECODE_ONLY` | `compute_gate_on_attention=false`; PP/CP, elastic EP and EPLB rejected; model-specific SP policy; role ranks equal DP x TP | Historical 0.30 V2-Lite 2A2F FULL 300-question comparison and 2A1F functional checks; current 0.32 Lite evidence is in the [validation record](https://github.com/vllm-project/afd-plugin/pull/507) | +| Ascend V2 + `CAMP2pAFDConnector` | Eager, `FULL`, or `FULL_DECODE_ONLY` native V2 ACL Graph | Eager or graph native DBO with two ubatches and Attention DP > 1; plain MLA graph DBO requires `FULL_DECODE_ONLY` | `compute_gate_on_attention` may be `false`, or `true` for DeepSeek-V4 at TP=1 without DBO/ubatching; PP/CP, elastic EP and EPLB rejected; model-specific SP policy; role ranks equal DP x TP | Historical 0.30 V2-Lite 2A2F FULL 300-question comparison and 2A1F functional checks; current 0.32 Lite evidence is in the [validation record](https://github.com/vllm-project/afd-plugin/pull/507) | | Ascend V2 + `CAMAsyncAFDConnector` | Eager Attention; layered FFN eager or FULL | Two native DBO microbatches on Attention, including DP1 and with FFN FULL; FFN stays outside the native DBO schedule | Experimental DeepSeek-V4 W4A8/W8A8 on A3/910C; `async_dp=true`, `compute_gate_on_attention=true`, and `dynamicQuant=1`; PP/CP unsupported; plugin-owned async CAM ops required | Historical DSV4 layered comparisons and limitations are in the [validation record](https://github.com/vllm-project/afd-plugin/pull/507); they predate removal of the legacy path | The 300-question Ascend evidence in this table is historical. The 0.32 diff --git a/tests/unit/compat/npu/test_sync_gate_feature_validation_cpu.py b/tests/unit/compat/npu/test_sync_gate_feature_validation_cpu.py new file mode 100644 index 000000000..aec7e0ecd --- /dev/null +++ b/tests/unit/compat/npu/test_sync_gate_feature_validation_cpu.py @@ -0,0 +1,159 @@ +# mypy: disable-error-code="attr-defined, name-defined" +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""CPU coverage for the sync gate-on-Attention feature contract. + +The public validator is the seam every AFD NPU run passes before execution, so it is +driven directly rather than through its private helpers. Only the two upstream modules +the validator reaches for -- the connector factory and the connector's extra-info type +-- are stand-ins; the validator itself is the real one. + +Keeping the assertions on the public entry matters: it fails both when the helper +starts rejecting supported shapes and when the entry stops calling the helper. +""" + +from __future__ import annotations + +import sys +import types +from types import SimpleNamespace + +import pytest + +from afd_plugin.compat.npu import feature_validation +from afd_plugin.config import CAMP2P_CONNECTOR + +_DSV4 = "deepseek_v4" + + +class _FakeExtraInfo: + """Stands in for the connector extra info the factory would build.""" + + def __init__(self) -> None: + self.validated = 0 + + def validate_supported(self) -> None: + self.validated += 1 + + +@pytest.fixture +def extra_info(monkeypatch): + info = _FakeExtraInfo() + factory = types.ModuleType("afd_plugin.connectors.factory") + factory.AFDConnectorFactory = SimpleNamespace( + parse_connector_extra_info=lambda connector, config: info, + ) + camp2p = types.ModuleType("afd_plugin.connectors.npu.camp2p") + camp2p.CAMP2PExtraInfo = _FakeExtraInfo + monkeypatch.setitem(sys.modules, "afd_plugin.connectors.factory", factory) + monkeypatch.setitem(sys.modules, "afd_plugin.connectors.npu.camp2p", camp2p) + return info + + +def _afd_config(*, role: str = "attention"): + return SimpleNamespace( + role=role, + connector=CAMP2P_CONNECTOR, + compute_gate_on_attention=True, + ) + + +def _vllm_config(*, tp: int = 1, dbo: bool = False, ubatching: bool = False): + hf_config = SimpleNamespace(model_type=_DSV4) + return SimpleNamespace( + model_config=SimpleNamespace( + hf_config=hf_config, + hf_text_config=SimpleNamespace(model_type=_DSV4), + ), + parallel_config=SimpleNamespace( + enable_dbo=dbo, + use_ubatching=ubatching, + num_ubatches=1, + tensor_parallel_size=tp, + ), + compilation_config=SimpleNamespace(cudagraph_mode=None), + speculative_config=None, + ) + + +@pytest.mark.parametrize("role", ["attention", "ffn"]) +def test_the_supported_shape_is_accepted_for_both_roles(extra_info, role): + """Gate-on-Attention at TP=1 on the sync connector must pass validation.""" + + feature_validation.fail_if_unsupported_npu_afd_features( + _vllm_config(), + afd_config=_afd_config(role=role), + ) + + assert extra_info.validated == 1 + + +@pytest.mark.parametrize( + ("kwargs", "match"), + [ + ({"dbo": True}, "DBO or ubatching"), + ({"ubatching": True}, "DBO or ubatching"), + ({"tp": 2}, "tensor-parallel size 1"), + ], +) +@pytest.mark.parametrize("role", ["attention", "ffn"]) +def test_unsupported_shapes_are_rejected(extra_info, kwargs, match, role): + with pytest.raises(RuntimeError, match=match): + feature_validation.fail_if_unsupported_npu_afd_features( + _vllm_config(**kwargs), + afd_config=_afd_config(role=role), + ) + + +def test_gate_on_ffn_keeps_its_previous_behaviour(extra_info): + """Turning the Attention-side gate off must not pick up the new restrictions.""" + + afd_config = SimpleNamespace( + role="ffn", + connector=CAMP2P_CONNECTOR, + compute_gate_on_attention=False, + ) + + feature_validation.fail_if_unsupported_npu_afd_features( + _vllm_config(tp=2), + afd_config=afd_config, + ) + + assert extra_info.validated == 1 + + +def _real_extra_info(): + """The connector's own type; importing it needs vLLM. + + ``camp2p`` pulls ``vllm.forward_context`` at module scope, so the real + class is only reachable where vLLM is installed. The validator cases + above use a stand-in and run everywhere; these two cover the class + itself and skip on a bare CPU host. + """ + + pytest.importorskip("torch") + pytest.importorskip("vllm") + from afd_plugin.connectors.npu.camp2p import CAMP2PExtraInfo + + return CAMP2PExtraInfo + + +def test_a_connector_level_gate_flag_does_not_block(): + """The switch is read from ``afd``; the connector-local copy has no say. + + Regression test for a rejection that used to live in ``validate_supported``: a + supported + shape was refused at startup whenever the flag also appeared under + ``connector_extra_config``. + """ + + _real_extra_info().from_mapping( + {"compute_gate_on_attention": True} + ).validate_supported() + + +def test_quant_mode_is_still_rejected(): + """The transport mode really is unsupported, so that check has to stay.""" + + with pytest.raises(RuntimeError, match="only quant_mode=0"): + _real_extra_info().from_mapping({"quant_mode": 4}).validate_supported() diff --git a/tests/unit/connectors/test_camp2p_connector.py b/tests/unit/connectors/test_camp2p_connector.py index 9cf18426e..d6e0e0809 100644 --- a/tests/unit/connectors/test_camp2p_connector.py +++ b/tests/unit/connectors/test_camp2p_connector.py @@ -23,13 +23,13 @@ AFDTransferState, ) from afd_plugin.connectors.metadata import AFDControlPayload, AFDDPMetadata -from afd_plugin.connectors.npu import camp2p as camp2p_module from afd_plugin.connectors.npu.camp2p import ( CAMP2pAFDConnector, CAMP2PExtraInfo, CAMP2PTransferState, build_camp2p_topology, ) +from afd_plugin.connectors.npu.camp2p import connector as camp2p_module from tests.unit.fakes.dp_metadata import DPMetadata as _FakeDPMetadata diff --git a/tests/unit/connectors/test_camp2p_token_ids.py b/tests/unit/connectors/test_camp2p_token_ids.py index adab709ce..7a563c23e 100644 --- a/tests/unit/connectors/test_camp2p_token_ids.py +++ b/tests/unit/connectors/test_camp2p_token_ids.py @@ -110,12 +110,12 @@ def make(name: str, **attributes: object) -> None: AFDTransferContext, AFDTransferMetadata, ) - from afd_plugin.connectors.npu import camp2p as camp2p_module from afd_plugin.connectors.npu.camp2p import ( CAMP2pAFDConnector, prepare_token_id_transfer, received_token_ids, ) + from afd_plugin.connectors.npu.camp2p import connector as camp2p_module class _CpuTorch: @@ -326,3 +326,188 @@ def fake_a2e(*args: Any) -> tuple[Any, ...]: ] assert calls[1][-1] == 0 assert without_ids.input_ids is None + + +def _routing(connector, rows, *, columns=None): + """Routing sized the way the operator is configured.""" + + topk = connector.num_experts_per_tok if columns is None else columns + ids = torch.arange(rows * topk, dtype=torch.int64).reshape(rows, topk) + weights = torch.full((rows, topk), 0.25, dtype=torch.bfloat16) + return ids, weights + + +def test_prepare_routing_transfer_packs_int32_ids_and_float32_weights(): + """The ids channel takes int32 ids and float32 weights, not the gate dtypes.""" + + ids = torch.tensor([[0, 1], [2, 3]], dtype=torch.int64) + weights = torch.full((2, 2), 0.25, dtype=torch.bfloat16) + + wire_ids, wire_weights = camp2p_module.prepare_routing_transfer( + ids, + weights, + expected_tokens=2, + top_k=2, + ) + + assert wire_ids.dtype == torch.int32 + assert wire_ids.tolist() == [[0, 1], [2, 3]] + assert wire_weights.dtype == torch.float32 + assert wire_weights.tolist() == [[0.25, 0.25], [0.25, 0.25]] + + +def test_prepare_routing_transfer_rejects_expert_count_mismatch(): + """The wire declares ``k``, so a narrower routing tensor must not be packed.""" + + ids = torch.zeros(3, 1, dtype=torch.int32) + weights = torch.zeros(3, 1, dtype=torch.float32) + + with pytest.raises(ValueError, match="k=2"): + camp2p_module.prepare_routing_transfer( + ids, + weights, + expected_tokens=3, + top_k=2, + ) + + +def test_prepare_routing_transfer_rejects_token_count_mismatch(): + ids = torch.zeros(2, 2, dtype=torch.int32) + weights = torch.zeros(2, 2, dtype=torch.float32) + + with pytest.raises(ValueError, match="declares 3"): + camp2p_module.prepare_routing_transfer( + ids, + weights, + expected_tokens=3, + top_k=2, + ) + + +def test_received_routing_trims_operator_padded_capacity(): + """Both channel tensors are cut together, so ids and weights stay paired.""" + + ids = torch.arange(12, dtype=torch.int32).reshape(4, 3) + weights = torch.arange(12, dtype=torch.float32).reshape(4, 3) + + topk_ids, topk_weights = camp2p_module.received_routing( + ids, + weights, + expected_tokens=2, + ) + + assert topk_ids.tolist() == [[0, 1, 2], [3, 4, 5]] + assert topk_weights.tolist() == [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]] + + +def test_received_routing_rejects_alignment_shortfall(): + ids = torch.zeros(2, 3, dtype=torch.int32) + weights = torch.zeros(2, 3, dtype=torch.float32) + + with pytest.raises(ValueError, match="routing rows"): + camp2p_module.received_routing(ids, weights, expected_tokens=3) + + +def test_send_attn_output_puts_routing_on_the_operator_ids_channel(monkeypatch): + """Attention-side routing must select the operator's ids mode and fill it.""" + + calls: list[tuple[Any, ...]] = [] + monkeypatch.setattr( + torch.ops.vllm, + "afd_camp2p_send_attn_output", + lambda *args: calls.append(args), + raising=False, + ) + monkeypatch.setattr(camp2p_module, "get_forward_context", SimpleNamespace) + connector = _connector(role="attention", rank=0) + hidden_states = torch.zeros(3, connector.hidden_size) + context = AFDTransferContext( + metadata=AFDTransferMetadata.create_attention_metadata( + layer_idx=0, + stage_idx=0, + seq_len=3, + ), + ) + ids, weights = _routing(connector, 3) + + connector.send_attn_output(hidden_states, context, routing=(ids, weights)) + + sent = calls[0] + # Trailing operator arguments: aiv_num, compute_gate, ids, scales. + assert sent[-3] == 1 + assert sent[-2].dtype == torch.int32 + assert sent[-2].tolist() == ids.tolist() + assert sent[-1].dtype == torch.float32 + assert sent[-1].tolist() == weights.to(torch.float32).tolist() + + +def test_send_attn_output_rejects_ids_and_routing_together(monkeypatch): + monkeypatch.setattr( + torch.ops.vllm, + "afd_camp2p_send_attn_output", + lambda *args: None, + raising=False, + ) + monkeypatch.setattr(camp2p_module, "get_forward_context", SimpleNamespace) + connector = _connector(role="attention", rank=0) + hidden_states = torch.zeros(3, connector.hidden_size) + context = AFDTransferContext( + metadata=AFDTransferMetadata.create_attention_metadata( + layer_idx=0, + stage_idx=0, + seq_len=3, + ), + ) + ids, weights = _routing(connector, 3) + + with pytest.raises(ValueError, match="both token ids"): + connector.send_attn_output( + hidden_states, + context, + input_ids=torch.tensor([7, 11, 13], dtype=torch.int64), + routing=(ids, weights), + ) + + +def test_recv_attn_output_reads_the_routing_slot(monkeypatch): + """The routing mode exposes the ids slot as paired top-k ids and weights.""" + + calls: list[tuple[Any, ...]] = [] + + def fake_a2e(*args: Any) -> tuple[Any, ...]: + calls.append(args) + tokens, topk = int(args[3]), int(args[5]) + ids = torch.arange(tokens * topk, dtype=torch.int32).reshape(tokens, topk) + weights = torch.full((tokens, topk), 0.5, dtype=torch.float32) + return ("hidden", ids, weights, "atten-batch", "active-mask") + + monkeypatch.setattr(torch.ops.afd_ascend, "a2e", fake_a2e, raising=False) + monkeypatch.setattr(camp2p_module, "torch", _CpuTorch()) + connector = _connector(role="ffn", rank=1) + connector.dp_metadata_list = {0: _FakeDPMetadata([2, 3, 5, 7])} + + payload = connector.recv_attn_output( + ubatch_idx=0, + layer_idx=0, + recv_routing=True, + ) + + # 10 tokens reach this FFN rank, in the same column order the gate produced. + assert calls[0][-1] == 1 + assert payload.input_ids is None + assert payload.topk_ids.shape == (10, connector.num_experts_per_tok) + assert payload.topk_ids[0].tolist() == list(range(connector.num_experts_per_tok)) + assert payload.topk_weights.shape == (10, connector.num_experts_per_tok) + assert payload.topk_weights[0].tolist() == [0.5] * connector.num_experts_per_tok + + +def test_recv_attn_output_rejects_two_wire_slot_modes(monkeypatch): + connector = _connector(role="ffn", rank=1) + + with pytest.raises(ValueError, match="exactly one"): + connector.recv_attn_output( + ubatch_idx=0, + layer_idx=0, + recv_input_ids=True, + recv_routing=True, + ) diff --git a/tests/unit/connectors/test_external_routing.py b/tests/unit/connectors/test_external_routing.py new file mode 100644 index 000000000..18dcfa66c --- /dev/null +++ b/tests/unit/connectors/test_external_routing.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""CPU tests for the Attention-side routing payload. + +The payload is pure tensor logic, so it is testable without an Ascend device. The +package resolves its connector lazily, so importing this module does not pull vLLM in. + +Rows carry rank- and position-specific values instead of a constant fill: a gather +or split that drops, reorders or mismatches a row has to change the observed +contents, not just the shape. +""" + +from __future__ import annotations + +import pytest + +torch = pytest.importorskip("torch") + +_TOP_K = 2 + + +@pytest.fixture(scope="module") +def routing_module(): + from afd_plugin.connectors.npu.camp2p import external_routing + + return external_routing + + +@pytest.fixture(scope="module") +def inputs_type(routing_module): + return routing_module.AFDExternalRoutingInputs + + +def _inputs(inputs_type, rows: int, rank: int = 0): + """Rows encode rank and position, so an ordering bug changes the values.""" + + base = rank * 100 + values = torch.arange(base, base + rows * _TOP_K) + return inputs_type( + topk_ids=values.to(torch.int32).reshape(rows, _TOP_K), + topk_weights=(values.to(torch.float32) / 10).reshape(rows, _TOP_K), + ) + + +_RANK_STRIDE = 100 + + +def _two_rank_gather(tensor): + """Stand-in collective whose ranks carry *different* rows. + + A real DP group gathers each rank's own tokens, so rank 1's shard is offset from + rank 0's. Repeating one shard would hide an implementation that reorders or drops + a rank. + """ + + return torch.cat([tensor, tensor + _RANK_STRIDE], dim=0) + + +def test_padding_appends_inert_rows_and_keeps_valid_ones(inputs_type): + inputs = _inputs(inputs_type, 2) + + padded = inputs.padded(2) + + assert padded.topk_ids[:2].tolist() == inputs.topk_ids.tolist() + assert padded.topk_weights[:2].tolist() == inputs.topk_weights.tolist() + assert padded.topk_ids[2:].tolist() == [[0, 0], [0, 0]] + assert padded.topk_weights[2:].tolist() == [[0.0, 0.0], [0.0, 0.0]] + assert padded.token_count == 4 + + +def test_gathering_keeps_rank_order_and_contents(inputs_type): + """Rank 0's rows come first, then rank 1's, with their own values intact.""" + + inputs = _inputs(inputs_type, 2, rank=0) + + gathered = inputs.gathered(_two_rank_gather) + + assert gathered.topk_ids.tolist() == ( + inputs.topk_ids.tolist() + (inputs.topk_ids + _RANK_STRIDE).tolist() + ) + assert gathered.topk_weights.tolist() == ( + inputs.topk_weights.tolist() + (inputs.topk_weights + _RANK_STRIDE).tolist() + ) + + +def test_split_rows_takes_one_rank_slice(inputs_type): + inputs = _inputs(inputs_type, 4) + + taken = inputs.split_rows(lambda tensor: torch.tensor_split(tensor, 2, dim=0)[1]) + + assert taken.topk_ids.tolist() == inputs.topk_ids.tolist()[2:] + assert taken.topk_weights.tolist() == inputs.topk_weights.tolist()[2:] + + +def test_non_matrix_routing_is_rejected(inputs_type): + with pytest.raises(ValueError, match="2-D"): + inputs_type( + topk_ids=torch.zeros(3, dtype=torch.int32), + topk_weights=torch.zeros(3, dtype=torch.float32), + ) + + +def test_mismatched_topk_shapes_are_rejected(inputs_type): + with pytest.raises(ValueError, match="same shape"): + inputs_type( + topk_ids=torch.zeros(2, 2, dtype=torch.int32), + topk_weights=torch.zeros(2, 3, dtype=torch.float32), + ) + + +@pytest.mark.parametrize("dtype", [torch.int32, torch.int64]) +def test_expert_indices_accept_both_index_dtypes(inputs_type, dtype): + """Both index widths are in the operator contract; neither may be rejected.""" + + inputs = inputs_type( + topk_ids=torch.zeros(2, 2, dtype=dtype), + topk_weights=torch.zeros(2, 2, dtype=torch.float32), + ) + + assert inputs.topk_ids.dtype == dtype + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.bool]) +def test_expert_indices_reject_non_index_dtypes(inputs_type, dtype): + with pytest.raises(ValueError, match="expert indices"): + inputs_type( + topk_ids=torch.zeros(2, 2, dtype=dtype), + topk_weights=torch.zeros(2, 2, dtype=torch.float32), + ) + + +def test_non_float_weights_are_rejected(inputs_type): + with pytest.raises(ValueError, match="gate weights"): + inputs_type( + topk_ids=torch.zeros(2, 2, dtype=torch.int32), + topk_weights=torch.zeros(2, 2, dtype=torch.int32), + ) diff --git a/tests/unit/model_executor/models/test_deepseek_v4_external_routing_moe_cpu.py b/tests/unit/model_executor/models/test_deepseek_v4_external_routing_moe_cpu.py new file mode 100644 index 000000000..ee16886cd --- /dev/null +++ b/tests/unit/model_executor/models/test_deepseek_v4_external_routing_moe_cpu.py @@ -0,0 +1,431 @@ +# mypy: disable-error-code="attr-defined, name-defined" +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""CPU tests for the AFD-owned external-routing MoE component. + +The component imports vLLM-Ascend types at module scope, so the tests +install stand-ins for those modules, load the real component from its file, and +drive it with fakes for the comm instance and the routed experts. + +What this covers: the routing reaches ``apply`` unchanged, only the ALLGATHER path +reduces during finalize, an unsupported comm method is rejected, comm instances are +reused across calls, and a failure in any stage leaves no per-call state behind. + +The real operators and graph execution stay with the green-zone suite. +""" + +from __future__ import annotations + +import importlib.util +import sys +import types +from dataclasses import dataclass +from pathlib import Path + +import pytest + +torch = pytest.importorskip("torch") + +_REPO_ROOT = Path(__file__).resolve().parents[4] +_MODULE_PATH = ( + _REPO_ROOT + / "afd_plugin" + / "model_executor" + / "models" + / "npu" + / "deepseek_v4_external_routing_moe.py" +) +_ROUTING_PATH = ( + _REPO_ROOT / "afd_plugin" / "connectors" / "npu" / "camp2p" / "external_routing.py" +) + + +class _MoECommType: + ALLGATHER = "ALLGATHER" + MC2 = "MC2" + ALLTOALL = "ALLTOALL" + FUSED_MC2 = "FUSED_MC2" + + +@dataclass(frozen=True) +class _PrepareOutput: + hidden_states: torch.Tensor + router_logits: torch.Tensor | None + mc2_mask: torch.Tensor | None + padded_hidden_states_shape: torch.Size | None + pertoken_scale: torch.Tensor | None = None + + +class _PrepareFinalizeBase: + def __init__(self, moe_config): + self.moe_config = moe_config + self.tp_rank = 0 + self.tp_size = 1 + + def prepare_external(self, hidden_states, routing, **kwargs): + return ( + _PrepareOutput( + hidden_states=hidden_states, + router_logits=None, + mc2_mask=torch.ones(8, dtype=torch.bool), + padded_hidden_states_shape=None, + pertoken_scale=None, + ), + routing, + ) + + +class _CommBase: + def __init__(self, moe_config): + self.moe_config = moe_config + self.prepare_finalize = self._get_prepare_finalize() + self.finalized: list[dict] = [] + + def _get_prepare_finalize(self): + return _PrepareFinalizeBase(self.moe_config) + + def finalize(self, **kwargs): + self.finalized.append(kwargs) + return kwargs["hidden_states"] + + +class _AllGatherComm(_CommBase): + pass + + +class _MC2Comm(_CommBase): + pass + + +def _moe_config(): + return types.SimpleNamespace(dp_size=1, pcp_size=1, tp_size=1, dp_group=None) + + +def _install_stub_modules(): + """Register the upstream modules the component imports from.""" + routing_spec = importlib.util.spec_from_file_location( + "afd_external_routing_under_test", _ROUTING_PATH + ) + assert routing_spec is not None and routing_spec.loader is not None + routing_module = importlib.util.module_from_spec(routing_spec) + routing_spec.loader.exec_module(routing_module) + + quant = types.ModuleType("vllm_ascend.quantization.quant_type") + quant.QuantType = type("QuantType", (), {"NONE": 0}) + + dataclass_module = types.ModuleType( + "vllm_ascend.ops.fused_moe.dataclass.prepare_finalize" + ) + dataclass_module.MoEPrepareOutput = _PrepareOutput + + prepare_module = types.ModuleType("vllm_ascend.ops.fused_moe.prepare_finalize") + prepare_module.PrepareAndFinalizeWithAllGather = _PrepareFinalizeBase + prepare_module.PrepareAndFinalizeWithMC2 = _PrepareFinalizeBase + + comm_module = types.ModuleType("vllm_ascend.ops.fused_moe.moe_comm_method") + comm_module.AllGatherCommImpl = _AllGatherComm + comm_module.MC2CommImpl = _MC2Comm + # The native selector is keyed by MoECommType and returns None when this + # configuration was not initialised with that method. + comm_by_type = { + _MoECommType.ALLGATHER: _AllGatherComm, + _MoECommType.MC2: _MC2Comm, + } + + def get_moe_comm_method(comm_type, moe_config=None): + impl = comm_by_type.get(comm_type) + return None if impl is None else impl(moe_config) + + comm_module.get_moe_comm_method = get_moe_comm_method + + moe_utils = types.ModuleType("vllm_ascend.ops.fused_moe.moe_utils") + moe_utils._pad_tokens_with_cat = lambda tensor, length: tensor + + parallel = types.ModuleType("vllm.distributed.parallel_state") + parallel.get_pcp_group = lambda: None + + enum_module = types.ModuleType("vllm_ascend.ascend_forward_context") + enum_module.MoECommType = _MoECommType + + context = enum_module + context._EXTRA_CTX = types.SimpleNamespace( + moe_comm_method=_AllGatherComm(_moe_config()), + moe_comm_type=_MoECommType.ALLGATHER, + mc2_mask=torch.ones(8, dtype=torch.bool), + padded_num_tokens=0, + max_tokens_across_dp=0, + max_tokens_across_pcp=0, + ) + + afd_routing = types.ModuleType("afd_plugin.connectors.npu.camp2p.external_routing") + afd_routing.AFDExternalRoutingInputs = routing_module.AFDExternalRoutingInputs + + modules = { + "vllm": types.ModuleType("vllm"), + "vllm.distributed": types.ModuleType("vllm.distributed"), + "vllm.distributed.parallel_state": parallel, + "vllm_ascend": types.ModuleType("vllm_ascend"), + "vllm_ascend.ops": types.ModuleType("vllm_ascend.ops"), + "vllm_ascend.ops.fused_moe": types.ModuleType("vllm_ascend.ops.fused_moe"), + "vllm_ascend.ops.fused_moe.dataclass": types.ModuleType( + "vllm_ascend.ops.fused_moe.dataclass" + ), + "vllm_ascend.ops.fused_moe.dataclass.prepare_finalize": dataclass_module, + "vllm_ascend.ops.fused_moe.prepare_finalize": prepare_module, + "vllm_ascend.ops.fused_moe.moe_comm_method": comm_module, + "vllm_ascend.ops.fused_moe.moe_utils": moe_utils, + "vllm_ascend.quantization": types.ModuleType("vllm_ascend.quantization"), + "vllm_ascend.quantization.quant_type": quant, + "vllm_ascend.ascend_forward_context": context, + "afd_plugin": types.ModuleType("afd_plugin"), + "afd_plugin.connectors": types.ModuleType("afd_plugin.connectors"), + "afd_plugin.connectors.npu": types.ModuleType("afd_plugin.connectors.npu"), + "afd_plugin.connectors.npu.camp2p.external_routing": afd_routing, + } + for name, module in modules.items(): + sys.modules[name] = module + + spec = importlib.util.spec_from_file_location( + "afd_deepseek_v4_external_routing_moe", _MODULE_PATH + ) + assert spec is not None and spec.loader is not None + component_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(component_module) + return component_module, context, comm_module + + +@pytest.fixture(scope="module") +def loaded(): + before = dict(sys.modules) + module, context, comm_module = _install_stub_modules() + yield module, context, comm_module + sys.modules.clear() + sys.modules.update(before) + + +class _Experts(torch.nn.Module): + def __init__(self, fail_at=None): + super().__init__() + self.moe_config = types.SimpleNamespace( + is_sequence_parallel=False, dp_size=1, pcp_size=1, tp_size=1, dp_group=None + ) + self.quant_type = 0 + self.ascend_pertoken_scale = "stale" + self.ascend_mc2_mask = "stale" + self.seen = [] + self.fail_at = fail_at + self.quant_method = self + + def apply( + self, *, layer, x, topk_weights, topk_ids, shared_experts, shared_experts_input + ): + if self.fail_at == "apply": + raise RuntimeError("apply failed") + self.seen.append((topk_ids.clone(), topk_weights.clone())) + return types.SimpleNamespace(routed_out=x + 1) + + +def _routing(component_module, offset, tokens=4): + ids = torch.arange(offset, offset + tokens, dtype=torch.int32).reshape(tokens, 1) + return component_module.AFDExternalRoutingInputs( + topk_ids=ids, topk_weights=torch.ones(tokens, 1) + ) + + +def test_the_routing_reaches_apply_unchanged(loaded): + module, context, _ = loaded + experts = _Experts() + component = module.AFDExternalRoutingMoE(experts) + context._EXTRA_CTX.moe_comm_method = _AllGatherComm(_moe_config()) + + hidden = torch.zeros(4, 2) + ids = torch.arange(4, dtype=torch.int32).reshape(4, 1) + weights = torch.ones(4, 1) + component.forward(hidden, topk_ids=ids, topk_weights=weights) + + assert torch.equal(experts.seen[-1][0], ids) + assert torch.equal(experts.seen[-1][1], weights) + + +@pytest.mark.parametrize( + "comm_cls,reduces", [(_AllGatherComm, True), (_MC2Comm, False)] +) +def test_only_allgather_reduces_during_finalize(loaded, comm_cls, reduces): + module, context, _ = loaded + comm = comm_cls(_moe_config()) + context._EXTRA_CTX.moe_comm_method = comm + experts = _Experts() + component = module.AFDExternalRoutingMoE(experts) + + component.forward( + torch.zeros(4, 2), + topk_ids=torch.zeros(4, 1, dtype=torch.int32), + topk_weights=torch.ones(4, 1), + ) + afd_comm = component._comm_for(context._EXTRA_CTX.moe_comm_type, comm) + assert afd_comm.finalized[-1]["reduce_results"] is reduces + + # finalize runs after the per-token state is cleared; record what it saw + seen = {} + original_finalize = afd_comm.finalize + + def recording_finalize(**kwargs): + seen["mask"] = experts.ascend_mc2_mask + seen["scale"] = experts.ascend_pertoken_scale + return original_finalize(**kwargs) + + afd_comm.finalize = recording_finalize + component.forward( + torch.zeros(4, 2), + topk_ids=torch.zeros(4, 1, dtype=torch.int32), + topk_weights=torch.ones(4, 1), + ) + assert seen["mask"] is None, "finalize ran with per-token state still set" + assert seen["scale"] is None, "finalize ran with per-token state still set" + assert context._EXTRA_CTX.moe_comm_method is comm, "the comm was not restored" + assert experts.ascend_mc2_mask is None, "per-call state leaked" + assert experts.ascend_pertoken_scale is None, "per-call state leaked" + + +def test_comm_is_reused_across_calls(loaded): + module, context, _ = loaded + native = _AllGatherComm(_moe_config()) + context._EXTRA_CTX.moe_comm_method = native + component = module.AFDExternalRoutingMoE(_Experts()) + + before = dict(component._candidates) + for _ in range(2): + component.forward( + torch.zeros(4, 2), + topk_ids=torch.zeros(4, 1, dtype=torch.int32), + topk_weights=torch.ones(4, 1), + ) + assert component._candidates == before, "a forward built a new comm" + + +def test_an_unsupported_comm_method_is_rejected(loaded): + module, context, _ = loaded + context._EXTRA_CTX.moe_comm_method = types.SimpleNamespace(moe_config=None) + component = module.AFDExternalRoutingMoE(_Experts()) + + with pytest.raises(RuntimeError, match="ALLGATHER and MC2"): + component.forward( + torch.zeros(4, 2), + topk_ids=torch.zeros(4, 1, dtype=torch.int32), + topk_weights=torch.ones(4, 1), + ) + + +def test_a_failure_in_apply_leaves_no_per_call_state(loaded): + module, context, _ = loaded + experts = _Experts(fail_at="apply") + native = _AllGatherComm(_moe_config()) + context._EXTRA_CTX.moe_comm_method = native + component = module.AFDExternalRoutingMoE(experts) + + with pytest.raises(RuntimeError, match="apply failed"): + component.forward( + torch.zeros(4, 2), + topk_ids=torch.zeros(4, 1, dtype=torch.int32), + topk_weights=torch.ones(4, 1), + ) + assert experts.ascend_pertoken_scale is None + assert experts.ascend_mc2_mask is None + assert context._EXTRA_CTX.moe_comm_method is native, "the comm was not restored" + + +class _GatherGroup: + def __init__(self, ranks): + self.ranks = ranks + + def all_gather(self, tensor, dim=0): + return torch.cat([tensor] * self.ranks, dim=dim) + + +def test_dp_gather_and_padding_align_the_routing(loaded): + """Padding then gathering must move the routing with the activation.""" + module, context, _ = loaded + group = _GatherGroup(2) + cfg = types.SimpleNamespace( + is_sequence_parallel=False, dp_size=2, pcp_size=1, tp_size=1, dp_group=group + ) + experts = _Experts() + experts.moe_config = cfg + context._EXTRA_CTX.max_tokens_across_dp = 6 + context._EXTRA_CTX.moe_comm_method = _AllGatherComm(cfg) + + component = module.AFDExternalRoutingMoE(experts) + ids = torch.arange(4, dtype=torch.int32).reshape(4, 1) + component.forward( + torch.zeros(4, 2), + topk_ids=ids, + topk_weights=torch.ones(4, 1), + ) + # padded from 4 to 6 rows, then gathered across 2 ranks + assert experts.seen[-1][0].shape[0] == 12 + + +def test_a_failure_during_prepare_clears_state_and_restores_comm(loaded): + module, context, _ = loaded + native = _AllGatherComm(_moe_config()) + context._EXTRA_CTX.moe_comm_method = native + experts = _Experts() + component = module.AFDExternalRoutingMoE(experts) + + def exploding_prepare(*args, **kwargs): + raise RuntimeError("prepare failed") + + component._comm_for( + context._EXTRA_CTX.moe_comm_type, native + ).prepare_finalize.prepare_external = exploding_prepare + with pytest.raises(RuntimeError, match="prepare failed"): + component.forward( + torch.zeros(4, 2), + topk_ids=torch.zeros(4, 1, dtype=torch.int32), + topk_weights=torch.ones(4, 1), + ) + assert experts.ascend_pertoken_scale is None + assert experts.ascend_mc2_mask is None + assert context._EXTRA_CTX.moe_comm_method is native, "the comm was not restored" + + +def test_a_missing_mc2_candidate_still_serves_allgather(loaded): + """A configuration initialised without MC2 must not need an MC2 candidate.""" + module, context, comm_module = loaded + original = comm_module.get_moe_comm_method + comm_module.get_moe_comm_method = lambda comm_type, moe_config=None: ( + None if comm_type == _MoECommType.MC2 else original(comm_type, moe_config) + ) + try: + experts = _Experts() + native = _AllGatherComm(experts.moe_config) + context._EXTRA_CTX.moe_comm_method = native + component = module.AFDExternalRoutingMoE(experts) + assert module.AFDMC2CommImpl not in component._candidates + component.forward( + torch.zeros(4, 2), + topk_ids=torch.zeros(4, 1, dtype=torch.int32), + topk_weights=torch.ones(4, 1), + ) + assert experts.seen, "the allgather candidate was not used" + finally: + comm_module.get_moe_comm_method = original + + +def test_the_comm_type_resolves_when_the_context_instance_is_unset(loaded): + """MRV2 can leave moe_comm_method unset; the comm type is always present.""" + module, context, _ = loaded + experts = _Experts() + context._EXTRA_CTX.moe_comm_type = _MoECommType.ALLGATHER + context._EXTRA_CTX.moe_comm_method = None + + component = module.AFDExternalRoutingMoE(experts) + component.forward( + torch.zeros(4, 2), + topk_ids=torch.zeros(4, 1, dtype=torch.int32), + topk_weights=torch.ones(4, 1), + ) + assert experts.seen, "expert execution did not run" + assert context._EXTRA_CTX.moe_comm_method is None, ( + "the unset instance was not restored" + ) diff --git a/tests/unit/model_executor/models/test_sync_attention_gate_proxy.py b/tests/unit/model_executor/models/test_sync_attention_gate_proxy.py new file mode 100644 index 000000000..ca1bf18da --- /dev/null +++ b/tests/unit/model_executor/models/test_sync_attention_gate_proxy.py @@ -0,0 +1,188 @@ +# mypy: disable-error-code="attr-defined, name-defined" +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project + +"""Wire protocol of the sync Attention-side gate shell. + +The shell's design claim is an ordering: send the routing, run the shared experts +while the FFN side works, then receive and merge. These tests assert that ordering +and the payload, using a fake connector. They prove the Python protocol; they do not +prove that the shared-expert work actually overlaps the remote work on a device, +which needs an NPU timeline. + +Like the other model-layer tests, this needs vLLM and is skipped on a CPU-only host. +""" + +from __future__ import annotations + +import sys +from types import ModuleType, SimpleNamespace + +import pytest + +torch = pytest.importorskip("torch") +# The adapter module needs this specific vLLM module, and a CPU-only host can have a +# partial vLLM install where `import vllm` succeeds but this one does not. +pytest.importorskip("vllm.forward_context") + +# The NPU adapter; the CUDA module of the same name has no sync gate shell. +from afd_plugin.model_executor.models.npu import deepseek_v4 as adapter # noqa: E402 + +TOP_K = 2 +HIDDEN = 4 +LAYER_IDX = 3 + + +class _RecordingConnector: + """Records the wire calls and the shared-expert run in one ordered list.""" + + def __init__(self, events: list[tuple]) -> None: + self.events = events + + def send_attn_output(self, hidden_states, context, **kwargs) -> None: + self.events.append(("send", hidden_states, context, kwargs)) + + def recv_ffn_output(self, *, ref_tensor, ubatch_idx): + self.events.append(("recv", ubatch_idx)) + return ref_tensor * 0.5 + + +def _fake_gate_topk(monkeypatch, events: list[tuple], topk_ids, topk_weights): + """Replace the gate helper with one that records and returns fixed routing. + + The forward does ``from package import deepseek_v4_attention_gate``. Two states + are possible: the submodule was already imported once, so the package carries a + real attribute, or it was not, so the import falls back to ``sys.modules``. + Setting both covers either one; ``raising=False`` because the attribute may not + exist yet. + """ + + import afd_plugin.model_executor.models.npu as npu_package + + helper = ModuleType( + "afd_plugin.model_executor.models.npu.deepseek_v4_attention_gate" + ) + + def compute_attention_gate_topk( + moe, hidden_states, *, input_ids=None, hidden_states_fp32=None + ): + del moe, input_ids, hidden_states_fp32 + events.append(("gate", int(hidden_states.shape[0]))) + return topk_weights, topk_ids + + helper.compute_attention_gate_topk = compute_attention_gate_topk # type: ignore[attr-defined] + monkeypatch.setitem( + sys.modules, + "afd_plugin.model_executor.models.npu.deepseek_v4_attention_gate", + helper, + ) + monkeypatch.setattr( + npu_package, + "deepseek_v4_attention_gate", + helper, + raising=False, + ) + + +def _shell(*, events, connector, shared_experts): + """Build the shell without its weight construction. + + The forward only needs the layer index and the shared-expert module, so the + linear layers the constructor would build are irrelevant here. + + ``shared_experts`` is a presence flag: True installs a recorder-backed module, + False leaves the slot empty. + """ + + del connector # the connector arrives through the forward metadata + shell = object.__new__(adapter.AFDDeepseekV4SyncAttentionGateRemoteMoE) + torch.nn.Module.__init__(shell) + shell.layer_idx = LAYER_IDX + shell.shared_experts = None + if shared_experts: + + class _Shared(torch.nn.Module): # type: ignore[name-defined] + def forward(self, hidden_states): + events.append(("shared", int(hidden_states.shape[0]))) + return torch.full_like(hidden_states, 2.0) + + shell.shared_experts = _Shared() + return shell + + +def _install_metadata(monkeypatch, connector, events): + afd_metadata = SimpleNamespace(connector=connector, stage_idx=7) + monkeypatch.setattr( + adapter, + "get_afd_metadata_from_forward_context", + lambda: afd_metadata, + ) + return afd_metadata + + +def test_forward_sends_routing_then_runs_shared_then_receives(monkeypatch): + """The shared experts must run between the send and the receive.""" + + events: list[tuple] = [] + connector = _RecordingConnector(events) + _install_metadata(monkeypatch, connector, events) + topk_ids = torch.tensor([[0, 1], [2, 3]], dtype=torch.int32) + topk_weights = torch.full((2, TOP_K), 0.5, dtype=torch.float32) + _fake_gate_topk(monkeypatch, events, topk_ids, topk_weights) + shell = _shell(events=events, connector=connector, shared_experts=True) + hidden_states = torch.full((2, HIDDEN), 8.0, dtype=torch.bfloat16) + + output = shell(hidden_states) + + assert [event[0] for event in events] == ["gate", "send", "shared", "recv"] + # The routing travels on the wire slot, not as token ids. + sent = events[1] + assert sent[3]["routing"][0] is topk_ids + assert sent[3]["routing"][1] is topk_weights + # The context identifies the layer and the stage the routing belongs to. + assert sent[2].metadata.layer_idx == LAYER_IDX + assert sent[2].metadata.seq_lens == [2] + assert events[3][1] == 7 # recv uses the metadata stage index + # routed (0.5 * input) plus shared (2.0). + assert torch.equal(output, hidden_states * 0.5 + 2.0) + + +def test_forward_without_shared_experts_returns_routed_only(monkeypatch): + """With no shared experts the output is the received routed tensor alone.""" + + events: list[tuple] = [] + connector = _RecordingConnector(events) + _install_metadata(monkeypatch, connector, events) + topk_ids = torch.zeros((2, TOP_K), dtype=torch.int32) + topk_weights = torch.ones((2, TOP_K), dtype=torch.float32) + _fake_gate_topk(monkeypatch, events, topk_ids, topk_weights) + shell = _shell(events=events, connector=connector, shared_experts=False) + hidden_states = torch.full((2, HIDDEN), 8.0, dtype=torch.bfloat16) + + output = shell(hidden_states) + + assert [event[0] for event in events] == ["gate", "send", "recv"] + assert torch.equal(output, hidden_states * 0.5) + + +def test_forward_refuses_a_context_without_afd_metadata(monkeypatch): + """A missing forward metadata is an error, not a silent activations-only send.""" + + events: list[tuple] = [] + monkeypatch.setattr( + adapter, + "get_afd_metadata_from_forward_context", + lambda: None, + ) + _fake_gate_topk( + monkeypatch, + events, + torch.zeros((2, TOP_K), dtype=torch.int32), + torch.ones((2, TOP_K), dtype=torch.float32), + ) + shell = _shell( + events=events, connector=_RecordingConnector(events), shared_experts=False + ) + + with pytest.raises(RuntimeError, match="AFD forward metadata"): + shell(torch.zeros(2, HIDDEN, dtype=torch.bfloat16)) diff --git a/tests/unit/test_sync_attention_gate_validation.py b/tests/unit/test_sync_attention_gate_validation.py new file mode 100644 index 000000000..d4680c4df --- /dev/null +++ b/tests/unit/test_sync_attention_gate_validation.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project + +"""Runner validation for the sync Attention-side gate prototype. + +The DBO and ubatching limits live in the NPU feature gate, which both workers run +before this validator, and are covered in +``compat/npu/test_sync_gate_feature_validation_cpu.py``. This module covers the +runner-level contract: a supported synchronous gate configuration must return +cleanly for either role. +""" + +import sys +from types import ModuleType, SimpleNamespace + +import pytest + +from afd_plugin import validation + + +def _runner_config(*, enable_dbo=False, use_ubatching=False): + """A ``VllmConfig`` surface complete enough for a clean return. + + Every field the validator reads on the accepted path is present, so the + "allowed" test can assert a real return instead of tolerating an exception. + """ + + return SimpleNamespace( + use_v2_model_runner=True, + model_config=SimpleNamespace(enforce_eager=True), + speculative_config=None, + parallel_config=SimpleNamespace( + enable_dbo=enable_dbo, + use_ubatching=use_ubatching, + num_ubatches=2, + pipeline_parallel_size=1, + prefill_context_parallel_size=1, + decode_context_parallel_size=1, + data_parallel_size=4, + tensor_parallel_size=1, + enable_expert_parallel=True, + enable_elastic_ep=False, + enable_eplb=False, + use_sequence_parallel_moe=False, + ), + compilation_config=SimpleNamespace( + pass_config=SimpleNamespace(enable_sp=False), + ), + ) + + +@pytest.fixture() +def sync_gate_afd_config(monkeypatch): + """Make the validator see the sync gate-on-Attention configuration.""" + + from afd_plugin.config import CAMP2P_CONNECTOR + + afd_config = SimpleNamespace( + connector=CAMP2P_CONNECTOR, + async_dp=False, + compute_gate_on_attention=True, + num_attention_ranks=4, + num_ffn_ranks=4, + ) + monkeypatch.setattr( + validation, + "parse_afd_config", + lambda _vllm_config, expected_role=None: afd_config, + ) + # The validator imports this lazily, and the real module path pulls vLLM in. + # Inject the leaf module so the accepted path can run on a CPU host. + for name, attributes in ( + ("afd_plugin.model_executor", {}), + ("afd_plugin.model_executor.models", {}), + ( + "afd_plugin.model_executor.models.model_utils", + {"has_afd_model_registration": lambda _model_config: True}, + ), + ): + module = ModuleType(name) + module.__path__ = [] + for attribute, value in attributes.items(): + setattr(module, attribute, value) + monkeypatch.setitem(sys.modules, name, module) + return afd_config + + +@pytest.mark.parametrize("role", ["attention", "ffn"]) +def test_validator_accepts_a_plain_sync_gate_config(sync_gate_afd_config, role): + """A supported configuration must return cleanly, not merely avoid one message.""" + + validation.validate_npu_model_runner_v2_config( + _runner_config(), + expected_role=role, + device_type="npu", + ) diff --git a/tests/unit/v1/worker/test_ffn_model_runner.py b/tests/unit/v1/worker/test_ffn_model_runner.py index 93a299f8a..a07eb8fb3 100644 --- a/tests/unit/v1/worker/test_ffn_model_runner.py +++ b/tests/unit/v1/worker/test_ffn_model_runner.py @@ -272,6 +272,12 @@ def _runner_with_connector_and_model(model, *, num_layers=1): runner.vllm_config = SimpleNamespace( # FFN keeps connector-driven execution within a GPU MRV2 deployment. use_v2_model_runner=True, + # A real VllmConfig always carries a model config. The forward context asks + # it for the MX scale algorithm, which reads exactly two fields: the + # architecture list and the HF text config's model type. Supplying the two + # keeps this fixture honest about the contract instead of omitting the whole + # object; both default to "not MiniMax M3", which is what these tests are. + model_config=SimpleNamespace(architectures=None, hf_text_config=None), parallel_config=SimpleNamespace( data_parallel_size=1, is_moe_model=True, diff --git a/tests/unit/v1/worker/test_npu_model_runner_v2.py b/tests/unit/v1/worker/test_npu_model_runner_v2.py index 62b47d734..5c60e7c57 100644 --- a/tests/unit/v1/worker/test_npu_model_runner_v2.py +++ b/tests/unit/v1/worker/test_npu_model_runner_v2.py @@ -644,7 +644,7 @@ def test_native_child_contexts_preserve_dp_and_own_ascend_kwargs(monkeypatch): assert result == contexts assert [ctx.dp_metadata for ctx in contexts] == dp assert contexts[1].attn_metadata is None - from afd_plugin.connectors.npu.camp2p import _get_group_ep + from afd_plugin.connectors.npu.camp2p.connector import _get_group_ep assert [ _get_group_ep(getattr(ctx, "ubatch_idx", 0), "stage0", "stage1", "") From 1f783fc9f14d22186da2047390bea88fdfe34d8d Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:17:13 +0200 Subject: [PATCH 31/42] test(ffn_faas): GPU FFN rank must honor --device-ids Add a regression test for the device-selection review finding: with --device-ids 2,3 and LOCAL_RANK=1 the standalone GPU FFN rank binds cuda:1 instead of GPU 3. vLLM only stores the resolved IDs in parallel_config.assigned_physical_gpu_ids; its GPU Worker registers them with set_assigned_physical_gpu_ids before mapping the local rank to a device, and the F path skips that step and selects the device before the config exists. The test drives DeepEPM2NFFNBackend.prepare from the native CLI and stops when the rank binds its GPU. It fails on this branch and passes once the config is built first and the mapping is registered before selection. Co-Authored-By: Claude Opus 5.5 Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- tests/unit/ffn_faas/test_gpu_adapter.py | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/unit/ffn_faas/test_gpu_adapter.py b/tests/unit/ffn_faas/test_gpu_adapter.py index f2113e605..92dcf1c97 100644 --- a/tests/unit/ffn_faas/test_gpu_adapter.py +++ b/tests/unit/ffn_faas/test_gpu_adapter.py @@ -9,14 +9,28 @@ import subprocess import sys from pathlib import Path +from unittest.mock import Mock import pytest from afd_plugin.ffn_faas.platforms import get_platform_adapter +from afd_plugin.ffn_faas.service import FFNService +from tests.unit.fakes.afdep import install_fake_afdep +from tests.unit.fakes.ffn_faas import write_qwen3_checkpoint REPO_ROOT = Path(__file__).resolve().parents[3] NATIVE_MODULES = ("torch", "vllm", "afdep", "deep_gemm") GRAPH = ("--compilation-config", '{"cudagraph_mode":"FULL_DECODE_ONLY"}') +# Hopper, the oldest architecture the DeepEP M2N connector accepts. +SM90_CAPABILITY = (9, 0) + + +class DeviceSelectionReachedError(Exception): + """Stops ``prepare`` at the moment the F rank binds its GPU.""" + + def __init__(self, device) -> None: + super().__init__(device) + self.device = device def _afd_config(**overrides) -> str: @@ -203,3 +217,43 @@ def test_cli_reports_a_bad_launch_as_a_usage_error(tmp_path): ) assert result.returncode == 2 assert "must agree" in result.stderr + + +def test_device_ids_place_the_rank_on_its_assigned_gpu( + launch_env, configure, monkeypatch, tmp_path +): + """``--device-ids 2,3`` puts F local rank 1 on GPU 3, as vLLM's Worker does. + + vLLM only records the resolved IDs in ``parallel_config``; its GPU Worker + registers them with ``set_assigned_physical_gpu_ids`` before mapping the + local rank to a device. The F rank must apply the same mapping, or it lands + on ``cuda:``, which may be an Attention GPU. The device itself + is never touched: binding it stops ``prepare``. + """ + torch = pytest.importorskip("torch") + platform_interface = pytest.importorskip("vllm.platforms.interface") + install_fake_afdep() + monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False) + # The mapping is process-wide in vLLM; start and end without one. + monkeypatch.setattr(platform_interface, "_assigned_physical_gpu_ids", None) + monkeypatch.setattr( + torch.cuda, "get_device_capability", lambda device=None: SM90_CAPABILITY + ) + + def stop_at_device_selection(device): + raise DeviceSelectionReachedError(device) + + monkeypatch.setattr(torch.accelerator, "set_device_index", stop_at_device_selection) + write_qwen3_checkpoint(tmp_path / "model") + launch, _ = configure( + "--enforce-eager", + "--device-ids", + "2,3", + afd={"connector_extra_config": {"expert_backend": "fused_experts"}}, + ) + + backend = launch.backend_factory() + with pytest.raises(DeviceSelectionReachedError) as selected: + backend.prepare(Mock(spec=FFNService)) + + assert selected.value.device == torch.device("cuda", 3) From 942c37bca63ddaf77fad4e1c97c39698dd58066d Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:23:27 +0200 Subject: [PATCH 32/42] fix(ffn_faas): apply --device-ids before selecting the GPU The standalone GPU FFN rank ignored --device-ids and bound cuda:, so FFN ranks could land on the Attention GPUs. vLLM only stores the resolved IDs in parallel_config.assigned_physical_gpu_ids; its GPU Worker registers them with set_assigned_physical_gpu_ids before mapping the local rank to a device. The F path selected the device before the config even existed. Build the F-only config first, then register the assignment and map the local rank through it, as the GPU Worker and the NPU adapter do. Building the config reads no weights, so a bad deployment still fails before the experts are loaded. Co-Authored-By: Claude Opus 5.5 Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- afd_plugin/ffn_faas/platforms/gpu/backend.py | 8 +++++--- afd_plugin/ffn_faas/platforms/gpu/environment.py | 15 ++++++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/afd_plugin/ffn_faas/platforms/gpu/backend.py b/afd_plugin/ffn_faas/platforms/gpu/backend.py index 6761dc499..9de1fd5e6 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/backend.py +++ b/afd_plugin/ffn_faas/platforms/gpu/backend.py @@ -54,8 +54,10 @@ def prepare(self, service: FFNService) -> None: Runs on the host thread, in the order that lets a bad deployment fail before the expert weights are read: - 1. select and check the GPU and the afdEP/DeepGEMM stack; - 2. build the F-only ``VllmConfig`` (AFD architecture, Triton MoE pin); + 1. build the F-only ``VllmConfig`` (AFD architecture, Triton MoE pin), + which resolves ``--device-ids``; no device or weight is touched; + 2. select and check the GPU through that assignment, and the + afdEP/DeepGEMM stack; 3. create the F-only NCCL world and vLLM's model-parallel groups; 4. construct the routed-experts runner, which runs the ModelRunnerV2 and CUDA-graph validation and creates the connector (no @@ -73,8 +75,8 @@ def prepare(self, service: FFNService) -> None: before the next native step, so a failure leaves it reachable. """ environment = self.environment - environment.initialize_device() config = environment.build_config(self.args) + environment.initialize_device(config) environment.initialize_compute(config) with set_current_vllm_config(config): self.runner = GPUFFNRoutedExpertsModelRunner(config, environment.device) diff --git a/afd_plugin/ffn_faas/platforms/gpu/environment.py b/afd_plugin/ffn_faas/platforms/gpu/environment.py index 8423d57d3..63b188482 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/environment.py +++ b/afd_plugin/ffn_faas/platforms/gpu/environment.py @@ -52,7 +52,7 @@ def __init__(self, settings: GPUFFNConfig) -> None: # NPU parallel: AscendFFNEnvironment.initialize_device. # Reuse: PARTIAL - launcher and borrowed-group checks are identical; # the device and native preflight are platform-specific. - def initialize_device(self) -> None: + def initialize_device(self, config: VllmConfig) -> None: """Select this rank's GPU after checking the launcher and native stack. Requires the ``torchrun`` environment to still match the settings and @@ -60,8 +60,9 @@ def initialize_device(self) -> None: so this process never adopts groups it does not own. Imports afdEP's DeepEP transport and, for the DeepGEMM expert backend, DeepGEMM, so a missing package fails before any weight is read. Maps ``LOCAL_RANK`` - to the visible GPU the way vLLM's GPU Worker does (honoring - ``CUDA_VISIBLE_DEVICES``) and requires SM90 or newer. + to the visible GPU the way vLLM's GPU Worker does: first registers the + ``--device-ids`` assignment that ``config`` resolved, then maps through + it (and ``CUDA_VISIBLE_DEVICES``). Requires SM90 or newer. """ self.require_uninitialized() self.require_launcher_environment() @@ -69,6 +70,7 @@ def initialize_device(self) -> None: settings = self.settings import torch from vllm.platforms import current_platform + from vllm.platforms.interface import set_assigned_physical_gpu_ids importlib.import_module(AFDEP_TRANSPORT_MODULE) expert_backend = settings.connector_extra_config.get("expert_backend") @@ -77,6 +79,13 @@ def initialize_device(self) -> None: # Importable only after the afdEP preflight above. from afd_plugin.connectors.gpu.deepep_m2n.connector import MIN_SM_MAJOR + # Same step as vLLM's GPU Worker.init_device: register --device-ids + # before mapping the local rank to a device. + assigned = config.parallel_config.assigned_physical_gpu_ids + if assigned is not None: + if settings.local_rank >= len(assigned): + raise ValueError("LOCAL_RANK is outside the --device-ids list") + set_assigned_physical_gpu_ids(assigned) visible_index = current_platform.logical_device_id_to_visible_device_id( settings.local_rank ) From dbccf656419d03fea392f9ff75a2cf5372266ef8 Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:41:01 +0200 Subject: [PATCH 33/42] test(ffn_faas): drop the --device-ids device-selection unit test Remove test_device_ids_place_the_rank_on_its_assigned_gpu and the helpers only it used from tests/unit/ffn_faas/test_gpu_adapter.py. Co-Authored-By: Claude Opus 5.5 Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- tests/unit/ffn_faas/test_gpu_adapter.py | 54 ------------------------- 1 file changed, 54 deletions(-) diff --git a/tests/unit/ffn_faas/test_gpu_adapter.py b/tests/unit/ffn_faas/test_gpu_adapter.py index 92dcf1c97..f2113e605 100644 --- a/tests/unit/ffn_faas/test_gpu_adapter.py +++ b/tests/unit/ffn_faas/test_gpu_adapter.py @@ -9,28 +9,14 @@ import subprocess import sys from pathlib import Path -from unittest.mock import Mock import pytest from afd_plugin.ffn_faas.platforms import get_platform_adapter -from afd_plugin.ffn_faas.service import FFNService -from tests.unit.fakes.afdep import install_fake_afdep -from tests.unit.fakes.ffn_faas import write_qwen3_checkpoint REPO_ROOT = Path(__file__).resolve().parents[3] NATIVE_MODULES = ("torch", "vllm", "afdep", "deep_gemm") GRAPH = ("--compilation-config", '{"cudagraph_mode":"FULL_DECODE_ONLY"}') -# Hopper, the oldest architecture the DeepEP M2N connector accepts. -SM90_CAPABILITY = (9, 0) - - -class DeviceSelectionReachedError(Exception): - """Stops ``prepare`` at the moment the F rank binds its GPU.""" - - def __init__(self, device) -> None: - super().__init__(device) - self.device = device def _afd_config(**overrides) -> str: @@ -217,43 +203,3 @@ def test_cli_reports_a_bad_launch_as_a_usage_error(tmp_path): ) assert result.returncode == 2 assert "must agree" in result.stderr - - -def test_device_ids_place_the_rank_on_its_assigned_gpu( - launch_env, configure, monkeypatch, tmp_path -): - """``--device-ids 2,3`` puts F local rank 1 on GPU 3, as vLLM's Worker does. - - vLLM only records the resolved IDs in ``parallel_config``; its GPU Worker - registers them with ``set_assigned_physical_gpu_ids`` before mapping the - local rank to a device. The F rank must apply the same mapping, or it lands - on ``cuda:``, which may be an Attention GPU. The device itself - is never touched: binding it stops ``prepare``. - """ - torch = pytest.importorskip("torch") - platform_interface = pytest.importorskip("vllm.platforms.interface") - install_fake_afdep() - monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False) - # The mapping is process-wide in vLLM; start and end without one. - monkeypatch.setattr(platform_interface, "_assigned_physical_gpu_ids", None) - monkeypatch.setattr( - torch.cuda, "get_device_capability", lambda device=None: SM90_CAPABILITY - ) - - def stop_at_device_selection(device): - raise DeviceSelectionReachedError(device) - - monkeypatch.setattr(torch.accelerator, "set_device_index", stop_at_device_selection) - write_qwen3_checkpoint(tmp_path / "model") - launch, _ = configure( - "--enforce-eager", - "--device-ids", - "2,3", - afd={"connector_extra_config": {"expert_backend": "fused_experts"}}, - ) - - backend = launch.backend_factory() - with pytest.raises(DeviceSelectionReachedError) as selected: - backend.prepare(Mock(spec=FFNService)) - - assert selected.value.device == torch.device("cuda", 3) From 9393311478e293a39edf1bf81e408fc4af7b3934 Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Wed, 11 Nov 2026 10:44:30 +0800 Subject: [PATCH 34/42] feat(npu): run standalone V4 FFN with native expert ownership (#490) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DeepSeek-V4 Flash W4A8 routed experts can run in independent F-only processes while retaining native expert-management context and the mature CAM execution path. F does not construct a complete CausalLM, vLLM Worker, upstream ModelRunner, EngineCore, request scheduler, sampling state or KV cache. This is a scoped independent-service delivery under #391. The Attention model, runner, connector, routing/shared experts, padding, DBO and shutdown paths keep their existing contracts. ## Implementation - The standalone entry point reuses vLLM's `FlexibleArgumentParser`, `EngineArgs.add_cli_args`, `EngineArgs.from_cli_args` and the existing AFD configuration validators. Graph mode uses `--compilation-config` / `--enforce-eager`; capacities use upstream options, and AFD topology / CAM options use `--additional-config`. Device selection uses native mapping. Only backend selection, management and receiver-shutdown controls add standalone flags. F remains independent of the serving lifecycle. - A CPU-only `FFNBackend`/platform-adapter interface separates the public CLI, Service and host loop from device/model implementation. Ascend/V4 loading, quantization, configuration and CAM composition live in `ffn_faas/platforms/npu/`. - Startup explicitly selects the qualified V4 adapter. `CAMRoutedModel` owns native loading, local placement checks, progress, storage accounting and cached-view lifetime. `V4ModelAdapter` supplies checkpoint names and native owner construction; both V4 description and adapter live in `v4_model.py`, with the native owner lazily loaded from `v4_weights.py`. There is no separate V4 execution runtime. The CAM backend has no V4 type dependency. CAM static placement guards live in platform settings, outside V4 metadata. This is a FaaS-only boundary cleanup, not second-model or quantization support. - `AscendFFNEnvironment` owns the selected device and F-only native compute/EP groups throughout execution. The original routed modules, expert-map managers and quantizers remain resident alongside the separate A/F CAM group. Native post-warmup CPU affinity policy is reused. - The routed-only container reuses the native factory, checkpoint loader and quantizer. Its execution-keyword contract is explicit; the unbound loader adaptation documents its pinned receiver contract and PATCH/delegation exception. Static guards reject EPLB, elastic EP and redundant experts. - Execution reuses `CAMAsyncAFDConnector`, original DS/DR/CS/CR, layered W4A8, paired startup and FULL graph pools. The FFN two-stream path and its core-budget options were removed to follow main (#550). There is no new FaaS connector, transport wire ABI or service control operator. Here `FFNBackend` is a lifecycle adapter for a complete runtime, composing the environment, routed owner, CAM connector and mature Async runtime. The public host loop calls `backend.execute_step()`; communication, compute and Graph execution remain in that runtime. This is not RFC #391's pure `compute(item)`/facts backend or a shared cross-platform, cross-connector WorkItem/StepPlan/Graph Step Loop. Preserving the mature CAM path is deliberate; a real GPU or synchronous connector integration should drive any future common loop. The environment, backend and host runner provide device/compute ownership, composition and host lifecycle without another Worker wrapper. See [the standalone service guide](https://github.com/vllm-project/afd-plugin/blob/053ffeae23e2a22f9d7cdaaa018a9eb430638067/docs/npu/STANDALONE_FFN_SERVICE.md) and the registered FFN runtime design for supported configuration and ownership. ## Validation Current main synchronization at `053ffea` incorporates main `e58fdec`, including its documentation reorganization. Conflict resolutions update the standalone guide, module ownership and moved links; FaaS implementation files are unchanged. Documentation hooks and local links pass. Targeted standalone and connector/session CPU results at `47c75d8` are reused: **120 passed, 6 skipped**. The skipped cases are the native-parser cases in the lean CPU environment; their earlier **6/6 pass** at `b26bdc3` is reused because the CLI implementation and fixtures are unchanged by this merge and use the same pinned vLLM source. Relevant Ruff, mypy 3.11, Markdown and other pre-commit hooks pass. No NPU, model, GSM8K or performance run was added for this main synchronization. The device and performance results below describe their recorded historical commits and execution conditions; they do not validate the updated runtime. ### Historical validation The 79 CPU cases and relevant checks below passed at `cfb72da`. The device regression below was run at `d3c6198`, using the recorded dependency stack and native binaries. The V4 checkpoint description and adapter remain together in `v4_model.py`. - 79 CPU cases pass, covering model/quantization rejection, local source selection, adapter-supplied routed layers, CPU-only first imports and host lifetime. The import gate precedes framework/fake imports and blocks both ordinary and dynamic torch/torch-npu/vLLM/Ascend imports. Twelve real subprocess isolation cases remain covered. Relevant mypy and pre-commit checks pass; metadata stubs do not qualify a second model. - One finite real 12A/4F regression on the original per-channel W4A8 checkpoint passed: A eager MRV2 TP4/DP3 with two native DBO microbatches on all 12 workers, and four F ranks with EP4, 43-layer FULL/two-stream execution (20 AIC/40 AIV compute, 8 AIV combine). - Three short warmups preceded six frozen 14,399–14,935-token requests at C6 and 17/31/257-token tails, each generating one token with prefix caching explicitly disabled. All nine business token IDs, generated-token log probabilities and top-5 log probabilities matched the existing mature CAM reference at `d136157` exactly. That reference was reused; mature AFD was not redeployed. Functional observations confirmed imports of the new `CAMRoutedModel` resource. - F-only EP4 and separate CAM16 groups remained live with the same native routed owner, expert-map managers and quantizers through the pending/failure boundary, with 64 local experts per layer. All four F ranks failed closed with owners retained. Whole-pair isolation left no owned or NPU process. Attention logged peer errors during F-first teardown; A's final zero exit and F's nonzero exit are not a graceful-drain proof. - The native extension and CANN package were reused without rebuilding. Applicable earlier operator-numerical and pinned-stack checks are retained, not counted as new candidate runs. No GSM8K or performance matrix was rerun. Pinned tested dependencies are vLLM `ced6857afa0ea7b2e3f0846a62e1394e90f15607`, vLLM-Ascend `8d4409d6256d8a6729140ddcc0d1889e3f96cdd6`, CANN 9.1.0 and torch-npu 2.10.0.post4. The finite regression ran on another A3 node after resource migration; its correctness results are separate from the earlier same-node performance comparison. Next-token equality is not a full-model accuracy claim. ## Historical clean prefill comparison These figures predate main's removal of FFN two-stream execution (#550). They remain evidence for the versions and conditions below, with no new performance claim for the current merged revision. The candidate at `9151609` was compared with the mature `vllm serve` FFN path on the same source/common clamp fix, native binaries, original W4A8 model and single A3 node. Both used 12A/4F, A TP4/DP3 with native DBO, F FULL/two-stream (20/40 compute and 8 combine), explicit prefix-cache disablement and natural expert routing. Product CLIs loaded no functional observer or profiler. Each measured round replayed the same 36 requests from six six-turn sessions at C6, with one output token. Input lengths were 14,399–58,462 tokens (median 27,951.5), totaling 1,071,401 prompt tokens per round. Frozen priorities, dependencies, domain routing and request payloads were preserved; generated answers did not enter later prompts. One full workload warmup per deployment was excluded. The ordered blocks were A1/A2, B1/B2, B3/B4, A3/A4; both B blocks shared one standalone deployment. | Service / change | Batch (s) | Mean TTFT (s) | Mean of round p95 (s) | | --- | ---: | ---: | ---: | | Mature AFD | 51.301 | 7.839 | 11.814 | | Standalone CAM F | 50.270 | 7.875 | 11.804 | | Standalone vs mature | -2.01% | +0.46% | -0.09% | The mature batch bracket drift was -2.19%; the observed batch advantage is not attributed to a particular change. All three mean metrics and observed round upper comparison bounds stayed below the 2% regression reference. The p95 column averages the per-round p95 values. The comparison measures the whole HTTP prefill path, including Attention and KV work; it is not isolated FFN timing. Four rounds per arm use one standalone deployment bracketed by two mature deployments, so repeats are correlated. The 2% reference is reported against observed variation, not claimed as a universal statistical noninferiority guarantee. Earlier custom-transport and prefix-cache-enabled scores are excluded. A first deployment failed after a long idle wait before any measured round; a later unused-status-port check failed before model launch. Those failures are retained, and each reported timed round is a complete successful replay. ## Lifecycle and remaining scope A pending CAM DR cannot be cancelled by host stop. Failed startup/receive or unfinished batches retain weights, graphs and communication owners until the entire paired deployment is isolated. Diagnostic or HTTP-cleanup failures cannot bypass the CLI's unsafe-session nonzero exit. This does not implement lossless bounded drain, EOS or independent hot restart. Resident expert context does not enable EPLB/EEP in the current static CAM layout. CUDA/other connector backends, synchronous ControlPlane, profiling management, cancellable receive and independent A/F dependency pins remain follow-up scope. Actual CAM pairing still follows native weight loading; a lightweight pre-loading handshake remains unimplemented. The earlier fused implementation and transport work remain preserved separately in #491 and the [pre-CAM snapshot](https://github.com/vllm-project/afd-plugin/commit/16e25f14afaa65e61a3e884427ee56288fe09cc7). They are not modified here or assumed compatible with this narrowed service. --------- Signed-off-by: jiaran-king --- afd_plugin/ffn_faas/__init__.py | 3 + afd_plugin/ffn_faas/__main__.py | 123 ++++++++ afd_plugin/ffn_faas/interfaces.py | 49 ++++ afd_plugin/ffn_faas/platforms/__init__.py | 20 ++ .../ffn_faas/platforms/ffn_model/__init__.py | 3 + .../ffn_model/deepseek_v4/__init__.py | 3 + .../ffn_model/deepseek_v4/npu/__init__.py | 3 + .../ffn_model/deepseek_v4/npu/v4_model.py | 221 ++++++++++++++ .../ffn_model/deepseek_v4/npu/v4_weights.py | 194 ++++++++++++ afd_plugin/ffn_faas/platforms/npu/__init__.py | 27 ++ afd_plugin/ffn_faas/platforms/npu/adapter.py | 90 ++++++ afd_plugin/ffn_faas/platforms/npu/backend.py | 112 +++++++ .../ffn_faas/platforms/npu/environment.py | 191 ++++++++++++ afd_plugin/ffn_faas/platforms/npu/model.py | 199 +++++++++++++ afd_plugin/ffn_faas/platforms/npu/settings.py | 69 +++++ afd_plugin/ffn_faas/runner.py | 87 ++++++ afd_plugin/ffn_faas/service.py | 192 ++++++++++++ .../v1/worker/npu/async_ffn_model_runner.py | 15 + csrc/npu/README.md | 10 + docs/design/module/ffn_runtime.md | 58 +++- docs/design/module/index.md | 2 +- docs/npu/STANDALONE_FFN_SERVICE.md | 151 ++++++++++ tests/npu/async_cam_runtime.py | 25 +- tests/npu/standalone_cam_attention.py | 192 ++++++++++++ tests/unit/fakes/ffn_faas.py | 270 +++++++++++++++++ tests/unit/ffn_faas/conftest.py | 14 + .../deepseek_v4/npu/test_v4_model.py | 110 +++++++ tests/unit/ffn_faas/test_main.py | 275 ++++++++++++++++++ tests/unit/ffn_faas/test_model.py | 32 ++ tests/unit/ffn_faas/test_runner.py | 92 ++++++ tests/unit/ffn_faas/test_service.py | 55 ++++ tests/unit/ffn_faas/test_settings.py | 157 ++++++++++ .../unit/v1/worker/test_async_ffn_runtime.py | 26 ++ 33 files changed, 3057 insertions(+), 13 deletions(-) create mode 100644 afd_plugin/ffn_faas/__init__.py create mode 100644 afd_plugin/ffn_faas/__main__.py create mode 100644 afd_plugin/ffn_faas/interfaces.py create mode 100644 afd_plugin/ffn_faas/platforms/__init__.py create mode 100644 afd_plugin/ffn_faas/platforms/ffn_model/__init__.py create mode 100644 afd_plugin/ffn_faas/platforms/ffn_model/deepseek_v4/__init__.py create mode 100644 afd_plugin/ffn_faas/platforms/ffn_model/deepseek_v4/npu/__init__.py create mode 100644 afd_plugin/ffn_faas/platforms/ffn_model/deepseek_v4/npu/v4_model.py create mode 100644 afd_plugin/ffn_faas/platforms/ffn_model/deepseek_v4/npu/v4_weights.py create mode 100644 afd_plugin/ffn_faas/platforms/npu/__init__.py create mode 100644 afd_plugin/ffn_faas/platforms/npu/adapter.py create mode 100644 afd_plugin/ffn_faas/platforms/npu/backend.py create mode 100644 afd_plugin/ffn_faas/platforms/npu/environment.py create mode 100644 afd_plugin/ffn_faas/platforms/npu/model.py create mode 100644 afd_plugin/ffn_faas/platforms/npu/settings.py create mode 100644 afd_plugin/ffn_faas/runner.py create mode 100644 afd_plugin/ffn_faas/service.py create mode 100644 docs/npu/STANDALONE_FFN_SERVICE.md create mode 100644 tests/npu/standalone_cam_attention.py create mode 100644 tests/unit/fakes/ffn_faas.py create mode 100644 tests/unit/ffn_faas/conftest.py create mode 100644 tests/unit/ffn_faas/platforms/ffn_model/deepseek_v4/npu/test_v4_model.py create mode 100644 tests/unit/ffn_faas/test_main.py create mode 100644 tests/unit/ffn_faas/test_model.py create mode 100644 tests/unit/ffn_faas/test_runner.py create mode 100644 tests/unit/ffn_faas/test_service.py create mode 100644 tests/unit/ffn_faas/test_settings.py diff --git a/afd_plugin/ffn_faas/__init__.py b/afd_plugin/ffn_faas/__init__.py new file mode 100644 index 000000000..aaac2bd74 --- /dev/null +++ b/afd_plugin/ffn_faas/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Standalone FFN process using the existing Async CAM backend.""" diff --git a/afd_plugin/ffn_faas/__main__.py b/afd_plugin/ffn_faas/__main__.py new file mode 100644 index 000000000..9aedb9936 --- /dev/null +++ b/afd_plugin/ffn_faas/__main__.py @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""External-launcher entry point for a standalone FFN platform adapter.""" + +from __future__ import annotations + +import argparse +import json +import os +import signal +import sys +import traceback +from contextlib import suppress +from pathlib import Path + +from afd_plugin.ffn_faas.interfaces import FFNPlatformAdapter +from afd_plugin.ffn_faas.platforms import SUPPORTED_BACKENDS, get_platform_adapter +from afd_plugin.ffn_faas.runner import FFNModelRunner +from afd_plugin.ffn_faas.service import FFNService + + +def _report_exception() -> None: + """Diagnostics must not choose how an unsafe native session exits.""" + with suppress(BaseException): + traceback.print_exc() + + +def create_parser(adapter: FFNPlatformAdapter) -> argparse.ArgumentParser: + parser = adapter.create_parser() + parser.description = __doc__ + parser.add_argument("command", choices=["serve"]) + parser.add_argument("model_tag", nargs="?", help="local model checkpoint") + parser.add_argument("--backend", choices=SUPPORTED_BACKENDS, default="ascend-cam") + parser.add_argument( + "--receiver-shutdown-timeout", + type=float, + default=5.0, + help="seconds to join the standalone receiver before process isolation", + ) + parser.add_argument("--status-host", default="127.0.0.1") + parser.add_argument("--status-port", type=int, default=29700) + parser.add_argument("--status-file", type=Path) + return parser + + +def main() -> int: + selection = argparse.ArgumentParser(add_help=False) + selection.add_argument( + "--backend", choices=SUPPORTED_BACKENDS, default="ascend-cam" + ) + selected, _ = selection.parse_known_args() + adapter = get_platform_adapter(selected.backend) + parser = create_parser(adapter) + args = parser.parse_args() + try: + launch = adapter.configure(args) + status_port = args.status_port + launch.local_rank if args.status_port else 0 + if not 0 <= status_port < 65536: + raise ValueError("invalid per-rank management port") + except (ValueError, OSError) as error: + parser.error(str(error)) + service = FFNService( + rank=launch.rank, + model_path=launch.model_path, + backend_name=launch.backend_name, + initial_status=launch.initial_status, + ) + address = service.start_http(args.status_host, status_port) + for signum in (signal.SIGINT, signal.SIGTERM): + signal.signal(signum, lambda signum, frame: service.request_signal_stop()) + print( + json.dumps( + {"rank": launch.rank, "management": address, "backend": launch.backend_name} + ), + flush=True, + ) + runner = FFNModelRunner( + service, launch.backend_factory, shutdown_timeout=launch.shutdown_timeout + ) + code = 0 + resources_released = False + try: + try: + runner.run() + except BaseException: + code = 1 + _report_exception() + try: + snapshot = service.snapshot() + # Capture the release proof before fallible reporting. If querying + # it fails, retain the conservative default through finalization. + resources_released = snapshot["resources_released"] is True + status = json.dumps(snapshot, indent=2) + "\n" + print(status, flush=True) + if args.status_file is not None: + args.status_file.parent.mkdir(parents=True, exist_ok=True) + args.status_file.with_name( + f"{args.status_file.stem}-rank-{launch.rank}.json" + ).write_text(status) + except BaseException: + code = 1 + _report_exception() + finally: + try: + service.close_http() + except BaseException: + code = 1 + _report_exception() + finally: + if not resources_released: + # Keep runner/backend owners reachable until process isolation, + # even if status writes, tracebacks, HTTP close or flush fail. + try: + for stream in (sys.stdout, sys.stderr): + with suppress(BaseException): + stream.flush() + finally: + os._exit(code or 1) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/afd_plugin/ffn_faas/interfaces.py b/afd_plugin/ffn_faas/interfaces.py new file mode 100644 index 000000000..9211cd407 --- /dev/null +++ b/afd_plugin/ffn_faas/interfaces.py @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""CPU-only boundaries between the service loop and a selected FFN platform.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Protocol + +if TYPE_CHECKING: + from argparse import ArgumentParser, Namespace + + from afd_plugin.ffn_faas.service import FFNService + + +class FFNBackend(Protocol): + """One rank's resources; failures retain the owner until process isolation.""" + + def prepare(self, service: FFNService) -> None: ... + + def start_receiver(self) -> None: ... + + def execute_step(self) -> None: ... + + def receiver_exited(self) -> None: ... + + def close(self) -> None: ... + + +@dataclass(frozen=True) +class FFNLaunch: + """Platform-independent information needed before starting the host loop.""" + + rank: int + local_rank: int + model_path: str + backend_name: str + shutdown_timeout: float + backend_factory: Callable[[], FFNBackend] + initial_status: Mapping[str, str | int | bool] = field(default_factory=dict) + + +class FFNPlatformAdapter(Protocol): + """Build the selected CLI and describe a launch without constructing a backend.""" + + def create_parser(self) -> ArgumentParser: ... + + def configure(self, args: Namespace) -> FFNLaunch: ... diff --git a/afd_plugin/ffn_faas/platforms/__init__.py b/afd_plugin/ffn_faas/platforms/__init__.py new file mode 100644 index 000000000..fa764be69 --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/__init__.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Select an implemented adapter without importing its tensor/runtime stack.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from afd_plugin.ffn_faas.interfaces import FFNPlatformAdapter + +SUPPORTED_BACKENDS = ("ascend-cam",) + + +def get_platform_adapter(name: str) -> FFNPlatformAdapter: + if name == "ascend-cam": + from afd_plugin.ffn_faas.platforms.npu.adapter import AscendCAMAdapter + + return AscendCAMAdapter() + raise ValueError(f"unsupported standalone FFN backend: {name}") diff --git a/afd_plugin/ffn_faas/platforms/ffn_model/__init__.py b/afd_plugin/ffn_faas/platforms/ffn_model/__init__.py new file mode 100644 index 000000000..2a445863e --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/ffn_model/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Model-specific FFN implementations grouped by model and backend.""" diff --git a/afd_plugin/ffn_faas/platforms/ffn_model/deepseek_v4/__init__.py b/afd_plugin/ffn_faas/platforms/ffn_model/deepseek_v4/__init__.py new file mode 100644 index 000000000..8e63e14ac --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/ffn_model/deepseek_v4/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""DeepSeek-V4 routed-expert implementations.""" diff --git a/afd_plugin/ffn_faas/platforms/ffn_model/deepseek_v4/npu/__init__.py b/afd_plugin/ffn_faas/platforms/ffn_model/deepseek_v4/npu/__init__.py new file mode 100644 index 000000000..ec934b79e --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/ffn_model/deepseek_v4/npu/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Ascend DeepSeek-V4 routed-expert parameter loading.""" diff --git a/afd_plugin/ffn_faas/platforms/ffn_model/deepseek_v4/npu/v4_model.py b/afd_plugin/ffn_faas/platforms/ffn_model/deepseek_v4/npu/v4_model.py new file mode 100644 index 000000000..fcfa2a6ff --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/ffn_model/deepseek_v4/npu/v4_model.py @@ -0,0 +1,221 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""CPU-only checkpoint description and loading adapter for ModelSlim V4 W4A8. + +Keep checkpoint names intact. Device loading must delegate packed conversion, +scales and compensation to the pinned Ascend W4A8 loader; this description does +not allocate tensors or initialize native runtime state. +""" + +from __future__ import annotations + +import json +import math +import re +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from vllm.config import VllmConfig + + from afd_plugin.ffn_faas.platforms.npu.model import RoutedWeightOwner + +PACKED_WEIGHTS_PER_BYTE = 2 +PROJECTIONS_PER_EXPERT = 3 +SOURCE_PARAMETERS_PER_PROJECTION = 4 +ENCODED_SCALE_BYTES = 8 +FLOAT_PARAMETER_BYTES = 4 +GATE_UP_PROJECTIONS = 2 + + +@dataclass(frozen=True) +class V4CheckpointDescription: + root: Path + hidden_size: int + intermediate_size: int + num_experts: int + top_k: int + model_layers: int + hash_layers: int + routed_scaling_factor: float + swiglu_limit: float + weight_map: Mapping[str, str] + routed_weight_map: Mapping[str, str] + + @property + def layer_indices(self) -> tuple[int, ...]: + # The pinned V4 decoder constructs routed MoE on every main layer. + # MTP checkpoint layers are outside this service's main-model contract. + return tuple(range(self.model_layers)) + + @classmethod + def read(cls, model_path: str | Path) -> V4CheckpointDescription: + root = Path(model_path).resolve(strict=True) + names = ( + "config.json", + "quant_model_weights.safetensors.index.json", + "quant_model_description.json", + ) + raw = {name: (root / name).read_bytes() for name in names} + config, index, quant = (json.loads(raw[name]) for name in names) + if config["architectures"] != ["DeepseekV4ForCausalLM"]: + raise ValueError("V4 FFN weights require DeepseekV4ForCausalLM") + if config["hidden_act"] != "silu" or config["scoring_func"] != "sqrtsoftplus": + raise ValueError("V4 FFN adapter requires SiLU and sqrtsoftplus routing") + if config.get("torch_dtype", "bfloat16") != "bfloat16": + raise ValueError("V4 FFN activation contract requires BF16") + if quant.get("group_size", 0) != 0: + raise ValueError("V4 FFN requires per-channel W4A8_DYNAMIC weights") + + def positive(name: str) -> int: + value = config[name] + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise ValueError(f"V4 checkpoint {name} must be a positive integer") + return value + + layers = positive("num_hidden_layers") + experts = positive("n_routed_experts") + top_k = positive("num_experts_per_tok") + hash_layers = config["num_hash_layers"] + if ( + not isinstance(hash_layers, int) + or isinstance(hash_layers, bool) + or not 0 <= hash_layers <= layers + or top_k > experts + ): + raise ValueError("invalid V4 Hash layer or routed top-k count") + scale, limit = ( + float(config["routed_scaling_factor"]), + float(config["swiglu_limit"]), + ) + if ( + not math.isfinite(scale) + or scale <= 0 + or not math.isfinite(limit) + or limit < 0 + ): + raise ValueError("invalid V4 routed scale or SwiGLU limit") + mapping = index["weight_map"] + if not isinstance(mapping, dict) or not all( + isinstance(name, str) and isinstance(shard, str) + for name, shard in mapping.items() + ): + raise ValueError("invalid V4 safetensors weight index") + for shard in set(mapping.values()): + if not (root / shard).resolve(strict=True).is_relative_to(root): + raise ValueError("V4 checkpoint shard is outside its root") + routed = {} + for name, shard in mapping.items(): + match = re.fullmatch( + r"layers\.(\d+)\.ffn\.experts\.(\d+)\.w([123])\." + r"(weight|weight_scale|weight_offset|scale_bias)", + name, + ) + if match is None or int(match[1]) >= layers: + continue + if int(match[2]) >= experts: + raise ValueError(f"V4 routed expert is outside the model: {name}") + if quant.get(name) != "W4A8_DYNAMIC": + raise ValueError(f"V4 routed tensor requires W4A8_DYNAMIC: {name}") + routed[name] = shard + # Fail before device allocation, including missing scales/compensation. + # This checks source names; it does not reproduce native packed mapping. + for layer in range(layers): + for expert in range(experts): + for projection in ("w1", "w2", "w3"): + for suffix in ( + "weight", + "weight_scale", + "weight_offset", + "scale_bias", + ): + name = ( + f"layers.{layer}.ffn.experts.{expert}.{projection}.{suffix}" + ) + if name not in routed: + raise ValueError( + f"V4 routed checkpoint tensor is missing: {name}" + ) + return cls( + root, + positive("hidden_size"), + positive("moe_intermediate_size"), + experts, + top_k, + layers, + hash_layers, + scale, + limit, + MappingProxyType(mapping), + MappingProxyType(routed), + ) + + +class V4ModelAdapter: + """Interpret only the validated V4 Flash per-channel ModelSlim contract.""" + + def __init__(self, checkpoint: V4CheckpointDescription) -> None: + self.checkpoint = checkpoint + self._sources: dict[int, list[tuple[int, str, str]]] = { + layer: [] for layer in checkpoint.layer_indices + } + for name, shard in checkpoint.routed_weight_map.items(): + fields = name.split(".", 5) + layer = int(fields[1]) + if layer in self._sources: + self._sources[layer].append((int(fields[4]), name, shard)) + + @property + def root(self) -> Path: + return self.checkpoint.root + + @property + def layer_indices(self) -> tuple[int, ...]: + return self.checkpoint.layer_indices + + @property + def num_experts(self) -> int: + return self.checkpoint.num_experts + + def estimated_parameter_bytes(self, local_experts: int) -> int: + # ModelSlim retains offsets as well as the executor's six tensor views. + # The resource records distinct actual storage sizes after native loading. + checkpoint = self.checkpoint + matrices = ( + PROJECTIONS_PER_EXPERT + * checkpoint.hidden_size + * checkpoint.intermediate_size + // PACKED_WEIGHTS_PER_BYTE + ) + channels = ( + GATE_UP_PROJECTIONS * checkpoint.intermediate_size + checkpoint.hidden_size + ) + auxiliary = channels * ( + ENCODED_SCALE_BYTES + FLOAT_PARAMETER_BYTES + FLOAT_PARAMETER_BYTES + ) + return len(checkpoint.layer_indices) * local_experts * (matrices + auxiliary) + + def local_sources( + self, layer_index: int, owned: frozenset[int] + ) -> list[tuple[str, str]]: + selected = [ + (name, shard) + for expert, name, shard in self._sources[layer_index] + if expert in owned + ] + required = ( + len(owned) * PROJECTIONS_PER_EXPERT * SOURCE_PARAMETERS_PER_PROJECTION + ) + if len(selected) != required: + raise ValueError("V4 layer is missing local native source parameters") + return selected + + def create_owner(self, config: VllmConfig) -> RoutedWeightOwner: + from afd_plugin.ffn_faas.platforms.ffn_model.deepseek_v4.npu.v4_weights import ( + V4RoutedWeightOwner, + ) + + return V4RoutedWeightOwner(self.checkpoint, config) diff --git a/afd_plugin/ffn_faas/platforms/ffn_model/deepseek_v4/npu/v4_weights.py b/afd_plugin/ffn_faas/platforms/ffn_model/deepseek_v4/npu/v4_weights.py new file mode 100644 index 000000000..45325c4e1 --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/ffn_model/deepseek_v4/npu/v4_weights.py @@ -0,0 +1,194 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Runtime-only routed parameter owner using the pinned native V4 loader. + +No native CausalLM/decoder constructor or forward is called. Only routed MoE +parameter modules are constructed; the native checkpoint loader and quantizer +keep ownership of name mapping, expert sharding and packed-weight conversion. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +import torch +from torch import nn +from vllm.config import VllmConfig +from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig +from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts +from vllm.model_executor.layers.fused_moe.router.fused_moe_router import FusedMoERouter +from vllm_ascend.models.deepseek_v4 import model as native + +from afd_plugin.ffn_faas.platforms.ffn_model.deepseek_v4.npu.v4_model import ( + V4CheckpointDescription, +) +from afd_plugin.ffn_faas.platforms.npu.settings import validate_static_experts +from afd_plugin.model_executor.npu.async_cam_layered import ( + AsyncCAMLayeredExecutor, + LayeredFFNWeights, +) + + +class RoutedParameterContainer(nn.Module): + """Factory hook that retains native parameters without a forward runner. + + The explicit keyword contract matches vLLM ced6857's FusedMoEFactory runner + hook. Changes fail at construction instead of disappearing in **kwargs. + No MoERunner, MC2 dispatcher or opposite-role module is constructed. + """ + + def __init__( + self, + *, + layer_name: str, + moe_config: FusedMoEConfig, + routed_experts: RoutedExperts, + router: FusedMoERouter, + enable_dbo: bool, + gate: nn.Module | None, + shared_expert_gate: nn.Module | None, + shared_experts: nn.Module | None, + routed_input_transform: nn.Module | None, + routed_output_transform: nn.Module | None, + routed_scaling_factor: float, + ) -> None: + super().__init__() + if ( + enable_dbo + or routed_scaling_factor != 1.0 + or any( + module is not None + for module in ( + gate, + shared_expert_gate, + shared_experts, + routed_input_transform, + routed_output_transform, + ) + ) + ): + raise ValueError("routed-only ownership cannot accept execution modules") + self.layer_name = layer_name + self.moe_config = moe_config + self.routed_experts = routed_experts + self.router = router + + +class V4RoutedWeightOwner(nn.Module): + """Preserve native parameter paths without constructing opposite-role data.""" + + def __init__( + self, checkpoint: V4CheckpointDescription, vllm_config: VllmConfig + ) -> None: + super().__init__() + self.checkpoint = checkpoint + self.config = vllm_config.model_config.hf_config + self.vllm_config = vllm_config + parallel = vllm_config.parallel_config + validate_static_experts( + enable_eplb=parallel.enable_eplb, + enable_elastic_ep=parallel.enable_elastic_ep, + redundant_experts=parallel.eplb_config.num_redundant_experts, + ) + self.num_redundant_experts = parallel.eplb_config.num_redundant_experts + self.model = nn.Module() + self.model.start_layer = 0 + self.model.end_layer = checkpoint.model_layers + self.model.layers = nn.ModuleDict() + + def create_layer(self, layer_index: int) -> nn.Module: + if layer_index not in self.checkpoint.layer_indices: + raise ValueError("V4 FFN weights must belong to a main-model layer") + layers = self.model.layers + key = str(layer_index) + if key in layers: + raise RuntimeError("V4 FFN layer weights are already constructed") + config = self.config + layer = nn.Module() + layer.layer_idx = layer_index + layer.is_moe_layer = True + layer.mlp = nn.Module() + layer.mlp.swiglu_limit = self.checkpoint.swiglu_limit + # Same routed parameter factory and prefixes as native DeepseekV4MoE. + # A already owns the gate, shared experts and the scaled probabilities. + layer.mlp.experts = FusedMoEFactory( + runner_cls=RoutedParameterContainer, + shared_experts=None, + gate=None, + num_experts=config.n_routed_experts, + top_k=config.num_experts_per_tok, + hidden_size=config.hidden_size, + intermediate_size=config.moe_intermediate_size, + renormalize=config.norm_topk_prob, + quant_config=self.vllm_config.quant_config, + prefix=f"model.layers.{layer_index}.mlp.experts", + scoring_func=config.scoring_func, + routed_scaling_factor=1.0, + swiglu_limit=config.swiglu_limit, + e_score_correction_bias=None, + enable_eplb=self.vllm_config.parallel_config.enable_eplb, + num_redundant_experts=self.num_redundant_experts, + is_sequence_parallel=False, + n_shared_experts=0, + hash_indices_table=None, + ) + layers[key] = layer + return layer + + # Upstream: vLLM-Ascend 8d4409d6256d8a6729140ddcc0d1889e3f96cdd6, + # vllm_ascend/models/deepseek_v4/model.py, AscendDeepseekV4ForCausalLM. + # Adaptation reason: reuse its expert-aware name mapping/weight_loader + # without constructing a CausalLM. The receiver contract is hf config, + # model.start_layer/end_layer/layers, named_parameters and redundant-expert + # count. Native factory construction supplies RoutedExperts' own mapping. + # Delegation exception: do not copy the large native loader. Keep this + # exact-signature adapter until upstream exposes a routed-only loader API; + # qualify dependency upgrades with real checkpoint loading and numerics. + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + # ### PATCH START: use the native loader on the routed-only owner. + return native.AscendDeepseekV4ForCausalLM.load_weights(self, weights) + # ### PATCH END: use the native loader on the routed-only owner. + + def finish_layer(self, layer_index: int) -> None: + routed = self.routed_experts(layer_index) + routed.quant_method.process_weights_after_loading(routed) + + def routed_experts(self, layer_index: int) -> RoutedExperts: + """Keep the native V4 module path inside the model adapter.""" + return self.model.layers[str(layer_index)].mlp.experts.routed_experts + + def weight_views(self) -> tuple[LayeredFFNWeights, ...]: + from vllm_ascend.quantization.quant_type import QuantType + + weights = [] + for layer in self.model.layers.values(): + routed = layer.mlp.experts.routed_experts + parallel = self.vllm_config.parallel_config + validate_static_experts( + enable_eplb=parallel.enable_eplb, + enable_elastic_ep=parallel.enable_elastic_ep, + redundant_experts=parallel.eplb_config.num_redundant_experts, + dynamic_eplb=routed.dynamic_eplb, + ) + scheme = routed.quant_method.quant_method + if scheme.quant_type != QuantType.W4A8: + raise ValueError("V4 FFN requires native W4A8 routed parameters") + view = LayeredFFNWeights( + layer_idx=layer.layer_idx, + w13=routed.w13_weight, + w2=routed.w2_weight, + w13_scale=routed.w13_weight_scale, + w2_scale=routed.w2_weight_scale, + w13_bias=routed.w13_scale_bias, + w2_bias=routed.w2_scale_bias, + per_channel=scheme.is_per_channel_weight, + swiglu_limit=self.checkpoint.swiglu_limit, + routed_scaling_factor=1.0, + quantization="W4A8", + ) + AsyncCAMLayeredExecutor._validate_layer(view) + weights.append(view) + if not weights or any(w.signature() != weights[0].signature() for w in weights): + raise ValueError("V4 FFN requires homogeneous native W4A8 layers") + return tuple(weights) diff --git a/afd_plugin/ffn_faas/platforms/npu/__init__.py b/afd_plugin/ffn_faas/platforms/npu/__init__.py new file mode 100644 index 000000000..2dfaa7c31 --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/npu/__init__.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Ascend CAM device and explicitly supported model selection.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from afd_plugin.ffn_faas.platforms.npu.model import CAMRoutedModel + + +def get_model(model_path: Path, ffn_ranks: int, ffn_rank: int) -> CAMRoutedModel: + """Select the one qualified model; incompatible checkpoints fail on CPU. + + This is a loading boundary, not a model registry or multi-model support claim. + Native CAM still requires its existing HF configuration and W4A8 contract. + """ + from afd_plugin.ffn_faas.platforms.ffn_model.deepseek_v4.npu.v4_model import ( + V4CheckpointDescription, + V4ModelAdapter, + ) + from afd_plugin.ffn_faas.platforms.npu.model import CAMRoutedModel + + adapter = V4ModelAdapter(V4CheckpointDescription.read(model_path)) + return CAMRoutedModel(adapter, ffn_ranks, ffn_rank) diff --git a/afd_plugin/ffn_faas/platforms/npu/adapter.py b/afd_plugin/ffn_faas/platforms/npu/adapter.py new file mode 100644 index 000000000..3a5879210 --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/npu/adapter.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Ascend/V4 CLI adaptation; importing this module remains CPU-only.""" + +from __future__ import annotations + +import os +from argparse import ArgumentParser, Namespace +from functools import partial +from pathlib import Path +from typing import TYPE_CHECKING + +from afd_plugin.config import AFD_ASYNC_CONNECTOR, parse_afd_config +from afd_plugin.ffn_faas.interfaces import FFNLaunch +from afd_plugin.ffn_faas.platforms.npu import get_model +from afd_plugin.ffn_faas.platforms.npu.settings import FFNConfig + +if TYPE_CHECKING: + from afd_plugin.ffn_faas.interfaces import FFNBackend + from afd_plugin.ffn_faas.platforms.npu.model import CAMRoutedModel + + +def create_backend( + model: CAMRoutedModel, settings: FFNConfig, args: Namespace +) -> FFNBackend: + from vllm_ascend.utils import bootstrap_custom_op_env + + bootstrap_custom_op_env() + from afd_plugin.ffn_faas.platforms.npu.backend import CAMFFNBackend + + return CAMFFNBackend(model, settings, args) + + +class AscendCAMAdapter: + """Keep model, quantization, topology and device options out of the host loop.""" + + def create_parser(self) -> ArgumentParser: + # Selected-platform imports: the Service/host entry stays CPU-only. + from vllm.engine.arg_utils import EngineArgs + from vllm.plugins import load_general_plugins + from vllm.utils.argparse_utils import FlexibleArgumentParser + + load_general_plugins() + parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) + # This process is already launched by torchrun, not a vLLM executor. + parser.set_defaults(distributed_executor_backend="external_launcher") + return parser + + def configure(self, args: Namespace) -> FFNLaunch: + if args.model_tag is not None: + # Match vLLM serve's positional-model precedence. + args.model = args.model_tag + afd = parse_afd_config(args.additional_config, expected_role="ffn") + if afd.connector != AFD_ASYNC_CONNECTOR: + raise ValueError("standalone Ascend FFN requires CAMAsyncAFDConnector") + rank, local_rank, world = ( + int(os.environ.get(key, default)) + for key, default in ( + ("RANK", "0"), + ("LOCAL_RANK", "0"), + ("WORLD_SIZE", "1"), + ) + ) + if world != afd.num_ffn_ranks or world != args.data_parallel_size: + raise ValueError( + "F-only WORLD_SIZE, --data-parallel-size and " + "additional_config.afd.num_ffn_ranks must agree" + ) + if args.data_parallel_rank is not None and args.data_parallel_rank != rank: + raise ValueError("--data-parallel-rank must match the F-only launcher RANK") + args.data_parallel_rank = rank + if args.distributed_executor_backend != "external_launcher": + raise ValueError("standalone FFN requires the external_launcher backend") + settings = FFNConfig( + model=Path(args.model), + ffn_ranks=world, + rank=rank, + local_rank=local_rank, + shutdown_timeout=args.receiver_shutdown_timeout, + ) + model = get_model(settings.model, settings.ffn_ranks, settings.rank) + return FFNLaunch( + rank=rank, + local_rank=local_rank, + model_path=str(model.root), + backend_name="CAMAsyncAFDConnector", + shutdown_timeout=settings.shutdown_timeout, + backend_factory=partial(create_backend, model, settings, args), + initial_status={"cam_paired": False}, + ) diff --git a/afd_plugin/ffn_faas/platforms/npu/backend.py b/afd_plugin/ffn_faas/platforms/npu/backend.py new file mode 100644 index 000000000..402cc2d74 --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/npu/backend.py @@ -0,0 +1,112 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Routed-only parameter ownership composed with the mature Async CAM runtime. + +Imported after the Ascend OPP bootstrap, inside the externally launched F process. +No Worker, upstream ModelRunner, EngineCore, scheduler or KV cache is constructed. +""" + +from __future__ import annotations + +from argparse import Namespace + +import torch +import torch_npu # noqa: F401 +from vllm.config import set_current_vllm_config +from vllm.engine.arg_utils import EngineArgs + +from afd_plugin.compat.npu import ( + fail_if_unsupported_npu_afd_features, + fix_all2all_backend_for_afd, +) +from afd_plugin.config import parse_afd_config +from afd_plugin.connectors.npu.async_cam import CAMAsyncAFDConnector +from afd_plugin.ffn_faas.platforms.npu.environment import AscendFFNEnvironment +from afd_plugin.ffn_faas.platforms.npu.model import CAMRoutedModel +from afd_plugin.ffn_faas.platforms.npu.settings import FFNConfig +from afd_plugin.ffn_faas.service import FFNService, ServicePhase +from afd_plugin.v1.worker.npu.async_ffn_model_runner import AsyncNPUFFNModelRunner +from afd_plugin.validation import validate_npu_model_runner_v2_config + + +class CAMFFNBackend: + """Keep weights and communication owners until receiver exit is proven.""" + + def __init__( + self, model: CAMRoutedModel, settings: FFNConfig, args: Namespace + ) -> None: + self.settings = settings + self.args = args + self.environment = AscendFFNEnvironment(settings) + self.model = model + self.runtime: AsyncNPUFFNModelRunner | None = None + self.connector: CAMAsyncAFDConnector | None = None + + def prepare(self, service: FFNService) -> None: + args = EngineArgs.from_cli_args(self.args) + config = args.create_engine_config() + # Keep the externally launched F-only ownership after platform defaults. + config.parallel_config.distributed_executor_backend = "external_launcher" + fix_all2all_backend_for_afd(config) + validate_npu_model_runner_v2_config( + config, expected_role="ffn", device_type="npu" + ) + fail_if_unsupported_npu_afd_features(config) + if config.speculative_config is not None: + raise ValueError("standalone V4 loads main-model routed layers only") + self.environment.initialize_device(config) + self.environment.initialize_compute(config) + with set_current_vllm_config(config): + self.model.load( + self.environment.device, config, progress=service.loading_progress + ) + service.phase( + ServicePhase.CONNECTING, + model_loaded=True, + parameter_bytes=self.model.parameter_bytes, + ) + self.connector = CAMAsyncAFDConnector( + rank=self.settings.rank, + local_rank=self.environment.device.index, + vllm_config=config, + afd_config=parse_afd_config(config, expected_role="ffn"), + role_rank=self.settings.rank, + ) + self.runtime = AsyncNPUFFNModelRunner( + config, self.environment.device, self.connector + ) + self.runtime.attach_model( + self.model, parameter_bytes=self.model.parameter_bytes + ) + # Reuse paired signatures, warmup, capture and Graph pools from + # the existing CAM implementation. + self.runtime.initialize_afd_connector() + self.environment.bind_host_threads(config) + service.phase( + ServicePhase.CONNECTING, + cam_paired=True, + compute_ep_world_size=self.environment.ep_group.world_size, + ) + + def start_receiver(self) -> None: + """Select the device once on the receiving thread, as the Worker does.""" + self.environment.activate_thread() + + def execute_step(self) -> None: + assert self.runtime is not None + self.runtime.execute_connector_driven_step() + if self.runtime.effective_mode == "eager": + # Preserve the mature eager loop's completion fence. + torch.npu.synchronize() + + def receiver_exited(self) -> None: + assert self.runtime is not None + self.runtime.mark_receiver_exited() + + def close(self) -> None: + assert self.runtime is not None + # The runtime rejects pending graphs before closing CAM. Only then may + # its borrowed parameter views and the original owner be released. + self.runtime.shutdown() + self.environment.close() + self.model.close() diff --git a/afd_plugin/ffn_faas/platforms/npu/environment.py b/afd_plugin/ffn_faas/platforms/npu/environment.py new file mode 100644 index 000000000..5a9f2065e --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/npu/environment.py @@ -0,0 +1,191 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Own the F-local Ascend device and native compute groups through execution. + +CAM owns a distinct A/F process group. Keeping the native F-only EP context +does not turn on EPLB/EEP or route any per-request work through its collectives. +""" + +from __future__ import annotations + +import logging +import os +from datetime import timedelta +from typing import TYPE_CHECKING + +from afd_plugin.ffn_faas.platforms.npu.settings import ( + FFNConfig, + validate_static_experts, +) + +if TYPE_CHECKING: + import torch + from vllm.config import VllmConfig + from vllm.distributed.parallel_state import GroupCoordinator + +COMPUTE_GROUP_TIMEOUT_SECONDS = 180 + + +class AscendFFNEnvironment: + """Never borrow unrelated groups or tear them down on a failed native seam.""" + + def __init__(self, settings: FFNConfig) -> None: + self.settings = settings + self._device: torch.device | None = None + self._ep_group: GroupCoordinator | None = None + self._groups_owned = False + + @property + def device(self) -> torch.device: + if self._device is None: + raise RuntimeError("Ascend FFN device has not been initialized") + return self._device + + @property + def ep_group(self) -> GroupCoordinator: + if self._ep_group is None: + raise RuntimeError("Ascend FFN expert group has not been initialized") + return self._ep_group + + def initialize_device(self, config: VllmConfig) -> None: + if self._groups_owned or self._device is not None: + raise RuntimeError("Ascend FFN environment is already initialized") + settings = self.settings + try: + world, rank = int(os.environ["WORLD_SIZE"]), int(os.environ["RANK"]) + except (KeyError, ValueError) as error: + raise ValueError( + "Ascend FFN requires an external F-only launcher" + ) from error + if world != settings.ffn_ranks or rank != settings.rank: + raise ValueError("Ascend FFN environment differs from the F-only launcher") + from vllm_ascend.utils import bootstrap_custom_op_env + + bootstrap_custom_op_env() + import torch + import torch_npu # noqa: F401 + from vllm.distributed.parallel_state import model_parallel_is_initialized + from vllm.platforms import current_platform + from vllm.platforms.interface import set_assigned_physical_gpu_ids + from vllm_ascend.utils import enable_custom_op + + from afd_plugin.compat.npu.ops import ensure_cam_async_ops_available + + if torch.distributed.is_initialized() or model_parallel_is_initialized(): + raise RuntimeError("Ascend FFN cannot borrow existing native groups") + ensure_cam_async_ops_available() + # Reuse the native Worker's mapping for --device-ids and visibility. + assigned = config.parallel_config.assigned_physical_gpu_ids + if assigned is not None: + if settings.local_rank >= len(assigned): + raise ValueError("LOCAL_RANK is outside the assigned device list") + set_assigned_physical_gpu_ids(assigned) + index = current_platform.logical_device_id_to_visible_device_id( + settings.local_rank + ) + self._device = torch.device(f"npu:{index}") + torch.npu.set_device(self._device) + # Native NPUModelRunner v1 installs this packing policy at import time. + # This service does not import that runner; native W4A8 still needs NZ. + torch.npu.config.allow_internal_format = True + if not enable_custom_op(): + raise RuntimeError("native V4 W4A8 custom operators are unavailable") + + def initialize_compute(self, config: VllmConfig) -> None: + if self._device is None or self._groups_owned: + raise RuntimeError("Ascend FFN compute initialization requires its device") + settings = self.settings + parallel = config.parallel_config + validate_static_experts( + enable_eplb=parallel.enable_eplb, + enable_elastic_ep=parallel.enable_elastic_ep, + redundant_experts=parallel.eplb_config.num_redundant_experts, + ) + from vllm.config import set_current_vllm_config + from vllm.distributed import ( + init_distributed_environment, + initialize_model_parallel, + ) + from vllm.distributed.parallel_state import ( + get_ep_group, + ) + from vllm_ascend.ascend_config import init_ascend_config + from vllm_ascend.distributed.parallel_state import init_ascend_model_parallel + from vllm_ascend.utils import register_ascend_customop + + with set_current_vllm_config(config): + register_ascend_customop(config) + init_ascend_config(config) + # Ownership starts before a collective can partially initialize. + self._groups_owned = True + init_distributed_environment( + world_size=settings.ffn_ranks, + rank=settings.rank, + local_rank=self.device.index, + distributed_init_method="env://", + backend="hccl", + timeout=timedelta(seconds=COMPUTE_GROUP_TIMEOUT_SECONDS), + ) + initialize_model_parallel( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + backend="hccl", + ) + init_ascend_model_parallel(parallel) + self._ep_group = get_ep_group() + if ( + self._ep_group.world_size != settings.ffn_ranks + or self._ep_group.rank_in_group != settings.rank + ): + raise ValueError("native V4 expert group differs from the F world") + + def activate_thread(self) -> None: + import torch + + torch.npu.set_device(self.device) + + def bind_host_threads(self, config: VllmConfig) -> None: + """Reuse the native Worker's post-warmup affinity policy, without a Worker. + + The receiver is created afterward and inherits the main-thread affinity. + Like native Worker startup, a binding failure is logged, not hidden as a + successful affinity proof. Qualification records actual /proc affinity. + """ + from vllm.platforms import current_platform + from vllm_ascend.ascend_config import get_ascend_config + from vllm_ascend.cpu_binding import bind_cpus + + if not get_ascend_config().enable_cpu_binding: + return + engram = config.engram_config + try: + bind_cpus( + self.settings.local_rank, + npu_id=current_platform.device_id_to_physical_device_id( + self.settings.local_rank + ), + migrate_memory=not (engram is not None and engram.cpu_offload), + ) + except Exception as error: + logging.getLogger(__name__).warning( + "Native CPU binding failed for F rank %s: %s", + self.settings.rank, + error, + ) + + def close(self) -> None: + """Only after CAM/executor completion; failures retain parameter owners.""" + if not self._groups_owned: + return + from vllm.distributed.parallel_state import ( + destroy_distributed_environment, + destroy_model_parallel, + ) + from vllm_ascend.distributed.parallel_state import destroy_ascend_model_parallel + + destroy_ascend_model_parallel() + destroy_model_parallel() + destroy_distributed_environment() + self._groups_owned = False + self._ep_group = None + self._device = None diff --git a/afd_plugin/ffn_faas/platforms/npu/model.py b/afd_plugin/ffn_faas/platforms/npu/model.py new file mode 100644 index 000000000..8209002d6 --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/npu/model.py @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Routed parameter lifetime for the existing Ascend CAM/W4A8 model seam. + +Model adapters supply checkpoint names and native construction at load time. +The mature runtime borrows cached weight views, without per-step adaptation. +This module is CPU-importable; native dependencies are loaded inside load(). +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable, Iterator +from contextlib import ExitStack +from pathlib import Path +from typing import TYPE_CHECKING, Protocol + +from afd_plugin.ffn_faas.platforms.npu.settings import validate_native_expert_map + +if TYPE_CHECKING: + import torch + from torch import nn + from vllm.config import VllmConfig + from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts + + from afd_plugin.model_executor.npu.async_cam_layered import LayeredFFNWeights + + +class RoutedWeightOwner(Protocol): + """Native routed modules retained until CAM and compute groups retire.""" + + def create_layer(self, layer_index: int) -> nn.Module: ... + def routed_experts(self, layer_index: int) -> RoutedExperts: ... + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: ... + def finish_layer(self, layer_index: int) -> None: ... + def weight_views(self) -> tuple[LayeredFFNWeights, ...]: ... + def parameters(self) -> Iterator[nn.Parameter]: ... + + +class RoutedModelAdapter(Protocol): + """Only facts and loading hooks consumed by this CAM integration.""" + + @property + def root(self) -> Path: ... + @property + def layer_indices(self) -> tuple[int, ...]: ... + @property + def num_experts(self) -> int: ... + def estimated_parameter_bytes(self, local_experts: int) -> int: ... + def local_sources( + self, layer_index: int, owned: frozenset[int] + ) -> list[tuple[str, str]]: ... + def create_owner(self, config: VllmConfig) -> RoutedWeightOwner: ... + + +class CAMRoutedModel: + """Own loading and native resources; lend the existing runtime its views.""" + + def __init__( + self, adapter: RoutedModelAdapter, ffn_ranks: int, ffn_rank: int + ) -> None: + if ffn_ranks <= 0 or adapter.num_experts % ffn_ranks: + raise ValueError("routed experts must divide across F ranks") + if not 0 <= ffn_rank < ffn_ranks: + raise ValueError("FFN rank is outside the F-only loading world") + self.adapter, self.ffn_ranks, self.ffn_rank = adapter, ffn_ranks, ffn_rank + self.local_experts = adapter.num_experts // ffn_ranks + self._owner: RoutedWeightOwner | None = None + self._views: tuple[LayeredFFNWeights, ...] | None = None + self._loading = False + self._loaded = False + self._parameter_bytes = adapter.estimated_parameter_bytes(self.local_experts) + + @property + def root(self) -> Path: + return self.adapter.root + + @property + def parameter_bytes(self) -> int: + return self._parameter_bytes + + @property + def native_view(self) -> tuple[LayeredFFNWeights, ...]: + if not self._loaded or self._views is None: + raise RuntimeError("FFN weights have not been loaded") + return self._views + + @property + def native_owner(self) -> RoutedWeightOwner: + if not self._loaded or self._owner is None: + raise RuntimeError("native routed parameters have not finished loading") + return self._owner + + def load( + self, + device: torch.device, + vllm_config: VllmConfig, + progress: Callable[[int, int], None] | None = None, + ) -> None: + if self._loading or self._owner is not None or self._loaded: + raise RuntimeError("FFN weights are already loading or loaded") + self._loading = True + try: + self._load_native(device, vllm_config, progress) + finally: + self._loading = False + + def _load_native( + self, + device: torch.device, + config: VllmConfig, + progress: Callable[[int, int], None] | None, + ) -> None: + import torch + from safetensors import safe_open + from vllm.config import set_current_vllm_config + from vllm.distributed.parallel_state import get_ep_group + + group = get_ep_group() + if group.world_size != self.ffn_ranks or group.rank_in_group != self.ffn_rank: + raise ValueError("native expert group differs from the F world") + adapter = self.adapter + total = len(adapter.layer_indices) * self.local_experts + completed = 0 + with set_current_vllm_config(config): + with torch.device(device): + owner = adapter.create_owner(config) + # A partial layer/packing failure must leave its owner reachable. + self._owner = owner + for layer_index in adapter.layer_indices: + with torch.device(device): + owner.create_layer(layer_index) + routed = owner.routed_experts(layer_index) + if routed.local_num_experts != self.local_experts: + raise ValueError( + "native local expert count differs from the F topology" + ) + mapping = routed.expert_map + owned = validate_native_expert_map( + num_experts=adapter.num_experts, + ffn_ranks=self.ffn_ranks, + ffn_rank=self.ffn_rank, + expert_map=mapping.cpu().tolist() if mapping is not None else None, + ) + selected = adapter.local_sources(layer_index, owned) + with ExitStack() as stack: + readers = { + shard: stack.enter_context( + safe_open( + str(adapter.root / shard), framework="pt", device="cpu" + ) + ) + for shard in sorted({shard for _, shard in selected}) + } + owner.load_weights( + (name, readers[shard].get_tensor(name)) + for name, shard in selected + ) + owner.finish_layer(layer_index) + # Startup-only fence: source mappings and packing temporaries + # can retire before the next layer, and progress means packed. + torch.npu.synchronize(device) + completed += len(owned) + if progress is not None: + progress(completed, total) + views = owner.weight_views() + if tuple(view.layer_idx for view in views) != adapter.layer_indices: + raise ValueError("native views do not match the routed model layers") + self._views = views + storage_sizes: dict[tuple[str, int], int] = {} + for parameter in owner.parameters(): + storage = parameter.untyped_storage() + key = (str(parameter.device), storage.data_ptr()) + storage_sizes[key] = storage.nbytes() + self._parameter_bytes = sum(storage_sizes.values()) + self._loaded = True + + def get_experts_layer_indices(self) -> tuple[int, ...]: + return self.adapter.layer_indices + + def get_async_cam_layered_weights(self) -> tuple[list[LayeredFFNWeights], str]: + return list(self.native_view), "" + + def compute_ffn_output( + self, + hidden_states: torch.Tensor, + layer_idx: int, + *, + group_list: torch.Tensor | None, + dynamic_scales: torch.Tensor | None, + ) -> torch.Tensor: + raise RuntimeError("standalone CAM requires the native layered W4A8 executor") + + def close(self) -> None: + """The backend retires execution and its compute environment first.""" + if self._loading: + raise RuntimeError("cannot close FFN weights during loading") + self._views = None + self._owner = None + self._loaded = False diff --git a/afd_plugin/ffn_faas/platforms/npu/settings.py b/afd_plugin/ffn_faas/platforms/npu/settings.py new file mode 100644 index 000000000..034623ebf --- /dev/null +++ b/afd_plugin/ffn_faas/platforms/npu/settings.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""CPU-only launch settings for the standalone CAM FFN process.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class FFNConfig: + model: Path + ffn_ranks: int + rank: int = 0 + local_rank: int = 0 + shutdown_timeout: float = 5.0 + + def __post_init__(self) -> None: + # Engine/connector definitions and validation belong to vLLM/AFD. + # Only external-launcher and host-shutdown facts live here. + if self.ffn_ranks <= 0 or not 0 <= self.rank < self.ffn_ranks: + raise ValueError("invalid F-only rank or world size") + if self.local_rank < 0: + raise ValueError("local rank must be nonnegative") + if not math.isfinite(self.shutdown_timeout) or self.shutdown_timeout <= 0: + raise ValueError("shutdown timeout must be finite and positive") + + +def validate_static_experts( + *, + enable_eplb: bool, + enable_elastic_ep: bool, + redundant_experts: int, + dynamic_eplb: bool = False, +) -> None: + """Keep native expert state resident without enabling unsupported relocation.""" + if enable_eplb or enable_elastic_ep or redundant_experts != 0 or dynamic_eplb: + raise ValueError( + "standalone CAM requires static expert placement; EPLB, elastic EP " + "and redundant experts are not supported" + ) + + +def validate_native_expert_map( + *, num_experts: int, ffn_ranks: int, ffn_rank: int, expert_map: list[int] | None +) -> frozenset[int]: + """Require the native sharding to match CAM's contiguous expert placement.""" + if ffn_ranks <= 0 or num_experts % ffn_ranks or not 0 <= ffn_rank < ffn_ranks: + raise ValueError("CAM experts require a divisible, valid F topology") + local_experts = num_experts // ffn_ranks + if expert_map is None: + if ffn_ranks != 1: + raise ValueError("sharded native experts require an explicit map") + # The native factory omits the map when every expert is local. + return frozenset(range(num_experts)) + if len(expert_map) != num_experts: + raise ValueError("native expert map has the wrong global expert count") + owned = frozenset(expert for expert, local in enumerate(expert_map) if local >= 0) + if sorted(expert_map[expert] for expert in owned) != list(range(local_experts)): + raise ValueError("native expert map is not a local bijection") + first = ffn_rank * local_experts + end = first + local_experts + if owned != frozenset(range(first, end)) or expert_map[first:end] != list( + range(local_experts) + ): + raise ValueError("native placement differs from contiguous CAM routing") + return owned diff --git a/afd_plugin/ffn_faas/runner.py b/afd_plugin/ffn_faas/runner.py new file mode 100644 index 000000000..199293ff5 --- /dev/null +++ b/afd_plugin/ffn_faas/runner.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Host lifecycle for one injected standalone FFN backend.""" + +from __future__ import annotations + +import math +import threading +from collections.abc import Callable + +from afd_plugin.ffn_faas.interfaces import FFNBackend +from afd_plugin.ffn_faas.service import FFNService, ServicePhase + +HOST_STATUS_POLL_SECONDS = 0.1 + + +class FFNModelRunner: + """Own the backend across stop requests and failed or pending receives.""" + + def __init__( + self, + service: FFNService, + backend_factory: Callable[[], FFNBackend], + *, + shutdown_timeout: float, + ) -> None: + if not math.isfinite(shutdown_timeout) or shutdown_timeout <= 0: + raise ValueError("shutdown timeout must be finite and positive") + self.service = service + self.backend_factory = backend_factory + self.shutdown_timeout = shutdown_timeout + self.backend: FFNBackend | None = None + self._thread: threading.Thread | None = None + self._done = threading.Event() + self._failure: BaseException | None = None + + def _serve(self) -> None: + assert self.backend is not None + try: + self.backend.start_receiver() + while not self.service.stop_requested.is_set(): + self.backend.execute_step() + self.service.completed_batches() + self.backend.receiver_exited() + except BaseException as error: + self._failure = error + finally: + self._done.set() + + def run(self) -> None: + try: + if self.service.stop_requested.is_set(): + self.service.phase(ServicePhase.STOPPED, resources_released=True) + return + self.service.phase(ServicePhase.LOADING) + # Retain the backend before any native initialization can fail. + self.backend = self.backend_factory() + self.backend.prepare(self.service) + try: + self.service.phase(ServicePhase.READY) + except InterruptedError: + self.backend.receiver_exited() + else: + self._thread = threading.Thread( + target=self._serve, name="ffn-receiver", daemon=True + ) + self._thread.start() + while not self.service.stop_requested.wait(HOST_STATUS_POLL_SECONDS): + if self._done.is_set(): + break + self._thread.join(timeout=self.shutdown_timeout) + if self._thread.is_alive(): + raise RuntimeError( + "FFN receive is pending; retain owners and isolate the " + "deployment. A host stop cannot cancel backend device work." + ) + if self._failure is not None: + raise RuntimeError( + "FFN receiver failed; retain resources" + ) from self._failure + self.backend.close() + self.service.phase(ServicePhase.STOPPED, resources_released=True) + except BaseException as error: + # No native close on a failed startup/receive: an enqueued task may + # still reference weights, Graph pools or communication buffers. + self.service.phase(ServicePhase.FAILED, error=str(error)) + raise diff --git a/afd_plugin/ffn_faas/service.py b/afd_plugin/ffn_faas/service.py new file mode 100644 index 000000000..71ae034a9 --- /dev/null +++ b/afd_plugin/ffn_faas/service.py @@ -0,0 +1,192 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""CPU-only service state and management HTTP; queries never touch a device.""" + +from __future__ import annotations + +import json +import os +import threading +import time +from collections.abc import Mapping +from enum import StrEnum +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final, cast + + +class ServicePhase(StrEnum): + STARTING = "starting" + LOADING = "loading" + CONNECTING = "connecting" + WARMUP = "warmup" + READY = "ready" + STOPPING = "stopping" + STOPPED = "stopped" + FAILED = "failed" + + +TERMINAL_PHASES: Final = frozenset({ServicePhase.STOPPED, ServicePhase.FAILED}) +SIGNAL_STOP_POLL_SECONDS: Final = 0.05 + + +class StopRequest(threading.Event): + """Signal handlers write one plain flag, without reentering Event locks.""" + + def __init__(self) -> None: + super().__init__() + self._signal_requested = False + + def request_from_signal(self) -> None: + self._signal_requested = True + + def is_set(self) -> bool: + return self._signal_requested or super().is_set() + + def wait(self, timeout: float | None = None) -> bool: + deadline = float("inf") if timeout is None else time.monotonic() + timeout + while not self.is_set(): + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + super().wait(min(SIGNAL_STOP_POLL_SECONDS, remaining)) + return True + + +class FFNService: + """Own lifecycle state; one external process owns one device runner.""" + + def __init__( + self, + *, + rank: int, + model_path: str, + backend_name: str = "unconfigured", + initial_status: Mapping[str, str | int | bool] | None = None, + ) -> None: + self.stop_requested = StopRequest() + self._lock = threading.Lock() + self._status: dict[str, str | int | bool] = { + "rank": rank, + "pid": os.getpid(), + "model_path": model_path, + "backend": backend_name, + "phase": ServicePhase.STARTING.value, + "completed_batches": 0, + "model_loaded": False, + "loaded_experts": 0, + "total_experts": 0, + "resources_released": False, + } + if initial_status: + if self._status.keys() & initial_status.keys(): + raise ValueError("backend status cannot override service-owned fields") + self._status.update(initial_status) + self._server: ThreadingHTTPServer | None = None + self._http_thread: threading.Thread | None = None + + def snapshot(self) -> dict[str, str | int | bool]: + with self._lock: + status = dict(self._status) + if ( + self.stop_requested.is_set() + and ServicePhase(cast(str, status["phase"])) not in TERMINAL_PHASES + ): + status["phase"] = ServicePhase.STOPPING.value + return status + + def phase(self, phase: ServicePhase, **details: str | int | bool) -> None: + with self._lock: + current = ServicePhase(cast(str, self._status["phase"])) + if current in TERMINAL_PHASES: + raise RuntimeError( + f"cannot leave terminal service state {current.value}" + ) + if self.stop_requested.is_set() and phase == ServicePhase.READY: + raise InterruptedError("FFN stop was requested before readiness") + self._status.update(details, phase=phase.value) + + def loading_progress(self, completed: int, total: int) -> None: + if self.stop_requested.is_set(): + raise InterruptedError("FFN stopped while loading expert weights") + with self._lock: + self._status.update(loaded_experts=completed, total_experts=total) + + def completed_batches(self, count: int = 1) -> None: + if count <= 0: + raise ValueError("completed work count must be positive") + with self._lock: + self._status["completed_batches"] = ( + int(self._status["completed_batches"]) + count + ) + + def request_stop(self) -> None: + # Request a host-loop stop. Backend completion, not this event, must + # establish device quiescence before resources can be released. + self.stop_requested.set() + with self._lock: + if ServicePhase(cast(str, self._status["phase"])) not in TERMINAL_PHASES: + self._status["phase"] = ServicePhase.STOPPING.value + + def request_signal_stop(self) -> None: + # Called on the interrupted main thread, which may already hold either + # the service lock or Event's condition lock. Never acquire either here. + self.stop_requested.request_from_signal() + + def start_http(self, host: str, port: int) -> tuple[str, int]: + service = self + + class Handler(BaseHTTPRequestHandler): + def respond(self, code: HTTPStatus, body: dict) -> None: + payload = (json.dumps(body, sort_keys=True) + "\n").encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def do_GET(self) -> None: + status = service.snapshot() + phase = ServicePhase(cast(str, status["phase"])) + if self.path == "/status": + self.respond(HTTPStatus.OK, status) + elif self.path in ("/ready", "/health"): + healthy = ( + phase == ServicePhase.READY + if self.path == "/ready" + else phase not in TERMINAL_PHASES + ) + self.respond( + HTTPStatus.OK if healthy else HTTPStatus.SERVICE_UNAVAILABLE, + status, + ) + else: + self.respond(HTTPStatus.NOT_FOUND, {"error": "unknown endpoint"}) + + def do_POST(self) -> None: + if self.path != "/stop": + self.respond(HTTPStatus.NOT_FOUND, {"error": "unknown endpoint"}) + return + service.request_stop() + self.respond(HTTPStatus.ACCEPTED, service.snapshot()) + + def log_message(self, format: str, *args: str) -> None: + pass + + self._server = ThreadingHTTPServer((host, port), Handler) + self._server.daemon_threads = True + self._http_thread = threading.Thread( + target=self._server.serve_forever, name="ffn-management", daemon=True + ) + self._http_thread.start() + address, bound_port = self._server.server_address[:2] + return str(address), int(bound_port) + + def close_http(self) -> None: + if self._server is not None: + self._server.shutdown() + self._server.server_close() + self._server = None + if self._http_thread is not None: + self._http_thread.join() + self._http_thread = None diff --git a/afd_plugin/v1/worker/npu/async_ffn_model_runner.py b/afd_plugin/v1/worker/npu/async_ffn_model_runner.py index 4ef9070e6..8ce1e175a 100644 --- a/afd_plugin/v1/worker/npu/async_ffn_model_runner.py +++ b/afd_plugin/v1/worker/npu/async_ffn_model_runner.py @@ -90,6 +90,21 @@ def __init__( self._failure: Exception | None = None self.prof = create_afd_npu_profiler("ffn") + def attach_model(self, model: FFNModel, *, parameter_bytes: int) -> None: + """Borrow an independently loaded model without constructing a Worker.""" + if self.model is not None or self.connector.is_initialized: + raise RuntimeError("Async FFN model must be attached before CAM startup") + if parameter_bytes < 0: + raise ValueError("model parameter size must be nonnegative") + self.model = model + self.model_memory_usage = parameter_bytes + + def mark_receiver_exited(self) -> None: + """Allow release only after the owner joined a successful receive loop.""" + if self._graph_pending: + raise RuntimeError("Async FFN has pending work; retain resources") + self._graph_receiver_exited = True + def load_model(self) -> None: # Import the loader only in the isolated, initialized NPU worker. from vllm.model_executor.model_loader import get_model_loader diff --git a/csrc/npu/README.md b/csrc/npu/README.md index 60a96e65d..0f1c8718c 100644 --- a/csrc/npu/README.md +++ b/csrc/npu/README.md @@ -167,10 +167,20 @@ from afd_plugin.compat.npu import ensure_afd_ascend_ops_loaded ensure_afd_ascend_ops_loaded() ``` +Prepare native and AFD `ASCEND_CUSTOM_OPP_PATH` vendor entries before launching +Python when using the layered operators. GE can initialize infer-shape +registrations while importing native modules; adding the AFD path afterward +cannot repair that process's missing registration. Preserve the CANN Python +and library paths from the activated environment. The +[standalone guide](../../docs/npu/STANDALONE_FFN_SERVICE.md#launch) shows the +paired vendor setup. + ## Current Scope A2E/E2A serve `CAMP2pAFDConnector`. `CAMAsyncAFDConnector` uses the plugin's 910C routed-only CAM operators; see the [operator contract](../../docs/design/npu/cam_async_routed_ops.md). +The standalone FFN entry point reuses this protocol and connector; see +[the standalone service guide](../../docs/npu/STANDALONE_FFN_SERVICE.md). `grouped_matmul_swiglu_quant_v2_layered` is a fused grouped-matmul + SiLU + dynamic-quantization CAM operator (A8W8 / A4W4 / diff --git a/docs/design/module/ffn_runtime.md b/docs/design/module/ffn_runtime.md index a8a9dc128..4b35c9074 100644 --- a/docs/design/module/ffn_runtime.md +++ b/docs/design/module/ffn_runtime.md @@ -6,6 +6,7 @@ owners: - "@hsliuustc0106" - "@jiangkuaixue123" primary_code_paths: + - "afd_plugin/ffn_faas/**" - "afd_plugin/v1/worker/ffn_metadata.py" - "afd_plugin/v1/worker/ffn_model_runner.py" - "afd_plugin/v1/worker/ffn_routed_experts_model_runner.py" @@ -26,6 +27,7 @@ depends_on: - "execution_platforms.md" - "compatibility_and_patches.md" validation_paths: + - "tests/unit/ffn_faas/test_*.py" - "tests/unit/model_executor/test_async_cam_layered.py" - "tests/npu/test_async_cam_layered_w8a8.py" - "tests/unit/v1/worker/test_ffn_metadata.py" @@ -33,12 +35,14 @@ validation_paths: - "tests/unit/v1/worker/test_ffn_routed_experts_model_runner.py" - "tests/unit/v1/worker/test_model_runner_v2.py" - "tests/unit/v1/worker/test_npu_runtime.py" + - "tests/unit/v1/worker/test_async_ffn_runtime.py" - "tests/unit/connectors/npu/test_async_cam_startup_wiring.py" - "tests/unit/connectors/npu/test_async_cam_startup.py" - "tests/unit/connectors/test_base.py" - "tests/unit/connectors/npu/test_async_cam_session.py" - "tests/unit/connectors/npu/test_async_cam.py" - "tests/npu/async_cam_runtime.py" + - "tests/npu/standalone_cam_attention.py" - "tests/e2e/models/deepseek_v4_flash/ffn_graph.py" - "tests/unit/compat/patches/test_engine_core.py" - "tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py" @@ -93,11 +97,63 @@ read-only `comm_args` property. ## Runtime selection -FFN is launched as a `vllm serve` process, and AFD config normalization selects +The traditional FFN is launched as a `vllm serve` process, and AFD config normalization selects its role-specific worker when `worker_cls="auto"`. It does not serve requests. Attention and FFN may be started in either order. Send API traffic only to Attention. +DeepSeek-V4 Flash per-channel W4A8 also has a standalone F entry point, +`python -m afd_plugin.ffn_faas serve`, implementing independent-service +ownership under [RFC #391](https://github.com/vllm-project/afd-plugin/issues/391). +It composes `CAMAsyncAFDConnector` and `AsyncNPUFFNModelRunner` without creating +a vLLM Worker, upstream ModelRunner, EngineCore, scheduler or KV cache. +Attention retains its existing model, connector and DBO path. Configuration, +parameter loading and quantization remain vLLM/vLLM-Ascend dependencies. + +The standalone Ascend CLI reuses vLLM engine argument definitions, parsing and +configuration construction. Existing AFD options remain in `additional_config`; +only host management and shutdown options are standalone-specific. The host +entry delegates parser creation to its selected platform, so CPU-only service +and host-loop imports remain independent of vLLM. + +The CPU-only host `FFNModelRunner` accepts an `FFNBackend` factory, retains +the backend before preparation, and owns the receiving thread and failure +boundary. `FFNService` manages host status and stop requests. Platform options +and CAM runtime composition live under `ffn_faas/platforms/npu/`. + +- `AscendFFNEnvironment` owns device initialization, F-only compute/EP groups + and native post-warmup CPU affinity. These groups coexist with the separate + A/F CAM group throughout execution. +- `CAMRoutedModel` manages parameter loading, local placement checks, storage + accounting and cached-view lifetime. Startup selects a `RoutedModelAdapter` + to provide checkpoint metadata and native owner construction. The backend + uses the common resource without model-specific branching. +- Model-specific code follows `ffn_faas/platforms/ffn_model///`. + Under `deepseek_v4/npu/`, `v4_model.py` contains V4 checkpoint metadata and + `V4ModelAdapter`. `v4_weights.py` contains the lazily imported native parameter + owner. Native routed modules, expert-map managers and quantizers remain + resident. The loader adapter delegates to the native V4 loader. Dependency + upgrades must preserve its documented receiver and factory contracts. +- Platform settings enforce static CAM placement and reject EPLB, elastic EP + and redundant experts. Retaining native expert state does not enable these + algorithms. + +`FFNBackend` is a runtime lifecycle interface. Its `execute_step()` delegates +communication, computation and Graph execution to the Async CAM +runtime. A shared cross-platform, cross-connector Step Loop and a pure +`compute(item)` backend are not implemented. The runtime consumes the existing +`LayeredFFNWeights` interface with W4A8 weights, without per-step model selection; +there is no +separate V4 execution runtime. + +`attach_model` lends independently loaded parameters to the runtime. After +receiver exit and device completion, shutdown releases CAM/Graph resources, +then native compute groups, and finally the routed owner. Failed or pending +native work retains all owners until whole-pair process isolation. A host +`/stop` request cannot cancel an outstanding CAM DR. +See the [standalone service guide](../../npu/STANDALONE_FFN_SERVICE.md) for +supported configuration, launch and shutdown behavior. + | Platform | Worker | Model runner | Current connectors | | --- | --- | --- | --- | | CUDA | `afd_plugin.v1.worker.AFDFFNWorker` | `GPUFFNModelRunner` | `P2pNcclAFDConnector` | diff --git a/docs/design/module/index.md b/docs/design/module/index.md index 80a302433..baba6dc12 100644 --- a/docs/design/module/index.md +++ b/docs/design/module/index.md @@ -86,7 +86,7 @@ contract. File-level entries deliberately resolve mixed directories. | --- | --- | | [Plugin boundary](plugin_boundary.md) | `afd_plugin/__init__.py`, `afd_plugin/config.py`, `afd_plugin/config_utils.py`, `afd_plugin/envs.py`, `afd_plugin/validation.py`, `afd_plugin/py.typed`, `afd_plugin/v1/__init__.py`, `afd_plugin/v1/worker/__init__.py`, `afd_plugin/v1/worker/npu/__init__.py`, `pyproject.toml` | | [Attention runtime](attention_runtime.md) | `afd_plugin/v1/worker/attention_metadata.py`, `afd_plugin/v1/worker/attention_model_runner_v2.py`, `afd_plugin/v1/worker/attention_worker.py`, `afd_plugin/v1/worker/ubatch_wrapper.py`, `afd_plugin/v1/worker/npu/attention_model_runner_v2.py`, `afd_plugin/v1/worker/npu/attention_worker.py`, `afd_plugin/v1/worker/npu/dspark_speculator.py` | -| [FFN runtime](ffn_runtime.md) | `afd_plugin/v1/worker/ffn_metadata.py`, `afd_plugin/v1/worker/ffn_model_runner.py`, `afd_plugin/v1/worker/ffn_routed_experts_model_runner.py`, `afd_plugin/v1/worker/ffn_worker.py`, `afd_plugin/v1/worker/npu/ffn_model_runner.py`, `afd_plugin/v1/worker/npu/async_ffn_model_runner.py`, `afd_plugin/v1/worker/npu/ffn_worker.py` | +| [FFN runtime](ffn_runtime.md) | `afd_plugin/ffn_faas/**`, `afd_plugin/v1/worker/ffn_metadata.py`, `afd_plugin/v1/worker/ffn_model_runner.py`, `afd_plugin/v1/worker/ffn_routed_experts_model_runner.py`, `afd_plugin/v1/worker/ffn_worker.py`, `afd_plugin/v1/worker/npu/ffn_model_runner.py`, `afd_plugin/v1/worker/npu/async_ffn_model_runner.py`, `afd_plugin/v1/worker/npu/ffn_worker.py` | | [Connector contracts](connector_contracts.md) | `afd_plugin/connectors/**/*.py`, `afd_plugin/connectors/npu/bin/**`, `afd_plugin/distributed/**/*.py` | | [Model integration](model_integration.md) | `afd_plugin/model_executor/**/*.py` | | [Execution platforms](execution_platforms.md) | `afd_plugin/compat/profiler.py`, `afd_plugin/compat/npu/forward_context.py`, `afd_plugin/compat/npu/ops.py`, `afd_plugin/compat/npu/profiler.py`, `afd_plugin/v1/worker/cuda_graph.py`, `afd_plugin/v1/worker/dbo.py`, `afd_plugin/v1/worker/npu/mla_graph.py`, `afd_plugin/v1/worker/npu/ubatch_runner.py`, `csrc/**`, `setup.py`, `MANIFEST.in` | diff --git a/docs/npu/STANDALONE_FFN_SERVICE.md b/docs/npu/STANDALONE_FFN_SERVICE.md new file mode 100644 index 000000000..56cbc8d09 --- /dev/null +++ b/docs/npu/STANDALONE_FFN_SERVICE.md @@ -0,0 +1,151 @@ +# Standalone CAM FFN service + +Run DeepSeek-V4 Flash routed experts in independent F-only processes with +`python -m afd_plugin.ffn_faas serve`. The service reuses native parameter +loading and the existing CAM execution path without constructing a complete +CausalLM, vLLM Worker, upstream ModelRunner, EngineCore, scheduler or KV cache. + +## Supported configuration + +- DeepSeek-V4 Flash with homogeneous per-channel ModelSlim W4A8 on A3/910C. +- Static expert parallelism with F TP=1. EPLB, elastic EP and redundant experts + are unsupported. +- Main-model routed MoE layers only. MTP layers, gates, shared experts and + Attention weights are not loaded by F. +- The existing Attention model, router, shared experts, padding, DBO hooks and + `CAMAsyncAFDConnector` are reused. F supports layered eager or FULL Graph + execution. + +Configuration, expert mapping, parameter loading and quantization depend on +vLLM/vLLM-Ascend. Other models, quantization schemes and device backends are +not supported by this entry point. + +## Architecture + +```text +CPU Service / host runner -> FFNBackend + Ascend CAM backend + AscendFFNEnvironment: device, F-only compute groups, CPU affinity + CAMRoutedModel: parameter loading, native owner and cached views + V4ModelAdapter / V4RoutedWeightOwner: V4 metadata and native loading + Async CAM runtime: A/F group, DS/DR/CS/CR, Graph and streams +``` + +The backend adapts the complete CAM runtime lifecycle. Communication, compute, +Graph execution remains in that runtime. See the +[FFN runtime design](../design/module/ffn_runtime.md#runtime-selection) for +module responsibilities and ownership. + +## Launch + +Keep the existing Attention launch and its `CAMAsyncAFDConnector` configuration. +See the [connector guide](CAM_ASYNC_CONNECTOR_USER_GUIDE.md) and +[eager/FULL runtime guide](../design/npu/async_ffn_runtime.md) for its existing contract. +It must use eager MRV2, Attention-side gating and dynamic quantization. Native +Attention DBO uses the existing integration contract; do not enable DBO on F. + +For a single-node 12A/4F deployment, with A using logical devices 0–11 and a +matching dispatch envelope of 2048 tokens: + +```bash +# Activate the existing CANN environment and keep its Python/library paths. +# GE must see both native vendor paths before the first torch-npu import. +afd_cam_vendor=/path/to/afd_plugin/_cann_ops_custom/vendors/afd-plugin +ascend_native_vendor=/path/to/vllm_ascend/_cann_ops_custom/vendors/custom_transformer +export ASCEND_CUSTOM_OPP_PATH="$ascend_native_vendor:$afd_cam_vendor${ASCEND_CUSTOM_OPP_PATH:+:$ASCEND_CUSTOM_OPP_PATH}" +export VLLM_USE_V2_MODEL_RUNNER=1 +export SOC_VERSION=910c +export VLLM_PLUGINS=ascend,afd,ascend_model,ascend_model_loader,ascend_kv_connector +# Keep BATCH_SIZE_FACTOR, HCCL/network settings and receive capacity consistent +# with the existing A launch and CAM runtime guide. +export ASCEND_RT_VISIBLE_DEVICES=12,13,14,15 +torchrun --nproc-per-node=4 --master-addr=127.0.0.1 --master-port=29500 \ + -m afd_plugin.ffn_faas serve /path/to/model --backend ascend-cam \ + --trust-remote-code --dtype bfloat16 --quantization ascend \ + --enable-expert-parallel --data-parallel-size 4 \ + --max-num-batched-tokens 2048 --max-num-seqs 256 \ + --compilation-config '{"cudagraph_mode":"FULL"}' \ + --additional-config '{ + "afd": { + "connector":"CAMAsyncAFDConnector","role":"ffn","async_dp":true, + "host":"127.0.0.1","port":29600, + "num_attention_ranks":12,"num_ffn_ranks":4, + "compute_gate_on_attention":true, + "connector_extra_config":{ + "dynamicQuant":1,"attn_ranks_per_dp":4 + } + }, + "enable_prefill_mc2":false,"enable_force_eplb":false,"enable_cpu_binding":true, + "ascend_fusion_config":{"fusion_ops_gmmswigluquant":true} + }' --status-port 29700 +``` + +Engine arguments use vLLM's `FlexibleArgumentParser`, `EngineArgs.add_cli_args` +and `EngineArgs.from_cli_args`; definitions, JSON/dotted options, defaults and +configuration validation come from the pinned upstream stack. The selected +Ascend CLI imports that layer, while importing the Service, host runner or +platform adapter module remains CPU-only. No serving engine is constructed. + +Use `--additional-config` for the existing AFD topology, connector and HCCL +options. Engine capacities and compilation are passed through +without rebuilding a second configuration. Only backend selection, management +endpoints, status files and shutdown control add standalone options. The old +`--full-graph`, `--attention-ranks`, `--attention-tp`, `--ffn-ranks`, `--devices` +and standalone stream/budget flags have been removed; update launch scripts to +the standard options above. Device selection uses the native `--device-ids` +mapping or `ASCEND_RT_VISIBLE_DEVICES` plus the launcher's `LOCAL_RANK`. + +The external launcher's `WORLD_SIZE`, `--data-parallel-size` and +`additional_config.afd.num_ffn_ranks` must agree. An omitted +`--data-parallel-rank` is taken from `RANK`; an explicit value must match it. +The executor defaults to `external_launcher` because torchrun already owns +these processes. Set `VLLM_USE_V2_MODEL_RUNNER=1` as on A. All other engine +options retain upstream defaults; the example explicitly selects the qualified +W4A8 profile. Reusing the parser does not qualify additional models, speculative +loading or serving/KV features for this service. + +The host receive-loop join uses `--receiver-shutdown-timeout` (default: five +seconds). It is separate from vLLM's `--shutdown-timeout`, whose EngineCore +shutdown semantics do not govern this standalone receiver. + +CAM creates the separate `[A0 ... A11, F0 ... F3]` world using the same host, +port, group name, topology, quantization, layer table and startup signatures as +traditional F. Use a different rendezvous port for the F-only compute world +and CAM. + +`--max-num-batched-tokens` must match A's dispatch envelope; it is not the +maximum request context length. F allocates no request KV cache. To use eager, +replace `--compilation-config` with `--enforce-eager`. FULL preserves paired +warmup, private graph pools and device layer selection. FFN two-stream +execution and its core-budget options were removed in #550; older launch +commands must omit those options. + +## Management and shutdown + +Each local F rank exposes a management endpoint on `--status-port + LOCAL_RANK`. +`--status-port 0` chooses an ephemeral port. + +- `/status` reports loading, CAM pairing, phase, completed submission batches + and resource-release state. Batches are not requests or full model forwards. +- `/health` reports host liveness. `/ready` becomes available after native + weights and paired CAM startup are prepared. Neither proves progress of an + outstanding device receive. +- `POST /stop`, SIGINT and SIGTERM request a stop between completed batches. + The host joins its receive loop for `--receiver-shutdown-timeout` seconds. + +The shutdown timeout does not cancel a CAM operator or prove device quiescence. +It applies after startup; native loading and rendezvous retain their existing +timeouts. CAM pairing follows native weight loading. An unresponsive startup +requires whole-deployment isolation. + +After receiver exit and device completion, the service releases CAM/Graph +resources, then native compute groups, and finally the routed parameter owner. +If native work fails or a receive remains pending, it reports failure with +`resources_released=false` and retains those owners until an immediate nonzero +process exit, bypassing Python destructors. Diagnostic and HTTP-cleanup errors +cannot bypass this exit. + +The operator must tear down the entire paired A/F deployment after such a +failure. A completed client request does not establish that the next +pre-submitted receive has finished. Graceful bounded drain, independent hot +restart and device cancellation are unsupported. diff --git a/tests/npu/async_cam_runtime.py b/tests/npu/async_cam_runtime.py index 98bf25799..74c7e3c59 100644 --- a/tests/npu/async_cam_runtime.py +++ b/tests/npu/async_cam_runtime.py @@ -296,16 +296,19 @@ def main(): print(json.dumps({"rank": rank, "stage": "connector_created"}), flush=True) if is_ffn: runner = AsyncNPUFFNModelRunner(config, device, connector) - runner.model = SyntheticModel( - device, - not args.per_group, - args.hidden_size, - args.intermediate_size, - None if args.weight_seed is None else args.weight_seed + rank * 10000, - args.routed_scale, - args.quantization, - args.swiglu_limit, - args.experts_per_rank, + runner.attach_model( + SyntheticModel( + device, + not args.per_group, + args.hidden_size, + args.intermediate_size, + None if args.weight_seed is None else args.weight_seed + rank * 10000, + args.routed_scale, + args.quantization, + args.swiglu_limit, + args.experts_per_rank, + ), + parameter_bytes=0, ) print(json.dumps({"rank": rank, "stage": "weights_ready"}), flush=True) runner.initialize_afd_connector() @@ -530,7 +533,7 @@ def finish_receiver(): ) AFDNPUFFNWorker._run_ffn_server_loop(worker) report["idle_shutdown"] = "PASS" - runner._graph_receiver_exited = True + runner.mark_receiver_exited() def serve(): nonlocal batch_index, data_replays diff --git a/tests/npu/standalone_cam_attention.py b/tests/npu/standalone_cam_attention.py new file mode 100644 index 000000000..38bcd63e1 --- /dev/null +++ b/tests/npu/standalone_cam_attention.py @@ -0,0 +1,192 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""One real CAM A peer for a separately launched routed-only FFN service. + +This finite functional check uses the original checkpoint on F and every main +MoE layer. It has no Attention model, KV cache, HTTP inference or performance +claim. Run an eager F service first with --output, then FULL F with +--reference pointing to that output. Terminate the whole paired deployment +after this client: the production service has no EOS or cancellable DR. +""" + +from __future__ import annotations + +import argparse +import json +import threading +from pathlib import Path +from types import SimpleNamespace + +from vllm_ascend.utils import bootstrap_custom_op_env + +bootstrap_custom_op_env() + +import torch # noqa: E402 +import torch_npu # noqa: E402, F401 + +from afd_plugin.compat.npu.ops import ensure_cam_async_ops_available # noqa: E402 +from afd_plugin.config import parse_afd_config # noqa: E402 +from afd_plugin.connectors.metadata import ( # noqa: E402 + AFDTransferContext, + AFDTransferMetadata, +) +from afd_plugin.connectors.npu.async_cam import CAMAsyncAFDConnector # noqa: E402 +from afd_plugin.ffn_faas.platforms.ffn_model.deepseek_v4.npu.v4_model import ( # noqa: E402 + V4CheckpointDescription, +) + + +def attention_config(checkpoint, args): + additional = { + "afd": { + "connector": "CAMAsyncAFDConnector", + "role": "attention", + "async_dp": True, + "host": "127.0.0.1", + "port": args.port, + "num_attention_ranks": 1, + "num_ffn_ranks": args.ffn_ranks, + "compute_gate_on_attention": True, + "connector_extra_config": { + "dynamicQuant": 1, + "attn_ranks_per_dp": 1, + "hccl_buffer_size": 64, + }, + }, + "enable_prefill_mc2": False, + "enable_force_eplb": False, + "enable_cpu_binding": True, + "ascend_fusion_config": {"fusion_ops_gmmswigluquant": True}, + } + return SimpleNamespace( + additional_config=additional, + model_config=SimpleNamespace( + dtype=torch.bfloat16, + enforce_eager=True, + hf_config=SimpleNamespace( + hidden_size=checkpoint.hidden_size, + num_experts_per_tok=checkpoint.top_k, + n_routed_experts=checkpoint.num_experts, + num_hidden_layers=checkpoint.model_layers, + architectures=["DeepseekV4ForCausalLM"], + model_type="deepseek_v4", + ), + ), + scheduler_config=SimpleNamespace(max_num_batched_tokens=32), + parallel_config=SimpleNamespace( + enable_dbo=False, + use_ubatching=False, + num_ubatches=2, + data_parallel_size=1, + tensor_parallel_size=1, + pipeline_parallel_size=1, + prefill_context_parallel_size=1, + decode_context_parallel_size=1, + enable_expert_parallel=True, + enable_elastic_ep=False, + enable_eplb=False, + use_sequence_parallel_moe=False, + ), + speculative_config=None, + use_v2_model_runner=True, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("model", type=Path) + parser.add_argument("--device", type=int, default=0) + parser.add_argument("--ffn-ranks", type=int, default=4) + parser.add_argument("--port", type=int, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--reference", type=Path) + parser.add_argument( + "--hold", action="store_true", help="retain CAM owners until controller exit" + ) + args = parser.parse_args() + checkpoint = V4CheckpointDescription.read(args.model) + config = attention_config(checkpoint, args) + args.output.parent.mkdir(parents=True, exist_ok=True) + ensure_cam_async_ops_available() + torch.npu.set_device(args.device) + connector = CAMAsyncAFDConnector( + 0, args.device, config, parse_afd_config(config), role_rank=0 + ) + records: list[dict[str, int]] = [] + report = { + "phase": "connecting", + "layers": records, + "performance_measured": False, + } + try: + connector.init_afd_connector() + report["mode"] = connector.graph_startup.mode + references = ( + torch.load(args.reference, weights_only=True) + if args.reference is not None + else None + ) + outputs = [] + # Every layer is selected by the existing device header. Alternating + # routes include zero local rows on other F ranks and capacity tails. + for index, layer in enumerate( + (*checkpoint.layer_indices, checkpoint.layer_indices[0]) + ): + tokens = (1, 7, 17)[layer % 3] + inputs = ( + torch.linspace(-2, 2, checkpoint.hidden_size) + .sin() + .repeat(tokens, 1) + .to(device=f"npu:{args.device}", dtype=torch.bfloat16) + ) + stride = checkpoint.num_experts // checkpoint.top_k + ids = torch.arange(checkpoint.top_k, dtype=torch.int32) + if layer % 2: + ids = (ids * stride + layer) % checkpoint.num_experts + ids = ids.repeat(tokens, 1).to(inputs.device) + weights = torch.full( + (tokens, checkpoint.top_k), + checkpoint.routed_scaling_factor / checkpoint.top_k, + device=inputs.device, + ) + context = AFDTransferContext(AFDTransferMetadata(layer, 0, [tokens])) + with torch.inference_mode(): + connector.send_attn_output( + inputs, context, topk_ids=ids, topk_weights=weights + ) + result = connector.recv_ffn_output(inputs, context=context) + done = torch.npu.Event() + done.record() + connector.wait_graph_device(done) + actual = result.cpu() + assert torch.isfinite(actual).all(), f"nonfinite layer {layer}" + if references is not None: + torch.testing.assert_close(actual, references[layer], rtol=0, atol=0) + if index < checkpoint.model_layers: + outputs.append(actual) + records.append({"layer": layer, "tokens": tokens}) + else: + # This extra completed transaction proves every F entered its + # next fixed-size batch. With no more A work, its remaining DRs + # stay pending: stop must retain owners, rather than race a + # legitimate exit at the preceding completed-batch boundary. + report["pending_batch_probe"] = {"layer": layer, "tokens": tokens} + if references is not None: + assert len(references) == len(outputs) + torch.save(outputs, args.output) + report.update(phase="passed", exact_eager_match=references is not None) + except BaseException as error: + report.update(phase="failed", error=repr(error)) + raise + finally: + # Do not close CAM while F may already have its next DR queued. The + # test controller tears down both roles together, just as deployment + # process isolation must do after a pending production receive. + args.output.with_suffix(".json").write_text(json.dumps(report, indent=2)) + print(json.dumps(report), flush=True) + if args.hold: + threading.Event().wait() + + +if __name__ == "__main__": + main() diff --git a/tests/unit/fakes/ffn_faas.py b/tests/unit/fakes/ffn_faas.py new file mode 100644 index 000000000..9c68a9232 --- /dev/null +++ b/tests/unit/fakes/ffn_faas.py @@ -0,0 +1,270 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Metadata fixtures and host-only lifecycle stand-ins for standalone FFN.""" + +from __future__ import annotations + +import atexit +import json +import sys +import threading +from argparse import ArgumentParser, Namespace +from contextlib import ExitStack +from dataclasses import dataclass +from pathlib import Path +from unittest.mock import patch + +from afd_plugin.ffn_faas.interfaces import FFNLaunch +from afd_plugin.ffn_faas.service import FFNService, ServicePhase + + +@dataclass(frozen=True) +class MetadataModelAdapter: + """CPU loading-boundary facts, never a qualified model or device stand-in.""" + + root: Path + layer_indices: tuple[int, ...] = (2, 7) + num_experts: int = 4 + + def estimated_parameter_bytes(self, local_experts: int) -> int: + return 1024 * local_experts + + def local_sources(self, layer_index, owned): + raise RuntimeError("metadata-only adapter has no checkpoint tensors") + + def create_owner(self, config): + raise RuntimeError("metadata-only adapter has no native parameter owner") + + +def write_v4_checkpoint(root: Path, *, layers: int = 2, experts: int = 4) -> Path: + root.mkdir(parents=True, exist_ok=True) + config = { + "architectures": ["DeepseekV4ForCausalLM"], + "hidden_act": "silu", + "scoring_func": "sqrtsoftplus", + "torch_dtype": "bfloat16", + "num_hidden_layers": layers, + "n_routed_experts": experts, + "num_experts_per_tok": 2, + "num_hash_layers": 1, + "hidden_size": 256, + "moe_intermediate_size": 256, + "routed_scaling_factor": 1.5, + "swiglu_limit": 10.0, + } + shard = "weights.safetensors" + weights = { + f"layers.{layer}.ffn.experts.{expert}.w{projection}.{suffix}": shard + for layer in range(layers) + for expert in range(experts) + for projection in (1, 2, 3) + for suffix in ("weight", "weight_scale", "weight_offset", "scale_bias") + } + (root / "config.json").write_text(json.dumps(config)) + (root / "quant_model_weights.safetensors.index.json").write_text( + json.dumps({"weight_map": weights}) + ) + (root / "quant_model_description.json").write_text( + json.dumps({"group_size": 0, **dict.fromkeys(weights, "W4A8_DYNAMIC")}) + ) + # Metadata tests never pass this placeholder to the native loader. + (root / "weights.safetensors").write_bytes(b"metadata-only fixture") + return root + + +class HostBackend: + """Exercise host ownership rules, without simulating device completion.""" + + def __init__( + self, *, stop_during_prepare: bool = False, fail: str = "", block: bool = False + ) -> None: + self.stop_during_prepare = stop_during_prepare + self.fail = fail + self.block = block + self.entered = threading.Event() + self.release = threading.Event() + self.finished = threading.Event() + self.released = False + self.service: FFNService | None = None + + def prepare(self, service: FFNService) -> None: + self.service = service + if self.fail == "prepare": + raise RuntimeError("prepare failed") + service.phase(ServicePhase.CONNECTING, model_loaded=True) + if self.stop_during_prepare: + service.request_stop() + + def start_receiver(self) -> None: + if self.fail == "activate": + raise RuntimeError("receiver activation failed") + + def execute_step(self) -> None: + assert self.service is not None + self.entered.set() + if self.block: + self.release.wait() + if self.fail == "execute": + self.finished.set() + raise RuntimeError("execute failed") + self.service.request_stop() + + def receiver_exited(self) -> None: + self.finished.set() + + def close(self) -> None: + if not self.finished.is_set(): + raise RuntimeError("host receiver has not exited") + if self.fail == "close": + raise RuntimeError("host-only resource release failed") + self.released = True + + +class HostPlatformAdapter: + """An independent CPU host contract; it has no model/device requirements.""" + + def create_parser(self) -> ArgumentParser: + return ArgumentParser() + + def configure(self, args) -> FFNLaunch: + return FFNLaunch( + rank=3, + local_rank=0, + model_path=str(args.model_tag), + backend_name="host-contract", + shutdown_timeout=args.receiver_shutdown_timeout, + backend_factory=HostBackend, + ) + + +def run_cpu_platform(model: str, status: str) -> int: + """Run the actual service/loop after the child entry installed its import gate.""" + from afd_plugin.ffn_faas import __main__ as cli + + args = [ + "ffn_faas", + "serve", + model, + "--backend", + "host-contract", + "--status-port", + "0", + "--status-file", + status, + "--receiver-shutdown-timeout", + "0.1", + ] + with ( + patch.object(cli, "SUPPORTED_BACKENDS", ("host-contract",)), + patch.object(cli, "get_platform_adapter", lambda _: HostPlatformAdapter()), + patch.object(sys, "argv", args), + ): + return cli.main() + + +def run_cli_host_case(model: str, marker: str, fault: str, released: bool) -> int: + """Run the real CLI exit path in a child; no native backend is constructed. + + The atexit marker distinguishes normal interpreter teardown from os._exit. + Faults start only after the host-only runner has reported its outcome. + """ + from afd_plugin.ffn_faas import __main__ as cli + + atexit.register(lambda: Path(marker).write_text("ordinary shutdown")) + args = [ + "ffn_faas", + "serve", + model, + "--backend", + "host-contract", + "--status-port", + "0", + ] + if fault == "status_file": + blocked = Path(marker).with_name("blocked-parent") + blocked.write_text("not a directory") + args += ["--status-file", str(blocked / "status.json")] + + with ExitStack() as patches: + + class FailedOutput: + def __init__(self, original, operation): + self.original, self.operation = original, operation + + def write(self, text): + if self.operation == "write": + raise BrokenPipeError("host-only reporting fault") + return self.original.write(text) + + def flush(self): + if self.operation == "flush": + raise BrokenPipeError("host-only flush fault") + return self.original.flush() + + class HostService(FFNService): + def snapshot(self) -> dict[str, str | int | bool]: + if fault == "snapshot": + raise OSError("host-only snapshot fault") + return super().snapshot() + + def close_http(self) -> None: + super().close_http() + if fault == "close_http": + raise OSError("host-only HTTP close fault") + + class HostRunner: + def __init__(self, service, backend_factory, *, shutdown_timeout): + self.service = service + + def run(self): + if released: + self.service.phase(ServicePhase.STOPPED, resources_released=True) + else: + self.service.phase(ServicePhase.FAILED, error="host-only failure") + if fault.startswith(("stdout_", "stderr_")): + stream_name, operation = fault.split("_", 1) + original = sys.stdout if stream_name == "stdout" else sys.stderr + patches.enter_context( + patch.object( + sys, stream_name, FailedOutput(original, operation) + ) + ) + if not released and fault != "returned_unreleased": + raise RuntimeError("host-only unreleased session") + + patches.enter_context( + patch.object(cli, "SUPPORTED_BACKENDS", ("host-contract",)) + ) + patches.enter_context( + patch.object(cli, "get_platform_adapter", lambda _: HostPlatformAdapter()) + ) + patches.enter_context(patch.object(cli, "FFNService", HostService)) + patches.enter_context(patch.object(cli, "FFNModelRunner", HostRunner)) + patches.enter_context(patch.object(sys, "argv", args)) + return cli.main() + + +def cam_cli_args(model: Path, **overrides) -> Namespace: + """Parsed engine arguments for CPU-only standalone launch admission tests.""" + values = { + "model": str(model), + "model_tag": None, + "data_parallel_size": 2, + "data_parallel_rank": None, + "distributed_executor_backend": "external_launcher", + "receiver_shutdown_timeout": 5.0, + "additional_config": { + "enable_cpu_binding": False, + "afd": { + "connector": "CAMAsyncAFDConnector", + "role": "ffn", + "async_dp": True, + "compute_gate_on_attention": True, + "num_attention_ranks": 12, + "num_ffn_ranks": 2, + "connector_extra_config": {"dynamicQuant": 1, "attn_ranks_per_dp": 4}, + }, + }, + } + values.update(overrides) + return Namespace(**values) diff --git a/tests/unit/ffn_faas/conftest.py b/tests/unit/ffn_faas/conftest.py new file mode 100644 index 000000000..d9b2cad2a --- /dev/null +++ b/tests/unit/ffn_faas/conftest.py @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Standalone FFN CPU fixtures.""" + +from __future__ import annotations + +import pytest + +from tests.unit.fakes.ffn_faas import write_v4_checkpoint + + +@pytest.fixture +def checkpoint_dir(tmp_path): + return write_v4_checkpoint(tmp_path / "model") diff --git a/tests/unit/ffn_faas/platforms/ffn_model/deepseek_v4/npu/test_v4_model.py b/tests/unit/ffn_faas/platforms/ffn_model/deepseek_v4/npu/test_v4_model.py new file mode 100644 index 000000000..f642349cc --- /dev/null +++ b/tests/unit/ffn_faas/platforms/ffn_model/deepseek_v4/npu/test_v4_model.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Native routed checkpoint metadata and contiguous placement contracts.""" + +from __future__ import annotations + +import json + +import pytest + +from afd_plugin.ffn_faas.platforms.ffn_model.deepseek_v4.npu.v4_model import ( + V4CheckpointDescription, + V4ModelAdapter, +) +from afd_plugin.ffn_faas.platforms.npu import get_model +from afd_plugin.ffn_faas.platforms.npu.environment import AscendFFNEnvironment +from afd_plugin.ffn_faas.platforms.npu.settings import ( + FFNConfig, +) +from tests.unit.fakes.vllm_config import npu_vllm_config + + +def test_main_routed_metadata_without_parameter_allocation(checkpoint_dir): + checkpoint = V4CheckpointDescription.read(checkpoint_dir) + assert checkpoint.layer_indices == (0, 1) + assert len(checkpoint.routed_weight_map) == 2 * 4 * 3 * 4 + model = get_model(checkpoint_dir, ffn_ranks=2, ffn_rank=1) + assert model.get_experts_layer_indices() == (0, 1) + assert model.parameter_bytes > 0 + with pytest.raises(RuntimeError, match="not been loaded"): + _ = model.native_view + + +@pytest.mark.parametrize( + "name,value", + [ + ("hidden_act", "gelu"), + ("swiglu_limit", -1), + ("hidden_size", 0), + ("num_experts_per_tok", 5), + ("num_hash_layers", 3), + ], +) +def test_incompatible_model_metadata_rejected(checkpoint_dir, name, value): + config_path = checkpoint_dir / "config.json" + config = json.loads(config_path.read_text()) + config[name] = value + config_path.write_text(json.dumps(config)) + with pytest.raises(ValueError): + V4CheckpointDescription.read(checkpoint_dir) + + +def test_missing_routed_source_rejected_before_loading(checkpoint_dir): + index_path = checkpoint_dir / "quant_model_weights.safetensors.index.json" + index = json.loads(index_path.read_text()) + del index["weight_map"]["layers.1.ffn.experts.3.w3.scale_bias"] + index_path.write_text(json.dumps(index)) + with pytest.raises(ValueError, match="missing"): + V4CheckpointDescription.read(checkpoint_dir) + + +def test_v4_adapter_selects_only_local_checkpoint_sources(checkpoint_dir): + adapter = V4ModelAdapter(V4CheckpointDescription.read(checkpoint_dir)) + sources = adapter.local_sources(1, frozenset({2, 3})) + assert len(sources) == 24 + assert ("layers.1.ffn.experts.2.w1.weight_scale", "weights.safetensors") in sources + assert all(name.startswith("layers.1.ffn.experts.") for name, _ in sources) + assert not any("experts.0." in name or "experts.1." in name for name, _ in sources) + + +@pytest.mark.parametrize("architecture", ["DeepseekV3ForCausalLM", "UnknownModel"]) +def test_model_selection_rejects_unqualified_architectures( + checkpoint_dir, architecture +): + config_path = checkpoint_dir / "config.json" + config = json.loads(config_path.read_text()) + config["architectures"] = [architecture] + config_path.write_text(json.dumps(config)) + with pytest.raises(ValueError, match="DeepseekV4ForCausalLM"): + get_model(checkpoint_dir, ffn_ranks=2, ffn_rank=0) + + +def test_model_selection_rejects_other_quantization_before_loading(checkpoint_dir): + quant_path = checkpoint_dir / "quant_model_description.json" + quant = json.loads(quant_path.read_text()) + quant["group_size"] = 128 + quant_path.write_text(json.dumps(quant)) + with pytest.raises(ValueError, match="per-channel"): + get_model(checkpoint_dir, ffn_ranks=2, ffn_rank=0) + + +def test_wrong_compute_world_rejected_before_native_imports( + checkpoint_dir, monkeypatch +): + settings = FFNConfig(checkpoint_dir, ffn_ranks=2) + environment = AscendFFNEnvironment(settings) + monkeypatch.setenv("RANK", "0") + monkeypatch.setenv("WORLD_SIZE", "1") + with pytest.raises(ValueError, match="differs from the F-only"): + environment.initialize_device(npu_vllm_config(role="ffn")) + + +def test_unused_compute_owner_can_close_without_importing_runtime(checkpoint_dir): + settings = FFNConfig(checkpoint_dir, ffn_ranks=1) + environment = AscendFFNEnvironment(settings) + environment.close() + with pytest.raises(RuntimeError, match="device has not been initialized"): + _ = environment.device + with pytest.raises(RuntimeError, match="expert group has not been initialized"): + _ = environment.ep_group diff --git a/tests/unit/ffn_faas/test_main.py b/tests/unit/ffn_faas/test_main.py new file mode 100644 index 000000000..07d483b4f --- /dev/null +++ b/tests/unit/ffn_faas/test_main.py @@ -0,0 +1,275 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Real child-process exit checks; these never construct a device backend.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + + +@pytest.fixture +def cli_case(checkpoint_dir, tmp_path): + def run(fault, released=False): + marker = tmp_path / "normal-shutdown" + script = ( + "import sys; from tests.unit.fakes.ffn_faas import run_cli_host_case; " + "raise SystemExit(run_cli_host_case(*sys.argv[1:4], " + "released=sys.argv[4]=='1'))" + ) + result = subprocess.run( + [ + sys.executable, + "-c", + script, + str(checkpoint_dir), + str(marker), + fault, + "1" if released else "0", + ], + cwd=Path(__file__).resolve().parents[3], + capture_output=True, + text=True, + timeout=15, + ) + return result, marker + + return run + + +@pytest.mark.parametrize( + "fault", + [ + "none", + "status_file", + "stdout_write", + "stdout_flush", + "stderr_write", + "stderr_flush", + "close_http", + "snapshot", + "returned_unreleased", + ], +) +def test_unreleased_cli_isolates_despite_reporting_failures(cli_case, fault): + result, marker = cli_case(fault) + assert result.returncode == 1, result.stderr + assert not marker.exists(), "unsafe session used normal interpreter teardown" + + +@pytest.mark.parametrize("fault", ["none", "status_file", "close_http"]) +def test_released_cli_keeps_normal_shutdown(cli_case, fault): + result, marker = cli_case(fault, released=True) + assert result.returncode == (0 if fault == "none" else 1), result.stderr + assert marker.read_text() == "ordinary shutdown" + + +@pytest.fixture +def cpu_platform_case(tmp_path): + status = tmp_path / "status.json" + model = tmp_path / "no-checkpoint-required" + # Install before importing even the fake: its module imports interfaces/service. + script = """ +import builtins +import importlib +import importlib.util +import runpy +import sys + +BLOCKED = ("torch", "torch_npu", "vllm", "vllm_ascend") +assert not any(name.split(".", 1)[0] in BLOCKED for name in sys.modules) + +ORIGINAL_IMPORT = builtins.__import__ +ORIGINAL_IMPORT_MODULE = importlib.import_module + +def require_cpu_import(name): + if name.split(".", 1)[0] in BLOCKED: + raise AssertionError(f"CPU host contract imported {name}") + +def cpu_import(name, *args, **kwargs): + require_cpu_import(name) + return ORIGINAL_IMPORT(name, *args, **kwargs) + +def cpu_import_module(name, package=None): + require_cpu_import(importlib.util.resolve_name(name, package)) + return ORIGINAL_IMPORT_MODULE(name, package) + +# Allow find_spec capability queries; block ordinary and dynamic module imports. +builtins.__import__ = cpu_import +importlib.import_module = cpu_import_module +if sys.argv[3]: + runpy.run_path(sys.argv[3]) +from tests.unit.fakes.ffn_faas import run_cpu_platform +raise SystemExit(run_cpu_platform(*sys.argv[1:3])) +""" + + def run(probe=None): + result = subprocess.run( + [sys.executable, "-c", script, str(model), str(status), str(probe or "")], + cwd=Path(__file__).resolve().parents[3], + capture_output=True, + text=True, + timeout=15, + ) + return result, model, status + + return run + + +def test_real_cli_accepts_an_independent_cpu_adapter(cpu_platform_case): + result, model, status = cpu_platform_case() + assert result.returncode == 0, result.stderr + snapshot = json.loads(status.with_name("status-rank-3.json").read_text()) + assert snapshot["backend"] == "host-contract" + assert "cam_paired" not in snapshot + assert snapshot["model_path"] == str(model) + assert snapshot["phase"] == "stopped" + assert snapshot["resources_released"] is True + + +def test_supported_model_selection_keeps_cpu_import_boundary( + cpu_platform_case, checkpoint_dir, tmp_path +): + probe = tmp_path / "select_model.py" + probe.write_text( + "from pathlib import Path\n" + "from afd_plugin.ffn_faas.platforms.npu import get_model\n" + f"model = get_model(Path({str(checkpoint_dir)!r}), 2, 1)\n" + "assert model.get_experts_layer_indices() == (0, 1)\n" + "assert model.parameter_bytes > 0\n" + "model.close()\n" + ) + result, _, _ = cpu_platform_case(probe) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + ("package", "statement"), + [ + ("torch", "import torch"), + ("torch_npu", "import torch_npu"), + ("vllm", "from vllm import config"), + ( + "vllm_ascend", + "from importlib import import_module; import_module('vllm_ascend.config')", + ), + ], +) +def test_cpu_entry_rejects_platform_imports( + cpu_platform_case, tmp_path, package, statement +): + # An importable stub ensures missing packages cannot make a broken gate pass. + stub = tmp_path / package + stub.mkdir() + (stub / "__init__.py").write_text("raise RuntimeError('platform stub ran')\n") + probe = tmp_path / "import_probe.py" + probe.write_text( + f"import importlib.util, sys\nsys.path.insert(0, {str(tmp_path)!r})\n" + f"assert importlib.util.find_spec({package!r}) is not None\n{statement}\n" + ) + result, _, _ = cpu_platform_case(probe) + assert result.returncode == 1 + assert f"CPU host contract imported {package}" in result.stderr + assert "platform stub ran" not in result.stderr + + +@pytest.fixture +def engine_parser(): + pytest.importorskip("vllm") + from afd_plugin.ffn_faas.__main__ import create_parser + from afd_plugin.ffn_faas.platforms.npu.adapter import AscendCAMAdapter + + return create_parser(AscendCAMAdapter()) + + +@pytest.fixture(params=[False, True], ids=["eager", "full"]) +def native_engine_case(engine_parser, request): + from vllm.engine.arg_utils import EngineArgs + + graph = request.param + config = { + "enable_cpu_binding": False, + "afd": { + "connector": "CAMAsyncAFDConnector", + "role": "ffn", + "connector_extra_config": {"hccl_buffer_size": 256}, + }, + } + mode = ( + ["--compilation-config", '{"cudagraph_mode":"FULL"}'] + if graph + else ["--enforce-eager"] + ) + parsed = engine_parser.parse_args( + [ + "serve", + "/model", + "--dtype", + "bfloat16", + "--max-num-batched-tokens", + "2K", + "--max-num-seqs", + "8", + "--data-parallel-size", + "4", + "--device-ids", + "4,5,6,7", + "--additional-config", + json.dumps(config), + "--shutdown-timeout", + "0", + "--receiver-shutdown-timeout", + "0.5", + *mode, + ] + ) + return ( + graph, + config, + EngineArgs.from_cli_args(parsed), + parsed.receiver_shutdown_timeout, + ) + + +def test_native_engine_options_preserve_configuration(native_engine_case): + graph, config, args, receiver_timeout = native_engine_case + assert args.max_num_batched_tokens == 2048 + assert args.max_num_seqs == 8 + assert args.dtype == "bfloat16" and args.data_parallel_size == 4 + assert args.device_ids == [4, 5, 6, 7] + assert args.additional_config == config + assert args.shutdown_timeout == 0 and receiver_timeout == 0.5 + assert args.enforce_eager is (not graph) + if graph: + assert args.compilation_config.cudagraph_mode.name == "FULL" + + +def test_native_dotted_json_options(engine_parser): + from vllm.engine.arg_utils import EngineArgs + + parsed = engine_parser.parse_args( + [ + "serve", + "/model", + "--additional-config.afd.role", + "ffn", + "--additional-config.afd.port", + "29600", + "--compilation-config.cudagraph_mode", + "FULL", + ] + ) + args = EngineArgs.from_cli_args(parsed) + assert args.additional_config == {"afd": {"role": "ffn", "port": 29600}} + assert args.compilation_config.cudagraph_mode.name == "FULL" + + +@pytest.mark.parametrize("flag", ["--full-graph", "--attention-ranks", "--devices"]) +def test_retired_standalone_engine_flags_are_rejected(engine_parser, flag): + with pytest.raises(SystemExit) as error: + engine_parser.parse_args(["serve", "/model", flag]) + assert error.value.code == 2 diff --git a/tests/unit/ffn_faas/test_model.py b/tests/unit/ffn_faas/test_model.py new file mode 100644 index 000000000..3621867c0 --- /dev/null +++ b/tests/unit/ffn_faas/test_model.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""CPU model-resource boundary; native loading is qualified on the NPU.""" + +from __future__ import annotations + +import pytest + +from afd_plugin.ffn_faas.platforms.npu.model import CAMRoutedModel +from tests.unit.fakes.ffn_faas import MetadataModelAdapter + + +def test_resource_uses_adapter_routed_layers_and_loading_estimate(tmp_path): + model = CAMRoutedModel(MetadataModelAdapter(tmp_path), ffn_ranks=2, ffn_rank=1) + assert model.root == tmp_path + assert model.get_experts_layer_indices() == (2, 7) + assert model.parameter_bytes == 2048 + with pytest.raises(RuntimeError, match="not been loaded"): + model.get_async_cam_layered_weights() + with pytest.raises(RuntimeError, match="not finished loading"): + _ = model.native_owner + model.close() + with pytest.raises(RuntimeError, match="not been loaded"): + _ = model.native_view + + +@pytest.mark.parametrize("world,rank", [(0, 0), (3, 0), (2, 2), (2, -1)]) +def test_resource_rejects_incompatible_f_topology_before_native_loading( + tmp_path, world, rank +): + with pytest.raises(ValueError): + CAMRoutedModel(MetadataModelAdapter(tmp_path), ffn_ranks=world, ffn_rank=rank) diff --git a/tests/unit/ffn_faas/test_runner.py b/tests/unit/ffn_faas/test_runner.py new file mode 100644 index 000000000..a3e475f8d --- /dev/null +++ b/tests/unit/ffn_faas/test_runner.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Host ownership checks; native receive/Graph behavior is device-qualified.""" + +from __future__ import annotations + +import threading + +import pytest + +from afd_plugin.ffn_faas.runner import FFNModelRunner +from afd_plugin.ffn_faas.service import FFNService +from tests.unit.fakes.ffn_faas import HostBackend + + +@pytest.fixture +def make_runner(tmp_path): + def build(backend): + service = FFNService(rank=0, model_path=str(tmp_path), backend_name="host-test") + return FFNModelRunner(service, lambda: backend, shutdown_timeout=0.01), service + + return build + + +@pytest.mark.parametrize("stop_during_prepare", [False, True]) +def test_successful_host_exit_releases_after_receiver_exit( + make_runner, stop_during_prepare +): + backend = HostBackend(stop_during_prepare=stop_during_prepare) + runner, service = make_runner(backend) + runner.run() + assert service.snapshot()["phase"] == "stopped" + assert service.snapshot()["resources_released"] is True + assert backend.released + + +@pytest.mark.parametrize("fail", ["prepare", "activate", "execute", "close"]) +def test_failed_native_boundary_retains_owners(make_runner, fail): + backend = HostBackend(fail=fail) + runner, service = make_runner(backend) + with pytest.raises(RuntimeError): + runner.run() + assert service.snapshot()["phase"] == "failed" + assert service.snapshot()["resources_released"] is False + assert not backend.released + + +def test_pending_host_receiver_is_not_closed(make_runner): + backend = HostBackend(block=True) + runner, service = make_runner(backend) + errors = [] + done = threading.Event() + + def run(): + try: + runner.run() + except RuntimeError as error: + errors.append(str(error)) + finally: + done.set() + + thread = threading.Thread(target=run) + thread.start() + try: + assert backend.entered.wait(timeout=5) + service.request_stop() + assert done.wait(timeout=5) + assert errors and "FFN receive is pending" in errors[0] + assert service.snapshot()["resources_released"] is False + assert not backend.released + finally: + backend.release.set() + thread.join() + assert backend.finished.wait(timeout=5) + + +def test_stop_before_backend_construction_needs_no_platform(tmp_path): + def unavailable(): + raise RuntimeError("no device backend is available") + + service = FFNService(rank=0, model_path=str(tmp_path)) + service.request_stop() + FFNModelRunner(service, unavailable, shutdown_timeout=0.1).run() + assert service.snapshot()["phase"] == "stopped" + assert service.snapshot()["resources_released"] is True + + +@pytest.mark.parametrize("timeout", [0.0, -1.0, float("nan"), float("inf")]) +def test_runner_rejects_unbounded_or_invalid_host_join(tmp_path, timeout): + service = FFNService(rank=0, model_path=str(tmp_path)) + with pytest.raises(ValueError, match="finite and positive"): + FFNModelRunner(service, HostBackend, shutdown_timeout=timeout) diff --git a/tests/unit/ffn_faas/test_service.py b/tests/unit/ffn_faas/test_service.py new file mode 100644 index 000000000..248fb57e0 --- /dev/null +++ b/tests/unit/ffn_faas/test_service.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Management HTTP remains CPU-only and does not promise device drain.""" + +from __future__ import annotations + +import json +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +import pytest + +from afd_plugin.ffn_faas.service import FFNService, ServicePhase + + +def test_readiness_and_host_stop_request(): + service = FFNService(rank=0, model_path="/model") + host, port = service.start_http("127.0.0.1", 0) + base = f"http://{host}:{port}" + try: + with pytest.raises(HTTPError) as unavailable: + urlopen(base + "/ready") + assert unavailable.value.code == 503 + service.phase(ServicePhase.READY, model_loaded=True, cam_paired=True) + with urlopen(base + "/ready") as response: + assert json.load(response)["phase"] == "ready" + with urlopen(Request(base + "/stop", method="POST")) as response: + status = json.load(response) + assert response.status == 202 + assert status["phase"] == "stopping" + assert status["resources_released"] is False + assert service.stop_requested.is_set() + with pytest.raises(HTTPError): + urlopen(base + "/ready") + service.phase(ServicePhase.FAILED, error="receive pending") + with pytest.raises(HTTPError) as failed: + urlopen(base + "/health") + assert failed.value.code == 503 + finally: + service.close_http() + + +def test_signal_stop_keeps_terminal_state(): + service = FFNService(rank=0, model_path="/model") + service.request_signal_stop() + assert service.snapshot()["phase"] == "stopping" + with pytest.raises(InterruptedError): + service.phase(ServicePhase.READY) + service.phase(ServicePhase.STOPPED, resources_released=True) + assert service.snapshot()["phase"] == "stopped" + + +def test_backend_metadata_cannot_claim_service_readiness(): + with pytest.raises(ValueError, match="service-owned fields"): + FFNService(rank=0, model_path="/model", initial_status={"phase": "ready"}) diff --git a/tests/unit/ffn_faas/test_settings.py b/tests/unit/ffn_faas/test_settings.py new file mode 100644 index 000000000..942f7705d --- /dev/null +++ b/tests/unit/ffn_faas/test_settings.py @@ -0,0 +1,157 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Standalone launch configuration preserves the existing CAM contract.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path + +import pytest + +from afd_plugin.ffn_faas.platforms.npu.adapter import AscendCAMAdapter +from afd_plugin.ffn_faas.platforms.npu.settings import ( + FFNConfig, + validate_native_expert_map, + validate_static_experts, +) +from tests.unit.fakes.ffn_faas import cam_cli_args + + +@pytest.fixture +def settings(): + return FFNConfig(Path("/model"), ffn_ranks=4) + + +@pytest.fixture +def launcher(monkeypatch): + monkeypatch.setenv("WORLD_SIZE", "2") + monkeypatch.setenv("RANK", "1") + monkeypatch.setenv("LOCAL_RANK", "1") + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + + +@pytest.mark.parametrize("positional", [False, True]) +def test_engine_configuration_keeps_user_values(checkpoint_dir, launcher, positional): + args = cam_cli_args(checkpoint_dir) + if positional: + args.model = "/unused-option-model" + args.model_tag = str(checkpoint_dir) + launch = AscendCAMAdapter().configure(args) + assert (launch.rank, launch.local_rank) == (1, 1) + assert launch.model_path == str(checkpoint_dir) + assert args.model == str(checkpoint_dir) + assert args.data_parallel_rank == 1 + assert args.additional_config["enable_cpu_binding"] is False + assert args.additional_config["afd"]["connector_extra_config"] == { + "dynamicQuant": 1, + "attn_ranks_per_dp": 4, + } + + +@pytest.mark.parametrize( + "change", + [ + {"data_parallel_size": 1}, + {"data_parallel_rank": 0}, + {"distributed_executor_backend": "mp"}, + {"receiver_shutdown_timeout": 0}, + ], +) +def test_launcher_rejects_inconsistent_engine_arguments( + checkpoint_dir, launcher, change +): + with pytest.raises(ValueError): + AscendCAMAdapter().configure(cam_cli_args(checkpoint_dir, **change)) + + +@pytest.mark.parametrize( + "field,value", [("port", 0), ("role", "attention"), ("num_ffn_ranks", 0)] +) +def test_cli_uses_existing_afd_validation(checkpoint_dir, launcher, field, value): + args = cam_cli_args(checkpoint_dir) + args.additional_config["afd"][field] = value + with pytest.raises(ValueError): + AscendCAMAdapter().configure(args) + + +@pytest.mark.parametrize( + "change", + [ + {"ffn_ranks": 0}, + {"rank": 4}, + {"local_rank": -1}, + {"shutdown_timeout": 0}, + {"shutdown_timeout": float("nan")}, + {"shutdown_timeout": float("inf")}, + ], +) +def test_invalid_host_launch_rejected(settings, change): + with pytest.raises(ValueError): + replace(settings, **change) + + +@pytest.mark.parametrize( + "rank,entries,owned", + [ + (0, [0, 1, -1, -1], {0, 1}), + (1, [-1, -1, 0, 1], {2, 3}), + ], +) +def test_native_expert_map_matches_cam(rank, entries, owned): + assert ( + validate_native_expert_map( + num_experts=4, ffn_ranks=2, ffn_rank=rank, expert_map=entries + ) + == owned + ) + + +@pytest.mark.parametrize( + "entries", + [ + [0, 0, -1, -1], + [-1, -1, 0, 1], + [1, 0, -1, -1], + [0, 1], + ], +) +def test_native_expert_map_mismatch_rejected(entries): + with pytest.raises(ValueError): + validate_native_expert_map( + num_experts=4, ffn_ranks=2, ffn_rank=0, expert_map=entries + ) + + +@pytest.mark.parametrize("ffn_ranks", [1, 2]) +def test_implicit_native_map_only_when_all_experts_are_local(ffn_ranks): + if ffn_ranks == 1: + assert validate_native_expert_map( + num_experts=4, ffn_ranks=1, ffn_rank=0, expert_map=None + ) == frozenset(range(4)) + else: + with pytest.raises(ValueError, match="explicit map"): + validate_native_expert_map( + num_experts=4, ffn_ranks=2, ffn_rank=0, expert_map=None + ) + + +@pytest.mark.parametrize( + "eplb,elastic,redundant,dynamic", + [ + (True, False, 0, False), + (False, True, 0, False), + (False, False, 1, False), + (False, False, 0, True), + ], +) +def test_static_cam_adapter_rejects_expert_relocation( + eplb, elastic, redundant, dynamic +): + with pytest.raises(ValueError, match="static expert placement"): + validate_static_experts( + enable_eplb=eplb, + enable_elastic_ep=elastic, + redundant_experts=redundant, + dynamic_eplb=dynamic, + ) diff --git a/tests/unit/v1/worker/test_async_ffn_runtime.py b/tests/unit/v1/worker/test_async_ffn_runtime.py index cd042b88d..ccae9a13a 100644 --- a/tests/unit/v1/worker/test_async_ffn_runtime.py +++ b/tests/unit/v1/worker/test_async_ffn_runtime.py @@ -106,6 +106,32 @@ def payload(layer, rows): ) +def test_independent_model_attachment_exposes_original_owner(harness): + model = harness.runner.get_model() + harness.runner.model = None + harness.connector._initialized = False + harness.runner.attach_model(model, parameter_bytes=123) + assert harness.runner.get_model() is model + + +@pytest.mark.parametrize("boundary", ["already_loaded", "paired", "negative_size"]) +def test_independent_attachment_rejects_invalid_lifetime(harness, boundary): + model = harness.runner.get_model() + if boundary != "already_loaded": + harness.runner.model = None + harness.connector._initialized = boundary == "paired" + with pytest.raises((RuntimeError, ValueError)): + harness.runner.attach_model( + model, parameter_bytes=-1 if boundary == "negative_size" else 123 + ) + + +def test_receiver_exit_rejects_pending_graph_work(harness): + harness.runner._graph_pending = True + with pytest.raises(RuntimeError, match="pending work; retain resources"): + harness.runner.mark_receiver_exited() + + def test_eager_preserves_layer_loop_device_metadata_and_return_headers(harness): h = harness first, zero = payload(5, 3), payload(2, 0) From 30d92466bb37e2dd5979d481873ce8f95224317c Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:23:07 +0200 Subject: [PATCH 35/42] docs(ffn_faas): drop NPU/GPU parallel comments from standalone FFN code Remove the "NPU parallel:" / "GPU parallel:" and "Reuse:" comment blocks that compared each standalone FFN class and method with its counterpart on the other platform, together with the dangling REFACTOR-CANDIDATE(R8) tag. The cross-references named file paths that no longer match main. Where a block carried a platform-local explanation that the docstrings did not already state, move it into the docstring: the adapter's fixed GPU constraints and handshake-adopted options, the backend lifecycle and release order, the per-step synchronize that makes execute_step fence-free, and the base-class versus platform split of both FFN environments. Comments and docstrings only; no behavior change. Co-Authored-By: Claude Opus 5.5 Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- afd_plugin/ffn_faas/platforms/gpu/__init__.py | 5 --- afd_plugin/ffn_faas/platforms/gpu/adapter.py | 18 +++++----- afd_plugin/ffn_faas/platforms/gpu/backend.py | 34 +++++++------------ .../ffn_faas/platforms/gpu/environment.py | 31 ++++------------- afd_plugin/ffn_faas/platforms/gpu/settings.py | 3 -- .../ffn_faas/platforms/npu/environment.py | 12 +++---- afd_plugin/v1/worker/ffn_model_runner.py | 6 ---- .../worker/ffn_routed_experts_model_runner.py | 4 --- 8 files changed, 33 insertions(+), 80 deletions(-) diff --git a/afd_plugin/ffn_faas/platforms/gpu/__init__.py b/afd_plugin/ffn_faas/platforms/gpu/__init__.py index 262c46548..8d763486d 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/__init__.py +++ b/afd_plugin/ffn_faas/platforms/gpu/__init__.py @@ -7,9 +7,4 @@ the launch. """ -# NPU parallel: afd_plugin/ffn_faas/platforms/npu/__init__.py (get_model -# selects the one qualified NPU model). -# Reuse: NO - the GPU platform passes the model path to vLLM and lets the AFD -# model registry select the FFN-only architecture. - GPU_DEEPEP_M2N_BACKEND = "gpu-deepep-m2n" diff --git a/afd_plugin/ffn_faas/platforms/gpu/adapter.py b/afd_plugin/ffn_faas/platforms/gpu/adapter.py index 16c7ce713..e8e587734 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/adapter.py +++ b/afd_plugin/ffn_faas/platforms/gpu/adapter.py @@ -33,10 +33,6 @@ from afd_plugin.ffn_faas.interfaces import FFNBackend -# NPU parallel: afd_plugin/ffn_faas/platforms/npu/adapter.py::create_backend -# (bootstraps the Ascend OPP environment, then imports the backend). -# Reuse: PARTIAL - the deferred-import factory shape is the same; GPU needs -# no environment bootstrap before importing torch. def create_backend(settings: GPUFFNConfig, args: Namespace) -> FFNBackend: """Import the GPU backend only when the host loop constructs it. @@ -72,13 +68,15 @@ def uses_cuda_graphs(args: Namespace) -> bool: return True -# NPU parallel: afd_plugin/ffn_faas/platforms/npu/adapter.py::AscendCAMAdapter. -# Reuse: YES - the shared launch module owns the native engine parser and the -# F-only launcher checks. GPU additionally fixes TP 1 with expert parallel, -# the execution mode and the lane count, and adopts Attention TP, capacity -# and the bucket ladder from the M2N handshake. class GPUDeepEPM2NAdapter: - """Keep GPU, topology and connector options out of the host loop.""" + """Keep GPU, topology and connector options out of the host loop. + + The native engine parser and the F-only launcher checks come from the + shared launch module. This adapter adds the GPU constraints: TP 1 with + expert parallelism, the execution mode and the DeepEP lane count. + Attention TP, capacity and the bucket ladder are not launch options; the + connector adopts them from the M2N handshake with Attention. + """ def create_parser(self) -> ArgumentParser: """Build vLLM's engine CLI with the F-only service's fixed defaults. diff --git a/afd_plugin/ffn_faas/platforms/gpu/backend.py b/afd_plugin/ffn_faas/platforms/gpu/backend.py index 9de1fd5e6..ba9bf830f 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/backend.py +++ b/afd_plugin/ffn_faas/platforms/gpu/backend.py @@ -28,16 +28,16 @@ ) -# NPU parallel: afd_plugin/ffn_faas/platforms/npu/backend.py::CAMFFNBackend. -# Reuse: PARTIAL - the same FFNBackend lifecycle (prepare phases, receiver -# thread activation, receiver-exit proof, release order runtime -> -# compute groups -> weights); the native pieces differ (NCCL/DeepEP M2N vs -# HCCL/CAM, lazy graph capture vs capture inside prepare) and GPU adds -# cancel_receive because the M2N control plane can end an idle receive. -# REFACTOR-CANDIDATE(R8): a backend base owning the phase sequence and the -# release-after-proof ordering, with platform hooks. class DeepEPM2NFFNBackend: - """Keep weights and communication owners until receiver exit is proven.""" + """Keep weights and communication owners until receiver exit is proven. + + Implements the ``FFNBackend`` lifecycle over DeepEP M2N: ``prepare`` runs + on the host thread; ``start_receiver``, ``execute_step`` and + ``receiver_exited`` run on the receiver thread; ``close`` then releases + the runtime, the compute groups and the weights, in that order. + ``cancel_receive`` can end an idle receive because the M2N control plane + owns an abortable NCCL group. + """ def __init__(self, settings: GPUFFNConfig, args: Namespace) -> None: """Hold the CPU-side launch facts; native work starts in ``prepare``.""" @@ -126,20 +126,16 @@ def prepare(self, service: FFNService) -> None: compute_ep_world_size=environment.ep_group.world_size, ) - # NPU parallel: CAMFFNBackend.start_receiver. - # Reuse: YES - identical delegation to the environment. def start_receiver(self) -> None: """Select the GPU once on the receiving thread, as the Worker does.""" self.environment.activate_thread() - # NPU parallel: CAMFFNBackend.execute_step. - # Reuse: PARTIAL - both delegate to the runner's - # execute_connector_driven_step; NPU adds its eager fence and never - # reports a cancel, GPU's runner already ends each step with a - # synchronize. def execute_step(self) -> bool: """Run one control payload's step through the shared GPU runner method. + The runner ends every step with a device synchronize, so this method + adds no fence of its own. + Returns ``False`` only when ``cancel_receive`` closed the control plane while the receive was idle. A closed control plane without a cancel request, and any other error (including a DeepEP timeout after @@ -150,8 +146,6 @@ def execute_step(self) -> bool: cancel_requested=self._cancel_requested.is_set ) - # NPU parallel: CAMFFNBackend.cancel_receive (a no-op there). - # Reuse: NO - GPU-only: the M2N control plane owns an abortable group. def cancel_receive(self) -> None: """End an idle control receive so a host stop can exit cleanly. @@ -168,15 +162,11 @@ def cancel_receive(self) -> None: if control_plane is not None: control_plane.close() - # NPU parallel: CAMFFNBackend.receiver_exited. - # Reuse: YES - identical delegation to the runner. def receiver_exited(self) -> None: """Record on the receiver thread that its loop ended cleanly.""" assert self.runner is not None self.runner.mark_receiver_exited() - # NPU parallel: CAMFFNBackend.close. - # Reuse: PARTIAL - same release order (runtime, compute groups, weights). def close(self) -> None: """Release CUDA graphs, DeepEP buffers, compute groups and weights. diff --git a/afd_plugin/ffn_faas/platforms/gpu/environment.py b/afd_plugin/ffn_faas/platforms/gpu/environment.py index 63b188482..845776d3c 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/environment.py +++ b/afd_plugin/ffn_faas/platforms/gpu/environment.py @@ -35,23 +35,19 @@ DBO_WORKSPACE_UBATCHES = 2 -# NPU parallel: afd_plugin/ffn_faas/platforms/npu/environment.py:: -# AscendFFNEnvironment. -# Reuse: PARTIAL - the shared base owns state, launcher checks, ownership -# flags, the EP-group check and close-state handling; NPU initializes HCCL -# explicitly plus Ascend groups and CPU binding, GPU reuses vLLM's GPU Worker -# distributed initializer, pins the routed MoE backend and selects the AFD -# model architecture. class GPUFFNEnvironment(FFNEnvironmentBase[GPUFFNConfig]): - """Never borrow unrelated groups or tear them down on a failed native seam.""" + """Never borrow unrelated groups or tear them down on a failed native seam. + + ``FFNEnvironmentBase`` owns the state, launcher checks, ownership flags, + the EP-group check and close-state handling. This class adds the GPU + device, the F-only ``VllmConfig`` (AFD routed-experts architecture and + pinned MoE backend) and vLLM's GPU Worker distributed initializer. + """ def __init__(self, settings: GPUFFNConfig) -> None: """Record the settings; nothing native happens until ``initialize_device``.""" super().__init__(settings, platform_name="GPU") - # NPU parallel: AscendFFNEnvironment.initialize_device. - # Reuse: PARTIAL - launcher and borrowed-group checks are identical; - # the device and native preflight are platform-specific. def initialize_device(self, config: VllmConfig) -> None: """Select this rank's GPU after checking the launcher and native stack. @@ -98,10 +94,6 @@ def initialize_device(self, config: VllmConfig) -> None: torch.accelerator.set_device_index(device) self._device = device - # NPU parallel: CAMFFNBackend.prepare (EngineArgs.from_cli_args, - # create_engine_config, re-asserted external_launcher). - # Reuse: YES - both build from the native engine CLI and re-assert the - # shared external launcher. def build_config(self, args: Namespace) -> VllmConfig: """Build the F-only ``VllmConfig`` exactly as the GPU FFN Worker sees it. @@ -135,10 +127,6 @@ def build_config(self, args: Namespace) -> VllmConfig: select_routed_moe_backend(config) return config - # NPU parallel: AscendFFNEnvironment.initialize_compute. - # Reuse: PARTIAL - ownership-before-collective and the EP-group check are - # identical; GPU calls vLLM's GPU Worker initializer instead of - # init_distributed_environment + initialize_model_parallel directly. def initialize_compute(self, config: VllmConfig) -> None: """Create the F-only NCCL world, model-parallel groups and workspace. @@ -188,8 +176,6 @@ def initialize_compute(self, config: VllmConfig) -> None: DBO_WORKSPACE_UBATCHES if config.parallel_config.enable_dbo else 1, ) - # NPU parallel: AscendFFNEnvironment.activate_thread. - # Reuse: YES (shape) - one call; only the device API differs. def activate_thread(self) -> None: """Select this rank's GPU on the calling (receiver) thread. @@ -200,9 +186,6 @@ def activate_thread(self) -> None: torch.cuda.set_device(self.device) - # NPU parallel: AscendFFNEnvironment.close. - # Reuse: PARTIAL - same order and ownership guard; NPU also destroys the - # Ascend model-parallel groups. def _destroy_groups(self) -> None: """Destroy the vLLM model-parallel groups and the F-only world. diff --git a/afd_plugin/ffn_faas/platforms/gpu/settings.py b/afd_plugin/ffn_faas/platforms/gpu/settings.py index 466fddd70..feb713e60 100644 --- a/afd_plugin/ffn_faas/platforms/gpu/settings.py +++ b/afd_plugin/ffn_faas/platforms/gpu/settings.py @@ -28,9 +28,6 @@ EAGER_MODE = "eager" -# NPU parallel: afd_plugin/ffn_faas/platforms/npu/settings.py::FFNConfig. -# Reuse: PARTIAL - the launch module owns rank and shutdown validation; GPU -# also fixes the payload format, the execution mode and the lane count. @dataclass(frozen=True) class GPUFFNConfig: """Frozen settings of one F rank in a standalone DeepEP M2N deployment.""" diff --git a/afd_plugin/ffn_faas/platforms/npu/environment.py b/afd_plugin/ffn_faas/platforms/npu/environment.py index 2fac1364c..b7e832310 100644 --- a/afd_plugin/ffn_faas/platforms/npu/environment.py +++ b/afd_plugin/ffn_faas/platforms/npu/environment.py @@ -24,13 +24,13 @@ COMPUTE_GROUP_TIMEOUT_SECONDS = 180 -# GPU parallel: afd_plugin/ffn_faas/platforms/gpu/environment.py:: -# GPUFFNEnvironment. -# Reuse: PARTIAL - the shared base owns state, launcher checks, ownership -# flags, the EP-group check and close-state handling; this class owns Ascend -# device selection, custom operators, HCCL groups and CPU binding. class AscendFFNEnvironment(FFNEnvironmentBase[FFNConfig]): - """Never borrow unrelated groups or tear them down on a failed native seam.""" + """Never borrow unrelated groups or tear them down on a failed native seam. + + ``FFNEnvironmentBase`` owns the state, launcher checks, ownership flags, + the EP-group check and close-state handling. This class owns Ascend + device selection, custom operators, HCCL groups and CPU binding. + """ def __init__(self, settings: FFNConfig) -> None: super().__init__(settings, platform_name="Ascend") diff --git a/afd_plugin/v1/worker/ffn_model_runner.py b/afd_plugin/v1/worker/ffn_model_runner.py index ab23cbe0e..011f53afd 100644 --- a/afd_plugin/v1/worker/ffn_model_runner.py +++ b/afd_plugin/v1/worker/ffn_model_runner.py @@ -152,12 +152,6 @@ def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]: def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: return None - # NPU parallel: afd_plugin/v1/worker/npu/async_ffn_model_runner.py:: - # AsyncNPUFFNModelRunner.execute_connector_driven_step (same name and - # bool return; the CAM step submits layered work items instead of - # receiving one control payload). - # Reuse: NO - the step bodies share no code; only the host contract - # (one call per batch, a bool result) is common. def execute_connector_driven_step( self, *, cancel_requested: Callable[[], bool] ) -> bool: diff --git a/afd_plugin/v1/worker/ffn_routed_experts_model_runner.py b/afd_plugin/v1/worker/ffn_routed_experts_model_runner.py index a5b8c09ab..7286263eb 100644 --- a/afd_plugin/v1/worker/ffn_routed_experts_model_runner.py +++ b/afd_plugin/v1/worker/ffn_routed_experts_model_runner.py @@ -161,10 +161,6 @@ def _take_local_experts(self, model: RoutedExpertsModel) -> None: for layer_idx in self.moe_layer_indices } - # NPU parallel: afd_plugin/v1/worker/npu/async_ffn_model_runner.py:: - # AsyncNPUFFNModelRunner.mark_receiver_exited (records the same proof; - # NPU rejects pending graph work, GPU fences the device instead). - # Reuse: PARTIAL - same contract, device-specific completion check. def mark_receiver_exited(self) -> None: """Record that the receive loop ended cleanly and its device work is done. From 671a8403c5ed826d9468b095ce127d9ef57d3437 Mon Sep 17 00:00:00 2001 From: "Songym@Huawei" <109567708+Pingzii@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:00:02 +0800 Subject: [PATCH 36/42] fix(npu): resolve connector classes through getattr in the factory (#567) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What this fixes `camp2p` resolves its public names through a package-level `__getattr__` (#503). The connector factory still read the class out of the module dictionary: ```python connector_cls = vars(module)[class_name] ``` For a lazily exported class that dictionary is empty, so every CAMP2P start fails before the worker can build the connector: ``` File "afd_plugin/connectors/factory.py", line 40, in loader connector_cls = vars(module)[class_name] KeyError: 'CAMP2pAFDConnector' ``` ## Change Read the class with `getattr`, which follows the `__getattr__` hook and behaves exactly as before for eagerly exported modules. The regression case moved into `tests/unit/connectors/test_p2p_connector.py` with the other factory tests: it registers a temporary connector backed by a module that only answers through `__getattr__`, and registers into a copied `_registry` so the temporary entry does not outlive the test. The missing-name case was dropped; the standalone test file is gone. ## Validation - The moved case fails with the old `vars()` lookup. - `tests/unit/connectors/test_p2p_connector.py` and the maintained CPU suite pass. - DCO, label and pre-commit pass. Signed-off-by: 宋元盟 Co-authored-by: 宋元盟 --- afd_plugin/connectors/factory.py | 5 ++- tests/unit/connectors/test_p2p_connector.py | 47 +++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/afd_plugin/connectors/factory.py b/afd_plugin/connectors/factory.py index 4cc19400d..20b0a08c7 100644 --- a/afd_plugin/connectors/factory.py +++ b/afd_plugin/connectors/factory.py @@ -37,7 +37,10 @@ def register_connector( def loader() -> type[AFDConnectorBase]: module = importlib.import_module(module_path) - connector_cls = vars(module)[class_name] + # ``getattr`` rather than ``vars(module)``: packages in this tree resolve + # their names through a module-level ``__getattr__``, so the class is not + # necessarily present in the module dictionary. + connector_cls = getattr(module, class_name) if not issubclass(connector_cls, AFDConnectorBase): raise TypeError( f"{module_path}.{class_name} is not an AFDConnectorBase", diff --git a/tests/unit/connectors/test_p2p_connector.py b/tests/unit/connectors/test_p2p_connector.py index dc6cb70e1..b4a3edbda 100644 --- a/tests/unit/connectors/test_p2p_connector.py +++ b/tests/unit/connectors/test_p2p_connector.py @@ -897,3 +897,50 @@ def test_p2p_subgroup_rendezvous_reuses_the_afd_world_store( def test_factory_rejects_unknown_connector(): with pytest.raises(ValueError, match="unsupported AFD connector type"): AFDConnectorFactory.get_connector_class("UnknownConnector") + + +class _LazilyExportedConnector(AFDConnectorBase): + """Stands in for a connector served only through module ``__getattr__``.""" + + def __init__(self, *args, **kwargs) -> None: # pragma: no cover - never built + raise NotImplementedError + + +@pytest.fixture +def isolated_connector_registry(monkeypatch): + """Keep a temporary registration out of the registry other tests share.""" + + monkeypatch.setattr( + AFDConnectorFactory, + "_registry", + dict(AFDConnectorFactory._registry), + ) + + +def test_factory_resolves_a_connector_served_only_by_package_getattr( + monkeypatch, isolated_connector_registry +): + """The factory must read lazily exported names out of ``__getattr__``.""" + + module = types.ModuleType("afd_lazily_exported_connector") + class_name = "_LazilyExportedConnector" + + def _resolve(name: str): + if name == class_name: + return _LazilyExportedConnector + raise AttributeError(name) + + # A module-level __getattr__ (PEP 562) is what the CAMP2P package uses. + module.__getattr__ = _resolve # type: ignore[method-assign] + monkeypatch.setitem(sys.modules, module.__name__, module) + + AFDConnectorFactory.register_connector( + "LazilyExportedConnector", + module.__name__, + class_name, + replace=True, + ) + + assert class_name not in vars(module), "the class must not be in the module dict" + resolved = AFDConnectorFactory.get_connector_class("LazilyExportedConnector") + assert resolved is _LazilyExportedConnector From 2bede024643ce4df7a04615fd4fbcc88d6d082d9 Mon Sep 17 00:00:00 2001 From: "Songym@Huawei" <109567708+Pingzii@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:00:43 +0800 Subject: [PATCH 37/42] fix(test): stop unit tests leaking modules, registries and guards across the session (#570) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #568. Replaces #569. The unit suite runs in one pytest session, so anything a test installs into `sys.modules` stays visible to every later test. Every stand-in below is now limited to the import that needs it and removed on both the success and the failure path. Test-only; no production file changes. ## What changes - **Sync gate proxy (#568)** — `pytest.importorskip("vllm_ascend")` next to the existing `vllm.forward_context` guard, so a host with vLLM but without vLLM-Ascend skips instead of failing during collection. The `torch_npu` stand-in stays for the adapter import only. `import afd_plugin` runs before the stand-in is installed, so model registration reflects the actual environment. - **Two `vllm` stubs** (`test_camp2p_token_ids.py`, `test_deepseek_v4_hash_ids.py`) — only the modules that are genuinely missing are installed; a pre-existing `vllm` module is never replaced, so it cannot outlive the context. - **CAMP2P cleanup** — modules the stub block loaded that still bind one of its objects are dropped, together with the parent package attribute, so the lazy package cannot serve a class built against the doubles. Modules that merely contain `None` are left alone. - **Component fixture** (`test_deepseek_v4_external_routing_moe_cpu.py`) — restores only the names it replaced, inside a `finally`; unrelated modules imported meanwhile survive. - **DeepEP** — the `afdep` stand-in lives only around the imports that need it, inside `try/finally`. It has a module spec, so `find_spec("afdep")` reports it correctly while installed and the package as missing afterwards. - **Async FFN runners** — adds explicit `vllm_ascend` import guards to the async FFN graph and runtime tests. ## Verification - A pre-installed `vllm` module survives both stubs; the connector is not left cached, and `camp2p` no longer serves a stub-bound class. - A module that only holds `None` is not evicted; a stand-in removed on a failing import leaves `afdep` and `torch_npu` absent. - `afd_plugin` imports with the stand-in installed and keeps a non-NPU DSV4 registration. - `pytest tests/unit --collect-only` collects with no errors; the affected files run 37 passed, 4 skipped locally. - Full `tests/unit` run on a CPU-only host: only the two `package/` cases fail, and they fail identically on `upstream/main` (checkout-specific, unrelated). --------- Signed-off-by: 宋元盟 Co-authored-by: 宋元盟 --- .../unit/connectors/test_camp2p_token_ids.py | 93 +++++++++++++------ .../connectors/test_deepep_m2n_connector.py | 50 +++++----- tests/unit/fakes/afdep.py | 22 +++++ ...st_deepseek_v4_external_routing_moe_cpu.py | 52 ++++++++++- .../models/test_sync_attention_gate_proxy.py | 34 ++++++- .../test_deepseek_v4_hash_ids.py | 37 ++++---- tests/unit/v1/worker/test_async_ffn_graph.py | 1 + .../unit/v1/worker/test_async_ffn_runtime.py | 1 + 8 files changed, 219 insertions(+), 71 deletions(-) diff --git a/tests/unit/connectors/test_camp2p_token_ids.py b/tests/unit/connectors/test_camp2p_token_ids.py index 7a563c23e..15dd97bcc 100644 --- a/tests/unit/connectors/test_camp2p_token_ids.py +++ b/tests/unit/connectors/test_camp2p_token_ids.py @@ -25,6 +25,7 @@ import sys import types from collections.abc import Iterator +from importlib.machinery import ModuleSpec from types import SimpleNamespace from typing import Any @@ -48,6 +49,32 @@ ) +def _drop_stub_bound_modules(before: set[str], stub_objects: set[int]) -> None: + """Forget modules the stub block loaded that still hold one of its objects. + + A cached module keeps the doubles in its globals, and the parent package keeps the + child module as an attribute, so both have to be dropped for a later import to build + the real thing. + """ + + for name in [ + name + for name in sys.modules + if name not in before and name.startswith("afd_plugin") + ]: + module = sys.modules[name] + if not any( + value is not None and id(value) in stub_objects + for value in vars(module).values() + ): + continue + sys.modules.pop(name, None) + parent_name, _, leaf = name.rpartition(".") + parent = sys.modules.get(parent_name) + if parent is not None and getattr(parent, leaf, None) is module: + delattr(parent, leaf) + + @contextlib.contextmanager def _vllm_stub() -> Iterator[None]: """Expose a minimal ``vllm`` surface for the duration of one import. @@ -67,42 +94,56 @@ def _vllm_stub() -> Iterator[None]: yield return - saved = {name: sys.modules.get(name) for name in missing} - - def make(name: str, **attributes: object) -> None: + builders: dict[str, dict[str, object]] = { + "vllm": {}, + "vllm.forward_context": { + "DPMetadata": type("DPMetadata", (), {}), + "get_forward_context": lambda: None, + }, + "vllm.logger": { + "init_logger": lambda *args, **kwargs: logging.getLogger("test") + }, + "vllm.utils": {}, + "vllm.utils.torch_utils": { + "direct_register_custom_op": lambda **kwargs: None, + "is_torch_equal_or_newer": lambda *args, **kwargs: True, + }, + "vllm.distributed": {}, + "vllm.distributed.parallel_state": { + "get_pcp_group": None, + "get_tensor_model_parallel_rank": None, + }, + } + + # Only the modules that are genuinely absent are installed. Overwriting a module + # that another test already imported would outlive this context: the restore + # below only knows about the names it added. + _imported_before = set(sys.modules) + for name in missing: module = types.ModuleType(name) module.__path__ = [] # type: ignore[attr-defined] - for attribute, value in attributes.items(): + # find_spec("vllm") runs at import time; a spec-less module makes it raise. + module.__spec__ = ModuleSpec(name, None) + for attribute, value in builders[name].items(): setattr(module, attribute, value) sys.modules[name] = module - make("vllm") - make( - "vllm.forward_context", - DPMetadata=type("DPMetadata", (), {}), - get_forward_context=lambda: None, - ) - make("vllm.logger", init_logger=lambda *args, **kwargs: logging.getLogger("test")) - make("vllm.utils") - make( - "vllm.utils.torch_utils", - direct_register_custom_op=lambda **kwargs: None, - is_torch_equal_or_newer=lambda *args, **kwargs: True, - ) - make("vllm.distributed") - make( - "vllm.distributed.parallel_state", - get_pcp_group=None, - get_tensor_model_parallel_rank=None, - ) + _stub_objects = { + id(value) + for name in missing + for attribute, value in vars(sys.modules[name]).items() + if not attribute.startswith("__") and value is not None + } try: yield finally: for name in missing: sys.modules.pop(name, None) - for name, module in saved.items(): - if module is not None: - sys.modules[name] = module + # A module imported while the stub was installed keeps the doubles in its + # globals, so it must not stay cached. Only modules the stub block loaded, and + # only those that bind one of the stub objects, are dropped; evicting their + # packages would rebuild the connector factory and leave two registries. + _drop_stub_bound_modules(_imported_before, _stub_objects) with _vllm_stub(): diff --git a/tests/unit/connectors/test_deepep_m2n_connector.py b/tests/unit/connectors/test_deepep_m2n_connector.py index eb9e9fbea..c151249aa 100644 --- a/tests/unit/connectors/test_deepep_m2n_connector.py +++ b/tests/unit/connectors/test_deepep_m2n_connector.py @@ -15,11 +15,6 @@ from vllm.platforms.interface import set_assigned_physical_gpu_ids # noqa: E402 -from tests.unit.fakes.afdep import FakeElasticBuffer, install_fake_afdep # noqa: E402 - -# The connector module imports afdEP's transport when it loads. -install_fake_afdep() - from afd_plugin.config import afd_config_from_mapping # noqa: E402 from afd_plugin.connectors import ( # noqa: E402 AFDConnectorFactory, @@ -29,23 +24,36 @@ AFDTransferMetadata, ) from afd_plugin.connectors.base import AFDControlPlaneClosedError # noqa: E402 -from afd_plugin.connectors.gpu.deepep_m2n import ( # noqa: E402 - connector as connector_module, -) -from afd_plugin.connectors.gpu.deepep_m2n.connector import ( # noqa: E402 - DeepEPM2NConfig, - bucket_ladder, - expert_weight_format, - local_handshake_info, - local_model_info, - plan_stage, - precompile_buckets, - resolve_capacity_tokens_per_rank, - resolve_num_lanes, - tokens_per_attention_rank, - validate_handshake, - validate_model_infos, +from tests.unit.fakes.afdep import ( # noqa: E402 + FakeElasticBuffer, + install_fake_afdep, + uninstall_fake_afdep, ) + +# The connector module imports afdEP's transport when it loads, so the stand-in has to +# exist for that import only. It is removed again on both paths: the bound names keep +# working, and the rest of the session sees the package as absent. +install_fake_afdep() +try: + from afd_plugin.connectors.gpu.deepep_m2n import ( # noqa: E402 + connector as connector_module, + ) + from afd_plugin.connectors.gpu.deepep_m2n.connector import ( # noqa: E402 + DeepEPM2NConfig, + bucket_ladder, + expert_weight_format, + local_handshake_info, + local_model_info, + plan_stage, + precompile_buckets, + resolve_capacity_tokens_per_rank, + resolve_num_lanes, + tokens_per_attention_rank, + validate_handshake, + validate_model_infos, + ) +finally: + uninstall_fake_afdep() from afd_plugin.connectors.gpu.deepep_m2n.experts import ( # noqa: E402 expert_row_ranges, ) diff --git a/tests/unit/fakes/afdep.py b/tests/unit/fakes/afdep.py index 1680d40c8..f7255da8f 100644 --- a/tests/unit/fakes/afdep.py +++ b/tests/unit/fakes/afdep.py @@ -10,6 +10,7 @@ from __future__ import annotations import sys +from importlib.machinery import ModuleSpec from importlib.util import find_spec from types import ModuleType @@ -52,6 +53,9 @@ def __init__(self, **kwargs: object) -> None: self.kwargs = kwargs +_FAKE_MARKER = "__afd_fake_afdep__" + + def install_fake_afdep() -> None: """Register the stand-in ``afdep.deepep`` unless afdEP is importable.""" if "afdep" in sys.modules or find_spec("afdep") is not None: @@ -62,5 +66,23 @@ def install_fake_afdep() -> None: deepep.DispatchHandle = type("DispatchHandle", (), {}) # type: ignore[attr-defined] deepep.ReceiveResult = type("ReceiveResult", (), {}) # type: ignore[attr-defined] package.deepep = deepep # type: ignore[attr-defined] + # A module without a spec makes a later find_spec("afdep") raise instead of + # reporting the package as missing. + package.__spec__ = ModuleSpec("afdep", None) + deepep.__spec__ = ModuleSpec("afdep.deepep", None) + setattr(package, _FAKE_MARKER, True) + setattr(deepep, _FAKE_MARKER, True) sys.modules["afdep"] = package sys.modules["afdep.deepep"] = deepep + + +def uninstall_fake_afdep() -> None: + """Remove the stand-in again, leaving a real afdEP install untouched. + + A stand-in left in ``sys.modules`` has no ``__spec__``, so a later + ``find_spec("afdep")`` raises instead of reporting "not installed". + """ + for name in ("afdep.deepep", "afdep"): + module = sys.modules.get(name) + if module is not None and getattr(module, _FAKE_MARKER, False): + sys.modules.pop(name, None) diff --git a/tests/unit/model_executor/models/test_deepseek_v4_external_routing_moe_cpu.py b/tests/unit/model_executor/models/test_deepseek_v4_external_routing_moe_cpu.py index ee16886cd..5cd65d9d0 100644 --- a/tests/unit/model_executor/models/test_deepseek_v4_external_routing_moe_cpu.py +++ b/tests/unit/model_executor/models/test_deepseek_v4_external_routing_moe_cpu.py @@ -20,6 +20,7 @@ import sys import types from dataclasses import dataclass +from importlib.machinery import ModuleSpec from pathlib import Path import pytest @@ -101,6 +102,28 @@ def _moe_config(): return types.SimpleNamespace(dp_size=1, pcp_size=1, tp_size=1, dp_group=None) +_STUBBED_MODULE_NAMES = ( + "vllm", + "vllm.distributed", + "vllm.distributed.parallel_state", + "vllm_ascend", + "vllm_ascend.ops", + "vllm_ascend.ops.fused_moe", + "vllm_ascend.ops.fused_moe.dataclass", + "vllm_ascend.ops.fused_moe.dataclass.prepare_finalize", + "vllm_ascend.ops.fused_moe.prepare_finalize", + "vllm_ascend.ops.fused_moe.moe_comm_method", + "vllm_ascend.ops.fused_moe.moe_utils", + "vllm_ascend.quantization", + "vllm_ascend.quantization.quant_type", + "vllm_ascend.ascend_forward_context", + "afd_plugin", + "afd_plugin.connectors", + "afd_plugin.connectors.npu", + "afd_plugin.connectors.npu.camp2p.external_routing", +) + + def _install_stub_modules(): """Register the upstream modules the component imports from.""" routing_spec = importlib.util.spec_from_file_location( @@ -182,7 +205,11 @@ def get_moe_comm_method(comm_type, moe_config=None): "afd_plugin.connectors.npu": types.ModuleType("afd_plugin.connectors.npu"), "afd_plugin.connectors.npu.camp2p.external_routing": afd_routing, } + assert set(modules) == set(_STUBBED_MODULE_NAMES) for name, module in modules.items(): + # find_spec() is called on these names at import time and raises on a + # module without a spec, so the stand-ins carry one. + module.__spec__ = ModuleSpec(name, None) sys.modules[name] = module spec = importlib.util.spec_from_file_location( @@ -194,13 +221,28 @@ def get_moe_comm_method(comm_type, moe_config=None): return component_module, context, comm_module +def _restore_modules(written: dict[str, types.ModuleType | None]) -> None: + """Put back only the names this fixture replaced, and drop the ones it added.""" + + for name, original in written.items(): + if original is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = original + + @pytest.fixture(scope="module") def loaded(): - before = dict(sys.modules) - module, context, comm_module = _install_stub_modules() - yield module, context, comm_module - sys.modules.clear() - sys.modules.update(before) + written = {name: sys.modules.get(name) for name in _STUBBED_MODULE_NAMES} + try: + module, context, comm_module = _install_stub_modules() + except BaseException: + _restore_modules(written) + raise + try: + yield module, context, comm_module + finally: + _restore_modules(written) class _Experts(torch.nn.Module): diff --git a/tests/unit/model_executor/models/test_sync_attention_gate_proxy.py b/tests/unit/model_executor/models/test_sync_attention_gate_proxy.py index ca1bf18da..486c31acc 100644 --- a/tests/unit/model_executor/models/test_sync_attention_gate_proxy.py +++ b/tests/unit/model_executor/models/test_sync_attention_gate_proxy.py @@ -16,6 +16,7 @@ from __future__ import annotations import sys +from importlib.machinery import ModuleSpec from types import ModuleType, SimpleNamespace import pytest @@ -24,9 +25,40 @@ # The adapter module needs this specific vLLM module, and a CPU-only host can have a # partial vLLM install where `import vllm` succeeds but this one does not. pytest.importorskip("vllm.forward_context") +# The adapter chain imports the shared-expert module, which loads vLLM-Ascend at module +# scope; without it the collection fails instead of skipping. +pytest.importorskip("vllm_ascend") + +# afd_plugin builds its model registry from ``find_spec("torch_npu")`` when the package +# is first imported, so it is imported before the stand-in exists; otherwise a process +# without the extension would register the NPU adapter for DSV4 and keep that path after +# the stand-in is gone. +import afd_plugin # noqa: E402,F401 + +# The adapter's import chain pulls the NPU extension in at module scope, but these +# tests only drive the Python wire protocol with fakes. A temporary stand-in keeps +# the module collectable on a CPU-only host; it is removed again so other modules +# still see the extension as missing and keep skipping. +try: + import torch_npu # noqa: F401 + + _stubbed_torch_npu = False +except ModuleNotFoundError: + _torch_npu_stub = ModuleType("torch_npu") + # ``find_spec`` is called on this name at import time, and a module without a + # spec makes it raise instead of reporting the extension as missing. + _torch_npu_stub.__spec__ = ModuleSpec("torch_npu", None) + sys.modules["torch_npu"] = _torch_npu_stub + _stubbed_torch_npu = True # The NPU adapter; the CUDA module of the same name has no sync gate shell. -from afd_plugin.model_executor.models.npu import deepseek_v4 as adapter # noqa: E402 +try: + from afd_plugin.model_executor.models.npu import ( + deepseek_v4 as adapter, # noqa: E402 + ) +finally: + if _stubbed_torch_npu: + del sys.modules["torch_npu"] TOP_K = 2 HIDDEN = 4 diff --git a/tests/unit/model_executor/test_deepseek_v4_hash_ids.py b/tests/unit/model_executor/test_deepseek_v4_hash_ids.py index f8bfc1d3f..161332c25 100644 --- a/tests/unit/model_executor/test_deepseek_v4_hash_ids.py +++ b/tests/unit/model_executor/test_deepseek_v4_hash_ids.py @@ -17,6 +17,7 @@ import sys import types from collections.abc import Iterator +from importlib.machinery import ModuleSpec from pathlib import Path from typing import Any @@ -50,34 +51,34 @@ def _vllm_stub() -> Iterator[None]: yield return - saved = {name: sys.modules.get(name) for name in missing} + builders: dict[str, dict[str, object]] = { + "vllm": {}, + "vllm.forward_context": { + "DPMetadata": type("DPMetadata", (), {}), + "ForwardContext": type("ForwardContext", (), {}), + "get_forward_context": lambda: None, + }, + "vllm.logger": { + "init_logger": lambda *args, **kwargs: logging.getLogger("test") + }, + } - def make(name: str, **attributes: object) -> None: + # Only the modules that are genuinely absent are installed: overwriting one that + # another test imported would survive this context, because the cleanup below + # only removes the names it added. + for name in missing: module = types.ModuleType(name) module.__path__ = [] # type: ignore[attr-defined] - for attribute, value in attributes.items(): + # find_spec("vllm") runs at import time; a spec-less module makes it raise. + module.__spec__ = ModuleSpec(name, None) + for attribute, value in builders[name].items(): setattr(module, attribute, value) sys.modules[name] = module - - make("vllm") - make( - "vllm.forward_context", - DPMetadata=type("DPMetadata", (), {}), - ForwardContext=type("ForwardContext", (), {}), - get_forward_context=lambda: None, - ) - make( - "vllm.logger", - init_logger=lambda *args, **kwargs: logging.getLogger("test"), - ) try: yield finally: for name in missing: sys.modules.pop(name, None) - for name, module in saved.items(): - if module is not None: - sys.modules[name] = module with _vllm_stub(): diff --git a/tests/unit/v1/worker/test_async_ffn_graph.py b/tests/unit/v1/worker/test_async_ffn_graph.py index 888119ac9..4454eeec0 100644 --- a/tests/unit/v1/worker/test_async_ffn_graph.py +++ b/tests/unit/v1/worker/test_async_ffn_graph.py @@ -10,6 +10,7 @@ pytest.importorskip("vllm") pytest.importorskip("torch_npu") +pytest.importorskip("vllm_ascend") import torch # noqa: E402 from afd_plugin.connectors.base import FFNExecutionSession # noqa: E402 diff --git a/tests/unit/v1/worker/test_async_ffn_runtime.py b/tests/unit/v1/worker/test_async_ffn_runtime.py index ccae9a13a..91c3db77a 100644 --- a/tests/unit/v1/worker/test_async_ffn_runtime.py +++ b/tests/unit/v1/worker/test_async_ffn_runtime.py @@ -12,6 +12,7 @@ pytest.importorskip("vllm") pytest.importorskip("torch_npu") +pytest.importorskip("vllm_ascend") import torch # noqa: E402 from afd_plugin.config import AFDConfig # noqa: E402 From f59d3ce238a80300a10f89af42d4a475e6836078 Mon Sep 17 00:00:00 2001 From: Ronen Kat <16743404+ronenkat@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:36:11 +0200 Subject: [PATCH 38/42] [GPU] Simplify AFD GPU recipes (#571) ## Purpose Unify the GPU `prefill_decode_colocation` recipes on one shared launcher. The existing recipes duplicate most of their launch logic, encode multi-pod placement differently, and vary ports, GPU order, and timeouts without a functional reason. This change reduces each recipe to its model, connector, default layout, DP/TP shape, mode, and `vllm serve` flags, while the shared launcher owns placement, AFD configuration, role startup, and overrides. The new launcher also makes every recipe runnable on one host or across pods from a single `LAYOUT`, directly or through `deploy-afd-k8s`. `AFD_DRY_RUN=1` prints the exact per-pod commands without starting vLLM. ## Issue - Related issue(s): #566 - Closing keyword, only if fully resolved: Closes #566 ## Scope - In scope: - GPU `prefill_decode_colocation` recipes for `P2pNcclAFDConnector` and `DeepEPM2NAFDConnector`. - The shared launcher and its user documentation. - `deploy-afd-k8s` and `run-vllm-bench-k8s` instructions needed to deploy and benchmark the new recipe shape. - Recipe, connector-guide, benchmark, and review-skill documentation updates. - Out of scope: - NPU recipes and runtime behavior. - `prefill_decode_disaggregation` recipes. - AFD plugin runtime code, connector implementations, and E2E gate definitions. ## Implementation Notes - Recipe scripts change from 19 files / 1,192 lines to 20 files / 374 lines. - Added `recipe/gpu/common/afd_colocation.sh` and `recipe/gpu/common/README.md`. Each personalized recipe now sets its model, connector, default `LAYOUT`, DP/TP defaults, mode, and serve flags before sourcing the shared launcher. - Restored explicit personalized recipes: - DeepSeek-V2-Lite and Qwen3-30B-A3B DeepEP M2N: `2a2f_{eager,graph,graph_dbo}_dp2tp1.sh` plus `2a2f_eager_dp1tp2.sh`. - Qwen3-235B-A22B-FP8 DeepEP M2N: `4a2f_{eager,graph,dbo}_dp4tp1.sh` plus `multipod_4a1f_4a1f_dbo.sh`. - DeepSeek-V2-Lite P2P NCCL: the existing seven personalized 2A2F, 4A4F, and two-pod graph/eager DBO recipes. - Qwen3.5-122B-A10B-FP8 P2P NCCL: retains `multipod_2a_2a_2f_graph.sh`. - The launcher derives device assignment, DP/TP flags, headless state, DP start ranks, AFD rank counts, and both `vllm serve` commands from `LAYOUT` and `POD=AFD_`. - Intentional command changes: - Attention API `18000`, FFN API `18001`, and AFD rendezvous `6239`. - FFN uses each pod's first GPUs and Attention uses the following GPUs. - Both roles use `--cpu-distributed-timeout-seconds 3600` and `afd_process_group_timeout_s: 3600`. - Both roles use `--trust-remote-code` and invoke `vllm serve` directly. - DeepEP M2N recipes use `--enable-ep-weight-filter` on both roles. - Qwen3-235B graph capture sizes add 12/24/48; DBO capture sizes add 384/512. - P2P DeepSeek-V2-Lite and Qwen3.5 tuning flags remain unchanged. - `deploy-afd-k8s` ships both the recipe and shared launcher in the recipe ConfigMap, sets `AFD_RECIPE_LIB` and `SERVE_HOST=0.0.0.0`, derives one pod per `LAYOUT` entry, and resolves Service ports from the launcher. - `tools/benchmarks/request_generator.sh` defaults to the colocation Attention port, `18000`; `run-vllm-bench-k8s` uses the same endpoint. - The common README documents that personalized recipes encode their own DP/TP defaults; `ATTN_TP` and `FFN_TP` should be changed only intentionally. ## Test Plan - CPU-only, no cluster: - Run `bash -n` and `shellcheck --shell=bash` on all 20 recipe wrappers and the shared launcher. - Run each recipe with `AFD_DRY_RUN=1`. - Assert mode flags, layouts, DP/TP flags, device assignments, rank counts, ports, and per-pod headless/start-rank behavior. - Compare old and new generated commands with a fake `vllm` executable. - Verify the single-pod `LAYOUT=2A2F` and two-pod `LAYOUT=4A0F,0A4F` configurations produce distinct Attention and FFN commands. - Static checks: - Run targeted pre-commit checks on changed recipe, documentation, and skill files. - Run `git diff --check`. - GPU smoke: - Deploy the renamed 2A2F graph + DBO DeepSeek P2P recipe through the two-file ConfigMap path used by `deploy-afd-k8s`. - Verify both roles start, the Attention health endpoint answers, and completions are correct. ## Test Result - All 20 wrappers passed `bash -n`, `shellcheck --shell=bash`, and `AFD_DRY_RUN=1`. - Dry-run assertions covered mode flags, layouts, DP/TP flags, device assignments, ports, rank counts, and per-pod headless/start-rank behavior. - A fake-`vllm` old/new comparison found no unintended command drift beyond the documented port, timeout, device-order, mode, and multi-pod changes. - `LAYOUT=2A2F` produced FFN on GPUs 0-1 and Attention on GPUs 2-3. `LAYOUT=4A0F,0A4F` produced Attention on pod `AFD_0` and FFN on pod `AFD_1`, each with DP4 and the expected rank counts. - Targeted pre-commit checks and `git diff --check` passed for the changed recipe and documentation files. - GPU smoke, 4x H100, vLLM `0.32.0`: the 2A2F graph + DBO DeepSeek P2P recipe ran through the `deploy-afd-k8s` ConfigMap path. Both roles became ready, `/health` answered, completion was correct, and 256/256 requests succeeded with ISL 256, OSL 128, and concurrency 64. ## Docs Impact - Files updated: - `recipe/gpu/common/README.md` and the GPU recipe READMEs. - `recipe/README.md`. - `docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md`. - `.agents/skills/deploy-afd-k8s/` and `.agents/skills/run-vllm-bench-k8s/SKILL.md`. - `.agents/skills/review-pr/references/features/disaggregation-topologies.md`. - `tools/benchmarks/request_generator.sh` default-port and usage documentation. - No `docs/design/module/` update is needed because plugin runtime contracts, E2E gate structure, and required E2E cases are unchanged. ---
Essential PR Checklist - [x] Purpose is clear and linked to public context when possible. - [x] Scope is bounded. - [x] Compatibility with vLLM v0.32.0 and, for NPU changes, the pinned vLLM-Ascend source is considered, as applicable. - [x] No changes are made to the vLLM source checkout. - [x] Plugin-owned classes or explicit dotted class paths are preferred over monkey patches. - [x] Any compat shim or monkey patch is isolated, idempotent, version-guarded, documented, and tested. - [x] Imports remain CPU-safe; CUDA-heavy work is delayed or GPU-gated. - [x] Validation evidence is included, including skipped GPU tests when applicable. - [x] Documentation impact is stated.
--------- Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- .agents/skills/deploy-afd-k8s/SKILL.md | 34 ++-- .../skills/deploy-afd-k8s/resources/deploy.md | 71 ++++--- .../resources/resolve-recipe.md | 187 ++++++------------ .../deploy-afd-k8s/resources/services.md | 30 ++- .../skills/deploy-afd-k8s/templates/pod.yaml | 28 +-- .../features/disaggregation-topologies.md | 7 +- .agents/skills/run-vllm-bench-k8s/SKILL.md | 6 +- docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md | 7 +- recipe/README.md | 10 +- recipe/gpu/DeepEPM2NAFDConnector/README.md | 143 ++++++++++---- .../deepseek_v2_lite/README.md | 60 +++--- .../2a2f_eager_dp1tp2.sh | 18 ++ .../2a2f_eager_dp2tp1.sh | 63 ++---- .../2a2f_graph_dbo_dp2tp1.sh | 73 ++----- .../2a2f_graph_dp1tp2.sh | 58 ------ .../2a2f_graph_dp2tp1.sh | 65 ++---- .../qwen3_235b_a22b_fp8/README.md | 119 ++++------- .../4a2f_dbo_dp4tp1.sh | 19 ++ .../4a2f_eager_dp4tp1.sh | 18 ++ .../4a2f_graph_dbo_dp4tp1.sh | 68 ------- .../4a2f_graph_dp4tp1.sh | 69 ++----- .../multipod_4a1f_4a1f_dbo.sh | 19 ++ .../multipod_4a1f_4a1f_graph.sh | 82 -------- .../qwen3_30b_a3b/README.md | 78 +++----- .../2a2f_eager_dp1tp2.sh | 18 ++ .../2a2f_eager_dp2tp1.sh | 61 ++---- .../2a2f_graph_dbo_dp2tp1.sh | 71 ++----- .../2a2f_graph_dp1tp2.sh | 56 ------ .../2a2f_graph_dp2tp1.sh | 63 ++---- recipe/gpu/P2pNcclAFDConnector/README.md | 41 ++-- .../deepseek_v2_lite/README.md | 97 ++++----- .../2a2f_eager_dbo_dp1tp2.sh | 64 ++---- .../2a2f_eager_dbo_dp2tp1.sh | 64 ++---- .../2a2f_graph_dbo_dp1tp2.sh | 70 ++----- .../2a2f_graph_dbo_dp2tp1.sh | 70 ++----- .../4a4f_eager_dbo_dp2tp2.sh | 64 ++---- .../4a4f_graph_dbo_dp2tp2.sh | 70 ++----- .../multipod_2a_2f_graph_dbo_dp1tp2.sh | 82 ++------ .../multipod_2a_2a_2f_graph.sh | 113 ++--------- recipe/gpu/common/README.md | 161 +++++++++++++++ recipe/gpu/common/afd_colocation.sh | 116 +++++++++++ tools/benchmarks/request_generator.sh | 6 +- 42 files changed, 1002 insertions(+), 1617 deletions(-) create mode 100644 recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_eager_dp1tp2.sh delete mode 100644 recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dp1tp2.sh create mode 100644 recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/4a2f_dbo_dp4tp1.sh create mode 100644 recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/4a2f_eager_dp4tp1.sh delete mode 100644 recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/4a2f_graph_dbo_dp4tp1.sh create mode 100644 recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_dbo.sh delete mode 100644 recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_graph.sh create mode 100644 recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_eager_dp1tp2.sh delete mode 100644 recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_graph_dp1tp2.sh create mode 100644 recipe/gpu/common/README.md create mode 100644 recipe/gpu/common/afd_colocation.sh diff --git a/.agents/skills/deploy-afd-k8s/SKILL.md b/.agents/skills/deploy-afd-k8s/SKILL.md index 8a528f701..a7ef9b68e 100644 --- a/.agents/skills/deploy-afd-k8s/SKILL.md +++ b/.agents/skills/deploy-afd-k8s/SKILL.md @@ -1,6 +1,6 @@ --- name: deploy-afd-k8s -description: Use when the user asks to deploy, serve, or stand up an AFD GPU recipe on Kubernetes/OpenShift from a recipe .sh script - creating the model PVC, recipe ConfigMap, serve pod(s), and Service(s), and waiting until the endpoint answers. Placement across pods is never asked for or assumed -- it is read directly off the recipe's own top-of-file comment block (however that block describes it, strict or prose): a recipe whose comment describes a per-pod placement deploys exactly that many pods with exactly that per-pod setup; a recipe without one deploys as a single pod. Do not use for local (non-k8s) serving, NPU recipes, prefill-decode disaggregation recipes, or E2E correctness testing (see run-e2e). +description: Use when the user asks to deploy, serve, or stand up an AFD GPU recipe on Kubernetes/OpenShift from a recipe .sh script - creating the model PVC, recipe ConfigMap, serve pod(s), and Service(s), and waiting until the endpoint answers. Placement across pods is never asked for or assumed -- it is the recipe's LAYOUT (the caller's setting, else the script's default): one pod per LAYOUT entry, a one-entry LAYOUT being a single pod. Do not use for local (non-k8s) serving, NPU recipes, prefill-decode disaggregation recipes, or E2E correctness testing (see run-e2e). --- # Deploy an AFD GPU recipe on Kubernetes @@ -15,13 +15,13 @@ On success this skill guarantees, in the target namespace: | | | |---|---| -| Endpoint | `http://vllm-service:${CLIENT_PORT}` (OpenAI-compatible, in-namespace; `CLIENT_PORT` defaults to `18305`) | +| Endpoint | `http://vllm-service:${CLIENT_PORT}` (OpenAI-compatible, in-namespace; `CLIENT_PORT` is the launcher's Attention port, `18000`) | | Serving | the model named by `MODEL_ID` | -| Pod(s) / Service | `vllm-pod`, or one pod per pod the recipe's header comment describes (named `vllm-pod-`); Service `vllm-service`, labels `app=afd-recipe,role=serve` | +| Pod(s) / Service | `vllm-pod`, or one pod per `LAYOUT` entry (named `vllm-pod-afd-`); Service `vllm-service`, labels `app=afd-recipe,role=serve` | | Model cache | `PVC_NAME`, mounted at `/models`, `HF_HOME=/models/.hf_home` | | Left running | yes, deliberately -- so weights stay warm for follow-up runs; step 5 asks the caller when to delete the pods | -Whenever a recipe's header spans more than one pod, two more internal-only +Whenever a recipe's `LAYOUT` spans more than one pod, two more internal-only Services may also exist (`vllm-ffn-p2p-service` always; `vllm-attn-dp-service` and/or `vllm-ffn-dp-service` only if that role is itself split across more than one pod) -- callers never talk to these directly. See @@ -38,15 +38,15 @@ than one pod) -- callers never talk to these directly. See NIXL-enabled image plus a proxy, and resolve a `SCRIPT_DIR`-relative path that this skill's flat script mount (step 4b) breaks. Ask the user for a different workflow for disaggregation. -- Recipes are hand-written shell scripts, not JSON configs -- there is no - generator step and no manifest to read. -- A recipe is **placement-driven** (more than one pod) exactly when its - top-of-file comment block describes a per-pod placement -- read that - block, don't infer shape from flags. Today's recipes spell it as one - `# For pod N: POD=...` line per pod, but the comment isn't guaranteed to - use that exact grammar going forward. See - [resolve-recipe.md](resources/resolve-recipe.md). -- Not for `tools/benchmarks/decode_bench_server.sh` (local, non-k8s). +- Recipes are short shell scripts on the shared launcher + `recipe/gpu/common/afd_colocation.sh` (see `recipe/gpu/common/README.md`); + the recipe ConfigMap ships both files (step 4b). +- Placement is `LAYOUT` (one pod per `AF` entry) and run-specific + settings are env (`ATTN_TP`, `FFN_TP`, `VLLM_ARGS`, `DECODE_BENCH`, ...); + see [resolve-recipe.md](resources/resolve-recipe.md). This skill only + serves; load for the decode-only benchmark + (`tools/benchmarks/request_generator.sh`) is run separately against the + endpoint. ## Requirements @@ -63,9 +63,9 @@ than one pod) -- callers never talk to these directly. See docker push "$IMAGE" ``` - The image supplies only the `afd-plugin` install. The recipe script is - mounted fresh from local disk on every run (step 4b), so editing or adding - a recipe never requires a rebuild. + The image supplies only the `afd-plugin` install. The recipe script and + the shared launcher are mounted fresh from local disk on every run (step + 4b), so editing or adding a recipe never requires a rebuild. - An `hf-token-secret` Secret with a `token` key, needed when `MODEL_ID` is a HF Hub repo id (not needed when weights are already staged on the PVC): ```bash @@ -81,7 +81,7 @@ exist **before** any Pod is deployed -- see | Resource | Covers | |---|---| -| [resolve-recipe.md](resources/resolve-recipe.md) | Step 1 (the only step that parses the recipe): read the recipe's serve-block fields and model id; read its placement (or lack of one) from its header comment -- never ask; record `GPU_COUNT` per pod. | +| [resolve-recipe.md](resources/resolve-recipe.md) | Step 1 (the only step that parses the recipe): model id, the `LAYOUT` plan (never ask), `GPU_COUNT` per pod, and the launcher's ports. | | [deploy.md](resources/deploy.md) | Steps 2-4f: take step 1's `GPU_COUNT`, confirm, PVC, ConfigMap, fsGroup, Services (step 4d -- which of the four, see [services.md](resources/services.md)), pod(s) via `templates/pod.yaml`, wait for Running and then for each pod's `Ready` condition. | | [report-and-teardown.md](resources/report-and-teardown.md) | Step 5: report the endpoint/model/nodes and ask when to delete the running pods. Step 6: teardown when the caller says so (pods/Services only -- never the PVC unless asked). | diff --git a/.agents/skills/deploy-afd-k8s/resources/deploy.md b/.agents/skills/deploy-afd-k8s/resources/deploy.md index 4bffc08c5..0a971ec66 100644 --- a/.agents/skills/deploy-afd-k8s/resources/deploy.md +++ b/.agents/skills/deploy-afd-k8s/resources/deploy.md @@ -3,8 +3,14 @@ ## 2. GPU_COUNT Use the `GPU_COUNT` recorded by [resolve-recipe.md](resolve-recipe.md) -- -one value for a plain recipe, one per plan entry. Don't re-read the script -here; all recipe parsing lives in step 1. +`a + f` per `LAYOUT` entry. Don't re-read the script here; all recipe +parsing lives in step 1: + +```bash +LAYOUT=${LAYOUT:-$(grep -oE 'LAYOUT:-[0-9AF,]+' "$RECIPE_SCRIPT_PATH" | cut -d- -f2)} +IFS=, read -ra ENTRIES <<<"$LAYOUT" +for e in "${ENTRIES[@]}"; do f=${e#*A}; echo "$e -> $(( ${e%%A*} + ${f%F} )) GPUs"; done +``` ## 3. Confirm before touching the cluster @@ -19,7 +25,7 @@ existing Pod(s)** with the same name(s) about to be deployed. AFD_PLUGIN_IMAGE= MODEL_ID= PVC_NAME= -CLIENT_PORT= # from the script's --port; default 18305 +CLIENT_PORT= # the launcher's ATTN_PORT (18000), see resolve-recipe.md RECIPE_SCRIPT_PATH= ``` @@ -72,11 +78,14 @@ rather than re-downloading. ## 4b. Recipe ConfigMap Lets the pod run any local recipe -- edited, uncommitted, or new -- without -rebuilding the image. Run from wherever `RECIPE_SCRIPT_PATH` resolves: +rebuilding the image. The recipe sources the shared launcher, so ship both; +`templates/pod.yaml` points the recipe at `/recipe/afd_colocation.sh` +(`AFD_RECIPE_LIB`). Run from the repository root: ```bash kubectl create configmap afd-recipe-script \ --from-file=recipe.sh="${RECIPE_SCRIPT_PATH}" \ + --from-file=afd_colocation.sh=recipe/gpu/common/afd_colocation.sh \ --dry-run=client -o yaml | kubectl apply -f - ``` @@ -108,17 +117,19 @@ recipe) against the one shared `templates/pod.yaml`: ```bash deploy_pod() { local pod_name="$1" gpu="$2" attn_role="$3" ffn_role="$4" extra_env_pairs="$5" - # extra_env_pairs: space-separated VAR=value tokens straight off this - # plan entry's header line (including POD=... itself), empty for a plain - # recipe. Build the spliced env-entries block for templates/pod.yaml. - # Leading (not trailing) newlines keep the `# ${TEMPLATE_EXTRA_ENV}` - # placeholder line a harmless comment when this is empty, and valid - # sibling list items when it isn't. - local extra_env="" - for pair in $extra_env_pairs; do + # extra_env_pairs: one VAR=value per line -- this plan entry's overrides + # (including POD=... itself) plus the caller's settings -- empty for a plain + # recipe. Values may hold spaces and JSON (VLLM_ARGS, DECODE_BENCH); they + # become single-quoted YAML strings. Leading (not trailing) newlines keep + # the `# ${TEMPLATE_EXTRA_ENV}` placeholder line a harmless comment when + # this is empty, and valid sibling list items when it isn't. + local extra_env="" pair val q="'" + while IFS= read -r pair; do + [ -n "$pair" ] || continue + val=${pair#*=} extra_env="$extra_env - - {name: ${pair%%=*}, value: \"${pair#*=}\"}" - done + - {name: ${pair%%=*}, value: '${val//$q/$q$q}'}" + done <<<"$extra_env_pairs" kubectl delete pod "$pod_name" --ignore-not-found @@ -131,26 +142,30 @@ deploy_pod() { } ``` -Plain recipe, one pod, both roles head, no overrides: +One-entry `LAYOUT` (e.g. the default `2A2F`), one pod, both roles head, +with or without the caller's settings: ```bash deploy_pod vllm-pod "$GPU_COUNT" head head "" +deploy_pod vllm-pod 4 head head $'VLLM_ARGS=--max-num-seqs 32' ``` -Placement-driven recipe, one call per entry parsed in -[resolve-recipe.md](resolve-recipe.md), e.g. the qwen3.5 3-pod plan: +Several entries, one call per entry parsed in +[resolve-recipe.md](resolve-recipe.md), e.g. `LAYOUT=2A0F,0A2F` (2 GPUs per +pod) with the decode-only benchmark, `LAYOUT=4A1F,4A1F` (5 GPUs per pod), +or the Qwen3.5 recipe's default `2A0F,2A0F,0A2F`: ```bash -deploy_pod vllm-pod-attention-0 2 head none "POD=ATTENTION_0" -deploy_pod vllm-pod-attention-1 2 worker none "POD=ATTENTION_1 ATTENTION_HEADLESS=1 ATTENTION_DP_START_RANK=2" -deploy_pod vllm-pod-ffn-0 2 none head "POD=FFN_0" -``` +SETTINGS=$'LAYOUT=2A0F,0A2F\nDECODE_BENCH={}\nAFD_REPO=/opt/afd-plugin' +deploy_pod vllm-pod-afd-0 2 head none $'POD=AFD_0\n'"$SETTINGS" +deploy_pod vllm-pod-afd-1 2 none head $'POD=AFD_1\n'"$SETTINGS" -`POD=AFD_` recipe, e.g. the DeepEP M2N 2 x 4A1F plan (5 GPUs per pod): +deploy_pod vllm-pod-afd-0 5 head head $'POD=AFD_0\nLAYOUT=4A1F,4A1F' +deploy_pod vllm-pod-afd-1 5 worker worker $'POD=AFD_1\nLAYOUT=4A1F,4A1F' -```bash -deploy_pod vllm-pod-afd-0 5 head head "POD=AFD_0" -deploy_pod vllm-pod-afd-1 5 worker worker "POD=AFD_1" +deploy_pod vllm-pod-afd-0 2 head none 'POD=AFD_0' +deploy_pod vllm-pod-afd-1 2 worker none 'POD=AFD_1' +deploy_pod vllm-pod-afd-2 2 none head 'POD=AFD_2' ``` ## 4f. Wait for Running, then for each pod's Ready condition @@ -198,6 +213,6 @@ done Only report the endpoint ready once every pod in the plan is `Ready` -- one pod becoming ready doesn't imply the others did. -Startup markers are keyed on the fixed log names every colocation recipe -uses today -- see `templates/pod.yaml`'s `ready_marker()` for the exact -lines. A recipe using different log names needs that template updated. +Startup markers are keyed on the launcher's fixed log names (`attn.log`, +`ffn.log`) -- see `templates/pod.yaml`'s `ready_marker()` for the exact +lines. diff --git a/.agents/skills/deploy-afd-k8s/resources/resolve-recipe.md b/.agents/skills/deploy-afd-k8s/resources/resolve-recipe.md index 1a7f5f0bb..7c2995ffa 100644 --- a/.agents/skills/deploy-afd-k8s/resources/resolve-recipe.md +++ b/.agents/skills/deploy-afd-k8s/resources/resolve-recipe.md @@ -1,4 +1,4 @@ -# 1. Resolve the recipe, its model, and its placement +# 1. Resolve the recipe, model, and pod resources Take the recipe path from the user, or ask. It must be a colocation recipe under `recipe/gpu/P2pNcclAFDConnector/**/prefill_decode_colocation/` or @@ -6,130 +6,69 @@ under `recipe/gpu/P2pNcclAFDConnector/**/prefill_decode_colocation/` or only; prefill/decode disaggregation recipes are out of scope -- see `SKILL.md` Scope). -Read the script and note, per `vllm serve` block (each backgrounded process -ending `> .log 2>&1 &`): - -- `CUDA_VISIBLE_DEVICES`, `--data-parallel-size`, `--tensor-parallel-size` -- the `"afd": {"role": ...}` block in `--additional-config` -- eager vs graph, and every other flag (`--enable-expert-parallel`, - `--max-num-seqs`, `--max-num-batched-tokens`, `--max-model-len`, - `--trust-remote-code`, host/port) - -Record `--max-num-batched-tokens` and report it to the caller later -- it -caps how much prefill work a single step can absorb. - -Resolve `MODEL_ID`: read the script's `MODEL_PATH` default. It is either a -HF Hub id (map the recipe's model directory name to one, e.g. -`deepseek_v2_lite` -> `deepseek-ai/DeepSeek-V2-Lite`; ask if the mapping -isn't obvious) or a **placeholder** path of the form -`/path/model_weights/` -- every colocation recipe uses exactly this -literal `/path/` stand-in today, regardless of what's actually staged on any -given cluster's PVC. Never pass this through verbatim: it names only the -model directory (``), not a real in-container path. Derive the real -`MODEL_ID` as `/`, where `` is the PVC mount -point fixed by `templates/pod.yaml` (`/models`) -- e.g. -`/path/model_weights/Qwen3.5-122B-A10B-FP8` resolves to -`/models/Qwen3.5-122B-A10B-FP8`. Before deploying, confirm that directory -actually exists on the target `PVC_NAME` (if it already exists, `kubectl -exec` into any pod already mounting it, e.g. `ls /models`, or spin up a -short-lived busybox pod mounting the PVC) -- don't rely on the recipe's -claim; a mismatch here fails fast inside the pod with a confusing -HF-repo-id validation error instead of a clear message. - -Tell the caller which form it is, and the resolved value if derived. - -Set `CLIENT_PORT` to the `--port` value on the attention (or sole, for a -single-pod recipe) `vllm serve` block -- `templates/pod.yaml` rebinds this -port off loopback and `vllm-service` (step 4d, see -[services.md](services.md)) exposes it, whatever value is actually there. - -## Read the placement plan -- never ask - -Look at the top-of-file comment block (above the first `vllm serve`/env-var -line). If it says nothing about per-pod placement, this is a plain -single-pod recipe: one pod, `POD` unset, both roles run unconditionally -- -continue to [deploy.md](deploy.md) with no plan. - -If it does describe a placement across multiple pods, read and interpret -that description directly. - -Example of a placement description: -``` -# For pod 1: POD=ATTENTION_0 -# For pod 2: POD=ATTENTION_1, ATTENTION_HEADLESS=1, ATTENTION_DP_START_RANK=2 -# For pod 3: POD=FFN_0 -``` - -Or -``` -"runs -on 3 pods: the first two run ATTENTION_0 and ATTENTION_1 -- the second one -headless, starting at data-parallel rank 2 -- and the third runs FFN_0" -``` - -Whatever the phrasing, read it and extract, per pod, in the order the -comment presents them: - -- that pod's `POD=` identity -- any further container-env overrides for that pod only (e.g. - `ATTENTION_HEADLESS=1`, `ATTENTION_DP_START_RANK=2`) - -The recipe already states ranks/ordering per pod -- there is nothing to -derive or ask about, regardless of how it's phrased. (A quick -`grep -E '^# For pod [0-9]+: '` is a fine way to spot today's convention at -a glance, but its absence doesn't mean "no plan" -- check the comment block -itself before concluding this is a plain single-pod recipe.) - -For each entry, derive (used by [deploy.md](deploy.md) and -[services.md](services.md)): - -- **k8s Pod name**: `vllm-pod-$(echo "$POD" | tr 'A-Z_' 'a-z-')`, e.g. - `vllm-pod-attention-0`. -- **`afd-attn-node-role`**: `none` unless `POD` starts with `ATTENTION_`, in - which case `worker` if that line's `ATTENTION_HEADLESS` override is `1`, - else `head`. -- **`afd-ffn-node-role`**: symmetric, `none` unless `POD` starts with `FFN_`. -- **`GPU_COUNT`**: the number of devices in the literal - `CUDA_VISIBLE_DEVICES=` of the `vllm serve` block this pod runs -- the one - inside the recipe's branch that tests `$POD` for this pod's role - (`ATTENTION` or `FFN`). Every pod sharing a role shares that literal, so - read it off the script; don't derive it from ranks or DP sizes. - -### Pods that run both roles: `POD=AFD_` - -DeepEP M2N multi-pod recipes (e.g. -`DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_graph.sh`) -list `POD=AFD_0`, `POD=AFD_1`, ... with no other per-pod overrides. Every -such pod runs one FFN and one Attention server and derives its DP flags -(headless, start rank) from its own index, so: - -- **`afd-attn-node-role`** and **`afd-ffn-node-role`**: both `head` for - `AFD_0`, both `worker` for every other `AFD_`. -- The pod shape comes from the script's `NUM_PODS`, `ATTN_PER_POD` and - `FFN_PER_POD` defaults (`NUM_PODS` must equal the number of entries). If - the user overrides them, pass the same values to every pod as container - env. -- **`GPU_COUNT`**: `ATTN_PER_POD + FFN_PER_POD`, the same for every pod. - These recipes build `CUDA_VISIBLE_DEVICES` with `$(seq ...)`, so the - literal-device rules above and below don't apply: - - ```bash - default() { grep -oE "$1:-[0-9]+" "$RECIPE_SCRIPT_PATH" | grep -oE '[0-9]+$'; } - GPU_COUNT=$(( ${ATTN_PER_POD:-$(default ATTN_PER_POD)} + ${FFN_PER_POD:-$(default FFN_PER_POD)} )) - ``` -- Ports are literals in the script, not `*_PORT:-N` defaults: the AFD - rendezvous port is the `"port": N` of the `afd` JSON, and the FFN and - Attention DP RPC ports are the last argument of the `role_dp "$F" ...` and - `role_dp "$A" ...` lines. - -## GPU_COUNT for a plain single-pod recipe - -The union of every literal `CUDA_VISIBLE_DEVICES=` in the script: +Every such recipe states its `MODEL_PATH`, default `LAYOUT`, mode, and serve +flags, then sources the shared launcher, +`recipe/gpu/common/afd_colocation.sh`. Treat the launcher as the execution +contract: do not reconstruct or parse its `vllm serve` commands to determine +Kubernetes resources. + +## Model + +Resolve `MODEL_ID` from the caller's `MODEL_PATH` if given, else the recipe's +default. It is either a HF Hub id or the placeholder +`/path/model_weights/` used by every colocation recipe. +Never pass this through verbatim: derive the real `MODEL_ID` as +`/models/` (`/models` is the PVC mount point fixed by +`templates/pod.yaml`), e.g. `/path/model_weights/Qwen3.5-122B-A10B-FP8` +resolves to `/models/Qwen3.5-122B-A10B-FP8`. Before deploying, confirm that +directory exists on the target `PVC_NAME` (`kubectl exec` into any pod +already mounting it, e.g. `ls /models`, or a short-lived busybox pod +mounting the PVC); a mismatch fails inside the pod with a confusing +HF-repo-id validation error. Tell the caller which form it is, and the +resolved value if derived. + +## Pod resource plan -- never ask + +Take `LAYOUT` from the caller's settings, else the recipe's default; never +ask. The selected `LAYOUT`, not a recipe filename or descriptive comment, +controls the plan. Split it by comma into one `AF` entry per pod. For +entry `k`, `a` is that pod's Attention GPU count and `f` its FFN GPU count: + +- **k8s Pod name**: `vllm-pod-afd-`. A one-entry `LAYOUT` (e.g. `2A2F`) + is a plain single pod: `vllm-pod`, no `POD` needed. +- **`GPU_COUNT`**: `a + f`, passed to `templates/pod.yaml` as `TEMPLATE_GPU` + so the container requests that many `nvidia.com/gpu` devices. +- **`afd-attn-node-role`**: `none` when `a` is 0, `head` on the first entry + with `a > 0`, `worker` on later ones. **`afd-ffn-node-role`**: the same + with `f`. For example, `2A0F,0A2F` gives `head none` then `none head`. +- **Container env**: every pod gets the selected `LAYOUT`, its own + `POD=AFD_`, and the caller's other recipe overrides unchanged. Leave + `HEAD_HOST`, `ATTN_HEAD_HOST`, and `FFN_HEAD_HOST` unset; their defaults + name the Services in [services.md](services.md). With `DECODE_BENCH`, also + set `AFD_REPO=/opt/afd-plugin`. + +The launcher owns role-to-device ordering, DP/TP command construction, and +startup validation of the placement. Record the pod/resource table for +[deploy.md](deploy.md), which uses it as-is. + +## Launcher ports + +Read the client and internal Service ports from the launcher's fixed assignments: ```bash -GPU_COUNT="$(grep -oE 'CUDA_VISIBLE_DEVICES=[0-9,]+' "$RECIPE_SCRIPT_PATH" \ - | cut -d= -f2 | tr ',' '\n' | sort -u | wc -l)" +RECIPE_LIB_PATH=recipe/gpu/common/afd_colocation.sh +lib_port() { + grep -oE "\b$1=[0-9]+" "$RECIPE_LIB_PATH" | head -1 | cut -d= -f2 +} + +CLIENT_PORT="$(lib_port ATTN_PORT)" +AFD_CONNECTOR_PORT="$(lib_port AFD_PORT)" +ATTN_DP_RPC_PORT="$(lib_port ATTN_DP_RPC_PORT)" +FFN_DP_RPC_PORT="$(lib_port FFN_DP_RPC_PORT)" ``` -This step is the only place that parses the recipe: -[deploy.md](deploy.md) uses the `GPU_COUNT` (and plan) recorded here as-is. +## Other deployment values + +Record the effective `--max-num-batched-tokens` -- the caller's `VLLM_ARGS` +value if overridden, else the recipe's `SERVE_ARGS` value -- for the final +report. diff --git a/.agents/skills/deploy-afd-k8s/resources/services.md b/.agents/skills/deploy-afd-k8s/resources/services.md index 761d23e65..94173686c 100644 --- a/.agents/skills/deploy-afd-k8s/resources/services.md +++ b/.agents/skills/deploy-afd-k8s/resources/services.md @@ -24,32 +24,28 @@ CLIENT_PORT="$CLIENT_PORT" envsubst '${CLIENT_PORT}' \ Internal only, carries AFD rendezvous traffic to whichever pod holds `afd-ffn-node-role: head`. Must exist before any multi-pod deploy, since the -recipe's own `AFD_CONNECTOR_HOST` default now points at this name directly. -Traffic to a headless Service goes straight to the pod IP, and there's one -rendezvous port, so a single static port entry is enough -- read -`AFD_CONNECTOR_PORT` off the recipe's own shell default: +launcher's default rendezvous host points at this name directly. Traffic to +a headless Service goes straight to the pod IP, and there's one rendezvous +port, so a single static port entry is enough (`AFD_CONNECTOR_PORT` from +[resolve-recipe.md](resolve-recipe.md#launcher-ports)): ```bash -AFD_CONNECTOR_PORT="$(grep -oE 'AFD_CONNECTOR_PORT:-[0-9]+' "$RECIPE_SCRIPT_PATH" | grep -oE '[0-9]+')" -AFD_CONNECTOR_PORT="$AFD_CONNECTOR_PORT" envsubst '${AFD_CONNECTOR_PORT}' \ +AFD_CONNECTOR_PORT="${AFD_CONNECTOR_PORT}" envsubst '${AFD_CONNECTOR_PORT}' \ < templates/service-ffn-p2p.yaml | kubectl apply -f - ``` ## `vllm-attn-dp-service` / `vllm-ffn-dp-service` -- iff that role is split -Only when **more than one** plan entry's `POD` carries the same role (i.e. -that role is split across pods, not just placed in its own single pod). -Every `POD=AFD_` entry carries both roles, so a plan of two or more -such entries needs both Services; read their ports (and the rendezvous port) -as [resolve-recipe.md](resolve-recipe.md) describes for those recipes rather -than with the `*_PORT:-N` greps below. -Carries that role's DP-RPC coordination traffic to its head pod. One per -split role, generic over role via `templates/service-dp-coord.yaml`: +Only when **more than one** `LAYOUT` entry has GPUs of that role (both +roles in `1A1F,1A1F`, Attention in `2A0F,2A0F,0A2F`, neither in +`2A0F,0A2F`). Carries that role's DP-RPC coordination traffic to its head +pod. One per split role, generic over role via +`templates/service-dp-coord.yaml`: ```bash -TEMPLATE_ROLE=attn -TEMPLATE_PORT="$(grep -oE 'ATTENTION_DP_RPC_PORT:-[0-9]+' "$RECIPE_SCRIPT_PATH" | grep -oE '[0-9]+')" -TEMPLATE_ROLE="$TEMPLATE_ROLE" TEMPLATE_PORT="$TEMPLATE_PORT" \ +TEMPLATE_ROLE=attn TEMPLATE_PORT="${ATTN_DP_RPC_PORT}" \ + envsubst '${TEMPLATE_ROLE} ${TEMPLATE_PORT}' < templates/service-dp-coord.yaml | kubectl apply -f - +TEMPLATE_ROLE=ffn TEMPLATE_PORT="${FFN_DP_RPC_PORT}" \ envsubst '${TEMPLATE_ROLE} ${TEMPLATE_PORT}' < templates/service-dp-coord.yaml | kubectl apply -f - ``` diff --git a/.agents/skills/deploy-afd-k8s/templates/pod.yaml b/.agents/skills/deploy-afd-k8s/templates/pod.yaml index 561f16f18..f43a0694e 100644 --- a/.agents/skills/deploy-afd-k8s/templates/pod.yaml +++ b/.agents/skills/deploy-afd-k8s/templates/pod.yaml @@ -33,33 +33,19 @@ spec: - -c - | set -euo pipefail - RECIPE_SCRIPT=/recipe/recipe.sh - mkdir -p /work/bin - # Recipes only ever call `uv run `; drop the `run` and exec - # the rest directly instead of invoking uv at all. - printf '#!/bin/bash\nshift\nexec "$@"\n' > /work/bin/uv - chmod +x /work/bin/uv - export PATH="/work/bin:$PATH" - - echo "=== launching $(basename "$RECIPE_SCRIPT") (POD=${POD:-} attn=${AFD_ATTN_NODE_ROLE} ffn=${AFD_FFN_NODE_ROLE}) ===" - - # Rebind the client-facing port off loopback: every `--host - # 127.0.0.1` on the branch is an API-server flag (the recipe binds - # :${TEMPLATE_CLIENT_PORT} on 127.0.0.1), but the Service needs - # 0.0.0.0. - RECIPE_BASENAME="$(basename "$RECIPE_SCRIPT" .sh)" - PATCHED_SCRIPT="/work/${RECIPE_BASENAME}.patched.sh" - sed 's/--host 127\.0\.0\.1/--host 0.0.0.0/' "$RECIPE_SCRIPT" > "$PATCHED_SCRIPT" + echo "=== launching recipe.sh (POD=${POD:-} attn=${AFD_ATTN_NODE_ROLE} ffn=${AFD_FFN_NODE_ROLE}) ===" + # The recipe sources the launcher from AFD_RECIPE_LIB and binds its + # API server to SERVE_HOST (0.0.0.0, so the Service can reach it). cd /work - bash "$PATCHED_SCRIPT" & + bash /recipe/recipe.sh & RECIPE_PID=$! echo "$RECIPE_PID" > /work/recipe.pid # Which log(s) this pod will actually produce, and this pod's own # readiness marker per log -- both keyed on this pod's node-role # labels, not re-derived from $POD. Fixed to the attn.log/ffn.log - # names every colocation recipe uses today. + # names of the colocation launcher. LOGS="" [ "$AFD_ATTN_NODE_ROLE" != "none" ] && LOGS="$LOGS attn.log" [ "$AFD_FFN_NODE_ROLE" != "none" ] && LOGS="$LOGS ffn.log" @@ -148,6 +134,10 @@ spec: value: "${TEMPLATE_ATTN_ROLE}" - name: AFD_FFN_NODE_ROLE value: "${TEMPLATE_FFN_ROLE}" + - name: AFD_RECIPE_LIB + value: "/recipe/afd_colocation.sh" + - name: SERVE_HOST + value: "0.0.0.0" # ${TEMPLATE_EXTRA_ENV} - name: HF_TOKEN valueFrom: diff --git a/.agents/skills/review-pr/references/features/disaggregation-topologies.md b/.agents/skills/review-pr/references/features/disaggregation-topologies.md index b5d920306..af959a048 100644 --- a/.agents/skills/review-pr/references/features/disaggregation-topologies.md +++ b/.agents/skills/review-pr/references/features/disaggregation-topologies.md @@ -14,10 +14,11 @@ recipes under `recipe/gpu/` and `recipe/npu/`. | Topology | Where it lives | | --- | --- | | Baseline (native DP/TP, no AFD) | `baseline-graph` E2E scenarios; native DP4/TP1/EP4 | -| CUDA NCCL P2P, colocation | `recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/` (2A2F and 4A4F, eager/graph, DBO variants, plus `multipod_2a_2f_graph_dbo_dp1tp2.sh`) | +| CUDA NCCL P2P, colocation | `recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/` (explicit 2A2F, 4A4F and two-pod graph/eager DBO recipes) | | CUDA NCCL P2P, disaggregation | `recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_disaggregation/` (2P1A1F eager/graph DBO) | -| CUDA NCCL P2P, Qwen3.5 multi-pod | `recipe/gpu/P2pNcclAFDConnector/qwen3_5_122b_a10b_fp8/prefill_decode_colocation/multipod_2a_2a_2f_graph.sh` | -| CUDA DeepEP M2N (routed experts) | `recipe/gpu/DeepEPM2NAFDConnector/{deepseek_v2_lite,qwen3_30b_a3b,qwen3_235b_a22b_fp8}/` (2A2F, and 4A2F for the 235B FP8 case) | +| CUDA NCCL P2P, Qwen3.5 multi-pod | `recipe/gpu/P2pNcclAFDConnector/qwen3_5_122b_a10b_fp8/prefill_decode_colocation/multipod_2a_2a_2f_graph.sh` (default `LAYOUT=2A0F,2A0F,0A2F`) | +| CUDA DeepEP M2N (routed experts) | `recipe/gpu/DeepEPM2NAFDConnector/{deepseek_v2_lite,qwen3_30b_a3b,qwen3_235b_a22b_fp8}/` (graph/eager/dbo, any `LAYOUT`) | +| GPU colocation launcher | `recipe/gpu/common/afd_colocation.sh`, shared by every GPU colocation recipe | | NPU CAM P2P disaggregation | `recipe/npu/CAMP2pAFDConnector/deepseek_v3_2/` (attention/FFN/baseline scripts) | | NPU async CAM | `recipe/npu/CAMAsyncAFDConnector/deepseek_v3_2/v0_26_accuracy/` and `deepseek_v4_flash/` (DSpark A/F) | | Gate matrices | `tests/e2e/models/*/` scenario lists; `docs/design/module/e2e_testing.md` | diff --git a/.agents/skills/run-vllm-bench-k8s/SKILL.md b/.agents/skills/run-vllm-bench-k8s/SKILL.md index e418a05fa..c52b400a0 100644 --- a/.agents/skills/run-vllm-bench-k8s/SKILL.md +++ b/.agents/skills/run-vllm-bench-k8s/SKILL.md @@ -42,10 +42,10 @@ folder as the list of possible recipes. Follow **`deploy-afd-k8s`** with `RECIPE_SCRIPT_PATH`. It returns `MODEL_ID`, `PVC_NAME`, `GPU_COUNT`, the serving node, the recipe's -`--max-num-batched-tokens`, and an endpoint at `http://vllm-service:18305`. +`--max-num-batched-tokens`, and an endpoint at `http://vllm-service:${CLIENT_PORT}` (18000). `vllm bench serve` is pointed at that endpoint with `HOST=vllm-service` -`PORT=18305`, and `MODEL_PATH=${MODEL_ID}` -- it must name exactly what the +`PORT=18000` (the launcher's Attention port), and `MODEL_PATH=${MODEL_ID}` -- it must name exactly what the pod serves, since the model id is sent in each request and used to load the tokenizer. @@ -195,7 +195,7 @@ spec: conc="${point##*:}" echo "=== sweep point rate=${rate} concurrency=${conc} ===" MODEL_PATH="${MODEL_ID}" \ - HOST=vllm-service PORT=18305 \ + HOST=vllm-service PORT=18000 \ DATASET_NAME="${DATASET_NAME}" \ INPUT_LEN="${INPUT_LEN}" OUTPUT_LEN="${OUTPUT_LEN}" \ NUM_PROMPTS="${NUM_PROMPTS}" \ diff --git a/docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md b/docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md index cc9aad6c3..76a2333bb 100644 --- a/docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md +++ b/docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md @@ -291,10 +291,9 @@ the headless pods. The FFN DP rank is the FFN's EP rank, so it decides which experts a pod loads; `--enable-ep-weight-filter` keeps each FFN from reading the other experts' weights. -The recipe -[`multipod_4a1f_4a1f_graph.sh`](../../recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/README.md#multi-pod-2-x-4a1f) -derives all of these flags for symmetric placements `N x AF` from the -pod index (`POD=AFD_`). +The [DeepEP M2N recipes](../../recipe/gpu/DeepEPM2NAFDConnector/README.md#layout) +derive all of these flags from a `LAYOUT` such as `2A0F,0A2F` or `4A1F,4A1F` +and the pod index (`POD=AFD_`). ## Minimal launch shape diff --git a/recipe/README.md b/recipe/README.md index 364b3be37..3e7cb9fd3 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -23,8 +23,9 @@ Recipes are organized by hardware backend, connector, and model: | --- | --- | --- | --- | --- | --- | | GPU | `P2pNcclAFDConnector` | DeepSeek-V2-Lite | Decode | Deployment examples; see the linked README for validation scope | [Launch examples](gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md) | | GPU (SM90+) | `DeepEPM2NAFDConnector` | DeepSeek-V2-Lite | Decode | Deployment examples matching the checked-in E2E layouts; hardware qualification is separate | [Launch examples](gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/README.md) | -| GPU (SM90+) | `DeepEPM2NAFDConnector` | Qwen3-30B-A3B | Decode | Same 2A2F layouts as the Qwen3 MoE DeepEP M2N E2E cases | [Launch examples](gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/README.md) | -| GPU (SM90+) | `DeepEPM2NAFDConnector` | Qwen3-235B-A22B-FP8 | Decode | 4A2F FP8-payload configuration run on H200 | [Launch examples](gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/README.md) | +| GPU (SM90+ for `DeepEPM2NAFDConnector`) | `DeepEPM2NAFDConnector` | Qwen3-30B-A3B | Prefill / decode; decode-only benchmark | 2A2F graph, eager, graph+DBO and eager Attention-TP2 recipes; multi-pod layouts remain configurable through `LAYOUT` | [Graph, eager, DBO](gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/README.md) | +| GPU (SM90+ for `DeepEPM2NAFDConnector`) | `DeepEPM2NAFDConnector` | Qwen3-235B-A22B-FP8 | Prefill / decode; decode-only benchmark | FP8-payload 4A2F eager, graph and DBO recipes, plus two-pod 4A1F DBO; H200-class FFN GPUs | [Graph, eager, DBO](gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/README.md) | +| GPU | `P2pNcclAFDConnector` | Qwen3.5-122B-A10B-FP8 | Decode | Three-pod graph recipe (`2A0F,2A0F,0A2F`); other layouts remain configurable through `LAYOUT` | [Graph](gpu/P2pNcclAFDConnector/qwen3_5_122b_a10b_fp8/prefill_decode_colocation/multipod_2a_2a_2f_graph.sh) | | Ascend NPU | `CAMP2pAFDConnector` | DeepSeek-V3.2 | Decode | Historical v0.19.1rc1 recipe; not qualified for the 0.32 target | [Synchronous decode](npu/CAMP2pAFDConnector/deepseek_v3_2/README.md) | | Ascend NPU | `CAMAsyncAFDConnector` | DeepSeek-V3.2 | Prefill / decode | Historical v0.19.1rc1/v0.26 recipes; standard V3.2 is unsupported by the current Async CAM path | [Async CAM](npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md) | | Ascend NPU | `CAMAsyncAFDConnector` | DeepSeek-V4-Flash W4A8/W8A8 DSpark | Separate P / decode, or colocated | v0.32; eager A TP1, F layered eager/FULL. W4A8 PD: native full-draft P TP4, A DP2, F EP4. W8A8 A/F: A DP2, F EP8; PD not qualified | [DSpark A/F](npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md) | @@ -41,3 +42,8 @@ directory should contain a `README.md` describing prerequisites, topology, launch commands, validation steps, and limitations. Keep model-specific launch scripts, configuration files, and result images in the same directory so the recipe can be moved or linked as one unit. + +GPU `prefill_decode_colocation` recipes are short scripts on the shared +[colocation launcher](gpu/common/README.md): a recipe sets its model, +connector, default `LAYOUT` and `vllm serve` flags, and the launcher handles +placement across GPUs and pods. Start a new one from an existing recipe. diff --git a/recipe/gpu/DeepEPM2NAFDConnector/README.md b/recipe/gpu/DeepEPM2NAFDConnector/README.md index bfbb024dd..7ca658db1 100644 --- a/recipe/gpu/DeepEPM2NAFDConnector/README.md +++ b/recipe/gpu/DeepEPM2NAFDConnector/README.md @@ -2,34 +2,38 @@ Launch scripts for running Attention-FFN Disaggregation (AFD) with `DeepEPM2NAFDConnector`, the expert-routed M2N connector for Hopper (SM90) and -newer GPUs. See +newer GPUs, with vLLM `v0.32.0`. See [`docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md`](../../../docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md) for installation, the configuration contract, the required vLLM settings, and -supported layouts. +the placements the connector accepts. -## Directory layout +This README covers what every recipe here shares; each model's `README.md` +lists only its defaults and model-specific notes. + +## Recipes + +Recipe names encode their layout, DP/TP shape, and mode. DeepSeek-V2-Lite and +Qwen3-30B-A3B provide `2a2f_{eager,graph,graph_dbo}_dp2tp1.sh` plus +`2a2f_eager_dp1tp2.sh`; Qwen3-235B-A22B-FP8 provides `4a2f` eager, graph and +DBO recipes plus `multipod_4a1f_4a1f_dbo.sh`. Each model README lists its +exact filenames and defaults. ```text . -├── deepseek_v2_lite/ -│ ├── README.md -│ └── prefill_decode_colocation/ # 2A2F: eager, graph, graph + DBO -├── qwen3_30b_a3b/ -│ ├── README.md -│ └── prefill_decode_colocation/ # 2A2F: eager, graph, graph + DBO, Attention TP2 -└── qwen3_235b_a22b_fp8/ - ├── README.md - └── prefill_decode_colocation/ # 4A2F FP8 payload: graph, graph + DBO; - # 2 pods x 4A1F (multipod_4a1f_4a1f_graph.sh) +├── deepseek_v2_lite/ # BF16 payload +├── qwen3_30b_a3b/ # BF16 payload +└── qwen3_235b_a22b_fp8/ # FP8 payload (fp8_block128_fp32) ``` -Each model directory holds its own scripts; open its `README.md` before -running anything in it. +Each script sets the model, its tuning and mode flags, and runs the shared +[colocation launcher](../common/README.md), which documents the layout, +every setting, and how to run a recipe directly or in a pod. ## Prerequisites -- SM90 or newer GPUs on one node: 4 for the DeepSeek-V2-Lite and - Qwen3-30B-A3B recipes, 6 for the Qwen3-235B-A22B-FP8 recipes. +- SM90 or newer GPUs. The DeepSeek and Qwen3-30B default `2A2F` layout uses + 4 GPUs; Qwen3-235B defaults to `4A2F` on 6 GPUs. See the model README for + memory needs. - vLLM `v0.32.0`, the plugin, afdEP-comm (the `afdep` extra) and DeepGEMM installed in one environment, with the elfutils headers available to DeepGEMM's JIT. See @@ -42,37 +46,100 @@ running anything in it. export AFDEP_NCCL_ROOT=$NCCL LD_LIBRARY_PATH=$NCCL/lib:${LD_LIBRARY_PATH:-} ``` +- Free ports `6239` (AFD rendezvous), `18000` (Attention API) and `18001` + (FFN, serves no requests); across pods also `13345` (Attention DP RPC) and + `13355` (FFN DP RPC). + The scripts call `vllm serve` directly rather than through `uv run`, so they use the environment that holds afdEP-comm and DeepGEMM. -## Running on a local host +## Layout -Run any script from the repository root, for example: +`LAYOUT` lists the pods, one `AF` entry each; see +[Layout](../common/README.md#layout) and +[Settings](../common/README.md#settings). Examples for this connector: +`4A2F` (one 6-GPU node), `2A0F,0A2F` and `1A1F,1A1F` (two pods), `4A1F,4A1F` +(two 5-GPU pods). `ATTN_TP=2` runs `2A2F`'s Attention as one DP1 x TP2 engine; +the FFN runs TP 1, and its DP rank (its EP rank) decides which experts it +loads. The connector checks the placement at startup and names the pods' +shapes when it rejects one; the +[supported layouts](../../../docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md#supported-layouts) +table lists which placements it accepts (for example `4A0F,0A2F` is not, its +pods differ in size). + +## Running ```bash export MODEL_PATH=/path/model_weights/Qwen3-30B-A3B bash recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_graph_dp2tp1.sh ``` -Each script starts the FFN server and then the Attention server in the -background and writes `ffn.log` and `attn.log` into the current directory. -Wait for `Application startup complete` in both logs, then send traffic to the -Attention server on port `18000`. The AFD rendezvous uses port `6239`; the FFN -server listens on `18001` but serves no requests. - -The first start on a node compiles transport and DeepGEMM kernels, so it is -slower than later starts. +Send traffic to the Attention server on port `18000` once `attn.log` prints +`Application startup complete`. The first start on a host compiles transport +and DeepGEMM kernels, so it is slower than later starts. Raise +`CUDAGRAPH_CAPTURE_SIZES` together with `--max-num-seqs`, so decode batches +stay in a captured size. Across pods, pass the fabric's NCCL settings (for +example `NCCL_IB_HCA`) through the environment; see +[Running directly](../common/README.md#running-directly) for hosts without +Kubernetes. ## Deploying on Kubernetes -The single-node recipes have no placement header, so the -**`deploy-afd-k8s`** skill (`.agents/skills/deploy-afd-k8s/SKILL.md`) deploys -each one as a single pod. The image must contain afdEP-comm and DeepGEMM. - -`multipod_4a1f_4a1f_graph.sh` carries a `POD=AFD_` header, so the skill -deploys it as one pod per entry, each running both roles. Every pod gets the -same `NUM_PODS`, `ATTN_PER_POD` and `FFN_PER_POD` (2, 4 and 1 by default), -which select any symmetric layout `N x AF`; the FFN uses each pod's -first `f` GPUs. See -[Multi-pod deployment](../../../docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md#multi-pod-deployment) -for the placements the connector accepts. +The **`deploy-afd-k8s`** skill (`.agents/skills/deploy-afd-k8s/SKILL.md`) +deploys a recipe as one pod per `LAYOUT` entry; see +[Running in a pod](../common/README.md#running-in-a-pod). The image must +contain afdEP-comm and DeepGEMM. + +## Checking a deployment + +```bash +curl -s http://127.0.0.1:18000/v1/completions \ + -H 'Content-Type: application/json' \ + -d '{"model":"'"$MODEL_PATH"'","prompt":"The capital of France is","max_tokens":12,"temperature":0}' +``` + +The answer should name Paris. For an accuracy check, run the matching E2E case +(`afd-deepep-m2n-*`); see +[`tests/e2e`](../../../tests/e2e/README.md#deepep-m2n-scenarios). + +## Decode-only benchmark + +`DECODE_BENCH` attaches the decode-only benchmark's KV connector +(`AFDDecodeBenchConnector`) to Attention and takes its +`kv_connector_extra_config` as JSON (`{}` is enough). The connector reports +every prompt token but the last as externally computed, so requests skip +prefill and decode at a context of the full prompt length over unwritten KV. +The connector lives in `tools/benchmarks`, which is not in the wheel: run from +the repository root or set `AFD_REPO`. + +```bash +export MODEL_PATH=/path/model_weights/Qwen3-30B-A3B +DECODE_BENCH='{}' \ + bash recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_graph_dbo_dp2tp1.sh + +INPUT_LEN=1024 OUTPUT_LEN=256 NUM_PROMPTS=2000 \ + MAX_CONCURRENCY=64 REQUEST_RATE=inf RESULT_DIR=/tmp/results/decode_bench \ + tools/benchmarks/request_generator.sh +``` + +Drive the Attention port (18000). The benchmark works in every layout and +mode; the connector is +[`tools/benchmarks/decode_bench.py`](../../../tools/benchmarks/decode_bench.py). + +## Configuration notes + +- Both roles set `compute_gate_on_attention: true` and run with + `--enable-expert-parallel` and `--enable-ep-weight-filter` (in each + recipe's `SERVE_ARGS`), so each FFN rank reads only its own experts. +- The FFN keeps vLLM's default `--moe-backend auto`, which the plugin pins to + `triton` so the expert weights stay in the layout the connector reads. +- The connector derives its dispatch buckets from the CUDA graph capture + sizes and compiles one transport kernel set per bucket at startup. The + dispatch capacity is the larger of `--max-num-batched-tokens` and the + largest capture size, so lowering `--max-num-batched-tokens` shrinks the + transport buffers. +- With DBO the connector uses two lanes; DBO must be on for both roles. +- `expert_num_sms` and `num_sms` split SMs between the DeepGEMM experts and + the transport kernels; pass them with `M2N_EXTRA_CONFIG`. See + [`connector_extra_config`](../../../docs/gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md#connector_extra_config). +- `FULL_DECODE_ONLY` is the only CUDA graph mode. diff --git a/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/README.md b/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/README.md index f2ff98ed8..6f04e2dcc 100644 --- a/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/README.md +++ b/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/README.md @@ -1,46 +1,42 @@ # DeepSeek-V2-Lite with DeepEPM2NAFDConnector -Launch scripts for DeepSeek-V2-Lite (BF16) on `DeepEPM2NAFDConnector` with -vLLM `v0.32.0`. They use the same layouts as the DeepSeek-V2-Lite DeepEP M2N -E2E cases (`test_deepseek_v2_lite_deepep_m2n` in -[`tests/e2e`](../../../../tests/e2e/README.md#deepep-m2n-scenarios)). - -## Prerequisites - -- 4 SM90 or newer GPUs on one node. -- The environment described in the - [connector recipes README](../README.md#prerequisites). -- DeepSeek-V2-Lite weights on disk. The scripts default to - `/path/model_weights/DeepSeek-V2-Lite`; override with `MODEL_PATH=...`. -- Free ports `6239` (AFD rendezvous), `18000` (Attention API) and `18001` - (FFN). - -## Layout - -`2A2F`: Attention on GPUs 0,1 (DP2, or DP1 x TP2) and FFN DP2 on GPUs 2,3. -FFN rank 0 owns routed experts 0-31 and FFN rank 1 owns routed experts 32-63. +DeepSeek-V2-Lite (BF16) recipes. The scripts, layouts, settings, multi-pod +runs and the decode-only benchmark are described once in the +[connector recipes README](../README.md); this page lists what is specific to +the model. ```text prefill_decode_colocation/ -├── 2a2f_eager_dp2tp1.sh # Attention DP2 x TP1, eager -├── 2a2f_graph_dp2tp1.sh # Attention DP2 x TP1, FULL_DECODE_ONLY graph -├── 2a2f_graph_dbo_dp2tp1.sh # Attention DP2 x TP1, graph + DBO -└── 2a2f_graph_dp1tp2.sh # Attention DP1 x TP2, FULL_DECODE_ONLY graph +├── 2a2f_eager_dp2tp1.sh # eager, FFN DP2/TP1, Attention DP2/TP1 +├── 2a2f_graph_dp2tp1.sh # graph, FFN DP2/TP1, Attention DP2/TP1 +├── 2a2f_graph_dbo_dp2tp1.sh # graph + DBO, FFN DP2/TP1, Attention DP2/TP1 +└── 2a2f_eager_dp1tp2.sh # eager, FFN DP2/TP1, Attention DP1/TP2 ``` -## Configuration notes +## Defaults + +| Setting | Value | +| --- | --- | +| `LAYOUT` | `2A2F`: FFN DP2 on GPUs 0,1, Attention DP2 on GPUs 2,3 | +| `--max-num-seqs` | 64 per Attention DP rank | +| `--max-num-batched-tokens` | 4096 | +| `--max-model-len` | 4096 | +| `CUDAGRAPH_CAPTURE_SIZES` | 8,16,32,64 | +| Payload | BF16 (no `connector_extra_config`); the FFN runs DeepGEMM experts | + +These match the DeepSeek-V2-Lite DeepEP M2N E2E cases +(`test_deepseek_v2_lite_deepep_m2n` in +[`tests/e2e`](../../../../tests/e2e/README.md#deepep-m2n-scenarios)); +`ATTN_TP=2` matches the Attention TP2 case. + +## Notes -- Both roles set `compute_gate_on_attention: true`, use the same rank counts, - and run with `--enable-expert-parallel`. The FFN runs TP 1. - Attention runs the gate, the router, the dense first layer and the two - shared experts; the FFN runs only the routed experts. -- With Attention TP 2 each TP rank routes and dispatches its slice of the + shared experts; the FFN runs only the routed experts. With two FFN ranks, + FFN rank 0 owns routed experts 0-31 and FFN rank 1 owns 32-63. +- With Attention TP each TP rank routes and dispatches its slice of the tokens and runs the shared experts on that slice with replicated weights; the TP group all-gathers the result. -- `connector_extra_config` is left empty, so the payload is BF16 and the FFN - runs DeepGEMM experts. -- The graph and DBO settings match the - [Qwen3-30B-A3B recipes](../qwen3_30b_a3b/README.md#configuration-notes). ## Limitations diff --git a/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_eager_dp1tp2.sh b/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_eager_dp1tp2.sh new file mode 100644 index 000000000..653d9b3ac --- /dev/null +++ b/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_eager_dp1tp2.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# DeepSeek-V2-Lite (BF16) on DeepEPM2NAFDConnector, 2A2F eager mode with +# Attention TP2: FFN DP2 x TP1 and Attention DP1 x TP2. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables + +AFD_CONNECTOR=${AFD_CONNECTOR:-DeepEPM2NAFDConnector} +LAYOUT=${LAYOUT:-2A2F} +MODEL_PATH=${MODEL_PATH:-/path/model_weights/DeepSeek-V2-Lite} +ATTN_TP=${ATTN_TP:-2} FFN_TP=${FFN_TP:-1} +MODE=eager +SERVE_ARGS=(--enable-ep-weight-filter --max-num-seqs 64 --max-num-batched-tokens 4096 --max-model-len 4096) + +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_eager_dp2tp1.sh b/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_eager_dp2tp1.sh index 3da8f033f..36b00f0bf 100644 --- a/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_eager_dp2tp1.sh +++ b/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_eager_dp2tp1.sh @@ -1,55 +1,18 @@ +#!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# DeepSeek-V2-Lite (BF16) on DeepEPM2NAFDConnector, 2A2F eager mode: +# FFN DP2 x TP1 and Attention DP2 x TP1. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables -# 2A2F eager: Attention DP2 x TP1 on GPUs 0,1; FFN DP2 on GPUs 2,3. - +AFD_CONNECTOR=${AFD_CONNECTOR:-DeepEPM2NAFDConnector} +LAYOUT=${LAYOUT:-2A2F} MODEL_PATH=${MODEL_PATH:-/path/model_weights/DeepSeek-V2-Lite} -export VLLM_USE_V2_MODEL_RUNNER=1 - -CUDA_VISIBLE_DEVICES=2,3 vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 2, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 4096 \ - --max-model-len 4096 \ - --enforce-eager \ - --host 127.0.0.1 \ - --port 18001 \ - --trust-remote-code > ffn.log 2>&1 & - -CUDA_VISIBLE_DEVICES=0,1 vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 2, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 4096 \ - --max-model-len 4096 \ - --enforce-eager \ - --host 127.0.0.1 \ - --port 18000 \ - --trust-remote-code > attn.log 2>&1 & +ATTN_TP=${ATTN_TP:-1} FFN_TP=${FFN_TP:-1} +MODE=eager +SERVE_ARGS=(--enable-ep-weight-filter --max-num-seqs 64 --max-num-batched-tokens 4096 --max-model-len 4096) -wait +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dbo_dp2tp1.sh b/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dbo_dp2tp1.sh index 0ea5a336e..10059bfa4 100644 --- a/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dbo_dp2tp1.sh +++ b/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dbo_dp2tp1.sh @@ -1,64 +1,19 @@ +#!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# DeepSeek-V2-Lite (BF16) on DeepEPM2NAFDConnector, 2A2F FULL_DECODE_ONLY +# CUDA graphs and DBO: FFN DP2 x TP1 and Attention DP2 x TP1. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables -# 2A2F graph + DBO: Attention DP2 x TP1 on GPUs 0,1; FFN DP2 on GPUs 2,3. -# DBO must be enabled on both roles. - +AFD_CONNECTOR=${AFD_CONNECTOR:-DeepEPM2NAFDConnector} +LAYOUT=${LAYOUT:-2A2F} MODEL_PATH=${MODEL_PATH:-/path/model_weights/DeepSeek-V2-Lite} -export VLLM_USE_V2_MODEL_RUNNER=1 - -CUDA_VISIBLE_DEVICES=2,3 vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 2, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 4096 \ - --max-model-len 4096 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64]}' \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 8 \ - --host 127.0.0.1 \ - --port 18001 \ - --trust-remote-code > ffn.log 2>&1 & - -CUDA_VISIBLE_DEVICES=0,1 vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 2, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 4096 \ - --max-model-len 4096 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64]}' \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 8 \ - --host 127.0.0.1 \ - --port 18000 \ - --trust-remote-code > attn.log 2>&1 & +ATTN_TP=${ATTN_TP:-1} FFN_TP=${FFN_TP:-1} +MODE=graph CUDAGRAPH_CAPTURE_SIZES=${CUDAGRAPH_CAPTURE_SIZES:-8,16,32,64} # ascending +SERVE_ARGS=(--enable-ep-weight-filter --max-num-seqs 64 --max-num-batched-tokens 4096 --max-model-len 4096 + --enable-dbo --dbo-decode-token-threshold 2 --dbo-prefill-token-threshold 8) -wait +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dp1tp2.sh b/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dp1tp2.sh deleted file mode 100644 index e08e1d695..000000000 --- a/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dp1tp2.sh +++ /dev/null @@ -1,58 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project - -# 2A2F graph with Attention TP: Attention DP1 x TP2 on GPUs 0,1; FFN DP2 x TP1 on -# GPUs 2,3. Each Attention TP rank dispatches its slice of the tokens. - -MODEL_PATH=${MODEL_PATH:-/path/model_weights/DeepSeek-V2-Lite} -export VLLM_USE_V2_MODEL_RUNNER=1 - -CUDA_VISIBLE_DEVICES=2,3 vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 2, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 4096 \ - --max-model-len 4096 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64]}' \ - --host 127.0.0.1 \ - --port 18001 \ - --trust-remote-code > ffn.log 2>&1 & - -CUDA_VISIBLE_DEVICES=0,1 vllm serve "$MODEL_PATH" \ - --data-parallel-size 1 \ - --tensor-parallel-size 2 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 2, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 4096 \ - --max-model-len 4096 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64]}' \ - --host 127.0.0.1 \ - --port 18000 \ - --trust-remote-code > attn.log 2>&1 & - -wait diff --git a/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dp2tp1.sh b/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dp2tp1.sh index c7f1801ad..a68079120 100644 --- a/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dp2tp1.sh +++ b/recipe/gpu/DeepEPM2NAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dp2tp1.sh @@ -1,57 +1,18 @@ +#!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# DeepSeek-V2-Lite (BF16) on DeepEPM2NAFDConnector, 2A2F FULL_DECODE_ONLY +# CUDA graphs: FFN DP2 x TP1 and Attention DP2 x TP1. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables -# 2A2F FULL_DECODE_ONLY graph: Attention DP2 x TP1 on GPUs 0,1; FFN DP2 on GPUs 2,3. - +AFD_CONNECTOR=${AFD_CONNECTOR:-DeepEPM2NAFDConnector} +LAYOUT=${LAYOUT:-2A2F} MODEL_PATH=${MODEL_PATH:-/path/model_weights/DeepSeek-V2-Lite} -export VLLM_USE_V2_MODEL_RUNNER=1 - -CUDA_VISIBLE_DEVICES=2,3 vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 2, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 4096 \ - --max-model-len 4096 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64]}' \ - --host 127.0.0.1 \ - --port 18001 \ - --trust-remote-code > ffn.log 2>&1 & - -CUDA_VISIBLE_DEVICES=0,1 vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 2, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 4096 \ - --max-model-len 4096 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64]}' \ - --host 127.0.0.1 \ - --port 18000 \ - --trust-remote-code > attn.log 2>&1 & +ATTN_TP=${ATTN_TP:-1} FFN_TP=${FFN_TP:-1} +MODE=graph CUDAGRAPH_CAPTURE_SIZES=${CUDAGRAPH_CAPTURE_SIZES:-8,16,32,64} # ascending +SERVE_ARGS=(--enable-ep-weight-filter --max-num-seqs 64 --max-num-batched-tokens 4096 --max-model-len 4096) -wait +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/README.md b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/README.md index c9bd1a761..0ddc9ace5 100644 --- a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/README.md +++ b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/README.md @@ -1,90 +1,51 @@ # Qwen3-235B-A22B-FP8 with DeepEPM2NAFDConnector -Launch scripts for Qwen3-235B-A22B-FP8 on `DeepEPM2NAFDConnector` with vLLM -`v0.30.0`, using the FP8 dispatch payload. - -## Prerequisites - -- 6 SM90 or newer GPUs on one node with enough memory for the model, or two - pods of 5 such GPUs on an RDMA fabric for the multi-pod recipe. These - configurations were run on H200. -- The environment described in the - [connector recipes README](../README.md#prerequisites). -- Qwen3-235B-A22B-FP8 weights on disk (block-128 FP8). The scripts default to - `/path/model_weights/Qwen3-235B-A22B-FP8`; override with `MODEL_PATH=...`. -- Free ports `6239` (AFD rendezvous), `18000` (Attention API) and `18001` - (FFN). - -## Layout - -`4A2F`: Attention DP4 x TP1 on GPUs 0-3 and FFN DP2 on GPUs 4,5. Each FFN rank -owns half of the 128 experts. +Qwen3-235B-A22B-FP8 recipes with the FP8 dispatch payload. The scripts, +layouts, settings, multi-pod runs and the decode-only benchmark are described +once in the [connector recipes README](../README.md); this page lists what is +specific to the model. ```text prefill_decode_colocation/ -├── 4a2f_graph_dp4tp1.sh # 256 sequences per Attention rank, FULL_DECODE_ONLY graph -├── 4a2f_graph_dbo_dp4tp1.sh # 512 sequences per Attention rank, graph + DBO -└── multipod_4a1f_4a1f_graph.sh # 2 pods x 4A1F, FULL_DECODE_ONLY graph (DBO optional) -``` - -### Multi-pod: 2 x 4A1F - -`multipod_4a1f_4a1f_graph.sh` runs on two pods of 5 GPUs. Each pod runs one -FFN rank on GPU 0 and four Attention ranks on GPUs 1-4 (Attention DP8 x TP1, -FFN DP2/EP2 in total), so every FFN GPU holds half of the experts, as in 4A2F. -Pod 0 leads both DP groups and hosts the AFD rendezvous; pod 1 starts both -servers headless and serves no requests. Send traffic to pod 0's port `18000`. - -Only `POD` differs between pods: - -```bash -# Pod 0 -POD=AFD_0 bash recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_graph.sh -# Pod 1 -POD=AFD_1 bash recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_graph.sh -``` - -On Kubernetes the script reaches pod 0 through the Services that the -`deploy-afd-k8s` skill creates. Without Kubernetes, set `HEAD_HOST` to pod 0's -address on both pods; pod 0 must accept ports `6239`, `13345` (Attention DP) -and `13355` (FFN DP). Pass the fabric's NCCL settings (for example -`NCCL_IB_HCA`) through the environment. - -`NUM_PODS`, `ATTN_PER_POD` and `FFN_PER_POD` select another symmetric layout -(`2 x 6A2F`, ...); the tuning values (`MAX_NUM_SEQS`, `MAX_NUM_BATCHED_TOKENS`, -`MAX_MODEL_LEN`, `CAPTURE_SIZES`, `PAYLOAD_FORMAT`) and DBO (`ENABLE_DBO=1`, -`DBO_DECODE_TOKEN_THRESHOLD`, `DBO_PREFILL_TOKEN_THRESHOLD`) are environment -variables with the 4A2F values as defaults. A placement the connector cannot -carry fails at startup with the pods' shapes in the error. - -## Configuration notes - -- `connector_extra_config` sets `payload_format: "fp8_block128_fp32"`. The - FFN must load the block-128 FP8 checkpoint; the startup handshake rejects a - mismatch between the payload format and the loaded expert weights. -- The FFN keeps vLLM's default `--moe-backend auto`, which the plugin pins to - `triton` so the expert weights stay in the layout the connector reads. -- `afd_process_group_timeout_s` and `--cpu-distributed-timeout-seconds` are - raised to 3600 s because loading the weights can take longer than the - default 120 s join timeout, especially from cold storage. -- The capture sizes cover `--max-num-seqs`. The connector derives its dispatch - buckets from them and compiles one transport kernel set per bucket at - startup. -- `expert_num_sms` (default `0`, all SMs) and `num_sms` tune how SMs are split - between DeepGEMM experts and the transport kernels; see the connector guide. - -## Checking a deployment - -```bash -curl -s http://127.0.0.1:18000/v1/completions \ - -H 'Content-Type: application/json' \ - -d '{"model":"'"$MODEL_PATH"'","prompt":"The capital of France is","max_tokens":12,"temperature":0}' +├── 4a2f_eager_dp4tp1.sh # eager, FFN DP2/TP1, Attention DP4/TP1 +├── 4a2f_graph_dp4tp1.sh # graph, FFN DP2/TP1, Attention DP4/TP1 +├── 4a2f_dbo_dp4tp1.sh # graph + DBO, FFN DP2/TP1, Attention DP4/TP1 +└── multipod_4a1f_4a1f_dbo.sh # 2 pods, graph + DBO, Attention DP8/TP1 ``` -The answer should name Paris. An FFN log line `Using TRITON Fp8 MoE backend` -confirms the expected expert-weight layout. +## Defaults + +| Setting | `4a2f_graph_dp4tp1.sh`, `4a2f_eager_dp4tp1.sh` | `4a2f_dbo_dp4tp1.sh`, `multipod_4a1f_4a1f_dbo.sh` | +| --- | --- | --- | +| `LAYOUT` | `4A2F` | `4A2F` / `4A1F,4A1F` | +| `--max-num-seqs` | 256 per Attention DP rank | 512 | +| `--max-num-batched-tokens` | 3072 | 3072 | +| `--max-model-len` | 4096 | 4096 | +| `CUDAGRAPH_CAPTURE_SIZES` | 8,12,16,24,32,48,64,96,128,192,256 | 8,16,32,64,96,128,192,256,384,512 | +| `connector_extra_config` | `"payload_format": "fp8_block128_fp32"` | same | + +Common layouts: + +| `LAYOUT` | GPUs | Notes | +| --- | --- | --- | +| `2A2F` | 4 on one host | each FFN GPU holds half of the 128 experts | +| `4A2F` | 6 on one host | Attention DP4 on GPUs 2-5 | +| `4A1F,4A1F` | 2 pods of 5 | Attention DP8, FFN DP2/EP2 | + +## Notes + +- Every layout with two FFN ranks puts about 110 GB of FP8 experts on each + FFN GPU, so it needs H200-class GPUs (about 140 GB); four or more FFN ranks + fit smaller GPUs. Attention ranks hold no experts + (`--enable-ep-weight-filter`). +- The FFN must load the block-128 FP8 checkpoint; the startup handshake + rejects a mismatch between the payload format and the loaded expert weights. +- `ATTN_TP=4` with `LAYOUT=4A2F` runs Attention as one DP1/TP4 engine; raise + `--max-num-seqs` and the capture sizes with it, since one engine then holds + every running request. +- Qwen3-235B holds 188 KiB of BF16 KV per token (94 layers, 4 KV heads of + 128); an H200 Attention rank has room for about 640K tokens. ## Limitations - Qwen3 MoE only; see the connector guide for the full list of requirements. -- `FULL_DECODE_ONLY` is the only CUDA graph mode. diff --git a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/4a2f_dbo_dp4tp1.sh b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/4a2f_dbo_dp4tp1.sh new file mode 100644 index 000000000..1ebd1b144 --- /dev/null +++ b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/4a2f_dbo_dp4tp1.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# Qwen3-235B-A22B-FP8 on DeepEPM2NAFDConnector, 4A2F FULL_DECODE_ONLY CUDA +# graphs and DBO: FFN DP2 x TP1 and Attention DP4 x TP1. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables + +AFD_CONNECTOR=${AFD_CONNECTOR:-DeepEPM2NAFDConnector} +LAYOUT=${LAYOUT:-4A2F} +MODEL_PATH=${MODEL_PATH:-/path/model_weights/Qwen3-235B-A22B-FP8} +CONNECTOR_CONFIG='"payload_format": "fp8_block128_fp32"' # FP8 payload +MODE=graph CUDAGRAPH_CAPTURE_SIZES=${CUDAGRAPH_CAPTURE_SIZES:-8,16,32,64,96,128,192,256,384,512} # ascending +SERVE_ARGS=(--enable-ep-weight-filter --max-num-seqs 512 --max-num-batched-tokens 3072 --max-model-len 4096 + --enable-dbo --dbo-decode-token-threshold 2 --dbo-prefill-token-threshold 8) + +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/4a2f_eager_dp4tp1.sh b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/4a2f_eager_dp4tp1.sh new file mode 100644 index 000000000..e196f5d76 --- /dev/null +++ b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/4a2f_eager_dp4tp1.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# Qwen3-235B-A22B-FP8 on DeepEPM2NAFDConnector, 4A2F eager mode: +# FFN DP2 x TP1 and Attention DP4 x TP1. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables + +AFD_CONNECTOR=${AFD_CONNECTOR:-DeepEPM2NAFDConnector} +LAYOUT=${LAYOUT:-4A2F} +MODEL_PATH=${MODEL_PATH:-/path/model_weights/Qwen3-235B-A22B-FP8} +CONNECTOR_CONFIG='"payload_format": "fp8_block128_fp32"' # FP8 payload +MODE=eager +SERVE_ARGS=(--enable-ep-weight-filter --max-num-seqs 256 --max-num-batched-tokens 3072 --max-model-len 4096) + +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/4a2f_graph_dbo_dp4tp1.sh b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/4a2f_graph_dbo_dp4tp1.sh deleted file mode 100644 index 37aef9314..000000000 --- a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/4a2f_graph_dbo_dp4tp1.sh +++ /dev/null @@ -1,68 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project - -# 4A2F graph + DBO, FP8 payload: Attention DP4 x TP1 on GPUs 0-3; FFN DP2 on -# GPUs 4,5. DBO must be enabled on both roles. - -MODEL_PATH=${MODEL_PATH:-/path/model_weights/Qwen3-235B-A22B-FP8} -export VLLM_USE_V2_MODEL_RUNNER=1 - -CUDA_VISIBLE_DEVICES=4,5 vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 4, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true, - "afd_process_group_timeout_s": 3600, - "connector_extra_config": {"payload_format": "fp8_block128_fp32"} - } - }' \ - --max-num-seqs 512 \ - --max-num-batched-tokens 3072 \ - --max-model-len 4096 \ - --max-cudagraph-capture-size 512 \ - --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64, 96, 128, 192, 256, 384, 512]}' \ - --cpu-distributed-timeout-seconds 3600 \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 8 \ - --host 127.0.0.1 \ - --port 18001 > ffn.log 2>&1 & - -CUDA_VISIBLE_DEVICES=0,1,2,3 vllm serve "$MODEL_PATH" \ - --data-parallel-size 4 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 4, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true, - "afd_process_group_timeout_s": 3600, - "connector_extra_config": {"payload_format": "fp8_block128_fp32"} - } - }' \ - --max-num-seqs 512 \ - --max-num-batched-tokens 3072 \ - --max-model-len 4096 \ - --max-cudagraph-capture-size 512 \ - --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64, 96, 128, 192, 256, 384, 512]}' \ - --cpu-distributed-timeout-seconds 3600 \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 8 \ - --host 127.0.0.1 \ - --port 18000 > attn.log 2>&1 & - -wait diff --git a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/4a2f_graph_dp4tp1.sh b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/4a2f_graph_dp4tp1.sh index 04491ad2a..c4b25f040 100644 --- a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/4a2f_graph_dp4tp1.sh +++ b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/4a2f_graph_dp4tp1.sh @@ -1,61 +1,18 @@ +#!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# Qwen3-235B-A22B-FP8 on DeepEPM2NAFDConnector, 4A2F FULL_DECODE_ONLY CUDA +# graphs: FFN DP2 x TP1 and Attention DP4 x TP1. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables -# 4A2F graph, FP8 payload: Attention DP4 x TP1 on GPUs 0-3; FFN DP2 on GPUs 4,5. - +AFD_CONNECTOR=${AFD_CONNECTOR:-DeepEPM2NAFDConnector} +LAYOUT=${LAYOUT:-4A2F} MODEL_PATH=${MODEL_PATH:-/path/model_weights/Qwen3-235B-A22B-FP8} -export VLLM_USE_V2_MODEL_RUNNER=1 - -CUDA_VISIBLE_DEVICES=4,5 vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 4, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true, - "afd_process_group_timeout_s": 3600, - "connector_extra_config": {"payload_format": "fp8_block128_fp32"} - } - }' \ - --max-num-seqs 256 \ - --max-num-batched-tokens 3072 \ - --max-model-len 4096 \ - --max-cudagraph-capture-size 256 \ - --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64, 96, 128, 192, 256]}' \ - --cpu-distributed-timeout-seconds 3600 \ - --host 127.0.0.1 \ - --port 18001 > ffn.log 2>&1 & - -CUDA_VISIBLE_DEVICES=0,1,2,3 vllm serve "$MODEL_PATH" \ - --data-parallel-size 4 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 4, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true, - "afd_process_group_timeout_s": 3600, - "connector_extra_config": {"payload_format": "fp8_block128_fp32"} - } - }' \ - --max-num-seqs 256 \ - --max-num-batched-tokens 3072 \ - --max-model-len 4096 \ - --max-cudagraph-capture-size 256 \ - --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64, 96, 128, 192, 256]}' \ - --cpu-distributed-timeout-seconds 3600 \ - --host 127.0.0.1 \ - --port 18000 > attn.log 2>&1 & +CONNECTOR_CONFIG='"payload_format": "fp8_block128_fp32"' # FP8 payload +MODE=graph CUDAGRAPH_CAPTURE_SIZES=${CUDAGRAPH_CAPTURE_SIZES:-8,12,16,24,32,48,64,96,128,192,256} # ascending +SERVE_ARGS=(--enable-ep-weight-filter --max-num-seqs 256 --max-num-batched-tokens 3072 --max-model-len 4096) -wait +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_dbo.sh b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_dbo.sh new file mode 100644 index 000000000..32d786a08 --- /dev/null +++ b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_dbo.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# Qwen3-235B-A22B-FP8 on DeepEPM2NAFDConnector, two 4A1F pods with +# FULL_DECODE_ONLY CUDA graphs and DBO: Attention DP8 x TP1, FFN DP2 x TP1. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables + +AFD_CONNECTOR=${AFD_CONNECTOR:-DeepEPM2NAFDConnector} +LAYOUT=${LAYOUT:-4A1F,4A1F} +MODEL_PATH=${MODEL_PATH:-/path/model_weights/Qwen3-235B-A22B-FP8} +CONNECTOR_CONFIG='"payload_format": "fp8_block128_fp32"' # FP8 payload +MODE=graph CUDAGRAPH_CAPTURE_SIZES=${CUDAGRAPH_CAPTURE_SIZES:-8,16,32,64,96,128,192,256,384,512} # ascending +SERVE_ARGS=(--enable-ep-weight-filter --max-num-seqs 512 --max-num-batched-tokens 3072 --max-model-len 4096 + --enable-dbo --dbo-decode-token-threshold 2 --dbo-prefill-token-threshold 8) + +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_graph.sh b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_graph.sh deleted file mode 100644 index ed48f9b2a..000000000 --- a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_235b_a22b_fp8/prefill_decode_colocation/multipod_4a1f_4a1f_graph.sh +++ /dev/null @@ -1,82 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project - -# Qwen3-235B-A22B-FP8 with DeepEPM2NAFDConnector, 2 x 4A1F, FULL_DECODE_ONLY -# CUDA graphs: 2 pods of 5 GPUs, each with FFN on GPU 0 and Attention DP4 x TP1 -# on GPUs 1-4 (Attention DP8, FFN DP2/EP2 in total). Send traffic to port -# 18000 of the POD=AFD_0 pod. -# -# Use .agents/skills/deploy-afd-k8s/SKILL.md to deploy this recipe -# on 2 pods using the setup below. -# For pod 1: POD=AFD_0 -# For pod 2: POD=AFD_1 -# -# NUM_PODS x (ATTN_PER_POD Attention + FFN_PER_POD FFN) sets any symmetric -# layout (2 x 1A1F, 3 x 6A2F, ...); give every pod the same values. HEAD_HOST -# points the AFD rendezvous and both DP leaders at pod 0 (runs without k8s). - -MODEL_PATH=${MODEL_PATH:-/path/model_weights/Qwen3-235B-A22B-FP8} -NUM_PODS=${NUM_PODS:-2} -ATTN_PER_POD=${ATTN_PER_POD:-4} -FFN_PER_POD=${FFN_PER_POD:-1} -PAYLOAD_FORMAT=${PAYLOAD_FORMAT:-fp8_block128_fp32} -MAX_NUM_SEQS=${MAX_NUM_SEQS:-256} # per Attention DP rank -MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-3072} -MAX_MODEL_LEN=${MAX_MODEL_LEN:-4096} -CAPTURE_SIZES=${CAPTURE_SIZES:-8,16,32,64,96,128,192,256} # ascending -ENABLE_DBO=${ENABLE_DBO:-0} -DBO_DECODE_TOKEN_THRESHOLD=${DBO_DECODE_TOKEN_THRESHOLD:-2} -DBO_PREFILL_TOKEN_THRESHOLD=${DBO_PREFILL_TOKEN_THRESHOLD:-2} -export VLLM_USE_V2_MODEL_RUNNER=1 - -K=${POD#AFD_} A=$ATTN_PER_POD F=$FFN_PER_POD - -# role_dp PER_POD ADDRESS RPC_PORT -> DP: this pod's DP flags for one role. -# Pod 0 leads and omits its start rank: vLLM reads an explicit start rank on a -# non-headless engine as hybrid load balancing and rejects the headless peers. -role_dp() { - DP=(--data-parallel-size $((NUM_PODS * $1)) --data-parallel-size-local "$1" - --data-parallel-address "$2" --data-parallel-rpc-port "$3") - if ((K > 0)); then DP+=(--headless --data-parallel-start-rank $((K * $1))); fi -} - -afd() { - cat < ffn.log 2>&1 & - -role_dp "$A" "${HEAD_HOST:-vllm-attn-dp-service}" 13345 -CUDA_VISIBLE_DEVICES=$(seq "$F" $((F + A - 1)) | paste -sd, -) vllm serve "$MODEL_PATH" \ - --additional-config "$(afd attention)" "${COMMON[@]}" "${DP[@]}" \ - --port 18000 > attn.log 2>&1 & - -wait diff --git a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/README.md b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/README.md index 184806b5e..817963a4d 100644 --- a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/README.md +++ b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/README.md @@ -1,59 +1,43 @@ # Qwen3-30B-A3B with DeepEPM2NAFDConnector -Launch scripts for Qwen3-30B-A3B (BF16) on `DeepEPM2NAFDConnector` with vLLM -`v0.30.0`. They use the same layouts as the Qwen3 MoE DeepEP M2N E2E cases -(`afd-deepep-m2n-*-2a2f` in [`tests/e2e`](../../../../tests/e2e/README.md#deepep-m2n-scenarios)). - -## Prerequisites - -- 4 SM90 or newer GPUs on one node. -- The environment described in the - [connector recipes README](../README.md#prerequisites). -- Qwen3-30B-A3B weights on disk. The scripts default to - `/path/model_weights/Qwen3-30B-A3B`; override with `MODEL_PATH=...`. -- Free ports `6239` (AFD rendezvous), `18000` (Attention API) and `18001` - (FFN). - -## Layout - -`2A2F`: Attention on GPUs 0,1 and FFN DP2 on GPUs 2,3. FFN rank 0 owns experts -0-63 and FFN rank 1 owns experts 64-127. +Qwen3-30B-A3B (BF16) recipes. The scripts, layouts, settings, multi-pod runs +and the decode-only benchmark are described once in the +[connector recipes README](../README.md); this page lists what is specific to +the model. ```text prefill_decode_colocation/ -├── 2a2f_eager_dp2tp1.sh # Attention DP2 x TP1, eager -├── 2a2f_graph_dp2tp1.sh # Attention DP2 x TP1, FULL_DECODE_ONLY graph -├── 2a2f_graph_dbo_dp2tp1.sh # Attention DP2 x TP1, graph + DBO -└── 2a2f_graph_dp1tp2.sh # Attention DP1 x TP2, FULL_DECODE_ONLY graph +├── 2a2f_eager_dp2tp1.sh # eager, FFN DP2/TP1, Attention DP2/TP1 +├── 2a2f_graph_dp2tp1.sh # graph, FFN DP2/TP1, Attention DP2/TP1 +├── 2a2f_graph_dbo_dp2tp1.sh # graph + DBO, FFN DP2/TP1, Attention DP2/TP1 +└── 2a2f_eager_dp1tp2.sh # eager, FFN DP2/TP1, Attention DP1/TP2 ``` -## Configuration notes - -- Both roles set `compute_gate_on_attention: true`, use the same rank counts, - and run with `--enable-expert-parallel`. The FFN runs TP 1. -- `connector_extra_config` is left empty, so the payload is BF16 and the FFN - runs DeepGEMM experts. -- The graph scripts capture batch sizes 8, 16, 32 and 64. The connector - derives its dispatch buckets from the capture sizes and compiles one - transport kernel set per bucket at startup. -- In the DBO script, both roles pass `--enable-dbo` with the same thresholds; - the connector then uses two lanes. -- The dispatch capacity is derived from `--max-num-batched-tokens` and the - largest capture size, so lowering `--max-num-batched-tokens` shrinks the - transport buffers. - -## Checking a deployment - -```bash -curl -s http://127.0.0.1:18000/v1/completions \ - -H 'Content-Type: application/json' \ - -d '{"model":"'"$MODEL_PATH"'","prompt":"The capital of France is","max_tokens":12,"temperature":0}' -``` +## Defaults + +| Setting | Value | +| --- | --- | +| `LAYOUT` | `2A2F`: FFN DP2 on GPUs 0,1, Attention DP2 on GPUs 2,3 | +| `--max-num-seqs` | 64 per Attention DP rank | +| `--max-num-batched-tokens` | 4096 | +| `--max-model-len` | 4096 | +| `CUDAGRAPH_CAPTURE_SIZES` | 8,16,32,64 | +| Payload | BF16 (no `connector_extra_config`); the FFN runs DeepGEMM experts | + +These match the Qwen3 MoE DeepEP M2N E2E cases (`afd-deepep-m2n-*-2a2f` and +the two-pod `2A0F,0A2F` and `1A1F,1A1F` cases in +[`tests/e2e`](../../../../tests/e2e/README.md#deepep-m2n-scenarios)); +`ATTN_TP=2` matches the Attention TP2 cases. + +## Notes -For an accuracy check, run the matching E2E case, for example -`afd-deepep-m2n-graph-2a2f`; see [`tests/e2e`](../../../../tests/e2e/README.md#deepep-m2n-scenarios). +- 4 GPUs of 80 GB (H100) or more fit `2A2F`. With two FFN ranks, FFN rank 0 + owns experts 0-63 and FFN rank 1 owns experts 64-127; the FFN count must + divide the 128 experts. +- Decode-only benchmark pool sizing: Qwen3-30B-A3B holds about 96 KiB of BF16 + KV per token, so 500 prompts of 1025 tokens need about 47 GiB per Attention + rank and fit an H100; 1000 prompts need about 94 GiB and need an H200. ## Limitations - Qwen3 MoE only; see the connector guide for the full list of requirements. -- `FULL_DECODE_ONLY` is the only CUDA graph mode. diff --git a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_eager_dp1tp2.sh b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_eager_dp1tp2.sh new file mode 100644 index 000000000..a70d42850 --- /dev/null +++ b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_eager_dp1tp2.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# Qwen3-30B-A3B (BF16) on DeepEPM2NAFDConnector, 2A2F eager mode with +# Attention TP2: FFN DP2 x TP1 and Attention DP1 x TP2. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables + +AFD_CONNECTOR=${AFD_CONNECTOR:-DeepEPM2NAFDConnector} +LAYOUT=${LAYOUT:-2A2F} +MODEL_PATH=${MODEL_PATH:-/path/model_weights/Qwen3-30B-A3B} +ATTN_TP=${ATTN_TP:-2} FFN_TP=${FFN_TP:-1} +MODE=eager +SERVE_ARGS=(--enable-ep-weight-filter --max-num-seqs 64 --max-num-batched-tokens 4096 --max-model-len 4096) + +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_eager_dp2tp1.sh b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_eager_dp2tp1.sh index 18b37e29a..c563ef9da 100644 --- a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_eager_dp2tp1.sh +++ b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_eager_dp2tp1.sh @@ -1,53 +1,18 @@ +#!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# Qwen3-30B-A3B (BF16) on DeepEPM2NAFDConnector, 2A2F eager mode: +# FFN DP2 x TP1 and Attention DP2 x TP1. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables -# 2A2F eager: Attention DP2 x TP1 on GPUs 0,1; FFN DP2 on GPUs 2,3. - +AFD_CONNECTOR=${AFD_CONNECTOR:-DeepEPM2NAFDConnector} +LAYOUT=${LAYOUT:-2A2F} MODEL_PATH=${MODEL_PATH:-/path/model_weights/Qwen3-30B-A3B} -export VLLM_USE_V2_MODEL_RUNNER=1 - -CUDA_VISIBLE_DEVICES=2,3 vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 2, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 4096 \ - --max-model-len 4096 \ - --enforce-eager \ - --host 127.0.0.1 \ - --port 18001 > ffn.log 2>&1 & - -CUDA_VISIBLE_DEVICES=0,1 vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 2, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 4096 \ - --max-model-len 4096 \ - --enforce-eager \ - --host 127.0.0.1 \ - --port 18000 > attn.log 2>&1 & +ATTN_TP=${ATTN_TP:-1} FFN_TP=${FFN_TP:-1} +MODE=eager +SERVE_ARGS=(--enable-ep-weight-filter --max-num-seqs 64 --max-num-batched-tokens 4096 --max-model-len 4096) -wait +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_graph_dbo_dp2tp1.sh b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_graph_dbo_dp2tp1.sh index 1736b681c..f08636036 100644 --- a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_graph_dbo_dp2tp1.sh +++ b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_graph_dbo_dp2tp1.sh @@ -1,62 +1,19 @@ +#!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# Qwen3-30B-A3B (BF16) on DeepEPM2NAFDConnector, 2A2F FULL_DECODE_ONLY CUDA +# graphs and DBO: FFN DP2 x TP1 and Attention DP2 x TP1. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables -# 2A2F graph + DBO: Attention DP2 x TP1 on GPUs 0,1; FFN DP2 on GPUs 2,3. -# DBO must be enabled on both roles. - +AFD_CONNECTOR=${AFD_CONNECTOR:-DeepEPM2NAFDConnector} +LAYOUT=${LAYOUT:-2A2F} MODEL_PATH=${MODEL_PATH:-/path/model_weights/Qwen3-30B-A3B} -export VLLM_USE_V2_MODEL_RUNNER=1 - -CUDA_VISIBLE_DEVICES=2,3 vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 2, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 4096 \ - --max-model-len 4096 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64]}' \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 8 \ - --host 127.0.0.1 \ - --port 18001 > ffn.log 2>&1 & - -CUDA_VISIBLE_DEVICES=0,1 vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 2, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 4096 \ - --max-model-len 4096 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64]}' \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 8 \ - --host 127.0.0.1 \ - --port 18000 > attn.log 2>&1 & +ATTN_TP=${ATTN_TP:-1} FFN_TP=${FFN_TP:-1} +MODE=graph CUDAGRAPH_CAPTURE_SIZES=${CUDAGRAPH_CAPTURE_SIZES:-8,16,32,64} # ascending +SERVE_ARGS=(--enable-ep-weight-filter --max-num-seqs 64 --max-num-batched-tokens 4096 --max-model-len 4096 + --enable-dbo --dbo-decode-token-threshold 2 --dbo-prefill-token-threshold 8) -wait +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_graph_dp1tp2.sh b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_graph_dp1tp2.sh deleted file mode 100644 index 447ace10f..000000000 --- a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_graph_dp1tp2.sh +++ /dev/null @@ -1,56 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project - -# 2A2F graph with Attention TP: Attention DP1 x TP2 on GPUs 0,1; FFN DP2 x TP1 on -# GPUs 2,3. Each Attention TP rank dispatches its slice of the tokens. - -MODEL_PATH=${MODEL_PATH:-/path/model_weights/Qwen3-30B-A3B} -export VLLM_USE_V2_MODEL_RUNNER=1 - -CUDA_VISIBLE_DEVICES=2,3 vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 2, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 4096 \ - --max-model-len 4096 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64]}' \ - --host 127.0.0.1 \ - --port 18001 > ffn.log 2>&1 & - -CUDA_VISIBLE_DEVICES=0,1 vllm serve "$MODEL_PATH" \ - --data-parallel-size 1 \ - --tensor-parallel-size 2 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 2, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 4096 \ - --max-model-len 4096 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64]}' \ - --host 127.0.0.1 \ - --port 18000 > attn.log 2>&1 & - -wait diff --git a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_graph_dp2tp1.sh b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_graph_dp2tp1.sh index 2660ec88d..85edcd752 100644 --- a/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_graph_dp2tp1.sh +++ b/recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_graph_dp2tp1.sh @@ -1,55 +1,18 @@ +#!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# Qwen3-30B-A3B (BF16) on DeepEPM2NAFDConnector, 2A2F FULL_DECODE_ONLY CUDA +# graphs: FFN DP2 x TP1 and Attention DP2 x TP1. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables -# 2A2F FULL_DECODE_ONLY graph: Attention DP2 x TP1 on GPUs 0,1; FFN DP2 on GPUs 2,3. - +AFD_CONNECTOR=${AFD_CONNECTOR:-DeepEPM2NAFDConnector} +LAYOUT=${LAYOUT:-2A2F} MODEL_PATH=${MODEL_PATH:-/path/model_weights/Qwen3-30B-A3B} -export VLLM_USE_V2_MODEL_RUNNER=1 - -CUDA_VISIBLE_DEVICES=2,3 vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 2, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 4096 \ - --max-model-len 4096 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64]}' \ - --host 127.0.0.1 \ - --port 18001 > ffn.log 2>&1 & - -CUDA_VISIBLE_DEVICES=0,1 vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "DeepEPM2NAFDConnector", - "host": "127.0.0.1", - "port": 6239, - "num_attention_ranks": 2, - "num_ffn_ranks": 2, - "compute_gate_on_attention": true - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 4096 \ - --max-model-len 4096 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64]}' \ - --host 127.0.0.1 \ - --port 18000 > attn.log 2>&1 & +ATTN_TP=${ATTN_TP:-1} FFN_TP=${FFN_TP:-1} +MODE=graph CUDAGRAPH_CAPTURE_SIZES=${CUDAGRAPH_CAPTURE_SIZES:-8,16,32,64} # ascending +SERVE_ARGS=(--enable-ep-weight-filter --max-num-seqs 64 --max-num-batched-tokens 4096 --max-model-len 4096) -wait +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/P2pNcclAFDConnector/README.md b/recipe/gpu/P2pNcclAFDConnector/README.md index 58862a6f0..8f958b62c 100644 --- a/recipe/gpu/P2pNcclAFDConnector/README.md +++ b/recipe/gpu/P2pNcclAFDConnector/README.md @@ -11,36 +11,31 @@ for the connector's rank mapping and configuration contract. ```text . ├── deepseek_v2_lite/ -│ ├── README.md -│ ├── prefill_decode_colocation/ # 2A2F, DP/TP variants, eager/graph +│ ├── prefill_decode_colocation/ # explicit 2A2F, 4A4F and two-pod DBO recipes │ └── prefill_decode_disaggregation/ # 2P1A1F -└── qwen3_5_122b_a10b_fp8/ - └── prefill_decode_colocation/ # multipod_2a_2a_2f_graph.sh: 4A2F split across 2 attention pods, graph +└── qwen3_5_122b_a10b_fp8/prefill_decode_colocation/ # multipod_2a_2a_2f_graph.sh ``` Each model directory holds its own recipe scripts; open its `README.md` -(where present) before running anything in it -- it documents prerequisites, -topology, ports, and known limitations for that model. +(where present) before running anything in it. -Recipes under `prefill_decode_colocation/` background an attention worker and -an FFN worker (no prefill split). Recipes under `prefill_decode_disaggregation/` -add a separate prefill stage and a proxy in front of it -- see the caveat in -"Deploying on Kubernetes" below. +The `prefill_decode_colocation/` recipes run an Attention server and an FFN +server (no prefill split) through the shared +[colocation launcher](../common/README.md), which documents the layout, +settings, ports, and how to run a recipe directly or in a pod. Recipes under +`prefill_decode_disaggregation/` add a separate prefill stage and a proxy in +front of it -- see the caveat in "Deploying on Kubernetes" below. ## Running on a local host -Run any script directly from the repository root, e.g.: - ```bash export MODEL_PATH=/path/model_weights/DeepSeek-V2-Lite bash recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dbo_dp1tp2.sh ``` -Each script backgrounds its workers and writes per-worker logs (`attn.log`, -`ffn.log`, and for disaggregation `afd_prefill*.log`) into the current -directory. Wait for `Application startup complete` in each log before -sending traffic. See the model README for exact prerequisites (GPU count, -weights, ports) and the benchmark command for that model. +A colocation recipe writes `attn.log` and `ffn.log` into the current +directory; send traffic to port `18000` once `attn.log` prints +`Application startup complete`. ## Deploying on Kubernetes @@ -49,15 +44,11 @@ to stand up a recipe from this folder on a Kubernetes/OpenShift cluster -- see that skill for the full contract (PVC/ConfigMap/pod/Service lifecycle, scope, and requirements). -The skill requires the recipe path (e.g. +The skill takes the recipe path (e.g. `recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dbo_dp1tp2.sh`) -plus the image, model id, and PVC name it needs. Placement is never asked -for: a recipe without a top-of-file placement comment deploys as a single -pod; a recipe whose header describes a per-pod placement (e.g. -`qwen3_5_122b_a10b_fp8/prefill_decode_colocation/multipod_2a_2a_2f_graph.sh`, -`deepseek_v2_lite/prefill_decode_colocation/multipod_2a_2f_graph_dbo_dp1tp2.sh`) -deploys exactly that many pods with exactly that per-pod setup -- see the -skill's `resolve-recipe.md` for the header convention. +plus the image, model id, and PVC name, and deploys one pod per `LAYOUT` +entry (for example the DeepSeek multipod recipe's default `2A0F,0A2F`, or +the Qwen3.5 recipe's default three pods). `prefill_decode_disaggregation` recipes are out of scope for the skill; run those locally instead. diff --git a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md index 2f18e18a9..accd39a42 100644 --- a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md +++ b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/README.md @@ -14,18 +14,18 @@ for tested E2E configurations. ## Prerequisites - Install [NIXL](https://github.com/ai-dynamo/nixl). -- Four GPUs for the `2a2f_*` and `2p1a1f_*` examples; the `4a4f_*` - examples require eight GPUs. The multipod example uses two GPUs per pod - across two pods. +- Four GPUs for the default `2A2F` colocation layout and the `2p1a1f_*` + examples; `LAYOUT=4A4F` needs eight GPUs, and `LAYOUT=2A0F,0A2F` two GPUs + on each of two pods. - vLLM `v0.32.0` and the `afd-plugin` package installed in the same environment (see repository root `AGENTS.md`). - DeepSeek-V2-Lite weights on disk. Colocation scripts default to `/path/model_weights/DeepSeek-V2-Lite`; disaggregation scripts default to `/home/models/hub/models--deepseek-ai--DeepSeek-V2-Lite/snapshots/604d5664dddd88a0433dbae533b7fe9472482de0`. Set `MODEL_PATH=...` to the local weights when launching. -- A free TCP port `6269` on `127.0.0.1` for the AFD p2p connector, and - HTTP ports `18305` and `18306` for local colocation, or - `18301`–`18305` for prefill/decode disaggregation. +- Free ports `6239` (AFD rendezvous), `18000` and `18001` (Attention and + FFN HTTP) for colocation, or `18301`–`18305` and `6269` for + prefill/decode disaggregation. ## Directory layout @@ -34,14 +34,14 @@ for tested E2E configurations. ├── prefill_decode_disaggregation/ # prefill_decode_disaggregation, 2P1A1F topology │ ├── 2p1a1f_eager_dbo.sh │ └── 2p1a1f_graph_dbo.sh -└── prefill_decode_colocation/ # prefill_decode_colocation, 2A2F/4A4F topology - ├── 2a2f_eager_dbo_dp1tp2.sh - ├── 2a2f_eager_dbo_dp2tp1.sh - ├── 2a2f_graph_dbo_dp1tp2.sh - ├── 2a2f_graph_dbo_dp2tp1.sh - ├── 4a4f_eager_dbo_dp2tp2.sh - ├── 4a4f_graph_dbo_dp2tp2.sh - └── multipod_2a_2f_graph_dbo_dp1tp2.sh # 2a2f_graph_dbo_dp1tp2, split across 2 k8s pods +└── prefill_decode_colocation/ # any LAYOUT, ATTN_TP, FFN_TP + ├── 2a2f_eager_dbo_dp2tp1.sh # 2A2F, eager + DBO, DP2/TP1 + ├── 2a2f_graph_dbo_dp2tp1.sh # 2A2F, graph + DBO, DP2/TP1 + ├── 2a2f_eager_dbo_dp1tp2.sh # 2A2F, eager + DBO, DP1/TP2 + ├── 2a2f_graph_dbo_dp1tp2.sh # 2A2F, graph + DBO, DP1/TP2 + ├── 4a4f_eager_dbo_dp2tp2.sh # 4A4F, eager + DBO, DP2/TP2 + ├── 4a4f_graph_dbo_dp2tp2.sh # 4A4F, graph + DBO, DP2/TP2 + └── multipod_2a_2f_graph_dbo_dp1tp2.sh # 2A0F,0A2F, graph + DBO, DP1/TP2 ``` ### 1. Prefill/Decode Disaggregation — `1a1f` @@ -58,31 +58,17 @@ for tested E2E configurations. ### 2. Prefill/Decode Colocation — `2a2f` -2 processes, two GPUs each: - -| GPUs | Role | Port | -|------|-----------|-------| -| 0, 1 | Attention | 18305 | -| 2, 3 | FFN | 18306 | - -The four `2a2f_*` variants cover the TP/DP cross product; `4a4f_*` doubles -both attention and FFN ranks (DP=2 on each side); `multipod_2a_2f_graph_dbo_dp1tp2.sh` -is the same `2a2f_graph_dbo_dp1tp2` config, but gated on a `POD` env var -(`ATTENTION_0` or `FFN_0`) for the `deploy-afd-k8s` skill to deploy it -across 2 pods (see repo-root `recipe/gpu/P2pNcclAFDConnector/README.md`); -its Attention and FFN HTTP ports can both be 18305 because they are in -separate pods -- -unlike the scripts above, it is **not** meant to be run directly: with -`POD` unset, neither `vllm serve` block executes, and its -`AFD_CONNECTOR_HOST` default (`vllm-ffn-p2p-service`) only resolves inside -the cluster the skill deploys it to: - -| File | DP | TP | -|---------------------------------------|----|----| -| `2a2f_*_dp1tp2.sh` | 1 | 2 | -| `2a2f_*_dp2tp1.sh` | 2 | 1 | -| `4a4f_*_dp2tp2.sh` | 2 | 2 | -| `multipod_2a_2f_graph_dbo_dp1tp2.sh` | 1 | 2 | +The colocation recipes run on the shared +[colocation launcher](../../common/README.md): `LAYOUT` places the ranks and +each script names its default DP/TP shape (FFN first, Attention second, and +Attention HTTP port 18000): + +| Script suffix | Attention / FFN | GPUs | +| --- | --- | --- | +| `2a2f_*_dp2tp1` | DP2 x TP1 each | 4 | +| `2a2f_*_dp1tp2` | DP1 x TP2 each | 4 | +| `4a4f_*_dp2tp2` | DP2 x TP2 each | 8 | +| `multipod_2a_2f_*_dp1tp2` | DP1 x TP2 each, one pod per role | 2 + 2 | ## Running @@ -96,7 +82,6 @@ before sending traffic. ```bash export MODEL_PATH=/path/model_weights/DeepSeek-V2-Lite -export VLLM_USE_V2_MODEL_RUNNER=1 bash recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dbo_dp1tp2.sh ``` @@ -122,7 +107,8 @@ bash tools/benchmarks/request_generator.sh The script runs `vllm bench serve`; it does not wait for readiness or send a separate smoke request. By default it sends 1024 random requests (1024 input tokens / 128 output tokens) at request rate 5 with `--max-concurrency 32` -against `127.0.0.1:18305`, and saves the JSON result to +against `127.0.0.1:18000` (the colocation Attention port; set `PORT=18305` +for the disaggregation proxy), and saves the JSON result to `$RESULT_DIR/$RESULT_FILENAME`. Override `HOST`, `PORT`, `MODEL_PATH`, `NUM_PROMPTS`, `REQUEST_RATE`, `MAX_CONCURRENCY`, `INPUT_LEN`, and `OUTPUT_LEN` for smaller smoke runs or larger throughput sweeps. @@ -130,7 +116,9 @@ for smaller smoke runs or larger throughput sweeps. ## Common AFD configuration Every AFD worker is wired through `--additional-config` with the same -shape; `role` differs between attention and FFN: +shape; `role` differs between attention and FFN. The colocation launcher +builds it from `LAYOUT` (the disaggregation scripts write it inline, on port +6269); print it with `AFD_DRY_RUN=1`: ```jsonc { @@ -138,9 +126,11 @@ shape; `role` differs between attention and FFN: "role": "attention", // or "ffn" "connector": "P2pNcclAFDConnector", "host": "127.0.0.1", - "port": 6269, - "num_attention_ranks": 1, // 2 in 2A2F - "num_ffn_ranks": 1 // 2 in 2A2F + "port": 6239, + "num_attention_ranks": 2, // Attention GPUs in LAYOUT + "num_ffn_ranks": 2, // FFN GPUs in LAYOUT + "compute_gate_on_attention": false, + "afd_process_group_timeout_s": 3600 } } ``` @@ -150,16 +140,7 @@ DBO (Dual Batch Overlap) is turned on for all examples with These threshold values are example defaults; tune them for the actual workload and deployment configuration. -### Switching eager → graph - -Graph mode replaces `--enforce-eager` with: - -```bash ---max-cudagraph-capture-size 64 ---compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", - "cudagraph_capture_sizes":[64]}' -``` - -The capture size of `64` is an example value; adjust -`--max-cudagraph-capture-size` and `cudagraph_capture_sizes` together based on -the actual batch sizes and available GPU memory. +The `graph_dbo` recipes capture `FULL_DECODE_ONLY` CUDA graphs of size `64` +(`CUDAGRAPH_CAPTURE_SIZES=64`); the `eager_dbo` recipes run +`--enforce-eager`. Adjust +`CUDAGRAPH_CAPTURE_SIZES` to the actual batch sizes and available GPU memory. diff --git a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_eager_dbo_dp1tp2.sh b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_eager_dbo_dp1tp2.sh index 1cac638bc..8c92d7cf4 100644 --- a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_eager_dbo_dp1tp2.sh +++ b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_eager_dbo_dp1tp2.sh @@ -1,55 +1,19 @@ +#!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# DeepSeek-V2-Lite (BF16) on P2pNcclAFDConnector, 2A2F eager mode and DBO: +# FFN DP1 x TP2 and Attention DP1 x TP2. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables +AFD_CONNECTOR=${AFD_CONNECTOR:-P2pNcclAFDConnector} +LAYOUT=${LAYOUT:-2A2F} MODEL_PATH=${MODEL_PATH:-/path/model_weights/DeepSeek-V2-Lite} -export VLLM_USE_V2_MODEL_RUNNER=1 +ATTN_TP=${ATTN_TP:-2} FFN_TP=${FFN_TP:-2} +MODE=eager +SERVE_ARGS=(--max-num-seqs 64 --max-num-batched-tokens 64 + --enable-dbo --dbo-decode-token-threshold 2 --dbo-prefill-token-threshold 12) -CUDA_VISIBLE_DEVICES=0,1 uv run vllm serve "$MODEL_PATH" \ - --data-parallel-size 1 \ - --tensor-parallel-size 2 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "P2pNcclAFDConnector", - "host": "127.0.0.1", - "port": 6269, - "num_attention_ranks": 2, - "num_ffn_ranks": 2 - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 64 \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 12 \ - --enforce-eager \ - --host 127.0.0.1 \ - --port 18305 \ - --trust-remote-code > attn.log 2>&1 & - -CUDA_VISIBLE_DEVICES=2,3 uv run vllm serve "$MODEL_PATH" \ - --data-parallel-size 1 \ - --tensor-parallel-size 2 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "P2pNcclAFDConnector", - "host": "127.0.0.1", - "port": 6269, - "num_attention_ranks": 2, - "num_ffn_ranks": 2 - } - }' \ - --max-num-seqs 64 \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 12 \ - --max-num-batched-tokens 64 \ - --enforce-eager \ - --host 127.0.0.1 \ - --port 18306 \ - --trust-remote-code > ffn.log 2>&1 & - -wait +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_eager_dbo_dp2tp1.sh b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_eager_dbo_dp2tp1.sh index edbab3812..ba600f9e1 100644 --- a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_eager_dbo_dp2tp1.sh +++ b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_eager_dbo_dp2tp1.sh @@ -1,55 +1,19 @@ +#!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# DeepSeek-V2-Lite (BF16) on P2pNcclAFDConnector, 2A2F eager mode and DBO: +# FFN DP2 x TP1 and Attention DP2 x TP1. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables +AFD_CONNECTOR=${AFD_CONNECTOR:-P2pNcclAFDConnector} +LAYOUT=${LAYOUT:-2A2F} MODEL_PATH=${MODEL_PATH:-/path/model_weights/DeepSeek-V2-Lite} -export VLLM_USE_V2_MODEL_RUNNER=1 +ATTN_TP=${ATTN_TP:-1} FFN_TP=${FFN_TP:-1} +MODE=eager +SERVE_ARGS=(--max-num-seqs 64 --max-num-batched-tokens 64 + --enable-dbo --dbo-decode-token-threshold 2 --dbo-prefill-token-threshold 12) -CUDA_VISIBLE_DEVICES=0,1 uv run vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "P2pNcclAFDConnector", - "host": "127.0.0.1", - "port": 6269, - "num_attention_ranks": 2, - "num_ffn_ranks": 2 - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 64 \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 12 \ - --enforce-eager \ - --host 127.0.0.1 \ - --port 18305 \ - --trust-remote-code > attn.log 2>&1 & - -CUDA_VISIBLE_DEVICES=2,3 uv run vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "P2pNcclAFDConnector", - "host": "127.0.0.1", - "port": 6269, - "num_attention_ranks": 2, - "num_ffn_ranks": 2 - } - }' \ - --max-num-seqs 64 \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 12 \ - --max-num-batched-tokens 64 \ - --enforce-eager \ - --host 127.0.0.1 \ - --port 18306 \ - --trust-remote-code > ffn.log 2>&1 & - -wait +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dbo_dp1tp2.sh b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dbo_dp1tp2.sh index e72893d22..bda209d64 100644 --- a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dbo_dp1tp2.sh +++ b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dbo_dp1tp2.sh @@ -1,61 +1,19 @@ +#!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# DeepSeek-V2-Lite (BF16) on P2pNcclAFDConnector, 2A2F FULL_DECODE_ONLY CUDA +# graphs and DBO: FFN DP1 x TP2 and Attention DP1 x TP2. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables +AFD_CONNECTOR=${AFD_CONNECTOR:-P2pNcclAFDConnector} +LAYOUT=${LAYOUT:-2A2F} MODEL_PATH=${MODEL_PATH:-/path/model_weights/DeepSeek-V2-Lite} -export VLLM_USE_V2_MODEL_RUNNER=1 +ATTN_TP=${ATTN_TP:-2} FFN_TP=${FFN_TP:-2} +MODE=graph CUDAGRAPH_CAPTURE_SIZES=${CUDAGRAPH_CAPTURE_SIZES:-64} # ascending +SERVE_ARGS=(--max-num-seqs 64 --max-num-batched-tokens 64 + --enable-dbo --dbo-decode-token-threshold 2 --dbo-prefill-token-threshold 12) -CUDA_VISIBLE_DEVICES=0,1 uv run vllm serve "$MODEL_PATH" \ - --data-parallel-size 1 \ - --tensor-parallel-size 2 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "P2pNcclAFDConnector", - "host": "127.0.0.1", - "port": 6269, - "num_attention_ranks": 2, - "num_ffn_ranks": 2 - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 64 \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 12 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{ - "cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes":[64] - }' \ - --host 127.0.0.1 \ - --port 18305 \ - --trust-remote-code > attn.log 2>&1 & - -CUDA_VISIBLE_DEVICES=2,3 uv run vllm serve "$MODEL_PATH" \ - --data-parallel-size 1 \ - --tensor-parallel-size 2 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "P2pNcclAFDConnector", - "host": "127.0.0.1", - "port": 6269, - "num_attention_ranks": 2, - "num_ffn_ranks": 2 - } - }' \ - --max-num-seqs 64 \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 12 \ - --max-num-batched-tokens 64 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{ - "cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes":[64] - }' \ - --host 127.0.0.1 \ - --port 18306 \ - --trust-remote-code > ffn.log 2>&1 & - -wait +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dbo_dp2tp1.sh b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dbo_dp2tp1.sh index c775780db..7baa8c391 100644 --- a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dbo_dp2tp1.sh +++ b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/2a2f_graph_dbo_dp2tp1.sh @@ -1,61 +1,19 @@ +#!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# DeepSeek-V2-Lite (BF16) on P2pNcclAFDConnector, 2A2F FULL_DECODE_ONLY CUDA +# graphs and DBO: FFN DP2 x TP1 and Attention DP2 x TP1. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables +AFD_CONNECTOR=${AFD_CONNECTOR:-P2pNcclAFDConnector} +LAYOUT=${LAYOUT:-2A2F} MODEL_PATH=${MODEL_PATH:-/path/model_weights/DeepSeek-V2-Lite} -export VLLM_USE_V2_MODEL_RUNNER=1 +ATTN_TP=${ATTN_TP:-1} FFN_TP=${FFN_TP:-1} +MODE=graph CUDAGRAPH_CAPTURE_SIZES=${CUDAGRAPH_CAPTURE_SIZES:-64} # ascending +SERVE_ARGS=(--max-num-seqs 64 --max-num-batched-tokens 64 + --enable-dbo --dbo-decode-token-threshold 2 --dbo-prefill-token-threshold 12) -CUDA_VISIBLE_DEVICES=0,1 uv run vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "P2pNcclAFDConnector", - "host": "127.0.0.1", - "port": 6269, - "num_attention_ranks": 2, - "num_ffn_ranks": 2 - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 64 \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 12 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{ - "cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes":[64] - }' \ - --host 127.0.0.1 \ - --port 18305 \ - --trust-remote-code > attn.log 2>&1 & - -CUDA_VISIBLE_DEVICES=2,3 uv run vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "P2pNcclAFDConnector", - "host": "127.0.0.1", - "port": 6269, - "num_attention_ranks": 2, - "num_ffn_ranks": 2 - } - }' \ - --max-num-seqs 64 \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 12 \ - --max-num-batched-tokens 64 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{ - "cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes":[64] - }' \ - --host 127.0.0.1 \ - --port 18306 \ - --trust-remote-code > ffn.log 2>&1 & - -wait +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/4a4f_eager_dbo_dp2tp2.sh b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/4a4f_eager_dbo_dp2tp2.sh index e2cbf9e71..28d718f90 100644 --- a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/4a4f_eager_dbo_dp2tp2.sh +++ b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/4a4f_eager_dbo_dp2tp2.sh @@ -1,55 +1,19 @@ +#!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# DeepSeek-V2-Lite (BF16) on P2pNcclAFDConnector, 4A4F eager mode and DBO: +# FFN DP2 x TP2 and Attention DP2 x TP2. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables +AFD_CONNECTOR=${AFD_CONNECTOR:-P2pNcclAFDConnector} +LAYOUT=${LAYOUT:-4A4F} MODEL_PATH=${MODEL_PATH:-/path/model_weights/DeepSeek-V2-Lite} -export VLLM_USE_V2_MODEL_RUNNER=1 +ATTN_TP=${ATTN_TP:-2} FFN_TP=${FFN_TP:-2} +MODE=eager +SERVE_ARGS=(--max-num-seqs 64 --max-num-batched-tokens 64 + --enable-dbo --dbo-decode-token-threshold 2 --dbo-prefill-token-threshold 12) -CUDA_VISIBLE_DEVICES=0,1,2,3 uv run vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 2 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "P2pNcclAFDConnector", - "host": "127.0.0.1", - "port": 6269, - "num_attention_ranks": 4, - "num_ffn_ranks": 4 - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 64 \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 12 \ - --enforce-eager \ - --host 127.0.0.1 \ - --port 18305 \ - --trust-remote-code > attn.log 2>&1 & - -CUDA_VISIBLE_DEVICES=4,5,6,7 uv run vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 2 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "P2pNcclAFDConnector", - "host": "127.0.0.1", - "port": 6269, - "num_attention_ranks": 4, - "num_ffn_ranks": 4 - } - }' \ - --max-num-seqs 64 \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 12 \ - --max-num-batched-tokens 64 \ - --enforce-eager \ - --host 127.0.0.1 \ - --port 18306 \ - --trust-remote-code > ffn.log 2>&1 & - -wait +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/4a4f_graph_dbo_dp2tp2.sh b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/4a4f_graph_dbo_dp2tp2.sh index cd87ccbb4..2ecb4e93f 100644 --- a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/4a4f_graph_dbo_dp2tp2.sh +++ b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/4a4f_graph_dbo_dp2tp2.sh @@ -1,61 +1,19 @@ +#!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# DeepSeek-V2-Lite (BF16) on P2pNcclAFDConnector, 4A4F FULL_DECODE_ONLY CUDA +# graphs and DBO: FFN DP2 x TP2 and Attention DP2 x TP2. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables +AFD_CONNECTOR=${AFD_CONNECTOR:-P2pNcclAFDConnector} +LAYOUT=${LAYOUT:-4A4F} MODEL_PATH=${MODEL_PATH:-/path/model_weights/DeepSeek-V2-Lite} -export VLLM_USE_V2_MODEL_RUNNER=1 +ATTN_TP=${ATTN_TP:-2} FFN_TP=${FFN_TP:-2} +MODE=graph CUDAGRAPH_CAPTURE_SIZES=${CUDAGRAPH_CAPTURE_SIZES:-64} # ascending +SERVE_ARGS=(--max-num-seqs 64 --max-num-batched-tokens 64 + --enable-dbo --dbo-decode-token-threshold 2 --dbo-prefill-token-threshold 12) -CUDA_VISIBLE_DEVICES=0,1,2,3 uv run vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 2 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "P2pNcclAFDConnector", - "host": "127.0.0.1", - "port": 6269, - "num_attention_ranks": 4, - "num_ffn_ranks": 4 - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 64 \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 12 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{ - "cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes":[64] - }' \ - --host 127.0.0.1 \ - --port 18305 \ - --trust-remote-code > attn.log 2>&1 & - -CUDA_VISIBLE_DEVICES=4,5,6,7 uv run vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 2 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "P2pNcclAFDConnector", - "host": "127.0.0.1", - "port": 6269, - "num_attention_ranks": 4, - "num_ffn_ranks": 4 - } - }' \ - --max-num-seqs 64 \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 12 \ - --max-num-batched-tokens 64 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{ - "cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes":[64] - }' \ - --host 127.0.0.1 \ - --port 18306 \ - --trust-remote-code > ffn.log 2>&1 & - -wait +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/multipod_2a_2f_graph_dbo_dp1tp2.sh b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/multipod_2a_2f_graph_dbo_dp1tp2.sh index b1a1f0b14..0a5dd95c0 100644 --- a/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/multipod_2a_2f_graph_dbo_dp1tp2.sh +++ b/recipe/gpu/P2pNcclAFDConnector/deepseek_v2_lite/prefill_decode_colocation/multipod_2a_2f_graph_dbo_dp1tp2.sh @@ -1,73 +1,19 @@ +#!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# DeepSeek-V2-Lite (BF16) on P2pNcclAFDConnector, two pods with 2A0F,0A2F, +# FULL_DECODE_ONLY CUDA graphs and DBO: each role is DP1 x TP2. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables -# Use .agents/skills/deploy-afd-k8s/SKILL.md to deploy this recipe -# on 2 pods using the setup below. -# For pod 1: POD=ATTENTION_0 -# For pod 2: POD=FFN_0 - +AFD_CONNECTOR=${AFD_CONNECTOR:-P2pNcclAFDConnector} +LAYOUT=${LAYOUT:-2A0F,0A2F} MODEL_PATH=${MODEL_PATH:-/path/model_weights/DeepSeek-V2-Lite} -export VLLM_USE_V2_MODEL_RUNNER=1 - -AFD_CONNECTOR_HOST=${AFD_CONNECTOR_HOST:-vllm-ffn-p2p-service} -AFD_CONNECTOR_PORT=${AFD_CONNECTOR_PORT:-6269} - -if [ "$POD" == "ATTENTION_0" ]; then - CUDA_VISIBLE_DEVICES=0,1 uv run vllm serve "$MODEL_PATH" \ - --data-parallel-size 1 \ - --tensor-parallel-size 2 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "P2pNcclAFDConnector", - "host": "'"${AFD_CONNECTOR_HOST}"'", - "port": '"${AFD_CONNECTOR_PORT}"', - "num_attention_ranks": 2, - "num_ffn_ranks": 2 - } - }' \ - --max-num-seqs 64 \ - --max-num-batched-tokens 64 \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 12 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{ - "cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes":[64] - }' \ - --host 127.0.0.1 \ - --port 18305 \ - --trust-remote-code > attn.log 2>&1 & -fi - -if [ "$POD" == "FFN_0" ]; then - CUDA_VISIBLE_DEVICES=0,1 uv run vllm serve "$MODEL_PATH" \ - --data-parallel-size 1 \ - --tensor-parallel-size 2 \ - --enable-expert-parallel \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "P2pNcclAFDConnector", - "host": "'"${AFD_CONNECTOR_HOST}"'", - "port": '"${AFD_CONNECTOR_PORT}"', - "num_attention_ranks": 2, - "num_ffn_ranks": 2 - } - }' \ - --max-num-seqs 64 \ - --enable-dbo \ - --dbo-decode-token-threshold 2 \ - --dbo-prefill-token-threshold 12 \ - --max-num-batched-tokens 64 \ - --max-cudagraph-capture-size 64 \ - --compilation-config '{ - "cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes":[64] - }' \ - --host 127.0.0.1 \ - --port 18305 \ - --trust-remote-code > ffn.log 2>&1 & -fi +ATTN_TP=${ATTN_TP:-2} FFN_TP=${FFN_TP:-2} +MODE=graph CUDAGRAPH_CAPTURE_SIZES=${CUDAGRAPH_CAPTURE_SIZES:-64} # ascending +SERVE_ARGS=(--max-num-seqs 64 --max-num-batched-tokens 64 + --enable-dbo --dbo-decode-token-threshold 2 --dbo-prefill-token-threshold 12) -wait +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/P2pNcclAFDConnector/qwen3_5_122b_a10b_fp8/prefill_decode_colocation/multipod_2a_2a_2f_graph.sh b/recipe/gpu/P2pNcclAFDConnector/qwen3_5_122b_a10b_fp8/prefill_decode_colocation/multipod_2a_2a_2f_graph.sh index ec63badbd..5582c73ee 100644 --- a/recipe/gpu/P2pNcclAFDConnector/qwen3_5_122b_a10b_fp8/prefill_decode_colocation/multipod_2a_2a_2f_graph.sh +++ b/recipe/gpu/P2pNcclAFDConnector/qwen3_5_122b_a10b_fp8/prefill_decode_colocation/multipod_2a_2a_2f_graph.sh @@ -1,100 +1,21 @@ +#!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project - -# Use .agents/skills/deploy-afd-k8s/SKILL.md to deploy this recipe -# on 3 pods using the setup below. -# For pod 1: POD=ATTENTION_0 -# For pod 2: POD=ATTENTION_1, ATTENTION_HEADLESS=1, ATTENTION_DP_START_RANK=2 -# For pod 3: POD=FFN_0 - +# +# Qwen3.5-122B-A10B-FP8 on P2pNcclAFDConnector, FULL_DECODE_ONLY CUDA graphs. +# Placement: one pod per comma-separated LAYOUT entry (default 2A0F,2A0F,0A2F), pod k +# running POD=AFD_ with the same LAYOUT; the default runs +# 4A2F on three 2-GPU pods (Attention DP4 over pods 0-1, FFN DP2 on pod 2); +# LAYOUT=4A2F runs it on one 6-GPU node. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables + +AFD_CONNECTOR=${AFD_CONNECTOR:-P2pNcclAFDConnector} +LAYOUT=${LAYOUT:-2A0F,2A0F,0A2F} MODEL_PATH=${MODEL_PATH:-/path/model_weights/Qwen3.5-122B-A10B-FP8} -export VLLM_USE_V2_MODEL_RUNNER=1 - -AFD_CONNECTOR_HOST=${AFD_CONNECTOR_HOST:-vllm-ffn-p2p-service} -AFD_CONNECTOR_PORT=${AFD_CONNECTOR_PORT:-1239} - -ATTENTION_DP_START_RANK=${ATTENTION_DP_START_RANK:-0} -ATTENTION_HEADLESS=${ATTENTION_HEADLESS:-0} -ATTENTION_DP_ADDRESS=${ATTENTION_DP_ADDRESS:-vllm-attn-dp-service} -ATTENTION_DP_RPC_PORT=${ATTENTION_DP_RPC_PORT:-13345} - -HEADLESS_ARGS=() -if [[ "$ATTENTION_HEADLESS" == "1" ]]; then - HEADLESS_ARGS=(--headless) -fi - -DP_START_RANK_ARGS=() -if [[ "$ATTENTION_DP_START_RANK" != "0" ]]; then - DP_START_RANK_ARGS=(--data-parallel-start-rank "$ATTENTION_DP_START_RANK") -fi - -if [[ "$POD" == ATTENTION_* ]]; then - CUDA_VISIBLE_DEVICES=0,1 uv run vllm serve "$MODEL_PATH" \ - --data-parallel-size 4 \ - --data-parallel-size-local 2 \ - "${DP_START_RANK_ARGS[@]}" \ - --data-parallel-address "$ATTENTION_DP_ADDRESS" \ - --data-parallel-rpc-port "$ATTENTION_DP_RPC_PORT" \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --dtype bfloat16 \ - --language-model-only \ - --max-model-len 114688 \ - --mamba-cache-mode align \ - --all2all-backend allgather_reducescatter \ - --seed 0 \ - --additional-config '{ - "afd": { - "role": "attention", - "connector": "P2pNcclAFDConnector", - "afd_process_group_timeout_s": 120, - "host": "'"${AFD_CONNECTOR_HOST}"'", - "port": '"${AFD_CONNECTOR_PORT}"', - "num_attention_ranks": 4, - "num_ffn_ranks": 2 - } - }' \ - --max-num-seqs 32 \ - --max-num-batched-tokens 8192 \ - --max-cudagraph-capture-size 32 \ - --compilation-config '{ - "cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes":[32] - }' \ - "${HEADLESS_ARGS[@]}" \ - --host 127.0.0.1 \ - --port 18305 > attn.log 2>&1 & -fi - -if [[ "$POD" == FFN_* ]]; then - CUDA_VISIBLE_DEVICES=0,1 uv run vllm serve "$MODEL_PATH" \ - --data-parallel-size 2 \ - --tensor-parallel-size 1 \ - --enable-expert-parallel \ - --dtype bfloat16 \ - --language-model-only \ - --max-model-len 114688 \ - --mamba-cache-mode align \ - --all2all-backend allgather_reducescatter \ - --seed 0 \ - --additional-config '{ - "afd": { - "role": "ffn", - "connector": "P2pNcclAFDConnector", - "afd_process_group_timeout_s": 120, - "host": "'"${AFD_CONNECTOR_HOST}"'", - "port": '"${AFD_CONNECTOR_PORT}"', - "num_attention_ranks": 4, - "num_ffn_ranks": 2 - } - }' \ - --max-num-seqs 32 \ - --max-num-batched-tokens 8192 \ - --max-cudagraph-capture-size 32 \ - --compilation-config '{ - "cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes":[32] - }' \ - --host 127.0.0.1 \ - --port 18305 > ffn.log 2>&1 & -fi +MODE=graph CUDAGRAPH_CAPTURE_SIZES=${CUDAGRAPH_CAPTURE_SIZES:-32} # ascending +SERVE_ARGS=(--dtype bfloat16 --language-model-only --max-model-len 114688 --mamba-cache-mode align + --all2all-backend allgather_reducescatter --seed 0 --max-num-seqs 32 --max-num-batched-tokens 8192) -wait +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" diff --git a/recipe/gpu/common/README.md b/recipe/gpu/common/README.md new file mode 100644 index 000000000..677e95df2 --- /dev/null +++ b/recipe/gpu/common/README.md @@ -0,0 +1,161 @@ +# Shared launcher for the GPU colocation recipes + +Every GPU `prefill_decode_colocation` recipe (`DeepEPM2NAFDConnector` and +`P2pNcclAFDConnector`) is a short script that states a model, a connector, a +default layout and its `vllm serve` flags, and then hands over to +[`afd_colocation.sh`](afd_colocation.sh). The launcher owns everything the +recipes share: the placement (`LAYOUT` and `POD` to GPUs and data-parallel +flags), the `afd` additional config, the decode-only benchmark hook, the +override variables, and the two `vllm serve` processes. + +## Writing a recipe + +```bash +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# Qwen3-30B-A3B (BF16) on DeepEPM2NAFDConnector, 2A2F graph + DBO. +# Placement: one pod per comma-separated LAYOUT entry (default 2A2F), pod k +# running POD=AFD_ with the same LAYOUT; FFN on each pod's first GPUs. +# Settings and running in a pod: recipe/gpu/common/README.md +# shellcheck disable=SC1090,SC2034 # the sourced launcher reads these variables + +AFD_CONNECTOR=${AFD_CONNECTOR:-DeepEPM2NAFDConnector} +LAYOUT=${LAYOUT:-2A2F} +MODEL_PATH=${MODEL_PATH:-/path/model_weights/Qwen3-30B-A3B} +ATTN_TP=${ATTN_TP:-1} FFN_TP=${FFN_TP:-1} +MODE=graph CUDAGRAPH_CAPTURE_SIZES=${CUDAGRAPH_CAPTURE_SIZES:-8,16,32,64} # ascending +SERVE_ARGS=(--enable-ep-weight-filter --max-num-seqs 64 --max-num-batched-tokens 4096 --max-model-len 4096 + --enable-dbo --dbo-decode-token-threshold 2 --dbo-prefill-token-threshold 8) + +source "${AFD_RECIPE_LIB:-$(dirname "${BASH_SOURCE[0]}")/../../../common/afd_colocation.sh}" +afd_colocation_serve "$@" +``` + +| Recipe variable | Meaning | +| --- | --- | +| `AFD_CONNECTOR` | Connector class. `P2pNcclAFDConnector` routes on the FFN and drops `CONNECTOR_CONFIG`; any other connector computes the gate on Attention. | +| `LAYOUT` | Default placement; see [Layout](#layout). | +| `MODEL_PATH` | Default weights directory or HF id. | +| `MODE` | `graph` (`FULL_DECODE_ONLY` CUDA graphs of `CUDAGRAPH_CAPTURE_SIZES`, the largest being the maximum capture size) or `eager` (`--enforce-eager`). | +| `SERVE_ARGS` | Array of model and tuning flags passed to both roles (DBO, `--enable-ep-weight-filter`, batch limits, dtype, ...). | +| `CONNECTOR_CONFIG` | Optional `connector_extra_config` members, for example `'"payload_format": "fp8_block128_fp32"'`. | + +The launcher always adds `--enable-expert-parallel`, `--trust-remote-code`, +`--cpu-distributed-timeout-seconds 3600` and `afd_process_group_timeout_s: +3600` (the roles can finish loading minutes apart). Its ports are fixed: +`18000` Attention API, `18001` FFN (serves no requests), `6239` AFD +rendezvous, `13345` and `13355` the Attention and FFN DP RPC ports across +pods. The default launcher path assumes the recipe sits at +`recipe/gpu////`. + +## Layout + +`LAYOUT` lists the pods in order, one `AF` entry each: `a` Attention +GPUs and `f` FFN GPUs. Inside a pod, the FFN takes the first `f` GPUs and the +Attention the next `a`; every pod runs the same script with the same +`LAYOUT` and its own `POD=AFD_` (`k` counts from 0; one pod needs no +`POD`). + +| `LAYOUT` | Pods | Per pod | +| --- | --- | --- | +| `2A2F` | 1 | FFN DP2 on GPUs 0,1, Attention DP2 on GPUs 2,3 | +| `4A2F` | 1 | FFN on GPUs 0,1, Attention DP4 on GPUs 2-5 | +| `2A0F,0A2F` | 2 | pod 0: Attention DP2; pod 1: FFN DP2 | +| `1A1F,1A1F` | 2 | FFN on GPU 0, Attention on GPU 1 | +| `2A0F,2A0F,0A2F` | 3 | pods 0-1: Attention DP4 (pod 1 headless); pod 2: FFN DP2 | + +`ATTN_TP` and `FFN_TP` split each pod's GPUs of that role into TP groups (they +must divide each pod's `a` or `f`); `num_attention_ranks` and `num_ffn_ranks` +still count GPUs. A role held by one pod gets only `--data-parallel-size`; a +role spread over pods is led by its first pod, and the others start headless +from their start rank. Each connector checks the placement at startup; see +its user guide for the layouts it accepts. + +## Settings + +Every setting is an environment variable; the defaults reproduce the recipe. + +| Variable | Default | Meaning | +| --- | --- | --- | +| `MODEL_PATH` | recipe | Weights directory or HF id. | +| `LAYOUT` | recipe | Pods and their GPUs; see [Layout](#layout). | +| `POD` | `AFD_0` | This pod's entry in `LAYOUT`, `AFD_`. | +| `ATTN_TP` / `FFN_TP` | recipe | TP size of the Attention / FFN role. | +| `CUDAGRAPH_CAPTURE_SIZES` | recipe | Ascending decode graph sizes (graph recipes). | +| `AFD_CONNECTOR` | recipe | Connector class, for example `P2pNcclAFDConnector` to run an M2N recipe on NCCL P2P. | +| `VLLM_ARGS` | empty | Extra `vllm serve` flags for both roles. | +| `ATTN_ARGS` / `FFN_ARGS` | empty | Extra `vllm serve` flags for one role. | +| `M2N_EXTRA_CONFIG` | empty | Extra `connector_extra_config` members, for example `'"expert_num_sms": 130'`. | +| `DECODE_BENCH` | empty | `kv_connector_extra_config` JSON of the decode-only benchmark connector on Attention (`{}` is enough); see [`tools/benchmarks`](../../../tools/benchmarks/). | +| `AFD_REPO` | current directory | Repository root that holds `tools/benchmarks` (with `DECODE_BENCH`). | +| `HEAD_HOST` | `127.0.0.1` on one pod | Address of pod 0 when it leads both roles (several pods, no Kubernetes). | +| `ATTN_HEAD_HOST` / `FFN_HEAD_HOST` | `HEAD_HOST`, else the Kubernetes Service names | Address of the first pod with Attention / FFN GPUs. The FFN head also hosts the AFD rendezvous. | +| `SERVE_HOST` | `127.0.0.1` | API server bind address (`0.0.0.0` to serve outside the pod). | +| `AFD_RECIPE_LIB` | next to the recipe in the repository | Path of `afd_colocation.sh`, for a recipe copied out of the repository. | +| `AFD_DRY_RUN` | empty | Print this pod's `vllm serve` commands instead of running them. | + +Each personalized recipe encodes the DP/TP shape named in its filename. The +`ATTN_TP` and `FFN_TP` defaults are therefore recipe-specific rather than +always `1`; set them only when intentionally changing that shape. + +`VLLM_ARGS`, `ATTN_ARGS`, `FFN_ARGS` and any arguments after the script name +come after the recipe's flags, so a repeated flag overrides its default. They +are split on whitespace without shell quoting, so pass JSON without spaces: + +```bash +VLLM_ARGS="--max-num-seqs 512" ATTN_ARGS="--no-enable-prefix-caching" \ + bash recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_graph_dp2tp1.sh +``` + +## Running directly + +From any directory (logs go to the current directory): + +```bash +export MODEL_PATH=/path/model_weights/Qwen3-30B-A3B +bash recipe/gpu/DeepEPM2NAFDConnector/qwen3_30b_a3b/prefill_decode_colocation/2a2f_graph_dp2tp1.sh +``` + +The script starts the FFN and Attention servers in the background, writes +`ffn.log` and `attn.log`, and waits. Send traffic to port `18000` once +`attn.log` prints `Application startup complete`. To see what a pod would +run without starting anything: + +```bash +AFD_DRY_RUN=1 LAYOUT=2A0F,0A2F POD=AFD_1 bash .../2a2f_graph_dp2tp1.sh +``` + +Several hosts without Kubernetes run the same script with their own `POD` +and the leaders' addresses: + +```bash +LAYOUT=2A0F,0A2F POD=AFD_0 ATTN_HEAD_HOST= FFN_HEAD_HOST= bash .../2a2f_graph_dp2tp1.sh +LAYOUT=2A0F,0A2F POD=AFD_1 ATTN_HEAD_HOST= FFN_HEAD_HOST= bash .../2a2f_graph_dp2tp1.sh +``` + +## Running in a pod + +A recipe needs `afd_colocation.sh` next to it. Ship both files and point +`AFD_RECIPE_LIB` at the launcher: + +- **`deploy-afd-k8s` skill** (`.agents/skills/deploy-afd-k8s/`): puts both + files into the `afd-recipe-script` ConfigMap (`recipe.sh` and + `afd_colocation.sh`), mounts it at `/recipe`, and sets + `AFD_RECIPE_LIB=/recipe/afd_colocation.sh` and `SERVE_HOST=0.0.0.0`. It + creates one pod per `LAYOUT` entry and the Services that the default head + host names resolve to. Edited recipes need no image rebuild. +- **A pod you manage**: copy both files and run the recipe with the launcher + path: + + ```bash + kubectl cp recipe/gpu/common/afd_colocation.sh :/work/afd_colocation.sh + kubectl cp .sh :/work/recipe.sh + kubectl exec -- bash -c 'cd /work && SERVE_HOST=0.0.0.0 \ + AFD_RECIPE_LIB=/work/afd_colocation.sh MODEL_PATH=/models/ bash recipe.sh' + ``` + +- **An image with this repository** (`docker/Dockerfile.ci` puts it in + `/opt/afd-plugin`): run the recipe in place, as the image's copy finds its + launcher: `bash /opt/afd-plugin/recipe/gpu///prefill_decode_colocation/.sh`. diff --git a/recipe/gpu/common/afd_colocation.sh b/recipe/gpu/common/afd_colocation.sh new file mode 100644 index 000000000..2abf78226 --- /dev/null +++ b/recipe/gpu/common/afd_colocation.sh @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +# +# Shared launcher of the GPU prefill_decode_colocation recipes: placement +# (LAYOUT/POD -> devices and DP flags), the AFD config and the two `vllm serve` +# processes. A recipe sets the variables below, sources this file and calls +# afd_colocation_serve "$@". See recipe/gpu/common/README.md. +# +# Set by the recipe: +# MODEL_PATH, AFD_CONNECTOR, LAYOUT model, connector class, default layout +# MODE graph (FULL_DECODE_ONLY CUDA graphs of +# CUDAGRAPH_CAPTURE_SIZES) or eager +# SERVE_ARGS array of model/tuning flags, both roles +# CONNECTOR_CONFIG optional connector_extra_config members +# Run settings (environment): POD, ATTN_TP, FFN_TP, VLLM_ARGS, ATTN_ARGS, +# FFN_ARGS, M2N_EXTRA_CONFIG, DECODE_BENCH, AFD_REPO, HEAD_HOST, +# ATTN_HEAD_HOST, FFN_HEAD_HOST, SERVE_HOST, AFD_DRY_RUN. + +ATTN_PORT=18000 FFN_PORT=18001 AFD_PORT=6239 ATTN_DP_RPC_PORT=13345 FFN_DP_RPC_PORT=13355 + +# role_dp TP DP_HOST DP_RPC_PORT ... -> TOTAL and this pod's DP +# flags in DP. A role spread over pods is led by its first pod; the others run +# headless from their start rank (the leader must omit it, or vLLM selects +# hybrid load balancing and rejects the headless peers). +role_dp() { + local tp=$1 host=$2 port=$3 i pods=0 start=0 lead=-1 + shift 3 + local n=("$@") + TOTAL=0 + for i in "${!n[@]}"; do + ((n[i])) || continue + ((n[i] % tp)) && { echo "TP $tp does not divide ${n[i]} ranks on pod $i" >&2; exit 1; } + ((lead < 0)) && lead=$i + ((i < K)) && start=$((start + n[i])) + pods=$((pods + 1)) TOTAL=$((TOTAL + n[i])) + done + DP=(--tensor-parallel-size "$tp" --data-parallel-size $((TOTAL / tp))) + ((pods > 1)) && DP+=(--data-parallel-size-local $((n[K] / tp)) + --data-parallel-address "$host" --data-parallel-rpc-port "$port") + ((pods > 1 && K != lead)) && DP+=(--headless --data-parallel-start-rank $((start / tp))) + return 0 +} + +afd_json() { # The AFD rendezvous runs on the pod that holds FFN rank 0. + cat <... +launch() { + local log=$1 devices + devices=$(seq "$2" $(($2 + $3 - 1)) | paste -sd, -) + shift 3 + if [[ -n ${AFD_DRY_RUN:-} ]]; then + echo "CUDA_VISIBLE_DEVICES=$devices vllm serve $(printf '%q ' "$@")> $log" + else + CUDA_VISIBLE_DEVICES=$devices vllm serve "$@" > "$log" 2>&1 & + fi +} + +afd_colocation_serve() { + export VLLM_USE_V2_MODEL_RUNNER=1 + local p + case ${MODE:-} in + graph) MODE_ARGS=(--max-cudagraph-capture-size "${CUDAGRAPH_CAPTURE_SIZES##*,}" + --compilation-config "{\"cudagraph_mode\": \"FULL_DECODE_ONLY\", \"cudagraph_capture_sizes\": [${CUDAGRAPH_CAPTURE_SIZES}]}") ;; + eager) MODE_ARGS=(--enforce-eager) ;; + *) echo "MODE must be graph or eager, got '${MODE:-}'" >&2; exit 1 ;; + esac + + GATE_ON_ATTENTION=true + CONNECTOR_CONFIG=${CONNECTOR_CONFIG:-}${M2N_EXTRA_CONFIG:+${CONNECTOR_CONFIG:+, }$M2N_EXTRA_CONFIG} + # P2P routes on the FFN and takes no connector_extra_config. + [[ $AFD_CONNECTOR == P2pNcclAFDConnector ]] && GATE_ON_ATTENTION=false CONNECTOR_CONFIG= + [[ -n $CONNECTOR_CONFIG ]] && CONNECTOR_CONFIG=", \"connector_extra_config\": {${CONNECTOR_CONFIG}}" + + IFS=, read -ra PODS <<<"$LAYOUT" + K=${POD:-AFD_0} && K=${K#AFD_} + if [[ ! $LAYOUT =~ ^[0-9]+A[0-9]+F(,[0-9]+A[0-9]+F)*$ || ! $K =~ ^[0-9]+$ ]] || ((K >= ${#PODS[@]})); then + echo "need LAYOUT=AF[,...] and POD=AFD_ in it: LAYOUT=$LAYOUT POD=${POD:-}" >&2 + exit 1 + fi + NUM_A=() NUM_F=() + for p in "${PODS[@]}"; do NUM_A+=("${p%%A*}") && p=${p#*A} && NUM_F+=("${p%F}"); done + ((${#PODS[@]} > 1)) || HEAD_HOST=${HEAD_HOST:-127.0.0.1} + + # Extra `vllm serve` flags go last, so they override the defaults. + read -ra ARGS <<<"${VLLM_ARGS:-}" && ARGS+=("$@") + read -ra ATTN_EXTRA <<<"${ATTN_ARGS:-}" + read -ra FFN_EXTRA <<<"${FFN_ARGS:-}" + if [[ -n ${DECODE_BENCH:-} ]]; then # tools/benchmarks is imported from the repository + ATTN_EXTRA=(--kv-transfer-config "{\"kv_connector\": \"AFDDecodeBenchConnector\", \"kv_connector_module_path\": \"tools.benchmarks.decode_bench\", \"kv_role\": \"kv_both\", \"kv_connector_extra_config\": ${DECODE_BENCH}}" "${ATTN_EXTRA[@]}") + export PYTHONPATH=${AFD_REPO:-$PWD}${PYTHONPATH:+:$PYTHONPATH} + fi + + role_dp "${FFN_TP:-1}" "${FFN_HEAD_HOST:-${HEAD_HOST:-vllm-ffn-dp-service}}" "$FFN_DP_RPC_PORT" "${NUM_F[@]}" + FFN_DP=("${DP[@]}") NUM_FFN=$TOTAL + role_dp "${ATTN_TP:-1}" "${ATTN_HEAD_HOST:-${HEAD_HOST:-vllm-attn-dp-service}}" "$ATTN_DP_RPC_PORT" "${NUM_A[@]}" + ATTN_DP=("${DP[@]}") NUM_ATTN=$TOTAL + + # The roles can finish loading minutes apart, hence the 3600 s timeouts. + COMMON=(--enable-expert-parallel --trust-remote-code --cpu-distributed-timeout-seconds 3600 + "${SERVE_ARGS[@]}" "${MODE_ARGS[@]}" --host "${SERVE_HOST:-127.0.0.1}") + + # FFN on this pod's GPUs [0, F), Attention on [F, F + A). + local F=${NUM_F[K]} A=${NUM_A[K]} + ((F)) && launch ffn.log 0 "$F" "$MODEL_PATH" --additional-config "$(afd_json ffn)" \ + "${COMMON[@]}" "${FFN_DP[@]}" "${ARGS[@]}" "${FFN_EXTRA[@]}" --port "$FFN_PORT" + ((A)) && launch attn.log "$F" "$A" "$MODEL_PATH" --additional-config "$(afd_json attention)" \ + "${COMMON[@]}" "${ATTN_DP[@]}" "${ARGS[@]}" "${ATTN_EXTRA[@]}" --port "$ATTN_PORT" + wait +} diff --git a/tools/benchmarks/request_generator.sh b/tools/benchmarks/request_generator.sh index a60dbbf7b..3ad8e3d45 100755 --- a/tools/benchmarks/request_generator.sh +++ b/tools/benchmarks/request_generator.sh @@ -8,13 +8,13 @@ # # Usage: # MODEL_PATH=/path/to/weights tools/benchmarks/request_generator.sh -# HOST=127.0.0.1 PORT=18305 INPUT_LEN=8192 OUTPUT_LEN=256 MAX_CONCURRENCY=64 \ +# HOST=127.0.0.1 PORT=18000 INPUT_LEN=8192 OUTPUT_LEN=256 MAX_CONCURRENCY=64 \ # MODEL_PATH=/path/to/weights tools/benchmarks/request_generator.sh # # Variables (default): # MODEL_PATH model weights dir/name (required) # HOST server host (127.0.0.1) -# PORT server port (18305) +# PORT server port (18000) # DATASET_NAME vllm bench dataset (random) # INPUT_LEN random input length (ISL) (1024) # OUTPUT_LEN random output length (OSL) (128) @@ -28,7 +28,7 @@ set -euo pipefail MODEL_PATH=${MODEL_PATH:?set MODEL_PATH to the model weights dir/name} HOST=${HOST:-127.0.0.1} -PORT=${PORT:-18305} +PORT=${PORT:-18000} DATASET_NAME=${DATASET_NAME:-random} INPUT_LEN=${INPUT_LEN:-1024} OUTPUT_LEN=${OUTPUT_LEN:-128} From 6497657fdb147e7dc833e7403d4af46e4ea43fc9 Mon Sep 17 00:00:00 2001 From: "Songym@Huawei" <109567708+Pingzii@users.noreply.github.com> Date: Fri, 13 Nov 2026 09:18:14 +0800 Subject: [PATCH 39/42] test: make the three platform-dependent unit tests portable (#574) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three unit tests fail on Windows for reasons unrelated to what they check: - `tests/unit/compat/npu/test_ascend_config_lifecycle.py` reads a source file with `Path.read_text()` and no encoding, so the GBK locale decodes UTF-8 bytes and raises `UnicodeDecodeError`; both reads now state UTF-8. - `tests/unit/package/test_cam_async_build.py` builds its manifest file list with native separators while setuptools works on posix manifest paths, so every required entry misses; the list now uses `as_posix()`. - `tests/unit/package/test_package.py` needs installation metadata to find the `vllm.general_plugins` entry point, which a source tree does not have; it now skips with that reason, and still checks the entry point when the distribution is installed. No production code changes. With these, full `tests/unit` on a Windows CPU host with `torch` and no `vLLM` is 900 passed / 89 skipped / 0 failed (previously 8 failed). Where these come from, for review context: - the six GBK failures: `tests/unit/compat/npu/test_ascend_config_lifecycle.py` arrived in #425 (`b7561d7`) with `ast.parse(path.read_text())` and no encoding, so it is latent everywhere except a locale that cannot decode UTF-8; - the manifest failure: #539 (`f2c79d8`) replaced the posix-normalised `required` set with `set(files.allfiles)`, whose entries carry native separators, so the comparison only holds where `/` is the separator. The #361 version of the test compared normalised paths and was correct; - the entry-point failure: the test first appeared as `tests/test_package.py` in #1 (`dffdb14`, "Add Phase 1 plugin skeleton") and was later moved to `tests/unit/package/test_package.py` by `d1fd84f`; it always needed installation metadata. None of the three involves the transports in #506; the tests are older than that branch. On verification scope: this PR's checks are label, pre-commit and DCO -- none of them installs the plugin or runs pytest. The numbers above come from a local run on this branch; the entry-point test exercising an installed build is therefore *not* covered by CI, before or after this change. Signed-off-by: 宋元盟 --- .../compat/npu/test_ascend_config_lifecycle.py | 4 ++-- tests/unit/package/test_cam_async_build.py | 4 +++- tests/unit/package/test_package.py | 16 ++++++++++++++-- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/tests/unit/compat/npu/test_ascend_config_lifecycle.py b/tests/unit/compat/npu/test_ascend_config_lifecycle.py index 756eb8187..5058a58ed 100644 --- a/tests/unit/compat/npu/test_ascend_config_lifecycle.py +++ b/tests/unit/compat/npu/test_ascend_config_lifecycle.py @@ -29,7 +29,7 @@ def load_functions(path, namespace, names): - tree = ast.parse(path.read_text()) + tree = ast.parse(path.read_text(encoding="utf-8")) tree.body = [ node for node in tree.body @@ -294,7 +294,7 @@ def test_async_shared_replication_only_overrides_effective_model_sp( self.assertEqual(vars(config), before) def test_factory_binds_already_loaded_native_scheduler_callers(self): - tree = ast.parse(FACTORY_PATH.read_text()) + tree = ast.parse(FACTORY_PATH.read_text(encoding="utf-8")) aliases = next( ast.literal_eval(node.value) for node in tree.body diff --git a/tests/unit/package/test_cam_async_build.py b/tests/unit/package/test_cam_async_build.py index 477bd3446..c1457eecc 100644 --- a/tests/unit/package/test_cam_async_build.py +++ b/tests/unit/package/test_cam_async_build.py @@ -72,8 +72,10 @@ def test_native_sources_survive_source_distribution_manifest(monkeypatch): root = Path(__file__).resolve().parents[3] monkeypatch.chdir(root) files = FileList() + # setuptools works on manifest paths, which are always posix; native + # separators would make every required entry miss on Windows. files.allfiles = [ - str(path.relative_to(root)) + path.relative_to(root).as_posix() for path in (root / "csrc").rglob("*") if path.is_file() ] diff --git a/tests/unit/package/test_package.py b/tests/unit/package/test_package.py index 8447094d6..d025f73c7 100644 --- a/tests/unit/package/test_package.py +++ b/tests/unit/package/test_package.py @@ -124,7 +124,19 @@ def test_register_afd_preserves_native_models_and_is_idempotent(registration): def test_entry_point_is_registered(): - entry_points = importlib.metadata.entry_points(group="vllm.general_plugins") - matches = [ep for ep in entry_points if ep.name == "afd"] + """The installed distribution has to expose the vLLM plugin entry point. + + Importing the package from a source tree does not register installation + metadata, so the entry point can only be checked on an installed build. + """ + try: + distribution = importlib.metadata.distribution("vllm-afd-plugin") + except importlib.metadata.PackageNotFoundError: + pytest.skip("the plugin is not installed in this environment") + matches = [ + entry_point + for entry_point in distribution.entry_points + if entry_point.group == "vllm.general_plugins" and entry_point.name == "afd" + ] assert matches assert matches[0].value == "afd_plugin:register_afd" From 26d8151209d420d505df1c0bd190c5a069dafc47 Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Fri, 13 Nov 2026 09:18:59 +0800 Subject: [PATCH 40/42] Refactor E2E scenarios into model-owned cases (#561) ## Purpose Move E2E scenario definitions from the shared runner into model-owned `cases.py` files. For migrated paths, a new combination of existing deployment, task and check capabilities requires no shared-module changes. This implements the existing-case migration from [RFC #549](https://github.com/vllm-project/afd-plugin/issues/549), preserving current acceptance rather than changing the coverage matrix. ## Issue - Related RFC: https://github.com/vllm-project/afd-plugin/issues/549 - This PR does not close the full RFC. ## Scope - Migrate all 48 existing single-deployment definitions: 38 CUDA and 10 NPU. - Migrate ten GPU multi-pod entries: four DeepSeek-V2-Lite and six Qwen3 MoE/DeepEP entries. This includes the three interleaved Qwen combinations already added to main by #555. - Preserve the 15 unique CI-selected node IDs, workloads, thresholds, concurrency and required evidence. - Retain the two NPU multi-pod entries and dedicated FFN/DSpark executors on their legacy paths. - No scenario additions/deletions relative to current main, cross-pod execution-evidence enhancement, runtime/compat changes, provisioning framework or CI pipeline changes. ## Implementation Notes - Immutable Case/Deployment/Role/Phase declarations keep model-specific choices in the model directory. Collection reads IDs; checkpoint-dependent settings resolve in the runner child. - Shared immutable eager/decode-graph presets construct common synchronous deployments for V2 on GPU/NPU, Qwen3 MoE and Qwen3.6. Models retain scenario tables, layouts, connector/capture/DBO choices and all workload/acceptance settings. Specialized DSV4 profiles remain model-local; no global scenario registry is introduced. - Deployment owns commands, connector startup order, readiness/liveness and cleanup. GPU pod coordination reuses this local executor while retaining rendezvous, placement, barriers, evaluator selection and verdict/abort propagation. - Tasks own workloads. The runner controls one observation window per phase; graph/DBO/Hash checks consume a shared snapshot. DSV4 GPU declares its two completion rounds and GSM8K as three phases. Existing native baseline evidence requirements are retained. - Structured records distinguish canonical/effective configuration, overrides, execution, coverage, cleanup and acceptance. Weakened GSM8K workloads remain debug runs and cannot qualify a fixed pytest entry; the verdict propagates to every pod. - DBO has one resolution rule for commands, per-role probe requirements and logging: explicit Role configuration overrides the Deployment default. Attention-only native DBO remains supported. HTTP client imports are delayed until task execution. - Align with `main@e58fdec`: NPU async CAM retains worker/service/runner shutdown budgets of 60/90/120 seconds. The worker environment override is preserved and recorded as effective configuration. Other profiles retain their existing budgets. - The zero-shared-diff extension rule applies to all single-deployment Cases and ten GPU multi-pod entries. Retained NPU multi-pod/dedicated paths remain outside that contract; each invocation uses one cleanup path. ## Test Plan Use CPU-safe collection, declaration/command comparisons, host integration with actual processes/sockets/signals, affected-file hooks, and the standard ready CI selection. Reuse earlier targeted device evidence for unchanged GPU paths; do not rerun the entire model/topology matrix. ## Test Result | Check | Result | | --- | --- | | Collection with HTTP client imports blocked | All 48 GPU node IDs and 10 NPU single-deployment IDs collect on a host without torch/vLLM | | Configuration/command comparison | All 48 single-deployment and the original seven GPU multi-pod declarations/argv remain identical to `8388dc3`; the three upstream-added Qwen interleaved combinations are retained | | GPU pod parity with latest main | All ten configurations / 30 role argv exactly match the legacy commands at `main@e58fdec` | | DBO follow-up, CPU job 7039 | Old source reproduces both specified regressions; fixed source passes 25 host checks, zero skips | | Latest main-alignment host run | Job 7048: 25 existing checks pass. Job 7051: the two shutdown cases reproduce old CLI 10 vs expected 90, then both pass after the fix; zero skips. All 27 host behaviors are covered across these runs, with verified owned cleanup and allocation release | | Shared-preset equivalence | Against `14add02`, all 48 single-deployment and seven GPU multi-pod Case declarations and argv match exactly, including workload, checks and cleanup settings | | Shared-preset host integration | CPU job 7175: all 27 existing host checks passed, zero failures/errors/skips; real processes, sockets, signals and TCPStore peers with stub service/evaluator CLIs. Owned process/port/IPC/shared-memory/scratch cleanup verified; allocation completed and released. Frozen test source matches commit `8388dc3` exactly | | Latest-main conflict integration | CPU job 7589: all eight affected multi-pod host checks passed, zero failures/errors/skips, including Qwen role-split and interleaved placement. Owned resource cleanup and allocation release verified; frozen source matches `a055cdc`. Other host checks are unchanged and retain job 7175's evidence. Affected-file pre-commit/mypy and diff checks passed | | Affected-file checks | Original 31-file checks passed; this six-file preset/docs update also passes pre-commit, including mypy/format/docs checks; diff hygiene passes | Host services/evaluators are stubs, and the probe-facing stub explicitly refuses device evidence. The two new host-only Cases use strict group cleanup for non-daemonizing stubs. Their initial run encountered a CPU Python without `os.pidfd_open`; only host-test isolation changed for the targeted recheck. Real NPU Case cleanup remains unchanged, and native CAM orphan/pidfd teardown is not qualified by these host checks. These checks verify harness configuration, process/RPC boundaries, error propagation and cleanup; they do not qualify accelerator execution. Earlier frozen-source hardware evidence includes 13 unique V2 P2P definitions, one DSV4 graph+DBO case (two ten-request rounds, 23/24 GSM8K answers, three four-rank windows and Hash pairing/freshness), a temporary two-GPU composition trial with zero shared-source edits, and one four-H20 interleaved multi-pod smoke on a single host. That smoke runs one seven-sample workload (2/7 correct at the original 0.27 threshold), with its result propagated to the peer. Every completed run has owned-resource cleanup and allocation-release evidence. These results are not a final-head full-matrix run, cross-node/RDMA qualification or additional NPU/DeepEP hardware qualification. Exact argv parity above refers to GPU multi-pod. Single-deployment comparison preserves CLI values/workloads; DSV4 budget flags move with Deployment ownership. The later DBO follow-up compares against `c176029`, and the shutdown follow-up compares with current main's protocol. Target pins: vLLM 0.32.0 (`5df63a7203cdb9328d824a27b682e77c7b9928be`) and vLLM-Ascend `bd42eea01d8312c4df469fabaed45f77a12df4d3`. Host/device evidence records its own frozen source; Linux runtime is Python 3.12.13 / torch 2.13.0+cu130. No model weights or runtime were downloaded/installed for this work. NPU device validation and unexecuted GPU combinations remain explicitly unqualified. ## Docs Impact Update the E2E README/module contract, multi-pod guide, GPU connector case link and run-e2e skill/Kubernetes example. RFC #549 and its bilingual design snapshot clarify shared presets versus model-owned scenarios; the coverage matrix is unchanged. Task-local reports, scratch scripts and raw validation output remain outside the repository. ---
Essential PR Checklist - [x] Purpose is clear and linked to the RFC. - [x] Scope is bounded. - [x] Pinned vLLM/vLLM-Ascend compatibility is considered; NPU hardware evidence is explicitly limited. - [x] No changes to upstream runtime source or plugin runtime/compat code. - [x] Plugin-owned execution paths are retained; no new runtime monkey patches or shims. - [x] Collection stays CPU-safe; device work remains in runner processes. - [x] Host/device results, skips and unexecuted combinations are distinguished. - [x] Documentation impact is stated.
--------- Signed-off-by: Zhou ziheng --- .agents/skills/run-e2e/SKILL.md | 12 +- .../skills/run-e2e/resources/k8-multi-pod.md | 45 +- docs/design/module/e2e_testing.md | 134 +++- docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md | 4 +- tests/conftest.py | 13 + tests/e2e/README.md | 146 +++- tests/e2e/accuracy/gsm8k.py | 85 +++ tests/e2e/case.py | 213 ++++++ tests/e2e/checks/__init__.py | 2 + tests/e2e/checks/mrv2.py | 372 +++++++++ tests/e2e/deployment.py | 436 +++++++++++ tests/e2e/environment.py | 146 +++- tests/e2e/models/deepseek_v2_lite/cases.py | 228 ++++++ .../deepseek_v2_lite/test_deepseek_v2_lite.py | 123 +-- .../test_deepseek_v2_lite_multi_pod.py | 50 +- tests/e2e/models/deepseek_v4_flash/cases.py | 269 +++++++ .../models/deepseek_v4_flash/completions.py | 71 +- .../deepseek_v4_flash/test_async_cam_npu.py | 59 +- .../e2e/models/deepseek_v4_flash/test_gpu.py | 63 +- .../deepseek_v4_flash/test_sync_camp2p_npu.py | 58 +- tests/e2e/models/qwen3_6/cases.py | 74 ++ tests/e2e/models/qwen3_6/test_qwen3_6.py | 62 +- tests/e2e/models/qwen3_moe/cases.py | 139 ++++ tests/e2e/models/qwen3_moe/test_qwen3_moe.py | 74 +- .../qwen3_moe/test_qwen3_moe_multi_pod.py | 57 +- tests/e2e/multi_pod/README.md | 512 ++++--------- tests/e2e/multi_pod/layout.py | 15 + tests/e2e/multi_pod/rendezvous.py | 6 +- tests/e2e/multi_pod/runner.py | 278 ++++++- tests/e2e/presets.py | 49 ++ tests/e2e/runner.py | 647 ++++++---------- tests/e2e/test_runner.py | 708 +++++++++++++++++- 32 files changed, 3853 insertions(+), 1297 deletions(-) create mode 100644 tests/e2e/case.py create mode 100644 tests/e2e/checks/__init__.py create mode 100644 tests/e2e/checks/mrv2.py create mode 100644 tests/e2e/deployment.py create mode 100644 tests/e2e/models/deepseek_v2_lite/cases.py create mode 100644 tests/e2e/models/deepseek_v4_flash/cases.py create mode 100644 tests/e2e/models/qwen3_6/cases.py create mode 100644 tests/e2e/models/qwen3_moe/cases.py create mode 100644 tests/e2e/presets.py diff --git a/.agents/skills/run-e2e/SKILL.md b/.agents/skills/run-e2e/SKILL.md index 22bb0c03a..b98eea221 100644 --- a/.agents/skills/run-e2e/SKILL.md +++ b/.agents/skills/run-e2e/SKILL.md @@ -18,7 +18,6 @@ To split a scenario's ranks across multiple pods/nodes on a Kubernetes cluster instead of running single-process, see `resources/k8-multi-pod.md`. -Each suite contains four gate scenarios: DeepSeek-V2-Lite on GPU has seven gate scenarios: - `baseline-graph` @@ -154,7 +153,7 @@ For Qwen3 MoE on GPU, run: ~~~bash python -m pytest -q -s \ - tests/e2e/models/qwen3_moe/test_qwen3_moe.py + tests/e2e/models/qwen3_moe/test_qwen3_moe.py::test_qwen3_moe ~~~ For Qwen3.6 MoE on CUDA, run: @@ -192,8 +191,13 @@ On cancellation, forward SIGTERM and allow over 90 seconds for cleanup. Success means all selected cases pass with zero skips: seven for the GPU DeepSeek gate; four for the NPU DeepSeek gate or GPU Qwen3 MoE suite; three for the GPU Qwen3.6 MoE suite. -The entire DeepSeek module includes local cases and collects 12 on GPU or -seven on NPU. Report the actual selected count, failed scenario, first +The DeepSeek P2P entry collects 13 cases on GPU or seven on NPU. Its separate +DeepEP M2N entry adds five GPU-only IDs even to an unfiltered NPU collection; +select the intended entry or node IDs rather than running an unfiltered module. +GPU multi-pod entries now invoke the model module's `multi-pod` subcommand; +follow the linked Kubernetes walkthrough for local checkpoint paths, per-pod +acceptance records and the existing workload-only evidence scope. +Report the actual selected count, failed scenario, first actionable error, and cleanup status. Any skip is a gate failure. ## Environment reference diff --git a/.agents/skills/run-e2e/resources/k8-multi-pod.md b/.agents/skills/run-e2e/resources/k8-multi-pod.md index 596f5e308..976ea0620 100644 --- a/.agents/skills/run-e2e/resources/k8-multi-pod.md +++ b/.agents/skills/run-e2e/resources/k8-multi-pod.md @@ -5,9 +5,9 @@ layout splits Attention and FFN ranks across more than one pod — optionally spread across nodes, for a genuine cross-node fabric test — rather than running everything in one process on one machine. -This walkthrough assumes the model under test is -`deepseek-ai/DeepSeek-V2-Lite`; substitute `MODEL`, `MODEL_PVC`, and the -scenario's topology if you're running a different model. +This walkthrough uses the model-owned DeepSeek-V2-Lite GPU entry. Select the +appropriate model module, local checkpoint and PVC for another existing suite; +do not infer model selection from the shared coordinator. ## How it works @@ -19,8 +19,8 @@ Attention/FFN ranks to launch) from its Kubernetes-assigned completion index, then rendezvous with its peers over a shared store before serving and evaluating GSM8K. No process outside the pods holds test state or drives the run: you apply the Service and Job once, and every pod decides -its own role, coordinates with its peers, and reports its own pass/fail -through its container exit code. +its own role, coordinates with its peers, and reports execution through its +container exit code and acceptance through its run record. ## Prerequisites @@ -75,10 +75,9 @@ both FFN ranks; that string becomes the runner's `--pod-layout` argument below, and its entry count becomes `NUM_PODS`, computed automatically from `POD_LAYOUT` in the Deploy script — you don't set it yourself. -The ranks across all entries must sum to the scenario's topology (a -`2a2f` scenario needs 2 Attention and 2 FFN ranks in total), and no single -pod's rank count for a role may exceed that role's TP size — a TP group -cannot span pods. +The ranks across all entries must sum to the scenario's topology (a `2a2f` +scenario needs 2 Attention and 2 FFN ranks in total). Each pod's rank count +for a role must be divisible by that role's TP size: a TP group cannot span pods. This assumes every pod requests the same number of GPUs (`GPUS_PER_POD` below): the layout can vary how many Attention/FFN ranks each pod carries, @@ -99,7 +98,7 @@ SCENARIO=afd-graph-2a2f POD_LAYOUT=2A0F,0A2F NUM_PODS=$(($(tr -cd ',' <<<"$POD_LAYOUT" | wc -c) + 1)) # entry count of POD_LAYOUT RUN_ID=$(date +%s) -MODEL=deepseek-ai/DeepSeek-V2-Lite +MODEL=/models # local checkpoint root on the pre-warmed model PVC GSM8K_OUTPUT_PATH=/work/gsm8k-results MODEL_PVC=deepseek-v2-lite-weights GPUS_PER_POD=2 @@ -161,7 +160,8 @@ spec: args: - python - -m - - tests.e2e.multi_pod.runner + - tests.e2e.models.deepseek_v2_lite.cases + - multi-pod - --scenario - ${SCENARIO} - --pod-layout @@ -170,7 +170,9 @@ spec: - ${RUN_ID} - --model - ${MODEL} - - --gsm8k-output-path + - --device-backend + - gpu + - --output-path - ${GSM8K_OUTPUT_PATH} - --store-host - ${JOB_NAME}-0.${JOB_NAME} @@ -247,15 +249,26 @@ kubectl get pods -n ${NAMESPACE} -l job-name=${JOB_NAME} \ kubectl logs -n ${NAMESPACE} ``` -The run passed only if every pod's exit code is `0`; a pod with no terminal -state yet (still running when the wait timed out) is not a pass. +Acceptance requires every pod's exit code to be `0` and every pod's `run.json` +to have `acceptance_passed=true`. A pod with no terminal state yet is not a pass. +The model CLI allows debug workloads, so CLI exit code alone is insufficient. +The pytest entries enforce that requirement through `run_case_entry()`. +`GSM8K_OUTPUT_PATH` is the run root: role logs and `run.json` are written there, +with evaluator artifacts under `evaluation/`. + +The ten existing GPU multi-pod entries retain workload, liveness and cleanup +checks. They do not collect cross-pod graph/DBO/Hash execution evidence. Qwen3 +DeepEP M2N uses `tests.e2e.models.qwen3_moe.cases multi-pod`, its declared +role-split/interleaved layouts and 100-sample workload; its connector runtime must already +be available. See [execution and maintenance](../../../../tests/e2e/multi_pod/README.md). ## DSV4 Flash on Ascend NPU `afd-dsv4-flash-async-cam-dp2tp4-ep8` splits its fixed 16-NPU deployment (Attention DP2/TP4, FFN DP8/TP1/EP8, async CAM) across two 8-NPU pods. Its -pass criterion is the ten concurrent chat completions, not GSM8K. This path -has unit coverage only; it has not yet run on NPU hardware. +pass criterion is the ten concurrent chat completions, not GSM8K. This retained +NPU path requires separate hardware evidence; GPU/host validation does not +qualify it. Start from the Deploy template above and change: diff --git a/docs/design/module/e2e_testing.md b/docs/design/module/e2e_testing.md index 946e13bbd..89077b760 100644 --- a/docs/design/module/e2e_testing.md +++ b/docs/design/module/e2e_testing.md @@ -5,6 +5,10 @@ status: normative owners: - "@yujuancao07" primary_code_paths: + - "tests/e2e/case.py" + - "tests/e2e/presets.py" + - "tests/e2e/deployment.py" + - "tests/e2e/checks/**" - "tests/e2e/runner.py" - "tests/e2e/cancellation.py" - "tests/e2e/multi_pod/**" @@ -51,7 +55,7 @@ verified_platform_refs: - "CUDA Qwen3 MoE" - "CUDA Qwen3.6 MoE" related_issues: [] -last_reviewed: 2026-11-09 +last_reviewed: 2026-11-10 --- # E2E testing @@ -67,13 +71,114 @@ operator tests, benchmarks, and performance tests are outside this scope. ## Structure +Single-deployment pytest entries use model-local `cases.py` builders for all +48 existing definitions: 38 CUDA and ten NPU. Stable node IDs and the 15 +existing CI selections stay unchanged. Collection reads IDs only. Builders +resolve checkpoint-dependent profile settings inside the runner subprocess; +the common `case_main` receives each model builder explicitly. + +`case.py` defines the common types; `presets.py` provides reusable synchronous +eager/decode-graph deployment construction across models and backends. V2 on +GPU/NPU, Qwen3 MoE and Qwen3.6 compose these presets with model-owned DP/TP +layouts, connector, capture size and DBO settings. V2 retains its explicit +comparison sequence-budget override. Presets select neither model IDs nor +scenarios and own no workload, threshold, evidence or cleanup requirements. +DSV4's specialized deployment configuration remains model-local. Shared +configuration does not generate a common scenario matrix across all models. + +`deployment.py` owns topology/resource binding, commands, connector launch +order, readiness, pre/post-evaluation liveness and cleanup. `runner.run_case` +executes declared phases through the serializable Task interface and writes +`run.json` after cleanup. GSM8K is shared; DSV4's concurrent answer/plumbing +semantics remain model-local. Its two concurrent rounds and GSM8K are phases, +including their existing sample, repetition and concurrency requirements. + +`checks/mrv2.py` prepares one worker probe and validates graph, DBO and Hash +inputs. The runner drives one reset/start/end per phase; checks consume a +shared immutable snapshot. Hash freshness compares declared windows. Every +required rank must report even in observation-only windows. Cases without +execution checks do not install the probe. + +The shared capabilities needed by existing DSV4 profiles are explicit: +per-role arguments/additional configuration and native DBO, connector extra +configuration, a pinned Attention rendezvous address, exact compilation +configuration, and `cleanup_profile`. Sync connectors start FFN then Attention; +async CAM starts Attention then FFN. The NPU async profile alone keeps its +scoped FFN process-ownership cleanup exception; other profiles are strict. +`Deployment.dbo` supplies a default, and an explicit `Role.dbo` overrides it. +`Role.dbo=None` inherits the default; `DboConfig()` keeps native thresholds. +`Deployment.dbo_for_role()` resolves this rule for commands, probe validation +and role logging. A DBO check requires it on every named role, so Attention-only +native DBO does not imply FFN DBO. +NPU async CAM declares 60-second worker, 90-second service and 120-second runner +shutdown budgets. The worker budget is an environment default, independent of +the service CLI timeout. Existing worker environment overrides are retained and +recorded in the effective configuration. Other profiles keep their declared +shutdown budgets and do not gain this worker default. + +The ten existing GPU multi-pod entries use model-local distributed Cases. +`multi_pod` derives local RoleSlots and coordinates addresses, launch barriers, +verdict and departure; `deployment.py` builds commands and owns local processes. +The original distributed workload, liveness and cleanup checks are retained; +no cross-pod MRV2/Hash evidence is added. Unsupported execution checks fail +before launch rather than being silently dropped. Every pod records the actual +evaluator configuration/results, its local outcome and cleanup. Qualification +requires every pod's acceptance entry to pass. + +The two NPU multi-pod entries and dedicated FFN/DSpark callers retain their +legacy path. Remaining CLI scenario configuration is NPU-only; GPU-specific +tables and evaluation/Hash branches have been removed. No cases are added or +removed. Implementation/collection does not establish device qualification: +report host, same-host distributed and cross-node/device execution separately. + +On migrated Case paths (all single-deployment entries and ten GPU multi-pod +entries), an existing-capability combination changes model declarations and +IDs only. +Changing Case fields, task/check behavior or deployment protocols is a shared +capability extension. A new connector must define launch order in deployment; +Cases do not override that order independently. Configuration conflicts in +deployment-owned flags fail before service startup. +Changing a preset affects its existing consumers and requires configuration +and command comparison plus affected integration checks. A new combination +using existing presets must leave the presets unchanged. +Retained NPU multi-pod and dedicated executors are outside this extension +contract until migrated. Case execution uses deployment-owned cleanup; +retained legacy execution uses its existing cleanup, never both for one run. + +Each Case run records canonical and effective settings, documented overrides, +resource bindings, role commands, phase results, evidence and cleanup outcome. +Configuration eligibility and execution success are independent. Weakened +sample counts or accuracy thresholds produce a debug run and cannot qualify a +fixed scenario. Permitted larger workloads preserve the model's existing +threshold rule. The model-module CLI can execute debug runs; the stable pytest +acceptance node additionally requires `acceptance_passed=true`. + +The GSM8K schema records the existing small-workload threshold separately: +V2 comparison debug runs still floor at 24 samples and use the declared smoke +threshold, while their canonical acceptance remains 128 samples at 0.27. +The host CLI integration covers this policy and rejects its qualification; +this field preserves an existing override rule rather than adding coverage. + +For migrated entries, the operator keeps CPU harness integration results, +actual device results, skipped and unexecuted entries separate. `--describe`, +collection and command comparisons do not establish device acceptance. A +retained old entry exits only after its workload, evidence, cleanup and active +callers have a validated replacement. + ```text tests/e2e/ -├── models//test_*.py # Test entries and case lists -├── accuracy/.py # Accuracy tool and result parsing -├── environment.py # Backend and role launch environment -├── runner.py # Service startup, evaluation, and cleanup -└── process_utils.py # Process-group termination and reaping +├── models//test_*.py # Stable pytest entries; launch a child +├── models//cases.py # Model-local declarations for migrated entries +├── case.py # Immutable scenario and deployment types +├── presets.py # Shared execution configuration; no scenario selection +├── runner.py # Case phase execution; remaining legacy entries +├── deployment.py # Case commands, service startup and cleanup +├── checks/mrv2.py # Host probe preparation and phase windows +├── mrv2_evidence.py # Worker-side probe +├── accuracy/.py # Shared workload and result validation +├── environment.py # Resource bindings and CLI environment +├── multi_pod/ # Peer coordination and placement; NPU legacy entry +└── process_utils.py # Process-group termination and reaping ``` New model files do not copy service startup, lm-eval, signal handling, or @@ -102,8 +207,8 @@ cleanup. Production code does not depend on the E2E harness. - `E2E-INV-007` — CI **MUST NOT** lower the sample count, accuracy threshold, or required case set. It **MUST** select cases by pytest node ID. - `E2E-INV-008` — A shared case **MUST** pass without skips on every platform - where CI selects it. New behavior **MUST** have focused unit tests and real - hardware evidence. + where CI selects it. New harness behavior **MUST** have affected integration + and real hardware evidence. ## Required coverage @@ -124,11 +229,14 @@ CUDA ready/merge CI selects seven DeepSeek-V2-Lite cases: the native baseline and the six DP2/TP2/DBO V2 cases. The V2 DP2 cases replace the three duplicate GPU 2A2F cases; the eager/DBO comparison retains 128 samples, concurrency 12 and the 0.27 accuracy threshold. The backend-neutral 2A2F IDs remain available -for NPU. The GPU module collects 13 cases, including 1A1F and 2A1F coverage -and the local-only `afd-v2-graph-dbo-tp2`; the NPU module collects seven. -Across the three GPU model suites, there are 20 cases: DeepSeek 13, Qwen3 -MoE four, and Qwen3.6 MoE three. The local-only TP2 Graph + DBO case is -outside the seven-case ready/merge CI gate. +for NPU. The `test_deepseek_v2_lite` entry collects 13 GPU cases, including +1A1F, 2A1F and the local-only `afd-v2-graph-dbo-tp2`; its NPU selection +collects seven. The P2P entries across DeepSeek, Qwen3 MoE and Qwen3.6 MoE +contain 20 CUDA cases (13 + 4 + 3). Separate DeepEP M2N entries add five GPU-only +cases each for DeepSeek and Qwen3 MoE. Unfiltered DeepSeek module collection +includes its five DeepEP IDs even on NPU; NPU runs select the P2P entry or +explicit node IDs. The local-only TP2 Graph + DBO case is outside the seven-case +ready/merge CI gate. GPU cases gate accuracy, service liveness and cleanup. AFD GPU graph/DBO cases additionally use a test-only worker extension to collect bounded per-rank diff --git a/docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md b/docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md index d87287845..bde5f1afe 100644 --- a/docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md +++ b/docs/gpu/NCCL_P2P_CONNECTOR_USER_GUIDE.md @@ -163,8 +163,8 @@ role-rank totals equal to DP x TP, and a registered AFD architecture. Elastic EP, EPLB, sequence-parallel MoE and compile SP are rejected. Eager and `FULL_DECODE_ONLY` graph execution are accepted. V2 DBO requires -exactly two microbatches and accepts Attention DP1 or DP > 1. The generic -[GPU E2E runner](../../tests/e2e/runner.py) defines eager/graph DP2 and TP2 +exactly two microbatches and accepts Attention DP1 or DP > 1. The model-local +[GPU E2E cases](../../tests/e2e/models/deepseek_v2_lite/cases.py) define eager/graph DP2 and TP2 scenarios, including `afd-v2-eager-dbo-dp2` and `afd-v2-graph-dbo-dp2`. See the [runtime matrix](../support.md#tested-runtime-matrix) diff --git a/tests/conftest.py b/tests/conftest.py index 1f7a46cf4..a805bfe58 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ from __future__ import annotations +import json import os import signal import subprocess @@ -95,6 +96,18 @@ def forward_cancellation(signum, _frame) -> None: signal.signal(signum, previous_handler) +def run_case_entry( + command: list[str], output_path: Path, env: dict[str, str] | None = None +) -> None: + """Require both successful execution and canonical acceptance at pytest gates.""" + run_runner(command, env=env) + record = json.loads((output_path / "run.json").read_text()) + assert record["acceptance_passed"], ( + "task execution succeeded but the fixed scenario did not qualify: " + f"{record['configuration']}" + ) + + def download_dataset(dataset_id: str, dataset_config: str | None = None) -> None: """Download/cache a Hugging Face dataset. diff --git a/tests/e2e/README.md b/tests/e2e/README.md index eda0c0784..4a2749616 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -12,15 +12,17 @@ The V2 DP2 cases replace the duplicate GPU `afd-eager-2a2f`, baseline and AFD eager, graph, and graph+DBO on 2A1F. Qwen3.6 MoE retains three: baseline and AFD eager and graph on 2A1F. GDN DBO is unsupported. -The DeepSeek-V2-Lite, Qwen3 MoE, and Qwen3.6 MoE GPU model suites collect -20 cases. The seven-case ready/merge CI gate excludes local-only cases such -as `afd-v2-graph-dbo-tp2`. The separate DSV4 Flash CUDA suite is described -below. - -The current suite has 13 DeepSeek cases, four Qwen3 MoE cases, and three -Qwen3.6 MoE cases. -Qwen3.6 GDN DBO is unsupported. Workload and threshold settings are -scenario-specific; see the scenario sections below. +The single-deployment inventory contains 48 definitions: 38 CUDA cases +(18 DeepSeek-V2-Lite, nine Qwen3 MoE, three Qwen3.6 MoE and eight DSV4 +Flash) and ten NPU cases (seven DeepSeek-V2-Lite and three DSV4 Flash). +The CUDA counts include the local DeepEP M2N entries. CI still selects the +same 15 unique node IDs across ready/merge and weekly gates. Definitions, +collected nodes and device acceptance results are different counts. + +The ten existing GPU multi-pod entries also use model-local Cases and the +shared local deployment executor. Their existing workload/liveness/cleanup +acceptance is retained; cross-pod execution evidence is not added. The two NPU +multi-pod entries and dedicated FFN/DSpark checks retain their existing path. Ordinary cases evaluate the first 7 GSM8K samples; DBO cases have a 24-sample minimum and 12 concurrent requests. The GPU DP2 eager/eager-DBO/ @@ -59,6 +61,113 @@ few-shot examples and a 4096-token maximum model length. See the [E2E testing design](../../docs/design/module/e2e_testing.md) before adding a model or case. +## Case migration and maintenance + +All 48 single-deployment entries now use model-local Python Cases through +their existing pytest node IDs. This changes the harness, not their accepted +workloads or coverage. It does not by itself establish device qualification: +report actual CUDA/NPU results, host integration, skips and unexecuted cases +separately. No existing case is removed in this migration. + +The extension rules below apply to the migrated Case paths: all single-deployment +entries, including NPU, and the ten GPU multi-pod entries. NPU multi-pod and +dedicated FFN/DSpark executors still use their retained legacy paths; they do not +yet have the same extension contract. + +- Define deployment, workload and checks in the model's `cases.py`. + Collection reads stable IDs; checkpoint and resource binding happens in the + child process. Shared code receives the model's builder explicitly. +- Reuse `presets.py` for common synchronous eager/decode-graph deployment + construction. V2 on GPU/NPU, Qwen3 MoE and Qwen3.6 share these presets. + Models supply their DP/TP layouts, backend, connector, capture size and DBO + settings; V2 keeps its comparison-specific sequence budget explicitly. + Presets contain no model IDs, scenario tables, workloads or acceptance rules. + DSV4 keeps its specialized deployment configuration in its model directory. +- Reuse existing fields, tasks, checks and connector protocols for a new + combination. Its shared-module diff must be empty. A missing field or + capability needs an explicit capability extension and affected verification. +- Keep sample counts, thresholds, required concurrency, evidence windows, + participating ranks and cleanup requirements with the Case. Do not infer + these from its ID in shared code. +- Preserve existing acceptance before deleting an old branch or helper. + Migrate active pytest, CLI and dedicated-script callers with their behavior. + +For an existing-capability combination on a migrated path, edit the model's +`cases.py` and its stable ID list; add CI/docs selection only where needed. +The common runner, +Case schema, presets, deployment, task/check implementations and probes must have zero +diff for that addition. Implement a new task as a frozen dataclass following +`Task`: `resolve()` applies documented overrides and reports ineligibility; +`execute()` runs the workload and validates its result. Its fields are recorded +as canonical/effective configuration. Model-specific response semantics stay +in the model directory. DSV4's existing two concurrent rounds plus GSM8K are +three declared phases, not runner branches. + +The runner resets/opens/closes each observation window once. Graph, DBO and +Hash checks consume the same phase snapshot; Hash freshness references the +preceding declared window. Observation-only phases still require every rank. +Pure eager/baseline cases without required checks do not load a probe; DSV4 +AFD eager intentionally retains its required Hash probe. + +Deployment owns connector order (sync FFN then Attention; async CAM Attention +then FFN), readiness, pre/post-workload liveness and cleanup. Its explicit +`cleanup_profile` retains the scoped NPU async CAM exception; ordinary cases +use strict cleanup. Per-role options, connector configuration, pinned Attention +address and exact compilation settings preserve the existing NPU profiles. +`Deployment.dbo` is the default for its roles. An explicit `Role.dbo` replaces +that default; `Role.dbo=None` inherits it. `DboConfig()` enables DBO with native +runtime thresholds, while an effective value of `None` leaves it disabled. +Commands, probe validation and role logging use `Deployment.dbo_for_role()`; +checks require DBO on each role they name. Async CAM can therefore enable DBO +on Attention alone without requiring it on FFN. + +GPU multi-pod entries use the model module's `multi-pod` subcommand and +`build_multi_pod_case()`. Placement and peer coordination stay in `multi_pod`; +the shared deployment executor builds and starts each local role. See the +[multi-pod execution guide](multi_pod/README.md) for the existing workload +settings, result paths and qualification limits. A pod layout does not silently +disable checks from a single-deployment Case. + +The legacy runner CLI and launch helpers remain for NPU and dedicated-script +callers. GPU-specific scenario tables, evaluation branches and Hash wrappers +have been retired from that executor. New GPU combinations extend model-local +declarations. Retire remaining legacy helpers only with their active callers. +The Case path uses the shared deployment cleanup; a legacy invocation uses its +retained cleanup. A single run does not execute both paths. + +The migrated module CLI also supports inspection and local debug execution: + +```bash +python -m tests.e2e.models.deepseek_v2_lite.cases \ + --scenario afd-v2-graph-dp2 --model /path/to/DeepSeek-V2-Lite \ + --device-backend gpu --devices 0,1,2,3 \ + --output-path /path/to/run --describe +``` + +Remove `--describe` to start the real services. Optional `--vllm-bin`, +`--api-port-base`, `--afd-port` and `--startup-timeout` bind local resources. +`AFD_E2E_API_PORT_BASE` and `AFD_E2E_AFD_PORT` provide the corresponding port +defaults. Device visibility still comes from the allocated environment. + +`run.json` records canonical/effective configuration, overrides, resource +bindings, commands, phase results and cleanup. `configuration.eligible` only +describes configuration; `execution.status` also reports actual execution. +Only `acceptance_passed=true` qualifies the fixed scenario. Lowering the +sample count or threshold with `AFD_GSM8K_LIMIT` / `AFD_GSM8K_THRESHOLD` marks +the run as debug, even if its workload succeeds. Increasing samples preserves +the existing larger-workload threshold. The module CLI allows debug execution; +the shared `run_case_entry()` pytest helper rejects an unqualified result. + +Required MRV2 checks prepare the probe before launch and store independent +per-phase evidence in `/mrv2.json`; role logs stay in the run directory. +Linux host harness checks use real child processes, sockets and signals with +stub service/evaluator CLIs. Multi-pod checks also require PyTorch's TCPStore. +They verify orchestration, not device execution: + +```bash +python -m pytest tests/e2e/test_runner.py +``` + ## Run Run from the repository root. The environment needs `vllm`, `pytest`, @@ -70,6 +179,13 @@ The selected test downloads/caches `openai/gsm8k` and its Hugging Face model when the backend model env var is unset. Point `HF_HOME` at a persistent cache if you want to reuse downloads across runs. +For an existing qualification, first locate the checkpoint from its recorded +run metadata and set the backend model path explicitly. A missing entry in one +cache directory does not establish that the model is absent or expand the +validation scope. When downloads are disallowed, use the existing model/dataset +cache with `HF_HUB_OFFLINE=1` and `HF_DATASETS_OFFLINE=1`; report missing assets +instead of preparing new downloads. + GPU: ```bash @@ -110,7 +226,7 @@ python -m pytest -q -s \ "tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py::test_deepseek_v2_lite[afd-graph-2a2f]" \ "tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py::test_deepseek_v2_lite[afd-graph-dbo-2a2f]" -# Qwen3 MoE (P2P cases; add the DeepEP M2N cases with -k deepep_m2n) +# Qwen3 MoE (P2P cases; run the DeepEP M2N entry separately) python -m pytest -q -s \ tests/e2e/models/qwen3_moe/test_qwen3_moe.py::test_qwen3_moe @@ -125,8 +241,10 @@ python -m pytest -q -s \ Success means seven GPU DeepSeek gate cases, four NPU DeepSeek gate cases, four GPU Qwen3 MoE P2P cases, five GPU Qwen3 MoE DeepEP M2N cases, or three GPU Qwen3.6 MoE cases, passed with zero skips. -Running the entire DeepSeek module collects 13 cases on GPU and seven on NPU, -including local-only cases. +The DeepSeek `test_deepseek_v2_lite` entry collects 13 cases on GPU and seven +on NPU, including local-only cases. The separate DeepEP M2N entry adds five +GPU-only cases. An unfiltered module collection includes those five IDs even +on NPU; select the P2P entry or the required node IDs for an NPU run. The Qwen3.5/3.6 adapter-family suite uses `Qwen/Qwen3.6-35B-A3B` and `P2pNcclAFDConnector` on CUDA. The runner now selects native ModelRunnerV2 @@ -407,6 +525,10 @@ service, and 120 seconds for the test runner to shut down. Equivalent external launchers should set `VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS=60` and pass `--shutdown-timeout 90` to both services; a service timeout alone does not change the worker deadline. Explicit worker environment overrides are retained. +The Case declares these three budgets through `worker_shutdown_timeout_s`, +`shutdown_timeout_s` and `termination_timeout_s`. The worker environment override +is recorded in the effective configuration and overrides alongside the canonical +60-second default. The retained legacy path uses the same default constants. Attention SIGKILL escalation remains a failure. FFN uses the existing scoped async CAM cleanup exception. diff --git a/tests/e2e/accuracy/gsm8k.py b/tests/e2e/accuracy/gsm8k.py index 39b60d4ab..99d511fd0 100644 --- a/tests/e2e/accuracy/gsm8k.py +++ b/tests/e2e/accuracy/gsm8k.py @@ -13,9 +13,12 @@ import sys import threading import time +from collections.abc import Mapping +from dataclasses import replace from pathlib import Path from types import FrameType +from tests.e2e.case import Gsm8kSettings, JsonValue from tests.e2e.process_utils import terminate_process_groups LM_EVAL_TIMEOUT_S = 7200 @@ -23,6 +26,88 @@ LM_EVAL_PROCESS_POLL_INTERVAL_S = 0.2 LM_EVAL_REAP_TIMEOUT_S = 5 LM_EVAL_READER_JOIN_TIMEOUT_S = 5 +GSM8K_FULL_SAMPLE_COUNT = 1319 +GSM8K_FULL_TIMEOUT_S = 8 * 60 * 60 +GSM8K_LIMIT_ENV = "AFD_GSM8K_LIMIT" +GSM8K_THRESHOLD_ENV = "AFD_GSM8K_THRESHOLD" + + +def resolve_settings( + canonical: Gsm8kSettings, environment: Mapping[str, str] +) -> tuple[Gsm8kSettings, tuple[str, ...]]: + """Apply documented local overrides and determine configuration eligibility.""" + limit = canonical.sample_limit + if GSM8K_LIMIT_ENV in environment: + raw = environment[GSM8K_LIMIT_ENV] + limit = None if raw == "all" else int(raw) + if limit is not None: + if limit < 1: + raise ValueError("GSM8K sample limit must be positive or 'all'") + limit = max(limit, canonical.minimum_samples) + threshold = canonical.threshold + if ( + canonical.smoke_max_samples is not None + and limit is not None + and limit <= canonical.smoke_max_samples + and canonical.smoke_threshold is not None + ): + threshold = canonical.smoke_threshold + if ( + canonical.smoke_max_samples is not None + and (limit is None or limit > canonical.smoke_max_samples) + and canonical.larger_sample_threshold is not None + ): + threshold = canonical.larger_sample_threshold + required_threshold = threshold + if GSM8K_THRESHOLD_ENV in environment: + threshold = float(environment[GSM8K_THRESHOLD_ENV]) + if not 0 <= threshold <= 1: + raise ValueError("GSM8K threshold must be a finite value between 0 and 1") + reasons = [] + canonical_count = canonical.sample_limit or GSM8K_FULL_SAMPLE_COUNT + actual_count = limit or GSM8K_FULL_SAMPLE_COUNT + if actual_count < canonical_count: + reasons.append(f"sample count reduced from {canonical_count} to {actual_count}") + if threshold < required_threshold: + reasons.append( + f"accuracy threshold reduced from {required_threshold} to {threshold}" + ) + return replace(canonical, sample_limit=limit, threshold=threshold), tuple(reasons) + + +def evaluate_settings( + settings: Gsm8kSettings, + *, + base_url: str, + model_name: str, + checkpoint: str, + output_path: Path, +) -> dict[str, JsonValue]: + """Execute the resolved Case workload and validate complete evaluator output.""" + results = _run_lm_eval( + base_url, + model_name, + output_path=str(output_path), + num_fewshot=settings.num_fewshot, + num_concurrent=settings.num_concurrent, + max_gen_toks=settings.max_gen_toks, + tokenizer=checkpoint, + limit=settings.sample_limit, + timeout_s=( + GSM8K_FULL_TIMEOUT_S if settings.sample_limit is None else LM_EVAL_TIMEOUT_S + ), + ) + count = _extract_gsm8k_sample_count(results) + expected = settings.sample_limit or GSM8K_FULL_SAMPLE_COUNT + if count != expected: + raise RuntimeError(f"GSM8K evaluated {count} samples; expected {expected}") + accuracy = _extract_gsm8k_accuracy(results) + if not accuracy >= settings.threshold: + raise RuntimeError( + f"GSM8K accuracy {accuracy:.4f} is below the required " + f"threshold {settings.threshold:.4f}" + ) + return {"sample_count": count, "accuracy": accuracy} def _finish_lm_eval_cleanup( diff --git a/tests/e2e/case.py b/tests/e2e/case.py new file mode 100644 index 000000000..13fc4ef59 --- /dev/null +++ b/tests/e2e/case.py @@ -0,0 +1,213 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Immutable declarations exchanged by model entries and the E2E executor. + +Importing declarations does not resolve checkpoints or import a device runtime. +New fields represent shared capabilities and require their own review. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Protocol, TypeAlias + +JsonValue: TypeAlias = ( + str | int | float | bool | None | list["JsonValue"] | dict[str, "JsonValue"] +) + +EXECUTION_EAGER = "eager" +EXECUTION_FULL_DECODE_GRAPH = "full_decode_graph" +CLEANUP_PROFILE_STRICT = "strict" +CLEANUP_PROFILE_NPU_ASYNC_CAM_LEGACY = "npu_async_cam_legacy" +BASELINE_ROLE = "baseline" +ATTENTION_ROLE = "attention" +FFN_ROLE = "ffn" +DEFAULT_GENERATED_TOKENS = 512 +DEFAULT_FEWSHOT = 8 +DEFAULT_API_PORT = 8000 +DEFAULT_CONNECTOR_PORT = 1239 +DEFAULT_STARTUP_TIMEOUT_S = 900 +DEFAULT_SHUTDOWN_TIMEOUT_S = 10 +DEFAULT_TERMINATION_TIMEOUT_S = 20 +WORKER_SHUTDOWN_TIMEOUT_ENV = "VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS" +NPU_ASYNC_VLLM_SHUTDOWN_TIMEOUT_S = 90 +NPU_ASYNC_WORKER_SHUTDOWN_TIMEOUT_S = 60 +NPU_ASYNC_TERMINATION_TIMEOUT_S = 120 + + +@dataclass(frozen=True) +class Role: + dp: int + tp: int = 1 + execution: str = EXECUTION_EAGER + dbo: DboConfig | None = None + vllm_args: tuple[str, ...] = () + additional_config: tuple[tuple[str, bool], ...] = () + + def __post_init__(self) -> None: + if self.dp < 1 or self.tp < 1: + raise ValueError("DP and TP must be positive") + if self.execution not in (EXECUTION_EAGER, EXECUTION_FULL_DECODE_GRAPH): + raise ValueError(f"unknown execution mode {self.execution!r}") + + @property + def rank_count(self) -> int: + return self.dp * self.tp + + @property + def uses_graph(self) -> bool: + return self.execution == EXECUTION_FULL_DECODE_GRAPH + + +@dataclass(frozen=True) +class DboConfig: + # None keeps the runtime default, as the existing async Attention path does. + decode_token_threshold: int | None = None + prefill_token_threshold: int | None = None + + +@dataclass(frozen=True) +class Deployment: + backend: str + attention: Role + ffn: Role | None = None + connector: str | None = None + cleanup_profile: str = CLEANUP_PROFILE_STRICT + dbo: DboConfig | None = None + cudagraph_capture_size: int | None = None + max_num_seqs: int | None = None + max_num_batched_tokens: int | None = None + compute_gate_on_attention: bool = False + connector_extra_config: tuple[tuple[str, int], ...] = () + pin_attention_address: bool = False + compilation_config: tuple[tuple[str, str | tuple[int, ...]], ...] = () + shutdown_timeout_s: int = DEFAULT_SHUTDOWN_TIMEOUT_S + # Existing worker environment overrides take precedence over this default. + worker_shutdown_timeout_s: int | None = None + termination_timeout_s: float = DEFAULT_TERMINATION_TIMEOUT_S + + @property + def is_baseline(self) -> bool: + return self.ffn is None + + @property + def roles(self) -> tuple[tuple[str, Role], ...]: + if self.ffn is None: + return ((BASELINE_ROLE, self.attention),) + return ((ATTENTION_ROLE, self.attention), (FFN_ROLE, self.ffn)) + + def dbo_for_role(self, role: str) -> DboConfig | None: + """A role's DBO overrides the deployment default; None inherits it.""" + override = dict(self.roles)[role].dbo + return self.dbo if override is None else override + + +@dataclass(frozen=True) +class Gsm8kSettings: + sample_limit: int | None + threshold: float + num_fewshot: int = DEFAULT_FEWSHOT + num_concurrent: int | None = None + max_gen_toks: int = DEFAULT_GENERATED_TOKENS + minimum_samples: int = 1 + # Existing smoke profiles raise their threshold for larger workloads. + smoke_max_samples: int | None = None + smoke_threshold: float | None = None + larger_sample_threshold: float | None = None + + +@dataclass(frozen=True) +class Gsm8kTask: + settings: Gsm8kSettings + + def resolve( + self, environment: Mapping[str, str] + ) -> tuple[Gsm8kTask, tuple[str, ...]]: + from tests.e2e.accuracy.gsm8k import resolve_settings + + settings, reasons = resolve_settings(self.settings, environment) + return Gsm8kTask(settings), reasons + + def execute( + self, *, base_url: str, model_name: str, checkpoint: str, output_path: Path + ) -> dict[str, JsonValue]: + from tests.e2e.accuracy.gsm8k import evaluate_settings + + return evaluate_settings( + self.settings, + base_url=base_url, + model_name=model_name, + checkpoint=checkpoint, + output_path=output_path, + ) + + +class Task(Protocol): + """A serializable task declaration, resolved and executed in the child.""" + + def resolve( + self, environment: Mapping[str, str] + ) -> tuple[Task, tuple[str, ...]]: ... + + def execute( + self, *, base_url: str, model_name: str, checkpoint: str, output_path: Path + ) -> dict[str, JsonValue]: ... + + +@dataclass(frozen=True) +class Mrv2Check: + roles: tuple[str, ...] + graph: bool = False + dbo: bool = False + + +@dataclass(frozen=True) +class HashInputCheck: + roles: tuple[str, ...] = (ATTENTION_ROLE, FFN_ROLE) + graph: bool = False + dbo: bool = False + compare_with: str | None = None + + +@dataclass(frozen=True) +class Phase: + name: str + task: Task + checks: tuple[Mrv2Check | HashInputCheck, ...] = () + + +@dataclass(frozen=True) +class Case: + id: str + model: str + deployment: Deployment + phases: tuple[Phase, ...] + served_model_name_prefix: str + common_vllm_args: tuple[str, ...] = () + + def __post_init__(self) -> None: + if not self.id or not self.phases: + raise ValueError("Case requires an ID and at least one phase") + names = [phase.name for phase in self.phases] + if len(names) != len(set(names)): + raise ValueError("phase names must be unique within a Case") + + +@dataclass(frozen=True) +class CaseEnvironment: + model: str + vllm_bin: str + device_backend: str + devices: tuple[str, ...] + output_path: Path + api_host: str = "127.0.0.1" + api_port_base: int = DEFAULT_API_PORT + afd_host: str = "127.0.0.1" + afd_port: int = DEFAULT_CONNECTOR_PORT + startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_S + + def __post_init__(self) -> None: + if len(self.devices) != len(set(self.devices)): + raise ValueError("devices must be unique") diff --git a/tests/e2e/checks/__init__.py b/tests/e2e/checks/__init__.py new file mode 100644 index 000000000..37277d264 --- /dev/null +++ b/tests/e2e/checks/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project diff --git a/tests/e2e/checks/mrv2.py b/tests/e2e/checks/mrv2.py new file mode 100644 index 000000000..7dd8b7b62 --- /dev/null +++ b/tests/e2e/checks/mrv2.py @@ -0,0 +1,372 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Host-side MRV2 probe preparation, per-phase windows and evidence checks.""" + +from __future__ import annotations + +import json +import re +import tempfile +import time +import urllib.request +from pathlib import Path + +from tests.e2e.case import Case, CaseEnvironment, HashInputCheck, Mrv2Check, Phase, Role +from tests.e2e.deployment import ServiceDeployment + +MRV2_EVIDENCE_RPC_TIMEOUT_S = 30 +MRV2_EVIDENCE_REPLAY_MIN_COUNT = 2 +PROCESS_POLL_INTERVAL_S = 0.2 +DBO_NUM_STAGES = 2 + + +class Mrv2Session: + """One shared probe per deployment, with independent immutable snapshots.""" + + def __init__(self, case: Case, environment: CaseEnvironment, run_id: str) -> None: + self.case = case + self.environment = environment + self.run_id = run_id + self.directory: tempfile.TemporaryDirectory[str] | None = None + self.start_marker: Path | None = None + self.hash_samples: dict[ + str, dict[tuple[int, int, int, int], set[tuple[int, ...]]] + ] = {} + layouts = dict(case.deployment.roles) + for phase in case.phases: + for check in phase.checks: + if not check.roles: + raise ValueError("MRV2 check requires at least one role") + for role in check.roles: + if role not in layouts: + raise ValueError(f"MRV2 requests absent role {role}") + if check.graph and not layouts[role].uses_graph: + raise ValueError( + f"graph evidence requested for eager role {role}" + ) + if check.dbo and case.deployment.dbo_for_role(role) is None: + raise ValueError( + f"DBO evidence requested for non-DBO role {role}" + ) + + def prepare(self) -> None: + if any(phase.checks for phase in self.case.phases): + self.directory = tempfile.TemporaryDirectory(prefix="afd-e2e-mrv2-") + self.start_marker = Path(self.directory.name) / "evaluation-start" + + @property + def arguments(self) -> tuple[str, ...]: + if self.start_marker is None: + return () + return ("--worker-extension-cls", "tests.e2e.mrv2_evidence.Worker") + + @property + def worker_environment(self) -> dict[str, str]: + if self.start_marker is None: + return {} + env = { + "VLLM_SERVER_DEV_MODE": "1", + "AFD_E2E_MRV2_EVIDENCE_START": str(self.start_marker), + } + if self.case.deployment.is_baseline: + env["AFD_E2E_NATIVE_BASELINE"] = "1" + hash_checks = [ + check + for phase in self.case.phases + for check in phase.checks + if isinstance(check, HashInputCheck) + ] + if hash_checks: + env["AFD_E2E_HASH_INPUT_IDS"] = "1" + env["AFD_E2E_HASH_REQUIRE_GRAPH"] = ( + "1" if any(check.graph for check in hash_checks) else "0" + ) + return env + + def rpc(self, method: str, window: str | None = None) -> None: + env = self.environment + request = urllib.request.Request( + f"http://{env.api_host}:{env.api_port_base}/collective_rpc", + data=json.dumps( + { + "method": method, + "timeout": MRV2_EVIDENCE_RPC_TIMEOUT_S, + "kwargs": {} if window is None else {"evaluation_window": window}, + } + ).encode(), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen( + request, timeout=MRV2_EVIDENCE_RPC_TIMEOUT_S + ) as reply: + reply.read() + + def start(self, phase: Phase) -> None: + if not phase.checks: + return + assert self.start_marker is not None + self.start_marker.with_name("evaluation-complete").unlink(missing_ok=True) + self.rpc("afd_reset_execution_evidence", phase.name) + pending = self.start_marker.with_suffix(".pending") + pending.write_text(phase.name) + pending.replace(self.start_marker) + + def finish( + self, phase: Phase, services: ServiceDeployment + ) -> tuple[tuple[str, str], ...]: + if not phase.checks: + return () + assert self.start_marker is not None + self.rpc("afd_log_execution_evidence") + self.start_marker.with_name("evaluation-complete").write_text(phase.name) + layouts = dict(self.case.deployment.roles) + expected = { + (role, dp, tp) + for check in phase.checks + for role in check.roles + for dp in range(layouts[role].dp) + for tp in range(layouts[role].tp) + } + deadline = time.monotonic() + MRV2_EVIDENCE_RPC_TIMEOUT_S + while True: + services.ensure_alive() + snapshot = tuple( + (role, line) + for role, line in services.execution_evidence + if f"window={phase.name} " in line + ) + observed = { + (role, int(match[1]), int(match[2])) + for role, line in snapshot + if (match := re.search(r"dp_rank=(\d+) tp_rank=(\d+)", line)) + is not None + } + if expected <= observed: + break + if time.monotonic() >= deadline: + raise RuntimeError( + f"missing MRV2 ranks in {phase.name}: {sorted(expected - observed)}" + ) + time.sleep(PROCESS_POLL_INTERVAL_S) + path = self.environment.output_path / phase.name + path.mkdir(parents=True, exist_ok=True) + (path / "mrv2.json").write_text( + json.dumps( + { + "run_id": self.run_id, + "case": self.case.id, + "phase": phase.name, + "evidence": snapshot, + }, + indent=2, + ) + + "\n" + ) + for check in phase.checks: + if isinstance(check, Mrv2Check): + assert_execution_evidence( + list(snapshot), + check, + role_layouts=layouts, + backend=self.case.deployment.backend, + asynchronous=self.case.deployment.connector + == "CAMAsyncAFDConnector", + ) + else: + samples = assert_hash_inputs( + list(snapshot), + role_layouts=layouts, + graph=check.graph, + dbo=check.dbo, + ) + if check.compare_with is not None: + assert_hash_freshness( + self.hash_samples[check.compare_with], + samples, + attention_ranks=layouts["attention"].rank_count, + dbo=check.dbo, + ) + self.hash_samples[phase.name] = samples + return snapshot + + def close(self) -> None: + if self.directory is not None: + self.directory.cleanup() + + +def assert_execution_evidence( + evidence: list[tuple[str, str]], + check: Mrv2Check, + *, + role_layouts: dict[str, Role], + backend: str, + asynchronous: bool = False, +) -> None: + if not check.graph and not check.dbo: + # Observation-only windows still require every rank in finish(). + return + metric = ( + "two_stage_graph_replays" + if check.dbo and check.graph + else "two_stage_steps" + if check.dbo + else "graph_replays" + ) + minimum = ( + MRV2_EVIDENCE_REPLAY_MIN_COUNT if metric == "two_stage_graph_replays" else 1 + ) + layout_kind = "consecutive_graph" if check.graph else "two_stage" + common_layouts: set[str] | None = None + for role in check.roles: + tp_size = role_layouts[role].tp + rank_count = role_layouts[role].rank_count + expected = { + (dp, tp) for dp in range(rank_count // tp_size) for tp in range(tp_size) + } + observed = set() + layouts: dict[str, set[tuple[int, int]]] = {} + for recorded_role, line in evidence: + if recorded_role != role: + continue + match = re.search( + rf"dp_rank=(\d+) tp_rank=(\d+).*\b{metric}=(\d+)(?: |$)", line + ) + if match is not None and int(match[3]) >= minimum: + rank = (int(match[1]), int(match[2])) + observed.add(rank) + if check.dbo and not asynchronous: + payload = json.loads(line.split(" layouts=", 1)[1]) + for layout in payload[layout_kind]: + key = json.dumps(layout, separators=(",", ":")) + layouts.setdefault(key, set()).add(rank) + if missing := expected - observed: + raise RuntimeError( + f"{backend.upper()} {role} lacks live " + f"{metric}>={minimum} evidence on ranks " + f"{sorted(missing)}" + ) + if check.dbo and not asynchronous: + complete_layouts = { + layout for layout, ranks in layouts.items() if expected <= ranks + } + common_layouts = ( + complete_layouts + if common_layouts is None + else common_layouts & complete_layouts + ) + if check.dbo and not asynchronous and not common_layouts: + raise RuntimeError( + f"{backend.upper()} DBO lacks matching two-stage layouts " + "across every participating rank" + + (" with consecutive graph replays" if check.graph else "") + ) + print( + f"[execution-coverage] live {metric}>={minimum} " + "confirmed on every participating rank" + ) + + +def assert_hash_inputs( + evidence: list[tuple[str, str]], + *, + role_layouts: dict[str, Role], + graph: bool, + dbo: bool, +) -> dict[tuple[int, int, int, int], set[tuple[int, ...]]]: + """Pair bounded successful live A/F samples in original control/peer order.""" + samples: dict[str, dict[tuple[int, int, int], dict]] = { + "attention": {}, + "ffn": {}, + } + observed_ranks: dict[str, set[tuple[int, int]]] = { + "attention": set(), + "ffn": set(), + } + for role, line in evidence: + match = re.search(r"dp_rank=(\d+) tp_rank=(\d+)", line) + assert match is not None + rank = (int(match[1]), int(match[2])) + payload = json.loads(line.split(" layouts=", 1)[1]) + for sample in payload["hash_ids"]: + if not sample["completed"] or sample["control"] <= 0: + raise RuntimeError( + "Hash ID sample did not complete a live control transaction" + ) + if graph and not sample["replayed"]: + raise RuntimeError("Hash ID sample lacks completed graph replay") + key = (sample["control"], sample["stage"], sample["peer"]) + if key in samples[role]: + raise RuntimeError(f"Duplicate {role} Hash ID sample: {key}") + if role == "attention" and sample["peer"] != rank[0]: + raise RuntimeError( + f"Attention Hash ID peer does not match DP rank: {key}" + ) + samples[role][key] = sample + observed_ranks[role].add(rank) + for role, ranks in observed_ranks.items(): + rank_count = role_layouts[role].rank_count + expected = {(dp, 0) for dp in range(rank_count)} + if ranks != expected: + raise RuntimeError( + f"{role} lacks bounded Hash ID samples on ranks {expected - ranks}" + ) + if samples["attention"].keys() != samples["ffn"].keys(): + raise RuntimeError("A/F Hash ID samples do not pair by control/stage/peer") + values: dict[tuple[int, int, int, int], set[tuple[int, ...]]] = {} + covered = set() + for key, attention in samples["attention"].items(): + ffn = samples["ffn"][key] + real = attention["real_tokens"] + bucket = attention["bucket"] + if ( + ffn["bucket"] != bucket + or len(ffn["input_ids"]) != bucket + or len(attention["input_ids"]) != real + or not 0 <= real <= bucket + or attention["input_ids"] != ffn["input_ids"][:real] + ): + raise RuntimeError( + f"Native A / actual Hash-layer F input IDs differ: {key}" + ) + if real: + _, stage, peer = key + covered.add((peer, stage)) + values.setdefault((peer, stage, bucket, real), set()).add( + tuple(attention["input_ids"]) + ) + stages = range(DBO_NUM_STAGES if dbo else 1) + required = { + (peer, stage) + for peer in range(role_layouts["attention"].rank_count) + for stage in stages + } + if missing := required - covered: + raise RuntimeError( + f"Hash ID samples lack real tokens for peer/stage {sorted(missing)}" + ) + print("[hash-inputs] native A and actual Hash-layer F IDs match on every rank") + return values + + +def assert_hash_freshness( + previous: dict[tuple[int, int, int, int], set[tuple[int, ...]]], + current: dict[tuple[int, int, int, int], set[tuple[int, ...]]], + *, + attention_ranks: int, + dbo: bool, +) -> None: + """Require new real IDs across rounds at the same padded and real size.""" + changed = { + (peer, stage) + for (peer, stage, bucket, real), values in current.items() + if (peer, stage, bucket, real) in previous + and values - previous[(peer, stage, bucket, real)] + } + stages = range(DBO_NUM_STAGES if dbo else 1) + required = {(peer, stage) for peer in range(attention_ranks) for stage in stages} + if missing := required - changed: + raise RuntimeError( + "Hash input IDs did not change across concurrent rounds with the same " + f"bucket and real token count: {sorted(missing)}" + ) + print("[hash-inputs] both concurrent windows show fresh IDs at the same bucket") diff --git a/tests/e2e/deployment.py b/tests/e2e/deployment.py new file mode 100644 index 000000000..1e92a1c1a --- /dev/null +++ b/tests/e2e/deployment.py @@ -0,0 +1,436 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Case deployment: protocol order, role commands, readiness and owned cleanup.""" + +from __future__ import annotations + +import json +import os +import shlex +import subprocess +import threading +import time +import urllib.error +import urllib.request +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path + +from tests.e2e.case import ( + CLEANUP_PROFILE_NPU_ASYNC_CAM_LEGACY, + CLEANUP_PROFILE_STRICT, + WORKER_SHUTDOWN_TIMEOUT_ENV, + Case, + CaseEnvironment, + DboConfig, + Deployment, + JsonValue, +) +from tests.e2e.multi_pod.layout import RoleSlot +from tests.e2e.process_utils import ( + kill_processes_matching_environment, + terminate_process_groups, +) + +REPO_ROOT = Path(__file__).resolve().parents[2] +PROCESS_POLL_INTERVAL_S = 0.2 +PROCESS_REAP_TIMEOUT_S = 5 +LOG_THREAD_JOIN_TIMEOUT_S = 2 +READY_POLL_INTERVAL_S = 2 +MRV2_EVIDENCE_PREFIX = "AFD E2E execution: " +NPU_ASYNC_PROCESS_KILL_TIMEOUT_S = 120 +# Starting the first role is not waiting for its connector rendezvous or API. +CONNECTOR_PROTOCOLS = ( + ("gpu", "P2pNcclAFDConnector", ("ffn", "attention")), + ("gpu", "DeepEPM2NAFDConnector", ("ffn", "attention")), + ("npu", "CAMP2pAFDConnector", ("ffn", "attention")), + ("npu", "CAMAsyncAFDConnector", ("attention", "ffn")), +) +DEPLOYMENT_ARGUMENTS = frozenset( + ( + "--config", + "--additional-config", + "--served-model-name", + "--data-parallel-size", + "--tensor-parallel-size", + "--data-parallel-size-local", + "--data-parallel-start-rank", + "--data-parallel-address", + "--data-parallel-rpc-port", + "--headless", + "--worker-extension-cls", + "--compilation-config", + "--enforce-eager", + "--max-num-seqs", + "--max-num-batched-tokens", + "--max-cudagraph-capture-size", + "--cudagraph-capture-sizes", + "--enable-dbo", + "--dbo-decode-token-threshold", + "--dbo-prefill-token-threshold", + "--shutdown-timeout", + "--host", + "--port", + ) +) + + +@dataclass(frozen=True) +class ResolvedDeployment: + case: Case + environment: CaseEnvironment + role_devices: tuple[tuple[str, tuple[str, ...]], ...] + launch_order: tuple[str, ...] + slots: tuple[RoleSlot, ...] = () + + def command(self, role: str, probe_arguments: tuple[str, ...] = ()) -> list[str]: + profile = self.case.deployment + layout = dict(profile.roles)[role] + dbo = profile.dbo_for_role(role) + env = self.environment + slot = next((slot for slot in self.slots if slot.role == role), None) + cmd = [ + env.vllm_bin, + "serve", + env.model, + "--shutdown-timeout", + str(profile.shutdown_timeout_s), + "--served-model-name", + f"{self.case.served_model_name_prefix}-{role}", + "--data-parallel-size", + str(layout.dp), + "--tensor-parallel-size", + str(layout.tp), + "--enable-expert-parallel", + ] + if profile.ffn is not None: + afd: dict[str, JsonValue] = { + "role": role, + "connector": profile.connector, + "host": env.afd_host if slot is None else slot.afd_host, + "port": env.afd_port, + "num_attention_ranks": profile.attention.rank_count, + "num_ffn_ranks": profile.ffn.rank_count, + } + if profile.connector == "CAMAsyncAFDConnector": + afd["async"] = True + if profile.compute_gate_on_attention: + afd["compute_gate_on_attention"] = True + if profile.connector_extra_config: + afd["connector_extra_config"] = dict(profile.connector_extra_config) + cmd.extend( + [ + "--enable-ep-weight-filter", + "--additional-config", + json.dumps( + {"afd": afd, **dict(layout.additional_config)}, + separators=(",", ":"), + ), + ] + ) + if slot is not None and slot.spans_pods: + cmd.extend( + [ + "--data-parallel-size-local", + str(slot.dp_size_local), + "--data-parallel-address", + slot.dp_address, + "--data-parallel-rpc-port", + str(slot.dp_rpc_port), + ] + ) + # A leader must omit start-rank: vLLM otherwise selects external LB + # and rejects its headless peers. + if slot.headless: + cmd.extend( + [ + "--data-parallel-start-rank", + str(slot.dp_start_rank), + "--headless", + ] + ) + elif role == "attention" and profile.pin_attention_address: + cmd.extend( + [ + "--data-parallel-address", + env.afd_host if slot is None else slot.dp_address, + ] + ) + cmd.extend(probe_arguments) + # Native baseline emits DBO before graph flags; preserve existing CLI. + if profile.is_baseline: + cmd.extend(dbo_arguments(dbo)) + if profile.max_num_seqs is not None: + cmd.extend(["--max-num-seqs", str(profile.max_num_seqs)]) + if profile.max_num_batched_tokens is not None: + cmd.extend( + ["--max-num-batched-tokens", str(profile.max_num_batched_tokens)] + ) + if profile.compilation_config: + cmd.extend( + [ + "--compilation-config", + json.dumps(dict(profile.compilation_config), separators=(",", ":")), + ] + ) + elif layout.uses_graph: + capture_size = str(profile.cudagraph_capture_size) + cmd.extend( + [ + "--max-cudagraph-capture-size", + capture_size, + "--cudagraph-capture-sizes", + capture_size, + "--compilation-config", + json.dumps( + {"cudagraph_mode": "FULL_DECODE_ONLY"}, separators=(",", ":") + ), + ] + ) + else: + cmd.append("--enforce-eager") + if not profile.is_baseline: + cmd.extend(dbo_arguments(dbo)) + port = env.api_port_base + (role == "ffn") + if slot is None or not slot.headless: + cmd.extend(["--host", env.api_host, "--port", str(port)]) + cmd.extend(layout.vllm_args) + cmd.extend(self.case.common_vllm_args) + return cmd + + +def dbo_arguments(dbo: DboConfig | None) -> list[str]: + if dbo is None: + return [] + flags = ["--enable-dbo"] + if dbo.decode_token_threshold is not None: + flags.extend(["--dbo-decode-token-threshold", str(dbo.decode_token_threshold)]) + if dbo.prefill_token_threshold is not None: + flags.extend( + ["--dbo-prefill-token-threshold", str(dbo.prefill_token_threshold)] + ) + return flags + + +def launch_order(profile: Deployment) -> tuple[str, ...]: + if profile.is_baseline: + return ("baseline",) + for backend, connector, order in CONNECTOR_PROTOCOLS: + if (profile.backend, profile.connector) == (backend, connector): + return order + raise ValueError( + f"unsupported connector protocol: {profile.backend}/{profile.connector}" + ) + + +def resolve( + case: Case, environment: CaseEnvironment, *, slots: tuple[RoleSlot, ...] = () +) -> ResolvedDeployment: + profile = case.deployment + if environment.device_backend != profile.backend: + raise ValueError( + f"Case requires {profile.backend}, got {environment.device_backend}" + ) + if profile.cleanup_profile not in ( + CLEANUP_PROFILE_STRICT, + CLEANUP_PROFILE_NPU_ASYNC_CAM_LEGACY, + ): + raise ValueError(f"unknown cleanup profile {profile.cleanup_profile!r}") + if profile.cleanup_profile == CLEANUP_PROFILE_NPU_ASYNC_CAM_LEGACY and ( + profile.backend != "npu" or profile.connector != "CAMAsyncAFDConnector" + ): + raise ValueError("legacy NPU async cleanup requires the NPU async CAM protocol") + if any(role.uses_graph for _, role in profile.roles) and ( + profile.cudagraph_capture_size is None or profile.cudagraph_capture_size < 1 + ): + raise ValueError("graph deployment requires a positive capture size") + arguments = ( + *case.common_vllm_args, + *(arg for _, layout in profile.roles for arg in layout.vllm_args), + ) + for argument in arguments: + name = argument.split("=", 1)[0].replace("_", "-") + if name in DEPLOYMENT_ARGUMENTS: + raise ValueError(f"{name} must be declared by Deployment or Check") + required = ( + sum(len(slot.devices) for slot in slots) + if slots + else sum(role.rank_count for _, role in profile.roles) + ) + if len(environment.devices) < required: + raise ValueError( + f"Case needs {required} devices, got {len(environment.devices)}" + ) + if slots: + role_bindings = tuple( + ( + slot.role, + tuple(environment.devices[int(device)] for device in slot.devices), + ) + for slot in slots + ) + return ResolvedDeployment( + case, environment, role_bindings, launch_order(profile), slots + ) + bound = [] + offset = 0 + for role, layout in profile.roles: + bound.append((role, environment.devices[offset : offset + layout.rank_count])) + offset += layout.rank_count + return ResolvedDeployment(case, environment, tuple(bound), launch_order(profile)) + + +class ServiceDeployment: + """The single owner of all local service processes for one deployment.""" + + def __init__( + self, + resolved: ResolvedDeployment, + *, + run_id: str, + probe_arguments: tuple[str, ...] = (), + probe_environment: dict[str, str] | None = None, + extra_environment: Mapping[str, str] | None = None, + ) -> None: + self.resolved = resolved + self.run_id = run_id + self.probe_arguments = probe_arguments + self.probe_environment = probe_environment or {} + self.extra_environment = extra_environment or {} + self.processes: list[subprocess.Popen[str]] = [] + self.processes_by_role: dict[str, subprocess.Popen[str]] = {} + self.log_threads: list[threading.Thread] = [] + self.execution_evidence: list[tuple[str, str]] = [] + + def start(self) -> None: + for role in self.resolved.launch_order: + self.start_role(role) + self.wait_ready() + + def start_role(self, role: str) -> None: + """Launch one local role; multi-pod owns barriers between these calls.""" + resolved = self.resolved + env = resolved.environment + role_devices = dict(resolved.role_devices) + env.output_path.mkdir(parents=True, exist_ok=True) + command = resolved.command(role, self.probe_arguments) + visible = ",".join(role_devices[role]) + process_env = os.environ.copy() + process_env.update(self.extra_environment) + backend = resolved.case.deployment.backend + process_env[ + "ASCEND_RT_VISIBLE_DEVICES" if backend == "npu" else "CUDA_VISIBLE_DEVICES" + ] = visible + process_env["VLLM_PLUGINS"] = ( + ("ascend" if backend == "npu" else "") + if resolved.case.deployment.is_baseline + else ("ascend,afd" if backend == "npu" else "afd") + ) + process_env["VLLM_USE_V2_MODEL_RUNNER"] = "1" + process_env["AFD_E2E_RUN_ID"] = self.run_id + process_env["AFD_E2E_PROCESS_ROLE"] = role + process_env["PYTHONUNBUFFERED"] = "1" + process_env.setdefault("VLLM_ENGINE_READY_TIMEOUT_S", "18000") + worker_timeout = resolved.case.deployment.worker_shutdown_timeout_s + if worker_timeout is not None: + process_env.setdefault(WORKER_SHUTDOWN_TIMEOUT_ENV, str(worker_timeout)) + process_env.pop("AFD_PLUGIN_EARLY_ENGINE_PATCH", None) + process_env.pop("AFD_E2E_HASH_INPUT_IDS", None) + process_env.pop("AFD_E2E_HASH_REQUIRE_GRAPH", None) + process_env.pop("AFD_E2E_NATIVE_BASELINE", None) + process_env.pop("AFD_E2E_MRV2_EVIDENCE_START", None) + if backend == "npu": + process_env.pop("VLLM_ASCEND_ENABLE_FLASHCOMM1", None) + if resolved.case.deployment.dbo_for_role(role) is not None: + process_env["VLLM_LOGGING_LEVEL"] = "DEBUG" + process_env.update(self.probe_environment) + inherited = process_env.get("PYTHONPATH", "") + process_env["PYTHONPATH"] = os.pathsep.join( + path for path in (str(REPO_ROOT), inherited) if path + ) + print(f"\n=== Starting {role}: {shlex.join(command)} ===", flush=True) + process = subprocess.Popen( + command, + cwd=REPO_ROOT, + env=process_env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + start_new_session=True, + ) + self.processes.append(process) + self.processes_by_role[role] = process + thread = threading.Thread(target=self.stream, args=(role, process), daemon=True) + self.log_threads.append(thread) + thread.start() + self.ensure_alive() + + def stream(self, role: str, process: subprocess.Popen[str]) -> None: + path = self.resolved.environment.output_path / f"{role}.log" + assert process.stdout is not None + with path.open("w") as log: + for line in process.stdout: + log.write(line) + log.flush() + print(f"[{role}] {line}", end="", flush=True) + if MRV2_EVIDENCE_PREFIX in line: + self.execution_evidence.append((role, line.strip())) + + def ensure_alive(self) -> None: + for process in self.processes: + if process.poll() is not None: + raise RuntimeError( + f"service {process.pid} exited with {process.returncode}" + ) + + def wait_ready( + self, *, path: str = "/v1/models", on_poll: Callable[[], None] | None = None + ) -> None: + env = self.resolved.environment + url = f"http://{env.api_host}:{env.api_port_base}{path}" + deadline = time.monotonic() + env.startup_timeout + last_error: BaseException | None = None + while time.monotonic() < deadline: + self.ensure_alive() + if on_poll is not None: + on_poll() + try: + with urllib.request.urlopen(url, timeout=5) as response: + if response.status == 200: + return + except (OSError, urllib.error.URLError) as exc: + last_error = exc + time.sleep(READY_POLL_INTERVAL_S) + raise TimeoutError(f"service did not become ready at {url}: {last_error}") + + def close(self) -> None: + legacy_async = ( + self.resolved.case.deployment.cleanup_profile + == CLEANUP_PROFILE_NPU_ASYNC_CAM_LEGACY + ) + ffn = self.processes_by_role.get("ffn") + failures = terminate_process_groups( + self.processes, + termination_timeout_s=self.resolved.case.deployment.termination_timeout_s, + poll_interval_s=PROCESS_POLL_INTERVAL_S, + reap_timeout_s=PROCESS_REAP_TIMEOUT_S, + deferred_sigkill_pgids=(ffn.pid,) + if legacy_async and ffn is not None + else (), + ) + if legacy_async: + failures.extend( + kill_processes_matching_environment( + {"AFD_E2E_RUN_ID": self.run_id, "AFD_E2E_PROCESS_ROLE": "ffn"}, + timeout_s=NPU_ASYNC_PROCESS_KILL_TIMEOUT_S, + poll_interval_s=PROCESS_POLL_INTERVAL_S, + process_name="FFN", + ) + ) + for thread in self.log_threads: + thread.join(timeout=LOG_THREAD_JOIN_TIMEOUT_S) + if thread.is_alive(): + failures.append("service log reader did not stop") + if failures: + raise RuntimeError("; ".join(failures)) diff --git a/tests/e2e/environment.py b/tests/e2e/environment.py index 359c0f577..9d270d99b 100644 --- a/tests/e2e/environment.py +++ b/tests/e2e/environment.py @@ -1,10 +1,89 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project -"""Environment parsing shared by local NPU E2E entrypoints.""" +"""Resource and environment parsing shared by E2E entrypoints.""" +import argparse import os +import sys from pathlib import Path +from tests.e2e.case import ( + DEFAULT_API_PORT, + DEFAULT_CONNECTOR_PORT, + DEFAULT_STARTUP_TIMEOUT_S, + CaseEnvironment, +) + + +def add_case_environment_arguments( + parser: argparse.ArgumentParser, + case_ids: tuple[str, ...], + *, + devices_required: bool = True, + startup_option: str = "--startup-timeout", +) -> None: + parser.add_argument("--scenario", choices=case_ids, required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--vllm-bin", default="vllm") + parser.add_argument("--device-backend", choices=("gpu", "npu"), required=True) + parser.add_argument("--devices", required=devices_required, default="") + parser.add_argument("--output-path", type=Path, required=True) + parser.add_argument("--api-host", default="127.0.0.1") + parser.add_argument( + "--api-port-base", + type=int, + default=int(os.environ.get("AFD_E2E_API_PORT_BASE", str(DEFAULT_API_PORT))), + ) + parser.add_argument("--afd-host", default="127.0.0.1") + parser.add_argument( + "--afd-port", + type=int, + default=int(os.environ.get("AFD_E2E_AFD_PORT", str(DEFAULT_CONNECTOR_PORT))), + ) + parser.add_argument( + startup_option, + dest="startup_timeout", + type=float, + default=DEFAULT_STARTUP_TIMEOUT_S, + ) + parser.add_argument( + "--describe", + action="store_true", + help="Print configuration and commands without starting services", + ) + + +def parse_case_environment( + case_ids: tuple[str, ...], argv: list[str] | None = None +) -> tuple[str, CaseEnvironment, bool]: + """Read resources in the subprocess; collection never calls this function.""" + parser = argparse.ArgumentParser(description="Run a model-declared E2E Case") + add_case_environment_arguments(parser, case_ids) + args = parser.parse_args(argv) + return ( + args.scenario, + case_environment(args), + args.describe, + ) + + +def case_environment(args: argparse.Namespace) -> CaseEnvironment: + """Bind parsed resources; model and placement decisions live elsewhere.""" + return CaseEnvironment( + model=str(Path(args.model).expanduser()), + vllm_bin=args.vllm_bin, + device_backend=args.device_backend, + devices=tuple( + value.strip() for value in args.devices.split(",") if value.strip() + ), + output_path=args.output_path, + api_host=args.api_host, + api_port_base=args.api_port_base, + afd_host=args.afd_host, + afd_port=args.afd_port, + startup_timeout=args.startup_timeout, + ) + def required_env(name: str) -> str: value = os.environ.get(name) @@ -13,6 +92,35 @@ def required_env(name: str) -> str: return value +def model_case_command( + module: str, + scenario: str, + output_path: Path, + *, + backend: str, + default_devices: tuple[str, ...] = ("0", "1", "2", "3"), +) -> list[str]: + """Bind a model entry to resources without selecting its topology or checks.""" + prefix = f"AFD_{backend.upper()}_E2E" + return [ + sys.executable, + "-m", + module, + "--scenario", + scenario, + "--model", + required_env(f"{prefix}_MODEL"), + "--vllm-bin", + os.environ.get(f"{prefix}_VLLM_BIN", "vllm"), + "--device-backend", + backend, + "--devices", + os.environ.get("AFD_E2E_DEVICES", ",".join(default_devices)), + "--output-path", + str(output_path), + ] + + def devices_from_env(name: str, expected_count: int) -> list[str]: devices = [item.strip() for item in required_env(name).split(",") if item.strip()] if len(devices) != expected_count: @@ -27,3 +135,39 @@ def prepend_env_paths(env: dict[str, str], name: str, *paths: Path) -> None: env[name] = os.pathsep.join( dict.fromkeys([*(str(path) for path in paths), *existing]), ) + + +def model_multi_pod_command(module: str, scenario: str, layout: str) -> list[str]: + """Bind an existing GPU pod to a model-owned case; provisioning stays outside.""" + command = [ + sys.executable, + "-m", + module, + "multi-pod", + "--scenario", + scenario, + "--pod-layout", + layout, + "--run-id", + required_env("AFD_E2E_RUN_ID"), + "--model", + required_env("AFD_GPU_E2E_MODEL"), + "--vllm-bin", + os.environ.get("AFD_GPU_E2E_VLLM_BIN", "vllm"), + "--device-backend", + "gpu", + "--output-path", + required_env("AFD_E2E_GSM8K_OUTPUT"), + "--store-host", + required_env("AFD_E2E_STORE_HOST"), + ] + for variable, option in ( + ("AFD_E2E_STORE_PORT", "--store-port"), + ("AFD_E2E_DEVICES", "--devices"), + ): + if value := os.environ.get(variable): + command.extend([option, value]) + for value in os.environ.get("AFD_E2E_POD_ENV", "").split(";"): + if value.strip(): + command.extend(["--pod-env", value.strip()]) + return command diff --git a/tests/e2e/models/deepseek_v2_lite/cases.py b/tests/e2e/models/deepseek_v2_lite/cases.py new file mode 100644 index 000000000..ca1f97e7e --- /dev/null +++ b/tests/e2e/models/deepseek_v2_lite/cases.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""DeepSeek-V2-Lite IDs and model-local Case construction. + +Collection imports IDs only. build_case executes in the runner subprocess. +GPU and NPU declarations share capabilities, not scenario selection in the runner. +""" + +from __future__ import annotations + +import sys +from dataclasses import replace + +from tests.e2e.case import ( + ATTENTION_ROLE, + FFN_ROLE, + Case, + CaseEnvironment, + DboConfig, + Gsm8kSettings, + Gsm8kTask, + Mrv2Check, + Phase, +) +from tests.e2e.presets import DECODE_GRAPH, EAGER + +V2_SCENARIOS = ( + "afd-v2-eager-1a1f", + "afd-v2-eager-dp2", + "afd-v2-eager-tp2", + "afd-v2-graph-1a1f", + "afd-v2-graph-dp2", + "afd-v2-graph-tp2", + "afd-v2-eager-dbo-dp2", + "afd-v2-graph-dbo-dp2", + "afd-v2-graph-dbo-tp2", +) +DEEPEP_M2N_SCENARIOS = ( + "afd-deepep-m2n-eager-2a2f", + "afd-deepep-m2n-graph-2a2f", + "afd-deepep-m2n-graph-dbo-2a2f", + "afd-deepep-m2n-eager-tp2-2a2f", + "afd-deepep-m2n-graph-tp2-2a2f", +) +GPU_IDS = ( + "baseline-graph", + *V2_SCENARIOS, + "afd-eager-2a1f", + "afd-graph-2a1f", + "afd-graph-dbo-2a1f", +) +NPU_IDS = ( + "baseline-graph", + "afd-eager-2a2f", + "afd-graph-2a2f", + "afd-graph-dbo-2a2f", + "afd-eager-2a1f", + "afd-graph-2a1f", + "afd-graph-dbo-2a1f", +) +V2_DBO_COMPARISON_SCENARIOS = frozenset( + ("afd-v2-eager-dp2", "afd-v2-eager-dbo-dp2", "afd-v2-graph-dbo-dp2"), +) +V2_SINGLE_RANK_SCENARIOS = frozenset(("afd-v2-eager-1a1f", "afd-v2-graph-1a1f")) +MULTI_POD_IDS = ("afd-graph-2a2f", "afd-graph-dbo-2a2f") +DEFAULT_GSM8K_SAMPLE_LIMIT = 7 +V2_DBO_SAMPLE_LIMIT = 128 +SMOKE_THRESHOLD = 0.25 +ACCURACY_THRESHOLD = 0.27 +SMOKE_MAX_SAMPLES = 24 +COMPARISON_CONCURRENCY = 12 +CAPTURE_SIZE = 8 +MAX_MODEL_LEN = 4096 +DBO_DECODE_THRESHOLD = 2 +DBO_PREFILL_THRESHOLD = 8 + + +def stable_ids(backend: str) -> tuple[str, ...]: + if backend == "gpu": + return GPU_IDS + if backend == "npu": + return NPU_IDS + raise ValueError(f"unknown backend {backend!r}") + + +def build_case(case_id: str, environment: CaseEnvironment) -> Case: + backend = environment.device_backend + allowed = stable_ids(backend) + (DEEPEP_M2N_SCENARIOS if backend == "gpu" else ()) + if case_id not in allowed: + raise ValueError(f"unsupported {backend} DeepSeek-V2-Lite case {case_id!r}") + return _build_case(case_id, environment) + + +def build_multi_pod_case(case_id: str, environment: CaseEnvironment) -> Case: + if environment.device_backend != "gpu" or case_id not in MULTI_POD_IDS: + raise ValueError(f"unsupported GPU multi-pod case {case_id!r}") + case = _build_case(case_id, environment) + dbo = case.deployment.dbo is not None + # These existing distributed cases gate accuracy, liveness and cleanup. + # Their original 0.27 threshold is independent of the single-host smoke rule. + return replace( + case, + phases=( + Phase( + "evaluation", + Gsm8kTask( + Gsm8kSettings( + sample_limit=SMOKE_MAX_SAMPLES + if dbo + else DEFAULT_GSM8K_SAMPLE_LIMIT, + threshold=ACCURACY_THRESHOLD, + num_concurrent=COMPARISON_CONCURRENCY if dbo else None, + minimum_samples=SMOKE_MAX_SAMPLES if dbo else 1, + ) + ), + ), + ), + ) + + +def _build_case(case_id: str, environment: CaseEnvironment) -> Case: + # The table is model-local. No shared module selects a model or case ID. + # baseline, graph, dbo, Attention DP/TP, FFN DP/TP + profiles = { + "baseline-graph": (True, True, False, 4, 1, 0, 1), + "afd-v2-eager-1a1f": (False, False, False, 1, 1, 1, 1), + "afd-v2-eager-dp2": (False, False, False, 2, 1, 2, 1), + "afd-v2-eager-tp2": (False, False, False, 1, 2, 1, 2), + "afd-v2-graph-1a1f": (False, True, False, 1, 1, 1, 1), + "afd-v2-graph-dp2": (False, True, False, 2, 1, 2, 1), + "afd-v2-graph-tp2": (False, True, False, 1, 2, 1, 2), + "afd-v2-eager-dbo-dp2": (False, False, True, 2, 1, 2, 1), + "afd-v2-graph-dbo-dp2": (False, True, True, 2, 1, 2, 1), + "afd-v2-graph-dbo-tp2": (False, True, True, 1, 2, 1, 2), + "afd-eager-2a1f": (False, False, False, 2, 1, 1, 1), + "afd-graph-2a1f": (False, True, False, 2, 1, 1, 1), + "afd-graph-dbo-2a1f": (False, True, True, 2, 1, 1, 1), + "afd-eager-2a2f": (False, False, False, 2, 1, 2, 1), + "afd-graph-2a2f": (False, True, False, 2, 1, 2, 1), + "afd-graph-dbo-2a2f": (False, True, True, 2, 1, 2, 1), + "afd-deepep-m2n-eager-2a2f": (False, False, False, 2, 1, 2, 1), + "afd-deepep-m2n-graph-2a2f": (False, True, False, 2, 1, 2, 1), + "afd-deepep-m2n-graph-dbo-2a2f": (False, True, True, 2, 1, 2, 1), + "afd-deepep-m2n-eager-tp2-2a2f": (False, False, False, 1, 2, 2, 1), + "afd-deepep-m2n-graph-tp2-2a2f": (False, True, False, 1, 2, 2, 1), + } + backend = environment.device_backend + baseline, graph, dbo, a_dp, a_tp, f_dp, f_tp = profiles[case_id] + execution = DECODE_GRAPH if graph else EAGER + comparison = case_id in V2_DBO_COMPARISON_SCENARIOS + deepep = case_id in DEEPEP_M2N_SCENARIOS + minimum_samples = SMOKE_MAX_SAMPLES if comparison or dbo else 1 + sample_limit = ( + V2_DBO_SAMPLE_LIMIT + if comparison + else max(DEFAULT_GSM8K_SAMPLE_LIMIT, minimum_samples) + ) + # The TP2 DBO case has always retained 0.27 even for a 24-sample run. + smoke = case_id != "afd-v2-graph-dbo-tp2" + deployment = execution.deployment( + backend=backend, + connector=( + None + if baseline + else "DeepEPM2NAFDConnector" + if deepep + else "P2pNcclAFDConnector" + if backend == "gpu" + else "CAMP2pAFDConnector" + ), + attention=(a_dp, a_tp), + ffn=None if baseline else (f_dp, f_tp), + capture_size=CAPTURE_SIZE, + compute_gate_on_attention=deepep, + dbo=DboConfig(DBO_DECODE_THRESHOLD, DBO_PREFILL_THRESHOLD) if dbo else None, + ) + if comparison: + deployment = replace(deployment, max_num_seqs=CAPTURE_SIZE) + return Case( + id=case_id, + model="DeepSeek-V2-Lite", + served_model_name_prefix="deepseek-v2-lite-afd", + deployment=deployment, + common_vllm_args=(f"--max-model-len={MAX_MODEL_LEN}",), + phases=( + Phase( + name="evaluation", + task=Gsm8kTask( + Gsm8kSettings( + sample_limit=sample_limit, + threshold=ACCURACY_THRESHOLD + if comparison or not smoke + else SMOKE_THRESHOLD, + num_concurrent=COMPARISON_CONCURRENCY + if comparison or dbo + else None, + minimum_samples=minimum_samples, + smoke_max_samples=SMOKE_MAX_SAMPLES if smoke else None, + smoke_threshold=SMOKE_THRESHOLD if smoke else None, + larger_sample_threshold=ACCURACY_THRESHOLD if smoke else None, + ), + ), + checks=( + (Mrv2Check((ATTENTION_ROLE, FFN_ROLE), graph=graph, dbo=dbo),) + if not baseline and (graph or dbo) + else () + ), + ), + ), + ) + + +if __name__ == "__main__": + if sys.argv[1:2] == ["multi-pod"]: + from tests.e2e.multi_pod.runner import case_main as multi_pod_case_main + + raise SystemExit( + multi_pod_case_main(build_multi_pod_case, MULTI_POD_IDS, sys.argv[2:]) + ) + # The shared entry receives the builder, and never imports this model. + from tests.e2e.runner import case_main + + raise SystemExit( + case_main( + build_case, + tuple(dict.fromkeys((*GPU_IDS, *NPU_IDS, *DEEPEP_M2N_SCENARIOS))), + ) + ) diff --git a/tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py b/tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py index fa46b4118..f171dcd47 100644 --- a/tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py +++ b/tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py @@ -5,7 +5,6 @@ from __future__ import annotations import os -import sys from collections.abc import Iterator from pathlib import Path @@ -15,46 +14,22 @@ download_dataset, download_model, preserve_environment_variable, - run_runner, + run_case_entry, ) -from tests.e2e.runner import ( +from tests.e2e.environment import model_case_command +from tests.e2e.models.deepseek_v2_lite.cases import ( DEEPEP_M2N_SCENARIOS, - DEFAULT_GSM8K_SAMPLE_LIMIT, - GSM8K_LIMIT_ENV, - GSM8K_THRESHOLD_ENV, - V2_DBO_COMPARISON_SCENARIOS, - V2_DBO_SAMPLE_LIMIT, - V2_SCENARIOS, V2_SINGLE_RANK_SCENARIOS, + stable_ids, ) GSM8K_DATASET_ID = "openai/gsm8k" GSM8K_DATASET_CONFIG = "main" DEEPSEEK_V2_LITE_REPO_ID = "deepseek-ai/DeepSeek-V2-Lite" -DEEPSEEK_V2_LITE_MAX_MODEL_LEN = 4096 -SMOKE_MIN_ACCURACY = 0.25 -SMOKE_MAX_SAMPLES = 24 -DEFAULT_DEVICE_IDS = ("0", "1", "2", "3") -ATTENTION_DEVICE_COUNT = 2 -AFD_FFN_DEVICE_COUNT = 1 -AFD_TWO_FFN_DEVICE_COUNT = 2 -BASELINE_DEVICE_COUNT = 4 -TWO_FFN_SCENARIO_SUFFIX = "-2a2f" # CI selects exact node IDs (E2E-INV-007). GPU uses the V2 cases for the # shared 2A2F coverage; NPU retains its existing scenario IDs and now runs # every case on native Ascend MRV2 through the shared harness. -SCENARIOS = ( - "baseline-graph", - *( - ("afd-eager-2a2f", "afd-graph-2a2f", "afd-graph-dbo-2a2f") - if os.environ.get("AFD_E2E_BACKEND") == "npu" - else V2_SCENARIOS - ), - # GPU weekly CI selects Graph + DBO; eager/graph remain local-only. - "afd-eager-2a1f", - "afd-graph-2a1f", - "afd-graph-dbo-2a1f", -) +SCENARIOS = stable_ids(os.environ.get("AFD_E2E_BACKEND", "gpu")) def _required_env(name: str) -> str: @@ -89,65 +64,15 @@ def prepare_e2e_assets() -> None: def build_runner_command(scenario: str, gsm8k_output_path: Path) -> list[str]: - backend = _required_env("AFD_E2E_BACKEND") - if backend == "gpu": - model = _required_env("AFD_GPU_E2E_MODEL") - vllm_bin = os.environ.get("AFD_GPU_E2E_VLLM_BIN", "vllm") - elif backend == "npu": - model = _required_env("AFD_NPU_E2E_MODEL") - vllm_bin = os.environ.get("AFD_NPU_E2E_VLLM_BIN", "vllm") - else: - raise RuntimeError("AFD_E2E_BACKEND must be 'gpu' or 'npu'") - - env_devices = os.environ.get("AFD_E2E_DEVICES") - if env_devices: - devices = [item.strip() for item in env_devices.split(",") if item.strip()] - elif scenario in V2_SINGLE_RANK_SCENARIOS: - devices = ["0", "1"] - else: - devices = list(DEFAULT_DEVICE_IDS) - if scenario == "baseline-graph": - attention_devices = devices[:BASELINE_DEVICE_COUNT] - ffn_devices = [] - elif scenario in V2_SINGLE_RANK_SCENARIOS: - attention_devices = devices[:1] - ffn_devices = devices[1:2] - else: - attention_devices = devices[:ATTENTION_DEVICE_COUNT] - ffn_device_count = ( - AFD_TWO_FFN_DEVICE_COUNT - if scenario.endswith(TWO_FFN_SCENARIO_SUFFIX) or scenario in V2_SCENARIOS - else AFD_FFN_DEVICE_COUNT - ) - ffn_devices = devices[ - ATTENTION_DEVICE_COUNT : ATTENTION_DEVICE_COUNT + ffn_device_count - ] - - command = [ - sys.executable, - "-m", - "tests.e2e.runner", - "--model", - model, - "--vllm-bin", - vllm_bin, - "--device-backend", - backend, - f"--common-vllm-arg=--max-model-len={DEEPSEEK_V2_LITE_MAX_MODEL_LEN}", - "--attention-devices", - ",".join(attention_devices), - ] - if scenario != "baseline-graph": - command.extend(["--ffn-devices", ",".join(ffn_devices)]) - command.extend( - [ - "--scenario", - scenario, - "--gsm8k-output-path", - str(gsm8k_output_path), - ], + return model_case_command( + "tests.e2e.models.deepseek_v2_lite.cases", + scenario, + gsm8k_output_path, + backend=_required_env("AFD_E2E_BACKEND"), + default_devices=("0", "1") + if scenario in V2_SINGLE_RANK_SCENARIOS + else ("0", "1", "2", "3"), ) - return command @pytest.fixture(scope="module", autouse=True) @@ -166,29 +91,11 @@ def _prepare_e2e_assets() -> Iterator[None]: yield -def _runner_env(scenario: str) -> dict[str, str]: - env = os.environ.copy() - default_limit = ( - V2_DBO_SAMPLE_LIMIT - if scenario in V2_DBO_COMPARISON_SCENARIOS - else DEFAULT_GSM8K_SAMPLE_LIMIT - ) - sample_limit = env.get(GSM8K_LIMIT_ENV, str(default_limit)) - if ( - scenario != "afd-v2-graph-dbo-tp2" - and sample_limit != "all" - and int(sample_limit) <= SMOKE_MAX_SAMPLES - ): - # Model smoke gates require 2/7 or 6/24 correct; larger runs keep 0.27. - env.setdefault(GSM8K_THRESHOLD_ENV, str(SMOKE_MIN_ACCURACY)) - return env - - @pytest.mark.e2e @pytest.mark.parametrize("scenario", SCENARIOS, ids=SCENARIOS) def test_deepseek_v2_lite(scenario: str, tmp_path: Path) -> None: command = build_runner_command(scenario, tmp_path / scenario) - run_runner(command, env=_runner_env(scenario)) + run_case_entry(command, tmp_path / scenario) @pytest.mark.e2e @@ -198,4 +105,4 @@ def test_deepseek_v2_lite(scenario: str, tmp_path: Path) -> None: @pytest.mark.parametrize("scenario", DEEPEP_M2N_SCENARIOS, ids=DEEPEP_M2N_SCENARIOS) def test_deepseek_v2_lite_deepep_m2n(scenario: str, tmp_path: Path) -> None: command = build_runner_command(scenario, tmp_path / scenario) - run_runner(command, env=_runner_env(scenario)) + run_case_entry(command, tmp_path / scenario) diff --git a/tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite_multi_pod.py b/tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite_multi_pod.py index bc3d9bf87..194148ee5 100644 --- a/tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite_multi_pod.py +++ b/tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite_multi_pod.py @@ -14,11 +14,13 @@ from __future__ import annotations import os -import sys +from pathlib import Path import pytest -from tests.conftest import run_runner +from tests.conftest import run_case_entry +from tests.e2e.environment import model_multi_pod_command +from tests.e2e.models.deepseek_v2_lite.cases import MULTI_POD_IDS # The layout is a second axis, orthogonal to the scenario id, so accuracy # evidence stays comparable with the single-host rows. @@ -26,13 +28,9 @@ "2pod-role-split": "2A0F,0A2F", "2pod-interleaved": "1A1F,1A1F", } -MULTI_POD_CASES = [ - ("afd-graph-2a2f", "2pod-role-split"), - ("afd-graph-2a2f", "2pod-interleaved"), - ("afd-graph-dbo-2a2f", "2pod-role-split"), - ("afd-graph-dbo-2a2f", "2pod-interleaved"), -] -DEEPSEEK_V2_LITE_MAX_MODEL_LEN = 4096 +MULTI_POD_CASES = tuple( + (scenario, layout) for scenario in MULTI_POD_IDS for layout in POD_LAYOUTS +) def _required_env(name: str) -> str: @@ -58,32 +56,9 @@ def build_runner_command(scenario: str, layout_name: str) -> list[str]: invocation should still get its own value, or a leftover process from a prior run on this same pod could be mistaken for part of this run. """ - layout = POD_LAYOUTS[layout_name] - command = [ - sys.executable, - "-m", - "tests.e2e.multi_pod.runner", - "--scenario", - scenario, - "--pod-layout", - layout, - "--run-id", - _required_env("AFD_E2E_RUN_ID"), - "--model", - _required_env("AFD_GPU_E2E_MODEL"), - "--gsm8k-output-path", - _required_env("AFD_E2E_GSM8K_OUTPUT"), - "--store-host", - _required_env("AFD_E2E_STORE_HOST"), - f"--common-vllm-arg=--max-model-len={DEEPSEEK_V2_LITE_MAX_MODEL_LEN}", - ] - store_port = os.environ.get("AFD_E2E_STORE_PORT") - if store_port: - command.extend(["--store-port", store_port]) - for value in os.environ.get("AFD_E2E_POD_ENV", "").split(";"): - if value.strip(): - command.extend(["--pod-env", value.strip()]) - return command + return model_multi_pod_command( + "tests.e2e.models.deepseek_v2_lite.cases", scenario, POD_LAYOUTS[layout_name] + ) @pytest.mark.e2e @@ -93,4 +68,7 @@ def build_runner_command(scenario: str, layout_name: str) -> list[str]: ids=[f"{scenario}-{layout}" for scenario, layout in MULTI_POD_CASES], ) def test_multi_pod(scenario: str, layout_name: str) -> None: - run_runner(build_runner_command(scenario, layout_name)) + run_case_entry( + build_runner_command(scenario, layout_name), + Path(_required_env("AFD_E2E_GSM8K_OUTPUT")), + ) diff --git a/tests/e2e/models/deepseek_v4_flash/cases.py b/tests/e2e/models/deepseek_v4_flash/cases.py new file mode 100644 index 000000000..868928052 --- /dev/null +++ b/tests/e2e/models/deepseek_v4_flash/cases.py @@ -0,0 +1,269 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""DSV4 single-deployment declarations, separate from dedicated/multi-pod runs. + +Importing IDs is safe. Checkpoint quantization is resolved only by build_case +in the runner child. Existing launch-profile helpers remain model-owned and +are also used by the unmigrated dedicated and multi-pod executors. +""" + +from __future__ import annotations + +import argparse +import json + +from tests.e2e.case import ( + CLEANUP_PROFILE_NPU_ASYNC_CAM_LEGACY, + CLEANUP_PROFILE_STRICT, + DEFAULT_SHUTDOWN_TIMEOUT_S, + EXECUTION_EAGER, + EXECUTION_FULL_DECODE_GRAPH, + NPU_ASYNC_TERMINATION_TIMEOUT_S, + NPU_ASYNC_VLLM_SHUTDOWN_TIMEOUT_S, + NPU_ASYNC_WORKER_SHUTDOWN_TIMEOUT_S, + Case, + CaseEnvironment, + DboConfig, + Deployment, + Gsm8kSettings, + Gsm8kTask, + HashInputCheck, + Mrv2Check, + Phase, + Role, +) +from tests.e2e.models.deepseek_v4_flash import config, gpu_config +from tests.e2e.models.deepseek_v4_flash.completions import CompletionTask + +GPU_BASIC_IDS = gpu_config.DSV4_GPU_BASIC_SCENARIOS +GPU_DBO_IDS = gpu_config.DSV4_GPU_DBO_SCENARIOS +NPU_ASYNC_IDS = (config.DSV4_ASYNC_CAM_SCENARIO,) +NPU_SYNC_IDS = config.DSV4_SYNC_CAMP2P_SCENARIOS +GSM8K_SAMPLES = 7 +GSM8K_THRESHOLD = 0.27 +DBO_CONCURRENCY = 12 +DBO_MINIMUM_SAMPLES = 2 * DBO_CONCURRENCY +DBO_DECODE_THRESHOLD = 2 +DBO_PREFILL_THRESHOLD = 8 + + +def _budget(arguments: list[str], name: str) -> int | None: + """Transfer a model profile's budget to its deployment declaration.""" + if name not in arguments: + return None + index = arguments.index(name) + value = int(arguments[index + 1]) + del arguments[index : index + 2] + return value + + +def _profile_arguments( + case_id: str, environment: CaseEnvironment +) -> argparse.Namespace: + args = argparse.Namespace( + scenario=case_id, + model=environment.model, + device_backend=environment.device_backend, + afd_host=environment.afd_host, + common_vllm_arg=[], + attention_vllm_arg=[], + ffn_vllm_arg=[], + completion_output_path=environment.output_path / "completions.json", + gsm8k_output_path=environment.output_path / "gsm8k", + use_decode_bench_connector=False, + ) + if environment.device_backend == "gpu": + gpu_config.configure_scenario(args) + elif case_id in NPU_ASYNC_IDS: + config.configure_scenario(args) + else: + config.configure_sync_camp2p_scenario(args) + return args + + +def build_case(case_id: str, environment: CaseEnvironment) -> Case: + if environment.device_backend == "gpu": + return _gpu_case(case_id, environment) + if environment.device_backend == "npu" and case_id in ( + *NPU_ASYNC_IDS, + *NPU_SYNC_IDS, + ): + return _npu_case(case_id, environment) + raise ValueError(f"unsupported DSV4 {environment.device_backend} case {case_id!r}") + + +def _gpu_case(case_id: str, environment: CaseEnvironment) -> Case: + baseline, graph, dbo, attention_ranks, ffn_ranks = ( + gpu_config.DSV4_GPU_SCENARIO_SETTINGS[case_id] + ) + args = _profile_arguments(case_id, environment) + max_num_seqs = _budget(args.common_vllm_arg, "--max-num-seqs") + max_num_batched_tokens = _budget(args.common_vllm_arg, "--max-num-batched-tokens") + execution = EXECUTION_FULL_DECODE_GRAPH if graph else EXECUTION_EAGER + roles = ("baseline",) if baseline else ("attention", "ffn") + checks = ( + (Mrv2Check(roles, graph=graph, dbo=dbo),) + if not baseline or graph or dbo + else () + ) + phases = [] + for index, name in enumerate(gpu_config.DSV4_GPU_HASH_WINDOWS): + hash_checks = ( + () + if baseline + else ( + HashInputCheck( + graph=graph, + dbo=dbo, + compare_with=gpu_config.DSV4_GPU_HASH_WINDOWS[index - 1] + if index + else None, + ), + ) + ) + phases.append( + Phase( + name, + CompletionTask( + first_request_index=index * config.DSV4_CONCURRENT_REQUESTS, + answer_repetitions=gpu_config.DSV4_GPU_ANSWER_REPETITIONS, + ), + (*checks, *hash_checks), + ) + ) + phases.append( + Phase( + "gsm8k", + Gsm8kTask( + Gsm8kSettings( + sample_limit=DBO_MINIMUM_SAMPLES if dbo else GSM8K_SAMPLES, + threshold=GSM8K_THRESHOLD, + num_concurrent=DBO_CONCURRENCY if dbo else None, + minimum_samples=DBO_MINIMUM_SAMPLES if dbo else 1, + ) + ), + checks, + ) + ) + return Case( + id=case_id, + model="DeepSeek-V4-Flash", + served_model_name_prefix="dsv4-flash-gpu", + deployment=Deployment( + backend="gpu", + attention=Role(attention_ranks, execution=execution), + ffn=None if baseline else Role(ffn_ranks, execution=execution), + connector=None if baseline else "P2pNcclAFDConnector", + dbo=DboConfig(DBO_DECODE_THRESHOLD, DBO_PREFILL_THRESHOLD) if dbo else None, + cudagraph_capture_size=gpu_config.DSV4_GPU_CAPTURE_SIZE if graph else None, + max_num_seqs=max_num_seqs, + max_num_batched_tokens=max_num_batched_tokens, + ), + common_vllm_args=tuple(args.common_vllm_arg), + phases=tuple(phases), + ) + + +def _npu_case(case_id: str, environment: CaseEnvironment) -> Case: + asynchronous = case_id in NPU_ASYNC_IDS + shape = None if asynchronous else config.DSV4_SYNC_SHAPES[case_id] + args = _profile_arguments(case_id, environment) + max_num_seqs = _budget(args.common_vllm_arg, "--max-num-seqs") + max_num_batched_tokens = _budget(args.common_vllm_arg, "--max-num-batched-tokens") + graph = shape is not None and config.sync_use_graph(shape) + execution = EXECUTION_FULL_DECODE_GRAPH if graph else EXECUTION_EAGER + attention_ranks = ( + config.DSV4_ATTENTION_RANKS if shape is None else shape.attention_ranks + ) + ffn_ranks = config.DSV4_FFN_RANKS if shape is None else shape.ffn_ranks + attention_tp = ( + config.DSV4_ATTENTION_TP_SIZE if shape is None else shape.attention_tp_size + ) + ffn_tp = 1 if shape is None else shape.ffn_tp_size + if asynchronous: + # The existing async path enables native DBO only on Attention, using + # upstream defaults; it does not claim a DBO execution acceptance check. + args.attention_vllm_arg.remove("--enable-dbo") + compilation = None if shape is None else config.sync_compilation_config(shape) + return Case( + id=case_id, + model="DeepSeek-V4-Flash", + served_model_name_prefix="dsv4-flash" if asynchronous else "dsv4-flash-sync", + deployment=Deployment( + backend="npu", + attention=Role( + attention_ranks // attention_tp, + attention_tp, + execution, + dbo=DboConfig() if asynchronous else None, + vllm_args=tuple(args.attention_vllm_arg), + additional_config=tuple( + config.additional_config( + "attention", async_cam=asynchronous + ).items() + ), + ), + ffn=Role( + ffn_ranks // ffn_tp, + ffn_tp, + execution, + vllm_args=tuple(args.ffn_vllm_arg), + additional_config=tuple( + config.additional_config("ffn", async_cam=asynchronous).items() + ), + ), + connector=args.afd_connector, + compute_gate_on_attention=args.compute_gate_on_attention, + connector_extra_config=tuple( + ( + json.loads(args.afd_connector_extra_config[0]) + if args.afd_connector_extra_config + else {} + ).items() + ), + pin_attention_address=args.attention_data_parallel_address is not None, + cleanup_profile=CLEANUP_PROFILE_NPU_ASYNC_CAM_LEGACY + if asynchronous + else CLEANUP_PROFILE_STRICT, + shutdown_timeout_s=NPU_ASYNC_VLLM_SHUTDOWN_TIMEOUT_S + if asynchronous + else DEFAULT_SHUTDOWN_TIMEOUT_S, + worker_shutdown_timeout_s=NPU_ASYNC_WORKER_SHUTDOWN_TIMEOUT_S + if asynchronous + else None, + termination_timeout_s=NPU_ASYNC_TERMINATION_TIMEOUT_S + if asynchronous + else config.DSV4_PROCESS_TERMINATION_TIMEOUT_S, + cudagraph_capture_size=( + shape.cudagraph_capture_size if shape is not None and graph else None + ), + compilation_config=tuple( + (key, tuple(value) if isinstance(value, list) else value) + for key, value in compilation.items() + ) + if compilation is not None + else (), + max_num_seqs=max_num_seqs, + max_num_batched_tokens=max_num_batched_tokens, + ), + common_vllm_args=tuple(args.common_vllm_arg), + phases=( + Phase( + "evaluation", + CompletionTask( + check_answer=True if shape is None else shape.check_answer + ), + (Mrv2Check(("attention", "ffn"), graph=True),) if graph else (), + ), + ), + ) + + +if __name__ == "__main__": + from tests.e2e.runner import case_main + + raise SystemExit( + case_main( + build_case, (*GPU_BASIC_IDS, *GPU_DBO_IDS, *NPU_ASYNC_IDS, *NPU_SYNC_IDS) + ) + ) diff --git a/tests/e2e/models/deepseek_v4_flash/completions.py b/tests/e2e/models/deepseek_v4_flash/completions.py index f3302688e..e6c790047 100644 --- a/tests/e2e/models/deepseek_v4_flash/completions.py +++ b/tests/e2e/models/deepseek_v4_flash/completions.py @@ -7,10 +7,11 @@ import asyncio import json import time +from collections.abc import Mapping +from dataclasses import dataclass from pathlib import Path -import httpx - +from tests.e2e.case import JsonValue from tests.e2e.models.deepseek_v4_flash.config import ( DSV4_COMPLETION_MAX_TOKENS, DSV4_CONCURRENT_REQUESTS, @@ -28,6 +29,11 @@ def evaluate_completions( check_answer: bool = True, answer_repetitions: int = 1, first_request_index: int = 0, + num_requests: int = DSV4_CONCURRENT_REQUESTS, + max_tokens: int = DSV4_COMPLETION_MAX_TOKENS, + timeout_s: float = DSV4_REQUEST_TIMEOUT_S, + first_operand: int = DSV4_PROMPT_FIRST_OPERAND, + second_operand: int = DSV4_PROMPT_SECOND_OPERAND, ) -> None: """Run one batch of ten concurrent requests and check every response. @@ -43,12 +49,16 @@ def evaluate_completions( exercises live decode after the answer's first token, rather than only checking a one-token answer computed by the eager prefill. """ + # Case declarations are imported during collection; load the client only + # when the runner executes this task. + import httpx + results: list[dict] = [ { "index": index, "prompt": ( - f"Compute {DSV4_PROMPT_FIRST_OPERAND + index} + " - f"{DSV4_PROMPT_SECOND_OPERAND}. " + f"Compute {first_operand + index} + " + f"{second_operand}. " + ( "Reply with just the number." if answer_repetitions == 1 @@ -58,9 +68,7 @@ def evaluate_completions( ), "error": "Request cancelled before completion", } - for index in range( - first_request_index, first_request_index + DSV4_CONCURRENT_REQUESTS - ) + for index in range(first_request_index, first_request_index + num_requests) ] async def request(client: httpx.AsyncClient, item: dict) -> None: @@ -72,7 +80,7 @@ async def request(client: httpx.AsyncClient, item: dict) -> None: "model": model, "messages": [{"role": "user", "content": item["prompt"]}], "temperature": 0.0, - "max_tokens": DSV4_COMPLETION_MAX_TOKENS, + "max_tokens": max_tokens, "chat_template_kwargs": {"thinking": False}, }, ) @@ -82,9 +90,7 @@ async def request(client: httpx.AsyncClient, item: dict) -> None: response.raise_for_status() item["response"] = response.json() validate_response(item["response"], check_answer=check_answer) - expected = ( - DSV4_PROMPT_FIRST_OPERAND + item["index"] + DSV4_PROMPT_SECOND_OPERAND - ) + expected = first_operand + item["index"] + second_operand expected_content = " ".join([str(expected)] * answer_repetitions) content = item["response"]["choices"][0]["message"]["content"].strip() if check_answer and content != expected_content: @@ -101,7 +107,7 @@ async def run_requests() -> None: # All requests are scheduled before yielding. Async socket operations # cancel immediately, so SIGTERM can reach the runner's service cleanup # without waiting for a thread blocked in a 300-second HTTP read. - async with httpx.AsyncClient(timeout=DSV4_REQUEST_TIMEOUT_S) as client: + async with httpx.AsyncClient(timeout=timeout_s) as client: await asyncio.gather(*(request(client, item) for item in results)) try: @@ -122,11 +128,48 @@ async def run_requests() -> None: if max(item["started_at"] for item in results) >= min( item["finished_at"] for item in results ): - raise RuntimeError("The ten completion requests did not overlap") + raise RuntimeError("The completion requests did not overlap") for item in results: content = item["response"]["choices"][0]["message"]["content"] print(f"Request {item['index'] + 1}: {item['prompt']} -> {content}") - print(f"Concurrent completions: {len(results)}/{DSV4_CONCURRENT_REQUESTS} passed") + print(f"Concurrent completions: {len(results)}/{num_requests} passed") + + +@dataclass(frozen=True) +class CompletionTask: + """The model's concurrent answer/plumbing contract, with recorded workload.""" + + check_answer: bool = True + answer_repetitions: int = 1 + first_request_index: int = 0 + num_requests: int = DSV4_CONCURRENT_REQUESTS + max_tokens: int = DSV4_COMPLETION_MAX_TOKENS + timeout_s: float = DSV4_REQUEST_TIMEOUT_S + first_operand: int = DSV4_PROMPT_FIRST_OPERAND + second_operand: int = DSV4_PROMPT_SECOND_OPERAND + + def resolve( + self, environment: Mapping[str, str] + ) -> tuple[CompletionTask, tuple[str, ...]]: + return self, () + + def execute( + self, *, base_url: str, model_name: str, checkpoint: str, output_path: Path + ) -> dict[str, JsonValue]: + evaluate_completions( + url=f"{base_url}/v1/chat/completions", + model=model_name, + output_path=output_path / "completions.json", + check_answer=self.check_answer, + answer_repetitions=self.answer_repetitions, + first_request_index=self.first_request_index, + num_requests=self.num_requests, + max_tokens=self.max_tokens, + timeout_s=self.timeout_s, + first_operand=self.first_operand, + second_operand=self.second_operand, + ) + return {"request_count": self.num_requests, "answer_checked": self.check_answer} def validate_response(result: dict, *, check_answer: bool = True) -> None: diff --git a/tests/e2e/models/deepseek_v4_flash/test_async_cam_npu.py b/tests/e2e/models/deepseek_v4_flash/test_async_cam_npu.py index 50d02550d..a40def0ce 100644 --- a/tests/e2e/models/deepseek_v4_flash/test_async_cam_npu.py +++ b/tests/e2e/models/deepseek_v4_flash/test_async_cam_npu.py @@ -5,13 +5,12 @@ from __future__ import annotations import os -import sys from pathlib import Path import pytest -from tests.conftest import run_runner -from tests.e2e.environment import devices_from_env, required_env +from tests.conftest import run_case_entry +from tests.e2e.environment import devices_from_env, model_case_command, required_env from tests.e2e.models.deepseek_v4_flash.config import ( DSV4_ASYNC_CAM_SCENARIO, DSV4_ATTENTION_RANKS, @@ -22,36 +21,26 @@ def build_runner_command(output_path: Path) -> list[str]: if required_env("AFD_E2E_BACKEND") != "npu": raise RuntimeError("DSV4 async CAM E2E requires AFD_E2E_BACKEND=npu") - devices = devices_from_env("AFD_E2E_DEVICES", DSV4_ATTENTION_RANKS + DSV4_FFN_RANKS) - return [ - sys.executable, - "-m", - "tests.e2e.runner", - "--model", - required_env("AFD_NPU_E2E_MODEL"), - "--vllm-bin", - os.environ.get("AFD_NPU_E2E_VLLM_BIN", "vllm"), - "--device-backend", - "npu", - "--attention-devices", - ",".join(devices[:DSV4_ATTENTION_RANKS]), - "--ffn-devices", - ",".join(devices[DSV4_ATTENTION_RANKS:]), - "--scenario", + devices_from_env("AFD_E2E_DEVICES", DSV4_ATTENTION_RANKS + DSV4_FFN_RANKS) + command = model_case_command( + "tests.e2e.models.deepseek_v4_flash.cases", DSV4_ASYNC_CAM_SCENARIO, - "--served-model-name-prefix", - "dsv4-flash", - "--afd-host", - required_env("HCCL_IF_IP"), - "--api-port-base", - os.environ.get("AFD_NPU_DSV4_E2E_API_PORT", "19280"), - "--afd-port", - os.environ.get("AFD_NPU_DSV4_E2E_AFD_PORT", "6455"), - "--startup-timeout", - os.environ.get("AFD_NPU_E2E_STARTUP_TIMEOUT", "1800"), - "--completion-output-path", - str(output_path), - ] + output_path, + backend="npu", + ) + command.extend( + [ + "--afd-host", + required_env("HCCL_IF_IP"), + "--api-port-base", + os.environ.get("AFD_NPU_DSV4_E2E_API_PORT", "19280"), + "--afd-port", + os.environ.get("AFD_NPU_DSV4_E2E_AFD_PORT", "6455"), + "--startup-timeout", + os.environ.get("AFD_NPU_E2E_STARTUP_TIMEOUT", "1800"), + ] + ) + return command def build_environment() -> dict[str, str]: @@ -88,7 +77,9 @@ def build_environment() -> dict[str, str]: @pytest.mark.slow @pytest.mark.parametrize("scenario", [DSV4_ASYNC_CAM_SCENARIO]) def test_deepseek_v4_flash_async_cam(scenario: str, tmp_path: Path) -> None: - run_runner( - build_runner_command(tmp_path / f"{scenario}.json"), + output_path = tmp_path / scenario + run_case_entry( + build_runner_command(output_path), + output_path, env=build_environment(), ) diff --git a/tests/e2e/models/deepseek_v4_flash/test_gpu.py b/tests/e2e/models/deepseek_v4_flash/test_gpu.py index aadaa60cd..c58b5bed6 100644 --- a/tests/e2e/models/deepseek_v4_flash/test_gpu.py +++ b/tests/e2e/models/deepseek_v4_flash/test_gpu.py @@ -5,19 +5,20 @@ from __future__ import annotations import os -import sys from pathlib import Path import pytest -from tests.conftest import download_dataset, run_runner -from tests.e2e.environment import devices_from_env, required_env +from tests.conftest import download_dataset, run_case_entry +from tests.e2e.environment import devices_from_env, model_case_command, required_env +from tests.e2e.models.deepseek_v4_flash.cases import ( + GPU_BASIC_IDS as DSV4_GPU_BASIC_SCENARIOS, +) +from tests.e2e.models.deepseek_v4_flash.cases import ( + GPU_DBO_IDS as DSV4_GPU_DBO_SCENARIOS, +) from tests.e2e.models.deepseek_v4_flash.gpu_config import ( - DSV4_GPU_BASIC_SCENARIOS, - DSV4_GPU_DBO_SCENARIOS, DSV4_GPU_DEVICE_COUNT, - DSV4_GPU_ROLE_RANKS, - DSV4_GPU_SCENARIO_SETTINGS, ) GSM8K_DATASET_ID = "openai/gsm8k" @@ -36,37 +37,19 @@ def prepare_e2e_assets() -> None: def build_runner_command(scenario: str, output_dir: Path) -> list[str]: - devices = devices_from_env("AFD_E2E_DEVICES", DSV4_GPU_DEVICE_COUNT) - baseline = DSV4_GPU_SCENARIO_SETTINGS[scenario][0] - command = [ - sys.executable, - "-m", - "tests.e2e.runner", - "--model", - required_env("AFD_GPU_E2E_MODEL"), - "--vllm-bin", - os.environ.get("AFD_GPU_E2E_VLLM_BIN", "vllm"), - "--device-backend", - "gpu", - "--attention-devices", - ",".join(devices[:DSV4_GPU_ROLE_RANKS]), - "--scenario", - scenario, - "--served-model-name-prefix", - "dsv4-flash-gpu", - "--completion-output-path", - str(output_dir / "completions.json"), - "--gsm8k-output-path", - str(output_dir / "gsm8k"), - "--startup-timeout", - os.environ.get("AFD_GPU_E2E_STARTUP_TIMEOUT", "1800"), - "--afd-port", - os.environ.get("AFD_GPU_DSV4_E2E_AFD_PORT", "6457"), - "--api-port-base", - os.environ.get("AFD_GPU_DSV4_E2E_API_PORT", "19480"), - ] - if not baseline: - command.extend(["--ffn-devices", ",".join(devices[DSV4_GPU_ROLE_RANKS:])]) + command = model_case_command( + "tests.e2e.models.deepseek_v4_flash.cases", scenario, output_dir, backend="gpu" + ) + command.extend( + [ + "--startup-timeout", + os.environ.get("AFD_GPU_E2E_STARTUP_TIMEOUT", "1800"), + "--afd-port", + os.environ.get("AFD_GPU_DSV4_E2E_AFD_PORT", "6457"), + "--api-port-base", + os.environ.get("AFD_GPU_DSV4_E2E_API_PORT", "19480"), + ] + ) return command @@ -74,7 +57,7 @@ def build_runner_command(scenario: str, output_dir: Path) -> list[str]: @pytest.mark.slow @pytest.mark.parametrize("scenario", DSV4_GPU_BASIC_SCENARIOS) def test_deepseek_v4_flash_gpu(scenario: str, tmp_path: Path) -> None: - run_runner(build_runner_command(scenario, tmp_path)) + run_case_entry(build_runner_command(scenario, tmp_path), tmp_path) @pytest.mark.e2e @@ -82,4 +65,4 @@ def test_deepseek_v4_flash_gpu(scenario: str, tmp_path: Path) -> None: @pytest.mark.parametrize("scenario", DSV4_GPU_DBO_SCENARIOS) def test_deepseek_v4_flash_gpu_dbo(scenario: str, tmp_path: Path) -> None: """Qualify DBO separately with the shared nonempty-stage execution oracle.""" - run_runner(build_runner_command(scenario, tmp_path)) + run_case_entry(build_runner_command(scenario, tmp_path), tmp_path) diff --git a/tests/e2e/models/deepseek_v4_flash/test_sync_camp2p_npu.py b/tests/e2e/models/deepseek_v4_flash/test_sync_camp2p_npu.py index 7b529df0e..0b643f4c1 100644 --- a/tests/e2e/models/deepseek_v4_flash/test_sync_camp2p_npu.py +++ b/tests/e2e/models/deepseek_v4_flash/test_sync_camp2p_npu.py @@ -14,13 +14,12 @@ from __future__ import annotations import os -import sys from pathlib import Path import pytest -from tests.conftest import run_runner -from tests.e2e.environment import devices_from_env, required_env +from tests.conftest import run_case_entry +from tests.e2e.environment import devices_from_env, model_case_command, required_env from tests.e2e.models.deepseek_v4_flash.config import ( DSV4_SYNC_CAMP2P_SCENARIOS, DSV4_SYNC_HCCL_BUFFER_SIZE_MB, @@ -46,36 +45,23 @@ def build_runner_command(scenario: str, output_path: Path) -> list[str]: shape = DSV4_SYNC_SHAPES[scenario] if required_env("AFD_E2E_BACKEND") != "npu": raise RuntimeError("DSV4 sync CAMP2P E2E requires AFD_E2E_BACKEND=npu") - devices = devices_from_env("AFD_E2E_DEVICES", shape.device_count) - return [ - sys.executable, - "-m", - "tests.e2e.runner", - "--model", - required_env("AFD_NPU_E2E_MODEL"), - "--vllm-bin", - os.environ.get("AFD_NPU_E2E_VLLM_BIN", "vllm"), - "--device-backend", - "npu", - "--attention-devices", - ",".join(devices[: shape.attention_ranks]), - "--ffn-devices", - ",".join(devices[shape.attention_ranks :]), - "--scenario", - scenario, - "--served-model-name-prefix", - "dsv4-flash-sync", - "--afd-host", - _afd_host(shape), - "--api-port-base", - os.environ.get("AFD_NPU_DSV4_SYNC_E2E_API_PORT", "19380"), - "--afd-port", - os.environ.get("AFD_NPU_DSV4_SYNC_E2E_AFD_PORT", "6456"), - "--startup-timeout", - os.environ.get("AFD_NPU_E2E_STARTUP_TIMEOUT", "1800"), - "--completion-output-path", - str(output_path), - ] + devices_from_env("AFD_E2E_DEVICES", shape.device_count) + command = model_case_command( + "tests.e2e.models.deepseek_v4_flash.cases", scenario, output_path, backend="npu" + ) + command.extend( + [ + "--afd-host", + _afd_host(shape), + "--api-port-base", + os.environ.get("AFD_NPU_DSV4_SYNC_E2E_API_PORT", "19380"), + "--afd-port", + os.environ.get("AFD_NPU_DSV4_SYNC_E2E_AFD_PORT", "6456"), + "--startup-timeout", + os.environ.get("AFD_NPU_E2E_STARTUP_TIMEOUT", "1800"), + ] + ) + return command def build_environment(scenario: str) -> dict[str, str]: @@ -139,7 +125,9 @@ def build_environment(scenario: str) -> dict[str, str]: @pytest.mark.slow @pytest.mark.parametrize("scenario", DSV4_SYNC_CAMP2P_SCENARIOS) def test_deepseek_v4_flash_sync_camp2p(scenario: str, tmp_path: Path) -> None: - run_runner( - build_runner_command(scenario, tmp_path / f"{scenario}.json"), + output_path = tmp_path / scenario + run_case_entry( + build_runner_command(scenario, output_path), + output_path, env=build_environment(scenario), ) diff --git a/tests/e2e/models/qwen3_6/cases.py b/tests/e2e/models/qwen3_6/cases.py new file mode 100644 index 000000000..1a73a9dc3 --- /dev/null +++ b/tests/e2e/models/qwen3_6/cases.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Model-local CUDA cases; importing IDs never resolves model assets.""" + +from __future__ import annotations + +from tests.e2e.case import ( + Case, + CaseEnvironment, + Gsm8kSettings, + Gsm8kTask, + Mrv2Check, + Phase, +) +from tests.e2e.presets import DECODE_GRAPH, EAGER + +SCENARIOS = ( + "baseline-graph", + "afd-eager-2a1f", + "afd-graph-2a1f", + # Native hybrid GDN does not support DBO. +) +GSM8K_SAMPLES = 7 +GSM8K_THRESHOLD = 0.27 +CAPTURE_SIZE = 8 +MAX_MODEL_LEN = 4096 + + +def build_case(case_id: str, environment: CaseEnvironment) -> Case: + if environment.device_backend != "gpu": + raise ValueError("Qwen3.6 MoE CUDA E2E requires the gpu backend") + profiles = { + "baseline-graph": (True, True, 4, 0), + "afd-eager-2a1f": (False, False, 2, 1), + "afd-graph-2a1f": (False, True, 2, 1), + } + baseline, graph, attention_ranks, ffn_ranks = profiles[case_id] + execution = DECODE_GRAPH if graph else EAGER + return Case( + id=case_id, + model="Qwen3.6 MoE", + served_model_name_prefix="qwen3-6-afd", + deployment=execution.deployment( + backend="gpu", + attention=(attention_ranks, 1), + ffn=None if baseline else (ffn_ranks, 1), + connector=None if baseline else "P2pNcclAFDConnector", + capture_size=CAPTURE_SIZE, + ), + common_vllm_args=( + f"--max-model-len={MAX_MODEL_LEN}", + "--language-model-only", + ), + phases=( + Phase( + "evaluation", + Gsm8kTask( + Gsm8kSettings( + sample_limit=GSM8K_SAMPLES, + threshold=GSM8K_THRESHOLD, + ) + ), + (Mrv2Check(("attention", "ffn"), graph=True),) + if not baseline and graph + else (), + ), + ), + ) + + +if __name__ == "__main__": + from tests.e2e.runner import case_main + + raise SystemExit(case_main(build_case, SCENARIOS)) diff --git a/tests/e2e/models/qwen3_6/test_qwen3_6.py b/tests/e2e/models/qwen3_6/test_qwen3_6.py index 6921e5f7f..85bc13ff4 100644 --- a/tests/e2e/models/qwen3_6/test_qwen3_6.py +++ b/tests/e2e/models/qwen3_6/test_qwen3_6.py @@ -5,7 +5,6 @@ from __future__ import annotations import os -import sys from collections.abc import Iterator from pathlib import Path @@ -15,23 +14,14 @@ download_dataset, download_model, preserve_environment_variable, - run_runner, + run_case_entry, ) +from tests.e2e.environment import model_case_command +from tests.e2e.models.qwen3_6.cases import SCENARIOS GSM8K_DATASET_ID = "openai/gsm8k" GSM8K_DATASET_CONFIG = "main" QWEN3_6_REPO_ID = "Qwen/Qwen3.6-35B-A3B" -QWEN3_6_MAX_MODEL_LEN = 4096 -DEFAULT_DEVICE_IDS = ("0", "1", "2", "3") -ATTENTION_DEVICE_COUNT = 2 -AFD_FFN_DEVICE_COUNT = 1 -BASELINE_DEVICE_COUNT = 4 -# Native MRV2 does not support DBO for Qwen3.6 hybrid GDN models. -SCENARIOS = ( - "baseline-graph", - "afd-eager-2a1f", - "afd-graph-2a1f", -) def _required_env(name: str) -> str: @@ -64,49 +54,9 @@ def build_runner_command(scenario: str, gsm8k_output_path: Path) -> list[str]: if backend != "gpu": raise RuntimeError("Qwen3.6 MoE E2E supports only the 'gpu' backend") - env_devices = os.environ.get("AFD_E2E_DEVICES") - devices = ( - [item.strip() for item in env_devices.split(",") if item.strip()] - if env_devices - else list(DEFAULT_DEVICE_IDS) - ) - if scenario == "baseline-graph": - attention_devices = devices[:BASELINE_DEVICE_COUNT] - ffn_devices = [] - else: - attention_devices = devices[:ATTENTION_DEVICE_COUNT] - ffn_devices = devices[ - ATTENTION_DEVICE_COUNT : ATTENTION_DEVICE_COUNT + AFD_FFN_DEVICE_COUNT - ] - - command = [ - sys.executable, - "-m", - "tests.e2e.runner", - "--model", - _required_env("AFD_GPU_E2E_MODEL"), - "--vllm-bin", - os.environ.get("AFD_GPU_E2E_VLLM_BIN", "vllm"), - "--device-backend", - backend, - f"--common-vllm-arg=--max-model-len={QWEN3_6_MAX_MODEL_LEN}", - "--common-vllm-arg=--language-model-only", - "--attention-devices", - ",".join(attention_devices), - ] - if scenario != "baseline-graph": - command.extend(["--ffn-devices", ",".join(ffn_devices)]) - command.extend( - [ - "--scenario", - scenario, - "--gsm8k-output-path", - str(gsm8k_output_path), - "--served-model-name-prefix", - "qwen3-6-afd", - ], + return model_case_command( + "tests.e2e.models.qwen3_6.cases", scenario, gsm8k_output_path, backend=backend ) - return command @pytest.fixture(scope="module", autouse=True) @@ -121,4 +71,4 @@ def _prepare_e2e_assets() -> Iterator[None]: @pytest.mark.parametrize("scenario", SCENARIOS, ids=SCENARIOS) def test_qwen3_6(scenario: str, tmp_path: Path) -> None: command = build_runner_command(scenario, tmp_path / scenario) - run_runner(command) + run_case_entry(command, tmp_path / scenario) diff --git a/tests/e2e/models/qwen3_moe/cases.py b/tests/e2e/models/qwen3_moe/cases.py new file mode 100644 index 000000000..c247a6be1 --- /dev/null +++ b/tests/e2e/models/qwen3_moe/cases.py @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Model-local CUDA cases; importing IDs never resolves model assets.""" + +from __future__ import annotations + +import sys +from dataclasses import replace + +from tests.e2e.case import ( + Case, + CaseEnvironment, + DboConfig, + Gsm8kSettings, + Gsm8kTask, + Mrv2Check, + Phase, +) +from tests.e2e.presets import DECODE_GRAPH, EAGER + +SCENARIOS = ( + "baseline-graph", + "afd-eager-2a1f", + "afd-graph-2a1f", + "afd-graph-dbo-2a1f", +) +DEEPEP_M2N_SCENARIOS = ( + "afd-deepep-m2n-eager-2a2f", + "afd-deepep-m2n-graph-2a2f", + "afd-deepep-m2n-graph-dbo-2a2f", + "afd-deepep-m2n-eager-tp2-2a2f", + "afd-deepep-m2n-graph-tp2-2a2f", +) +MULTI_POD_IDS = ( + "afd-deepep-m2n-eager-2a2f", + "afd-deepep-m2n-graph-2a2f", + "afd-deepep-m2n-graph-dbo-2a2f", +) +MULTI_POD_GSM8K_SAMPLES = 100 + +GSM8K_SAMPLES = 7 +GSM8K_THRESHOLD = 0.27 +DBO_CONCURRENCY = 12 +DBO_MINIMUM_SAMPLES = 2 * DBO_CONCURRENCY +DBO_DECODE_THRESHOLD = 2 +DBO_PREFILL_THRESHOLD = 8 +CAPTURE_SIZE = 8 +MAX_MODEL_LEN = 4096 + + +def build_multi_pod_case(case_id: str, environment: CaseEnvironment) -> Case: + if case_id not in MULTI_POD_IDS: + raise ValueError(f"unsupported GPU multi-pod case {case_id!r}") + case = build_case(case_id, environment) + dbo = case.deployment.dbo is not None + return replace( + case, + # Preserve the served name used by the existing distributed entry. + served_model_name_prefix="deepseek-v2-lite-afd", + phases=( + Phase( + "evaluation", + Gsm8kTask( + Gsm8kSettings( + sample_limit=MULTI_POD_GSM8K_SAMPLES, + threshold=GSM8K_THRESHOLD, + num_concurrent=DBO_CONCURRENCY if dbo else None, + minimum_samples=DBO_MINIMUM_SAMPLES if dbo else 1, + ) + ), + ), + ), + ) + + +def build_case(case_id: str, environment: CaseEnvironment) -> Case: + if environment.device_backend != "gpu": + raise ValueError("Qwen3 MoE CUDA E2E requires the gpu backend") + profiles = { + "baseline-graph": (True, True, False, 4, 1, 0, 1), + "afd-eager-2a1f": (False, False, False, 2, 1, 1, 1), + "afd-graph-2a1f": (False, True, False, 2, 1, 1, 1), + "afd-graph-dbo-2a1f": (False, True, True, 2, 1, 1, 1), + "afd-deepep-m2n-eager-2a2f": (False, False, False, 2, 1, 2, 1), + "afd-deepep-m2n-graph-2a2f": (False, True, False, 2, 1, 2, 1), + "afd-deepep-m2n-graph-dbo-2a2f": (False, True, True, 2, 1, 2, 1), + "afd-deepep-m2n-eager-tp2-2a2f": (False, False, False, 1, 2, 2, 1), + "afd-deepep-m2n-graph-tp2-2a2f": (False, True, False, 1, 2, 2, 1), + } + baseline, graph, dbo, a_dp, a_tp, f_dp, f_tp = profiles[case_id] + execution = DECODE_GRAPH if graph else EAGER + deepep = case_id in DEEPEP_M2N_SCENARIOS + return Case( + id=case_id, + model="Qwen3 MoE", + served_model_name_prefix="qwen3-moe-afd", + deployment=execution.deployment( + backend="gpu", + attention=(a_dp, a_tp), + ffn=None if baseline else (f_dp, f_tp), + connector=None + if baseline + else "DeepEPM2NAFDConnector" + if deepep + else "P2pNcclAFDConnector", + compute_gate_on_attention=deepep, + dbo=DboConfig(DBO_DECODE_THRESHOLD, DBO_PREFILL_THRESHOLD) if dbo else None, + capture_size=CAPTURE_SIZE, + ), + common_vllm_args=(f"--max-model-len={MAX_MODEL_LEN}",), + phases=( + Phase( + "evaluation", + Gsm8kTask( + Gsm8kSettings( + sample_limit=DBO_MINIMUM_SAMPLES if dbo else GSM8K_SAMPLES, + threshold=GSM8K_THRESHOLD, + num_concurrent=DBO_CONCURRENCY if dbo else None, + minimum_samples=DBO_MINIMUM_SAMPLES if dbo else 1, + ) + ), + (Mrv2Check(("attention", "ffn"), graph=graph, dbo=dbo),) + if not baseline and (graph or dbo) + else (), + ), + ), + ) + + +if __name__ == "__main__": + if sys.argv[1:2] == ["multi-pod"]: + from tests.e2e.multi_pod.runner import case_main as multi_pod_case_main + + raise SystemExit( + multi_pod_case_main(build_multi_pod_case, MULTI_POD_IDS, sys.argv[2:]) + ) + from tests.e2e.runner import case_main + + raise SystemExit(case_main(build_case, (*SCENARIOS, *DEEPEP_M2N_SCENARIOS))) diff --git a/tests/e2e/models/qwen3_moe/test_qwen3_moe.py b/tests/e2e/models/qwen3_moe/test_qwen3_moe.py index ec02ccab9..b3416fb9f 100644 --- a/tests/e2e/models/qwen3_moe/test_qwen3_moe.py +++ b/tests/e2e/models/qwen3_moe/test_qwen3_moe.py @@ -5,7 +5,6 @@ from __future__ import annotations import os -import sys from collections.abc import Iterator from pathlib import Path @@ -15,29 +14,14 @@ download_dataset, download_model, preserve_environment_variable, - run_runner, + run_case_entry, ) -from tests.e2e.runner import DEEPEP_M2N_SCENARIOS +from tests.e2e.environment import model_case_command +from tests.e2e.models.qwen3_moe.cases import DEEPEP_M2N_SCENARIOS, SCENARIOS GSM8K_DATASET_ID = "openai/gsm8k" GSM8K_DATASET_CONFIG = "main" QWEN3_MOE_REPO_ID = "Qwen/Qwen3-30B-A3B" -QWEN3_MOE_MAX_MODEL_LEN = 4096 -DEFAULT_DEVICE_IDS = ("0", "1", "2", "3") -ATTENTION_DEVICE_COUNT = 2 -AFD_FFN_DEVICE_COUNT = 1 -DEEPEP_M2N_FFN_DEVICE_COUNT = 2 -BASELINE_DEVICE_COUNT = 4 -SCENARIOS = ( - "baseline-graph", - "afd-eager-2a1f", - "afd-graph-2a1f", - "afd-graph-dbo-2a1f", -) -# DeepEP M2N (routing on Attention) needs EP over at least two FFN ranks, so -# its scenarios (DEEPEP_M2N_SCENARIOS) run 2A2F. CI selects test ids -# explicitly and does not run them; running the whole module (pytest -m e2e -# tests/e2e/models/qwen3_moe) does, and they need DeepEP and 4 GPUs. def _required_env(name: str) -> str: @@ -70,53 +54,9 @@ def build_runner_command(scenario: str, gsm8k_output_path: Path) -> list[str]: if backend != "gpu": raise RuntimeError("Qwen3 MoE E2E supports only the 'gpu' backend") - env_devices = os.environ.get("AFD_E2E_DEVICES") - devices = ( - [item.strip() for item in env_devices.split(",") if item.strip()] - if env_devices - else list(DEFAULT_DEVICE_IDS) - ) - if scenario == "baseline-graph": - attention_devices = devices[:BASELINE_DEVICE_COUNT] - ffn_devices = [] - else: - ffn_device_count = ( - DEEPEP_M2N_FFN_DEVICE_COUNT - if scenario in DEEPEP_M2N_SCENARIOS - else AFD_FFN_DEVICE_COUNT - ) - attention_devices = devices[:ATTENTION_DEVICE_COUNT] - ffn_devices = devices[ - ATTENTION_DEVICE_COUNT : ATTENTION_DEVICE_COUNT + ffn_device_count - ] - - command = [ - sys.executable, - "-m", - "tests.e2e.runner", - "--model", - _required_env("AFD_GPU_E2E_MODEL"), - "--vllm-bin", - os.environ.get("AFD_GPU_E2E_VLLM_BIN", "vllm"), - "--device-backend", - backend, - f"--common-vllm-arg=--max-model-len={QWEN3_MOE_MAX_MODEL_LEN}", - "--attention-devices", - ",".join(attention_devices), - ] - if scenario != "baseline-graph": - command.extend(["--ffn-devices", ",".join(ffn_devices)]) - command.extend( - [ - "--scenario", - scenario, - "--gsm8k-output-path", - str(gsm8k_output_path), - "--served-model-name-prefix", - "qwen3-moe-afd", - ], + return model_case_command( + "tests.e2e.models.qwen3_moe.cases", scenario, gsm8k_output_path, backend=backend ) - return command @pytest.fixture(scope="module", autouse=True) @@ -131,11 +71,11 @@ def _prepare_e2e_assets() -> Iterator[None]: @pytest.mark.parametrize("scenario", SCENARIOS, ids=SCENARIOS) def test_qwen3_moe(scenario: str, tmp_path: Path) -> None: command = build_runner_command(scenario, tmp_path / scenario) - run_runner(command) + run_case_entry(command, tmp_path / scenario) @pytest.mark.e2e @pytest.mark.parametrize("scenario", DEEPEP_M2N_SCENARIOS, ids=DEEPEP_M2N_SCENARIOS) def test_qwen3_moe_deepep_m2n(scenario: str, tmp_path: Path) -> None: command = build_runner_command(scenario, tmp_path / scenario) - run_runner(command) + run_case_entry(command, tmp_path / scenario) diff --git a/tests/e2e/models/qwen3_moe/test_qwen3_moe_multi_pod.py b/tests/e2e/models/qwen3_moe/test_qwen3_moe_multi_pod.py index 5918c0669..3461dac31 100644 --- a/tests/e2e/models/qwen3_moe/test_qwen3_moe_multi_pod.py +++ b/tests/e2e/models/qwen3_moe/test_qwen3_moe_multi_pod.py @@ -21,8 +21,8 @@ from __future__ import annotations import os -import sys from collections.abc import Iterator +from pathlib import Path import pytest @@ -30,31 +30,23 @@ download_dataset, download_model, preserve_environment_variable, - run_runner, + run_case_entry, ) -from tests.e2e.runner import GSM8K_LIMIT_ENV +from tests.e2e.environment import model_multi_pod_command +from tests.e2e.models.qwen3_moe.cases import MULTI_POD_IDS GSM8K_DATASET_ID = "openai/gsm8k" GSM8K_DATASET_CONFIG = "main" QWEN3_MOE_REPO_ID = "Qwen/Qwen3-30B-A3B" -QWEN3_MOE_MAX_MODEL_LEN = 4096 -# The default sample limit of these cases; AFD_GSM8K_LIMIT overrides it. -GSM8K_SAMPLE_LIMIT = 100 # The layout is a second axis, orthogonal to the scenario id, so accuracy # evidence stays comparable with the single-host rows. POD_LAYOUTS = { "2pod-role-split": "2A0F,0A2F", "2pod-interleaved": "1A1F,1A1F", } -MULTI_POD_CASES = [ - (scenario, layout) - for layout in POD_LAYOUTS - for scenario in ( - "afd-deepep-m2n-eager-2a2f", - "afd-deepep-m2n-graph-2a2f", - "afd-deepep-m2n-graph-dbo-2a2f", - ) -] +MULTI_POD_CASES = tuple( + (scenario, layout) for layout in POD_LAYOUTS for scenario in MULTI_POD_IDS +) def _required_env(name: str) -> str: @@ -85,31 +77,9 @@ def build_runner_command(scenario: str, layout_name: str) -> list[str]: only supplies what it cannot infer: the scenario, the shared rendezvous store host, and where to write results. """ - command = [ - sys.executable, - "-m", - "tests.e2e.multi_pod.runner", - "--scenario", - scenario, - "--pod-layout", - POD_LAYOUTS[layout_name], - "--run-id", - _required_env("AFD_E2E_RUN_ID"), - "--model", - _required_env("AFD_GPU_E2E_MODEL"), - "--gsm8k-output-path", - _required_env("AFD_E2E_GSM8K_OUTPUT"), - "--store-host", - _required_env("AFD_E2E_STORE_HOST"), - f"--common-vllm-arg=--max-model-len={QWEN3_MOE_MAX_MODEL_LEN}", - ] - store_port = os.environ.get("AFD_E2E_STORE_PORT") - if store_port: - command.extend(["--store-port", store_port]) - for value in os.environ.get("AFD_E2E_POD_ENV", "").split(";"): - if value.strip(): - command.extend(["--pod-env", value.strip()]) - return command + return model_multi_pod_command( + "tests.e2e.models.qwen3_moe.cases", scenario, POD_LAYOUTS[layout_name] + ) @pytest.mark.e2e @@ -119,6 +89,7 @@ def build_runner_command(scenario: str, layout_name: str) -> list[str]: ids=[f"{scenario}-{layout}" for scenario, layout in MULTI_POD_CASES], ) def test_multi_pod_deepep_m2n(scenario: str, layout_name: str) -> None: - env = os.environ.copy() - env.setdefault(GSM8K_LIMIT_ENV, str(GSM8K_SAMPLE_LIMIT)) - run_runner(build_runner_command(scenario, layout_name), env=env) + run_case_entry( + build_runner_command(scenario, layout_name), + Path(_required_env("AFD_E2E_GSM8K_OUTPUT")), + ) diff --git a/tests/e2e/multi_pod/README.md b/tests/e2e/multi_pod/README.md index 904ec1bdb..4de425428 100644 --- a/tests/e2e/multi_pod/README.md +++ b/tests/e2e/multi_pod/README.md @@ -1,373 +1,157 @@ -# `tests/e2e/multi_pod/runner.py` walkthrough +# Multi-pod E2E execution -This document explains how `runner.py` executes, using -`tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite_multi_pod.py` as the -worked example. It is a companion to the code, not a replacement for it — -line numbers refer to `runner.py` as of this writing and may drift. +Each existing pod runs one slice of a model-declared GPU Case. The model owns +its topology and acceptance workload; `multi_pod` owns rank placement and peer +coordination; `deployment.py` owns this pod's service processes. Pods or +containers must already exist. See the +[Kubernetes walkthrough](../../../.agents/skills/run-e2e/resources/k8-multi-pod.md) +for provisioning, resources and completion handling. -## The core idea +## Entry and ownership -A "multi-pod" AFD E2E scenario splits Attention and FFN ranks across more -than one pod (or container). Something has to decide, per pod, which ranks -it should launch, how those ranks should be wired to their peers in other -pods, and when it is safe to evaluate. `runner.py` is that "something" — but -deliberately with **no master process**. The same program, with the same -argv, runs once inside *every* participating pod. Each copy: - -1. figures out its own identity from its environment (pod 0? pod 1? …), -2. derives the *entire* global plan locally, with a pure function, from - inputs every pod was given identically, -3. reads out only the slice of that plan that belongs to it, -4. coordinates with its peers through a shared rendezvous store (not through - each other directly), and -5. tears down only the children it personally started. - -This is why the module docstring says "nothing outside the pods holds test -state." A k8s Job (or a Docker container group) can restart, reschedule, or -be inspected independently, and nothing needs to be told what the plan was — -it just needs the same launch inputs again. - -## How the test reaches the runner - -`test_deepseek_v2_lite_multi_pod.py` does **not** call any function in -`runner.py` directly. Instead: - -- `MULTI_POD_CASES` pairs a scenario id (e.g. `afd-graph-2a2f`) with a layout - name (e.g. `2pod-role-split` → pod layout string `"2A0F,0A2F"`). -- `build_runner_command()` builds an argv for `python -m - tests.e2e.multi_pod.runner` — the scenario, the pod layout, a run id, the - model, output paths, and the shared rendezvous store host — pulling most of - it from required environment variables (`AFD_E2E_RUN_ID`, - `AFD_GPU_E2E_MODEL`, `AFD_E2E_GSM8K_OUTPUT`, `AFD_E2E_STORE_HOST`, …). -- `test_multi_pod()` hands that argv to `run_runner()` (`tests/conftest.py`), - which `subprocess.Popen`s it in a new process group, forwards - SIGTERM/SIGINT into that group, and raises if it exits non-zero. - -Critically, the test file's own docstring spells out the deployment -assumption: **the pods must already exist**. A human or external deployment -tool creates N pods/containers and invokes this same pytest node once inside -each of them; see the -[multi-pod deployment instructions](../../../.agents/skills/run-e2e/resources/k8-multi-pod.md). -`pytest tests/e2e/models/deepseek_v2_lite/ -test_deepseek_v2_lite_multi_pod.py::test_multi_pod[...]` running inside pod 0 -and the identical invocation running inside pod 1 together *are* one -multi-pod E2E run. `test_multi_pod()` only supplies the inputs the in-pod -runner cannot infer from its own pod's environment (scenario, layout, store -host); everything pod-specific (index, peer addresses) is resolved by the -runner itself. - -So conceptually: +The ten existing GPU entries are unchanged: four DeepSeek-V2-Lite +scenario/layout pairs and six Qwen3 MoE DeepEP M2N pairs. Their pytest +entrypoints call `model_multi_pod_command()` and `run_case_entry()`: ```text -external provisioning (k8s Job / container launcher) - └─ pod 0: pytest ... test_multi_pod[...] → runner.py --pod-layout 2A0F,0A2F ... - └─ pod 1: pytest ... test_multi_pod[...] → runner.py --pod-layout 2A0F,0A2F ... +models//test_*_multi_pod.py + -> python -m tests.e2e.models..cases multi-pod ... + -> model-owned build_multi_pod_case() + -> multi_pod/runner.case_main() / run_case() + -> layout.py: logical topology -> this pod's RoleSlots + -> rendezvous.py: TCPStore barriers, abort and verdict + -> deployment.py: role commands, local start/readiness/cleanup + -> declared Task: workload and result validation on the evaluator ``` -The two `runner.py` invocations rendezvous with each other over a -`torch.distributed.TCPStore`, not through pytest or the driver. - -## `runner.py` code walkthrough - -### Imports and constants (lines 1–76) - -The runner deliberately imports its "pure" helpers (`identity.py`, -`layout.py`) separately from the rendezvous module, which is the only one -that pulls in `torch.distributed`. The docstring of `identity.py` calls this -out explicitly: identity resolution has to stay unit-testable without a GPU -or a cluster. - -The barrier name constants (`ADDRESS_BARRIER`, `LAUNCHED_BARRIER`, -`SERVING_BARRIER`, `VERDICT_BARRIER`, `TEARDOWN_BARRIER`) name the five -synchronization points every pod passes through, in order — they show up -again in `run_pod()`. -`ROLE_LABELS` is just for readable log prefixes (`ATTN`, `FFN`, `BASELINE`). - -### `PodProcess` (lines 79–85) - -A tiny frozen dataclass pairing a locally-launched `subprocess.Popen` with -its role and its log-line label. `processes: list[PodProcess]` is threaded -through `main()` → `run_pod()` → `launch_slot()` and is what cleanup and the -liveness checks iterate over. - -### `main()` (lines 88–156) — phases 1–5 and the top-level shape - -This is the entry point (`if __name__ == "__main__": sys.exit(main())`, -line 442). Reading it top to bottom: - -1. **Parse args, configure the scenario, parse the layout.** - `configure_scenario(args)` (imported from the single-host - `tests/e2e/runner.py`) is the same function the single-host runner uses — - it turns a scenario id like `afd-graph-2a2f` into concrete settings - (`args.num_attention_ranks`, `args.cuda_graph_full_decode_only`, - `args.afd_connector`, …). Reusing it is what keeps a multi-pod scenario's - *logical* topology (rank counts, connector, DBO, CUDA graph) identical to - its single-host counterpart — only pod placement differs. - `PodLayout.parse(args.pod_layout)` turns `"2A0F,0A2F"` into a tuple of - `PodSpec(attention=2, ffn=0)` / `PodSpec(attention=0, ffn=2)`. - `Topology.from_args(args)` captures the scenario's *logical* shape - (`RoleTopology` per role, connector, `baseline`), independent of how it is - laid out across pods. `validate_layout()` cross-checks the two: the - layout's total per-role rank counts must match the topology's, and no - role's local rank count within a pod may violate that role's TP size - (a TP group cannot span pods). - The runner then sets `args.collect_mrv2_execution_evidence = False`. For - graph and DBO scenarios the single-host runner proves the requested - execution path, not just accuracy: it loads the `tests.e2e.mrv2_evidence` - worker extension, opens the evaluation window through Attention's - `/collective_rpc` and a local marker file the FFN worker polls, and checks - every rank's summary line for graph replays or two-stage DBO steps. No pod - can drive that window for the others, so multi-pod graph and DBO scenarios - (e.g. `afd-graph-2a2f`, `afd-graph-dbo-2a2f`) launch without the extension - and pass on accuracy alone. **TODO:** share the window and the per-rank - summaries through the rendezvous store and check the merged result before - the verdict. - -2. **Resolve this pod's index.** `resolve_pod_index(layout.num_pods)` (from - `identity.py`) checks, in order: an explicit `AFD_E2E_POD_INDEX` - override, the k8s Indexed Job's `JOB_COMPLETION_INDEX`, then falls back to - the trailing integer of `HOSTNAME` (for hand-applied manifests or the - container launchers that name containers `..-0`, `..-1`, …). - -3. **Wait for the rendezvous store's DNS to resolve, then connect.** - `wait_for_address(args.store_host, args.store_dns_timeout)` exists - because a freshly-created pod's DNS record can lag its own start by a few - seconds; without pre-resolving, that shows up as an unexplained - `TCPStore` connection failure rather than a named "still waiting on DNS" - state. `Rendezvous(...)` then opens a `TCPStore` — pod 0 is always the - store's master (`is_master=pod_index == 0`), which is independent of - which pod *leads* a role or evaluates. - -4. **`rendezvous.verify_agreement(layout.canonical(), args.scenario)`.** - Pod 0 publishes the canonical layout string and scenario id; every other - pod reads them back and raises immediately if its own values differ. This - turns a misconfigured deployment (e.g. one pod launched with a different - `--pod-layout`) into a fast, named failure instead of a barrier timeout - thirty minutes later. - -5. **Resolve every pod's address, plan, and print.** - `resolve_addresses()` (see below) gives every pod the same ordered list - of peer addresses. `plan(topology, layout, addresses)[pod_index]` calls - the pure planning function from `layout.py` and keeps only this pod's - `PodPlan`. `print_plan()` logs it for the pod's own console/log stream. - -6. **Stale-process pre-flight.** `find_stale_run_markers(args.run_id)` scans - `/proc` for local processes whose `AFD_E2E_RUN_ID` environment variable - does not match this run's id — a leftover process from a previous run - still holding a GPU/NPU device or a port. If found, the pod publishes an - abort (so peers unwind quickly) and raises, rather than letting the - symptom surface later as an unexplained rendezvous hang. - -7. **Run the pod body under `cancellable_run(cleanup)`.** - `cancellable_run` (from `tests/e2e/cancellation.py`) is shared with the - single-host runner and gives both the same SIGTERM/SIGINT precedence: a - signal during the body unwinds immediately into cleanup; a signal that - arrives *during* cleanup is deferred until cleanup finishes; the - resulting `SystemExit` then wins over a cleanup error, which in turn wins - over a body error. `cleanup()` (defined as a closure over `processes`, - `log_threads`, and `rendezvous`) calls `terminate_processes()` on every - local child, then joins the log-streaming threads, and best-effort - publishes its own outcome to the store under `phase/{pod_index}/cleanup` - (`publish_quietly` — a store failure during teardown must never mask the - run's actual result). If the body (`run_pod`) raises, `main()` also - publishes an abort naming which pod failed, so peers still blocked on a - barrier or on the verdict key unwind within seconds instead of waiting - out their full timeout. - -8. On success, prints a `PASSED` line tagged with this pod's index, scenario, - and layout, and returns `0`. - -9. **Release the store, whatever the outcome.** Everything after connecting - runs inside a `try`/`finally` that calls `leave_rendezvous()`. Pod 0's - process *is* the store server, so it must outlive every peer's last store - call: a barrier only proves that every pod *arrived*, and a peer may still - be asleep in the barrier's poll interval when pod 0 returns from it. So - after its own cleanup each peer calls `Rendezvous.depart()` (its last - store write), and pod 0 calls `wait_for_departures()` before exiting, - bounded by `departure_timeout(args)` — the scenario's teardown budget - plus `DEPARTURE_GRACE_S`. A straggler (a peer that died or never - connected) is logged, never turned into a different result; another - arrival barrier could not replace this, since the last pod to arrive - always leaves while the others still sleep. - -### `run_pod()` (lines 159–259) — phases 6–11, the ordered protocol +For example, invoke the following in each of the two pre-existing pods, with +its own output directory and device visibility: -This is the heart of the coordination logic, and its own comments number the -phases (6 through 11) that continue from `main()`'s 1–5. Every phase either -launches something local or waits on a named rendezvous barrier; **all** -barrier waits pass `on_poll=assert_local_processes_alive`, so a barrier wait -never blocks silently past a local child dying — it notices and raises -within one poll interval. - -- **Phases 6–7 — ordered launch.** Which role starts first depends on the - connector: `uses_async_connector(args)` (true for `CAMAsyncAFDConnector`) - means Attention hosts the AFD rendezvous and must come up first; - otherwise FFN does (mirrors `RENDEZVOUS_ROLE_BY_CONNECTOR` in - `layout.py`). The runner launches this pod's slot for the *first* role - kind (`launch_slot`, if this pod holds that role), then waits on a - `f"{first_kind}-launched"` barrier before any pod starts its *second* - role. This matters because the AFD connector's own rendezvous protocol - needs the answering side present before the initiating side dials it — - getting the launch order wrong here would produce connector-level - timeouts unrelated to this runner's own barriers. - -- **Phase 8 — "launched" barrier.** Once both of this pod's local role - processes (if any) are started, it publishes `phase/{pod_index}/launched` - and waits for every pod. The code comment is explicit that "launched" - only means every pod's *children are alive*, not that they are serving — - a weaker, faster-to-reach checkpoint than readiness. - -- **Phase 9 — readiness via `/health`.** Only the pod holding the - *non-headless* Attention slot is polled for `/health` — and the comment - explains why that's sufficient: the Attention leader's API server binds - only after the AFD connector rendezvous completes, which itself requires - every FFN rank to be present. So one health check transitively proves the - whole distributed AFD world formed. An FFN engine core never builds an API - server at all (it enters a busy loop — see - `afd_plugin/compat/patches/engine_core.py:_run_ffn_busy_loop`), and a - headless DP slot starts no server by construction, so for both of those - the phase-8 liveness check was already the honest assertion. After the - optional health poll, the pod publishes `phase/{pod_index}/serving` and - waits on the `SERVING_BARRIER`. - -- **Phase 10 — exactly one evaluator.** `pod_plan.is_evaluator` is true only - for the pod holding the Attention role's leader rank (set in - `layout.plan()`). That one pod runs the accuracy/completion check against - its **own local** API through `run_scenario_evaluation`, the same - scenario-to-evaluator mapping the single-host `tests/e2e/runner.py` uses - (teardown likewise shares `process_termination_timeout`), publishes the - verdict string to - the store (`VERDICT_KEY`), and re-raises on failure after recording - `"fail: {exc}"` and publishing an abort. Every *other* pod instead calls - `rendezvous.wait_for_key(VERDICT_KEY, ...)` and blocks until that key - appears (or a peer aborts). - -- **Phase 11 — every pod holds the same verdict.** All pods (evaluator - included) pass through one more barrier (`VERDICT_BARRIER`) so that a slow - non-evaluator pod can't be torn down by its driver before it has actually - read the verdict. A final `assert_local_processes_alive()` catches a child - that died in the narrow window after the last health/liveness check, and - the pod raises if its verdict isn't `PASS_VERDICT` — this is what makes - the process's own exit code (and thus the k8s Job / Docker container exit - code, and thus `test_multi_pod()`'s subprocess result) reflect pass/fail - for the whole distributed run, not just this pod's local view. - -- **Phase 12 — teardown barrier.** A passing pod waits on - `TEARDOWN_BARRIER` before returning into cleanup, so no pod stops its - roles until every pod has passed its final liveness check. Stopping a - local role can take down its AFD peers on other pods, which a pod still - checking would misreport as a failed run. This is the only barrier - *without* `on_poll`: from here on a child exiting is expected. - -### `launch_slot()` (lines 262–290) - -Builds and starts exactly one role's process for this pod. It picks -`build_baseline_command` or `build_vllm_command(args, role=slot.role, -slot=slot)` — both imported unchanged from the single-host runner — and -passes the `RoleSlot` through so `build_vllm_command` can add the -multi-pod-only flags (`--data-parallel-size-local`, -`--data-parallel-address`, `--data-parallel-rpc-port`, plus -`--data-parallel-start-rank` and `--headless` on non-leader slots) *only* -when `slot.spans_pods` is true. The leader omits its start rank (always 0): -vLLM 0.30 reads an explicit start rank on a non-headless engine as hybrid -load balancing, which turns external when the local DP size is 1 and then -rejects the headless peers. `slot.spans_pods` is what keeps a one-pod layout -producing the exact same argv, character for character, as the single-host -runner — making a 1-pod layout a genuine control case rather than a -different code path pretending to be one. After starting the process, it is -appended to `processes`, its stdout is piped through `stream_output()` onto -a daemon thread prefixed with a pod/role label, and it is immediately polled -once to fail fast if it exited before even reaching the timeout-based checks -later. - -### `wait_for_health()` (lines 293–325) - -A small polling loop against `/health` (not `/v1/models`) on the Attention -API port, calling the caller-supplied `on_poll` liveness check on every -iteration (including the first, before the first request) so a dead child is -never masked by an HTTP-level retry. Raises `TimeoutError` naming the last -observed error if the deadline passes. - -### `deferred_sigkill_pgids()` (lines 328–335) - -Delegates to `uses_npu_async_process_cleanup(args)` (shared with the -single-host runner) to decide whether this pod's FFN process needs the NPU -async teardown allowance — FFN workers using the async CAM connector on NPU -can sit in uninterruptible HCCL teardown for tens of seconds after SIGTERM, -so their process groups get a deferred, longer-patience SIGKILL rather than -being force-killed on the same schedule as everything else. This mirrors -the equivalent logic in the single-host `tests/e2e/runner.py:main()` -verbatim, just scoped to *this pod's* FFN process instead of the whole run's. - -### `resolve_addresses()` (lines 338–363) - -Three ways to learn every pod's address, tried in order: - -1. **Explicit list** — `--pod-addresses` or the `AFD_E2E_POD_ADDRESSES` - environment variable, comma-separated. Used when the driver already - knows every pod's address up front. -2. **A template** — `--pod-address-template` with an `{index}` field (e.g. a - StatefulSet-style DNS name `afd-e2e-abc-{index}.afd-e2e-abc`). This - *skips* the address-exchange barrier entirely, since the addresses are - derivable without any coordination. -3. **Rendezvous exchange** — the fallback: this pod publishes its own - `local_address()` under `addr/{pod_index}`, waits on `ADDRESS_BARRIER` - for every pod to do the same, then reads all of them back in order. - `local_address()` prefers `HCCL_IF_IP` (an Ascend pod's HCCL interface, - which HCCL and CAM bind to), then `POD_IP`, then the hostname. - -### `publish_quietly()` and `print_plan()` (lines 366–388) - -`publish_quietly` wraps a single `rendezvous.set()` in a `try/except -Exception`, printing rather than raising — used only from `cleanup()`, -where the docstring/comment is direct: teardown's own outcome must never -fail the process, because the exit code the driver observes is what -actually reports the run's result. `print_plan` is pure logging: this pod's -address, its peer list, whether it's the evaluator, and per-slot device/DP -placement — useful for debugging a hung run from pod logs alone. - -### `parse_args()` (lines 391–439) - -Adds the multi-pod-specific CLI surface on top of -`add_scenario_arguments(parser)` (shared with the single-host runner, which -supplies `--model`, `--scenario`, connector/graph/DBO flags, etc. — but -deliberately *not* device selection, since that's derived differently by -each runner). The multi-pod-only arguments are `--pod-layout`, `--run-id`, -`--store-host` / `--store-port` / `--store-dns-timeout`, the three address -sources for `resolve_addresses()`, `--pod-env` (extra `KEY=VALUE`s merged -into every launched process's environment), and the three timeout knobs -(`--launch-timeout`, `--serving-timeout`, `--verdict-timeout`) that bound -each of the barriers above. - -## Where the shared logic actually lives - -A recurring theme above: most of what looks like "the test's logic" — -scenario configuration, vLLM command construction, GSM8K/completion -evaluation, environment building, process start/stream/terminate — is -**not** duplicated in `runner.py`. It is imported unchanged from the -single-host `tests/e2e/runner.py`. `runner.py` (this module) only adds what -is genuinely different about running as one pod among several: - -- identity resolution (`identity.py`), -- deriving a global plan and reading out one pod's slice of it - (`layout.py`), -- and cross-pod coordination through a shared store (`rendezvous.py`). - -This is also why `layout.RoleSlot` is imported directly into -`tests/e2e/runner.py` (`build_vllm_command(..., slot: RoleSlot | None = -None)`) rather than the multi-pod runner reimplementing command -construction: a `slot=None` call from the single-host runner and a -`slot=` call from here are meant to produce identical -output whenever that slot doesn't actually span pods. - -## Provisioning: who creates the pods in the first place +```bash +python -m tests.e2e.models.deepseek_v2_lite.cases multi-pod \ + --scenario afd-graph-2a2f --pod-layout 2A0F,0A2F \ + --run-id example-run --model /models/DeepSeek-V2-Lite \ + --device-backend gpu --output-path /work/results \ + --store-host afd-e2e-0.afd-e2e +``` -`runner.py` assumes N pods/containers already exist and that it is being -invoked once inside each. Provision them manually or through external -deployment tooling, following the -[multi-pod deployment instructions](../../../.agents/skills/run-e2e/resources/k8-multi-pod.md). -This repository does not provide a `tests/e2e/multi_pod/driver/` package. -The provisioner creates the pods/containers, invokes the same pytest node in -each, collects exit codes and logs, and removes the deployment afterward. -Service launch order, readiness, evaluation, and child-process teardown are -owned by the in-pod runner, as walked through above. +The original NPU DSV4 multi-pod entries still use the retained +`python -m tests.e2e.multi_pod.runner` CLI and NPU launch helpers. The shared +coordination sequence is the same. NPU migration is separate work. + +## Identity, placement and resources + +`resolve_pod_index()` uses `AFD_E2E_POD_INDEX`, then `JOB_COMPLETION_INDEX`, +then a trailing index in `HOSTNAME`. Pod 0 owns the TCPStore; the Attention +leader owns evaluation. These need not be the same pod. + +A layout such as `2A0F,0A2F` is independent of the Case's logical topology. +Per-role totals must match the Case, and each pod must hold a whole number of +TP groups: a TP group cannot span pods. Qwen3 DeepEP M2N retains the role-split +and interleaved layouts from main. Its transport requires equal rank counts +per NVLink domain, with FFN ranks filling the first domains or the same FFN +count in every domain. The connector's transport mapping owns these rules; +this migration adds no transport capability. +Baseline placement across multiple pods remains unsupported. + +`plan()` derives every pod's plan from the same topology, layout and addresses. +Each pod uses only its own `RoleSlot`s. A role spanning pods receives local DP +size, rendezvous address and RPC port. Only headless peers receive the DP start +rank; emitting it on the leader selects a different vLLM load-balancing mode +that rejects headless peers. Headless commands do not reserve HTTP ports. + +Slot devices are positions within the pod's allocated device list. `--devices` +or `AFD_E2E_DEVICES` can bind that list explicitly; otherwise the GPU CLI uses +inherited `CUDA_VISIBLE_DEVICES`, or the pod's local ordinal list when unset. +The service launcher partitions that list by role. + +Address resolution uses, in order: + +1. `--pod-addresses` or `AFD_E2E_POD_ADDRESSES`; +2. `--pod-address-template` with an `{index}` field, without an exchange barrier; +3. each pod's published address and an address-exchange barrier. + +The advertised local address prefers `HCCL_IF_IP`, then `POD_IP`, then hostname. +DNS resolution of the store host is bounded before TCPStore connection. +Pod 0 publishes the scenario ID and canonical layout; peers reject disagreement. +GPU execution binds this pod's allocated resources and cleans only its launched +groups. A different run marker alone does not prove stale ownership or block a +run on a shared host. The retained NPU path keeps its existing stale-run check +for its isolated pod environment; it does not clean up another run's resources. + +## Coordination sequence + +1. Read the model Case, validate placement, resolve identity and addresses. +2. Start local slots for the connector's first role; all pods pass its launched + barrier before starting the second role. GPU connector order comes from + `deployment.py`, not scenario-name branches in the coordinator. +3. Pass the launched barrier. This proves local children are alive, not ready. +4. The non-headless Attention leader waits for `/health`; other slots report + liveness. FFN busy loops and headless workers do not expose an HTTP API. + All pods then pass the serving barrier. +5. Exactly one evaluator executes the declared phases against its local API. + It publishes the workload results, effective configuration and verdict. + Peers consume those results rather than run the workload again. +6. Pass the verdict barrier and check local liveness once more. +7. Pass the teardown barrier before stopping any local role. It intentionally + omits liveness polling: a peer may now legitimately start shutting down. +8. Clean up this pod's owned process groups, reap leaders and join log readers. +9. Non-master pods publish departure after their last store access. Pod 0 waits + for departures with a bounded grace period, then releases the store. + +Barrier and verdict waits poll peer aborts and local liveness. A local failure +publishes an abort so peers waiting on coordination can unwind. Cleanup uses +`cancellable_run()`: cancellation is deferred during cleanup and takes priority +on exit, followed by cleanup errors and the original execution error. Cleanup +status publication is best effort and cannot mask those errors. + +Departure is different from arrival: an arrival barrier cannot prove that a +sleeping peer has finished its last store read. Pod 0 therefore outlives peers' +explicit departure messages; stragglers are reported after the bounded wait. +Do not automatically restart a pod into an already-advanced barrier sequence. + +## Acceptance and records + +Migration preserves existing GPU multi-pod acceptance: + +| Suite | Workload | Threshold | Execution probes | +| --- | --- | --- | --- | +| V2 graph | 7 GSM8K samples | 0.27 | None | +| V2 graph + DBO | 24 GSM8K samples, concurrency 12 | 0.27 | None | +| Qwen3 DeepEP M2N | 100 GSM8K samples; DBO concurrency 12 | 0.27 | None | + +These Cases explicitly declare no execution checks. Accuracy, local service +liveness, peer coordination and cleanup are checked; graph/DBO/Hash execution +across pods is not proved. The executor rejects a Case requesting unsupported +cross-pod execution checks instead of silently dropping them. + +Each pod writes role logs and `run.json` under its output directory. The +evaluator writes task artifacts under the phase directory (currently +`evaluation/`). `AFD_E2E_GSM8K_OUTPUT` supplies that root to the pytest entry. +Records identify the local plan/devices, actual evaluator settings and results, +coverage scope, local execution and cleanup. A weakened evaluator configuration +produces a debug verdict on every pod. The CLI may complete that workload, but +`run_case_entry()` rejects it as acceptance. + +The deployment passes only when every pod's pytest entry succeeds with +`acceptance_passed=true`. One pod's record does not prove peer cleanup. The +retained NPU CLI keeps its existing completion and scoped async-FFN cleanup +policy, including its longer process-group wait and marked-process cleanup. + +## Extending and validating + +For the migrated GPU Case path, add existing-capability combinations through +model declarations, IDs and layout selection. Do not add model/scenario branches +to shared coordination, commands, tasks or probes. Preserve the model's distributed acceptance settings explicitly; +single-deployment defaults are not automatically interchangeable. +The retained NPU executor is outside this extension contract until migrated. +GPU Case runs use deployment-owned cleanup; legacy NPU runs keep their existing +cleanup path. The coordinator does not run both for the same invocation. + +`tests/e2e/test_runner.py` exercises two real runner processes and a real +TCPStore, with stub service/evaluator CLIs. These host integrations require +PyTorch for TCPStore, but do not load models or establish device correctness. +They cover role-split/interleaved placement, the Qwen workload, debug verdict +propagation, startup/evaluator failure, cancellation and owned cleanup. +Real-device validation is reported separately, including whether placement was +same-host or cross-node. This migration adds no cross-pod execution evidence. diff --git a/tests/e2e/multi_pod/layout.py b/tests/e2e/multi_pod/layout.py index 0282ec923..0c31159b9 100644 --- a/tests/e2e/multi_pod/layout.py +++ b/tests/e2e/multi_pod/layout.py @@ -14,6 +14,8 @@ from collections.abc import Sequence from dataclasses import dataclass +from tests.e2e.case import Case + ATTENTION_ROLE = "attention" FFN_ROLE = "ffn" BASELINE_ROLE = "baseline" @@ -148,6 +150,19 @@ def rendezvous_role(self) -> str | None: ) return rendezvous_role + @classmethod + def from_case(cls, case: Case) -> Topology: + profile = case.deployment + return cls( + attention=RoleTopology(profile.attention.rank_count, profile.attention.tp), + ffn=RoleTopology( + 0 if profile.ffn is None else profile.ffn.rank_count, + 1 if profile.ffn is None else profile.ffn.tp, + ), + connector=profile.connector or "", + baseline=profile.is_baseline, + ) + @classmethod def from_args(cls, args: argparse.Namespace) -> Topology: """Build a topology from a ``configure_scenario``-populated namespace.""" diff --git a/tests/e2e/multi_pod/rendezvous.py b/tests/e2e/multi_pod/rendezvous.py index 64e34e1e6..2497a53c0 100644 --- a/tests/e2e/multi_pod/rendezvous.py +++ b/tests/e2e/multi_pod/rendezvous.py @@ -13,8 +13,6 @@ from collections.abc import Callable from datetime import timedelta -from torch.distributed import TCPStore - DEFAULT_STORE_PORT = 29500 STORE_CONNECT_TIMEOUT_S = 600 BARRIER_POLL_INTERVAL_S = 2.0 @@ -27,6 +25,7 @@ VERDICT_KEY = "verdict" ABORT_COUNT_KEY = "count/abort" PASS_VERDICT = "pass" +DEBUG_VERDICT = "debug" class PeerFailureError(RuntimeError): @@ -54,6 +53,9 @@ def __init__( connect_timeout_s: float = STORE_CONNECT_TIMEOUT_S, poll_interval_s: float = BARRIER_POLL_INTERVAL_S, ) -> None: + # Runtime-only dependency: case collection and CLI inspection need no torch. + from torch.distributed import TCPStore + if not 0 <= pod_index < num_pods: raise ValueError( f"pod index {pod_index} is outside 0..{num_pods - 1}", diff --git a/tests/e2e/multi_pod/runner.py b/tests/e2e/multi_pod/runner.py index aa3c7420f..6a8792a33 100644 --- a/tests/e2e/multi_pod/runner.py +++ b/tests/e2e/multi_pod/runner.py @@ -12,6 +12,7 @@ from __future__ import annotations import argparse +import json import os import subprocess import sys @@ -20,9 +21,12 @@ import urllib.error import urllib.request from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import asdict, dataclass, replace +from tests.e2e import deployment from tests.e2e.cancellation import cancellable_run +from tests.e2e.case import Case, CaseEnvironment, JsonValue +from tests.e2e.environment import add_case_environment_arguments, case_environment from tests.e2e.multi_pod.identity import ( POD_ADDRESSES_ENV, find_stale_run_markers, @@ -42,6 +46,7 @@ validate_layout, ) from tests.e2e.multi_pod.rendezvous import ( + DEBUG_VERDICT, DEFAULT_STORE_PORT, PASS_VERDICT, VERDICT_KEY, @@ -57,6 +62,7 @@ build_env, build_vllm_command, configure_scenario, + prepare_case, print_command, process_termination_timeout, run_scenario_evaluation, @@ -166,7 +172,34 @@ def cleanup() -> None: with cancellable_run(cleanup): try: - run_pod(args, rendezvous, pod_plan, processes, log_threads) + + def check_alive() -> None: + for entry in processes: + if entry.process.poll() is not None: + raise RuntimeError( + f"{entry.label} exited with {entry.process.returncode}" + ) + + def evaluate() -> str: + run_scenario_evaluation(args) + return PASS_VERDICT + + run_pod( + args, + rendezvous, + pod_plan, + launch_order=(ATTENTION_ROLE, FFN_ROLE) + if uses_async_connector(args) + else (FFN_ROLE, ATTENTION_ROLE), + start_slot=lambda slot: launch_slot( + args, pod_plan, slot, processes, log_threads + ), + check_alive=check_alive, + wait_ready=lambda slot, check: wait_for_health( + args, slot, args.serving_timeout, on_poll=check + ), + evaluate=evaluate, + ) except BaseException as exc: rendezvous.publish_abort(f"pod {pod_index}: {exc}") raise @@ -185,34 +218,30 @@ def run_pod( args: argparse.Namespace, rendezvous: Rendezvous, pod_plan: PodPlan, - processes: list[PodProcess], - log_threads: list[threading.Thread], -) -> None: + *, + launch_order: tuple[str, ...], + start_slot: Callable[[RoleSlot], None], + check_alive: Callable[[], None], + wait_ready: Callable[[RoleSlot, Callable[[], None]], None], + evaluate: Callable[[], str], +) -> str: """Phases 6-12: ordered launch, readiness, evaluation, verdict, teardown.""" pod_index = pod_plan.index def assert_local_processes_alive() -> None: - for entry in processes: - returncode = entry.process.poll() - if returncode is not None: - message = ( - f"pod {pod_index} {entry.label} exited " - f"(rc={returncode}) during the run" - ) - rendezvous.publish_abort(message) - raise RuntimeError(message) + try: + check_alive() + except BaseException as exc: + rendezvous.publish_abort(f"pod {pod_index}: {exc}") + raise # Phases 6-7: the connector's rendezvous role launches first, and every pod # waits for it before the other role starts. - first_kind, second_kind = ( - (ATTENTION_ROLE, FFN_ROLE) - if uses_async_connector(args) - else (FFN_ROLE, ATTENTION_ROLE) - ) + first_kind, second_kind = launch_order for kind, barrier in ((first_kind, f"{first_kind}-launched"), (second_kind, None)): slot = pod_plan.slot(kind) if slot is not None: - launch_slot(args, pod_plan, slot, processes, log_threads) + start_slot(slot) if barrier is not None: rendezvous.barrier( barrier, @@ -238,12 +267,7 @@ def assert_local_processes_alive() -> None: # for both the honest assertion is liveness, published at phase 8. serving_slot = pod_plan.slot(ATTENTION_ROLE) if serving_slot is not None and not serving_slot.headless: - wait_for_health( - args, - serving_slot, - args.serving_timeout, - on_poll=assert_local_processes_alive, - ) + wait_ready(serving_slot, assert_local_processes_alive) rendezvous.set(f"phase/{pod_index}/serving", "ok") rendezvous.barrier( SERVING_BARRIER, @@ -254,14 +278,13 @@ def assert_local_processes_alive() -> None: # Phase 10: exactly one pod evaluates, against its own local API. if pod_plan.is_evaluator: try: - run_scenario_evaluation(args) + verdict = evaluate() except BaseException as exc: verdict = f"fail: {exc}" rendezvous.set(VERDICT_KEY, verdict) rendezvous.publish_abort(verdict) raise - rendezvous.set(VERDICT_KEY, PASS_VERDICT) - verdict = PASS_VERDICT + rendezvous.set(VERDICT_KEY, verdict) else: verdict = rendezvous.wait_for_key( VERDICT_KEY, @@ -277,7 +300,7 @@ def assert_local_processes_alive() -> None: ) assert_local_processes_alive() print(f"[pod-{pod_index}] verdict: {verdict}", flush=True) - if verdict != PASS_VERDICT: + if verdict not in (PASS_VERDICT, DEBUG_VERDICT): raise RuntimeError(f"pod {pod_index} run failed: {verdict}") # Phase 12: no pod tears down until every pod has passed its last liveness @@ -285,6 +308,7 @@ def assert_local_processes_alive() -> None: # still checking would report as a failed run. No on_poll here: from now on # a child exiting is expected. rendezvous.barrier(TEARDOWN_BARRIER, args.verdict_timeout) + return verdict def launch_slot( @@ -446,11 +470,189 @@ def print_plan(pod_plan: PodPlan, addresses: list[str]) -> None: ) -def parse_args() -> argparse.Namespace: +def run_case( + case: Case, environment: CaseEnvironment, args: argparse.Namespace +) -> dict: + """Coordinate an existing GPU case; no cross-pod execution probes are added.""" + if any(phase.checks for phase in case.phases): + raise ValueError( + "multi-pod execution checks require cross-pod evidence support" + ) + layout = PodLayout.parse(args.pod_layout) + topology = Topology.from_case(case) + validate_layout(topology, layout) + pod_index = resolve_pod_index(layout.num_pods) + wait_for_address(args.store_host, args.store_dns_timeout) + rendezvous = Rendezvous( + host=args.store_host, + port=args.store_port, + pod_index=pod_index, + num_pods=layout.num_pods, + ) + try: + rendezvous.verify_agreement(layout.canonical(), case.id) + addresses = resolve_addresses(args, rendezvous, layout.num_pods, pod_index) + pod_plan = plan(topology, layout, addresses)[pod_index] + print_plan(pod_plan, addresses) + environment = replace( + environment, devices=environment.devices or pod_plan.devices + ) + effective, record = prepare_case(case, environment, slots=pod_plan.slots) + resolved = deployment.resolve(effective, environment, slots=pod_plan.slots) + record["run_id"] = pod_e2e_run_id(args.run_id, pod_index) + record["pod"] = { + **asdict(pod_plan), + "layout": layout.canonical(), + "role_devices": dict(resolved.role_devices), + } + record["coverage"] = ["task_result", "local_liveness", "local_cleanup"] + services = deployment.ServiceDeployment( + resolved, + run_id=record["run_id"], + extra_environment=parse_key_values(args.pod_env, option="--pod-env"), + ) + environment.output_path.mkdir(parents=True, exist_ok=True) + + def cleanup() -> None: + try: + services.close() + except BaseException as exc: + record["cleanup"] = {"status": "failed", "error": str(exc)} + publish_quietly(rendezvous, f"phase/{pod_index}/cleanup", str(exc)) + raise + else: + record["cleanup"] = {"status": "passed"} + publish_quietly(rendezvous, f"phase/{pod_index}/cleanup", "ok") + finally: + record["processes"] = [ + {"pid": process.pid, "returncode": process.poll()} + for process in services.processes + ] + + def evaluate() -> str: + for phase in effective.phases: + phase_record: dict[str, JsonValue] = { + "name": phase.name, + "status": "running", + "checks": [], + } + record["phases"].append(phase_record) + phase_record["result"] = phase.task.execute( + base_url=f"http://{environment.api_host}:{environment.api_port_base}", + model_name=f"{case.served_model_name_prefix}-attention", + checkpoint=environment.model, + output_path=environment.output_path / phase.name, + ) + phase_record["status"] = "passed" + # This is workload/configuration result propagation, not per-rank + # graph/DBO evidence. Every pod records what the evaluator ran. + rendezvous.set( + "evaluation/result", + json.dumps( + { + "effective": record["effective"], + "configuration": record["configuration"], + "phases": record["phases"], + } + ), + ) + return ( + PASS_VERDICT if record["configuration"]["eligible"] else DEBUG_VERDICT + ) + + try: + with cancellable_run(cleanup): + record["execution"] = {"status": "running"} + try: + verdict = run_pod( + args, + rendezvous, + pod_plan, + launch_order=resolved.launch_order, + start_slot=lambda slot: services.start_role(slot.role), + check_alive=services.ensure_alive, + wait_ready=lambda slot, check: services.wait_ready( + path="/health", + on_poll=check, + ), + evaluate=evaluate, + ) + evaluation = json.loads( + rendezvous.wait_for_key( + "evaluation/result", + args.verdict_timeout, + ) + ) + record.update(evaluation) + except BaseException as exc: + rendezvous.publish_abort(f"pod {pod_index}: {exc}") + raise + record["execution"] = {"status": "passed"} + record["acceptance_passed"] = verdict == PASS_VERDICT + return record + except BaseException as exc: + status = "cancelled" if isinstance(exc, SystemExit) else "failed" + record["execution"] = {"status": status, "error": str(exc)} + for phase_record in record["phases"]: + if phase_record["status"] == "running": + phase_record.update(status=status, error=str(exc)) + raise + finally: + (environment.output_path / "run.json").write_text( + json.dumps(record, indent=2) + "\n" + ) + print( + f"[pod-{pod_index}] CASE {case.id}: {record['execution']['status']}; " + f"acceptance_passed={record['acceptance_passed']}", + flush=True, + ) + except BaseException as exc: + rendezvous.publish_abort(f"pod {pod_index}: {exc}") + raise + finally: + leave_rendezvous( + rendezvous, case.deployment.termination_timeout_s + DEPARTURE_GRACE_S + ) + + +def case_main( + build_case: Callable[[str, CaseEnvironment], Case], + case_ids: tuple[str, ...], + argv: list[str] | None = None, +) -> int: parser = argparse.ArgumentParser( - description="Run one pod's slice of a multi-pod AFD E2E scenario.", + description="Run one pod of a model-declared GPU case" ) - add_scenario_arguments(parser) + add_case_environment_arguments( + parser, + case_ids, + devices_required=False, + startup_option="--serving-timeout", + ) + add_pod_arguments(parser) + parser.set_defaults(startup_timeout=1800) + args = parser.parse_args(argv) + args.serving_timeout = args.startup_timeout + environment = case_environment(args) + if not environment.devices and os.environ.get("CUDA_VISIBLE_DEVICES"): + environment = replace( + environment, devices=tuple(os.environ["CUDA_VISIBLE_DEVICES"].split(",")) + ) + case = build_case(args.scenario, environment) + if args.describe: + print( + json.dumps( + {"canonical": asdict(case), "bindings": asdict(environment)}, + default=str, + indent=2, + ) + ) + else: + run_case(case, environment, args) + return 0 + + +def add_pod_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--pod-layout", required=True, @@ -492,8 +694,16 @@ def parse_args() -> argparse.Namespace: help="KEY=VALUE added to every launched vLLM process environment.", ) parser.add_argument("--launch-timeout", type=float, default=900) - parser.add_argument("--serving-timeout", type=float, default=1800) parser.add_argument("--verdict-timeout", type=float, default=3600) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run the retained NPU multi-pod entry.", + ) + add_scenario_arguments(parser) + add_pod_arguments(parser) + parser.add_argument("--serving-timeout", type=float, default=1800) return parser.parse_args() diff --git a/tests/e2e/presets.py b/tests/e2e/presets.py new file mode 100644 index 000000000..5d929d53b --- /dev/null +++ b/tests/e2e/presets.py @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Reusable execution configuration; models select cases and acceptance settings.""" + +from dataclasses import dataclass + +from tests.e2e.case import ( + EXECUTION_EAGER, + EXECUTION_FULL_DECODE_GRAPH, + DboConfig, + Deployment, + Role, +) + + +@dataclass(frozen=True) +class ExecutionPreset: + """A shared execution mode for both roles of a synchronous deployment.""" + + execution: str + + def deployment( + self, + *, + backend: str, + attention: tuple[int, int], + ffn: tuple[int, int] | None, + connector: str | None, + capture_size: int, + dbo: DboConfig | None = None, + compute_gate_on_attention: bool = False, + ) -> Deployment: + """Layouts are (DP, TP); models may replace explicit budget fields.""" + graph = self.execution == EXECUTION_FULL_DECODE_GRAPH + return Deployment( + backend=backend, + attention=Role(*attention, execution=self.execution), + ffn=Role(*ffn, execution=self.execution) if ffn is not None else None, + connector=connector, + dbo=dbo, + compute_gate_on_attention=compute_gate_on_attention, + cudagraph_capture_size=capture_size if graph else None, + max_num_seqs=capture_size if graph else None, + max_num_batched_tokens=capture_size if ffn is None else None, + ) + + +EAGER = ExecutionPreset(EXECUTION_EAGER) +DECODE_GRAPH = ExecutionPreset(EXECUTION_FULL_DECODE_GRAPH) diff --git a/tests/e2e/runner.py b/tests/e2e/runner.py index 5ee36469f..8bcb1862b 100644 --- a/tests/e2e/runner.py +++ b/tests/e2e/runner.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project -"""Run fixed baseline and AFD E2E scenarios on real hardware.""" +"""Execute model-declared Cases and retain the existing NPU CLI callers.""" from __future__ import annotations @@ -16,18 +16,36 @@ import time import urllib.error import urllib.request -from collections.abc import Mapping +import uuid +from collections.abc import Callable, Mapping +from dataclasses import asdict, replace from pathlib import Path from typing import Any +from tests.e2e import deployment from tests.e2e.accuracy.gsm8k import ( _extract_gsm8k_accuracy, _extract_gsm8k_sample_count, _run_lm_eval, ) from tests.e2e.cancellation import cancellable_run +from tests.e2e.case import ( + NPU_ASYNC_TERMINATION_TIMEOUT_S, + NPU_ASYNC_VLLM_SHUTDOWN_TIMEOUT_S, + NPU_ASYNC_WORKER_SHUTDOWN_TIMEOUT_S, + WORKER_SHUTDOWN_TIMEOUT_ENV, + Case, + CaseEnvironment, + JsonValue, + Mrv2Check, + Role, +) +from tests.e2e.checks.mrv2 import ( + Mrv2Session, + assert_execution_evidence, +) +from tests.e2e.environment import parse_case_environment from tests.e2e.models.deepseek_v4_flash import config as dsv4_config -from tests.e2e.models.deepseek_v4_flash import gpu_config as dsv4_gpu_config from tests.e2e.models.deepseek_v4_flash.completions import evaluate_completions from tests.e2e.models.deepseek_v4_flash.config import ( DSV4_ASYNC_CAM_SCENARIO, @@ -40,10 +58,6 @@ DSV4_SYNC_SHAPES, sync_shape, ) -from tests.e2e.models.deepseek_v4_flash.gpu_config import ( - DSV4_GPU_SCENARIO_SETTINGS, - DSV4_GPU_SCENARIOS, -) from tests.e2e.multi_pod.layout import RoleSlot from tests.e2e.process_utils import ( kill_processes_matching_environment, @@ -58,44 +72,6 @@ # Graph capture default for the scenarios that do not carry their own launch # profile; the DSV4 synchronous profiles override it. DEFAULT_CUDAGRAPH_CAPTURE_SIZE = 8 -V2_SCENARIOS = ( - "afd-v2-eager-1a1f", - "afd-v2-eager-dp2", - "afd-v2-eager-tp2", - "afd-v2-graph-1a1f", - "afd-v2-graph-dp2", - "afd-v2-graph-tp2", - "afd-v2-eager-dbo-dp2", - "afd-v2-graph-dbo-dp2", - "afd-v2-graph-dbo-tp2", -) -# DeepEP M2N routes on Attention (gate and top-k there) and dispatches each -# token straight to the FFN ranks owning its experts. -DEEPEP_M2N_CONNECTOR = "DeepEPM2NAFDConnector" -DEEPEP_M2N_SCENARIOS = ( - "afd-deepep-m2n-eager-2a2f", - "afd-deepep-m2n-graph-2a2f", - "afd-deepep-m2n-graph-dbo-2a2f", - "afd-deepep-m2n-eager-tp2-2a2f", - "afd-deepep-m2n-graph-tp2-2a2f", -) -# Attention runs DP1 x TP2: each TP rank dispatches its slice of the tokens. -# FFN stays DP2 x TP1 (experts are not TP-sharded under DeepEP M2N). -DEEPEP_M2N_ATTENTION_TP_SCENARIOS = frozenset( - ("afd-deepep-m2n-eager-tp2-2a2f", "afd-deepep-m2n-graph-tp2-2a2f"), -) -DEEPEP_M2N_ATTENTION_TP_SIZE = 2 -V2_DBO_COMPARISON_SCENARIOS = frozenset( - ("afd-v2-eager-dp2", "afd-v2-eager-dbo-dp2", "afd-v2-graph-dbo-dp2"), -) -V2_DBO_SAMPLE_LIMIT = 128 -V2_DBO_MAX_NUM_SEQS = 8 -V2_SINGLE_RANK_SCENARIOS = frozenset( - ("afd-v2-eager-1a1f", "afd-v2-graph-1a1f"), -) -V2_TENSOR_PARALLEL_SCENARIOS = frozenset( - ("afd-v2-eager-tp2", "afd-v2-graph-tp2", "afd-v2-graph-dbo-tp2"), -) E2E_RUN_ID_ENV = "AFD_E2E_RUN_ID" E2E_PROCESS_ROLE_ENV = "AFD_E2E_PROCESS_ROLE" PROCESS_TERMINATION_TIMEOUT_S = 20 @@ -107,9 +83,6 @@ NPU_ASYNC_PROCESS_KILL_TIMEOUT_S = 120 LOG_THREAD_JOIN_TIMEOUT_S = 2 VLLM_SHUTDOWN_TIMEOUT_S = 10 -NPU_ASYNC_VLLM_SHUTDOWN_TIMEOUT_S = 90 -NPU_ASYNC_WORKER_SHUTDOWN_TIMEOUT_S = 60 -NPU_ASYNC_TERMINATION_TIMEOUT_S = 120 DEFAULT_GSM8K_SAMPLE_LIMIT = 7 GSM8K_NUM_FEWSHOT = 8 GSM8K_FULL_SAMPLE_COUNT = 1319 @@ -227,42 +200,15 @@ def cleanup() -> None: wait_for_openai_api(args, processes) ensure_processes_alive(processes) - if args.scenario in DSV4_GPU_SCENARIOS: - round_samples = [] - for round_index in range(dsv4_gpu_config.DSV4_GPU_CONCURRENT_ROUNDS): - window = f"concurrent-{round_index + 1}" - start_evaluation_window(args, mrv2_evidence_start, window) - run_concurrent_completion_evaluation(args, round_index=round_index) - round_samples.append( - finish_evaluation_window( - args, - mrv2_evidence_start, - window, - mrv2_execution_evidence, - processes, - ) - ) - if needs_hash_input_evidence(args): - assert_hash_input_freshness(round_samples, args) - start_evaluation_window(args, mrv2_evidence_start, "gsm8k") - run_gsm8k_evaluation(args) - finish_evaluation_window( - args, - mrv2_evidence_start, - "gsm8k", - mrv2_execution_evidence, - processes, - ) - else: - start_evaluation_window(args, mrv2_evidence_start, "evaluation") - run_scenario_evaluation(args) - finish_evaluation_window( - args, - mrv2_evidence_start, - "evaluation", - mrv2_execution_evidence, - processes, - ) + start_evaluation_window(args, mrv2_evidence_start, "evaluation") + run_scenario_evaluation(args) + finish_evaluation_window( + args, + mrv2_evidence_start, + "evaluation", + mrv2_execution_evidence, + processes, + ) ensure_processes_alive(processes) @@ -317,9 +263,6 @@ def add_scenario_arguments(parser: argparse.ArgumentParser) -> None: "afd-graph-2a2f", "afd-graph-dbo-2a2f", *DSV4_SCENARIOS, - *DSV4_GPU_SCENARIOS, - *V2_SCENARIOS, - *DEEPEP_M2N_SCENARIOS, ], required=True, help="Fixed E2E scenario to run.", @@ -412,10 +355,8 @@ def add_scenario_arguments(parser: argparse.ArgumentParser) -> None: def configure_scenario(args: argparse.Namespace) -> None: - """Set topology and features for the selected fixed scenario.""" - is_dsv4 = args.scenario in DSV4_SCENARIOS - is_dsv4_gpu = args.scenario in DSV4_GPU_SCENARIOS - scenario_settings = { + """Configure retained NPU callers; GPU cases are model-owned declarations.""" + settings = { "baseline-graph": (True, True, False, 4, 0), "afd-eager-2a1f": (False, False, False, 2, 1), "afd-graph-2a1f": (False, True, False, 2, 1), @@ -431,91 +372,51 @@ def configure_scenario(args: argparse.Namespace) -> None: DSV4_ATTENTION_RANKS, DSV4_FFN_RANKS, ), - "afd-v2-eager-1a1f": (False, False, False, 1, 1), - "afd-v2-eager-dp2": (False, False, False, 2, 2), - "afd-v2-eager-tp2": (False, False, False, 2, 2), - "afd-v2-graph-1a1f": (False, True, False, 1, 1), - "afd-v2-graph-dp2": (False, True, False, 2, 2), - "afd-v2-graph-tp2": (False, True, False, 2, 2), - "afd-v2-eager-dbo-dp2": (False, False, True, 2, 2), - "afd-v2-graph-dbo-dp2": (False, True, True, 2, 2), - "afd-v2-graph-dbo-tp2": (False, True, True, 2, 2), - **DSV4_GPU_SCENARIO_SETTINGS, - "afd-deepep-m2n-eager-2a2f": (False, False, False, 2, 2), - "afd-deepep-m2n-graph-2a2f": (False, True, False, 2, 2), - "afd-deepep-m2n-graph-dbo-2a2f": (False, True, True, 2, 2), - "afd-deepep-m2n-eager-tp2-2a2f": (False, False, False, 2, 2), - "afd-deepep-m2n-graph-tp2-2a2f": (False, True, False, 2, 2), } - for sync_scenario, sync_profile in DSV4_SYNC_SHAPES.items(): - # The A5 script's native DBO stays off for these scenarios: a split batch - # is the current suspect for the DSA operator tiling failure on that - # host. - scenario_settings[sync_scenario] = ( + for scenario, profile in DSV4_SYNC_SHAPES.items(): + # The A5 host profile keeps native DBO off: split batches are the + # current suspect for its DSA operator tiling failure. + settings[scenario] = ( False, - dsv4_config.sync_use_graph(sync_profile), + dsv4_config.sync_use_graph(profile), False, - sync_profile.attention_ranks, - sync_profile.ffn_ranks, + profile.attention_ranks, + profile.ffn_ranks, ) - baseline, use_graph, enable_dbo, attention_ranks, ffn_ranks = scenario_settings[ - args.scenario - ] + baseline, graph, dbo, attention_ranks, ffn_ranks = settings[args.scenario] args.baseline = baseline - args.cuda_graph_full_decode_only = use_graph - args.enable_dbo = enable_dbo + args.cuda_graph_full_decode_only = graph + args.enable_dbo = dbo args.num_attention_ranks = attention_ranks args.num_ffn_ranks = ffn_ranks args.attention_data_parallel_address = None - # A runner that cannot drive the evidence window turns this off. args.collect_mrv2_execution_evidence = True args.tp_size = 1 - active_sync_profile = sync_shape(args.scenario) - if args.scenario == DSV4_ASYNC_CAM_SCENARIO: - args.attention_tp_size = DSV4_ATTENTION_TP_SIZE - elif active_sync_profile is not None: - args.attention_tp_size = active_sync_profile.attention_tp_size - elif args.scenario in V2_TENSOR_PARALLEL_SCENARIOS: - args.attention_tp_size = 2 - elif args.scenario in DEEPEP_M2N_ATTENTION_TP_SCENARIOS: - args.attention_tp_size = DEEPEP_M2N_ATTENTION_TP_SIZE - else: - args.attention_tp_size = 1 - if args.scenario in V2_TENSOR_PARALLEL_SCENARIOS: - args.ffn_tp_size = 2 - elif active_sync_profile is not None: - # A synchronous DSV4 profile sizes the FFN side exactly like its - # Attention side: A5 shards by data parallel with expert parallelism, - # A3 by tensor parallel. - args.ffn_tp_size = active_sync_profile.ffn_tp_size - else: - args.ffn_tp_size = 1 - if not is_dsv4 and args.gsm8k_output_path is None: + profile = sync_shape(args.scenario) + args.attention_tp_size = ( + DSV4_ATTENTION_TP_SIZE + if args.scenario == DSV4_ASYNC_CAM_SCENARIO + else profile.attention_tp_size + if profile is not None + else 1 + ) + args.ffn_tp_size = profile.ffn_tp_size if profile is not None else 1 + if args.scenario not in DSV4_SCENARIOS and args.gsm8k_output_path is None: raise ValueError("--gsm8k-output-path is required for GSM8K scenarios") - if args.scenario in DEEPEP_M2N_SCENARIOS: - args.afd_connector = DEEPEP_M2N_CONNECTOR - args.compute_gate_on_attention = True if args.scenario == DSV4_ASYNC_CAM_SCENARIO: dsv4_config.configure_scenario(args) elif args.scenario in DSV4_SYNC_CAMP2P_SCENARIOS: dsv4_config.configure_sync_camp2p_scenario(args) - elif is_dsv4_gpu: - dsv4_gpu_config.configure_scenario(args) - if use_graph: + if graph: args.cudagraph_capture_size = ( - active_sync_profile.cudagraph_capture_size - if active_sync_profile is not None - else dsv4_gpu_config.DSV4_GPU_CAPTURE_SIZE - if is_dsv4_gpu + profile.cudagraph_capture_size + if profile is not None else DEFAULT_CUDAGRAPH_CAPTURE_SIZE ) if args.scenario == NPU_EAGER_DBO_SCENARIO: - # This opt-in eager cell qualifies DBO at the native 0.32 thresholds. args.dbo_decode_token_threshold = NPU_EAGER_DBO_DECODE_THRESHOLD args.dbo_prefill_token_threshold = NPU_EAGER_DBO_PREFILL_THRESHOLD - elif enable_dbo: - # vLLM validates that both DBO thresholds are at least the - # microbatch count (2), so a decode threshold of 1 is rejected. + elif dbo: args.dbo_decode_token_threshold = 2 args.dbo_prefill_token_threshold = 8 @@ -545,11 +446,7 @@ def validate_topology( f"--ffn-devices must contain exactly {args.num_ffn_ranks} device", ) if args.baseline: - baseline_ranks = ( - dsv4_gpu_config.DSV4_GPU_ROLE_RANKS - if args.scenario in DSV4_GPU_SCENARIOS - else 4 - ) + baseline_ranks = 4 if args.num_attention_ranks != baseline_ranks or args.num_ffn_ranks != 0: raise ValueError( f"baseline E2E requires {baseline_ranks} Attention ranks " @@ -558,16 +455,10 @@ def validate_topology( if role_tp_size(args, "attention") != 1: raise ValueError("baseline E2E requires Attention TP=1") return - if args.scenario in V2_SCENARIOS and args.device_backend != "gpu": - raise ValueError("ModelRunnerV2 E2E scenarios require GPU") if args.scenario == NPU_EAGER_DBO_SCENARIO and args.device_backend != "npu": raise ValueError("Lite eager DBO scenario requires NPU") - if args.scenario in DEEPEP_M2N_SCENARIOS and args.device_backend != "gpu": - raise ValueError("DeepEP M2N E2E scenarios require GPU") if args.scenario in DSV4_SCENARIOS and args.device_backend != "npu": raise ValueError("DSV4 scenarios require NPU") - if args.scenario in DSV4_GPU_SCENARIOS and args.device_backend != "gpu": - raise ValueError("CUDA DSV4 scenarios require GPU") for role, rank_count in ( ("attention", args.num_attention_ranks), ("ffn", args.num_ffn_ranks), @@ -610,15 +501,14 @@ def build_baseline_command(args: argparse.Namespace) -> list[str]: cmd.extend(dbo_vllm_arguments(args)) if args.cuda_graph_full_decode_only: capture_size = str(args.cudagraph_capture_size) - if args.scenario not in DSV4_GPU_SCENARIOS: - cmd.extend( - [ - "--max-num-seqs", - capture_size, - "--max-num-batched-tokens", - capture_size, - ], - ) + cmd.extend( + [ + "--max-num-seqs", + capture_size, + "--max-num-batched-tokens", + capture_size, + ], + ) cmd.extend( [ "--max-cudagraph-capture-size", @@ -752,15 +642,6 @@ def build_vllm_command( ) if needs_mrv2_execution_evidence(args): cmd.extend(["--worker-extension-cls", "tests.e2e.mrv2_evidence.Worker"]) - if args.scenario in V2_DBO_COMPARISON_SCENARIOS: - # Keep request concurrency identical across the no-DBO, eager DBO - # and graph DBO comparison; retain native chunked-prefill defaults. - cmd.extend( - [ - "--max-num-seqs", - str(V2_DBO_MAX_NUM_SEQS), - ], - ) profile_compilation_config = ( None if sync_profile is None @@ -781,8 +662,7 @@ def build_vllm_command( # profiles do) keeps it: the capture size only has to cover it. max_num_seqs_args = ( [] - if args.scenario in V2_DBO_COMPARISON_SCENARIOS - or any(arg == "--max-num-seqs" for arg in args.common_vllm_arg) + if any(arg == "--max-num-seqs" for arg in args.common_vllm_arg) else ["--max-num-seqs", capture_size] ) cmd.extend( @@ -918,15 +798,12 @@ def run_gsm8k_evaluation(args: argparse.Namespace) -> None: """Run the configured GSM8K workload against the scenario's public API.""" if args.gsm8k_output_path is None: raise RuntimeError("--gsm8k-output-path is required for GSM8K scenarios") - is_v2_dbo_comparison = args.scenario in V2_DBO_COMPARISON_SCENARIOS configured_limit = os.environ.get( GSM8K_LIMIT_ENV, - str( - V2_DBO_SAMPLE_LIMIT if is_v2_dbo_comparison else DEFAULT_GSM8K_SAMPLE_LIMIT - ), + str(DEFAULT_GSM8K_SAMPLE_LIMIT), ) sample_limit = None if configured_limit == "all" else int(configured_limit) - if (args.enable_dbo or is_v2_dbo_comparison) and sample_limit is not None: + if args.enable_dbo and sample_limit is not None: sample_limit = max(sample_limit, DBO_EVAL_MIN_SAMPLES) expected_sample_count = ( GSM8K_FULL_SAMPLE_COUNT if sample_limit is None else sample_limit @@ -935,7 +812,7 @@ def run_gsm8k_evaluation(args: argparse.Namespace) -> None: {"timeout_s": FULL_GSM8K_TIMEOUT_S} if sample_limit is None else {} ) scenario_options = {} - if args.enable_dbo or is_v2_dbo_comparison: + if args.enable_dbo: scenario_options["num_concurrent"] = DBO_EVAL_NUM_CONCURRENT role = "baseline" if args.baseline else "attention" results = _run_lm_eval( @@ -977,16 +854,11 @@ def execution_evidence_roles(args: argparse.Namespace) -> tuple[str, ...]: def needs_mrv2_execution_evidence(args: argparse.Namespace) -> bool: - observe_native = args.baseline and args.scenario in DSV4_GPU_SCENARIOS return ( - needs_hash_input_evidence(args) - or (not args.baseline or observe_native) + not args.baseline and (args.cuda_graph_full_decode_only or uses_dbo_execution(args)) - ) and args.collect_mrv2_execution_evidence - - -def needs_hash_input_evidence(args: argparse.Namespace) -> bool: - return args.scenario in DSV4_GPU_SCENARIOS and not args.baseline + and args.collect_mrv2_execution_evidence + ) def start_evaluation_window( @@ -1018,12 +890,7 @@ def finish_evaluation_window( scoped = [(role, line) for role, line in evidence if f"window={window} " in line] if args.cuda_graph_full_decode_only or uses_dbo_execution(args): assert_mrv2_execution_evidence(scoped, args) - return ( - assert_hash_input_evidence(scoped, args) - if needs_hash_input_evidence(args) - and window in dsv4_gpu_config.DSV4_GPU_HASH_WINDOWS - else {} - ) + return {} def mrv2_execution_evidence_rpc( @@ -1090,204 +957,35 @@ def wait_for_mrv2_execution_summaries( def assert_mrv2_execution_evidence( evidence: list[tuple[str, str]], args: argparse.Namespace ) -> None: - metric = ( - "two_stage_graph_replays" - if uses_dbo_execution(args) and args.cuda_graph_full_decode_only - else "two_stage_steps" - if uses_dbo_execution(args) - else "graph_replays" - ) - minimum = ( - MRV2_EVIDENCE_REPLAY_MIN_COUNT if metric == "two_stage_graph_replays" else 1 - ) - layout_kind = ( - "consecutive_graph" if args.cuda_graph_full_decode_only else "two_stage" - ) - common_layouts: set[str] | None = None - for role in execution_evidence_roles(args): - tp_size = role_tp_size(args, role) - rank_count = args.num_ffn_ranks if role == "ffn" else args.num_attention_ranks - expected = { - (dp, tp) for dp in range(rank_count // tp_size) for tp in range(tp_size) - } - observed = set() - layouts: dict[str, set[tuple[int, int]]] = {} - for recorded_role, line in evidence: - if recorded_role != role: - continue - match = re.search( - rf"dp_rank=(\d+) tp_rank=(\d+).*\b{metric}=(\d+)(?: |$)", line - ) - if match is not None and int(match[3]) >= minimum: - rank = (int(match[1]), int(match[2])) - observed.add(rank) - if uses_dbo_execution(args) and not uses_async_connector(args): - payload = json.loads(line.split(" layouts=", 1)[1]) - for layout in payload[layout_kind]: - key = json.dumps(layout, separators=(",", ":")) - layouts.setdefault(key, set()).add(rank) - if missing := expected - observed: - raise RuntimeError( - f"{args.device_backend.upper()} {role} lacks live " - f"{metric}>={minimum} evidence on ranks " - f"{sorted(missing)}" - ) - if uses_dbo_execution(args) and not uses_async_connector(args): - complete_layouts = { - layout for layout, ranks in layouts.items() if expected <= ranks - } - common_layouts = ( - complete_layouts - if common_layouts is None - else common_layouts & complete_layouts - ) - if ( - uses_dbo_execution(args) - and not uses_async_connector(args) - and not common_layouts - ): - raise RuntimeError( - f"{args.device_backend.upper()} DBO lacks matching two-stage layouts " - "across every participating rank" - + ( - " with consecutive graph replays" - if args.cuda_graph_full_decode_only - else "" - ) - ) - print( - f"[execution-coverage] live {metric}>={minimum} " - "confirmed on every participating rank" - ) - - -def assert_hash_input_evidence( - evidence: list[tuple[str, str]], args: argparse.Namespace -) -> dict[tuple[int, int, int, int], set[tuple[int, ...]]]: - """Pair bounded successful live A/F samples in original control/peer order.""" - samples: dict[str, dict[tuple[int, int, int], dict]] = { - "attention": {}, - "ffn": {}, - } - observed_ranks: dict[str, set[tuple[int, int]]] = { - "attention": set(), - "ffn": set(), - } - for role, line in evidence: - match = re.search(r"dp_rank=(\d+) tp_rank=(\d+)", line) - assert match is not None - rank = (int(match[1]), int(match[2])) - payload = json.loads(line.split(" layouts=", 1)[1]) - for sample in payload["hash_ids"]: - if not sample["completed"] or sample["control"] <= 0: - raise RuntimeError( - "Hash ID sample did not complete a live control transaction" - ) - if args.cuda_graph_full_decode_only and not sample["replayed"]: - raise RuntimeError("Hash ID sample lacks completed graph replay") - key = (sample["control"], sample["stage"], sample["peer"]) - if key in samples[role]: - raise RuntimeError(f"Duplicate {role} Hash ID sample: {key}") - if role == "attention" and sample["peer"] != rank[0]: - raise RuntimeError( - f"Attention Hash ID peer does not match DP rank: {key}" - ) - samples[role][key] = sample - observed_ranks[role].add(rank) - for role, ranks in observed_ranks.items(): - rank_count = args.num_ffn_ranks if role == "ffn" else args.num_attention_ranks - expected = {(dp, 0) for dp in range(rank_count)} - if ranks != expected: - raise RuntimeError( - f"{role} lacks bounded Hash ID samples on ranks {expected - ranks}" - ) - if samples["attention"].keys() != samples["ffn"].keys(): - raise RuntimeError("A/F Hash ID samples do not pair by control/stage/peer") - values: dict[tuple[int, int, int, int], set[tuple[int, ...]]] = {} - covered = set() - for key, attention in samples["attention"].items(): - ffn = samples["ffn"][key] - real = attention["real_tokens"] - bucket = attention["bucket"] - if ( - ffn["bucket"] != bucket - or len(ffn["input_ids"]) != bucket - or len(attention["input_ids"]) != real - or not 0 <= real <= bucket - or attention["input_ids"] != ffn["input_ids"][:real] - ): - raise RuntimeError( - f"Native A / actual Hash-layer F input IDs differ: {key}" - ) - if real: - _, stage, peer = key - covered.add((peer, stage)) - values.setdefault((peer, stage, bucket, real), set()).add( - tuple(attention["input_ids"]) - ) - stages = range(DBO_EVAL_NUM_UBATCHES if args.enable_dbo else 1) - required = { - (peer, stage) for peer in range(args.num_attention_ranks) for stage in stages - } - if missing := required - covered: - raise RuntimeError( - f"Hash ID samples lack real tokens for peer/stage {sorted(missing)}" + roles = execution_evidence_roles(args) + layouts = { + role: Role( + dp=(args.num_ffn_ranks if role == "ffn" else args.num_attention_ranks) + // role_tp_size(args, role), + tp=role_tp_size(args, role), ) - print("[hash-inputs] native A and actual Hash-layer F IDs match on every rank") - return values - - -def assert_hash_input_freshness( - rounds: list[dict[tuple[int, int, int, int], set[tuple[int, ...]]]], - args: argparse.Namespace, -) -> None: - """Require new real IDs across rounds at the same padded and real size.""" - previous, current = rounds - changed = { - (peer, stage) - for (peer, stage, bucket, real), values in current.items() - if (peer, stage, bucket, real) in previous - and values - previous[(peer, stage, bucket, real)] - } - stages = range(DBO_EVAL_NUM_UBATCHES if args.enable_dbo else 1) - required = { - (peer, stage) for peer in range(args.num_attention_ranks) for stage in stages + for role in roles } - if missing := required - changed: - raise RuntimeError( - "Hash input IDs did not change across concurrent rounds with the same " - f"bucket and real token count: {sorted(missing)}" - ) - print("[hash-inputs] both concurrent windows show fresh IDs at the same bucket") + assert_execution_evidence( + evidence, + Mrv2Check( + roles, graph=args.cuda_graph_full_decode_only, dbo=uses_dbo_execution(args) + ), + role_layouts=layouts, + backend=args.device_backend, + asynchronous=uses_async_connector(args), + ) -def run_concurrent_completion_evaluation( - args: argparse.Namespace, *, round_index: int | None = None -) -> None: +def run_concurrent_completion_evaluation(args: argparse.Namespace) -> None: profile = sync_shape(args.scenario) role = "baseline" if args.baseline else "attention" - output_path = Path(args.completion_output_path) - if round_index is not None: - output_path = output_path.with_stem( - f"{output_path.stem}-round-{round_index + 1}" - ) evaluate_completions( url=f"http://{args.api_host}:{attention_api_port(args)}/v1/chat/completions", model=served_model_name(args, role), - output_path=output_path, - # A synchronous profile may state that its host does not answer reliably - # yet, and then the oracle checks the concurrent plumbing only. + output_path=Path(args.completion_output_path), + # Preserve the NPU profile's answer/plumbing acceptance policy. check_answer=True if profile is None else profile.check_answer, - first_request_index=( - 0 - if round_index is None - else round_index * dsv4_config.DSV4_CONCURRENT_REQUESTS - ), - answer_repetitions=( - dsv4_gpu_config.DSV4_GPU_ANSWER_REPETITIONS - if args.scenario in DSV4_GPU_SCENARIOS - else 1 - ), ) @@ -1306,14 +1004,8 @@ def build_env( env.setdefault("VLLM_ENGINE_READY_TIMEOUT_S", "18000") env[visible_devices_env_name(args.device_backend)] = visible_devices env["VLLM_USE_V2_MODEL_RUNNER"] = "1" - if needs_hash_input_evidence(args): - env["AFD_E2E_HASH_INPUT_IDS"] = "1" - env["AFD_E2E_HASH_REQUIRE_GRAPH"] = ( - "1" if args.cuda_graph_full_decode_only else "0" - ) - else: - env.pop("AFD_E2E_HASH_INPUT_IDS", None) - env.pop("AFD_E2E_HASH_REQUIRE_GRAPH", None) + env.pop("AFD_E2E_HASH_INPUT_IDS", None) + env.pop("AFD_E2E_HASH_REQUIRE_GRAPH", None) if needs_mrv2_execution_evidence(args): env["VLLM_SERVER_DEV_MODE"] = "1" if args.baseline: @@ -1338,7 +1030,7 @@ def build_env( env.pop("VLLM_ASCEND_ENABLE_FLASHCOMM1", None) if args.afd_async and not args.baseline: env.setdefault( - "VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS", + WORKER_SHUTDOWN_TIMEOUT_ENV, str(NPU_ASYNC_WORKER_SHUTDOWN_TIMEOUT_S), ) env.pop("AFD_PLUGIN_EARLY_ENGINE_PATCH", None) @@ -1483,5 +1175,154 @@ def shell_quote(value: str) -> str: return "'" + value.replace("'", "'\"'\"'") + "'" +def prepare_case( + case: Case, environment: CaseEnvironment, *, slots: tuple[RoleSlot, ...] = () +) -> tuple[Case, dict]: + """Resolve documented overrides without mutating the canonical declaration.""" + effective_phases = [] + reasons: list[str] = [] + for phase in case.phases: + task, phase_reasons = phase.task.resolve(os.environ) + effective_phases.append(replace(phase, task=task)) + reasons.extend(f"{phase.name}: {reason}" for reason in phase_reasons) + profile = case.deployment + override_names: tuple[str, ...] = (GSM8K_LIMIT_ENV, GSM8K_THRESHOLD_ENV) + if profile.worker_shutdown_timeout_s is not None: + override_names += (WORKER_SHUTDOWN_TIMEOUT_ENV,) + worker_timeout = os.environ.get(WORKER_SHUTDOWN_TIMEOUT_ENV) + if worker_timeout is not None: + profile = replace(profile, worker_shutdown_timeout_s=int(worker_timeout)) + effective = replace(case, deployment=profile, phases=tuple(effective_phases)) + resolved = deployment.resolve(effective, environment, slots=slots) + probe_arguments = ( + ("--worker-extension-cls", "tests.e2e.mrv2_evidence.Worker") + if any(phase.checks for phase in effective.phases) + else () + ) + record = { + "case_id": case.id, + "run_id": uuid.uuid4().hex, + "canonical": asdict(case), + "effective": asdict(effective), + "bindings": { + **asdict(environment), + "output_path": str(environment.output_path), + }, + "overrides": { + name: os.environ[name] for name in override_names if name in os.environ + }, + "configuration": { + "eligible": not reasons, + "classification": "debug" if reasons else "qualified", + "reasons": reasons, + }, + "commands": { + role: resolved.command(role, probe_arguments) + for role, _ in resolved.role_devices + }, + "execution": {"status": "not_started"}, + "cleanup": {"status": "not_started"}, + "phases": [], + "acceptance_passed": False, + } + return effective, record + + +def run_case(case: Case, environment: CaseEnvironment) -> dict: + """Execute one deployment; success follows task, evidence and cleanup.""" + effective, record = prepare_case(case, environment) + resolved = deployment.resolve(effective, environment) + probe = Mrv2Session(effective, environment, record["run_id"]) + services = deployment.ServiceDeployment(resolved, run_id=record["run_id"]) + environment.output_path.mkdir(parents=True, exist_ok=True) + + def cleanup() -> None: + failures = [] + try: + services.close() + except BaseException as exc: + failures.append(str(exc)) + finally: + try: + probe.close() + except BaseException as exc: + failures.append(str(exc)) + record["processes"] = [ + {"pid": process.pid, "returncode": process.poll()} + for process in services.processes + ] + if failures: + record["cleanup"] = {"status": "failed", "errors": failures} + raise RuntimeError("; ".join(failures)) + record["cleanup"] = {"status": "passed"} + + try: + with cancellable_run(cleanup): + probe.prepare() + services.probe_arguments = probe.arguments + services.probe_environment = probe.worker_environment + services.start() + services.ensure_alive() + record["execution"] = {"status": "running"} + for phase in effective.phases: + phase_record: dict[str, JsonValue] = { + "name": phase.name, + "status": "running", + "checks": [asdict(check) for check in phase.checks], + } + record["phases"].append(phase_record) + probe.start(phase) + role = "baseline" if effective.deployment.is_baseline else "attention" + phase_record["result"] = phase.task.execute( + base_url=f"http://{environment.api_host}:{environment.api_port_base}", + model_name=f"{case.served_model_name_prefix}-{role}", + checkpoint=environment.model, + output_path=environment.output_path / phase.name, + ) + snapshot = probe.finish(phase, services) + phase_record["evidence_records"] = len(snapshot) + phase_record["status"] = "passed" + services.ensure_alive() + record["execution"] = {"status": "passed"} + record["acceptance_passed"] = record["configuration"]["eligible"] + return record + except BaseException as exc: + record["execution"] = { + "status": "cancelled" if isinstance(exc, SystemExit) else "failed", + "error": str(exc), + } + for phase_record in record["phases"]: + if phase_record["status"] == "running": + phase_record["status"] = record["execution"]["status"] + phase_record["error"] = str(exc) + raise + finally: + (environment.output_path / "run.json").write_text( + json.dumps(record, indent=2) + "\n" + ) + print( + f"\nE2E CASE {case.id}: {record['execution']['status']}; " + f"configuration={record['configuration']['classification']}; " + f"acceptance_passed={record['acceptance_passed']}", + flush=True, + ) + + +def case_main( + build_case: Callable[[str, CaseEnvironment], Case], + case_ids: tuple[str, ...], + argv: list[str] | None = None, +) -> int: + """The model injects its builder; shared code never imports the model.""" + case_id, environment, describe = parse_case_environment(case_ids, argv) + case = build_case(case_id, environment) + if describe: + _, record = prepare_case(case, environment) + print(json.dumps(record, indent=2)) + else: + run_case(case, environment) + return 0 + + if __name__ == "__main__": sys.exit(main()) diff --git a/tests/e2e/test_runner.py b/tests/e2e/test_runner.py index 818e46c61..2bd27bd11 100644 --- a/tests/e2e/test_runner.py +++ b/tests/e2e/test_runner.py @@ -7,37 +7,64 @@ import json import os import signal +import socket import subprocess import sys import threading +import urllib.error +from dataclasses import replace from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import pytest +from tests.e2e import runner +from tests.e2e.case import ( + CLEANUP_PROFILE_STRICT, + CaseEnvironment, + DboConfig, + Mrv2Check, + Phase, +) +from tests.e2e.models.deepseek_v2_lite import cases as v2_cases +from tests.e2e.models.deepseek_v4_flash import cases as dsv4_cases from tests.e2e.models.deepseek_v4_flash import test_async_cam_npu as entrypoint +from tests.e2e.models.deepseek_v4_flash.completions import CompletionTask CONCURRENT_REQUESTS = 10 CANCEL_SCRIPT = """ import json +import subprocess import sys +from dataclasses import replace from pathlib import Path from tests.e2e import runner +from tests.e2e.deployment import ServiceDeployment +from tests.e2e.models.deepseek_v4_flash.cases import build_case, NPU_ASYNC_IDS pid_path = Path(sys.argv.pop(1)) -pids = [] -start = runner.start_process - -def start_service(name, command, env): - process = start(name, [sys.executable, '-c', 'import time; time.sleep(60)'], env) - pids.append(process.pid) - pid_path.write_text(json.dumps(pids)) - return process - -runner.start_process = start_service -runner.wait_for_openai_api = lambda *args: None -sys.exit(runner.main()) + +def start_services(self): + for role in self.resolved.launch_order: + process = subprocess.Popen( + [sys.executable, '-c', 'import time; time.sleep(60)'], + start_new_session=True, + ) + self.processes.append(process) + self.processes_by_role[role] = process + pid_path.write_text(json.dumps([p.pid for p in self.processes])) + +def host_case(case_id, environment): + case = build_case(case_id, environment) + return replace(case, id="host-completion-cancellation", deployment=replace( + case.deployment, cleanup_profile="strict", termination_timeout_s=1, + )) + +# Isolate only device-service startup; execute the real task, signals and +# process cleanup. This host-only run does not qualify an NPU scenario. +ServiceDeployment.start = start_services +sys.exit(runner.case_main(host_case, NPU_ASYNC_IDS)) """ @@ -91,7 +118,11 @@ def do_POST(self): for pid in json.loads(pid_path.read_text()): with pytest.raises(ProcessLookupError): os.kill(pid, 0) - results = json.loads((tmp_path / "responses.json").read_text()) + results = json.loads( + ( + tmp_path / "responses.json" / "evaluation" / "completions.json" + ).read_text() + ) assert len(results) == CONCURRENT_REQUESTS assert all("cancelled" in row["error"] for row in results) finally: @@ -106,3 +137,654 @@ def do_POST(self): server.shutdown() server.server_close() server_thread.join(timeout=5) + + +HOST_SERVICE = r""" +import json +import os +import re +import signal +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +arguments = sys.argv[1:] +model = arguments[arguments.index("--served-model-name") + 1] +with open(os.environ["AFD_HOST_PIDS"], "a") as log: + log.write(str(os.getpid()) + "\n") +Path(os.environ["AFD_HOST_PIDS"]).with_name(f"{model}.json").write_text(json.dumps({ + "arguments": arguments, + "logging_level": os.environ.get("VLLM_LOGGING_LEVEL"), + "worker_shutdown_timeout": os.environ.get("VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS"), +})) +if ( + (os.environ.get("AFD_HOST_OUTCOME") == "startup-failure" + and model.endswith("attention")) + or (os.environ.get("AFD_HOST_OUTCOME") == "peer-startup-failure" + and os.environ.get("AFD_E2E_POD_INDEX") == "1" + and model.endswith("ffn")) +): + raise SystemExit(3) +if os.environ.get("AFD_HOST_OUTCOME") == "cleanup-failure": + signal.signal(signal.SIGTERM, signal.SIG_IGN) +if "--headless" in arguments or model.endswith("-ffn"): + # Connector-driven FFN processes have no HTTP API. + signal.pause() + raise SystemExit(0) +port = int(arguments[arguments.index("--port") + 1]) + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write(json.dumps({"data": []}).encode()) + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + if self.path == "/collective_rpc": + # Host services expose no device execution evidence. + self.send_error(503, "host fixture provides no device evidence") + return + prompt = body["messages"][0]["content"] + operands = re.search(r"Compute (\d+) \+ (\d+)", prompt) + repeated = re.search(r"repeated (\d+) times", prompt) + count = int(repeated[1]) if repeated else 1 + answer = str(int(operands[1]) + int(operands[2])) + if os.environ.get("AFD_HOST_OUTCOME") == "wrong-answer": + answer = "incorrect" + choice = {"message": {"content": " ".join([answer] * count)}, + "finish_reason": "stop"} + response = {"choices": [choice]} + self.send_response(200) + self.end_headers() + self.wfile.write(json.dumps(response).encode()) + + def log_message(self, *args): + pass + +ThreadingHTTPServer(("127.0.0.1", port), Handler).serve_forever() +""" + +HOST_EVALUATOR = r""" +import argparse +import json +import os +import signal +import sys +from pathlib import Path + +parser = argparse.ArgumentParser() +parser.add_argument("--model") +parser.add_argument("--model_args") +parser.add_argument("--tasks") +parser.add_argument("--output_path") +parser.add_argument("--log_samples", action="store_true") +parser.add_argument("--num_fewshot", type=int) +parser.add_argument("--limit", type=int, default=1319) +args = parser.parse_args() +outcome = os.environ.get("AFD_HOST_OUTCOME") +if outcome == "cancel": + print("AFD_HOST_EVALUATOR_WAITING", flush=True) + signal.pause() +if outcome == "evaluator-error": + raise SystemExit(7) +output = Path(args.output_path) +output.mkdir(parents=True, exist_ok=True) +result = { + "results": {"gsm8k": {"exact_match,strict-match": 1.0}}, + "n-samples": {"gsm8k": {"effective": args.limit - (outcome == "partial-result")}}, +} +(output / "results.json").write_text(json.dumps(result)) +(output / "invocation.json").write_text(json.dumps(vars(args))) +print(json.dumps(result), flush=True) +""" + + +@pytest.fixture +def host_case_runtime(monkeypatch, tmp_path): + """Real child-process CLI boundaries; this fixture supplies no device evidence.""" + executable = tmp_path / "host-vllm" + executable.write_text(f"#!{sys.executable}\n" + HOST_SERVICE) + executable.chmod(0o700) + package = tmp_path / "stubs" / "lm_eval" + package.mkdir(parents=True) + (package / "__init__.py").write_text("") + (package / "__main__.py").write_text(HOST_EVALUATOR) + monkeypatch.setenv( + "PYTHONPATH", os.pathsep.join((str(package.parent), os.getcwd())) + ) + monkeypatch.setenv("AFD_HOST_PIDS", str(tmp_path / "service-pids")) + monkeypatch.delenv("AFD_GSM8K_LIMIT", raising=False) + monkeypatch.delenv("AFD_GSM8K_THRESHOLD", raising=False) + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + return CaseEnvironment( + model="/models/host-test", + vllm_bin=str(executable), + device_backend="gpu", + devices=("0", "1", "2", "3"), + output_path=tmp_path / "result", + api_port_base=port, + startup_timeout=10, + ) + + +@pytest.mark.parametrize( + "worker_override", + [None, "75"], + ids=["default-worker-timeout", "explicit-worker-timeout"], +) +def test_async_cam_shutdown_settings_reach_services_and_workers( + monkeypatch, host_case_runtime, worker_override +): + if worker_override is None: + monkeypatch.delenv("VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS", raising=False) + else: + monkeypatch.setenv("VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS", worker_override) + environment = replace( + host_case_runtime, + device_backend="npu", + devices=tuple(str(index) for index in range(16)), + ) + case = dsv4_cases.build_case(dsv4_cases.NPU_ASYNC_IDS[0], environment) + # Host stubs never daemonize. Exercise owned-group cleanup here, not the + # real CAM orphan/pidfd teardown, and keep this result out of device coverage. + case = replace( + case, + id="host-async-shutdown", + deployment=replace(case.deployment, cleanup_profile=CLEANUP_PROFILE_STRICT), + ) + record = runner.run_case(case, environment) + assert record["acceptance_passed"] is True + assert record["cleanup"]["status"] == "passed" + for role in ("attention", "ffn"): + invocation = json.loads( + ( + environment.output_path.parent + / f"{case.served_model_name_prefix}-{role}.json" + ).read_text() + ) + arguments = invocation["arguments"] + assert arguments[arguments.index("--shutdown-timeout") + 1] == "90" + assert invocation["worker_shutdown_timeout"] == (worker_override or "60") + assert record["canonical"]["deployment"]["worker_shutdown_timeout_s"] == 60 + assert record["effective"]["deployment"]["worker_shutdown_timeout_s"] == int( + worker_override or "60" + ) + assert record["effective"]["deployment"]["termination_timeout_s"] == 120 + if worker_override is not None: + assert ( + record["overrides"]["VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS"] + == worker_override + ) + for service in record["processes"]: + with pytest.raises(ProcessLookupError): + os.kill(service["pid"], 0) + + +@pytest.mark.parametrize( + "default_dbo,attention_dbo,check_roles,expected", + [ + pytest.param( + None, + DboConfig(), + ("attention",), + {"attention": (None, None), "ffn": None}, + id="attention-only-native-defaults", + ), + pytest.param( + DboConfig(2, 8), + DboConfig(4, 16), + ("attention", "ffn"), + {"attention": (4, 16), "ffn": (2, 8)}, + id="role-override-and-inherited-default", + ), + pytest.param(None, DboConfig(), ("ffn",), None, id="reject-disabled-role"), + ], +) +def test_role_dbo_reaches_service_cli_before_requiring_device_evidence( + monkeypatch, host_case_runtime, default_dbo, attention_dbo, check_roles, expected +): # long-test: exercise configuration through launched service CLIs and the probe RPC + environment = host_case_runtime + monkeypatch.setenv("VLLM_LOGGING_LEVEL", "INFO") + source = v2_cases.build_case("afd-v2-eager-dp2", environment) + case = replace( + source, + deployment=replace( + source.deployment, + dbo=default_dbo, + attention=replace(source.deployment.attention, dbo=attention_dbo), + ), + phases=( + Phase( + "evaluation", source.phases[0].task, (Mrv2Check(check_roles, dbo=True),) + ), + ), + ) + if expected is None: + with pytest.raises(ValueError, match="DBO evidence requested"): + runner.run_case(case, environment) + assert not (environment.output_path.parent / "service-pids").exists() + return + + # The real probe RPC must be reached; no host fixture can qualify DBO. + with pytest.raises(urllib.error.HTTPError, match="503"): + runner.run_case(case, environment) + record = json.loads((environment.output_path / "run.json").read_text()) + assert record["acceptance_passed"] is False + assert record["cleanup"]["status"] == "passed" + for role, thresholds in expected.items(): + invocation = json.loads( + ( + environment.output_path.parent + / f"{case.served_model_name_prefix}-{role}.json" + ).read_text() + ) + arguments = invocation["arguments"] + assert ("--enable-dbo" in arguments) == (thresholds is not None) + if thresholds is not None: + assert ( + tuple( + int(arguments[arguments.index(flag) + 1]) + if flag in arguments + else None + for flag in ( + "--dbo-decode-token-threshold", + "--dbo-prefill-token-threshold", + ) + ) + == thresholds + ) + assert invocation["logging_level"] == ( + "DEBUG" if thresholds is not None else "INFO" + ) + for service in record["processes"]: + with pytest.raises(ProcessLookupError): + os.kill(service["pid"], 0) + + +@pytest.mark.parametrize( + "outcome,limit,threshold,error,eligible", + [ + ("pass", None, None, None, True), + ("debug-samples", "3", None, None, False), + ("debug-threshold", None, "0.1", None, False), + ("expanded", "28", None, None, True), + ("partial-result", None, None, "expected 7", True), + ("evaluator-error", None, None, "exited with code 7", True), + ("cleanup-failure", None, None, "forced SIGKILL", True), + ("startup-failure", None, None, "exited with 3", True), + ], +) +def test_case_runs_public_clis_and_records_qualification( + monkeypatch, + host_case_runtime, + outcome, + limit, + threshold, + error, + eligible, +): # long-test: integrates service launch, evaluator CLI, failure and resource cleanup + environment = host_case_runtime + monkeypatch.setenv("AFD_HOST_OUTCOME", outcome) + if limit is not None: + monkeypatch.setenv("AFD_GSM8K_LIMIT", limit) + if threshold is not None: + monkeypatch.setenv("AFD_GSM8K_THRESHOLD", threshold) + source_id = "afd-v2-eager-dp2" if outcome == "startup-failure" else "baseline-graph" + source = v2_cases.build_case(source_id, environment) + case = replace( + source, + id="host-harness", + deployment=replace(source.deployment, termination_timeout_s=0.5), + ) + if error: + with pytest.raises(RuntimeError, match=error): + runner.run_case(case, environment) + else: + runner.run_case(case, environment) + + record = json.loads((environment.output_path / "run.json").read_text()) + assert record["configuration"]["eligible"] is eligible + assert record["execution"]["status"] == ("failed" if error else "passed") + assert record["acceptance_passed"] == (eligible and error is None) + assert record["cleanup"]["status"] == ( + "failed" if outcome == "cleanup-failure" else "passed" + ) + for service in record["processes"]: + with pytest.raises(ProcessLookupError): + os.kill(service["pid"], 0) + if error is None: + invocation = json.loads( + (environment.output_path / "evaluation" / "invocation.json").read_text() + ) + settings = record["effective"]["phases"][0]["task"]["settings"] + assert invocation["limit"] == (int(limit) if limit else 7) + assert invocation["num_fewshot"] == 8 + assert "max_gen_toks=512" in invocation["model_args"] + assert settings["threshold"] == ( + float(threshold) if threshold else (0.27 if limit == "28" else 0.25) + ) + + +def test_case_comparison_debug_preserves_floor_without_qualifying( + monkeypatch, host_case_runtime +): + """Exercise the published evaluator CLI with the existing debug policy.""" + monkeypatch.setenv("AFD_GSM8K_LIMIT", "7") + environment = host_case_runtime + case = replace( + v2_cases.build_case("afd-v2-eager-dp2", environment), + id="host-comparison-debug", + ) + record = runner.run_case(case, environment) + canonical = record["canonical"]["phases"][0]["task"]["settings"] + effective = record["effective"]["phases"][0]["task"]["settings"] + assert (canonical["sample_limit"], canonical["threshold"]) == (128, 0.27) + assert (effective["sample_limit"], effective["threshold"]) == (24, 0.25) + assert record["execution"]["status"] == "passed" + assert record["configuration"]["eligible"] is False + assert record["acceptance_passed"] is False + assert record["cleanup"]["status"] == "passed" + invocation = json.loads( + (environment.output_path / "evaluation" / "invocation.json").read_text() + ) + assert invocation["limit"] == 24 + assert "num_concurrent=12" in invocation["model_args"] + for service in record["processes"]: + with pytest.raises(ProcessLookupError): + os.kill(service["pid"], 0) + + +CANCEL_CASE_SCRIPT = """ +import os +from dataclasses import replace +from pathlib import Path +from tests.e2e.case import CaseEnvironment +from tests.e2e.models.deepseek_v2_lite.cases import build_case +from tests.e2e.runner import run_case + +env = CaseEnvironment( + model="/models/host-test", vllm_bin=os.environ["AFD_HOST_VLLM"], + device_backend="gpu", devices=("0", "1", "2", "3"), + output_path=Path(os.environ["AFD_HOST_OUTPUT"]), + api_port_base=int(os.environ["AFD_HOST_PORT"]), startup_timeout=10, +) +case = replace(build_case("baseline-graph", env), id="host-harness") +run_case(case, env) +""" + + +@pytest.mark.parametrize("signum", [signal.SIGTERM, signal.SIGINT]) +def test_case_cancellation_reaps_evaluator_and_service( + monkeypatch, + host_case_runtime, + signum, +): # long-test: cancellation crosses evaluator, runner and owned service process groups + environment = host_case_runtime + monkeypatch.setenv("AFD_HOST_OUTCOME", "cancel") + monkeypatch.setenv("AFD_HOST_VLLM", environment.vllm_bin) + monkeypatch.setenv("AFD_HOST_OUTPUT", str(environment.output_path)) + monkeypatch.setenv("AFD_HOST_PORT", str(environment.api_port_base)) + waiting = threading.Event() + process = subprocess.Popen( + [sys.executable, "-c", CANCEL_CASE_SCRIPT], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) + lines = [] + + def read_output(): + assert process.stdout is not None + for line in process.stdout: + lines.append(line) + if "AFD_HOST_EVALUATOR_WAITING" in line: + waiting.set() + + thread = threading.Thread(target=read_output, daemon=True) + thread.start() + try: + assert waiting.wait(timeout=15), "".join(lines) + process.send_signal(signum) + process.wait(timeout=15) + thread.join(timeout=5) + assert process.returncode == 128 + signum, "".join(lines) + record = json.loads((environment.output_path / "run.json").read_text()) + assert record["execution"]["status"] == "cancelled" + assert record["cleanup"]["status"] == "passed" + assert record["acceptance_passed"] is False + for service in record["processes"]: + with pytest.raises(ProcessLookupError): + os.kill(service["pid"], 0) + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=5) + pid_path = environment.output_path.parent / "service-pids" + if pid_path.exists(): + for pid in pid_path.read_text().splitlines(): + with contextlib.suppress(ProcessLookupError): + os.killpg(int(pid), signal.SIGKILL) + + +@pytest.mark.parametrize("outcome", ["pass", "wrong-answer"]) +def test_case_sequences_model_tasks_and_gsm8k( + monkeypatch, host_case_runtime, outcome +): # long-test: real concurrent HTTP task, evaluator subprocess and cleanup + environment = host_case_runtime + monkeypatch.setenv("AFD_HOST_OUTCOME", outcome) + source = v2_cases.build_case("baseline-graph", environment) + case = replace( + source, + id="host-model-tasks", + phases=( + Phase("first", CompletionTask(answer_repetitions=8)), + Phase( + "second", CompletionTask(first_request_index=10, answer_repetitions=8) + ), + replace(source.phases[0], name="gsm8k"), + ), + ) + if outcome == "wrong-answer": + with pytest.raises(RuntimeError, match="wrong answer"): + runner.run_case(case, environment) + else: + runner.run_case(case, environment) + record = json.loads((environment.output_path / "run.json").read_text()) + assert record["cleanup"]["status"] == "passed" + assert record["acceptance_passed"] is (outcome == "pass") + if outcome == "pass": + assert [phase["name"] for phase in record["phases"]] == [ + "first", + "second", + "gsm8k", + ] + for name, start in (("first", 0), ("second", 10)): + results = json.loads( + (environment.output_path / name / "completions.json").read_text() + ) + assert [item["index"] for item in results] == list(range(start, start + 10)) + assert all("error" not in item for item in results) + assert record["phases"][-1]["result"]["sample_count"] == 7 + else: + assert [phase["name"] for phase in record["phases"]] == ["first"] + assert not (environment.output_path / "second").exists() + assert not (environment.output_path / "gsm8k").exists() + + +@pytest.fixture +def multi_pod_runtime(host_case_runtime, tmp_path): + """Real TCPStore peers and owned service/evaluator CLIs; no device simulation.""" + + def run(model, scenario, layout, outcome): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + store_port = listener.getsockname()[1] + environment = host_case_runtime + children = [] + threads = [] + lines: list[list[str]] = [[], []] + waiting = threading.Event() + other_run = None + if model == "deepseek_v2_lite" and layout == "2A0F,0A2F" and outcome == "pass": + # A different marker is not proof of a stale process or ownership + # of this deployment's devices/ports on a shared host. + other_run = subprocess.Popen( + [sys.executable, "-c", "import signal; signal.pause()"], + env={**os.environ, "AFD_E2E_RUN_ID": "host-independent-run"}, + start_new_session=True, + ) + + def read_output(index, child): + for line in child.stdout: + lines[index].append(line) + if "AFD_HOST_EVALUATOR_WAITING" in line: + waiting.set() + + try: + for index in range(2): + child_env = os.environ.copy() + child_env.update( + AFD_E2E_POD_INDEX=str(index), + AFD_HOST_OUTCOME=outcome, + AFD_HOST_PIDS=str(tmp_path / f"pod-{index}-pids"), + ) + # Only the evaluator receives a weakened override. Its debug + # verdict and effective workload must reach the other pod too. + if outcome == "debug" and index == 0: + child_env["AFD_GSM8K_LIMIT"] = "3" + command = [ + sys.executable, + "-m", + f"tests.e2e.models.{model}.cases", + "multi-pod", + "--scenario", + scenario, + "--model", + environment.model, + "--vllm-bin", + environment.vllm_bin, + "--device-backend", + "gpu", + "--devices", + "0,1", + "--output-path", + str(tmp_path / f"pod-{index}"), + "--api-port-base", + str(environment.api_port_base), + "--pod-layout", + layout, + "--run-id", + tmp_path.name, + "--store-host", + "127.0.0.1", + "--store-port", + str(store_port), + "--pod-addresses", + "127.0.0.1,127.0.0.1", + "--launch-timeout", + "15", + "--serving-timeout", + "15", + "--verdict-timeout", + "20", + ] + child = subprocess.Popen( + command, + env=child_env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) + children.append(child) + thread = threading.Thread( + target=read_output, args=(index, child), daemon=True + ) + thread.start() + threads.append(thread) + if outcome == "cancel": + assert waiting.wait(timeout=25), "".join(sum(lines, [])) + children[0].send_signal(signal.SIGTERM) + for child in children: + child.wait(timeout=45) + for thread in threads: + thread.join(timeout=5) + for index in range(2): + assert (tmp_path / f"pod-{index}" / "run.json").exists(), "".join( + sum(lines, []) + ) + records = [ + json.loads((tmp_path / f"pod-{index}" / "run.json").read_text()) + for index in range(2) + ] + # Assert before emergency fixture cleanup so it cannot hide a leak. + for record in records: + for service in record["processes"]: + with pytest.raises(ProcessLookupError): + os.kill(service["pid"], 0) + if other_run is not None: + assert other_run.poll() is None + return children, records, lines + finally: + for child in children: + if child.poll() is None: + os.killpg(child.pid, signal.SIGKILL) + child.wait(timeout=5) + if other_run is not None: + other_run.terminate() + other_run.wait(timeout=5) + for index in range(2): + pids = tmp_path / f"pod-{index}-pids" + if pids.exists(): + for pid in pids.read_text().splitlines(): + with contextlib.suppress(ProcessLookupError): + os.killpg(int(pid), signal.SIGKILL) + + return run + + +@pytest.mark.parametrize( + "model,scenario,layout,outcome,samples", + [ + ("deepseek_v2_lite", "afd-graph-2a2f", "2A0F,0A2F", "pass", 7), + ("deepseek_v2_lite", "afd-graph-2a2f", "1A1F,1A1F", "pass", 7), + ("qwen3_moe", "afd-deepep-m2n-graph-2a2f", "2A0F,0A2F", "pass", 100), + ("qwen3_moe", "afd-deepep-m2n-graph-2a2f", "1A1F,1A1F", "pass", 100), + ("deepseek_v2_lite", "afd-graph-2a2f", "2A0F,0A2F", "debug", 3), + ( + "deepseek_v2_lite", + "afd-graph-2a2f", + "1A1F,1A1F", + "peer-startup-failure", + None, + ), + ("deepseek_v2_lite", "afd-graph-2a2f", "2A0F,0A2F", "evaluator-error", None), + ("deepseek_v2_lite", "afd-graph-2a2f", "2A0F,0A2F", "cancel", None), + ], +) +def test_multi_pod_case_coordinates_workload_and_owned_cleanup( + multi_pod_runtime, model, scenario, layout, outcome, samples +): + children, records, lines = multi_pod_runtime(model, scenario, layout, outcome) + succeeded = samples is not None + for child, record, log in zip(children, records, lines, strict=True): + assert (child.returncode == 0) is succeeded, "".join(log) + assert record["cleanup"]["status"] == "passed" + assert record["acceptance_passed"] is (outcome == "pass") + assert record["coverage"] == ["task_result", "local_liveness", "local_cleanup"] + if succeeded: + assert record["execution"]["status"] == "passed" + assert record["configuration"]["eligible"] is (outcome == "pass") + assert record["phases"][0]["result"]["sample_count"] == samples + assert record["phases"][0]["checks"] == [] + if layout == "1A1F,1A1F": + headless = records[1]["commands"] + assert all("--headless" in command for command in headless.values()) + assert all("--port" not in command for command in headless.values()) + if outcome == "cancel": + assert children[0].returncode == 128 + signal.SIGTERM + assert records[0]["execution"]["status"] == "cancelled" From 66203361bdece97ec3cf957ee00260da462bb9e2 Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Fri, 13 Nov 2026 15:45:36 +0800 Subject: [PATCH 41/42] [NPU] Enable native target and DSpark draft Attention graphs (#545) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Purpose Enable native decode graphs for Async CAM DeepSeek-V4 target Attention and DSpark draft Attention. Reuse the vLLM/vLLM-Ascend graph managers and install AFD metadata at capture/replay boundaries for remote expert calls. ## Issue Related to #499; complements the F-side FULL support merged in #532. ## Scope - Target native graphs with an eager or native-graph DSpark draft; matching target/draft TP and independent F execution settings. - Preserve Attention-local FlashComm1/SP and native PD. P uses the upstream complete draft model and existing KV transfer; no new PD protocol is added. - Accept native `FULL`/`FULL_DECODE_ONLY` resolution. Remove graph-specific restrictions on compilation mode and model-free speculation; existing model-based draft/MTP checks remain. Device qualification uses mode 0. - Keep DBO excluded and draft graph dependent on target graph, as native target capture owns draft capture. Unequal target/draft TP, native kernels, quantization changes and lifecycle work remain outside this PR. ## Implementation Notes - Stage target metadata after native capture input preparation and before replay, preserving real and padded token counts. Send synchronous DP control only when a connector has a control plane; Async CAM has none. - Scope draft metadata to its own descriptor and query width. Reuse the existing physical KV block-size adapter and restore temporary methods/context on normal and exceptional exits. - Native managers retain ownership of graph buffers, buckets, attention/KV updates, sampling and fallback. No new ModelRunner, F executor or communication protocol is introduced. Preserve main's #500 routing fix. - Integrate on main `e58fdec5876782b9c6a03683416611deea42af92`, retaining its MegaMoE path and FFN two-stream removal. No F-runner, connector or native-operator changes are included in the PR diff. ## Test Plan / Test Result The conflict resolution at `fb4095d` merges main `26d8151` and preserves both Async CAM Attention graph validation and the synchronous CAMP2p Attention-side routing constraints from #503. Focused CPU checks passed **123 cases**, with **35 dependency-gated skips**. Changed-file pre-commit hooks and `git diff --check` passed. NPU results below are prior evidence; no new device run was performed for this merge. After the earlier main synchronization at `7fca98e`, focused Async CAM connector/session CPU checks pass **40 cases**; **3 graph test modules** are skipped because the host lacks native dependencies. Documentation hooks, local links and diff hygiene pass. Conflict resolutions retain the Graph capability descriptions and main's documentation reorganization. Attention graph implementation files are unchanged. The earlier **112 passed, 33 dependency-gated skips** and Ruff results at `822779f` are reused. The merge retains the graph control-plane test fields and main's `attention_step` connector context. Six pinned-runtime feature-validation checks and the earlier metadata, restoration, signature and device results below are reused pre-merge evidence. No new NPU experiment was run for conflict resolution. Configuration tests continue to admit PD/SP/native options and retain the model-based draft boundary and DBO rejection. Retained NPU comparisons used the pre-merge main-integrated execution code (`5afe9dd89314a36850cbb19e9a4e40d9706c3b83`, with the admission gates removed): | Combination | Fixed functional requests | Greedy output vs same-config eager | Successful target / draft replays per A rank | Padded draft replays per A rank | | --- | ---: | --- | ---: | ---: | | SP / FlashComm1 | 13 | All token-for-token identical | 73 / 86 | 86 | | Native PD + SP / FlashComm1 | 10 | All token-for-token identical | 82 / 83 | 83 | Actual SP flags were checked on both target and draft. PD used `MooncakeHybridConnector` with native full-draft P TP4. Cases cover one-token, 127/128/129-token and chunked 513-token prompts, concurrency and request reuse. PD eager/graph both recorded 405 drafted and 263 accepted tokens. In the SP comparison, the totals were 475/347 and 465/345 respectively; outputs still matched exactly, and concurrent scheduling was not fixed between arms. The pinned native configuration resolver was also executed directly: requested `FULL` resolves to `FULL_DECODE_ONLY` for DSV4's `UNIFORM_BATCH` backend. This was a configuration check, not another model experiment. Compilation mode 3 and model-free speculation admission were checked at the configuration seam only; their device execution is not qualified here. One formal measurement per arm used W4A8, A DP1/TP4, F EP4/TP1 FULL, five speculative tokens, 32 requests, 128 input and 128 output tokens, concurrency 4 and four client warmups. Each arm completed 32/32 requests and 4,096 output tokens. Functional replay probes were removed before timing; their counters remained unchanged afterward. No profiling ran during measurement. | Combination | Mean TPOT, eager → dual graph | Output throughput, eager → dual graph | Mean TTFT, eager → dual graph | | --- | ---: | ---: | ---: | | SP / FlashComm1 | 38.40 → 14.83 ms | 86.89 → 221.53 token/s | 412.96 → 254.95 ms | | Native PD + SP / FlashComm1 | 36.15 → 12.22 ms | 83.85 → 226.11 token/s | 1041.01 → 531.84 ms | Retained earlier non-PD, SP-off evidence: nine finite requests matched the reference token-for-token, with 51 target and 60 draft replays per A rank. Its single performance comparison measured TPOT 32.20 → 14.63 ms, throughput 102.62 → 226.70 token/s and TTFT 393.14 → 234.11 ms. That earlier measurement used `140369b873c34779d0d53a8af82aced3c47484eb` on an older main baseline; it was reused rather than rerun. Do not compare absolute numbers across the separate deployment experiments or attribute the combined gain to one graph. Pinned stack: vLLM `5df63a7203cdb9328d824a27b682e77c7b9928be`, vLLM-Ascend `bd42eea01d8312c4df469fabaed45f77a12df4d3`, CANN 9.2 and torch-npu 2.10.0.post7 on Ascend 910C/A3. Existing native binaries were reused unchanged. MegaMoE with A graphs, W8A8 A graphs, other KV connectors/topologies, GPU execution, DBO and task-level model accuracy are not newly qualified. Successful cleanup of owned processes is not a new graceful-shutdown qualification. ## Docs Impact Update the existing Attention, model/platform, Async CAM/FFN and DSpark recipe contracts. Distinguish actual implementation dependencies from device validation coverage. Experiment scripts, probes, raw logs and private evidence stay outside the patch. ---
Essential PR Checklist - [x] Purpose is clear and linked to public context. - [x] Scope is bounded. - [x] Compatibility with the pinned vLLM 0.32 / Ascend pair is considered. - [x] No upstream source checkouts are changed. - [x] Existing plugin classes and native graph managers are reused. - [x] Scoped native hooks are restored, documented and signature-checked. - [x] Runtime imports retain NPU boundaries and dependency guards. - [x] Executed, reused and unverified validation paths are distinguished. - [x] Documentation impact is stated.
--------- Signed-off-by: jiaran-king Signed-off-by: Zhou ziheng --- afd_plugin/compat/npu/feature_validation.py | 47 +++--- .../v1/worker/attention_model_runner_v2.py | 13 +- .../worker/npu/attention_model_runner_v2.py | 18 ++- afd_plugin/v1/worker/npu/dspark_graph.py | 149 ++++++++++++++++++ afd_plugin/v1/worker/npu/dspark_speculator.py | 108 +++++++++---- afd_plugin/validation.py | 27 +++- docs/design/module/attention_runtime.md | 14 +- docs/design/module/execution_platforms.md | 37 +++-- docs/design/module/model_integration.md | 10 +- docs/design/npu/async_ffn_runtime.md | 14 +- docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md | 49 +++++- recipe/README.md | 2 +- .../deepseek_v4_flash/README.md | 49 ++++-- .../compat/npu/test_feature_validation.py | 62 +++++++- tests/unit/compat/patches/test_signatures.py | 36 +++++ tests/unit/config/test_validation.py | 102 +++++++++++- tests/unit/fakes/npu_attention.py | 54 +++++++ tests/unit/fakes/vllm_config.py | 33 +++- tests/unit/v1/worker/npu/test_dspark_graph.py | 105 ++++++++++++ .../v1/worker/test_npu_model_runner_v2.py | 76 ++++++++- 20 files changed, 875 insertions(+), 130 deletions(-) create mode 100644 afd_plugin/v1/worker/npu/dspark_graph.py create mode 100644 tests/unit/fakes/npu_attention.py create mode 100644 tests/unit/v1/worker/npu/test_dspark_graph.py diff --git a/afd_plugin/compat/npu/feature_validation.py b/afd_plugin/compat/npu/feature_validation.py index 73edc9990..80f8de270 100644 --- a/afd_plugin/compat/npu/feature_validation.py +++ b/afd_plugin/compat/npu/feature_validation.py @@ -14,7 +14,11 @@ is_afd_async_dp, parse_afd_config, ) -from afd_plugin.validation import cam_receive_capacity, requested_ffn_mode +from afd_plugin.validation import ( + cam_receive_capacity, + requested_ffn_mode, + validate_async_cam_attention_graph, +) if TYPE_CHECKING: from vllm.config import VllmConfig @@ -231,14 +235,13 @@ def _fail_if_unsupported_npu_afd_async_features( ) if not bool(vllm_config.model_config.enforce_eager): if afd_config.is_attention_server: - raise RuntimeError( - "CAMAsyncAFDConnector supports only eager Attention execution" - ) - requested_ffn_mode(vllm_config) - if not vllm_config.use_v2_model_runner: - raise RuntimeError("Async FFN FULL requires MRV2") - if parallel_config.tensor_parallel_size != 1: - raise RuntimeError("Async FFN FULL requires F TP=1") + validate_async_cam_attention_graph(vllm_config) + else: + requested_ffn_mode(vllm_config) + if not vllm_config.use_v2_model_runner: + raise RuntimeError("Async FFN FULL requires MRV2") + if parallel_config.tensor_parallel_size != 1: + raise RuntimeError("Async FFN FULL requires F TP=1") if bool(parallel_config.enable_dbo) or bool(parallel_config.use_ubatching): if not afd_config.is_attention_server: raise RuntimeError("Async CAM DBO/ubatching requires the Attention role") @@ -262,26 +265,32 @@ def _fail_if_unsupported_npu_afd_async_features( raise RuntimeError("AFD DSpark supports only DeepSeek-V4 on NPU") # The shared MRV2 validator owns PP/CP and static EP constraints. # Keep only the additional target/draft restrictions here. - # F may replay its expert transaction in FULL; Attention and native - # draft proposal execution stay eager. + # Both graphs use the native decode path. Draft capture is owned + # by native target capture, so draft-only graph mode is unsupported. + # F keeps its independent TP1 expert execution. if ( - ( - afd_config.is_attention_server - and not vllm_config.model_config.enforce_eager + speculative_config.draft_parallel_config.tensor_parallel_size + != parallel_config.tensor_parallel_size + or ( + afd_config.is_ffn_server + and parallel_config.tensor_parallel_size != 1 ) - or not speculative_config.enforce_eager - or parallel_config.tensor_parallel_size != 1 - or speculative_config.draft_parallel_config.tensor_parallel_size != 1 ): raise RuntimeError( - "AFD DSpark requires eager Attention/draft and target/draft TP=1" + "AFD DSpark requires matching target/draft TP and F TP=1" ) + if ( + afd_config.is_attention_server + and not speculative_config.enforce_eager + and vllm_config.model_config.enforce_eager + ): + raise RuntimeError("DSpark draft Graph requires target Attention graph") elif afd_config.is_attention_server and ( speculative_config.use_eagle() or speculative_config.uses_draft_model() ): raise RuntimeError( "CAMAsyncAFDConnector does not support native draft models or MTP; " - "only DSV4 DSpark eager is supported", + "only DSV4 DSpark is supported", ) factor_value = os.environ.get("BATCH_SIZE_FACTOR", "1") diff --git a/afd_plugin/v1/worker/attention_model_runner_v2.py b/afd_plugin/v1/worker/attention_model_runner_v2.py index 82eefc156..3ff495685 100644 --- a/afd_plugin/v1/worker/attention_model_runner_v2.py +++ b/afd_plugin/v1/worker/attention_model_runner_v2.py @@ -197,8 +197,8 @@ def _use_afd_capture_input_preparation( # Patch reason: native ModelCudaGraphManager.capture calls this exact # module symbol once before each warmup/formal-capture forward. - # Patch functionality: preserve native preparation and publish the - # matching AFD event before the forward starts. + # Patch functionality: preserve native preparation and stage the matching + # AFD metadata before the forward; synchronous connectors also send control. # Signature: matches vLLM v0.32.0 prepare_inputs_to_capture exactly, # including the trailing pcp_manager parameter. # Upstream source: vllm/v1/worker/gpu/cudagraph_utils.py, @@ -237,10 +237,11 @@ def prepare_inputs_to_capture( None, int(desc.num_tokens), ) - runner.send_dp_metadata( - runner.build_capture_dp_metadata(int(desc.num_tokens)), - None, - ) + if runner.connector.control_plane is not None: + runner.send_dp_metadata( + runner.build_capture_dp_metadata(int(desc.num_tokens)), + None, + ) runner._afd_suppress_metadata_send = True return attention_state # ### PATCH END: stage one exact AFD capture event. diff --git a/afd_plugin/v1/worker/npu/attention_model_runner_v2.py b/afd_plugin/v1/worker/npu/attention_model_runner_v2.py index 1c2b8214d..8f791cca7 100644 --- a/afd_plugin/v1/worker/npu/attention_model_runner_v2.py +++ b/afd_plugin/v1/worker/npu/attention_model_runner_v2.py @@ -82,10 +82,10 @@ def _use_afd_fullgraph_replay_hook( original_run_fullgraph = manager.run_fullgraph manager_state[_AFD_FULLGRAPH_HOOK_MARKER] = True - # Patch reason: native FULL replay bypasses ForwardContext creation, so the - # execute-scoped AFD provider cannot publish runtime control. - # Patch functionality: wrap only this manager instance and publish one - # ordinary padded control payload immediately before each native replay. + # Patch reason: native FULL replay bypasses the ordinary forward, so its + # attention-parameter update needs fresh AFD real/padded token metadata. + # Patch functionality: stage that metadata on this manager's replay scope; + # synchronous connectors also publish one padded control payload. # Signature: matches vLLM v0.32.0 ModelCudaGraphManager.run_fullgraph exactly. # Upstream source: vllm/v1/worker/gpu/cudagraph_utils.py, # ModelCudaGraphManager.run_fullgraph; commit @@ -119,10 +119,11 @@ def run_fullgraph( runner._is_warmup = False runner._afd_is_graph_capturing = False runner._afd_is_graph_replaying = True - runner.send_dp_metadata( - runner.build_capture_dp_metadata(padded_tokens), - None, - ) + if runner.connector.control_plane is not None: + runner.send_dp_metadata( + runner.build_capture_dp_metadata(padded_tokens), + None, + ) result = original_run_fullgraph(desc) finally: runner._afd_is_graph_replaying = previous_is_graph_replaying @@ -269,6 +270,7 @@ def __init__( self.vllm_config, self.device, afd_context=self._afd_draft_context, + afd_runner=self, ) self.speculator.update_stream = self.update_stream except BaseException: diff --git a/afd_plugin/v1/worker/npu/dspark_graph.py b/afd_plugin/v1/worker/npu/dspark_graph.py new file mode 100644 index 000000000..e17e1990e --- /dev/null +++ b/afd_plugin/v1/worker/npu/dspark_graph.py @@ -0,0 +1,149 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Install AFD metadata around native draft graph capture and replay.""" + +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from types import MethodType +from typing import TYPE_CHECKING + +import torch +from vllm.config import CUDAGraphMode +from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.worker.gpu.block_table import BlockTables +from vllm.v1.worker.gpu.cudagraph_utils import ( + AttentionState, + BatchExecutionDescriptor, +) +from vllm.v1.worker.gpu.input_batch import InputBuffers +from vllm.v1.worker.gpu.spec_decode.dflash import cudagraph as dflash_cudagraph +from vllm.v1.worker.utils import AttentionGroup +from vllm_ascend.worker.v2.spec_decode.dflash.aclgraph import DFlashAclGraphManager + +from afd_plugin.v1.worker.attention_model_runner_v2 import _AFDCaptureEventTracker + +if TYPE_CHECKING: + from afd_plugin.v1.worker.npu.attention_model_runner_v2 import ( + AFDNPUAttentionModelRunnerV2, + ) + + +@contextmanager +def use_afd_draft_capture( + runner: AFDNPUAttentionModelRunnerV2, + manager: DFlashAclGraphManager, +) -> Iterator[None]: + """Associate every native draft capture forward with its own AFD batch.""" + descs = manager._capture_descs.get(CUDAGraphMode.FULL, [])[ + : manager._max_full_descs_to_capture + ] + if not runner.speculative_config.enforce_eager and not descs: + raise RuntimeError("AFD DSpark draft FULL has no native capture descriptors") + tracker = _AFDCaptureEventTracker(descs) + original_prepare = dflash_cudagraph._prepare_dflash_inputs_to_capture + + # Upstream: vLLM 5df63a72, dflash/cudagraph.py; Ascend bd42eea0 delegates + # draft capture here. Signature matches the native preparation function. + # Native input/KV preparation remains in the saved function; only AFD + # metadata is added. Remove when draft capture exposes a plugin callback. + def prepare_inputs( + num_reqs: int, + num_tokens: int, + input_buffers: InputBuffers, + block_tables: BlockTables, + attn_groups: list[list[AttentionGroup]], + kv_cache_config: KVCacheConfig, + max_model_len: int, + skip_attn: bool, + causal: bool | Mapping[int, bool], + ) -> AttentionState: + # ### PATCH START: stage descriptor-local draft metadata. + state = original_prepare( + num_reqs, + num_tokens, + input_buffers, + block_tables, + attn_groups, + kv_cache_config, + max_model_len, + skip_attn, + causal, + ) + desc, is_warmup = tracker.consume(num_reqs, num_tokens) + runner._afd_pending_metadata = runner.build_afd_metadata(None, desc.num_tokens) + runner._afd_suppress_metadata_send = True + runner._is_warmup = is_warmup + runner._afd_is_graph_capturing = not is_warmup + runner._afd_is_graph_replaying = False + return state + # ### PATCH END: stage descriptor-local draft metadata. + + previous_capturing = runner._afd_is_graph_capturing + previous_replaying = runner._afd_is_graph_replaying + dflash_cudagraph._prepare_dflash_inputs_to_capture = prepare_inputs + try: + yield + tracker.assert_complete() + finally: + dflash_cudagraph._prepare_dflash_inputs_to_capture = original_prepare + runner._afd_is_graph_capturing = previous_capturing + runner._afd_is_graph_replaying = previous_replaying + + +@contextmanager +def use_afd_draft_replay( + runner: AFDNPUAttentionModelRunnerV2, + manager: DFlashAclGraphManager, + real_tokens: int, +) -> Iterator[None]: + """Preserve native replay while staging the draft's real and padded rows.""" + state = manager.__dict__ + had_override = "run_fullgraph" in state + previous_override = state.get("run_fullgraph") + original_replay = manager.run_fullgraph + + # Upstream: Ascend bd42eea0 DFlashAclGraphManager.run_fullgraph. + # Signature matches upstream. Delegate graph replay and attention updates; + # add only AFD's model-local metadata. Remove when the native manager has + # an instance-local pre-replay callback. + def run_fullgraph( + self, + desc: BatchExecutionDescriptor, + ) -> torch.Tensor | tuple[torch.Tensor, list[torch.Tensor]]: + # ### PATCH START: install draft-local replay metadata. + previous = ( + runner._afd_pending_metadata, + runner._afd_suppress_metadata_send, + runner._is_warmup, + runner._afd_is_graph_capturing, + runner._afd_is_graph_replaying, + ) + try: + metadata = runner.build_afd_metadata(None, real_tokens) + metadata.tokens_lens = [desc.num_tokens] + runner._afd_pending_metadata = metadata + runner._afd_suppress_metadata_send = True + runner._is_warmup = False + runner._afd_is_graph_capturing = False + runner._afd_is_graph_replaying = True + return original_replay(desc) + finally: + ( + runner._afd_pending_metadata, + runner._afd_suppress_metadata_send, + runner._is_warmup, + runner._afd_is_graph_capturing, + runner._afd_is_graph_replaying, + ) = previous + # ### PATCH END: install draft-local replay metadata. + + try: + manager.run_fullgraph = MethodType(run_fullgraph, manager) + yield + finally: + if had_override: + manager.run_fullgraph = previous_override + else: + del manager.run_fullgraph diff --git a/afd_plugin/v1/worker/npu/dspark_speculator.py b/afd_plugin/v1/worker/npu/dspark_speculator.py index 0de54a59b..41940c87e 100644 --- a/afd_plugin/v1/worker/npu/dspark_speculator.py +++ b/afd_plugin/v1/worker/npu/dspark_speculator.py @@ -4,19 +4,30 @@ from __future__ import annotations -from collections.abc import Callable -from contextlib import AbstractContextManager -from typing import Any +from collections.abc import Callable, Iterator +from contextlib import AbstractContextManager, contextmanager, nullcontext +from typing import TYPE_CHECKING, Any, cast import torch from vllm.config import VllmConfig from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.spec_decode.dflash import speculator as dflash_speculator +from vllm_ascend.worker.v2.spec_decode.dflash.aclgraph import DFlashAclGraphManager from vllm_ascend.worker.v2.spec_decode.dflash.speculator import ( prepare_dflash_inputs_factory, ) from vllm_ascend.worker.v2.spec_decode.dspark.speculator import AscendDSparkSpeculator +from afd_plugin.v1.worker.npu.dspark_graph import ( + use_afd_draft_capture, + use_afd_draft_replay, +) + +if TYPE_CHECKING: + from afd_plugin.v1.worker.npu.attention_model_runner_v2 import ( + AFDNPUAttentionModelRunnerV2, + ) + class AFDDSparkSpeculator(AscendDSparkSpeculator): def __init__( @@ -25,9 +36,37 @@ def __init__( device: torch.device, *, afd_context: Callable[[bool, bool], AbstractContextManager[None]], + afd_runner: AFDNPUAttentionModelRunnerV2, ) -> None: super().__init__(vllm_config, device) self.afd_context = afd_context + self.afd_runner = afd_runner + + @contextmanager + def _physical_input_preparation(self, *, enabled: bool = True) -> Iterator[None]: + original_prepare = dflash_speculator.prepare_dflash_inputs + if enabled: + dflash_speculator.prepare_dflash_inputs = self._afd_prepare_inputs + try: + yield + finally: + dflash_speculator.prepare_dflash_inputs = original_prepare + + # Upstream: vLLM 5df63a72 DFlashSpeculator.capture, inherited by Ascend + # bd42eea0 DSpark. Signature matches. Delegate native context/KV and Markov + # capture, adding draft-local AFD metadata and the physical block size. + # Remove when native capture exposes metadata and physical-size hooks. + def capture(self) -> None: + # ### PATCH START: scope native draft capture. + assert self.query_cudagraph_manager is not None + manager = cast(DFlashAclGraphManager, self.query_cudagraph_manager) + with ( + self.afd_context(True, False), + self._physical_input_preparation(), + use_afd_draft_capture(self.afd_runner, manager), + ): + super().capture() + # ### PATCH END: scope native draft capture. # Upstream: Ascend bd42eea0, AscendDSparkSpeculator.set_attn. # Its input factory captures the global cache_config.block_size, which can @@ -88,30 +127,41 @@ def propose( num_speculative_tokens: int | None = None, ) -> torch.Tensor: # ### PATCH START: scope only the native draft proposal - prepare_inputs = dflash_speculator.prepare_dflash_inputs - if not (dummy_run and skip_attn_for_dummy_run): - dflash_speculator.prepare_dflash_inputs = self._afd_prepare_inputs - try: - with self.afd_context(dummy_run, is_profile): - return super().propose( - input_batch, - attn_metadata, - slot_mappings, - last_hidden_states, - aux_hidden_states, - num_sampled, - num_rejected, - last_sampled, - next_prefill_tokens, - temperature, - seeds, - None, - dummy_run, - skip_attn_for_dummy_run, - mm_inputs, - is_profile=is_profile, - num_speculative_tokens=num_speculative_tokens, - ) - finally: - dflash_speculator.prepare_dflash_inputs = prepare_inputs + manager = self.query_cudagraph_manager + # Native memory profiling runs before KV/graph-manager initialization. + replay_scope = ( + use_afd_draft_replay( + self.afd_runner, + cast(DFlashAclGraphManager, manager), + input_batch.num_reqs * self.num_query_per_req, + ) + if manager is not None + else nullcontext() + ) + with ( + self.afd_context(dummy_run, is_profile), + self._physical_input_preparation( + enabled=not (dummy_run and skip_attn_for_dummy_run) + ), + replay_scope, + ): + return super().propose( + input_batch, + attn_metadata, + slot_mappings, + last_hidden_states, + aux_hidden_states, + num_sampled, + num_rejected, + last_sampled, + next_prefill_tokens, + temperature, + seeds, + None, + dummy_run, + skip_attn_for_dummy_run, + mm_inputs, + is_profile=is_profile, + num_speculative_tokens=num_speculative_tokens, + ) # ### PATCH END: scope only the native draft proposal diff --git a/afd_plugin/validation.py b/afd_plugin/validation.py index 7d586b3a5..b2ef9b0eb 100644 --- a/afd_plugin/validation.py +++ b/afd_plugin/validation.py @@ -89,6 +89,25 @@ def validate_graph_capacity( raise ValueError("Async CAM graph source rows exceed the uint16 protocol bound") +def validate_async_cam_attention_graph(vllm_config: VllmConfig) -> None: + """Validate AFD dependencies for native Async CAM Attention graphs.""" + model = vllm_config.model_config + parallel = vllm_config.parallel_config + if ( + not vllm_config.use_v2_model_runner + or model.hf_text_config.model_type != "deepseek_v4" + ): + raise RuntimeError("Async CAM Attention graph requires MRV2 DeepSeek-V4") + if parallel.enable_dbo or parallel.use_ubatching: + raise RuntimeError("Async CAM Attention graph does not support DBO/ubatching") + if ( + parallel.pipeline_parallel_size, + parallel.prefill_context_parallel_size, + parallel.decode_context_parallel_size, + ) != (1, 1, 1): + raise RuntimeError("Async CAM Attention graph does not support PP or CP") + + def validate_gpu_model_runner_v2_config( vllm_config: VllmConfig, *, @@ -207,11 +226,9 @@ def validate_npu_model_runner_v2_config( ) if not vllm_config.model_config.enforce_eager: if expected_role == "attention": - raise RuntimeError( - "AFD NPU ModelRunnerV2 Async CAM requires eager execution " - "on Attention" - ) - requested_ffn_mode(vllm_config) + validate_async_cam_attention_graph(vllm_config) + else: + requested_ffn_mode(vllm_config) elif afd_config.connector != CAMP2P_CONNECTOR: raise RuntimeError( "AFD NPU ModelRunnerV2 requires either synchronous CAMP2pAFDConnector " diff --git a/docs/design/module/attention_runtime.md b/docs/design/module/attention_runtime.md index ae6b48f28..57188eeda 100644 --- a/docs/design/module/attention_runtime.md +++ b/docs/design/module/attention_runtime.md @@ -13,6 +13,7 @@ primary_code_paths: - "afd_plugin/v1/worker/npu/attention_model_runner_v2.py" - "afd_plugin/v1/worker/npu/attention_worker.py" - "afd_plugin/v1/worker/npu/dspark_speculator.py" + - "afd_plugin/v1/worker/npu/dspark_graph.py" related_code_paths: - "afd_plugin/connectors/**" - "afd_plugin/model_executor/**" @@ -24,6 +25,7 @@ depends_on: - "execution_platforms.md" - "compatibility_and_patches.md" validation_paths: + - "tests/unit/v1/worker/npu/test_dspark_graph.py" - "tests/unit/v1/worker/test_attention_model_runner.py" - "tests/unit/v1/worker/test_model_runner_v2.py" - "tests/unit/v1/worker/test_npu_runtime.py" @@ -166,7 +168,7 @@ Both platforms require V2. The paired deployment constraints are: | Excluded features | Elastic EP, EPLB, sequence-parallel MoE, and compile SP | Elastic EP and EPLB; model-specific SP restrictions | Elastic EP, EPLB, native draft models/MTP except DSV4 DSpark, finegrained TP, and explicit shared-expert DP | | Sequence parallelism | Unsupported | Model-specific; synchronous DeepSeek-V2/V3 SP is rejected | Attention-local FlashComm1/SP supported; disabled on FFN; no PCP/DCP | | Native DBO | Exactly two microbatches, Attention DP1 or DP > 1 | Eager or graph; exactly two microbatches, Attention DP > 1 | Eager only; exactly two microbatches at Attention DP1 or DP > 1 | -| Graph execution | Eager or `FULL_DECODE_ONLY` | Eager, `FULL`, or `FULL_DECODE_ONLY`; plain MLA graph DBO requires `FULL_DECODE_ONLY` | Eager Attention; independent FFN also supports the narrow W4A8/W8A8 FULL contract | +| Graph execution | Eager or `FULL_DECODE_ONLY` | Eager, `FULL`, or `FULL_DECODE_ONLY`; plain MLA graph DBO requires `FULL_DECODE_ONLY` | Eager Attention or DSV4 target native graph without DBO; native `FULL`/`FULL_DECODE_ONLY` resolution; Attention-local SP and native PD may remain enabled; DSpark draft may be eager or native Graph; independent FFN supports the narrow W4A8/W8A8 FULL contract | | Model | Registered AFD architecture; DeepEP M2N is limited to Qwen3 MoE or DeepSeek-V2 | Registered AFD architecture | DeepSeek-V4 with layered W4A8/W8A8 FFN experts and `dynamicQuant=1` | Both roles validate the paired V2 topology, but only Attention uses the native @@ -335,12 +337,14 @@ shutdown; Ascend and other models keep native preparation and profiling. Ascend V2 supports exactly two native microbatches: synchronous CAMP2P requires Attention DP > 1 and supports graph DBO, while Async CAM uses eager -Attention and also supports Attention DP1. Plain MLA graph DBO requires +Attention for DBO and also supports Attention DP1. Its separate target +DSV4 decode graph profile excludes DBO. Plain MLA graph DBO requires `FULL_DECODE_ONLY`. -For full-graph capture, the V2 runner -wraps native input preparation and publishes one warmup and one capture -payload per descriptor. CUDA two-stage preparation installs separate AFD +For full-graph capture, the V2 runner wraps native input preparation and +stages descriptor-matched AFD metadata. Connectors with a control plane also +publish one warmup and one capture payload per descriptor; Async CAM does not. +CUDA two-stage preparation installs separate AFD metadata on both native stage contexts and sends their complete layout before eager execution or FULL replay. Single-stage FULL replay publishes its padded control shape through an execute-scoped manager hook; two-stage replay reuses diff --git a/docs/design/module/execution_platforms.md b/docs/design/module/execution_platforms.md index b52d2ade5..1992482ca 100644 --- a/docs/design/module/execution_platforms.md +++ b/docs/design/module/execution_platforms.md @@ -285,11 +285,12 @@ groups and also keeps Attention TP peers aligned. ### ACL Graph and NPU Graph NPU Attention follows the upstream ACL graph dispatcher while adding AFD -metadata and control-plane coordination. Native MRV2 owns single/two-stage -graph descriptors, warmup/capture, persistent outputs and replay. AFD retains -formal per-stage MLA registries by the complete native descriptor, including -LoRA specialization, and publishes one matching FFN control payload before -each warmup, capture or live replay. +metadata and, for synchronous CAMP2P, control-plane coordination. Native MRV2 +owns single/two-stage graph descriptors, warmup/capture, persistent outputs and +replay. AFD retains formal per-stage MLA registries by the complete native descriptor, including +LoRA specialization. Synchronous CAMP2P publishes one matching FFN control +payload before each warmup, capture or live replay; Async CAM stages local +AFD metadata without sending DP control messages. For MLA DBO full graphs, each stage records its own upstream `GraphParams`. `merge_mla_graph_params()` validates identical layer order and record counts, @@ -305,12 +306,16 @@ register no FIA workspace, so those models, DeepSeek V4 among them, take the generic two-stage path with no MLA registries. With synchronous CAMP2P, the NPU V2 runner supports eager, `FULL`, and -`FULL_DECODE_ONLY`; Async CAM Attention is eager-only. Like CUDA V2, -it publishes descriptor-matched warmup/capture control outside formal graph -capture and installs an instance-scoped pre-replay hook because native full -replay creates no `ForwardContext`. Native `NPUUBatchRunner` owns preparation, -execution and merge for eager and graph DBO. Plain MLA graph DBO requires -`FULL_DECODE_ONLY`; SFA/DSA +`FULL_DECODE_ONLY`; Async CAM additionally admits DSV4 target native +`FULL` or `FULL_DECODE_ONLY`, with native mode resolution and no DBO. DSpark +draft may be eager or native Graph. Attention-local FlashComm1/SP and native PD +can remain enabled. Device qualification uses compilation mode 0. +Synchronous CAMP2P publishes descriptor-matched warmup/capture control outside +formal graph capture. Async CAM has no control plane and only stages local +AFD metadata. Both install an instance-scoped pre-replay hook because native +full replay creates no `ForwardContext`. Native `NPUUBatchRunner` owns +preparation, execution and merge for eager and graph DBO. Plain MLA graph DBO +requires `FULL_DECODE_ONLY`; SFA/DSA accepted modes still require model-specific qualification. Native two-stage capture shares the ordinary descriptor's FIA workspace; replay merges captured records with current metadata for Ascend updates. Each child carries its actual stage ID for the opaque CAMP2P HCCL @@ -318,13 +323,13 @@ group lookup. Async CAM enables eager native DBO at DP1; it has no separate MoE stage pipeline. Async FFN FULL accepts eager Attention DBO; FFN stays outside Attention's native DBO thread schedule. -The FFN runner owns a separate ACL graph cache keyed by stage token counts and -A/F topology. Warmup runs the eager FFN path. Formal capture updates connector -state through `connector.control_plane` before entering `torch.npu.graph(...)`, +The synchronous FFN runner owns a separate ACL graph cache keyed by stage token +counts and A/F topology. Warmup runs the eager FFN path. Formal capture updates +connector state through `connector.control_plane` before entering `torch.npu.graph(...)`, so replay contains only model and data-plane operations. Replay additionally requires the control payload's `is_graph_replaying=true`; an unknown key or eager payload executes eagerly. -CAM async has no control plane and uses the independent -[Async FFN runtime](../npu/async_ffn_runtime.md). +Async CAM has no control plane and uses the separate complete-transaction +graph described in the [Async FFN runtime guide](../npu/async_ffn_runtime.md). ### Ascend native operators and packaging diff --git a/docs/design/module/model_integration.md b/docs/design/module/model_integration.md index 236f49611..7af50ca4a 100644 --- a/docs/design/module/model_integration.md +++ b/docs/design/module/model_integration.md @@ -85,7 +85,7 @@ registers lazy AFD wrapper paths under `AFD`-prefixed aliases. | `DeepseekV3ForCausalLM` | `AFDDeepseekV3ForCausalLM` | `AFDDeepseekV3ForCausalLM` | | `DeepseekV32ForCausalLM` | `AFDDeepseekV32ForCausalLM` | `AFDDeepseekV3ForCausalLM` | | `DeepseekV4ForCausalLM` | `AFDDeepseekV4ForCausalLM` | `AFDDeepseekV4ForCausalLM` | -| `DSparkDraftModel` | Unsupported | `AFDDSparkDeepseekV4ForCausalLM` (Async CAM: eager A, eager/FULL F) | +| `DSparkDraftModel` | Unsupported | `AFDDSparkDeepseekV4ForCausalLM` (Async CAM: eager/native decode Graph A, eager/FULL F) | | `GlmMoeDsaForCausalLM` | `AFDGlmMoeDsaForCausalLM` | `AFDGlmMoeDsaForCausalLM` | | `Qwen3MoeForCausalLM` | `AFDQwen3MoeForCausalLM` | `AFDQwen3MoeForCausalLM` | | `Qwen3_5MoeForConditionalGeneration` | `AFDQwen3_5MoeForConditionalGeneration` | `AFDQwen3_5MoeForConditionalGeneration` | @@ -176,8 +176,12 @@ installs its own AFD metadata and opts out of cross-DP synchronization; native sampling, rejection, heads and context-KV updates are inherited. The [0.32 recipe](../../../recipe/npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md) describes A/F serving. In PD deployments, Prefill uses the upstream complete -draft model and KV connector. Attention and native draft proposal execution stay -eager. F may use the existing FULL transaction graph with one homogeneous W4A8 or W8A8 +draft model and KV connector. Attention may use native graphs with or without +PD and Attention-local FlashComm1/SP, with matching target/draft TP and DBO +disabled. The draft can remain eager or use its own native graph alongside the +target graph. Native resolution selects the effective mode for `FULL` or +`FULL_DECODE_ONLY`; device qualification uses compilation mode 0. +F may use the existing FULL transaction graph with one homogeneous W4A8 or W8A8 table covering target and draft; selection remains on device. DSpark excludes DBO. diff --git a/docs/design/npu/async_ffn_runtime.md b/docs/design/npu/async_ffn_runtime.md index 299dab9c9..33eb5d9bb 100644 --- a/docs/design/npu/async_ffn_runtime.md +++ b/docs/design/npu/async_ffn_runtime.md @@ -1,7 +1,7 @@ # MRV2 Async CAM FFN eager/FULL runtime FULL captures one routed-expert transaction: dispatch receive (DR), -W4A8/W8A8 FFN, and combine send (CS). Attention remains MRV2 eager. +W4A8/W8A8 FFN, and combine send (CS). Attention uses MRV2 eager or native target decode graphs. `AsyncNPUFFNModelRunner` owns Async CAM model loading, eager/FULL transactions and receive-safe resource release independently of the synchronous CAMP2P @@ -12,7 +12,13 @@ and receive-safe resource release independently of the synchronous CAMP2P The target is the pinned vLLM 0.32 / vLLM-Ascend stack on A3/910C, with homogeneous DeepSeek-V4 W4A8 or W8A8 layers, dynamicQuant=1, static expert placement, F TP=1 and Attention-side gating. Both roles select -`VLLM_USE_V2_MODEL_RUNNER=1`. Attention keeps `--enforce-eager`. +`VLLM_USE_V2_MODEL_RUNNER=1`. Attention normally keeps `--enforce-eager`; the +[target Attention graph](../../npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md#deepseek-v4-target-attention-graph) +profile accepts `FULL` or `FULL_DECODE_ONLY`, with native graph-mode resolution. +Attention-local FlashComm1/SP and native PD can be combined with the graphs. +DSpark draft may remain eager or use its native graph alongside target graph. +Device qualification uses compilation mode 0; other compilation modes have not +been validated. Other model-based drafts and MTP remain unsupported. Attention may enable native DBO under the existing two-microbatch constraints. Enable native DBO only on Attention; FFN consumes CAM work items without joining Attention's DBO thread schedule. @@ -28,8 +34,8 @@ decode-only graph modes are rejected. DeepSeek-V4 DSpark uses this same F graph for target and draft routed experts. The combined layer table and startup agreement cover target0–42/draft43–45; all layers must share the same W4A8 or W8A8 layout. Native P loads the complete -draft model. Target/draft Attention and the native speculative configuration -remain eager. DSpark excludes DBO; see the +draft model. Target Attention may use the native decode graph; the draft can +remain eager or use its native query Graph. DSpark excludes DBO; see the [DSpark recipe](../../../recipe/npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md). W8A8 reuses the loaded INT8 NZ weights, FP32 W13 scales and BF16 W2 scales. diff --git a/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md b/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md index df23e1314..543b137a7 100644 --- a/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md +++ b/docs/npu/CAM_ASYNC_CONNECTOR_USER_GUIDE.md @@ -17,11 +17,12 @@ Current runtime constraints and validation evidence are summarized in the ## When to use this connector -Use this connector for eager asynchronous Ascend inference with these requirements: +Use this connector for asynchronous Ascend inference with these requirements: - plugin-owned Ascend operators are built and installed on every node; - Attention performs MoE gating before dispatch to FFN ranks; -- Attention execution is eager and AFD async-DP is enabled; **`async_dp=true` is required**; +- Attention uses eager execution or the DeepSeek-V4 graph profile below; + **`async_dp=true` is required**; - regular prefill and autoregressive decode steps use the same CAM path; - optional ubatching uses exactly two native MRV2 DBO microbatches, including Attention DP1. @@ -33,8 +34,8 @@ Attention DP replicas make that decision independently; FFN is connector-driven. FFN also supports the experimental complete-transaction FULL mode described in [Async FFN runtime](../design/npu/async_ffn_runtime.md#selection-and-startup). -FULL requires MRV2 on both roles and accepts Attention-side DBO under the -existing constraints. Attention graph execution remains unsupported. +FULL requires MRV2 on both roles and accepts eager Attention-side DBO under +the existing constraints. The target Attention graph profile is described below. ## ModelRunnerV2 deployment @@ -45,7 +46,7 @@ Attention uses native Ascend ModelRunnerV2; FFN uses the independent The V2 Async CAM contract is: - DeepSeek-V4 with homogeneous W4A8 or W8A8 routed experts on Ascend 910C/A3; -- `--enforce-eager` on Attention and `--enable-expert-parallel` on both roles, with +- eager Attention or the target graph profile below, and `--enable-expert-parallel` on both roles, with `async_dp=true` and `compute_gate_on_attention=true`; - on FFN, select eager with `--enforce-eager`; for FULL, omit `--enforce-eager` and pass `--compilation-config '{"cudagraph_mode":"FULL"}'`, following the narrow @@ -67,6 +68,41 @@ per-ubatch model forwards, and output merge. Native splitting can divide one request across microbatches. Native draft-model/MTP paths other than DeepSeek-V4 DSpark are unsupported. +### DeepSeek-V4 target Attention graph + +For a DeepSeek-V4 deployment, replace Attention's `--enforce-eager` +with native ACL decode graph selection: + +```bash +--compilation-config '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}' +``` + +The profile requires PP=PCP=DCP=1, static expert placement, +and DBO/ubatching disabled. Attention-local FlashComm1/SP and native PD may remain +enabled. DSpark may stay enabled: `speculative_config.enforce_eager=true` keeps the +draft eager; `false` enables its native query Graph alongside target Graph. Target and draft use matching +TP widths. Other model-based drafts and MTP remain unsupported. PD uses the +existing upstream KV connector; the W4A8 device comparison covers +`MooncakeHybridConnector` with a native full-draft P. Draft Graph requires target Graph because +native target capture owns the draft capture lifecycle. +Attention DPs remain independent. Each Attention TP group uses the existing +CAM token split and output gather; set `attn_ranks_per_dp` to Attention TP +on both roles. + +Native Ascend owns graph inputs, batch buckets, attention-parameter updates and +replay. Ineligible batches, including prefill, retain native eager fallback. +`FULL` is also accepted and resolved by the native attention backend; the +current DSV4 backend resolves it to `FULL_DECODE_ONLY`. The device qualification +uses compilation mode 0, which disables Dynamo while retaining ACL graph +capture. Other compilation modes have not been validated by this qualification. +AFD stages local real/padded token metadata before capture/replay; Async CAM +does not send synchronous DP control messages. Existing synchronous connectors +retain their control-plane behavior. + +Keep the existing FFN execution settings. Transport warmup and F +service readiness use the existing startup protocol. This option introduces no +new graph owner, native kernel or shutdown protocol. + ## CAM async data flow CAM async removes synchronization between data-parallel replicas and avoids the @@ -276,7 +312,8 @@ actual vendor library path. Missing source operators fail startup. ## Current limitations -- Attention is eager. FFN FULL targets homogeneous W4A8 or W8A8 layers on A3/910C; +- Attention is eager except for the DeepSeek-V4 native decode graph profile above. + FFN FULL targets homogeneous W4A8 or W8A8 layers on A3/910C; see [supported configuration](../design/npu/async_ffn_runtime.md#selection-and-startup) and [validation commands](../design/npu/async_ffn_runtime.md#validation). Broader models and topologies require separate validation. diff --git a/recipe/README.md b/recipe/README.md index 3e7cb9fd3..694418b0d 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -28,7 +28,7 @@ Recipes are organized by hardware backend, connector, and model: | GPU | `P2pNcclAFDConnector` | Qwen3.5-122B-A10B-FP8 | Decode | Three-pod graph recipe (`2A0F,2A0F,0A2F`); other layouts remain configurable through `LAYOUT` | [Graph](gpu/P2pNcclAFDConnector/qwen3_5_122b_a10b_fp8/prefill_decode_colocation/multipod_2a_2a_2f_graph.sh) | | Ascend NPU | `CAMP2pAFDConnector` | DeepSeek-V3.2 | Decode | Historical v0.19.1rc1 recipe; not qualified for the 0.32 target | [Synchronous decode](npu/CAMP2pAFDConnector/deepseek_v3_2/README.md) | | Ascend NPU | `CAMAsyncAFDConnector` | DeepSeek-V3.2 | Prefill / decode | Historical v0.19.1rc1/v0.26 recipes; standard V3.2 is unsupported by the current Async CAM path | [Async CAM](npu/CAMAsyncAFDConnector/deepseek_v3_2/README.md) | -| Ascend NPU | `CAMAsyncAFDConnector` | DeepSeek-V4-Flash W4A8/W8A8 DSpark | Separate P / decode, or colocated | v0.32; eager A TP1, F layered eager/FULL. W4A8 PD: native full-draft P TP4, A DP2, F EP4. W8A8 A/F: A DP2, F EP8; PD not qualified | [DSpark A/F](npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md) | +| Ascend NPU | `CAMAsyncAFDConnector` | DeepSeek-V4-Flash W4A8/W8A8 DSpark | Separate P / decode, or colocated | v0.32; F layered eager/FULL. W4A8 target/draft native decode Graph: A DP1/TP4, F EP4, with Attention-local SP and with/without PD; native full-draft P TP4 for PD. Earlier eager W4A8 PD: A DP2/TP1, F EP4. W8A8 eager A/F: A DP2/TP1, F EP8; A Graph and PD not qualified | [DSpark A/F](npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md) | Open the model-level README before running a recipe. It documents the required hardware and runtime baseline, topology, environment variables, launch order, diff --git a/recipe/npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md b/recipe/npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md index 48d51b85a..fec550967 100644 --- a/recipe/npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md +++ b/recipe/npu/CAMAsyncAFDConnector/deepseek_v4_flash/README.md @@ -34,11 +34,14 @@ and network interface. A TP1/DP1 and F EP4 use five distinct DIEs. MODEL=/absolute/path/to/dsv4-flash-dspark-w4a8 LOCAL_IP=REPLACE_WITH_COMMUNICATION_IP INTERFACE=eth0 -NUM_A=1 +A_DP=1 +A_TP=1 +NUM_A=$((A_DP * A_TP)) NUM_F=4 A_DEVICES=0 F_DEVICES=4,5,6,7 FFN_EXEC_ARGS=(--enforce-eager) +A_EXEC_ARGS=(--enforce-eager) export VLLM_USE_V2_MODEL_RUNNER=1 VLLM_ASCEND_ENABLE_FLASHCOMM1=0 export BATCH_SIZE_FACTOR=0.00390625 @@ -61,14 +64,36 @@ COMMON_ARGS=( --no-disable-hybrid-kv-cache-manager ) DSPARK_CONFIG="{\"method\":\"dspark\",\"model\":\"$MODEL\",\"num_speculative_tokens\":5,\"enforce_eager\":true}" +A_DSPARK_CONFIG="$DSPARK_CONFIG" ``` -For DP2 set `NUM_A=2` and `A_DEVICES=0,1`. The W4A8 example uses F EP4. +For DP2/TP1 set `A_DP=2`, recompute `NUM_A`, and use `A_DEVICES=0,1`. The W4A8 example uses F EP4. For a W8A8 checkpoint, size F for its larger weights; the validated W8A8 configuration uses `NUM_F=8` and `F_DEVICES=8,9,10,11,12,13,14,15`. -Target and draft Attention use eager execution. F uses layered eager or layered -FULL, with one combined target/draft weight table. Layered eager can be slower -for small decode batches, so measure the chosen deployment. +Target and draft use matching Attention TP widths; F remains TP1 with EP. +Target Attention may use eager execution or the native decode graph. +`A_DSPARK_CONFIG` selects native draft eager/Graph execution on A; F's +`DSPARK_CONFIG` and expert execution settings remain independent. To enable both +Attention graphs, set the following before launching. Keep `enforce_eager=true` +in `A_DSPARK_CONFIG` when only the target graph is wanted. + +```bash +A_EXEC_ARGS=(--compilation-config '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}') +A_DSPARK_CONFIG="{\"method\":\"dspark\",\"model\":\"$MODEL\",\"num_speculative_tokens\":5,\"enforce_eager\":false}" +``` + +Attention-local FlashComm1/SP and native PD may remain enabled with both graphs. +For PD, keep the native full-draft P and KV-transfer configuration described +below. The W4A8 graph comparison covers A DP1/TP4, F EP4, and native P TP4 when +using `MooncakeHybridConnector`. Compilation mode 0 is the qualified example; +other compilation modes have not been validated here. + +Native capture buckets are normalized separately for target verification and +draft queries; choose buckets covering both query widths and `max-num-seqs`. +For A TP4, set `A_TP=4`, `A_DP=1`, recompute `NUM_A`, and use four distinct +A devices. Both roles use the same A TP width for CAM topology. F uses layered +eager or layered FULL, with one combined target/draft weight table. Layered eager can be slower for +small decode batches, so measure the chosen deployment. For F FULL, replace the F settings above before launching: @@ -78,8 +103,8 @@ FFN_EXEC_ARGS=(--compilation-config '{"cudagraph_mode":"FULL"}') F captures one DR → layered FFN → CS transaction and repeatedly replays it for both target and draft layers. The bounded replay group is independent of request -and target/draft phase boundaries. A and the native draft configuration retain -eager execution. FULL requires matching W4A8 or W8A8 layouts on all layers; unsupported +and target/draft phase boundaries. Attention target/draft graph choices are +independent of F's execution mode. FULL requires matching W4A8 or W8A8 layouts on all layers; unsupported graph modes fail startup. DSpark does not support DBO. Layered W4A8 forwards the model's `swiglu_limit` for both target and draft @@ -106,17 +131,17 @@ vllm serve "$MODEL" "${COMMON_ARGS[@]}" "${FFN_EXEC_ARGS[@]}" \ \"host\":\"$LOCAL_IP\",\"port\":12499, \"num_attention_ranks\":$NUM_A,\"num_ffn_ranks\":$NUM_F, \"compute_gate_on_attention\":true, - \"connector_extra_config\":{\"dynamicQuant\":1,\"attn_ranks_per_dp\":1} + \"connector_extra_config\":{\"dynamicQuant\":1,\"attn_ranks_per_dp\":$A_TP} } }" >ffn.log 2>&1 & ``` ```bash ASCEND_RT_VISIBLE_DEVICES="$A_DEVICES" VLLM_PLUGINS=ascend,afd \ -vllm serve "$MODEL" "${COMMON_ARGS[@]}" --enforce-eager \ +vllm serve "$MODEL" "${COMMON_ARGS[@]}" "${A_EXEC_ARGS[@]}" \ --port 7100 --data-parallel-rpc-port 17100 \ - --tensor-parallel-size 1 --data-parallel-size "$NUM_A" \ - --all2all-backend allgather_reducescatter --speculative-config "$DSPARK_CONFIG" \ + --tensor-parallel-size "$A_TP" --data-parallel-size "$A_DP" \ + --all2all-backend allgather_reducescatter --speculative-config "$A_DSPARK_CONFIG" \ --additional-config "{ \"enable_cpu_binding\":false,\"enable_force_eplb\":false, \"enable_dsa_cp\":false,\"multistream_dsv4_dsa_overlap\":false, @@ -125,7 +150,7 @@ vllm serve "$MODEL" "${COMMON_ARGS[@]}" --enforce-eager \ \"host\":\"$LOCAL_IP\",\"port\":12499, \"num_attention_ranks\":$NUM_A,\"num_ffn_ranks\":$NUM_F, \"compute_gate_on_attention\":true, - \"connector_extra_config\":{\"dynamicQuant\":1,\"attn_ranks_per_dp\":1} + \"connector_extra_config\":{\"dynamicQuant\":1,\"attn_ranks_per_dp\":$A_TP} } }" >attention.log 2>&1 & ``` diff --git a/tests/unit/compat/npu/test_feature_validation.py b/tests/unit/compat/npu/test_feature_validation.py index 93ed73baf..cda824302 100644 --- a/tests/unit/compat/npu/test_feature_validation.py +++ b/tests/unit/compat/npu/test_feature_validation.py @@ -19,7 +19,10 @@ from tests.unit.fakes.vllm_config import ( # noqa: E402 async_moe_config as _async_moe_config, ) -from tests.unit.fakes.vllm_config import npu_dspark_config # noqa: E402 +from tests.unit.fakes.vllm_config import ( # noqa: E402 + npu_async_cam_target_graph_config, + npu_dspark_config, +) from tests.unit.fakes.vllm_config import ( # noqa: E402 npu_vllm_config as _vllm_config, ) @@ -35,7 +38,7 @@ [ ("attention", True, True, "deepseek_v4", None), ("ffn", True, True, "deepseek_v4", None), - ("attention", False, True, "deepseek_v4", "DSpark requires eager"), + ("attention", False, True, "deepseek_v4", "draft Graph requires target"), ( "attention", True, @@ -43,13 +46,19 @@ "deepseek_v2", "requires DeepSeek V4", ), + ("ffn", False, True, "deepseek_v4", None), ("ffn", True, False, "deepseek_v4", None), - ("attention", True, False, "deepseek_v4", "eager Attention"), + ("ffn", False, False, "deepseek_v4", None), + ("attention", True, False, "deepseek_v4", None), + ("attention", False, False, "deepseek_v4", None), ], ) def test_dspark_feature_boundaries(role, draft_eager, target_eager, model_type, error): config = npu_dspark_config(role=role, draft_eager=draft_eager) config.model_config.enforce_eager = target_eager + if role == "attention" and not target_eager: + config.compilation_config.mode = 0 + config.compilation_config.cudagraph_mode.name = "FULL_DECODE_ONLY" config.model_config.hf_text_config.model_type = model_type if error: with pytest.raises(RuntimeError, match=error): @@ -58,6 +67,16 @@ def test_dspark_feature_boundaries(role, draft_eager, target_eager, model_type, fail_if_unsupported_npu_afd_features(config) +@pytest.mark.parametrize("draft_eager", [True, False]) +def test_dspark_target_graph_reuses_tp4(monkeypatch, draft_eager): + monkeypatch.setenv("VLLM_ASCEND_ENABLE_FLASHCOMM1", "0") + config = npu_dspark_config(tp_size=4, draft_eager=draft_eager) + config.model_config.enforce_eager = False + config.compilation_config.mode = 0 + config.compilation_config.cudagraph_mode.name = "FULL_DECODE_ONLY" + fail_if_unsupported_npu_afd_features(config) + + @pytest.mark.parametrize( "mode", ["FULL_DECODE_ONLY", "PIECEWISE", "FULL_AND_PIECEWISE"] ) @@ -173,7 +192,7 @@ def _mla_dbo(cudagraph_mode, **model_fields): ), pytest.param( _config_with(model={"enforce_eager": False}, **ASYNC_CAM), - "only eager", + "FULL_DECODE_ONLY", id="async-graph", ), pytest.param( @@ -277,6 +296,41 @@ def test_async_cam_rejects_models_without_layered_ffn(role, model_type): fail_if_unsupported_npu_afd_features(config) +@pytest.mark.parametrize("dp_size", [1, 2]) +@pytest.mark.parametrize("tp_size", [1, 2]) +def test_async_cam_target_graph_feature_admission(monkeypatch, dp_size, tp_size): + monkeypatch.setenv("VLLM_ASCEND_ENABLE_FLASHCOMM1", "1") + config = npu_async_cam_target_graph_config( + data_parallel_size=dp_size, + tensor_parallel_size=tp_size, + num_attention_ranks=dp_size * tp_size, + ) + config.kv_transfer_config = object() + config.parallel_config.use_sequence_parallel_moe = True + fail_if_unsupported_npu_afd_features(config) + + +@pytest.mark.parametrize( + "method,uses_draft,error", + [ + ("mtp", True, "native draft models or MTP"), + ("ngram", False, None), + ], +) +def test_async_cam_target_graph_feature_preserves_draft_model_boundary( + monkeypatch, method, uses_draft, error +): + monkeypatch.setenv("VLLM_ASCEND_ENABLE_FLASHCOMM1", "0") + config = npu_async_cam_target_graph_config() + config.speculative_config = SimpleNamespace( + method=method, + use_eagle=lambda: False, + uses_draft_model=lambda: uses_draft, + ) + with pytest.raises(RuntimeError, match=error) if error else nullcontext(): + fail_if_unsupported_npu_afd_features(config) + + def test_npu_afd_num_ubatches_reads_two_ubatch_dbo(): assert npu_afd_num_ubatches(_vllm_config(**TWO_UBATCH_DBO)) == 2 diff --git a/tests/unit/compat/patches/test_signatures.py b/tests/unit/compat/patches/test_signatures.py index 0b7bd68c4..d9ebad4bb 100644 --- a/tests/unit/compat/patches/test_signatures.py +++ b/tests/unit/compat/patches/test_signatures.py @@ -237,6 +237,22 @@ def _use_installed_patch(binding, symbol, marker_name, install): "fullgraph-replay", id="fullgraph-replay", ), + pytest.param( + "vllm.v1.worker.gpu.spec_decode.dflash.cudagraph", + None, + "_prepare_dflash_inputs_to_capture", + None, + "npu-draft-capture", + id="npu-draft-capture", + ), + pytest.param( + "vllm_ascend.worker.v2.spec_decode.dflash.aclgraph", + "DFlashAclGraphManager", + "run_fullgraph", + None, + "npu-draft-replay", + id="npu-draft-replay", + ), pytest.param( "vllm_ascend.ascend_config", None, @@ -352,6 +368,26 @@ def test_patch_preserves_upstream_signature( tracker = adapter._AFDCaptureEventTracker([]) scope = adapter._use_afd_capture_input_preparation(None, tracker) + elif scope_name in {"npu-draft-capture", "npu-draft-replay"}: + from vllm.config import CUDAGraphMode + + from afd_plugin.v1.worker.npu import dspark_graph as adapter + + if scope_name == "npu-draft-capture": + runner = SimpleNamespace( + speculative_config=SimpleNamespace(enforce_eager=True), + _afd_is_graph_capturing=False, + _afd_is_graph_replaying=False, + ) + manager = SimpleNamespace( + _capture_descs={CUDAGraphMode.FULL: []}, + _max_full_descs_to_capture=None, + ) + scope = adapter.use_afd_draft_capture(runner, manager) + else: + native_class = getattr(target, owner) + binding = native_class.__new__(native_class) + scope = adapter.use_afd_draft_replay(SimpleNamespace(), binding, 1) elif scope_name == "camp2p-dispatch": from afd_plugin.v1.worker.npu import attention_model_runner_v2 as adapter diff --git a/tests/unit/config/test_validation.py b/tests/unit/config/test_validation.py index 6d8a53e0c..f61a7f7df 100644 --- a/tests/unit/config/test_validation.py +++ b/tests/unit/config/test_validation.py @@ -18,7 +18,11 @@ validate_gpu_model_runner_v2_config, validate_npu_model_runner_v2_config, ) -from tests.unit.fakes.vllm_config import mrv2_vllm_config +from tests.unit.fakes.vllm_config import ( + mrv2_vllm_config, + npu_async_cam_target_graph_config, + npu_dspark_config, +) MODEL_UTILS_MODULE = "afd_plugin.model_executor.models.model_utils" @@ -156,6 +160,97 @@ def test_gpu_validator_accepts_target_config_without_removed_enable_sp(mrv2_conf ) +@pytest.mark.parametrize("dp_size", [1, 2]) +@pytest.mark.parametrize("tp_size", [1, 2]) +def test_async_cam_target_graph_accepts_independent_attention_dps( + monkeypatch, mrv2_config, dp_size, tp_size +): + monkeypatch.setenv("VLLM_ASCEND_ENABLE_FLASHCOMM1", "0") + config = npu_async_cam_target_graph_config( + data_parallel_size=dp_size, + tensor_parallel_size=tp_size, + num_attention_ranks=dp_size * tp_size, + ) + validate_npu_model_runner_v2_config( + config, expected_role="attention", device_type="npu" + ) + + +@pytest.mark.parametrize("draft_eager", [True, False]) +def test_async_cam_target_graph_accepts_dspark_tp4( + monkeypatch, mrv2_config, draft_eager +): + monkeypatch.setenv("VLLM_ASCEND_ENABLE_FLASHCOMM1", "0") + config = npu_async_cam_target_graph_config( + tensor_parallel_size=4, num_attention_ranks=4 + ) + config.speculative_config = npu_dspark_config( + tp_size=4, draft_eager=draft_eager + ).speculative_config + validate_npu_model_runner_v2_config( + config, expected_role="attention", device_type="npu" + ) + + +@pytest.mark.parametrize( + "path,value,error", + [ + ("use_v2_model_runner", False, "ModelRunnerV2"), + ("model_config.hf_text_config.model_type", "deepseek_v2", "DeepSeek-V4"), + ("parallel_config.enable_dbo", True, "DBO/ubatching"), + ("parallel_config.use_ubatching", True, "DBO/ubatching"), + ("parallel_config.pipeline_parallel_size", 2, "PP or CP"), + ("parallel_config.prefill_context_parallel_size", 2, "PP or CP"), + ("parallel_config.decode_context_parallel_size", 2, "PP or CP"), + ("compilation_config.cudagraph_mode.name", "NONE", "supports ACL graph modes"), + ( + "compilation_config.cudagraph_mode.name", + "PIECEWISE", + "supports ACL graph modes", + ), + ], +) +def test_async_cam_target_graph_rejects_unsupported_configuration( + monkeypatch, mrv2_config, path, value, error +): + monkeypatch.setenv("VLLM_ASCEND_ENABLE_FLASHCOMM1", "0") + config = npu_async_cam_target_graph_config() + *parents, name = path.split(".") + target = config + for parent in parents: + target = getattr(target, parent) + setattr(target, name, value) + with pytest.raises(RuntimeError, match=error): + validate_npu_model_runner_v2_config( + config, expected_role="attention", device_type="npu" + ) + + +@pytest.mark.parametrize( + "path,value", + [ + ("kv_transfer_config", object()), + ("parallel_config.use_sequence_parallel_moe", True), + ("speculative_config", SimpleNamespace(method="ngram")), + ("compilation_config.cudagraph_mode.name", "FULL"), + ("compilation_config.mode", 3), + ], +) +def test_async_cam_target_graph_preserves_native_options( + monkeypatch, mrv2_config, path, value +): + monkeypatch.setenv("VLLM_ASCEND_ENABLE_FLASHCOMM1", "1") + config = npu_async_cam_target_graph_config() + *parents, name = path.split(".") + target = config + for parent in parents: + target = getattr(target, parent) + setattr(target, name, value) + validate_npu_model_runner_v2_config( + config, expected_role="attention", device_type="npu" + ) + + @pytest.mark.parametrize( "role,tp,error", [ @@ -655,8 +750,8 @@ def test_gpu_v2_validator_outcome(mrv2_config, monkeypatch, overrides, error): ), pytest.param( {**_NPU_ASYNC, "model.enforce_eager": False}, - _err("eager"), - id="reject-async-cam-graph", + _err("MRV2 DeepSeek-V4"), + id="reject-async-cam-non-dsv4-graph", ), pytest.param( {**_NPU_ASYNC, "par.pipeline_parallel_size": 2}, @@ -697,6 +792,7 @@ def test_gpu_v2_validator_outcome(mrv2_config, monkeypatch, overrides, error): ) def test_npu_v2_validator_outcome(mrv2_config, overrides, error): config, call = _validator_call(overrides) + config.model_config.hf_text_config = config.model_config.hf_config with _expect(error): validate_npu_model_runner_v2_config( config, diff --git a/tests/unit/fakes/npu_attention.py b/tests/unit/fakes/npu_attention.py new file mode 100644 index 000000000..80e8ae9d0 --- /dev/null +++ b/tests/unit/fakes/npu_attention.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Native Ascend metadata seams without device allocation.""" + +from types import SimpleNamespace + +from tests.unit.fakes.vllm_config import npu_vllm_config + + +def metadata_runner(): + from afd_plugin.v1.worker.npu.attention_model_runner_v2 import ( + AFDNPUAttentionModelRunnerV2, + ) + + runner = object.__new__(AFDNPUAttentionModelRunnerV2) + runner.vllm_config = npu_vllm_config() + runner.vllm_config.kv_transfer_config = None + runner.speculative_config = SimpleNamespace(enforce_eager=False) + runner.connector = SimpleNamespace(control_plane=None) + runner._afd_pending_metadata = None + runner._afd_suppress_metadata_send = False + runner._afd_transaction_counter = 0 + runner._is_warmup = runner._afd_is_profile = False + runner._afd_is_graph_capturing = runner._afd_is_graph_replaying = False + return runner + + +def forward_context(num_tokens): + return SimpleNamespace( + batch_descriptor=SimpleNamespace(num_tokens=num_tokens), + attn_metadata={}, + additional_kwargs={}, + ubatch_slices=None, + ) + + +class DraftGraphManager: + def __init__(self, runner, descs=(), *, fail=False): + from vllm.config import CUDAGraphMode + + self.runner = runner + self._capture_descs = {CUDAGraphMode.FULL: list(descs)} + self._max_full_descs_to_capture = None + self.fail = fail + self.emitted = [] + + def run_fullgraph(self, desc): + context = forward_context(desc.num_tokens) + self.runner.install_afd_metadata_on_forward_context(context) + metadata = context.additional_kwargs["afd_metadata"] + self.emitted.append(metadata) + if self.fail: + raise RuntimeError("native replay failed") + return metadata diff --git a/tests/unit/fakes/vllm_config.py b/tests/unit/fakes/vllm_config.py index e67a400da..c374efebd 100644 --- a/tests/unit/fakes/vllm_config.py +++ b/tests/unit/fakes/vllm_config.py @@ -45,6 +45,7 @@ def npu_parallel_config(**overrides): "pipeline_parallel_size": 1, "use_sequence_parallel_moe": False, "enable_eplb": False, + "enable_elastic_ep": False, "prefill_context_parallel_size": 1, "decode_context_parallel_size": 1, "dbo_decode_token_threshold": 1, @@ -101,10 +102,34 @@ def npu_vllm_config( fast_moe_cold_start=False, ), speculative_config=speculative_config, + kv_transfer_config=None, scheduler_config=SimpleNamespace(max_num_batched_tokens=16), ) +def npu_async_cam_target_graph_config(**parallel_overrides): + config = npu_vllm_config( + connector="CAMAsyncAFDConnector", + async_dp=True, + compute_gate_on_attention=True, + cudagraph_mode="FULL_DECODE_ONLY", + extra_config={ + "dynamicQuant": 1, + "attn_ranks_per_dp": parallel_overrides.get("tensor_parallel_size", 1), + }, + enable_expert_parallel=True, + **parallel_overrides, + ) + config.model_config.enforce_eager = False + config.model_config.hf_config = config.model_config.hf_text_config = ( + SimpleNamespace( + model_type="deepseek_v4", architectures=["DeepseekV4ForCausalLM"] + ) + ) + config.compilation_config.mode = 0 + return config + + def async_moe_config( *, role="attention", @@ -253,7 +278,7 @@ def m2n_vllm_config( ) -def npu_dspark_config(*, role="attention", draft_eager=True): +def npu_dspark_config(*, role="attention", draft_eager=True, tp_size=1): hf = SimpleNamespace( architectures=["DSparkDraftModel"], vocab_size=16, @@ -266,12 +291,14 @@ def npu_dspark_config(*, role="attention", draft_eager=True): connector="CAMAsyncAFDConnector", async_dp=True, compute_gate_on_attention=True, - extra_config={"dynamicQuant": 1}, + tensor_parallel_size=tp_size, + num_attention_ranks=tp_size, + extra_config={"dynamicQuant": 1, "attn_ranks_per_dp": tp_size}, speculative_config=SimpleNamespace( method="dspark", enforce_eager=draft_eager, num_speculative_tokens=5, - draft_parallel_config=SimpleNamespace(tensor_parallel_size=1), + draft_parallel_config=SimpleNamespace(tensor_parallel_size=tp_size), draft_model_config=SimpleNamespace(hf_config=hf, hf_text_config=hf), ), ) diff --git a/tests/unit/v1/worker/npu/test_dspark_graph.py b/tests/unit/v1/worker/npu/test_dspark_graph.py new file mode 100644 index 000000000..42884a3b0 --- /dev/null +++ b/tests/unit/v1/worker/npu/test_dspark_graph.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project +"""Draft graph scopes preserve independent AFD batches and native preparation.""" + +from contextlib import nullcontext + +import pytest + +pytest.importorskip("torch") +pytest.importorskip("vllm") +pytest.importorskip("vllm_ascend") + +from vllm.config import CUDAGraphMode # noqa: E402 +from vllm.v1.worker.gpu.cudagraph_utils import ( # noqa: E402 + AttentionState, + BatchExecutionDescriptor, +) + +from afd_plugin.v1.worker.npu import dspark_graph # noqa: E402 +from tests.unit.fakes.npu_attention import ( # noqa: E402 + DraftGraphManager, + forward_context, + metadata_runner, +) + + +@pytest.mark.parametrize("fail", [False, True]) +def test_draft_replay_isolates_padding_and_restores_target_context(fail): + runner = metadata_runner() + target = forward_context(12) + runner.install_afd_metadata_on_forward_context(target) + target_metadata = target.additional_kwargs["afd_metadata"] + manager = DraftGraphManager(runner, fail=fail) + desc = BatchExecutionDescriptor(CUDAGraphMode.FULL, num_tokens=15, num_reqs=3) + expected = ( + pytest.raises(RuntimeError, match="native replay failed") + if fail + else nullcontext() + ) + with ( + expected, + runner._afd_draft_context(False, False), + dspark_graph.use_afd_draft_replay(runner, manager, real_tokens=10), + ): + manager.run_fullgraph(desc) + draft = manager.emitted[0] + assert draft.tokens_lens == [15] + assert draft.tokens_unpadded_lens == [10] + assert draft.transaction_id != target_metadata.transaction_id + following = forward_context(12) + runner.install_afd_metadata_on_forward_context(following) + assert following.additional_kwargs["afd_metadata"] is target_metadata + manager.fail = False + assert manager.run_fullgraph(desc) is target_metadata + + +@pytest.mark.parametrize("fail", [False, True]) +def test_draft_capture_creates_distinct_batches_and_restores_native_prepare( + monkeypatch, fail +): + runner = metadata_runner() + desc = BatchExecutionDescriptor(CUDAGraphMode.FULL, num_tokens=10, num_reqs=2) + manager = DraftGraphManager(runner, [desc]) + native_result = AttentionState({}, {}) + monkeypatch.setattr( + dspark_graph.dflash_cudagraph, + "_prepare_dflash_inputs_to_capture", + lambda *args, **kwargs: native_result, + ) + emitted = [] + expected = ( + pytest.raises(RuntimeError, match="native capture failed") + if fail + else nullcontext() + ) + with ( + expected, + runner._afd_draft_context(True, False), + dspark_graph.use_afd_draft_capture(runner, manager), + ): + for _ in range(2): + result = dspark_graph.dflash_cudagraph._prepare_dflash_inputs_to_capture( + 2, 10, None, None, None, None, 4096, False, False + ) + assert result is native_result + context = forward_context(10) + runner.install_afd_metadata_on_forward_context(context) + emitted.append(context.additional_kwargs["afd_metadata"]) + if fail: + raise RuntimeError("native capture failed") + assert [item.tokens_lens for item in emitted] == [[10], [10]] + assert emitted[0].transaction_id != emitted[1].transaction_id + assert ( + dspark_graph.dflash_cudagraph._prepare_dflash_inputs_to_capture() + is native_result + ) + + +def test_draft_full_rejects_missing_native_capture_descriptors(): + runner = metadata_runner() + with ( + pytest.raises(RuntimeError, match="no native capture descriptors"), + dspark_graph.use_afd_draft_capture(runner, DraftGraphManager(runner)), + ): + pass diff --git a/tests/unit/v1/worker/test_npu_model_runner_v2.py b/tests/unit/v1/worker/test_npu_model_runner_v2.py index 5c60e7c57..7cb0283c8 100644 --- a/tests/unit/v1/worker/test_npu_model_runner_v2.py +++ b/tests/unit/v1/worker/test_npu_model_runner_v2.py @@ -57,7 +57,9 @@ def _runner(*, async_dp=False): async_dp=async_dp, num_attention_ranks=2, num_ffn_ranks=2 ) runner.connector = SimpleNamespace( - extra_info=SimpleNamespace(), attention_step=nullcontext + extra_info=SimpleNamespace(), + control_plane=None if async_dp else object(), + attention_step=nullcontext, ) runner.cudagraph_manager = None runner.ubatch_runner = None @@ -200,10 +202,11 @@ def update(backend, stream, forward, tokens, config, speculative): @pytest.mark.parametrize("profile_only", [False, True]) @pytest.mark.parametrize("raise_in_native", [False, True]) -def test_capture_forwards_new_inputs_and_restores( - monkeypatch, profile_only, raise_in_native -): # long-test: capture lifecycle - runner = _runner() +@pytest.mark.parametrize("async_dp", [False, True]) +def test_capture_forwards_new_inputs_and_restores( # long-test: capture lifecycle + monkeypatch, profile_only, raise_in_native, async_dp +): + runner = _runner(async_dp=async_dp) descs = [ cudagraph_utils.BatchExecutionDescriptor( cg_mode=CUDAGraphMode.FULL, num_reqs=1, num_tokens=8 @@ -311,11 +314,72 @@ def native_capture(self, *, profile_only=False): ] if not profile_only: expected_events.extend([(32, True, False), (32, False, True)]) - assert events == expected_events + assert events == ([] if async_dp else expected_events) assert cudagraph_utils.prepare_inputs_to_capture is native_prepare assert original_prepare is not native_prepare +@pytest.mark.parametrize("async_dp", [False, True]) +@pytest.mark.parametrize("fail_replay", [False, True]) +def test_replay_real_and_padded_metadata( # long-test: native replay/context seam + monkeypatch, async_dp, fail_replay +): + runner = _runner(async_dp=async_dp) + original_metadata = runner._afd_pending_metadata + runner._afd_transaction_counter = 0 + runner.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY + runner.build_capture_dp_metadata = lambda tokens: tokens + sent = [] + runner.send_dp_metadata = lambda tokens, _slices: sent.append(tokens) + emitted = [] + desc = cudagraph_utils.BatchExecutionDescriptor( + CUDAGraphMode.FULL, num_tokens=8, num_reqs=8 + ) + + def context(): + return SimpleNamespace( + batch_descriptor=None, + attn_metadata={}, + additional_kwargs={}, + ubatch_slices=None, + ) + + class Manager: + def run_fullgraph(self, descriptor): + update_context = context() + runner.install_afd_metadata_on_forward_context(update_context) + emitted.append(update_context.additional_kwargs["afd_metadata"]) + if fail_replay: + raise RuntimeError("native replay failed") + return "native-output" + + runner.cudagraph_manager = Manager() + + def native_execute(self, *args, **kwargs): + assert should_skip_dp_coordination() is async_dp + return self.cudagraph_manager.run_fullgraph(desc) + + monkeypatch.setattr(NPUModelRunner, "execute_model", native_execute) + monkeypatch.setattr(npu_v2, "step_afd_npu_profiler", lambda _prof: None) + expected = ( + pytest.raises(RuntimeError, match="native replay failed") + if fail_replay + else nullcontext() + ) + with expected: + assert ( + runner.execute_model(SimpleNamespace(total_num_scheduled_tokens=5)) + == "native-output" + ) + assert emitted[0].tokens_lens == [8] + assert emitted[0].tokens_unpadded_lens == [5] + assert sent == ([] if async_dp else [8]) + following = context() + runner.install_afd_metadata_on_forward_context(following) + assert following.additional_kwargs["afd_metadata"] is original_metadata + assert not should_skip_dp_coordination() + + @pytest.mark.parametrize("num_tokens", [0, 1, 7]) @pytest.mark.parametrize("dp_rank", [0, 1]) def test_async_execute_uses_native_local_dp_dispatch(monkeypatch, num_tokens, dp_rank): From 67eeb015caece15169e089fcf34a53bdde64a737 Mon Sep 17 00:00:00 2001 From: ronenkat <16743404+ronenkat@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:29:54 +0200 Subject: [PATCH 42/42] test(e2e): port the standalone FFN service cases to model-owned cases #561 moved E2E scenarios into model-owned Case/Deployment declarations, which left the branch's legacy runner path for the standalone FFN service unreachable. Port the four afd-deepep-m2n-faas-*-2a2f cases to tests/e2e/models/qwen3_moe/cases.py. Deployment.ffn_service declares that the FFN role runs the standalone FFN service under torchrun instead of `vllm serve`. The deployment builds its command from the same AFD config as the vLLM FFN role, waits for every rank's /ready before the Attention API, and finish() ends a passing case in the service's stop order (/stop on every rank, then Attention) and requires a clean launcher exit and released resources per rank. Cleanup sweeps the torchrun workers by run environment, as they live outside the launcher's process group. Execution evidence is required from Attention only; the service has no vLLM Worker. Co-Authored-By: Claude Opus 5.5 Signed-off-by: ronenkat <16743404+ronenkat@users.noreply.github.com> --- docs/design/module/e2e_testing.md | 9 +- tests/e2e/README.md | 27 ++- tests/e2e/case.py | 22 ++ tests/e2e/deployment.py | 221 +++++++++++++++++-- tests/e2e/models/qwen3_moe/cases.py | 59 +++-- tests/e2e/models/qwen3_moe/test_qwen3_moe.py | 17 +- tests/e2e/runner.py | 1 + 7 files changed, 314 insertions(+), 42 deletions(-) diff --git a/docs/design/module/e2e_testing.md b/docs/design/module/e2e_testing.md index 89077b760..e3be92e16 100644 --- a/docs/design/module/e2e_testing.md +++ b/docs/design/module/e2e_testing.md @@ -105,6 +105,12 @@ configuration, a pinned Attention rendezvous address, exact compilation configuration, and `cleanup_profile`. Sync connectors start FFN then Attention; async CAM starts Attention then FFN. The NPU async profile alone keeps its scoped FFN process-ownership cleanup exception; other profiles are strict. +`Deployment.ffn_service` replaces the FFN role's `vllm serve` with the +standalone FFN service under `torchrun` on one host. Readiness adds every +rank's `/ready`; a passing case ends with `/stop` on every rank, then +Attention, and requires a clean launcher exit and released resources per rank. +Its torchrun workers sit outside the launcher's process group, so cleanup also +sweeps FFN processes by run environment. `Deployment.dbo` supplies a default, and an explicit `Role.dbo` overrides it. `Role.dbo=None` inherits the default; `DboConfig()` keeps native thresholds. `Deployment.dbo_for_role()` resolves this rule for commands, probe validation @@ -233,7 +239,8 @@ for NPU. The `test_deepseek_v2_lite` entry collects 13 GPU cases, including 1A1F, 2A1F and the local-only `afd-v2-graph-dbo-tp2`; its NPU selection collects seven. The P2P entries across DeepSeek, Qwen3 MoE and Qwen3.6 MoE contain 20 CUDA cases (13 + 4 + 3). Separate DeepEP M2N entries add five GPU-only -cases each for DeepSeek and Qwen3 MoE. Unfiltered DeepSeek module collection +cases each for DeepSeek and Qwen3 MoE, and a Qwen3 MoE standalone FFN service +entry adds four. Unfiltered DeepSeek module collection includes its five DeepEP IDs even on NPU; NPU runs select the P2P entry or explicit node IDs. The local-only TP2 Graph + DBO case is outside the seven-case ready/merge CI gate. diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 4a2749616..d651f9432 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -12,10 +12,11 @@ The V2 DP2 cases replace the duplicate GPU `afd-eager-2a2f`, baseline and AFD eager, graph, and graph+DBO on 2A1F. Qwen3.6 MoE retains three: baseline and AFD eager and graph on 2A1F. GDN DBO is unsupported. -The single-deployment inventory contains 48 definitions: 38 CUDA cases -(18 DeepSeek-V2-Lite, nine Qwen3 MoE, three Qwen3.6 MoE and eight DSV4 +The single-deployment inventory contains 52 definitions: 42 CUDA cases +(18 DeepSeek-V2-Lite, 13 Qwen3 MoE, three Qwen3.6 MoE and eight DSV4 Flash) and ten NPU cases (seven DeepSeek-V2-Lite and three DSV4 Flash). -The CUDA counts include the local DeepEP M2N entries. CI still selects the +The CUDA counts include the local DeepEP M2N and standalone FFN service +entries. CI still selects the same 15 unique node IDs across ready/merge and weekly gates. Definitions, collected nodes and device acceptance results are different counts. @@ -112,7 +113,12 @@ AFD eager intentionally retains its required Hash probe. Deployment owns connector order (sync FFN then Attention; async CAM Attention then FFN), readiness, pre/post-workload liveness and cleanup. Its explicit `cleanup_profile` retains the scoped NPU async CAM exception; ordinary cases -use strict cleanup. Per-role options, connector configuration, pinned Attention +use strict cleanup. `Deployment.ffn_service` runs the FFN role as the +standalone FFN service (`torchrun -m afd_plugin.ffn_faas serve`) on one host: +the runner waits for every rank's `/ready`, ends the case with `/stop` on +every rank followed by Attention shutdown, requires the launcher to exit 0 and +every rank's status file to report released resources, and sweeps the +torchrun workers by environment on cleanup. Per-role options, connector configuration, pinned Attention address and exact compilation settings preserve the existing NPU profiles. `Deployment.dbo` is the default for its roles. An explicit `Role.dbo` replaces that default; `Role.dbo=None` inherits it. `DboConfig()` enables DBO with native @@ -294,6 +300,19 @@ python -m pytest -q -s \ tests/e2e/models/qwen3_moe/test_qwen3_moe.py::test_qwen3_moe_deepep_m2n ``` +`test_qwen3_moe_deepep_m2n_ffn_service` runs the eager, graph, graph + DBO +and Attention TP2 graph deployments (`afd-deepep-m2n-faas-*-2a2f`) with the +FFN side served by the +[standalone GPU FFN service](../../docs/gpu/STANDALONE_FFN_SERVICE.md) +instead of `vllm serve`. Attention launches unchanged; execution evidence is +required from Attention only, because the service has no vLLM Worker. It has +the same requirements and is not run by CI. + +```bash +python -m pytest -q -s \ + tests/e2e/models/qwen3_moe/test_qwen3_moe.py::test_qwen3_moe_deepep_m2n_ffn_service +``` + `tests/e2e/models/qwen3_moe/test_qwen3_moe_multi_pod.py` runs the eager, graph and graph + DBO cases on two pods in two layouts: one role per pod (`2A0F,0A2F`) and one Attention and one FFN rank per pod (`1A1F,1A1F`, which diff --git a/tests/e2e/case.py b/tests/e2e/case.py index 13fc4ef59..b6ab5f2c1 100644 --- a/tests/e2e/case.py +++ b/tests/e2e/case.py @@ -35,6 +35,27 @@ NPU_ASYNC_VLLM_SHUTDOWN_TIMEOUT_S = 90 NPU_ASYNC_WORKER_SHUTDOWN_TIMEOUT_S = 60 NPU_ASYNC_TERMINATION_TIMEOUT_S = 120 +# Independently started services must pair within the AFD rendezvous timeout on +# both roles; the standalone FFN service loads its experts while Attention loads. +FFN_SERVICE_AFD_TIMEOUT_S = 600 +FFN_SERVICE_RECEIVER_SHUTDOWN_TIMEOUT_S = 10 +FFN_SERVICE_STOP_TIMEOUT_S = 120 + + +@dataclass(frozen=True) +class FfnService: + """The FFN role runs the standalone FFN service instead of ``vllm serve``. + + ``torchrun -m afd_plugin.ffn_faas serve --backend `` starts one + process per FFN rank on the local host. Attention launches unchanged. The + service reports per-rank readiness on its management endpoint and must + stop cleanly: ``/stop`` on every rank while idle, then Attention. + """ + + backend: str + afd_timeout_s: int = FFN_SERVICE_AFD_TIMEOUT_S + receiver_shutdown_timeout_s: int = FFN_SERVICE_RECEIVER_SHUTDOWN_TIMEOUT_S + stop_timeout_s: float = FFN_SERVICE_STOP_TIMEOUT_S @dataclass(frozen=True) @@ -87,6 +108,7 @@ class Deployment: # Existing worker environment overrides take precedence over this default. worker_shutdown_timeout_s: int | None = None termination_timeout_s: float = DEFAULT_TERMINATION_TIMEOUT_S + ffn_service: FfnService | None = None @property def is_baseline(self) -> bool: diff --git a/tests/e2e/deployment.py b/tests/e2e/deployment.py index 1e92a1c1a..f7328a703 100644 --- a/tests/e2e/deployment.py +++ b/tests/e2e/deployment.py @@ -8,6 +8,7 @@ import os import shlex import subprocess +import sys import threading import time import urllib.error @@ -17,8 +18,10 @@ from pathlib import Path from tests.e2e.case import ( + ATTENTION_ROLE, CLEANUP_PROFILE_NPU_ASYNC_CAM_LEGACY, CLEANUP_PROFILE_STRICT, + FFN_ROLE, WORKER_SHUTDOWN_TIMEOUT_ENV, Case, CaseEnvironment, @@ -39,6 +42,13 @@ READY_POLL_INTERVAL_S = 2 MRV2_EVIDENCE_PREFIX = "AFD E2E execution: " NPU_ASYNC_PROCESS_KILL_TIMEOUT_S = 120 +DECODE_GRAPH_COMPILATION_CONFIG = {"cudagraph_mode": "FULL_DECODE_ONLY"} +# The standalone FFN service's torchrun (F-only) world listens next to the AFD +# rendezvous port; its rank r management endpoint is the FFN API port + r. +FFN_SERVICE_MASTER_PORT_OFFSET = 1 +FFN_SERVICE_STATUS_FILE = "ffn-service.json" +FFN_SERVICE_HTTP_TIMEOUT_S = 5 +FFN_SERVICE_PROCESS_KILL_TIMEOUT_S = 120 # Starting the first role is not waiting for its connector rendezvous or API. CONNECTOR_PROTOCOLS = ( ("gpu", "P2pNcclAFDConnector", ("ffn", "attention")), @@ -85,6 +95,9 @@ class ResolvedDeployment: def command(self, role: str, probe_arguments: tuple[str, ...] = ()) -> list[str]: profile = self.case.deployment + if role == FFN_ROLE and profile.ffn_service is not None: + # The service has no vLLM Worker to host the evidence extension. + return self.ffn_service_command() layout = dict(profile.roles)[role] dbo = profile.dbo_for_role(role) env = self.environment @@ -104,26 +117,15 @@ def command(self, role: str, probe_arguments: tuple[str, ...] = ()) -> list[str] "--enable-expert-parallel", ] if profile.ffn is not None: - afd: dict[str, JsonValue] = { - "role": role, - "connector": profile.connector, - "host": env.afd_host if slot is None else slot.afd_host, - "port": env.afd_port, - "num_attention_ranks": profile.attention.rank_count, - "num_ffn_ranks": profile.ffn.rank_count, - } - if profile.connector == "CAMAsyncAFDConnector": - afd["async"] = True - if profile.compute_gate_on_attention: - afd["compute_gate_on_attention"] = True - if profile.connector_extra_config: - afd["connector_extra_config"] = dict(profile.connector_extra_config) cmd.extend( [ "--enable-ep-weight-filter", "--additional-config", json.dumps( - {"afd": afd, **dict(layout.additional_config)}, + { + "afd": self.afd_config(role, slot), + **dict(layout.additional_config), + }, separators=(",", ":"), ), ] @@ -182,9 +184,7 @@ def command(self, role: str, probe_arguments: tuple[str, ...] = ()) -> list[str] "--cudagraph-capture-sizes", capture_size, "--compilation-config", - json.dumps( - {"cudagraph_mode": "FULL_DECODE_ONLY"}, separators=(",", ":") - ), + json.dumps(DECODE_GRAPH_COMPILATION_CONFIG, separators=(",", ":")), ] ) else: @@ -198,6 +198,100 @@ def command(self, role: str, probe_arguments: tuple[str, ...] = ()) -> list[str] cmd.extend(self.case.common_vllm_args) return cmd + def afd_config(self, role: str, slot: RoleSlot | None) -> dict[str, JsonValue]: + """The ``additional_config["afd"]`` one role's service starts with.""" + profile = self.case.deployment + assert profile.ffn is not None + env = self.environment + afd: dict[str, JsonValue] = { + "role": role, + "connector": profile.connector, + "host": env.afd_host if slot is None else slot.afd_host, + "port": env.afd_port, + "num_attention_ranks": profile.attention.rank_count, + "num_ffn_ranks": profile.ffn.rank_count, + } + if profile.connector == "CAMAsyncAFDConnector": + afd["async"] = True + if profile.compute_gate_on_attention: + afd["compute_gate_on_attention"] = True + if profile.connector_extra_config: + afd["connector_extra_config"] = dict(profile.connector_extra_config) + if profile.ffn_service is not None: + afd["afd_process_group_timeout_s"] = profile.ffn_service.afd_timeout_s + return afd + + def ffn_service_command(self) -> list[str]: + """``torchrun`` command of the standalone FFN service on this host. + + One process per FFN rank. The AFD configuration, execution mode and + DBO match the FFN half of the ``vllm serve`` deployment, so the + unchanged Attention command pairs with it. Request-shaping arguments + (``common_vllm_args``, batch and capture budgets) belong to Attention, + whose control payloads drive every FFN step and graph capture. + """ + profile = self.case.deployment + service, ffn = profile.ffn_service, profile.ffn + assert service is not None and ffn is not None + env = self.environment + cmd = [ + sys.executable, + "-m", + "torch.distributed.run", + f"--nproc-per-node={ffn.rank_count}", + "--nnodes=1", + f"--master-addr={env.afd_host}", + f"--master-port={env.afd_port + FFN_SERVICE_MASTER_PORT_OFFSET}", + "-m", + "afd_plugin.ffn_faas", + "serve", + env.model, + "--backend", + service.backend, + "--data-parallel-size", + str(ffn.dp), + "--tensor-parallel-size", + str(ffn.tp), + "--enable-expert-parallel", + "--additional-config", + json.dumps( + { + "afd": self.afd_config(FFN_ROLE, None), + **dict(ffn.additional_config), + }, + separators=(",", ":"), + ), + "--receiver-shutdown-timeout", + str(service.receiver_shutdown_timeout_s), + "--status-host", + env.api_host, + "--status-port", + str(self.ffn_service_status_port(0)), + "--status-file", + str(self.ffn_service_status_file), + ] + if ffn.uses_graph: + cmd.extend( + [ + "--compilation-config", + json.dumps(DECODE_GRAPH_COMPILATION_CONFIG, separators=(",", ":")), + ] + ) + else: + cmd.append("--enforce-eager") + cmd.extend(dbo_arguments(profile.dbo_for_role(FFN_ROLE))) + cmd.extend(ffn.vllm_args) + return cmd + + def ffn_service_status_port(self, rank: int) -> int: + """Management port of one FFN service rank (the FFN API port + rank).""" + return self.environment.api_port_base + 1 + rank + + @property + def ffn_service_status_file(self) -> Path: + """Base status file; rank r writes ``ffn-service-rank-.json``.""" + return self.environment.output_path / FFN_SERVICE_STATUS_FILE + def dbo_arguments(dbo: DboConfig | None) -> list[str]: if dbo is None: @@ -240,6 +334,8 @@ def resolve( profile.backend != "npu" or profile.connector != "CAMAsyncAFDConnector" ): raise ValueError("legacy NPU async cleanup requires the NPU async CAM protocol") + if profile.ffn_service is not None and (profile.ffn is None or slots): + raise ValueError("the FFN service runs a local FFN role on one host") if any(role.uses_graph for _, role in profile.roles) and ( profile.cudagraph_capture_size is None or profile.cudagraph_capture_size < 1 ): @@ -305,8 +401,88 @@ def __init__( def start(self) -> None: for role in self.resolved.launch_order: self.start_role(role) + if self.resolved.case.deployment.ffn_service is not None: + self.wait_ffn_service_ready() self.wait_ready() + def wait_ffn_service_ready(self) -> None: + """Wait until every FFN service rank reports ``/ready`` (paired with A). + + ``/ready`` means paired and receiving; CUDA graphs are captured later, + driven by Attention, whose API comes up only after that. + """ + profile = self.resolved.case.deployment + assert profile.ffn is not None + env = self.resolved.environment + pending = set(range(profile.ffn.rank_count)) + deadline = time.monotonic() + env.startup_timeout + last_error: BaseException | None = None + while pending and time.monotonic() < deadline: + self.ensure_alive() + for rank in sorted(pending): + try: + with urllib.request.urlopen( + self.ffn_service_url(rank, "/ready"), + timeout=FFN_SERVICE_HTTP_TIMEOUT_S, + ) as response: + if response.status == 200: + pending.discard(rank) + except (OSError, urllib.error.URLError) as exc: + last_error = exc + if pending: + time.sleep(READY_POLL_INTERVAL_S) + if pending: + raise TimeoutError( + f"FFN service ranks {sorted(pending)} were not ready: {last_error}" + ) + + def ffn_service_url(self, rank: int, path: str) -> str: + env = self.resolved.environment + port = self.resolved.ffn_service_status_port(rank) + return f"http://{env.api_host}:{port}{path}" + + def finish(self) -> None: + """Stop a deployment whose protocol defines a clean stop; else no-op. + + The standalone FFN service stops idle: ``/stop`` on every rank cancels + its idle control receive, then Attention shuts down right away, since + releasing DeepEP buffers is a barrier over every M2N rank. The service + launcher must exit 0 and every rank must report ``stopped`` with + released resources. Other deployments stop in ``close``. + """ + profile = self.resolved.case.deployment + service = profile.ffn_service + if service is None: + return + assert profile.ffn is not None + for rank in range(profile.ffn.rank_count): + request = urllib.request.Request( + self.ffn_service_url(rank, "/stop"), method="POST" + ) + with urllib.request.urlopen(request, timeout=FFN_SERVICE_HTTP_TIMEOUT_S): + pass + failures = terminate_process_groups( + [self.processes_by_role[ATTENTION_ROLE]], + termination_timeout_s=profile.termination_timeout_s, + poll_interval_s=PROCESS_POLL_INTERVAL_S, + reap_timeout_s=PROCESS_REAP_TIMEOUT_S, + ) + if failures: + raise RuntimeError("; ".join(failures)) + returncode = self.processes_by_role[FFN_ROLE].wait( + timeout=service.stop_timeout_s + ) + if returncode != 0: + raise RuntimeError(f"FFN service exited with {returncode}") + status_file = self.resolved.ffn_service_status_file + for rank in range(profile.ffn.rank_count): + rank_file = status_file.with_name(f"{status_file.stem}-rank-{rank}.json") + status = json.loads(rank_file.read_text()) + if status["phase"] != "stopped" or status["resources_released"] is not True: + raise RuntimeError( + f"FFN service rank {rank} did not stop cleanly: {status}" + ) + def start_role(self, role: str) -> None: """Launch one local role; multi-pod owns barriers between these calls.""" resolved = self.resolved @@ -419,11 +595,16 @@ def close(self) -> None: if legacy_async and ffn is not None else (), ) - if legacy_async: + # torchrun starts the FFN service workers outside the launcher's process + # group, so the service also needs the environment-matched sweep. + ffn_service = self.resolved.case.deployment.ffn_service is not None + if legacy_async or ffn_service: failures.extend( kill_processes_matching_environment( {"AFD_E2E_RUN_ID": self.run_id, "AFD_E2E_PROCESS_ROLE": "ffn"}, - timeout_s=NPU_ASYNC_PROCESS_KILL_TIMEOUT_S, + timeout_s=FFN_SERVICE_PROCESS_KILL_TIMEOUT_S + if ffn_service + else NPU_ASYNC_PROCESS_KILL_TIMEOUT_S, poll_interval_s=PROCESS_POLL_INTERVAL_S, process_name="FFN", ) diff --git a/tests/e2e/models/qwen3_moe/cases.py b/tests/e2e/models/qwen3_moe/cases.py index c247a6be1..46a3f1f4b 100644 --- a/tests/e2e/models/qwen3_moe/cases.py +++ b/tests/e2e/models/qwen3_moe/cases.py @@ -11,6 +11,7 @@ Case, CaseEnvironment, DboConfig, + FfnService, Gsm8kSettings, Gsm8kTask, Mrv2Check, @@ -31,6 +32,16 @@ "afd-deepep-m2n-eager-tp2-2a2f", "afd-deepep-m2n-graph-tp2-2a2f", ) +# The same DeepEP M2N deployments with the FFN side served by the standalone FFN +# service (torchrun -m afd_plugin.ffn_faas) instead of `vllm serve`; Attention +# launches exactly as in the mirrored case. +DEEPEP_M2N_FFN_SERVICE_SCENARIOS = ( + "afd-deepep-m2n-faas-eager-2a2f", + "afd-deepep-m2n-faas-graph-2a2f", + "afd-deepep-m2n-faas-graph-dbo-2a2f", + "afd-deepep-m2n-faas-graph-tp2-2a2f", +) +FFN_SERVICE_BACKEND = "gpu-deepep-m2n" MULTI_POD_IDS = ( "afd-deepep-m2n-eager-2a2f", "afd-deepep-m2n-graph-2a2f", @@ -86,27 +97,38 @@ def build_case(case_id: str, environment: CaseEnvironment) -> Case: "afd-deepep-m2n-graph-dbo-2a2f": (False, True, True, 2, 1, 2, 1), "afd-deepep-m2n-eager-tp2-2a2f": (False, False, False, 1, 2, 2, 1), "afd-deepep-m2n-graph-tp2-2a2f": (False, True, False, 1, 2, 2, 1), + "afd-deepep-m2n-faas-eager-2a2f": (False, False, False, 2, 1, 2, 1), + "afd-deepep-m2n-faas-graph-2a2f": (False, True, False, 2, 1, 2, 1), + "afd-deepep-m2n-faas-graph-dbo-2a2f": (False, True, True, 2, 1, 2, 1), + "afd-deepep-m2n-faas-graph-tp2-2a2f": (False, True, False, 1, 2, 2, 1), } baseline, graph, dbo, a_dp, a_tp, f_dp, f_tp = profiles[case_id] execution = DECODE_GRAPH if graph else EAGER - deepep = case_id in DEEPEP_M2N_SCENARIOS + ffn_service = case_id in DEEPEP_M2N_FFN_SERVICE_SCENARIOS + deepep = case_id in DEEPEP_M2N_SCENARIOS or ffn_service + deployment = execution.deployment( + backend="gpu", + attention=(a_dp, a_tp), + ffn=None if baseline else (f_dp, f_tp), + connector=None + if baseline + else "DeepEPM2NAFDConnector" + if deepep + else "P2pNcclAFDConnector", + compute_gate_on_attention=deepep, + dbo=DboConfig(DBO_DECODE_THRESHOLD, DBO_PREFILL_THRESHOLD) if dbo else None, + capture_size=CAPTURE_SIZE, + ) + if ffn_service: + deployment = replace(deployment, ffn_service=FfnService(FFN_SERVICE_BACKEND)) + # The FFN service has no vLLM Worker to host the evidence extension; + # Attention's evidence covers the paired graph/DBO path. + evidence_roles = ("attention",) if ffn_service else ("attention", "ffn") return Case( id=case_id, model="Qwen3 MoE", served_model_name_prefix="qwen3-moe-afd", - deployment=execution.deployment( - backend="gpu", - attention=(a_dp, a_tp), - ffn=None if baseline else (f_dp, f_tp), - connector=None - if baseline - else "DeepEPM2NAFDConnector" - if deepep - else "P2pNcclAFDConnector", - compute_gate_on_attention=deepep, - dbo=DboConfig(DBO_DECODE_THRESHOLD, DBO_PREFILL_THRESHOLD) if dbo else None, - capture_size=CAPTURE_SIZE, - ), + deployment=deployment, common_vllm_args=(f"--max-model-len={MAX_MODEL_LEN}",), phases=( Phase( @@ -119,7 +141,7 @@ def build_case(case_id: str, environment: CaseEnvironment) -> Case: minimum_samples=DBO_MINIMUM_SAMPLES if dbo else 1, ) ), - (Mrv2Check(("attention", "ffn"), graph=graph, dbo=dbo),) + (Mrv2Check(evidence_roles, graph=graph, dbo=dbo),) if not baseline and (graph or dbo) else (), ), @@ -136,4 +158,9 @@ def build_case(case_id: str, environment: CaseEnvironment) -> Case: ) from tests.e2e.runner import case_main - raise SystemExit(case_main(build_case, (*SCENARIOS, *DEEPEP_M2N_SCENARIOS))) + raise SystemExit( + case_main( + build_case, + (*SCENARIOS, *DEEPEP_M2N_SCENARIOS, *DEEPEP_M2N_FFN_SERVICE_SCENARIOS), + ) + ) diff --git a/tests/e2e/models/qwen3_moe/test_qwen3_moe.py b/tests/e2e/models/qwen3_moe/test_qwen3_moe.py index b3416fb9f..5ceb051fc 100644 --- a/tests/e2e/models/qwen3_moe/test_qwen3_moe.py +++ b/tests/e2e/models/qwen3_moe/test_qwen3_moe.py @@ -17,7 +17,11 @@ run_case_entry, ) from tests.e2e.environment import model_case_command -from tests.e2e.models.qwen3_moe.cases import DEEPEP_M2N_SCENARIOS, SCENARIOS +from tests.e2e.models.qwen3_moe.cases import ( + DEEPEP_M2N_FFN_SERVICE_SCENARIOS, + DEEPEP_M2N_SCENARIOS, + SCENARIOS, +) GSM8K_DATASET_ID = "openai/gsm8k" GSM8K_DATASET_CONFIG = "main" @@ -79,3 +83,14 @@ def test_qwen3_moe(scenario: str, tmp_path: Path) -> None: def test_qwen3_moe_deepep_m2n(scenario: str, tmp_path: Path) -> None: command = build_runner_command(scenario, tmp_path / scenario) run_case_entry(command, tmp_path / scenario) + + +@pytest.mark.e2e +@pytest.mark.parametrize( + "scenario", + DEEPEP_M2N_FFN_SERVICE_SCENARIOS, + ids=DEEPEP_M2N_FFN_SERVICE_SCENARIOS, +) +def test_qwen3_moe_deepep_m2n_ffn_service(scenario: str, tmp_path: Path) -> None: + command = build_runner_command(scenario, tmp_path / scenario) + run_case_entry(command, tmp_path / scenario) diff --git a/tests/e2e/runner.py b/tests/e2e/runner.py index 8bcb1862b..78aeedf58 100644 --- a/tests/e2e/runner.py +++ b/tests/e2e/runner.py @@ -1283,6 +1283,7 @@ def cleanup() -> None: phase_record["evidence_records"] = len(snapshot) phase_record["status"] = "passed" services.ensure_alive() + services.finish() record["execution"] = {"status": "passed"} record["acceptance_passed"] = record["configuration"]["eligible"] return record