diff --git a/.github/filters.yaml b/.github/filters.yaml index 15d863745fb2..13759209d7b5 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -91,11 +91,11 @@ core: - 'lib/**' - 'tests/**' - 'components/src/dynamo/router/**' - - 'components/src/dynamo/thunderagent_router/**' - - 'components/src/dynamo/mocker/**' - - 'components/src/dynamo/replay/**' + - 'dingo/thunderagent_router/**' + - 'dingo/mocker/**' + - 'dingo/replay/**' - 'components/src/dynamo/frontend/**' - - 'components/src/dynamo/common/**' + - 'dingo/common/**' - 'components/src/dynamo/gpu_memory_service/**' # TODO(will): Move TokenSpeed to a dedicated framework filter once CI has # TokenSpeed Docker images and backend-specific build/test jobs. @@ -120,7 +120,7 @@ operator: - 'deploy/snapshot/**' - 'docs/kubernetes/api-reference.md' - 'deploy/helm/charts/platform/**' - - 'components/src/dynamo/profiler/utils/dgdr_*' + - 'dingo/profiler/utils/dgdr_*' snapshot: - *ci @@ -146,10 +146,10 @@ deploy: planner: - 'container/templates/planner.Dockerfile' - - 'components/src/dynamo/planner/**' - - 'components/src/dynamo/global_planner/**' - - 'components/src/dynamo/profiler/**' - - 'components/src/dynamo/global_router/**' + - 'dingo/planner/**' + - 'dingo/global_planner/**' + - 'dingo/profiler/**' + - 'dingo/global_router/**' # Power-aware planner (PR #9369): per-node power agent component - 'deploy/power-agent/**' - 'deploy/helm/charts/power-agent/**' @@ -191,11 +191,11 @@ frontend: - 'container/deps/*' - 'container/compliance/**' - 'components/src/dynamo/router/**' - - 'components/src/dynamo/thunderagent_router/**' - - 'components/src/dynamo/mocker/**' - - 'components/src/dynamo/replay/**' + - 'dingo/thunderagent_router/**' + - 'dingo/mocker/**' + - 'dingo/replay/**' - 'components/src/dynamo/frontend/**' - - 'components/src/dynamo/common/**' + - 'dingo/common/**' - 'deploy/inference-gateway/**' - 'container/templates/frontend.Dockerfile' - '!**/*.md' diff --git a/.github/labeler.yml b/.github/labeler.yml index 15c80d01af6b..5d77e35ec7ad 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -58,9 +58,9 @@ frontend: planner: - changed-files: - any-glob-to-any-file: - - components/src/dynamo/planner/** - - components/src/dynamo/profiler/** - - components/src/dynamo/global_planner/** + - dingo/planner/** + - dingo/profiler/** + - dingo/global_planner/** # Deployment labels deployment::k8s: @@ -98,8 +98,8 @@ xpu: multimodal: - changed-files: - any-glob-to-any-file: - - components/src/dynamo/common/memory/multimodal_embedding_cache_manager.py - - components/src/dynamo/common/multimodal/** + - dingo/common/memory/multimodal_embedding_cache_manager.py + - dingo/common/multimodal/** - components/src/dynamo/vllm/omni/** - components/src/dynamo/vllm/multimodal_handlers/** - components/src/dynamo/vllm/multimodal_utils/** diff --git a/.gitignore b/.gitignore index 77294e4c53df..8b4da449b13e 100644 --- a/.gitignore +++ b/.gitignore @@ -54,9 +54,9 @@ CMakeCache.txt *_pb2.pyi # Planner plugin framework: stubs are checked in so test/build environments # don't need grpcio-tools or a protoc step just to import the module. -!components/src/dynamo/planner/plugins/proto/v1/plugin_pb2.py -!components/src/dynamo/planner/plugins/proto/v1/plugin_pb2_grpc.py -!components/src/dynamo/planner/plugins/proto/v1/plugin_pb2.pyi +!dingo/planner/plugins/proto/v1/plugin_pb2.py +!dingo/planner/plugins/proto/v1/plugin_pb2_grpc.py +!dingo/planner/plugins/proto/v1/plugin_pb2.pyi *.svg !docs/assets/**/*.svg diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 05bc0c413b13..c70c536e8d63 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ # limitations under the License. default_install_hook_types: [pre-commit, commit-msg] -exclude: ^(src/grpc_generated|.*\.patch$|.*/connect/.*\.py|components/src/dynamo/planner/plugins/proto/v1/plugin_pb2(_grpc)?\.pyi?$) +exclude: ^(src/grpc_generated|.*\.patch$|.*/connect/.*\.py|dingo/planner/plugins/proto/v1/plugin_pb2(_grpc)?\.pyi?$) repos: - repo: https://github.com/timothycrosley/isort rev: 5.12.0 diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index 8d8321d91d7f..000000000000 --- a/AGENTS.md +++ /dev/null @@ -1,11 +0,0 @@ - - -- Keep changes focused and reviewable. -- Use Conventional Commit PR titles: `type(scope): summary`. Accepted types: - `feat`, `fix`, `docs`, `test`, `ci`, `refactor`, `perf`, `chore`, `revert`, - `style`, and `build`. -- PR descriptions must include `Summary` and `Validation`. -- Sign every commit with DCO: `git commit -s`. diff --git a/CODEOWNERS b/CODEOWNERS deleted file mode 100644 index 9e0ebe57f9bb..000000000000 --- a/CODEOWNERS +++ /dev/null @@ -1,67 +0,0 @@ -# CODEOWNERS file for Dynamo -# -# DO NOT ADD ANY INDIVIDUALS. The groups here are all teams: https://github.com/orgs/ai-dynamo/teams - -*.rs @ai-dynamo/dynamo-rust-codeowners -Cargo.toml @ai-dynamo/dynamo-rust-codeowners - -# Python Libraries -*.py @ai-dynamo/python-codeowners @ai-dynamo/Devops - -# Container/Environments -/container/ @ai-dynamo/Devops @ai-dynamo/dynamo-rust-codeowners @ai-dynamo/python-codeowners @ai-dynamo/dynamo-deploy-codeowners - -# Examples -/examples/ @ai-dynamo/Devops @ai-dynamo/dynamo-rust-codeowners @ai-dynamo/python-codeowners @ai-dynamo/dynamo-deploy-codeowners - -# Dynamo deploy -/deploy/ @ai-dynamo/dynamo-deploy-codeowners -/examples/*/deploy/ @ai-dynamo/dynamo-deploy-codeowners -/examples/backends/*/deploy/ @ai-dynamo/dynamo-deploy-codeowners -# CI/CD -/.github/ @ai-dynamo/Devops -/.github/workflows/*.ps1 @ai-dynamo/Devops -CODEOWNERS @ai-dynamo/Devops - -# Planner -/components/src/dynamo/planner/ @ai-dynamo/python-codeowners @ai-dynamo/Devops -/components/src/dynamo/global_router/ @ai-dynamo/python-codeowners @ai-dynamo/Devops -/components/src/dynamo/global_planner/ @ai-dynamo/python-codeowners @ai-dynamo/Devops -/examples/global_planner/ @ai-dynamo/python-codeowners @ai-dynamo/Devops -/components/src/dynamo/profiler/ @ai-dynamo/python-codeowners @ai-dynamo/Devops - -# recipes - -/recipes/ @ai-dynamo/python-codeowners @ai-dynamo/Devops @ai-dynamo/dynamo-deploy-codeowners - -# Legal -/ATTRIBUTIONS-Rust.md @ai-dynamo/Devops -/ATTRIBUTIONS.md @ai-dynamo/Devops -/CODE_OF_CONDUCT.md @ai-dynamo/Devops -/CONTRIBUTING.md @ai-dynamo/Devops -/LICENSE @ai-dynamo/Devops -/SECURITY.md @ai-dynamo/Devops - -# KVBM v2 -/lib/memory/ @grahamking @ryanolson @oandreeva-nv -/lib/kvbm-*/ @ai-dynamo/kvbm-v2 -/lib/kvbm-logical/ @ai-dynamo/kvbm-v2 @PeaBrane - -# Claude Code -/.claude/ @ishandhanani @alec-flowers @MatejKosec @dagil-nvidia @athreesh @nv-tusharma @ayushag-nv - -# Parsers — tool-calling + reasoning parser correctness -/lib/parsers/ @ai-dynamo/dynamo-parser-codeowners -/lib/bindings/python/rust/parsers.rs @ai-dynamo/dynamo-parser-codeowners -/lib/bindings/python/tests/test_parsers.py @ai-dynamo/dynamo-parser-codeowners -/lib/llm/tests/test_reasoning_parser.rs @ai-dynamo/dynamo-parser-codeowners -/lib/llm/tests/test_streaming_tool_parsers.rs @ai-dynamo/dynamo-parser-codeowners - -# Docs — the docs group owns documentation surfaces: the docs site (docs/, fern/) -# and the Markdown inside examples/ and recipes/. Code teams above still own all -# non-Markdown (code, deploy configs) in examples/ and recipes/; only the prose -# routes to the docs group. Placed last so these rules win (last match wins). -/docs/ @ai-dynamo/dynamo-docs -/fern/ @ai-dynamo/dynamo-docs -/examples/**/*.md @ai-dynamo/dynamo-docs -/recipes/**/*.md @ai-dynamo/dynamo-docs diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md deleted file mode 100644 index 93ed46b21055..000000000000 --- a/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,134 +0,0 @@ - -# Contributor Covenant Code of Conduct - -## Our Pledge - -We as members, contributors, and leaders pledge to make participation in our -community a harassment-free experience for everyone, regardless of age, body -size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socio-economic status, -nationality, personal appearance, race, caste, color, religion, or sexual -identity and orientation. - -We pledge to act and interact in ways that contribute to an open, welcoming, -diverse, inclusive, and healthy community. - -## Our Standards - -Examples of behavior that contributes to a positive environment for our -community include: - -* Demonstrating empathy and kindness toward other people -* Being respectful of differing opinions, viewpoints, and experiences -* Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologizing to those affected by our mistakes, - and learning from the experience -* Focusing on what is best not just for us as individuals, but for the overall - community - -Examples of unacceptable behavior include: - -* The use of sexualized language or imagery, and sexual attention or advances of - any kind -* Trolling, insulting or derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or email address, - without their explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Enforcement Responsibilities - -Community leaders are responsible for clarifying and enforcing our standards of -acceptable behavior and will take appropriate and fair corrective action in -response to any behavior that they deem inappropriate, threatening, offensive, -or harmful. - -Community leaders have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are -not aligned to this Code of Conduct, and will communicate reasons for moderation -decisions when appropriate. - -## Scope - -This Code of Conduct applies within all community spaces, and also applies when -an individual is officially representing the community in public spaces. -Examples of representing our community include using an official e-mail address, -posting via an official social media account, or acting as an appointed -representative at an online or offline event. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported through Github's report functionality. - -All complaints will be reviewed and investigated promptly and fairly by the -administrators of this repository. - -All community leaders are obligated to respect the privacy and security of the -reporter of any incident. - -## Enforcement Guidelines - -Community leaders will follow these Community Impact Guidelines in determining -the consequences for any action they deem in violation of this Code of Conduct: - -### 1. Correction - -**Community Impact**: Use of inappropriate language or other behavior deemed -unprofessional or unwelcome in the community. - -**Consequence**: A private, written warning from community leaders, providing -clarity around the nature of the violation and an explanation of why the -behavior was inappropriate. A public apology may be requested. - -### 2. Warning - -**Community Impact**: A violation through a single incident or series of -actions. - -**Consequence**: A warning with consequences for continued behavior. No -interaction with the people involved, including unsolicited interaction with -those enforcing the Code of Conduct, for a specified period of time. This -includes avoiding interactions in community spaces as well as external channels -like social media. Violating these terms may lead to a temporary or permanent -ban. - -### 3. Temporary Ban - -**Community Impact**: A serious violation of community standards, including -sustained inappropriate behavior. - -**Consequence**: A temporary ban from any sort of interaction or public -communication with the community for a specified period of time. No public or -private interaction with the people involved, including unsolicited interaction -with those enforcing the Code of Conduct, is allowed during this period. -Violating these terms may lead to a permanent ban. - -### 4. Permanent Ban - -**Community Impact**: Demonstrating a pattern of violation of community -standards, including sustained inappropriate behavior, harassment of an -individual, or aggression toward or disparagement of classes of individuals. - -**Consequence**: A permanent ban from any sort of public interaction within the -community. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 2.1, available at -[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. - -Community Impact Guidelines were inspired by -[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. - -For answers to common questions about this code of conduct, see the FAQ at -[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at -[https://www.contributor-covenant.org/translations][translations]. - -[homepage]: https://www.contributor-covenant.org -[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html -[Mozilla CoC]: https://github.com/mozilla/diversity -[FAQ]: https://www.contributor-covenant.org/faq -[translations]: https://www.contributor-covenant.org/translations \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 497448336a0b..000000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,44 +0,0 @@ - - -# Contributing to Dynamo - -Thank you for your interest in contributing to Dynamo! - -For the full contribution guide — including how to get started, build from source, submit PRs, and what to expect during review — see: - -**[Contribution Guide](https://docs.nvidia.com/dynamo/getting-started/contribution-guide)** (docs site) - -Or view the source: [`docs/contribution-guide.md`](docs/contribution-guide.md) - -## Quick Links - -- [Good first issues](https://github.com/ai-dynamo/dynamo/labels/good-first-issue) -- [Help wanted](https://github.com/ai-dynamo/dynamo/labels/help-wanted) -- [Open a bug report](https://github.com/ai-dynamo/dynamo/issues/new?template=bug_report.yml) -- [Propose a feature](https://github.com/ai-dynamo/dynamo/issues/new?template=feature_request.yml) -- [Design Proposals](https://github.com/ai-dynamo/enhancements) -- [GitHub Discussions](https://github.com/ai-dynamo/dynamo/discussions) -- [CNCF Slack (`#ai-dynamo`)](https://communityinviter.com/apps/cloud-native/cncf) -- [Discord](https://discord.gg/D92uqZRjCZ) -- [Office Hours](https://www.youtube.com/playlist?list=PL5B692fm6--tgryKu94h2Zb7jTFM3Go4X) -- [Community Meetings](https://docs.google.com/document/d/1uR8xD_hlYGwV6QspvSc36k1H-wo1BUcVmFbHH9xlXd8/view) ([Youtube](https://www.youtube.com/@ai-dynamo-community)) -- Weekly (Wed 10:30 AM PT) development community meetings -- [Dynamo Day Recordings](https://nvevents.nvidia.com/dynamoday) - -Dynamo requires all contributions to be signed off with the [Developer Certificate of Origin (DCO)](https://developercertificate.org/). This certifies that you have the right to submit your contribution under the project's [Apache 2.0 license](https://github.com/ai-dynamo/dynamo/blob/main/LICENSE). - -By contributing, you agree that your contributions will be licensed under the [Apache 2.0 License](https://github.com/ai-dynamo/dynamo/blob/main/LICENSE). diff --git a/DCO.md b/DCO.md deleted file mode 100644 index 47b33de097f6..000000000000 --- a/DCO.md +++ /dev/null @@ -1,81 +0,0 @@ -# ✅ Fixing DCO Check Failures - -The **Developer Certificate of Origin (DCO)** check ensures all commits are signed off. - If your PR fails the DCO check, here’s how to fix it. - ---- - -## 🖥️ Option 1: Fix via GitHub Web Editor - ⚠️ Works only if your PR has 1 commit. - -1. Go to your **Pull Request** → **Commits** tab. -2. Click the **⋯ menu** → **Edit commit message**. -3. Add this line at the end of the commit message: - - ```text - Signed-off-by: Your Name - ``` -4. Save changes → GitHub will create a new commit with sign-off. -5. Re-run the DCO check. - -## 📦 Option 2: Fix via GitHub Desktop - -1. Open your branch in GitHub Desktop. -2. Go to Repository → Repository Settings → Commit Behavior. -3. Check ✅ Always sign-off commits. -4. Amend the last commit: - - Right-click the commit → Amend Commit. - - Save again with sign-off enabled. -5. Push with force (if required): - ``` - git push --force-with-lease - ``` - -## 💻 Option 3: Fix via CLI (Multiple Commits) - -1. Enable sign-off in your config: - ``` - git config --global user.name "Your Name" - git config --global user.email "your.email@example.com" - ``` -2. Re-sign commits interactively: - ``` - git rebase -i HEAD~N - ``` - Replace N with the number of commits to fix. - Mark commits as edit, then run: - ``` - git commit --amend --signoff - git rebase --continue - ``` -3. Push with: - ``` - git push --force-with-lease - ``` - -## 🔀 If Your Branch Is Messy After Syncing with main - -- Simplest fix: squash all commits into a single new signed commit (via Desktop or CLI). -- Alternatively, ask a maintainer to Squash and Merge with a sign-off on merge. - -## ✨ Pro Tips -This ensures you’ll never fail DCO again. - -- Use the -s flag when committing from CLI: - ``` - git commit -s -m "Your commit message" - ``` -- Turn on Always sign-off commits in your client (GitHub Desktop or Git CLI). - 1. GitHub Desktop - Turn on Always sign-off commits in your client. - - 2. Git CLI - You can always sign-off commits automatically using a commit template (NOTE: This will only work if you use enter your commit message interactively with `git commit`, and will _not_ work with `git commit -m ""`): - 1. Create ~/.git-commit-template.txt with: - ``` - Signed-off-by: Your Name - ``` - 2. Tell Git to use it: - ``` - git config --global commit.template ~/.git-commit-template.txt - ``` diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 1ed6128b8bf7..000000000000 --- a/LICENSE +++ /dev/null @@ -1,210 +0,0 @@ - NOTICE: The test data files under ./lib/llm/tests/data/deepseek-v3.2 are - derived from the DeepSeek-V3.2 model repository (originally developed by - DeepSeek). The original files were obtained from: - https://huggingface.co/deepseek-ai/DeepSeek-V3.2 (commit c69397ecfd1fd142e90e3fbad51f4c7e40b9f3d3) - These files are licensed under the MIT License. The full text of the MIT - License can be found in ./lib/llm/tests/data/deepseek-v3.2/LICENSE. - The rest of this codebase is licensed under the Apache License 2.0 as - described below. - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/README.zh-CN.md b/README.zh-CN.md deleted file mode 100644 index 1a060da91fa9..000000000000 --- a/README.zh-CN.md +++ /dev/null @@ -1,275 +0,0 @@ - - -![Dynamo 横幅](./docs/assets/img/dynamo-frontpage-banner.png) - -[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -[![GitHub Release](https://img.shields.io/github/v/release/ai-dynamo/dynamo)](https://github.com/ai-dynamo/dynamo/releases/latest) -[![PyPI](https://img.shields.io/pypi/v/ai-dynamo)](https://pypi.org/project/ai-dynamo/) -[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/ai-dynamo/dynamo) -[![Discord](https://dcbadge.limes.pink/api/server/D92uqZRjCZ?style=flat)](https://discord.gg/D92uqZRjCZ) -![Community Contributors](https://img.shields.io/badge/community_contributors-70%2B-brightgreen) - -| **[文档](https://docs.nvidia.com/dynamo/)** | **[路线图](https://github.com/ai-dynamo/dynamo/issues/5506)** | **[配方](https://github.com/ai-dynamo/dynamo/tree/main/recipes)** | **[示例](https://github.com/ai-dynamo/dynamo/tree/main/examples)** | **[预构建容器](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/collections/ai-dynamo)** | **[摘要](docs/digest/index.mdx)** | **[设计提案](https://github.com/ai-dynamo/enhancements)** | **[如何贡献](#社区与贡献)** | - - -

- English | 简体中文 -

- -# Dynamo - - -> [!NOTE] -> **DeepSeek-V4 首日配方已可用。** [DeepSeek-V4-Pro](recipes/deepseek-v4/deepseek-v4-pro/) 和 [DeepSeek-V4-Flash](recipes/deepseek-v4/deepseek-v4-flash/) 的 Kubernetes 部署路径已经过测试，并已在 **vLLM** 与 **SGLang** 两个后端合并到 main；预构建的 SGLang 容器镜像也已发布到 NGC。 - -**开源的数据中心级推理栈。** Dynamo 是位于推理引擎之上的编排层。它不会取代 SGLang、TensorRT-LLM 或 vLLM，而是把它们组织成一个协同工作的多节点推理系统。分离式服务、智能路由、多层 KV 缓存和自动扩缩容协同工作，为 LLM、推理、多模态和视频生成工作负载最大化吞吐并最小化延迟。 - -使用 Rust 构建以获得性能，使用 Python 扩展以获得灵活性。 - -## 何时使用 Dynamo - -- 你正在跨 **多个 GPU 或节点** 提供 LLM 服务，并且需要协调它们 -- 你希望使用 **KV 感知路由** 来避免重复的预填充计算 -- 你需要 **独立扩缩容预填充和解码**（分离式服务） -- 你希望通过 **自动扩缩容** 在最低总体拥有成本（TCO）下满足延迟 SLA -- 你需要在启动新副本时获得 **快速冷启动** - -如果你只是在单个 GPU 上运行单个模型，那么单独使用推理引擎通常已经足够。 - -**功能支持概览：** - -| | [SGLang](https://docs.nvidia.com/dynamo/backends/sg-lang) | [TensorRT-LLM](https://docs.nvidia.com/dynamo/backends/tensor-rt-llm) | [vLLM](https://docs.nvidia.com/dynamo/backends/v-llm) | -|---|:----:|:----------:|:--:| -| [**分离式服务**](docs/design-docs/disagg-serving.zh-CN.md) | ✅ | ✅ | ✅ | -| [**KV 感知路由**](docs/components/router/README.zh-CN.md) | ✅ | ✅ | ✅ | -| [**基于 SLA 的 Planner**](docs/components/planner/planner-guide.zh-CN.md) | ✅ | ✅ | ✅ | -| [**KVBM**](docs/components/kvbm/README.zh-CN.md) | 🚧 | ✅ | ✅ | -| [**多模态**](https://docs.nvidia.com/dynamo/user-guides/multimodal) | ✅ | ✅ | ✅ | -| [**工具调用**](docs/tool-calling/README.zh-CN.md) | ✅ | ✅ | ✅ | - -> **[完整功能矩阵 →](https://docs.nvidia.com/dynamo/resources/feature-matrix)** — LoRA、请求迁移、推测解码以及功能之间的交互。 - -## 关键结果 - -| 结果 | 背景 | -|--------|------| -| 单 GPU 吞吐提升 **7x** | 在 GB200 NVL72 上使用 Dynamo 运行 DeepSeek R1，相比未使用 Dynamo 的 B200（[InferenceX](https://inferencex.semianalysis.com/)） | -| 模型启动速度提升 **7x** | ModelExpress 权重流式加载（H200 上的 DeepSeek-V3） | -| 首 token 时间快 **2x** | KV 感知路由，Qwen3-Coder 480B（[Baseten 基准](https://www.baseten.co/blog/how-baseten-achieved-2x-faster-inference-with-nvidia-dynamo/)） | -| SLA 违约减少 **80%** | Planner 自动扩缩容，同时 TCO 降低 5%（[Alibaba APSARA 2025 @ 2:50:00](https://yunqi.aliyun.com/2025/session?agendaId=6062)） | -| 吞吐提升 **750x** | GB300 NVL72 上的 DeepSeek-R1（[InferenceXv2](https://inferencex.semianalysis.com/)） | - - -## Dynamo 的作用 - -大多数推理引擎优化的是单个 GPU 或单个节点。Dynamo 是 **位于这些引擎之上的编排层**，它把一组 GPU 转化为协同工作的推理系统。 - -

- Dynamo 架构概览 -

- -**[架构深入解析 →](docs/design-docs/architecture.zh-CN.md)** - -### 核心能力 - -| 能力 | 作用 | 价值 | -|------|------|------| -| [**分离式预填充/解码**](docs/design-docs/disagg-serving.zh-CN.md) | 将预填充和解码拆分为可独立扩缩容的 GPU 池 | 最大化 GPU 利用率；每个阶段都运行在针对其工作负载调优的硬件上 | -| [**KV 感知路由**](docs/components/router/README.zh-CN.md) | 根据 worker 负载和 KV 缓存重叠度路由请求 | 消除冗余预填充计算，TTFT 快 2x | -| [**KV Block Manager (KVBM)**](docs/components/kvbm/README.zh-CN.md) | 在 GPU → CPU → SSD → 远程存储之间卸载 KV 缓存 | 将有效上下文长度扩展到 GPU 显存之外 | -| [**ModelExpress**](https://github.com/ai-dynamo/modelexpress) | 通过 NIXL/NVLink 在 GPU 之间流式传输模型权重 | 新副本冷启动快 7x | -| [**Planner**](docs/components/planner/planner-guide.zh-CN.md) | 由 SLA 驱动的自动扩缩容器，可分析工作负载并调整资源池规模 | 以最低总体拥有成本（TCO）满足延迟目标 | -| [**Grove**](https://github.com/ai-dynamo/grove) | 面向拓扑感知 gang scheduling 的 K8s operator（NVL72） | 在机架、主机和 NUMA 节点之间优化放置工作负载 | -| [**AIConfigurator**](https://github.com/ai-dynamo/aiconfigurator) | 在数秒内模拟 10K+ 部署配置 | 无需消耗 GPU 小时即可找到最优服务配置 | -| [**容错**](docs/fault-tolerance/request-migration.zh-CN.md) | 金丝雀健康检查 + 运行中请求迁移 | worker 可以失败，但用户请求不应失败 | - -### 1.0 新功能 - -- **零配置部署（[DGDR](https://docs.nvidia.com/dynamo/kubernetes-deployment/deploy-models/dgdr-reference)）** *(beta)：* 在一个 YAML 中指定模型、硬件和 SLA；AIConfigurator 自动分析工作负载，Planner 优化拓扑，然后由 Dynamo 完成部署 -- **Agentic inference：** 按请求提供延迟优先级、预期输出长度和缓存固定 TTL 等提示。集成 [LangChain](https://docs.langchain.com/oss/python/integrations/chat/nvidia_ai_endpoints#use-with-nvidia-dynamo) + [NeMo Agent Toolkit](https://github.com/NVIDIA/NeMo-Agent-Toolkit) -- **多模态 E/P/D：** 带 embedding cache 的分离式 encode/prefill/decode；图像工作负载 TTFT 快 30% -- **视频生成：** 原生支持 [FastVideo](https://github.com/hao-ai-lab/FastVideo) + [SGLang Diffusion](https://lmsys.org/blog/2026-02-16-sglang-diffusion-advanced-optimizations/)；单张 B200 上实现实时 1080p -- **K8s Inference Gateway 插件：** 在标准 Kubernetes gateway 内提供 KV 感知路由 -- **存储层 KV 卸载：** 支持 S3/Azure blob，并通过全局 KV 事件提供集群级缓存可见性 - -## 快速开始 - -### 选项 A：容器（最快） - -```bash -# 拉取预构建容器（SGLang 示例） -docker run --gpus all --network host --rm -it nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.1 - -# 在容器内启动 frontend 和 worker -python3 -m dingo.frontend --http-port 8000 --discovery-backend file > /dev/null 2>&1 & -python3 -m dingo.sglang --model-path Qwen/Qwen3-0.6B --discovery-backend file & - -# 发送请求 -curl -s localhost:8000/v1/chat/completions -H "Content-Type: application/json" -d '{ - "model": "Qwen/Qwen3-0.6B", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100 -}' | jq -``` - -另有 [`tensorrtllm-runtime:1.2.1`](https://docs.nvidia.com/dynamo/resources/release-artifacts) 和 [`vllm-runtime:1.2.1`](https://docs.nvidia.com/dynamo/resources/release-artifacts) 可用。 - -### 选项 B：从 PyPI 安装 - -安装 [uv](https://github.com/astral-sh/uv)（`curl -LsSf https://astral.sh/uv/install.sh | sh`），然后运行： - -```bash -uv pip install --prerelease=allow "ai-dynamo[sglang]" # 或 [vllm] -``` - -> **注意：** TensorRT-LLM 需要配合 `--extra-index-url https://pypi.nvidia.com` 使用 `pip`。TRT-LLM 专用说明请参阅[安装指南](docs/getting-started/local-installation.zh-CN.md)。 - -然后按上面的方式启动 frontend 和一个 worker。系统依赖和后端专用说明请参阅[完整安装指南](docs/getting-started/local-installation.zh-CN.md)。 - -### 选项 C：Kubernetes（推荐） - -对于生产级多节点集群，安装 [Dynamo Platform](https://docs.nvidia.com/dynamo/kubernetes-deployment/start-here/installation-guide)，并使用单个 manifest 部署： - -```yaml -# 零配置部署：指定模型 + SLA，剩余工作由 Dynamo 处理 -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeploymentRequest -metadata: - name: my-model -spec: - model: Qwen/Qwen3-0.6B - backend: vllm - sla: - ttft: 200.0 # ms - itl: 20.0 # ms - autoApply: true -``` - -常见模型的预构建配方： - -| 模型 | 框架 | 模式 | 配方 | -|------|------|------|------| -| Llama-3-70B | vLLM | 聚合式 | [查看](recipes/llama-3-70b/vllm/) | -| DeepSeek-R1 | SGLang | 分离式 | [查看](recipes/deepseek-r1/sglang/) | -| Qwen3-32B-FP8 | TensorRT-LLM | 聚合式 | [查看](recipes/qwen3-32b-fp8/trtllm/) | - -完整列表见 [recipes/](recipes/README.md)。云平台专用指南：[AWS EKS](docs/kubernetes/cloud-providers/eks/eks.md) · [Google GKE](docs/kubernetes/cloud-providers/gke/gke.md) · [Azure AKS](docs/kubernetes/cloud-providers/aks/aks.md) · [Amazon ECS](docs/kubernetes/cloud-providers/ecs/ecs.md) - -## 从源码构建 - -适用于希望在本地构建和开发的贡献者。详情请参阅[完整构建指南](docs/getting-started/building-from-source.zh-CN.md)。 - -```bash -# 安装系统依赖（Ubuntu 24.04） -sudo apt install -y build-essential libhwloc-dev libudev-dev pkg-config libclang-dev protobuf-compiler python3-dev cmake - -# 安装 Rust -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh && source $HOME/.cargo/env - -# 创建虚拟环境并构建 -uv venv dynamo && source dynamo/bin/activate -uv pip install pip 'maturin[patchelf]' -cd lib/bindings/python && maturin develop --uv && cd $PROJECT_ROOT -uv pip install -e lib/gpu_memory_service -uv pip install -e . -``` - -> VSCode/Cursor 用户：预配置开发环境请参阅 [`.devcontainer`](.devcontainer/README.md)。 - -## 社区与贡献 - -Dynamo 采用 OSS 优先的开放开发模式。我们欢迎各种形式的贡献。 - -- **[贡献指南](docs/contribution-guide.zh-CN.md)** — 如何贡献代码、文档和配方 -- **[设计提案](https://github.com/ai-dynamo/enhancements)** — 重大功能的 RFC -- **[Office Hours](https://www.youtube.com/playlist?list=PL5B692fm6--tgryKu94h2Zb7jTFM3Go4X)** — 双周会议 -- **[社区会议](https://docs.google.com/document/d/1uR8xD_hlYGwV6QspvSc36k1H-wo1BUcVmFbHH9xlXd8/view)** ([Youtube](https://www.youtube.com/@ai-dynamo-community)) – 每周（Wed 10:30 AM PT）开发者社区会议 -- **[Discord](https://discord.gg/D92uqZRjCZ)** — 与团队和社区交流 -- **[Dynamo Day 录像](https://nvevents.nvidia.com/dynamoday)** — 来自生产用户的深入分享 - -## 最新动态 - -- [03/15] [Dynamo 1.0 发布：生产就绪，并获得强劲社区采用](https://developer.nvidia.com/blog/introducing-nvidia-dynamo-a-low-latency-distributed-inference-framework-for-scaling-reasoning-ai-models/) -- [03/15] [NVIDIA Blackwell Ultra 在 MLPerf 中创下新的推理纪录](https://developer.nvidia.com/blog/nvidia-blackwell-ultra-sets-new-inference-records-in-mlperf-debut/) -- [03/15] [NVIDIA Blackwell 在 SemiAnalysis InferenceMax 基准中领先](https://developer.nvidia.com/blog/nvidia-blackwell-leads-on-new-semianalysis-inferencemax-benchmarks/) -- [12/05] [Moonshot AI 的 Kimi K2 在 GB200 上借助 Dynamo 实现 10x 推理加速](https://quantumzeitgeist.com/kimi-k2-nvidia-ai-ai-breakthrough/) -- [12/02] [Mistral AI 使用 Dynamo 让 Mistral Large 3 推理速度提升 10x](https://www.marktechpost.com/2025/12/02/nvidia-and-mistral-ai-bring-10x-faster-inference-for-the-mistral-3-family-on-gb200-nvl72-gpu-systems/) -- [11/20] [Dell 将 PowerScale 与 NIXL 集成，使 TTFT 快 19x](https://www.dell.com/en-us/dt/corporate/newsroom/announcements/detailpage.press-releases~usa~2025~11~dell-technologies-and-nvidia-advance-enterprise-ai-innovation.htm) - -
-较早动态 - -Dynamo 提供完整的基准测试工具： - -- **[基准测试指南](docs/benchmarks/benchmarking.md)** – 使用 AIPerf 比较部署拓扑 -- **[SLA 驱动部署](docs/components/planner/planner-guide.zh-CN.md)** – 优化部署以满足 SLA 要求 - -## Frontend OpenAPI 规范 - -兼容 OpenAI 的 frontend 会在 `/openapi.json` 暴露 OpenAPI 3 规范。无需运行服务器即可生成： - -```bash -cargo run -p dynamo-llm --bin generate-frontend-openapi -``` - -该命令会写入 `docs/reference/api/openapi.json`。 - -## 服务发现与消息传递 - -Dynamo 使用 TCP 进行组件间通信。在 Kubernetes 上，原生资源（[CRDs + EndpointSlices](docs/kubernetes/service-discovery.md)）负责服务发现。对大多数部署来说，外部服务是可选的： - -| 部署 | etcd | NATS | 说明 | -|------|------|------|------| -| **本地开发** | ❌ 不需要 | ❌ 不需要 | 传入 `--discovery-backend file`；vLLM 还需要 `--kv-events-config '{"enable_kv_cache_events": false}'` | -| **Kubernetes** | ❌ 不需要 | ❌ 不需要 | K8s 原生服务发现；TCP 请求平面 | - -> **注意：** KV 感知路由不需要 NATS。需要基于事件的 cache 状态跟踪时可以启用 KV event；如果不需要外部事件基础设施，可以使用 `--no-router-kv-events` 进行基于预测的路由。 - -对于选择使用 etcd 或 NATS JetStream 模式的 Slurm 或其他分布式部署： - -- [etcd](https://etcd.io/) 可以直接以 `./etcd` 运行。 -- [nats](https://nats.io/) 需要启用 JetStream：`nats-server -js`。 - -快速启动二者：`docker compose -f dev/docker-compose.yml up -d` - -## 更多动态 - -- [11/20] [Dell 将 PowerScale 与 Dynamo 的 NIXL 集成，使 TTFT 快 19x](https://www.dell.com/en-us/dt/corporate/newsroom/announcements/detailpage.press-releases~usa~2025~11~dell-technologies-and-nvidia-advance-enterprise-ai-innovation.htm) -- [11/20] [WEKA 与 NVIDIA 合作，为 Dynamo 提供 KV 缓存存储方案](https://siliconangle.com/2025/11/20/nvidia-weka-kv-cache-solution-ai-inferencing-sc25/) -- [11/13] [Dynamo Office Hours 播放列表](https://www.youtube.com/playlist?list=PL5B692fm6--tgryKu94h2Zb7jTFM3Go4X) -- [10/16] [Baseten 如何借助 NVIDIA Dynamo 实现 2x 推理加速](https://www.baseten.co/blog/how-baseten-achieved-2x-faster-inference-with-nvidia-dynamo/) -- [12/01] [InfoQ：NVIDIA Dynamo 简化 LLM 推理的 Kubernetes 部署](https://www.infoq.com/news/2025/12/nvidia-dynamo-kubernetes/) - -
- -## 参考资料 - -- **[支持矩阵](https://docs.nvidia.com/dynamo/resources/support-matrix)** — 硬件、操作系统、CUDA 和后端版本 -- **[功能矩阵](https://docs.nvidia.com/dynamo/resources/feature-matrix)** — 详细后端兼容性 -- **[发布产物](https://docs.nvidia.com/dynamo/resources/release-artifacts)** — 容器、wheel、Helm chart -- **[服务发现](https://docs.nvidia.com/dynamo/kubernetes-deployment/advanced-platform/service-discovery)** — K8s 原生、etcd 与基于文件的服务发现对比 -- **[基准测试指南](https://docs.nvidia.com/dynamo/user-guides/benchmarking)** — 使用 AIPerf 比较部署拓扑 - - -[disagg]: docs/design-docs/disagg-serving.zh-CN.md -[kv-routing]: docs/components/router/README.zh-CN.md -[planner]: docs/components/planner/planner-guide.zh-CN.md -[kvbm]: docs/components/kvbm/README.zh-CN.md -[migration]: docs/fault-tolerance/request-migration.zh-CN.md -[lora]: examples/backends/vllm/deploy/lora/README.md -[tools]: docs/tool-calling/README.zh-CN.md diff --git a/SECURITY.md b/SECURITY.md deleted file mode 100644 index f56e6311ffb3..000000000000 --- a/SECURITY.md +++ /dev/null @@ -1,31 +0,0 @@ - - -# Report a Security Vulnerability - -To report a potential security vulnerability in any NVIDIA product, please use either: -* This web form: [Security Vulnerability Submission Form](https://www.nvidia.com/en-us/support/submit-security-vulnerability/), or -* Send email to: [NVIDIA PSIRT](mailto:psirt@nvidia.com) - -If reporting a potential vulnerability via email, please encrypt it using NVIDIA’s public PGP key ([see PGP Key page](https://www.nvidia.com/en-us/security/pgp-key/)) and include the following information: -1. Product/Driver name and version/branch that contains the vulnerability -2. Type of vulnerability (code execution, denial of service, buffer overflow, etc.) -3. Instructions to reproduce the vulnerability -4. Proof-of-concept or exploit code -5. Potential impact of the vulnerability, including how an attacker could exploit the vulnerability - -See https://www.nvidia.com/en-us/security/ for past NVIDIA Security Bulletins and Notices. diff --git a/benchmarks/frontend/README.md b/benchmarks/frontend/README.md index d0a2d1a18f6c..815041f9f853 100644 --- a/benchmarks/frontend/README.md +++ b/benchmarks/frontend/README.md @@ -43,7 +43,7 @@ Local mode starts a mocker backend and frontend process on the current machine, **Prerequisites:** -- `dynamo.mocker` and `dingo.frontend` installed (from the Dynamo repo) +- `dingo.mocker` and `dingo.frontend` installed (from the Dynamo repo) - `aiperf` installed and on `$PATH` - A HuggingFace model accessible locally (default: `Qwen/Qwen3-0.6B`) diff --git a/benchmarks/frontend/dgd/templates/mocker.yaml b/benchmarks/frontend/dgd/templates/mocker.yaml index 8cacdddd3951..aa69844879d3 100644 --- a/benchmarks/frontend/dgd/templates/mocker.yaml +++ b/benchmarks/frontend/dgd/templates/mocker.yaml @@ -64,7 +64,7 @@ ${WORKER_IMAGE_PULL_SECRETS_BLOCK} - -c args: - | - python3 -m dynamo.mocker \ + python3 -m dingo.mocker \ --model-path "${MODEL_PATH}" \ --model-name "${MODEL_NAME}" \ --num-workers ${NUM_WORKERS} \ diff --git a/benchmarks/frontend/scripts/run_perf.sh b/benchmarks/frontend/scripts/run_perf.sh index cb22ffde9a63..d137f136da02 100755 --- a/benchmarks/frontend/scripts/run_perf.sh +++ b/benchmarks/frontend/scripts/run_perf.sh @@ -15,7 +15,7 @@ # Output: artifacts/obs_YYYYMMDD_HHMMSS/ with subdirs for each data source. # # Prerequisites: -# - dynamo.mocker and dingo.frontend installed +# - dingo.mocker and dingo.frontend installed # - aiperf installed # - Optional: nsys, perf, bpftrace, flamegraph tools (auto-detected) @@ -409,7 +409,7 @@ for MN in "${MODEL_NAMES[@]}"; do fi MN_SAFE="${MN//\//_}" - HF_HUB_OFFLINE=1 DYN_SYSTEM_PORT=$WORKER_PORT DYN_EVENT_PLANE="$EVENT_PLANE" python -m dynamo.mocker "${MOCKER_ARGS[@]}" \ + HF_HUB_OFFLINE=1 DYN_SYSTEM_PORT=$WORKER_PORT DYN_EVENT_PLANE="$EVENT_PLANE" python -m dingo.mocker "${MOCKER_ARGS[@]}" \ > "$OUTPUT_DIR/logs/mocker_${MN_SAFE}_${i}.log" 2>&1 & ALL_PIDS+=($!) echo " Worker $WORKER_IDX ($MN #$i): PID ${ALL_PIDS[-1]}, port $WORKER_PORT" diff --git a/benchmarks/mocker/bench_aic_rust_callback.py b/benchmarks/mocker/bench_aic_rust_callback.py index 84df04c7c341..60876dc46cdd 100644 --- a/benchmarks/mocker/bench_aic_rust_callback.py +++ b/benchmarks/mocker/bench_aic_rust_callback.py @@ -57,8 +57,8 @@ import sys import time -from dynamo.mocker import MockEngineArgs -from dynamo.replay import run_synthetic_trace_replay +from dingo.mocker import MockEngineArgs +from dingo.replay import run_synthetic_trace_replay # AIC tuple — matches components/.../tests/unit/test_replay_aic_parity.py so the # perf database is one already exercised by CI. diff --git a/benchmarks/multimodal/embedding_transfer/protocol.py b/benchmarks/multimodal/embedding_transfer/protocol.py index 1811010c96b0..922b99f8b5dd 100644 --- a/benchmarks/multimodal/embedding_transfer/protocol.py +++ b/benchmarks/multimodal/embedding_transfer/protocol.py @@ -3,8 +3,8 @@ from pydantic import BaseModel -from dynamo.common.constants import EmbeddingTransferMode -from dynamo.common.multimodal.embedding_transfer import TransferRequest +from dingo.common.constants import EmbeddingTransferMode +from dingo.common.multimodal.embedding_transfer import TransferRequest class TransferConfig(BaseModel): diff --git a/benchmarks/multimodal/embedding_transfer/receiver.py b/benchmarks/multimodal/embedding_transfer/receiver.py index 1048acf9ba7e..4cdab1b14190 100644 --- a/benchmarks/multimodal/embedding_transfer/receiver.py +++ b/benchmarks/multimodal/embedding_transfer/receiver.py @@ -7,7 +7,7 @@ import uvloop from protocol import BatchTransferRequest, EmbeddingTransferMode, TransferConfig -from dynamo.common.multimodal.embedding_transfer import ( +from dingo.common.multimodal.embedding_transfer import ( LocalEmbeddingReceiver, NixlReadEmbeddingReceiver, NixlWriteEmbeddingReceiver, diff --git a/benchmarks/multimodal/embedding_transfer/sender.py b/benchmarks/multimodal/embedding_transfer/sender.py index 56522eddb71d..3b5c083602ac 100644 --- a/benchmarks/multimodal/embedding_transfer/sender.py +++ b/benchmarks/multimodal/embedding_transfer/sender.py @@ -8,7 +8,7 @@ import uvloop from protocol import BatchTransferRequest, EmbeddingTransferMode, TransferConfig -from dynamo.common.multimodal.embedding_transfer import ( +from dingo.common.multimodal.embedding_transfer import ( LocalEmbeddingSender, NixlReadEmbeddingSender, NixlWriteEmbeddingSender, diff --git a/benchmarks/multimodal/http/runner.py b/benchmarks/multimodal/http/runner.py index 38d2fec8a546..2bcca8192cbd 100644 --- a/benchmarks/multimodal/http/runner.py +++ b/benchmarks/multimodal/http/runner.py @@ -17,7 +17,7 @@ import uuid from dataclasses import dataclass -from dynamo.common.http import close_http_client, fetch_bytes +from dingo.common.http import close_http_client, fetch_bytes @dataclass diff --git a/benchmarks/multimodal/http/sweep.py b/benchmarks/multimodal/http/sweep.py index 63fa4f082b77..cf65e2481893 100644 --- a/benchmarks/multimodal/http/sweep.py +++ b/benchmarks/multimodal/http/sweep.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Sweep aiohttp vs httpx through dynamo.common.http. +"""Sweep aiohttp vs httpx through dingo.common.http. Rate-limited emitter. For each (request_rate, server-processing-time-mean-ms) pair, brings up a local media server with that processing-time-mean and diff --git a/benchmarks/request_trace/README.md b/benchmarks/request_trace/README.md index 8a170518c802..9173cbd48db2 100644 --- a/benchmarks/request_trace/README.md +++ b/benchmarks/request_trace/README.md @@ -51,7 +51,7 @@ request can be replayed by Dynamo mock workers. Pass the original JSONL or JSONL.GZ shards directly to the replay harness: ```bash -python -m dynamo.replay /tmp/dynamo-request-trace.*.jsonl.gz \ +python -m dingo.replay /tmp/dynamo-request-trace.*.jsonl.gz \ --trace-format dynamo \ --replay-mode offline \ --router-mode kv_router \ diff --git a/benchmarks/router/README.md b/benchmarks/router/README.md index 1b3779aabddd..ec4d2ceedb76 100644 --- a/benchmarks/router/README.md +++ b/benchmarks/router/README.md @@ -111,13 +111,13 @@ NAMESPACE="test-disagg" MODEL="Qwen/Qwen3-0.6B" # Terminal 1: Decode mockers (2 workers) -python -m dynamo.mocker --model-path "$MODEL" \ +python -m dingo.mocker --model-path "$MODEL" \ --endpoint "dyn://${NAMESPACE}.backend.generate" \ --disaggregation-mode decode --num-workers 2 \ --speedup-ratio 10 --block-size 16 # Terminal 2: Prefill mockers (2 workers) -python -m dynamo.mocker --model-path "$MODEL" \ +python -m dingo.mocker --model-path "$MODEL" \ --endpoint "dyn://${NAMESPACE}.prefill.generate" \ --disaggregation-mode prefill --num-workers 2 \ --speedup-ratio 10 --block-size 16 diff --git a/benchmarks/router/run_engines.sh b/benchmarks/router/run_engines.sh index 6b1a302647fa..14d5993d53c7 100755 --- a/benchmarks/router/run_engines.sh +++ b/benchmarks/router/run_engines.sh @@ -178,7 +178,7 @@ if [ "$USE_MOCKERS" = true ]; then fi MOCKER_ARGS+=("${EXTRA_ARGS[@]}") - python -m dynamo.mocker "${MOCKER_ARGS[@]}" & + python -m dingo.mocker "${MOCKER_ARGS[@]}" & PIDS+=($!) echo "Started mocker with $NUM_WORKERS workers (PID: $!)" else diff --git a/codespell.txt b/codespell.txt deleted file mode 100644 index 8b3d68a5abab..000000000000 --- a/codespell.txt +++ /dev/null @@ -1,2 +0,0 @@ -dynmo->dynamo -dynemo->dynamo diff --git a/components/src/dynamo/planner/plugins/proto/v1/plugin_pb2.py b/components/src/dynamo/planner/plugins/proto/v1/plugin_pb2.py deleted file mode 100644 index c9709331726f..000000000000 --- a/components/src/dynamo/planner/plugins/proto/v1/plugin_pb2.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: dynamo/planner/plugins/proto/v1/plugin.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,dynamo/planner/plugins/proto/v1/plugin.proto\x12\x18\x64ynamo.planner.plugin.v1\"\xdc\x02\n\x0fRegisterRequest\x12\x11\n\tplugin_id\x18\x01 \x01(\t\x12\x13\n\x0bplugin_type\x18\x02 \x01(\t\x12\x10\n\x08priority\x18\x03 \x01(\r\x12\x10\n\x08\x65ndpoint\x18\x04 \x01(\t\x12\x0f\n\x07version\x18\x05 \x01(\t\x12\"\n\x1a\x65xecution_interval_seconds\x18\x06 \x01(\x02\x12\x39\n\x0bhold_policy\x18\x07 \x01(\x0e\x32$.dynamo.planner.plugin.v1.HoldPolicy\x12\r\n\x05needs\x18\x08 \x03(\t\x12\x18\n\x10protocol_version\x18\t \x01(\t\x12\x12\n\nauth_token\x18\n \x01(\t\x12 \n\x18requires_produced_fields\x18\r \x03(\t\x12\"\n\x1aobservation_window_seconds\x18\x0e \x01(\x02J\x04\x08\x0b\x10\x0cJ\x04\x08\x0c\x10\r\"`\n\x10RegisterResponse\x12\x10\n\x08\x61\x63\x63\x65pted\x18\x01 \x01(\x08\x12\x15\n\rreject_reason\x18\x02 \x01(\t\x12#\n\x1bnegotiated_protocol_version\x18\x03 \x01(\t\"9\n\x10HeartbeatRequest\x12\x11\n\tplugin_id\x18\x01 \x01(\t\x12\x12\n\nauth_token\x18\x02 \x01(\t\"\x1f\n\x11HeartbeatResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\"J\n\x11UnregisterRequest\x12\x11\n\tplugin_id\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\x12\x12\n\nauth_token\x18\x03 \x01(\t\" \n\x12UnregisterResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\"D\n\x12ListPluginsRequest\x12\x14\n\x0cstage_filter\x18\x01 \x01(\t\x12\x18\n\x10include_disabled\x18\x02 \x01(\x08\"L\n\x13ListPluginsResponse\x12\x35\n\x07plugins\x18\x01 \x03(\x0b\x32$.dynamo.planner.plugin.v1.PluginInfo\"\xc0\x02\n\nPluginInfo\x12\x11\n\tplugin_id\x18\x01 \x01(\t\x12\x13\n\x0bplugin_type\x18\x02 \x01(\t\x12\x10\n\x08priority\x18\x03 \x01(\r\x12\x0f\n\x07version\x18\x04 \x01(\t\x12\x18\n\x10protocol_version\x18\x05 \x01(\t\x12\x0f\n\x07\x65nabled\x18\x06 \x01(\x08\x12\x12\n\nis_builtin\x18\x07 \x01(\x08\x12\x11\n\ttransport\x18\x08 \x01(\t\x12=\n\rcircuit_state\x18\t \x01(\x0e\x32&.dynamo.planner.plugin.v1.CircuitState\x12\x19\n\x11\x65valuations_total\x18\n \x01(\x04\x12 \n\x18last_call_at_seconds_ago\x18\x0b \x01(\x01\x12\x19\n\x11\x63\x61\x63he_age_seconds\x18\x0c \x01(\x01\"\x89\x03\n\x0fPipelineContext\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65\x63ision_id\x18\x02 \x01(\t\x12\x44\n\x0cobservations\x18\x03 \x01(\x0b\x32).dynamo.planner.plugin.v1.ObservationDataH\x00\x88\x01\x01\x12\x42\n\x0bpredictions\x18\x04 \x01(\x0b\x32(.dynamo.planner.plugin.v1.PredictionDataH\x01\x88\x01\x01\x12@\n\x08proposal\x18\x05 \x01(\x0b\x32).dynamo.planner.plugin.v1.ScalingProposalH\x02\x88\x01\x01\x12\x43\n\x0b\x63onstrained\x18\x06 \x01(\x0b\x32).dynamo.planner.plugin.v1.ScalingProposalH\x03\x88\x01\x01\x42\x0f\n\r_observationsB\x0e\n\x0c_predictionsB\x0b\n\t_proposalB\x0e\n\x0c_constrained\"\xe3\x01\n\x0fObservationData\x12>\n\x07traffic\x18\x01 \x01(\x0b\x32(.dynamo.planner.plugin.v1.TrafficMetricsH\x00\x88\x01\x01\x12\x33\n\x03\x66pm\x18\x02 \x01(\x0b\x32!.dynamo.planner.plugin.v1.FpmDataH\x01\x88\x01\x01\x12;\n\x07workers\x18\x03 \x01(\x0b\x32%.dynamo.planner.plugin.v1.WorkerStateH\x02\x88\x01\x01\x42\n\n\x08_trafficB\x06\n\x04_fpmB\n\n\x08_workers\"\xa7\x01\n\x0eTrafficMetrics\x12\x12\n\nduration_s\x18\x01 \x01(\x01\x12\x0f\n\x07num_req\x18\x02 \x01(\x01\x12\x0b\n\x03isl\x18\x03 \x01(\x01\x12\x0b\n\x03osl\x18\x04 \x01(\x01\x12\x18\n\x0bkv_hit_rate\x18\x05 \x01(\x01H\x00\x88\x01\x01\x12\x1a\n\raccept_length\x18\x06 \x01(\x01H\x01\x88\x01\x01\x42\x0e\n\x0c_kv_hit_rateB\x10\n\x0e_accept_length\"\x94\x02\n\x07\x46pmData\x12N\n\x0fprefill_engines\x18\x01 \x03(\x0b\x32\x35.dynamo.planner.plugin.v1.FpmData.PrefillEnginesEntry\x12L\n\x0e\x64\x65\x63ode_engines\x18\x02 \x03(\x0b\x32\x34.dynamo.planner.plugin.v1.FpmData.DecodeEnginesEntry\x1a\x35\n\x13PrefillEnginesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\x1a\x34\n\x12\x44\x65\x63odeEnginesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\"\xdf\x02\n\x0bWorkerState\x12\x1a\n\rready_prefill\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x19\n\x0cready_decode\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1d\n\x10\x65xpected_prefill\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x1c\n\x0f\x65xpected_decode\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12(\n\x1bprefill_scaling_in_progress\x18\x05 \x01(\x08H\x04\x88\x01\x01\x12\'\n\x1a\x64\x65\x63ode_scaling_in_progress\x18\x06 \x01(\x08H\x05\x88\x01\x01\x42\x10\n\x0e_ready_prefillB\x0f\n\r_ready_decodeB\x13\n\x11_expected_prefillB\x12\n\x10_expected_decodeB\x1e\n\x1c_prefill_scaling_in_progressB\x1d\n\x1b_decode_scaling_in_progress\"\xb2\x02\n\x0ePredictionData\x12\x1e\n\x11predicted_num_req\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x1a\n\rpredicted_isl\x18\x02 \x01(\x01H\x01\x88\x01\x01\x12\x1a\n\rpredicted_osl\x18\x03 \x01(\x01H\x02\x88\x01\x01\x12\x0e\n\x06source\x18\x04 \x01(\t\x12\"\n\x15predicted_kv_hit_rate\x18\x05 \x01(\x01H\x03\x88\x01\x01\x12$\n\x17predicted_accept_length\x18\x06 \x01(\x01H\x04\x88\x01\x01\x42\x14\n\x12_predicted_num_reqB\x10\n\x0e_predicted_islB\x10\n\x0e_predicted_oslB\x18\n\x16_predicted_kv_hit_rateB\x1a\n\x18_predicted_accept_length\"m\n\x0fScalingProposal\x12:\n\x07targets\x18\x01 \x03(\x0b\x32).dynamo.planner.plugin.v1.ComponentTarget\x12\x0e\n\x06reason\x18\x02 \x01(\t\x12\x0e\n\x06source\x18\x03 \x01(\t\"\x8d\x01\n\x0f\x43omponentTarget\x12\x1a\n\x12sub_component_type\x18\x01 \x01(\t\x12\x15\n\x08replicas\x18\x03 \x01(\x05H\x00\x88\x01\x01\x12\x34\n\x04type\x18\x04 \x01(\x0e\x32&.dynamo.planner.plugin.v1.OverrideTypeB\x0b\n\t_replicasJ\x04\x08\x02\x10\x03\"\\\n\x0eOverrideResult\x12:\n\x07targets\x18\x01 \x03(\x0b\x32).dynamo.planner.plugin.v1.ComponentTarget\x12\x0e\n\x06reason\x18\x02 \x01(\t\"\x0e\n\x0c\x41\x63\x63\x65ptResult\"\x1e\n\x0cRejectResult\x12\x0e\n\x06reason\x18\x01 \x01(\t\"Q\n\x13PredictStageRequest\x12:\n\x07\x63ontext\x18\x01 \x01(\x0b\x32).dynamo.planner.plugin.v1.PipelineContext\"t\n\x14PredictStageResponse\x12=\n\x0bpredictions\x18\x01 \x01(\x0b\x32(.dynamo.planner.plugin.v1.PredictionData\x12\x0e\n\x06reason\x18\x02 \x01(\t\x12\r\n\x05\x66inal\x18\x03 \x01(\x08\"Q\n\x13ProposeStageRequest\x12:\n\x07\x63ontext\x18\x01 \x01(\x0b\x32).dynamo.planner.plugin.v1.PipelineContext\"\xe1\x01\n\x14ProposeStageResponse\x12\x38\n\x06\x61\x63\x63\x65pt\x18\x01 \x01(\x0b\x32&.dynamo.planner.plugin.v1.AcceptResultH\x00\x12<\n\x08override\x18\x02 \x01(\x0b\x32(.dynamo.planner.plugin.v1.OverrideResultH\x00\x12\x38\n\x06reject\x18\x03 \x01(\x0b\x32&.dynamo.planner.plugin.v1.RejectResultH\x00\x12\r\n\x05\x66inal\x18\x04 \x01(\x08\x42\x08\n\x06result\"\x8f\x01\n\x15ReconcileStageRequest\x12:\n\x07\x63ontext\x18\x01 \x01(\x0b\x32).dynamo.planner.plugin.v1.PipelineContext\x12:\n\tproposals\x18\x02 \x03(\x0b\x32\'.dynamo.planner.plugin.v1.ProposeResult\"\xf0\x01\n\rProposeResult\x12\x11\n\tplugin_id\x18\x01 \x01(\t\x12\x38\n\x06\x61\x63\x63\x65pt\x18\x02 \x01(\x0b\x32&.dynamo.planner.plugin.v1.AcceptResultH\x00\x12<\n\x08override\x18\x03 \x01(\x0b\x32(.dynamo.planner.plugin.v1.OverrideResultH\x00\x12\x38\n\x06reject\x18\x04 \x01(\x0b\x32&.dynamo.planner.plugin.v1.RejectResultH\x00\x12\x10\n\x08priority\x18\x05 \x01(\rB\x08\n\x06result\"\xe3\x01\n\x16ReconcileStageResponse\x12\x38\n\x06\x61\x63\x63\x65pt\x18\x01 \x01(\x0b\x32&.dynamo.planner.plugin.v1.AcceptResultH\x00\x12<\n\x08override\x18\x02 \x01(\x0b\x32(.dynamo.planner.plugin.v1.OverrideResultH\x00\x12\x38\n\x06reject\x18\x03 \x01(\x0b\x32&.dynamo.planner.plugin.v1.RejectResultH\x00\x12\r\n\x05\x66inal\x18\x04 \x01(\x08\x42\x08\n\x06result\"S\n\x15\x43onstrainStageRequest\x12:\n\x07\x63ontext\x18\x01 \x01(\x0b\x32).dynamo.planner.plugin.v1.PipelineContext\"\xe3\x01\n\x16\x43onstrainStageResponse\x12\x38\n\x06\x61\x63\x63\x65pt\x18\x01 \x01(\x0b\x32&.dynamo.planner.plugin.v1.AcceptResultH\x00\x12<\n\x08override\x18\x02 \x01(\x0b\x32(.dynamo.planner.plugin.v1.OverrideResultH\x00\x12\x38\n\x06reject\x18\x03 \x01(\x0b\x32&.dynamo.planner.plugin.v1.RejectResultH\x00\x12\r\n\x05\x66inal\x18\x04 \x01(\x08\x42\x08\n\x06result\"\x9e\x01\n\x10\x42ootstrapRequest\x12\x16\n\x0e\x62ootstrap_data\x18\x01 \x01(\x0c\x12\x44\n\x05hints\x18\x02 \x03(\x0b\x32\x35.dynamo.planner.plugin.v1.BootstrapRequest.HintsEntry\x1a,\n\nHintsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"0\n\x11\x42ootstrapResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0cResetRequest\x12\x0e\n\x06reason\x18\x01 \x01(\t\",\n\rResetResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t*1\n\nHoldPolicy\x12\x14\n\x10\x41\x43\x43\x45PT_WHEN_IDLE\x10\x00\x12\r\n\tHOLD_LAST\x10\x01*3\n\x0c\x43ircuitState\x12\n\n\x06\x43LOSED\x10\x00\x12\x08\n\x04OPEN\x10\x01\x12\r\n\tHALF_OPEN\x10\x02*2\n\x0cOverrideType\x12\x07\n\x03SET\x10\x00\x12\x0c\n\x08\x41T_LEAST\x10\x01\x12\x0b\n\x07\x41T_MOST\x10\x02\x32\xae\x03\n\x0ePluginRegistry\x12\x61\n\x08Register\x12).dynamo.planner.plugin.v1.RegisterRequest\x1a*.dynamo.planner.plugin.v1.RegisterResponse\x12\x64\n\tHeartbeat\x12*.dynamo.planner.plugin.v1.HeartbeatRequest\x1a+.dynamo.planner.plugin.v1.HeartbeatResponse\x12g\n\nUnregister\x12+.dynamo.planner.plugin.v1.UnregisterRequest\x1a,.dynamo.planner.plugin.v1.UnregisterResponse\x12j\n\x0bListPlugins\x12,.dynamo.planner.plugin.v1.ListPluginsRequest\x1a-.dynamo.planner.plugin.v1.ListPluginsResponse2y\n\rPredictPlugin\x12h\n\x07Predict\x12-.dynamo.planner.plugin.v1.PredictStageRequest\x1a..dynamo.planner.plugin.v1.PredictStageResponse2y\n\rProposePlugin\x12h\n\x07Propose\x12-.dynamo.planner.plugin.v1.ProposeStageRequest\x1a..dynamo.planner.plugin.v1.ProposeStageResponse2\x81\x01\n\x0fReconcilePlugin\x12n\n\tReconcile\x12/.dynamo.planner.plugin.v1.ReconcileStageRequest\x1a\x30.dynamo.planner.plugin.v1.ReconcileStageResponse2\x81\x01\n\x0f\x43onstrainPlugin\x12n\n\tConstrain\x12/.dynamo.planner.plugin.v1.ConstrainStageRequest\x1a\x30.dynamo.planner.plugin.v1.ConstrainStageResponse2\xd1\x01\n\x0fPluginLifecycle\x12\x64\n\tBootstrap\x12*.dynamo.planner.plugin.v1.BootstrapRequest\x1a+.dynamo.planner.plugin.v1.BootstrapResponse\x12X\n\x05Reset\x12&.dynamo.planner.plugin.v1.ResetRequest\x1a\'.dynamo.planner.plugin.v1.ResetResponseb\x06proto3') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'dynamo.planner.plugins.proto.v1.plugin_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - _FPMDATA_PREFILLENGINESENTRY._options = None - _FPMDATA_PREFILLENGINESENTRY._serialized_options = b'8\001' - _FPMDATA_DECODEENGINESENTRY._options = None - _FPMDATA_DECODEENGINESENTRY._serialized_options = b'8\001' - _BOOTSTRAPREQUEST_HINTSENTRY._options = None - _BOOTSTRAPREQUEST_HINTSENTRY._serialized_options = b'8\001' - _HOLDPOLICY._serialized_start=5066 - _HOLDPOLICY._serialized_end=5115 - _CIRCUITSTATE._serialized_start=5117 - _CIRCUITSTATE._serialized_end=5168 - _OVERRIDETYPE._serialized_start=5170 - _OVERRIDETYPE._serialized_end=5220 - _REGISTERREQUEST._serialized_start=75 - _REGISTERREQUEST._serialized_end=423 - _REGISTERRESPONSE._serialized_start=425 - _REGISTERRESPONSE._serialized_end=521 - _HEARTBEATREQUEST._serialized_start=523 - _HEARTBEATREQUEST._serialized_end=580 - _HEARTBEATRESPONSE._serialized_start=582 - _HEARTBEATRESPONSE._serialized_end=613 - _UNREGISTERREQUEST._serialized_start=615 - _UNREGISTERREQUEST._serialized_end=689 - _UNREGISTERRESPONSE._serialized_start=691 - _UNREGISTERRESPONSE._serialized_end=723 - _LISTPLUGINSREQUEST._serialized_start=725 - _LISTPLUGINSREQUEST._serialized_end=793 - _LISTPLUGINSRESPONSE._serialized_start=795 - _LISTPLUGINSRESPONSE._serialized_end=871 - _PLUGININFO._serialized_start=874 - _PLUGININFO._serialized_end=1194 - _PIPELINECONTEXT._serialized_start=1197 - _PIPELINECONTEXT._serialized_end=1590 - _OBSERVATIONDATA._serialized_start=1593 - _OBSERVATIONDATA._serialized_end=1820 - _TRAFFICMETRICS._serialized_start=1823 - _TRAFFICMETRICS._serialized_end=1990 - _FPMDATA._serialized_start=1993 - _FPMDATA._serialized_end=2269 - _FPMDATA_PREFILLENGINESENTRY._serialized_start=2162 - _FPMDATA_PREFILLENGINESENTRY._serialized_end=2215 - _FPMDATA_DECODEENGINESENTRY._serialized_start=2217 - _FPMDATA_DECODEENGINESENTRY._serialized_end=2269 - _WORKERSTATE._serialized_start=2272 - _WORKERSTATE._serialized_end=2623 - _PREDICTIONDATA._serialized_start=2626 - _PREDICTIONDATA._serialized_end=2932 - _SCALINGPROPOSAL._serialized_start=2934 - _SCALINGPROPOSAL._serialized_end=3043 - _COMPONENTTARGET._serialized_start=3046 - _COMPONENTTARGET._serialized_end=3187 - _OVERRIDERESULT._serialized_start=3189 - _OVERRIDERESULT._serialized_end=3281 - _ACCEPTRESULT._serialized_start=3283 - _ACCEPTRESULT._serialized_end=3297 - _REJECTRESULT._serialized_start=3299 - _REJECTRESULT._serialized_end=3329 - _PREDICTSTAGEREQUEST._serialized_start=3331 - _PREDICTSTAGEREQUEST._serialized_end=3412 - _PREDICTSTAGERESPONSE._serialized_start=3414 - _PREDICTSTAGERESPONSE._serialized_end=3530 - _PROPOSESTAGEREQUEST._serialized_start=3532 - _PROPOSESTAGEREQUEST._serialized_end=3613 - _PROPOSESTAGERESPONSE._serialized_start=3616 - _PROPOSESTAGERESPONSE._serialized_end=3841 - _RECONCILESTAGEREQUEST._serialized_start=3844 - _RECONCILESTAGEREQUEST._serialized_end=3987 - _PROPOSERESULT._serialized_start=3990 - _PROPOSERESULT._serialized_end=4230 - _RECONCILESTAGERESPONSE._serialized_start=4233 - _RECONCILESTAGERESPONSE._serialized_end=4460 - _CONSTRAINSTAGEREQUEST._serialized_start=4462 - _CONSTRAINSTAGEREQUEST._serialized_end=4545 - _CONSTRAINSTAGERESPONSE._serialized_start=4548 - _CONSTRAINSTAGERESPONSE._serialized_end=4775 - _BOOTSTRAPREQUEST._serialized_start=4778 - _BOOTSTRAPREQUEST._serialized_end=4936 - _BOOTSTRAPREQUEST_HINTSENTRY._serialized_start=4892 - _BOOTSTRAPREQUEST_HINTSENTRY._serialized_end=4936 - _BOOTSTRAPRESPONSE._serialized_start=4938 - _BOOTSTRAPRESPONSE._serialized_end=4986 - _RESETREQUEST._serialized_start=4988 - _RESETREQUEST._serialized_end=5018 - _RESETRESPONSE._serialized_start=5020 - _RESETRESPONSE._serialized_end=5064 - _PLUGINREGISTRY._serialized_start=5223 - _PLUGINREGISTRY._serialized_end=5653 - _PREDICTPLUGIN._serialized_start=5655 - _PREDICTPLUGIN._serialized_end=5776 - _PROPOSEPLUGIN._serialized_start=5778 - _PROPOSEPLUGIN._serialized_end=5899 - _RECONCILEPLUGIN._serialized_start=5902 - _RECONCILEPLUGIN._serialized_end=6031 - _CONSTRAINPLUGIN._serialized_start=6034 - _CONSTRAINPLUGIN._serialized_end=6163 - _PLUGINLIFECYCLE._serialized_start=6166 - _PLUGINLIFECYCLE._serialized_end=6375 -# @@protoc_insertion_point(module_scope) diff --git a/components/src/dynamo/profiler/README.md b/components/src/dynamo/profiler/README.md deleted file mode 100644 index 62bd9615cfd7..000000000000 --- a/components/src/dynamo/profiler/README.md +++ /dev/null @@ -1,13 +0,0 @@ - - -# Profiler - -Documentation for the Dynamo Profiler has moved to [docs/components/profiler/](../../../../docs/components/profiler/README.md). - -- [Profiler Overview](../../../../docs/components/profiler/README.md) -- [Profiler Guide](../../../../docs/components/profiler/profiler-guide.md) -- [Profiler Examples](../../../../docs/components/profiler/profiler-examples.md) diff --git a/container/README.md b/container/README.md index e2407a94fb2a..642fc8bb6624 100644 --- a/container/README.md +++ b/container/README.md @@ -66,7 +66,7 @@ Below is a summary of the general file structure for the framework Dockerfile st **Note on `/workspace/` COPY set:** - Common to all three frameworks: `tests`, `examples`, `dev`, `components/src/dynamo/{common,frontend,}` -- vLLM and TRT-LLM additionally copy `lib`; SGLang and TRT-LLM additionally copy `deploy` and `components/src/dynamo/mocker`; SGLang additionally copies `recipes`. +- vLLM and TRT-LLM additionally copy `lib`; SGLang and TRT-LLM additionally copy `deploy` and `dingo/mocker`; SGLang additionally copies `recipes`. - See each framework's `templates/_runtime.Dockerfile` for the exact list. diff --git a/container/deps/requirements.common.txt b/container/deps/requirements.common.txt index bf346d63c62b..6f896377d417 100644 --- a/container/deps/requirements.common.txt +++ b/container/deps/requirements.common.txt @@ -23,7 +23,7 @@ imageio-ffmpeg>=0.6.0 # binary skipped per --no-binary directive at top of file matplotlib==3.10.7 msgspec==0.19.0 nvidia-ml-py<=13.580.65 # NVIDIA/CUDA related, may vary by driver version -# NVTX Python bindings used by dynamo.common.utils.nvtx_utils for DYN_NVTX=1 profiling. +# NVTX Python bindings used by dingo.common.utils.nvtx_utils for DYN_NVTX=1 profiling. nvtx==0.2.14 opentelemetry-api<=1.38.0 # Pinned for tracing support opentelemetry-exporter-otlp<=1.38.0 diff --git a/container/launch_message/frontend.txt b/container/launch_message/frontend.txt index 9727c27e1799..d4c9845a7702 100644 --- a/container/launch_message/frontend.txt +++ b/container/launch_message/frontend.txt @@ -51,7 +51,7 @@ Benefits: Quick Start: Start mocker with custom configuration: -> python -m dynamo.mocker \ +> python -m dingo.mocker \ --model-path TinyLlama/TinyLlama-1.1B-Chat-v1.0 \ --num-gpu-blocks-override 8192 \ --block-size 16 \ diff --git a/container/templates/args.Dockerfile b/container/templates/args.Dockerfile index 8acc1a89d69b..b889ce001d0c 100644 --- a/container/templates/args.Dockerfile +++ b/container/templates/args.Dockerfile @@ -88,10 +88,10 @@ ARG FRONTEND_IMAGE={{ context.dynamo.frontend_image }} {% endif %} {% if target == "planner" %} -ARG PLANNER_BUILD_IMAGE={{ context.dynamo.planner_build_image }} -ARG PLANNER_BUILD_IMAGE_TAG={{ context.dynamo.planner_build_image_tag }} -ARG PLANNER_RUNTIME_IMAGE={{ context.dynamo.planner_runtime_image }} -ARG PLANNER_RUNTIME_IMAGE_TAG={{ context.dynamo.planner_runtime_image_tag }} +ARG PLANNER_BUILD_IMAGE={{ context.dingo.planner_build_image }} +ARG PLANNER_BUILD_IMAGE_TAG={{ context.dingo.planner_build_image_tag }} +ARG PLANNER_RUNTIME_IMAGE={{ context.dingo.planner_runtime_image }} +ARG PLANNER_RUNTIME_IMAGE_TAG={{ context.dingo.planner_runtime_image_tag }} {% endif %} {% if framework == "vllm" -%} diff --git a/container/templates/planner.Dockerfile b/container/templates/planner.Dockerfile index b7c736ae9451..b82e52e9d599 100644 --- a/container/templates/planner.Dockerfile +++ b/container/templates/planner.Dockerfile @@ -83,9 +83,9 @@ RUN --mount=type=bind,source=./container/deps/requirements.planner.txt,target=/t # Copy only the subset of the repository needed for planner/profiler service # startup and the component-local planner-family test suites. COPY --chmod=664 --chown=dynamo:0 pyproject.toml /workspace/pyproject.toml -COPY --chmod=775 --chown=dynamo:0 components/src/dynamo/planner /workspace/components/src/dynamo/planner -COPY --chmod=775 --chown=dynamo:0 components/src/dynamo/profiler /workspace/components/src/dynamo/profiler -COPY --chmod=775 --chown=dynamo:0 components/src/dynamo/global_planner /workspace/components/src/dynamo/global_planner +COPY --chmod=775 --chown=dynamo:0 dingo/planner /workspace/dingo/planner +COPY --chmod=775 --chown=dynamo:0 dingo/profiler /workspace/dingo/profiler +COPY --chmod=775 --chown=dynamo:0 dingo/global_planner /workspace/dingo/global_planner COPY --chmod=775 --chown=dynamo:0 deploy /workspace/deploy COPY --chmod=775 --chown=dynamo:0 dev /workspace/dev COPY --chmod=775 --chown=dynamo:0 examples /workspace/examples diff --git a/container/templates/sglang_runtime.Dockerfile b/container/templates/sglang_runtime.Dockerfile index ae3350a723ed..6473d79a9e8d 100644 --- a/container/templates/sglang_runtime.Dockerfile +++ b/container/templates/sglang_runtime.Dockerfile @@ -138,7 +138,7 @@ RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \ # Install nvtx pinned in container/deps/requirements.common.txt so DYN_NVTX=1 # profiling works in all targets (runtime, dev, local-dev) — see -# components/src/dynamo/common/utils/nvtx_utils.py. --no-deps preserves the +# dingo/common/utils/nvtx_utils.py. --no-deps preserves the # upstream lmsysorg/sglang Python stack. RUN --mount=type=bind,source=./container/deps/requirements.common.txt,target=/tmp/requirements.common.txt \ --mount=type=cache,target=/root/.cache/pip,sharing=locked \ @@ -161,10 +161,10 @@ COPY --chmod=775 --chown=dynamo:0 tests /workspace/tests COPY --chmod=775 --chown=dynamo:0 examples /workspace/examples COPY --chmod=775 --chown=dynamo:0 deploy /workspace/deploy COPY --chmod=775 --chown=dynamo:0 dev /workspace/dev -COPY --chmod=775 --chown=dynamo:0 components/src/dynamo/common /workspace/components/src/dynamo/common +COPY --chmod=775 --chown=dynamo:0 dingo/common /workspace/dingo/common COPY --chmod=775 --chown=dynamo:0 dingo/frontend /workspace/dingo/frontend COPY --chmod=775 --chown=dynamo:0 dingo/sglang /workspace/dingo/sglang -COPY --chmod=775 --chown=dynamo:0 components/src/dynamo/mocker /workspace/components/src/dynamo/mocker +COPY --chmod=775 --chown=dynamo:0 dingo/mocker /workspace/dingo/mocker COPY --chmod=775 --chown=dynamo:0 recipes/ /workspace/recipes/ COPY --chmod=664 --chown=dynamo:0 LICENSE /workspace/ diff --git a/container/templates/vllm_runtime.Dockerfile b/container/templates/vllm_runtime.Dockerfile index 8aaf7646c357..d462599e1ddd 100644 --- a/container/templates/vllm_runtime.Dockerfile +++ b/container/templates/vllm_runtime.Dockerfile @@ -283,7 +283,7 @@ USER dynamo COPY --chmod=775 --chown=dynamo:0 tests /workspace/tests COPY --chmod=775 --chown=dynamo:0 examples /workspace/examples COPY --chmod=775 --chown=dynamo:0 dev /workspace/dev -COPY --chmod=775 --chown=dynamo:0 components/src/dynamo/common /workspace/components/src/dynamo/common +COPY --chmod=775 --chown=dynamo:0 dingo/common /workspace/dingo/common COPY --chmod=775 --chown=dynamo:0 dingo/frontend /workspace/dingo/frontend COPY --chmod=775 --chown=dynamo:0 dingo/vllm /workspace/dingo/vllm COPY --chown=dynamo:0 lib /workspace/lib diff --git a/deny.toml b/deny.toml deleted file mode 100644 index aec12205b935..000000000000 --- a/deny.toml +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# cargo-deny plugin for linting dependencies -# https://github.com/EmbarkStudios/cargo-deny/blob/main/deny.template.toml - -[graph] -# Dynamo CI targets Linux; avoid checking target-specific dependencies for -# unsupported platforms such as Windows. -targets = [ - "x86_64-unknown-linux-gnu", - "aarch64-unknown-linux-gnu", -] - -[licenses] -confidence-threshold = 0.93 -allow = [ - "MIT-0", - "MIT", - "Apache-2.0", - "Apache-2.0 WITH LLVM-exception", - "ISC", - "0BSD", - "BSD-2-Clause", - "BSD-3-Clause", - "OpenSSL", - "Unicode-3.0", - "BSL-1.0", - "MPL-2.0", - "CDLA-Permissive-2.0", - "Zlib", - "NCSA", - "LGPL-3.0", - "LGPL-3.0-only", - "CC0-1.0", - "Unicode-DFS-2016", - "WTFPL" -] - -# TODO exceptions -# MIT: https://github.com/guidance-ai/llguidance -# "llguidance", -# MIT: https://github.com/guidance-ai/llguidance/toktrie -# "toktrie", -# MIT: https://github.com/guidance-ai/llguidance/toktrie_hf_tokenizers -# "toktrie_hf_tokenizers", - -[[licenses.clarify]] - -name = "ring" -expression = "MIT AND ISC AND OpenSSL" -license-files = [ - { path = "LICENSE", hash = 0xbd0eed23 } -] - -[bans] -deny = [ - # Ensure we don't depend on openssl - { name = "native-tls" }, - { name = "openssl-sys" }, -] diff --git a/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeploymentrequests.yaml index 8f4b7a273478..debf2f515100 100644 --- a/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeploymentrequests.yaml @@ -565,7 +565,7 @@ spec: planner: description: |- Planner contains the raw Planner configuration passed to the Planner service. - Its schema is defined by dynamo.planner.config.planner_config.PlannerConfig. + Its schema is defined by dingo.planner.config.planner_config.PlannerConfig. See https://docs.dynamo.nvidia.com/dynamo/components/planner/planner-guide#plannerconfig-reference. DGDR passes this object through without field-level validation; the Planner service validates it at startup. diff --git a/deploy/operator/api/scripts/generate_pydantic_from_go.py b/deploy/operator/api/scripts/generate_pydantic_from_go.py index 5981a0d8efa1..371ac946c4db 100755 --- a/deploy/operator/api/scripts/generate_pydantic_from_go.py +++ b/deploy/operator/api/scripts/generate_pydantic_from_go.py @@ -37,12 +37,12 @@ # Planner-specific types are the canonical hand-written source of truth. _IMPORT_OVERRIDES: dict[str, tuple[str, str, bool]] = { "PlannerPreDeploymentSweepMode": ( - "dynamo.planner.config.planner_config", + "dingo.planner.config.planner_config", "PlannerPreDeploymentSweepMode", True, ), "PlannerConfig": ( - "dynamo.planner.config.planner_config", + "dingo.planner.config.planner_config", "PlannerConfig", True, ), diff --git a/deploy/operator/api/scripts/validate_pydantic_models.py b/deploy/operator/api/scripts/validate_pydantic_models.py index 25af8629025b..c5eac0724801 100755 --- a/deploy/operator/api/scripts/validate_pydantic_models.py +++ b/deploy/operator/api/scripts/validate_pydantic_models.py @@ -47,28 +47,33 @@ def _repo_root() -> Path: return start -_components_src = _repo_root() / "components" / "src" - -# In the operator Docker build the context is deploy/operator/ only — components/src -# is not copied in. The generated files are already committed, so skip validation. -if not _components_src.exists(): +_root = _repo_root() +_components_src = _root / "components" / "src" +_profiler_src = _root / "dingo" / "profiler" + +# In the operator Docker build the context is deploy/operator/ only — the Python +# component sources are not copied in. The generated files are already committed, +# so skip validation. +if not _components_src.exists() or not _profiler_src.exists(): print( - f"Note: {_components_src} not found (operator-only build context). " + "Note: Python component sources not found (operator-only build context). " "Skipping Pydantic validation tests." ) sys.exit(0) -# Add the components src to path so we can import the generated models +# The planner remains under components/src while the profiler is part of the +# repository-root dingo package. sys.path.insert(0, str(_components_src)) +sys.path.insert(0, str(_root)) # --------------------------------------------------------------------------- -# Stub dynamo.runtime.logging and bypass the heavy dynamo.planner.__init__ +# Stub dynamo.runtime.logging and bypass the heavy dingo.planner.__init__ # before importing any dynamo module. # # dynamo itself must be a namespace-like package (has __path__) so that -# Python's import machinery can traverse down to dynamo.profiler from the -# filesystem. dynamo.planner is pre-registered as a stub to skip its heavy -# __init__.py, while still allowing dynamo.planner.config.* to load normally. +# dingo.planner is pre-registered as a stub to skip its heavy __init__.py, +# while still allowing dingo.planner.config.* to load normally. The lightweight +# dingo package is imported normally from the repository root. # --------------------------------------------------------------------------- _dynamo_path = str(_components_src / "dynamo") _planner_path = str(_components_src / "dynamo" / "planner") @@ -87,14 +92,14 @@ def _repo_root() -> Path: _logging_mod.configure_dynamo_logging = lambda *args, **kwargs: None # type: ignore[attr-defined] sys.modules["dynamo.runtime.logging"] = _logging_mod -_planner_mod = types.ModuleType("dynamo.planner") +_planner_mod = types.ModuleType("dingo.planner") _planner_mod.__path__ = [_planner_path] # type: ignore[attr-defined] -_planner_mod.__package__ = "dynamo.planner" -sys.modules["dynamo.planner"] = _planner_mod +_planner_mod.__package__ = "dingo.planner" +sys.modules["dingo.planner"] = _planner_mod import pydantic # noqa: E402 -from dynamo.profiler.utils.dgdr_v1beta1_types import ( # noqa: E402 +from dingo.profiler.utils.dgdr_v1beta1_types import ( # noqa: E402 BackendType, DeploymentInfoStatus, DGDRPhase, diff --git a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go index acd3314b1b9d..76d7364ff4d4 100644 --- a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go @@ -334,7 +334,7 @@ type KVRouterSpec struct { // FeaturesSpec controls optional Dynamo platform features in the generated deployment. type FeaturesSpec struct { // Planner contains the raw Planner configuration passed to the Planner service. - // Its schema is defined by dynamo.planner.config.planner_config.PlannerConfig. + // Its schema is defined by dingo.planner.config.planner_config.PlannerConfig. // See https://docs.dynamo.nvidia.com/dynamo/components/planner/planner-guide#plannerconfig-reference. // DGDR passes this object through without field-level validation; the Planner // service validates it at startup. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 8f4b7a273478..debf2f515100 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -565,7 +565,7 @@ spec: planner: description: |- Planner contains the raw Planner configuration passed to the Planner service. - Its schema is defined by dynamo.planner.config.planner_config.PlannerConfig. + Its schema is defined by dingo.planner.config.planner_config.PlannerConfig. See https://docs.dynamo.nvidia.com/dynamo/components/planner/planner-guide#plannerconfig-reference. DGDR passes this object through without field-level validation; the Planner service validates it at startup. diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go index 00b6ace6160e..c83d9b4ca50b 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go @@ -1511,7 +1511,7 @@ func (r *DynamoGraphDeploymentRequestReconciler) createProfilingJob(ctx context. profilerContainer := corev1.Container{ Name: ContainerNameProfiler, Image: imageName, - Command: []string{"python", "-m", "dynamo.profiler"}, + Command: []string{"python", "-m", "dingo.profiler"}, Args: profilerArgs, Env: profilerEnv, VolumeMounts: volumeMounts, diff --git a/deploy/operator/internal/controller/profiling_job_overrides_test.go b/deploy/operator/internal/controller/profiling_job_overrides_test.go index 6d586ee5b122..ba5f5c4885b1 100644 --- a/deploy/operator/internal/controller/profiling_job_overrides_test.go +++ b/deploy/operator/internal/controller/profiling_job_overrides_test.go @@ -56,7 +56,7 @@ func baseJob() *batchv1.Job { { Name: "profiler", Image: "profiler:latest", - Command: []string{"python", "-m", "dynamo.profiler"}, + Command: []string{"python", "-m", "dingo.profiler"}, Env: []corev1.EnvVar{ {Name: "OUTPUT_DIR", Value: "/output"}, }, diff --git a/deploy/utils/dynamo_deployment.py b/deploy/utils/dynamo_deployment.py index e1e1f97520b8..9b00f4ffa843 100644 --- a/deploy/utils/dynamo_deployment.py +++ b/deploy/utils/dynamo_deployment.py @@ -734,6 +734,6 @@ async def main(): # run with: -# uv run components/src/dynamo/profiler/utils/dynamo_deployment.py -n mo-dyn -f ./examples/vllm/deploy/agg.yaml -l ./client_logs +# uv run dingo/profiler/utils/dynamo_deployment.py -n mo-dyn -f ./examples/vllm/deploy/agg.yaml -l ./client_logs if __name__ == "__main__": asyncio.run(main()) diff --git a/dev/sanity_check.py b/dev/sanity_check.py index 90c67f8bc560..da701e58f36e 100755 --- a/dev/sanity_check.py +++ b/dev/sanity_check.py @@ -27,7 +27,7 @@ IMPORTANT: This script is STANDALONE and uses only Python stdlib (no Dynamo components). Why: Must work before Dynamo is built/installed (CI, fresh containers, build failures). -This tool is for pre-deployment validation; dynamo.common.config_dump is for runtime. +This tool is for pre-deployment validation; dingo.common.config_dump is for runtime. Hard-coded paths: Uses defaults (e.g., ~/.cache/huggingface/hub) for predictable behavior even when environment variables are misconfigured. See class docs for details. @@ -668,7 +668,7 @@ def _suppress_planner_warnings(self) -> None: """Suppress Prometheus endpoint warnings from planner module during import testing.""" # The planner module logs a warning about Prometheus endpoint when imported # outside of a Kubernetes cluster. Suppress this for cleaner output. - planner_logger = logging.getLogger("dynamo.planner.config.defaults") + planner_logger = logging.getLogger("dingo.planner.config.defaults") planner_logger.setLevel(logging.ERROR) # Also suppress the defaults._get_default_prometheus_endpoint logger defaults_logger = logging.getLogger("defaults._get_default_prometheus_endpoint") @@ -1806,7 +1806,7 @@ class HuggingFaceInfo(NodeInfo): This class directly uses ~/.cache/huggingface/hub instead of reading environment variables because this tool must work reliably in all environments, including when environment variables are misconfigured or not set. For dynamic configuration that - respects all HF environment variables, use dynamo.common.config_dump at runtime. + respects all HF environment variables, use dingo.common.config_dump at runtime. """ def __init__(self, thorough_check: bool = False): @@ -3163,7 +3163,7 @@ def _discover_framework_components(self, workspace_dir: Optional[str]) -> list: Returns: List of framework component module names - Example: ['dingo.frontend', 'dynamo.planner', 'dingo.vllm', 'dingo.sglang'] + Example: ['dingo.frontend', 'dingo.planner', 'dingo.vllm', 'dingo.sglang'] Note: Scans components/src/dynamo/... directory for modules with __init__.py files. """ diff --git a/components/README.md b/dingo/README.md similarity index 97% rename from components/README.md rename to dingo/README.md index a066e4e3f767..7ab2ddfbc870 100644 --- a/components/README.md +++ b/dingo/README.md @@ -32,7 +32,7 @@ Dynamo supports multiple inference engines, each with their own deployment confi Each engine provides launch and deploy scripts for different deployment patterns in the [examples](../examples/backends/) folder. -### [Frontend](../dingo/frontend/) +### [Frontend](frontend/) The frontend component provides the HTTP API layer and request processing: @@ -41,7 +41,7 @@ The frontend component provides the HTTP API layer and request processing: - **Router** - Routes requests to appropriate workers based on load and KV cache state - **Auto-discovery** - Automatically discovers and registers available workers -### [Planner](src/dynamo/planner/) +### [Planner](planner/) The planner component monitors system state and dynamically adjusts worker allocation: diff --git a/components/src/dynamo/common/__init__.py b/dingo/common/__init__.py similarity index 100% rename from components/src/dynamo/common/__init__.py rename to dingo/common/__init__.py diff --git a/components/src/dynamo/common/backend/AGENTS.md b/dingo/common/backend/AGENTS.md similarity index 100% rename from components/src/dynamo/common/backend/AGENTS.md rename to dingo/common/backend/AGENTS.md diff --git a/components/src/dynamo/common/backend/CLAUDE.md b/dingo/common/backend/CLAUDE.md similarity index 99% rename from components/src/dynamo/common/backend/CLAUDE.md rename to dingo/common/backend/CLAUDE.md index 1ca099e472d0..9a78cd8772d9 100644 --- a/components/src/dynamo/common/backend/CLAUDE.md +++ b/dingo/common/backend/CLAUDE.md @@ -250,7 +250,7 @@ pass, KV transfer) nest under the framework's `engine.generate` span. Splat `telemetry.engine_trace_kwargs(context)` into the inference-engine call: ```python -from dynamo.common.backend import telemetry +from dingo.common.backend import telemetry # vLLM / TRT-LLM: default kwarg name `trace_headers`, unconditional gen = self.engine_client.generate( diff --git a/components/src/dynamo/common/backend/README.md b/dingo/common/backend/README.md similarity index 98% rename from components/src/dynamo/common/backend/README.md rename to dingo/common/backend/README.md index aded7bc61e3e..b5b7687d1ec9 100644 --- a/components/src/dynamo/common/backend/README.md +++ b/dingo/common/backend/README.md @@ -51,7 +51,7 @@ Worker <-- runtime integration (worker.py) ### Running the sample engine ```bash -python -m dynamo.common.backend.sample_main \ +python -m dingo.common.backend.sample_main \ --model-name test-model \ --namespace dynamo \ --component sample \ @@ -83,7 +83,7 @@ Each `unified_main.py` calls `run(MyLLMEngine)` from the common Subclass `LLMEngine` and implement the required methods: ```python -from dynamo.common.backend import LLMEngine, EngineConfig, LlmRegistration, WorkerConfig +from dingo.common.backend import LLMEngine, EngineConfig, LlmRegistration, WorkerConfig class MyEngine(LLMEngine): @classmethod @@ -135,7 +135,7 @@ Then create an entry point: ```python # my_backend/unified_main.py -from dynamo.common.backend.run import run +from dingo.common.backend.run import run from my_backend.llm_engine import MyEngine def main(): @@ -298,7 +298,7 @@ examples/backends/trtllm/launch/disagg.sh --unified ### Helpers -`dynamo.common.backend.disagg` ships small utilities engines can call +`dingo.common.backend.disagg` ships small utilities engines can call directly: `enforce_prefill_max_tokens(request)`, `extract_prefill_result(request)`, and `require_prefill_result(request, mode)`. These are optional — engines @@ -337,7 +337,7 @@ Use `ZmqSource` when the engine already emits Dynamo-compatible KV events on a ZMQ socket: ```python -from dynamo.common.backend.publisher import ZmqSource +from dingo.common.backend.publisher import ZmqSource async def kv_event_sources(self): return [ @@ -349,7 +349,7 @@ Use `PushSource` when the engine needs a live publisher and drives `publish_stored()` / `publish_removed()` from its own thread: ```python -from dynamo.common.backend.publisher import PushSource +from dingo.common.backend.publisher import PushSource def _on_kv_publisher_ready(self, publisher): self._kv_publisher = publisher @@ -380,11 +380,11 @@ publish loop observe the shutdown signal before resources are released. The framework opens an `engine.generate` span around every `generate()` call (see the Rust backend-common README for the full attribute table). Engine code reaches the recording surface through the -`dynamo.common.backend.telemetry` facade, which mirrors the OpenTelemetry +`dingo.common.backend.telemetry` facade, which mirrors the OpenTelemetry `Span` API — no Dynamo-specific vocabulary: ```python -from dynamo.common.backend import telemetry +from dingo.common.backend import telemetry async def generate(self, request, context): # Trace headers for the downstream inference engine (W3C traceparent). @@ -519,7 +519,7 @@ Lifecycle and runtime: bootstrap address, vLLM and TRT-LLM use an engine-internal handshake. See [Disaggregated Serving](#disaggregated-serving) below. - **Logprobs** — selected-token + top-k logprob extraction and - streaming, sourced from `dynamo.common.backend.logprobs` and used by + streaming, sourced from `dingo.common.backend.logprobs` and used by both unified engines and the legacy handlers (which now delegate here). vLLM/TRT-LLM share an extractor; SGLang has a cumulative-array variant. The sample engine and Rust mocker emit synthetic logprobs diff --git a/components/src/dynamo/common/backend/__init__.py b/dingo/common/backend/__init__.py similarity index 100% rename from components/src/dynamo/common/backend/__init__.py rename to dingo/common/backend/__init__.py diff --git a/components/src/dynamo/common/backend/disagg.py b/dingo/common/backend/disagg.py similarity index 95% rename from components/src/dynamo/common/backend/disagg.py rename to dingo/common/backend/disagg.py index 85cee3ed38d2..557a3656adab 100644 --- a/components/src/dynamo/common/backend/disagg.py +++ b/dingo/common/backend/disagg.py @@ -24,8 +24,8 @@ from typing import Any, Optional -from dynamo.common.backend.engine import GenerateRequest -from dynamo.common.constants import DisaggregationMode +from dingo.common.backend.engine import GenerateRequest +from dingo.common.constants import DisaggregationMode def enforce_prefill_max_tokens(request: GenerateRequest) -> GenerateRequest: diff --git a/components/src/dynamo/common/backend/dp_rank.py b/dingo/common/backend/dp_rank.py similarity index 96% rename from components/src/dynamo/common/backend/dp_rank.py rename to dingo/common/backend/dp_rank.py index be5a43a55236..65bafa79d045 100644 --- a/components/src/dynamo/common/backend/dp_rank.py +++ b/dingo/common/backend/dp_rank.py @@ -14,7 +14,7 @@ import logging from typing import Any, Optional, cast -from dynamo.common.backend.engine import GenerateRequest +from dingo.common.backend.engine import GenerateRequest logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/common/backend/engine.py b/dingo/common/backend/engine.py similarity index 98% rename from components/src/dynamo/common/backend/engine.py rename to dingo/common/backend/engine.py index 069fc712bc26..e63be75c84e2 100644 --- a/components/src/dynamo/common/backend/engine.py +++ b/dingo/common/backend/engine.py @@ -12,7 +12,7 @@ from typing_extensions import Required from dynamo._core import Context -from dynamo.common.constants import DisaggregationMode +from dingo.common.constants import DisaggregationMode from .publisher import KvEventSource @@ -41,14 +41,14 @@ class GenerateRequest(TypedDict, total=False): Disaggregated-serving keys (``prefill_result``, ``bootstrap_info``) are set by the frontend's PrefillRouter on decode requests; engines - read them via ``dynamo.common.backend.disagg`` helpers. + read them via ``dingo.common.backend.disagg`` helpers. Multimodal keys (``multi_modal_data``, ``mm_processor_kwargs``, ``mm_routing_info``) are populated by the frontend preprocessor when the request carries media. ``encoder_result`` is set by the frontend when forwarding a request from an Encode worker to a downstream Prefill/Aggregated peer; engines read it via - :func:`dynamo.common.backend.multimodal.require_encoder_result`. All + :func:`dingo.common.backend.multimodal.require_encoder_result`. All four are object-shaped (``dict``) by contract. ``model`` carries the requested model name (set by the Rust @@ -84,12 +84,12 @@ class GenerateChunk(TypedDict, total=False): PrefillRouter to forward to the decode peer. When the caller requested logprobs, chunks may also carry ``log_probs`` and ``top_logprobs`` aligned to ``token_ids`` — see - :mod:`dynamo.common.backend.logprobs`. + :mod:`dingo.common.backend.logprobs`. Encode terminals carry ``encoder_result`` (an opaque object the frontend forwards onto the downstream ``PreprocessedRequest.encoder_result``). Construct with - :func:`dynamo.common.backend.multimodal.encoder_terminal_chunk`. + :func:`dingo.common.backend.multimodal.encoder_terminal_chunk`. """ token_ids: Required[list[int]] @@ -259,7 +259,7 @@ async def cleanup(self) -> None: async def register_prometheus(self, metrics: "EngineMetrics") -> None: """Bridge a vendor-prefixed Prometheus registry into the runtime's ``/metrics`` output via :func:`metrics.add_expfmt_callback`. Default - no-op. See :mod:`dynamo.common.backend.metrics` for helpers. Do not + no-op. See :mod:`dingo.common.backend.metrics` for helpers. Do not retain ``metrics`` past return. Framework-owned lifecycle + per-rank gauges @@ -396,7 +396,7 @@ async def logits_processor_spec(self) -> "LogitsProcessorSpec | None": # passes the request through as a JSON ``dict`` and serializes each yielded # object back — no Rust request type (the modality-neutral trade-off). # Canonical field schemas: NvCreateImageRequest/NvImagesResponse in -# dynamo.common.protocols.image_protocol (videos: video_protocol). +# dingo.common.protocols.image_protocol (videos: video_protocol). RawRequest = dict[str, Any] RawResponseChunk = dict[str, Any] diff --git a/components/src/dynamo/common/backend/health_check.py b/dingo/common/backend/health_check.py similarity index 100% rename from components/src/dynamo/common/backend/health_check.py rename to dingo/common/backend/health_check.py diff --git a/components/src/dynamo/common/backend/logprobs.py b/dingo/common/backend/logprobs.py similarity index 100% rename from components/src/dynamo/common/backend/logprobs.py rename to dingo/common/backend/logprobs.py diff --git a/components/src/dynamo/common/backend/metrics.py b/dingo/common/backend/metrics.py similarity index 99% rename from components/src/dynamo/common/backend/metrics.py rename to dingo/common/backend/metrics.py index 35b72b9ac064..cb954aaeef3d 100644 --- a/components/src/dynamo/common/backend/metrics.py +++ b/dingo/common/backend/metrics.py @@ -27,7 +27,7 @@ import tempfile from typing import TYPE_CHECKING, Optional -from dynamo.common.utils.prometheus import gather_with_labels +from dingo.common.utils.prometheus import gather_with_labels if TYPE_CHECKING: from prometheus_client import CollectorRegistry diff --git a/components/src/dynamo/common/backend/multimodal.py b/dingo/common/backend/multimodal.py similarity index 95% rename from components/src/dynamo/common/backend/multimodal.py rename to dingo/common/backend/multimodal.py index f3f74bf41db4..20635621cb5b 100644 --- a/components/src/dynamo/common/backend/multimodal.py +++ b/dingo/common/backend/multimodal.py @@ -4,7 +4,7 @@ """Contract helpers for the unified-backend multimodal wire shape. Three small helpers, parallel to the prefill/decode pair in -``dynamo.common.backend.disagg``: +``dingo.common.backend.disagg``: * :func:`extract_multimodal_kwargs` — pull ``multi_modal_data`` and ``mm_processor_kwargs`` off a request into a flat ``**kwargs`` dict @@ -28,8 +28,8 @@ from typing import Any, Optional -from dynamo.common.backend.engine import GenerateChunk, GenerateRequest -from dynamo.common.constants import DisaggregationMode +from dingo.common.backend.engine import GenerateChunk, GenerateRequest +from dingo.common.constants import DisaggregationMode def extract_multimodal_kwargs(request: GenerateRequest) -> Optional[dict[str, Any]]: @@ -59,7 +59,7 @@ def require_encoder_result( """Return the request's ``encoder_result`` dict, raising on missing or wrongly shaped. - Analog of :func:`dynamo.common.backend.disagg.require_prefill_result`. + Analog of :func:`dingo.common.backend.disagg.require_prefill_result`. The frontend is expected to forward the Encode worker's terminal-chunk ``encoder_result`` onto the downstream ``PreprocessedRequest.encoder_result``; missing means the request diff --git a/components/src/dynamo/common/backend/publisher.py b/dingo/common/backend/publisher.py similarity index 100% rename from components/src/dynamo/common/backend/publisher.py rename to dingo/common/backend/publisher.py diff --git a/components/src/dynamo/common/backend/run.py b/dingo/common/backend/run.py similarity index 95% rename from components/src/dynamo/common/backend/run.py rename to dingo/common/backend/run.py index 7fb1fadbb8bb..5879735106b3 100644 --- a/components/src/dynamo/common/backend/run.py +++ b/dingo/common/backend/run.py @@ -6,7 +6,7 @@ Each backend's ``unified_main.py`` calls :func:`run` with its ``LLMEngine`` subclass. Example:: - from dynamo.common.backend.run import run + from dingo.common.backend.run import run from dingo.vllm.llm_engine import VllmLLMEngine def main(): diff --git a/components/src/dynamo/common/backend/sample_diffusion_engine.py b/dingo/common/backend/sample_diffusion_engine.py similarity index 100% rename from components/src/dynamo/common/backend/sample_diffusion_engine.py rename to dingo/common/backend/sample_diffusion_engine.py diff --git a/components/src/dynamo/common/backend/sample_diffusion_main.py b/dingo/common/backend/sample_diffusion_main.py similarity index 58% rename from components/src/dynamo/common/backend/sample_diffusion_main.py rename to dingo/common/backend/sample_diffusion_main.py index 368b5a643fa8..783422e0c3cd 100644 --- a/components/src/dynamo/common/backend/sample_diffusion_main.py +++ b/dingo/common/backend/sample_diffusion_main.py @@ -4,11 +4,11 @@ """Entry point for the sample diffusion backend (CPU-only). Usage: - python -m dynamo.common.backend.sample_diffusion_main --model-name sample-diffusion-model + python -m dingo.common.backend.sample_diffusion_main --model-name sample-diffusion-model """ -from dynamo.common.backend.run import run -from dynamo.common.backend.sample_diffusion_engine import SampleDiffusionEngine +from dingo.common.backend.run import run +from dingo.common.backend.sample_diffusion_engine import SampleDiffusionEngine def main(): diff --git a/components/src/dynamo/common/backend/sample_engine.py b/dingo/common/backend/sample_engine.py similarity index 99% rename from components/src/dynamo/common/backend/sample_engine.py rename to dingo/common/backend/sample_engine.py index af2bb3b1b673..fb913fc88738 100644 --- a/components/src/dynamo/common/backend/sample_engine.py +++ b/dingo/common/backend/sample_engine.py @@ -14,7 +14,7 @@ from typing import Any, Optional from dynamo._core import Context -from dynamo.common.constants import DisaggregationMode +from dingo.common.constants import DisaggregationMode from dynamo.llm import KvEventPublisher from . import telemetry diff --git a/components/src/dynamo/common/backend/sample_main.py b/dingo/common/backend/sample_main.py similarity index 60% rename from components/src/dynamo/common/backend/sample_main.py rename to dingo/common/backend/sample_main.py index 2c44ed741550..2c9e099ccae9 100644 --- a/components/src/dynamo/common/backend/sample_main.py +++ b/dingo/common/backend/sample_main.py @@ -4,11 +4,11 @@ """Entry point for the sample backend. Usage: - python -m dynamo.common.backend.sample_main --model-name test-model + python -m dingo.common.backend.sample_main --model-name test-model """ -from dynamo.common.backend.run import run -from dynamo.common.backend.sample_engine import SampleLLMEngine +from dingo.common.backend.run import run +from dingo.common.backend.sample_engine import SampleLLMEngine def main(): diff --git a/components/src/dynamo/common/backend/telemetry.py b/dingo/common/backend/telemetry.py similarity index 98% rename from components/src/dynamo/common/backend/telemetry.py rename to dingo/common/backend/telemetry.py index 2c1cd3686faa..d2e3db40c9d8 100644 --- a/components/src/dynamo/common/backend/telemetry.py +++ b/dingo/common/backend/telemetry.py @@ -11,7 +11,7 @@ Example:: - from dynamo.common.backend import telemetry + from dingo.common.backend import telemetry span = telemetry.current_span(context) span.set_attribute("ttft_ms", 42.0) diff --git a/components/src/dynamo/common/backend/tests/test_backend_bindings.py b/dingo/common/backend/tests/test_backend_bindings.py similarity index 92% rename from components/src/dynamo/common/backend/tests/test_backend_bindings.py rename to dingo/common/backend/tests/test_backend_bindings.py index 1b7841ff429c..124eee4aced6 100644 --- a/components/src/dynamo/common/backend/tests/test_backend_bindings.py +++ b/dingo/common/backend/tests/test_backend_bindings.py @@ -4,7 +4,7 @@ """Contract tests for the ``dynamo._core.backend`` PyO3 bindings. These tests verify the Rust → Python binding surface that -``dynamo.common.backend.Worker`` delegates to. They DO NOT exercise the +``dingo.common.backend.Worker`` delegates to. They DO NOT exercise the full lifecycle (which would require etcd, NATS, and a running event loop) — that's covered by the Rust unit tests in ``lib/backend-common/src/worker.rs``. Here we just pin down the Python @@ -132,7 +132,7 @@ def test_worker_config_accepts_disaggregation_mode(): @pytest.mark.unified def test_python_worker_config_from_runtime_config_copies_parser_settings(): - from dynamo.common.backend.worker import WorkerConfig + from dingo.common.backend.worker import WorkerConfig runtime_cfg = MagicMock() runtime_cfg.namespace = "test" @@ -169,7 +169,7 @@ def test_python_worker_config_from_runtime_config_copies_parser_settings(): @pytest.mark.unified def test_python_worker_config_from_runtime_config_applies_defaults_when_fields_absent(): - from dynamo.common.backend.worker import WorkerConfig + from dingo.common.backend.worker import WorkerConfig class _BareRuntime: namespace = "ns" @@ -191,7 +191,7 @@ class _BareRuntime: @pytest.mark.unified def test_python_worker_config_from_runtime_config_overrides_win(): - from dynamo.common.backend.worker import WorkerConfig + from dingo.common.backend.worker import WorkerConfig class _WithComponent: namespace = "ns" @@ -209,8 +209,8 @@ class _WithComponent: @pytest.mark.unified def test_python_worker_config_picks_up_disaggregation_mode_from_runtime_config(): - from dynamo.common.backend.worker import WorkerConfig - from dynamo.common.constants import DisaggregationMode + from dingo.common.backend.worker import WorkerConfig + from dingo.common.constants import DisaggregationMode class _Prefill: namespace = "ns" @@ -226,8 +226,8 @@ class _Prefill: @pytest.mark.unified def test_python_worker_config_falls_back_to_serving_mode_for_sglang(): - from dynamo.common.backend.worker import WorkerConfig - from dynamo.common.constants import DisaggregationMode + from dingo.common.backend.worker import WorkerConfig + from dingo.common.constants import DisaggregationMode class _Sglang: # SGLang stores the resolved mode under `serving_mode` rather than @@ -245,8 +245,8 @@ class _Sglang: @pytest.mark.unified def test_python_worker_config_defaults_to_aggregated_when_runtime_lacks_mode(): - from dynamo.common.backend.worker import WorkerConfig - from dynamo.common.constants import DisaggregationMode + from dingo.common.backend.worker import WorkerConfig + from dingo.common.constants import DisaggregationMode class _NoMode: namespace = "ns" @@ -262,12 +262,12 @@ class _NoMode: def test_python_worker_config_coerces_foreign_disaggregation_mode_enum_by_name(): """Foreign enum on `runtime_cfg` (e.g. TRT-LLM's local `DisaggregationMode`) is coerced by `.name` when a member with the - same name exists on `dynamo.common.constants.DisaggregationMode`. + same name exists on `dingo.common.constants.DisaggregationMode`. An explicit `disaggregation_mode=` override still wins.""" import enum - from dynamo.common.backend.worker import WorkerConfig - from dynamo.common.constants import DisaggregationMode + from dingo.common.backend.worker import WorkerConfig + from dingo.common.constants import DisaggregationMode class _ForeignMode(enum.Enum): AGGREGATED = "prefill_and_decode" @@ -296,7 +296,7 @@ class _RuntimeWithForeignMode: def test_python_worker_config_rejects_unrecognized_disaggregation_mode_value(): """A non-enum or unrecognized name on `runtime_cfg.disaggregation_mode` raises TypeError so a typo-string can't silently degrade to AGG.""" - from dynamo.common.backend.worker import WorkerConfig + from dingo.common.backend.worker import WorkerConfig class _RuntimeWithStringMode: namespace = "ns" @@ -311,12 +311,12 @@ class _RuntimeWithStringMode: @pytest.mark.unified def test_python_worker_config_translates_all_disagg_modes(): - """Every variant of dynamo.common.constants.DisaggregationMode must map + """Every variant of dingo.common.constants.DisaggregationMode must map to a Rust binding value -- including ENCODE, which gained unified-path support. Regression for the prior `NotImplementedError` behavior where ENCODE was rejected at translation time.""" - from dynamo.common.backend.worker import _to_rust_disaggregation_mode - from dynamo.common.constants import DisaggregationMode + from dingo.common.backend.worker import _to_rust_disaggregation_mode + from dingo.common.constants import DisaggregationMode rust_mode_for = { DisaggregationMode.AGGREGATED: backend.DisaggregationMode.Aggregated, @@ -334,7 +334,7 @@ def test_python_worker_config_round_trips_route_to_encoder(): -> Rust pyclass without being silently dropped at any layer. vLLM is the only Python backend with the field today; SGLang/TRT-LLM get False via the getattr default until they add the field.""" - from dynamo.common.backend.worker import WorkerConfig + from dingo.common.backend.worker import WorkerConfig # Simulated vLLM-style runtime config that exposes the field. class _RuntimeWithRoute: diff --git a/components/src/dynamo/common/backend/tests/test_disagg_helpers.py b/dingo/common/backend/tests/test_disagg_helpers.py similarity index 95% rename from components/src/dynamo/common/backend/tests/test_disagg_helpers.py rename to dingo/common/backend/tests/test_disagg_helpers.py index 2a2d2092ae1a..c78deb3f44e9 100644 --- a/components/src/dynamo/common/backend/tests/test_disagg_helpers.py +++ b/dingo/common/backend/tests/test_disagg_helpers.py @@ -7,12 +7,12 @@ import pytest -from dynamo.common.backend.disagg import ( +from dingo.common.backend.disagg import ( enforce_prefill_max_tokens, extract_prefill_result, require_prefill_result, ) -from dynamo.common.constants import DisaggregationMode +from dingo.common.constants import DisaggregationMode pytestmark = [pytest.mark.unit, pytest.mark.gpu_0, pytest.mark.pre_merge] diff --git a/components/src/dynamo/common/backend/tests/test_dp_rank.py b/dingo/common/backend/tests/test_dp_rank.py similarity index 94% rename from components/src/dynamo/common/backend/tests/test_dp_rank.py rename to dingo/common/backend/tests/test_dp_rank.py index aeb09d0ff7f5..af97612f873b 100644 --- a/components/src/dynamo/common/backend/tests/test_dp_rank.py +++ b/dingo/common/backend/tests/test_dp_rank.py @@ -7,7 +7,7 @@ import pytest -from dynamo.common.backend.dp_rank import forced_dp_rank, validate_global_dp_rank +from dingo.common.backend.dp_rank import forced_dp_rank, validate_global_dp_rank pytestmark = [pytest.mark.unit, pytest.mark.gpu_0, pytest.mark.pre_merge] diff --git a/components/src/dynamo/common/backend/tests/test_engine.py b/dingo/common/backend/tests/test_engine.py similarity index 99% rename from components/src/dynamo/common/backend/tests/test_engine.py rename to dingo/common/backend/tests/test_engine.py index 6db5b623b648..bba9eca81b40 100644 --- a/components/src/dynamo/common/backend/tests/test_engine.py +++ b/dingo/common/backend/tests/test_engine.py @@ -23,7 +23,7 @@ exc_type=ImportError, ) -from dynamo.common.backend.engine import ( # noqa: E402 +from dingo.common.backend.engine import ( # noqa: E402 DYN_ENABLE_TEST_LOGITS_PROCESSOR, EngineConfig, ForcedTokenSequenceSpec, @@ -37,7 +37,7 @@ resolve_test_logits_processor_spec, serialize_logits_processor_entries, ) -from dynamo.common.constants import DisaggregationMode # noqa: E402 +from dingo.common.constants import DisaggregationMode # noqa: E402 # NOTE: `dynamo.logits_processing.examples` pulls in torch, which is not a # base/common dependency, so it is imported lazily inside the one test that diff --git a/components/src/dynamo/common/backend/tests/test_health_check.py b/dingo/common/backend/tests/test_health_check.py similarity index 98% rename from components/src/dynamo/common/backend/tests/test_health_check.py rename to dingo/common/backend/tests/test_health_check.py index 6017aedd586c..64f99e839a2e 100644 --- a/components/src/dynamo/common/backend/tests/test_health_check.py +++ b/dingo/common/backend/tests/test_health_check.py @@ -9,7 +9,7 @@ import pytest -from dynamo.common.backend.health_check import ( +from dingo.common.backend.health_check import ( HEALTH_CHECK_KEY, bos_token_id_or, build_health_check_payload, diff --git a/components/src/dynamo/common/backend/tests/test_logprobs.py b/dingo/common/backend/tests/test_logprobs.py similarity index 99% rename from components/src/dynamo/common/backend/tests/test_logprobs.py rename to dingo/common/backend/tests/test_logprobs.py index b8000864a76e..736d7ac46034 100644 --- a/components/src/dynamo/common/backend/tests/test_logprobs.py +++ b/dingo/common/backend/tests/test_logprobs.py @@ -15,7 +15,7 @@ import pytest -from dynamo.common.backend.logprobs import ( +from dingo.common.backend.logprobs import ( DYN_SGL_ALLOW_TOP_LOGPROBS_ENV, build_sglang_logprob_kwargs, extract_from_completion_output, diff --git a/components/src/dynamo/common/backend/tests/test_metrics_helpers.py b/dingo/common/backend/tests/test_metrics_helpers.py similarity index 94% rename from components/src/dynamo/common/backend/tests/test_metrics_helpers.py rename to dingo/common/backend/tests/test_metrics_helpers.py index e10827e37f2f..02e2b60200fc 100644 --- a/components/src/dynamo/common/backend/tests/test_metrics_helpers.py +++ b/dingo/common/backend/tests/test_metrics_helpers.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Behavioural tests for engine-facing helpers in -``dynamo.common.backend.metrics``.""" +``dingo.common.backend.metrics``.""" from __future__ import annotations @@ -12,7 +12,7 @@ import pytest from prometheus_client import CollectorRegistry, Gauge -from dynamo.common.backend.metrics import gather_with_labels, register_global_registry +from dingo.common.backend.metrics import gather_with_labels, register_global_registry pytestmark = [pytest.mark.unit, pytest.mark.gpu_0, pytest.mark.pre_merge] diff --git a/components/src/dynamo/common/backend/tests/test_multimodal_helpers.py b/dingo/common/backend/tests/test_multimodal_helpers.py similarity index 98% rename from components/src/dynamo/common/backend/tests/test_multimodal_helpers.py rename to dingo/common/backend/tests/test_multimodal_helpers.py index 3731bc7800dd..f70905592c3b 100644 --- a/components/src/dynamo/common/backend/tests/test_multimodal_helpers.py +++ b/dingo/common/backend/tests/test_multimodal_helpers.py @@ -13,12 +13,12 @@ import pytest -from dynamo.common.backend.multimodal import ( +from dingo.common.backend.multimodal import ( encoder_terminal_chunk, extract_multimodal_kwargs, require_encoder_result, ) -from dynamo.common.constants import DisaggregationMode +from dingo.common.constants import DisaggregationMode pytestmark = [pytest.mark.unit, pytest.mark.gpu_0, pytest.mark.pre_merge] diff --git a/components/src/dynamo/common/backend/tests/test_publisher.py b/dingo/common/backend/tests/test_publisher.py similarity index 96% rename from components/src/dynamo/common/backend/tests/test_publisher.py rename to dingo/common/backend/tests/test_publisher.py index 97b00ad4de4a..d02ed28854e5 100644 --- a/components/src/dynamo/common/backend/tests/test_publisher.py +++ b/dingo/common/backend/tests/test_publisher.py @@ -16,13 +16,13 @@ ) from dynamo._core import Context # noqa: E402 -from dynamo.common.backend.engine import ( # noqa: E402 +from dingo.common.backend.engine import ( # noqa: E402 EngineConfig, GenerateChunk, GenerateRequest, LLMEngine, ) -from dynamo.common.backend.publisher import ( # noqa: E402 +from dingo.common.backend.publisher import ( # noqa: E402 ComponentSnapshot, PushSource, ZmqSource, @@ -90,7 +90,7 @@ async def test_vllm_kv_event_sources_return_one_zmq_source_per_dp_rank(monkeypat mod = pytest.importorskip( "dingo.vllm.llm_engine", reason="vLLM backend dependencies not installed" ) - from dynamo.common.constants import DisaggregationMode + from dingo.common.constants import DisaggregationMode engine = mod.VllmLLMEngine.__new__(mod.VllmLLMEngine) engine.engine_args = SimpleNamespace( diff --git a/components/src/dynamo/common/backend/tests/test_sample_diffusion_engine.py b/dingo/common/backend/tests/test_sample_diffusion_engine.py similarity index 95% rename from components/src/dynamo/common/backend/tests/test_sample_diffusion_engine.py rename to dingo/common/backend/tests/test_sample_diffusion_engine.py index aae3f95cc1d0..1bbf504eba60 100644 --- a/components/src/dynamo/common/backend/tests/test_sample_diffusion_engine.py +++ b/dingo/common/backend/tests/test_sample_diffusion_engine.py @@ -18,9 +18,9 @@ reason="dynamo._core.backend not built — run `maturin develop` first", ) -from dynamo.common.backend.engine import RawEngine # noqa: E402 -from dynamo.common.backend.health_check import is_probe # noqa: E402 -from dynamo.common.backend.sample_diffusion_engine import ( # noqa: E402 +from dingo.common.backend.engine import RawEngine # noqa: E402 +from dingo.common.backend.health_check import is_probe # noqa: E402 +from dingo.common.backend.sample_diffusion_engine import ( # noqa: E402 SampleDiffusionEngine, ) from dynamo.llm import ModelInput # noqa: E402 diff --git a/components/src/dynamo/common/backend/tests/test_sample_engine.py b/dingo/common/backend/tests/test_sample_engine.py similarity index 98% rename from components/src/dynamo/common/backend/tests/test_sample_engine.py rename to dingo/common/backend/tests/test_sample_engine.py index 0706b550ac3e..e4d308de14ff 100644 --- a/components/src/dynamo/common/backend/tests/test_sample_engine.py +++ b/dingo/common/backend/tests/test_sample_engine.py @@ -14,9 +14,9 @@ reason="dynamo._core.backend not built — run `maturin develop` first", ) -from dynamo.common.backend.publisher import PushSource # noqa: E402 -from dynamo.common.backend.sample_engine import SampleLLMEngine # noqa: E402 -from dynamo.common.constants import DisaggregationMode # noqa: E402 +from dingo.common.backend.publisher import PushSource # noqa: E402 +from dingo.common.backend.sample_engine import SampleLLMEngine # noqa: E402 +from dingo.common.constants import DisaggregationMode # noqa: E402 pytestmark = [ pytest.mark.unit, diff --git a/components/src/dynamo/common/backend/tests/test_telemetry.py b/dingo/common/backend/tests/test_telemetry.py similarity index 99% rename from components/src/dynamo/common/backend/tests/test_telemetry.py rename to dingo/common/backend/tests/test_telemetry.py index 820d42e5002a..954e59d928f7 100644 --- a/components/src/dynamo/common/backend/tests/test_telemetry.py +++ b/dingo/common/backend/tests/test_telemetry.py @@ -20,7 +20,7 @@ import pytest -from dynamo.common.backend import telemetry +from dingo.common.backend import telemetry pytestmark = [pytest.mark.unit, pytest.mark.gpu_0, pytest.mark.pre_merge] diff --git a/components/src/dynamo/common/backend/worker.py b/dingo/common/backend/worker.py similarity index 98% rename from components/src/dynamo/common/backend/worker.py rename to dingo/common/backend/worker.py index d4f3fa1a384a..0e1eee40716a 100644 --- a/components/src/dynamo/common/backend/worker.py +++ b/dingo/common/backend/worker.py @@ -26,7 +26,7 @@ from typing import Optional from dynamo._core import backend as _backend -from dynamo.common.constants import DisaggregationMode +from dingo.common.constants import DisaggregationMode from dynamo.llm import ModelInput from dynamo.runtime.logging import configure_dynamo_logging @@ -62,7 +62,7 @@ def add_signal_handler(sig, callback, *args): loop.add_signal_handler = add_signal_handler # type: ignore[assignment] -# Map the user-facing `dynamo.common.constants.DisaggregationMode` to the +# Map the user-facing `dingo.common.constants.DisaggregationMode` to the # Rust enum. All four modes (AGGREGATED, PREFILL, DECODE, ENCODE) are # supported by the unified abstraction. _DISAGG_MODE_TO_RUST = { @@ -97,7 +97,7 @@ def _coerce_disagg_mode(value) -> DisaggregationMode: return DisaggregationMode[value.name] raise TypeError( f"disaggregation_mode is {type(value).__name__}({value!r}); " - "expected dynamo.common.constants.DisaggregationMode" + "expected dingo.common.constants.DisaggregationMode" ) diff --git a/components/src/dynamo/common/config_dump/__init__.py b/dingo/common/config_dump/__init__.py similarity index 78% rename from components/src/dynamo/common/config_dump/__init__.py rename to dingo/common/config_dump/__init__.py index 7353e5525a45..f375d76b0dc3 100644 --- a/components/src/dynamo/common/config_dump/__init__.py +++ b/dingo/common/config_dump/__init__.py @@ -8,14 +8,14 @@ for debugging and diagnostics purposes. """ -from dynamo.common.config_dump.config_dumper import ( +from dingo.common.config_dump.config_dumper import ( add_config_dump_args, dump_config, get_config_dump, register_encoder, ) -from dynamo.common.config_dump.environment import get_environment_vars -from dynamo.common.config_dump.system_info import ( +from dingo.common.config_dump.environment import get_environment_vars +from dingo.common.config_dump.system_info import ( get_gpu_info, get_runtime_info, get_system_info, diff --git a/components/src/dynamo/common/config_dump/config_dumper.py b/dingo/common/config_dump/config_dumper.py similarity index 99% rename from components/src/dynamo/common/config_dump/config_dumper.py rename to dingo/common/config_dump/config_dumper.py index 8d7b1f9c5e3d..1a834d72adf8 100644 --- a/components/src/dynamo/common/config_dump/config_dumper.py +++ b/dingo/common/config_dump/config_dumper.py @@ -62,7 +62,7 @@ def _get_vllm_version() -> Optional[str]: def _get_dynamo_version() -> str: """Get Dynamo version.""" try: - from dynamo.common import __version__ + from dingo.common import __version__ except Exception: __version__ = "0.0.0+unknown" diff --git a/components/src/dynamo/common/config_dump/environment.py b/dingo/common/config_dump/environment.py similarity index 100% rename from components/src/dynamo/common/config_dump/environment.py rename to dingo/common/config_dump/environment.py diff --git a/components/src/dynamo/common/config_dump/system_info.py b/dingo/common/config_dump/system_info.py similarity index 100% rename from components/src/dynamo/common/config_dump/system_info.py rename to dingo/common/config_dump/system_info.py diff --git a/components/src/dynamo/common/configuration/__init__.py b/dingo/common/configuration/__init__.py similarity index 100% rename from components/src/dynamo/common/configuration/__init__.py rename to dingo/common/configuration/__init__.py diff --git a/components/src/dynamo/common/configuration/arg_group.py b/dingo/common/configuration/arg_group.py similarity index 100% rename from components/src/dynamo/common/configuration/arg_group.py rename to dingo/common/configuration/arg_group.py diff --git a/components/src/dynamo/common/configuration/config_base.py b/dingo/common/configuration/config_base.py similarity index 100% rename from components/src/dynamo/common/configuration/config_base.py rename to dingo/common/configuration/config_base.py diff --git a/components/src/dynamo/common/configuration/groups/__init__.py b/dingo/common/configuration/groups/__init__.py similarity index 100% rename from components/src/dynamo/common/configuration/groups/__init__.py rename to dingo/common/configuration/groups/__init__.py diff --git a/components/src/dynamo/common/configuration/groups/aic_perf_args.py b/dingo/common/configuration/groups/aic_perf_args.py similarity index 96% rename from components/src/dynamo/common/configuration/groups/aic_perf_args.py rename to dingo/common/configuration/groups/aic_perf_args.py index 1a723012e0d2..c784d7bb4c52 100644 --- a/components/src/dynamo/common/configuration/groups/aic_perf_args.py +++ b/dingo/common/configuration/groups/aic_perf_args.py @@ -5,9 +5,9 @@ from typing import Optional -from dynamo.common.configuration.arg_group import ArgGroup -from dynamo.common.configuration.config_base import ConfigBase -from dynamo.common.configuration.utils import add_argument +from dingo.common.configuration.arg_group import ArgGroup +from dingo.common.configuration.config_base import ConfigBase +from dingo.common.configuration.utils import add_argument _AIC_PERF_FIELDS: tuple[str, ...] = ( "aic_backend", diff --git a/components/src/dynamo/common/configuration/groups/frontend_decoding_args.py b/dingo/common/configuration/groups/frontend_decoding_args.py similarity index 96% rename from components/src/dynamo/common/configuration/groups/frontend_decoding_args.py rename to dingo/common/configuration/groups/frontend_decoding_args.py index 0b79a1ab87d3..ea62cea66331 100644 --- a/components/src/dynamo/common/configuration/groups/frontend_decoding_args.py +++ b/dingo/common/configuration/groups/frontend_decoding_args.py @@ -16,7 +16,7 @@ flag plumbing is shared. """ -from dynamo.common.configuration.utils import add_negatable_bool_argument +from dingo.common.configuration.utils import add_negatable_bool_argument def add_frontend_decoding_arg(g, *, env_prefix: str) -> None: diff --git a/components/src/dynamo/common/configuration/groups/http_args.py b/dingo/common/configuration/groups/http_args.py similarity index 95% rename from components/src/dynamo/common/configuration/groups/http_args.py rename to dingo/common/configuration/groups/http_args.py index 1ad1f74e31b8..a4c06ff14b80 100644 --- a/components/src/dynamo/common/configuration/groups/http_args.py +++ b/dingo/common/configuration/groups/http_args.py @@ -1,13 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Configuration for the ``dynamo.common.http`` client. +"""Configuration for the ``dingo.common.http`` client. :class:`HttpConfigBase` carries the operator-tunable knobs; :class:`HttpArgGroup` registers the matching ``--http-*`` CLI flags and ``DYN_HTTP_*`` env vars. :func:`from_env` is an env-only construction path used by the singleton client in -``dynamo.common.http`` (whose primary callers don't own an argparse). +``dingo.common.http`` (whose primary callers don't own an argparse). Legacy ``DYN_MM_HTTP_*`` env vars are still honored for backward compatibility with deployments that predate the rename — see @@ -26,16 +26,16 @@ # typing_extensions back-port (added as an explicit dep in pyproject.toml). from typing_extensions import Self -from dynamo.common.configuration.arg_group import ArgGroup -from dynamo.common.configuration.config_base import ConfigBase -from dynamo.common.configuration.utils import add_argument, nullable_float +from dingo.common.configuration.arg_group import ArgGroup +from dingo.common.configuration.config_base import ConfigBase +from dingo.common.configuration.utils import add_argument, nullable_float logger = logging.getLogger(__name__) # Map of canonical env var → list of legacy aliases (priority order). # Only the six env vars that shipped via the now-deleted -# ``dynamo.common.multimodal.http_client`` module are kept for back +# ``dingo.common.multimodal.http_client`` module are kept for back # compat. ``DYN_HTTP_TIMEOUT`` accepts ``DYN_MM_HTTP_READ_TIMEOUT`` # because the read-vs-total semantic was clarified during the rename. _LEGACY_ENV_ALIASES: dict[str, tuple[str, ...]] = { diff --git a/components/src/dynamo/common/configuration/groups/kv_router_args.py b/dingo/common/configuration/groups/kv_router_args.py similarity index 99% rename from components/src/dynamo/common/configuration/groups/kv_router_args.py rename to dingo/common/configuration/groups/kv_router_args.py index 0beeecd39e04..a8dd19198efb 100644 --- a/components/src/dynamo/common/configuration/groups/kv_router_args.py +++ b/dingo/common/configuration/groups/kv_router_args.py @@ -15,9 +15,9 @@ import warnings from typing import Optional -from dynamo.common.configuration.arg_group import ArgGroup -from dynamo.common.configuration.config_base import ConfigBase -from dynamo.common.configuration.utils import ( +from dingo.common.configuration.arg_group import ArgGroup +from dingo.common.configuration.config_base import ConfigBase +from dingo.common.configuration.utils import ( add_argument, add_negatable_bool_argument, nullable_float, diff --git a/components/src/dynamo/common/configuration/groups/router_args.py b/dingo/common/configuration/groups/router_args.py similarity index 98% rename from components/src/dynamo/common/configuration/groups/router_args.py rename to dingo/common/configuration/groups/router_args.py index 51ab63c15a16..814c29fccc94 100644 --- a/components/src/dynamo/common/configuration/groups/router_args.py +++ b/dingo/common/configuration/groups/router_args.py @@ -14,9 +14,9 @@ import logging from typing import Any, Optional -from dynamo.common.configuration.arg_group import ArgGroup -from dynamo.common.configuration.config_base import ConfigBase -from dynamo.common.configuration.utils import ( +from dingo.common.configuration.arg_group import ArgGroup +from dingo.common.configuration.config_base import ConfigBase +from dingo.common.configuration.utils import ( add_argument, add_negatable_bool_argument, nullable_float, diff --git a/components/src/dynamo/common/configuration/groups/runtime_args.py b/dingo/common/configuration/groups/runtime_args.py similarity index 97% rename from components/src/dynamo/common/configuration/groups/runtime_args.py rename to dingo/common/configuration/groups/runtime_args.py index ab646b1f8737..0a8f4b8e45de 100644 --- a/components/src/dynamo/common/configuration/groups/runtime_args.py +++ b/dingo/common/configuration/groups/runtime_args.py @@ -7,11 +7,11 @@ from typing import List, Optional from dynamo._core import get_reasoning_parser_names, get_tool_parser_names -from dynamo.common.configuration.arg_group import ArgGroup -from dynamo.common.configuration.config_base import ConfigBase -from dynamo.common.configuration.utils import add_argument, add_negatable_bool_argument -from dynamo.common.utils.namespace import get_worker_namespace -from dynamo.common.utils.output_modalities import OutputModality +from dingo.common.configuration.arg_group import ArgGroup +from dingo.common.configuration.config_base import ConfigBase +from dingo.common.configuration.utils import add_argument, add_negatable_bool_argument +from dingo.common.utils.namespace import get_worker_namespace +from dingo.common.utils.output_modalities import OutputModality class DynamoRuntimeConfig(ConfigBase): diff --git a/components/src/dynamo/common/configuration/utils.py b/dingo/common/configuration/utils.py similarity index 100% rename from components/src/dynamo/common/configuration/utils.py rename to dingo/common/configuration/utils.py diff --git a/components/src/dynamo/common/constants.py b/dingo/common/constants.py similarity index 100% rename from components/src/dynamo/common/constants.py rename to dingo/common/constants.py diff --git a/components/src/dynamo/common/engine_monitor.py b/dingo/common/engine_monitor.py similarity index 97% rename from components/src/dynamo/common/engine_monitor.py rename to dingo/common/engine_monitor.py index 4604977feb20..e7c2f10d04c0 100644 --- a/components/src/dynamo/common/engine_monitor.py +++ b/dingo/common/engine_monitor.py @@ -10,7 +10,7 @@ from dataclasses import dataclass from typing import Optional -from dynamo.common.configuration.utils import env_or_default +from dingo.common.configuration.utils import env_or_default logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/common/forward_pass_metrics.py b/dingo/common/forward_pass_metrics.py similarity index 99% rename from components/src/dynamo/common/forward_pass_metrics.py rename to dingo/common/forward_pass_metrics.py index f656ee7bbeb0..4055a2c878c5 100644 --- a/components/src/dynamo/common/forward_pass_metrics.py +++ b/dingo/common/forward_pass_metrics.py @@ -27,7 +27,7 @@ in the parent process handles event plane transport and discovery registration automatically. -See ``dynamo.common.recv_forward_pass_metrics`` for a standalone +See ``dingo.common.recv_forward_pass_metrics`` for a standalone consumer example. TODO: add metrics for TrtLLM/SGLang diff --git a/components/src/dynamo/common/http/README.md b/dingo/common/http/README.md similarity index 99% rename from components/src/dynamo/common/http/README.md rename to dingo/common/http/README.md index de026f13decc..6fc31a450f8f 100644 --- a/components/src/dynamo/common/http/README.md +++ b/dingo/common/http/README.md @@ -1,4 +1,4 @@ -# `dynamo.common.http` +# `dingo.common.http` HTTP fetch client. Backend-neutral facade (`fetch_bytes` / `close_http_client`) over an `HttpClient` ABC with diff --git a/components/src/dynamo/common/http/__init__.py b/dingo/common/http/__init__.py similarity index 98% rename from components/src/dynamo/common/http/__init__.py rename to dingo/common/http/__init__.py index e28d1ffee93a..927e62ed94a3 100644 --- a/components/src/dynamo/common/http/__init__.py +++ b/dingo/common/http/__init__.py @@ -28,7 +28,7 @@ import os from typing import Optional -from dynamo.common.configuration.groups.http_args import ( +from dingo.common.configuration.groups.http_args import ( HttpArgGroup, HttpConfigBase, _apply_legacy_env_aliases, diff --git a/components/src/dynamo/common/http/aiohttp_client.py b/dingo/common/http/aiohttp_client.py similarity index 100% rename from components/src/dynamo/common/http/aiohttp_client.py rename to dingo/common/http/aiohttp_client.py diff --git a/components/src/dynamo/common/http/base.py b/dingo/common/http/base.py similarity index 98% rename from components/src/dynamo/common/http/base.py rename to dingo/common/http/base.py index 22c5724def33..bcfce6ad7d6b 100644 --- a/components/src/dynamo/common/http/base.py +++ b/dingo/common/http/base.py @@ -15,7 +15,7 @@ import asyncio from typing import Optional -from dynamo.common.configuration.groups.http_args import HttpConfigBase, from_env +from dingo.common.configuration.groups.http_args import HttpConfigBase, from_env from .url_validator import ( _MAX_REDIRECTS, diff --git a/components/src/dynamo/common/http/httpx_client.py b/dingo/common/http/httpx_client.py similarity index 100% rename from components/src/dynamo/common/http/httpx_client.py rename to dingo/common/http/httpx_client.py diff --git a/components/src/dynamo/common/http/url_validator.py b/dingo/common/http/url_validator.py similarity index 100% rename from components/src/dynamo/common/http/url_validator.py rename to dingo/common/http/url_validator.py diff --git a/components/src/dynamo/common/lora/__init__.py b/dingo/common/lora/__init__.py similarity index 100% rename from components/src/dynamo/common/lora/__init__.py rename to dingo/common/lora/__init__.py diff --git a/components/src/dynamo/common/lora/manager.py b/dingo/common/lora/manager.py similarity index 98% rename from components/src/dynamo/common/lora/manager.py rename to dingo/common/lora/manager.py index aa6eab54810e..c96575669a91 100644 --- a/components/src/dynamo/common/lora/manager.py +++ b/dingo/common/lora/manager.py @@ -11,8 +11,8 @@ from pathlib import Path from typing import Any, Dict, Optional, Protocol -from dynamo.common.lora.once import OnceLock -from dynamo.common.utils.env import env_bool +from dingo.common.lora.once import OnceLock +from dingo.common.utils.env import env_bool from dynamo.llm import LoRADownloader logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/common/lora/once.py b/dingo/common/lora/once.py similarity index 100% rename from components/src/dynamo/common/lora/once.py rename to dingo/common/lora/once.py diff --git a/components/src/dynamo/common/memory/__init__.py b/dingo/common/memory/__init__.py similarity index 80% rename from components/src/dynamo/common/memory/__init__.py rename to dingo/common/memory/__init__.py index d600f85a82c0..fb070b11d911 100644 --- a/components/src/dynamo/common/memory/__init__.py +++ b/dingo/common/memory/__init__.py @@ -3,7 +3,7 @@ """Memory management utilities for Dynamo components.""" -from dynamo.common.memory.multimodal_embedding_cache_manager import ( +from dingo.common.memory.multimodal_embedding_cache_manager import ( MultimodalEmbeddingCacheManager, ) diff --git a/components/src/dynamo/common/memory/multimodal_embedding_cache_manager.py b/dingo/common/memory/multimodal_embedding_cache_manager.py similarity index 100% rename from components/src/dynamo/common/memory/multimodal_embedding_cache_manager.py rename to dingo/common/memory/multimodal_embedding_cache_manager.py diff --git a/components/src/dynamo/common/metadata_upload.py b/dingo/common/metadata_upload.py similarity index 98% rename from components/src/dynamo/common/metadata_upload.py rename to dingo/common/metadata_upload.py index a29f21a9cbbb..a8c2929ab538 100644 --- a/components/src/dynamo/common/metadata_upload.py +++ b/dingo/common/metadata_upload.py @@ -30,7 +30,7 @@ def _backend_metadata_upload_settings(request: dict[str, Any]) -> dict[str, Any] async def _upload_bytes(url: str, storage_path: str, data: bytes) -> str: try: - from dynamo.common.storage import get_fs, upload_to_fs + from dingo.common.storage import get_fs, upload_to_fs except ImportError as exc: raise RuntimeError( "Metadata upload requires fsspec support. " diff --git a/components/src/dynamo/common/model_fetch.py b/dingo/common/model_fetch.py similarity index 98% rename from components/src/dynamo/common/model_fetch.py rename to dingo/common/model_fetch.py index 72b2b2020896..85b671906439 100644 --- a/components/src/dynamo/common/model_fetch.py +++ b/dingo/common/model_fetch.py @@ -10,7 +10,7 @@ import sys from multiprocessing.connection import Connection -from dynamo.common.snapshot.lifecycle import ( +from dingo.common.snapshot.lifecycle import ( SENTINEL_POLL_INTERVAL_SEC, is_snapshot_enabled, ) diff --git a/components/src/dynamo/common/multimodal/__init__.py b/dingo/common/multimodal/__init__.py similarity index 81% rename from components/src/dynamo/common/multimodal/__init__.py rename to dingo/common/multimodal/__init__.py index f99aa7c249d2..03a6231cef1d 100644 --- a/components/src/dynamo/common/multimodal/__init__.py +++ b/dingo/common/multimodal/__init__.py @@ -5,10 +5,10 @@ from collections.abc import Callable -from dynamo.common.constants import EmbeddingTransferMode -from dynamo.common.multimodal.async_encoder_cache import AsyncEncoderCache -from dynamo.common.multimodal.audio_loader import AudioLoader -from dynamo.common.multimodal.embedding_transfer import ( +from dingo.common.constants import EmbeddingTransferMode +from dingo.common.multimodal.async_encoder_cache import AsyncEncoderCache +from dingo.common.multimodal.audio_loader import AudioLoader +from dingo.common.multimodal.embedding_transfer import ( AbstractEmbeddingReceiver, AbstractEmbeddingSender, LocalEmbeddingReceiver, @@ -19,8 +19,8 @@ NixlWriteEmbeddingSender, TransferRequest, ) -from dynamo.common.multimodal.image_loader import ImageLoader -from dynamo.common.multimodal.video_loader import VideoLoader +from dingo.common.multimodal.image_loader import ImageLoader +from dingo.common.multimodal.video_loader import VideoLoader EMBEDDING_SENDER_FACTORIES: dict[ EmbeddingTransferMode, Callable[[], AbstractEmbeddingSender] diff --git a/components/src/dynamo/common/multimodal/async_encoder_cache.py b/dingo/common/multimodal/async_encoder_cache.py similarity index 98% rename from components/src/dynamo/common/multimodal/async_encoder_cache.py rename to dingo/common/multimodal/async_encoder_cache.py index f4f7add1156b..dcb38dbf4654 100644 --- a/components/src/dynamo/common/multimodal/async_encoder_cache.py +++ b/dingo/common/multimodal/async_encoder_cache.py @@ -19,7 +19,7 @@ import logging from typing import Awaitable, Callable, Dict, Optional -from dynamo.common.memory.multimodal_embedding_cache_manager import ( +from dingo.common.memory.multimodal_embedding_cache_manager import ( CachedEmbedding, MultimodalEmbeddingCacheManager, ) diff --git a/components/src/dynamo/common/multimodal/audio_loader.py b/dingo/common/multimodal/audio_loader.py similarity index 96% rename from components/src/dynamo/common/multimodal/audio_loader.py rename to dingo/common/multimodal/audio_loader.py index c0095f767a3f..363ba405d2aa 100644 --- a/components/src/dynamo/common/multimodal/audio_loader.py +++ b/dingo/common/multimodal/audio_loader.py @@ -8,10 +8,10 @@ import numpy as np -from dynamo.common.http import fetch_bytes -from dynamo.common.http.url_validator import UrlValidationPolicy, validate_media_url -from dynamo.common.utils import nvtx_utils as _nvtx -from dynamo.common.utils.runtime import run_async +from dingo.common.http import fetch_bytes +from dingo.common.http.url_validator import UrlValidationPolicy, validate_media_url +from dingo.common.utils import nvtx_utils as _nvtx +from dingo.common.utils.runtime import run_async logger = logging.getLogger(__name__) @@ -34,7 +34,7 @@ def _create_nixl_connector() -> Any: async def read_decoded_media_via_nixl(*args: Any, **kwargs: Any) -> Any: try: - from dynamo.common.utils.media_nixl import ( + from dingo.common.utils.media_nixl import ( read_decoded_media_via_nixl as _read_decoded_media_via_nixl, ) except ImportError as exc: diff --git a/components/src/dynamo/common/multimodal/embedding_transfer.py b/dingo/common/multimodal/embedding_transfer.py similarity index 99% rename from components/src/dynamo/common/multimodal/embedding_transfer.py rename to dingo/common/multimodal/embedding_transfer.py index d5326df49517..d994f4377f8e 100644 --- a/components/src/dynamo/common/multimodal/embedding_transfer.py +++ b/dingo/common/multimodal/embedding_transfer.py @@ -18,8 +18,8 @@ from pydantic import BaseModel from safetensors import torch as safetensors_torch -from dynamo.common.utils import nvtx_utils as _nvtx -from dynamo.common.utils.runtime import run_async +from dingo.common.utils import nvtx_utils as _nvtx +from dingo.common.utils.runtime import run_async logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/common/multimodal/image_loader.py b/dingo/common/multimodal/image_loader.py similarity index 98% rename from components/src/dynamo/common/multimodal/image_loader.py rename to dingo/common/multimodal/image_loader.py index f1c8ec68e955..f95de1859d97 100644 --- a/components/src/dynamo/common/multimodal/image_loader.py +++ b/dingo/common/multimodal/image_loader.py @@ -13,8 +13,8 @@ from PIL import Image -from dynamo.common.utils import nvtx_utils as _nvtx -from dynamo.common.utils.runtime import run_async +from dingo.common.utils import nvtx_utils as _nvtx +from dingo.common.utils.runtime import run_async from ..http import HttpError, HttpStatusError, HttpTimeoutError, fetch_bytes from ..http.url_validator import UrlValidationPolicy, validate_media_url @@ -40,7 +40,7 @@ def _create_nixl_connector() -> Any: async def read_decoded_media_via_nixl(*args: Any, **kwargs: Any) -> Any: try: - from dynamo.common.utils.media_nixl import ( + from dingo.common.utils.media_nixl import ( read_decoded_media_via_nixl as _read_decoded_media_via_nixl, ) except ImportError as exc: diff --git a/components/src/dynamo/common/multimodal/media_connector.py b/dingo/common/multimodal/media_connector.py similarity index 96% rename from components/src/dynamo/common/multimodal/media_connector.py rename to dingo/common/multimodal/media_connector.py index 73d54848c3e8..7fb971fd3f59 100644 --- a/components/src/dynamo/common/multimodal/media_connector.py +++ b/dingo/common/multimodal/media_connector.py @@ -11,7 +11,7 @@ Usage: # Import this module to register the connector, then set the env var: - import dynamo.common.multimodal.media_connector # noqa: F401 + import dingo.common.multimodal.media_connector # noqa: F401 os.environ["VLLM_MEDIA_CONNECTOR"] = "dynamo" """ @@ -19,7 +19,7 @@ from PIL import Image -from dynamo.common.multimodal.image_loader import ImageLoader +from dingo.common.multimodal.image_loader import ImageLoader logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/common/multimodal/mm_kwargs_transfer.py b/dingo/common/multimodal/mm_kwargs_transfer.py similarity index 99% rename from components/src/dynamo/common/multimodal/mm_kwargs_transfer.py rename to dingo/common/multimodal/mm_kwargs_transfer.py index 76c112f74de5..edbf9c29ad74 100644 --- a/components/src/dynamo/common/multimodal/mm_kwargs_transfer.py +++ b/dingo/common/multimodal/mm_kwargs_transfer.py @@ -34,8 +34,8 @@ import torch from pydantic import BaseModel -from dynamo.common.utils import nvtx_utils as _nvtx -from dynamo.common.utils.runtime import run_async +from dingo.common.utils import nvtx_utils as _nvtx +from dingo.common.utils.runtime import run_async logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/common/multimodal/routing_utils.py b/dingo/common/multimodal/routing_utils.py similarity index 100% rename from components/src/dynamo/common/multimodal/routing_utils.py rename to dingo/common/multimodal/routing_utils.py diff --git a/components/src/dynamo/common/multimodal/video_loader.py b/dingo/common/multimodal/video_loader.py similarity index 97% rename from components/src/dynamo/common/multimodal/video_loader.py rename to dingo/common/multimodal/video_loader.py index a5b18590a1cb..c372d4ea5f33 100644 --- a/components/src/dynamo/common/multimodal/video_loader.py +++ b/dingo/common/multimodal/video_loader.py @@ -21,9 +21,9 @@ import numpy as np -from dynamo.common.http import fetch_bytes -from dynamo.common.http.url_validator import UrlValidationPolicy, validate_media_url -from dynamo.common.utils.runtime import run_async +from dingo.common.http import fetch_bytes +from dingo.common.http.url_validator import UrlValidationPolicy, validate_media_url +from dingo.common.utils.runtime import run_async logger = logging.getLogger(__name__) @@ -46,7 +46,7 @@ def _create_nixl_connector() -> Any: async def read_decoded_media_via_nixl(*args: Any, **kwargs: Any) -> Any: try: - from dynamo.common.utils.media_nixl import ( + from dingo.common.utils.media_nixl import ( read_decoded_media_via_nixl as _read_decoded_media_via_nixl, ) except ImportError as exc: diff --git a/components/src/dynamo/common/protocols/__init__.py b/dingo/common/protocols/__init__.py similarity index 90% rename from components/src/dynamo/common/protocols/__init__.py rename to dingo/common/protocols/__init__.py index 55f2125b21e8..c1976f030e12 100644 --- a/components/src/dynamo/common/protocols/__init__.py +++ b/dingo/common/protocols/__init__.py @@ -7,7 +7,7 @@ - video_protocol: NvCreateVideoRequest, NvVideosResponse for video generation """ -from dynamo.common.protocols.video_protocol import ( +from dingo.common.protocols.video_protocol import ( NvCreateVideoRequest, NvVideosResponse, VideoData, diff --git a/components/src/dynamo/common/protocols/audio_protocol.py b/dingo/common/protocols/audio_protocol.py similarity index 100% rename from components/src/dynamo/common/protocols/audio_protocol.py rename to dingo/common/protocols/audio_protocol.py diff --git a/components/src/dynamo/common/protocols/image_protocol.py b/dingo/common/protocols/image_protocol.py similarity index 100% rename from components/src/dynamo/common/protocols/image_protocol.py rename to dingo/common/protocols/image_protocol.py diff --git a/components/src/dynamo/common/protocols/video_protocol.py b/dingo/common/protocols/video_protocol.py similarity index 100% rename from components/src/dynamo/common/protocols/video_protocol.py rename to dingo/common/protocols/video_protocol.py diff --git a/components/src/dynamo/common/recv_forward_pass_metrics.py b/dingo/common/recv_forward_pass_metrics.py similarity index 96% rename from components/src/dynamo/common/recv_forward_pass_metrics.py rename to dingo/common/recv_forward_pass_metrics.py index 6e61191ee91e..dc7f81ad931d 100644 --- a/components/src/dynamo/common/recv_forward_pass_metrics.py +++ b/dingo/common/recv_forward_pass_metrics.py @@ -15,16 +15,16 @@ Usage: # recv mode (default) - python -m dynamo.common.recv_forward_pass_metrics \\ + python -m dingo.common.recv_forward_pass_metrics \\ --namespace dynamo --component backend --endpoint generate # tracking mode (poll every 2 seconds) - python -m dynamo.common.recv_forward_pass_metrics \\ + python -m dingo.common.recv_forward_pass_metrics \\ --namespace dynamo --component backend --endpoint generate \\ --mode tracking --poll-interval 2.0 # recv mode with plot saving - python -m dynamo.common.recv_forward_pass_metrics \\ + python -m dingo.common.recv_forward_pass_metrics \\ --namespace dynamo --component backend --endpoint generate \\ --save-plot metrics.png """ @@ -40,7 +40,7 @@ import matplotlib.pyplot as plt import msgspec -from dynamo.common.forward_pass_metrics import ForwardPassMetrics, decode +from dingo.common.forward_pass_metrics import ForwardPassMetrics, decode from dynamo.runtime import DistributedRuntime from dynamo.runtime.logging import configure_dynamo_logging diff --git a/components/src/dynamo/common/rl/__init__.py b/dingo/common/rl/__init__.py similarity index 100% rename from components/src/dynamo/common/rl/__init__.py rename to dingo/common/rl/__init__.py diff --git a/components/src/dynamo/common/rl/admin.py b/dingo/common/rl/admin.py similarity index 100% rename from components/src/dynamo/common/rl/admin.py rename to dingo/common/rl/admin.py diff --git a/components/src/dynamo/common/snapshot/__init__.py b/dingo/common/snapshot/__init__.py similarity index 100% rename from components/src/dynamo/common/snapshot/__init__.py rename to dingo/common/snapshot/__init__.py diff --git a/components/src/dynamo/common/snapshot/constants.py b/dingo/common/snapshot/constants.py similarity index 100% rename from components/src/dynamo/common/snapshot/constants.py rename to dingo/common/snapshot/constants.py diff --git a/components/src/dynamo/common/snapshot/lifecycle.py b/dingo/common/snapshot/lifecycle.py similarity index 99% rename from components/src/dynamo/common/snapshot/lifecycle.py rename to dingo/common/snapshot/lifecycle.py index ba67f1358e80..615c099465fb 100644 --- a/components/src/dynamo/common/snapshot/lifecycle.py +++ b/dingo/common/snapshot/lifecycle.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import Any, Generic, TypeVar -from dynamo.common.snapshot.constants import ( +from dingo.common.snapshot.constants import ( READY_FOR_SNAPSHOT_FILE, RESTORE_COMPLETE_FILE, SNAPSHOT_COMPLETE_FILE, diff --git a/components/src/dynamo/common/snapshot/restore_context.py b/dingo/common/snapshot/restore_context.py similarity index 98% rename from components/src/dynamo/common/snapshot/restore_context.py rename to dingo/common/snapshot/restore_context.py index e86f9f362d51..dc611617eef9 100644 --- a/components/src/dynamo/common/snapshot/restore_context.py +++ b/dingo/common/snapshot/restore_context.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import Awaitable, Callable, Mapping, TypeVar -from dynamo.common.snapshot.constants import ( +from dingo.common.snapshot.constants import ( KUBERNETES_OPTIONAL_ENV_NAMES, KUBERNETES_REQUIRED_ENV_NAMES, RESTORE_RUNTIME_ENV_NAMES, @@ -100,7 +100,7 @@ def parse_snapshot_restore_runtime_config(argv: list[str] | None) -> object: import argparse - from dynamo.common.configuration.groups.runtime_args import ( + from dingo.common.configuration.groups.runtime_args import ( DynamoRuntimeArgGroup, DynamoRuntimeConfig, ) diff --git a/components/src/dynamo/common/storage.py b/dingo/common/storage.py similarity index 100% rename from components/src/dynamo/common/storage.py rename to dingo/common/storage.py diff --git a/components/src/dynamo/common/tests/configuration/test_kv_router_args.py b/dingo/common/tests/configuration/test_kv_router_args.py similarity index 99% rename from components/src/dynamo/common/tests/configuration/test_kv_router_args.py rename to dingo/common/tests/configuration/test_kv_router_args.py index aa25de62dd83..7c02a557afd3 100644 --- a/components/src/dynamo/common/tests/configuration/test_kv_router_args.py +++ b/dingo/common/tests/configuration/test_kv_router_args.py @@ -6,11 +6,11 @@ import pytest -from dynamo.common.configuration.groups.aic_perf_args import ( +from dingo.common.configuration.groups.aic_perf_args import ( AicPerfArgGroup, AicPerfConfigBase, ) -from dynamo.common.configuration.groups.kv_router_args import ( +from dingo.common.configuration.groups.kv_router_args import ( KvRouterArgGroup, KvRouterConfigBase, ) diff --git a/components/src/dynamo/common/tests/configuration/test_utils.py b/dingo/common/tests/configuration/test_utils.py similarity index 99% rename from components/src/dynamo/common/tests/configuration/test_utils.py rename to dingo/common/tests/configuration/test_utils.py index bd2e6020848d..001334fdb0ee 100644 --- a/components/src/dynamo/common/tests/configuration/test_utils.py +++ b/dingo/common/tests/configuration/test_utils.py @@ -6,7 +6,7 @@ import pytest -from dynamo.common.configuration.utils import ( +from dingo.common.configuration.utils import ( add_argument, add_negatable_bool_argument, env_or_default, diff --git a/components/src/dynamo/common/tests/http/conftest.py b/dingo/common/tests/http/conftest.py similarity index 92% rename from components/src/dynamo/common/tests/http/conftest.py rename to dingo/common/tests/http/conftest.py index 0bd56223c909..5a6efdbbdd8b 100644 --- a/components/src/dynamo/common/tests/http/conftest.py +++ b/dingo/common/tests/http/conftest.py @@ -12,7 +12,7 @@ import pytest_asyncio -from dynamo.common.http import close_http_client +from dingo.common.http import close_http_client @pytest_asyncio.fixture(autouse=True) diff --git a/components/src/dynamo/common/tests/http/test_aiohttp_client.py b/dingo/common/tests/http/test_aiohttp_client.py similarity index 97% rename from components/src/dynamo/common/tests/http/test_aiohttp_client.py rename to dingo/common/tests/http/test_aiohttp_client.py index fa467e2c2aed..a54c469a7bb9 100644 --- a/components/src/dynamo/common/tests/http/test_aiohttp_client.py +++ b/dingo/common/tests/http/test_aiohttp_client.py @@ -22,9 +22,9 @@ import pytest from yarl import URL -from dynamo.common import http as mm_http -from dynamo.common.http import AiohttpClient -from dynamo.common.http.url_validator import UrlValidationPolicy +from dingo.common import http as mm_http +from dingo.common.http import AiohttpClient +from dingo.common.http.url_validator import UrlValidationPolicy pytestmark = [ pytest.mark.asyncio, diff --git a/components/src/dynamo/common/tests/http/test_http_facade.py b/dingo/common/tests/http/test_http_facade.py similarity index 95% rename from components/src/dynamo/common/tests/http/test_http_facade.py rename to dingo/common/tests/http/test_http_facade.py index 49cc941f33fc..431b150c526b 100644 --- a/components/src/dynamo/common/tests/http/test_http_facade.py +++ b/dingo/common/tests/http/test_http_facade.py @@ -13,9 +13,9 @@ import pytest -from dynamo.common import http as mm_http -from dynamo.common.http import AiohttpClient, HttpxClient, from_env -from dynamo.common.http.url_validator import UrlValidationError, UrlValidationPolicy +from dingo.common import http as mm_http +from dingo.common.http import AiohttpClient, HttpxClient, from_env +from dingo.common.http.url_validator import UrlValidationError, UrlValidationPolicy pytestmark = [ pytest.mark.asyncio, @@ -60,7 +60,7 @@ async def test_invalid_backend_raises(monkeypatch) -> None: async def test_legacy_mm_http_env_var_still_honored(monkeypatch) -> None: """Legacy ``DYN_MM_HTTP_*`` env vars from the deleted - ``dynamo.common.multimodal.http_client`` module still take effect. + ``dingo.common.multimodal.http_client`` module still take effect. """ monkeypatch.delenv("DYN_HTTP_MAX_CONNECTIONS", raising=False) monkeypatch.setenv("DYN_MM_HTTP_MAX_CONNECTIONS", "77") diff --git a/components/src/dynamo/common/tests/http/test_httpx_client.py b/dingo/common/tests/http/test_httpx_client.py similarity index 96% rename from components/src/dynamo/common/tests/http/test_httpx_client.py rename to dingo/common/tests/http/test_httpx_client.py index b152e63a67d1..4568d8f78b57 100644 --- a/components/src/dynamo/common/tests/http/test_httpx_client.py +++ b/dingo/common/tests/http/test_httpx_client.py @@ -20,9 +20,9 @@ import httpx import pytest -from dynamo.common import http as mm_http -from dynamo.common.http import HttpxClient -from dynamo.common.http.url_validator import UrlValidationPolicy +from dingo.common import http as mm_http +from dingo.common.http import HttpxClient +from dingo.common.http.url_validator import UrlValidationPolicy pytestmark = [ pytest.mark.asyncio, diff --git a/components/src/dynamo/common/tests/http/test_url_validator.py b/dingo/common/tests/http/test_url_validator.py similarity index 96% rename from components/src/dynamo/common/tests/http/test_url_validator.py rename to dingo/common/tests/http/test_url_validator.py index d37651734ea9..a9e7395e89af 100644 --- a/components/src/dynamo/common/tests/http/test_url_validator.py +++ b/dingo/common/tests/http/test_url_validator.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for ``dynamo.common.http.url_validator``. +"""Unit tests for ``dingo.common.http.url_validator``. These cover scheme / IP / hostname / path / redirect logic in isolation of the media loaders, so they run quickly with no network and no vLLM imports. @@ -15,7 +15,7 @@ import pytest -from dynamo.common.http.url_validator import ( +from dingo.common.http.url_validator import ( UrlValidationError, UrlValidationPolicy, is_blocked_ip, @@ -183,7 +183,7 @@ def _impl(host: str, *_args: Any, **_kwargs: Any): async def test_validate_url_rejects_host_resolving_to_private_ip() -> None: with patch( - "dynamo.common.http.url_validator.socket.getaddrinfo", + "dingo.common.http.url_validator.socket.getaddrinfo", side_effect=_fake_getaddrinfo(["10.0.0.5"]), ): with pytest.raises(UrlValidationError, match="blocked IP"): @@ -193,7 +193,7 @@ async def test_validate_url_rejects_host_resolving_to_private_ip() -> None: async def test_validate_url_rejects_host_if_any_ip_is_private() -> None: # Even if the host resolves to a public IP too, any blocked IP is fatal. with patch( - "dynamo.common.http.url_validator.socket.getaddrinfo", + "dingo.common.http.url_validator.socket.getaddrinfo", side_effect=_fake_getaddrinfo(["8.8.8.8", "169.254.169.254"]), ): with pytest.raises(UrlValidationError, match="169.254.169.254"): @@ -202,7 +202,7 @@ async def test_validate_url_rejects_host_if_any_ip_is_private() -> None: async def test_validate_url_accepts_public_host() -> None: with patch( - "dynamo.common.http.url_validator.socket.getaddrinfo", + "dingo.common.http.url_validator.socket.getaddrinfo", side_effect=_fake_getaddrinfo(["93.184.216.34"]), ): await validate_url("https://example.com/x.png", STRICT_HTTPS) @@ -210,7 +210,7 @@ async def test_validate_url_accepts_public_host() -> None: async def test_validate_url_resolution_failure_raises() -> None: with patch( - "dynamo.common.http.url_validator.socket.getaddrinfo", + "dingo.common.http.url_validator.socket.getaddrinfo", side_effect=socket.gaierror("nodename nor servname provided"), ): with pytest.raises(UrlValidationError, match="Could not resolve"): @@ -219,7 +219,7 @@ async def test_validate_url_resolution_failure_raises() -> None: async def test_validate_url_skips_resolution_when_private_allowed() -> None: # In developer mode we short-circuit DNS to keep tests deterministic. - with patch("dynamo.common.http.url_validator.socket.getaddrinfo") as resolver: + with patch("dingo.common.http.url_validator.socket.getaddrinfo") as resolver: await validate_url("https://example.com/x.png", PERMISSIVE) resolver.assert_not_called() diff --git a/components/src/dynamo/common/tests/lora/test_manager.py b/dingo/common/tests/lora/test_manager.py similarity index 94% rename from components/src/dynamo/common/tests/lora/test_manager.py rename to dingo/common/tests/lora/test_manager.py index dee7b1f8a70c..4ba5afc132bc 100644 --- a/components/src/dynamo/common/tests/lora/test_manager.py +++ b/dingo/common/tests/lora/test_manager.py @@ -1,16 +1,16 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for dynamo.common.lora.manager.get_lora_manager singleton.""" +"""Unit tests for dingo.common.lora.manager.get_lora_manager singleton.""" import threading import time import pytest -from dynamo.common.lora import manager as manager_module -from dynamo.common.lora.manager import get_lora_manager -from dynamo.common.lora.once import OnceLock +from dingo.common.lora import manager as manager_module +from dingo.common.lora.manager import get_lora_manager +from dingo.common.lora.once import OnceLock pytestmark = [ pytest.mark.unit, diff --git a/components/src/dynamo/common/tests/lora/test_once.py b/dingo/common/tests/lora/test_once.py similarity index 97% rename from components/src/dynamo/common/tests/lora/test_once.py rename to dingo/common/tests/lora/test_once.py index 2ae74ee688c1..8a64fa427833 100644 --- a/components/src/dynamo/common/tests/lora/test_once.py +++ b/dingo/common/tests/lora/test_once.py @@ -1,14 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for dynamo.common.lora.once module.""" +"""Unit tests for dingo.common.lora.once module.""" import threading import time import pytest -from dynamo.common.lora.once import OnceLock +from dingo.common.lora.once import OnceLock pytestmark = [ pytest.mark.unit, diff --git a/components/src/dynamo/common/tests/memory/test_embedding_cache_metrics.py b/dingo/common/tests/memory/test_embedding_cache_metrics.py similarity index 98% rename from components/src/dynamo/common/tests/memory/test_embedding_cache_metrics.py rename to dingo/common/tests/memory/test_embedding_cache_metrics.py index 59ea79dba268..0049ae77e446 100644 --- a/components/src/dynamo/common/tests/memory/test_embedding_cache_metrics.py +++ b/dingo/common/tests/memory/test_embedding_cache_metrics.py @@ -8,11 +8,11 @@ import pytest import torch -from dynamo.common.memory.multimodal_embedding_cache_manager import ( +from dingo.common.memory.multimodal_embedding_cache_manager import ( CachedEmbedding, MultimodalEmbeddingCacheManager, ) -from dynamo.common.utils.prometheus import ( +from dingo.common.utils.prometheus import ( EmbeddingCacheMetrics, register_embedding_cache_metrics, ) diff --git a/components/src/dynamo/common/tests/memory/test_multimodal_embedding_cache_manager.py b/dingo/common/tests/memory/test_multimodal_embedding_cache_manager.py similarity index 99% rename from components/src/dynamo/common/tests/memory/test_multimodal_embedding_cache_manager.py rename to dingo/common/tests/memory/test_multimodal_embedding_cache_manager.py index b30f1abb3e76..4957ddb475db 100644 --- a/components/src/dynamo/common/tests/memory/test_multimodal_embedding_cache_manager.py +++ b/dingo/common/tests/memory/test_multimodal_embedding_cache_manager.py @@ -6,7 +6,7 @@ import pytest import torch -from dynamo.common.memory.multimodal_embedding_cache_manager import ( +from dingo.common.memory.multimodal_embedding_cache_manager import ( CachedEmbedding, CacheMutation, MultimodalEmbeddingCacheManager, diff --git a/components/src/dynamo/common/tests/multimodal/conftest.py b/dingo/common/tests/multimodal/conftest.py similarity index 92% rename from components/src/dynamo/common/tests/multimodal/conftest.py rename to dingo/common/tests/multimodal/conftest.py index a14e77edbfb7..3f995b256e8e 100644 --- a/components/src/dynamo/common/tests/multimodal/conftest.py +++ b/dingo/common/tests/multimodal/conftest.py @@ -12,7 +12,7 @@ import pytest_asyncio -from dynamo.common.http import close_http_client +from dingo.common.http import close_http_client @pytest_asyncio.fixture(autouse=True) diff --git a/components/src/dynamo/common/tests/multimodal/test_async_encoder_cache.py b/dingo/common/tests/multimodal/test_async_encoder_cache.py similarity index 98% rename from components/src/dynamo/common/tests/multimodal/test_async_encoder_cache.py rename to dingo/common/tests/multimodal/test_async_encoder_cache.py index a2312763abc5..bd07eb275952 100644 --- a/components/src/dynamo/common/tests/multimodal/test_async_encoder_cache.py +++ b/dingo/common/tests/multimodal/test_async_encoder_cache.py @@ -8,11 +8,11 @@ import pytest import torch -from dynamo.common.memory.multimodal_embedding_cache_manager import ( +from dingo.common.memory.multimodal_embedding_cache_manager import ( CachedEmbedding, MultimodalEmbeddingCacheManager, ) -from dynamo.common.multimodal.async_encoder_cache import AsyncEncoderCache +from dingo.common.multimodal.async_encoder_cache import AsyncEncoderCache # Total runtime ~0.75s — no need for parallel marker. pytestmark = [ diff --git a/components/src/dynamo/common/tests/multimodal/test_audio_loader.py b/dingo/common/tests/multimodal/test_audio_loader.py similarity index 95% rename from components/src/dynamo/common/tests/multimodal/test_audio_loader.py rename to dingo/common/tests/multimodal/test_audio_loader.py index 5a6eb4a188cc..bca9bef75992 100644 --- a/components/src/dynamo/common/tests/multimodal/test_audio_loader.py +++ b/dingo/common/tests/multimodal/test_audio_loader.py @@ -6,9 +6,9 @@ import numpy as np import pytest -import dynamo.common.multimodal.audio_loader as audio_loader_module -from dynamo.common.http.url_validator import UrlValidationPolicy -from dynamo.common.multimodal.audio_loader import AudioLoader +import dingo.common.multimodal.audio_loader as audio_loader_module +from dingo.common.http.url_validator import UrlValidationPolicy +from dingo.common.multimodal.audio_loader import AudioLoader pytestmark = [ pytest.mark.unit, diff --git a/components/src/dynamo/common/tests/multimodal/test_embedding_transfer.py b/dingo/common/tests/multimodal/test_embedding_transfer.py similarity index 99% rename from components/src/dynamo/common/tests/multimodal/test_embedding_transfer.py rename to dingo/common/tests/multimodal/test_embedding_transfer.py index bf1d1ff3e89a..6f2fed98ea55 100644 --- a/components/src/dynamo/common/tests/multimodal/test_embedding_transfer.py +++ b/dingo/common/tests/multimodal/test_embedding_transfer.py @@ -11,7 +11,7 @@ import pytest import torch -from dynamo.common.multimodal.embedding_transfer import ( +from dingo.common.multimodal.embedding_transfer import ( LocalEmbeddingReceiver, LocalEmbeddingSender, NixlReadEmbeddingReceiver, diff --git a/components/src/dynamo/common/tests/multimodal/test_image_loader.py b/dingo/common/tests/multimodal/test_image_loader.py similarity index 97% rename from components/src/dynamo/common/tests/multimodal/test_image_loader.py rename to dingo/common/tests/multimodal/test_image_loader.py index 94d7b8045445..7e99df93f491 100644 --- a/components/src/dynamo/common/tests/multimodal/test_image_loader.py +++ b/dingo/common/tests/multimodal/test_image_loader.py @@ -23,9 +23,9 @@ import pytest from PIL import Image -from dynamo.common.http import HttpStatusError, HttpTimeoutError -from dynamo.common.http.url_validator import UrlValidationPolicy -from dynamo.common.multimodal.image_loader import URL_VARIANT_KEY, ImageLoader +from dingo.common.http import HttpStatusError, HttpTimeoutError +from dingo.common.http.url_validator import UrlValidationPolicy +from dingo.common.multimodal.image_loader import URL_VARIANT_KEY, ImageLoader pytestmark = [ pytest.mark.asyncio, @@ -34,7 +34,7 @@ pytest.mark.pre_merge, ] -_FETCH_BYTES_PATH = "dynamo.common.multimodal.image_loader.fetch_bytes" +_FETCH_BYTES_PATH = "dingo.common.multimodal.image_loader.fetch_bytes" def _make_png_bytes() -> bytes: diff --git a/components/src/dynamo/common/tests/multimodal/test_media_connector.py b/dingo/common/tests/multimodal/test_media_connector.py similarity index 96% rename from components/src/dynamo/common/tests/multimodal/test_media_connector.py rename to dingo/common/tests/multimodal/test_media_connector.py index a1b3c46b72d5..7d58acf8657f 100644 --- a/components/src/dynamo/common/tests/multimodal/test_media_connector.py +++ b/dingo/common/tests/multimodal/test_media_connector.py @@ -6,7 +6,7 @@ import pytest from PIL import Image -from dynamo.common.multimodal.image_loader import ImageLoader +from dingo.common.multimodal.image_loader import ImageLoader pytestmark = [ pytest.mark.unit, diff --git a/components/src/dynamo/common/tests/multimodal/test_mm_kwargs_transfer.py b/dingo/common/tests/multimodal/test_mm_kwargs_transfer.py similarity index 98% rename from components/src/dynamo/common/tests/multimodal/test_mm_kwargs_transfer.py rename to dingo/common/tests/multimodal/test_mm_kwargs_transfer.py index 4285942866fa..8c67339f1a9c 100644 --- a/components/src/dynamo/common/tests/multimodal/test_mm_kwargs_transfer.py +++ b/dingo/common/tests/multimodal/test_mm_kwargs_transfer.py @@ -8,7 +8,7 @@ import pytest -from dynamo.common.multimodal.mm_kwargs_transfer import ( +from dingo.common.multimodal.mm_kwargs_transfer import ( MmKwargsNixlSender, MmKwargsShmReceiver, MmKwargsShmSender, @@ -312,7 +312,7 @@ class TestMmKwargsNixlReceiverDescriptorValidation: def test_acquire_descriptor_raises_on_none_data_ref(self): """Pre-allocated descriptor with None _data_ref raises RuntimeError.""" - from dynamo.common.multimodal.mm_kwargs_transfer import MmKwargsNixlReceiver + from dingo.common.multimodal.mm_kwargs_transfer import MmKwargsNixlReceiver # Create a receiver with a mocked pool receiver = MmKwargsNixlReceiver.__new__(MmKwargsNixlReceiver) @@ -346,7 +346,7 @@ async def test_multi_image_nixl_receive_preserves_order(self): import torch - from dynamo.common.multimodal.mm_kwargs_transfer import MmKwargsNixlReceiver + from dingo.common.multimodal.mm_kwargs_transfer import MmKwargsNixlReceiver # Prepare 3 pickled items with distinct content items = [ diff --git a/components/src/dynamo/common/tests/multimodal/test_routing_utils.py b/dingo/common/tests/multimodal/test_routing_utils.py similarity index 98% rename from components/src/dynamo/common/tests/multimodal/test_routing_utils.py rename to dingo/common/tests/multimodal/test_routing_utils.py index 4e69ae6ad013..b424afb19632 100644 --- a/components/src/dynamo/common/tests/multimodal/test_routing_utils.py +++ b/dingo/common/tests/multimodal/test_routing_utils.py @@ -7,7 +7,7 @@ import pytest -from dynamo.common.multimodal.routing_utils import ( +from dingo.common.multimodal.routing_utils import ( build_mm_routing_info_from_features, pad_value_for_mm_hash, ) diff --git a/components/src/dynamo/common/tests/multimodal/test_video_loader.py b/dingo/common/tests/multimodal/test_video_loader.py similarity index 94% rename from components/src/dynamo/common/tests/multimodal/test_video_loader.py rename to dingo/common/tests/multimodal/test_video_loader.py index 7158cb80a5bc..fffda6dee8e3 100644 --- a/components/src/dynamo/common/tests/multimodal/test_video_loader.py +++ b/dingo/common/tests/multimodal/test_video_loader.py @@ -6,9 +6,9 @@ import numpy as np import pytest -import dynamo.common.multimodal.video_loader as video_loader_module -from dynamo.common.http.url_validator import UrlValidationPolicy -from dynamo.common.multimodal.video_loader import VideoLoader +import dingo.common.multimodal.video_loader as video_loader_module +from dingo.common.http.url_validator import UrlValidationPolicy +from dingo.common.multimodal.video_loader import VideoLoader pytestmark = [ pytest.mark.unit, diff --git a/components/src/dynamo/common/tests/test_audio_protocol.py b/dingo/common/tests/test_audio_protocol.py similarity index 97% rename from components/src/dynamo/common/tests/test_audio_protocol.py rename to dingo/common/tests/test_audio_protocol.py index 17c5813c5351..3fff292e162f 100644 --- a/components/src/dynamo/common/tests/test_audio_protocol.py +++ b/dingo/common/tests/test_audio_protocol.py @@ -1,12 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for dynamo.common.protocols.audio_protocol module.""" +"""Unit tests for dingo.common.protocols.audio_protocol module.""" import pytest from pydantic import ValidationError -from dynamo.common.protocols.audio_protocol import ( +from dingo.common.protocols.audio_protocol import ( AudioData, NvAudioSpeechResponse, NvCreateAudioSpeechRequest, diff --git a/components/src/dynamo/common/tests/test_rl_admin.py b/dingo/common/tests/test_rl_admin.py similarity index 99% rename from components/src/dynamo/common/tests/test_rl_admin.py rename to dingo/common/tests/test_rl_admin.py index 176005cd2de5..cfcff8192814 100644 --- a/components/src/dynamo/common/tests/test_rl_admin.py +++ b/dingo/common/tests/test_rl_admin.py @@ -7,7 +7,7 @@ import pytest -from dynamo.common.rl import ( +from dingo.common.rl import ( RLAdminValidationError, RLRouteRegistry, first_endpoint_response, diff --git a/components/src/dynamo/common/tests/test_snapshot_lifecycle.py b/dingo/common/tests/test_snapshot_lifecycle.py similarity index 96% rename from components/src/dynamo/common/tests/test_snapshot_lifecycle.py rename to dingo/common/tests/test_snapshot_lifecycle.py index 5da5b5c74b87..b98076e2a917 100644 --- a/components/src/dynamo/common/tests/test_snapshot_lifecycle.py +++ b/dingo/common/tests/test_snapshot_lifecycle.py @@ -6,12 +6,12 @@ import pytest -from dynamo.common.snapshot.constants import ( +from dingo.common.snapshot.constants import ( READY_FOR_SNAPSHOT_FILE, RESTORE_COMPLETE_FILE, SNAPSHOT_CONTROL_DIR_ENV, ) -from dynamo.common.snapshot.lifecycle import SnapshotConfig +from dingo.common.snapshot.lifecycle import SnapshotConfig pytestmark = [pytest.mark.unit, pytest.mark.gpu_0, pytest.mark.pre_merge] diff --git a/components/src/dynamo/common/tests/test_snapshot_restore_context.py b/dingo/common/tests/test_snapshot_restore_context.py similarity index 98% rename from components/src/dynamo/common/tests/test_snapshot_restore_context.py rename to dingo/common/tests/test_snapshot_restore_context.py index c4bfba0d76d4..a39a04dd1d70 100644 --- a/components/src/dynamo/common/tests/test_snapshot_restore_context.py +++ b/dingo/common/tests/test_snapshot_restore_context.py @@ -7,7 +7,7 @@ import pytest -from dynamo.common.snapshot.constants import ( +from dingo.common.snapshot.constants import ( KUBERNETES_OPTIONAL_ENV_NAMES, KUBERNETES_REQUIRED_ENV_NAMES, RESTORE_RUNTIME_ENV_NAMES, @@ -15,7 +15,7 @@ SNAPSHOT_RESTORE_CONTEXT_FILE, SNAPSHOT_RESTORE_STANDBY_ENV, ) -from dynamo.common.snapshot.restore_context import ( +from dingo.common.snapshot.restore_context import ( apply_snapshot_restore_env, refresh_snapshot_restore_config, ) diff --git a/components/src/dynamo/common/tests/test_storage.py b/dingo/common/tests/test_storage.py similarity index 94% rename from components/src/dynamo/common/tests/test_storage.py rename to dingo/common/tests/test_storage.py index 3b05bb3f2739..0c19bdb50cac 100644 --- a/components/src/dynamo/common/tests/test_storage.py +++ b/dingo/common/tests/test_storage.py @@ -1,13 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for dynamo.common.storage module.""" +"""Unit tests for dingo.common.storage module.""" from unittest.mock import MagicMock, patch import pytest -from dynamo.common.storage import get_fs, get_media_url, upload_to_fs +from dingo.common.storage import get_fs, get_media_url, upload_to_fs pytestmark = [ pytest.mark.unit, @@ -46,8 +46,8 @@ def test_no_protocol_defaults_to_file(self, tmp_path): def test_s3_url_protocol(self): """Test s3:// URL extracts correct protocol and bucket path.""" - with patch("dynamo.common.storage.fsspec.filesystem") as mock_fsspec, patch( - "dynamo.common.storage.DirFileSystem" + with patch("dingo.common.storage.fsspec.filesystem") as mock_fsspec, patch( + "dingo.common.storage.DirFileSystem" ) as mock_dirfs: mock_inner_fs = MagicMock(protocol="s3") mock_fsspec.return_value = mock_inner_fs @@ -59,8 +59,8 @@ def test_s3_url_protocol(self): def test_gs_url_protocol(self): """Test gs:// URL extracts correct protocol and path.""" - with patch("dynamo.common.storage.fsspec.filesystem") as mock_fsspec, patch( - "dynamo.common.storage.DirFileSystem" + with patch("dingo.common.storage.fsspec.filesystem") as mock_fsspec, patch( + "dingo.common.storage.DirFileSystem" ) as mock_dirfs: mock_inner_fs = MagicMock(protocol="gs") mock_fsspec.return_value = mock_inner_fs diff --git a/components/src/dynamo/common/tests/test_video_protocol.py b/dingo/common/tests/test_video_protocol.py similarity index 97% rename from components/src/dynamo/common/tests/test_video_protocol.py rename to dingo/common/tests/test_video_protocol.py index 8d10b99658b2..3aa41c07ad79 100644 --- a/components/src/dynamo/common/tests/test_video_protocol.py +++ b/dingo/common/tests/test_video_protocol.py @@ -1,12 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for dynamo.common.protocols.video_protocol module.""" +"""Unit tests for dingo.common.protocols.video_protocol module.""" import pytest from pydantic import ValidationError -from dynamo.common.protocols.video_protocol import ( +from dingo.common.protocols.video_protocol import ( NvCreateVideoRequest, NvVideosResponse, VideoData, diff --git a/components/src/dynamo/common/tests/test_video_utils.py b/dingo/common/tests/test_video_utils.py similarity index 81% rename from components/src/dynamo/common/tests/test_video_utils.py rename to dingo/common/tests/test_video_utils.py index fddb373412af..256a00ae8e02 100644 --- a/components/src/dynamo/common/tests/test_video_utils.py +++ b/dingo/common/tests/test_video_utils.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for dynamo.common.utils.video_utils module.""" +"""Unit tests for dingo.common.utils.video_utils module.""" from unittest.mock import MagicMock, patch @@ -42,10 +42,10 @@ def _mock_iio_v2(self): return iio, writer def test_mp4_selects_h264_nvenc_codec(self): - from dynamo.common.utils.video_utils import encode_to_video_bytes + from dingo.common.utils.video_utils import encode_to_video_bytes iio = self._mock_iio_v3() - with patch("dynamo.common.utils.video_utils.io") as mock_io, patch( + with patch("dingo.common.utils.video_utils.io") as mock_io, patch( "imageio.v3", iio, create=True ), patch.dict("sys.modules", {"imageio.v3": iio}): buf = MagicMock() @@ -60,10 +60,10 @@ def test_mp4_selects_h264_nvenc_codec(self): assert kwargs.get("fps") == 8 def test_webm_selects_libvpx_vp9_codec(self): - from dynamo.common.utils.video_utils import encode_to_video_bytes + from dingo.common.utils.video_utils import encode_to_video_bytes iio = self._mock_iio_v3() - with patch("dynamo.common.utils.video_utils.io") as mock_io, patch( + with patch("dingo.common.utils.video_utils.io") as mock_io, patch( "imageio.v3", iio, create=True ), patch.dict("sys.modules", {"imageio.v3": iio}): buf = MagicMock() @@ -77,10 +77,10 @@ def test_webm_selects_libvpx_vp9_codec(self): assert kwargs.get("codec") == "libvpx-vp9" def test_mp4_passes_extension_to_imwrite(self): - from dynamo.common.utils.video_utils import encode_to_video_bytes + from dingo.common.utils.video_utils import encode_to_video_bytes iio = self._mock_iio_v3() - with patch("dynamo.common.utils.video_utils.io") as mock_io, patch( + with patch("dingo.common.utils.video_utils.io") as mock_io, patch( "imageio.v3", iio, create=True ), patch.dict("sys.modules", {"imageio.v3": iio}): buf = MagicMock() @@ -93,10 +93,10 @@ def test_mp4_passes_extension_to_imwrite(self): assert kwargs.get("extension") == ".mp4" def test_webm_passes_extension_to_imwrite(self): - from dynamo.common.utils.video_utils import encode_to_video_bytes + from dingo.common.utils.video_utils import encode_to_video_bytes iio = self._mock_iio_v3() - with patch("dynamo.common.utils.video_utils.io") as mock_io, patch( + with patch("dingo.common.utils.video_utils.io") as mock_io, patch( "imageio.v3", iio, create=True ), patch.dict("sys.modules", {"imageio.v3": iio}): buf = MagicMock() @@ -109,10 +109,10 @@ def test_webm_passes_extension_to_imwrite(self): assert kwargs.get("extension") == ".webm" def test_unsupported_format_raises_value_error(self): - from dynamo.common.utils.video_utils import encode_to_video_bytes + from dingo.common.utils.video_utils import encode_to_video_bytes iio = self._mock_iio_v3() - with patch("dynamo.common.utils.video_utils.io") as mock_io, patch( + with patch("dingo.common.utils.video_utils.io") as mock_io, patch( "imageio.v3", iio, create=True ), patch.dict("sys.modules", {"imageio.v3": iio}): mock_io.BytesIO.return_value = MagicMock() @@ -122,11 +122,11 @@ def test_unsupported_format_raises_value_error(self): encode_to_video_bytes(make_frames(), output_format="avi") def test_returns_bytes_from_buffer(self): - from dynamo.common.utils.video_utils import encode_to_video_bytes + from dingo.common.utils.video_utils import encode_to_video_bytes expected = b"\x00\x01\x02" iio = self._mock_iio_v3() - with patch("dynamo.common.utils.video_utils.io") as mock_io, patch( + with patch("dingo.common.utils.video_utils.io") as mock_io, patch( "imageio.v3", iio, create=True ), patch.dict("sys.modules", {"imageio.v3": iio}): buf = MagicMock() @@ -139,10 +139,10 @@ def test_returns_bytes_from_buffer(self): def test_v2_api_fallback_writes_all_frames(self): """When imageio.v3.imwrite is absent, falls back to get_writer loop.""" - from dynamo.common.utils.video_utils import encode_to_video_bytes + from dingo.common.utils.video_utils import encode_to_video_bytes iio_v2, writer = self._mock_iio_v2() - with patch("dynamo.common.utils.video_utils.io") as mock_io, patch( + with patch("dingo.common.utils.video_utils.io") as mock_io, patch( "imageio.v3", iio_v2, create=True ), patch.dict("sys.modules", {"imageio.v3": iio_v2}): buf = MagicMock() diff --git a/components/src/dynamo/common/utils/__init__.py b/dingo/common/utils/__init__.py similarity index 96% rename from components/src/dynamo/common/utils/__init__.py rename to dingo/common/utils/__init__.py index a9dbaecf6486..37fa82c43f14 100644 --- a/components/src/dynamo/common/utils/__init__.py +++ b/dingo/common/utils/__init__.py @@ -14,7 +14,7 @@ - prometheus: Prometheus metrics collection and logging utilities """ -from dynamo.common.utils import ( +from dingo.common.utils import ( endpoint_types, engine_response, namespace, diff --git a/components/src/dynamo/common/utils/endpoint_types.py b/dingo/common/utils/endpoint_types.py similarity index 100% rename from components/src/dynamo/common/utils/endpoint_types.py rename to dingo/common/utils/endpoint_types.py diff --git a/components/src/dynamo/common/utils/engine_response.py b/dingo/common/utils/engine_response.py similarity index 100% rename from components/src/dynamo/common/utils/engine_response.py rename to dingo/common/utils/engine_response.py diff --git a/components/src/dynamo/common/utils/env.py b/dingo/common/utils/env.py similarity index 100% rename from components/src/dynamo/common/utils/env.py rename to dingo/common/utils/env.py diff --git a/components/src/dynamo/common/utils/graceful_shutdown.py b/dingo/common/utils/graceful_shutdown.py similarity index 100% rename from components/src/dynamo/common/utils/graceful_shutdown.py rename to dingo/common/utils/graceful_shutdown.py diff --git a/components/src/dynamo/common/utils/image_utils.py b/dingo/common/utils/image_utils.py similarity index 100% rename from components/src/dynamo/common/utils/image_utils.py rename to dingo/common/utils/image_utils.py diff --git a/components/src/dynamo/common/utils/input_params.py b/dingo/common/utils/input_params.py similarity index 100% rename from components/src/dynamo/common/utils/input_params.py rename to dingo/common/utils/input_params.py diff --git a/components/src/dynamo/common/utils/label_injecting_collector.py b/dingo/common/utils/label_injecting_collector.py similarity index 100% rename from components/src/dynamo/common/utils/label_injecting_collector.py rename to dingo/common/utils/label_injecting_collector.py diff --git a/components/src/dynamo/common/utils/media_nixl.py b/dingo/common/utils/media_nixl.py similarity index 100% rename from components/src/dynamo/common/utils/media_nixl.py rename to dingo/common/utils/media_nixl.py diff --git a/components/src/dynamo/common/utils/namespace.py b/dingo/common/utils/namespace.py similarity index 100% rename from components/src/dynamo/common/utils/namespace.py rename to dingo/common/utils/namespace.py diff --git a/components/src/dynamo/common/utils/nvtx_utils.py b/dingo/common/utils/nvtx_utils.py similarity index 98% rename from components/src/dynamo/common/utils/nvtx_utils.py rename to dingo/common/utils/nvtx_utils.py index 37cc7c3b450c..eff8f6ca230b 100644 --- a/components/src/dynamo/common/utils/nvtx_utils.py +++ b/dingo/common/utils/nvtx_utils.py @@ -7,7 +7,7 @@ Usage — same syntax as the bare nvtx module: - from dynamo.common.utils import nvtx_utils as _nvtx + from dingo.common.utils import nvtx_utils as _nvtx # Manual range (needed when the range spans async yields or has conditional end) rng = _nvtx.start_range("my:range", color="blue") diff --git a/components/src/dynamo/common/utils/output_modalities.py b/dingo/common/utils/output_modalities.py similarity index 93% rename from components/src/dynamo/common/utils/output_modalities.py rename to dingo/common/utils/output_modalities.py index 3c9dcbeb3173..86e23eb823eb 100644 --- a/components/src/dynamo/common/utils/output_modalities.py +++ b/dingo/common/utils/output_modalities.py @@ -6,9 +6,9 @@ from pydantic import BaseModel -from dynamo.common.protocols.audio_protocol import NvCreateAudioSpeechRequest -from dynamo.common.protocols.image_protocol import NvCreateImageRequest -from dynamo.common.protocols.video_protocol import NvCreateVideoRequest +from dingo.common.protocols.audio_protocol import NvCreateAudioSpeechRequest +from dingo.common.protocols.image_protocol import NvCreateImageRequest +from dingo.common.protocols.video_protocol import NvCreateVideoRequest from dynamo.llm import ModelType diff --git a/components/src/dynamo/common/utils/paths.py b/dingo/common/utils/paths.py similarity index 100% rename from components/src/dynamo/common/utils/paths.py rename to dingo/common/utils/paths.py diff --git a/components/src/dynamo/common/utils/prometheus.py b/dingo/common/utils/prometheus.py similarity index 99% rename from components/src/dynamo/common/utils/prometheus.py rename to dingo/common/utils/prometheus.py index 875e514bf7bb..3209e7af7315 100644 --- a/components/src/dynamo/common/utils/prometheus.py +++ b/dingo/common/utils/prometheus.py @@ -28,7 +28,7 @@ if TYPE_CHECKING: from prometheus_client import CollectorRegistry - from dynamo.common.memory import MultimodalEmbeddingCacheManager + from dingo.common.memory import MultimodalEmbeddingCacheManager # Auto-label injection: always injects dynamo_namespace, dynamo_component, dynamo_endpoint labels # into engine metrics based on the endpoint hierarchy. @@ -272,7 +272,7 @@ def get_prometheus_expfmt( # called by SGLang's engine initialization. Importing at the top of this file would # trigger prometheus_client initialization too early (before PROMETHEUS_MULTIPROC_DIR # is set), breaking multiprocess metrics collection. - from dynamo.common.utils.label_injecting_collector import ( + from dingo.common.utils.label_injecting_collector import ( LabelInjectingCollector, ) @@ -341,7 +341,7 @@ def gather_with_labels( source metrics win over auto-labels of the same name. Lives here (as a leaf utility next to :func:`get_prometheus_expfmt`) - so ``dynamo.common.backend.metrics`` can import it without dragging + so ``dingo.common.backend.metrics`` can import it without dragging in the broader backend module. """ return get_prometheus_expfmt( diff --git a/components/src/dynamo/common/utils/runtime.py b/dingo/common/utils/runtime.py similarity index 100% rename from components/src/dynamo/common/utils/runtime.py rename to dingo/common/utils/runtime.py diff --git a/components/src/dynamo/common/utils/structural_tag.py b/dingo/common/utils/structural_tag.py similarity index 100% rename from components/src/dynamo/common/utils/structural_tag.py rename to dingo/common/utils/structural_tag.py diff --git a/components/src/dynamo/common/utils/tests/test_env.py b/dingo/common/utils/tests/test_env.py similarity index 92% rename from components/src/dynamo/common/utils/tests/test_env.py rename to dingo/common/utils/tests/test_env.py index de8dbc973569..22b57e48830d 100644 --- a/components/src/dynamo/common/utils/tests/test_env.py +++ b/dingo/common/utils/tests/test_env.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for dynamo.common.utils.env.""" +"""Unit tests for dingo.common.utils.env.""" import pytest -from dynamo.common.utils.env import env_bool +from dingo.common.utils.env import env_bool pytestmark = [ pytest.mark.unit, diff --git a/components/src/dynamo/common/utils/tests/test_graceful_shutdown.py b/dingo/common/utils/tests/test_graceful_shutdown.py similarity index 98% rename from components/src/dynamo/common/utils/tests/test_graceful_shutdown.py rename to dingo/common/utils/tests/test_graceful_shutdown.py index 2fcc8ed8a6a1..88ae460f05d2 100644 --- a/components/src/dynamo/common/utils/tests/test_graceful_shutdown.py +++ b/dingo/common/utils/tests/test_graceful_shutdown.py @@ -27,7 +27,7 @@ # Module loading: import graceful_shutdown without triggering the full dynamo # package (which requires dynamo.llm, CUDA, etc.) # -# We cannot do `from dynamo.common.utils import graceful_shutdown` because the +# We cannot do `from dingo.common.utils import graceful_shutdown` because the # dynamo package __init__ transitively imports dynamo._core, which is a native # extension (PyO3) requiring CUDA/NIXL libraries that are not available in # unit test environments. Instead, we stub dynamo._core and load the module @@ -46,7 +46,7 @@ def _load_graceful_shutdown(): spec = importlib.util.spec_from_file_location( - "dynamo.common.utils.graceful_shutdown", + "dingo.common.utils.graceful_shutdown", _GRACEFUL_SHUTDOWN_PATH, ) mod = importlib.util.module_from_spec(spec) diff --git a/components/src/dynamo/common/utils/tests/test_inject_reasoning_content.py b/dingo/common/utils/tests/test_inject_reasoning_content.py similarity index 97% rename from components/src/dynamo/common/utils/tests/test_inject_reasoning_content.py rename to dingo/common/utils/tests/test_inject_reasoning_content.py index 4703bca794bd..12a3bf12a484 100644 --- a/components/src/dynamo/common/utils/tests/test_inject_reasoning_content.py +++ b/dingo/common/utils/tests/test_inject_reasoning_content.py @@ -12,7 +12,7 @@ import pytest -from dynamo.common.utils.input_params import _inject_reasoning_content +from dingo.common.utils.input_params import _inject_reasoning_content # Total runtime ~0.04s — no need for parallel marker. pytestmark = [ @@ -191,7 +191,7 @@ def test_injects_when_template_ignores_reasoning(self): ) tokenizer.apply_chat_template = MagicMock(return_value="rendered") - from dynamo.common.utils.input_params import InputParamManager + from dingo.common.utils.input_params import InputParamManager mgr = InputParamManager(tokenizer) request = { @@ -223,7 +223,7 @@ def test_skips_injection_when_template_handles_reasoning(self): ) tokenizer.apply_chat_template = MagicMock(return_value="rendered") - from dynamo.common.utils.input_params import InputParamManager + from dingo.common.utils.input_params import InputParamManager mgr = InputParamManager(tokenizer) request = { diff --git a/components/src/dynamo/common/utils/tests/test_label_injecting_collector.py b/dingo/common/utils/tests/test_label_injecting_collector.py similarity index 94% rename from components/src/dynamo/common/utils/tests/test_label_injecting_collector.py rename to dingo/common/utils/tests/test_label_injecting_collector.py index a909b0975253..df0469eb4f63 100644 --- a/components/src/dynamo/common/utils/tests/test_label_injecting_collector.py +++ b/dingo/common/utils/tests/test_label_injecting_collector.py @@ -24,7 +24,7 @@ class TestLabelInjectingCollector: def test_counter_label_injection(self): """Test injecting labels into Counter metrics""" - from dynamo.common.utils.label_injecting_collector import ( + from dingo.common.utils.label_injecting_collector import ( LabelInjectingCollector, ) @@ -65,7 +65,7 @@ def test_counter_label_injection(self): def test_gauge_label_injection(self): """Test injecting labels into Gauge metrics""" - from dynamo.common.utils.label_injecting_collector import ( + from dingo.common.utils.label_injecting_collector import ( LabelInjectingCollector, ) @@ -100,7 +100,7 @@ def test_gauge_label_injection(self): def test_histogram_preserves_le_label(self): """Test that histogram 'le' label is preserved and not overwritten""" - from dynamo.common.utils.label_injecting_collector import ( + from dingo.common.utils.label_injecting_collector import ( LabelInjectingCollector, ) @@ -153,7 +153,7 @@ def test_histogram_preserves_le_label(self): def test_summary_preserves_quantile_label(self): """Test that summary 'quantile' label is preserved""" - from dynamo.common.utils.label_injecting_collector import ( + from dingo.common.utils.label_injecting_collector import ( LabelInjectingCollector, ) @@ -200,7 +200,7 @@ def test_summary_preserves_quantile_label(self): def test_multiple_labels_injection(self): """Test injecting multiple labels at once""" - from dynamo.common.utils.label_injecting_collector import ( + from dingo.common.utils.label_injecting_collector import ( LabelInjectingCollector, ) @@ -235,7 +235,7 @@ def test_multiple_labels_injection(self): def test_merge_with_existing_labels(self): """Test injecting labels into metrics that already have labels""" - from dynamo.common.utils.label_injecting_collector import ( + from dingo.common.utils.label_injecting_collector import ( LabelInjectingCollector, ) @@ -288,7 +288,7 @@ def test_merge_with_existing_labels(self): def test_existing_label_not_overwritten(self): """Test that existing labels take precedence over injected labels""" - from dynamo.common.utils.label_injecting_collector import ( + from dingo.common.utils.label_injecting_collector import ( LabelInjectingCollector, ) @@ -319,7 +319,7 @@ def test_existing_label_not_overwritten(self): def test_empty_labels_raises_error(self): """Test that empty labels dict raises ValueError""" - from dynamo.common.utils.label_injecting_collector import ( + from dingo.common.utils.label_injecting_collector import ( LabelInjectingCollector, ) @@ -331,7 +331,7 @@ def test_empty_labels_raises_error(self): def test_reserved_label_le_raises_error(self): """Test that trying to inject reserved label 'le' raises ValueError""" - from dynamo.common.utils.label_injecting_collector import ( + from dingo.common.utils.label_injecting_collector import ( LabelInjectingCollector, ) @@ -346,7 +346,7 @@ def test_reserved_label_le_raises_error(self): def test_reserved_label_quantile_raises_error(self): """Test that trying to inject reserved label 'quantile' raises ValueError""" - from dynamo.common.utils.label_injecting_collector import ( + from dingo.common.utils.label_injecting_collector import ( LabelInjectingCollector, ) @@ -361,7 +361,7 @@ def test_reserved_label_quantile_raises_error(self): def test_multiple_metrics_all_get_labels(self): """Test that all metrics in registry get injected labels""" - from dynamo.common.utils.label_injecting_collector import ( + from dingo.common.utils.label_injecting_collector import ( LabelInjectingCollector, ) @@ -396,7 +396,7 @@ def test_multiple_metrics_all_get_labels(self): def test_timestamp_preservation(self): """Test that timestamps are preserved""" - from dynamo.common.utils.label_injecting_collector import ( + from dingo.common.utils.label_injecting_collector import ( LabelInjectingCollector, ) diff --git a/components/src/dynamo/common/utils/tests/test_topology.py b/dingo/common/utils/tests/test_topology.py similarity index 100% rename from components/src/dynamo/common/utils/tests/test_topology.py rename to dingo/common/utils/tests/test_topology.py diff --git a/components/src/dynamo/common/utils/time_section.py b/dingo/common/utils/time_section.py similarity index 100% rename from components/src/dynamo/common/utils/time_section.py rename to dingo/common/utils/time_section.py diff --git a/components/src/dynamo/common/utils/topology.py b/dingo/common/utils/topology.py similarity index 100% rename from components/src/dynamo/common/utils/topology.py rename to dingo/common/utils/topology.py diff --git a/components/src/dynamo/common/utils/video_utils.py b/dingo/common/utils/video_utils.py similarity index 100% rename from components/src/dynamo/common/utils/video_utils.py rename to dingo/common/utils/video_utils.py diff --git a/dingo/frontend/CLAUDE.md b/dingo/frontend/CLAUDE.md deleted file mode 100644 index 119b2070eefe..000000000000 --- a/dingo/frontend/CLAUDE.md +++ /dev/null @@ -1,16 +0,0 @@ -# Frontend Configuration Boundary - -`dingo.frontend` parses CLI arguments and environment defaults into -`FrontendConfig`. After that point, treat `FrontendConfig` as the source of -truth for frontend-owned settings. - -- Do not write parsed `FrontendConfig` values back to `os.environ` for Rust to - re-read. That creates a cyclic Python -> env -> Rust contract where CLI - overrides can diverge from Rust behavior. -- Pass frontend-owned values through explicit PyO3 binding parameters or - config structs. -- Env fallbacks may remain for standalone/direct Rust entrypoints, but an - explicit Python binding value must win once the frontend has parsed config. -- If a setting must cross Python serialization, PyO3, RPC, or process - boundaries, add an explicit transport path at that boundary rather than - relying on shared process environment state. diff --git a/dingo/frontend/frontend_args.py b/dingo/frontend/frontend_args.py index 1474cfc95306..57b1068d0285 100644 --- a/dingo/frontend/frontend_args.py +++ b/dingo/frontend/frontend_args.py @@ -6,21 +6,21 @@ import pathlib from typing import Any, Dict, Optional -from dynamo.common.config_dump import register_encoder -from dynamo.common.configuration.arg_group import ArgGroup -from dynamo.common.configuration.groups.aic_perf_args import ( +from dingo.common.config_dump import register_encoder +from dingo.common.configuration.arg_group import ArgGroup +from dingo.common.configuration.groups.aic_perf_args import ( AicPerfArgGroup, AicPerfConfigBase, ) -from dynamo.common.configuration.groups.kv_router_args import ( +from dingo.common.configuration.groups.kv_router_args import ( KvRouterArgGroup, KvRouterConfigBase, ) -from dynamo.common.configuration.groups.router_args import ( +from dingo.common.configuration.groups.router_args import ( RouterArgGroup, RouterConfigBase, ) -from dynamo.common.configuration.utils import ( +from dingo.common.configuration.utils import ( add_argument, add_negatable_bool_argument, env_or_default, diff --git a/dingo/frontend/main.py b/dingo/frontend/main.py index ce28d47e09b9..96191557da5a 100644 --- a/dingo/frontend/main.py +++ b/dingo/frontend/main.py @@ -27,7 +27,7 @@ import uvloop -from dynamo.common.config_dump import dump_config +from dingo.common.config_dump import dump_config from dynamo.llm import ( AicPerfConfig, EngineType, diff --git a/dingo/frontend/vllm_processor.py b/dingo/frontend/vllm_processor.py index 6f01a4a6f0bc..161d75b4f49e 100644 --- a/dingo/frontend/vllm_processor.py +++ b/dingo/frontend/vllm_processor.py @@ -28,13 +28,13 @@ from vllm.v1.engine.parallel_sampling import ParentRequest from dynamo._internal import ModelDeploymentCard -from dynamo.common.multimodal.mm_kwargs_transfer import ( +from dingo.common.multimodal.mm_kwargs_transfer import ( MmKwargsNixlSender, MmKwargsSender, MmKwargsShmSender, ) -from dynamo.common.multimodal.routing_utils import build_mm_routing_info_from_features -from dynamo.common.utils import nvtx_utils as _nvtx +from dingo.common.multimodal.routing_utils import build_mm_routing_info_from_features +from dingo.common.utils import nvtx_utils as _nvtx from dingo.frontend.frontend_args import FrontendConfig from dynamo.llm import ModelCardInstanceId, PythonAsyncEngine, RoutedEngine @@ -833,7 +833,7 @@ async def chat_engine_factory( # This eliminates data URI encoding overhead entirely. if os.environ.get("VLLM_MEDIA_CONNECTOR") != "dynamo": os.environ["VLLM_MEDIA_CONNECTOR"] = "dynamo" - import dynamo.common.multimodal.media_connector # noqa: F401 + import dingo.common.multimodal.media_connector # noqa: F401 input_processor = InputProcessor(vllm_config) tokenizer = input_processor.get_tokenizer() diff --git a/components/src/dynamo/global_planner/README.md b/dingo/global_planner/README.md similarity index 90% rename from components/src/dynamo/global_planner/README.md rename to dingo/global_planner/README.md index 6a7046642710..8074ee0e4443 100644 --- a/components/src/dynamo/global_planner/README.md +++ b/dingo/global_planner/README.md @@ -32,7 +32,7 @@ That is fine for isolated deployments, but it becomes awkward when you want one ## Terminology -- **SLA Planner**: The normal `dynamo.planner` component that computes desired replica counts from SLA targets, profiles, and/or metrics. +- **SLA Planner**: The normal `dingo.planner` component that computes desired replica counts from SLA targets, profiles, and/or metrics. - **Local planner**: An instance of that planner running inside one DGD or one pool. - **GlobalPlanner**: The centralized execution and policy layer that receives scale requests from local planners and applies them to target DGDs. - **Hierarchical planner**: An architecture term, not a separate binary. In practice it means multiple local planners feeding one `GlobalPlanner`, often together with `GlobalRouter`. @@ -65,28 +65,28 @@ Given `DYN_NAMESPACE=`, this component serves: ```bash # Accept scale requests from any namespace -DYN_NAMESPACE=global-infra python -m dynamo.global_planner +DYN_NAMESPACE=global-infra python -m dingo.global_planner ``` ```bash # Restrict requests to specific planner namespaces -DYN_NAMESPACE=global-infra python -m dynamo.global_planner \ +DYN_NAMESPACE=global-infra python -m dingo.global_planner \ --managed-namespaces app-ns-1 app-ns-2 ``` ```bash # Dry-run mode (no Kubernetes updates) -DYN_NAMESPACE=global-infra python -m dynamo.global_planner --no-operation +DYN_NAMESPACE=global-infra python -m dingo.global_planner --no-operation ``` ```bash # Enforce a maximum total GPU budget across managed pools -DYN_NAMESPACE=global-infra python -m dynamo.global_planner --max-total-gpus 16 +DYN_NAMESPACE=global-infra python -m dingo.global_planner --max-total-gpus 16 ``` ```bash # Fixed-total deployment: pin the cluster at 16 GPUs with cross-pool transfers -DYN_NAMESPACE=global-infra python -m dynamo.global_planner \ +DYN_NAMESPACE=global-infra python -m dingo.global_planner \ --min-total-gpus 16 --max-total-gpus 16 ``` @@ -156,7 +156,7 @@ When the selected partners span multiple DGDs: - **Cross-DGD partners** (different DGD) get their own per-DGD patch. - **Direction-aware order**: scale-down DGDs apply first (freeing GPUs), then scale-up DGDs. If any second-or-later patch fails, the first has already landed and the system self-corrects from the new state on the next tick. -This scope is needed for "multiple agg pools sharing a budget" deployments such as [`examples/global_planner/global-planner-gpu-budget.yaml`](../../../../examples/global_planner/global-planner-gpu-budget.yaml). +This scope is needed for "multiple agg pools sharing a budget" deployments such as [`examples/global_planner/global-planner-gpu-budget.yaml`](../../examples/global_planner/global-planner-gpu-budget.yaml). **Tolerance for asymmetric pools.** When two paired pools have different `resources.limits.gpu` per replica, a single-worker step cannot always exactly cancel. Paired transfers may land up to `max(gpu_per_replica across the paired pools)` **below** `min` so the pair can still rebalance in whole-worker steps. `max` is a hard cluster-capacity bound and is never relaxed — pairs whose post-transfer total would exceed `max` are denied. Standalone (non-paired) requests must stay strictly within `[min, max]`. @@ -166,8 +166,8 @@ This scope is needed for "multiple agg pools sharing a budget" deployments such ## Related Documentation -- [Planner Guide](../../../../docs/components/planner/planner-guide.md) — Planner configuration and deployment workflow -- [Global Planner Deployment Guide](../../../../docs/components/planner/global-planner.md) — Deployment patterns for `GlobalPlanner`, including multi-model coordination and single-endpoint multi-pool workflows -- [Planner Design](../../../../docs/design-docs/planner-design.md) — Planner architecture and algorithms +- [Planner Guide](../../docs/components/planner/planner-guide.md) — Planner configuration and deployment workflow +- [Global Planner Deployment Guide](../../docs/components/planner/global-planner.md) — Deployment patterns for `GlobalPlanner`, including multi-model coordination and single-endpoint multi-pool workflows +- [Planner Design](../../docs/design-docs/planner-design.md) — Planner architecture and algorithms Planners delegate to this service when planner config uses `environment: "global-planner"` and sets `global_planner_namespace`. diff --git a/components/src/dynamo/global_planner/__init__.py b/dingo/global_planner/__init__.py similarity index 85% rename from components/src/dynamo/global_planner/__init__.py rename to dingo/global_planner/__init__.py index b0d751c60392..30e31f232858 100644 --- a/components/src/dynamo/global_planner/__init__.py +++ b/dingo/global_planner/__init__.py @@ -15,7 +15,7 @@ - GlobalPlanner is stateless and can scale horizontally Usage: - DYN_NAMESPACE=global-infra python -m dynamo.global_planner \ + DYN_NAMESPACE=global-infra python -m dingo.global_planner \ --managed-namespaces app-ns-1 app-ns-2 """ @@ -23,4 +23,4 @@ "ScaleRequestHandler", ] -from dynamo.global_planner.scale_handler import ScaleRequestHandler +from dingo.global_planner.scale_handler import ScaleRequestHandler diff --git a/components/src/dynamo/global_planner/__main__.py b/dingo/global_planner/__main__.py similarity index 93% rename from components/src/dynamo/global_planner/__main__.py rename to dingo/global_planner/__main__.py index acc9a82b1959..a2078e817201 100644 --- a/components/src/dynamo/global_planner/__main__.py +++ b/dingo/global_planner/__main__.py @@ -7,10 +7,10 @@ Entry point for the GlobalPlanner component. Usage: - DYN_NAMESPACE=global-infra python -m dynamo.global_planner + DYN_NAMESPACE=global-infra python -m dingo.global_planner With authorization: - DYN_NAMESPACE=global-infra python -m dynamo.global_planner \\ + DYN_NAMESPACE=global-infra python -m dingo.global_planner \\ --managed-namespaces app-ns-1 app-ns-2 """ @@ -20,8 +20,8 @@ from pydantic import BaseModel -from dynamo.global_planner.argparse_config import create_global_planner_parser -from dynamo.global_planner.scale_handler import ScaleRequestHandler +from dingo.global_planner.argparse_config import create_global_planner_parser +from dingo.global_planner.scale_handler import ScaleRequestHandler from dynamo.runtime import DistributedRuntime, dynamo_worker from dynamo.runtime.logging import configure_dynamo_logging @@ -132,7 +132,7 @@ async def health_check(request: HealthCheckRequest): # never register the health endpoint, so system_health would never flip # to Ready and the operator-injected HTTP probes on :system/live and # :system/health would 503 indefinitely. Run concurrently via - # asyncio.gather; pattern matches components/src/dynamo/planner/__main__.py. + # asyncio.gather; pattern matches dingo/planner/__main__.py. # # Passing health_check_payload to the health endpoint registers it as a # health-check target so system_health flips to Ready once the endpoint diff --git a/components/src/dynamo/global_planner/argparse_config.py b/dingo/global_planner/argparse_config.py similarity index 93% rename from components/src/dynamo/global_planner/argparse_config.py rename to dingo/global_planner/argparse_config.py index c842e932af29..a53507dc3b36 100644 --- a/components/src/dynamo/global_planner/argparse_config.py +++ b/dingo/global_planner/argparse_config.py @@ -18,14 +18,14 @@ def create_global_planner_parser() -> argparse.ArgumentParser: epilog=""" Examples: # Simple deployment (accept all namespaces) - DYN_NAMESPACE=global-infra python -m dynamo.global_planner + DYN_NAMESPACE=global-infra python -m dingo.global_planner # With authorization - DYN_NAMESPACE=global-infra python -m dynamo.global_planner \\ + DYN_NAMESPACE=global-infra python -m dingo.global_planner \\ --managed-namespaces app-ns-1 app-ns-2 app-ns-3 # Custom environment - DYN_NAMESPACE=global-infra python -m dynamo.global_planner \\ + DYN_NAMESPACE=global-infra python -m dingo.global_planner \\ --environment=kubernetes """, ) diff --git a/components/src/dynamo/global_planner/scale_handler.py b/dingo/global_planner/scale_handler.py similarity index 99% rename from components/src/dynamo/global_planner/scale_handler.py rename to dingo/global_planner/scale_handler.py index fd5d3c7675ee..4c75c53226d9 100644 --- a/components/src/dynamo/global_planner/scale_handler.py +++ b/dingo/global_planner/scale_handler.py @@ -11,10 +11,10 @@ from dataclasses import dataclass from typing import Iterator, Optional -from dynamo.planner import KubernetesConnector, SubComponentType, TargetReplica -from dynamo.planner.connectors.kubernetes_api import KubernetesAPI -from dynamo.planner.connectors.protocol import ScaleRequest, ScaleResponse, ScaleStatus -from dynamo.planner.core import budget +from dingo.planner import KubernetesConnector, SubComponentType, TargetReplica +from dingo.planner.connectors.kubernetes_api import KubernetesAPI +from dingo.planner.connectors.protocol import ScaleRequest, ScaleResponse, ScaleStatus +from dingo.planner.core import budget from dynamo.runtime import DistributedRuntime, dynamo_endpoint logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/global_planner/tests/unit/test_main_endpoint_registration.py b/dingo/global_planner/tests/unit/test_main_endpoint_registration.py similarity index 97% rename from components/src/dynamo/global_planner/tests/unit/test_main_endpoint_registration.py rename to dingo/global_planner/tests/unit/test_main_endpoint_registration.py index 7ca48494add9..fe9b7b80e2aa 100644 --- a/components/src/dynamo/global_planner/tests/unit/test_main_endpoint_registration.py +++ b/dingo/global_planner/tests/unit/test_main_endpoint_registration.py @@ -20,7 +20,7 @@ import pytest -from dynamo.global_planner import __main__ as gp_main +from dingo.global_planner import __main__ as gp_main pytestmark = [ pytest.mark.gpu_0, @@ -67,7 +67,7 @@ def make_endpoint(name: str) -> MagicMock: with patch.dict( os.environ, {"DYN_NAMESPACE": "gp-ns", "POD_NAMESPACE": "default"} - ), patch("dynamo.global_planner.__main__.ScaleRequestHandler") as mock_handler_cls: + ), patch("dingo.global_planner.__main__.ScaleRequestHandler") as mock_handler_cls: mock_handler_cls.return_value = MagicMock() # main never returns on its own (it awaits the endpoint futures diff --git a/components/src/dynamo/global_planner/tests/unit/test_scale_request_handler.py b/dingo/global_planner/tests/unit/test_scale_request_handler.py similarity index 98% rename from components/src/dynamo/global_planner/tests/unit/test_scale_request_handler.py rename to dingo/global_planner/tests/unit/test_scale_request_handler.py index a5eadd6430ba..68da78b0a013 100644 --- a/components/src/dynamo/global_planner/tests/unit/test_scale_request_handler.py +++ b/dingo/global_planner/tests/unit/test_scale_request_handler.py @@ -8,9 +8,9 @@ import pytest -from dynamo.global_planner.scale_handler import PoolIntent, ScaleRequestHandler -from dynamo.planner import SubComponentType, TargetReplica -from dynamo.planner.connectors.protocol import ScaleRequest +from dingo.global_planner.scale_handler import PoolIntent, ScaleRequestHandler +from dingo.planner import SubComponentType, TargetReplica +from dingo.planner.connectors.protocol import ScaleRequest pytestmark = [ pytest.mark.gpu_0, @@ -47,7 +47,7 @@ async def test_handler_authorization_success(mock_runtime): # Mock KubernetesConnector with patch( - "dynamo.global_planner.scale_handler.KubernetesConnector" + "dingo.global_planner.scale_handler.KubernetesConnector" ) as mock_connector_cls: mock_connector = AsyncMock() mock_connector_cls.return_value = mock_connector @@ -140,7 +140,7 @@ async def test_handler_multiple_dgds(mock_runtime): ) with patch( - "dynamo.global_planner.scale_handler.KubernetesConnector" + "dingo.global_planner.scale_handler.KubernetesConnector" ) as mock_connector_cls: mock_connector = AsyncMock() mock_connector_cls.return_value = mock_connector @@ -182,7 +182,7 @@ async def test_handler_error_handling(mock_runtime): ) with patch( - "dynamo.global_planner.scale_handler.KubernetesConnector" + "dingo.global_planner.scale_handler.KubernetesConnector" ) as mock_connector_cls: mock_connector = AsyncMock() mock_connector_cls.return_value = mock_connector @@ -247,9 +247,9 @@ async def test_populate_connectors_explicit_mode(mock_runtime): ) with ( - patch("dynamo.global_planner.scale_handler.KubernetesAPI") as mock_kube_cls, + patch("dingo.global_planner.scale_handler.KubernetesAPI") as mock_kube_cls, patch( - "dynamo.global_planner.scale_handler.KubernetesConnector" + "dingo.global_planner.scale_handler.KubernetesConnector" ) as mock_connector_cls, ): mock_kube = MagicMock() @@ -281,9 +281,9 @@ async def test_populate_connectors_implicit_mode(mock_runtime): ) with ( - patch("dynamo.global_planner.scale_handler.KubernetesAPI") as mock_kube_cls, + patch("dingo.global_planner.scale_handler.KubernetesAPI") as mock_kube_cls, patch( - "dynamo.global_planner.scale_handler.KubernetesConnector" + "dingo.global_planner.scale_handler.KubernetesConnector" ) as mock_connector_cls, ): mock_kube = MagicMock() @@ -322,7 +322,7 @@ async def test_handler_blocking_mode(mock_runtime): ) with patch( - "dynamo.global_planner.scale_handler.KubernetesConnector" + "dingo.global_planner.scale_handler.KubernetesConnector" ) as mock_connector_cls: mock_connector = AsyncMock() mock_connector_cls.return_value = mock_connector @@ -637,7 +637,7 @@ async def test_cross_dgd_pair_second_patch_failure_self_corrects(mock_runtime, c connector_b.set_component_replicas.side_effect = Exception("simulated K8s failure") req = _scale_req(dgd="dgd-a", caller_ns="default-dgd-a", prefill=2) - with caplog.at_level(_logging.ERROR, logger="dynamo.global_planner.scale_handler"): + with caplog.at_level(_logging.ERROR, logger="dingo.global_planner.scale_handler"): results = await _run(handler, req) # Overall response: the request-side patch succeeded; response path reports # success because the request-side was applied. The cross-DGD partner @@ -889,9 +889,9 @@ async def test_initial_below_floor_logs_warning(mock_runtime): semantics — and the floor only blocks explicit scale-downs going forward (covered by the standalone-deny tests above).""" with ( - patch("dynamo.global_planner.scale_handler.KubernetesAPI") as mock_kube_cls, + patch("dingo.global_planner.scale_handler.KubernetesAPI") as mock_kube_cls, patch( - "dynamo.global_planner.scale_handler.KubernetesConnector" + "dingo.global_planner.scale_handler.KubernetesConnector" ) as mock_connector_cls, ): mock_kube = MagicMock() @@ -907,7 +907,7 @@ async def test_initial_below_floor_logs_warning(mock_runtime): ) mock_connector_cls.return_value = mock_connector - with patch("dynamo.global_planner.scale_handler.logger") as mock_logger: + with patch("dingo.global_planner.scale_handler.logger") as mock_logger: ScaleRequestHandler( runtime=mock_runtime, managed_namespaces=["default-my-dgd"], diff --git a/components/src/dynamo/global_router/README.md b/dingo/global_router/README.md similarity index 99% rename from components/src/dynamo/global_router/README.md rename to dingo/global_router/README.md index ed490a95090f..c2b00f3f001e 100644 --- a/components/src/dynamo/global_router/README.md +++ b/dingo/global_router/README.md @@ -74,7 +74,7 @@ Global Router (registers as Chat + Completions) ## Usage ```bash -python -m dynamo.global_router \ +python -m dingo.global_router \ --config path/to/global_router_config.json \ --model-name Qwen/Qwen3-0.6B \ --namespace dynamo diff --git a/components/src/dynamo/global_router/__init__.py b/dingo/global_router/__init__.py similarity index 100% rename from components/src/dynamo/global_router/__init__.py rename to dingo/global_router/__init__.py diff --git a/components/src/dynamo/global_router/__main__.py b/dingo/global_router/__main__.py similarity index 98% rename from components/src/dynamo/global_router/__main__.py rename to dingo/global_router/__main__.py index 0bd5a1478bea..a2c82406b360 100644 --- a/components/src/dynamo/global_router/__main__.py +++ b/dingo/global_router/__main__.py @@ -4,7 +4,7 @@ """ Global Router Service for Hierarchical Routing -Usage: python -m dynamo.global_router --config --model-name +Usage: python -m dingo.global_router --config --model-name This service routes requests to local routers in different namespaces based on a grid-based pool selection strategy. It supports two modes: diff --git a/components/src/dynamo/global_router/backend_args.py b/dingo/global_router/backend_args.py similarity index 94% rename from components/src/dynamo/global_router/backend_args.py rename to dingo/global_router/backend_args.py index ef00fb276431..66aed277ad55 100644 --- a/components/src/dynamo/global_router/backend_args.py +++ b/dingo/global_router/backend_args.py @@ -5,9 +5,9 @@ from typing import Optional -from dynamo.common.configuration.arg_group import ArgGroup -from dynamo.common.configuration.config_base import ConfigBase -from dynamo.common.configuration.utils import add_argument, env_or_default +from dingo.common.configuration.arg_group import ArgGroup +from dingo.common.configuration.config_base import ConfigBase +from dingo.common.configuration.utils import add_argument, env_or_default class DynamoGlobalRouterArgGroup(ArgGroup): diff --git a/components/src/dynamo/global_router/handler.py b/dingo/global_router/handler.py similarity index 100% rename from components/src/dynamo/global_router/handler.py rename to dingo/global_router/handler.py diff --git a/components/src/dynamo/global_router/pool_selection.py b/dingo/global_router/pool_selection.py similarity index 100% rename from components/src/dynamo/global_router/pool_selection.py rename to dingo/global_router/pool_selection.py diff --git a/components/src/dynamo/global_router/tests/test_agg_routing.py b/dingo/global_router/tests/test_agg_routing.py similarity index 99% rename from components/src/dynamo/global_router/tests/test_agg_routing.py rename to dingo/global_router/tests/test_agg_routing.py index 5fad0b8a5ee2..9bddbb6804f6 100644 --- a/components/src/dynamo/global_router/tests/test_agg_routing.py +++ b/dingo/global_router/tests/test_agg_routing.py @@ -8,7 +8,7 @@ import pytest -from dynamo.global_router.pool_selection import ( +from dingo.global_router.pool_selection import ( AggPoolSelectionStrategy, GlobalRouterConfig, PriorityPoolOverride, diff --git a/components/src/dynamo/global_router/tests/test_priority_retry.py b/dingo/global_router/tests/test_priority_retry.py similarity index 99% rename from components/src/dynamo/global_router/tests/test_priority_retry.py rename to dingo/global_router/tests/test_priority_retry.py index 327e0a36efa4..6c9f626fd8a2 100644 --- a/components/src/dynamo/global_router/tests/test_priority_retry.py +++ b/dingo/global_router/tests/test_priority_retry.py @@ -9,7 +9,7 @@ import pytest -from dynamo.global_router.handler import GlobalRouterHandler +from dingo.global_router.handler import GlobalRouterHandler pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/global_router/tests/test_priority_routing.py b/dingo/global_router/tests/test_priority_routing.py similarity index 99% rename from components/src/dynamo/global_router/tests/test_priority_routing.py rename to dingo/global_router/tests/test_priority_routing.py index a166d05c34eb..623974be9e7e 100644 --- a/components/src/dynamo/global_router/tests/test_priority_routing.py +++ b/dingo/global_router/tests/test_priority_routing.py @@ -8,7 +8,7 @@ import pytest -from dynamo.global_router.pool_selection import ( +from dingo.global_router.pool_selection import ( DecodePoolSelectionStrategy, GlobalRouterConfig, PrefillPoolSelectionStrategy, diff --git a/components/src/dynamo/mocker/README.md b/dingo/mocker/README.md similarity index 100% rename from components/src/dynamo/mocker/README.md rename to dingo/mocker/README.md diff --git a/components/src/dynamo/mocker/__init__.py b/dingo/mocker/__init__.py similarity index 100% rename from components/src/dynamo/mocker/__init__.py rename to dingo/mocker/__init__.py diff --git a/components/src/dynamo/mocker/__main__.py b/dingo/mocker/__main__.py similarity index 83% rename from components/src/dynamo/mocker/__main__.py rename to dingo/mocker/__main__.py index f39460df7a3a..07a87e5dd372 100644 --- a/components/src/dynamo/mocker/__main__.py +++ b/dingo/mocker/__main__.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from dynamo.mocker.main import main +from dingo.mocker.main import main if __name__ == "__main__": main() diff --git a/components/src/dynamo/mocker/aic_session.py b/dingo/mocker/aic_session.py similarity index 100% rename from components/src/dynamo/mocker/aic_session.py rename to dingo/mocker/aic_session.py diff --git a/components/src/dynamo/mocker/args.py b/dingo/mocker/args.py similarity index 99% rename from components/src/dynamo/mocker/args.py rename to dingo/mocker/args.py index 7e3159ec5242..1a5c62eced3b 100644 --- a/components/src/dynamo/mocker/args.py +++ b/dingo/mocker/args.py @@ -7,7 +7,7 @@ import tempfile from pathlib import Path -from dynamo.common.utils.namespace import get_worker_namespace +from dingo.common.utils.namespace import get_worker_namespace from . import __version__ diff --git a/components/src/dynamo/mocker/config.py b/dingo/mocker/config.py similarity index 99% rename from components/src/dynamo/mocker/config.py rename to dingo/mocker/config.py index 5e69a990f1a6..cf7460130d40 100644 --- a/components/src/dynamo/mocker/config.py +++ b/dingo/mocker/config.py @@ -11,9 +11,9 @@ DEFAULT_MEM_FRACTION_STATIC, estimate_num_gpu_blocks, ) -from dynamo.common.utils.topology import apply_topology_config +from dingo.common.utils.topology import apply_topology_config from dynamo.llm import ModelRuntimeConfig -from dynamo.mocker import MockEngineArgs, ReasoningConfig, SglangArgs, TrtllmArgs +from dingo.mocker import MockEngineArgs, ReasoningConfig, SglangArgs, TrtllmArgs _DEFAULT_NUM_GPU_BLOCKS = 16384 _DEFAULT_MAX_NUM_SEQS = 256 diff --git a/components/src/dynamo/mocker/main.py b/dingo/mocker/main.py similarity index 98% rename from components/src/dynamo/mocker/main.py rename to dingo/mocker/main.py index edc9691b4c2d..7bbb7ccb9c89 100644 --- a/components/src/dynamo/mocker/main.py +++ b/dingo/mocker/main.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Usage: `python -m dynamo.mocker --model-path /data/models/Qwen3-0.6B` +# Usage: `python -m dingo.mocker --model-path /data/models/Qwen3-0.6B` # Now supports vLLM-style individual arguments for MockEngineArgs import argparse @@ -15,7 +15,7 @@ os.environ.setdefault("DYN_COMPUTE_THREADS", "0") -from dynamo.common.utils.runtime import create_runtime +from dingo.common.utils.runtime import create_runtime from dynamo.llm import EngineType, EntrypointArgs, fetch_model, make_engine, run_input from dynamo.runtime.logging import configure_dynamo_logging diff --git a/components/src/dynamo/mocker/tests/unit/test_config.py b/dingo/mocker/tests/unit/test_config.py similarity index 98% rename from components/src/dynamo/mocker/tests/unit/test_config.py rename to dingo/mocker/tests/unit/test_config.py index c3406c958dd3..1d323ca30208 100644 --- a/components/src/dynamo/mocker/tests/unit/test_config.py +++ b/dingo/mocker/tests/unit/test_config.py @@ -11,8 +11,8 @@ import pytest from dynamo.llm import EngineType, EntrypointArgs -from dynamo.mocker import MockEngineArgs -from dynamo.mocker.args import parse_args +from dingo.mocker import MockEngineArgs +from dingo.mocker.args import parse_args MODULE_PATH = Path(__file__).resolve().parents[2] / "config.py" SPEC = importlib.util.spec_from_file_location("dynamo_mocker_config", MODULE_PATH) @@ -435,7 +435,7 @@ def test_build_mocker_engine_args_preserves_explicit_max_model_len(): def test_replay_engine_args_keeps_max_model_len_explicit_only(): - import dynamo.replay.main as replay_main + import dingo.replay.main as replay_main engine_args = replay_main._load_engine_args( json.dumps( @@ -450,7 +450,7 @@ def test_replay_engine_args_keeps_max_model_len_explicit_only(): def test_replay_engine_args_preserves_explicit_max_model_len(): - import dynamo.replay.main as replay_main + import dingo.replay.main as replay_main engine_args = replay_main._load_engine_args( json.dumps( @@ -466,7 +466,7 @@ def test_replay_engine_args_preserves_explicit_max_model_len(): def test_replay_engine_args_compute_kv_bytes_for_g3_before_validation(monkeypatch): - import dynamo.replay.main as replay_main + import dingo.replay.main as replay_main calls = [] @@ -495,7 +495,7 @@ def fake_compute_kv_bytes_per_token(model_path, kv_cache_dtype="auto"): def test_replay_engine_args_compute_kv_bytes_for_g4_before_validation(monkeypatch): - import dynamo.replay.main as replay_main + import dingo.replay.main as replay_main calls = [] @@ -529,7 +529,7 @@ def test_get_kv_cache_dtype_bytes_supports_int8(): # transfer latency are overstated. from types import SimpleNamespace - from dynamo.mocker.utils.kv_cache import get_kv_cache_dtype_bytes + from dingo.mocker.utils.kv_cache import get_kv_cache_dtype_bytes cfg = SimpleNamespace(dtype="bfloat16") assert get_kv_cache_dtype_bytes(cfg, "int8") == 1 @@ -540,7 +540,7 @@ def test_get_kv_cache_dtype_bytes_supports_int8(): def test_replay_engine_args_forwards_aic_kv_cache_dtype(monkeypatch): # Offload KV-byte estimation must use the configured (normalized) KV dtype, # not always "auto". - import dynamo.replay.main as replay_main + import dingo.replay.main as replay_main calls = [] diff --git a/components/src/dynamo/mocker/utils/kv_cache.py b/dingo/mocker/utils/kv_cache.py similarity index 95% rename from components/src/dynamo/mocker/utils/kv_cache.py rename to dingo/mocker/utils/kv_cache.py index 7e76dd3875d5..d3d87c34f444 100644 --- a/components/src/dynamo/mocker/utils/kv_cache.py +++ b/dingo/mocker/utils/kv_cache.py @@ -6,7 +6,7 @@ from transformers import AutoConfig -from dynamo.profiler.utils.model_info import get_model_info +from dingo.profiler.utils.model_info import get_model_info logger = logging.getLogger(__name__) @@ -65,7 +65,7 @@ def compute_kv_bytes_per_token( Formula: num_layers * 2 (K+V) * num_kv_heads * head_dim * dtype_bytes - Uses get_model_info from dynamo.profiler for robust detection of num_kv_heads + Uses get_model_info from dingo.profiler for robust detection of num_kv_heads across different model architectures. Args: diff --git a/components/src/dynamo/mocker/utils/planner_profiler_perf_data_converter.py b/dingo/mocker/utils/planner_profiler_perf_data_converter.py similarity index 97% rename from components/src/dynamo/mocker/utils/planner_profiler_perf_data_converter.py rename to dingo/mocker/utils/planner_profiler_perf_data_converter.py index 3880a6308391..72be2ca535d7 100644 --- a/components/src/dynamo/mocker/utils/planner_profiler_perf_data_converter.py +++ b/dingo/mocker/utils/planner_profiler_perf_data_converter.py @@ -4,7 +4,7 @@ """ Convert planner profiler results to mocker-compatible NPZ format. -Uses the FPM-based regression models from ``dynamo.planner.core.perf_model`` +Uses the FPM-based regression models from ``dingo.planner.core.perf_model`` to evaluate prefill TTFT and decode ITL on a regular grid, producing the lookup tables that the mocker uses for latency simulation. @@ -55,11 +55,11 @@ def convert_profile_results_to_npz( Returns: Path to the generated NPZ file. """ - from dynamo.planner.core.perf_model import ( + from dingo.planner.core.perf_model import ( DecodeRegressionModel, PrefillRegressionModel, ) - from dynamo.planner.monitoring.perf_metrics import ( + from dingo.planner.monitoring.perf_metrics import ( _convert_decode_profiling, _convert_prefill_profiling, ) diff --git a/components/src/dynamo/planner/README.md b/dingo/planner/README.md similarity index 100% rename from components/src/dynamo/planner/README.md rename to dingo/planner/README.md diff --git a/components/src/dynamo/planner/__init__.py b/dingo/planner/__init__.py similarity index 63% rename from components/src/dynamo/planner/__init__.py rename to dingo/planner/__init__.py index 83d9f364c826..192a1b0aa040 100644 --- a/components/src/dynamo/planner/__init__.py +++ b/dingo/planner/__init__.py @@ -12,16 +12,16 @@ "WorkerInfo", ] -from dynamo.planner.config.defaults import ( +from dingo.planner.config.defaults import ( SLAPlannerDefaults, SubComponentType, TargetReplica, ) -from dynamo.planner.connectors.base import PlannerConnector -from dynamo.planner.connectors.global_planner import GlobalPlannerConnector -from dynamo.planner.connectors.kubernetes import KubernetesConnector -from dynamo.planner.connectors.virtual import VirtualConnector -from dynamo.planner.monitoring.worker_info import WorkerInfo +from dingo.planner.connectors.base import PlannerConnector +from dingo.planner.connectors.global_planner import GlobalPlannerConnector +from dingo.planner.connectors.kubernetes import KubernetesConnector +from dingo.planner.connectors.virtual import VirtualConnector +from dingo.planner.monitoring.worker_info import WorkerInfo try: from ._version import __version__ diff --git a/components/src/dynamo/planner/__main__.py b/dingo/planner/__main__.py similarity index 96% rename from components/src/dynamo/planner/__main__.py rename to dingo/planner/__main__.py index 92e65e199668..f06d6d02b9fe 100644 --- a/components/src/dynamo/planner/__main__.py +++ b/dingo/planner/__main__.py @@ -20,8 +20,8 @@ from pydantic import BaseModel -from dynamo.planner.config.planner_config import PlannerConfig -from dynamo.planner.core.adapters import ( +from dingo.planner.config.planner_config import PlannerConfig +from dingo.planner.core.adapters import ( AggPlanner, DecodePlanner, DisaggPlanner, diff --git a/components/src/dynamo/planner/config/__init__.py b/dingo/planner/config/__init__.py similarity index 100% rename from components/src/dynamo/planner/config/__init__.py rename to dingo/planner/config/__init__.py diff --git a/components/src/dynamo/planner/config/aic_interpolation_spec.py b/dingo/planner/config/aic_interpolation_spec.py similarity index 89% rename from components/src/dynamo/planner/config/aic_interpolation_spec.py rename to dingo/planner/config/aic_interpolation_spec.py index 68643d25790c..9be5323452d2 100644 --- a/components/src/dynamo/planner/config/aic_interpolation_spec.py +++ b/dingo/planner/config/aic_interpolation_spec.py @@ -13,14 +13,14 @@ from pydantic import BaseModel, Field -from dynamo.planner.config.parallelization import PickedParallelConfig +from dingo.planner.config.parallelization import PickedParallelConfig class AICInterpolationSpec(BaseModel): """Everything the planner needs to reproduce the rapid-mode AIC sweep. The picks come straight from AIC's picker DataFrame (via - :func:`dynamo.profiler.utils.profile_common.picked_config_from_row`) so + :func:`dingo.profiler.utils.profile_common.picked_config_from_row`) so any AIC-valid pick is representable. """ diff --git a/components/src/dynamo/planner/config/backend_components.py b/dingo/planner/config/backend_components.py similarity index 100% rename from components/src/dynamo/planner/config/backend_components.py rename to dingo/planner/config/backend_components.py diff --git a/components/src/dynamo/planner/config/defaults.py b/dingo/planner/config/defaults.py similarity index 98% rename from components/src/dynamo/planner/config/defaults.py rename to dingo/planner/config/defaults.py index 1e2a85d1701d..f8f2a89a74d3 100644 --- a/components/src/dynamo/planner/config/defaults.py +++ b/dingo/planner/config/defaults.py @@ -31,7 +31,7 @@ class BasePlannerDefaults: # GPU floor for the local planner (per-DGD scope). -1 disables. # When set alongside max_gpu_budget (with min == max), pins the total # and the planner only redistributes replicas between pools. - # See dynamo.planner.core.budget.proportional_clamp_pair for the + # See dingo.planner.core.budget.proportional_clamp_pair for the # tolerance band semantics. min_gpu_budget = -1 min_endpoint = 1 # applies to both decode and prefill diff --git a/components/src/dynamo/planner/config/parallelization.py b/dingo/planner/config/parallelization.py similarity index 98% rename from components/src/dynamo/planner/config/parallelization.py rename to dingo/planner/config/parallelization.py index 78c0d0e74f43..ffdb03c807e7 100644 --- a/components/src/dynamo/planner/config/parallelization.py +++ b/dingo/planner/config/parallelization.py @@ -6,7 +6,7 @@ ``PickedParallelConfig`` stores the full ``(tp, pp, dp, moe_tp, moe_ep)`` tuple that AIConfigurator's picker emits. Both the profiler (which picks) and the planner (which consumes the pick to bootstrap perf models) need this type, so -it lives under ``dynamo.planner.config`` rather than the profiler tree. +it lives under ``dingo.planner.config`` rather than the profiler tree. It is a pydantic ``BaseModel`` so it serialises cleanly into the planner ConfigMap as part of ``AICInterpolationSpec``. diff --git a/components/src/dynamo/planner/config/planner_config.py b/dingo/planner/config/planner_config.py similarity index 98% rename from components/src/dynamo/planner/config/planner_config.py rename to dingo/planner/config/planner_config.py index 26e96ac6917a..9e8c1b91954d 100644 --- a/components/src/dynamo/planner/config/planner_config.py +++ b/dingo/planner/config/planner_config.py @@ -32,11 +32,11 @@ model_validator, ) -from dynamo.planner.config.aic_interpolation_spec import AICInterpolationSpec -from dynamo.planner.config.defaults import SLAPlannerDefaults -from dynamo.planner.config.parallelization import PickedParallelConfig -from dynamo.planner.plugins.registry.config import PluginRegistrationConfig -from dynamo.planner.plugins.types import HoldPolicy +from dingo.planner.config.aic_interpolation_spec import AICInterpolationSpec +from dingo.planner.config.defaults import SLAPlannerDefaults +from dingo.planner.config.parallelization import PickedParallelConfig +from dingo.planner.plugins.registry.config import PluginRegistrationConfig +from dingo.planner.plugins.types import HoldPolicy logger = logging.getLogger(__name__) @@ -344,7 +344,7 @@ class SchedulingConfig(BaseModel): class PlannerConfig(BaseModel): """Pydantic configuration for the Dynamo Planner. - Defines the JSON/YAML config consumed by ``python -m dynamo.planner``. + Defines the JSON/YAML config consumed by ``python -m dingo.planner``. Defaults are sourced from SLAPlannerDefaults. """ diff --git a/components/src/dynamo/planner/connectors/__init__.py b/dingo/planner/connectors/__init__.py similarity index 100% rename from components/src/dynamo/planner/connectors/__init__.py rename to dingo/planner/connectors/__init__.py diff --git a/components/src/dynamo/planner/connectors/base.py b/dingo/planner/connectors/base.py similarity index 95% rename from components/src/dynamo/planner/connectors/base.py rename to dingo/planner/connectors/base.py index 8fdd400fc42e..4400c4966ac8 100644 --- a/components/src/dynamo/planner/connectors/base.py +++ b/dingo/planner/connectors/base.py @@ -15,7 +15,7 @@ from abc import ABC, abstractmethod -from dynamo.planner.config.defaults import SubComponentType +from dingo.planner.config.defaults import SubComponentType # TODO: add ability to scale component to X replicas diff --git a/components/src/dynamo/planner/connectors/global_planner.py b/dingo/planner/connectors/global_planner.py similarity index 97% rename from components/src/dynamo/planner/connectors/global_planner.py rename to dingo/planner/connectors/global_planner.py index 26026e5af469..df232a53cffa 100644 --- a/components/src/dynamo/planner/connectors/global_planner.py +++ b/dingo/planner/connectors/global_planner.py @@ -11,19 +11,19 @@ from kubernetes.client import ApiException from kubernetes.config.config_exception import ConfigException -from dynamo.planner.config.defaults import SubComponentType, TargetReplica -from dynamo.planner.connectors.base import PlannerConnector -from dynamo.planner.connectors.kubernetes import KubernetesConnector -from dynamo.planner.connectors.protocol import ScaleRequest, ScaleStatus -from dynamo.planner.connectors.remote_client import RemotePlannerClient -from dynamo.planner.errors import ( +from dingo.planner.config.defaults import SubComponentType, TargetReplica +from dingo.planner.connectors.base import PlannerConnector +from dingo.planner.connectors.kubernetes import KubernetesConnector +from dingo.planner.connectors.protocol import ScaleRequest, ScaleStatus +from dingo.planner.connectors.remote_client import RemotePlannerClient +from dingo.planner.errors import ( DeploymentModelNameMismatchError, DeploymentValidationError, EmptyTargetReplicasError, ModelNameNotFoundError, UserProvidedModelNameMismatchError, ) -from dynamo.planner.monitoring.worker_info import ( +from dingo.planner.monitoring.worker_info import ( WorkerInfo, build_worker_info_from_defaults, ) diff --git a/components/src/dynamo/planner/connectors/kubernetes.py b/dingo/planner/connectors/kubernetes.py similarity index 98% rename from components/src/dynamo/planner/connectors/kubernetes.py rename to dingo/planner/connectors/kubernetes.py index de38dc7cd1e3..1e7992f98ee4 100644 --- a/components/src/dynamo/planner/connectors/kubernetes.py +++ b/dingo/planner/connectors/kubernetes.py @@ -18,20 +18,20 @@ import os from typing import Optional -from dynamo.planner.config.defaults import SubComponentType, TargetReplica -from dynamo.planner.connectors.base import PlannerConnector -from dynamo.planner.connectors.kubernetes_api import ( +from dingo.planner.config.defaults import SubComponentType, TargetReplica +from dingo.planner.connectors.base import PlannerConnector +from dingo.planner.connectors.kubernetes_api import ( DYNAMO_WORKER_METADATA_API_VERSION, NVIDIA_API_GROUP, KubernetesAPI, ) -from dynamo.planner.connectors.mdc import ( +from dingo.planner.connectors.mdc import ( MdcEntry, is_model_card, select_entry, worker_info_from_mdc, ) -from dynamo.planner.errors import ( +from dingo.planner.errors import ( DeploymentModelNameMismatchError, DeploymentValidationError, EmptyTargetReplicasError, @@ -39,12 +39,12 @@ PlannerError, UserProvidedModelNameMismatchError, ) -from dynamo.planner.monitoring.dgd_services import ( +from dingo.planner.monitoring.dgd_services import ( get_component_from_type_or_name, get_component_type, get_components_by_name, ) -from dynamo.planner.monitoring.worker_info import ( +from dingo.planner.monitoring.worker_info import ( WorkerInfo, build_worker_info_from_defaults, ) diff --git a/components/src/dynamo/planner/connectors/kubernetes_api.py b/dingo/planner/connectors/kubernetes_api.py similarity index 99% rename from components/src/dynamo/planner/connectors/kubernetes_api.py rename to dingo/planner/connectors/kubernetes_api.py index 07ae061a7fb4..489f628a1aef 100644 --- a/components/src/dynamo/planner/connectors/kubernetes_api.py +++ b/dingo/planner/connectors/kubernetes_api.py @@ -20,8 +20,8 @@ from kubernetes import client, config from kubernetes.config.config_exception import ConfigException -from dynamo.planner.errors import DynamoGraphDeploymentNotFoundError -from dynamo.planner.monitoring.dgd_services import ( +from dingo.planner.errors import DynamoGraphDeploymentNotFoundError +from dingo.planner.monitoring.dgd_services import ( Service, get_component_type, get_components_by_name, diff --git a/components/src/dynamo/planner/connectors/mdc.py b/dingo/planner/connectors/mdc.py similarity index 97% rename from components/src/dynamo/planner/connectors/mdc.py rename to dingo/planner/connectors/mdc.py index ea776c727266..a3e43d589ffb 100644 --- a/components/src/dynamo/planner/connectors/mdc.py +++ b/dingo/planner/connectors/mdc.py @@ -17,8 +17,8 @@ from dataclasses import dataclass, field from typing import Any, Callable, Optional, Protocol -from dynamo.planner.config.defaults import SubComponentType -from dynamo.planner.monitoring.worker_info import ( +from dingo.planner.config.defaults import SubComponentType +from dingo.planner.monitoring.worker_info import ( WorkerInfo, build_worker_info_from_defaults, ) diff --git a/components/src/dynamo/planner/connectors/protocol.py b/dingo/planner/connectors/protocol.py similarity index 96% rename from components/src/dynamo/planner/connectors/protocol.py rename to dingo/planner/connectors/protocol.py index 3e3b9c91db46..e6d8c83a101b 100644 --- a/components/src/dynamo/planner/connectors/protocol.py +++ b/dingo/planner/connectors/protocol.py @@ -8,7 +8,7 @@ from pydantic import BaseModel -from dynamo.planner.config.defaults import TargetReplica +from dingo.planner.config.defaults import TargetReplica class ScaleStatus(str, Enum): diff --git a/components/src/dynamo/planner/connectors/remote_client.py b/dingo/planner/connectors/remote_client.py similarity index 97% rename from components/src/dynamo/planner/connectors/remote_client.py rename to dingo/planner/connectors/remote_client.py index c489cb7b4754..7a384621d236 100644 --- a/components/src/dynamo/planner/connectors/remote_client.py +++ b/dingo/planner/connectors/remote_client.py @@ -7,8 +7,8 @@ import logging from dynamo._core import Client -from dynamo.planner.config.defaults import SubComponentType -from dynamo.planner.connectors.protocol import ScaleRequest, ScaleResponse +from dingo.planner.config.defaults import SubComponentType +from dingo.planner.connectors.protocol import ScaleRequest, ScaleResponse from dynamo.runtime import DistributedRuntime logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/planner/connectors/virtual.py b/dingo/planner/connectors/virtual.py similarity index 96% rename from components/src/dynamo/planner/connectors/virtual.py rename to dingo/planner/connectors/virtual.py index e3b2dfc3dde9..fb716b22fc51 100644 --- a/components/src/dynamo/planner/connectors/virtual.py +++ b/dingo/planner/connectors/virtual.py @@ -7,11 +7,11 @@ from typing import TYPE_CHECKING, Optional from dynamo._core import VirtualConnectorCoordinator -from dynamo.planner.config.defaults import SubComponentType, TargetReplica -from dynamo.planner.connectors.base import PlannerConnector -from dynamo.planner.connectors.mdc import MdcEntry, select_entry, worker_info_from_mdc -from dynamo.planner.errors import EmptyTargetReplicasError -from dynamo.planner.monitoring.worker_info import ( +from dingo.planner.config.defaults import SubComponentType, TargetReplica +from dingo.planner.connectors.base import PlannerConnector +from dingo.planner.connectors.mdc import MdcEntry, select_entry, worker_info_from_mdc +from dingo.planner.errors import EmptyTargetReplicasError +from dingo.planner.monitoring.worker_info import ( WorkerInfo, build_worker_info_from_defaults, ) diff --git a/components/src/dynamo/planner/core/__init__.py b/dingo/planner/core/__init__.py similarity index 84% rename from components/src/dynamo/planner/core/__init__.py rename to dingo/planner/core/__init__.py index 63b2bb13fde8..f1e36290bc5a 100644 --- a/components/src/dynamo/planner/core/__init__.py +++ b/dingo/planner/core/__init__.py @@ -1,8 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from dynamo.planner.core.state_machine import PlannerScalingState -from dynamo.planner.core.types import ( +from dingo.planner.core.state_machine import PlannerScalingState +from dingo.planner.core.types import ( EngineCapabilities, FpmObservations, PlannerEffects, diff --git a/components/src/dynamo/planner/core/adapters.py b/dingo/planner/core/adapters.py similarity index 97% rename from components/src/dynamo/planner/core/adapters.py rename to dingo/planner/core/adapters.py index b693b4a20ec1..26b78e91c904 100644 --- a/components/src/dynamo/planner/core/adapters.py +++ b/dingo/planner/core/adapters.py @@ -10,10 +10,10 @@ import logging -from dynamo.planner.config.defaults import SubComponentType, TargetReplica -from dynamo.planner.core.base import NativePlannerBase -from dynamo.planner.core.types import PlannerEffects -from dynamo.planner.monitoring.perf_metrics import ( +from dingo.planner.config.defaults import SubComponentType, TargetReplica +from dingo.planner.core.base import NativePlannerBase +from dingo.planner.core.types import PlannerEffects +from dingo.planner.monitoring.perf_metrics import ( PreDeploymentMetricsUnavailableError, fetch_pre_deployment_metrics, ) diff --git a/components/src/dynamo/planner/core/base.py b/dingo/planner/core/base.py similarity index 96% rename from components/src/dynamo/planner/core/base.py rename to dingo/planner/core/base.py index 1b0ac92b213d..f35925b1e6f2 100644 --- a/components/src/dynamo/planner/core/base.py +++ b/dingo/planner/core/base.py @@ -6,7 +6,7 @@ This module contains **zero decision logic**. It only gathers data from the outside world (Prometheus, FPM subscribers, K8s connectors) and applies scaling decisions back. All scaling logic lives in -:class:`~dynamo.planner.core.state_machine.PlannerScalingState` and is exposed +:class:`~dingo.planner.core.state_machine.PlannerScalingState` and is exposed through builtin orchestrator plugins. Subclasses (PrefillPlanner, DecodePlanner, AggPlanner, DisaggPlanner) set @@ -26,15 +26,15 @@ from kubernetes.config.config_exception import ConfigException from prometheus_client import start_http_server -from dynamo.planner.config.backend_components import WORKER_COMPONENT_NAMES -from dynamo.planner.config.defaults import SubComponentType, TargetReplica -from dynamo.planner.config.planner_config import PlannerConfig -from dynamo.planner.connectors.global_planner import GlobalPlannerConnector -from dynamo.planner.connectors.kubernetes import KubernetesConnector -from dynamo.planner.connectors.virtual import VirtualConnector -from dynamo.planner.core.budget import _initialize_gpu_counts -from dynamo.planner.core.engine_protocol import EngineProtocol -from dynamo.planner.core.types import ( +from dingo.planner.config.backend_components import WORKER_COMPONENT_NAMES +from dingo.planner.config.defaults import SubComponentType, TargetReplica +from dingo.planner.config.planner_config import PlannerConfig +from dingo.planner.connectors.global_planner import GlobalPlannerConnector +from dingo.planner.connectors.kubernetes import KubernetesConnector +from dingo.planner.connectors.virtual import VirtualConnector +from dingo.planner.core.budget import _initialize_gpu_counts +from dingo.planner.core.engine_protocol import EngineProtocol +from dingo.planner.core.types import ( EngineCapabilities, FpmObservations, PlannerEffects, @@ -45,16 +45,16 @@ WorkerCapabilities, WorkerCounts, ) -from dynamo.planner.errors import PlannerError -from dynamo.planner.monitoring.diagnostics_recorder import DiagnosticsRecorder -from dynamo.planner.monitoring.live_dashboard import start_live_dashboard -from dynamo.planner.monitoring.planner_metrics import PlannerPrometheusMetrics -from dynamo.planner.monitoring.traffic_metrics import Metrics, PrometheusAPIClient -from dynamo.planner.monitoring.worker_info import WorkerInfo, resolve_worker_info -from dynamo.planner.offline.trace_data import extract_metrics_from_mooncake +from dingo.planner.errors import PlannerError +from dingo.planner.monitoring.diagnostics_recorder import DiagnosticsRecorder +from dingo.planner.monitoring.live_dashboard import start_live_dashboard +from dingo.planner.monitoring.planner_metrics import PlannerPrometheusMetrics +from dingo.planner.monitoring.traffic_metrics import Metrics, PrometheusAPIClient +from dingo.planner.monitoring.worker_info import WorkerInfo, resolve_worker_info +from dingo.planner.offline.trace_data import extract_metrics_from_mooncake if TYPE_CHECKING: - from dynamo.common.forward_pass_metrics import ForwardPassMetrics + from dingo.common.forward_pass_metrics import ForwardPassMetrics from dynamo.llm import FpmEventSubscriber from dynamo.runtime import DistributedRuntime @@ -228,7 +228,7 @@ async def _install_benchmark_fpms( pass one or both depending on ``fetch_pre_deployment_metrics`` outcomes; AggPlanner passes ``agg_fpms``. """ - from dynamo.planner.plugins.orchestrator.engine_adapter import ( + from dingo.planner.plugins.orchestrator.engine_adapter import ( OrchestratorEngineAdapter, ) @@ -250,7 +250,7 @@ def _ensure_engine(self) -> EngineProtocol: self.prefill_worker_info, self.decode_worker_info, ) - from dynamo.planner.plugins.orchestrator.engine_adapter import ( + from dingo.planner.plugins.orchestrator.engine_adapter import ( OrchestratorEngineAdapter, ) @@ -281,7 +281,7 @@ def _load_predictor_warmup_observations( return None async def _bootstrap_engine_plugins_if_needed(self) -> None: - from dynamo.planner.plugins.orchestrator.engine_adapter import ( + from dingo.planner.plugins.orchestrator.engine_adapter import ( OrchestratorEngineAdapter, ) @@ -534,7 +534,7 @@ def _refresh_worker_info_from_connector(self) -> None: def _decode_fpm_bytes( self, subscriber: Optional[FpmEventSubscriber] ) -> dict[tuple[str, int], ForwardPassMetrics]: - from dynamo.common.forward_pass_metrics import decode as decode_fpm + from dingo.common.forward_pass_metrics import decode as decode_fpm if subscriber is None: return {} diff --git a/components/src/dynamo/planner/core/budget.py b/dingo/planner/core/budget.py similarity index 99% rename from components/src/dynamo/planner/core/budget.py rename to dingo/planner/core/budget.py index 3fd1b7942b85..31098abbb895 100644 --- a/components/src/dynamo/planner/core/budget.py +++ b/dingo/planner/core/budget.py @@ -32,8 +32,8 @@ import math from typing import Iterable -from dynamo.planner.config.planner_config import PlannerConfig -from dynamo.planner.errors import DeploymentValidationError +from dingo.planner.config.planner_config import PlannerConfig +from dingo.planner.errors import DeploymentValidationError from dynamo.runtime.logging import configure_dynamo_logging configure_dynamo_logging() diff --git a/components/src/dynamo/planner/core/engine_protocol.py b/dingo/planner/core/engine_protocol.py similarity index 94% rename from components/src/dynamo/planner/core/engine_protocol.py rename to dingo/planner/core/engine_protocol.py index 479852e1f5ce..252b7f33d164 100644 --- a/components/src/dynamo/planner/core/engine_protocol.py +++ b/dingo/planner/core/engine_protocol.py @@ -15,7 +15,7 @@ from typing import Protocol, runtime_checkable -from dynamo.planner.core.types import PlannerEffects, ScheduledTick, TickInput +from dingo.planner.core.types import PlannerEffects, ScheduledTick, TickInput @runtime_checkable diff --git a/components/src/dynamo/planner/core/load/__init__.py b/dingo/planner/core/load/__init__.py similarity index 100% rename from components/src/dynamo/planner/core/load/__init__.py rename to dingo/planner/core/load/__init__.py diff --git a/components/src/dynamo/planner/core/load/predictors.py b/dingo/planner/core/load/predictors.py similarity index 99% rename from components/src/dynamo/planner/core/load/predictors.py rename to dingo/planner/core/load/predictors.py index b139e2ce4784..99fc1eef023a 100644 --- a/components/src/dynamo/planner/core/load/predictors.py +++ b/dingo/planner/core/load/predictors.py @@ -27,7 +27,7 @@ from filterpy.kalman import KalmanFilter from prophet import Prophet -from dynamo.planner.config.planner_config import PlannerConfig +from dingo.planner.config.planner_config import PlannerConfig from dynamo.runtime.logging import configure_dynamo_logging configure_dynamo_logging() diff --git a/components/src/dynamo/planner/core/load_scaling.py b/dingo/planner/core/load_scaling.py similarity index 99% rename from components/src/dynamo/planner/core/load_scaling.py rename to dingo/planner/core/load_scaling.py index 769b322a5124..d47465b09f72 100644 --- a/components/src/dynamo/planner/core/load_scaling.py +++ b/dingo/planner/core/load_scaling.py @@ -13,10 +13,10 @@ import logging from typing import TYPE_CHECKING, Optional -from dynamo.planner.core.types import FpmObservations, ScalingDecision +from dingo.planner.core.types import FpmObservations, ScalingDecision if TYPE_CHECKING: - from dynamo.common.forward_pass_metrics import ForwardPassMetrics + from dingo.common.forward_pass_metrics import ForwardPassMetrics logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/planner/core/perf_model/__init__.py b/dingo/planner/core/perf_model/__init__.py similarity index 57% rename from components/src/dynamo/planner/core/perf_model/__init__.py rename to dingo/planner/core/perf_model/__init__.py index 0be96c0b465f..ed8a5ce7743c 100644 --- a/components/src/dynamo/planner/core/perf_model/__init__.py +++ b/dingo/planner/core/perf_model/__init__.py @@ -1,10 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from dynamo.planner.core.perf_model.agg import AggRegressionModel -from dynamo.planner.core.perf_model.decode import DecodeRegressionModel -from dynamo.planner.core.perf_model.prefill import PrefillRegressionModel -from dynamo.planner.core.perf_model.rust_adapter import ( +from dingo.planner.core.perf_model.agg import AggRegressionModel +from dingo.planner.core.perf_model.decode import DecodeRegressionModel +from dingo.planner.core.perf_model.prefill import PrefillRegressionModel +from dingo.planner.core.perf_model.rust_adapter import ( PlannerEngineCapacity, PlannerEnginePerfModel, ) diff --git a/components/src/dynamo/planner/core/perf_model/agg.py b/dingo/planner/core/perf_model/agg.py similarity index 99% rename from components/src/dynamo/planner/core/perf_model/agg.py rename to dingo/planner/core/perf_model/agg.py index 2c831c948f32..9057aa392d90 100644 --- a/components/src/dynamo/planner/core/perf_model/agg.py +++ b/dingo/planner/core/perf_model/agg.py @@ -12,8 +12,8 @@ import numpy as np -from dynamo.common.forward_pass_metrics import ForwardPassMetrics -from dynamo.planner.core.perf_model.base import ( +from dingo.common.forward_pass_metrics import ForwardPassMetrics +from dingo.planner.core.perf_model.base import ( _BaseRegressionModel, _clamp_kv_hit_rate, _MovingAverage, diff --git a/components/src/dynamo/planner/core/perf_model/base.py b/dingo/planner/core/perf_model/base.py similarity index 99% rename from components/src/dynamo/planner/core/perf_model/base.py rename to dingo/planner/core/perf_model/base.py index e223409b842f..20bcb87c4fa8 100644 --- a/components/src/dynamo/planner/core/perf_model/base.py +++ b/dingo/planner/core/perf_model/base.py @@ -16,7 +16,7 @@ import numpy as np from sklearn.linear_model import LinearRegression -from dynamo.common.forward_pass_metrics import ForwardPassMetrics +from dingo.common.forward_pass_metrics import ForwardPassMetrics logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/planner/core/perf_model/decode.py b/dingo/planner/core/perf_model/decode.py similarity index 97% rename from components/src/dynamo/planner/core/perf_model/decode.py rename to dingo/planner/core/perf_model/decode.py index ba23a33a28c9..0f218073b3f6 100644 --- a/components/src/dynamo/planner/core/perf_model/decode.py +++ b/dingo/planner/core/perf_model/decode.py @@ -11,8 +11,8 @@ import numpy as np -from dynamo.common.forward_pass_metrics import ForwardPassMetrics -from dynamo.planner.core.perf_model.base import _BaseRegressionModel, _MovingAverage +from dingo.common.forward_pass_metrics import ForwardPassMetrics +from dingo.planner.core.perf_model.base import _BaseRegressionModel, _MovingAverage logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/planner/core/perf_model/prefill.py b/dingo/planner/core/perf_model/prefill.py similarity index 98% rename from components/src/dynamo/planner/core/perf_model/prefill.py rename to dingo/planner/core/perf_model/prefill.py index 073203947718..53a285dc24c7 100644 --- a/components/src/dynamo/planner/core/perf_model/prefill.py +++ b/dingo/planner/core/perf_model/prefill.py @@ -12,8 +12,8 @@ import numpy as np -from dynamo.common.forward_pass_metrics import ForwardPassMetrics -from dynamo.planner.core.perf_model.base import ( +from dingo.common.forward_pass_metrics import ForwardPassMetrics +from dingo.planner.core.perf_model.base import ( _BaseRegressionModel, _clamp_kv_hit_rate, _MovingAverage, diff --git a/components/src/dynamo/planner/core/perf_model/rust_adapter.py b/dingo/planner/core/perf_model/rust_adapter.py similarity index 97% rename from components/src/dynamo/planner/core/perf_model/rust_adapter.py rename to dingo/planner/core/perf_model/rust_adapter.py index 87d721967d8f..e2d8678e5acc 100644 --- a/components/src/dynamo/planner/core/perf_model/rust_adapter.py +++ b/dingo/planner/core/perf_model/rust_adapter.py @@ -17,16 +17,16 @@ from dataclasses import dataclass from typing import Any, Optional -from dynamo.common.forward_pass_metrics import ( +from dingo.common.forward_pass_metrics import ( FPM_VERSION, ForwardPassMetrics, QueuedRequestMetrics, ScheduledRequestMetrics, ) -from dynamo.planner.config.parallelization import PickedParallelConfig -from dynamo.planner.config.planner_config import AICPerfModelSpec, PlannerConfig -from dynamo.planner.core.perf_model.base import _clamp_kv_hit_rate -from dynamo.planner.core.types import EngineCapabilities +from dingo.planner.config.parallelization import PickedParallelConfig +from dingo.planner.config.planner_config import AICPerfModelSpec, PlannerConfig +from dingo.planner.core.perf_model.base import _clamp_kv_hit_rate +from dingo.planner.core.types import EngineCapabilities logger = logging.getLogger(__name__) @@ -38,12 +38,12 @@ RustEnginePerfOptions: Any = None try: # pragma: no cover - availability depends on the optional Rust feature. - from dynamo.mocker import AicEngineConfig as _AicEngineConfig - from dynamo.mocker import EngineCapacityRequest as _EngineCapacityRequest - from dynamo.mocker import EnginePerfLimits as _EnginePerfLimits - from dynamo.mocker import OptimizationTarget as _OptimizationTarget - from dynamo.mocker import RustEnginePerfModel as _RustEnginePerfModel - from dynamo.mocker import RustEnginePerfOptions as _RustEnginePerfOptions + from dingo.mocker import AicEngineConfig as _AicEngineConfig + from dingo.mocker import EngineCapacityRequest as _EngineCapacityRequest + from dingo.mocker import EnginePerfLimits as _EnginePerfLimits + from dingo.mocker import OptimizationTarget as _OptimizationTarget + from dingo.mocker import RustEnginePerfModel as _RustEnginePerfModel + from dingo.mocker import RustEnginePerfOptions as _RustEnginePerfOptions AicEngineConfig = _AicEngineConfig EngineCapacityRequest = _EngineCapacityRequest diff --git a/components/src/dynamo/planner/core/state_machine.py b/dingo/planner/core/state_machine.py similarity index 98% rename from components/src/dynamo/planner/core/state_machine.py rename to dingo/planner/core/state_machine.py index 8b9b24ca0278..359408a9fc85 100644 --- a/components/src/dynamo/planner/core/state_machine.py +++ b/dingo/planner/core/state_machine.py @@ -22,15 +22,15 @@ import math from typing import TYPE_CHECKING, Optional -from dynamo.planner.config.planner_config import PlannerConfig -from dynamo.planner.core.budget import ( +from dingo.planner.config.planner_config import PlannerConfig +from dingo.planner.core.budget import ( proportional_clamp_pair, proportional_clamp_single, ) -from dynamo.planner.core.load_scaling import LoadScalingMixin -from dynamo.planner.core.perf_model import PlannerEnginePerfModel -from dynamo.planner.core.throughput_scaling import ThroughputScalingMixin -from dynamo.planner.core.types import ( +from dingo.planner.core.load_scaling import LoadScalingMixin +from dingo.planner.core.perf_model import PlannerEnginePerfModel +from dingo.planner.core.throughput_scaling import ThroughputScalingMixin +from dingo.planner.core.types import ( FpmObservations, ScalingDecision, TickDiagnostics, @@ -40,7 +40,7 @@ ) if TYPE_CHECKING: - from dynamo.common.forward_pass_metrics import ForwardPassMetrics + from dingo.common.forward_pass_metrics import ForwardPassMetrics logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/planner/core/throughput_scaling.py b/dingo/planner/core/throughput_scaling.py similarity index 99% rename from components/src/dynamo/planner/core/throughput_scaling.py rename to dingo/planner/core/throughput_scaling.py index 980a819f338b..bc9039d05f07 100644 --- a/components/src/dynamo/planner/core/throughput_scaling.py +++ b/dingo/planner/core/throughput_scaling.py @@ -14,7 +14,7 @@ import math from typing import Optional -from dynamo.planner.core.types import ScalingDecision +from dingo.planner.core.types import ScalingDecision logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/planner/core/types.py b/dingo/planner/core/types.py similarity index 99% rename from components/src/dynamo/planner/core/types.py rename to dingo/planner/core/types.py index ec73d8141df3..58091bd35693 100644 --- a/components/src/dynamo/planner/core/types.py +++ b/dingo/planner/core/types.py @@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: - from dynamo.common.forward_pass_metrics import ForwardPassMetrics + from dingo.common.forward_pass_metrics import ForwardPassMetrics @dataclass diff --git a/components/src/dynamo/planner/errors.py b/dingo/planner/errors.py similarity index 100% rename from components/src/dynamo/planner/errors.py rename to dingo/planner/errors.py diff --git a/components/src/dynamo/planner/examples/__init__.py b/dingo/planner/examples/__init__.py similarity index 100% rename from components/src/dynamo/planner/examples/__init__.py rename to dingo/planner/examples/__init__.py diff --git a/components/src/dynamo/planner/examples/external_plugin/README.md b/dingo/planner/examples/external_plugin/README.md similarity index 97% rename from components/src/dynamo/planner/examples/external_plugin/README.md rename to dingo/planner/examples/external_plugin/README.md index bd4f97055251..e52e51049a02 100644 --- a/components/src/dynamo/planner/examples/external_plugin/README.md +++ b/dingo/planner/examples/external_plugin/README.md @@ -24,7 +24,7 @@ can stand in for any stage in a smoke deployment. ## Run locally ```bash -python -m dynamo.planner.examples.external_plugin.reference_runner \ +python -m dingo.planner.examples.external_plugin.reference_runner \ --listen=0.0.0.0:9099 \ --stage=predict \ --plugin-id=ext-predict \ diff --git a/components/src/dynamo/planner/examples/external_plugin/__init__.py b/dingo/planner/examples/external_plugin/__init__.py similarity index 100% rename from components/src/dynamo/planner/examples/external_plugin/__init__.py rename to dingo/planner/examples/external_plugin/__init__.py diff --git a/components/src/dynamo/planner/examples/external_plugin/reference_runner.py b/dingo/planner/examples/external_plugin/reference_runner.py similarity index 98% rename from components/src/dynamo/planner/examples/external_plugin/reference_runner.py rename to dingo/planner/examples/external_plugin/reference_runner.py index 3ce768dbbf3c..e5a49341082f 100644 --- a/components/src/dynamo/planner/examples/external_plugin/reference_runner.py +++ b/dingo/planner/examples/external_plugin/reference_runner.py @@ -46,8 +46,8 @@ import grpc -from dynamo.planner.plugins.proto.v1 import plugin_pb2 as pb -from dynamo.planner.plugins.proto.v1 import plugin_pb2_grpc as pbg +from dingo.planner.plugins.proto.v1 import plugin_pb2 as pb +from dingo.planner.plugins.proto.v1 import plugin_pb2_grpc as pbg # --------------------------------------------------------------------------- # Per-stage Servicer implementations diff --git a/components/src/dynamo/planner/monitoring/__init__.py b/dingo/planner/monitoring/__init__.py similarity index 100% rename from components/src/dynamo/planner/monitoring/__init__.py rename to dingo/planner/monitoring/__init__.py diff --git a/components/src/dynamo/planner/monitoring/aic_estimator.py b/dingo/planner/monitoring/aic_estimator.py similarity index 100% rename from components/src/dynamo/planner/monitoring/aic_estimator.py rename to dingo/planner/monitoring/aic_estimator.py diff --git a/components/src/dynamo/planner/monitoring/aic_interpolation.py b/dingo/planner/monitoring/aic_interpolation.py similarity index 95% rename from components/src/dynamo/planner/monitoring/aic_interpolation.py rename to dingo/planner/monitoring/aic_interpolation.py index a472b85419c4..85ee7cf0a547 100644 --- a/components/src/dynamo/planner/monitoring/aic_interpolation.py +++ b/dingo/planner/monitoring/aic_interpolation.py @@ -35,13 +35,13 @@ import logging -from dynamo.common.forward_pass_metrics import ( +from dingo.common.forward_pass_metrics import ( ForwardPassMetrics, ScheduledRequestMetrics, ) -from dynamo.planner.config.aic_interpolation_spec import AICInterpolationSpec -from dynamo.planner.config.defaults import SubComponentType -from dynamo.planner.config.parallelization import ( +from dingo.planner.config.aic_interpolation_spec import AICInterpolationSpec +from dingo.planner.config.defaults import SubComponentType +from dingo.planner.config.parallelization import ( PickedParallelConfig, picked_to_aic_model_config_kwargs, ) @@ -49,7 +49,7 @@ # aic_estimator itself lazy-imports aiconfigurator, so importing the wrapper # class at module load time does NOT pull in the optional dependency — # ImportError only materialises when the class is instantiated. -from dynamo.planner.monitoring.aic_estimator import AIConfiguratorPerfEstimator +from dingo.planner.monitoring.aic_estimator import AIConfiguratorPerfEstimator logger = logging.getLogger(__name__) @@ -65,7 +65,7 @@ def run_aic_interpolation( Lazy-imports ``aiconfigurator`` — callers should catch ``ImportError`` and fall back to the file-based loader if the dependency is missing. """ - from dynamo.planner.monitoring.aic_estimator import AIConfiguratorPerfEstimator + from dingo.planner.monitoring.aic_estimator import AIConfiguratorPerfEstimator estimator = AIConfiguratorPerfEstimator( hf_id=spec.hf_id, diff --git a/components/src/dynamo/planner/monitoring/dgd_services.py b/dingo/planner/monitoring/dgd_services.py similarity index 97% rename from components/src/dynamo/planner/monitoring/dgd_services.py rename to dingo/planner/monitoring/dgd_services.py index 7b599755e75d..338012ba8843 100644 --- a/components/src/dynamo/planner/monitoring/dgd_services.py +++ b/dingo/planner/monitoring/dgd_services.py @@ -19,9 +19,9 @@ from pydantic import BaseModel -from dynamo.common.utils.runtime import parse_endpoint -from dynamo.planner.config.defaults import SubComponentType -from dynamo.planner.errors import DuplicateSubComponentError, SubComponentNotFoundError +from dingo.common.utils.runtime import parse_endpoint +from dingo.planner.config.defaults import SubComponentType +from dingo.planner.errors import DuplicateSubComponentError, SubComponentNotFoundError from dynamo.runtime.logging import configure_dynamo_logging configure_dynamo_logging() diff --git a/components/src/dynamo/planner/monitoring/diagnostics_recorder.py b/dingo/planner/monitoring/diagnostics_recorder.py similarity index 99% rename from components/src/dynamo/planner/monitoring/diagnostics_recorder.py rename to dingo/planner/monitoring/diagnostics_recorder.py index 2d7ca08352d4..a05a799728fe 100644 --- a/components/src/dynamo/planner/monitoring/diagnostics_recorder.py +++ b/dingo/planner/monitoring/diagnostics_recorder.py @@ -23,9 +23,9 @@ import plotly.graph_objects as go # type: ignore[import-untyped] from plotly.subplots import make_subplots # type: ignore[import-untyped] -from dynamo.planner.config.planner_config import PlannerConfig -from dynamo.planner.core.types import PlannerEffects, TickDiagnostics, TickInput -from dynamo.planner.monitoring.traffic_metrics import Metrics +from dingo.planner.config.planner_config import PlannerConfig +from dingo.planner.core.types import PlannerEffects, TickDiagnostics, TickInput +from dingo.planner.monitoring.traffic_metrics import Metrics logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/planner/monitoring/live_dashboard.py b/dingo/planner/monitoring/live_dashboard.py similarity index 95% rename from components/src/dynamo/planner/monitoring/live_dashboard.py rename to dingo/planner/monitoring/live_dashboard.py index a68c4d1a3810..6ef9ce928877 100644 --- a/components/src/dynamo/planner/monitoring/live_dashboard.py +++ b/dingo/planner/monitoring/live_dashboard.py @@ -15,7 +15,7 @@ from aiohttp import web if TYPE_CHECKING: - from dynamo.planner.monitoring.diagnostics_recorder import DiagnosticsRecorder + from dingo.planner.monitoring.diagnostics_recorder import DiagnosticsRecorder logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/planner/monitoring/perf_metrics.py b/dingo/planner/monitoring/perf_metrics.py similarity index 97% rename from components/src/dynamo/planner/monitoring/perf_metrics.py rename to dingo/planner/monitoring/perf_metrics.py index f9cea1afec31..2d73d8f62387 100644 --- a/components/src/dynamo/planner/monitoring/perf_metrics.py +++ b/dingo/planner/monitoring/perf_metrics.py @@ -19,13 +19,13 @@ import numpy as np -from dynamo.common.forward_pass_metrics import ( +from dingo.common.forward_pass_metrics import ( ForwardPassMetrics, ScheduledRequestMetrics, ) -from dynamo.planner.config.aic_interpolation_spec import AICInterpolationSpec -from dynamo.planner.config.defaults import SubComponentType -from dynamo.planner.monitoring.worker_info import WorkerInfo +from dingo.planner.config.aic_interpolation_spec import AICInterpolationSpec +from dingo.planner.config.defaults import SubComponentType +from dingo.planner.monitoring.worker_info import WorkerInfo logger = logging.getLogger(__name__) @@ -114,7 +114,7 @@ def _try_aic_interpolation( component_type: SubComponentType, ) -> list[ForwardPassMetrics]: """Delegate to the AIC sweep. Separated so the ImportError is catchable.""" - from dynamo.planner.monitoring.aic_interpolation import run_aic_interpolation + from dingo.planner.monitoring.aic_interpolation import run_aic_interpolation return run_aic_interpolation(aic_spec, component_type) diff --git a/components/src/dynamo/planner/monitoring/planner_metrics.py b/dingo/planner/monitoring/planner_metrics.py similarity index 100% rename from components/src/dynamo/planner/monitoring/planner_metrics.py rename to dingo/planner/monitoring/planner_metrics.py diff --git a/components/src/dynamo/planner/monitoring/traffic_metrics.py b/dingo/planner/monitoring/traffic_metrics.py similarity index 100% rename from components/src/dynamo/planner/monitoring/traffic_metrics.py rename to dingo/planner/monitoring/traffic_metrics.py diff --git a/components/src/dynamo/planner/monitoring/worker_info.py b/dingo/planner/monitoring/worker_info.py similarity index 98% rename from components/src/dynamo/planner/monitoring/worker_info.py rename to dingo/planner/monitoring/worker_info.py index 2f7825f59c70..35dba83a172d 100644 --- a/components/src/dynamo/planner/monitoring/worker_info.py +++ b/dingo/planner/monitoring/worker_info.py @@ -17,8 +17,8 @@ from dataclasses import dataclass from typing import Any, Optional -from dynamo.planner.config.backend_components import WORKER_COMPONENT_NAMES -from dynamo.planner.config.defaults import SubComponentType +from dingo.planner.config.backend_components import WORKER_COMPONENT_NAMES +from dingo.planner.config.defaults import SubComponentType logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/planner/offline/__init__.py b/dingo/planner/offline/__init__.py similarity index 100% rename from components/src/dynamo/planner/offline/__init__.py rename to dingo/planner/offline/__init__.py diff --git a/components/src/dynamo/planner/offline/replay_adapter.py b/dingo/planner/offline/replay_adapter.py similarity index 98% rename from components/src/dynamo/planner/offline/replay_adapter.py rename to dingo/planner/offline/replay_adapter.py index bf416d6a9fd9..1d4e921dd138 100644 --- a/components/src/dynamo/planner/offline/replay_adapter.py +++ b/dingo/planner/offline/replay_adapter.py @@ -37,14 +37,14 @@ from dataclasses import dataclass, field from typing import Any, Optional -from dynamo.common.forward_pass_metrics import ( +from dingo.common.forward_pass_metrics import ( ForwardPassMetrics, QueuedRequestMetrics, ScheduledRequestMetrics, ) -from dynamo.planner.config.planner_config import PlannerConfig -from dynamo.planner.core.engine_protocol import EngineProtocol -from dynamo.planner.core.types import ( +from dingo.planner.config.planner_config import PlannerConfig +from dingo.planner.core.engine_protocol import EngineProtocol +from dingo.planner.core.types import ( FpmObservations, PlannerEffects, ScheduledTick, @@ -54,10 +54,10 @@ WorkerCapabilities, WorkerCounts, ) -from dynamo.planner.monitoring.diagnostics_recorder import DiagnosticsRecorder -from dynamo.planner.monitoring.traffic_metrics import Metrics -from dynamo.planner.plugins.clock import VirtualClock -from dynamo.planner.plugins.orchestrator.engine_adapter import OrchestratorEngineAdapter +from dingo.planner.monitoring.diagnostics_recorder import DiagnosticsRecorder +from dingo.planner.monitoring.traffic_metrics import Metrics +from dingo.planner.plugins.clock import VirtualClock +from dingo.planner.plugins.orchestrator.engine_adapter import OrchestratorEngineAdapter logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/planner/offline/trace_data.py b/dingo/planner/offline/trace_data.py similarity index 100% rename from components/src/dynamo/planner/offline/trace_data.py rename to dingo/planner/offline/trace_data.py diff --git a/components/src/dynamo/planner/plugins/__init__.py b/dingo/planner/plugins/__init__.py similarity index 100% rename from components/src/dynamo/planner/plugins/__init__.py rename to dingo/planner/plugins/__init__.py diff --git a/components/src/dynamo/planner/plugins/_proto_bridge.py b/dingo/planner/plugins/_proto_bridge.py similarity index 99% rename from components/src/dynamo/planner/plugins/_proto_bridge.py rename to dingo/planner/plugins/_proto_bridge.py index cad393c2af45..a1a2459c1c41 100644 --- a/components/src/dynamo/planner/plugins/_proto_bridge.py +++ b/dingo/planner/plugins/_proto_bridge.py @@ -32,8 +32,8 @@ from google.protobuf.message import Message from pydantic import BaseModel -from dynamo.planner.plugins import types as pyd -from dynamo.planner.plugins.proto.v1 import plugin_pb2 as pb +from dingo.planner.plugins import types as pyd +from dingo.planner.plugins.proto.v1 import plugin_pb2 as pb PydT = TypeVar("PydT", bound=BaseModel) diff --git a/components/src/dynamo/planner/plugins/builtins/__init__.py b/dingo/planner/plugins/builtins/__init__.py similarity index 85% rename from components/src/dynamo/planner/plugins/builtins/__init__.py rename to dingo/planner/plugins/builtins/__init__.py index c96af4163c4b..770303982c3c 100644 --- a/components/src/dynamo/planner/plugins/builtins/__init__.py +++ b/dingo/planner/plugins/builtins/__init__.py @@ -3,7 +3,7 @@ """Builtin local-planner plugins.""" -from dynamo.planner.plugins.builtins.local_planner import ( +from dingo.planner.plugins.builtins.local_planner import ( BuiltinLoadPredict, BuiltinLoadPropose, BuiltinThroughputPropose, diff --git a/components/src/dynamo/planner/plugins/builtins/local_planner.py b/dingo/planner/plugins/builtins/local_planner.py similarity index 96% rename from components/src/dynamo/planner/plugins/builtins/local_planner.py rename to dingo/planner/plugins/builtins/local_planner.py index 22f1f40f7a13..ff164706809c 100644 --- a/components/src/dynamo/planner/plugins/builtins/local_planner.py +++ b/dingo/planner/plugins/builtins/local_planner.py @@ -8,18 +8,18 @@ import logging import math -from dynamo.common.forward_pass_metrics import ForwardPassMetrics -from dynamo.common.forward_pass_metrics import decode as decode_fpm -from dynamo.planner.config.planner_config import PlannerConfig -from dynamo.planner.core.load.predictors import LOAD_PREDICTORS -from dynamo.planner.core.state_machine import PlannerScalingState -from dynamo.planner.core.types import ( +from dingo.common.forward_pass_metrics import ForwardPassMetrics +from dingo.common.forward_pass_metrics import decode as decode_fpm +from dingo.planner.config.planner_config import PlannerConfig +from dingo.planner.core.load.predictors import LOAD_PREDICTORS +from dingo.planner.core.state_machine import PlannerScalingState +from dingo.planner.core.types import ( FpmObservations, ScalingDecision, TrafficObservation, WorkerCapabilities, ) -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.types import ( AcceptResult, ComponentTarget, OverrideResult, diff --git a/components/src/dynamo/planner/plugins/clock.py b/dingo/planner/plugins/clock.py similarity index 100% rename from components/src/dynamo/planner/plugins/clock.py rename to dingo/planner/plugins/clock.py diff --git a/components/src/dynamo/planner/plugins/merge/__init__.py b/dingo/planner/plugins/merge/__init__.py similarity index 87% rename from components/src/dynamo/planner/plugins/merge/__init__.py rename to dingo/planner/plugins/merge/__init__.py index 058fbab1debf..b38682123e3e 100644 --- a/components/src/dynamo/planner/plugins/merge/__init__.py +++ b/dingo/planner/plugins/merge/__init__.py @@ -20,9 +20,9 @@ awaits plugin RPCs — the algorithmic logic itself is synchronous. """ -from dynamo.planner.plugins.merge.chain_augment import chain_augment -from dynamo.planner.plugins.merge.type_aware import type_aware_merge -from dynamo.planner.plugins.merge.types import ( +from dingo.planner.plugins.merge.chain_augment import chain_augment +from dingo.planner.plugins.merge.type_aware import type_aware_merge +from dingo.planner.plugins.merge.types import ( ChainAugmentOutcome, ComponentKey, MergeOutcome, diff --git a/components/src/dynamo/planner/plugins/merge/chain_augment.py b/dingo/planner/plugins/merge/chain_augment.py similarity index 98% rename from components/src/dynamo/planner/plugins/merge/chain_augment.py rename to dingo/planner/plugins/merge/chain_augment.py index 3e9b3b9b57dd..63988f46d0ee 100644 --- a/components/src/dynamo/planner/plugins/merge/chain_augment.py +++ b/dingo/planner/plugins/merge/chain_augment.py @@ -74,11 +74,11 @@ import logging from typing import Any, Optional, Sequence -from dynamo.planner.plugins.merge.types import ( +from dingo.planner.plugins.merge.types import ( ChainAugmentOutcome, PredictPluginCallable, ) -from dynamo.planner.plugins.types import PipelineContext, PredictionData +from dingo.planner.plugins.types import PipelineContext, PredictionData log = logging.getLogger(__name__) diff --git a/components/src/dynamo/planner/plugins/merge/type_aware.py b/dingo/planner/plugins/merge/type_aware.py similarity index 98% rename from components/src/dynamo/planner/plugins/merge/type_aware.py rename to dingo/planner/plugins/merge/type_aware.py index 6a6c56cbbaec..8a0c406c72f7 100644 --- a/components/src/dynamo/planner/plugins/merge/type_aware.py +++ b/dingo/planner/plugins/merge/type_aware.py @@ -39,8 +39,8 @@ import math from typing import Mapping, Sequence -from dynamo.planner.plugins.merge.types import ComponentKey, MergeOutcome, PluginResult -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.merge.types import ComponentKey, MergeOutcome, PluginResult +from dingo.planner.plugins.types import ( ComponentTarget, OverrideResult, OverrideType, diff --git a/components/src/dynamo/planner/plugins/merge/types.py b/dingo/planner/plugins/merge/types.py similarity index 99% rename from components/src/dynamo/planner/plugins/merge/types.py rename to dingo/planner/plugins/merge/types.py index 113b859664b1..aaa8a2357a44 100644 --- a/components/src/dynamo/planner/plugins/merge/types.py +++ b/dingo/planner/plugins/merge/types.py @@ -25,7 +25,7 @@ from dataclasses import dataclass, field from typing import Optional, Protocol, Union, runtime_checkable -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.types import ( AcceptResult, OverrideResult, PipelineContext, diff --git a/components/src/dynamo/planner/plugins/orchestrator/__init__.py b/dingo/planner/plugins/orchestrator/__init__.py similarity index 75% rename from components/src/dynamo/planner/plugins/orchestrator/__init__.py rename to dingo/planner/plugins/orchestrator/__init__.py index 27dc47ee0f78..af184adffc19 100644 --- a/components/src/dynamo/planner/plugins/orchestrator/__init__.py +++ b/dingo/planner/plugins/orchestrator/__init__.py @@ -9,8 +9,8 @@ / RECONCILE / CONSTRAIN) per tick and emits an EXECUTE decision. """ -from dynamo.planner.plugins.orchestrator.orchestrator import LocalPlannerOrchestrator -from dynamo.planner.plugins.orchestrator.pipeline import PipelineOutcome, run_pipeline +from dingo.planner.plugins.orchestrator.orchestrator import LocalPlannerOrchestrator +from dingo.planner.plugins.orchestrator.pipeline import PipelineOutcome, run_pipeline __all__ = [ "LocalPlannerOrchestrator", diff --git a/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py b/dingo/planner/plugins/orchestrator/engine_adapter.py similarity index 97% rename from components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py rename to dingo/planner/plugins/orchestrator/engine_adapter.py index 1af513200feb..440b4440774a 100644 --- a/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py +++ b/dingo/planner/plugins/orchestrator/engine_adapter.py @@ -62,17 +62,17 @@ import logging from typing import TYPE_CHECKING, Any, Optional, Sequence -from dynamo.common.forward_pass_metrics import encode as _encode_fpm_record +from dingo.common.forward_pass_metrics import encode as _encode_fpm_record if TYPE_CHECKING: import grpc.aio -from dynamo.planner.core.budget import ( +from dingo.planner.core.budget import ( proportional_clamp_pair, proportional_clamp_single, ) -from dynamo.planner.core.state_machine import PlannerScalingState -from dynamo.planner.core.types import ( +from dingo.planner.core.state_machine import PlannerScalingState +from dingo.planner.core.types import ( FpmObservations, PlannerEffects, ScalingDecision, @@ -83,21 +83,21 @@ WorkerCapabilities, WorkerCounts, ) -from dynamo.planner.plugins.builtins import ( +from dingo.planner.plugins.builtins import ( BuiltinLoadPredict, BuiltinLoadPropose, BuiltinThroughputPropose, ) -from dynamo.planner.plugins.clock import Clock, VirtualClock, WallClock -from dynamo.planner.plugins.merge.types import ComponentKey -from dynamo.planner.plugins.orchestrator.orchestrator import LocalPlannerOrchestrator -from dynamo.planner.plugins.registry.auth import AllowUnauthenticatedAuth -from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker -from dynamo.planner.plugins.registry.config import build_auth_validator -from dynamo.planner.plugins.registry.server import PluginRegistryServer -from dynamo.planner.plugins.scheduler import PluginScheduler -from dynamo.planner.plugins.transport.config import make_transport_for_endpoint -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.clock import Clock, VirtualClock, WallClock +from dingo.planner.plugins.merge.types import ComponentKey +from dingo.planner.plugins.orchestrator.orchestrator import LocalPlannerOrchestrator +from dingo.planner.plugins.registry.auth import AllowUnauthenticatedAuth +from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker +from dingo.planner.plugins.registry.config import build_auth_validator +from dingo.planner.plugins.registry.server import PluginRegistryServer +from dingo.planner.plugins.scheduler import PluginScheduler +from dingo.planner.plugins.transport.config import make_transport_for_endpoint +from dingo.planner.plugins.types import ( FpmData, ObservationData, PipelineContext, @@ -179,7 +179,7 @@ def __init__( # names on REGISTRY and the second would raise "Duplicated # timeseries" — we tolerate that by falling back to ``None`` so # emission becomes a no-op instead of crashing. - from dynamo.planner.monitoring.planner_metrics import PluginFrameworkMetrics + from dingo.planner.monitoring.planner_metrics import PluginFrameworkMetrics self._plugin_framework_metrics: Optional[PluginFrameworkMetrics] try: @@ -379,7 +379,7 @@ def _load_in_process_plugins_from_config(self) -> None: if not specs: self._in_process_plugins_loaded = True return - from dynamo.planner.plugins.orchestrator.in_process_loader import ( + from dingo.planner.plugins.orchestrator.in_process_loader import ( load_in_process_plugins, ) @@ -427,7 +427,7 @@ async def _maybe_start_gateway(self) -> None: return # Local import keeps the gateway module out of the cold-start # import chain for deployments that never enable it. - from dynamo.planner.plugins.registry.gateway import start_gateway_server + from dingo.planner.plugins.registry.gateway import start_gateway_server grpc_server, actual_listen = await start_gateway_server( self._orchestrator.registry, @@ -952,7 +952,7 @@ def _encode_fpm(obs: Optional[FpmObservations]) -> Optional[FpmData]: - per-engine map key = ``f"{worker_id}/{dp_rank}"`` (flat str since proto3 ``map`` can't carry a tuple key) - per-engine map value = msgpack-encoded ``ForwardPassMetrics`` - via the canonical ``dynamo.common.forward_pass_metrics.encode`` + via the canonical ``dingo.common.forward_pass_metrics.encode`` helper (shared module-level encoder) so cross-language plugins decode with any standard msgpack library and the wire format stays in lock-step with the rest of dynamo's FPM serialization. diff --git a/components/src/dynamo/planner/plugins/orchestrator/in_process_loader.py b/dingo/planner/plugins/orchestrator/in_process_loader.py similarity index 93% rename from components/src/dynamo/planner/plugins/orchestrator/in_process_loader.py rename to dingo/planner/plugins/orchestrator/in_process_loader.py index 33e29ae0f69b..e81d1390e287 100644 --- a/components/src/dynamo/planner/plugins/orchestrator/in_process_loader.py +++ b/dingo/planner/plugins/orchestrator/in_process_loader.py @@ -18,9 +18,9 @@ import logging from typing import Sequence -from dynamo.planner.plugins.orchestrator.orchestrator import LocalPlannerOrchestrator -from dynamo.planner.plugins.registry.config import InProcessPluginSpec -from dynamo.planner.plugins.types import HoldPolicy +from dingo.planner.plugins.orchestrator.orchestrator import LocalPlannerOrchestrator +from dingo.planner.plugins.registry.config import InProcessPluginSpec +from dingo.planner.plugins.types import HoldPolicy log = logging.getLogger(__name__) diff --git a/components/src/dynamo/planner/plugins/orchestrator/orchestrator.py b/dingo/planner/plugins/orchestrator/orchestrator.py similarity index 95% rename from components/src/dynamo/planner/plugins/orchestrator/orchestrator.py rename to dingo/planner/plugins/orchestrator/orchestrator.py index 9649e58703b1..a45911e0fb40 100644 --- a/components/src/dynamo/planner/plugins/orchestrator/orchestrator.py +++ b/dingo/planner/plugins/orchestrator/orchestrator.py @@ -34,17 +34,17 @@ import logging from typing import Any, Mapping, Optional, Sequence -from dynamo.planner.core.types import TrafficObservation, WorkerCapabilities -from dynamo.planner.monitoring.planner_metrics import PluginFrameworkMetrics -from dynamo.planner.plugins.clock import Clock -from dynamo.planner.plugins.merge.types import ComponentKey -from dynamo.planner.plugins.orchestrator.pipeline import PipelineOutcome, run_pipeline -from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker -from dynamo.planner.plugins.registry.server import PluginRegistryServer -from dynamo.planner.plugins.registry.types import RegisteredPlugin -from dynamo.planner.plugins.scheduler import PluginScheduler -from dynamo.planner.plugins.transport.errors import PluginUnknownMethodError -from dynamo.planner.plugins.types import ( +from dingo.planner.core.types import TrafficObservation, WorkerCapabilities +from dingo.planner.monitoring.planner_metrics import PluginFrameworkMetrics +from dingo.planner.plugins.clock import Clock +from dingo.planner.plugins.merge.types import ComponentKey +from dingo.planner.plugins.orchestrator.pipeline import PipelineOutcome, run_pipeline +from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker +from dingo.planner.plugins.registry.server import PluginRegistryServer +from dingo.planner.plugins.registry.types import RegisteredPlugin +from dingo.planner.plugins.scheduler import PluginScheduler +from dingo.planner.plugins.transport.errors import PluginUnknownMethodError +from dingo.planner.plugins.types import ( BootstrapRequest, HoldPolicy, ListPluginsRequest, diff --git a/components/src/dynamo/planner/plugins/orchestrator/pipeline.py b/dingo/planner/plugins/orchestrator/pipeline.py similarity index 97% rename from components/src/dynamo/planner/plugins/orchestrator/pipeline.py rename to dingo/planner/plugins/orchestrator/pipeline.py index 65475b1e7572..4a76f7567554 100644 --- a/components/src/dynamo/planner/plugins/orchestrator/pipeline.py +++ b/dingo/planner/plugins/orchestrator/pipeline.py @@ -44,9 +44,9 @@ from dataclasses import dataclass, field from typing import Literal, Mapping, Optional -from dynamo.planner.monitoring.planner_metrics import PluginFrameworkMetrics -from dynamo.planner.plugins.clock import Clock -from dynamo.planner.plugins.merge import ( +from dingo.planner.monitoring.planner_metrics import PluginFrameworkMetrics +from dingo.planner.plugins.clock import Clock +from dingo.planner.plugins.merge import ( ChainAugmentOutcome, ComponentKey, MergeOutcome, @@ -54,11 +54,11 @@ chain_augment, type_aware_merge, ) -from dynamo.planner.plugins.merge.types import PredictPluginCallable -from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker, CircuitState -from dynamo.planner.plugins.registry.types import RegisteredPlugin -from dynamo.planner.plugins.scheduler import PluginScheduler -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.merge.types import PredictPluginCallable +from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker, CircuitState +from dingo.planner.plugins.registry.types import RegisteredPlugin +from dingo.planner.plugins.scheduler import PluginScheduler +from dingo.planner.plugins.types import ( AcceptResult, ComponentTarget, ConstrainStageRequest, @@ -548,9 +548,9 @@ def _result_label(pr: PluginResult) -> str: need the full mix should sum ``plugin_override_active{override_type=...}`` instead. """ - from dynamo.planner.plugins.types import AcceptResult as _AcceptResult - from dynamo.planner.plugins.types import OverrideResult as _OverrideResult - from dynamo.planner.plugins.types import RejectResult as _RejectResult + from dingo.planner.plugins.types import AcceptResult as _AcceptResult + from dingo.planner.plugins.types import OverrideResult as _OverrideResult + from dingo.planner.plugins.types import RejectResult as _RejectResult r = pr.result if isinstance(r, _RejectResult): @@ -586,7 +586,7 @@ def _set_circuit_state( auto-transition OPEN → HALF_OPEN after cooldown) and pins the gauge so dashboards display the live view even for plugins not evaluated this tick.""" - from dynamo.planner.monitoring.planner_metrics import ( + from dingo.planner.monitoring.planner_metrics import ( CIRCUIT_STATE_CLOSED, CIRCUIT_STATE_HALF_OPEN, CIRCUIT_STATE_OPEN, @@ -624,8 +624,8 @@ def _emit_override_active( Plugins that returned ACCEPT or REJECT-but-not-winning leave the gauge at all-zero — that's the correct "evaluated, no override" state.""" - from dynamo.planner.plugins.types import OverrideResult as _OverrideResult - from dynamo.planner.plugins.types import RejectResult as _RejectResult + from dingo.planner.plugins.types import OverrideResult as _OverrideResult + from dingo.planner.plugins.types import RejectResult as _RejectResult # Reset every plugin we ATTEMPTED this tick (triggered + inherited), # not just those that produced a result. A plugin whose call raised is @@ -687,7 +687,7 @@ def _emit_clamps_and_rejects( the counter tracks "REJECT happened", independent of what the orchestrator does next. """ - from dynamo.planner.plugins.types import RejectResult as _RejectResult + from dingo.planner.plugins.types import RejectResult as _RejectResult # -- clamp counters ---------------------------------------------------- clamp_counter = None diff --git a/components/src/dynamo/planner/plugins/proto/__init__.py b/dingo/planner/plugins/proto/__init__.py similarity index 100% rename from components/src/dynamo/planner/plugins/proto/__init__.py rename to dingo/planner/plugins/proto/__init__.py diff --git a/components/src/dynamo/planner/plugins/proto/v1/README.md b/dingo/planner/plugins/proto/v1/README.md similarity index 96% rename from components/src/dynamo/planner/plugins/proto/v1/README.md rename to dingo/planner/plugins/proto/v1/README.md index 3e159af99cf5..753ccdca80bc 100644 --- a/components/src/dynamo/planner/plugins/proto/v1/README.md +++ b/dingo/planner/plugins/proto/v1/README.md @@ -48,22 +48,21 @@ Generated stubs (`plugin_pb2.py`, `plugin_pb2_grpc.py`, `plugin_pb2.pyi`) are **checked into git** so that test/build environments don't need `grpcio-tools` installed just to import the module. The repo-wide `.gitignore` excludes `*_pb2.py` / `*_pb2.pyi` for other consumers; the -planner stubs are explicitly negated with `!components/src/dynamo/planner/plugins/proto/v1/*`. +planner stubs are explicitly negated with `!dingo/planner/plugins/proto/v1/*`. Regenerate locally when you edit `plugin.proto`: ```bash -# Regenerate all three stubs (run from components/src/) -cd components/src +# Regenerate all three stubs (run from the repository root) python -m grpc_tools.protoc \ --python_out=. --grpc_python_out=. --pyi_out=. --proto_path=. \ - dynamo/planner/plugins/proto/v1/plugin.proto + dingo/planner/plugins/proto/v1/plugin.proto # protoc strips the SPDX header — re-prepend it on generated Python stubs. SPDX=$'# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n# SPDX-License-Identifier: Apache-2.0\n' -for f in dynamo/planner/plugins/proto/v1/plugin_pb2.py \ - dynamo/planner/plugins/proto/v1/plugin_pb2_grpc.py \ - dynamo/planner/plugins/proto/v1/plugin_pb2.pyi; do +for f in dingo/planner/plugins/proto/v1/plugin_pb2.py \ + dingo/planner/plugins/proto/v1/plugin_pb2_grpc.py \ + dingo/planner/plugins/proto/v1/plugin_pb2.pyi; do printf '%s%s' "$SPDX" "$(cat "$f")" > "$f" done ``` @@ -225,7 +224,7 @@ is higher priority than authority override. `FpmData.prefill_engines` / `decode_engines` are `map`. Each value is a **msgspec/msgpack-encoded** `ForwardPassMetrics` record (see -`dynamo.common.forward_pass_metrics`). Wire format is standard msgpack, so +`dingo.common.forward_pass_metrics`). Wire format is standard msgpack, so cross-language plugins decode with any msgpack library (Go's vmihailenco/msgpack, Rust's rmp-serde, JS @msgpack/msgpack, etc.) plus knowledge of the `ForwardPassMetrics` struct layout. diff --git a/components/src/dynamo/planner/plugins/proto/v1/__init__.py b/dingo/planner/plugins/proto/v1/__init__.py similarity index 95% rename from components/src/dynamo/planner/plugins/proto/v1/__init__.py rename to dingo/planner/plugins/proto/v1/__init__.py index 03c6245b0a8c..30733da5fcfa 100644 --- a/components/src/dynamo/planner/plugins/proto/v1/__init__.py +++ b/dingo/planner/plugins/proto/v1/__init__.py @@ -16,7 +16,7 @@ # CI's pre-commit ``pytest-marker-report`` collects test modules without # running their bodies. Several tests import ``plugin_pb2`` / # ``plugin_pb2_grpc`` at module top via ``from -# dynamo.planner.plugins.proto.v1 import plugin_pb2``. When the generated +# dingo.planner.plugins.proto.v1 import plugin_pb2``. When the generated # stubs are *not on disk* (which is the case in a fresh pre-commit virtualenv # that hasn't yet executed the container build's ``protoc`` step), that # import raises ``ImportError`` and the test fails *collection*, taking the @@ -50,7 +50,7 @@ def __getattr__(self, name: str): # type: ignore[no-untyped-def] raise AttributeError(name) msg = ( f"{self.__name__}.{name} not available — generated proto stub " - "missing. Run protoc per components/src/dynamo/planner/plugins/" + "missing. Run protoc per dingo/planner/plugins/" "proto/v1/README.md to generate plugin_pb2.py / plugin_pb2_grpc.py." ) dummy = type(name, (), {"__init__": lambda self, *a, **kw: (_ for _ in ()).throw(RuntimeError(msg))}) # type: ignore[arg-type] @@ -82,7 +82,7 @@ def __getattr__(self, name: str): # type: ignore[no-untyped-def] _placeholder = _PlaceholderModule(_fq) _placeholder.__doc__ = ( f"Pre-generation placeholder for {_fq}. Run protoc per " - "``components/src/dynamo/planner/plugins/proto/v1/README.md`` " + "``dingo/planner/plugins/proto/v1/README.md`` " "to generate the real module." ) sys.modules[_fq] = _placeholder diff --git a/components/src/dynamo/planner/plugins/proto/v1/plugin.proto b/dingo/planner/plugins/proto/v1/plugin.proto similarity index 98% rename from components/src/dynamo/planner/plugins/proto/v1/plugin.proto rename to dingo/planner/plugins/proto/v1/plugin.proto index c9a9f4f39bd3..35efd5c4d5f7 100644 --- a/components/src/dynamo/planner/plugins/proto/v1/plugin.proto +++ b/dingo/planner/plugins/proto/v1/plugin.proto @@ -18,6 +18,7 @@ syntax = "proto3"; +// Keep the public gRPC wire package stable across the Python source move. package dynamo.planner.plugin.v1; // ============================================================================ @@ -200,7 +201,7 @@ enum CircuitState { // Pipeline context (flows through all stages) // // Observation types are isomorphic to the existing TickInput boundary types -// in components/src/dynamo/planner/core/types.py. FPM data is passed in its +// in dingo/planner/core/types.py. FPM data is passed in its // native msgspec encoding to avoid duplicating the ForwardPassMetrics schema. // ============================================================================ @@ -267,7 +268,7 @@ message TrafficMetrics // Mirrors FpmObservations (types.py). // // Wire format: each map value is a msgspec/msgpack-encoded -// ForwardPassMetrics record (see ``dynamo.common.forward_pass_metrics``). +// ForwardPassMetrics record (see ``dingo.common.forward_pass_metrics``). // Cross-language plugins decode via any standard msgpack library (Go's // vmihailenco/msgpack, Rust's rmp-serde, JS @msgpack/msgpack, etc.) plus // knowledge of the ForwardPassMetrics struct layout. @@ -339,7 +340,7 @@ message PredictionData } // Aligns wire format with existing ScaleRequest.target_replicas -// (components/src/dynamo/planner/connectors/protocol.py). +// (dingo/planner/connectors/protocol.py). // Used as the output of RECONCILE/CONSTRAIN: each ComponentTarget's // `type` field is unused here (only `replicas` matters). message ScalingProposal diff --git a/dingo/planner/plugins/proto/v1/plugin_pb2.py b/dingo/planner/plugins/proto/v1/plugin_pb2.py new file mode 100644 index 000000000000..3a7b25809e76 --- /dev/null +++ b/dingo/planner/plugins/proto/v1/plugin_pb2.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: dingo/planner/plugins/proto/v1/plugin.proto +# Protobuf Python Version: 5.27.2 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 5, + 27, + 2, + '', + 'dingo/planner/plugins/proto/v1/plugin.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n+dingo/planner/plugins/proto/v1/plugin.proto\x12\x18\x64ynamo.planner.plugin.v1\"\xdc\x02\n\x0fRegisterRequest\x12\x11\n\tplugin_id\x18\x01 \x01(\t\x12\x13\n\x0bplugin_type\x18\x02 \x01(\t\x12\x10\n\x08priority\x18\x03 \x01(\r\x12\x10\n\x08\x65ndpoint\x18\x04 \x01(\t\x12\x0f\n\x07version\x18\x05 \x01(\t\x12\"\n\x1a\x65xecution_interval_seconds\x18\x06 \x01(\x02\x12\x39\n\x0bhold_policy\x18\x07 \x01(\x0e\x32$.dynamo.planner.plugin.v1.HoldPolicy\x12\r\n\x05needs\x18\x08 \x03(\t\x12\x18\n\x10protocol_version\x18\t \x01(\t\x12\x12\n\nauth_token\x18\n \x01(\t\x12 \n\x18requires_produced_fields\x18\r \x03(\t\x12\"\n\x1aobservation_window_seconds\x18\x0e \x01(\x02J\x04\x08\x0b\x10\x0cJ\x04\x08\x0c\x10\r\"`\n\x10RegisterResponse\x12\x10\n\x08\x61\x63\x63\x65pted\x18\x01 \x01(\x08\x12\x15\n\rreject_reason\x18\x02 \x01(\t\x12#\n\x1bnegotiated_protocol_version\x18\x03 \x01(\t\"9\n\x10HeartbeatRequest\x12\x11\n\tplugin_id\x18\x01 \x01(\t\x12\x12\n\nauth_token\x18\x02 \x01(\t\"\x1f\n\x11HeartbeatResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\"J\n\x11UnregisterRequest\x12\x11\n\tplugin_id\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\x12\x12\n\nauth_token\x18\x03 \x01(\t\" \n\x12UnregisterResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\"D\n\x12ListPluginsRequest\x12\x14\n\x0cstage_filter\x18\x01 \x01(\t\x12\x18\n\x10include_disabled\x18\x02 \x01(\x08\"L\n\x13ListPluginsResponse\x12\x35\n\x07plugins\x18\x01 \x03(\x0b\x32$.dynamo.planner.plugin.v1.PluginInfo\"\xc0\x02\n\nPluginInfo\x12\x11\n\tplugin_id\x18\x01 \x01(\t\x12\x13\n\x0bplugin_type\x18\x02 \x01(\t\x12\x10\n\x08priority\x18\x03 \x01(\r\x12\x0f\n\x07version\x18\x04 \x01(\t\x12\x18\n\x10protocol_version\x18\x05 \x01(\t\x12\x0f\n\x07\x65nabled\x18\x06 \x01(\x08\x12\x12\n\nis_builtin\x18\x07 \x01(\x08\x12\x11\n\ttransport\x18\x08 \x01(\t\x12=\n\rcircuit_state\x18\t \x01(\x0e\x32&.dynamo.planner.plugin.v1.CircuitState\x12\x19\n\x11\x65valuations_total\x18\n \x01(\x04\x12 \n\x18last_call_at_seconds_ago\x18\x0b \x01(\x01\x12\x19\n\x11\x63\x61\x63he_age_seconds\x18\x0c \x01(\x01\"\x89\x03\n\x0fPipelineContext\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65\x63ision_id\x18\x02 \x01(\t\x12\x44\n\x0cobservations\x18\x03 \x01(\x0b\x32).dynamo.planner.plugin.v1.ObservationDataH\x00\x88\x01\x01\x12\x42\n\x0bpredictions\x18\x04 \x01(\x0b\x32(.dynamo.planner.plugin.v1.PredictionDataH\x01\x88\x01\x01\x12@\n\x08proposal\x18\x05 \x01(\x0b\x32).dynamo.planner.plugin.v1.ScalingProposalH\x02\x88\x01\x01\x12\x43\n\x0b\x63onstrained\x18\x06 \x01(\x0b\x32).dynamo.planner.plugin.v1.ScalingProposalH\x03\x88\x01\x01\x42\x0f\n\r_observationsB\x0e\n\x0c_predictionsB\x0b\n\t_proposalB\x0e\n\x0c_constrained\"\xe3\x01\n\x0fObservationData\x12>\n\x07traffic\x18\x01 \x01(\x0b\x32(.dynamo.planner.plugin.v1.TrafficMetricsH\x00\x88\x01\x01\x12\x33\n\x03\x66pm\x18\x02 \x01(\x0b\x32!.dynamo.planner.plugin.v1.FpmDataH\x01\x88\x01\x01\x12;\n\x07workers\x18\x03 \x01(\x0b\x32%.dynamo.planner.plugin.v1.WorkerStateH\x02\x88\x01\x01\x42\n\n\x08_trafficB\x06\n\x04_fpmB\n\n\x08_workers\"\xa7\x01\n\x0eTrafficMetrics\x12\x12\n\nduration_s\x18\x01 \x01(\x01\x12\x0f\n\x07num_req\x18\x02 \x01(\x01\x12\x0b\n\x03isl\x18\x03 \x01(\x01\x12\x0b\n\x03osl\x18\x04 \x01(\x01\x12\x18\n\x0bkv_hit_rate\x18\x05 \x01(\x01H\x00\x88\x01\x01\x12\x1a\n\raccept_length\x18\x06 \x01(\x01H\x01\x88\x01\x01\x42\x0e\n\x0c_kv_hit_rateB\x10\n\x0e_accept_length\"\x94\x02\n\x07\x46pmData\x12N\n\x0fprefill_engines\x18\x01 \x03(\x0b\x32\x35.dynamo.planner.plugin.v1.FpmData.PrefillEnginesEntry\x12L\n\x0e\x64\x65\x63ode_engines\x18\x02 \x03(\x0b\x32\x34.dynamo.planner.plugin.v1.FpmData.DecodeEnginesEntry\x1a\x35\n\x13PrefillEnginesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\x1a\x34\n\x12\x44\x65\x63odeEnginesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\"\xdf\x02\n\x0bWorkerState\x12\x1a\n\rready_prefill\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x19\n\x0cready_decode\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1d\n\x10\x65xpected_prefill\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x1c\n\x0f\x65xpected_decode\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12(\n\x1bprefill_scaling_in_progress\x18\x05 \x01(\x08H\x04\x88\x01\x01\x12\'\n\x1a\x64\x65\x63ode_scaling_in_progress\x18\x06 \x01(\x08H\x05\x88\x01\x01\x42\x10\n\x0e_ready_prefillB\x0f\n\r_ready_decodeB\x13\n\x11_expected_prefillB\x12\n\x10_expected_decodeB\x1e\n\x1c_prefill_scaling_in_progressB\x1d\n\x1b_decode_scaling_in_progress\"\xb2\x02\n\x0ePredictionData\x12\x1e\n\x11predicted_num_req\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x1a\n\rpredicted_isl\x18\x02 \x01(\x01H\x01\x88\x01\x01\x12\x1a\n\rpredicted_osl\x18\x03 \x01(\x01H\x02\x88\x01\x01\x12\x0e\n\x06source\x18\x04 \x01(\t\x12\"\n\x15predicted_kv_hit_rate\x18\x05 \x01(\x01H\x03\x88\x01\x01\x12$\n\x17predicted_accept_length\x18\x06 \x01(\x01H\x04\x88\x01\x01\x42\x14\n\x12_predicted_num_reqB\x10\n\x0e_predicted_islB\x10\n\x0e_predicted_oslB\x18\n\x16_predicted_kv_hit_rateB\x1a\n\x18_predicted_accept_length\"m\n\x0fScalingProposal\x12:\n\x07targets\x18\x01 \x03(\x0b\x32).dynamo.planner.plugin.v1.ComponentTarget\x12\x0e\n\x06reason\x18\x02 \x01(\t\x12\x0e\n\x06source\x18\x03 \x01(\t\"\x8d\x01\n\x0f\x43omponentTarget\x12\x1a\n\x12sub_component_type\x18\x01 \x01(\t\x12\x15\n\x08replicas\x18\x03 \x01(\x05H\x00\x88\x01\x01\x12\x34\n\x04type\x18\x04 \x01(\x0e\x32&.dynamo.planner.plugin.v1.OverrideTypeB\x0b\n\t_replicasJ\x04\x08\x02\x10\x03\"\\\n\x0eOverrideResult\x12:\n\x07targets\x18\x01 \x03(\x0b\x32).dynamo.planner.plugin.v1.ComponentTarget\x12\x0e\n\x06reason\x18\x02 \x01(\t\"\x0e\n\x0c\x41\x63\x63\x65ptResult\"\x1e\n\x0cRejectResult\x12\x0e\n\x06reason\x18\x01 \x01(\t\"Q\n\x13PredictStageRequest\x12:\n\x07\x63ontext\x18\x01 \x01(\x0b\x32).dynamo.planner.plugin.v1.PipelineContext\"t\n\x14PredictStageResponse\x12=\n\x0bpredictions\x18\x01 \x01(\x0b\x32(.dynamo.planner.plugin.v1.PredictionData\x12\x0e\n\x06reason\x18\x02 \x01(\t\x12\r\n\x05\x66inal\x18\x03 \x01(\x08\"Q\n\x13ProposeStageRequest\x12:\n\x07\x63ontext\x18\x01 \x01(\x0b\x32).dynamo.planner.plugin.v1.PipelineContext\"\xe1\x01\n\x14ProposeStageResponse\x12\x38\n\x06\x61\x63\x63\x65pt\x18\x01 \x01(\x0b\x32&.dynamo.planner.plugin.v1.AcceptResultH\x00\x12<\n\x08override\x18\x02 \x01(\x0b\x32(.dynamo.planner.plugin.v1.OverrideResultH\x00\x12\x38\n\x06reject\x18\x03 \x01(\x0b\x32&.dynamo.planner.plugin.v1.RejectResultH\x00\x12\r\n\x05\x66inal\x18\x04 \x01(\x08\x42\x08\n\x06result\"\x8f\x01\n\x15ReconcileStageRequest\x12:\n\x07\x63ontext\x18\x01 \x01(\x0b\x32).dynamo.planner.plugin.v1.PipelineContext\x12:\n\tproposals\x18\x02 \x03(\x0b\x32\'.dynamo.planner.plugin.v1.ProposeResult\"\xf0\x01\n\rProposeResult\x12\x11\n\tplugin_id\x18\x01 \x01(\t\x12\x38\n\x06\x61\x63\x63\x65pt\x18\x02 \x01(\x0b\x32&.dynamo.planner.plugin.v1.AcceptResultH\x00\x12<\n\x08override\x18\x03 \x01(\x0b\x32(.dynamo.planner.plugin.v1.OverrideResultH\x00\x12\x38\n\x06reject\x18\x04 \x01(\x0b\x32&.dynamo.planner.plugin.v1.RejectResultH\x00\x12\x10\n\x08priority\x18\x05 \x01(\rB\x08\n\x06result\"\xe3\x01\n\x16ReconcileStageResponse\x12\x38\n\x06\x61\x63\x63\x65pt\x18\x01 \x01(\x0b\x32&.dynamo.planner.plugin.v1.AcceptResultH\x00\x12<\n\x08override\x18\x02 \x01(\x0b\x32(.dynamo.planner.plugin.v1.OverrideResultH\x00\x12\x38\n\x06reject\x18\x03 \x01(\x0b\x32&.dynamo.planner.plugin.v1.RejectResultH\x00\x12\r\n\x05\x66inal\x18\x04 \x01(\x08\x42\x08\n\x06result\"S\n\x15\x43onstrainStageRequest\x12:\n\x07\x63ontext\x18\x01 \x01(\x0b\x32).dynamo.planner.plugin.v1.PipelineContext\"\xe3\x01\n\x16\x43onstrainStageResponse\x12\x38\n\x06\x61\x63\x63\x65pt\x18\x01 \x01(\x0b\x32&.dynamo.planner.plugin.v1.AcceptResultH\x00\x12<\n\x08override\x18\x02 \x01(\x0b\x32(.dynamo.planner.plugin.v1.OverrideResultH\x00\x12\x38\n\x06reject\x18\x03 \x01(\x0b\x32&.dynamo.planner.plugin.v1.RejectResultH\x00\x12\r\n\x05\x66inal\x18\x04 \x01(\x08\x42\x08\n\x06result\"\x9e\x01\n\x10\x42ootstrapRequest\x12\x16\n\x0e\x62ootstrap_data\x18\x01 \x01(\x0c\x12\x44\n\x05hints\x18\x02 \x03(\x0b\x32\x35.dynamo.planner.plugin.v1.BootstrapRequest.HintsEntry\x1a,\n\nHintsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"0\n\x11\x42ootstrapResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0cResetRequest\x12\x0e\n\x06reason\x18\x01 \x01(\t\",\n\rResetResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t*1\n\nHoldPolicy\x12\x14\n\x10\x41\x43\x43\x45PT_WHEN_IDLE\x10\x00\x12\r\n\tHOLD_LAST\x10\x01*3\n\x0c\x43ircuitState\x12\n\n\x06\x43LOSED\x10\x00\x12\x08\n\x04OPEN\x10\x01\x12\r\n\tHALF_OPEN\x10\x02*2\n\x0cOverrideType\x12\x07\n\x03SET\x10\x00\x12\x0c\n\x08\x41T_LEAST\x10\x01\x12\x0b\n\x07\x41T_MOST\x10\x02\x32\xae\x03\n\x0ePluginRegistry\x12\x61\n\x08Register\x12).dynamo.planner.plugin.v1.RegisterRequest\x1a*.dynamo.planner.plugin.v1.RegisterResponse\x12\x64\n\tHeartbeat\x12*.dynamo.planner.plugin.v1.HeartbeatRequest\x1a+.dynamo.planner.plugin.v1.HeartbeatResponse\x12g\n\nUnregister\x12+.dynamo.planner.plugin.v1.UnregisterRequest\x1a,.dynamo.planner.plugin.v1.UnregisterResponse\x12j\n\x0bListPlugins\x12,.dynamo.planner.plugin.v1.ListPluginsRequest\x1a-.dynamo.planner.plugin.v1.ListPluginsResponse2y\n\rPredictPlugin\x12h\n\x07Predict\x12-.dynamo.planner.plugin.v1.PredictStageRequest\x1a..dynamo.planner.plugin.v1.PredictStageResponse2y\n\rProposePlugin\x12h\n\x07Propose\x12-.dynamo.planner.plugin.v1.ProposeStageRequest\x1a..dynamo.planner.plugin.v1.ProposeStageResponse2\x81\x01\n\x0fReconcilePlugin\x12n\n\tReconcile\x12/.dynamo.planner.plugin.v1.ReconcileStageRequest\x1a\x30.dynamo.planner.plugin.v1.ReconcileStageResponse2\x81\x01\n\x0f\x43onstrainPlugin\x12n\n\tConstrain\x12/.dynamo.planner.plugin.v1.ConstrainStageRequest\x1a\x30.dynamo.planner.plugin.v1.ConstrainStageResponse2\xd1\x01\n\x0fPluginLifecycle\x12\x64\n\tBootstrap\x12*.dynamo.planner.plugin.v1.BootstrapRequest\x1a+.dynamo.planner.plugin.v1.BootstrapResponse\x12X\n\x05Reset\x12&.dynamo.planner.plugin.v1.ResetRequest\x1a\'.dynamo.planner.plugin.v1.ResetResponseb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'dingo.planner.plugins.proto.v1.plugin_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_FPMDATA_PREFILLENGINESENTRY']._loaded_options = None + _globals['_FPMDATA_PREFILLENGINESENTRY']._serialized_options = b'8\001' + _globals['_FPMDATA_DECODEENGINESENTRY']._loaded_options = None + _globals['_FPMDATA_DECODEENGINESENTRY']._serialized_options = b'8\001' + _globals['_BOOTSTRAPREQUEST_HINTSENTRY']._loaded_options = None + _globals['_BOOTSTRAPREQUEST_HINTSENTRY']._serialized_options = b'8\001' + _globals['_HOLDPOLICY']._serialized_start=5065 + _globals['_HOLDPOLICY']._serialized_end=5114 + _globals['_CIRCUITSTATE']._serialized_start=5116 + _globals['_CIRCUITSTATE']._serialized_end=5167 + _globals['_OVERRIDETYPE']._serialized_start=5169 + _globals['_OVERRIDETYPE']._serialized_end=5219 + _globals['_REGISTERREQUEST']._serialized_start=74 + _globals['_REGISTERREQUEST']._serialized_end=422 + _globals['_REGISTERRESPONSE']._serialized_start=424 + _globals['_REGISTERRESPONSE']._serialized_end=520 + _globals['_HEARTBEATREQUEST']._serialized_start=522 + _globals['_HEARTBEATREQUEST']._serialized_end=579 + _globals['_HEARTBEATRESPONSE']._serialized_start=581 + _globals['_HEARTBEATRESPONSE']._serialized_end=612 + _globals['_UNREGISTERREQUEST']._serialized_start=614 + _globals['_UNREGISTERREQUEST']._serialized_end=688 + _globals['_UNREGISTERRESPONSE']._serialized_start=690 + _globals['_UNREGISTERRESPONSE']._serialized_end=722 + _globals['_LISTPLUGINSREQUEST']._serialized_start=724 + _globals['_LISTPLUGINSREQUEST']._serialized_end=792 + _globals['_LISTPLUGINSRESPONSE']._serialized_start=794 + _globals['_LISTPLUGINSRESPONSE']._serialized_end=870 + _globals['_PLUGININFO']._serialized_start=873 + _globals['_PLUGININFO']._serialized_end=1193 + _globals['_PIPELINECONTEXT']._serialized_start=1196 + _globals['_PIPELINECONTEXT']._serialized_end=1589 + _globals['_OBSERVATIONDATA']._serialized_start=1592 + _globals['_OBSERVATIONDATA']._serialized_end=1819 + _globals['_TRAFFICMETRICS']._serialized_start=1822 + _globals['_TRAFFICMETRICS']._serialized_end=1989 + _globals['_FPMDATA']._serialized_start=1992 + _globals['_FPMDATA']._serialized_end=2268 + _globals['_FPMDATA_PREFILLENGINESENTRY']._serialized_start=2161 + _globals['_FPMDATA_PREFILLENGINESENTRY']._serialized_end=2214 + _globals['_FPMDATA_DECODEENGINESENTRY']._serialized_start=2216 + _globals['_FPMDATA_DECODEENGINESENTRY']._serialized_end=2268 + _globals['_WORKERSTATE']._serialized_start=2271 + _globals['_WORKERSTATE']._serialized_end=2622 + _globals['_PREDICTIONDATA']._serialized_start=2625 + _globals['_PREDICTIONDATA']._serialized_end=2931 + _globals['_SCALINGPROPOSAL']._serialized_start=2933 + _globals['_SCALINGPROPOSAL']._serialized_end=3042 + _globals['_COMPONENTTARGET']._serialized_start=3045 + _globals['_COMPONENTTARGET']._serialized_end=3186 + _globals['_OVERRIDERESULT']._serialized_start=3188 + _globals['_OVERRIDERESULT']._serialized_end=3280 + _globals['_ACCEPTRESULT']._serialized_start=3282 + _globals['_ACCEPTRESULT']._serialized_end=3296 + _globals['_REJECTRESULT']._serialized_start=3298 + _globals['_REJECTRESULT']._serialized_end=3328 + _globals['_PREDICTSTAGEREQUEST']._serialized_start=3330 + _globals['_PREDICTSTAGEREQUEST']._serialized_end=3411 + _globals['_PREDICTSTAGERESPONSE']._serialized_start=3413 + _globals['_PREDICTSTAGERESPONSE']._serialized_end=3529 + _globals['_PROPOSESTAGEREQUEST']._serialized_start=3531 + _globals['_PROPOSESTAGEREQUEST']._serialized_end=3612 + _globals['_PROPOSESTAGERESPONSE']._serialized_start=3615 + _globals['_PROPOSESTAGERESPONSE']._serialized_end=3840 + _globals['_RECONCILESTAGEREQUEST']._serialized_start=3843 + _globals['_RECONCILESTAGEREQUEST']._serialized_end=3986 + _globals['_PROPOSERESULT']._serialized_start=3989 + _globals['_PROPOSERESULT']._serialized_end=4229 + _globals['_RECONCILESTAGERESPONSE']._serialized_start=4232 + _globals['_RECONCILESTAGERESPONSE']._serialized_end=4459 + _globals['_CONSTRAINSTAGEREQUEST']._serialized_start=4461 + _globals['_CONSTRAINSTAGEREQUEST']._serialized_end=4544 + _globals['_CONSTRAINSTAGERESPONSE']._serialized_start=4547 + _globals['_CONSTRAINSTAGERESPONSE']._serialized_end=4774 + _globals['_BOOTSTRAPREQUEST']._serialized_start=4777 + _globals['_BOOTSTRAPREQUEST']._serialized_end=4935 + _globals['_BOOTSTRAPREQUEST_HINTSENTRY']._serialized_start=4891 + _globals['_BOOTSTRAPREQUEST_HINTSENTRY']._serialized_end=4935 + _globals['_BOOTSTRAPRESPONSE']._serialized_start=4937 + _globals['_BOOTSTRAPRESPONSE']._serialized_end=4985 + _globals['_RESETREQUEST']._serialized_start=4987 + _globals['_RESETREQUEST']._serialized_end=5017 + _globals['_RESETRESPONSE']._serialized_start=5019 + _globals['_RESETRESPONSE']._serialized_end=5063 + _globals['_PLUGINREGISTRY']._serialized_start=5222 + _globals['_PLUGINREGISTRY']._serialized_end=5652 + _globals['_PREDICTPLUGIN']._serialized_start=5654 + _globals['_PREDICTPLUGIN']._serialized_end=5775 + _globals['_PROPOSEPLUGIN']._serialized_start=5777 + _globals['_PROPOSEPLUGIN']._serialized_end=5898 + _globals['_RECONCILEPLUGIN']._serialized_start=5901 + _globals['_RECONCILEPLUGIN']._serialized_end=6030 + _globals['_CONSTRAINPLUGIN']._serialized_start=6033 + _globals['_CONSTRAINPLUGIN']._serialized_end=6162 + _globals['_PLUGINLIFECYCLE']._serialized_start=6165 + _globals['_PLUGINLIFECYCLE']._serialized_end=6374 +# @@protoc_insertion_point(module_scope) diff --git a/components/src/dynamo/planner/plugins/proto/v1/plugin_pb2.pyi b/dingo/planner/plugins/proto/v1/plugin_pb2.pyi similarity index 99% rename from components/src/dynamo/planner/plugins/proto/v1/plugin_pb2.pyi rename to dingo/planner/plugins/proto/v1/plugin_pb2.pyi index a32285d93c2b..748d5fcdcfda 100644 --- a/components/src/dynamo/planner/plugins/proto/v1/plugin_pb2.pyi +++ b/dingo/planner/plugins/proto/v1/plugin_pb2.pyi @@ -1,6 +1,5 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - from google.protobuf.internal import containers as _containers from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper from google.protobuf import descriptor as _descriptor diff --git a/components/src/dynamo/planner/plugins/proto/v1/plugin_pb2_grpc.py b/dingo/planner/plugins/proto/v1/plugin_pb2_grpc.py similarity index 73% rename from components/src/dynamo/planner/plugins/proto/v1/plugin_pb2_grpc.py rename to dingo/planner/plugins/proto/v1/plugin_pb2_grpc.py index 27ea08dba000..c52071c6fd7a 100644 --- a/components/src/dynamo/planner/plugins/proto/v1/plugin_pb2_grpc.py +++ b/dingo/planner/plugins/proto/v1/plugin_pb2_grpc.py @@ -5,7 +5,7 @@ import grpc import warnings -from dynamo.planner.plugins.proto.v1 import plugin_pb2 as dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2 +from dingo.planner.plugins.proto.v1 import plugin_pb2 as dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2 GRPC_GENERATED_VERSION = '1.67.1' GRPC_VERSION = grpc.__version__ @@ -20,7 +20,7 @@ if _version_not_supported: raise RuntimeError( f'The grpc package installed is at version {GRPC_VERSION},' - + f' but the generated code in dynamo/planner/plugins/proto/v1/plugin_pb2_grpc.py depends on' + + f' but the generated code in dingo/planner/plugins/proto/v1/plugin_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' @@ -42,23 +42,23 @@ def __init__(self, channel): """ self.Register = channel.unary_unary( '/dynamo.planner.plugin.v1.PluginRegistry/Register', - request_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.RegisterRequest.SerializeToString, - response_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.RegisterResponse.FromString, + request_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.RegisterRequest.SerializeToString, + response_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.RegisterResponse.FromString, _registered_method=True) self.Heartbeat = channel.unary_unary( '/dynamo.planner.plugin.v1.PluginRegistry/Heartbeat', - request_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.HeartbeatRequest.SerializeToString, - response_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.HeartbeatResponse.FromString, + request_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.HeartbeatRequest.SerializeToString, + response_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.HeartbeatResponse.FromString, _registered_method=True) self.Unregister = channel.unary_unary( '/dynamo.planner.plugin.v1.PluginRegistry/Unregister', - request_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.UnregisterRequest.SerializeToString, - response_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.UnregisterResponse.FromString, + request_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.UnregisterRequest.SerializeToString, + response_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.UnregisterResponse.FromString, _registered_method=True) self.ListPlugins = channel.unary_unary( '/dynamo.planner.plugin.v1.PluginRegistry/ListPlugins', - request_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ListPluginsRequest.SerializeToString, - response_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ListPluginsResponse.FromString, + request_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ListPluginsRequest.SerializeToString, + response_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ListPluginsResponse.FromString, _registered_method=True) @@ -105,23 +105,23 @@ def add_PluginRegistryServicer_to_server(servicer, server): rpc_method_handlers = { 'Register': grpc.unary_unary_rpc_method_handler( servicer.Register, - request_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.RegisterRequest.FromString, - response_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.RegisterResponse.SerializeToString, + request_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.RegisterRequest.FromString, + response_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.RegisterResponse.SerializeToString, ), 'Heartbeat': grpc.unary_unary_rpc_method_handler( servicer.Heartbeat, - request_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.HeartbeatRequest.FromString, - response_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.HeartbeatResponse.SerializeToString, + request_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.HeartbeatRequest.FromString, + response_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.HeartbeatResponse.SerializeToString, ), 'Unregister': grpc.unary_unary_rpc_method_handler( servicer.Unregister, - request_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.UnregisterRequest.FromString, - response_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.UnregisterResponse.SerializeToString, + request_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.UnregisterRequest.FromString, + response_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.UnregisterResponse.SerializeToString, ), 'ListPlugins': grpc.unary_unary_rpc_method_handler( servicer.ListPlugins, - request_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ListPluginsRequest.FromString, - response_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ListPluginsResponse.SerializeToString, + request_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ListPluginsRequest.FromString, + response_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ListPluginsResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( @@ -153,8 +153,8 @@ def Register(request, request, target, '/dynamo.planner.plugin.v1.PluginRegistry/Register', - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.RegisterRequest.SerializeToString, - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.RegisterResponse.FromString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.RegisterRequest.SerializeToString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.RegisterResponse.FromString, options, channel_credentials, insecure, @@ -180,8 +180,8 @@ def Heartbeat(request, request, target, '/dynamo.planner.plugin.v1.PluginRegistry/Heartbeat', - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.HeartbeatRequest.SerializeToString, - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.HeartbeatResponse.FromString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.HeartbeatRequest.SerializeToString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.HeartbeatResponse.FromString, options, channel_credentials, insecure, @@ -207,8 +207,8 @@ def Unregister(request, request, target, '/dynamo.planner.plugin.v1.PluginRegistry/Unregister', - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.UnregisterRequest.SerializeToString, - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.UnregisterResponse.FromString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.UnregisterRequest.SerializeToString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.UnregisterResponse.FromString, options, channel_credentials, insecure, @@ -234,8 +234,8 @@ def ListPlugins(request, request, target, '/dynamo.planner.plugin.v1.PluginRegistry/ListPlugins', - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ListPluginsRequest.SerializeToString, - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ListPluginsResponse.FromString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ListPluginsRequest.SerializeToString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ListPluginsResponse.FromString, options, channel_credentials, insecure, @@ -262,8 +262,8 @@ def __init__(self, channel): """ self.Predict = channel.unary_unary( '/dynamo.planner.plugin.v1.PredictPlugin/Predict', - request_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.PredictStageRequest.SerializeToString, - response_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.PredictStageResponse.FromString, + request_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.PredictStageRequest.SerializeToString, + response_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.PredictStageResponse.FromString, _registered_method=True) @@ -285,8 +285,8 @@ def add_PredictPluginServicer_to_server(servicer, server): rpc_method_handlers = { 'Predict': grpc.unary_unary_rpc_method_handler( servicer.Predict, - request_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.PredictStageRequest.FromString, - response_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.PredictStageResponse.SerializeToString, + request_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.PredictStageRequest.FromString, + response_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.PredictStageResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( @@ -318,8 +318,8 @@ def Predict(request, request, target, '/dynamo.planner.plugin.v1.PredictPlugin/Predict', - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.PredictStageRequest.SerializeToString, - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.PredictStageResponse.FromString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.PredictStageRequest.SerializeToString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.PredictStageResponse.FromString, options, channel_credentials, insecure, @@ -342,8 +342,8 @@ def __init__(self, channel): """ self.Propose = channel.unary_unary( '/dynamo.planner.plugin.v1.ProposePlugin/Propose', - request_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ProposeStageRequest.SerializeToString, - response_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ProposeStageResponse.FromString, + request_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ProposeStageRequest.SerializeToString, + response_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ProposeStageResponse.FromString, _registered_method=True) @@ -361,8 +361,8 @@ def add_ProposePluginServicer_to_server(servicer, server): rpc_method_handlers = { 'Propose': grpc.unary_unary_rpc_method_handler( servicer.Propose, - request_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ProposeStageRequest.FromString, - response_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ProposeStageResponse.SerializeToString, + request_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ProposeStageRequest.FromString, + response_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ProposeStageResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( @@ -390,8 +390,8 @@ def Propose(request, request, target, '/dynamo.planner.plugin.v1.ProposePlugin/Propose', - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ProposeStageRequest.SerializeToString, - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ProposeStageResponse.FromString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ProposeStageRequest.SerializeToString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ProposeStageResponse.FromString, options, channel_credentials, insecure, @@ -414,8 +414,8 @@ def __init__(self, channel): """ self.Reconcile = channel.unary_unary( '/dynamo.planner.plugin.v1.ReconcilePlugin/Reconcile', - request_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ReconcileStageRequest.SerializeToString, - response_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ReconcileStageResponse.FromString, + request_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ReconcileStageRequest.SerializeToString, + response_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ReconcileStageResponse.FromString, _registered_method=True) @@ -433,8 +433,8 @@ def add_ReconcilePluginServicer_to_server(servicer, server): rpc_method_handlers = { 'Reconcile': grpc.unary_unary_rpc_method_handler( servicer.Reconcile, - request_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ReconcileStageRequest.FromString, - response_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ReconcileStageResponse.SerializeToString, + request_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ReconcileStageRequest.FromString, + response_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ReconcileStageResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( @@ -462,8 +462,8 @@ def Reconcile(request, request, target, '/dynamo.planner.plugin.v1.ReconcilePlugin/Reconcile', - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ReconcileStageRequest.SerializeToString, - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ReconcileStageResponse.FromString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ReconcileStageRequest.SerializeToString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ReconcileStageResponse.FromString, options, channel_credentials, insecure, @@ -486,8 +486,8 @@ def __init__(self, channel): """ self.Constrain = channel.unary_unary( '/dynamo.planner.plugin.v1.ConstrainPlugin/Constrain', - request_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ConstrainStageRequest.SerializeToString, - response_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ConstrainStageResponse.FromString, + request_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ConstrainStageRequest.SerializeToString, + response_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ConstrainStageResponse.FromString, _registered_method=True) @@ -505,8 +505,8 @@ def add_ConstrainPluginServicer_to_server(servicer, server): rpc_method_handlers = { 'Constrain': grpc.unary_unary_rpc_method_handler( servicer.Constrain, - request_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ConstrainStageRequest.FromString, - response_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ConstrainStageResponse.SerializeToString, + request_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ConstrainStageRequest.FromString, + response_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ConstrainStageResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( @@ -534,8 +534,8 @@ def Constrain(request, request, target, '/dynamo.planner.plugin.v1.ConstrainPlugin/Constrain', - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ConstrainStageRequest.SerializeToString, - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ConstrainStageResponse.FromString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ConstrainStageRequest.SerializeToString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ConstrainStageResponse.FromString, options, channel_credentials, insecure, @@ -567,13 +567,13 @@ def __init__(self, channel): """ self.Bootstrap = channel.unary_unary( '/dynamo.planner.plugin.v1.PluginLifecycle/Bootstrap', - request_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.BootstrapRequest.SerializeToString, - response_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.BootstrapResponse.FromString, + request_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.BootstrapRequest.SerializeToString, + response_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.BootstrapResponse.FromString, _registered_method=True) self.Reset = channel.unary_unary( '/dynamo.planner.plugin.v1.PluginLifecycle/Reset', - request_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ResetRequest.SerializeToString, - response_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ResetResponse.FromString, + request_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ResetRequest.SerializeToString, + response_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ResetResponse.FromString, _registered_method=True) @@ -611,13 +611,13 @@ def add_PluginLifecycleServicer_to_server(servicer, server): rpc_method_handlers = { 'Bootstrap': grpc.unary_unary_rpc_method_handler( servicer.Bootstrap, - request_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.BootstrapRequest.FromString, - response_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.BootstrapResponse.SerializeToString, + request_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.BootstrapRequest.FromString, + response_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.BootstrapResponse.SerializeToString, ), 'Reset': grpc.unary_unary_rpc_method_handler( servicer.Reset, - request_deserializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ResetRequest.FromString, - response_serializer=dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ResetResponse.SerializeToString, + request_deserializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ResetRequest.FromString, + response_serializer=dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ResetResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( @@ -654,8 +654,8 @@ def Bootstrap(request, request, target, '/dynamo.planner.plugin.v1.PluginLifecycle/Bootstrap', - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.BootstrapRequest.SerializeToString, - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.BootstrapResponse.FromString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.BootstrapRequest.SerializeToString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.BootstrapResponse.FromString, options, channel_credentials, insecure, @@ -681,8 +681,8 @@ def Reset(request, request, target, '/dynamo.planner.plugin.v1.PluginLifecycle/Reset', - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ResetRequest.SerializeToString, - dynamo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ResetResponse.FromString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ResetRequest.SerializeToString, + dingo_dot_planner_dot_plugins_dot_proto_dot_v1_dot_plugin__pb2.ResetResponse.FromString, options, channel_credentials, insecure, diff --git a/components/src/dynamo/planner/plugins/registry/README.md b/dingo/planner/plugins/registry/README.md similarity index 100% rename from components/src/dynamo/planner/plugins/registry/README.md rename to dingo/planner/plugins/registry/README.md diff --git a/components/src/dynamo/planner/plugins/registry/__init__.py b/dingo/planner/plugins/registry/__init__.py similarity index 89% rename from components/src/dynamo/planner/plugins/registry/__init__.py rename to dingo/planner/plugins/registry/__init__.py index 99ffb54227c0..2086aefc87a2 100644 --- a/components/src/dynamo/planner/plugins/registry/__init__.py +++ b/dingo/planner/plugins/registry/__init__.py @@ -18,8 +18,8 @@ transport/clock primitives into the planner pipeline. """ -from dynamo.planner.plugins.registry.errors import AuthError, RegistryError -from dynamo.planner.plugins.registry.types import ( +from dingo.planner.plugins.registry.errors import AuthError, RegistryError +from dingo.planner.plugins.registry.types import ( RegisteredPlugin, derive_transport_type, ) diff --git a/components/src/dynamo/planner/plugins/registry/auth/__init__.py b/dingo/planner/plugins/registry/auth/__init__.py similarity index 80% rename from components/src/dynamo/planner/plugins/registry/auth/__init__.py rename to dingo/planner/plugins/registry/auth/__init__.py index 91b92a2317ec..f9445741ea13 100644 --- a/components/src/dynamo/planner/plugins/registry/auth/__init__.py +++ b/dingo/planner/plugins/registry/auth/__init__.py @@ -15,13 +15,13 @@ K8s ServiceAccount tokens and SPIFFE JWT-SVIDs land in a follow-up PR. """ -from dynamo.planner.plugins.registry.auth.base import ( +from dingo.planner.plugins.registry.auth.base import ( AllowUnauthenticatedAuth, AuthIdentity, AuthValidator, ) -from dynamo.planner.plugins.registry.auth.multi import MultiSourceAuth -from dynamo.planner.plugins.registry.auth.static_secret import StaticSecretAuth +from dingo.planner.plugins.registry.auth.multi import MultiSourceAuth +from dingo.planner.plugins.registry.auth.static_secret import StaticSecretAuth __all__ = [ "AuthValidator", diff --git a/components/src/dynamo/planner/plugins/registry/auth/base.py b/dingo/planner/plugins/registry/auth/base.py similarity index 97% rename from components/src/dynamo/planner/plugins/registry/auth/base.py rename to dingo/planner/plugins/registry/auth/base.py index 51c3d2733678..56588fe79f8b 100644 --- a/components/src/dynamo/planner/plugins/registry/auth/base.py +++ b/dingo/planner/plugins/registry/auth/base.py @@ -16,7 +16,7 @@ from dataclasses import dataclass, field from typing import Literal -from dynamo.planner.plugins.registry.errors import AuthError +from dingo.planner.plugins.registry.errors import AuthError log = logging.getLogger(__name__) diff --git a/components/src/dynamo/planner/plugins/registry/auth/multi.py b/dingo/planner/plugins/registry/auth/multi.py similarity index 92% rename from components/src/dynamo/planner/plugins/registry/auth/multi.py rename to dingo/planner/plugins/registry/auth/multi.py index a0da6f2b383d..811585218506 100644 --- a/components/src/dynamo/planner/plugins/registry/auth/multi.py +++ b/dingo/planner/plugins/registry/auth/multi.py @@ -18,8 +18,8 @@ from typing import Sequence -from dynamo.planner.plugins.registry.auth.base import AuthIdentity, AuthValidator -from dynamo.planner.plugins.registry.errors import AuthError +from dingo.planner.plugins.registry.auth.base import AuthIdentity, AuthValidator +from dingo.planner.plugins.registry.errors import AuthError class MultiSourceAuth(AuthValidator): diff --git a/components/src/dynamo/planner/plugins/registry/auth/static_secret.py b/dingo/planner/plugins/registry/auth/static_secret.py similarity index 95% rename from components/src/dynamo/planner/plugins/registry/auth/static_secret.py rename to dingo/planner/plugins/registry/auth/static_secret.py index 96897b1a4e07..62e73f56aeb2 100644 --- a/components/src/dynamo/planner/plugins/registry/auth/static_secret.py +++ b/dingo/planner/plugins/registry/auth/static_secret.py @@ -15,8 +15,8 @@ import hmac from typing import Mapping -from dynamo.planner.plugins.registry.auth.base import AuthIdentity, AuthValidator -from dynamo.planner.plugins.registry.errors import AuthError +from dingo.planner.plugins.registry.auth.base import AuthIdentity, AuthValidator +from dingo.planner.plugins.registry.errors import AuthError class StaticSecretAuth(AuthValidator): diff --git a/components/src/dynamo/planner/plugins/registry/circuit_breaker.py b/dingo/planner/plugins/registry/circuit_breaker.py similarity index 98% rename from components/src/dynamo/planner/plugins/registry/circuit_breaker.py rename to dingo/planner/plugins/registry/circuit_breaker.py index 9a794fd97a1c..cabc860a1664 100644 --- a/components/src/dynamo/planner/plugins/registry/circuit_breaker.py +++ b/dingo/planner/plugins/registry/circuit_breaker.py @@ -33,8 +33,8 @@ from dataclasses import dataclass from typing import Callable -from dynamo.planner.plugins.clock import Clock -from dynamo.planner.plugins.types import CircuitState +from dingo.planner.plugins.clock import Clock +from dingo.planner.plugins.types import CircuitState log = logging.getLogger(__name__) diff --git a/components/src/dynamo/planner/plugins/registry/config.py b/dingo/planner/plugins/registry/config.py similarity index 92% rename from components/src/dynamo/planner/plugins/registry/config.py rename to dingo/planner/plugins/registry/config.py index 70193a697742..f3d36cc03e6d 100644 --- a/components/src/dynamo/planner/plugins/registry/config.py +++ b/dingo/planner/plugins/registry/config.py @@ -33,7 +33,7 @@ from pydantic import BaseModel, ConfigDict, Field -from dynamo.planner.plugins.transport.config import TransportConfig +from dingo.planner.plugins.transport.config import TransportConfig # ``PluginRegistryServer`` (and through it ``plugin_pb2`` / ``plugin_pb2_grpc``) # is only needed at *runtime* by the build helpers below — not by the @@ -43,11 +43,11 @@ # generated proto stubs to be present on disk (the stubs are generated at # install / dev-time only). if TYPE_CHECKING: - from dynamo.planner.plugins.clock import Clock - from dynamo.planner.plugins.registry.auth.base import AuthValidator - from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker - from dynamo.planner.plugins.registry.server import PluginRegistryServer - from dynamo.planner.plugins.transport.base import PluginTransport + from dingo.planner.plugins.clock import Clock + from dingo.planner.plugins.registry.auth.base import AuthValidator + from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker + from dingo.planner.plugins.registry.server import PluginRegistryServer + from dingo.planner.plugins.transport.base import PluginTransport log = logging.getLogger(__name__) @@ -166,7 +166,7 @@ def build_auth_validator(config: AuthConfig) -> "AuthValidator": # Heavy auth/registry imports deferred to call time so this module # stays importable before generated plugin_pb2 stubs exist (see # TYPE_CHECKING block at module top). - from dynamo.planner.plugins.registry.auth import ( + from dingo.planner.plugins.registry.auth import ( AllowUnauthenticatedAuth, MultiSourceAuth, StaticSecretAuth, @@ -212,8 +212,8 @@ def build_registry_from_config( # Heavy registry imports deferred to call time so this module stays # importable before generated plugin_pb2 stubs exist (see TYPE_CHECKING # block at module top). - from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker - from dynamo.planner.plugins.registry.server import PluginRegistryServer + from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker + from dingo.planner.plugins.registry.server import PluginRegistryServer auth = build_auth_validator(config.auth) cb = CircuitBreaker(clock) @@ -243,7 +243,7 @@ def _transport_factory_shim( third positional argument; the registry's factory protocol is ``(plugin_id, endpoint, *, in_process_instance=None)``.""" # Heavy transport import deferred (see TYPE_CHECKING block above). - from dynamo.planner.plugins.transport.config import make_transport_for_endpoint + from dingo.planner.plugins.transport.config import make_transport_for_endpoint return make_transport_for_endpoint( plugin_id, diff --git a/components/src/dynamo/planner/plugins/registry/errors.py b/dingo/planner/plugins/registry/errors.py similarity index 100% rename from components/src/dynamo/planner/plugins/registry/errors.py rename to dingo/planner/plugins/registry/errors.py diff --git a/components/src/dynamo/planner/plugins/registry/gateway.py b/dingo/planner/plugins/registry/gateway.py similarity index 96% rename from components/src/dynamo/planner/plugins/registry/gateway.py rename to dingo/planner/plugins/registry/gateway.py index c35589292e91..2e02f52adee6 100644 --- a/components/src/dynamo/planner/plugins/registry/gateway.py +++ b/dingo/planner/plugins/registry/gateway.py @@ -33,11 +33,11 @@ import grpc -from dynamo.planner.plugins._proto_bridge import proto_to_pydantic, pydantic_to_proto -from dynamo.planner.plugins.proto.v1 import plugin_pb2 as pb -from dynamo.planner.plugins.proto.v1 import plugin_pb2_grpc as pbg -from dynamo.planner.plugins.registry.server import PluginRegistryServer -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins._proto_bridge import proto_to_pydantic, pydantic_to_proto +from dingo.planner.plugins.proto.v1 import plugin_pb2 as pb +from dingo.planner.plugins.proto.v1 import plugin_pb2_grpc as pbg +from dingo.planner.plugins.registry.server import PluginRegistryServer +from dingo.planner.plugins.types import ( HeartbeatRequest, HeartbeatResponse, RegisterRequest, diff --git a/components/src/dynamo/planner/plugins/registry/server.py b/dingo/planner/plugins/registry/server.py similarity index 98% rename from components/src/dynamo/planner/plugins/registry/server.py rename to dingo/planner/plugins/registry/server.py index 35ab301a930c..44f6ec820077 100644 --- a/components/src/dynamo/planner/plugins/registry/server.py +++ b/dingo/planner/plugins/registry/server.py @@ -35,16 +35,16 @@ from packaging.version import InvalidVersion, Version -from dynamo.planner.plugins.clock import Clock -from dynamo.planner.plugins.registry.auth.base import AuthValidator -from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker -from dynamo.planner.plugins.registry.errors import AuthError -from dynamo.planner.plugins.registry.types import ( +from dingo.planner.plugins.clock import Clock +from dingo.planner.plugins.registry.auth.base import AuthValidator +from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker +from dingo.planner.plugins.registry.errors import AuthError +from dingo.planner.plugins.registry.types import ( RegisteredPlugin, derive_transport_type, ) -from dynamo.planner.plugins.transport.base import PluginTransport -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.transport.base import PluginTransport +from dingo.planner.plugins.types import ( HoldPolicy, ListPluginsRequest, PluginInfo, diff --git a/components/src/dynamo/planner/plugins/registry/types.py b/dingo/planner/plugins/registry/types.py similarity index 97% rename from components/src/dynamo/planner/plugins/registry/types.py rename to dingo/planner/plugins/registry/types.py index ad75b30f14f7..ec1249a69524 100644 --- a/components/src/dynamo/planner/plugins/registry/types.py +++ b/dingo/planner/plugins/registry/types.py @@ -25,8 +25,8 @@ from dataclasses import dataclass, field from typing import Literal -from dynamo.planner.plugins.transport.base import PluginTransport -from dynamo.planner.plugins.types import HoldPolicy +from dingo.planner.plugins.transport.base import PluginTransport +from dingo.planner.plugins.types import HoldPolicy TransportType = Literal["in_process", "grpc"] diff --git a/components/src/dynamo/planner/plugins/scheduler.py b/dingo/planner/plugins/scheduler.py similarity index 97% rename from components/src/dynamo/planner/plugins/scheduler.py rename to dingo/planner/plugins/scheduler.py index ea2d0fb03598..963530084d1b 100644 --- a/components/src/dynamo/planner/plugins/scheduler.py +++ b/dingo/planner/plugins/scheduler.py @@ -50,14 +50,14 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Optional -from dynamo.planner.plugins.clock import Clock -from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker -from dynamo.planner.plugins.registry.types import RegisteredPlugin -from dynamo.planner.plugins.types import HoldPolicy, OverrideResult +from dingo.planner.plugins.clock import Clock +from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker +from dingo.planner.plugins.registry.types import RegisteredPlugin +from dingo.planner.plugins.types import HoldPolicy, OverrideResult if TYPE_CHECKING: - from dynamo.planner.monitoring.planner_metrics import PluginFrameworkMetrics - from dynamo.planner.plugins.registry.server import PluginRegistryServer + from dingo.planner.monitoring.planner_metrics import PluginFrameworkMetrics + from dingo.planner.plugins.registry.server import PluginRegistryServer log = logging.getLogger(__name__) diff --git a/components/src/dynamo/planner/plugins/transport/README.md b/dingo/planner/plugins/transport/README.md similarity index 100% rename from components/src/dynamo/planner/plugins/transport/README.md rename to dingo/planner/plugins/transport/README.md diff --git a/components/src/dynamo/planner/plugins/transport/__init__.py b/dingo/planner/plugins/transport/__init__.py similarity index 82% rename from components/src/dynamo/planner/plugins/transport/__init__.py rename to dingo/planner/plugins/transport/__init__.py index 64a4b84f32db..1b0db5947494 100644 --- a/components/src/dynamo/planner/plugins/transport/__init__.py +++ b/dingo/planner/plugins/transport/__init__.py @@ -16,15 +16,15 @@ from typing import TYPE_CHECKING, Any -from dynamo.planner.plugins.transport.base import PluginTransport -from dynamo.planner.plugins.transport.errors import ( +from dingo.planner.plugins.transport.base import PluginTransport +from dingo.planner.plugins.transport.errors import ( PluginCallError, PluginConnectionError, PluginSerializationError, PluginTimeoutError, PluginUnknownMethodError, ) -from dynamo.planner.plugins.transport.in_process import InProcessTransport +from dingo.planner.plugins.transport.in_process import InProcessTransport # ``GrpcTransport`` is eagerly *re-exported* via PEP 562 lazy # ``__getattr__`` so ``from ... transport import GrpcTransport`` still @@ -34,7 +34,7 @@ # generation. See ``planner_config.py`` lazy-import refactor for the matching # change at the registry layer. if TYPE_CHECKING: - from dynamo.planner.plugins.transport.grpc_remote import GrpcTransport + from dingo.planner.plugins.transport.grpc_remote import GrpcTransport __all__ = [ "PluginTransport", @@ -50,7 +50,7 @@ def __getattr__(name: str) -> Any: if name == "GrpcTransport": - from dynamo.planner.plugins.transport.grpc_remote import GrpcTransport + from dingo.planner.plugins.transport.grpc_remote import GrpcTransport return GrpcTransport raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/components/src/dynamo/planner/plugins/transport/_grpc_base.py b/dingo/planner/plugins/transport/_grpc_base.py similarity index 97% rename from components/src/dynamo/planner/plugins/transport/_grpc_base.py rename to dingo/planner/plugins/transport/_grpc_base.py index c12d00cce676..ed7d9142966d 100644 --- a/components/src/dynamo/planner/plugins/transport/_grpc_base.py +++ b/dingo/planner/plugins/transport/_grpc_base.py @@ -20,10 +20,10 @@ from google.protobuf.message import Message as ProtoMessage from pydantic import BaseModel -from dynamo.planner.plugins._proto_bridge import proto_to_pydantic, pydantic_to_proto -from dynamo.planner.plugins.transport._method_dispatch import StubDispatcher -from dynamo.planner.plugins.transport.base import PluginTransport -from dynamo.planner.plugins.transport.errors import ( +from dingo.planner.plugins._proto_bridge import proto_to_pydantic, pydantic_to_proto +from dingo.planner.plugins.transport._method_dispatch import StubDispatcher +from dingo.planner.plugins.transport.base import PluginTransport +from dingo.planner.plugins.transport.errors import ( PluginCallError, PluginConnectionError, PluginSerializationError, diff --git a/components/src/dynamo/planner/plugins/transport/_method_dispatch.py b/dingo/planner/plugins/transport/_method_dispatch.py similarity index 97% rename from components/src/dynamo/planner/plugins/transport/_method_dispatch.py rename to dingo/planner/plugins/transport/_method_dispatch.py index 4149ea2bdf5e..ae4e390a434c 100644 --- a/components/src/dynamo/planner/plugins/transport/_method_dispatch.py +++ b/dingo/planner/plugins/transport/_method_dispatch.py @@ -20,7 +20,7 @@ import grpc -from dynamo.planner.plugins.proto.v1 import plugin_pb2_grpc as pbg +from dingo.planner.plugins.proto.v1 import plugin_pb2_grpc as pbg # Method name → (stub class, method attribute on stub instance) _METHOD_STUB_MAP: dict[str, tuple[type[Any], str]] = { diff --git a/components/src/dynamo/planner/plugins/transport/base.py b/dingo/planner/plugins/transport/base.py similarity index 100% rename from components/src/dynamo/planner/plugins/transport/base.py rename to dingo/planner/plugins/transport/base.py diff --git a/components/src/dynamo/planner/plugins/transport/config.py b/dingo/planner/plugins/transport/config.py similarity index 95% rename from components/src/dynamo/planner/plugins/transport/config.py rename to dingo/planner/plugins/transport/config.py index 96bbd7f1d301..92f738d22719 100644 --- a/components/src/dynamo/planner/plugins/transport/config.py +++ b/dingo/planner/plugins/transport/config.py @@ -22,8 +22,8 @@ from pydantic import BaseModel, ConfigDict, Field -from dynamo.planner.plugins.clock import Clock, VirtualClock, WallClock -from dynamo.planner.plugins.transport.base import PluginTransport +from dingo.planner.plugins.clock import Clock, VirtualClock, WallClock +from dingo.planner.plugins.transport.base import PluginTransport # ``GrpcTransport`` import deferred to ``make_transport_for_endpoint`` # where it's actually constructed. Module-top import would pull in @@ -104,7 +104,7 @@ def make_transport_for_endpoint( if endpoint.startswith("inproc://"): # Local import keeps the module top free of plugin_pb2-dependent # transports — see comment at top of file. - from dynamo.planner.plugins.transport.in_process import InProcessTransport + from dingo.planner.plugins.transport.in_process import InProcessTransport if in_process_instance is None: raise ValueError( @@ -117,7 +117,7 @@ def make_transport_for_endpoint( if endpoint.startswith("grpc://"): # Same deferred-import pattern as ``InProcessTransport`` above. - from dynamo.planner.plugins.transport.grpc_remote import GrpcTransport + from dingo.planner.plugins.transport.grpc_remote import GrpcTransport if not config.allow_insecure_grpc: raise ValueError( diff --git a/components/src/dynamo/planner/plugins/transport/errors.py b/dingo/planner/plugins/transport/errors.py similarity index 100% rename from components/src/dynamo/planner/plugins/transport/errors.py rename to dingo/planner/plugins/transport/errors.py diff --git a/components/src/dynamo/planner/plugins/transport/grpc_remote.py b/dingo/planner/plugins/transport/grpc_remote.py similarity index 97% rename from components/src/dynamo/planner/plugins/transport/grpc_remote.py rename to dingo/planner/plugins/transport/grpc_remote.py index f8288fbb52c5..6e536e8380e2 100644 --- a/components/src/dynamo/planner/plugins/transport/grpc_remote.py +++ b/dingo/planner/plugins/transport/grpc_remote.py @@ -14,7 +14,7 @@ import grpc -from dynamo.planner.plugins.transport._grpc_base import ( +from dingo.planner.plugins.transport._grpc_base import ( _GrpcTransportBase, grpc_channel_options, ) diff --git a/components/src/dynamo/planner/plugins/transport/in_process.py b/dingo/planner/plugins/transport/in_process.py similarity index 97% rename from components/src/dynamo/planner/plugins/transport/in_process.py rename to dingo/planner/plugins/transport/in_process.py index 02e20319a64f..a5e966dba4d4 100644 --- a/components/src/dynamo/planner/plugins/transport/in_process.py +++ b/dingo/planner/plugins/transport/in_process.py @@ -15,8 +15,8 @@ import inspect from typing import Any -from dynamo.planner.plugins.transport.base import PluginTransport -from dynamo.planner.plugins.transport.errors import ( +from dingo.planner.plugins.transport.base import PluginTransport +from dingo.planner.plugins.transport.errors import ( PluginCallError, PluginConnectionError, PluginTimeoutError, diff --git a/components/src/dynamo/planner/plugins/types.py b/dingo/planner/plugins/types.py similarity index 99% rename from components/src/dynamo/planner/plugins/types.py rename to dingo/planner/plugins/types.py index e728bd7c7213..8ab0da2ad2c5 100644 --- a/components/src/dynamo/planner/plugins/types.py +++ b/dingo/planner/plugins/types.py @@ -4,7 +4,7 @@ """Pydantic v2 mirror of the v1 plugin proto messages. These classes are 1:1 with the proto messages in -``dynamo.planner.plugins.proto.v1.plugin_pb2``; field names, types, and +``dingo.planner.plugins.proto.v1.plugin_pb2``; field names, types, and optional-ness must match exactly. **Why a Pydantic mirror?** diff --git a/components/src/dynamo/planner/tests/config/__init__.py b/dingo/planner/tests/config/__init__.py similarity index 100% rename from components/src/dynamo/planner/tests/config/__init__.py rename to dingo/planner/tests/config/__init__.py diff --git a/components/src/dynamo/planner/tests/config/test_scheduling_config.py b/dingo/planner/tests/config/test_scheduling_config.py similarity index 98% rename from components/src/dynamo/planner/tests/config/test_scheduling_config.py rename to dingo/planner/tests/config/test_scheduling_config.py index a40c29efa5be..0ef326a4c7ae 100644 --- a/components/src/dynamo/planner/tests/config/test_scheduling_config.py +++ b/dingo/planner/tests/config/test_scheduling_config.py @@ -9,7 +9,7 @@ import yaml from pydantic import ValidationError -from dynamo.planner.config.planner_config import PlannerConfig, SchedulingConfig +from dingo.planner.config.planner_config import PlannerConfig, SchedulingConfig pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/planner/tests/conftest.py b/dingo/planner/tests/conftest.py similarity index 100% rename from components/src/dynamo/planner/tests/conftest.py rename to dingo/planner/tests/conftest.py diff --git a/components/src/dynamo/planner/tests/core/__init__.py b/dingo/planner/tests/core/__init__.py similarity index 100% rename from components/src/dynamo/planner/tests/core/__init__.py rename to dingo/planner/tests/core/__init__.py diff --git a/components/src/dynamo/planner/tests/core/test_tick_diagnostics_extended.py b/dingo/planner/tests/core/test_tick_diagnostics_extended.py similarity index 99% rename from components/src/dynamo/planner/tests/core/test_tick_diagnostics_extended.py rename to dingo/planner/tests/core/test_tick_diagnostics_extended.py index c8e989bcace6..5c07b020ee63 100644 --- a/components/src/dynamo/planner/tests/core/test_tick_diagnostics_extended.py +++ b/dingo/planner/tests/core/test_tick_diagnostics_extended.py @@ -29,7 +29,7 @@ import pytest -from dynamo.planner.core.types import TickDiagnostics +from dingo.planner.core.types import TickDiagnostics pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D/disagg.yaml b/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D/disagg.yaml similarity index 100% rename from components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D/disagg.yaml rename to dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D/disagg.yaml diff --git a/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D/selected_decode_interpolation/raw_data.npz b/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D/selected_decode_interpolation/raw_data.npz similarity index 100% rename from components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D/selected_decode_interpolation/raw_data.npz rename to dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D/selected_decode_interpolation/raw_data.npz diff --git a/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D/selected_prefill_interpolation/raw_data.npz b/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D/selected_prefill_interpolation/raw_data.npz similarity index 100% rename from components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D/selected_prefill_interpolation/raw_data.npz rename to dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D/selected_prefill_interpolation/raw_data.npz diff --git a/components/src/dynamo/planner/tests/integration/test_external_plugin_e2e.py b/dingo/planner/tests/integration/test_external_plugin_e2e.py similarity index 97% rename from components/src/dynamo/planner/tests/integration/test_external_plugin_e2e.py rename to dingo/planner/tests/integration/test_external_plugin_e2e.py index 76c0b6afec1c..eb7472f7fe1a 100644 --- a/components/src/dynamo/planner/tests/integration/test_external_plugin_e2e.py +++ b/dingo/planner/tests/integration/test_external_plugin_e2e.py @@ -45,20 +45,20 @@ import grpc import pytest -from dynamo.planner.plugins.clock import WallClock -from dynamo.planner.plugins.merge.types import ComponentKey -from dynamo.planner.plugins.orchestrator.orchestrator import LocalPlannerOrchestrator -from dynamo.planner.plugins.proto.v1 import plugin_pb2 as pb -from dynamo.planner.plugins.proto.v1 import plugin_pb2_grpc as pbg -from dynamo.planner.plugins.registry.auth.base import AllowUnauthenticatedAuth -from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker -from dynamo.planner.plugins.registry.server import PluginRegistryServer -from dynamo.planner.plugins.scheduler import PluginScheduler -from dynamo.planner.plugins.transport.config import ( +from dingo.planner.plugins.clock import WallClock +from dingo.planner.plugins.merge.types import ComponentKey +from dingo.planner.plugins.orchestrator.orchestrator import LocalPlannerOrchestrator +from dingo.planner.plugins.proto.v1 import plugin_pb2 as pb +from dingo.planner.plugins.proto.v1 import plugin_pb2_grpc as pbg +from dingo.planner.plugins.registry.auth.base import AllowUnauthenticatedAuth +from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker +from dingo.planner.plugins.registry.server import PluginRegistryServer +from dingo.planner.plugins.scheduler import PluginScheduler +from dingo.planner.plugins.transport.config import ( TransportConfig, make_transport_for_endpoint, ) -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.types import ( HoldPolicy, ListPluginsRequest, PipelineContext, diff --git a/components/src/dynamo/planner/tests/integration/test_virtual_connector.py b/dingo/planner/tests/integration/test_virtual_connector.py similarity index 97% rename from components/src/dynamo/planner/tests/integration/test_virtual_connector.py rename to dingo/planner/tests/integration/test_virtual_connector.py index eb8ff3cb18f1..69bfd4475215 100644 --- a/components/src/dynamo/planner/tests/integration/test_virtual_connector.py +++ b/dingo/planner/tests/integration/test_virtual_connector.py @@ -11,7 +11,7 @@ import pytest from dynamo._core import DistributedRuntime, VirtualConnectorClient -from dynamo.planner import SubComponentType, TargetReplica, VirtualConnector +from dingo.planner import SubComponentType, TargetReplica, VirtualConnector pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/planner/tests/manual/.gitignore b/dingo/planner/tests/manual/.gitignore similarity index 100% rename from components/src/dynamo/planner/tests/manual/.gitignore rename to dingo/planner/tests/manual/.gitignore diff --git a/components/src/dynamo/planner/tests/manual/README.md b/dingo/planner/tests/manual/README.md similarity index 91% rename from components/src/dynamo/planner/tests/manual/README.md rename to dingo/planner/tests/manual/README.md index f1da00bf6108..6e83515a41e9 100644 --- a/components/src/dynamo/planner/tests/manual/README.md +++ b/dingo/planner/tests/manual/README.md @@ -52,7 +52,7 @@ This directory contains comprehensive tests for validating the SLA planner's sca ### Test Types -1. **Unit Tests** (`components/src/dynamo/planner/tests/unit/test_load_based_scaling.py` + plugin/orchestrator tests) - Test the mathematical formulas and builtin pipeline behavior in isolation +1. **Unit Tests** (`dingo/planner/tests/unit/test_load_based_scaling.py` + plugin/orchestrator tests) - Test the mathematical formulas and builtin pipeline behavior in isolation 2. **End-to-End Tests** (`scaling/run_scaling_test.sh`) - Test complete workflow including Kubernetes deployment, load generation, and pod scaling validation 3. **End-to-End Perf Tests** (see instructions below) - Compare performance (goodput and goodput/GPU) on deployments with and without sla planner @@ -63,7 +63,7 @@ This directory contains comprehensive tests for validating the SLA planner's sca Test the replica calculation logic without requiring Kubernetes: ```bash -PYTHONPATH=components/src python -m pytest components/src/dynamo/planner/tests/unit/test_load_based_scaling.py components/src/dynamo/planner/tests/plugins/orchestrator -v +PYTHONPATH=components/src python -m pytest dingo/planner/tests/unit/test_load_based_scaling.py dingo/planner/tests/plugins/orchestrator -v ``` **Note**: The unit tests automatically mock external dependencies (prometheus_client, runtime modules) to ensure they can run in isolation without requiring the full Dynamo environment. @@ -91,13 +91,13 @@ The main test scenario validates prefill scaling for H200 with 1P1D -> 2P1D conf Run the test with: ```bash -components/src/dynamo/planner/tests/manual/scaling/run_scaling_test.sh --namespace +dingo/planner/tests/manual/scaling/run_scaling_test.sh --namespace ``` -To save results to `components/src/dynamo/planner/tests/e2e_scaling_results` instead of `/tmp`: +To save results to `dingo/planner/tests/e2e_scaling_results` instead of `/tmp`: ```bash -components/src/dynamo/planner/tests/manual/scaling/run_scaling_test.sh --namespace --save-results +dingo/planner/tests/manual/scaling/run_scaling_test.sh --namespace --save-results ``` ### Instructions for End-to-End Perf Tests diff --git a/components/src/dynamo/planner/tests/manual/figures/dryrun_plot.png b/dingo/planner/tests/manual/figures/dryrun_plot.png similarity index 100% rename from components/src/dynamo/planner/tests/manual/figures/dryrun_plot.png rename to dingo/planner/tests/manual/figures/dryrun_plot.png diff --git a/components/src/dynamo/planner/tests/manual/figures/sla_planner_perf.png b/dingo/planner/tests/manual/figures/sla_planner_perf.png similarity index 100% rename from components/src/dynamo/planner/tests/manual/figures/sla_planner_perf.png rename to dingo/planner/tests/manual/figures/sla_planner_perf.png diff --git a/components/src/dynamo/planner/tests/manual/perf_test_configs/agg_8b.yaml b/dingo/planner/tests/manual/perf_test_configs/agg_8b.yaml similarity index 100% rename from components/src/dynamo/planner/tests/manual/perf_test_configs/agg_8b.yaml rename to dingo/planner/tests/manual/perf_test_configs/agg_8b.yaml diff --git a/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_2p2d.yaml b/dingo/planner/tests/manual/perf_test_configs/disagg_8b_2p2d.yaml similarity index 100% rename from components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_2p2d.yaml rename to dingo/planner/tests/manual/perf_test_configs/disagg_8b_2p2d.yaml diff --git a/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_3p1d.yaml b/dingo/planner/tests/manual/perf_test_configs/disagg_8b_3p1d.yaml similarity index 100% rename from components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_3p1d.yaml rename to dingo/planner/tests/manual/perf_test_configs/disagg_8b_3p1d.yaml diff --git a/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_planner.yaml b/dingo/planner/tests/manual/perf_test_configs/disagg_8b_planner.yaml similarity index 96% rename from components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_planner.yaml rename to dingo/planner/tests/manual/perf_test_configs/disagg_8b_planner.yaml index abba3a95c243..ce5cbae2d93d 100644 --- a/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_planner.yaml +++ b/dingo/planner/tests/manual/perf_test_configs/disagg_8b_planner.yaml @@ -79,10 +79,10 @@ spec: command: - python3 - -m - - dynamo.planner + - dingo.planner args: - --config - - '{"environment": "kubernetes", "backend": "vllm", "optimization_target": "sla", "ttft_ms": 200, "itl_ms": 10, "profile_results_dir": "/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D/", "throughput_adjustment_interval_seconds": 60, "metric_reporting_prometheus_port": 9085}' + - '{"environment": "kubernetes", "backend": "vllm", "optimization_target": "sla", "ttft_ms": 200, "itl_ms": 10, "profile_results_dir": "/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D/", "throughput_adjustment_interval_seconds": 60, "metric_reporting_prometheus_port": 9085}' VllmDecodeWorker: envFromSecret: hf-token-secret componentType: worker diff --git a/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_tp2.yaml b/dingo/planner/tests/manual/perf_test_configs/disagg_8b_tp2.yaml similarity index 100% rename from components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_tp2.yaml rename to dingo/planner/tests/manual/perf_test_configs/disagg_8b_tp2.yaml diff --git a/components/src/dynamo/planner/tests/manual/perf_test_configs/image_cache_daemonset.yaml b/dingo/planner/tests/manual/perf_test_configs/image_cache_daemonset.yaml similarity index 100% rename from components/src/dynamo/planner/tests/manual/perf_test_configs/image_cache_daemonset.yaml rename to dingo/planner/tests/manual/perf_test_configs/image_cache_daemonset.yaml diff --git a/components/src/dynamo/planner/tests/manual/scaling/disagg_planner_load.yaml b/dingo/planner/tests/manual/scaling/disagg_planner_load.yaml similarity index 98% rename from components/src/dynamo/planner/tests/manual/scaling/disagg_planner_load.yaml rename to dingo/planner/tests/manual/scaling/disagg_planner_load.yaml index d9b0ff18a078..275d277cdda3 100644 --- a/components/src/dynamo/planner/tests/manual/scaling/disagg_planner_load.yaml +++ b/dingo/planner/tests/manual/scaling/disagg_planner_load.yaml @@ -30,7 +30,7 @@ spec: command: - python3 - -m - - dynamo.planner + - dingo.planner args: - --config - '{"environment": "kubernetes", "backend": "vllm", "optimization_target": "sla", "enable_load_scaling": true, "enable_throughput_scaling": false, "pre_deployment_sweeping_mode": "none", "load_adjustment_interval_seconds": 5, "load_min_observations": 5}' diff --git a/components/src/dynamo/planner/tests/manual/scaling/disagg_planner_throughput.yaml b/dingo/planner/tests/manual/scaling/disagg_planner_throughput.yaml similarity index 92% rename from components/src/dynamo/planner/tests/manual/scaling/disagg_planner_throughput.yaml rename to dingo/planner/tests/manual/scaling/disagg_planner_throughput.yaml index 8e9cb973ce14..33d91bacee9a 100644 --- a/components/src/dynamo/planner/tests/manual/scaling/disagg_planner_throughput.yaml +++ b/dingo/planner/tests/manual/scaling/disagg_planner_throughput.yaml @@ -22,10 +22,10 @@ spec: command: - python3 - -m - - dynamo.planner + - dingo.planner args: - --config - - '{"environment": "kubernetes", "backend": "vllm", "optimization_target": "sla", "enable_load_scaling": false, "enable_throughput_scaling": true, "pre_deployment_sweeping_mode": "none", "load_adjustment_interval_seconds": 5, "throughput_adjustment_interval_seconds": 60, "profile_results_dir": "/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' + - '{"environment": "kubernetes", "backend": "vllm", "optimization_target": "sla", "enable_load_scaling": false, "enable_throughput_scaling": true, "pre_deployment_sweeping_mode": "none", "load_adjustment_interval_seconds": 5, "throughput_adjustment_interval_seconds": 60, "profile_results_dir": "/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' VllmDecodeWorker: envFromSecret: hf-token-secret componentType: worker diff --git a/components/src/dynamo/planner/tests/manual/scaling/run_scaling_test.sh b/dingo/planner/tests/manual/scaling/run_scaling_test.sh similarity index 97% rename from components/src/dynamo/planner/tests/manual/scaling/run_scaling_test.sh rename to dingo/planner/tests/manual/scaling/run_scaling_test.sh index 65682b9c61c6..ce73ef2a09ee 100755 --- a/components/src/dynamo/planner/tests/manual/scaling/run_scaling_test.sh +++ b/dingo/planner/tests/manual/scaling/run_scaling_test.sh @@ -72,7 +72,7 @@ check_prerequisites() { if ! command -v aiperf &> /dev/null; then log_error "aiperf not found. This tool is required for load generation." - log_error "Follow components/src/dynamo/planner/tests/manual/README.md for setup." + log_error "Follow dingo/planner/tests/manual/README.md for setup." exit 1 fi @@ -182,7 +182,7 @@ run_test() { local test_args="--namespace $NAMESPACE --mode $MODE --base-url http://localhost:$LOCAL_PORT" if [ "$SAVE_RESULTS" = true ]; then test_args="$test_args --save-results" - log_info "Results will be saved to components/src/dynamo/planner/tests/e2e_scaling_results" + log_info "Results will be saved to dingo/planner/tests/e2e_scaling_results" fi $python_cmd "$TEST_FILE" $test_args diff --git a/components/src/dynamo/planner/tests/manual/scaling/scaling_e2e.py b/dingo/planner/tests/manual/scaling/scaling_e2e.py similarity index 98% rename from components/src/dynamo/planner/tests/manual/scaling/scaling_e2e.py rename to dingo/planner/tests/manual/scaling/scaling_e2e.py index a8d1aa9cc91d..753746812602 100644 --- a/components/src/dynamo/planner/tests/manual/scaling/scaling_e2e.py +++ b/dingo/planner/tests/manual/scaling/scaling_e2e.py @@ -17,7 +17,7 @@ from dataclasses import dataclass from typing import Any, Dict, List, Optional, Tuple -from dynamo.planner.tests.unit.load_generator import LoadGenerator +from dingo.planner.tests.unit.load_generator import LoadGenerator logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" @@ -311,7 +311,7 @@ async def main(): "--save-results", action="store_true", help=( - "Save results to components/src/dynamo/planner/tests/e2e_scaling_results " + "Save results to dingo/planner/tests/e2e_scaling_results " "instead of /tmp" ), ) diff --git a/components/src/dynamo/planner/tests/manual/unit/load_test.sh b/dingo/planner/tests/manual/unit/load_test.sh similarity index 100% rename from components/src/dynamo/planner/tests/manual/unit/load_test.sh rename to dingo/planner/tests/manual/unit/load_test.sh diff --git a/components/src/dynamo/planner/tests/monitoring/__init__.py b/dingo/planner/tests/monitoring/__init__.py similarity index 100% rename from components/src/dynamo/planner/tests/monitoring/__init__.py rename to dingo/planner/tests/monitoring/__init__.py diff --git a/components/src/dynamo/planner/tests/monitoring/test_decision_state_enums.py b/dingo/planner/tests/monitoring/test_decision_state_enums.py similarity index 98% rename from components/src/dynamo/planner/tests/monitoring/test_decision_state_enums.py rename to dingo/planner/tests/monitoring/test_decision_state_enums.py index ee6f09a48c9c..9e6052b80168 100644 --- a/components/src/dynamo/planner/tests/monitoring/test_decision_state_enums.py +++ b/dingo/planner/tests/monitoring/test_decision_state_enums.py @@ -12,7 +12,7 @@ import pytest from prometheus_client import CollectorRegistry, Enum -from dynamo.planner.monitoring.planner_metrics import ( +from dingo.planner.monitoring.planner_metrics import ( LOAD_DECISION_STATES, THROUGHPUT_DECISION_STATES, ) diff --git a/components/src/dynamo/planner/tests/monitoring/test_plugin_framework_metrics.py b/dingo/planner/tests/monitoring/test_plugin_framework_metrics.py similarity index 99% rename from components/src/dynamo/planner/tests/monitoring/test_plugin_framework_metrics.py rename to dingo/planner/tests/monitoring/test_plugin_framework_metrics.py index 34b35d0dd33c..4d0b319667cd 100644 --- a/components/src/dynamo/planner/tests/monitoring/test_plugin_framework_metrics.py +++ b/dingo/planner/tests/monitoring/test_plugin_framework_metrics.py @@ -8,7 +8,7 @@ import pytest from prometheus_client import CollectorRegistry -from dynamo.planner.monitoring.planner_metrics import ( +from dingo.planner.monitoring.planner_metrics import ( CIRCUIT_STATE_CLOSED, CIRCUIT_STATE_HALF_OPEN, CIRCUIT_STATE_OPEN, diff --git a/components/src/dynamo/planner/tests/offline/__init__.py b/dingo/planner/tests/offline/__init__.py similarity index 100% rename from components/src/dynamo/planner/tests/offline/__init__.py rename to dingo/planner/tests/offline/__init__.py diff --git a/components/src/dynamo/planner/tests/offline/test_replay_adapter_fpm.py b/dingo/planner/tests/offline/test_replay_adapter_fpm.py similarity index 96% rename from components/src/dynamo/planner/tests/offline/test_replay_adapter_fpm.py rename to dingo/planner/tests/offline/test_replay_adapter_fpm.py index 15595ec34e75..0e625308f3de 100644 --- a/components/src/dynamo/planner/tests/offline/test_replay_adapter_fpm.py +++ b/dingo/planner/tests/offline/test_replay_adapter_fpm.py @@ -18,20 +18,20 @@ import pytest -from dynamo.mocker import MockEngineArgs -from dynamo.planner.config.planner_config import PlannerConfig -from dynamo.planner.core.types import ( +from dingo.mocker import MockEngineArgs +from dingo.planner.config.planner_config import PlannerConfig +from dingo.planner.core.types import ( EngineCapabilities, ScheduledTick, WorkerCapabilities, ) -from dynamo.planner.offline.replay_adapter import ( +from dingo.planner.offline.replay_adapter import ( ReplayPlannerAdapter, _build_fpm_from_dict, _merge_traffic, ) -from dynamo.planner.plugins.orchestrator.engine_adapter import OrchestratorEngineAdapter -from dynamo.replay.main import _engine_caps +from dingo.planner.plugins.orchestrator.engine_adapter import OrchestratorEngineAdapter +from dingo.replay.main import _engine_caps pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/planner/tests/offline/test_trace_data.py b/dingo/planner/tests/offline/test_trace_data.py similarity index 97% rename from components/src/dynamo/planner/tests/offline/test_trace_data.py rename to dingo/planner/tests/offline/test_trace_data.py index d09eb77a18e3..49085fea2da5 100644 --- a/components/src/dynamo/planner/tests/offline/test_trace_data.py +++ b/dingo/planner/tests/offline/test_trace_data.py @@ -18,7 +18,7 @@ import pytest -from dynamo.planner.offline.trace_data import extract_metrics_from_mooncake +from dingo.planner.offline.trace_data import extract_metrics_from_mooncake pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/planner/tests/plugins/__init__.py b/dingo/planner/tests/plugins/__init__.py similarity index 100% rename from components/src/dynamo/planner/tests/plugins/__init__.py rename to dingo/planner/tests/plugins/__init__.py diff --git a/components/src/dynamo/planner/tests/plugins/clock/__init__.py b/dingo/planner/tests/plugins/clock/__init__.py similarity index 100% rename from components/src/dynamo/planner/tests/plugins/clock/__init__.py rename to dingo/planner/tests/plugins/clock/__init__.py diff --git a/components/src/dynamo/planner/tests/plugins/clock/test_clocks.py b/dingo/planner/tests/plugins/clock/test_clocks.py similarity index 98% rename from components/src/dynamo/planner/tests/plugins/clock/test_clocks.py rename to dingo/planner/tests/plugins/clock/test_clocks.py index 91015fae5289..fff918629fec 100644 --- a/components/src/dynamo/planner/tests/plugins/clock/test_clocks.py +++ b/dingo/planner/tests/plugins/clock/test_clocks.py @@ -10,7 +10,7 @@ import pytest -from dynamo.planner.plugins.clock import VirtualClock, WallClock +from dingo.planner.plugins.clock import VirtualClock, WallClock pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/planner/tests/plugins/merge/__init__.py b/dingo/planner/tests/plugins/merge/__init__.py similarity index 100% rename from components/src/dynamo/planner/tests/plugins/merge/__init__.py rename to dingo/planner/tests/plugins/merge/__init__.py diff --git a/components/src/dynamo/planner/tests/plugins/merge/test_chain_augment.py b/dingo/planner/tests/plugins/merge/test_chain_augment.py similarity index 99% rename from components/src/dynamo/planner/tests/plugins/merge/test_chain_augment.py rename to dingo/planner/tests/plugins/merge/test_chain_augment.py index 5506f5df3841..1167e053a73a 100644 --- a/components/src/dynamo/planner/tests/plugins/merge/test_chain_augment.py +++ b/dingo/planner/tests/plugins/merge/test_chain_augment.py @@ -26,8 +26,8 @@ import pytest -from dynamo.planner.plugins.merge import chain_augment -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.merge import chain_augment +from dingo.planner.plugins.types import ( PipelineContext, PredictionData, PredictStageResponse, diff --git a/components/src/dynamo/planner/tests/plugins/merge/test_type_aware_basic.py b/dingo/planner/tests/plugins/merge/test_type_aware_basic.py similarity index 98% rename from components/src/dynamo/planner/tests/plugins/merge/test_type_aware_basic.py rename to dingo/planner/tests/plugins/merge/test_type_aware_basic.py index e75bd90da5b5..da89b03ff571 100644 --- a/components/src/dynamo/planner/tests/plugins/merge/test_type_aware_basic.py +++ b/dingo/planner/tests/plugins/merge/test_type_aware_basic.py @@ -24,13 +24,13 @@ import pytest -from dynamo.planner.plugins.merge import ( +from dingo.planner.plugins.merge import ( ComponentKey, MergeOutcome, PluginResult, type_aware_merge, ) -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.types import ( AcceptResult, ComponentTarget, OverrideResult, diff --git a/components/src/dynamo/planner/tests/plugins/merge/test_type_aware_clamp_tracking.py b/dingo/planner/tests/plugins/merge/test_type_aware_clamp_tracking.py similarity index 97% rename from components/src/dynamo/planner/tests/plugins/merge/test_type_aware_clamp_tracking.py rename to dingo/planner/tests/plugins/merge/test_type_aware_clamp_tracking.py index 7507fe8c1329..212a282681ad 100644 --- a/components/src/dynamo/planner/tests/plugins/merge/test_type_aware_clamp_tracking.py +++ b/dingo/planner/tests/plugins/merge/test_type_aware_clamp_tracking.py @@ -12,8 +12,8 @@ import pytest -from dynamo.planner.plugins.merge import ComponentKey, PluginResult, type_aware_merge -from dynamo.planner.plugins.types import ComponentTarget, OverrideResult, OverrideType +from dingo.planner.plugins.merge import ComponentKey, PluginResult, type_aware_merge +from dingo.planner.plugins.types import ComponentTarget, OverrideResult, OverrideType pytestmark = [ pytest.mark.gpu_0, @@ -230,7 +230,7 @@ def override_for(key, plugin_id, priority, ot, replicas): def test_short_circuit_leaves_clamped_empty(): - from dynamo.planner.plugins.types import RejectResult + from dingo.planner.plugins.types import RejectResult outcome = type_aware_merge( [ diff --git a/components/src/dynamo/planner/tests/plugins/merge/test_type_aware_constrain.py b/dingo/planner/tests/plugins/merge/test_type_aware_constrain.py similarity index 96% rename from components/src/dynamo/planner/tests/plugins/merge/test_type_aware_constrain.py rename to dingo/planner/tests/plugins/merge/test_type_aware_constrain.py index e9ec114aaab6..19ff7802a463 100644 --- a/components/src/dynamo/planner/tests/plugins/merge/test_type_aware_constrain.py +++ b/dingo/planner/tests/plugins/merge/test_type_aware_constrain.py @@ -17,13 +17,13 @@ import pytest -from dynamo.planner.plugins.merge import ( +from dingo.planner.plugins.merge import ( ComponentKey, MergeOutcome, PluginResult, type_aware_merge, ) -from dynamo.planner.plugins.types import ComponentTarget, OverrideResult, OverrideType +from dingo.planner.plugins.types import ComponentTarget, OverrideResult, OverrideType pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/planner/tests/plugins/merge/test_type_aware_short_circuit.py b/dingo/planner/tests/plugins/merge/test_type_aware_short_circuit.py similarity index 98% rename from components/src/dynamo/planner/tests/plugins/merge/test_type_aware_short_circuit.py rename to dingo/planner/tests/plugins/merge/test_type_aware_short_circuit.py index 86c139bda4ed..c5b7890adc3b 100644 --- a/components/src/dynamo/planner/tests/plugins/merge/test_type_aware_short_circuit.py +++ b/dingo/planner/tests/plugins/merge/test_type_aware_short_circuit.py @@ -22,8 +22,8 @@ import pytest -from dynamo.planner.plugins.merge import ComponentKey, PluginResult, type_aware_merge -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.merge import ComponentKey, PluginResult, type_aware_merge +from dingo.planner.plugins.types import ( ComponentTarget, OverrideResult, OverrideType, diff --git a/components/src/dynamo/planner/tests/plugins/merge/test_type_aware_worked_examples.py b/dingo/planner/tests/plugins/merge/test_type_aware_worked_examples.py similarity index 96% rename from components/src/dynamo/planner/tests/plugins/merge/test_type_aware_worked_examples.py rename to dingo/planner/tests/plugins/merge/test_type_aware_worked_examples.py index 1c2291d0ca16..acd9f3a0e608 100644 --- a/components/src/dynamo/planner/tests/plugins/merge/test_type_aware_worked_examples.py +++ b/dingo/planner/tests/plugins/merge/test_type_aware_worked_examples.py @@ -20,8 +20,8 @@ import pytest -from dynamo.planner.plugins.merge import ComponentKey, PluginResult, type_aware_merge -from dynamo.planner.plugins.types import ComponentTarget, OverrideResult, OverrideType +from dingo.planner.plugins.merge import ComponentKey, PluginResult, type_aware_merge +from dingo.planner.plugins.types import ComponentTarget, OverrideResult, OverrideType pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/planner/tests/plugins/orchestrator/__init__.py b/dingo/planner/tests/plugins/orchestrator/__init__.py similarity index 100% rename from components/src/dynamo/planner/tests/plugins/orchestrator/__init__.py rename to dingo/planner/tests/plugins/orchestrator/__init__.py diff --git a/components/src/dynamo/planner/tests/plugins/orchestrator/_fake_in_process_plugin.py b/dingo/planner/tests/plugins/orchestrator/_fake_in_process_plugin.py similarity index 88% rename from components/src/dynamo/planner/tests/plugins/orchestrator/_fake_in_process_plugin.py rename to dingo/planner/tests/plugins/orchestrator/_fake_in_process_plugin.py index 2779a0a40fca..470d0d25c73c 100644 --- a/components/src/dynamo/planner/tests/plugins/orchestrator/_fake_in_process_plugin.py +++ b/dingo/planner/tests/plugins/orchestrator/_fake_in_process_plugin.py @@ -9,7 +9,7 @@ from __future__ import annotations -from dynamo.planner.plugins.types import AcceptResult, ProposeStageResponse +from dingo.planner.plugins.types import AcceptResult, ProposeStageResponse class FakePlugin: diff --git a/components/src/dynamo/planner/tests/plugins/orchestrator/conftest.py b/dingo/planner/tests/plugins/orchestrator/conftest.py similarity index 87% rename from components/src/dynamo/planner/tests/plugins/orchestrator/conftest.py rename to dingo/planner/tests/plugins/orchestrator/conftest.py index 891a50d0a806..b8e229e6c3ed 100644 --- a/components/src/dynamo/planner/tests/plugins/orchestrator/conftest.py +++ b/dingo/planner/tests/plugins/orchestrator/conftest.py @@ -10,13 +10,13 @@ import pytest -from dynamo.planner.plugins.clock import VirtualClock -from dynamo.planner.plugins.orchestrator.orchestrator import LocalPlannerOrchestrator -from dynamo.planner.plugins.registry.auth import AllowUnauthenticatedAuth -from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker -from dynamo.planner.plugins.registry.server import PluginRegistryServer -from dynamo.planner.plugins.scheduler import PluginScheduler -from dynamo.planner.plugins.transport.config import ( +from dingo.planner.plugins.clock import VirtualClock +from dingo.planner.plugins.orchestrator.orchestrator import LocalPlannerOrchestrator +from dingo.planner.plugins.registry.auth import AllowUnauthenticatedAuth +from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker +from dingo.planner.plugins.registry.server import PluginRegistryServer +from dingo.planner.plugins.scheduler import PluginScheduler +from dingo.planner.plugins.transport.config import ( TransportConfig, make_transport_for_endpoint, ) diff --git a/components/src/dynamo/planner/tests/plugins/orchestrator/test_concurrency.py b/dingo/planner/tests/plugins/orchestrator/test_concurrency.py similarity index 98% rename from components/src/dynamo/planner/tests/plugins/orchestrator/test_concurrency.py rename to dingo/planner/tests/plugins/orchestrator/test_concurrency.py index a1b02b0c2342..4f3577423f0a 100644 --- a/components/src/dynamo/planner/tests/plugins/orchestrator/test_concurrency.py +++ b/dingo/planner/tests/plugins/orchestrator/test_concurrency.py @@ -20,8 +20,8 @@ import pytest -from dynamo.planner.plugins.merge.types import ComponentKey -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.merge.types import ComponentKey +from dingo.planner.plugins.types import ( AcceptResult, CircuitState, ComponentTarget, diff --git a/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py b/dingo/planner/tests/plugins/orchestrator/test_engine_adapter.py similarity index 96% rename from components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py rename to dingo/planner/tests/plugins/orchestrator/test_engine_adapter.py index 46ef19256bed..1e291d0ce9f8 100644 --- a/components/src/dynamo/planner/tests/plugins/orchestrator/test_engine_adapter.py +++ b/dingo/planner/tests/plugins/orchestrator/test_engine_adapter.py @@ -24,8 +24,8 @@ import pytest -from dynamo.planner.config.planner_config import PlannerConfig -from dynamo.planner.core.types import ( +from dingo.planner.config.planner_config import PlannerConfig +from dingo.planner.core.types import ( EngineCapabilities, FpmObservations, ScheduledTick, @@ -34,11 +34,11 @@ WorkerCapabilities, WorkerCounts, ) -from dynamo.planner.plugins.clock import VirtualClock -from dynamo.planner.plugins.merge.types import ChainAugmentOutcome -from dynamo.planner.plugins.orchestrator.engine_adapter import OrchestratorEngineAdapter -from dynamo.planner.plugins.orchestrator.pipeline import PipelineOutcome -from dynamo.planner.plugins.types import ComponentTarget, ScalingProposal +from dingo.planner.plugins.clock import VirtualClock +from dingo.planner.plugins.merge.types import ChainAugmentOutcome +from dingo.planner.plugins.orchestrator.engine_adapter import OrchestratorEngineAdapter +from dingo.planner.plugins.orchestrator.pipeline import PipelineOutcome +from dingo.planner.plugins.types import ComponentTarget, ScalingProposal pytestmark = [ pytest.mark.gpu_0, @@ -86,7 +86,7 @@ def _agg_config_custom_intervals( def _agg_config_with_in_process_plugin() -> PlannerConfig: - from dynamo.planner.plugins.registry.config import ( + from dingo.planner.plugins.registry.config import ( InProcessPluginSpec, PluginRegistrationConfig, ) @@ -101,7 +101,7 @@ def _agg_config_with_in_process_plugin() -> PlannerConfig: in_process_plugins=[ InProcessPluginSpec.model_validate( { - "module": "dynamo.planner.tests.plugins.orchestrator._fake_in_process_plugin", + "module": "dingo.planner.tests.plugins.orchestrator._fake_in_process_plugin", "class": "FakePlugin", "plugin_id": "configured_in_process", "plugin_type": "propose", @@ -148,7 +148,7 @@ def _disagg_config_sla_no_budget() -> PlannerConfig: def _make_fpm(worker_id: str = "w1", dp_rank: int = 0): - from dynamo.common.forward_pass_metrics import ( + from dingo.common.forward_pass_metrics import ( ForwardPassMetrics, QueuedRequestMetrics, ScheduledRequestMetrics, @@ -173,8 +173,8 @@ def _make_fpm(worker_id: str = "w1", dp_rank: int = 0): @pytest.mark.asyncio async def test_builtin_load_propose_refreshes_throughput_bound_before_load(): - from dynamo.planner.plugins.builtins.local_planner import BuiltinLoadPropose - from dynamo.planner.plugins.types import ( + from dingo.planner.plugins.builtins.local_planner import BuiltinLoadPropose + from dingo.planner.plugins.types import ( ObservationData, PipelineContext, PredictionData, @@ -231,7 +231,7 @@ def _build_real_regression(cfg: PlannerConfig, caps: WorkerCapabilities, kind: s reuse that construction so the test exercises the real type (which only exposes ``add_observations``, never the singular ``add_observation``). """ - from dynamo.planner.core.state_machine import PlannerScalingState + from dingo.planner.core.state_machine import PlannerScalingState state = PlannerScalingState(cfg, caps) return getattr(state, f"_{kind}_regression") @@ -410,7 +410,7 @@ def test_tick_input_to_context_maps_observations_and_fpm(): # The ingress glue: TickInput → PipelineContext.observations, including # the FPM msgpack encoding external plugins decode. Asserts field mapping # (traffic + worker scaling flags) and that the FPM bytes round-trip. - from dynamo.common.forward_pass_metrics import decode as _fpm_decode + from dingo.common.forward_pass_metrics import decode as _fpm_decode adapter = OrchestratorEngineAdapter(_agg_config_throughput_on(), _caps()) ti = TickInput( @@ -479,7 +479,7 @@ def test_orchestrator_path_honours_configured_protocol_version_range(): config, making any non-default range silently ineffective on the gateway). """ - from dynamo.planner.config.planner_config import PluginRegistrationConfig + from dingo.planner.config.planner_config import PluginRegistrationConfig config = PlannerConfig( mode="agg", @@ -510,7 +510,7 @@ async def test_tick_propagates_pipeline_execute_action_to_diagnostics(): populated prediction / load / throughput fields — the three execute-action fields were silently dropped. """ - from dynamo.planner.plugins.orchestrator.pipeline import PipelineOutcome + from dingo.planner.plugins.orchestrator.pipeline import PipelineOutcome adapter = OrchestratorEngineAdapter(_agg_config_throughput_on(), _caps()) @@ -990,7 +990,7 @@ def test_default_clock_is_wallclock(): refactor that flips it doesn't silently break production cadence tracking. """ - from dynamo.planner.plugins.clock import WallClock + from dingo.planner.plugins.clock import WallClock adapter = OrchestratorEngineAdapter(_agg_config_throughput_on(), _caps()) assert isinstance(adapter._clock, WallClock) @@ -1015,9 +1015,9 @@ def test_lazy_traffic_due_check_uses_monotonic_not_wall_epoch(): ``traffic_consumers_due`` and ``need_traffic_metrics`` would be ``True``. Fixed: monotonic projection correctly skips the plugin. """ - from dynamo.planner.plugins.clock import Clock - from dynamo.planner.plugins.registry.types import RegisteredPlugin - from dynamo.planner.plugins.types import HoldPolicy + from dingo.planner.plugins.clock import Clock + from dingo.planner.plugins.registry.types import RegisteredPlugin + from dingo.planner.plugins.types import HoldPolicy class _FixedMonoClock(Clock): """Wall-vs-monotonic drift simulator — not a VirtualClock so the @@ -1091,8 +1091,8 @@ def test_lazy_traffic_pull_matches_dot_path_sub_paths_of_observations_traffic(): trigger the pull — the trailing ``.`` in the prefix-match is load-bearing. Locks both branches. """ - from dynamo.planner.plugins.registry.types import RegisteredPlugin - from dynamo.planner.plugins.types import HoldPolicy + from dingo.planner.plugins.registry.types import RegisteredPlugin + from dingo.planner.plugins.types import HoldPolicy def _make_traffic_plugin(plugin_id: str, needs: list[str]) -> RegisteredPlugin: plugin = RegisteredPlugin( diff --git a/components/src/dynamo/planner/tests/plugins/orchestrator/test_in_process_loader.py b/dingo/planner/tests/plugins/orchestrator/test_in_process_loader.py similarity index 94% rename from components/src/dynamo/planner/tests/plugins/orchestrator/test_in_process_loader.py rename to dingo/planner/tests/plugins/orchestrator/test_in_process_loader.py index d0df0b051a6a..0c45852fb5b8 100644 --- a/components/src/dynamo/planner/tests/plugins/orchestrator/test_in_process_loader.py +++ b/dingo/planner/tests/plugins/orchestrator/test_in_process_loader.py @@ -7,10 +7,10 @@ import pytest -from dynamo.planner.plugins.orchestrator.in_process_loader import ( +from dingo.planner.plugins.orchestrator.in_process_loader import ( load_in_process_plugins, ) -from dynamo.planner.plugins.registry.config import InProcessPluginSpec +from dingo.planner.plugins.registry.config import InProcessPluginSpec pytestmark = [ pytest.mark.gpu_0, @@ -20,7 +20,7 @@ ] -FAKE_PLUGIN_MODULE = "dynamo.planner.tests.plugins.orchestrator._fake_in_process_plugin" +FAKE_PLUGIN_MODULE = "dingo.planner.tests.plugins.orchestrator._fake_in_process_plugin" def test_loader_registers_plugin_from_module_path(ctx_factory): @@ -70,7 +70,7 @@ def test_loader_raises_on_unknown_module(ctx_factory): ctx = ctx_factory() spec = InProcessPluginSpec.model_validate( { - "module": "dynamo.planner.tests.plugins.orchestrator.does_not_exist", + "module": "dingo.planner.tests.plugins.orchestrator.does_not_exist", "class": "Missing", "plugin_id": "x", "plugin_type": "propose", diff --git a/components/src/dynamo/planner/tests/plugins/orchestrator/test_orchestrator_lifecycle.py b/dingo/planner/tests/plugins/orchestrator/test_orchestrator_lifecycle.py similarity index 98% rename from components/src/dynamo/planner/tests/plugins/orchestrator/test_orchestrator_lifecycle.py rename to dingo/planner/tests/plugins/orchestrator/test_orchestrator_lifecycle.py index ff62ae2ff011..4a3c43664ba7 100644 --- a/components/src/dynamo/planner/tests/plugins/orchestrator/test_orchestrator_lifecycle.py +++ b/dingo/planner/tests/plugins/orchestrator/test_orchestrator_lifecycle.py @@ -7,8 +7,8 @@ import pytest -from dynamo.planner.plugins.merge.types import ComponentKey -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.merge.types import ComponentKey +from dingo.planner.plugins.types import ( ComponentTarget, OverrideResult, OverrideType, diff --git a/components/src/dynamo/planner/tests/plugins/orchestrator/test_pipeline.py b/dingo/planner/tests/plugins/orchestrator/test_pipeline.py similarity index 99% rename from components/src/dynamo/planner/tests/plugins/orchestrator/test_pipeline.py rename to dingo/planner/tests/plugins/orchestrator/test_pipeline.py index c744e863a982..dc8d2bdd4f87 100644 --- a/components/src/dynamo/planner/tests/plugins/orchestrator/test_pipeline.py +++ b/dingo/planner/tests/plugins/orchestrator/test_pipeline.py @@ -23,8 +23,8 @@ import pytest -from dynamo.planner.plugins.merge.types import ComponentKey -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.merge.types import ComponentKey +from dingo.planner.plugins.types import ( AcceptResult, ComponentTarget, ConstrainStageResponse, @@ -652,7 +652,7 @@ def test_pipeline_py_has_no_stage_level_wait_for(): (``_body()``), not an ``asyncio.gather``. Per-plugin timeouts already live in ``PluginTransport.call``; a stage-level wait_for would double- count the budget.""" - from dynamo.planner.plugins.orchestrator import pipeline as _pipeline_module + from dingo.planner.plugins.orchestrator import pipeline as _pipeline_module source_path = pathlib.Path(_pipeline_module.__file__) assert source_path.exists(), f"pipeline source not found at {source_path}" diff --git a/components/src/dynamo/planner/tests/plugins/orchestrator/test_pipeline_metrics.py b/dingo/planner/tests/plugins/orchestrator/test_pipeline_metrics.py similarity index 99% rename from components/src/dynamo/planner/tests/plugins/orchestrator/test_pipeline_metrics.py rename to dingo/planner/tests/plugins/orchestrator/test_pipeline_metrics.py index 4d0f2995bdee..71e12f308690 100644 --- a/components/src/dynamo/planner/tests/plugins/orchestrator/test_pipeline_metrics.py +++ b/dingo/planner/tests/plugins/orchestrator/test_pipeline_metrics.py @@ -13,12 +13,12 @@ import pytest from prometheus_client import CollectorRegistry -from dynamo.planner.monitoring.planner_metrics import ( +from dingo.planner.monitoring.planner_metrics import ( CIRCUIT_STATE_CLOSED, PluginFrameworkMetrics, ) -from dynamo.planner.plugins.merge.types import ComponentKey -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.merge.types import ComponentKey +from dingo.planner.plugins.types import ( AcceptResult, ComponentTarget, ConstrainStageResponse, diff --git a/components/src/dynamo/planner/tests/plugins/proto/__init__.py b/dingo/planner/tests/plugins/proto/__init__.py similarity index 100% rename from components/src/dynamo/planner/tests/plugins/proto/__init__.py rename to dingo/planner/tests/plugins/proto/__init__.py diff --git a/components/src/dynamo/planner/tests/plugins/proto/test_round_trip.py b/dingo/planner/tests/plugins/proto/test_round_trip.py similarity index 99% rename from components/src/dynamo/planner/tests/plugins/proto/test_round_trip.py rename to dingo/planner/tests/plugins/proto/test_round_trip.py index a49e31573949..50a5c323a043 100644 --- a/components/src/dynamo/planner/tests/plugins/proto/test_round_trip.py +++ b/dingo/planner/tests/plugins/proto/test_round_trip.py @@ -15,13 +15,13 @@ import pytest -from dynamo.planner.plugins import types as pyd -from dynamo.planner.plugins._proto_bridge import ( +from dingo.planner.plugins import types as pyd +from dingo.planner.plugins._proto_bridge import ( _PYD_TO_PROTO, proto_to_pydantic, pydantic_to_proto, ) -from dynamo.planner.plugins.proto.v1 import plugin_pb2 as pb +from dingo.planner.plugins.proto.v1 import plugin_pb2 as pb pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/planner/tests/plugins/registry/__init__.py b/dingo/planner/tests/plugins/registry/__init__.py similarity index 100% rename from components/src/dynamo/planner/tests/plugins/registry/__init__.py rename to dingo/planner/tests/plugins/registry/__init__.py diff --git a/components/src/dynamo/planner/tests/plugins/registry/auth/__init__.py b/dingo/planner/tests/plugins/registry/auth/__init__.py similarity index 100% rename from components/src/dynamo/planner/tests/plugins/registry/auth/__init__.py rename to dingo/planner/tests/plugins/registry/auth/__init__.py diff --git a/components/src/dynamo/planner/tests/plugins/registry/auth/test_allow_unauthenticated.py b/dingo/planner/tests/plugins/registry/auth/test_allow_unauthenticated.py similarity index 88% rename from components/src/dynamo/planner/tests/plugins/registry/auth/test_allow_unauthenticated.py rename to dingo/planner/tests/plugins/registry/auth/test_allow_unauthenticated.py index 682543ad6d1a..974ac71e9c8e 100644 --- a/components/src/dynamo/planner/tests/plugins/registry/auth/test_allow_unauthenticated.py +++ b/dingo/planner/tests/plugins/registry/auth/test_allow_unauthenticated.py @@ -13,7 +13,7 @@ import pytest -from dynamo.planner.plugins.registry.auth import AllowUnauthenticatedAuth +from dingo.planner.plugins.registry.auth import AllowUnauthenticatedAuth pytestmark = [ pytest.mark.gpu_0, @@ -25,7 +25,7 @@ def test_construction_emits_warning(caplog): with caplog.at_level( - logging.WARNING, logger="dynamo.planner.plugins.registry.auth.base" + logging.WARNING, logger="dingo.planner.plugins.registry.auth.base" ): AllowUnauthenticatedAuth() warnings = [r for r in caplog.records if r.levelno == logging.WARNING] diff --git a/components/src/dynamo/planner/tests/plugins/registry/auth/test_multi.py b/dingo/planner/tests/plugins/registry/auth/test_multi.py similarity index 95% rename from components/src/dynamo/planner/tests/plugins/registry/auth/test_multi.py rename to dingo/planner/tests/plugins/registry/auth/test_multi.py index 2e5c3f908ad0..81d275d6e45e 100644 --- a/components/src/dynamo/planner/tests/plugins/registry/auth/test_multi.py +++ b/dingo/planner/tests/plugins/registry/auth/test_multi.py @@ -7,12 +7,12 @@ import pytest -from dynamo.planner.plugins.registry.auth import ( +from dingo.planner.plugins.registry.auth import ( AuthIdentity, AuthValidator, MultiSourceAuth, ) -from dynamo.planner.plugins.registry.errors import AuthError +from dingo.planner.plugins.registry.errors import AuthError pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/planner/tests/plugins/registry/auth/test_static_secret.py b/dingo/planner/tests/plugins/registry/auth/test_static_secret.py similarity index 96% rename from components/src/dynamo/planner/tests/plugins/registry/auth/test_static_secret.py rename to dingo/planner/tests/plugins/registry/auth/test_static_secret.py index 69c775d3fa02..4c938ab20b0c 100644 --- a/components/src/dynamo/planner/tests/plugins/registry/auth/test_static_secret.py +++ b/dingo/planner/tests/plugins/registry/auth/test_static_secret.py @@ -7,8 +7,8 @@ import pytest -from dynamo.planner.plugins.registry.auth import StaticSecretAuth -from dynamo.planner.plugins.registry.errors import AuthError +from dingo.planner.plugins.registry.auth import StaticSecretAuth +from dingo.planner.plugins.registry.errors import AuthError pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/planner/tests/plugins/registry/test_circuit_breaker.py b/dingo/planner/tests/plugins/registry/test_circuit_breaker.py similarity index 96% rename from components/src/dynamo/planner/tests/plugins/registry/test_circuit_breaker.py rename to dingo/planner/tests/plugins/registry/test_circuit_breaker.py index a44c95d4bbd7..034aed92b196 100644 --- a/components/src/dynamo/planner/tests/plugins/registry/test_circuit_breaker.py +++ b/dingo/planner/tests/plugins/registry/test_circuit_breaker.py @@ -7,9 +7,9 @@ import pytest -from dynamo.planner.plugins.clock import VirtualClock -from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker -from dynamo.planner.plugins.types import CircuitState +from dingo.planner.plugins.clock import VirtualClock +from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker +from dingo.planner.plugins.types import CircuitState pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/planner/tests/plugins/registry/test_config.py b/dingo/planner/tests/plugins/registry/test_config.py similarity index 92% rename from components/src/dynamo/planner/tests/plugins/registry/test_config.py rename to dingo/planner/tests/plugins/registry/test_config.py index 8d37d7eab4ba..d9cfb6afe882 100644 --- a/components/src/dynamo/planner/tests/plugins/registry/test_config.py +++ b/dingo/planner/tests/plugins/registry/test_config.py @@ -10,16 +10,16 @@ import pytest from pydantic import ValidationError -from dynamo.planner.plugins.clock import VirtualClock -from dynamo.planner.plugins.registry.auth import MultiSourceAuth -from dynamo.planner.plugins.registry.config import ( +from dingo.planner.plugins.clock import VirtualClock +from dingo.planner.plugins.registry.auth import MultiSourceAuth +from dingo.planner.plugins.registry.config import ( AuthConfig, InProcessPluginSpec, PluginRegistrationConfig, build_auth_validator, build_registry_from_config, ) -from dynamo.planner.plugins.registry.server import PluginRegistryServer +from dingo.planner.plugins.registry.server import PluginRegistryServer pytestmark = [ pytest.mark.gpu_0, @@ -48,7 +48,7 @@ def test_static_secret_only_builds_multi_with_one_source(): def test_static_secret_empty_secrets_logs_warning(caplog): with caplog.at_level( - logging.WARNING, logger="dynamo.planner.plugins.registry.config" + logging.WARNING, logger="dingo.planner.plugins.registry.config" ): build_auth_validator( AuthConfig(trusted_sources=["static_secret"], static_secrets={}) @@ -106,8 +106,8 @@ def test_build_registry_from_config_returns_server_and_breaker(): @pytest.mark.asyncio async def test_build_registry_propagates_protocol_versions(): - from dynamo.planner.plugins.transport.config import TransportConfig - from dynamo.planner.plugins.types import RegisterRequest + from dingo.planner.plugins.transport.config import TransportConfig + from dingo.planner.plugins.types import RegisterRequest config = PluginRegistrationConfig( auth=AuthConfig(trusted_sources=["allow_unauthenticated"]), diff --git a/components/src/dynamo/planner/tests/plugins/registry/test_external_bootstrap.py b/dingo/planner/tests/plugins/registry/test_external_bootstrap.py similarity index 95% rename from components/src/dynamo/planner/tests/plugins/registry/test_external_bootstrap.py rename to dingo/planner/tests/plugins/registry/test_external_bootstrap.py index 858d4e1dd7ed..179046e5d0ca 100644 --- a/components/src/dynamo/planner/tests/plugins/registry/test_external_bootstrap.py +++ b/dingo/planner/tests/plugins/registry/test_external_bootstrap.py @@ -28,20 +28,20 @@ import pytest -from dynamo.planner.config.planner_config import ExternalPluginEntry -from dynamo.planner.plugins.clock import VirtualClock -from dynamo.planner.plugins.orchestrator.orchestrator import LocalPlannerOrchestrator -from dynamo.planner.plugins.registry.auth.base import ( +from dingo.planner.config.planner_config import ExternalPluginEntry +from dingo.planner.plugins.clock import VirtualClock +from dingo.planner.plugins.orchestrator.orchestrator import LocalPlannerOrchestrator +from dingo.planner.plugins.registry.auth.base import ( AllowUnauthenticatedAuth, AuthIdentity, AuthValidator, ) -from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker -from dynamo.planner.plugins.registry.errors import AuthError -from dynamo.planner.plugins.registry.server import PluginRegistryServer -from dynamo.planner.plugins.scheduler import PluginScheduler -from dynamo.planner.plugins.transport.base import PluginTransport -from dynamo.planner.plugins.types import HoldPolicy, ListPluginsRequest +from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker +from dingo.planner.plugins.registry.errors import AuthError +from dingo.planner.plugins.registry.server import PluginRegistryServer +from dingo.planner.plugins.scheduler import PluginScheduler +from dingo.planner.plugins.transport.base import PluginTransport +from dingo.planner.plugins.types import HoldPolicy, ListPluginsRequest pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/planner/tests/plugins/registry/test_gateway.py b/dingo/planner/tests/plugins/registry/test_gateway.py similarity index 91% rename from components/src/dynamo/planner/tests/plugins/registry/test_gateway.py rename to dingo/planner/tests/plugins/registry/test_gateway.py index 9747ad2ee47f..f7b058ac2ae5 100644 --- a/components/src/dynamo/planner/tests/plugins/registry/test_gateway.py +++ b/dingo/planner/tests/plugins/registry/test_gateway.py @@ -21,14 +21,14 @@ import grpc import pytest -from dynamo.planner.plugins.clock import VirtualClock -from dynamo.planner.plugins.proto.v1 import plugin_pb2 as pb -from dynamo.planner.plugins.registry.auth import AuthIdentity, AuthValidator -from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker -from dynamo.planner.plugins.registry.errors import AuthError -from dynamo.planner.plugins.registry.gateway import PluginRegistryGatewayServicer -from dynamo.planner.plugins.registry.server import PluginRegistryServer -from dynamo.planner.plugins.transport.base import PluginTransport +from dingo.planner.plugins.clock import VirtualClock +from dingo.planner.plugins.proto.v1 import plugin_pb2 as pb +from dingo.planner.plugins.registry.auth import AuthIdentity, AuthValidator +from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker +from dingo.planner.plugins.registry.errors import AuthError +from dingo.planner.plugins.registry.gateway import PluginRegistryGatewayServicer +from dingo.planner.plugins.registry.server import PluginRegistryServer +from dingo.planner.plugins.transport.base import PluginTransport pytestmark = [ pytest.mark.gpu_0, @@ -112,7 +112,7 @@ async def _register(server, plugin_id="p1", auth_token="A"): ) # Use the in-process method (already exercised in test_server.py); the # gateway's Register path goes through this same method too. - from dynamo.planner.plugins._proto_bridge import proto_to_pydantic + from dingo.planner.plugins._proto_bridge import proto_to_pydantic resp = await server.register(proto_to_pydantic(req)) assert resp.accepted is True @@ -225,8 +225,8 @@ async def test_start_gateway_server_raises_when_port_zero(): bad address, etc). The helper must catch that BEFORE starting the server so operators see a clear RuntimeError instead of a silently running gateway that accepts no connections.""" - from dynamo.planner.plugins.registry import gateway as gw_mod - from dynamo.planner.plugins.registry.gateway import start_gateway_server + from dingo.planner.plugins.registry import gateway as gw_mod + from dingo.planner.plugins.registry.gateway import start_gateway_server server, _ = _make_servicer() @@ -287,7 +287,7 @@ async def test_start_gateway_server_refuses_plaintext_tcp_without_allow_insecure closed — the gateway receives plugins' shared-secret auth tokens, so a plaintext TCP bind would leak them. Mirrors the outbound allow_insecure_grpc gate. The check raises BEFORE any bind.""" - from dynamo.planner.plugins.registry.gateway import start_gateway_server + from dingo.planner.plugins.registry.gateway import start_gateway_server server, _ = _make_servicer() with pytest.raises(RuntimeError, match="refusing to bind plaintext"): @@ -298,8 +298,8 @@ async def test_start_gateway_server_refuses_plaintext_tcp_without_allow_insecure async def test_start_gateway_server_allows_plaintext_tcp_when_opted_in(): """allow_insecure=True permits the plaintext TCP bind (operator accepted the risk); it must not hit the fail-closed guard.""" - from dynamo.planner.plugins.registry import gateway as gw_mod - from dynamo.planner.plugins.registry.gateway import start_gateway_server + from dingo.planner.plugins.registry import gateway as gw_mod + from dingo.planner.plugins.registry.gateway import start_gateway_server server, _ = _make_servicer() diff --git a/components/src/dynamo/planner/tests/plugins/registry/test_integration.py b/dingo/planner/tests/plugins/registry/test_integration.py similarity index 96% rename from components/src/dynamo/planner/tests/plugins/registry/test_integration.py rename to dingo/planner/tests/plugins/registry/test_integration.py index ebcfa62f5aca..ec376a5f24dc 100644 --- a/components/src/dynamo/planner/tests/plugins/registry/test_integration.py +++ b/dingo/planner/tests/plugins/registry/test_integration.py @@ -27,15 +27,15 @@ import pytest -from dynamo.planner.plugins.clock import VirtualClock -from dynamo.planner.plugins.registry.config import ( +from dingo.planner.plugins.clock import VirtualClock +from dingo.planner.plugins.registry.config import ( AuthConfig, PluginRegistrationConfig, build_registry_from_config, ) -from dynamo.planner.plugins.scheduler import PluginScheduler -from dynamo.planner.plugins.transport.base import PluginTransport -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.scheduler import PluginScheduler +from dingo.planner.plugins.transport.base import PluginTransport +from dingo.planner.plugins.types import ( CircuitState, ComponentTarget, HoldPolicy, @@ -78,7 +78,7 @@ def _factory(plugin_id, endpoint, config, *, in_process_instance=None): # ``registry.config`` defers its ``make_transport_for_endpoint`` import # to call time, so we monkeypatch at the *source* module. monkeypatch.setattr( - "dynamo.planner.plugins.transport.config.make_transport_for_endpoint", + "dingo.planner.plugins.transport.config.make_transport_for_endpoint", _factory, ) diff --git a/components/src/dynamo/planner/tests/plugins/registry/test_list_plugins.py b/dingo/planner/tests/plugins/registry/test_list_plugins.py similarity index 92% rename from components/src/dynamo/planner/tests/plugins/registry/test_list_plugins.py rename to dingo/planner/tests/plugins/registry/test_list_plugins.py index 1b46be385654..67a7a4a981ef 100644 --- a/components/src/dynamo/planner/tests/plugins/registry/test_list_plugins.py +++ b/dingo/planner/tests/plugins/registry/test_list_plugins.py @@ -13,13 +13,13 @@ import pytest -from dynamo.planner.plugins.clock import VirtualClock -from dynamo.planner.plugins.registry.auth import AllowUnauthenticatedAuth -from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker -from dynamo.planner.plugins.registry.server import PluginRegistryServer -from dynamo.planner.plugins.scheduler import PluginScheduler -from dynamo.planner.plugins.transport.base import PluginTransport -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.clock import VirtualClock +from dingo.planner.plugins.registry.auth import AllowUnauthenticatedAuth +from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker +from dingo.planner.plugins.registry.server import PluginRegistryServer +from dingo.planner.plugins.scheduler import PluginScheduler +from dingo.planner.plugins.transport.base import PluginTransport +from dingo.planner.plugins.types import ( CircuitState, ComponentTarget, HoldPolicy, diff --git a/components/src/dynamo/planner/tests/plugins/registry/test_server.py b/dingo/planner/tests/plugins/registry/test_server.py similarity index 97% rename from components/src/dynamo/planner/tests/plugins/registry/test_server.py rename to dingo/planner/tests/plugins/registry/test_server.py index a18ced24ebe8..08ba51450277 100644 --- a/components/src/dynamo/planner/tests/plugins/registry/test_server.py +++ b/dingo/planner/tests/plugins/registry/test_server.py @@ -9,17 +9,17 @@ import pytest -from dynamo.planner.plugins.clock import VirtualClock -from dynamo.planner.plugins.registry.auth import ( +from dingo.planner.plugins.clock import VirtualClock +from dingo.planner.plugins.registry.auth import ( AuthIdentity, AuthValidator, StaticSecretAuth, ) -from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker -from dynamo.planner.plugins.registry.errors import AuthError -from dynamo.planner.plugins.registry.server import PluginRegistryServer -from dynamo.planner.plugins.transport.base import PluginTransport -from dynamo.planner.plugins.types import HoldPolicy, ListPluginsRequest, RegisterRequest +from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker +from dingo.planner.plugins.registry.errors import AuthError +from dingo.planner.plugins.registry.server import PluginRegistryServer +from dingo.planner.plugins.transport.base import PluginTransport +from dingo.planner.plugins.types import HoldPolicy, ListPluginsRequest, RegisterRequest pytestmark = [ pytest.mark.gpu_0, @@ -568,7 +568,7 @@ async def test_register_resets_circuit_breaker_for_plugin_id(): cb.record_failure("p1") # threshold default 5 -> OPEN await server.register(_req()) # After register the breaker state should be fresh. - from dynamo.planner.plugins.types import CircuitState + from dingo.planner.plugins.types import CircuitState assert cb.state("p1") == CircuitState.CLOSED @@ -580,6 +580,6 @@ async def test_unregister_resets_circuit_breaker(): for _ in range(5): cb.record_failure("p1") await server.unregister("p1") - from dynamo.planner.plugins.types import CircuitState + from dingo.planner.plugins.types import CircuitState assert cb.state("p1") == CircuitState.CLOSED diff --git a/components/src/dynamo/planner/tests/plugins/scheduler/__init__.py b/dingo/planner/tests/plugins/scheduler/__init__.py similarity index 100% rename from components/src/dynamo/planner/tests/plugins/scheduler/__init__.py rename to dingo/planner/tests/plugins/scheduler/__init__.py diff --git a/components/src/dynamo/planner/tests/plugins/scheduler/test_active_set.py b/dingo/planner/tests/plugins/scheduler/test_active_set.py similarity index 96% rename from components/src/dynamo/planner/tests/plugins/scheduler/test_active_set.py rename to dingo/planner/tests/plugins/scheduler/test_active_set.py index ccda04830a3d..e1524fc34ff5 100644 --- a/components/src/dynamo/planner/tests/plugins/scheduler/test_active_set.py +++ b/dingo/planner/tests/plugins/scheduler/test_active_set.py @@ -7,13 +7,13 @@ import pytest -from dynamo.planner.plugins.clock import VirtualClock -from dynamo.planner.plugins.registry.auth import AllowUnauthenticatedAuth -from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker -from dynamo.planner.plugins.registry.server import PluginRegistryServer -from dynamo.planner.plugins.scheduler import PluginScheduler -from dynamo.planner.plugins.transport.base import PluginTransport -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.clock import VirtualClock +from dingo.planner.plugins.registry.auth import AllowUnauthenticatedAuth +from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker +from dingo.planner.plugins.registry.server import PluginRegistryServer +from dingo.planner.plugins.scheduler import PluginScheduler +from dingo.planner.plugins.transport.base import PluginTransport +from dingo.planner.plugins.types import ( ComponentTarget, HoldPolicy, OverrideResult, diff --git a/components/src/dynamo/planner/tests/plugins/scheduler/test_cache_invalidation.py b/dingo/planner/tests/plugins/scheduler/test_cache_invalidation.py similarity index 93% rename from components/src/dynamo/planner/tests/plugins/scheduler/test_cache_invalidation.py rename to dingo/planner/tests/plugins/scheduler/test_cache_invalidation.py index fef5a9fb130b..cae76dfada5c 100644 --- a/components/src/dynamo/planner/tests/plugins/scheduler/test_cache_invalidation.py +++ b/dingo/planner/tests/plugins/scheduler/test_cache_invalidation.py @@ -12,13 +12,13 @@ import pytest -from dynamo.planner.plugins.clock import VirtualClock -from dynamo.planner.plugins.registry.auth import AllowUnauthenticatedAuth -from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker -from dynamo.planner.plugins.registry.server import PluginRegistryServer -from dynamo.planner.plugins.scheduler import PluginScheduler -from dynamo.planner.plugins.transport.base import PluginTransport -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.clock import VirtualClock +from dingo.planner.plugins.registry.auth import AllowUnauthenticatedAuth +from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker +from dingo.planner.plugins.registry.server import PluginRegistryServer +from dingo.planner.plugins.scheduler import PluginScheduler +from dingo.planner.plugins.transport.base import PluginTransport +from dingo.planner.plugins.types import ( ComponentTarget, HoldPolicy, OverrideResult, diff --git a/components/src/dynamo/planner/tests/plugins/scheduler/test_phase_alignment.py b/dingo/planner/tests/plugins/scheduler/test_phase_alignment.py similarity index 93% rename from components/src/dynamo/planner/tests/plugins/scheduler/test_phase_alignment.py rename to dingo/planner/tests/plugins/scheduler/test_phase_alignment.py index 2ba38c21888e..4004f8dfc02c 100644 --- a/components/src/dynamo/planner/tests/plugins/scheduler/test_phase_alignment.py +++ b/dingo/planner/tests/plugins/scheduler/test_phase_alignment.py @@ -23,12 +23,12 @@ import pytest -from dynamo.planner.plugins.clock import VirtualClock -from dynamo.planner.plugins.registry.auth import AllowUnauthenticatedAuth -from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker -from dynamo.planner.plugins.registry.server import PluginRegistryServer -from dynamo.planner.plugins.transport.base import PluginTransport -from dynamo.planner.plugins.types import HoldPolicy, RegisterRequest +from dingo.planner.plugins.clock import VirtualClock +from dingo.planner.plugins.registry.auth import AllowUnauthenticatedAuth +from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker +from dingo.planner.plugins.registry.server import PluginRegistryServer +from dingo.planner.plugins.transport.base import PluginTransport +from dingo.planner.plugins.types import HoldPolicy, RegisterRequest pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/planner/tests/plugins/scheduler/test_requires_produced_fields.py b/dingo/planner/tests/plugins/scheduler/test_requires_produced_fields.py similarity index 95% rename from components/src/dynamo/planner/tests/plugins/scheduler/test_requires_produced_fields.py rename to dingo/planner/tests/plugins/scheduler/test_requires_produced_fields.py index 440d330c0103..df1e31a65fef 100644 --- a/components/src/dynamo/planner/tests/plugins/scheduler/test_requires_produced_fields.py +++ b/dingo/planner/tests/plugins/scheduler/test_requires_produced_fields.py @@ -19,13 +19,13 @@ import pytest -from dynamo.planner.plugins.clock import VirtualClock -from dynamo.planner.plugins.registry.auth import AllowUnauthenticatedAuth -from dynamo.planner.plugins.registry.circuit_breaker import CircuitBreaker -from dynamo.planner.plugins.registry.server import PluginRegistryServer -from dynamo.planner.plugins.scheduler import PluginScheduler -from dynamo.planner.plugins.transport.base import PluginTransport -from dynamo.planner.plugins.types import ( +from dingo.planner.plugins.clock import VirtualClock +from dingo.planner.plugins.registry.auth import AllowUnauthenticatedAuth +from dingo.planner.plugins.registry.circuit_breaker import CircuitBreaker +from dingo.planner.plugins.registry.server import PluginRegistryServer +from dingo.planner.plugins.scheduler import PluginScheduler +from dingo.planner.plugins.transport.base import PluginTransport +from dingo.planner.plugins.types import ( HoldPolicy, ObservationData, PipelineContext, diff --git a/components/src/dynamo/planner/tests/plugins/transport/__init__.py b/dingo/planner/tests/plugins/transport/__init__.py similarity index 100% rename from components/src/dynamo/planner/tests/plugins/transport/__init__.py rename to dingo/planner/tests/plugins/transport/__init__.py diff --git a/components/src/dynamo/planner/tests/plugins/transport/test_config.py b/dingo/planner/tests/plugins/transport/test_config.py similarity index 94% rename from components/src/dynamo/planner/tests/plugins/transport/test_config.py rename to dingo/planner/tests/plugins/transport/test_config.py index 971cc3a83f64..0d775ba16d2e 100644 --- a/components/src/dynamo/planner/tests/plugins/transport/test_config.py +++ b/dingo/planner/tests/plugins/transport/test_config.py @@ -7,9 +7,9 @@ import pytest -from dynamo.planner.plugins.clock import VirtualClock, WallClock -from dynamo.planner.plugins.transport import GrpcTransport, InProcessTransport -from dynamo.planner.plugins.transport.config import ( +from dingo.planner.plugins.clock import VirtualClock, WallClock +from dingo.planner.plugins.transport import GrpcTransport, InProcessTransport +from dingo.planner.plugins.transport.config import ( ClockConfig, TransportConfig, make_clock, @@ -125,7 +125,7 @@ def test_grpc_channel_options_honours_kwargs(): """The channel-options builder must echo its kwargs into the ``grpc.keepalive_time_ms`` / ``grpc.max_*_message_length`` entries so the GrpcTransport channel uses the right values.""" - from dynamo.planner.plugins.transport._grpc_base import grpc_channel_options + from dingo.planner.plugins.transport._grpc_base import grpc_channel_options opts = dict( grpc_channel_options(keepalive_time_ms=7_777, max_message_size_bytes=4096) diff --git a/components/src/dynamo/planner/tests/plugins/transport/test_in_process.py b/dingo/planner/tests/plugins/transport/test_in_process.py similarity index 98% rename from components/src/dynamo/planner/tests/plugins/transport/test_in_process.py rename to dingo/planner/tests/plugins/transport/test_in_process.py index dfce8552ab90..43a779f06a83 100644 --- a/components/src/dynamo/planner/tests/plugins/transport/test_in_process.py +++ b/dingo/planner/tests/plugins/transport/test_in_process.py @@ -9,7 +9,7 @@ import pytest -from dynamo.planner.plugins.transport import ( +from dingo.planner.plugins.transport import ( InProcessTransport, PluginCallError, PluginConnectionError, diff --git a/components/src/dynamo/planner/tests/plugins/transport/test_transport_contract.py b/dingo/planner/tests/plugins/transport/test_transport_contract.py similarity index 98% rename from components/src/dynamo/planner/tests/plugins/transport/test_transport_contract.py rename to dingo/planner/tests/plugins/transport/test_transport_contract.py index 179518e0c21c..9b4e3f551895 100644 --- a/components/src/dynamo/planner/tests/plugins/transport/test_transport_contract.py +++ b/dingo/planner/tests/plugins/transport/test_transport_contract.py @@ -21,9 +21,9 @@ import grpc import pytest -from dynamo.planner.plugins.proto.v1 import plugin_pb2 as pb -from dynamo.planner.plugins.proto.v1 import plugin_pb2_grpc as pbg -from dynamo.planner.plugins.transport import ( +from dingo.planner.plugins.proto.v1 import plugin_pb2 as pb +from dingo.planner.plugins.proto.v1 import plugin_pb2_grpc as pbg +from dingo.planner.plugins.transport import ( GrpcTransport, InProcessTransport, PluginConnectionError, diff --git a/components/src/dynamo/planner/tests/unit/load_generator.py b/dingo/planner/tests/unit/load_generator.py similarity index 99% rename from components/src/dynamo/planner/tests/unit/load_generator.py rename to dingo/planner/tests/unit/load_generator.py index cee2d2de0a4c..844384042595 100644 --- a/components/src/dynamo/planner/tests/unit/load_generator.py +++ b/dingo/planner/tests/unit/load_generator.py @@ -353,7 +353,7 @@ async def main(): parser.add_argument( "--save-results", action="store_true", - help="Save results to components/src/dynamo/planner/tests/data instead of /tmp", + help="Save results to dingo/planner/tests/data instead of /tmp", ) args = parser.parse_args() diff --git a/components/src/dynamo/planner/tests/unit/test_advisory_mode.py b/dingo/planner/tests/unit/test_advisory_mode.py similarity index 93% rename from components/src/dynamo/planner/tests/unit/test_advisory_mode.py rename to dingo/planner/tests/unit/test_advisory_mode.py index f0c4968b72b8..032d14342110 100644 --- a/components/src/dynamo/planner/tests/unit/test_advisory_mode.py +++ b/dingo/planner/tests/unit/test_advisory_mode.py @@ -5,7 +5,7 @@ import pytest -from dynamo.planner.config.defaults import SLAPlannerDefaults +from dingo.planner.config.defaults import SLAPlannerDefaults pytestmark = [ pytest.mark.gpu_0, @@ -41,7 +41,7 @@ def _stub_heavy_deps(): "FpmEventSubscriber": MagicMock, "FpmEventRelay": MagicMock, }, - "dynamo.common.forward_pass_metrics": { + "dingo.common.forward_pass_metrics": { "ForwardPassMetrics": MagicMock, "ScheduledRequestMetrics": MagicMock, }, @@ -65,7 +65,7 @@ def test_default_is_false(self): class TestPlannerConfigAdvisory: def test_config_with_advisory(self): - from dynamo.planner.config.planner_config import PlannerConfig + from dingo.planner.config.planner_config import PlannerConfig config = PlannerConfig.model_construct( mode="agg", @@ -74,7 +74,7 @@ def test_config_with_advisory(self): assert config.advisory is True def test_config_default_is_false(self): - from dynamo.planner.config.planner_config import PlannerConfig + from dingo.planner.config.planner_config import PlannerConfig config = PlannerConfig.model_construct(mode="agg") assert config.advisory is False diff --git a/components/src/dynamo/planner/tests/unit/test_aic_interpolation.py b/dingo/planner/tests/unit/test_aic_interpolation.py similarity index 98% rename from components/src/dynamo/planner/tests/unit/test_aic_interpolation.py rename to dingo/planner/tests/unit/test_aic_interpolation.py index 43e3bbc9b49a..ccb0ddb7a4b9 100644 --- a/components/src/dynamo/planner/tests/unit/test_aic_interpolation.py +++ b/dingo/planner/tests/unit/test_aic_interpolation.py @@ -12,13 +12,13 @@ import pytest -from dynamo.planner.config.aic_interpolation_spec import AICInterpolationSpec -from dynamo.planner.config.defaults import SubComponentType -from dynamo.planner.config.parallelization import ( +from dingo.planner.config.aic_interpolation_spec import AICInterpolationSpec +from dingo.planner.config.defaults import SubComponentType +from dingo.planner.config.parallelization import ( PickedParallelConfig, picked_to_aic_model_config_kwargs, ) -from dynamo.planner.monitoring import aic_interpolation as aic_mod +from dingo.planner.monitoring import aic_interpolation as aic_mod pytestmark = [ pytest.mark.gpu_0, @@ -71,7 +71,7 @@ def _patch_estimator( cls = MagicMock(return_value=instance) return ( patch( - "dynamo.planner.monitoring.aic_estimator.AIConfiguratorPerfEstimator", + "dingo.planner.monitoring.aic_estimator.AIConfiguratorPerfEstimator", cls, ), instance, diff --git a/components/src/dynamo/planner/tests/unit/test_budget.py b/dingo/planner/tests/unit/test_budget.py similarity index 99% rename from components/src/dynamo/planner/tests/unit/test_budget.py rename to dingo/planner/tests/unit/test_budget.py index 4423c80b77e5..052e3e60a833 100644 --- a/components/src/dynamo/planner/tests/unit/test_budget.py +++ b/dingo/planner/tests/unit/test_budget.py @@ -2,11 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 """Unit tests for the shared GPU budget primitives in -``dynamo.planner.core.budget``.""" +``dingo.planner.core.budget``.""" import pytest -from dynamo.planner.core.budget import ( +from dingo.planner.core.budget import ( bounds_for_total, compute_tolerance, proportional_clamp_pair, diff --git a/components/src/dynamo/planner/tests/unit/test_diagnostics_recorder.py b/dingo/planner/tests/unit/test_diagnostics_recorder.py similarity index 98% rename from components/src/dynamo/planner/tests/unit/test_diagnostics_recorder.py rename to dingo/planner/tests/unit/test_diagnostics_recorder.py index b7e7de865847..de7458bb65a1 100644 --- a/components/src/dynamo/planner/tests/unit/test_diagnostics_recorder.py +++ b/dingo/planner/tests/unit/test_diagnostics_recorder.py @@ -21,15 +21,15 @@ pytest.skip("msgspec required for FPM data", allow_module_level=True) try: - from dynamo.common.forward_pass_metrics import ( + from dingo.common.forward_pass_metrics import ( ForwardPassMetrics, QueuedRequestMetrics, ScheduledRequestMetrics, ) except ImportError: pytest.skip("forward_pass_metrics not available", allow_module_level=True) -from dynamo.planner.config.planner_config import PlannerConfig -from dynamo.planner.core.types import ( +from dingo.planner.config.planner_config import PlannerConfig +from dingo.planner.core.types import ( FpmObservations, PlannerEffects, ScalingDecision, @@ -37,8 +37,8 @@ TickInput, WorkerCounts, ) -from dynamo.planner.monitoring.diagnostics_recorder import DiagnosticsRecorder -from dynamo.planner.monitoring.traffic_metrics import Metrics +from dingo.planner.monitoring.diagnostics_recorder import DiagnosticsRecorder +from dingo.planner.monitoring.traffic_metrics import Metrics pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/planner/tests/unit/test_kube.py b/dingo/planner/tests/unit/test_kube.py similarity index 98% rename from components/src/dynamo/planner/tests/unit/test_kube.py rename to dingo/planner/tests/unit/test_kube.py index 8f6fe244f064..184f5c58fa9f 100644 --- a/components/src/dynamo/planner/tests/unit/test_kube.py +++ b/dingo/planner/tests/unit/test_kube.py @@ -19,8 +19,8 @@ import pytest from kubernetes import client -from dynamo.planner.connectors.kubernetes_api import KubernetesAPI -from dynamo.planner.errors import DynamoGraphDeploymentNotFoundError +from dingo.planner.connectors.kubernetes_api import KubernetesAPI +from dingo.planner.errors import DynamoGraphDeploymentNotFoundError pytestmark = [ pytest.mark.gpu_0, @@ -32,7 +32,7 @@ @pytest.fixture def mock_config(): - with patch("dynamo.planner.connectors.kubernetes_api.config") as mock: + with patch("dingo.planner.connectors.kubernetes_api.config") as mock: mock.load_incluster_config = MagicMock() yield mock @@ -40,7 +40,7 @@ def mock_config(): @pytest.fixture def mock_custom_api(): with patch( - "dynamo.planner.connectors.kubernetes_api.client.CustomObjectsApi" + "dingo.planner.connectors.kubernetes_api.client.CustomObjectsApi" ) as mock: yield mock.return_value @@ -48,7 +48,7 @@ def mock_custom_api(): @pytest.fixture def mock_namespace(): with patch( - "dynamo.planner.connectors.kubernetes_api.get_current_k8s_namespace", + "dingo.planner.connectors.kubernetes_api.get_current_k8s_namespace", return_value="default", ) as mock: yield mock diff --git a/components/src/dynamo/planner/tests/unit/test_kubernetes_connector.py b/dingo/planner/tests/unit/test_kubernetes_connector.py similarity index 99% rename from components/src/dynamo/planner/tests/unit/test_kubernetes_connector.py rename to dingo/planner/tests/unit/test_kubernetes_connector.py index 262439f15e45..fd8e9e07b4b4 100644 --- a/components/src/dynamo/planner/tests/unit/test_kubernetes_connector.py +++ b/dingo/planner/tests/unit/test_kubernetes_connector.py @@ -18,9 +18,9 @@ import pytest -from dynamo.planner.config.defaults import SubComponentType, TargetReplica -from dynamo.planner.connectors.kubernetes import KubernetesConnector -from dynamo.planner.errors import ( +from dingo.planner.config.defaults import SubComponentType, TargetReplica +from dingo.planner.connectors.kubernetes import KubernetesConnector +from dingo.planner.errors import ( DeploymentModelNameMismatchError, DeploymentValidationError, DuplicateSubComponentError, @@ -29,7 +29,7 @@ ModelNameNotFoundError, SubComponentNotFoundError, ) -from dynamo.planner.monitoring.dgd_services import ( +from dingo.planner.monitoring.dgd_services import ( Service, get_component_from_type_or_name, ) @@ -63,7 +63,7 @@ def mock_kube_api_class(mock_kube_api): def kubernetes_connector(mock_kube_api_class, monkeypatch): # Patch the KubernetesAPI class before instantiating the connector monkeypatch.setattr( - "dynamo.planner.connectors.kubernetes.KubernetesAPI", mock_kube_api_class + "dingo.planner.connectors.kubernetes.KubernetesAPI", mock_kube_api_class ) with patch.dict(os.environ, {"DYN_PARENT_DGD_K8S_NAME": "test-graph"}): connector = KubernetesConnector("test-dynamo-namespace") @@ -100,7 +100,7 @@ def _deployment(*components): def test_kubernetes_connector_no_env_var(): - with patch("dynamo.planner.connectors.kubernetes.KubernetesAPI"): + with patch("dingo.planner.connectors.kubernetes.KubernetesAPI"): with pytest.raises(DeploymentValidationError) as exc_info: KubernetesConnector("test-dynamo-namespace") diff --git a/components/src/dynamo/planner/tests/unit/test_load_based_scaling.py b/dingo/planner/tests/unit/test_load_based_scaling.py similarity index 98% rename from components/src/dynamo/planner/tests/unit/test_load_based_scaling.py rename to dingo/planner/tests/unit/test_load_based_scaling.py index 370b927ecf00..705e1fbfa565 100644 --- a/components/src/dynamo/planner/tests/unit/test_load_based_scaling.py +++ b/dingo/planner/tests/unit/test_load_based_scaling.py @@ -20,21 +20,21 @@ pytest.skip("msgspec required for FPM tests", allow_module_level=True) try: - from dynamo.common.forward_pass_metrics import ( + from dingo.common.forward_pass_metrics import ( ForwardPassMetrics, QueuedRequestMetrics, ScheduledRequestMetrics, ) except ImportError: pytest.skip("forward_pass_metrics not available", allow_module_level=True) -from dynamo.planner.config.planner_config import PlannerConfig -from dynamo.planner.core.base import NativePlannerBase -from dynamo.planner.core.perf_model import ( +from dingo.planner.config.planner_config import PlannerConfig +from dingo.planner.core.base import NativePlannerBase +from dingo.planner.core.perf_model import ( AggRegressionModel, DecodeRegressionModel, PrefillRegressionModel, ) -from dynamo.planner.monitoring.worker_info import WorkerInfo +from dingo.planner.monitoring.worker_info import WorkerInfo pytestmark = [ pytest.mark.gpu_0, @@ -774,8 +774,8 @@ def _make_planner(self, require_prefill=False, require_decode=True): # tests). KubernetesConnector.__init__ loads ~/.kube/config and reads # DYN_PARENT_DGD_K8S_NAME; stub both so this runs in plain pytest envs. with ( - patch("dynamo.planner.core.base.PlannerPrometheusMetrics") as mock_metrics, - patch("dynamo.planner.connectors.kubernetes.KubernetesAPI"), + patch("dingo.planner.core.base.PlannerPrometheusMetrics") as mock_metrics, + patch("dingo.planner.connectors.kubernetes.KubernetesAPI"), patch.dict(os.environ, {"DYN_PARENT_DGD_K8S_NAME": "test-graph"}), ): mock_metrics.return_value = Mock() diff --git a/components/src/dynamo/planner/tests/unit/test_load_generator.py b/dingo/planner/tests/unit/test_load_generator.py similarity index 100% rename from components/src/dynamo/planner/tests/unit/test_load_generator.py rename to dingo/planner/tests/unit/test_load_generator.py diff --git a/components/src/dynamo/planner/tests/unit/test_load_predictors.py b/dingo/planner/tests/unit/test_load_predictors.py similarity index 97% rename from components/src/dynamo/planner/tests/unit/test_load_predictors.py rename to dingo/planner/tests/unit/test_load_predictors.py index 0afe13d0b196..af8777a94670 100644 --- a/components/src/dynamo/planner/tests/unit/test_load_predictors.py +++ b/dingo/planner/tests/unit/test_load_predictors.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Unit tests for load predictor classes in dynamo.planner.core.load.predictors.""" +"""Unit tests for load predictor classes in dingo.planner.core.load.predictors.""" import math from datetime import datetime, timedelta @@ -22,7 +22,7 @@ import pandas as pd import pytest -from dynamo.planner.core.load.predictors import ( +from dingo.planner.core.load.predictors import ( ConstantPredictor, KalmanPredictor, ProphetPredictor, @@ -175,7 +175,7 @@ def fake_predict(df): mock_model.predict.side_effect = fake_predict with patch( - "dynamo.planner.core.load.predictors.Prophet", + "dingo.planner.core.load.predictors.Prophet", return_value=mock_model, ): predictor.predict_next() @@ -212,7 +212,7 @@ def fake_predict(df): mock_model.predict.side_effect = fake_predict with patch( - "dynamo.planner.core.load.predictors.Prophet", + "dingo.planner.core.load.predictors.Prophet", return_value=mock_model, ): predictor.predict_next() @@ -248,7 +248,7 @@ def fake_predict(df): mock_model.predict.side_effect = fake_predict with patch( - "dynamo.planner.core.load.predictors.Prophet", + "dingo.planner.core.load.predictors.Prophet", return_value=mock_model, ): predictor.predict_next() @@ -283,7 +283,7 @@ def test_predict_next_returns_non_negative_raw_mode(self): mock_model.predict.return_value = self._mock_forecast_df(-5.0) with patch( - "dynamo.planner.core.load.predictors.Prophet", + "dingo.planner.core.load.predictors.Prophet", return_value=mock_model, ): result = predictor.predict_next() @@ -301,7 +301,7 @@ def test_predict_next_returns_non_negative_log1p_mode(self): mock_model.predict.return_value = self._mock_forecast_df(-100.0) with patch( - "dynamo.planner.core.load.predictors.Prophet", + "dingo.planner.core.load.predictors.Prophet", return_value=mock_model, ): result = predictor.predict_next() @@ -395,7 +395,7 @@ def fake_predict(df): mock_model.predict.side_effect = fake_predict with patch( - "dynamo.planner.core.load.predictors.Prophet", + "dingo.planner.core.load.predictors.Prophet", return_value=mock_model, ): predictor.predict_next() diff --git a/components/src/dynamo/planner/tests/unit/test_mdc.py b/dingo/planner/tests/unit/test_mdc.py similarity index 99% rename from components/src/dynamo/planner/tests/unit/test_mdc.py rename to dingo/planner/tests/unit/test_mdc.py index 7e0da41e9995..3c059ecba984 100644 --- a/components/src/dynamo/planner/tests/unit/test_mdc.py +++ b/dingo/planner/tests/unit/test_mdc.py @@ -11,8 +11,8 @@ import pytest -from dynamo.planner.config.defaults import SubComponentType -from dynamo.planner.connectors.mdc import ( +from dingo.planner.config.defaults import SubComponentType +from dingo.planner.connectors.mdc import ( MdcEntry, is_model_card, is_prefill_card, diff --git a/components/src/dynamo/planner/tests/unit/test_metric_publication.py b/dingo/planner/tests/unit/test_metric_publication.py similarity index 94% rename from components/src/dynamo/planner/tests/unit/test_metric_publication.py rename to dingo/planner/tests/unit/test_metric_publication.py index 9980a57cb21f..9486e72c6f75 100644 --- a/components/src/dynamo/planner/tests/unit/test_metric_publication.py +++ b/dingo/planner/tests/unit/test_metric_publication.py @@ -21,15 +21,15 @@ import pytest -from dynamo.planner.config.planner_config import PlannerConfig -from dynamo.planner.core.base import NativePlannerBase -from dynamo.planner.core.types import ( +from dingo.planner.config.planner_config import PlannerConfig +from dingo.planner.core.base import NativePlannerBase +from dingo.planner.core.types import ( ScheduledTick, TickDiagnostics, TickInput, WorkerCounts, ) -from dynamo.planner.monitoring.planner_metrics import PREFIX, PlannerPrometheusMetrics +from dingo.planner.monitoring.planner_metrics import PREFIX, PlannerPrometheusMetrics pytestmark = [ pytest.mark.gpu_0, @@ -48,9 +48,9 @@ def _make_planner(prometheus_enabled: bool = True) -> NativePlannerBase: test; the actual port value is never used for I/O. """ with patch( - "dynamo.planner.core.base.PlannerPrometheusMetrics" - ) as mock_metrics, patch("dynamo.planner.core.base.start_http_server"), patch( - "dynamo.planner.connectors.kubernetes.KubernetesAPI" + "dingo.planner.core.base.PlannerPrometheusMetrics" + ) as mock_metrics, patch("dingo.planner.core.base.start_http_server"), patch( + "dingo.planner.connectors.kubernetes.KubernetesAPI" ), patch.dict( os.environ, {"DYN_PARENT_DGD_K8S_NAME": "test-graph"} ): @@ -239,7 +239,7 @@ def test_scale_down_refused_consolidation_state_registered(self): # Mocked Enum's state() doesn't validate, so guard against the # registration list drifting away from the diag stamps by checking # the constant directly. - from dynamo.planner.monitoring.planner_metrics import LOAD_DECISION_STATES + from dingo.planner.monitoring.planner_metrics import LOAD_DECISION_STATES assert "scale_down_refused_consolidation" in LOAD_DECISION_STATES @@ -326,9 +326,9 @@ def _make_planner_with_port( """ port = 9091 if prometheus_enabled else 0 with patch( - "dynamo.planner.core.base.PlannerPrometheusMetrics" - ) as mock_metrics, patch("dynamo.planner.core.base.start_http_server"), patch( - "dynamo.planner.connectors.kubernetes.KubernetesAPI" + "dingo.planner.core.base.PlannerPrometheusMetrics" + ) as mock_metrics, patch("dingo.planner.core.base.start_http_server"), patch( + "dingo.planner.connectors.kubernetes.KubernetesAPI" ), patch.dict( os.environ, {"DYN_PARENT_DGD_K8S_NAME": "test-graph"} ): @@ -398,8 +398,8 @@ class TestPlannerPrometheusMetricsHasSlaTargetGauges: def test_metrics_object_has_sla_target_gauge_attributes(self): """Instance attributes sla_target_ttft_ms and sla_target_itl_ms must exist.""" with patch( - "dynamo.planner.monitoring.planner_metrics.Gauge" - ) as mock_gauge, patch("dynamo.planner.monitoring.planner_metrics.Enum"): + "dingo.planner.monitoring.planner_metrics.Gauge" + ) as mock_gauge, patch("dingo.planner.monitoring.planner_metrics.Enum"): mock_gauge.return_value = MagicMock() metrics = PlannerPrometheusMetrics() diff --git a/components/src/dynamo/planner/tests/unit/test_parallelization_label.py b/dingo/planner/tests/unit/test_parallelization_label.py similarity index 96% rename from components/src/dynamo/planner/tests/unit/test_parallelization_label.py rename to dingo/planner/tests/unit/test_parallelization_label.py index 290059911491..0ee015e43971 100644 --- a/components/src/dynamo/planner/tests/unit/test_parallelization_label.py +++ b/dingo/planner/tests/unit/test_parallelization_label.py @@ -22,8 +22,8 @@ ] try: - from dynamo.planner.config.parallelization import PickedParallelConfig - from dynamo.profiler.utils.aic_dataframe import make_parallel_label + from dingo.planner.config.parallelization import PickedParallelConfig + from dingo.profiler.utils.aic_dataframe import make_parallel_label except ImportError as e: pytest.skip(f"Skip (missing dependency): {e}", allow_module_level=True) diff --git a/components/src/dynamo/planner/tests/unit/test_perf_metrics_priority.py b/dingo/planner/tests/unit/test_perf_metrics_priority.py similarity index 97% rename from components/src/dynamo/planner/tests/unit/test_perf_metrics_priority.py rename to dingo/planner/tests/unit/test_perf_metrics_priority.py index 87f8f94cf19f..8558651031da 100644 --- a/components/src/dynamo/planner/tests/unit/test_perf_metrics_priority.py +++ b/dingo/planner/tests/unit/test_perf_metrics_priority.py @@ -14,10 +14,10 @@ import pytest -from dynamo.planner.config.aic_interpolation_spec import AICInterpolationSpec -from dynamo.planner.config.defaults import SubComponentType -from dynamo.planner.config.parallelization import PickedParallelConfig -from dynamo.planner.monitoring import perf_metrics as pm +from dingo.planner.config.aic_interpolation_spec import AICInterpolationSpec +from dingo.planner.config.defaults import SubComponentType +from dingo.planner.config.parallelization import PickedParallelConfig +from dingo.planner.monitoring import perf_metrics as pm pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/planner/tests/unit/test_planner_config.py b/dingo/planner/tests/unit/test_planner_config.py similarity index 97% rename from components/src/dynamo/planner/tests/unit/test_planner_config.py rename to dingo/planner/tests/unit/test_planner_config.py index eb9b9dcc00b6..ba8c53b05b9a 100644 --- a/components/src/dynamo/planner/tests/unit/test_planner_config.py +++ b/dingo/planner/tests/unit/test_planner_config.py @@ -6,8 +6,8 @@ import pytest from pydantic import ValidationError -from dynamo.planner.config.parallelization import PickedParallelConfig -from dynamo.planner.config.planner_config import PlannerConfig +from dingo.planner.config.parallelization import PickedParallelConfig +from dingo.planner.config.planner_config import PlannerConfig pytestmark = [ pytest.mark.gpu_0, diff --git a/components/src/dynamo/planner/tests/unit/test_prometheus.py b/dingo/planner/tests/unit/test_prometheus.py similarity index 99% rename from components/src/dynamo/planner/tests/unit/test_prometheus.py rename to dingo/planner/tests/unit/test_prometheus.py index f46428890e06..88cc14e8a53c 100644 --- a/components/src/dynamo/planner/tests/unit/test_prometheus.py +++ b/dingo/planner/tests/unit/test_prometheus.py @@ -20,7 +20,7 @@ import pytest from dynamo import prometheus_names -from dynamo.planner.monitoring.traffic_metrics import ( +from dingo.planner.monitoring.traffic_metrics import ( FrontendMetric, FrontendMetricContainer, PrometheusAPIClient, diff --git a/components/src/dynamo/planner/tests/unit/test_remote_planner.py b/dingo/planner/tests/unit/test_remote_planner.py similarity index 95% rename from components/src/dynamo/planner/tests/unit/test_remote_planner.py rename to dingo/planner/tests/unit/test_remote_planner.py index 79a3fb227144..8d276882fa3e 100644 --- a/components/src/dynamo/planner/tests/unit/test_remote_planner.py +++ b/dingo/planner/tests/unit/test_remote_planner.py @@ -12,12 +12,12 @@ import pytest -from dynamo.planner import SubComponentType, TargetReplica -from dynamo.planner.connectors.global_planner import GlobalPlannerConnector -from dynamo.planner.connectors.protocol import ScaleRequest, ScaleResponse, ScaleStatus -from dynamo.planner.connectors.remote_client import RemotePlannerClient -from dynamo.planner.errors import DeploymentValidationError, EmptyTargetReplicasError -from dynamo.planner.monitoring.worker_info import WorkerInfo +from dingo.planner import SubComponentType, TargetReplica +from dingo.planner.connectors.global_planner import GlobalPlannerConnector +from dingo.planner.connectors.protocol import ScaleRequest, ScaleResponse, ScaleStatus +from dingo.planner.connectors.remote_client import RemotePlannerClient +from dingo.planner.errors import DeploymentValidationError, EmptyTargetReplicasError +from dingo.planner.monitoring.worker_info import WorkerInfo async def _async_responses(*items): @@ -232,7 +232,7 @@ async def test_connector_initialization(connector, connector_runtime): assert connector.remote_client is None with patch( - "dynamo.planner.connectors.global_planner.RemotePlannerClient" + "dingo.planner.connectors.global_planner.RemotePlannerClient" ) as mock_client_class: mock_client = MagicMock() mock_client_class.return_value = mock_client @@ -307,7 +307,7 @@ async def test_connector_set_replicas_rejected(connector): mock_client.send_scale_request = AsyncMock(return_value=mock_response) connector.remote_client = mock_client - with patch("dynamo.planner.connectors.global_planner.logger") as mock_logger: + with patch("dingo.planner.connectors.global_planner.logger") as mock_logger: # Must not raise — REJECTED is a legitimate business outcome. await connector.set_component_replicas(target_replicas, blocking=False) @@ -435,7 +435,7 @@ def test_connector_model_name_and_predicted_load(connector_runtime): # Without model name — force the local connector init to fail so we # exercise the fallback deterministically. with patch( - "dynamo.planner.connectors.global_planner.KubernetesConnector", + "dingo.planner.connectors.global_planner.KubernetesConnector", side_effect=DeploymentValidationError(["forced test failure"]), ): c2 = GlobalPlannerConnector( @@ -486,7 +486,7 @@ def test_connector_get_worker_info_falls_back_on_local_init_failure(connector_ru doesn't depend on the caller's kube config. """ with patch( - "dynamo.planner.connectors.global_planner.KubernetesConnector", + "dingo.planner.connectors.global_planner.KubernetesConnector", side_effect=DeploymentValidationError(["forced test failure"]), ): c = GlobalPlannerConnector( @@ -532,7 +532,7 @@ def test_connector_get_actual_worker_counts_out_of_cluster_fallback(connector_ru out-of-cluster callers aren't blocked. """ with patch( - "dynamo.planner.connectors.global_planner.KubernetesConnector", + "dingo.planner.connectors.global_planner.KubernetesConnector", side_effect=DeploymentValidationError(["forced test failure"]), ): c = GlobalPlannerConnector( @@ -566,7 +566,7 @@ def test_connector_get_worker_runtime_namespace_out_of_cluster_fallback( ): """Fallback to base namespace when no pool-local KubernetesConnector exists.""" with patch( - "dynamo.planner.connectors.global_planner.KubernetesConnector", + "dingo.planner.connectors.global_planner.KubernetesConnector", side_effect=DeploymentValidationError(["forced test failure"]), ): c = GlobalPlannerConnector( diff --git a/components/src/dynamo/planner/tests/unit/test_rust_perf_adapter.py b/dingo/planner/tests/unit/test_rust_perf_adapter.py similarity index 97% rename from components/src/dynamo/planner/tests/unit/test_rust_perf_adapter.py rename to dingo/planner/tests/unit/test_rust_perf_adapter.py index 861cb40034bb..5df71f3ee0d5 100644 --- a/components/src/dynamo/planner/tests/unit/test_rust_perf_adapter.py +++ b/dingo/planner/tests/unit/test_rust_perf_adapter.py @@ -8,7 +8,7 @@ import pytest try: - from dynamo.common.forward_pass_metrics import ( + from dingo.common.forward_pass_metrics import ( ForwardPassMetrics, QueuedRequestMetrics, ScheduledRequestMetrics, @@ -16,11 +16,11 @@ except ImportError: pytest.skip("forward_pass_metrics not available", allow_module_level=True) -from dynamo.planner.config.parallelization import PickedParallelConfig -from dynamo.planner.config.planner_config import AICPerfModelSpec, PlannerConfig -from dynamo.planner.core.perf_model import rust_adapter -from dynamo.planner.core.perf_model.rust_adapter import PlannerEnginePerfModel -from dynamo.planner.core.types import EngineCapabilities +from dingo.planner.config.parallelization import PickedParallelConfig +from dingo.planner.config.planner_config import AICPerfModelSpec, PlannerConfig +from dingo.planner.core.perf_model import rust_adapter +from dingo.planner.core.perf_model.rust_adapter import PlannerEnginePerfModel +from dingo.planner.core.types import EngineCapabilities pytestmark = [ pytest.mark.gpu_0, diff --git a/dingo/profiler/README.md b/dingo/profiler/README.md new file mode 100644 index 000000000000..5b48f4f0d6ca --- /dev/null +++ b/dingo/profiler/README.md @@ -0,0 +1,13 @@ + + +# Profiler + +Documentation for the Dynamo Profiler has moved to [docs/components/profiler/](../../docs/components/profiler/README.md). + +- [Profiler Overview](../../docs/components/profiler/README.md) +- [Profiler Guide](../../docs/components/profiler/profiler-guide.md) +- [Profiler Examples](../../docs/components/profiler/profiler-examples.md) diff --git a/components/src/dynamo/profiler/__init__.py b/dingo/profiler/__init__.py similarity index 100% rename from components/src/dynamo/profiler/__init__.py rename to dingo/profiler/__init__.py diff --git a/components/src/dynamo/profiler/__main__.py b/dingo/profiler/__main__.py similarity index 96% rename from components/src/dynamo/profiler/__main__.py rename to dingo/profiler/__main__.py index f86f7a0c07b1..5fcc69afc06a 100644 --- a/components/src/dynamo/profiler/__main__.py +++ b/dingo/profiler/__main__.py @@ -18,9 +18,9 @@ Usage:: - python -m dynamo.profiler --config - python -m dynamo.profiler --config '{"model": "Qwen/Qwen3-32B", ...}' - python -m dynamo.profiler --config /path/to/dgdr_spec.yaml + python -m dingo.profiler --config + python -m dingo.profiler --config '{"model": "Qwen/Qwen3-32B", ...}' + python -m dingo.profiler --config /path/to/dgdr_spec.yaml """ import argparse @@ -34,7 +34,7 @@ import yaml -from dynamo.profiler.utils.dgdr_v1beta1_types import DynamoGraphDeploymentRequestSpec +from dingo.profiler.utils.dgdr_v1beta1_types import DynamoGraphDeploymentRequestSpec from .profile_sla import run_profile from .utils.profile_common import ( diff --git a/components/src/dynamo/profiler/interpolation.py b/dingo/profiler/interpolation.py similarity index 92% rename from components/src/dynamo/profiler/interpolation.py rename to dingo/profiler/interpolation.py index ca72805a07de..4084c27f9fa6 100644 --- a/components/src/dynamo/profiler/interpolation.py +++ b/dingo/profiler/interpolation.py @@ -18,7 +18,7 @@ Rapid-mode interpolation is no longer generated by the profiler: the planner runs AIConfigurator in-process at bootstrap (``planner/monitoring/aic_interpolation.py``) and the mocker pulls AIC perf data at runtime via ``--aic-perf-model`` flags -injected by :func:`dynamo.profiler.utils.dgd_generation.generate_mocker_config`. +injected by :func:`dingo.profiler.utils.dgd_generation.generate_mocker_config`. This module only handles the thorough path (real GPUs → NPZ on disk). """ @@ -28,20 +28,20 @@ import yaml from deploy.utils.dynamo_deployment import DeploymentFailedError, DynamoDeploymentClient -from dynamo.planner.config.planner_config import PlannerPreDeploymentSweepMode -from dynamo.profiler.utils.config_modifiers import CONFIG_MODIFIERS -from dynamo.profiler.utils.config_modifiers.parallelization_mapping import ( +from dingo.planner.config.planner_config import PlannerPreDeploymentSweepMode +from dingo.profiler.utils.config_modifiers import CONFIG_MODIFIERS +from dingo.profiler.utils.config_modifiers.parallelization_mapping import ( PickedParallelConfig, ) -from dynamo.profiler.utils.defaults import EngineType -from dynamo.profiler.utils.dgdr_v1beta1_types import DynamoGraphDeploymentRequestSpec -from dynamo.profiler.utils.profile_common import ( +from dingo.profiler.utils.defaults import EngineType +from dingo.profiler.utils.dgdr_v1beta1_types import DynamoGraphDeploymentRequestSpec +from dingo.profiler.utils.profile_common import ( ProfilerOperationalConfig, inject_tolerations_into_dgd, pick_decode_component, ) -from dynamo.profiler.utils.profile_decode import profile_decode -from dynamo.profiler.utils.profile_prefill import profile_prefill +from dingo.profiler.utils.profile_decode import profile_decode +from dingo.profiler.utils.profile_prefill import profile_prefill logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/profiler/profile_endpoint.py b/dingo/profiler/profile_endpoint.py similarity index 94% rename from components/src/dynamo/profiler/profile_endpoint.py rename to dingo/profiler/profile_endpoint.py index f98f63dbfab2..883352e251da 100644 --- a/components/src/dynamo/profiler/profile_endpoint.py +++ b/dingo/profiler/profile_endpoint.py @@ -5,9 +5,9 @@ import logging import os -from dynamo.profiler.utils.defaults import EngineType -from dynamo.profiler.utils.profile_decode import profile_decode -from dynamo.profiler.utils.profile_prefill import profile_prefill +from dingo.profiler.utils.defaults import EngineType +from dingo.profiler.utils.profile_decode import profile_decode +from dingo.profiler.utils.profile_prefill import profile_prefill logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) diff --git a/components/src/dynamo/profiler/profile_sla.py b/dingo/profiler/profile_sla.py similarity index 96% rename from components/src/dynamo/profiler/profile_sla.py rename to dingo/profiler/profile_sla.py index 8dbdc95f86c0..8fc1e4e69c9b 100644 --- a/components/src/dynamo/profiler/profile_sla.py +++ b/dingo/profiler/profile_sla.py @@ -24,30 +24,30 @@ from aiconfigurator.sdk.utils import get_model_config_from_model_path from deploy.utils.dynamo_deployment import cleanup_remaining_deployments -from dynamo.profiler.interpolation import run_interpolation -from dynamo.profiler.rapid import run_rapid -from dynamo.profiler.thorough import run_thorough -from dynamo.profiler.utils.config_modifiers import CONFIG_MODIFIERS -from dynamo.profiler.utils.config_modifiers.parallelization_mapping import ( +from dingo.profiler.interpolation import run_interpolation +from dingo.profiler.rapid import run_rapid +from dingo.profiler.thorough import run_thorough +from dingo.profiler.utils.config_modifiers import CONFIG_MODIFIERS +from dingo.profiler.utils.config_modifiers.parallelization_mapping import ( PickedParallelConfig, ) -from dynamo.profiler.utils.config_modifiers.protocol import apply_dgd_overrides -from dynamo.profiler.utils.defaults import SearchStrategy -from dynamo.profiler.utils.dgd_generation import ( +from dingo.profiler.utils.config_modifiers.protocol import apply_dgd_overrides +from dingo.profiler.utils.defaults import SearchStrategy +from dingo.profiler.utils.dgd_generation import ( assemble_final_config, build_aic_interpolation_spec, build_aic_perf_model_spec, ) -from dynamo.profiler.utils.dgdr_v1beta1_types import ( +from dingo.profiler.utils.dgdr_v1beta1_types import ( BackendType, DynamoGraphDeploymentRequestSpec, ProfilingPhase, ) -from dynamo.profiler.utils.dgdr_validate import ( +from dingo.profiler.utils.dgdr_validate import ( valid_dgdr_spec, validate_dgdr_dynamo_features, ) -from dynamo.profiler.utils.profile_common import ( +from dingo.profiler.utils.profile_common import ( ProfilerOperationalConfig, determine_picking_mode, get_profiling_job_tolerations, @@ -58,7 +58,7 @@ warn_and_update_sla, warn_gpu_shortage, ) -from dynamo.profiler.utils.profiler_status import ProfilerStatus, write_profiler_status +from dingo.profiler.utils.profiler_status import ProfilerStatus, write_profiler_status logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/profiler/rapid.py b/dingo/profiler/rapid.py similarity index 98% rename from components/src/dynamo/profiler/rapid.py rename to dingo/profiler/rapid.py index 6576bc3f6fd9..1645e8679daf 100644 --- a/components/src/dynamo/profiler/rapid.py +++ b/dingo/profiler/rapid.py @@ -25,9 +25,9 @@ from aiconfigurator.generator.naive import build_naive_generator_params from aiconfigurator.sdk.task import TaskConfig, TaskRunner -from dynamo.profiler.utils.config import clamp_total_gpus_to_budget -from dynamo.profiler.utils.dgdr_v1beta1_types import DynamoGraphDeploymentRequestSpec -from dynamo.profiler.utils.profile_common import ( +from dingo.profiler.utils.config import clamp_total_gpus_to_budget +from dingo.profiler.utils.dgdr_v1beta1_types import DynamoGraphDeploymentRequestSpec +from dingo.profiler.utils.profile_common import ( derive_backend_image, needs_profile_data, resolve_model_path, diff --git a/components/src/dynamo/profiler/tests/data/configs/10_thorough_override_security_context.yaml b/dingo/profiler/tests/data/configs/10_thorough_override_security_context.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/10_thorough_override_security_context.yaml rename to dingo/profiler/tests/data/configs/10_thorough_override_security_context.yaml diff --git a/components/src/dynamo/profiler/tests/data/configs/11_auto_rapid_no_planner_no_load.yaml b/dingo/profiler/tests/data/configs/11_auto_rapid_no_planner_no_load.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/11_auto_rapid_no_planner_no_load.yaml rename to dingo/profiler/tests/data/configs/11_auto_rapid_no_planner_no_load.yaml diff --git a/components/src/dynamo/profiler/tests/data/configs/12_thorough_moe_qwen3_30b_a3b_sglang.yaml b/dingo/profiler/tests/data/configs/12_thorough_moe_qwen3_30b_a3b_sglang.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/12_thorough_moe_qwen3_30b_a3b_sglang.yaml rename to dingo/profiler/tests/data/configs/12_thorough_moe_qwen3_30b_a3b_sglang.yaml diff --git a/components/src/dynamo/profiler/tests/data/configs/1_rapid_supported_no_planner_no_load.yaml b/dingo/profiler/tests/data/configs/1_rapid_supported_no_planner_no_load.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/1_rapid_supported_no_planner_no_load.yaml rename to dingo/profiler/tests/data/configs/1_rapid_supported_no_planner_no_load.yaml diff --git a/components/src/dynamo/profiler/tests/data/configs/2_rapid_supported_no_planner_with_load.yaml b/dingo/profiler/tests/data/configs/2_rapid_supported_no_planner_with_load.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/2_rapid_supported_no_planner_with_load.yaml rename to dingo/profiler/tests/data/configs/2_rapid_supported_no_planner_with_load.yaml diff --git a/components/src/dynamo/profiler/tests/data/configs/2b_rapid_supported_pvc_no_planner_with_load.yaml b/dingo/profiler/tests/data/configs/2b_rapid_supported_pvc_no_planner_with_load.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/2b_rapid_supported_pvc_no_planner_with_load.yaml rename to dingo/profiler/tests/data/configs/2b_rapid_supported_pvc_no_planner_with_load.yaml diff --git a/components/src/dynamo/profiler/tests/data/configs/2c_rapid_supported_e2e_latency.yaml b/dingo/profiler/tests/data/configs/2c_rapid_supported_e2e_latency.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/2c_rapid_supported_e2e_latency.yaml rename to dingo/profiler/tests/data/configs/2c_rapid_supported_e2e_latency.yaml diff --git a/components/src/dynamo/profiler/tests/data/configs/2d_rapid_both_concurrency_and_rate_error.yaml b/dingo/profiler/tests/data/configs/2d_rapid_both_concurrency_and_rate_error.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/2d_rapid_both_concurrency_and_rate_error.yaml rename to dingo/profiler/tests/data/configs/2d_rapid_both_concurrency_and_rate_error.yaml diff --git a/components/src/dynamo/profiler/tests/data/configs/3_rapid_supported_planner_rapid_sweep.yaml b/dingo/profiler/tests/data/configs/3_rapid_supported_planner_rapid_sweep.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/3_rapid_supported_planner_rapid_sweep.yaml rename to dingo/profiler/tests/data/configs/3_rapid_supported_planner_rapid_sweep.yaml diff --git a/components/src/dynamo/profiler/tests/data/configs/3b_rapid_supported_planner_rapid_sweep_mocker.yaml b/dingo/profiler/tests/data/configs/3b_rapid_supported_planner_rapid_sweep_mocker.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/3b_rapid_supported_planner_rapid_sweep_mocker.yaml rename to dingo/profiler/tests/data/configs/3b_rapid_supported_planner_rapid_sweep_mocker.yaml diff --git a/components/src/dynamo/profiler/tests/data/configs/4_rapid_unsupported_no_planner.yaml b/dingo/profiler/tests/data/configs/4_rapid_unsupported_no_planner.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/4_rapid_unsupported_no_planner.yaml rename to dingo/profiler/tests/data/configs/4_rapid_unsupported_no_planner.yaml diff --git a/components/src/dynamo/profiler/tests/data/configs/5_rapid_unsupported_planner.yaml b/dingo/profiler/tests/data/configs/5_rapid_unsupported_planner.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/5_rapid_unsupported_planner.yaml rename to dingo/profiler/tests/data/configs/5_rapid_unsupported_planner.yaml diff --git a/components/src/dynamo/profiler/tests/data/configs/5b_rapid_unsupported_planner_throughput_error.yaml b/dingo/profiler/tests/data/configs/5b_rapid_unsupported_planner_throughput_error.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/5b_rapid_unsupported_planner_throughput_error.yaml rename to dingo/profiler/tests/data/configs/5b_rapid_unsupported_planner_throughput_error.yaml diff --git a/components/src/dynamo/profiler/tests/data/configs/6_thorough_no_planner_with_load.yaml b/dingo/profiler/tests/data/configs/6_thorough_no_planner_with_load.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/6_thorough_no_planner_with_load.yaml rename to dingo/profiler/tests/data/configs/6_thorough_no_planner_with_load.yaml diff --git a/components/src/dynamo/profiler/tests/data/configs/7_thorough_planner_rapid_sweep.yaml b/dingo/profiler/tests/data/configs/7_thorough_planner_rapid_sweep.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/7_thorough_planner_rapid_sweep.yaml rename to dingo/profiler/tests/data/configs/7_thorough_planner_rapid_sweep.yaml diff --git a/components/src/dynamo/profiler/tests/data/configs/7b_thorough_planner_thorough_sweep.yaml b/dingo/profiler/tests/data/configs/7b_thorough_planner_thorough_sweep.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/7b_thorough_planner_thorough_sweep.yaml rename to dingo/profiler/tests/data/configs/7b_thorough_planner_thorough_sweep.yaml diff --git a/components/src/dynamo/profiler/tests/data/configs/8_thorough_empty_candidates.yaml b/dingo/profiler/tests/data/configs/8_thorough_empty_candidates.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/8_thorough_empty_candidates.yaml rename to dingo/profiler/tests/data/configs/8_thorough_empty_candidates.yaml diff --git a/components/src/dynamo/profiler/tests/data/configs/9a_thorough_dsr1_sglang_overrides.yaml b/dingo/profiler/tests/data/configs/9a_thorough_dsr1_sglang_overrides.yaml similarity index 100% rename from components/src/dynamo/profiler/tests/data/configs/9a_thorough_dsr1_sglang_overrides.yaml rename to dingo/profiler/tests/data/configs/9a_thorough_dsr1_sglang_overrides.yaml diff --git a/components/src/dynamo/profiler/tests/integration/test_profile_sla_dgdr.py b/dingo/profiler/tests/integration/test_profile_sla_dgdr.py similarity index 96% rename from components/src/dynamo/profiler/tests/integration/test_profile_sla_dgdr.py rename to dingo/profiler/tests/integration/test_profile_sla_dgdr.py index 5d20d9200bc1..4f0048e4e266 100644 --- a/components/src/dynamo/profiler/tests/integration/test_profile_sla_dgdr.py +++ b/dingo/profiler/tests/integration/test_profile_sla_dgdr.py @@ -28,13 +28,13 @@ ] try: - from dynamo.profiler.profile_sla import run_profile - from dynamo.profiler.utils.dgdr_v1beta1_types import ( + from dingo.profiler.profile_sla import run_profile + from dingo.profiler.utils.dgdr_v1beta1_types import ( BackendType, DynamoGraphDeploymentRequestSpec, SearchStrategy, ) - from dynamo.profiler.utils.profile_common import ProfilerOperationalConfig + from dingo.profiler.utils.profile_common import ProfilerOperationalConfig except ImportError as _e: pytest.skip(f"Skip testing (refactor in progress): {_e}", allow_module_level=True) @@ -364,17 +364,17 @@ def _make_thorough_patches(backend: str = "trtllm"): svc_name = _DECODE_SVC_NAMES.get(backend, "decode") return [ patch( - "dynamo.profiler.thorough.DynamoDeploymentClient", + "dingo.profiler.thorough.DynamoDeploymentClient", side_effect=lambda **kw: _mock_deployment_client(), ), - patch("dynamo.profiler.thorough.get_prefill_ttft", return_value=50.0), + patch("dingo.profiler.thorough.get_prefill_ttft", return_value=50.0), patch( - "dynamo.profiler.thorough.get_decode_itl_and_thpt_per_gpu", + "dingo.profiler.thorough.get_decode_itl_and_thpt_per_gpu", return_value=(8.0, 125.0), ), - patch("dynamo.profiler.thorough.get_num_request_range", return_value=[1, 4, 8]), + patch("dingo.profiler.thorough.get_num_request_range", return_value=[1, 4, 8]), patch( - "dynamo.profiler.thorough.pick_decode_component", + "dingo.profiler.thorough.pick_decode_component", return_value=svc_name, ), ] @@ -386,7 +386,7 @@ def _make_thorough_patches(backend: str = "trtllm"): def _patch_kv_cache_log(backend: str = "trtllm"): """Patch get_kv_cache_size_from_dynamo_log on the real config modifier.""" - from dynamo.profiler.utils.config_modifiers import CONFIG_MODIFIERS + from dingo.profiler.utils.config_modifiers import CONFIG_MODIFIERS real_modifier = CONFIG_MODIFIERS[backend] return patch.object( @@ -438,19 +438,19 @@ def mock_profile_decode(work_dir, *args, **kwargs): interp_patches = [ patch( - "dynamo.profiler.interpolation.DynamoDeploymentClient", + "dingo.profiler.interpolation.DynamoDeploymentClient", side_effect=lambda **kw: _mock_deployment_client(), ), patch( - "dynamo.profiler.interpolation.profile_prefill", + "dingo.profiler.interpolation.profile_prefill", side_effect=mock_profile_prefill, ), patch( - "dynamo.profiler.interpolation.profile_decode", + "dingo.profiler.interpolation.profile_decode", side_effect=mock_profile_decode, ), patch( - "dynamo.profiler.interpolation.pick_decode_component", + "dingo.profiler.interpolation.pick_decode_component", return_value="TRTLLMWorker", ), ] diff --git a/components/src/dynamo/profiler/tests/test_replay_optimize.py b/dingo/profiler/tests/test_replay_optimize.py similarity index 99% rename from components/src/dynamo/profiler/tests/test_replay_optimize.py rename to dingo/profiler/tests/test_replay_optimize.py index d489a59caa7a..b48439e404a5 100644 --- a/components/src/dynamo/profiler/tests/test_replay_optimize.py +++ b/dingo/profiler/tests/test_replay_optimize.py @@ -14,11 +14,11 @@ try: from dynamo.llm import KvRouterConfig - from dynamo.mocker import MockEngineArgs + from dingo.mocker import MockEngineArgs except ImportError: pytest.skip("dynamo mocker bindings not available", allow_module_level=True) -from dynamo.profiler.utils import replay_optimize -from dynamo.profiler.utils.replay_optimize import ( +from dingo.profiler.utils import replay_optimize +from dingo.profiler.utils.replay_optimize import ( DenseAggReplayState, DenseReplayState, EngineSpec, @@ -198,7 +198,7 @@ def fake_run_trace_replay(trace_file, **kwargs): return {"output_throughput_tok_s": 1.0} monkeypatch.setattr( - "dynamo.profiler.utils.replay_optimize.evaluate.run_trace_replay", + "dingo.profiler.utils.replay_optimize.evaluate.run_trace_replay", fake_run_trace_replay, ) @@ -982,7 +982,7 @@ def test_evaluate_state_prefers_normalized_metrics_over_report_payload() -> None ) with patch( - "dynamo.profiler.utils.replay_optimize.evaluate._run_replay_for_state", + "dingo.profiler.utils.replay_optimize.evaluate._run_replay_for_state", return_value={ "output_throughput_tok_s": "11.0", "score": -1.0, @@ -1024,7 +1024,7 @@ def test_evaluate_agg_state_prefers_normalized_metrics_over_report_payload() -> ) with patch( - "dynamo.profiler.utils.replay_optimize.evaluate._run_agg_replay_for_state", + "dingo.profiler.utils.replay_optimize.evaluate._run_agg_replay_for_state", return_value={ "output_throughput_tok_s": "24.0", "score": -1.0, diff --git a/components/src/dynamo/profiler/tests/unit/test_aic_dataframe.py b/dingo/profiler/tests/unit/test_aic_dataframe.py similarity index 98% rename from components/src/dynamo/profiler/tests/unit/test_aic_dataframe.py rename to dingo/profiler/tests/unit/test_aic_dataframe.py index e88cf489b50c..90f96461c562 100644 --- a/components/src/dynamo/profiler/tests/unit/test_aic_dataframe.py +++ b/dingo/profiler/tests/unit/test_aic_dataframe.py @@ -21,7 +21,7 @@ ] try: - from dynamo.profiler.utils.aic_dataframe import make_parallel_label + from dingo.profiler.utils.aic_dataframe import make_parallel_label except ImportError as e: pytest.skip(f"Skip (missing dependency): {e}", allow_module_level=True) diff --git a/components/src/dynamo/profiler/tests/unit/test_dgd_generation_aic.py b/dingo/profiler/tests/unit/test_dgd_generation_aic.py similarity index 96% rename from components/src/dynamo/profiler/tests/unit/test_dgd_generation_aic.py rename to dingo/profiler/tests/unit/test_dgd_generation_aic.py index 570eba4f013c..24bf2195e069 100644 --- a/components/src/dynamo/profiler/tests/unit/test_dgd_generation_aic.py +++ b/dingo/profiler/tests/unit/test_dgd_generation_aic.py @@ -6,20 +6,20 @@ import pytest try: - from dynamo.planner.config.aic_interpolation_spec import AICInterpolationSpec - from dynamo.planner.config.parallelization import PickedParallelConfig - from dynamo.planner.config.planner_config import ( + from dingo.planner.config.aic_interpolation_spec import AICInterpolationSpec + from dingo.planner.config.parallelization import PickedParallelConfig + from dingo.planner.config.planner_config import ( PlannerConfig, PlannerPreDeploymentSweepMode, ) - from dynamo.profiler.utils.dgd_generation import ( + from dingo.profiler.utils.dgd_generation import ( _build_planner_config, _inject_mocker_aic_args, build_aic_interpolation_spec, build_aic_perf_model_spec, enable_vllm_benchmark_mode, ) - from dynamo.profiler.utils.dgdr_v1beta1_types import ( + from dingo.profiler.utils.dgdr_v1beta1_types import ( DynamoGraphDeploymentRequestSpec, FeaturesSpec, MockerSpec, @@ -362,7 +362,7 @@ def test_no_spec_leaves_aic_interpolation_none(self): class TestNeedsProfileDataRapid: def test_rapid_planner_only_returns_false(self): """Planner-only rapid: no files needed; planner will use aic_spec.""" - from dynamo.profiler.utils.profile_common import needs_profile_data + from dingo.profiler.utils.profile_common import needs_profile_data planner = PlannerConfig( enable_throughput_scaling=True, @@ -375,7 +375,7 @@ def test_rapid_planner_only_returns_false(self): def test_thorough_planner_returns_true(self): """Thorough still needs files.""" - from dynamo.profiler.utils.profile_common import needs_profile_data + from dingo.profiler.utils.profile_common import needs_profile_data planner = PlannerConfig( enable_throughput_scaling=True, @@ -388,7 +388,7 @@ def test_thorough_planner_returns_true(self): def test_none_planner_only_returns_false(self): """Planner-only none mode can warm from native AIC or live FPMs.""" - from dynamo.profiler.utils.profile_common import needs_profile_data + from dingo.profiler.utils.profile_common import needs_profile_data planner = PlannerConfig( enable_throughput_scaling=True, @@ -401,7 +401,7 @@ def test_none_planner_only_returns_false(self): def test_mocker_rapid_returns_false(self): """Mocker + rapid: mocker pulls AIC perf data at runtime; no NPZ files.""" - from dynamo.profiler.utils.profile_common import needs_profile_data + from dingo.profiler.utils.profile_common import needs_profile_data planner = PlannerConfig( enable_throughput_scaling=True, @@ -414,7 +414,7 @@ def test_mocker_rapid_returns_false(self): def test_mocker_thorough_returns_true(self): """Mocker + thorough: mocker consumes real-GPU NPZ.""" - from dynamo.profiler.utils.profile_common import needs_profile_data + from dingo.profiler.utils.profile_common import needs_profile_data planner = PlannerConfig( enable_throughput_scaling=True, diff --git a/components/src/dynamo/profiler/tests/unit/test_dgd_generation_planner_sla.py b/dingo/profiler/tests/unit/test_dgd_generation_planner_sla.py similarity index 94% rename from components/src/dynamo/profiler/tests/unit/test_dgd_generation_planner_sla.py rename to dingo/profiler/tests/unit/test_dgd_generation_planner_sla.py index 2fef0a1852dd..9fa634786565 100644 --- a/components/src/dynamo/profiler/tests/unit/test_dgd_generation_planner_sla.py +++ b/dingo/profiler/tests/unit/test_dgd_generation_planner_sla.py @@ -12,10 +12,10 @@ import pytest try: - from dynamo.planner.config.defaults import SLAPlannerDefaults - from dynamo.planner.config.planner_config import PlannerConfig - from dynamo.profiler.utils.dgd_generation import _build_planner_config - from dynamo.profiler.utils.dgdr_v1beta1_types import ( + from dingo.planner.config.defaults import SLAPlannerDefaults + from dingo.planner.config.planner_config import PlannerConfig + from dingo.profiler.utils.dgd_generation import _build_planner_config + from dingo.profiler.utils.dgdr_v1beta1_types import ( DynamoGraphDeploymentRequestSpec, FeaturesSpec, GPUSKUType, @@ -23,7 +23,7 @@ SLASpec, WorkloadSpec, ) - from dynamo.profiler.utils.dgdr_validate import valid_dgdr_spec + from dingo.profiler.utils.dgdr_validate import valid_dgdr_spec except ImportError as e: pytest.skip(f"Missing dependency: {e}", allow_module_level=True) diff --git a/components/src/dynamo/profiler/tests/unit/test_get_num_request_range.py b/dingo/profiler/tests/unit/test_get_num_request_range.py similarity index 95% rename from components/src/dynamo/profiler/tests/unit/test_get_num_request_range.py rename to dingo/profiler/tests/unit/test_get_num_request_range.py index b2699f08d5f9..e784ecf4edee 100644 --- a/components/src/dynamo/profiler/tests/unit/test_get_num_request_range.py +++ b/dingo/profiler/tests/unit/test_get_num_request_range.py @@ -9,8 +9,8 @@ import pytest -from dynamo.profiler.utils.defaults import DECODE_MAX_CONCURRENCY -from dynamo.profiler.utils.profile_decode import get_num_request_range +from dingo.profiler.utils.defaults import DECODE_MAX_CONCURRENCY +from dingo.profiler.utils.profile_decode import get_num_request_range pytestmark = [ pytest.mark.pre_merge, diff --git a/components/src/dynamo/profiler/tests/unit/test_helpers_profile_sla.py b/dingo/profiler/tests/unit/test_helpers_profile_sla.py similarity index 95% rename from components/src/dynamo/profiler/tests/unit/test_helpers_profile_sla.py rename to dingo/profiler/tests/unit/test_helpers_profile_sla.py index c2b80292724b..6fce09676994 100644 --- a/components/src/dynamo/profiler/tests/unit/test_helpers_profile_sla.py +++ b/dingo/profiler/tests/unit/test_helpers_profile_sla.py @@ -24,24 +24,24 @@ ] try: - from dynamo.planner.config.planner_config import ( + from dingo.planner.config.planner_config import ( PlannerConfig, PlannerPreDeploymentSweepMode, ) - from dynamo.profiler.profile_sla import ( + from dingo.profiler.profile_sla import ( _extract_profiler_params, _write_final_output, ) - from dynamo.profiler.utils.config_modifiers.parallelization_mapping import ( + from dingo.profiler.utils.config_modifiers.parallelization_mapping import ( PickedParallelConfig, ) - from dynamo.profiler.utils.defaults import SearchStrategy - from dynamo.profiler.utils.dgd_generation import ( + from dingo.profiler.utils.defaults import SearchStrategy + from dingo.profiler.utils.dgd_generation import ( add_profile_data_to_config, assemble_final_config, enable_planner_worker_scaling_adapters, ) - from dynamo.profiler.utils.dgdr_v1beta1_types import ( + from dingo.profiler.utils.dgdr_v1beta1_types import ( DynamoGraphDeploymentRequestSpec, FeaturesSpec, HardwareSpec, @@ -49,11 +49,11 @@ SLASpec, WorkloadSpec, ) - from dynamo.profiler.utils.dgdr_validate import ( + from dingo.profiler.utils.dgdr_validate import ( valid_dgdr_spec, validate_dgdr_dynamo_features, ) - from dynamo.profiler.utils.profile_common import ProfilerOperationalConfig + from dingo.profiler.utils.profile_common import ProfilerOperationalConfig except ImportError as e: pytest.skip(f"Skip (missing dependency): {e}", allow_module_level=True) @@ -449,7 +449,7 @@ def test_none_config_dry_run_writes_empty_yaml_and_returns_true(self, tmp_path): # assemble_final_config # --------------------------------------------------------------------------- -_DGD_GEN = "dynamo.profiler.utils.dgd_generation" +_DGD_GEN = "dingo.profiler.utils.dgd_generation" class TestAssembleFinalConfig: @@ -973,7 +973,7 @@ def test_naive_fallback_resolved_backend_auto(self): try: from unittest.mock import patch - from dynamo.profiler.rapid import _run_naive_fallback + from dingo.profiler.rapid import _run_naive_fallback except ImportError as e: pytest.skip(f"Missing dependency: {e}") @@ -981,11 +981,11 @@ def test_naive_fallback_resolved_backend_auto(self): with ( patch( - "dynamo.profiler.rapid.build_naive_generator_params", + "dingo.profiler.rapid.build_naive_generator_params", return_value={}, ), patch( - "dynamo.profiler.rapid.generate_backend_artifacts", + "dingo.profiler.rapid.generate_backend_artifacts", return_value={}, ), ): @@ -1020,7 +1020,7 @@ def test_naive_fallback_resolved_backend_concrete(self): try: from unittest.mock import patch - from dynamo.profiler.rapid import _run_naive_fallback + from dingo.profiler.rapid import _run_naive_fallback except ImportError as e: pytest.skip(f"Missing dependency: {e}") @@ -1028,11 +1028,11 @@ def test_naive_fallback_resolved_backend_concrete(self): with ( patch( - "dynamo.profiler.rapid.build_naive_generator_params", + "dingo.profiler.rapid.build_naive_generator_params", return_value={}, ), patch( - "dynamo.profiler.rapid.generate_backend_artifacts", + "dingo.profiler.rapid.generate_backend_artifacts", return_value={}, ), ): @@ -1054,18 +1054,18 @@ def test_naive_fallback_chosen_exp_is_agg(self): try: from unittest.mock import patch - from dynamo.profiler.rapid import _run_naive_fallback + from dingo.profiler.rapid import _run_naive_fallback except ImportError as e: pytest.skip(f"Missing dependency: {e}") dgdr = _make_dgdr(backend="vllm") with ( patch( - "dynamo.profiler.rapid.build_naive_generator_params", + "dingo.profiler.rapid.build_naive_generator_params", return_value={}, ), patch( - "dynamo.profiler.rapid.generate_backend_artifacts", + "dingo.profiler.rapid.generate_backend_artifacts", return_value={}, ), ): @@ -1102,11 +1102,11 @@ def test_run_profile_skips_interpolation_when_agg(self, tmp_path): import asyncio from unittest.mock import AsyncMock, patch - from dynamo.planner.config.planner_config import ( + from dingo.planner.config.planner_config import ( PlannerPreDeploymentSweepMode, ) - from dynamo.profiler.profile_sla import run_profile - from dynamo.profiler.utils.dgdr_v1beta1_types import FeaturesSpec + from dingo.profiler.profile_sla import run_profile + from dingo.profiler.utils.dgdr_v1beta1_types import FeaturesSpec except ImportError as e: pytest.skip(f"Missing dependency: {e}") @@ -1135,7 +1135,7 @@ def test_run_profile_skips_interpolation_when_agg(self, tmp_path): "resolved_backend": "vllm", } - _PROFILE_SLA = "dynamo.profiler.profile_sla" + _PROFILE_SLA = "dingo.profiler.profile_sla" with ( patch( f"{_PROFILE_SLA}._extract_profiler_params", @@ -1150,7 +1150,7 @@ def test_run_profile_skips_interpolation_when_agg(self, tmp_path): 2000.0, 50.0, __import__( - "dynamo.profiler.utils.defaults", fromlist=["SearchStrategy"] + "dingo.profiler.utils.defaults", fromlist=["SearchStrategy"] ).SearchStrategy.RAPID, "autoscale", ), @@ -1200,11 +1200,11 @@ def test_run_profile_calls_interpolation_with_resolved_backend_for_disagg( import asyncio from unittest.mock import AsyncMock, patch - from dynamo.planner.config.planner_config import ( + from dingo.planner.config.planner_config import ( PlannerPreDeploymentSweepMode, ) - from dynamo.profiler.profile_sla import run_profile - from dynamo.profiler.utils.dgdr_v1beta1_types import FeaturesSpec + from dingo.profiler.profile_sla import run_profile + from dingo.profiler.utils.dgdr_v1beta1_types import FeaturesSpec except ImportError as e: pytest.skip(f"Missing dependency: {e}") @@ -1239,7 +1239,7 @@ def test_run_profile_calls_interpolation_with_resolved_backend_for_disagg( "resolved_backend": "vllm", } - _PROFILE_SLA = "dynamo.profiler.profile_sla" + _PROFILE_SLA = "dingo.profiler.profile_sla" with ( patch( f"{_PROFILE_SLA}._extract_profiler_params", @@ -1254,7 +1254,7 @@ def test_run_profile_calls_interpolation_with_resolved_backend_for_disagg( 2000.0, 50.0, __import__( - "dynamo.profiler.utils.defaults", fromlist=["SearchStrategy"] + "dingo.profiler.utils.defaults", fromlist=["SearchStrategy"] ).SearchStrategy.RAPID, "autoscale", ), @@ -1312,7 +1312,7 @@ class TestValidateDgdServiceNameLengths: """Unit tests for _validate_dgd_service_name_lengths.""" try: - from dynamo.profiler.profile_sla import _validate_dgd_service_name_lengths + from dingo.profiler.profile_sla import _validate_dgd_service_name_lengths except ImportError: pass @@ -1328,7 +1328,7 @@ def _make_dgdr(self) -> "DynamoGraphDeploymentRequestSpec": def test_passes_when_combined_length_at_limit(self, monkeypatch): """Names summing to exactly 45 should not raise.""" - from dynamo.profiler.profile_sla import _validate_dgd_service_name_lengths + from dingo.profiler.profile_sla import _validate_dgd_service_name_lengths # dgdr_name="a" * 9 → dgd_name="a"*9 + "-dgd" = 13 chars; svc = 32 chars → 45 total monkeypatch.setenv("DGDR_NAME", "a" * 9) @@ -1338,7 +1338,7 @@ def test_passes_when_combined_length_at_limit(self, monkeypatch): def test_raises_when_combined_length_exceeds_limit(self, monkeypatch): """Names summing to 46 should raise ValueError with detail.""" - from dynamo.profiler.profile_sla import _validate_dgd_service_name_lengths + from dingo.profiler.profile_sla import _validate_dgd_service_name_lengths # dgdr_name="a"*9 → dgd_name=13; svc=33 → combined=46 monkeypatch.setenv("DGDR_NAME", "a" * 9) @@ -1349,7 +1349,7 @@ def test_raises_when_combined_length_exceeds_limit(self, monkeypatch): def test_raises_lists_all_violations(self, monkeypatch): """All offending service names should appear in the error message.""" - from dynamo.profiler.profile_sla import _validate_dgd_service_name_lengths + from dingo.profiler.profile_sla import _validate_dgd_service_name_lengths # dgdr_name="a"*27 → dgd_name=31 chars; TRTLLM names (18-19) → combined 49-50 > 45 monkeypatch.setenv("DGDR_NAME", "a" * 27) @@ -1366,7 +1366,7 @@ def test_raises_lists_all_violations(self, monkeypatch): def test_fallback_to_config_metadata_name_when_dgdr_name_unset(self, monkeypatch): """Without DGDR_NAME, the name in final_config.metadata.name is used.""" - from dynamo.profiler.profile_sla import _validate_dgd_service_name_lengths + from dingo.profiler.profile_sla import _validate_dgd_service_name_lengths monkeypatch.delenv("DGDR_NAME", raising=False) dgdr = self._make_dgdr() @@ -1376,7 +1376,7 @@ def test_fallback_to_config_metadata_name_when_dgdr_name_unset(self, monkeypatch def test_fallback_raises_when_config_name_causes_violation(self, monkeypatch): """Without DGDR_NAME, a long metadata.name still triggers a violation.""" - from dynamo.profiler.profile_sla import _validate_dgd_service_name_lengths + from dingo.profiler.profile_sla import _validate_dgd_service_name_lengths monkeypatch.delenv("DGDR_NAME", raising=False) dgdr = self._make_dgdr() @@ -1394,7 +1394,7 @@ def test_skips_when_neither_dgdr_name_nor_config_name_available( """No DGDR_NAME and no metadata.name in config → debug log, no raise.""" import logging - from dynamo.profiler.profile_sla import _validate_dgd_service_name_lengths + from dingo.profiler.profile_sla import _validate_dgd_service_name_lengths monkeypatch.delenv("DGDR_NAME", raising=False) dgdr = self._make_dgdr() @@ -1405,8 +1405,8 @@ def test_skips_when_neither_dgdr_name_nor_config_name_available( def test_respects_dgd_name_override(self, monkeypatch): """User-supplied DGD name override is used instead of dgdr_name + '-dgd'.""" - from dynamo.profiler.profile_sla import _validate_dgd_service_name_lengths - from dynamo.profiler.utils.dgdr_v1beta1_types import OverridesSpec + from dingo.profiler.profile_sla import _validate_dgd_service_name_lengths + from dingo.profiler.utils.dgdr_v1beta1_types import OverridesSpec monkeypatch.setenv("DGDR_NAME", "a" * 9) # Override makes dgd_name very long; "x"*43 + "svc"(3) = 46 > 45 diff --git a/components/src/dynamo/profiler/tests/unit/test_helpers_rapid.py b/dingo/profiler/tests/unit/test_helpers_rapid.py similarity index 87% rename from components/src/dynamo/profiler/tests/unit/test_helpers_rapid.py rename to dingo/profiler/tests/unit/test_helpers_rapid.py index 0b163f5db98e..97693a1d3d34 100644 --- a/components/src/dynamo/profiler/tests/unit/test_helpers_rapid.py +++ b/dingo/profiler/tests/unit/test_helpers_rapid.py @@ -22,8 +22,8 @@ ] try: - from dynamo.profiler.rapid import _run_default_sim, _run_naive_fallback - from dynamo.profiler.utils.dgdr_v1beta1_types import ( + from dingo.profiler.rapid import _run_default_sim, _run_naive_fallback + from dingo.profiler.utils.dgdr_v1beta1_types import ( DynamoGraphDeploymentRequestSpec, FeaturesSpec, HardwareSpec, @@ -76,11 +76,11 @@ def test_returns_expected_structure(self): dgdr = _make_dgdr() with ( patch( - "dynamo.profiler.rapid.build_naive_generator_params", + "dingo.profiler.rapid.build_naive_generator_params", return_value=copy.deepcopy(_FAKE_GENERATOR_PARAMS), ), patch( - "dynamo.profiler.rapid.generate_backend_artifacts", + "dingo.profiler.rapid.generate_backend_artifacts", return_value={}, ), ): @@ -108,11 +108,11 @@ def test_empty_artifacts_yields_none_dgd_config(self): dgdr = _make_dgdr() with ( patch( - "dynamo.profiler.rapid.build_naive_generator_params", + "dingo.profiler.rapid.build_naive_generator_params", return_value=copy.deepcopy(_FAKE_GENERATOR_PARAMS), ), patch( - "dynamo.profiler.rapid.generate_backend_artifacts", + "dingo.profiler.rapid.generate_backend_artifacts", return_value={}, ), ): @@ -140,11 +140,11 @@ def fake_generate(params, backend, use_dynamo_generator=False): with ( patch( - "dynamo.profiler.rapid.build_naive_generator_params", + "dingo.profiler.rapid.build_naive_generator_params", return_value=copy.deepcopy(_FAKE_GENERATOR_PARAMS), ), patch( - "dynamo.profiler.rapid.generate_backend_artifacts", + "dingo.profiler.rapid.generate_backend_artifacts", side_effect=fake_generate, ), ): @@ -170,11 +170,11 @@ def fake_generate(params, backend, use_dynamo_generator=False): with ( patch( - "dynamo.profiler.rapid.build_naive_generator_params", + "dingo.profiler.rapid.build_naive_generator_params", return_value=copy.deepcopy(_FAKE_GENERATOR_PARAMS), ), patch( - "dynamo.profiler.rapid.generate_backend_artifacts", + "dingo.profiler.rapid.generate_backend_artifacts", side_effect=fake_generate, ), ): @@ -202,13 +202,13 @@ def _execute_return(self, chosen="disagg", ttft=100.0, tpot=10.0): def test_returns_required_keys(self): dgdr = _make_dgdr() with ( - patch("dynamo.profiler.rapid.build_default_task_configs", return_value={}), + patch("dingo.profiler.rapid.build_default_task_configs", return_value={}), patch( - "dynamo.profiler.rapid._execute_task_configs", + "dingo.profiler.rapid._execute_task_configs", return_value=self._execute_return(), ), patch( - "dynamo.profiler.rapid._generate_dgd_from_pick", + "dingo.profiler.rapid._generate_dgd_from_pick", return_value={"kind": "DGD"}, ), ): @@ -247,11 +247,11 @@ def fake_execute(task_configs, mode, top_n, **kwargs): return self._execute_return() with ( - patch("dynamo.profiler.rapid.build_default_task_configs", return_value={}), + patch("dingo.profiler.rapid.build_default_task_configs", return_value={}), patch( - "dynamo.profiler.rapid._execute_task_configs", side_effect=fake_execute + "dingo.profiler.rapid._execute_task_configs", side_effect=fake_execute ), - patch("dynamo.profiler.rapid._generate_dgd_from_pick", return_value=None), + patch("dingo.profiler.rapid._generate_dgd_from_pick", return_value=None), ): _run_default_sim( dgdr, @@ -283,11 +283,11 @@ def fake_execute(task_configs, mode, top_n, **kwargs): return self._execute_return() with ( - patch("dynamo.profiler.rapid.build_default_task_configs", return_value={}), + patch("dingo.profiler.rapid.build_default_task_configs", return_value={}), patch( - "dynamo.profiler.rapid._execute_task_configs", side_effect=fake_execute + "dingo.profiler.rapid._execute_task_configs", side_effect=fake_execute ), - patch("dynamo.profiler.rapid._generate_dgd_from_pick", return_value=None), + patch("dingo.profiler.rapid._generate_dgd_from_pick", return_value=None), ): _run_default_sim( dgdr, @@ -312,12 +312,12 @@ def test_latencies_extracted_from_chosen_exp(self): """best_latencies come from the chosen experiment's entry.""" dgdr = _make_dgdr() with ( - patch("dynamo.profiler.rapid.build_default_task_configs", return_value={}), + patch("dingo.profiler.rapid.build_default_task_configs", return_value={}), patch( - "dynamo.profiler.rapid._execute_task_configs", + "dingo.profiler.rapid._execute_task_configs", return_value=self._execute_return(ttft=123.0, tpot=7.0), ), - patch("dynamo.profiler.rapid._generate_dgd_from_pick", return_value=None), + patch("dingo.profiler.rapid._generate_dgd_from_pick", return_value=None), ): result = _run_default_sim( dgdr, @@ -349,12 +349,12 @@ class TestRunDefaultSimForceDisagg: def _call_default_sim(self, dgdr, execute_return_value): with ( - patch("dynamo.profiler.rapid.build_default_task_configs", return_value={}), + patch("dingo.profiler.rapid.build_default_task_configs", return_value={}), patch( - "dynamo.profiler.rapid._execute_task_configs", + "dingo.profiler.rapid._execute_task_configs", return_value=execute_return_value, ), - patch("dynamo.profiler.rapid._generate_dgd_from_pick", return_value=None), + patch("dingo.profiler.rapid._generate_dgd_from_pick", return_value=None), ): return _run_default_sim( dgdr, diff --git a/components/src/dynamo/profiler/tests/unit/test_helpers_thorough.py b/dingo/profiler/tests/unit/test_helpers_thorough.py similarity index 91% rename from components/src/dynamo/profiler/tests/unit/test_helpers_thorough.py rename to dingo/profiler/tests/unit/test_helpers_thorough.py index 4b03ca533b7a..a73dc86ff695 100644 --- a/components/src/dynamo/profiler/tests/unit/test_helpers_thorough.py +++ b/dingo/profiler/tests/unit/test_helpers_thorough.py @@ -21,9 +21,9 @@ ] try: - from dynamo.profiler.thorough import _pick_thorough_best_config - from dynamo.profiler.utils.aic_dataframe import build_decode_row, build_prefill_row - from dynamo.profiler.utils.dgdr_v1beta1_types import ( + from dingo.profiler.thorough import _pick_thorough_best_config + from dingo.profiler.utils.aic_dataframe import build_decode_row, build_prefill_row + from dingo.profiler.utils.dgdr_v1beta1_types import ( DynamoGraphDeploymentRequestSpec, HardwareSpec, SLASpec, @@ -112,7 +112,7 @@ def test_autoscale_calls_pick_autoscale(self): mock_result = _mock_result() with patch( - "dynamo.profiler.thorough.pick_autoscale", return_value=mock_result + "dingo.profiler.thorough.pick_autoscale", return_value=mock_result ) as mock_pick: result = _pick_thorough_best_config( prefill_df, @@ -136,7 +136,7 @@ def test_load_match_uses_request_latency_when_set(self): dgdr = _make_dgdr(workload=WorkloadSpec(isl=4000, osl=1000, requestRate=5.0)) with patch( - "dynamo.profiler.thorough.pick_load_match", return_value=_mock_result() + "dingo.profiler.thorough.pick_load_match", return_value=_mock_result() ) as mock_pick: _pick_thorough_best_config( prefill_df, @@ -163,7 +163,7 @@ def test_load_match_falls_back_to_target_tpot(self): dgdr = _make_dgdr() with patch( - "dynamo.profiler.thorough.pick_load_match", return_value=_mock_result() + "dingo.profiler.thorough.pick_load_match", return_value=_mock_result() ) as mock_pick: _pick_thorough_best_config( prefill_df, @@ -188,7 +188,7 @@ def test_default_uses_request_latency_when_set(self): dgdr = _make_dgdr() with patch( - "dynamo.profiler.thorough.pick_default", return_value=_mock_result() + "dingo.profiler.thorough.pick_default", return_value=_mock_result() ) as mock_pick: _pick_thorough_best_config( prefill_df, @@ -214,7 +214,7 @@ def test_default_falls_back_to_target_tpot(self): dgdr = _make_dgdr() with patch( - "dynamo.profiler.thorough.pick_default", return_value=_mock_result() + "dingo.profiler.thorough.pick_default", return_value=_mock_result() ) as mock_pick: _pick_thorough_best_config( prefill_df, @@ -239,7 +239,7 @@ def test_load_match_omits_workload_kwargs_when_no_workload(self): dgdr = _make_dgdr() # no requestRate or concurrency with patch( - "dynamo.profiler.thorough.pick_load_match", return_value=_mock_result() + "dingo.profiler.thorough.pick_load_match", return_value=_mock_result() ) as mock_pick: _pick_thorough_best_config( prefill_df, diff --git a/components/src/dynamo/profiler/tests/unit/test_main_parse_config.py b/dingo/profiler/tests/unit/test_main_parse_config.py similarity index 93% rename from components/src/dynamo/profiler/tests/unit/test_main_parse_config.py rename to dingo/profiler/tests/unit/test_main_parse_config.py index d70688e4ccc7..d840a92e11c2 100644 --- a/components/src/dynamo/profiler/tests/unit/test_main_parse_config.py +++ b/dingo/profiler/tests/unit/test_main_parse_config.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for profiler ``--config`` parsing in ``dynamo.profiler.__main__``.""" +"""Unit tests for profiler ``--config`` parsing in ``dingo.profiler.__main__``.""" import json @@ -16,7 +16,7 @@ ] try: - import dynamo.profiler.__main__ as profiler_main + import dingo.profiler.__main__ as profiler_main except ImportError: pytest.skip("dynamo.llm bindings not available", allow_module_level=True) diff --git a/components/src/dynamo/profiler/tests/unit/test_model_info.py b/dingo/profiler/tests/unit/test_model_info.py similarity index 91% rename from components/src/dynamo/profiler/tests/unit/test_model_info.py rename to dingo/profiler/tests/unit/test_model_info.py index 5adb7ceef1d0..aa5c16c372f7 100644 --- a/components/src/dynamo/profiler/tests/unit/test_model_info.py +++ b/dingo/profiler/tests/unit/test_model_info.py @@ -15,7 +15,7 @@ ] try: - from dynamo.profiler.utils.model_info import get_mamba_cache_align_block_size + from dingo.profiler.utils.model_info import get_mamba_cache_align_block_size except ImportError as e: pytest.skip(f"Skip (missing dependency): {e}", allow_module_level=True) diff --git a/components/src/dynamo/profiler/tests/unit/test_pareto.py b/dingo/profiler/tests/unit/test_pareto.py similarity index 91% rename from components/src/dynamo/profiler/tests/unit/test_pareto.py rename to dingo/profiler/tests/unit/test_pareto.py index cdec4d582029..ce9b721e422e 100644 --- a/components/src/dynamo/profiler/tests/unit/test_pareto.py +++ b/dingo/profiler/tests/unit/test_pareto.py @@ -3,7 +3,7 @@ import pytest -from dynamo.profiler.utils.pareto import compute_pareto +from dingo.profiler.utils.pareto import compute_pareto pytestmark = [ pytest.mark.pre_merge, diff --git a/components/src/dynamo/profiler/tests/unit/test_planner_image_selection.py b/dingo/profiler/tests/unit/test_planner_image_selection.py similarity index 94% rename from components/src/dynamo/profiler/tests/unit/test_planner_image_selection.py rename to dingo/profiler/tests/unit/test_planner_image_selection.py index 0abcde031f79..a780d35c1020 100644 --- a/components/src/dynamo/profiler/tests/unit/test_planner_image_selection.py +++ b/dingo/profiler/tests/unit/test_planner_image_selection.py @@ -11,15 +11,15 @@ ] try: - from dynamo.profiler.utils.config import update_image - from dynamo.profiler.utils.dgd_generation import add_planner_to_config - from dynamo.profiler.utils.dgdr_v1beta1_types import ( + from dingo.profiler.utils.config import update_image + from dingo.profiler.utils.dgd_generation import add_planner_to_config + from dingo.profiler.utils.dgdr_v1beta1_types import ( DynamoGraphDeploymentRequestSpec, HardwareSpec, SLASpec, WorkloadSpec, ) - from dynamo.profiler.utils.profile_common import ( + from dingo.profiler.utils.profile_common import ( derive_backend_image, derive_planner_image, ) diff --git a/components/src/dynamo/profiler/tests/unit/test_profile_sla_auto_backend.py b/dingo/profiler/tests/unit/test_profile_sla_auto_backend.py similarity index 91% rename from components/src/dynamo/profiler/tests/unit/test_profile_sla_auto_backend.py rename to dingo/profiler/tests/unit/test_profile_sla_auto_backend.py index 0a5f61ce8498..0dce2e3d4995 100644 --- a/components/src/dynamo/profiler/tests/unit/test_profile_sla_auto_backend.py +++ b/dingo/profiler/tests/unit/test_profile_sla_auto_backend.py @@ -6,7 +6,7 @@ import pytest try: - from dynamo.profiler.rapid import _DEFAULT_NAIVE_BACKEND + from dingo.profiler.rapid import _DEFAULT_NAIVE_BACKEND except ImportError: pytest.skip("dynamo.llm bindings not available", allow_module_level=True) @@ -25,7 +25,7 @@ def test_autoscale_sim_resolves_auto_to_default() -> None: """ import inspect - from dynamo.profiler.rapid import _run_autoscale_sim + from dingo.profiler.rapid import _run_autoscale_sim src = inspect.getsource(_run_autoscale_sim) # The function must guard against "auto" before TaskConfig is constructed. @@ -43,7 +43,7 @@ def test_autoscale_sim_returns_resolved_backend() -> None: """ import inspect - from dynamo.profiler.rapid import _run_autoscale_sim + from dingo.profiler.rapid import _run_autoscale_sim src = inspect.getsource(_run_autoscale_sim) assert ( @@ -60,7 +60,7 @@ def test_naive_fallback_resolves_auto_to_default() -> None: """ import inspect - from dynamo.profiler.rapid import _run_naive_fallback + from dingo.profiler.rapid import _run_naive_fallback src = inspect.getsource(_run_naive_fallback) assert ( @@ -78,7 +78,7 @@ def test_default_sim_returns_resolved_backend() -> None: """ import inspect - from dynamo.profiler.rapid import _run_default_sim + from dingo.profiler.rapid import _run_default_sim src = inspect.getsource(_run_default_sim) assert ( diff --git a/components/src/dynamo/profiler/tests/unit/test_profiler_protocol.py b/dingo/profiler/tests/unit/test_profiler_protocol.py similarity index 94% rename from components/src/dynamo/profiler/tests/unit/test_profiler_protocol.py rename to dingo/profiler/tests/unit/test_profiler_protocol.py index 7d78c2251638..d175cd8c7b1e 100644 --- a/components/src/dynamo/profiler/tests/unit/test_profiler_protocol.py +++ b/dingo/profiler/tests/unit/test_profiler_protocol.py @@ -18,20 +18,20 @@ ] try: - from dynamo.profiler.utils.config_modifiers import CONFIG_MODIFIERS - from dynamo.profiler.utils.config_modifiers.parallelization_mapping import ( + from dingo.profiler.utils.config_modifiers import CONFIG_MODIFIERS + from dingo.profiler.utils.config_modifiers.parallelization_mapping import ( PickedParallelConfig, ) - from dynamo.profiler.utils.config_modifiers.protocol import ( + from dingo.profiler.utils.config_modifiers.protocol import ( BaseConfigModifier, apply_dgd_overrides, ) - from dynamo.profiler.utils.defaults import EngineType, SearchStrategy - from dynamo.profiler.utils.dgdr_v1beta1_types import ( + from dingo.profiler.utils.defaults import EngineType, SearchStrategy + from dingo.profiler.utils.dgdr_v1beta1_types import ( DynamoGraphDeploymentRequestSpec, OverridesSpec, ) - from dynamo.profiler.utils.profile_common import ProfilerOperationalConfig + from dingo.profiler.utils.profile_common import ProfilerOperationalConfig except ImportError: pytest.skip("dynamo.llm bindings not available", allow_module_level=True) @@ -190,7 +190,7 @@ def test_vllm_mamba_align_raises_max_num_batched_tokens() -> None: ] with patch( - "dynamo.profiler.utils.config_modifiers.vllm.get_mamba_cache_align_block_size", + "dingo.profiler.utils.config_modifiers.vllm.get_mamba_cache_align_block_size", return_value=8320, ): result = modifier._apply_mamba_cache_align_token_floor(args, "nemotron") @@ -208,7 +208,7 @@ def test_vllm_mamba_align_skips_without_explicit_align_mode() -> None: ] with patch( - "dynamo.profiler.utils.config_modifiers.vllm.get_mamba_cache_align_block_size" + "dingo.profiler.utils.config_modifiers.vllm.get_mamba_cache_align_block_size" ) as mock_floor: result = modifier._apply_mamba_cache_align_token_floor(args, "llama") @@ -242,7 +242,7 @@ def test_vllm_model_runtime_constraints_update_decode_config() -> None: } with patch( - "dynamo.profiler.utils.config_modifiers.vllm.get_mamba_cache_align_block_size", + "dingo.profiler.utils.config_modifiers.vllm.get_mamba_cache_align_block_size", return_value=8320, ): result = modifier.apply_model_runtime_constraints(config, "nemotron") @@ -492,7 +492,7 @@ def test_apply_dgd_overrides_missing_service_skipped_with_warning(caplog) -> Non } with caplog.at_level( - logging.WARNING, logger="dynamo.profiler.utils.config_modifiers.protocol" + logging.WARNING, logger="dingo.profiler.utils.config_modifiers.protocol" ): result = apply_dgd_overrides(dgd_config, overrides) @@ -544,7 +544,7 @@ async def test_run_profile_applies_dgd_overrides_before_interpolation( deployed without extraPodSpec.tolerations, causing them to stay Pending on GPU nodes with nvidia.com/gpu:NoSchedule taints. """ - from dynamo.profiler.profile_sla import run_profile + from dingo.profiler.profile_sla import run_profile base_dgd = copy.deepcopy(_BASE_DGD) dgdr = DynamoGraphDeploymentRequestSpec( @@ -568,14 +568,14 @@ async def _fake_interpolation(dgdr_arg, ops_arg, disagg_config, *args, **kwargs) } with ( - patch("dynamo.profiler.profile_sla.valid_dgdr_spec"), - patch("dynamo.profiler.profile_sla.validate_dgdr_dynamo_features"), + patch("dingo.profiler.profile_sla.valid_dgdr_spec"), + patch("dingo.profiler.profile_sla.validate_dgdr_dynamo_features"), patch( - "dynamo.profiler.profile_sla.check_model_hardware_support", + "dingo.profiler.profile_sla.check_model_hardware_support", return_value=True, ), patch( - "dynamo.profiler.profile_sla._extract_profiler_params", + "dingo.profiler.profile_sla._extract_profiler_params", return_value=( "test/model", "vllm", @@ -591,7 +591,7 @@ async def _fake_interpolation(dgdr_arg, ops_arg, disagg_config, *args, **kwargs) ), ), patch( - "dynamo.profiler.profile_sla._execute_strategy", + "dingo.profiler.profile_sla._execute_strategy", new=AsyncMock( return_value=( pick_result, @@ -602,25 +602,25 @@ async def _fake_interpolation(dgdr_arg, ops_arg, disagg_config, *args, **kwargs) ) ), ), - patch("dynamo.profiler.profile_sla.needs_profile_data", return_value=True), + patch("dingo.profiler.profile_sla.needs_profile_data", return_value=True), patch( - "dynamo.profiler.profile_sla.run_interpolation", + "dingo.profiler.profile_sla.run_interpolation", new=_fake_interpolation, ), patch( - "dynamo.profiler.profile_sla.assemble_final_config", + "dingo.profiler.profile_sla.assemble_final_config", return_value=copy.deepcopy(base_dgd), ), - patch("dynamo.profiler.profile_sla._write_final_output", return_value=True), - patch("dynamo.profiler.profile_sla.write_profiler_status"), + patch("dingo.profiler.profile_sla._write_final_output", return_value=True), + patch("dingo.profiler.profile_sla.write_profiler_status"), patch( - "dynamo.profiler.profile_sla.cleanup_remaining_deployments", + "dingo.profiler.profile_sla.cleanup_remaining_deployments", new=AsyncMock(), ), ): with caplog.at_level( logging.WARNING, - logger="dynamo.profiler.utils.config_modifiers.protocol", + logger="dingo.profiler.utils.config_modifiers.protocol", ): await run_profile(dgdr, ops) diff --git a/components/src/dynamo/profiler/tests/unit/test_rapid_planner_replicas.py b/dingo/profiler/tests/unit/test_rapid_planner_replicas.py similarity index 94% rename from components/src/dynamo/profiler/tests/unit/test_rapid_planner_replicas.py rename to dingo/profiler/tests/unit/test_rapid_planner_replicas.py index f30bf9ec793f..0a86b688a15e 100644 --- a/components/src/dynamo/profiler/tests/unit/test_rapid_planner_replicas.py +++ b/dingo/profiler/tests/unit/test_rapid_planner_replicas.py @@ -34,8 +34,8 @@ try: from aiconfigurator.sdk.task import TaskConfig - from dynamo.profiler.rapid import _generate_dgd_from_pick - from dynamo.profiler.utils.dgdr_v1beta1_types import ( + from dingo.profiler.rapid import _generate_dgd_from_pick + from dingo.profiler.utils.dgdr_v1beta1_types import ( DynamoGraphDeploymentRequestSpec, HardwareSpec, ModelCacheSpec, @@ -116,11 +116,11 @@ def fake_task_config_to_generator_config( with ( patch( - "dynamo.profiler.rapid.task_config_to_generator_config", + "dingo.profiler.rapid.task_config_to_generator_config", side_effect=fake_task_config_to_generator_config, ), patch( - "dynamo.profiler.rapid.generate_backend_artifacts", + "dingo.profiler.rapid.generate_backend_artifacts", return_value={"k8s_deploy.yaml": ""}, ), ): @@ -171,11 +171,11 @@ def fake_task_config_to_generator_config( with ( patch( - "dynamo.profiler.rapid.task_config_to_generator_config", + "dingo.profiler.rapid.task_config_to_generator_config", side_effect=fake_task_config_to_generator_config, ), patch( - "dynamo.profiler.rapid.generate_backend_artifacts", + "dingo.profiler.rapid.generate_backend_artifacts", return_value={"k8s_deploy.yaml": ""}, ), ): diff --git a/components/src/dynamo/profiler/tests/unit/test_replay_aic_parity.py b/dingo/profiler/tests/unit/test_replay_aic_parity.py similarity index 98% rename from components/src/dynamo/profiler/tests/unit/test_replay_aic_parity.py rename to dingo/profiler/tests/unit/test_replay_aic_parity.py index 341c837e0d0e..37befd78b8e9 100644 --- a/components/src/dynamo/profiler/tests/unit/test_replay_aic_parity.py +++ b/dingo/profiler/tests/unit/test_replay_aic_parity.py @@ -5,8 +5,8 @@ import pytest -from dynamo.mocker import MockEngineArgs -from dynamo.replay import run_synthetic_trace_replay +from dingo.mocker import MockEngineArgs +from dingo.replay import run_synthetic_trace_replay # run_synthetic_trace_replay constructs the Rust AIC callback, which imports # aiconfigurator.sdk.engine (Phase 1.5 compile_engine API). Skip if absent — diff --git a/components/src/dynamo/profiler/tests/unit/test_resolve_model_path.py b/dingo/profiler/tests/unit/test_resolve_model_path.py similarity index 88% rename from components/src/dynamo/profiler/tests/unit/test_resolve_model_path.py rename to dingo/profiler/tests/unit/test_resolve_model_path.py index 058b54c55af1..f5cab77bd779 100644 --- a/components/src/dynamo/profiler/tests/unit/test_resolve_model_path.py +++ b/dingo/profiler/tests/unit/test_resolve_model_path.py @@ -17,20 +17,20 @@ ] try: - from dynamo.profiler.rapid import ( + from dingo.profiler.rapid import ( _generate_dgd_from_pick, _run_autoscale_sim, _run_default_sim, ) - from dynamo.profiler.thorough import run_thorough - from dynamo.profiler.utils.dgdr_v1beta1_types import ( + from dingo.profiler.thorough import run_thorough + from dingo.profiler.utils.dgdr_v1beta1_types import ( DynamoGraphDeploymentRequestSpec, HardwareSpec, ModelCacheSpec, SLASpec, WorkloadSpec, ) - from dynamo.profiler.utils.profile_common import ( + from dingo.profiler.utils.profile_common import ( ProfilerOperationalConfig, resolve_model_path, ) @@ -177,13 +177,13 @@ def test_default_sim_uses_local_path_when_pvc_mounted(self, tmp_path): with ( patch( - "dynamo.profiler.rapid.build_default_task_configs", return_value={} + "dingo.profiler.rapid.build_default_task_configs", return_value={} ) as mock_build, patch( - "dynamo.profiler.rapid._execute_task_configs", + "dingo.profiler.rapid._execute_task_configs", return_value=self._execute_return(), ), - patch("dynamo.profiler.rapid._generate_dgd_from_pick", return_value=None), + patch("dingo.profiler.rapid._generate_dgd_from_pick", return_value=None), ): _run_default_sim( dgdr, @@ -207,13 +207,13 @@ def test_default_sim_uses_hf_id_when_no_pvc(self): with ( patch( - "dynamo.profiler.rapid.build_default_task_configs", return_value={} + "dingo.profiler.rapid.build_default_task_configs", return_value={} ) as mock_build, patch( - "dynamo.profiler.rapid._execute_task_configs", + "dingo.profiler.rapid._execute_task_configs", return_value=self._execute_return(), ), - patch("dynamo.profiler.rapid._generate_dgd_from_pick", return_value=None), + patch("dingo.profiler.rapid._generate_dgd_from_pick", return_value=None), ): _run_default_sim( dgdr, @@ -240,9 +240,9 @@ def test_autoscale_sim_uses_local_path_when_pvc_mounted(self, tmp_path): runner = MagicMock() runner.run.return_value = {"pareto_df": pd.DataFrame()} with ( - patch("dynamo.profiler.rapid.TaskConfig") as mock_task_config, - patch("dynamo.profiler.rapid.TaskRunner", return_value=runner), - patch("dynamo.profiler.rapid._generate_dgd_from_pick", return_value=None), + patch("dingo.profiler.rapid.TaskConfig") as mock_task_config, + patch("dingo.profiler.rapid.TaskRunner", return_value=runner), + patch("dingo.profiler.rapid._generate_dgd_from_pick", return_value=None), ): _run_autoscale_sim( dgdr, _HF_ID, "h200_sxm", "trtllm", 8, 4000, 1000, 2000.0, 50.0, None @@ -257,9 +257,9 @@ def test_autoscale_sim_uses_hf_id_when_no_pvc(self): runner = MagicMock() runner.run.return_value = {"pareto_df": pd.DataFrame()} with ( - patch("dynamo.profiler.rapid.TaskConfig") as mock_task_config, - patch("dynamo.profiler.rapid.TaskRunner", return_value=runner), - patch("dynamo.profiler.rapid._generate_dgd_from_pick", return_value=None), + patch("dingo.profiler.rapid.TaskConfig") as mock_task_config, + patch("dingo.profiler.rapid.TaskRunner", return_value=runner), + patch("dingo.profiler.rapid._generate_dgd_from_pick", return_value=None), ): _run_autoscale_sim( dgdr, _HF_ID, "h200_sxm", "trtllm", 8, 4000, 1000, 2000.0, 50.0, None @@ -304,11 +304,11 @@ def _capture_generate(self, dgdr, cfg) -> MagicMock: task_configs = {"disagg": self._task_config()} with ( patch( - "dynamo.profiler.rapid.task_config_to_generator_config", + "dingo.profiler.rapid.task_config_to_generator_config", return_value=cfg, ), patch( - "dynamo.profiler.rapid.generate_backend_artifacts", + "dingo.profiler.rapid.generate_backend_artifacts", return_value={"k8s_deploy.yaml": ""}, ) as mock_generate, ): @@ -353,15 +353,15 @@ async def _capture_enumerate(self, dgdr, output_dir) -> MagicMock: ops = ProfilerOperationalConfig(output_dir=str(output_dir)) with ( patch( - "dynamo.profiler.thorough.enumerate_profiling_configs", + "dingo.profiler.thorough.enumerate_profiling_configs", return_value=([], []), ) as mock_enumerate, patch( - "dynamo.profiler.thorough._benchmark_prefill_candidates", + "dingo.profiler.thorough._benchmark_prefill_candidates", new=AsyncMock(return_value=pd.DataFrame()), ), patch( - "dynamo.profiler.thorough._benchmark_decode_candidates", + "dingo.profiler.thorough._benchmark_decode_candidates", new=AsyncMock(return_value=pd.DataFrame()), ), ): @@ -409,24 +409,24 @@ async def _capture_task_config(self, dgdr, output_dir) -> MagicMock: nonempty_df = pd.DataFrame([{"tp": 1}]) with ( patch( - "dynamo.profiler.thorough.enumerate_profiling_configs", + "dingo.profiler.thorough.enumerate_profiling_configs", return_value=([], []), ), patch( - "dynamo.profiler.thorough._benchmark_prefill_candidates", + "dingo.profiler.thorough._benchmark_prefill_candidates", new=AsyncMock(return_value=nonempty_df), ), patch( - "dynamo.profiler.thorough._benchmark_decode_candidates", + "dingo.profiler.thorough._benchmark_decode_candidates", new=AsyncMock(return_value=nonempty_df), ), patch( - "dynamo.profiler.thorough._pick_thorough_best_config", + "dingo.profiler.thorough._pick_thorough_best_config", return_value={"best_config_df": pd.DataFrame()}, ), - patch("dynamo.profiler.thorough.TaskConfig") as mock_task_config, + patch("dingo.profiler.thorough.TaskConfig") as mock_task_config, patch( - "dynamo.profiler.thorough._generate_dgd_from_pick", + "dingo.profiler.thorough._generate_dgd_from_pick", return_value=None, ), ): diff --git a/components/src/dynamo/profiler/thorough.py b/dingo/profiler/thorough.py similarity index 97% rename from components/src/dynamo/profiler/thorough.py rename to dingo/profiler/thorough.py index d481b2f01ce4..26e0f7b17523 100644 --- a/components/src/dynamo/profiler/thorough.py +++ b/dingo/profiler/thorough.py @@ -25,25 +25,25 @@ from aiconfigurator.sdk.task import TaskConfig from deploy.utils.dynamo_deployment import DeploymentFailedError, DynamoDeploymentClient -from dynamo.profiler.rapid import _generate_dgd_from_pick -from dynamo.profiler.utils.aic_dataframe import ( +from dingo.profiler.rapid import _generate_dgd_from_pick +from dingo.profiler.utils.aic_dataframe import ( build_decode_row, build_disagg_df_from_static, build_prefill_row, make_parallel_label, ) -from dynamo.profiler.utils.aiperf import ( +from dingo.profiler.utils.aiperf import ( get_decode_itl_and_thpt_per_gpu, get_prefill_ttft, ) -from dynamo.profiler.utils.config_modifiers import CONFIG_MODIFIERS -from dynamo.profiler.utils.config_modifiers.protocol import apply_dgd_overrides -from dynamo.profiler.utils.dgdr_v1beta1_types import ( +from dingo.profiler.utils.config_modifiers import CONFIG_MODIFIERS +from dingo.profiler.utils.config_modifiers.protocol import apply_dgd_overrides +from dingo.profiler.utils.dgdr_v1beta1_types import ( DynamoGraphDeploymentRequestSpec, ModelCacheSpec, ProfilingPhase, ) -from dynamo.profiler.utils.profile_common import ( +from dingo.profiler.utils.profile_common import ( ProfilerOperationalConfig, derive_backend_image, get_profiling_job_tolerations, @@ -51,8 +51,8 @@ pick_decode_component, resolve_model_path, ) -from dynamo.profiler.utils.profile_decode import get_num_request_range -from dynamo.profiler.utils.profiler_status import ProfilerStatus, write_profiler_status +from dingo.profiler.utils.profile_decode import get_num_request_range +from dingo.profiler.utils.profiler_status import ProfilerStatus, write_profiler_status logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/profiler/utils/__init__.py b/dingo/profiler/utils/__init__.py similarity index 100% rename from components/src/dynamo/profiler/utils/__init__.py rename to dingo/profiler/utils/__init__.py diff --git a/components/src/dynamo/profiler/utils/aic_dataframe.py b/dingo/profiler/utils/aic_dataframe.py similarity index 100% rename from components/src/dynamo/profiler/utils/aic_dataframe.py rename to dingo/profiler/utils/aic_dataframe.py diff --git a/components/src/dynamo/profiler/utils/aiperf.py b/dingo/profiler/utils/aiperf.py similarity index 99% rename from components/src/dynamo/profiler/utils/aiperf.py rename to dingo/profiler/utils/aiperf.py index f943650c0827..942a7096eabc 100644 --- a/components/src/dynamo/profiler/utils/aiperf.py +++ b/dingo/profiler/utils/aiperf.py @@ -20,7 +20,7 @@ import subprocess from typing import Optional, Tuple -from dynamo.profiler.utils.defaults import ( +from dingo.profiler.utils.defaults import ( AIPERF_PREFILL_ATTN_DP_NUM_REQ_RATIO, AIPERF_PREFILL_BENCHMARK_OSL, AIPERF_WARMUP_REQUEST_PER_DP_RANK, diff --git a/components/src/dynamo/profiler/utils/config.py b/dingo/profiler/utils/config.py similarity index 98% rename from components/src/dynamo/profiler/utils/config.py rename to dingo/profiler/utils/config.py index 2285283de0e9..5afba857f0b4 100644 --- a/components/src/dynamo/profiler/utils/config.py +++ b/dingo/profiler/utils/config.py @@ -23,9 +23,9 @@ from pydantic import BaseModel -from dynamo.common.utils.paths import get_workspace_dir -from dynamo.planner.config.backend_components import WORKER_COMPONENT_NAMES -from dynamo.planner.config.defaults import SubComponentType +from dingo.common.utils.paths import get_workspace_dir +from dingo.planner.config.backend_components import WORKER_COMPONENT_NAMES +from dingo.planner.config.defaults import SubComponentType logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -103,8 +103,8 @@ class DgdPlannerServiceConfig(BaseModel): extraPodSpec: PodSpec = PodSpec( mainContainer=Container( image="my-registry/dynamo-planner:my-tag", # placeholder - workingDir=f"{get_workspace_dir()}/components/src/dynamo/planner", - command=["python3", "-m", "dynamo.planner"], + workingDir=f"{get_workspace_dir()}/dingo/planner", + command=["python3", "-m", "dingo.planner"], args=[], ) ) diff --git a/components/src/dynamo/profiler/utils/config_modifiers/__init__.py b/dingo/profiler/utils/config_modifiers/__init__.py similarity index 81% rename from components/src/dynamo/profiler/utils/config_modifiers/__init__.py rename to dingo/profiler/utils/config_modifiers/__init__.py index 623528f40486..7f738df14337 100644 --- a/components/src/dynamo/profiler/utils/config_modifiers/__init__.py +++ b/dingo/profiler/utils/config_modifiers/__init__.py @@ -16,12 +16,12 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from dynamo.profiler.utils.config_modifiers.protocol import ( + from dingo.profiler.utils.config_modifiers.protocol import ( ConfigModifierProtocol, ) -from dynamo.profiler.utils.config_modifiers.sglang import SGLangConfigModifier -from dynamo.profiler.utils.config_modifiers.vllm import VllmV1ConfigModifier +from dingo.profiler.utils.config_modifiers.sglang import SGLangConfigModifier +from dingo.profiler.utils.config_modifiers.vllm import VllmV1ConfigModifier CONFIG_MODIFIERS: dict[str, type["ConfigModifierProtocol"]] = { "vllm": VllmV1ConfigModifier, diff --git a/components/src/dynamo/profiler/utils/config_modifiers/parallelization_mapping.py b/dingo/profiler/utils/config_modifiers/parallelization_mapping.py similarity index 96% rename from components/src/dynamo/profiler/utils/config_modifiers/parallelization_mapping.py rename to dingo/profiler/utils/config_modifiers/parallelization_mapping.py index ecb5a702ae94..b345d0b77df8 100644 --- a/components/src/dynamo/profiler/utils/config_modifiers/parallelization_mapping.py +++ b/dingo/profiler/utils/config_modifiers/parallelization_mapping.py @@ -4,7 +4,7 @@ """Profiler-internal parallelization helpers. ``PickedParallelConfig`` has been relocated to -``dynamo.planner.config.parallelization`` so both the profiler and the planner +``dingo.planner.config.parallelization`` so both the profiler and the planner can share it. It is re-exported here for back-compat. """ @@ -13,10 +13,10 @@ from dataclasses import dataclass from enum import Enum -from dynamo.planner.config.defaults import SubComponentType -from dynamo.planner.config.parallelization import PickedParallelConfig -from dynamo.profiler.utils.defaults import PREFILL_MAX_NUM_TOKENS -from dynamo.profiler.utils.model_info import MOE_ADDITIONAL_TP_ARCHITECTURES, ModelInfo +from dingo.planner.config.defaults import SubComponentType +from dingo.planner.config.parallelization import PickedParallelConfig +from dingo.profiler.utils.defaults import PREFILL_MAX_NUM_TOKENS +from dingo.profiler.utils.model_info import MOE_ADDITIONAL_TP_ARCHITECTURES, ModelInfo __all__ = [ "ParallelizationStrategy", diff --git a/components/src/dynamo/profiler/utils/config_modifiers/protocol.py b/dingo/profiler/utils/config_modifiers/protocol.py similarity index 99% rename from components/src/dynamo/profiler/utils/config_modifiers/protocol.py rename to dingo/profiler/utils/config_modifiers/protocol.py index 5e14d28a94e9..6132ebf5c648 100644 --- a/components/src/dynamo/profiler/utils/config_modifiers/protocol.py +++ b/dingo/profiler/utils/config_modifiers/protocol.py @@ -20,8 +20,8 @@ from typing import Any, Protocol, Tuple from uuid import uuid4 -from dynamo.planner.config.defaults import SubComponentType -from dynamo.profiler.utils.config import ( +from dingo.planner.config.defaults import SubComponentType +from dingo.profiler.utils.config import ( Config, Container, PodSpec, @@ -32,7 +32,7 @@ setup_worker_service_resources, update_image, ) -from dynamo.profiler.utils.defaults import EngineType +from dingo.profiler.utils.defaults import EngineType logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/profiler/utils/config_modifiers/sglang.py b/dingo/profiler/utils/config_modifiers/sglang.py similarity index 98% rename from components/src/dynamo/profiler/utils/config_modifiers/sglang.py rename to dingo/profiler/utils/config_modifiers/sglang.py index 22fdbb6034b5..78da40ba9c4b 100644 --- a/components/src/dynamo/profiler/utils/config_modifiers/sglang.py +++ b/dingo/profiler/utils/config_modifiers/sglang.py @@ -8,8 +8,8 @@ import yaml -from dynamo.planner.config.defaults import SubComponentType -from dynamo.profiler.utils.config import ( +from dingo.planner.config.defaults import SubComponentType +from dingo.profiler.utils.config import ( Config, append_argument, break_arguments, @@ -21,8 +21,8 @@ update_image, validate_and_get_worker_args, ) -from dynamo.profiler.utils.config_modifiers.protocol import BaseConfigModifier -from dynamo.profiler.utils.defaults import ( +from dingo.profiler.utils.config_modifiers.protocol import BaseConfigModifier +from dingo.profiler.utils.defaults import ( DYNAMO_RUN_DEFAULT_PORT, EngineType, resolve_deploy_path, diff --git a/components/src/dynamo/profiler/utils/config_modifiers/vllm.py b/dingo/profiler/utils/config_modifiers/vllm.py similarity index 98% rename from components/src/dynamo/profiler/utils/config_modifiers/vllm.py rename to dingo/profiler/utils/config_modifiers/vllm.py index 032c84e82eb2..1bb267bde74b 100644 --- a/components/src/dynamo/profiler/utils/config_modifiers/vllm.py +++ b/dingo/profiler/utils/config_modifiers/vllm.py @@ -8,8 +8,8 @@ import yaml from pydantic import ValidationError -from dynamo.planner.config.defaults import SubComponentType -from dynamo.profiler.utils.config import ( +from dingo.planner.config.defaults import SubComponentType +from dingo.profiler.utils.config import ( Config, append_argument, break_arguments, @@ -21,13 +21,13 @@ update_image, validate_and_get_worker_args, ) -from dynamo.profiler.utils.config_modifiers.protocol import BaseConfigModifier -from dynamo.profiler.utils.defaults import ( +from dingo.profiler.utils.config_modifiers.protocol import BaseConfigModifier +from dingo.profiler.utils.defaults import ( DYNAMO_RUN_DEFAULT_PORT, EngineType, resolve_deploy_path, ) -from dynamo.profiler.utils.model_info import get_mamba_cache_align_block_size +from dingo.profiler.utils.model_info import get_mamba_cache_align_block_size logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) diff --git a/components/src/dynamo/profiler/utils/defaults.py b/dingo/profiler/utils/defaults.py similarity index 96% rename from components/src/dynamo/profiler/utils/defaults.py rename to dingo/profiler/utils/defaults.py index 20813d27baf0..ffde1cae31e8 100644 --- a/components/src/dynamo/profiler/utils/defaults.py +++ b/dingo/profiler/utils/defaults.py @@ -22,7 +22,7 @@ def resolve_deploy_path(rel_path: str) -> str: Uses get_workspace_dir() which handles repo root, container, and env var cases. """ - from dynamo.common.utils.paths import get_workspace_dir + from dingo.common.utils.paths import get_workspace_dir return os.path.join(get_workspace_dir(), rel_path) diff --git a/components/src/dynamo/profiler/utils/dgd_generation.py b/dingo/profiler/utils/dgd_generation.py similarity index 98% rename from components/src/dynamo/profiler/utils/dgd_generation.py rename to dingo/profiler/utils/dgd_generation.py index b4d635a90bb1..057b4512e2a7 100644 --- a/components/src/dynamo/profiler/utils/dgd_generation.py +++ b/dingo/profiler/utils/dgd_generation.py @@ -22,23 +22,23 @@ import numpy as np import yaml -from dynamo.common.utils.paths import get_workspace_dir -from dynamo.planner.config.aic_interpolation_spec import AICInterpolationSpec -from dynamo.planner.config.backend_components import ( +from dingo.common.utils.paths import get_workspace_dir +from dingo.planner.config.aic_interpolation_spec import AICInterpolationSpec +from dingo.planner.config.backend_components import ( MockerComponentName, VllmComponentName, ) -from dynamo.planner.config.parallelization import ( +from dingo.planner.config.parallelization import ( PickedParallelConfig, picked_to_aic_model_config_kwargs, ) -from dynamo.planner.config.planner_config import ( +from dingo.planner.config.planner_config import ( AICPerfModelSpec, PlannerConfig, PlannerPreDeploymentSweepMode, ) -from dynamo.profiler.utils.config import DgdPlannerServiceConfig, set_argument_value -from dynamo.profiler.utils.profile_common import ( +from dingo.profiler.utils.config import DgdPlannerServiceConfig, set_argument_value +from dingo.profiler.utils.profile_common import ( ProfilerOperationalConfig, derive_planner_image, is_mocker_enabled, diff --git a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py b/dingo/profiler/utils/dgdr_v1beta1_types.py similarity index 97% rename from components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py rename to dingo/profiler/utils/dgdr_v1beta1_types.py index ae2ec415ae0d..051d98f870ff 100644 --- a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py +++ b/dingo/profiler/utils/dgdr_v1beta1_types.py @@ -27,7 +27,7 @@ from pydantic import BaseModel, Field, model_validator # Import canonical planner types - do NOT redefine them here. -from dynamo.planner.config.planner_config import ( # noqa: F401 (re-exported) +from dingo.planner.config.planner_config import ( # noqa: F401 (re-exported) PlannerConfig, PlannerPreDeploymentSweepMode, ) @@ -204,7 +204,7 @@ class FeaturesSpec(BaseModel): planner: Optional[PlannerConfig] = Field( default=None, - description="Planner contains the raw Planner configuration passed to the Planner service. Its schema is defined by dynamo.planner.config.planner_config.PlannerConfig. See https://docs.dynamo.nvidia.com/dynamo/components/planner/planner-guide#plannerconfig-reference. DGDR passes this object through without field-level validation; the Planner service validates it at startup. The presence of this field (non-null) enables the planner in the generated DGD.", + description="Planner contains the raw Planner configuration passed to the Planner service. Its schema is defined by dingo.planner.config.planner_config.PlannerConfig. See https://docs.dynamo.nvidia.com/dynamo/components/planner/planner-guide#plannerconfig-reference. DGDR passes this object through without field-level validation; the Planner service validates it at startup. The presence of this field (non-null) enables the planner in the generated DGD.", ) mocker: Optional[MockerSpec] = Field( default=None, diff --git a/components/src/dynamo/profiler/utils/dgdr_validate.py b/dingo/profiler/utils/dgdr_validate.py similarity index 95% rename from components/src/dynamo/profiler/utils/dgdr_validate.py rename to dingo/profiler/utils/dgdr_validate.py index b32353368344..4a90c8612911 100644 --- a/components/src/dynamo/profiler/utils/dgdr_validate.py +++ b/dingo/profiler/utils/dgdr_validate.py @@ -27,14 +27,14 @@ import logging -from dynamo.planner.config.planner_config import PlannerPreDeploymentSweepMode -from dynamo.profiler.utils.defaults import SearchStrategy -from dynamo.profiler.utils.dgdr_v1beta1_types import ( +from dingo.planner.config.planner_config import PlannerPreDeploymentSweepMode +from dingo.profiler.utils.defaults import SearchStrategy +from dingo.profiler.utils.dgdr_v1beta1_types import ( DynamoGraphDeploymentRequestSpec, SLASpec, WorkloadSpec, ) -from dynamo.profiler.utils.profile_common import is_planner_enabled +from dingo.profiler.utils.profile_common import is_planner_enabled logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/profiler/utils/model_info.py b/dingo/profiler/utils/model_info.py similarity index 100% rename from components/src/dynamo/profiler/utils/model_info.py rename to dingo/profiler/utils/model_info.py diff --git a/components/src/dynamo/profiler/utils/pareto.py b/dingo/profiler/utils/pareto.py similarity index 100% rename from components/src/dynamo/profiler/utils/pareto.py rename to dingo/profiler/utils/pareto.py diff --git a/components/src/dynamo/profiler/utils/plot.py b/dingo/profiler/utils/plot.py similarity index 99% rename from components/src/dynamo/profiler/utils/plot.py rename to dingo/profiler/utils/plot.py index 15ef25913c43..ca2734191440 100644 --- a/components/src/dynamo/profiler/utils/plot.py +++ b/dingo/profiler/utils/plot.py @@ -22,8 +22,8 @@ from matplotlib import cm from scipy.interpolate import griddata -from dynamo.profiler.utils.defaults import DEFAULT_GPU_COST_PER_HOUR -from dynamo.profiler.utils.pareto import compute_pareto +from dingo.profiler.utils.defaults import DEFAULT_GPU_COST_PER_HOUR +from dingo.profiler.utils.pareto import compute_pareto logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) diff --git a/components/src/dynamo/profiler/utils/profile_common.py b/dingo/profiler/utils/profile_common.py similarity index 98% rename from components/src/dynamo/profiler/utils/profile_common.py rename to dingo/profiler/utils/profile_common.py index ef5c96713ef7..fb871a5770c7 100644 --- a/components/src/dynamo/profiler/utils/profile_common.py +++ b/dingo/profiler/utils/profile_common.py @@ -22,11 +22,11 @@ import pandas as pd -from dynamo.planner.config.planner_config import PlannerPreDeploymentSweepMode -from dynamo.profiler.utils.config_modifiers.parallelization_mapping import ( +from dingo.planner.config.planner_config import PlannerPreDeploymentSweepMode +from dingo.profiler.utils.config_modifiers.parallelization_mapping import ( PickedParallelConfig, ) -from dynamo.profiler.utils.dgdr_v1beta1_types import ( +from dingo.profiler.utils.dgdr_v1beta1_types import ( DynamoGraphDeploymentRequestSpec, ProfilingPhase, ) diff --git a/components/src/dynamo/profiler/utils/profile_decode.py b/dingo/profiler/utils/profile_decode.py similarity index 95% rename from components/src/dynamo/profiler/utils/profile_decode.py rename to dingo/profiler/utils/profile_decode.py index 33c227bf7e1a..65637050dab7 100644 --- a/components/src/dynamo/profiler/utils/profile_decode.py +++ b/dingo/profiler/utils/profile_decode.py @@ -6,9 +6,9 @@ import numpy as np -from dynamo.profiler.utils.aiperf import get_decode_itl_and_thpt_per_gpu -from dynamo.profiler.utils.defaults import DECODE_MAX_CONCURRENCY -from dynamo.profiler.utils.plot import plot_decode_3d_surface +from dingo.profiler.utils.aiperf import get_decode_itl_and_thpt_per_gpu +from dingo.profiler.utils.defaults import DECODE_MAX_CONCURRENCY +from dingo.profiler.utils.plot import plot_decode_3d_surface logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) diff --git a/components/src/dynamo/profiler/utils/profile_prefill.py b/dingo/profiler/utils/profile_prefill.py similarity index 96% rename from components/src/dynamo/profiler/utils/profile_prefill.py rename to dingo/profiler/utils/profile_prefill.py index 2ef3581aad0f..b12df62f9502 100644 --- a/components/src/dynamo/profiler/utils/profile_prefill.py +++ b/dingo/profiler/utils/profile_prefill.py @@ -6,8 +6,8 @@ import numpy as np -from dynamo.profiler.utils.aiperf import get_prefill_ttft -from dynamo.profiler.utils.plot import plot_prefill_interpolation +from dingo.profiler.utils.aiperf import get_prefill_ttft +from dingo.profiler.utils.plot import plot_prefill_interpolation logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) diff --git a/components/src/dynamo/profiler/utils/profiler_status.py b/dingo/profiler/utils/profiler_status.py similarity index 96% rename from components/src/dynamo/profiler/utils/profiler_status.py rename to dingo/profiler/utils/profiler_status.py index a35b3dfb54f2..7b5ab32543cd 100755 --- a/components/src/dynamo/profiler/utils/profiler_status.py +++ b/dingo/profiler/utils/profiler_status.py @@ -16,7 +16,7 @@ import yaml -from dynamo.profiler.utils.dgdr_v1beta1_types import ProfilingPhase +from dingo.profiler.utils.dgdr_v1beta1_types import ProfilingPhase logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/profiler/utils/replay_optimize/__init__.py b/dingo/profiler/utils/replay_optimize/__init__.py similarity index 100% rename from components/src/dynamo/profiler/utils/replay_optimize/__init__.py rename to dingo/profiler/utils/replay_optimize/__init__.py diff --git a/components/src/dynamo/profiler/utils/replay_optimize/aic.py b/dingo/profiler/utils/replay_optimize/aic.py similarity index 100% rename from components/src/dynamo/profiler/utils/replay_optimize/aic.py rename to dingo/profiler/utils/replay_optimize/aic.py diff --git a/components/src/dynamo/profiler/utils/replay_optimize/bench.py b/dingo/profiler/utils/replay_optimize/bench.py similarity index 100% rename from components/src/dynamo/profiler/utils/replay_optimize/bench.py rename to dingo/profiler/utils/replay_optimize/bench.py diff --git a/components/src/dynamo/profiler/utils/replay_optimize/constants.py b/dingo/profiler/utils/replay_optimize/constants.py similarity index 100% rename from components/src/dynamo/profiler/utils/replay_optimize/constants.py rename to dingo/profiler/utils/replay_optimize/constants.py diff --git a/components/src/dynamo/profiler/utils/replay_optimize/engine_args.py b/dingo/profiler/utils/replay_optimize/engine_args.py similarity index 98% rename from components/src/dynamo/profiler/utils/replay_optimize/engine_args.py rename to dingo/profiler/utils/replay_optimize/engine_args.py index e2d89291ba00..22c6aa14c508 100644 --- a/components/src/dynamo/profiler/utils/replay_optimize/engine_args.py +++ b/dingo/profiler/utils/replay_optimize/engine_args.py @@ -8,7 +8,7 @@ from typing import Any, Literal from dynamo.llm import KvRouterConfig -from dynamo.mocker import MockEngineArgs +from dingo.mocker import MockEngineArgs from .constants import AIC_BACKEND_VERSIONS diff --git a/components/src/dynamo/profiler/utils/replay_optimize/evaluate.py b/dingo/profiler/utils/replay_optimize/evaluate.py similarity index 99% rename from components/src/dynamo/profiler/utils/replay_optimize/evaluate.py rename to dingo/profiler/utils/replay_optimize/evaluate.py index 72e9f793f2c2..6576490247e8 100644 --- a/components/src/dynamo/profiler/utils/replay_optimize/evaluate.py +++ b/dingo/profiler/utils/replay_optimize/evaluate.py @@ -17,8 +17,8 @@ from typing import Any from dynamo.llm import KvRouterConfig -from dynamo.mocker import MockEngineArgs -from dynamo.replay import run_synthetic_trace_replay, run_trace_replay +from dingo.mocker import MockEngineArgs +from dingo.replay import run_synthetic_trace_replay, run_trace_replay from .engine_args import ( _build_agg_candidate_engine_args, diff --git a/components/src/dynamo/profiler/utils/replay_optimize/example.py b/dingo/profiler/utils/replay_optimize/example.py similarity index 99% rename from components/src/dynamo/profiler/utils/replay_optimize/example.py rename to dingo/profiler/utils/replay_optimize/example.py index 5239fbb38513..2c8c2760b164 100644 --- a/components/src/dynamo/profiler/utils/replay_optimize/example.py +++ b/dingo/profiler/utils/replay_optimize/example.py @@ -6,7 +6,7 @@ import argparse from collections.abc import Sequence -from dynamo.profiler.utils.replay_optimize import ( +from dingo.profiler.utils.replay_optimize import ( EngineSpec, HardwareSpec, ReplayOptimizeSpec, diff --git a/components/src/dynamo/profiler/utils/replay_optimize/logging.py b/dingo/profiler/utils/replay_optimize/logging.py similarity index 100% rename from components/src/dynamo/profiler/utils/replay_optimize/logging.py rename to dingo/profiler/utils/replay_optimize/logging.py diff --git a/components/src/dynamo/profiler/utils/replay_optimize/models.py b/dingo/profiler/utils/replay_optimize/models.py similarity index 100% rename from components/src/dynamo/profiler/utils/replay_optimize/models.py rename to dingo/profiler/utils/replay_optimize/models.py diff --git a/components/src/dynamo/profiler/utils/replay_optimize/scoring.py b/dingo/profiler/utils/replay_optimize/scoring.py similarity index 100% rename from components/src/dynamo/profiler/utils/replay_optimize/scoring.py rename to dingo/profiler/utils/replay_optimize/scoring.py diff --git a/components/src/dynamo/profiler/utils/replay_optimize/search.py b/dingo/profiler/utils/replay_optimize/search.py similarity index 100% rename from components/src/dynamo/profiler/utils/replay_optimize/search.py rename to dingo/profiler/utils/replay_optimize/search.py diff --git a/components/src/dynamo/profiler/utils/replay_optimize/specs.py b/dingo/profiler/utils/replay_optimize/specs.py similarity index 99% rename from components/src/dynamo/profiler/utils/replay_optimize/specs.py rename to dingo/profiler/utils/replay_optimize/specs.py index 2cbb19a4a94d..4ea517c0a99a 100644 --- a/components/src/dynamo/profiler/utils/replay_optimize/specs.py +++ b/dingo/profiler/utils/replay_optimize/specs.py @@ -35,7 +35,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from dynamo.profiler.utils.dgdr_v1beta1_types import BackendType, GPUSKUType +from dingo.profiler.utils.dgdr_v1beta1_types import BackendType, GPUSKUType from .constants import ( AIC_BACKEND_VERSIONS, diff --git a/components/src/dynamo/profiler/utils/search_space_autogen.py b/dingo/profiler/utils/search_space_autogen.py similarity index 98% rename from components/src/dynamo/profiler/utils/search_space_autogen.py rename to dingo/profiler/utils/search_space_autogen.py index a5995f558929..d11e6e8a2111 100644 --- a/components/src/dynamo/profiler/utils/search_space_autogen.py +++ b/dingo/profiler/utils/search_space_autogen.py @@ -8,8 +8,8 @@ import yaml -from dynamo.profiler.utils.config_modifiers import CONFIG_MODIFIERS -from dynamo.profiler.utils.model_info import ModelInfo, get_model_info +from dingo.profiler.utils.config_modifiers import CONFIG_MODIFIERS +from dingo.profiler.utils.model_info import ModelInfo, get_model_info logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) diff --git a/components/src/dynamo/profiler/webui/data_template.json b/dingo/profiler/webui/data_template.json similarity index 100% rename from components/src/dynamo/profiler/webui/data_template.json rename to dingo/profiler/webui/data_template.json diff --git a/components/src/dynamo/profiler/webui/select_config.py b/dingo/profiler/webui/select_config.py similarity index 98% rename from components/src/dynamo/profiler/webui/select_config.py rename to dingo/profiler/webui/select_config.py index 135e830f26e1..7c94a60eb04b 100644 --- a/components/src/dynamo/profiler/webui/select_config.py +++ b/dingo/profiler/webui/select_config.py @@ -6,7 +6,7 @@ import queue from typing import Any -from dynamo.profiler.webui.utils import ( +from dingo.profiler.webui.utils import ( add_profiling_error, clear_profiling_errors, create_gradio_interface, diff --git a/components/src/dynamo/profiler/webui/utils.py b/dingo/profiler/webui/utils.py similarity index 99% rename from components/src/dynamo/profiler/webui/utils.py rename to dingo/profiler/webui/utils.py index b6d4fd5765c9..fdf2dc25fcdd 100644 --- a/components/src/dynamo/profiler/webui/utils.py +++ b/dingo/profiler/webui/utils.py @@ -21,12 +21,12 @@ load_profiling_javascript, ) -from dynamo.profiler.utils.dgd_generation import ( +from dingo.profiler.utils.dgd_generation import ( generate_decode_service_config_preview, generate_prefill_decode_services_config_preview, generate_prefill_service_config_preview, ) -from dynamo.profiler.utils.pareto import compute_pareto +from dingo.profiler.utils.pareto import compute_pareto logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/replay/__init__.py b/dingo/replay/__init__.py similarity index 73% rename from components/src/dynamo/replay/__init__.py rename to dingo/replay/__init__.py index 776f538950d9..40b557fbc115 100644 --- a/components/src/dynamo/replay/__init__.py +++ b/dingo/replay/__init__.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from dynamo.replay.api import run_synthetic_trace_replay, run_trace_replay +from dingo.replay.api import run_synthetic_trace_replay, run_trace_replay __all__ = ["run_synthetic_trace_replay", "run_trace_replay"] diff --git a/components/src/dynamo/replay/__main__.py b/dingo/replay/__main__.py similarity index 84% rename from components/src/dynamo/replay/__main__.py rename to dingo/replay/__main__.py index 13646779aa65..005913b8b6d0 100644 --- a/components/src/dynamo/replay/__main__.py +++ b/dingo/replay/__main__.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from dynamo.replay.main import main +from dingo.replay.main import main if __name__ == "__main__": raise SystemExit(main()) diff --git a/components/src/dynamo/replay/api.py b/dingo/replay/api.py similarity index 98% rename from components/src/dynamo/replay/api.py rename to dingo/replay/api.py index e37b89e36928..e37bada3a5b4 100644 --- a/components/src/dynamo/replay/api.py +++ b/dingo/replay/api.py @@ -74,7 +74,7 @@ def run_trace_replay( # Planner-in-the-loop: the Rust bridge owns the sim loop and calls back into # the Python planner adapter once per PlannerTick (main._run_planner_replay), # returning a ReplayPlannerReport (its .trace_report matches the static dict). - from dynamo.replay.main import _run_planner_replay + from dingo.replay.main import _run_planner_replay return _run_planner_replay( trace_file=trace_files[0], @@ -161,7 +161,7 @@ def run_synthetic_trace_replay( raise ValueError( "planner_config replay only supports replay_mode='offline'" ) - from dynamo.replay.main import SyntheticWorkload, _run_planner_replay + from dingo.replay.main import SyntheticWorkload, _run_planner_replay return _run_planner_replay( trace_file=None, diff --git a/components/src/dynamo/replay/main.py b/dingo/replay/main.py similarity index 98% rename from components/src/dynamo/replay/main.py rename to dingo/replay/main.py index fb7d2d883173..facb7937a498 100644 --- a/components/src/dynamo/replay/main.py +++ b/dingo/replay/main.py @@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Protocol, cast if TYPE_CHECKING: - from dynamo.planner.core.types import EngineCapabilities + from dingo.planner.core.types import EngineCapabilities from dynamo._internal.aic import ( DEFAULT_GPU_MEMORY_UTILIZATION, @@ -23,15 +23,15 @@ _normalize_aic_quant_mode, estimate_num_gpu_blocks, ) -from dynamo.common.forward_pass_metrics import ( +from dingo.common.forward_pass_metrics import ( ForwardPassMetrics, ScheduledRequestMetrics, ) from dynamo.llm import AicPerfConfig, KvRouterConfig -from dynamo.mocker import MockEngineArgs -from dynamo.mocker.utils.kv_cache import compute_kv_bytes_per_token -from dynamo.replay import run_synthetic_trace_replay, run_trace_replay -from dynamo.replay.reporting import format_report_table, write_report_json +from dingo.mocker import MockEngineArgs +from dingo.mocker.utils.kv_cache import compute_kv_bytes_per_token +from dingo.replay import run_synthetic_trace_replay, run_trace_replay +from dingo.replay.reporting import format_report_table, write_report_json class PlannerProfileDataResult(Protocol): @@ -75,7 +75,7 @@ def resolve_planner_profile_data( return SimpleNamespace(npz_path=planner_profile_data) try: - module = importlib.import_module("dynamo.mocker.args") + module = importlib.import_module("dingo.mocker.args") except ImportError: return SimpleNamespace( npz_path=None, @@ -288,7 +288,7 @@ def _load_aic_perf_config(args: argparse.Namespace): def _engine_caps(args: MockEngineArgs) -> EngineCapabilities: """Derive EngineCapabilities from MockEngineArgs.""" - from dynamo.planner.core.types import EngineCapabilities + from dingo.planner.core.types import EngineCapabilities max_kv_tokens = args.num_gpu_blocks * args.block_size return EngineCapabilities( @@ -424,10 +424,10 @@ def _run_planner_replay( # Fix the polynomial model to incorporate batch_size, or gate disagg # SLA mode on having a non-polynomial perf model. """ - from dynamo.mocker import PlannerReplayBridge - from dynamo.planner.config.planner_config import PlannerConfig - from dynamo.planner.core.types import WorkerCapabilities - from dynamo.planner.offline.replay_adapter import ReplayPlannerAdapter + from dingo.mocker import PlannerReplayBridge + from dingo.planner.config.planner_config import PlannerConfig + from dingo.planner.core.types import WorkerCapabilities + from dingo.planner.offline.replay_adapter import ReplayPlannerAdapter planner_config = PlannerConfig.from_config_arg(planner_config_arg) planner_config.advisory = True @@ -696,7 +696,7 @@ def _run_planner_replay( def main(argv: Sequence[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m dynamo.replay") + parser = argparse.ArgumentParser(prog="python -m dingo.replay") parser.add_argument("trace_files", nargs="*") parser.add_argument("--extra-engine-args") parser.add_argument("--prefill-engine-args") diff --git a/components/src/dynamo/replay/reporting.py b/dingo/replay/reporting.py similarity index 100% rename from components/src/dynamo/replay/reporting.py rename to dingo/replay/reporting.py diff --git a/dingo/router/CLAUDE.md b/dingo/router/CLAUDE.md index d318048e0208..daa9249830ec 100644 --- a/dingo/router/CLAUDE.md +++ b/dingo/router/CLAUDE.md @@ -46,7 +46,7 @@ logic but do not share the same serialization, RPC, or process boundaries. ``` Examples: `python -m dingo.router` and - `python -m dynamo.thunderagent_router`. These own a binding-level + `python -m dingo.thunderagent_router`. These own a binding-level `KvRouter` in another process. ### Guidance diff --git a/dingo/router/README.md b/dingo/router/README.md index 7ba20f035cfe..445b87c28c09 100644 --- a/dingo/router/README.md +++ b/dingo/router/README.md @@ -47,7 +47,7 @@ Clients call the `generate` endpoint to stream completions, call `best_worker_id > > Use this manual setup if you need explicit control over prefill routing configuration or want to manage prefill and decode routers separately. -For an integrated frontend disaggregated example, see [`examples/backends/vllm/launch/disagg_router.sh`](/examples/backends/vllm/launch/disagg_router.sh). For explicit multi-router composition, see the [Global Router README](/components/src/dynamo/global_router/README.md). +For an integrated frontend disaggregated example, see [`examples/backends/vllm/launch/disagg_router.sh`](/examples/backends/vllm/launch/disagg_router.sh). For explicit multi-router composition, see the [Global Router README](/dingo/global_router/README.md). ```bash # Start frontend router for decode workers diff --git a/dingo/router/args.py b/dingo/router/args.py index 83eb808b53d3..9d96634356ac 100644 --- a/dingo/router/args.py +++ b/dingo/router/args.py @@ -6,16 +6,16 @@ import argparse from typing import Optional -from dynamo.common.configuration.arg_group import ArgGroup -from dynamo.common.configuration.groups.aic_perf_args import ( +from dingo.common.configuration.arg_group import ArgGroup +from dingo.common.configuration.groups.aic_perf_args import ( AicPerfArgGroup, AicPerfConfigBase, ) -from dynamo.common.configuration.groups.kv_router_args import ( +from dingo.common.configuration.groups.kv_router_args import ( KvRouterArgGroup, KvRouterConfigBase, ) -from dynamo.common.configuration.utils import add_argument, add_negatable_bool_argument +from dingo.common.configuration.utils import add_argument, add_negatable_bool_argument from dynamo.llm import AicPerfConfig, KvRouterConfig diff --git a/dingo/sglang/__main__.py b/dingo/sglang/__main__.py index 19990ee47a36..e9a64da96957 100644 --- a/dingo/sglang/__main__.py +++ b/dingo/sglang/__main__.py @@ -7,7 +7,7 @@ os.environ["PYTHONHASHSEED"] = "0" if __name__ == "__main__": - from dynamo.common.snapshot.restore_context import maybe_run_restore_standby_mode + from dingo.common.snapshot.restore_context import maybe_run_restore_standby_mode # Check before importing dingo.sglang.main: restore standby mode must # capture env and hold without importing SGLang or constructing backend state. diff --git a/dingo/sglang/args.py b/dingo/sglang/args.py index fd458bc3f1b7..b819db588f98 100644 --- a/dingo/sglang/args.py +++ b/dingo/sglang/args.py @@ -17,16 +17,16 @@ from sglang.srt.server_args import ServerArgs from sglang.srt.server_args_config_parser import ConfigArgumentMerger -from dynamo.common.config_dump import register_encoder -from dynamo.common.configuration.groups import DynamoRuntimeConfig -from dynamo.common.configuration.groups.runtime_args import DynamoRuntimeArgGroup -from dynamo.common.constants import DisaggregationMode -from dynamo.common.model_fetch import fetch_model -from dynamo.common.snapshot.lifecycle import ( +from dingo.common.config_dump import register_encoder +from dingo.common.configuration.groups import DynamoRuntimeConfig +from dingo.common.configuration.groups.runtime_args import DynamoRuntimeArgGroup +from dingo.common.constants import DisaggregationMode +from dingo.common.model_fetch import fetch_model +from dingo.common.snapshot.lifecycle import ( configure_snapshot_capture_env, is_snapshot_enabled, ) -from dynamo.common.utils.runtime import parse_endpoint +from dingo.common.utils.runtime import parse_endpoint from dynamo.runtime.logging import configure_dynamo_logging from dingo.sglang._compat import enable_disjoint_streaming_output from dingo.sglang.backend_args import DynamoSGLangArgGroup, DynamoSGLangConfig diff --git a/dingo/sglang/backend_args.py b/dingo/sglang/backend_args.py index 4de9103dcb89..f89bd89176b6 100644 --- a/dingo/sglang/backend_args.py +++ b/dingo/sglang/backend_args.py @@ -10,13 +10,13 @@ import warnings from typing import Optional -from dynamo.common.configuration.arg_group import ArgGroup -from dynamo.common.configuration.config_base import ConfigBase -from dynamo.common.configuration.groups.frontend_decoding_args import ( +from dingo.common.configuration.arg_group import ArgGroup +from dingo.common.configuration.config_base import ConfigBase +from dingo.common.configuration.groups.frontend_decoding_args import ( add_frontend_decoding_arg, ) -from dynamo.common.configuration.utils import add_argument, add_negatable_bool_argument -from dynamo.common.constants import EmbeddingTransferMode +from dingo.common.configuration.utils import add_argument, add_negatable_bool_argument +from dingo.common.constants import EmbeddingTransferMode from . import __version__ diff --git a/dingo/sglang/init_diffusion.py b/dingo/sglang/init_diffusion.py index 028100b8cc61..18b3a4cc6ea8 100644 --- a/dingo/sglang/init_diffusion.py +++ b/dingo/sglang/init_diffusion.py @@ -8,8 +8,8 @@ import sglang as sgl -from dynamo.common.storage import get_fs -from dynamo.common.utils.endpoint_types import parse_endpoint_types +from dingo.common.storage import get_fs +from dingo.common.utils.endpoint_types import parse_endpoint_types from dynamo.llm import WorkerType from dynamo.runtime import DistributedRuntime from dingo.sglang.args import Config diff --git a/dingo/sglang/init_embedding.py b/dingo/sglang/init_embedding.py index 420ad59f56cf..bb58464675cd 100644 --- a/dingo/sglang/init_embedding.py +++ b/dingo/sglang/init_embedding.py @@ -7,7 +7,7 @@ import sglang as sgl -from dynamo.common.utils.prometheus import register_engine_metrics_callback +from dingo.common.utils.prometheus import register_engine_metrics_callback from dynamo.llm import ModelInput, ModelType, WorkerType from dynamo.runtime import DistributedRuntime from dingo.sglang.args import Config diff --git a/dingo/sglang/init_llm.py b/dingo/sglang/init_llm.py index b555925aa7fc..2b220a767964 100644 --- a/dingo/sglang/init_llm.py +++ b/dingo/sglang/init_llm.py @@ -10,8 +10,8 @@ import sglang as sgl from sglang.srt.observability.trace import set_global_trace_level -from dynamo.common.constants import DisaggregationMode -from dynamo.common.utils.endpoint_types import parse_endpoint_types +from dingo.common.constants import DisaggregationMode +from dingo.common.utils.endpoint_types import parse_endpoint_types from dynamo.llm import ModelInput, ModelType, WorkerType from dynamo.runtime import DistributedRuntime from dingo.sglang.args import Config diff --git a/dingo/sglang/init_multimodal.py b/dingo/sglang/init_multimodal.py index c2b28351c795..e04842cf333d 100644 --- a/dingo/sglang/init_multimodal.py +++ b/dingo/sglang/init_multimodal.py @@ -8,8 +8,8 @@ import sglang as sgl from dynamo import prometheus_names -from dynamo.common.constants import DisaggregationMode -from dynamo.common.utils.prometheus import register_embedding_cache_metrics +from dingo.common.constants import DisaggregationMode +from dingo.common.utils.prometheus import register_embedding_cache_metrics from dynamo.llm import ( ModelInput, ModelType, diff --git a/dingo/sglang/llm_engine.py b/dingo/sglang/llm_engine.py index da51a0f21863..dfb0dc534d49 100644 --- a/dingo/sglang/llm_engine.py +++ b/dingo/sglang/llm_engine.py @@ -33,10 +33,10 @@ from sglang.srt.utils.network import get_local_ip_auto, get_zmq_socket from dynamo._core import Context -from dynamo.common.backend import logprobs as _shared_logprobs -from dynamo.common.backend import telemetry -from dynamo.common.backend.dp_rank import forced_dp_rank, validate_global_dp_rank -from dynamo.common.backend.engine import ( +from dingo.common.backend import logprobs as _shared_logprobs +from dingo.common.backend import telemetry +from dingo.common.backend.dp_rank import forced_dp_rank, validate_global_dp_rank +from dingo.common.backend.engine import ( DYN_ENABLE_TEST_LOGITS_PROCESSOR, EngineConfig, GenerateChunk, @@ -48,16 +48,16 @@ logits_processors_for_request, resolve_test_logits_processor_spec, ) -from dynamo.common.backend.health_check import ( +from dingo.common.backend.health_check import ( bos_token_id_or, build_health_check_payload, is_probe, ) -from dynamo.common.backend.publisher import ComponentSnapshot, KvEventSource, ZmqSource -from dynamo.common.backend.worker import WorkerConfig -from dynamo.common.constants import DisaggregationMode -from dynamo.common.utils.input_params import InputParamManager -from dynamo.common.utils.structural_tag import serialize_structural_tag +from dingo.common.backend.publisher import ComponentSnapshot, KvEventSource, ZmqSource +from dingo.common.backend.worker import WorkerConfig +from dingo.common.constants import DisaggregationMode +from dingo.common.utils.input_params import InputParamManager +from dingo.common.utils.structural_tag import serialize_structural_tag from dynamo.llm import ModelInput from dingo.sglang._disagg import ( SGLANG_WORKER_GROUP_ID_KEY, @@ -914,7 +914,7 @@ async def register_prometheus(self, metrics: "EngineMetrics") -> None: if self.server_args.enable_metrics: from prometheus_client import CollectorRegistry, multiprocess - from dynamo.common.backend.metrics import register_engine_registry + from dingo.common.backend.metrics import register_engine_registry sgl_registry = CollectorRegistry() multiprocess.MultiProcessCollector(sgl_registry) diff --git a/dingo/sglang/logits_processing/adapter.py b/dingo/sglang/logits_processing/adapter.py index d9ba8af4b725..a6d62ce3f0ff 100644 --- a/dingo/sglang/logits_processing/adapter.py +++ b/dingo/sglang/logits_processing/adapter.py @@ -59,7 +59,7 @@ from sglang.srt.sampling.custom_logit_processor import CustomLogitProcessor -from dynamo.common.backend.engine import ( +from dingo.common.backend.engine import ( ForcedTokenSequenceSpec, LogitsProcessorEntry, deserialize_logits_processor_entries, diff --git a/dingo/sglang/main.py b/dingo/sglang/main.py index 28ca9b8dc79b..691b7e07c236 100644 --- a/dingo/sglang/main.py +++ b/dingo/sglang/main.py @@ -7,13 +7,13 @@ import uvloop -from dynamo.common.config_dump import dump_config -from dynamo.common.constants import DisaggregationMode -from dynamo.common.snapshot.restore_context import ( +from dingo.common.config_dump import dump_config +from dingo.common.constants import DisaggregationMode +from dingo.common.snapshot.restore_context import ( parse_snapshot_restore_runtime_config, refresh_snapshot_restore_config, ) -from dynamo.common.utils.runtime import create_runtime +from dingo.common.utils.runtime import create_runtime from dynamo.runtime.logging import configure_dynamo_logging from dingo.sglang.args import parse_args from dingo.sglang.init_diffusion import ( diff --git a/dingo/sglang/protocol.py b/dingo/sglang/protocol.py index 42e709151e01..df8831e2330f 100644 --- a/dingo/sglang/protocol.py +++ b/dingo/sglang/protocol.py @@ -6,8 +6,8 @@ from pydantic import BaseModel, ConfigDict, Field from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest -from dynamo.common.multimodal import TransferRequest -from dynamo.common.protocols.image_protocol import ImageNvExt +from dingo.common.multimodal import TransferRequest +from dingo.common.protocols.image_protocol import ImageNvExt TokenIdType = int diff --git a/dingo/sglang/publisher.py b/dingo/sglang/publisher.py index 80892d966157..12fd13fec61d 100644 --- a/dingo/sglang/publisher.py +++ b/dingo/sglang/publisher.py @@ -18,7 +18,7 @@ if TYPE_CHECKING: from prometheus_client import CollectorRegistry -from dynamo.common.utils.prometheus import ( +from dingo.common.utils.prometheus import ( LLMBackendMetrics, register_engine_metrics_callback, ) diff --git a/dingo/sglang/register.py b/dingo/sglang/register.py index 9d843f2776a7..b850780550d2 100644 --- a/dingo/sglang/register.py +++ b/dingo/sglang/register.py @@ -13,8 +13,8 @@ from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from dynamo._core import Endpoint -from dynamo.common.utils.output_modalities import get_output_modalities -from dynamo.common.utils.topology import apply_topology_config +from dingo.common.utils.output_modalities import get_output_modalities +from dingo.common.utils.topology import apply_topology_config from dynamo.llm import ( MediaDecoder, MediaFetcher, diff --git a/dingo/sglang/request_handlers/embedding/metrics.py b/dingo/sglang/request_handlers/embedding/metrics.py index 823beabdbb39..70d2b50e676c 100644 --- a/dingo/sglang/request_handlers/embedding/metrics.py +++ b/dingo/sglang/request_handlers/embedding/metrics.py @@ -11,7 +11,7 @@ Histograms are also bound to a caller-provided ``CollectorRegistry`` (NOT the default global ``REGISTRY``) because the Dynamo SGLang worker exposes metrics on ``/metrics`` only when the registry has been attached via -``dynamo.common.utils.prometheus.register_engine_metrics_callback``. The +``dingo.common.utils.prometheus.register_engine_metrics_callback``. The init function below should be called from ``init_embedding.py`` after engine setup, paired with that callback registration. """ @@ -51,7 +51,7 @@ def init_embedding_metrics(registry: "CollectorRegistry") -> None: the function without colliding with prior production registration. The caller is responsible for wiring ``registry`` to the worker's ``/metrics`` endpoint via - ``dynamo.common.utils.prometheus.register_engine_metrics_callback``. + ``dingo.common.utils.prometheus.register_engine_metrics_callback``. """ from prometheus_client import Histogram diff --git a/dingo/sglang/request_handlers/handler_base.py b/dingo/sglang/request_handlers/handler_base.py index e80d7fee018c..e95079b3c7b8 100644 --- a/dingo/sglang/request_handlers/handler_base.py +++ b/dingo/sglang/request_handlers/handler_base.py @@ -26,11 +26,11 @@ from sglang.srt.utils.network import NetworkAddress, get_local_ip_auto from dynamo._core import Context -from dynamo.common.constants import DisaggregationMode -from dynamo.common.lora.manager import get_lora_manager -from dynamo.common.utils.endpoint_types import parse_endpoint_types -from dynamo.common.utils.input_params import InputParamManager -from dynamo.common.utils.structural_tag import serialize_structural_tag +from dingo.common.constants import DisaggregationMode +from dingo.common.lora.manager import get_lora_manager +from dingo.common.utils.endpoint_types import parse_endpoint_types +from dingo.common.utils.input_params import InputParamManager +from dingo.common.utils.structural_tag import serialize_structural_tag from dynamo.llm import ( KvEventPublisher, ModelInput, diff --git a/dingo/sglang/request_handlers/image_diffusion/image_diffusion_handler.py b/dingo/sglang/request_handlers/image_diffusion/image_diffusion_handler.py index 86f36839d32b..746527a81583 100644 --- a/dingo/sglang/request_handlers/image_diffusion/image_diffusion_handler.py +++ b/dingo/sglang/request_handlers/image_diffusion/image_diffusion_handler.py @@ -14,8 +14,8 @@ from PIL import Image from dynamo._core import Context -from dynamo.common.protocols.image_protocol import ImageNvExt -from dynamo.common.storage import upload_to_fs +from dingo.common.protocols.image_protocol import ImageNvExt +from dingo.common.storage import upload_to_fs from dingo.sglang.args import Config from dingo.sglang.protocol import CreateImageRequest, ImageData, ImagesResponse from dingo.sglang.publisher import DynamoSglangPublisher diff --git a/dingo/sglang/request_handlers/llm/decode_handler.py b/dingo/sglang/request_handlers/llm/decode_handler.py index 5ef956e0ec18..82d7e16e8e92 100644 --- a/dingo/sglang/request_handlers/llm/decode_handler.py +++ b/dingo/sglang/request_handlers/llm/decode_handler.py @@ -10,11 +10,11 @@ from PIL.Image import Image as PILImage from dynamo._core import Context -from dynamo.common.backend import logprobs as _shared_logprobs -from dynamo.common.constants import DisaggregationMode -from dynamo.common.metadata_upload import MetadataUploader -from dynamo.common.multimodal.image_loader import ImageLoader -from dynamo.common.utils.engine_response import normalize_finish_reason +from dingo.common.backend import logprobs as _shared_logprobs +from dingo.common.constants import DisaggregationMode +from dingo.common.metadata_upload import MetadataUploader +from dingo.common.multimodal.image_loader import ImageLoader +from dingo.common.utils.engine_response import normalize_finish_reason from dingo.sglang._compat import filter_supported_async_generate_kwargs from dingo.sglang.args import Config from dingo.sglang.publisher import DynamoSglangPublisher diff --git a/dingo/sglang/request_handlers/multimodal/encode_worker_handler.py b/dingo/sglang/request_handlers/multimodal/encode_worker_handler.py index eb9de4c5c92b..fe284a6c413d 100644 --- a/dingo/sglang/request_handlers/multimodal/encode_worker_handler.py +++ b/dingo/sglang/request_handlers/multimodal/encode_worker_handler.py @@ -20,12 +20,12 @@ from transformers import AutoTokenizer from dynamo._core import Client, Context -from dynamo.common.memory.multimodal_embedding_cache_manager import ( +from dingo.common.memory.multimodal_embedding_cache_manager import ( CachedEmbedding, MultimodalEmbeddingCacheManager, ) -from dynamo.common.multimodal import EMBEDDING_SENDER_FACTORIES -from dynamo.common.utils import nvtx_utils as _nvtx +from dingo.common.multimodal import EMBEDDING_SENDER_FACTORIES +from dingo.common.utils import nvtx_utils as _nvtx from dynamo.llm import MultimodalEmbeddingCachePublisher from dingo.sglang.args import Config from dingo.sglang.protocol import ( diff --git a/dingo/sglang/request_handlers/multimodal/worker_handler.py b/dingo/sglang/request_handlers/multimodal/worker_handler.py index 7120166583da..744da46eb2e9 100644 --- a/dingo/sglang/request_handlers/multimodal/worker_handler.py +++ b/dingo/sglang/request_handlers/multimodal/worker_handler.py @@ -10,10 +10,10 @@ import torch from dynamo._core import Client, Context -from dynamo.common.constants import DisaggregationMode, EmbeddingTransferMode -from dynamo.common.multimodal import EMBEDDING_RECEIVER_FACTORIES, TransferRequest -from dynamo.common.utils import nvtx_utils as _nvtx -from dynamo.common.utils.engine_response import normalize_finish_reason +from dingo.common.constants import DisaggregationMode, EmbeddingTransferMode +from dingo.common.multimodal import EMBEDDING_RECEIVER_FACTORIES, TransferRequest +from dingo.common.utils import nvtx_utils as _nvtx +from dingo.common.utils.engine_response import normalize_finish_reason from dingo.sglang.args import Config from dingo.sglang.protocol import ( DisaggSglangMultimodalRequest, diff --git a/dingo/sglang/request_handlers/video_generation/video_generation_handler.py b/dingo/sglang/request_handlers/video_generation/video_generation_handler.py index 2bb990b34cfc..08a0f4eb8a6d 100644 --- a/dingo/sglang/request_handlers/video_generation/video_generation_handler.py +++ b/dingo/sglang/request_handlers/video_generation/video_generation_handler.py @@ -12,7 +12,7 @@ import torch from dynamo._core import Context -from dynamo.common.storage import upload_to_fs +from dingo.common.storage import upload_to_fs from dingo.sglang.args import Config from dingo.sglang.protocol import ( CreateVideoRequest, diff --git a/dingo/sglang/shutdown.py b/dingo/sglang/shutdown.py index cad58f276ef4..60a4cdf33059 100644 --- a/dingo/sglang/shutdown.py +++ b/dingo/sglang/shutdown.py @@ -9,7 +9,7 @@ from typing import Any, Awaitable, Callable, DefaultDict from dynamo._core import DistributedRuntime -from dynamo.common.utils.graceful_shutdown import graceful_shutdown_with_discovery +from dingo.common.utils.graceful_shutdown import graceful_shutdown_with_discovery SignalCallback = Callable[..., Any] diff --git a/dingo/sglang/snapshot.py b/dingo/sglang/snapshot.py index b280bd9eed4a..acfbde1df67a 100644 --- a/dingo/sglang/snapshot.py +++ b/dingo/sglang/snapshot.py @@ -11,7 +11,7 @@ import sglang as sgl -from dynamo.common.snapshot.lifecycle import ( +from dingo.common.snapshot.lifecycle import ( EngineSnapshotController, SnapshotConfig, configure_snapshot_capture_env, diff --git a/dingo/sglang/tests/test_fpm_contract.py b/dingo/sglang/tests/test_fpm_contract.py index 746a39010357..90db8d294802 100644 --- a/dingo/sglang/tests/test_fpm_contract.py +++ b/dingo/sglang/tests/test_fpm_contract.py @@ -33,8 +33,8 @@ def test_sglang_fpm_decodes_with_dynamo_schema(): ) from sglang.srt.observability.forward_pass_metrics import encode as sglang_encode - from dynamo.common.forward_pass_metrics import ForwardPassMetrics as DynamoFPM - from dynamo.common.forward_pass_metrics import decode as dynamo_decode + from dingo.common.forward_pass_metrics import ForwardPassMetrics as DynamoFPM + from dingo.common.forward_pass_metrics import decode as dynamo_decode sglang_fpm = SglangFPM( version=1, @@ -96,9 +96,9 @@ def test_sglang_fpm_field_order_matches_dynamo(): ScheduledRequestMetrics as SglangScheduled, ) - from dynamo.common.forward_pass_metrics import ForwardPassMetrics as DynamoFPM - from dynamo.common.forward_pass_metrics import QueuedRequestMetrics as DynamoQueued - from dynamo.common.forward_pass_metrics import ( + from dingo.common.forward_pass_metrics import ForwardPassMetrics as DynamoFPM + from dingo.common.forward_pass_metrics import QueuedRequestMetrics as DynamoQueued + from dingo.common.forward_pass_metrics import ( ScheduledRequestMetrics as DynamoScheduled, ) diff --git a/dingo/sglang/tests/test_sglang_decode_handler.py b/dingo/sglang/tests/test_sglang_decode_handler.py index 04bba76486f3..b4938abb8201 100644 --- a/dingo/sglang/tests/test_sglang_decode_handler.py +++ b/dingo/sglang/tests/test_sglang_decode_handler.py @@ -7,7 +7,7 @@ import pytest -from dynamo.common.metadata_upload import MetadataUploader +from dingo.common.metadata_upload import MetadataUploader from dingo.sglang.request_handlers.llm.decode_handler import ( DecodeWorkerHandler, _extract_sglang_stop_reason, diff --git a/dingo/sglang/tests/test_sglang_disagg.py b/dingo/sglang/tests/test_sglang_disagg.py index 8e87b11bf7c3..6a6d6aca63be 100644 --- a/dingo/sglang/tests/test_sglang_disagg.py +++ b/dingo/sglang/tests/test_sglang_disagg.py @@ -9,7 +9,7 @@ pytest.importorskip("sglang", reason="sglang not installed in this container") -from dynamo.common.constants import DisaggregationMode # noqa: E402 +from dingo.common.constants import DisaggregationMode # noqa: E402 from dingo.sglang.llm_engine import SglangLLMEngine # noqa: E402 pytestmark = [ diff --git a/dingo/sglang/tests/test_sglang_frontend_decoding.py b/dingo/sglang/tests/test_sglang_frontend_decoding.py index 289f0474e36d..ceb13e723924 100644 --- a/dingo/sglang/tests/test_sglang_frontend_decoding.py +++ b/dingo/sglang/tests/test_sglang_frontend_decoding.py @@ -9,7 +9,7 @@ import pytest from PIL import Image -from dynamo.common.constants import DisaggregationMode, EmbeddingTransferMode +from dingo.common.constants import DisaggregationMode, EmbeddingTransferMode from dingo.sglang.backend_args import DynamoSGLangConfig from dingo.sglang.request_handlers.llm.decode_handler import DecodeWorkerHandler diff --git a/dingo/sglang/tests/test_sglang_logits_runtime.py b/dingo/sglang/tests/test_sglang_logits_runtime.py index a3883b21d866..bda9c6380ec7 100644 --- a/dingo/sglang/tests/test_sglang_logits_runtime.py +++ b/dingo/sglang/tests/test_sglang_logits_runtime.py @@ -7,7 +7,7 @@ wires entries into `custom_params` / the `custom_logit_processor` kwarg. The shared spec-entry policy (gating, serialization) is tested in -`dynamo.common.backend.tests.test_engine`. These tests exercise the SGLang +`dingo.common.backend.tests.test_engine`. These tests exercise the SGLang realizer with CPU tensors, so they need `sglang` + `torch` importable but no GPU.""" @@ -22,7 +22,7 @@ torch = pytest.importorskip("torch") pytest.importorskip("sglang.srt.sampling.custom_logit_processor") -from dynamo.common.backend.engine import ( # noqa: E402 +from dingo.common.backend.engine import ( # noqa: E402 ForcedTokenSequenceSpec, serialize_logits_processor_entries, ) diff --git a/dingo/sglang/tests/test_sglang_multimodal_embedding_cache.py b/dingo/sglang/tests/test_sglang_multimodal_embedding_cache.py index 61161310467f..7093f9ec27d4 100644 --- a/dingo/sglang/tests/test_sglang_multimodal_embedding_cache.py +++ b/dingo/sglang/tests/test_sglang_multimodal_embedding_cache.py @@ -11,11 +11,11 @@ import pytest import torch -from dynamo.common.memory.multimodal_embedding_cache_manager import ( +from dingo.common.memory.multimodal_embedding_cache_manager import ( CachedEmbedding, MultimodalEmbeddingCacheManager, ) -from dynamo.common.multimodal import TransferRequest +from dingo.common.multimodal import TransferRequest from dingo.sglang.request_handlers.multimodal.encode_worker_handler import ( Modality, MultimodalEncodeWorkerHandler, diff --git a/dingo/sglang/tests/test_sglang_prometheus_utils.py b/dingo/sglang/tests/test_sglang_prometheus_utils.py index 0bbaa6d0586c..2276808eb7b2 100644 --- a/dingo/sglang/tests/test_sglang_prometheus_utils.py +++ b/dingo/sglang/tests/test_sglang_prometheus_utils.py @@ -7,7 +7,7 @@ import pytest -from dynamo.common.utils.prometheus import get_prometheus_expfmt +from dingo.common.utils.prometheus import get_prometheus_expfmt pytestmark = [ pytest.mark.unit, diff --git a/dingo/sglang/tests/test_sglang_trace_propagation.py b/dingo/sglang/tests/test_sglang_trace_propagation.py index 7d67439f53be..f1ca05e0a289 100644 --- a/dingo/sglang/tests/test_sglang_trace_propagation.py +++ b/dingo/sglang/tests/test_sglang_trace_propagation.py @@ -10,7 +10,7 @@ import pytest -from dynamo.common.constants import DisaggregationMode +from dingo.common.constants import DisaggregationMode # Guard the engine import directly: package metadata can be present without # submodules, and native-lib loads (e.g., libcuda.so.1) raise `ImportError` diff --git a/dingo/sglang/tests/test_sglang_unified_probe.py b/dingo/sglang/tests/test_sglang_unified_probe.py index 50f5ca71d674..758444cec425 100644 --- a/dingo/sglang/tests/test_sglang_unified_probe.py +++ b/dingo/sglang/tests/test_sglang_unified_probe.py @@ -35,7 +35,7 @@ def is_stopped(self) -> bool: def _build_prefill_engine(stream_factory): - from dynamo.common.constants import DisaggregationMode + from dingo.common.constants import DisaggregationMode from dingo.sglang.llm_engine import SglangLLMEngine server_args = SimpleNamespace(skip_tokenizer_init=True) diff --git a/dingo/sglang/tests/test_sglang_unit.py b/dingo/sglang/tests/test_sglang_unit.py index d61a0e495703..675061ac18de 100644 --- a/dingo/sglang/tests/test_sglang_unit.py +++ b/dingo/sglang/tests/test_sglang_unit.py @@ -13,7 +13,7 @@ from sglang.srt.disaggregation.utils import FAKE_BOOTSTRAP_HOST import dingo.sglang._compat as sglang_compat -from dynamo.common.constants import EmbeddingTransferMode +from dingo.common.constants import EmbeddingTransferMode from dingo.sglang._compat import ( ensure_sglang_top_level_exports, filter_supported_async_generate_kwargs, @@ -818,7 +818,7 @@ async def test_lora_registration_model_type_gate( """ from unittest.mock import AsyncMock, MagicMock - from dynamo.common.constants import DisaggregationMode + from dingo.common.constants import DisaggregationMode from dingo.sglang.request_handlers import handler_base from dingo.sglang.request_handlers.handler_base import LoraMixin diff --git a/dingo/sglang/unified_diffusion.py b/dingo/sglang/unified_diffusion.py index 066ac322fb5a..98e9c6f67b6c 100644 --- a/dingo/sglang/unified_diffusion.py +++ b/dingo/sglang/unified_diffusion.py @@ -18,10 +18,10 @@ from typing import Any, Optional from dynamo._core import Context -from dynamo.common.backend.engine import DiffusionEngine, EngineConfig -from dynamo.common.backend.worker import WorkerConfig -from dynamo.common.constants import DisaggregationMode as CommonDisaggregationMode -from dynamo.common.storage import get_fs +from dingo.common.backend.engine import DiffusionEngine, EngineConfig +from dingo.common.backend.worker import WorkerConfig +from dingo.common.constants import DisaggregationMode as CommonDisaggregationMode +from dingo.common.storage import get_fs from dynamo.llm import ModelInput from dingo.sglang.args import parse_args diff --git a/dingo/sglang/unified_diffusion_main.py b/dingo/sglang/unified_diffusion_main.py index 9d01e4285667..a48637462545 100644 --- a/dingo/sglang/unified_diffusion_main.py +++ b/dingo/sglang/unified_diffusion_main.py @@ -10,7 +10,7 @@ The token-pipeline / diffusion-LLM entry point is dingo.sglang.unified_main. """ -from dynamo.common.backend.run import run +from dingo.common.backend.run import run from dingo.sglang.unified_diffusion import SglangDiffusionEngine diff --git a/dingo/sglang/unified_main.py b/dingo/sglang/unified_main.py index a2f421dfdb5d..b2e12104038f 100644 --- a/dingo/sglang/unified_main.py +++ b/dingo/sglang/unified_main.py @@ -10,7 +10,7 @@ and feature gap details. """ -from dynamo.common.backend.run import run +from dingo.common.backend.run import run from dingo.sglang.llm_engine import SglangLLMEngine diff --git a/components/src/dynamo/thunderagent_router/README.md b/dingo/thunderagent_router/README.md similarity index 97% rename from components/src/dynamo/thunderagent_router/README.md rename to dingo/thunderagent_router/README.md index 397ecee31581..95582406d358 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/dingo/thunderagent_router/README.md @@ -1,4 +1,4 @@ -# `dynamo.thunderagent_router` (experimental) +# `dingo.thunderagent_router` (experimental) > **Experimental — not a released component.** Run it from a source checkout > (see [Install](#install)), not from a `pip install ai-dynamo`. The CLI @@ -30,7 +30,7 @@ cd dynamo uv pip install -e . ``` -`python -m dynamo.thunderagent_router` then resolves against that checkout. +`python -m dingo.thunderagent_router` then resolves against that checkout. ## Usage @@ -45,7 +45,7 @@ python -m dingo.vllm \ "enable_kv_cache_events":true}' # 2. Start the ThunderAgent router pointing at the worker endpoint -python -m dynamo.thunderagent_router \ +python -m dingo.thunderagent_router \ --endpoint dynamo.backend.generate \ --model-name \ --router-block-size 16 \ @@ -107,7 +107,7 @@ One script brings up both TP4 workers, the program-aware router, and the frontend on `:8100`: ```bash -bash components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh +bash dingo/thunderagent_router/run_minimax_8xh100.sh ``` First launch JIT-warms the FP8 kernels — wait for `curl localhost:8100/v1/models` diff --git a/components/src/dynamo/thunderagent_router/__init__.py b/dingo/thunderagent_router/__init__.py similarity index 76% rename from components/src/dynamo/thunderagent_router/__init__.py rename to dingo/thunderagent_router/__init__.py index cd0a5ed482bc..00cd522f0632 100644 --- a/components/src/dynamo/thunderagent_router/__init__.py +++ b/dingo/thunderagent_router/__init__.py @@ -3,6 +3,6 @@ """ThunderAgent program scheduler inside a Dynamo router service.""" -from dynamo.thunderagent_router.router import PauseDecision, ThunderAgentScheduler +from dingo.thunderagent_router.router import PauseDecision, ThunderAgentScheduler __all__ = ["PauseDecision", "ThunderAgentScheduler"] diff --git a/components/src/dynamo/thunderagent_router/__main__.py b/dingo/thunderagent_router/__main__.py similarity index 98% rename from components/src/dynamo/thunderagent_router/__main__.py rename to dingo/thunderagent_router/__main__.py index 77be33749770..fde0228c11ab 100644 --- a/components/src/dynamo/thunderagent_router/__main__.py +++ b/dingo/thunderagent_router/__main__.py @@ -4,7 +4,7 @@ """Standalone ThunderAgent router service. Usage: - python -m dynamo.thunderagent_router \\ + python -m dingo.thunderagent_router \\ --endpoint dingo.vllm.generate \\ --router-block-size 64 @@ -30,14 +30,14 @@ ) from dynamo.runtime import DistributedRuntime, dynamo_worker from dynamo.runtime.logging import configure_dynamo_logging -from dynamo.thunderagent_router.args import ( +from dingo.thunderagent_router.args import ( ThunderAgentRouterConfig, build_aic_perf_config, build_kv_router_config, parse_args, ) -from dynamo.thunderagent_router.capacity import WorkerCapacityProvider -from dynamo.thunderagent_router.router import ThunderAgentScheduler +from dingo.thunderagent_router.capacity import WorkerCapacityProvider +from dingo.thunderagent_router.router import ThunderAgentScheduler configure_dynamo_logging() logger = logging.getLogger(__name__) diff --git a/components/src/dynamo/thunderagent_router/args.py b/dingo/thunderagent_router/args.py similarity index 98% rename from components/src/dynamo/thunderagent_router/args.py rename to dingo/thunderagent_router/args.py index 95166b9b8187..2d314ee5daf7 100644 --- a/components/src/dynamo/thunderagent_router/args.py +++ b/dingo/thunderagent_router/args.py @@ -8,15 +8,15 @@ import argparse from typing import Optional -from dynamo.common.configuration.arg_group import ArgGroup -from dynamo.common.configuration.utils import add_argument +from dingo.common.configuration.arg_group import ArgGroup +from dingo.common.configuration.utils import add_argument from dingo.router.args import ( DynamoRouterArgGroup, DynamoRouterConfig, build_aic_perf_config, build_kv_router_config, ) -from dynamo.thunderagent_router.router import ThunderAgentConfig +from dingo.thunderagent_router.router import ThunderAgentConfig class ThunderAgentRouterConfig(DynamoRouterConfig): diff --git a/components/src/dynamo/thunderagent_router/capacity.py b/dingo/thunderagent_router/capacity.py similarity index 100% rename from components/src/dynamo/thunderagent_router/capacity.py rename to dingo/thunderagent_router/capacity.py diff --git a/components/src/dynamo/thunderagent_router/program_state.py b/dingo/thunderagent_router/program_state.py similarity index 100% rename from components/src/dynamo/thunderagent_router/program_state.py rename to dingo/thunderagent_router/program_state.py diff --git a/components/src/dynamo/thunderagent_router/router.py b/dingo/thunderagent_router/router.py similarity index 99% rename from components/src/dynamo/thunderagent_router/router.py rename to dingo/thunderagent_router/router.py index f87e395ab73c..509b8cf6cf41 100644 --- a/components/src/dynamo/thunderagent_router/router.py +++ b/dingo/thunderagent_router/router.py @@ -16,8 +16,8 @@ from dataclasses import dataclass from typing import Optional -from dynamo.thunderagent_router.capacity import WorkerCapacityProvider -from dynamo.thunderagent_router.program_state import ( +from dingo.thunderagent_router.capacity import WorkerCapacityProvider +from dingo.thunderagent_router.program_state import ( Program, ProgramLifecycle, ProgramStatus, diff --git a/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh b/dingo/thunderagent_router/run_minimax_8xh100.sh similarity index 98% rename from components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh rename to dingo/thunderagent_router/run_minimax_8xh100.sh index b45cb957d367..2369dce8657b 100755 --- a/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh +++ b/dingo/thunderagent_router/run_minimax_8xh100.sh @@ -32,7 +32,7 @@ DYN_SYSTEM_PORT=8082 CUDA_VISIBLE_DEVICES=4,5,6,7 python -m dingo.vllm \ # Program-aware router: registers the model handler and forwards the parser so # MiniMax's XML reaches the agent as OpenAI tool_calls. -python -m dynamo.thunderagent_router \ +python -m dingo.thunderagent_router \ --endpoint dingo.vllm.generate \ --model-name "$MODEL" \ --dyn-tool-call-parser minimax_m2 \ diff --git a/components/src/dynamo/thunderagent_router/tests/__init__.py b/dingo/thunderagent_router/tests/__init__.py similarity index 100% rename from components/src/dynamo/thunderagent_router/tests/__init__.py rename to dingo/thunderagent_router/tests/__init__.py diff --git a/components/src/dynamo/thunderagent_router/tests/test_capacity.py b/dingo/thunderagent_router/tests/test_capacity.py similarity index 97% rename from components/src/dynamo/thunderagent_router/tests/test_capacity.py rename to dingo/thunderagent_router/tests/test_capacity.py index 67f5a4364e0d..7ab52983f309 100644 --- a/components/src/dynamo/thunderagent_router/tests/test_capacity.py +++ b/dingo/thunderagent_router/tests/test_capacity.py @@ -10,7 +10,7 @@ import pytest -from dynamo.thunderagent_router.capacity import WorkerCapacityProvider +from dingo.thunderagent_router.capacity import WorkerCapacityProvider pytestmark = [pytest.mark.pre_merge, pytest.mark.unit, pytest.mark.gpu_0] diff --git a/components/src/dynamo/thunderagent_router/tests/test_program_state.py b/dingo/thunderagent_router/tests/test_program_state.py similarity index 96% rename from components/src/dynamo/thunderagent_router/tests/test_program_state.py rename to dingo/thunderagent_router/tests/test_program_state.py index a40657ab8d9a..2bafd909f7a8 100644 --- a/components/src/dynamo/thunderagent_router/tests/test_program_state.py +++ b/dingo/thunderagent_router/tests/test_program_state.py @@ -7,7 +7,7 @@ import pytest -from dynamo.thunderagent_router.program_state import ( +from dingo.thunderagent_router.program_state import ( ProgramLifecycle, ProgramStatus, ProgramTable, diff --git a/components/src/dynamo/thunderagent_router/tests/test_router.py b/dingo/thunderagent_router/tests/test_router.py similarity index 98% rename from components/src/dynamo/thunderagent_router/tests/test_router.py rename to dingo/thunderagent_router/tests/test_router.py index d551e9825491..43f90b6f3393 100644 --- a/components/src/dynamo/thunderagent_router/tests/test_router.py +++ b/dingo/thunderagent_router/tests/test_router.py @@ -12,8 +12,8 @@ import pytest -from dynamo.thunderagent_router.program_state import ProgramLifecycle, ProgramStatus -from dynamo.thunderagent_router.router import ThunderAgentConfig, ThunderAgentScheduler +from dingo.thunderagent_router.program_state import ProgramLifecycle, ProgramStatus +from dingo.thunderagent_router.router import ThunderAgentConfig, ThunderAgentScheduler pytestmark = [pytest.mark.pre_merge, pytest.mark.unit, pytest.mark.gpu_0] diff --git a/dingo/vllm/__main__.py b/dingo/vllm/__main__.py index 02ebe8765ed5..ba1a4ad64a53 100644 --- a/dingo/vllm/__main__.py +++ b/dingo/vllm/__main__.py @@ -7,7 +7,7 @@ os.environ["PYTHONHASHSEED"] = "0" if __name__ == "__main__": - from dynamo.common.snapshot.restore_context import maybe_run_restore_standby_mode + from dingo.common.snapshot.restore_context import maybe_run_restore_standby_mode # Check before importing dingo.vllm.main: restore standby mode must capture # env and hold without importing vLLM or constructing backend/runtime state. diff --git a/dingo/vllm/args.py b/dingo/vllm/args.py index 05850356e332..7c9ec183a23c 100644 --- a/dingo/vllm/args.py +++ b/dingo/vllm/args.py @@ -17,12 +17,12 @@ except ImportError: from vllm.utils.argparse_utils import FlexibleArgumentParser -from dynamo.common.config_dump import register_encoder -from dynamo.common.configuration.groups.runtime_args import ( +from dingo.common.config_dump import register_encoder +from dingo.common.configuration.groups.runtime_args import ( DynamoRuntimeArgGroup, DynamoRuntimeConfig, ) -from dynamo.common.utils.runtime import parse_endpoint +from dingo.common.utils.runtime import parse_endpoint from dingo.vllm.backend_args import DynamoVllmArgGroup, DynamoVllmConfig from dingo.vllm.constants import DisaggregationMode diff --git a/dingo/vllm/backend_args.py b/dingo/vllm/backend_args.py index bdc477e604bc..3719142d5018 100644 --- a/dingo/vllm/backend_args.py +++ b/dingo/vllm/backend_args.py @@ -9,12 +9,12 @@ import warnings from typing import Optional, Union -from dynamo.common.configuration.arg_group import ArgGroup -from dynamo.common.configuration.config_base import ConfigBase -from dynamo.common.configuration.groups.frontend_decoding_args import ( +from dingo.common.configuration.arg_group import ArgGroup +from dingo.common.configuration.config_base import ConfigBase +from dingo.common.configuration.groups.frontend_decoding_args import ( add_frontend_decoding_arg, ) -from dynamo.common.configuration.utils import add_argument, add_negatable_bool_argument +from dingo.common.configuration.utils import add_argument, add_negatable_bool_argument from . import __version__ from .constants import DisaggregationMode, EmbeddingTransferMode diff --git a/dingo/vllm/constants.py b/dingo/vllm/constants.py index da9b6833613e..dc24a36a4b38 100644 --- a/dingo/vllm/constants.py +++ b/dingo/vllm/constants.py @@ -3,10 +3,10 @@ """Constants for vLLM backend. -DisaggregationMode is defined in dynamo.common.constants and re-exported here +DisaggregationMode is defined in dingo.common.constants and re-exported here so that existing imports from dingo.vllm.constants continue to work. """ -from dynamo.common.constants import DisaggregationMode, EmbeddingTransferMode +from dingo.common.constants import DisaggregationMode, EmbeddingTransferMode __all__ = ["DisaggregationMode", "EmbeddingTransferMode"] diff --git a/dingo/vllm/engine_monitor.py b/dingo/vllm/engine_monitor.py index d8668bf989ee..0b2e7fff7af8 100644 --- a/dingo/vllm/engine_monitor.py +++ b/dingo/vllm/engine_monitor.py @@ -11,7 +11,7 @@ from vllm.v1.engine.async_llm import AsyncLLM from vllm.v1.engine.exceptions import EngineDeadError -from dynamo.common.engine_monitor import EngineHealthMonitorConfig +from dingo.common.engine_monitor import EngineHealthMonitorConfig from dynamo.runtime import DistributedRuntime from dynamo.runtime.logging import configure_dynamo_logging diff --git a/dingo/vllm/handlers.py b/dingo/vllm/handlers.py index 5c9b0095f4af..46ab13f6525d 100644 --- a/dingo/vllm/handlers.py +++ b/dingo/vllm/handlers.py @@ -45,38 +45,38 @@ from vllm.v1.engine.exceptions import EngineDeadError from dynamo._core import Context -from dynamo.common.backend import logprobs as _shared_logprobs -from dynamo.common.lora.manager import LoRAInfo, get_lora_manager -from dynamo.common.memory.multimodal_embedding_cache_manager import ( +from dingo.common.backend import logprobs as _shared_logprobs +from dingo.common.lora.manager import LoRAInfo, get_lora_manager +from dingo.common.memory.multimodal_embedding_cache_manager import ( MultimodalEmbeddingCacheManager, ) -from dynamo.common.multimodal.audio_loader import AudioLoader -from dynamo.common.multimodal.embedding_transfer import ( +from dingo.common.multimodal.audio_loader import AudioLoader +from dingo.common.multimodal.embedding_transfer import ( LocalEmbeddingReceiver, NixlReadEmbeddingReceiver, NixlWriteEmbeddingReceiver, ) -from dynamo.common.multimodal.image_loader import ImageLoader -from dynamo.common.multimodal.mm_kwargs_transfer import ( +from dingo.common.multimodal.image_loader import ImageLoader +from dingo.common.multimodal.mm_kwargs_transfer import ( MmKwargsNixlReceiver, MmKwargsReceiver, MmKwargsShmReceiver, MmKwargsShmTransferMetadata, MmKwargsTransferMetadata, ) -from dynamo.common.multimodal.video_loader import VideoLoader -from dynamo.common.rl import ( +from dingo.common.multimodal.video_loader import VideoLoader +from dingo.common.rl import ( RLAdminValidationError, RLRouteRegistry, env_bool, require_lora_load_request, require_lora_unload_request, ) -from dynamo.common.utils import nvtx_utils as _nvtx -from dynamo.common.utils.engine_response import normalize_finish_reason -from dynamo.common.utils.input_params import InputParamManager -from dynamo.common.utils.structural_tag import serialize_structural_tag -from dynamo.common.utils.time_section import time_and_log_code_section +from dingo.common.utils import nvtx_utils as _nvtx +from dingo.common.utils.engine_response import normalize_finish_reason +from dingo.common.utils.input_params import InputParamManager +from dingo.common.utils.structural_tag import serialize_structural_tag +from dingo.common.utils.time_section import time_and_log_code_section from dynamo.llm import ( KvEventPublisher, ModelInput, diff --git a/dingo/vllm/instrumented_scheduler.py b/dingo/vllm/instrumented_scheduler.py index 9283c3fe0a83..010bdea90335 100644 --- a/dingo/vllm/instrumented_scheduler.py +++ b/dingo/vllm/instrumented_scheduler.py @@ -100,7 +100,7 @@ from vllm.v1.core.sched.output import CachedRequestData, NewRequestData, SchedulerOutput from vllm.v1.request import Request, RequestStatus -from dynamo.common.forward_pass_metrics import ( +from dingo.common.forward_pass_metrics import ( ForwardPassMetrics, QueuedRequestMetrics, ScheduledRequestMetrics, diff --git a/dingo/vllm/llm_engine.py b/dingo/vllm/llm_engine.py index adfe843e7847..726f21271e83 100644 --- a/dingo/vllm/llm_engine.py +++ b/dingo/vllm/llm_engine.py @@ -26,11 +26,11 @@ from vllm.v1.metrics.stats import IterationStats, SchedulerStats from dynamo._core import Context -from dynamo.common.backend import logprobs as _shared_logprobs -from dynamo.common.backend import telemetry -from dynamo.common.backend.disagg import require_prefill_result -from dynamo.common.backend.dp_rank import forced_dp_rank, validate_global_dp_rank -from dynamo.common.backend.engine import ( +from dingo.common.backend import logprobs as _shared_logprobs +from dingo.common.backend import telemetry +from dingo.common.backend.disagg import require_prefill_result +from dingo.common.backend.dp_rank import forced_dp_rank, validate_global_dp_rank +from dingo.common.backend.engine import ( DYN_ENABLE_TEST_LOGITS_PROCESSOR, EngineConfig, GenerateChunk, @@ -42,18 +42,18 @@ logits_processors_for_request, resolve_test_logits_processor_spec, ) -from dynamo.common.backend.health_check import ( +from dingo.common.backend.health_check import ( bos_token_id_or, build_health_check_payload, ) -from dynamo.common.backend.metrics import ( +from dingo.common.backend.metrics import ( ensure_prometheus_multiproc_dir, register_global_registry, ) -from dynamo.common.backend.publisher import ComponentSnapshot, KvEventSource, ZmqSource -from dynamo.common.backend.worker import WorkerConfig -from dynamo.common.constants import DisaggregationMode -from dynamo.common.lora.manager import LoRAInfo, get_lora_manager +from dingo.common.backend.publisher import ComponentSnapshot, KvEventSource, ZmqSource +from dingo.common.backend.worker import WorkerConfig +from dingo.common.constants import DisaggregationMode +from dingo.common.lora.manager import LoRAInfo, get_lora_manager from dynamo.llm import ( ModelInput, ModelRuntimeConfig, diff --git a/dingo/vllm/logits_processing/adapter.py b/dingo/vllm/logits_processing/adapter.py index 2aebdebedb37..ef13be6ecd09 100644 --- a/dingo/vllm/logits_processing/adapter.py +++ b/dingo/vllm/logits_processing/adapter.py @@ -19,7 +19,7 @@ serialized entries onto a request's `SamplingParams`. The shared generation-stage gating and per-request freshness policy live in -``dynamo.common.backend.engine``; this module only translates the +``dingo.common.backend.engine``; this module only translates the backend-neutral entries into vLLM's mechanism. API assumptions (vLLM v1, ~0.11+; not introspectable in CI without a GPU): @@ -47,7 +47,7 @@ RequestLogitsProcessor, ) -from dynamo.common.backend.engine import ( +from dingo.common.backend.engine import ( ForcedTokenSequenceSpec, LogitsProcessorEntry, deserialize_logits_processor_entries, diff --git a/dingo/vllm/main.py b/dingo/vllm/main.py index 93b759a3c90c..f02064cda2eb 100644 --- a/dingo/vllm/main.py +++ b/dingo/vllm/main.py @@ -24,19 +24,19 @@ from vllm.v1.engine.async_llm import AsyncLLM from vllm.v1.metrics.prometheus import setup_multiprocess_prometheus -from dynamo.common.config_dump import dump_config -from dynamo.common.model_fetch import fetch_model -from dynamo.common.snapshot.restore_context import ( +from dingo.common.config_dump import dump_config +from dingo.common.model_fetch import fetch_model +from dingo.common.snapshot.restore_context import ( parse_snapshot_restore_runtime_config, refresh_snapshot_restore_config, ) -from dynamo.common.utils.graceful_shutdown import install_signal_handlers -from dynamo.common.utils.prometheus import ( +from dingo.common.utils.graceful_shutdown import install_signal_handlers +from dingo.common.utils.prometheus import ( LLMBackendMetrics, register_engine_metrics_callback, ) -from dynamo.common.utils.runtime import create_runtime -from dynamo.common.utils.topology import apply_topology_config +from dingo.common.utils.runtime import create_runtime +from dingo.common.utils.topology import apply_topology_config from dynamo.llm import ( KvEventPublisher, MediaDecoder, diff --git a/dingo/vllm/multimodal_handlers/encode_worker_handler.py b/dingo/vllm/multimodal_handlers/encode_worker_handler.py index 745b60d09c2e..5daecaa38d6e 100644 --- a/dingo/vllm/multimodal_handlers/encode_worker_handler.py +++ b/dingo/vllm/multimodal_handlers/encode_worker_handler.py @@ -13,14 +13,14 @@ from vllm.engine.arg_utils import AsyncEngineArgs import dynamo.nixl_connect as connect -from dynamo.common.multimodal import ( +from dingo.common.multimodal import ( LocalEmbeddingSender, NixlReadEmbeddingSender, NixlWriteEmbeddingSender, ) -from dynamo.common.multimodal.embedding_transfer import AbstractEmbeddingSender -from dynamo.common.utils import nvtx_utils as _nvtx -from dynamo.common.utils.time_section import time_and_log_code_section +from dingo.common.multimodal.embedding_transfer import AbstractEmbeddingSender +from dingo.common.utils import nvtx_utils as _nvtx +from dingo.common.utils.time_section import time_and_log_code_section from dynamo.runtime import DistributedRuntime from ..constants import EmbeddingTransferMode diff --git a/dingo/vllm/multimodal_utils/__init__.py b/dingo/vllm/multimodal_utils/__init__.py index dba9d0f6b717..1963d0324dac 100644 --- a/dingo/vllm/multimodal_utils/__init__.py +++ b/dingo/vllm/multimodal_utils/__init__.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from dynamo.common.multimodal.image_loader import ImageLoader +from dingo.common.multimodal.image_loader import ImageLoader from dingo.vllm.multimodal_utils.chat_message_utils import extract_user_text from dingo.vllm.multimodal_utils.embed_assembler import build_mixed_embeds from dingo.vllm.multimodal_utils.encode_utils import ( diff --git a/dingo/vllm/multimodal_utils/prefill_worker_utils.py b/dingo/vllm/multimodal_utils/prefill_worker_utils.py index 912008e044a5..b9209f502e73 100644 --- a/dingo/vllm/multimodal_utils/prefill_worker_utils.py +++ b/dingo/vllm/multimodal_utils/prefill_worker_utils.py @@ -9,15 +9,15 @@ import torch from vllm.sampling_params import SamplingParams as VllmSamplingParams -from dynamo.common.memory.multimodal_embedding_cache_manager import ( +from dingo.common.memory.multimodal_embedding_cache_manager import ( CachedEmbedding, MultimodalEmbeddingCacheManager, ) -from dynamo.common.multimodal.embedding_transfer import ( +from dingo.common.multimodal.embedding_transfer import ( AbstractEmbeddingReceiver, LocalEmbeddingReceiver, ) -from dynamo.common.utils.time_section import time_and_log_code_section +from dingo.common.utils.time_section import time_and_log_code_section from dynamo.runtime import Client from .encode_utils import get_embedding_hash diff --git a/dingo/vllm/multimodal_utils/protocol.py b/dingo/vllm/multimodal_utils/protocol.py index 2f1f9622fc13..c3d703ea29f4 100644 --- a/dingo/vllm/multimodal_utils/protocol.py +++ b/dingo/vllm/multimodal_utils/protocol.py @@ -36,7 +36,7 @@ from vllm.sampling_params import SamplingParams from vllm.v1.metrics.stats import RequestStateStats -from dynamo.common.multimodal.embedding_transfer import TransferRequest +from dingo.common.multimodal.embedding_transfer import TransferRequest class Request(BaseModel): diff --git a/dingo/vllm/omni/args.py b/dingo/vllm/omni/args.py index ab5ca480dd7a..4fb5be8c3913 100644 --- a/dingo/vllm/omni/args.py +++ b/dingo/vllm/omni/args.py @@ -18,12 +18,12 @@ except ImportError: from vllm.utils.argparse_utils import FlexibleArgumentParser -from dynamo.common.configuration.arg_group import ArgGroup -from dynamo.common.configuration.groups.runtime_args import ( +from dingo.common.configuration.arg_group import ArgGroup +from dingo.common.configuration.groups.runtime_args import ( DynamoRuntimeArgGroup, DynamoRuntimeConfig, ) -from dynamo.common.configuration.utils import add_argument, add_negatable_bool_argument +from dingo.common.configuration.utils import add_argument, add_negatable_bool_argument logger = logging.getLogger(__name__) diff --git a/dingo/vllm/omni/audio_handler.py b/dingo/vllm/omni/audio_handler.py index 3a2a15a92238..6a400aa85d7b 100644 --- a/dingo/vllm/omni/audio_handler.py +++ b/dingo/vllm/omni/audio_handler.py @@ -13,8 +13,8 @@ from vllm_omni.inputs.data import OmniTextPrompt -from dynamo.common.protocols.audio_protocol import NvCreateAudioSpeechRequest -from dynamo.common.utils.output_modalities import RequestType +from dingo.common.protocols.audio_protocol import NvCreateAudioSpeechRequest +from dingo.common.utils.output_modalities import RequestType logger = logging.getLogger(__name__) diff --git a/dingo/vllm/omni/base_handler.py b/dingo/vllm/omni/base_handler.py index 625423e884f5..e7a81337cd1b 100644 --- a/dingo/vllm/omni/base_handler.py +++ b/dingo/vllm/omni/base_handler.py @@ -18,8 +18,8 @@ DiffusionParallelConfig = None # type: ignore[assignment, misc] from dynamo._core import Context -from dynamo.common.protocols.audio_protocol import NvAudioSpeechResponse -from dynamo.common.utils.output_modalities import RequestType +from dingo.common.protocols.audio_protocol import NvAudioSpeechResponse +from dingo.common.utils.output_modalities import RequestType from dingo.vllm.handlers import BaseWorkerHandler, build_sampling_params logger = logging.getLogger(__name__) diff --git a/dingo/vllm/omni/main.py b/dingo/vllm/omni/main.py index ed61c619a8ab..7a0bf78d7d81 100644 --- a/dingo/vllm/omni/main.py +++ b/dingo/vllm/omni/main.py @@ -10,11 +10,11 @@ import uvloop from dynamo import prometheus_names -from dynamo.common.config_dump import dump_config -from dynamo.common.storage import get_fs -from dynamo.common.utils.graceful_shutdown import install_signal_handlers -from dynamo.common.utils.output_modalities import get_output_modalities -from dynamo.common.utils.runtime import create_runtime +from dingo.common.config_dump import dump_config +from dingo.common.storage import get_fs +from dingo.common.utils.graceful_shutdown import install_signal_handlers +from dingo.common.utils.output_modalities import get_output_modalities +from dingo.common.utils.runtime import create_runtime from dynamo.llm import ModelInput, ModelType, WorkerType, fetch_model, register_model from dynamo.runtime import DistributedRuntime from dynamo.runtime.logging import configure_dynamo_logging diff --git a/dingo/vllm/omni/omni_handler.py b/dingo/vllm/omni/omni_handler.py index 934bc66cd217..54a8d3425223 100644 --- a/dingo/vllm/omni/omni_handler.py +++ b/dingo/vllm/omni/omni_handler.py @@ -12,12 +12,12 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt from dynamo._core import Context -from dynamo.common.multimodal import ImageLoader -from dynamo.common.protocols.audio_protocol import NvCreateAudioSpeechRequest -from dynamo.common.protocols.image_protocol import ImageNvExt, NvCreateImageRequest -from dynamo.common.protocols.video_protocol import NvCreateVideoRequest, VideoNvExt -from dynamo.common.utils.output_modalities import RequestType, parse_request_type -from dynamo.common.utils.video_utils import compute_num_frames, parse_size +from dingo.common.multimodal import ImageLoader +from dingo.common.protocols.audio_protocol import NvCreateAudioSpeechRequest +from dingo.common.protocols.image_protocol import ImageNvExt, NvCreateImageRequest +from dingo.common.protocols.video_protocol import NvCreateVideoRequest, VideoNvExt +from dingo.common.utils.output_modalities import RequestType, parse_request_type +from dingo.common.utils.video_utils import compute_num_frames, parse_size from dynamo.llm.exceptions import EngineShutdown from dingo.vllm.omni.audio_handler import AudioGenerationHandler from dingo.vllm.omni.base_handler import BaseOmniHandler diff --git a/dingo/vllm/omni/output_formatter.py b/dingo/vllm/omni/output_formatter.py index 9816bd3f69a5..7c03159d7a78 100644 --- a/dingo/vllm/omni/output_formatter.py +++ b/dingo/vllm/omni/output_formatter.py @@ -22,13 +22,13 @@ import torch from diffusers.utils.export_utils import export_to_video -from dynamo.common.protocols.audio_protocol import AudioData, NvAudioSpeechResponse -from dynamo.common.protocols.image_protocol import ImageData, NvImagesResponse -from dynamo.common.protocols.video_protocol import NvVideosResponse, VideoData -from dynamo.common.storage import upload_to_fs -from dynamo.common.utils.engine_response import normalize_finish_reason -from dynamo.common.utils.output_modalities import RequestType -from dynamo.common.utils.video_utils import normalize_video_frames +from dingo.common.protocols.audio_protocol import AudioData, NvAudioSpeechResponse +from dingo.common.protocols.image_protocol import ImageData, NvImagesResponse +from dingo.common.protocols.video_protocol import NvVideosResponse, VideoData +from dingo.common.storage import upload_to_fs +from dingo.common.utils.engine_response import normalize_finish_reason +from dingo.common.utils.output_modalities import RequestType +from dingo.common.utils.video_utils import normalize_video_frames logger = logging.getLogger(__name__) diff --git a/dingo/vllm/omni/stage_router.py b/dingo/vllm/omni/stage_router.py index a42119fd228b..5f3c7daf5bae 100644 --- a/dingo/vllm/omni/stage_router.py +++ b/dingo/vllm/omni/stage_router.py @@ -11,8 +11,8 @@ from vllm_omni.entrypoints.utils import load_and_resolve_stage_configs from dynamo import prometheus_names -from dynamo.common.storage import get_fs -from dynamo.common.utils.output_modalities import ( +from dingo.common.storage import get_fs +from dingo.common.utils.output_modalities import ( RequestType, get_output_modalities, parse_request_type, diff --git a/dingo/vllm/omni/utils.py b/dingo/vllm/omni/utils.py index 742a68290adc..8cd42add7d54 100644 --- a/dingo/vllm/omni/utils.py +++ b/dingo/vllm/omni/utils.py @@ -11,8 +11,8 @@ from vllm_omni.entrypoints.stage_utils import shm_read_bytes from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt -from dynamo.common.utils.output_modalities import RequestType, parse_request_type -from dynamo.common.utils.video_utils import compute_num_frames, parse_size +from dingo.common.utils.output_modalities import RequestType, parse_request_type +from dingo.common.utils.video_utils import compute_num_frames, parse_size DEFAULT_IMAGE_SIZE = "1024x1024" DEFAULT_VIDEO_SIZE = "832x480" diff --git a/dingo/vllm/publisher.py b/dingo/vllm/publisher.py index ff1113b307d7..e25087596f96 100644 --- a/dingo/vllm/publisher.py +++ b/dingo/vllm/publisher.py @@ -10,7 +10,7 @@ from vllm.v1.metrics.loggers import StatLoggerBase from vllm.v1.metrics.stats import IterationStats, SchedulerStats -from dynamo.common.utils.prometheus import LLMBackendMetrics +from dingo.common.utils.prometheus import LLMBackendMetrics from dynamo.llm import WorkerMetricsPublisher from dynamo.runtime import Endpoint diff --git a/dingo/vllm/snapshot.py b/dingo/vllm/snapshot.py index 84bd2d6e9c1e..8aadc8f7654b 100644 --- a/dingo/vllm/snapshot.py +++ b/dingo/vllm/snapshot.py @@ -5,7 +5,7 @@ import logging from collections.abc import Callable -from dynamo.common.snapshot.lifecycle import ( +from dingo.common.snapshot.lifecycle import ( EngineSnapshotController, SnapshotConfig, configure_snapshot_capture_env, diff --git a/dingo/vllm/tests/multimodal_utils/test_vllm_embedding_loader.py b/dingo/vllm/tests/multimodal_utils/test_vllm_embedding_loader.py index 69a4c9e72f7f..5f0c4c9e2d9f 100644 --- a/dingo/vllm/tests/multimodal_utils/test_vllm_embedding_loader.py +++ b/dingo/vllm/tests/multimodal_utils/test_vllm_embedding_loader.py @@ -8,7 +8,7 @@ import pytest import torch -from dynamo.common.memory.multimodal_embedding_cache_manager import ( +from dingo.common.memory.multimodal_embedding_cache_manager import ( CachedEmbedding, MultimodalEmbeddingCacheManager, ) diff --git a/dingo/vllm/tests/omni/test_audio_handler.py b/dingo/vllm/tests/omni/test_audio_handler.py index 521618f8b9de..4b4b5d4f9add 100644 --- a/dingo/vllm/tests/omni/test_audio_handler.py +++ b/dingo/vllm/tests/omni/test_audio_handler.py @@ -8,8 +8,8 @@ import pytest try: - from dynamo.common.protocols.audio_protocol import NvCreateAudioSpeechRequest - from dynamo.common.utils.output_modalities import RequestType + from dingo.common.protocols.audio_protocol import NvCreateAudioSpeechRequest + from dingo.common.utils.output_modalities import RequestType from dingo.vllm.omni.audio_handler import AudioGenerationHandler except ImportError: pytest.skip("vLLM omni dependencies not available", allow_module_level=True) diff --git a/dingo/vllm/tests/omni/test_omni_handler.py b/dingo/vllm/tests/omni/test_omni_handler.py index e571eee2b36c..38a0ff2076c5 100644 --- a/dingo/vllm/tests/omni/test_omni_handler.py +++ b/dingo/vllm/tests/omni/test_omni_handler.py @@ -12,10 +12,10 @@ from vllm.sampling_params import SamplingParams from vllm_omni.inputs.data import OmniDiffusionSamplingParams - from dynamo.common.protocols.audio_protocol import NvCreateAudioSpeechRequest - from dynamo.common.protocols.image_protocol import NvCreateImageRequest - from dynamo.common.protocols.video_protocol import NvCreateVideoRequest, VideoNvExt - from dynamo.common.utils.output_modalities import RequestType + from dingo.common.protocols.audio_protocol import NvCreateAudioSpeechRequest + from dingo.common.protocols.image_protocol import NvCreateImageRequest + from dingo.common.protocols.video_protocol import NvCreateVideoRequest, VideoNvExt + from dingo.common.utils.output_modalities import RequestType from dingo.vllm.omni.audio_handler import AudioGenerationHandler from dingo.vllm.omni.omni_handler import EngineInputs, OmniHandler from dingo.vllm.omni.utils import build_original_prompt, parse_omni_request diff --git a/dingo/vllm/tests/omni/test_omni_stage_router.py b/dingo/vllm/tests/omni/test_omni_stage_router.py index c0278f32caa9..1fc2b5830a49 100644 --- a/dingo/vllm/tests/omni/test_omni_stage_router.py +++ b/dingo/vllm/tests/omni/test_omni_stage_router.py @@ -9,7 +9,7 @@ import pytest -from dynamo.common.utils.output_modalities import RequestType +from dingo.common.utils.output_modalities import RequestType try: from dingo.vllm.omni import stage_router diff --git a/dingo/vllm/tests/omni/test_output_formatter.py b/dingo/vllm/tests/omni/test_output_formatter.py index 41d608c21738..e5975e65ce74 100644 --- a/dingo/vllm/tests/omni/test_output_formatter.py +++ b/dingo/vllm/tests/omni/test_output_formatter.py @@ -153,7 +153,7 @@ async def test_multiple_images(self): class TestDiffusionFormatterImage: @pytest.mark.asyncio async def test_chat_completion_format(self): - from dynamo.common.utils.output_modalities import RequestType + from dingo.common.utils.output_modalities import RequestType f = _make_diffusion_formatter() img = MagicMock() @@ -166,7 +166,7 @@ async def test_chat_completion_format(self): @pytest.mark.asyncio async def test_image_generation_b64_format(self): - from dynamo.common.utils.output_modalities import RequestType + from dingo.common.utils.output_modalities import RequestType f = _make_diffusion_formatter() img = MagicMock() @@ -181,7 +181,7 @@ async def test_image_generation_b64_format(self): @pytest.mark.asyncio async def test_image_generation_default_format_returns_b64(self): - from dynamo.common.utils.output_modalities import RequestType + from dingo.common.utils.output_modalities import RequestType f = _make_diffusion_formatter() img = MagicMock() @@ -196,7 +196,7 @@ async def test_image_generation_default_format_returns_b64(self): @pytest.mark.asyncio async def test_empty_images_returns_error(self): - from dynamo.common.utils.output_modalities import RequestType + from dingo.common.utils.output_modalities import RequestType f = _make_diffusion_formatter() chunk = await f._encode_image( @@ -208,7 +208,7 @@ async def test_empty_images_returns_error(self): class TestDiffusionFormatterVideo: @pytest.mark.asyncio async def test_empty_frames_returns_none(self): - from dynamo.common.utils.output_modalities import RequestType + from dingo.common.utils.output_modalities import RequestType f = _make_diffusion_formatter() stage = MagicMock() @@ -362,7 +362,7 @@ class TestOutputFormatter: @pytest.mark.asyncio async def test_routes_text(self): - from dynamo.common.utils.output_modalities import RequestType + from dingo.common.utils.output_modalities import RequestType from dingo.vllm.omni.output_formatter import OutputFormatter f = OutputFormatter(model_name="test-model") @@ -376,7 +376,7 @@ async def test_routes_text(self): @pytest.mark.asyncio async def test_routes_image(self): - from dynamo.common.utils.output_modalities import RequestType + from dingo.common.utils.output_modalities import RequestType from dingo.vllm.omni.output_formatter import OutputFormatter f = OutputFormatter(model_name="test-model") @@ -394,7 +394,7 @@ async def test_routes_image(self): async def test_routes_audio(self): import numpy as np - from dynamo.common.utils.output_modalities import RequestType + from dingo.common.utils.output_modalities import RequestType from dingo.vllm.omni.output_formatter import OutputFormatter f = OutputFormatter(model_name="test-model") @@ -526,7 +526,7 @@ def _patches(self): @pytest.mark.asyncio async def test_video_url_response_format(self): - from dynamo.common.utils.output_modalities import RequestType + from dingo.common.utils.output_modalities import RequestType from dingo.vllm.omni.output_formatter import DiffusionFormatter f = DiffusionFormatter(model_name="test", media_fs=None, media_http_url=None) @@ -553,7 +553,7 @@ async def test_video_url_response_format(self): async def test_video_b64_response_format(self): import base64 - from dynamo.common.utils.output_modalities import RequestType + from dingo.common.utils.output_modalities import RequestType from dingo.vllm.omni.output_formatter import DiffusionFormatter f = DiffusionFormatter(model_name="test", media_fs=None, media_http_url=None) @@ -580,7 +580,7 @@ async def test_video_b64_response_format(self): @pytest.mark.asyncio async def test_video_default_response_format_is_url(self): """Omitting response_format defaults to url.""" - from dynamo.common.utils.output_modalities import RequestType + from dingo.common.utils.output_modalities import RequestType from dingo.vllm.omni.output_formatter import DiffusionFormatter f = DiffusionFormatter(model_name="test", media_fs=None, media_http_url=None) diff --git a/dingo/vllm/tests/test_vllm_delta_streaming.py b/dingo/vllm/tests/test_vllm_delta_streaming.py index c0c98c58f349..82f02fc51342 100644 --- a/dingo/vllm/tests/test_vllm_delta_streaming.py +++ b/dingo/vllm/tests/test_vllm_delta_streaming.py @@ -6,7 +6,7 @@ import pytest from vllm.sampling_params import RequestOutputKind, SamplingParams -from dynamo.common.constants import DisaggregationMode +from dingo.common.constants import DisaggregationMode from dingo.vllm.handlers import BaseWorkerHandler, build_sampling_params pytestmark = [ diff --git a/dingo/vllm/tests/test_vllm_engine.py b/dingo/vllm/tests/test_vllm_engine.py index 6bc6c47d4aac..9b8e52702b9d 100644 --- a/dingo/vllm/tests/test_vllm_engine.py +++ b/dingo/vllm/tests/test_vllm_engine.py @@ -210,7 +210,7 @@ async def _check_from_args_propagates_disaggregation_mode_to_worker_config( ``WorkerConfig`` the unified Worker sees, and onto the engine instance so ``generate()`` can branch on it. Without this hookup the prefill role would silently degrade to aggregated.""" - from dynamo.common.constants import DisaggregationMode + from dingo.common.constants import DisaggregationMode from dingo.vllm.llm_engine import VllmLLMEngine extra_args: list[str] = [] diff --git a/dingo/vllm/tests/test_vllm_engine_monitor_stats.py b/dingo/vllm/tests/test_vllm_engine_monitor_stats.py index dea09df2b273..eb2716899f21 100644 --- a/dingo/vllm/tests/test_vllm_engine_monitor_stats.py +++ b/dingo/vllm/tests/test_vllm_engine_monitor_stats.py @@ -8,7 +8,7 @@ import pytest -from dynamo.common.engine_monitor import EngineHealthMonitorConfig +from dingo.common.engine_monitor import EngineHealthMonitorConfig from dingo.vllm.engine_monitor import VllmEngineMonitor pytestmark = [ diff --git a/dingo/vllm/tests/test_vllm_logits_runtime.py b/dingo/vllm/tests/test_vllm_logits_runtime.py index 0be3698ad043..1a8bed2264a1 100644 --- a/dingo/vllm/tests/test_vllm_logits_runtime.py +++ b/dingo/vllm/tests/test_vllm_logits_runtime.py @@ -7,7 +7,7 @@ the startup registration helper. The shared spec-entry policy (generation-stage gating, serialization) is -tested in `dynamo.common.backend.tests.test_engine`. These tests exercise +tested in `dingo.common.backend.tests.test_engine`. These tests exercise the vLLM realizer in isolation with mocked `SamplingParams`, so they need `vllm` + `torch` importable but no GPU.""" @@ -24,7 +24,7 @@ torch = pytest.importorskip("torch") pytest.importorskip("vllm.v1.sample.logits_processor") -from dynamo.common.backend.engine import ( # noqa: E402 +from dingo.common.backend.engine import ( # noqa: E402 ForcedTokenSequenceSpec, serialize_logits_processor_entries, ) diff --git a/dingo/vllm/tests/test_vllm_lora.py b/dingo/vllm/tests/test_vllm_lora.py index 23b6fa33c96c..cb5e6d6e352e 100644 --- a/dingo/vllm/tests/test_vllm_lora.py +++ b/dingo/vllm/tests/test_vllm_lora.py @@ -18,8 +18,8 @@ pytest.importorskip("vllm.usage.usage_lib") pytest.importorskip("vllm.v1.engine.async_llm") -from dynamo.common.constants import DisaggregationMode # noqa: E402 -from dynamo.common.lora.manager import LoRAInfo # noqa: E402 +from dingo.common.constants import DisaggregationMode # noqa: E402 +from dingo.common.lora.manager import LoRAInfo # noqa: E402 from dynamo.llm import ModelType, WorkerType # noqa: E402 from dingo.vllm import llm_engine as llm_engine_mod # noqa: E402 from dingo.vllm.llm_engine import VllmLLMEngine # noqa: E402 diff --git a/dingo/vllm/tests/test_vllm_prometheus_utils.py b/dingo/vllm/tests/test_vllm_prometheus_utils.py index 1d3318b2d1a3..658198699589 100644 --- a/dingo/vllm/tests/test_vllm_prometheus_utils.py +++ b/dingo/vllm/tests/test_vllm_prometheus_utils.py @@ -7,7 +7,7 @@ import pytest -from dynamo.common.utils.prometheus import get_prometheus_expfmt +from dingo.common.utils.prometheus import get_prometheus_expfmt pytestmark = [ pytest.mark.unit, diff --git a/dingo/vllm/tests/test_vllm_trace_propagation.py b/dingo/vllm/tests/test_vllm_trace_propagation.py index 0e2e6cfe61d4..e4850ed50568 100644 --- a/dingo/vllm/tests/test_vllm_trace_propagation.py +++ b/dingo/vllm/tests/test_vllm_trace_propagation.py @@ -11,7 +11,7 @@ import pytest -from dynamo.common.constants import DisaggregationMode +from dingo.common.constants import DisaggregationMode # Guard the engine import directly: a partial vLLM install (e.g., metadata # present but `vllm.usage` missing) lets `find_spec("vllm")` pass while the diff --git a/dingo/vllm/tests/test_vllm_unit.py b/dingo/vllm/tests/test_vllm_unit.py index e1d271242f02..fdc7b82ddec9 100644 --- a/dingo/vllm/tests/test_vllm_unit.py +++ b/dingo/vllm/tests/test_vllm_unit.py @@ -332,7 +332,7 @@ def test_parse_args_does_not_track_logprobs_mode_presence(mock_vllm_cli): def test_unified_from_args_applies_rl_logprobs_default(monkeypatch): - from dynamo.common.constants import DisaggregationMode as CommonDisaggregationMode + from dingo.common.constants import DisaggregationMode as CommonDisaggregationMode from dingo.vllm import llm_engine config = SimpleNamespace( @@ -368,7 +368,7 @@ async def run_from_args(): def test_unified_generate_passes_enable_rl_to_sampling_params(monkeypatch): - from dynamo.common.constants import DisaggregationMode as CommonDisaggregationMode + from dingo.common.constants import DisaggregationMode as CommonDisaggregationMode from dingo.vllm import llm_engine captured: dict[str, bool] = {} @@ -412,7 +412,7 @@ async def run_generate(): @pytest.mark.asyncio async def test_unified_start_returns_normalized_served_model_name(monkeypatch): """Return the Dynamo-normalized served model name from EngineConfig.""" - from dynamo.common.constants import DisaggregationMode as CommonDisaggregationMode + from dingo.common.constants import DisaggregationMode as CommonDisaggregationMode from dingo.vllm import llm_engine served_model_name = "Qwen/Qwen3-0.6B" @@ -1007,7 +1007,7 @@ async def test_health_check_decode_opts_out_with_warning(): @pytest.mark.asyncio async def test_health_check_aggregated_returns_canary(): - from dynamo.common.backend.health_check import HEALTH_CHECK_KEY + from dingo.common.backend.health_check import HEALTH_CHECK_KEY from dingo.vllm.llm_engine import VllmLLMEngine engine = VllmLLMEngine( diff --git a/dingo/vllm/tests/test_vllm_worker_handler.py b/dingo/vllm/tests/test_vllm_worker_handler.py index d72c109f41bd..9e83eaf15777 100644 --- a/dingo/vllm/tests/test_vllm_worker_handler.py +++ b/dingo/vllm/tests/test_vllm_worker_handler.py @@ -19,7 +19,7 @@ import torch import dingo.vllm.handlers as mod -from dynamo.common.memory.multimodal_embedding_cache_manager import ( +from dingo.common.memory.multimodal_embedding_cache_manager import ( MultimodalEmbeddingCacheManager, ) from dingo.vllm.multimodal_utils.protocol import ( diff --git a/dingo/vllm/unified_main.py b/dingo/vllm/unified_main.py index 174ce5ee8f61..9b909e2c0ff8 100644 --- a/dingo/vllm/unified_main.py +++ b/dingo/vllm/unified_main.py @@ -10,7 +10,7 @@ and feature gap details. """ -from dynamo.common.backend.run import run +from dingo.common.backend.run import run from dingo.vllm.llm_engine import VllmLLMEngine diff --git a/dingo/vllm/unified_omni.py b/dingo/vllm/unified_omni.py index 320ecda68c05..ccbda3642586 100644 --- a/dingo/vllm/unified_omni.py +++ b/dingo/vllm/unified_omni.py @@ -25,10 +25,10 @@ from typing import Any, Optional from dynamo._core import Context -from dynamo.common.backend.engine import EngineConfig, RawEngine -from dynamo.common.backend.worker import WorkerConfig -from dynamo.common.constants import DisaggregationMode as CommonDisaggregationMode -from dynamo.common.storage import get_fs +from dingo.common.backend.engine import EngineConfig, RawEngine +from dingo.common.backend.worker import WorkerConfig +from dingo.common.constants import DisaggregationMode as CommonDisaggregationMode +from dingo.common.storage import get_fs from dynamo.llm import ModelInput from dingo.vllm.omni.args import parse_omni_args diff --git a/dingo/vllm/unified_omni_main.py b/dingo/vllm/unified_omni_main.py index f43b16ea9698..61fdfaec2462 100644 --- a/dingo/vllm/unified_omni_main.py +++ b/dingo/vllm/unified_omni_main.py @@ -8,7 +8,7 @@ --output-modalities {image,video,audio} [vLLM-Omni args...] """ -from dynamo.common.backend.run import run +from dingo.common.backend.run import run from dingo.vllm.unified_omni import VllmOmniEngine diff --git a/dingo/vllm/worker_factory.py b/dingo/vllm/worker_factory.py index df76fb0df9e8..bca198927438 100644 --- a/dingo/vllm/worker_factory.py +++ b/dingo/vllm/worker_factory.py @@ -16,9 +16,9 @@ from vllm.v1.engine.async_llm import AsyncLLM from dynamo import prometheus_names -from dynamo.common.rl import first_endpoint_response, register_rl_routes -from dynamo.common.utils.endpoint_types import parse_endpoint_types -from dynamo.common.utils.prometheus import ( +from dingo.common.rl import first_endpoint_response, register_rl_routes +from dingo.common.utils.endpoint_types import parse_endpoint_types +from dingo.common.utils.prometheus import ( LLMBackendMetrics, register_embedding_cache_metrics, ) diff --git a/docs/agents/agent-replay.md b/docs/agents/agent-replay.md index 37ff205e32d5..a46c7c53b538 100644 --- a/docs/agents/agent-replay.md +++ b/docs/agents/agent-replay.md @@ -139,7 +139,7 @@ Fixed-schedule behavior is required to preserve recorded arrival times, although Pass the captured shards directly to simulated workers: ```bash -python -m dynamo.replay /tmp/agent-run/request-trace.*.jsonl.gz \ +python -m dingo.replay /tmp/agent-run/request-trace.*.jsonl.gz \ --trace-format dynamo \ --replay-mode offline \ --router-mode kv_router \ diff --git a/docs/agents/thunderagent-router.md b/docs/agents/thunderagent-router.md index 80f2c4e9f72a..f70243945543 100644 --- a/docs/agents/thunderagent-router.md +++ b/docs/agents/thunderagent-router.md @@ -5,9 +5,9 @@ title: ThunderAgent Program Scheduler subtitle: Program-level scheduling with tool-boundary pause/resume on top of KV-aware routing --- -> **Experimental — not a released component.** Run it from a source checkout, not from a `pip install ai-dynamo`. The CLI flags, session headers, and lifecycle hooks are all unstable and will change. Build and launch specifics live next to the code in [`components/src/dynamo/thunderagent_router/README.md`](../../components/src/dynamo/thunderagent_router/README.md). +> **Experimental — not a released component.** Run it from a source checkout, not from a `pip install ai-dynamo`. The CLI flags, session headers, and lifecycle hooks are all unstable and will change. Build and launch specifics live next to the code in [`dingo/thunderagent_router/README.md`](https://github.com/ai-dynamo/dynamo/blob/main/dingo/thunderagent_router/README.md). -`dynamo.thunderagent_router` is a standalone Dynamo router that schedules at the granularity of an agent run — the whole `LLM turn → tool call → next turn` loop — instead of individual requests. It wraps Dynamo's native KV router and adds a program-level scheduler with tool-boundary pause/resume on top of KV-aware routing, porting the scheduler from the [ThunderAgent](https://arxiv.org/abs/2602.13692) paper (Kang et al., 2026). +`dingo.thunderagent_router` is a standalone Dynamo router that schedules at the granularity of an agent run — the whole `LLM turn → tool call → next turn` loop — instead of individual requests. It wraps Dynamo's native KV router and adds a program-level scheduler with tool-boundary pause/resume on top of KV-aware routing, porting the scheduler from the [ThunderAgent](https://arxiv.org/abs/2602.13692) paper (Kang et al., 2026). ## The Problem @@ -64,7 +64,7 @@ Pause/resume is driven by per-worker utilization — the program working set as > **Constraint:** `pause-target <= pause-threshold`. The service rejects configs that violate it (along with `0 <= resume-hysteresis <= pause-threshold` and `0 <= soft-demote-threshold <= pause-threshold`). -All `KvRouter` flags from `dingo.router` (`--router-temperature`, `--use-kv-events`, `--router-track-output-blocks`, …) are also accepted and forwarded. See the [folder README](../../components/src/dynamo/thunderagent_router/README.md) for the remaining service flags (`--endpoint`, `--model-name`, `--model-path`, tool-call and reasoning parsers). +All `KvRouter` flags from `dingo.router` (`--router-temperature`, `--use-kv-events`, `--router-track-output-blocks`, …) are also accepted and forwarded. See the [folder README](https://github.com/ai-dynamo/dynamo/blob/main/dingo/thunderagent_router/README.md) for the remaining service flags (`--endpoint`, `--model-name`, `--model-path`, tool-call and reasoning parsers). ## Architecture @@ -75,7 +75,7 @@ All `KvRouter` flags from `dingo.router` (`--router-temperature`, `--use-kv-even │ chat completions, with session headers ▼ ┌─────────────────────────────────────────────────────────────┐ -│ dynamo.thunderagent_router (this service) │ +│ dingo.thunderagent_router (this service) │ │ - ProgramTable: session_id → ProgramState │ │ - admission gate: before_request → was_paused? │ │ - scheduler loop (every scheduler_interval_seconds): │ @@ -122,13 +122,13 @@ Paused program (tokens=) Resumed program -> worker= (tokens=) ``` -Enable these by lowering the log level for `dynamo.thunderagent_router`. They give the exact program identities behind each INFO summary count. +Enable these by lowering the log level for `dingo.thunderagent_router`. They give the exact program identities behind each INFO summary count. For per-request tracing (token counts, cache hits, worker placement), the router also integrates with [Agent Tracing](agent-tracing.md#enable-output): set `DYN_REQUEST_TRACE=1` on the frontend to land a `request_end` record per LLM call. Harness tool-event spans are separate: they require `DYN_REQUEST_TRACE_TOOL_EVENTS_ZMQ_ENDPOINT` plus a configured publisher. ## Reproducing the MiniMax-M2 Results -The headline numbers (program-aware scheduling vs KV-routing-only on the same hardware, ~12-16% throughput improvement on SWE-bench-Lite with two TP4 MiniMax-M2 replicas on a single 8×H100 node) and the exact launch/repro commands live in the [folder README](../../components/src/dynamo/thunderagent_router/README.md). +The headline numbers (program-aware scheduling vs KV-routing-only on the same hardware, ~12-16% throughput improvement on SWE-bench-Lite with two TP4 MiniMax-M2 replicas on a single 8×H100 node) and the exact launch/repro commands live in the [folder README](https://github.com/ai-dynamo/dynamo/blob/main/dingo/thunderagent_router/README.md). ## References diff --git a/docs/api/nixl-connect/README.md b/docs/api/nixl-connect/README.md index 3f19caab7098..85ba5ff26c6c 100644 --- a/docs/api/nixl-connect/README.md +++ b/docs/api/nixl-connect/README.md @@ -140,11 +140,11 @@ flowchart LR #### Code Examples -See [NixlReadEmbeddingSender](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/common/multimodal/embedding_transfer.py), +See [NixlReadEmbeddingSender](https://github.com/ai-dynamo/dynamo/blob/main/dingo/common/multimodal/embedding_transfer.py), for how they coordinate directly with the Encode Worker by creating a [`ReadableOperation`](readable-operation.md), sending the operation's metadata via Dynamo's round-robin dispatcher, and awaiting the operation for completion before making use of the transferred data. -See [NixlReadEmbeddingReceiver](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/common/multimodal/embedding_transfer.py), +See [NixlReadEmbeddingReceiver](https://github.com/ai-dynamo/dynamo/blob/main/dingo/common/multimodal/embedding_transfer.py), for how the resulting embeddings are registered with the NIXL subsystem by creating a [`Descriptor`](descriptor.md), a [`ReadOperation`](read-operation.md) is created using the metadata provided by the requesting worker, and the worker awaits for the data transfer to complete for yielding a response. diff --git a/docs/backends/sglang/sglang-logits-processing.md b/docs/backends/sglang/sglang-logits-processing.md index d364db08a41a..0440a9c495aa 100644 --- a/docs/backends/sglang/sglang-logits-processing.md +++ b/docs/backends/sglang/sglang-logits-processing.md @@ -14,7 +14,7 @@ Logits processors let you modify the next-token logits at every decoding step (e ### How it works - **Interface**: Implement `dynamo.logits_processing.BaseLogitsProcessor`, which defines `__call__(input_ids, logits)` and modifies `logits` in-place. -- **Shared spec layer**: The engine declares a backend-neutral `LogitsProcessorSpec` (see `dynamo.common.backend.engine`). The shared `logits_processors_for_request` helper owns the generation-stage gating (activate only on `AGGREGATED` / `DECODE`) and the per-request freshness policy. +- **Shared spec layer**: The engine declares a backend-neutral `LogitsProcessorSpec` (see `dingo.common.backend.engine`). The shared `logits_processors_for_request` helper owns the generation-stage gating (activate only on `AGGREGATED` / `DECODE`) and the per-request freshness policy. - **SGLang mechanism**: SGLang gates the feature behind the server flag `--enable-custom-logit-processor`, takes a *serialized* processor class as a top-level `async_generate` argument (`custom_logit_processor`), and hands that processor a **batch** logits tensor plus a `custom_param_list` (one `sampling_params["custom_params"]` dict per request in the batch). Dynamo's adapter lives at `dingo.sglang.logits_processing.adapter`. ### Quick test: HelloWorld processor @@ -42,7 +42,7 @@ The quick test targets aggregated deployments. In disaggregated mode the prefill ### How the unified backend wires this up -The unified SGLang engine threads logits processors through the shared spec layer in `dynamo.common.backend.engine` and the per-backend realizer at `dingo.sglang.logits_processing.adapter`: +The unified SGLang engine threads logits processors through the shared spec layer in `dingo.common.backend.engine` and the per-backend realizer at `dingo.sglang.logits_processing.adapter`: - `from_args()` sets `server_args.enable_custom_logit_processor = True` and `server_args.skip_tokenizer_init = False` when the env hook is on **and** the worker is a generation role — after user overrides, so an explicit `skip_tokenizer_init=True` can't starve the hook. PREFILL keeps its configured flags. - `start()` resolves a `LogitsProcessorSpec` once via `resolve_test_logits_processor_spec`, tokenizing `"Hello world!"` into a `ForcedTokenSequenceSpec`. `None` when the env var is off or on a non-generation role. diff --git a/docs/backends/sglang/sglang-observability.md b/docs/backends/sglang/sglang-observability.md index 28d9cda447a9..1670c32cb2e7 100644 --- a/docs/backends/sglang/sglang-observability.md +++ b/docs/backends/sglang/sglang-observability.md @@ -224,7 +224,7 @@ Dynamo RPC (NATS transport) | v SGLang Handler (Python) - dynamo.common.utils.otel_tracing.build_trace_headers(context) + dingo.common.utils.otel_tracing.build_trace_headers(context) builds W3C traceparent: "00-{trace_id}-{span_id}-01" | v @@ -239,7 +239,7 @@ SGLang Engine (internal spans attached to same trace) ``` Key implementation files: -- `components/src/dynamo/common/utils/otel_tracing.py` - W3C `traceparent` header builder +- `dingo/common/utils/otel_tracing.py` - W3C `traceparent` header builder - `dingo/sglang/request_handlers/handler_base.py:71-84` - Extracts trace context from Dynamo `Context` object - `dingo/sglang/request_handlers/llm/decode_handler.py` - Passes `external_trace_header` and `rid=trace_id` to `engine.async_generate()` @@ -495,4 +495,4 @@ This is useful for automated benchmarking pipelines where you want to capture me - Dynamo runtime metrics (prefixed with `dynamo_*`) are available at the same `/metrics` endpoint alongside SGLang metrics - Implementation: `lib/runtime/src/metrics.rs` (Rust runtime metrics) - Metric names: `lib/runtime/src/metrics/prometheus_names.rs` (metric name constants) - - Integration code: `components/src/dynamo/common/utils/prometheus.py` - Prometheus utilities and callback registration + - Integration code: `dingo/common/utils/prometheus.py` - Prometheus utilities and callback registration diff --git a/docs/backends/vllm/vllm-logits-processing.md b/docs/backends/vllm/vllm-logits-processing.md index 183cb1004031..7b73a34c54e8 100644 --- a/docs/backends/vllm/vllm-logits-processing.md +++ b/docs/backends/vllm/vllm-logits-processing.md @@ -14,7 +14,7 @@ Logits processors let you modify the next-token logits at every decoding step (e ### How it works - **Interface**: Implement `dynamo.logits_processing.BaseLogitsProcessor`, which defines `__call__(input_ids, logits)` and modifies `logits` in-place. -- **Shared spec layer**: The engine declares a backend-neutral `LogitsProcessorSpec` (see `dynamo.common.backend.engine`). The shared `logits_processors_for_request` helper owns the generation-stage gating (activate only on `AGGREGATED` / `DECODE`) and the per-request freshness policy. +- **Shared spec layer**: The engine declares a backend-neutral `LogitsProcessorSpec` (see `dingo.common.backend.engine`). The shared `logits_processors_for_request` helper owns the generation-stage gating (activate only on `AGGREGATED` / `DECODE`) and the per-request freshness policy. - **vLLM mechanism**: Unlike TRT-LLM, vLLM cannot accept a live per-request callable. Its custom logits processors are *engine-loaded*: a class is registered at engine init and instantiated once, then called for every batch. Per-request activation rides on `SamplingParams.extra_args` (vLLM's `vllm_xargs`). Dynamo's adapter lives at `dingo.vllm.logits_processing.adapter`. ### Quick test: HelloWorld processor @@ -42,7 +42,7 @@ The quick test targets aggregated deployments. In disaggregated mode the prefill ### How the unified backend wires this up -The unified vLLM engine threads logits processors through the shared spec layer in `dynamo.common.backend.engine` and the per-backend realizer at `dingo.vllm.logits_processing.adapter`: +The unified vLLM engine threads logits processors through the shared spec layer in `dingo.common.backend.engine` and the per-backend realizer at `dingo.vllm.logits_processing.adapter`: - `start()` registers the engine-loaded adapter (`DynamoVllmLogitsProcessor`) onto `engine_args.logits_processors` **before** building the engine config — but only when the env hook is on and the worker is a generation role (`AGGREGATED` / `DECODE`). Production paths leave `logits_processors` untouched. After the engine (and tokenizer) is up, it resolves a `LogitsProcessorSpec` once via `resolve_test_logits_processor_spec`, tokenizing `"Hello world!"` into a `ForcedTokenSequenceSpec` with the token IDs already resolved. `None` when the env var is off or on a non-generation role. - `generate()` calls `logits_processors_for_request(spec, disaggregation_mode=...)` to get the per-request entry list (empty on PREFILL or when spec is `None`), then `activate_logits_processors(sampling_params, entries)` serializes the entries into `sampling_params.extra_args["dynamo_logits"]`. diff --git a/docs/backends/vllm/vllm-observability.md b/docs/backends/vllm/vllm-observability.md index 4370333ecc93..c4564c16f7c7 100644 --- a/docs/backends/vllm/vllm-observability.md +++ b/docs/backends/vllm/vllm-observability.md @@ -141,4 +141,4 @@ Troubleshooting LMCache-related metrics and logs is documented in: - Dynamo runtime metrics (prefixed with `dynamo_*`) are available at the same `/metrics` endpoint alongside vLLM metrics - Implementation: `lib/runtime/src/metrics.rs` (Rust runtime metrics) - Metric names: `lib/runtime/src/metrics/prometheus_names.rs` (metric name constants) - - Integration code: `components/src/dynamo/common/utils/prometheus.py` - Prometheus utilities and callback registration + - Integration code: `dingo/common/utils/prometheus.py` - Prometheus utilities and callback registration diff --git a/docs/benchmarks/benchmarking.md b/docs/benchmarks/benchmarking.md index aef63505601e..68136a4d4863 100644 --- a/docs/benchmarks/benchmarking.md +++ b/docs/benchmarks/benchmarking.md @@ -420,7 +420,7 @@ kubectl get endpoints -n $NAMESPACE ## Testing with DynoSim / Mocker -For development and testing purposes, Dynamo provides DynoSim and the [mocker backend](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/mocker) to simulate LLM inference without requiring actual GPU resources. This is useful for: +For development and testing purposes, Dynamo provides DynoSim and the [mocker backend](https://github.com/ai-dynamo/dynamo/blob/main/dingo/mocker) to simulate LLM inference without requiring actual GPU resources. This is useful for: - **Testing deployments** without expensive GPU infrastructure - **Developing and debugging** router, planner, or frontend logic diff --git a/docs/benchmarks/kv-router-ab-testing.md b/docs/benchmarks/kv-router-ab-testing.md index d58b4b5eb29d..1168c29163bf 100644 --- a/docs/benchmarks/kv-router-ab-testing.md +++ b/docs/benchmarks/kv-router-ab-testing.md @@ -389,7 +389,7 @@ For this A/B comparison, we use the [**Mooncake FAST'25 Toolagent Trace**](https These two requests share blocks 46–57 (12 blocks × 512 tokens = ~6,144 tokens of shared prefix) — a tool agent continuing the same session with accumulated context. Each hash ID represents a **512-token block**, and the hash includes both the current block and all preceding blocks, preserving the pattern of prefix reuse while protecting user privacy. The **KV Smart Router** routes requests with matching hash IDs to the same worker, maximizing cache hits. -If you reproduce this benchmark with `python -m dynamo.replay`, keep that dataset fact separate from +If you reproduce this benchmark with `python -m dingo.replay`, keep that dataset fact separate from the replay engine configuration: - use `--trace-block-size 512` for the Mooncake/toolagent trace itself diff --git a/docs/components/frontend/frontend-guide.md b/docs/components/frontend/frontend-guide.md index c83354a5e738..885e27058655 100644 --- a/docs/components/frontend/frontend-guide.md +++ b/docs/components/frontend/frontend-guide.md @@ -156,7 +156,6 @@ Backends auto-register with the frontend when they call `register_model()`. Supp - [vLLM Backend](../../backends/vllm/README.md) - [SGLang Backend](../../backends/sglang/README.md) -- [TensorRT-LLM Backend](../../backends/trtllm/README.md) ## See Also diff --git a/docs/components/planner/global-planner.md b/docs/components/planner/global-planner.md index 35dc508d12dd..6a963c52e022 100644 --- a/docs/components/planner/global-planner.md +++ b/docs/components/planner/global-planner.md @@ -21,7 +21,7 @@ Without `GlobalPlanner`, each DGD's local planner scales only its own deployment ## Terminology -- **Planner**: The `dynamo.planner` component that computes desired replica counts to maintain latency SLAs. See the [Planner overview](README.md). +- **Planner**: The `dingo.planner` component that computes desired replica counts to maintain latency SLAs. See the [Planner overview](README.md). - **Local Planner**: A pool-local instance of the Planner running inside a single DGD. - **Global Planner**: The centralized execution and policy layer that receives scale requests from local planners. - **Single-endpoint multi-pool deployment**: One model endpoint backed by multiple DGDs for the same model. This pattern uses both `GlobalRouter` and `GlobalPlanner`. @@ -227,7 +227,7 @@ GlobalPlanner: command: - python3 - -m - - dynamo.global_planner + - dingo.global_planner args: - --managed-namespaces - ${K8S_NAMESPACE}-gp-prefill-0 @@ -264,7 +264,7 @@ The planner inside each pool must be configured for `global-planner` mode so it "ttft": 2000, "prefill_engine_num_gpu": 2, "model_name": "${MODEL_NAME}", - "profile_results_dir": "/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D" + "profile_results_dir": "/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D" } ``` @@ -348,7 +348,7 @@ Clients can pass request targets through `extra_args`: } ``` -For more details, see [Global Router README](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/global_router/README.md). +For more details, see [Global Router README](https://github.com/ai-dynamo/dynamo/blob/main/dingo/global_router/README.md). ## Step 5: Deploy In Order @@ -422,6 +422,6 @@ This keeps profiling and pool selection simple while still giving you one public - [Planner Guide](planner-guide.md) — Planner configuration reference - [DGDR Examples](../../kubernetes/dgdr-examples.md) — DGDR examples for generating per-pool configs - [Profiler Guide](../profiler/profiler-guide.md) — Pre-deployment profiling workflow -- [Global Planner README](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/global_planner/README.md) — Centralized scale execution -- [Global Router README](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/global_router/README.md) — Cross-pool request routing +- [Global Planner README](https://github.com/ai-dynamo/dynamo/blob/main/dingo/global_planner/README.md) — Centralized scale execution +- [Global Router README](https://github.com/ai-dynamo/dynamo/blob/main/dingo/global_router/README.md) — Cross-pool request routing - [vLLM global planner example](https://github.com/ai-dynamo/dynamo/blob/main/examples/global_planner/global-planner-vllm-test.yaml) — End-to-end reference manifest diff --git a/docs/components/router/router-disaggregated-serving.md b/docs/components/router/router-disaggregated-serving.md index 44c57d0af541..cae3838d1c6a 100644 --- a/docs/components/router/router-disaggregated-serving.md +++ b/docs/components/router/router-disaggregated-serving.md @@ -64,7 +64,7 @@ await prefill_endpoint.serve_endpoint(prefill_handler.generate) ``` >[!Note] -> The automatic disaggregated routing setup described here is currently supported by the integrated `dingo.frontend` path. It is not provided as a single turnkey mode by the standalone Python router (`python -m dingo.router`). If you build this topology with standalone routers, you must launch and connect the prefill and decode routing stages yourself and handle request handoff, including the `disaggregated_params` returned by prefill. For an advanced reference, see the [Global Router](https://github.com/ai-dynamo/dynamo/tree/main/components/src/dynamo/global_router), which composes local prefill and decode router pools explicitly. +> The automatic disaggregated routing setup described here is currently supported by the integrated `dingo.frontend` path. It is not provided as a single turnkey mode by the standalone Python router (`python -m dingo.router`). If you build this topology with standalone routers, you must launch and connect the prefill and decode routing stages yourself and handle request handoff, including the `disaggregated_params` returned by prefill. For an advanced reference, see the [Global Router](https://github.com/ai-dynamo/dynamo/tree/main/dingo/global_router), which composes local prefill and decode router pools explicitly. ## Request Flow diff --git a/docs/components/router/router-examples.md b/docs/components/router/router-examples.md index 469e078467ad..c8076d478cf8 100644 --- a/docs/components/router/router-examples.md +++ b/docs/components/router/router-examples.md @@ -110,7 +110,6 @@ For basic Kubernetes deployment with the KV Router, see the [Kubernetes Deployme ### Complete K8s Examples -- [TRT-LLM aggregated router example](https://github.com/ai-dynamo/dynamo/blob/main/examples/backends/trtllm/deploy/agg_router.yaml) - [vLLM aggregated router example](https://github.com/ai-dynamo/dynamo/blob/main/examples/backends/vllm/deploy/agg_router.yaml) - [SGLang aggregated router example](https://github.com/ai-dynamo/dynamo/blob/main/examples/backends/sglang/deploy/agg_router.yaml) - [Kubernetes deployment guide](../../kubernetes/README.md) @@ -296,7 +295,7 @@ For full documentation on implementing KV event publishing for custom inference For deployments with multiple worker pools, the **Global Router** enables hierarchical routing by sitting between the frontend and local routers. It selects the appropriate pool for each request based on configurable policies, supporting disaggregated topologies where pools are tuned for different workload characteristics. -- **Component details**: [`components/src/dynamo/global_router/`](https://github.com/ai-dynamo/dynamo/tree/main/components/src/dynamo/global_router/) +- **Component details**: [`dingo/global_router/`](https://github.com/ai-dynamo/dynamo/tree/main/dingo/global_router/) - **Example**: [`examples/global_planner/`](https://github.com/ai-dynamo/dynamo/tree/main/examples/global_planner/) ## See Also diff --git a/docs/design-docs/planner-design.md b/docs/design-docs/planner-design.md index 7ae434b7fc49..ff2d5e0325eb 100644 --- a/docs/design-docs/planner-design.md +++ b/docs/design-docs/planner-design.md @@ -223,7 +223,7 @@ The interpolators use the profiling sweep granularity to determine precision. Fi ## Initialization -The `python -m dynamo.planner` entrypoint loads `PlannerConfig`, constructs +The `python -m dingo.planner` entrypoint loads `PlannerConfig`, constructs the mode-specific planner wrapper, and then initializes the selected connector. The runtime base validates worker topology, discovers worker capabilities, installs any available pre-deployment FPMs into the perf model, @@ -262,7 +262,7 @@ Per-engine FPM queue depths from `_collect_fpm()` are exported as labeled Promet ### Regression Models Three specialized regression models live under -`components/src/dynamo/planner/core/perf_model/`: +`dingo/planner/core/perf_model/`: - **PrefillRegressionModel**: 1D regression `sum_prefill_tokens -> wall_time`. Estimates TTFT by simulating chunked prefill scheduling (chunks of `max_num_batched_tokens`). - **DecodeRegressionModel**: 1D regression `sum_decode_kv_tokens -> wall_time`. Estimates ITL for total decode load (scheduled + queued + avg decode length). - **AggRegressionModel**: 2D regression `(sum_prefill_tokens, sum_decode_kv_tokens) -> wall_time`. Estimates both TTFT (simulated prefill with piggybacked decode) and ITL (decode with average piggybacked prefill). @@ -298,13 +298,13 @@ In aggregated mode (`--mode agg`), engines handle both prefill and decode via ch | File / package | Purpose | | --------------- | ------- | -| `components/src/dynamo/planner/core/base.py` | Runtime I/O loop: gathers observations and applies scaling effects. | -| `components/src/dynamo/planner/core/state_machine.py` | Shared builtin scaling state used by local planner plugins. | -| `components/src/dynamo/planner/core/load_scaling.py` | FPM-driven load scaling algorithm. | -| `components/src/dynamo/planner/core/throughput_scaling.py` | Prediction-driven throughput scaling algorithm. | -| `components/src/dynamo/planner/plugins/builtins/` | Builtin plugins that expose the local planner algorithms to the pipeline. | -| `components/src/dynamo/planner/plugins/orchestrator/` | PREDICT -> PROPOSE -> RECONCILE -> CONSTRAIN pipeline driver and engine adapter. | -| `components/src/dynamo/planner/plugins/proto/v1/` | Public gRPC/proto plugin API. | -| `components/src/dynamo/planner/monitoring/` | Prometheus, diagnostics reports, live dashboard, and worker metadata. | -| `components/src/dynamo/planner/connectors/` | K8s, virtual, global-planner, and remote connector implementations. | -| `components/src/dynamo/planner/config/` | PlannerConfig schema, defaults, backend component names, and profiling bootstrap specs. | +| `dingo/planner/core/base.py` | Runtime I/O loop: gathers observations and applies scaling effects. | +| `dingo/planner/core/state_machine.py` | Shared builtin scaling state used by local planner plugins. | +| `dingo/planner/core/load_scaling.py` | FPM-driven load scaling algorithm. | +| `dingo/planner/core/throughput_scaling.py` | Prediction-driven throughput scaling algorithm. | +| `dingo/planner/plugins/builtins/` | Builtin plugins that expose the local planner algorithms to the pipeline. | +| `dingo/planner/plugins/orchestrator/` | PREDICT -> PROPOSE -> RECONCILE -> CONSTRAIN pipeline driver and engine adapter. | +| `dingo/planner/plugins/proto/v1/` | Public gRPC/proto plugin API. | +| `dingo/planner/monitoring/` | Prometheus, diagnostics reports, live dashboard, and worker metadata. | +| `dingo/planner/connectors/` | K8s, virtual, global-planner, and remote connector implementations. | +| `dingo/planner/config/` | PlannerConfig schema, defaults, backend component names, and profiling bootstrap specs. | diff --git a/docs/design-docs/planner-design.zh-CN.md b/docs/design-docs/planner-design.zh-CN.md index 3a6f552da302..4d5338a46c0c 100644 --- a/docs/design-docs/planner-design.zh-CN.md +++ b/docs/design-docs/planner-design.zh-CN.md @@ -158,7 +158,7 @@ Interpolators 使用 profiling sweep granularity 来决定精度。granularity ## 初始化 -`python -m dynamo.planner` entrypoint 加载 `PlannerConfig`，构造 mode-specific planner wrapper，然后初始化所选 connector。运行时 base 会验证 worker topology、发现 worker capabilities、把可用的 pre-deployment FPM 安装到 perf model、启动 builtin 和已配置 plugins，并进入 tick loop。 +`python -m dingo.planner` entrypoint 加载 `PlannerConfig`，构造 mode-specific planner wrapper，然后初始化所选 connector。运行时 base 会验证 worker topology、发现 worker capabilities、把可用的 pre-deployment FPM 安装到 perf model、启动 builtin 和已配置 plugins，并进入 tick loop。 ## Performance Considerations @@ -193,7 +193,7 @@ Interpolators 使用 profiling sweep granularity 来决定精度。granularity ### Regression Models -三个专用 regression models 位于 `components/src/dynamo/planner/core/perf_model/`： +三个专用 regression models 位于 `dingo/planner/core/perf_model/`： - **PrefillRegressionModel**：1D regression `sum_prefill_tokens -> wall_time`。通过模拟 chunked prefill scheduling（chunk 大小为 `max_num_batched_tokens`）估算 TTFT。 - **DecodeRegressionModel**：1D regression `sum_decode_kv_tokens -> wall_time`。估算 total decode load（scheduled + queued + avg decode length）的 ITL。 @@ -229,13 +229,13 @@ Interpolators 使用 profiling sweep granularity 来决定精度。granularity | File / package | Purpose | | --------------- | ------- | -| `components/src/dynamo/planner/core/base.py` | Runtime I/O loop：收集 observations 并应用 scaling effects。 | -| `components/src/dynamo/planner/core/state_machine.py` | 本地 planner plugins 使用的 shared builtin scaling state。 | -| `components/src/dynamo/planner/core/load_scaling.py` | FPM-driven load scaling algorithm。 | -| `components/src/dynamo/planner/core/throughput_scaling.py` | Prediction-driven throughput scaling algorithm。 | -| `components/src/dynamo/planner/plugins/builtins/` | 把 local planner algorithms 暴露给 pipeline 的 builtin plugins。 | -| `components/src/dynamo/planner/plugins/orchestrator/` | PREDICT -> PROPOSE -> RECONCILE -> CONSTRAIN pipeline driver 和 engine adapter。 | -| `components/src/dynamo/planner/plugins/proto/v1/` | Public gRPC/proto plugin API。 | -| `components/src/dynamo/planner/monitoring/` | Prometheus、diagnostics reports、live dashboard 和 worker metadata。 | -| `components/src/dynamo/planner/connectors/` | K8s、virtual、global-planner 和 remote connector implementations。 | -| `components/src/dynamo/planner/config/` | PlannerConfig schema、defaults、backend component names 和 profiling bootstrap specs。 | +| `dingo/planner/core/base.py` | Runtime I/O loop：收集 observations 并应用 scaling effects。 | +| `dingo/planner/core/state_machine.py` | 本地 planner plugins 使用的 shared builtin scaling state。 | +| `dingo/planner/core/load_scaling.py` | FPM-driven load scaling algorithm。 | +| `dingo/planner/core/throughput_scaling.py` | Prediction-driven throughput scaling algorithm。 | +| `dingo/planner/plugins/builtins/` | 把 local planner algorithms 暴露给 pipeline 的 builtin plugins。 | +| `dingo/planner/plugins/orchestrator/` | PREDICT -> PROPOSE -> RECONCILE -> CONSTRAIN pipeline driver 和 engine adapter。 | +| `dingo/planner/plugins/proto/v1/` | Public gRPC/proto plugin API。 | +| `dingo/planner/monitoring/` | Prometheus、diagnostics reports、live dashboard 和 worker metadata。 | +| `dingo/planner/connectors/` | K8s、virtual、global-planner 和 remote connector implementations。 | +| `dingo/planner/config/` | PlannerConfig schema、defaults、backend component names 和 profiling bootstrap specs。 | diff --git a/docs/development/unified-backends.md b/docs/development/unified-backends.md index e734a9e1b57f..11d91e8f0e68 100644 --- a/docs/development/unified-backends.md +++ b/docs/development/unified-backends.md @@ -57,7 +57,7 @@ custom backend in a `DynamoGraphDeployment` and follow the ## Python Implementation > **New — Dynamo's unified backend.** This guide covers the new > **unified backend** infrastructure in -> [`dynamo.common.backend`](https://github.com/ai-dynamo/dynamo/tree/main/components/src/dynamo/common/backend): +> [`dingo.common.backend`](https://github.com/ai-dynamo/dynamo/tree/main/dingo/common/backend): > a shared `LLMEngine` ABC that vLLM, SGLang, TRT-LLM, and a sample > engine already implement, and that any custom Python engine can plug > into the same way. For the Rust version of the same contract, use the @@ -74,18 +74,18 @@ custom backend in a `DynamoGraphDeployment` and follow the This guide walks through building a Python backend for an inference engine that plugs into Dynamo's distributed runtime via -`dynamo.common.backend`. A "unified backend" is a Python entry point +`dingo.common.backend`. A "unified backend" is a Python entry point that implements the shared `LLMEngine` ABC and lets the framework own runtime lifecycle (signal handling, model registration, graceful shutdown, cancellation monitoring) — your code just owns inference. Your backend lives in its own package and **does not need to be part of the dynamo repository**. It depends on `ai-dynamo` from PyPI (or -the git source) and imports `dynamo.common.backend`. The steps below +the git source) and imports `dingo.common.backend`. The steps below assume you're starting a fresh package in your own repo. The reference example is the **sample engine** at -[`sample_engine.py`](../../components/src/dynamo/common/backend/sample_engine.py) +[`sample_engine.py`](../../dingo/common/backend/sample_engine.py) — a complete, runnable implementation under 120 lines. Read it alongside this guide. @@ -93,9 +93,9 @@ alongside this guide. - This guide — step-by-step walkthrough for someone starting a new backend from scratch. -- [`LLMEngine` ABC docstrings](../../components/src/dynamo/common/backend/engine.py) +- [`LLMEngine` ABC docstrings](../../dingo/common/backend/engine.py) — authoritative method-by-method contract. -- [Package README](../../components/src/dynamo/common/backend/README.md) +- [Package README](../../dingo/common/backend/README.md) — in-tree reference: `GenerateRequest` / `GenerateChunk` field definitions, per-engine cancellation cookbook (vLLM / SGLang / TRT-LLM), full `DynamoException` table, file index, and the @@ -108,7 +108,7 @@ contract — what every engine on the unified path gets — plus the gaps that apply to all three engines. Per-engine specifics (vLLM sleep/wake, SGLang diffusion, TRT-LLM custom logits processors, etc.) live in the -[package README](../../components/src/dynamo/common/backend/README.md#feature-gaps). +[package README](../../dingo/common/backend/README.md#feature-gaps). **Supported today** @@ -182,14 +182,14 @@ A backend is two things: 1. **An engine class** that subclasses `LLMEngine` — owns the model, accepts preprocessed token requests, streams output chunks. 2. **A `main.py` entry point** — a three-line shim that hands the - engine class to `run()` from `dynamo.common.backend.run`, which + engine class to `run()` from `dingo.common.backend.run`, which drives the lifecycle. -The `dynamo.common.backend` package handles everything else: signal +The `dingo.common.backend` package handles everything else: signal handling, distributed runtime setup, model registration with discovery, the serving loop, graceful shutdown, cancellation monitoring, and error chain wrapping. (The lifecycle state machine -actually lives in Rust; `dynamo.common.backend.Worker` is a thin +actually lives in Rust; `dingo.common.backend.Worker` is a thin Python shim over it.) ```text @@ -234,7 +234,7 @@ name = "my-backend" version = "0.1.0" requires-python = ">=3.11" dependencies = [ - # ai-dynamo bundles dynamo.common.backend. Pin to the release whose + # ai-dynamo bundles dingo.common.backend. Pin to the release whose # LLMEngine contract you tested against — the surface is still beta # and may change between releases. "ai-dynamo>=1.2.0", @@ -278,7 +278,7 @@ import asyncio from collections.abc import AsyncGenerator from dynamo._core import Context -from dynamo.common.backend import ( +from dingo.common.backend import ( EngineConfig, GenerateChunk, GenerateRequest, @@ -394,7 +394,7 @@ request. Called concurrently for multiple in-flight requests. **Contract** (chunk shape is defined by the `GenerateChunk` TypedDict — see -[Request / Response Types](../../components/src/dynamo/common/backend/README.md#request--response-types) +[Request / Response Types](../../dingo/common/backend/README.md#request--response-types) in the package README for the field reference): - Every chunk carries `token_ids` and `index` (use `0` for single @@ -514,7 +514,7 @@ Use `ZmqSource` when the engine already emits Dynamo-compatible KV events on a ZMQ socket, as vLLM and SGLang do: ```python -from dynamo.common.backend.publisher import ZmqSource +from dingo.common.backend.publisher import ZmqSource async def kv_event_sources(self): return [ @@ -528,7 +528,7 @@ Use `PushSource` when the engine needs a live publisher object and drives TRT-LLM backend is the reference implementation for this path: ```python -from dynamo.common.backend.publisher import PushSource +from dingo.common.backend.publisher import PushSource def _on_kv_publisher_ready(self, publisher): self._kv_publisher = publisher @@ -559,7 +559,7 @@ Three lines. ```python # src/my_backend/main.py -from dynamo.common.backend.run import run +from dingo.common.backend.run import run from .engine import MyBackend @@ -584,7 +584,7 @@ Pair this with the `[project.scripts]` entry from Step 1's **Errors**: the framework wraps non-`DynamoException` errors raised from `generate()` (or lifecycle methods) as `Unknown`. For typed error reporting, raise a `DynamoException` subclass directly from -[`dynamo.llm.exceptions`](../../components/src/dynamo/common/backend/README.md#error-handling) +[`dynamo.llm.exceptions`](../../dingo/common/backend/README.md#error-handling) — it propagates unchanged through the Rust bridge: ```python @@ -625,7 +625,7 @@ pip install -e ".[dev]" ``` The sample engine has a unit-test -[suite](../../components/src/dynamo/common/backend/tests/test_engine.py) +[suite](../../dingo/common/backend/tests/test_engine.py) that you can copy as a starting point. The shape of a useful test: ```python @@ -726,11 +726,11 @@ the framework configures `tracing` from `DYN_LOG`. ### Python reference: sample engine -[`sample_engine.py`](../../components/src/dynamo/common/backend/sample_engine.py) +[`sample_engine.py`](../../dingo/common/backend/sample_engine.py) is the canonical minimal reference. Run it as-is: ```bash -python -m dynamo.common.backend.sample_main --model-name test-model +python -m dingo.common.backend.sample_main --model-name test-model ``` It generates rotating token IDs with no ML dependencies, so it's a @@ -763,11 +763,11 @@ Before shipping: ### Python see also -- [`LLMEngine` ABC](../../components/src/dynamo/common/backend/engine.py) +- [`LLMEngine` ABC](../../dingo/common/backend/engine.py) — authoritative contract. -- [Package README](../../components/src/dynamo/common/backend/README.md) +- [Package README](../../dingo/common/backend/README.md) — feature gaps, error model, request/response contract. -- [Sample engine](../../components/src/dynamo/common/backend/sample_engine.py) +- [Sample engine](../../dingo/common/backend/sample_engine.py) — example user guide. - Rust tab on this page — the Rust counterpart, same contract, lower-level. @@ -834,7 +834,7 @@ contract — what every engine on the unified path gets, whether written in Rust directly or plugged in from Python via the PyO3 `Worker` shim. Per-engine specifics (vLLM sleep/wake, SGLang diffusion, TRT-LLM custom logits processors, etc.) live in the -[Python package README](../../components/src/dynamo/common/backend/README.md#feature-gaps). +[Python package README](../../dingo/common/backend/README.md#feature-gaps). **Supported today** @@ -904,7 +904,7 @@ Request handling: | Feature | What's missing | |---------|----------------| -| `cum_log_probs` response wire | Completion-side `log_probs` / `top_logprobs` are populated on the unified path for vLLM, SGLang, and TRT-LLM (shared helpers in `components/src/dynamo/common/backend/logprobs.py`). Prompt-side logprobs ride on the final chunk's `LLMEngineOutput.engine_data["prompt_logprobs"]` (consumed by `prompt_logprobs_from_engine_data` in the response builders). `cum_log_probs` is still not emitted. | +| `cum_log_probs` response wire | Completion-side `log_probs` / `top_logprobs` are populated on the unified path for vLLM, SGLang, and TRT-LLM (shared helpers in `dingo/common/backend/logprobs.py`). Prompt-side logprobs ride on the final chunk's `LLMEngineOutput.engine_data["prompt_logprobs"]` (consumed by `prompt_logprobs_from_engine_data` in the response builders). `cum_log_probs` is still not emitted. | | Text-in-text-out mode | `ModelInput::Text` is rejected at startup — `Tokens` only | | Multimodal | Images / video / embeddings, NIXL embedding transfer, separate encode workers; `ENCODE` disaggregation role | | Diffusion | Image (FLUX), video (Wan2.1), LLM diffusion (DLLM) workers; no diffusion engine, MediaOutput, or media scheduling on the unified path | @@ -1609,7 +1609,7 @@ Before shipping: - [Conformance kit](../../lib/backend-common/src/testing.rs) — `run_conformance`, `mock_context`, `cancelling_context`. - [Mocker backend](../backends/mocker_backend/README.md) — example user guide. -- [Python sibling](../../components/src/dynamo/common/backend/README.md) +- [Python sibling](../../dingo/common/backend/README.md) — Python ABC layered over this crate. diff --git a/docs/dynosim/README.md b/docs/dynosim/README.md index 9960565ab807..1de7da5f2f6d 100644 --- a/docs/dynosim/README.md +++ b/docs/dynosim/README.md @@ -18,9 +18,9 @@ Use DynoSim when you want to answer questions such as: | Component | Entry Point | Role | |---|---|---| -| DynoSim run | `python -m dynamo.replay` | Runs one workload against one simulated Dynamo configuration and emits metrics plus a report | -| DynoSim sweep | `dynamo.profiler.utils.replay_optimize` | Sweeps many simulation trials across TP shape, worker split, router knobs, SLA constraints, and GPU budget | -| Live simulation with Mocker | `python -m dynamo.mocker` | Runs simulated workers inside a live Dynamo deployment path, including worker registration and KV event publishing | +| DynoSim run | `python -m dingo.replay` | Runs one workload against one simulated Dynamo configuration and emits metrics plus a report | +| DynoSim sweep | `dingo.profiler.utils.replay_optimize` | Sweeps many simulation trials across TP shape, worker split, router knobs, SLA constraints, and GPU budget | +| Live simulation with Mocker | `python -m dingo.mocker` | Runs simulated workers inside a live Dynamo deployment path, including worker registration and KV event publishing | | Mocker core | `lib/mocker` | Models engine scheduling, KV allocation, prefix caching, preemption, and timing | | AIC | AI Configurator SDK | Supplies calibrated timing and candidate-shape data for supported model/backend/GPU tuples | | Planner simulation | `--planner-config` on DynoSim runs | Runs Planner decisions in the simulation loop to study scaling behavior and SLA compliance | diff --git a/docs/dynosim/mocker.md b/docs/dynosim/mocker.md index 8f0920c268ca..79067ec8edb0 100644 --- a/docs/dynosim/mocker.md +++ b/docs/dynosim/mocker.md @@ -30,17 +30,17 @@ The mocker simulates: ```bash # Launch a single mocker worker -python -m dynamo.mocker --model-path Qwen/Qwen3-0.6B +python -m dingo.mocker --model-path Qwen/Qwen3-0.6B # Launch with custom KV cache configuration -python -m dynamo.mocker \ +python -m dingo.mocker \ --model-path Qwen/Qwen3-0.6B \ --num-gpu-blocks-override 8192 \ --block-size 64 \ --max-num-seqs 256 # Launch with timing speedup for faster testing -python -m dynamo.mocker \ +python -m dingo.mocker \ --model-path Qwen/Qwen3-0.6B \ --speedup-ratio 10.0 ``` @@ -49,13 +49,13 @@ python -m dynamo.mocker \ ```bash # Launch prefill worker -python -m dynamo.mocker \ +python -m dingo.mocker \ --model-path Qwen/Qwen3-0.6B \ --disaggregation-mode prefill \ --bootstrap-ports 50100 # Launch decode worker (in another terminal) -python -m dynamo.mocker \ +python -m dingo.mocker \ --model-path Qwen/Qwen3-0.6B \ --disaggregation-mode decode ``` @@ -64,7 +64,7 @@ python -m dynamo.mocker \ ```bash # Launch 4 mocker workers sharing the same tokio runtime -python -m dynamo.mocker \ +python -m dingo.mocker \ --model-path Qwen/Qwen3-0.6B \ --num-workers 4 ``` @@ -130,7 +130,7 @@ python -m dynamo.mocker \ ## DynoSim Runs -Mocker also powers DynoSim runs through the dedicated `python -m dynamo.replay` CLI, which exposes +Mocker also powers DynoSim runs through the dedicated `python -m dingo.replay` CLI, which exposes `offline|online`, `round_robin|kv_router`, `arrival_speedup_ratio`, closed-loop concurrency admission, synthetic workload generation, and offline disaggregated prefill/decode simulation directly: @@ -140,7 +140,7 @@ runs use `--extra-engine-args`. Offline disaggregated runs instead use `--num-prefill-workers` and `--num-decode-workers`. ```bash -python -m dynamo.replay /path/to/mooncake_trace.jsonl \ +python -m dingo.replay /path/to/mooncake_trace.jsonl \ --num-workers 4 \ --replay-mode offline \ --router-mode kv_router \ @@ -154,7 +154,7 @@ python -m dynamo.replay /path/to/mooncake_trace.jsonl \ The same CLI also supports synthetic workloads without a trace file: ```bash -python -m dynamo.replay \ +python -m dingo.replay \ --input-tokens 5000 \ --output-tokens 500 \ --request-count 1000 \ @@ -169,7 +169,7 @@ python -m dynamo.replay \ Synthetic workloads also support shared-prefix and multi-turn tests: ```bash -python -m dynamo.replay \ +python -m dingo.replay \ --input-tokens 5000 \ --output-tokens 500 \ --request-count 200 \ @@ -221,7 +221,7 @@ Offline DynoSim runs also support disaggregated `kv_router` mode. In that mode: Example: ```bash -python -m dynamo.replay \ +python -m dingo.replay \ --input-tokens 4096 \ --output-tokens 256 \ --request-count 100 \ @@ -251,9 +251,9 @@ It also accepts older raw-data directories containing: - `decode_raw_data.json` ```bash -python -m dynamo.mocker \ +python -m dingo.mocker \ --model-path nvidia/Llama-3.1-8B-Instruct-FP8 \ - --planner-profile-data components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D \ + --planner-profile-data dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D \ --speedup-ratio 1.0 ``` @@ -264,7 +264,7 @@ To use the AIC SDK for latency prediction: ```bash uv pip install '.[mocker]' -python -m dynamo.mocker \ +python -m dingo.mocker \ --model-path nvidia/Llama-3.1-8B-Instruct-FP8 \ --engine-type vllm \ --aic-perf-model \ @@ -275,8 +275,8 @@ The AIC model automatically uses `--model-path` and `--engine-type` to select th Important notes: -- AIC is opt-in. If you do not pass `--aic-perf-model`, `python -m dynamo.mocker` does not use AIC. -- `python -m dynamo.replay` has two separate AIC surfaces: +- AIC is opt-in. If you do not pass `--aic-perf-model`, `python -m dingo.mocker` does not use AIC. +- `python -m dingo.replay` has two separate AIC surfaces: - engine timing AIC through `--extra-engine-args` / staged engine JSON - router-side prefill-load AIC through top-level `--aic-*` flags plus `router_prefill_load_model="aic"` in `--router-config` - The Python AIC session bridge is now shared with the live KV router path via the internal `dynamo._internal.aic` module. Mocker CLI behavior is unchanged; this just removes duplicate AIC session code. @@ -292,14 +292,14 @@ mocked worker timing model itself to come from AIC. For aggregated DynoSim runs, engine timing AIC still comes from `--extra-engine-args`: ```bash -python -m dynamo.replay /path/to/trace.jsonl \ +python -m dingo.replay /path/to/trace.jsonl \ --extra-engine-args '{"aic_backend":"vllm","aic_system":"h200_sxm","aic_model_path":"nvidia/Llama-3.1-8B-Instruct-FP8","aic_tp_size":1}' ``` For offline disaggregated DynoSim runs, pass the staged engine configs instead: ```bash -python -m dynamo.replay /path/to/trace.jsonl \ +python -m dingo.replay /path/to/trace.jsonl \ --replay-mode offline \ --router-mode kv_router \ --prefill-engine-args '{"worker_type":"prefill","aic_backend":"vllm","aic_system":"h200_sxm","aic_model_path":"nvidia/Llama-3.1-8B-Instruct-FP8","aic_tp_size":1,"block_size":512}' \ @@ -308,12 +308,12 @@ python -m dynamo.replay /path/to/trace.jsonl \ --num-decode-workers 6 ``` -The `aic_backend` field enables the AIC perf model and should match `engine_type` (`"vllm"` or `"sglang"`). The `aic_model_path` field is the equivalent of `--model-path` in `dynamo.mocker`. +The `aic_backend` field enables the AIC perf model and should match `engine_type` (`"vllm"` or `"sglang"`). The `aic_model_path` field is the equivalent of `--model-path` in `dingo.mocker`. DynoSim router-side AIC prompt-load modeling is configured separately with top-level flags: ```bash -python -m dynamo.replay /path/to/trace.jsonl \ +python -m dingo.replay /path/to/trace.jsonl \ --replay-mode offline \ --router-mode kv_router \ --num-workers 4 \ @@ -336,7 +336,7 @@ the decode-stage router keeps prompt tracking disabled. Example `--reasoning` configuration: ```bash -python -m dynamo.mocker \ +python -m dingo.mocker \ --model-path Qwen/Qwen3-0.6B \ --reasoning '{"start_thinking_token_id":123,"end_thinking_token_id":456,"thinking_ratio":0.6}' ``` diff --git a/docs/dynosim/planner-benchmarking.md b/docs/dynosim/planner-benchmarking.md index f1bd8012d05b..320bb534194f 100644 --- a/docs/dynosim/planner-benchmarking.md +++ b/docs/dynosim/planner-benchmarking.md @@ -14,7 +14,7 @@ For the general mechanics of DynoSim runs (input format, arrival speedup, router ### Build Install the Dynamo Python components and build the Rust runtime bindings. The -`python -m dynamo.replay` CLI is part of the Python components, and it imports the +`python -m dingo.replay` CLI is part of the Python components, and it imports the runtime `_core` module from the bindings package: ```bash @@ -94,7 +94,7 @@ curl -sLO https://raw.githubusercontent.com/kvcache-ai/Mooncake/main/FAST25-rele Run agg (2 workers, TP=1): ```bash -python -m dynamo.replay traces/mooncake_fast25/toolagent_trace.jsonl \ +python -m dingo.replay traces/mooncake_fast25/toolagent_trace.jsonl \ --planner-config '{ "mode": "agg", "optimization_target": "sla", @@ -112,7 +112,7 @@ python -m dynamo.replay traces/mooncake_fast25/toolagent_trace.jsonl \ Run disagg (1P1D, TP=1): ```bash -python -m dynamo.replay traces/mooncake_fast25/toolagent_trace.jsonl \ +python -m dingo.replay traces/mooncake_fast25/toolagent_trace.jsonl \ --planner-config '{ "mode": "disagg", "optimization_target": "sla", @@ -151,7 +151,7 @@ run_one() { else extra=$(printf '{"aic_backend":"vllm","aic_system":"h200_sxm","aic_model_path":"nvidia/Llama-3.1-8B-Instruct-FP8","aic_tp_size":1,"startup_time":%d}' "$s") fi - python -m dynamo.replay "$TRACE" \ + python -m dingo.replay "$TRACE" \ --planner-config "$(printf '{"mode":"agg","optimization_target":"sla","ttft_ms":1500,"itl_ms":50,"enable_throughput_scaling":true,"enable_load_scaling":true,"pre_deployment_sweeping_mode":"rapid","throughput_adjustment_interval_seconds":300,"load_adjustment_interval_seconds":10,"prefill_engine_num_gpu":1,"decode_engine_num_gpu":1,"report_filename":"%s"}' "$name")" \ --extra-engine-args "$extra" \ --num-workers 2 --arrival-speedup-ratio 1.0 \ diff --git a/docs/dynosim/runs.md b/docs/dynosim/runs.md index 506218a98ecd..79c561294aa3 100644 --- a/docs/dynosim/runs.md +++ b/docs/dynosim/runs.md @@ -6,14 +6,14 @@ subtitle: Run one trace or synthetic workload through a simulated Dynamo configu --- A DynoSim run evaluates one workload against one simulated Dynamo configuration. The current CLI is -`python -m dynamo.replay`, which prints an AIPerf-style summary table, writes the full report JSON +`python -m dingo.replay`, which prints an AIPerf-style summary table, writes the full report JSON to disk, and exposes `offline|online`, `round_robin|kv_router`, `arrival_speedup_ratio`, closed-loop concurrency, and synthetic workload inputs directly. The command keeps the existing `replay` name for now. The docs use "DynoSim run" for the product concept: one workload, one simulated configuration, one report. -Unlike normal `dynamo.mocker` usage, offline mode does not launch workers, register endpoints, or +Unlike normal `dingo.mocker` usage, offline mode does not launch workers, register endpoints, or require NATS, etcd, or a frontend. Online mode does exercise the live mock-worker runtime path. Use DynoSim runs when you want to: @@ -73,7 +73,7 @@ See [`lib/mocker/src/replay/offline/README.md`](../../lib/mocker/src/replay/offl Run an offline DynoSim trial through the dedicated CLI: ```bash -python -m dynamo.replay /path/to/mooncake_trace.jsonl \ +python -m dingo.replay /path/to/mooncake_trace.jsonl \ --num-workers 4 \ --replay-mode offline \ --router-mode round_robin \ @@ -85,7 +85,7 @@ python -m dynamo.replay /path/to/mooncake_trace.jsonl \ Run a synthetic DynoSim trial through the same CLI when you want fixed request shapes without a trace file: ```bash -python -m dynamo.replay \ +python -m dingo.replay \ --input-tokens 5000 \ --output-tokens 500 \ --request-count 1000 \ @@ -101,7 +101,7 @@ Run a synthetic workload when you want shared-prefix or multi-turn structure wit file: ```bash -python -m dynamo.replay \ +python -m dingo.replay \ --input-tokens 5000 \ --output-tokens 500 \ --request-count 200 \ @@ -115,7 +115,7 @@ python -m dynamo.replay \ --report-json /tmp/dynosim-report.json ``` -`python -m dynamo.replay` prints an AIPerf-style summary table to stdout and writes the full +`python -m dingo.replay` prints an AIPerf-style summary table to stdout and writes the full report JSON to disk. ## Input Format @@ -170,7 +170,7 @@ block size from the records and builds the standard or agentic in-memory model based on `agent_context`. It does not create an intermediate Mooncake file. ```bash -python -m dynamo.replay /tmp/dynamo-request-trace.*.jsonl.gz \ +python -m dingo.replay /tmp/dynamo-request-trace.*.jsonl.gz \ --trace-format dynamo \ --replay-mode offline \ --router-mode kv_router \ @@ -208,7 +208,7 @@ child requests spawned by this row, and `prefix_reset` marks the first row in a Run it with: ```bash -python -m dynamo.replay /path/to/agentic-mooncake.jsonl \ +python -m dingo.replay /path/to/agentic-mooncake.jsonl \ --trace-format agentic_mooncake \ --trace-block-size 128 \ --replay-mode offline \ @@ -235,7 +235,7 @@ Dynamo request traces embed their trace block size. DynoSim derives it when ## DynoSim Surfaces -### `python -m dynamo.replay` +### `python -m dingo.replay` The dedicated DynoSim CLI exposes: @@ -280,7 +280,7 @@ Defaults: Example: ```bash -python -m dynamo.replay /path/to/mooncake_trace.jsonl \ +python -m dingo.replay /path/to/mooncake_trace.jsonl \ --replay-mode online \ --router-mode kv_router \ --num-workers 4 \ @@ -348,7 +348,7 @@ Synthetic mode bypasses trace loading and generates in-memory requests with fixe lengths and optional synthetic arrival spacing: ```bash -python -m dynamo.replay \ +python -m dingo.replay \ --input-tokens 5000 \ --output-tokens 500 \ --request-count 200 \ @@ -381,7 +381,7 @@ Default trace mode preserves the timestamps from the trace and simulates arrival those timestamps: ```bash -python -m dynamo.replay /path/to/mooncake_trace.jsonl \ +python -m dingo.replay /path/to/mooncake_trace.jsonl \ --replay-mode offline \ --num-workers 4 \ --trace-block-size 512 \ @@ -399,7 +399,7 @@ Use `--replay-concurrency` to ignore first-turn trace arrival timing and keep a requests in flight: ```bash -python -m dynamo.replay /path/to/mooncake_trace.jsonl \ +python -m dingo.replay /path/to/mooncake_trace.jsonl \ --replay-mode offline \ --num-workers 4 \ --replay-concurrency 16 @@ -421,7 +421,7 @@ is useful when you want the run to include live request dispatch, live output ha same async KV-event propagation model used by the current router integration. ```bash -python -m dynamo.replay /path/to/mooncake_trace.jsonl \ +python -m dingo.replay /path/to/mooncake_trace.jsonl \ --replay-mode online \ --router-mode kv_router \ --num-workers 4 \ @@ -436,7 +436,7 @@ Use `--arrival-speedup-ratio` to compress or stretch the trace arrival process w mocker compute model. Larger values make arrivals happen sooner relative to the original trace. ```bash -python -m dynamo.replay /path/to/mooncake_trace.jsonl \ +python -m dingo.replay /path/to/mooncake_trace.jsonl \ --replay-mode offline \ --num-workers 4 \ --arrival-speedup-ratio 5 \ @@ -466,7 +466,7 @@ To compare queue policies manually, keep the same trace and engine args fixed an `router_queue_policy` inside `--router-config`: ```bash -python -m dynamo.replay /path/to/mooncake_trace.jsonl \ +python -m dingo.replay /path/to/mooncake_trace.jsonl \ --replay-mode offline \ --router-mode kv_router \ --num-workers 4 \ @@ -474,7 +474,7 @@ python -m dynamo.replay /path/to/mooncake_trace.jsonl \ --extra-engine-args '{"block_size":64}' \ --router-config '{"router_queue_policy":"fcfs"}' -python -m dynamo.replay /path/to/mooncake_trace.jsonl \ +python -m dingo.replay /path/to/mooncake_trace.jsonl \ --replay-mode offline \ --router-mode kv_router \ --num-workers 4 \ @@ -489,7 +489,7 @@ an expected production default. To enable router-side AIC prefill-load modeling during simulation: ```bash -python -m dynamo.replay /path/to/mooncake_trace.jsonl \ +python -m dingo.replay /path/to/mooncake_trace.jsonl \ --replay-mode offline \ --router-mode kv_router \ --num-workers 4 \ @@ -528,9 +528,9 @@ The report contains: - output-token-throughput-per-user summaries The dedicated DynoSim CLI returns the same report schema as the Python APIs -`dynamo.replay.run_trace_replay(...)` and `dynamo.replay.run_synthetic_trace_replay(...)`. +`dingo.replay.run_trace_replay(...)` and `dingo.replay.run_synthetic_trace_replay(...)`. -If `--report-json` is not provided, `python -m dynamo.replay` writes a timestamped +If `--report-json` is not provided, `python -m dingo.replay` writes a timestamped `dynamo_replay_report_*.json` file in the current working directory. ## Constraints @@ -559,7 +559,7 @@ If you violate those constraints, DynoSim fails immediately with a validation er ## Practical Notes -- `python -m dynamo.replay` requires exactly one of: +- `python -m dingo.replay` requires exactly one of: either a trace file, or all of `--input-tokens`, `--output-tokens`, and `--request-count` - `--replay-concurrency` works with both trace-file and synthetic workloads - mocker compute-speed knobs such as `speedup_ratio` still affect simulated timing when passed via diff --git a/docs/dynosim/sweeps.md b/docs/dynosim/sweeps.md index 260013394bec..7ff41f8bd36a 100644 --- a/docs/dynosim/sweeps.md +++ b/docs/dynosim/sweeps.md @@ -7,7 +7,7 @@ subtitle: Search simulated deployment candidates across topology, router, and SL A DynoSim sweep runs many simulated trials across candidate topologies, router settings, and timing-model inputs, then ranks the results against SLA constraints and GPU budget. Use sweeps when a single [DynoSim run](runs.md) is not enough and you want to search the design space before validating on real GPUs. -The current Python API is `dynamo.profiler.utils.replay_optimize`. The docs use "DynoSim sweep" as the product term while keeping the existing implementation name for now. +The current Python API is `dingo.profiler.utils.replay_optimize`. The docs use "DynoSim sweep" as the product term while keeping the existing implementation name for now. ## What It Answers @@ -121,7 +121,7 @@ export DYN_LOG='info,dynamo_kv_router::scheduling::selector=warn' The canonical starting point is the checked-in driver script: ```bash -.venv/bin/python components/src/dynamo/profiler/utils/replay_optimize/example.py \ +.venv/bin/python dingo/profiler/utils/replay_optimize/example.py \ --max-parallel-evals 4 ``` @@ -165,7 +165,7 @@ Only add those when the experiment is specifically about scheduler limits. To run against a Mooncake-style trace instead of the synthetic workload: ```bash -.venv/bin/python components/src/dynamo/profiler/utils/replay_optimize/example.py \ +.venv/bin/python dingo/profiler/utils/replay_optimize/example.py \ --trace-file /path/to/mooncake_trace.jsonl \ --arrival-speedup-ratio 1.0 \ --max-parallel-evals 4 @@ -182,7 +182,7 @@ curl -sL \ Then run: ```bash -.venv/bin/python components/src/dynamo/profiler/utils/replay_optimize/example.py \ +.venv/bin/python dingo/profiler/utils/replay_optimize/example.py \ --trace-file /tmp/toolagent_trace.jsonl \ --arrival-speedup-ratio 1.0 \ --max-parallel-evals 4 diff --git a/docs/features/diffusion/README.md b/docs/features/diffusion/README.md index 760d1e7c7ea0..5a8e61f1fc2c 100644 --- a/docs/features/diffusion/README.md +++ b/docs/features/diffusion/README.md @@ -26,5 +26,4 @@ For deployment guides, configuration, and examples for each backend: - **[vLLM-Omni](../../backends/vllm/vllm-omni.md)** - **[SGLang Diffusion](../../backends/sglang/sglang-diffusion.md)** -- **[TRT-LLM Diffusion](../../backends/trtllm/trtllm-diffusion.md)** - **[FastVideo (custom worker)](fastvideo.md)** diff --git a/docs/features/diffusion/fastvideo.md b/docs/features/diffusion/fastvideo.md index 1cbbfcff4563..456c5cde414a 100644 --- a/docs/features/diffusion/fastvideo.md +++ b/docs/features/diffusion/fastvideo.md @@ -9,7 +9,7 @@ sidebar-title: FastVideo This guide covers deploying [FastVideo](https://github.com/hao-ai-lab/FastVideo) text-to-video generation on Dynamo using a custom worker (`worker.py`) exposed through the `/v1/videos` endpoint. > [!NOTE] -> Dynamo also supports diffusion through built-in backends: [SGLang Diffusion](../../backends/sglang/sglang-diffusion.md) (LLM diffusion, image, video), [vLLM-Omni](../../backends/vllm/vllm-omni.md) (text-to-image, text-to-video), and [TRT-LLM Diffusion](../../backends/trtllm/trtllm-diffusion.md) (text-to-image, text-to-video). See the [Diffusion Overview](README.md) for the full support matrix. +> Dynamo also supports diffusion through built-in backends: [SGLang Diffusion](../../backends/sglang/sglang-diffusion.md) (LLM diffusion, image, video) and [vLLM-Omni](../../backends/vllm/vllm-omni.md) (text-to-image, text-to-video). See the [Diffusion Overview](README.md) for the full support matrix. ## Overview @@ -282,5 +282,4 @@ The example source lives at [`examples/diffusers/`](https://github.com/ai-dynamo - [vLLM-Omni Text-to-Image](../../backends/vllm/vllm-omni.md#text-to-image) — vLLM-Omni image generation - [SGLang Video Generation](../../backends/sglang/sglang-diffusion.md#video-generation) — SGLang video generation worker - [SGLang Image Diffusion](../../backends/sglang/sglang-diffusion.md#image-diffusion) — SGLang image diffusion worker -- [TRT-LLM Diffusion](../../backends/trtllm/trtllm-diffusion.md#quick-start) — TensorRT-LLM diffusion quick start - [Diffusion Overview](README.md) — Full backend support matrix diff --git a/docs/features/disaggregated-serving/README.md b/docs/features/disaggregated-serving/README.md index 59ef40b04502..0c65ec57641c 100644 --- a/docs/features/disaggregated-serving/README.md +++ b/docs/features/disaggregated-serving/README.md @@ -278,7 +278,6 @@ transfer settings: | Backend | Examples | |---|---| | vLLM | [Deployment examples](https://github.com/ai-dynamo/dynamo/tree/main/examples/backends/vllm/deploy), including `disagg.yaml`, `disagg_router.yaml`, and `disagg_planner.yaml` | -| TensorRT-LLM | [Deployment examples](https://github.com/ai-dynamo/dynamo/tree/main/examples/backends/trtllm/deploy), including disaggregated, router, and planner variants | | SGLang | [Deployment examples](https://github.com/ai-dynamo/dynamo/tree/main/examples/backends/sglang/deploy), including NIXL-based disaggregated serving | ## Operational Notes diff --git a/docs/features/lora/README.md b/docs/features/lora/README.md index aa2bd8c9bb04..542e9ed21523 100644 --- a/docs/features/lora/README.md +++ b/docs/features/lora/README.md @@ -49,7 +49,7 @@ flowchart TD The LoRA system consists of: - **Rust Core** (`lib/llm/src/lora/`): High-performance downloading, caching, and validation -- **Python Manager** (`components/src/dynamo/common/lora/`): Extensible wrapper with custom source support +- **Python Manager** (`dingo/common/lora/`): Extensible wrapper with custom source support - **Worker Handlers** (`dingo/vllm/handlers.py`): Load/unload API and inference integration ## Quick Start diff --git a/docs/features/multimodal/README.md b/docs/features/multimodal/README.md index f19bbcb306cf..2bdf3adcc78e 100644 --- a/docs/features/multimodal/README.md +++ b/docs/features/multimodal/README.md @@ -8,7 +8,7 @@ subtitle: Deploy multimodal models with image, video, and audio support in Dynam Dynamo supports multimodal inference across multiple LLM backends, enabling models to process images, video, and audio alongside text. -**Security Requirement**: Multimodal processing must be explicitly enabled at startup. See the relevant backend documentation ([vLLM](multimodal-vllm.md), [SGLang](multimodal-sglang.md), [TRT-LLM](multimodal-trtllm.md)) for the necessary flags. This prevents unintended processing of multimodal data from untrusted sources. +**Security Requirement**: Multimodal processing must be explicitly enabled at startup. See the relevant backend documentation ([vLLM](multimodal-vllm.md), [SGLang](multimodal-sglang.md)) for the necessary flags. This prevents unintended processing of multimodal data from untrusted sources. ```mermaid @@ -43,7 +43,6 @@ Dynamo provides support for improving latency and throughput for vision-and-lang | Stack | Image | Video | Audio | |-------|-------|-------|-------| | **[vLLM](multimodal-vllm.md)** | ✅ | 🧪 | 🧪 | -| **[TRT-LLM](multimodal-trtllm.md)** | ✅ | ❌ | ❌ | | **[SGLang](multimodal-sglang.md)** | ✅ | 🧪 | ❌ | **Status:** ✅ Supported | 🧪 Experimental | ❌ Not supported @@ -51,7 +50,7 @@ Dynamo provides support for improving latency and throughput for vision-and-lang ## Security: URL Validation All multimodal loaders route remote fetches through a shared URL policy -(`dynamo.common.multimodal.url_validator`). Only +(`dingo.common.multimodal.url_validator`). Only `https://` and `data:` URLs are allowed by default, private / internal IPs are blocked, and local file access is disabled. Every HTTP redirect hop is re-validated against the policy. @@ -72,7 +71,6 @@ Two environment variables loosen the defaults for non-public deployments: Reference implementations for deploying multimodal models: - [vLLM multimodal examples](https://github.com/ai-dynamo/dynamo/tree/main/examples/backends/vllm/launch) (image, video) -- [TRT-LLM multimodal examples](https://github.com/ai-dynamo/dynamo/tree/main/examples/backends/trtllm/launch) - [SGLang multimodal examples](https://github.com/ai-dynamo/dynamo/tree/main/examples/backends/sglang/launch) ## Backend Documentation @@ -80,5 +78,4 @@ Reference implementations for deploying multimodal models: Detailed deployment guides, configuration, and examples for each backend: - **[vLLM Multimodal](multimodal-vllm.md)** -- **[TensorRT-LLM Multimodal](multimodal-trtllm.md)** - **[SGLang Multimodal](multimodal-sglang.md)** diff --git a/docs/features/multimodal/embedding-cache.md b/docs/features/multimodal/embedding-cache.md index d55aa3d894a9..1525cf97e7e4 100644 --- a/docs/features/multimodal/embedding-cache.md +++ b/docs/features/multimodal/embedding-cache.md @@ -25,14 +25,13 @@ If your workload consists entirely of unique multimodal content, the cache provi | Backend | Aggregated | Disaggregated (E/PD) | Notes | |---------|------------|----------------------|-------| | **vLLM** | ✅ | ✅ | Aggregated uses vLLM-native `ec_both`; disaggregated uses Dynamo `EmbeddingCacheManager` | -| **TRT-LLM** | ❌ | ✅ | Dynamo `MultimodalEmbeddingCacheManager` in PD worker | | **SGLang** | ❌ | ✅ | Dynamo `MultimodalEmbeddingCacheManager` in the encode worker | This support requires vLLM `0.17.0` or newer. ## How It Works -In vLLM/TRT-LLM disaggregated flows, the prefill worker owns the CPU-side LRU cache. On a hit, the encode worker is skipped entirely. On a miss, the encode worker produces the embedding, transfers it via NIXL, and the prefill worker saves it to the cache. +In vLLM disaggregated flows, the prefill worker owns the CPU-side LRU cache. On a hit, the encode worker is skipped entirely. On a miss, the encode worker produces the embedding, transfers it via NIXL, and the prefill worker saves it to the cache. In SGLang E/PD, the encode worker owns the cache and skips re-encoding on cache hits before forwarding the cached image or video embeddings downstream. @@ -53,13 +52,6 @@ cd $DYNAMO_HOME/examples/backends/vllm bash launch/disagg_multimodal_e_pd.sh --multimodal-embedding-cache-capacity-gb 10 ``` -**Launch (TRT-LLM):** - -```bash -cd $DYNAMO_HOME/examples/backends/trtllm -./launch/disagg_e_pd.sh --multimodal-embedding-cache-capacity-gb 10 -``` - ## Configuration | Parameter | Description | Default | @@ -68,4 +60,4 @@ cd $DYNAMO_HOME/examples/backends/trtllm Set the capacity based on your expected working set of unique multimodal content. A larger cache holds more embeddings but consumes more host memory. -See the backend-specific documentation ([vLLM](multimodal-vllm.md#embedding-cache), [TRT-LLM](multimodal-trtllm.md#embedding-cache)) for more details. +See the [vLLM backend documentation](multimodal-vllm.md#embedding-cache) for more details. diff --git a/docs/features/multimodal/encoder-disaggregation.md b/docs/features/multimodal/encoder-disaggregation.md index 943f28fa07c9..cf1418f531cf 100644 --- a/docs/features/multimodal/encoder-disaggregation.md +++ b/docs/features/multimodal/encoder-disaggregation.md @@ -30,7 +30,6 @@ For simple deployments or development/testing, the aggregated (EPD) pattern is e | Backend | E/PD | E/P/D | Notes | |---------|------|-------|-------| | **vLLM** | ✅ | ✅ | Separate encode worker currently handles `image_url` inputs; `video_url` inputs stay on the prefill/PD path | -| **TRT-LLM** | ❌ | ✅ | Supports image URLs (via `MultimodalEncoder`) and pre-computed embeddings (via NIXL) | | **SGLang** | ✅ | ✅ | NIXL for embeddings; bootstrap mechanism for P/D KV transfer | ## Deployment Patterns @@ -67,18 +66,6 @@ bash launch/disagg_multimodal_e_pd.sh --model "Qwen/Qwen3-VL-30B-A3B-Instruct-FP bash launch/disagg_multimodal_epd.sh --model "Qwen/Qwen3-VL-30B-A3B-Instruct-FP8" ``` -### TRT-LLM - -```bash -cd $DYNAMO_HOME/examples/backends/trtllm - -# E/PD -bash launch/disagg_e_pd.sh - -# E/P/D -./launch/epd_multimodal_image_and_embeddings.sh -``` - ### SGLang ```bash @@ -91,4 +78,4 @@ cd $DYNAMO_HOME/examples/backends/sglang ./launch/multimodal_disagg.sh ``` -See the backend-specific documentation ([vLLM](multimodal-vllm.md), [TRT-LLM](multimodal-trtllm.md), [SGLang](multimodal-sglang.md)) for full configuration details and component flags. +See the backend-specific documentation ([vLLM](multimodal-vllm.md), [SGLang](multimodal-sglang.md)) for full configuration details and component flags. diff --git a/docs/features/multimodal/multimodal-kv-routing.md b/docs/features/multimodal/multimodal-kv-routing.md index 5ff976da9ee4..c34ee71e3d58 100644 --- a/docs/features/multimodal/multimodal-kv-routing.md +++ b/docs/features/multimodal/multimodal-kv-routing.md @@ -7,7 +7,7 @@ subtitle: Route multimodal requests to workers with the best KV cache overlap ## Overview -Multimodal KV routing extends Dynamo's KV-aware router to account for image content when computing cache overlap scores. An image hash (`mm_hash`) is computed per request — in the Rust frontend by default for vLLM backends, by vLLM's own processor when the chat-processor variant is enabled, or by a dedicated MM router worker for TRT-LLM backends — and included in per-block routing metadata. The KV router then selects the backend worker with the highest cache overlap, including overlap on image embedding blocks. +Multimodal KV routing extends Dynamo's KV-aware router to account for image content when computing cache overlap scores. An image hash (`mm_hash`) is computed per request — in the Rust frontend by default for vLLM backends or by vLLM's own processor when the chat-processor variant is enabled — and included in per-block routing metadata. The KV router then selects the backend worker with the highest cache overlap, including overlap on image embedding blocks. Repeated requests containing the same image are routed to the worker that already has the corresponding KV cache blocks, maximizing prefix cache reuse. @@ -29,7 +29,6 @@ Without MM-aware routing, the standard router treats image token blocks as opaqu |---------|------|-----------|-------| | **vLLM** | Rust frontend (default) | ✅ | Uses `llm-multimodal` crate for image-token counting + placeholder expansion. Supported models tracked below. | | **vLLM** | Python chat-processor (`--dyn-chat-processor vllm --router-mode kv`) | ✅ | Uses vLLM's own multimodal processor — supports any VLM that vLLM supports. | -| **TRT-LLM** | — | ✅ | Uses dedicated MM Router Worker. Requires `--publish-events-and-metrics` on TRT-LLM workers. | | **SGLang** | Rust frontend (default) | ✅ (\*) | Uses `llm-multimodal` crate for image-token counting; engaged automatically when the worker reports `backend_framework="sglang"`. | (\*) The SGLang Rust-frontend path substitutes per-image `pad_value` tokens in the routing-side view so SGLang's RadixAttention prefix cache key (`MM_PAD_SHIFT_VALUE + mm_hash % 2^30`) matches byte-for-byte. Requires the sglang fork with the `mm_hashes` field on `GenerateReqInput` ([sgl-project/sglang#25300](https://github.com/sgl-project/sglang/pull/25300)). @@ -88,19 +87,6 @@ Frontend (vLLM processor + KV router) → Backend Workers Use this variant (`--dyn-chat-processor=vllm`) when you want the frontend to run vLLM's HF image processor in-process and ship pre-processed `mm_kwargs` to the selected worker via shared memory or NIXL RDMA, so the backend skips the HF processor entirely. See the [Transfer Mode Details](#transfer-mode-details-vllm-only) section below for the `DYNAMO_MM_TRANSFER` flags. -### TRT-LLM - -```text -Frontend (round-robin) → MM Router Worker → Backend Workers - │ - ├─ Download image - ├─ Compute mm_hash - ├─ Build per-block MM metadata - └─ KvRouter selects best worker -``` - -For TRT-LLM, a dedicated MM Router Worker sits between the frontend and backend workers. See the [TRT-LLM MM Router README](https://github.com/ai-dynamo/dynamo/tree/main/examples/backends/trtllm/mm_router_worker/README.md) for setup instructions. - ### SGLang ```text @@ -192,15 +178,6 @@ delivery channel between frontend and worker. | `DYNAMO_MM_TRANSFER` | `shm` | Transfer mode for pre-processed mm_kwargs: `shm` (shared memory, same-node), `nixl` (RDMA, cross-node) | | `DYNAMO_DISABLE_NIXL_MM` | unset | Set to `1` to disable mm_kwargs transfer entirely (backend re-processes images from URLs) | -### TRT-LLM - -```bash -cd $DYNAMO_HOME/examples/backends/trtllm/mm_router_worker -./launch.sh -``` - -See the [TRT-LLM MM Router README](https://github.com/ai-dynamo/dynamo/tree/main/examples/backends/trtllm/mm_router_worker/README.md) for full setup instructions and configuration options. - ### SGLang ```bash @@ -262,9 +239,8 @@ On the second identical request the same worker wins with high overlap (e.g. `13 Applies to the `--dyn-chat-processor=vllm` launch (`agg_multimodal_router_chat_processor.sh`), **not** the default Rust frontend path. In the chat-processor variant the frontend runs the HF image processor in-process and ships the pre-processed `mm_kwargs` to the selected backend worker so the backend can skip re-processing; the `DYNAMO_MM_TRANSFER` environment variable controls how that payload is transferred. -The default Rust frontend path doesn't run the HF processor or pre-render `mm_kwargs` — it forwards only `mm_hashes`, and each worker re-processes the image itself. TRT-LLM backends similarly re-run their own preprocessing and don't honor `DYNAMO_MM_TRANSFER`. +The default Rust frontend path doesn't run the HF processor or pre-render `mm_kwargs` — it forwards only `mm_hashes`, and each worker re-processes the image itself. - **`shm`** (default): POSIX shared memory via a `/dev/shm` segment. Intended for same-node deployments, where frontend and backend share the host filesystem. If the backend can't access the segment (e.g., running on a different node), it falls back to re-processing the image from the URL. - **`nixl`**: NIXL RDMA transfer. Required for cross-node deployments where `/dev/shm` is not shared between frontend and backend. Works across nodes over InfiniBand or TCP (whichever UCX selects). - **`DYNAMO_DISABLE_NIXL_MM=1`**: Disables pre-processed mm_kwargs transfer entirely. The backend downloads and processes images itself from the original URLs. Useful for debugging or when transfer overhead exceeds re-processing cost. - diff --git a/docs/getting-started/examples.md b/docs/getting-started/examples.md index bbe9cd5da5e9..32e1baa1ad76 100644 --- a/docs/getting-started/examples.md +++ b/docs/getting-started/examples.md @@ -24,9 +24,3 @@ Presents examples and reference implementations for deploying Large Language Mod Presents examples and reference implementations for deploying Large Language Models (LLMs) in various configurations with SGLang. [View SGLang Backend Guide](../backends/sglang/README.md) - -## TensorRT-LLM - -Presents examples and reference implementations for deploying Large Language Models (LLMs) in various configurations with TensorRT-LLM. - -[View TensorRT-LLM Backend Guide](../backends/trtllm/README.md) diff --git a/docs/getting-started/local-installation.md b/docs/getting-started/local-installation.md index 0acf433bc8b3..5120f7cd7c8d 100644 --- a/docs/getting-started/local-installation.md +++ b/docs/getting-started/local-installation.md @@ -53,7 +53,7 @@ To run frontend and worker in the same container, either: See [Release Artifacts](../reference/release-artifacts.md#container-images) for available versions and backend guides for run instructions: [SGLang](../backends/sglang/README.md) | -[TensorRT-LLM](../backends/trtllm/README.md) | [vLLM](../backends/vllm/README.md) +[vLLM](../backends/vllm/README.md) ### Option B: Install from PyPI @@ -89,8 +89,7 @@ pip install --pre --extra-index-url https://pypi.nvidia.com "ai-dynamo[trtllm]" TensorRT-LLM requires `pip` due to a transitive Git URL dependency that `uv` doesn't resolve. We recommend using the TensorRT-LLM container for -broader compatibility. See the [TRT-LLM backend guide](../backends/trtllm/README.md) -for details. +broader compatibility. **vLLM** diff --git a/docs/getting-started/local-installation.zh-CN.md b/docs/getting-started/local-installation.zh-CN.md index 78685f02e53c..ec2ac390d591 100644 --- a/docs/getting-started/local-installation.zh-CN.md +++ b/docs/getting-started/local-installation.zh-CN.md @@ -54,7 +54,7 @@ docker run --gpus all --network host --rm -it nvcr.io/nvidia/ai-dynamo/vllm-runt 如需查看可用版本，请参阅[发布产物](../reference/release-artifacts.md#container-images)； 如需运行说明，请参阅各后端指南：[SGLang](../backends/sglang/README.md) | -[TensorRT-LLM](../backends/trtllm/README.md) | [vLLM](../backends/vllm/README.md) +[vLLM](../backends/vllm/README.md) ### 选项 B：从 PyPI 安装 @@ -90,7 +90,6 @@ pip install --pre --extra-index-url https://pypi.nvidia.com "ai-dynamo[trtllm]" 由于传递性 Git URL 依赖项 `uv` 无法解析，TensorRT-LLM 需要使用 `pip`。 为获得更广泛的兼容性，我们建议使用 TensorRT-LLM 容器。 -详情请参阅 [TRT-LLM 后端指南](../backends/trtllm/README.md)。 **vLLM** diff --git a/docs/index.yml b/docs/index.yml index 19aaf5be2627..ec1a72bb263d 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -329,8 +329,6 @@ navigation: path: features/multimodal/multimodal-kv-routing.md - page: SGLang Multimodal path: features/multimodal/multimodal-sglang.md - - page: TensorRT-LLM Multimodal - path: features/multimodal/multimodal-trtllm.md - page: vLLM Multimodal path: features/multimodal/multimodal-vllm.md - section: Diffusion @@ -365,19 +363,6 @@ navigation: path: backends/sglang/sglang-observability.md - page: Agentic Workloads path: backends/sglang/agents.md - - section: TensorRT-LLM - path: backends/trtllm/README.md - contents: - - page: Reference Guide - path: backends/trtllm/trtllm-reference-guide.md - - page: Examples - path: backends/trtllm/trtllm-examples.md - - page: Observability - path: backends/trtllm/trtllm-observability.md - - page: Diffusion (Experimental) - path: backends/trtllm/trtllm-diffusion.md - - page: Known Issues and Mitigations - path: backends/trtllm/trtllm-known-issues.md - section: vLLM path: backends/vllm/README.md contents: @@ -570,25 +555,6 @@ navigation: # -- Frontend (hidden sub-pages) -- - page: NVIDIA Request Extensions (nvext) path: components/frontend/nvext.md - # -- Backend detail pages -- - - section: TensorRT-LLM Details - contents: - - page: Building a Custom Container - path: backends/trtllm/trtllm-building-custom-container.md - - page: KV Cache Transfer - path: backends/trtllm/trtllm-kv-cache-transfer.md - - page: Logits Processing - path: backends/trtllm/trtllm-logits-processing.md - - page: DP Rank Routing - path: backends/trtllm/trtllm-dp-rank-routing.md - - page: Multinode Examples - path: backends/trtllm/multinode/trtllm-multinode-examples.md - - page: Llama4 + Eagle - path: backends/trtllm/trtllm-llama4-plus-eagle.md - - page: Gemma3 Sliding Window - path: backends/trtllm/trtllm-gemma3-sliding-window-attention.md - - page: GPT-OSS - path: backends/trtllm/trtllm-gpt-oss.md # -- Features (hidden sub-pages) -- - section: Speculative Decoding path: features/speculative-decoding/README.md diff --git a/docs/kubernetes/api-reference.md b/docs/kubernetes/api-reference.md index ba6e53c75b37..0315effee2e4 100644 --- a/docs/kubernetes/api-reference.md +++ b/docs/kubernetes/api-reference.md @@ -2452,7 +2452,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `planner` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | Planner contains the raw Planner configuration passed to the Planner service.
Its schema is defined by dynamo.planner.config.planner_config.PlannerConfig.
See https://docs.dynamo.nvidia.com/dynamo/components/planner/planner-guide#plannerconfig-reference.
DGDR passes this object through without field-level validation; the Planner
service validates it at startup.
The presence of this field (non-null) enables the planner in the generated DGD. | | Type: object
Optional: \{\}
| +| `planner` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | Planner contains the raw Planner configuration passed to the Planner service.
Its schema is defined by dingo.planner.config.planner_config.PlannerConfig.
See https://docs.dynamo.nvidia.com/dynamo/components/planner/planner-guide#plannerconfig-reference.
DGDR passes this object through without field-level validation; the Planner
service validates it at startup.
The presence of this field (non-null) enables the planner in the generated DGD. | | Type: object
Optional: \{\}
| | `mocker` _[MockerSpec](#mockerspec)_ | Mocker configures the simulated (mocker) backend for testing without GPUs. | | Optional: \{\}
| diff --git a/docs/kubernetes/autoscaling.md b/docs/kubernetes/autoscaling.md index 8545302b2def..bd1bd85450b8 100644 --- a/docs/kubernetes/autoscaling.md +++ b/docs/kubernetes/autoscaling.md @@ -171,7 +171,6 @@ The recommended way to deploy Planner is via `DynamoGraphDeploymentRequest` (DGD Example configurations with Planner: - `examples/backends/vllm/deploy/disagg_planner.yaml` - `examples/backends/sglang/deploy/disagg_planner.yaml` -- `examples/backends/trtllm/deploy/disagg_planner.yaml` For more details, see the [SLA Planner documentation](../components/planner/planner-guide.md). diff --git a/docs/kubernetes/deployment/multinode-deployment.md b/docs/kubernetes/deployment/multinode-deployment.md index 317cd70a4cab..b4f932ab1085 100644 --- a/docs/kubernetes/deployment/multinode-deployment.md +++ b/docs/kubernetes/deployment/multinode-deployment.md @@ -309,7 +309,6 @@ To enable compilation cache, add a volume mount with `useAsCompilationCache: tru For additional support and examples, see the working multinode configurations in: - **SGLang**: [examples/backends/sglang/deploy/](https://github.com/ai-dynamo/dynamo/tree/main/examples/backends/sglang/deploy/README.md) -- **TensorRT-LLM**: [examples/backends/trtllm/deploy/](https://github.com/ai-dynamo/dynamo/tree/main/examples/backends/trtllm/deploy/README.md) - **vLLM**: [examples/backends/vllm/deploy/](https://github.com/ai-dynamo/dynamo/tree/main/examples/backends/vllm/deploy/README.md) These examples demonstrate proper usage of the `multinode` section with corresponding `gpu` limits and correct `tp-size` configuration. diff --git a/docs/kubernetes/dgdr.md b/docs/kubernetes/dgdr.md index 06b8895399db..f5597b327b8e 100644 --- a/docs/kubernetes/dgdr.md +++ b/docs/kubernetes/dgdr.md @@ -287,7 +287,6 @@ Worker KV-event flags are backend-specific. For cross-backend behavior, see |---|---|---| | vLLM | [vLLM Reference Guide](../backends/vllm/vllm-reference-guide.md#argument-reference), [vLLM Examples](../backends/vllm/vllm-examples.md#aggregated-serving-with-kv-routing) | `--enable-prefix-caching` and `--kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}'` on the aggregated worker or disaggregated prefill worker | | SGLang | [SGLang KV Events](../backends/sglang/sglang-reference-guide.md#kv-events), [SGLang Examples](../backends/sglang/sglang-examples.md#aggregated-serving-with-kv-routing) | `--kv-events-config` with the SGLang event endpoint | -| TRT-LLM | [TRT-LLM DP Rank Routing](../backends/trtllm/trtllm-dp-rank-routing.md#enabling-dp-rank-routing), [TRT-LLM Observability](../backends/trtllm/trtllm-observability.md) | `--publish-events-and-metrics` | In Kubernetes deployments the Dynamo runtime normally uses Kubernetes discovery and the NATS event plane. Some backends, such as vLLM and SGLang, diff --git a/docs/kubernetes/disagg-communication-guide.md b/docs/kubernetes/disagg-communication-guide.md index c4e99b91b4e1..2102bc4acfd7 100644 --- a/docs/kubernetes/disagg-communication-guide.md +++ b/docs/kubernetes/disagg-communication-guide.md @@ -821,4 +821,3 @@ resources: - [Disaggregated Serving Architecture](../design-docs/disagg-serving.md) - [AIConfigurator Deployment Guide](../features/disaggregated-serving/README.md) - [NIXL Benchmark Deployment](../../deploy/pre-deployment/nixl/README.md) -- [KV Cache Transfer Methods](../backends/trtllm/trtllm-kv-cache-transfer.md) diff --git a/docs/observability/metrics-comparison.md b/docs/observability/metrics-comparison.md index caec04ad1a4d..a9716f80b919 100644 --- a/docs/observability/metrics-comparison.md +++ b/docs/observability/metrics-comparison.md @@ -12,7 +12,6 @@ For Dynamo's own runtime metrics (`dynamo_*`), see the [Metrics Guide](metrics.m - [vLLM Observability](../backends/vllm/vllm-observability.md) - [SGLang Observability](../backends/sglang/sglang-observability.md) -- [TensorRT-LLM Observability](../backends/trtllm/trtllm-observability.md) | Framework | Metric Prefix | Unique Metrics | Version Tested | Required Flags | |-----------|---------------|----------------|----------------|----------------| @@ -145,4 +144,3 @@ These are **pass-through metrics from the engines themselves** — Dynamo expose | | KV transfer total (MB) | - | `kv_transfer_total_mb` | - | | | KV transfer bytes | - | - | `kv_transfer_bytes` | | | KV transfer success | - | - | `kv_transfer_success_total` | - diff --git a/docs/observability/metrics.md b/docs/observability/metrics.md index 9f55650984c2..7d9c99e112ed 100644 --- a/docs/observability/metrics.md +++ b/docs/observability/metrics.md @@ -86,7 +86,7 @@ Dynamo exposes several categories of metrics: - **Frontend Metrics** (`dynamo_frontend_*`) - Request handling, token processing, and latency measurements - **Component Metrics** (`dynamo_component_*`) - Request counts, processing times, byte transfers, and system uptime - **Specialized Component Metrics** (e.g., `dynamo_preprocessor_*`) - Component-specific metrics -- **Engine Metrics** (Pass-through) - Backend engines expose their own metrics: [vLLM](../backends/vllm/vllm-observability.md) (`vllm:*`), [SGLang](../backends/sglang/sglang-observability.md) (`sglang:*`), [TensorRT-LLM](../backends/trtllm/trtllm-observability.md) (`trtllm_*`) +- **Engine Metrics** (Pass-through) - Backend engines expose their own metrics: [vLLM](../backends/vllm/vllm-observability.md) (`vllm:*`) and [SGLang](../backends/sglang/sglang-observability.md) (`sglang:*`) ## Runtime Hierarchy @@ -172,7 +172,7 @@ Values you will see in the `dynamo_component` label on `dynamo_component_*` seri | Value | Meaning | |-------|---------| | `router` | The standalone KV router (`python -m dingo.router`). | -| `Planner` | The planner component (`python -m dynamo.planner`). Note the capital `P`. | +| `Planner` | The planner component (`python -m dingo.planner`). Note the capital `P`. | | `prefill` | The prefill worker in disaggregated serving (all backends). | | `backend` | The decode worker in disaggregated serving for all backends, **and** the combined worker for vLLM in aggregated mode. | | `encode` | The encode worker for vLLM, SGLang, and TRT-LLM. | diff --git a/docs/observability/request-tracing.md b/docs/observability/request-tracing.md index 80ee06476d44..cadaceab3a64 100644 --- a/docs/observability/request-tracing.md +++ b/docs/observability/request-tracing.md @@ -169,7 +169,7 @@ these shape checks or create an agent-only fallback row. Pass `dynamo.request.trace.v1` JSONL or JSONL.GZ shards directly to replay: ```bash -python -m dynamo.replay /tmp/dynamo-request-trace.*.jsonl.gz \ +python -m dingo.replay /tmp/dynamo-request-trace.*.jsonl.gz \ --trace-format dynamo \ --replay-mode offline \ --router-mode kv_router \ diff --git a/docs/reference/feature-matrix.md b/docs/reference/feature-matrix.md index 1c21cd278b33..9d8b09d047c3 100644 --- a/docs/reference/feature-matrix.md +++ b/docs/reference/feature-matrix.md @@ -86,7 +86,7 @@ SGLang is optimized for high-throughput serving with fast primitives, providing TensorRT-LLM delivers maximum inference performance and optimization, with full KVBM integration and robust disaggregated serving support. -*Source: [docs/backends/trtllm/README.md][trtllm-readme]* +*Source: TensorRT-LLM backend documentation* | Feature | Disaggregated Serving | KV-Aware Routing | SLA-Based Planner | KV Block Manager | Multimodal | Request Migration | Request Cancellation | LoRA | Tool Calling | Speculative Decoding | | :--- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | @@ -102,7 +102,7 @@ TensorRT-LLM delivers maximum inference performance and optimization, with full | **Speculative Decoding** | ✅ | ✅ | — | ✅ | — | ✅ | ✅ | | ✅ | — | > **Notes:** -> 1. **Multimodal Disaggregation**: Supports **EP/D** (Traditional) and **E/P/D** (Full Disaggregation) image flows, including image URLs and pre-computed embeddings. ([Source][mm-trtllm]) +> 1. **Multimodal Disaggregation**: Supports **EP/D** (Traditional) and **E/P/D** (Full Disaggregation) image flows, including image URLs and pre-computed embeddings. > 2. **Multimodal + KV-Aware Routing**: Image-aware KV routing is supported through the dedicated TRT-LLM MM Router Worker. It requires KV event publishing on the TRT-LLM workers. ([Source][mm-kv-routing]) > 3. **Request Cancellation**: Due to known issues, the TensorRT-LLM engine is temporarily not notified of request cancellations, meaning allocated resources for cancelled requests are not freed. @@ -112,7 +112,6 @@ TensorRT-LLM delivers maximum inference performance and optimization, with full {/* Backend READMEs — paths relative to rendered URL /resources/feature-matrix */} [vllm-readme]: ../backends/v-llm [sglang-readme]: ../backends/sg-lang -[trtllm-readme]: ../backends/tensor-rt-llm {/* Design Docs */} [disagg]: ../design-docs/disaggregated-serving @@ -125,7 +124,6 @@ TensorRT-LLM delivers maximum inference performance and optimization, with full {/* Multimodal */} [mm]: ../user-guides/multimodal [mm-vllm]: ../features/multimodal/multimodal-vllm.md -[mm-trtllm]: ../features/multimodal/multimodal-trtllm.md [mm-sglang]: ../features/multimodal/multimodal-sglang.md [mm-kv-routing]: ../features/multimodal/multimodal-kv-routing.md diff --git a/docs/reference/release-artifacts.md b/docs/reference/release-artifacts.md index d2de3308d5ca..46ea6c573a72 100644 --- a/docs/reference/release-artifacts.md +++ b/docs/reference/release-artifacts.md @@ -77,7 +77,7 @@ We recommend using the TensorRT-LLM NGC container instead of the `ai-dynamo[trtl ### Container Images (NGC) > [!TIP] -> For detailed run instructions, see the backend-specific guides: [vLLM](../backends/vllm/README.md) | [SGLang](../backends/sglang/README.md) | [TensorRT-LLM](../backends/trtllm/README.md) +> For detailed run instructions, see the backend-specific guides: [vLLM](../backends/vllm/README.md) | [SGLang](../backends/sglang/README.md) ```bash # Runtime containers diff --git a/examples/backends/mocker/deploy/agg.yaml b/examples/backends/mocker/deploy/agg.yaml index 1cb1c505c993..be7010c7eb74 100644 --- a/examples/backends/mocker/deploy/agg.yaml +++ b/examples/backends/mocker/deploy/agg.yaml @@ -3,7 +3,7 @@ # NOTE: There is no dedicated `mocker-runtime` image. The published # `dynamo-planner` image includes the ai-dynamo wheel, ai-dynamo-runtime, and -# planner/profiler dependencies needed by `python3 -m dynamo.mocker`. +# planner/profiler dependencies needed by `python3 -m dingo.mocker`. # Replace `my-tag` below with the Dynamo release tag you deploy. apiVersion: nvidia.com/v1alpha1 kind: DynamoGraphDeployment @@ -29,7 +29,7 @@ spec: command: - python3 - -m - - dynamo.mocker + - dingo.mocker args: - --model-path - nvidia/Llama-3.1-8B-Instruct-FP8 @@ -38,4 +38,4 @@ spec: - --speedup-ratio - "1.0" - --planner-profile-data - - /workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D + - /workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D diff --git a/examples/backends/mocker/deploy/disagg.yaml b/examples/backends/mocker/deploy/disagg.yaml index 42d46bdad6c7..19cd7aea1e06 100644 --- a/examples/backends/mocker/deploy/disagg.yaml +++ b/examples/backends/mocker/deploy/disagg.yaml @@ -3,7 +3,7 @@ # NOTE: There is no dedicated `mocker-runtime` image. The published # `dynamo-planner` image includes the ai-dynamo wheel, ai-dynamo-runtime, and -# planner/profiler dependencies needed by `python3 -m dynamo.mocker`. +# planner/profiler dependencies needed by `python3 -m dingo.mocker`. # Replace `my-tag` below with the Dynamo release tag you deploy. apiVersion: nvidia.com/v1alpha1 kind: DynamoGraphDeployment @@ -29,7 +29,7 @@ spec: command: - python3 - -m - - dynamo.mocker + - dingo.mocker args: - --model-path - nvidia/Llama-3.1-8B-Instruct-FP8 @@ -38,7 +38,7 @@ spec: - --speedup-ratio - "1.0" - --planner-profile-data - - /workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D + - /workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D - --disaggregation-mode - prefill decode: @@ -53,7 +53,7 @@ spec: command: - python3 - -m - - dynamo.mocker + - dingo.mocker args: - --model-path - nvidia/Llama-3.1-8B-Instruct-FP8 @@ -62,6 +62,6 @@ spec: - --speedup-ratio - "1.0" - --planner-profile-data - - /workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D + - /workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D - --disaggregation-mode - decode diff --git a/examples/backends/mocker/deploy/v1beta1/agg.yaml b/examples/backends/mocker/deploy/v1beta1/agg.yaml index 4493a166e345..60e79db05e87 100644 --- a/examples/backends/mocker/deploy/v1beta1/agg.yaml +++ b/examples/backends/mocker/deploy/v1beta1/agg.yaml @@ -3,7 +3,7 @@ # NOTE: There is no dedicated `mocker-runtime` image. The published # `dynamo-planner` image includes the ai-dynamo wheel, ai-dynamo-runtime, and -# planner/profiler dependencies needed by `python3 -m dynamo.mocker`. +# planner/profiler dependencies needed by `python3 -m dingo.mocker`. # Replace `my-tag` below with the Dynamo release tag you deploy. apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment @@ -31,11 +31,11 @@ spec: - --speedup-ratio - "1.0" - --planner-profile-data - - /workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D + - /workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D command: - python3 - -m - - dynamo.mocker + - dingo.mocker envFrom: - secretRef: name: hf-token-secret diff --git a/examples/backends/mocker/deploy/v1beta1/disagg.yaml b/examples/backends/mocker/deploy/v1beta1/disagg.yaml index 32b46039abaf..ddb40b8608da 100644 --- a/examples/backends/mocker/deploy/v1beta1/disagg.yaml +++ b/examples/backends/mocker/deploy/v1beta1/disagg.yaml @@ -3,7 +3,7 @@ # NOTE: There is no dedicated `mocker-runtime` image. The published # `dynamo-planner` image includes the ai-dynamo wheel, ai-dynamo-runtime, and -# planner/profiler dependencies needed by `python3 -m dynamo.mocker`. +# planner/profiler dependencies needed by `python3 -m dingo.mocker`. # Replace `my-tag` below with the Dynamo release tag you deploy. apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment @@ -31,13 +31,13 @@ spec: - --speedup-ratio - "1.0" - --planner-profile-data - - /workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D + - /workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D - --disaggregation-mode - decode command: - python3 - -m - - dynamo.mocker + - dingo.mocker envFrom: - secretRef: name: hf-token-secret @@ -58,13 +58,13 @@ spec: - --speedup-ratio - "1.0" - --planner-profile-data - - /workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D + - /workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D - --disaggregation-mode - prefill command: - python3 - -m - - dynamo.mocker + - dingo.mocker envFrom: - secretRef: name: hf-token-secret diff --git a/examples/backends/sample/launch/agg.sh b/examples/backends/sample/launch/agg.sh index 1768c0dddf2a..33e048ce33d6 100755 --- a/examples/backends/sample/launch/agg.sh +++ b/examples/backends/sample/launch/agg.sh @@ -44,7 +44,7 @@ print_launch_banner "Launching Sample Aggregated Serving" "$MODEL_NAME" "$HTTP_P python3 -m dingo.frontend & # run sample worker -python3 -m dynamo.common.backend.sample_main \ +python3 -m dingo.common.backend.sample_main \ --model-name "$MODEL_NAME" \ "${EXTRA_ARGS[@]}" & diff --git a/examples/backends/sample/launch/agg_diffusion.sh b/examples/backends/sample/launch/agg_diffusion.sh index aaf61d0a7fa3..76591ee56ecf 100755 --- a/examples/backends/sample/launch/agg_diffusion.sh +++ b/examples/backends/sample/launch/agg_diffusion.sh @@ -46,7 +46,7 @@ print_launch_banner "Launching Sample Diffusion Aggregated Serving" "$MODEL_NAME python3 -m dingo.frontend & # run sample diffusion worker (serves /v1/images/generations) -python3 -m dynamo.common.backend.sample_diffusion_main \ +python3 -m dingo.common.backend.sample_diffusion_main \ --model-name "$MODEL_NAME" \ --endpoint-types images \ "${EXTRA_ARGS[@]}" & diff --git a/examples/backends/sample/launch/disagg.sh b/examples/backends/sample/launch/disagg.sh index de81f6d1deef..3441500b9127 100755 --- a/examples/backends/sample/launch/disagg.sh +++ b/examples/backends/sample/launch/disagg.sh @@ -59,7 +59,7 @@ python3 -m dingo.frontend & # Distinct --component name keeps the two workers visible separately in # discovery, mirroring the per-role components in vLLM/SGLang/TRT-LLM. DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT1:-8081} \ -python3 -m dynamo.common.backend.sample_main \ +python3 -m dingo.common.backend.sample_main \ --model-name "$MODEL_NAME" \ --component sample-prefill \ --disaggregation-mode prefill \ @@ -67,7 +67,7 @@ python3 -m dynamo.common.backend.sample_main \ # run decode worker DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT2:-8082} \ -python3 -m dynamo.common.backend.sample_main \ +python3 -m dingo.common.backend.sample_main \ --model-name "$MODEL_NAME" \ --component sample-decode \ --disaggregation-mode decode \ diff --git a/examples/backends/sample/launch/multimodal_agg.sh b/examples/backends/sample/launch/multimodal_agg.sh index 83a4a1cdedc0..5530a4a4aa8f 100755 --- a/examples/backends/sample/launch/multimodal_agg.sh +++ b/examples/backends/sample/launch/multimodal_agg.sh @@ -43,7 +43,7 @@ done echo "Running direct aggregated-worker multimodal handoff smoke with $MODEL_NAME" DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT:-8081} \ -python3 -m dynamo.common.backend.sample_main \ +python3 -m dingo.common.backend.sample_main \ --model-name "$MODEL_NAME" \ --namespace "$NAMESPACE" \ --component "$COMPONENT" \ diff --git a/examples/backends/sample/launch/multimodal_disagg.sh b/examples/backends/sample/launch/multimodal_disagg.sh index fd5d3f70556e..66ab300e7ce8 100755 --- a/examples/backends/sample/launch/multimodal_disagg.sh +++ b/examples/backends/sample/launch/multimodal_disagg.sh @@ -45,7 +45,7 @@ done echo "Running direct Encode -> Prefill -> Decode multimodal handoff smoke with $MODEL_NAME" DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT1:-8081} \ -python3 -m dynamo.common.backend.sample_main \ +python3 -m dingo.common.backend.sample_main \ --model-name "$MODEL_NAME" \ --namespace "$NAMESPACE" \ --component "$ENCODE_COMPONENT" \ @@ -54,7 +54,7 @@ python3 -m dynamo.common.backend.sample_main \ "${EXTRA_ARGS[@]}" & DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT2:-8082} \ -python3 -m dynamo.common.backend.sample_main \ +python3 -m dingo.common.backend.sample_main \ --model-name "$MODEL_NAME" \ --namespace "$NAMESPACE" \ --component "$PREFILL_COMPONENT" \ @@ -64,7 +64,7 @@ python3 -m dynamo.common.backend.sample_main \ "${EXTRA_ARGS[@]}" & DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT3:-8083} \ -python3 -m dynamo.common.backend.sample_main \ +python3 -m dingo.common.backend.sample_main \ --model-name "$MODEL_NAME" \ --namespace "$NAMESPACE" \ --component "$DECODE_COMPONENT" \ diff --git a/examples/backends/sglang/deploy/disagg_planner.yaml b/examples/backends/sglang/deploy/disagg_planner.yaml index 421f19eb26a0..64bbf531920b 100644 --- a/examples/backends/sglang/deploy/disagg_planner.yaml +++ b/examples/backends/sglang/deploy/disagg_planner.yaml @@ -30,7 +30,7 @@ spec: command: - python3 - -m - - dynamo.planner + - dingo.planner args: - --config - '{"environment": "kubernetes", "backend": "sglang", "optimization_target": "sla", "enable_throughput_scaling": true, "enable_load_scaling": true, "pre_deployment_sweeping_mode": "none", "throughput_adjustment_interval_seconds": 60, "load_adjustment_interval_seconds": 5}' diff --git a/examples/backends/sglang/deploy/v1beta1/disagg_planner.yaml b/examples/backends/sglang/deploy/v1beta1/disagg_planner.yaml index d4a3093dcdaf..934aec59d436 100644 --- a/examples/backends/sglang/deploy/v1beta1/disagg_planner.yaml +++ b/examples/backends/sglang/deploy/v1beta1/disagg_planner.yaml @@ -27,7 +27,7 @@ spec: command: - python3 - -m - - dynamo.planner + - dingo.planner envFrom: - secretRef: name: hf-token-secret diff --git a/examples/backends/vllm/deploy/disagg_planner.yaml b/examples/backends/vllm/deploy/disagg_planner.yaml index 91360b1ba21e..fda0f3cbef7a 100644 --- a/examples/backends/vllm/deploy/disagg_planner.yaml +++ b/examples/backends/vllm/deploy/disagg_planner.yaml @@ -29,7 +29,7 @@ spec: command: - python3 - -m - - dynamo.planner + - dingo.planner args: - --config - '{"environment": "kubernetes", "backend": "vllm", "optimization_target": "sla", "enable_throughput_scaling": true, "enable_load_scaling": true, "pre_deployment_sweeping_mode": "none", "throughput_adjustment_interval_seconds": 60, "load_adjustment_interval_seconds": 5}' diff --git a/examples/backends/vllm/deploy/v1beta1/disagg_planner.yaml b/examples/backends/vllm/deploy/v1beta1/disagg_planner.yaml index 86f03de961e7..6c80a4dc076e 100644 --- a/examples/backends/vllm/deploy/v1beta1/disagg_planner.yaml +++ b/examples/backends/vllm/deploy/v1beta1/disagg_planner.yaml @@ -27,7 +27,7 @@ spec: command: - python3 - -m - - dynamo.planner + - dingo.planner # Planner image selection: # Dynamo >= 1.1.0: use the dedicated planner image # nvcr.io/nvidia/ai-dynamo/dynamo-planner: diff --git a/examples/global_planner/README.md b/examples/global_planner/README.md index 4736b2b1c47b..21aecbaa3b46 100644 --- a/examples/global_planner/README.md +++ b/examples/global_planner/README.md @@ -161,6 +161,6 @@ This is why planner configs and router endpoints use the full `${K8S_NAMESPACE}- ## Further Reading - [Global Planner Deployment Guide](../../docs/components/planner/global-planner.md) -- [Global Planner README](../../components/src/dynamo/global_planner/README.md) +- [Global Planner README](../../dingo/global_planner/README.md) - [Planner Configuration Guide](../../docs/components/planner/planner-guide.md) -- [Global Router README](../../components/src/dynamo/global_router/README.md) +- [Global Router README](../../dingo/global_router/README.md) diff --git a/examples/global_planner/global-planner-gpu-budget.yaml b/examples/global_planner/global-planner-gpu-budget.yaml index 9ab807ed2376..f71ad5df6823 100644 --- a/examples/global_planner/global-planner-gpu-budget.yaml +++ b/examples/global_planner/global-planner-gpu-budget.yaml @@ -70,7 +70,7 @@ spec: command: - python3 - -m - - dynamo.global_planner + - dingo.global_planner args: - --max-total-gpus - "${MAX_TOTAL_GPUS}" @@ -165,10 +165,10 @@ spec: command: - python3 - -m - - dynamo.planner + - dingo.planner args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"disagg","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","ttft_ms":2000,"itl_ms":200,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"decode_engine_num_gpu":1,"model_name":"${MODEL_A}","profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"disagg","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","ttft_ms":2000,"itl_ms":200,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"decode_engine_num_gpu":1,"model_name":"${MODEL_A}","profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' --- # ── Model B: self-contained disagg serving DGD ────────────────────────────── apiVersion: nvidia.com/v1alpha1 @@ -260,7 +260,7 @@ spec: command: - python3 - -m - - dynamo.planner + - dingo.planner args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"disagg","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","ttft_ms":2000,"itl_ms":200,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"decode_engine_num_gpu":1,"model_name":"${MODEL_B}","profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"disagg","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","ttft_ms":2000,"itl_ms":200,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"decode_engine_num_gpu":1,"model_name":"${MODEL_B}","profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' diff --git a/examples/global_planner/global-planner-mocker-test.yaml b/examples/global_planner/global-planner-mocker-test.yaml index 78fb377a73e4..e7baf568507f 100644 --- a/examples/global_planner/global-planner-mocker-test.yaml +++ b/examples/global_planner/global-planner-mocker-test.yaml @@ -87,7 +87,7 @@ spec: command: - python3 - -m - - dynamo.global_router + - dingo.global_router args: - --config - /config/global_router_config.json @@ -111,7 +111,7 @@ spec: command: - python3 - -m - - dynamo.global_planner + - dingo.global_planner args: - --no-operation --- @@ -155,7 +155,7 @@ spec: command: - python3 - -m - - dynamo.mocker + - dingo.mocker args: - --model-path - nvidia/Llama-3.1-8B-Instruct-FP8 @@ -164,7 +164,7 @@ spec: - --speedup-ratio - "5.0" - --planner-profile-data - - /workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D + - /workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D - --is-prefill-worker Planner: @@ -178,10 +178,10 @@ spec: command: - python3 - -m - - dynamo.planner + - dingo.planner args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"mocker","mode":"prefill","optimization_target":"sla","throughput_metrics_source":"router","throughput_adjustment_interval_seconds":30,"ttft_ms":2000,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D","model_name":"nvidia/Llama-3.1-8B-Instruct-FP8"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"mocker","mode":"prefill","optimization_target":"sla","throughput_metrics_source":"router","throughput_adjustment_interval_seconds":30,"ttft_ms":2000,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D","model_name":"nvidia/Llama-3.1-8B-Instruct-FP8"}' --- apiVersion: nvidia.com/v1alpha1 kind: DynamoGraphDeployment @@ -223,7 +223,7 @@ spec: command: - python3 - -m - - dynamo.mocker + - dingo.mocker args: - --model-path - nvidia/Llama-3.1-8B-Instruct-FP8 @@ -232,7 +232,7 @@ spec: - --speedup-ratio - "5.0" - --planner-profile-data - - /workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D + - /workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D - --is-prefill-worker Planner: @@ -246,10 +246,10 @@ spec: command: - python3 - -m - - dynamo.planner + - dingo.planner args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"mocker","mode":"prefill","optimization_target":"sla","throughput_metrics_source":"router","throughput_adjustment_interval_seconds":30,"ttft_ms":2000,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D","model_name":"nvidia/Llama-3.1-8B-Instruct-FP8"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"mocker","mode":"prefill","optimization_target":"sla","throughput_metrics_source":"router","throughput_adjustment_interval_seconds":30,"ttft_ms":2000,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D","model_name":"nvidia/Llama-3.1-8B-Instruct-FP8"}' --- apiVersion: nvidia.com/v1alpha1 kind: DynamoGraphDeployment @@ -292,7 +292,7 @@ spec: command: - python3 - -m - - dynamo.mocker + - dingo.mocker args: - --model-path - nvidia/Llama-3.1-8B-Instruct-FP8 @@ -301,7 +301,7 @@ spec: - --speedup-ratio - "5.0" - --planner-profile-data - - /workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D + - /workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D Planner: componentType: planner @@ -314,10 +314,10 @@ spec: command: - python3 - -m - - dynamo.planner + - dingo.planner args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"mocker","mode":"decode","optimization_target":"sla","throughput_metrics_source":"router","throughput_adjustment_interval_seconds":30,"itl_ms":200,"max_gpu_budget":-1,"decode_engine_num_gpu":1,"profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D","model_name":"nvidia/Llama-3.1-8B-Instruct-FP8"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"mocker","mode":"decode","optimization_target":"sla","throughput_metrics_source":"router","throughput_adjustment_interval_seconds":30,"itl_ms":200,"max_gpu_budget":-1,"decode_engine_num_gpu":1,"profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D","model_name":"nvidia/Llama-3.1-8B-Instruct-FP8"}' --- apiVersion: nvidia.com/v1alpha1 kind: DynamoGraphDeployment @@ -360,7 +360,7 @@ spec: command: - python3 - -m - - dynamo.mocker + - dingo.mocker args: - --model-path - nvidia/Llama-3.1-8B-Instruct-FP8 @@ -369,7 +369,7 @@ spec: - --speedup-ratio - "5.0" - --planner-profile-data - - /workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D + - /workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D Planner: componentType: planner @@ -382,7 +382,7 @@ spec: command: - python3 - -m - - dynamo.planner + - dingo.planner args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"mocker","mode":"decode","optimization_target":"sla","throughput_metrics_source":"router","throughput_adjustment_interval_seconds":30,"itl_ms":200,"max_gpu_budget":-1,"decode_engine_num_gpu":1,"profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D","model_name":"nvidia/Llama-3.1-8B-Instruct-FP8"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"mocker","mode":"decode","optimization_target":"sla","throughput_metrics_source":"router","throughput_adjustment_interval_seconds":30,"itl_ms":200,"max_gpu_budget":-1,"decode_engine_num_gpu":1,"profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D","model_name":"nvidia/Llama-3.1-8B-Instruct-FP8"}' diff --git a/examples/global_planner/global-planner-vllm-test.yaml b/examples/global_planner/global-planner-vllm-test.yaml index 44ed62d68d19..08add45ab084 100644 --- a/examples/global_planner/global-planner-vllm-test.yaml +++ b/examples/global_planner/global-planner-vllm-test.yaml @@ -134,7 +134,7 @@ spec: command: - python3 - -m - - dynamo.global_router + - dingo.global_router args: - --config - /config/global_router_config.json @@ -158,7 +158,7 @@ spec: command: - python3 - -m - - dynamo.global_planner + - dingo.global_planner --- apiVersion: nvidia.com/v1alpha1 kind: DynamoGraphDeployment @@ -230,10 +230,10 @@ spec: command: - python3 - -m - - dynamo.planner + - dingo.planner args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"prefill","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","ttft_ms":2000,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"model_name":"${MODEL_NAME}","profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"prefill","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","ttft_ms":2000,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"model_name":"${MODEL_NAME}","profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' --- apiVersion: nvidia.com/v1alpha1 kind: DynamoGraphDeployment @@ -305,10 +305,10 @@ spec: command: - python3 - -m - - dynamo.planner + - dingo.planner args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"prefill","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","ttft_ms":2000,"max_gpu_budget":-1,"prefill_engine_num_gpu":2,"model_name":"${MODEL_NAME}","profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"prefill","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","ttft_ms":2000,"max_gpu_budget":-1,"prefill_engine_num_gpu":2,"model_name":"${MODEL_NAME}","profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' --- apiVersion: nvidia.com/v1alpha1 kind: DynamoGraphDeployment @@ -380,7 +380,7 @@ spec: command: - python3 - -m - - dynamo.planner + - dingo.planner args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"decode","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","itl_ms":200,"max_gpu_budget":-1,"decode_engine_num_gpu":1,"model_name":"${MODEL_NAME}","profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"decode","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","itl_ms":200,"max_gpu_budget":-1,"decode_engine_num_gpu":1,"model_name":"${MODEL_NAME}","profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' diff --git a/examples/global_planner/v1beta1/global-planner-gpu-budget.yaml b/examples/global_planner/v1beta1/global-planner-gpu-budget.yaml index de33789ada62..0caa94057c65 100644 --- a/examples/global_planner/v1beta1/global-planner-gpu-budget.yaml +++ b/examples/global_planner/v1beta1/global-planner-gpu-budget.yaml @@ -71,7 +71,7 @@ spec: command: - python3 - -m - - dynamo.global_planner + - dingo.global_planner image: ${DYNAMO_IMAGE} name: main replicas: 1 @@ -105,11 +105,11 @@ spec: containers: - args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"disagg","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","ttft_ms":2000,"itl_ms":200,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"decode_engine_num_gpu":1,"model_name":"${MODEL_A}","profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"disagg","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","ttft_ms":2000,"itl_ms":200,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"decode_engine_num_gpu":1,"model_name":"${MODEL_A}","profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' command: - python3 - -m - - dynamo.planner + - dingo.planner image: ${DYNAMO_IMAGE} name: main replicas: 1 @@ -206,11 +206,11 @@ spec: containers: - args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"disagg","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","ttft_ms":2000,"itl_ms":200,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"decode_engine_num_gpu":1,"model_name":"${MODEL_B}","profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"disagg","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","ttft_ms":2000,"itl_ms":200,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"decode_engine_num_gpu":1,"model_name":"${MODEL_B}","profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' command: - python3 - -m - - dynamo.planner + - dingo.planner image: ${DYNAMO_IMAGE} name: main replicas: 1 diff --git a/examples/global_planner/v1beta1/global-planner-mocker-test.yaml b/examples/global_planner/v1beta1/global-planner-mocker-test.yaml index 23624f9a7aa8..1a4876207a8e 100644 --- a/examples/global_planner/v1beta1/global-planner-mocker-test.yaml +++ b/examples/global_planner/v1beta1/global-planner-mocker-test.yaml @@ -81,7 +81,7 @@ spec: command: - python3 - -m - - dynamo.global_planner + - dingo.global_planner image: ${DYNAMO_IMAGE} name: main imagePullSecrets: @@ -102,7 +102,7 @@ spec: command: - python3 - -m - - dynamo.global_router + - dingo.global_router image: ${DYNAMO_IMAGE} name: main volumeMounts: @@ -157,12 +157,12 @@ spec: - --speedup-ratio - "5.0" - --planner-profile-data - - /workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D + - /workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D - --is-prefill-worker command: - python3 - -m - - dynamo.mocker + - dingo.mocker image: ${DYNAMO_IMAGE} name: main workingDir: /workspace @@ -176,11 +176,11 @@ spec: containers: - args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"mocker","mode":"prefill","optimization_target":"sla","throughput_metrics_source":"router","throughput_adjustment_interval_seconds":30,"ttft_ms":2000,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D","model_name":"nvidia/Llama-3.1-8B-Instruct-FP8"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"mocker","mode":"prefill","optimization_target":"sla","throughput_metrics_source":"router","throughput_adjustment_interval_seconds":30,"ttft_ms":2000,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D","model_name":"nvidia/Llama-3.1-8B-Instruct-FP8"}' command: - python3 - -m - - dynamo.planner + - dingo.planner image: ${DYNAMO_IMAGE} name: main imagePullSecrets: @@ -227,12 +227,12 @@ spec: - --speedup-ratio - "5.0" - --planner-profile-data - - /workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D + - /workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D - --is-prefill-worker command: - python3 - -m - - dynamo.mocker + - dingo.mocker image: ${DYNAMO_IMAGE} name: main workingDir: /workspace @@ -246,11 +246,11 @@ spec: containers: - args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"mocker","mode":"prefill","optimization_target":"sla","throughput_metrics_source":"router","throughput_adjustment_interval_seconds":30,"ttft_ms":2000,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D","model_name":"nvidia/Llama-3.1-8B-Instruct-FP8"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"mocker","mode":"prefill","optimization_target":"sla","throughput_metrics_source":"router","throughput_adjustment_interval_seconds":30,"ttft_ms":2000,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D","model_name":"nvidia/Llama-3.1-8B-Instruct-FP8"}' command: - python3 - -m - - dynamo.planner + - dingo.planner image: ${DYNAMO_IMAGE} name: main imagePullSecrets: @@ -298,11 +298,11 @@ spec: - --speedup-ratio - "5.0" - --planner-profile-data - - /workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D + - /workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D command: - python3 - -m - - dynamo.mocker + - dingo.mocker image: ${DYNAMO_IMAGE} name: main workingDir: /workspace @@ -316,11 +316,11 @@ spec: containers: - args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"mocker","mode":"decode","optimization_target":"sla","throughput_metrics_source":"router","throughput_adjustment_interval_seconds":30,"itl_ms":200,"max_gpu_budget":-1,"decode_engine_num_gpu":1,"profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D","model_name":"nvidia/Llama-3.1-8B-Instruct-FP8"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"mocker","mode":"decode","optimization_target":"sla","throughput_metrics_source":"router","throughput_adjustment_interval_seconds":30,"itl_ms":200,"max_gpu_budget":-1,"decode_engine_num_gpu":1,"profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D","model_name":"nvidia/Llama-3.1-8B-Instruct-FP8"}' command: - python3 - -m - - dynamo.planner + - dingo.planner image: ${DYNAMO_IMAGE} name: main imagePullSecrets: @@ -368,11 +368,11 @@ spec: - --speedup-ratio - "5.0" - --planner-profile-data - - /workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D + - /workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D command: - python3 - -m - - dynamo.mocker + - dingo.mocker image: ${DYNAMO_IMAGE} name: main workingDir: /workspace @@ -386,11 +386,11 @@ spec: containers: - args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"mocker","mode":"decode","optimization_target":"sla","throughput_metrics_source":"router","throughput_adjustment_interval_seconds":30,"itl_ms":200,"max_gpu_budget":-1,"decode_engine_num_gpu":1,"profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D","model_name":"nvidia/Llama-3.1-8B-Instruct-FP8"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"mocker","mode":"decode","optimization_target":"sla","throughput_metrics_source":"router","throughput_adjustment_interval_seconds":30,"itl_ms":200,"max_gpu_budget":-1,"decode_engine_num_gpu":1,"profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D","model_name":"nvidia/Llama-3.1-8B-Instruct-FP8"}' command: - python3 - -m - - dynamo.planner + - dingo.planner image: ${DYNAMO_IMAGE} name: main imagePullSecrets: diff --git a/examples/global_planner/v1beta1/global-planner-vllm-test.yaml b/examples/global_planner/v1beta1/global-planner-vllm-test.yaml index 28063450646b..d16b8501e28f 100644 --- a/examples/global_planner/v1beta1/global-planner-vllm-test.yaml +++ b/examples/global_planner/v1beta1/global-planner-vllm-test.yaml @@ -114,7 +114,7 @@ spec: - command: - python3 - -m - - dynamo.global_planner + - dingo.global_planner image: ${DYNAMO_IMAGE} name: main imagePullSecrets: @@ -145,7 +145,7 @@ spec: command: - python3 - -m - - dynamo.global_router + - dingo.global_router image: ${DYNAMO_IMAGE} name: main volumeMounts: @@ -200,11 +200,11 @@ spec: containers: - args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"prefill","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","ttft_ms":2000,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"model_name":"${MODEL_NAME}","profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"prefill","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","ttft_ms":2000,"max_gpu_budget":-1,"prefill_engine_num_gpu":1,"model_name":"${MODEL_NAME}","profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' command: - python3 - -m - - dynamo.planner + - dingo.planner image: ${DYNAMO_IMAGE} name: main imagePullSecrets: @@ -281,11 +281,11 @@ spec: containers: - args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"prefill","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","ttft_ms":2000,"max_gpu_budget":-1,"prefill_engine_num_gpu":2,"model_name":"${MODEL_NAME}","profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"prefill","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","ttft_ms":2000,"max_gpu_budget":-1,"prefill_engine_num_gpu":2,"model_name":"${MODEL_NAME}","profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' command: - python3 - -m - - dynamo.planner + - dingo.planner image: ${DYNAMO_IMAGE} name: main imagePullSecrets: @@ -363,11 +363,11 @@ spec: containers: - args: - --config - - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"decode","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","itl_ms":200,"max_gpu_budget":-1,"decode_engine_num_gpu":1,"model_name":"${MODEL_NAME}","profile_results_dir":"/workspace/components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' + - '{"environment":"global-planner","global_planner_namespace":"${K8S_NAMESPACE}-gp-ctrl","backend":"vllm","mode":"decode","optimization_target":"sla","enable_load_scaling":false,"enable_throughput_scaling":true,"throughput_metrics_source":"router","itl_ms":200,"max_gpu_budget":-1,"decode_engine_num_gpu":1,"model_name":"${MODEL_NAME}","profile_results_dir":"/workspace/dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D"}' command: - python3 - -m - - dynamo.planner + - dingo.planner image: ${DYNAMO_IMAGE} name: main imagePullSecrets: diff --git a/fern/components/profiler/profiler_guide.md b/fern/components/profiler/profiler_guide.md index 8ef9c4111b8c..e911300efc3e 100644 --- a/fern/components/profiler/profiler_guide.md +++ b/fern/components/profiler/profiler_guide.md @@ -588,4 +588,4 @@ kubectl create secret docker-registry nvcr-imagepullsecret \ - [DGDR Examples](../../../docs/components/profiler/profiler-examples.md) - Complete DGDR YAML examples - [DGDR API Reference](/docs/kubernetes/api-reference.md) - DGDR specification -- [Profiler Arguments Reference](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py) - Full Configuration Reference +- [Profiler Arguments Reference](https://github.com/ai-dynamo/dynamo/blob/main/dingo/profiler/utils/dgdr_v1beta1_types.py) - Full Configuration Reference diff --git a/hatch_build.py b/hatch_build.py index f9b99da0e7db..ed72f760ee69 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -24,12 +24,25 @@ def get_components(): if os.path.isdir(item_path) and not item.startswith("."): components.append(item_path) - if not components: - raise RuntimeError(f"No components found in directory: {components_dir}") + for component_name in ( + "common", + "frontend", + "global_planner", + "global_router", + "mocker", + "planner", + "profiler", + "replay", + "thunderagent_router", + ): + dingo_component = os.path.join(root, "dingo", component_name) + if os.path.isdir(dingo_component): + components.append(dingo_component) - dingo_frontend = os.path.join(root, "dingo", "frontend") - if os.path.isdir(dingo_frontend): - components.append(dingo_frontend) + if not components: + raise RuntimeError( + f"No components found in {components_dir} or {os.path.join(root, 'dingo')}" + ) return components diff --git a/lib/backend-common/CLAUDE.md b/lib/backend-common/CLAUDE.md index c0dd900a3b63..2ae3d8119479 100644 --- a/lib/backend-common/CLAUDE.md +++ b/lib/backend-common/CLAUDE.md @@ -230,7 +230,7 @@ aggregates it when present. `usage(prompt, completion)` computes - **Non-generic `Worker`, `EngineAdapter`, and `run()`.** All hold `Arc`. This is load-bearing for the PyO3 path: - `components/src/dynamo/common/backend/worker.py` is a thin shim + `dingo/common/backend/worker.py` is a thin shim over `dynamo._core.backend.Worker` (this crate), so Python engines plug in through the same `Arc` slot via a `PyLLMEngine` adapter. @@ -567,7 +567,7 @@ Also available: `testing::mock_context()` and Shipped: `dynamo._core.backend.Worker` is a PyO3 binding that hands a `PyLLMEngine` (Python-implemented) into the same `Arc` slot the Rust path uses. The Python-side -`dynamo.common.backend.Worker` (`components/src/dynamo/common/backend/worker.py`) +`dingo.common.backend.Worker` (`dingo/common/backend/worker.py`) is a thin wrapper that drives this. The lifecycle state machine, signal handling, and graceful-shutdown orchestrator live entirely in this crate — Python adds no lifecycle logic. diff --git a/lib/backend-common/README.md b/lib/backend-common/README.md index 55bb68443da7..0b68b9184848 100644 --- a/lib/backend-common/README.md +++ b/lib/backend-common/README.md @@ -13,9 +13,9 @@ SPDX-License-Identifier: Apache-2.0 > logprobs. Multimodal, diffusion (image/video/DLLM), LoRA, engine > routes (pause/resume, profiling, weight updates), text-in-text-out, > and snapshot/CRIU are still on the non-unified path. See the -> [Python package README](../../components/src/dynamo/common/backend/README.md#feature-gaps) +> [Python package README](../../dingo/common/backend/README.md#feature-gaps) > for the per-engine matrix. The Python `Worker` -> ([`dynamo.common.backend`](../../components/src/dynamo/common/backend/)) +> ([`dingo.common.backend`](../../dingo/common/backend/)) > is a thin shim over this crate. > **Looking for a walkthrough?** Start with @@ -417,7 +417,7 @@ lib/backend-common/ The Python `Worker` shim that drives this crate from `dynamo.*.unified_main` entry points lives at -[`components/src/dynamo/common/backend/worker.py`](../../components/src/dynamo/common/backend/worker.py). +[`dingo/common/backend/worker.py`](../../dingo/common/backend/worker.py). ## See Also @@ -426,7 +426,7 @@ entry points lives at - [`CLAUDE.md`](CLAUDE.md) — design notes (rationale, invariants, Phase 2 PyO3 plans). - [Mocker example](examples/mocker/) — reference engine + compose stack. -- [Python sibling](../../components/src/dynamo/common/backend/README.md) - — `dynamo.common.backend`, the Python ABC layered over this crate. +- [Python sibling](../../dingo/common/backend/README.md) + — `dingo.common.backend`, the Python ABC layered over this crate. - [DEP #8251](https://github.com/ai-dynamo/dynamo/issues/8251) — Backend Interface proposal and ongoing status. diff --git a/lib/backend-common/src/worker.rs b/lib/backend-common/src/worker.rs index 43c0c055eb9e..2f316096be85 100644 --- a/lib/backend-common/src/worker.rs +++ b/lib/backend-common/src/worker.rs @@ -403,7 +403,7 @@ impl Worker { /// cannot be reused after `run()` returns. /// /// Shutdown sequence (mirrors `graceful_shutdown_with_discovery` in - /// `components/src/dynamo/common/utils/graceful_shutdown.py`): + /// `dingo/common/utils/graceful_shutdown.py`): /// 1. `endpoint.unregister_endpoint_instance()` — router stops routing. /// 2. Sleep `DYN_GRACEFUL_SHUTDOWN_GRACE_PERIOD_SECS` (default 5s) to /// let in-flight router decisions complete. diff --git a/lib/bindings/python/rust/backend.rs b/lib/bindings/python/rust/backend.rs index c62d27998284..155fc3b103bf 100644 --- a/lib/bindings/python/rust/backend.rs +++ b/lib/bindings/python/rust/backend.rs @@ -536,7 +536,7 @@ impl Worker { // Initialize logging now that tokio context is available. Mirrors // the DistributedRuntime init path — required so workers using - // `dynamo.common.backend.Worker` directly (without constructing a + // `dingo.common.backend.Worker` directly (without constructing a // DistributedRuntime first) install the tracing + OTLP exporter // layers. Without this, OTEL_EXPORT_ENABLED workers emit no // logs and no spans. @@ -791,7 +791,7 @@ impl PyEngineCore { let bound = result.bind(py); // Accept either the Rust EngineConfig pyclass or any Python // object exposing the canonical attribute names (e.g. the - // `dynamo.common.backend.EngineConfig` dataclass). + // `dingo.common.backend.EngineConfig` dataclass). if let Ok(cfg) = bound.extract::() { return Ok(cfg.inner); } @@ -1532,7 +1532,7 @@ fn depythonize_kv_source(item: &Bound<'_, PyAny>) -> PyResult { // `Arc` as the existing Python pyclass, and // invokes the engine-supplied callback. The callback is // declared sync on the Python side (see `PushSource.on_ready` - // in `dynamo.common.backend.publisher`), so no asyncio + // in `dingo.common.backend.publisher`), so no asyncio // round-trip is needed here. let on_ready_obj: PyObject = item.getattr("on_ready")?.into(); let on_ready: OnPublisherReady = Box::new(move |publisher| { diff --git a/lib/bindings/python/rust/context.rs b/lib/bindings/python/rust/context.rs index 8164c4c3d57f..74f9f77e0828 100644 --- a/lib/bindings/python/rust/context.rs +++ b/lib/bindings/python/rust/context.rs @@ -5,7 +5,7 @@ // trace identity, and span access for engine observability. // // Engine code reaches the observability surface through the -// `dynamo.common.backend.telemetry` facade, which is itself a thin wrapper +// `dingo.common.backend.telemetry` facade, which is itself a thin wrapper // over [`Context::current_span`] / [`Context::start_span`]. Both return a // unified [`SpanProxy`] handle whose `set_attribute` / `add_event` / // `set_status` operations mirror the OTel `Span` API. @@ -53,7 +53,7 @@ fn warn_bridge_missing_once(method: &str) { /// /// The span is private — engine code reaches it via [`Context::current_span`] /// (the auto-span proxy) or [`Context::start_span`] (a child span). The -/// facade in `dynamo.common.backend.telemetry` is a one-line wrapper around +/// facade in `dingo.common.backend.telemetry` is a one-line wrapper around /// those methods. #[derive(Clone)] #[pyclass] @@ -356,7 +356,7 @@ impl Context { /// `SpanProxy`; when no parent was plumbed in (test contexts) or the /// OTel bridge isn't installed, the proxy is a silent no-op. /// - /// Prefer the `dynamo.common.backend.telemetry.current_span(context)` + /// Prefer the `dingo.common.backend.telemetry.current_span(context)` /// facade in engine code — it's the documented surface. fn current_span(&self) -> SpanProxy { match &self.span { @@ -382,7 +382,7 @@ impl Context { /// /// Returns a no-op proxy when no parent was plumbed in or the bridge /// isn't installed. Prefer the - /// `dynamo.common.backend.telemetry.start_span(context, name)` facade + /// `dingo.common.backend.telemetry.start_span(context, name)` facade /// in engine code. #[pyo3(signature = (name, attrs=None))] fn start_span(&self, name: &str, attrs: Option<&Bound<'_, PyDict>>) -> PyResult { diff --git a/lib/bindings/python/rust/errors.rs b/lib/bindings/python/rust/errors.rs index ec414adfd2f1..8d6dcdd945ad 100644 --- a/lib/bindings/python/rust/errors.rs +++ b/lib/bindings/python/rust/errors.rs @@ -136,7 +136,7 @@ define_dynamo_exceptions!( /// Read `(code, message)` off a Python exception carrying an HTTP-style /// status. Accepts `.code` (matches [`HttpError`] in `http.rs`) or `.status` -/// (matches `dynamo.common.http.HttpStatusError`) plus `.message`. +/// (matches `dingo.common.http.HttpStatusError`) plus `.message`. /// /// SECURITY: `.message` is forwarded verbatim to clients on 4xx responses /// (HTTP protocol contract). Python callers must ensure it contains no diff --git a/lib/bindings/python/rust/llm/replay.rs b/lib/bindings/python/rust/llm/replay.rs index cfdb300768d3..d78a656519b3 100644 --- a/lib/bindings/python/rust/llm/replay.rs +++ b/lib/bindings/python/rust/llm/replay.rs @@ -1779,7 +1779,7 @@ fn populate_missing_offload_kv_bytes_per_token( .extract()?; let kv_cache_dtype = kv_cache_dtype.as_deref().unwrap_or("auto"); - let kv_cache_module = py.import("dynamo.mocker.utils.kv_cache")?; + let kv_cache_module = py.import("dingo.mocker.utils.kv_cache")?; let kv_bytes_per_token = kv_cache_module .getattr("compute_kv_bytes_per_token")? .call1((model_path, kv_cache_dtype))? diff --git a/lib/bindings/python/src/dynamo/_core.pyi b/lib/bindings/python/src/dynamo/_core.pyi index 9aadc3ae2958..4386ff050c06 100644 --- a/lib/bindings/python/src/dynamo/_core.pyi +++ b/lib/bindings/python/src/dynamo/_core.pyi @@ -539,7 +539,7 @@ class Context: (test contexts) or the OTel bridge isn't installed. Engines normally reach this through - ``dynamo.common.backend.telemetry.current_span(context)``. + ``dingo.common.backend.telemetry.current_span(context)``. """ ... @@ -552,7 +552,7 @@ class Context: ``__exit__`` / ``close()`` / drop. Engines normally reach this through - ``dynamo.common.backend.telemetry.start_span(context, name)``. + ``dingo.common.backend.telemetry.start_span(context, name)``. """ ... diff --git a/lib/bindings/python/tests/replay/replay_utils.py b/lib/bindings/python/tests/replay/replay_utils.py index a8fb2b94f823..566fb8fef819 100644 --- a/lib/bindings/python/tests/replay/replay_utils.py +++ b/lib/bindings/python/tests/replay/replay_utils.py @@ -11,7 +11,7 @@ import pytest from dynamo.llm import KvRouterConfig -from dynamo.mocker import MockEngineArgs +from dingo.mocker import MockEngineArgs MOONCAKE_TRACE_FIRST20 = """{"timestamp": 0, "input_length": 6755, "output_length": 500, "hash_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]} {"timestamp": 0, "input_length": 7319, "output_length": 490, "hash_ids": [0, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]} @@ -378,7 +378,7 @@ def _run_aic_static_point(backend_name: str, isl: int, osl: int, batch_size: int def _planner_profile_data_dir_path() -> Path: return ( Path(__file__).resolve().parents[5] - / "components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D" + / "dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D" ) @@ -400,7 +400,7 @@ def _run_replay_cli(tmp_path, *args): [ sys.executable, "-m", - "dynamo.replay", + "dingo.replay", *args, ], capture_output=True, diff --git a/lib/bindings/python/tests/replay/test_replay_aic_capacity.py b/lib/bindings/python/tests/replay/test_replay_aic_capacity.py index c38ec9029d75..c180ace35344 100644 --- a/lib/bindings/python/tests/replay/test_replay_aic_capacity.py +++ b/lib/bindings/python/tests/replay/test_replay_aic_capacity.py @@ -6,9 +6,9 @@ import pytest import dynamo._internal.aic as aic -import dynamo.replay.main as replay_main -from dynamo.mocker import MockEngineArgs, PlannerReplayBridge -from dynamo.replay import run_synthetic_trace_replay +import dingo.replay.main as replay_main +from dingo.mocker import MockEngineArgs, PlannerReplayBridge +from dingo.replay import run_synthetic_trace_replay from .replay_utils import _write_trace_and_args diff --git a/lib/bindings/python/tests/replay/test_replay_cli.py b/lib/bindings/python/tests/replay/test_replay_cli.py index 1a95718aa307..b89c6ab84f43 100644 --- a/lib/bindings/python/tests/replay/test_replay_cli.py +++ b/lib/bindings/python/tests/replay/test_replay_cli.py @@ -6,7 +6,7 @@ import pytest -import dynamo.replay.main as replay_main +import dingo.replay.main as replay_main from .replay_utils import ( _assert_basic_report_counts, diff --git a/lib/bindings/python/tests/replay/test_replay_planner_load_modes.py b/lib/bindings/python/tests/replay/test_replay_planner_load_modes.py index c31e049a676d..0f235a926268 100644 --- a/lib/bindings/python/tests/replay/test_replay_planner_load_modes.py +++ b/lib/bindings/python/tests/replay/test_replay_planner_load_modes.py @@ -14,8 +14,8 @@ import pytest -from dynamo.mocker import MockEngineArgs, PlannerReplayBridge -from dynamo.replay import run_synthetic_trace_replay, run_trace_replay +from dingo.mocker import MockEngineArgs, PlannerReplayBridge +from dingo.replay import run_synthetic_trace_replay, run_trace_replay from .replay_utils import _vllm_args, _write_trace_and_args diff --git a/lib/bindings/python/tests/replay/test_replay_policy_plumbing.py b/lib/bindings/python/tests/replay/test_replay_policy_plumbing.py index 58e1672086f4..62dcda8784b9 100644 --- a/lib/bindings/python/tests/replay/test_replay_policy_plumbing.py +++ b/lib/bindings/python/tests/replay/test_replay_policy_plumbing.py @@ -5,8 +5,8 @@ import pytest -import dynamo.replay.api as replay_api -import dynamo.replay.main as replay_main +import dingo.replay.api as replay_api +import dingo.replay.main as replay_main from dynamo.llm import KvRouterConfig pytestmark = [ diff --git a/lib/bindings/python/tests/replay/test_replay_smoke.py b/lib/bindings/python/tests/replay/test_replay_smoke.py index e2cc1d816d9f..755992d7e358 100644 --- a/lib/bindings/python/tests/replay/test_replay_smoke.py +++ b/lib/bindings/python/tests/replay/test_replay_smoke.py @@ -6,9 +6,9 @@ import pytest -from dynamo.mocker import MockEngineArgs -from dynamo.replay import run_synthetic_trace_replay, run_trace_replay -from dynamo.replay.reporting import format_report_table, write_report_json +from dingo.mocker import MockEngineArgs +from dingo.replay import run_synthetic_trace_replay, run_trace_replay +from dingo.replay.reporting import format_report_table, write_report_json from .replay_utils import ( _assert_basic_report_counts, @@ -449,7 +449,7 @@ def test_run_trace_replay_accepts_partial_extra_engine_args_json(tmp_path, repla def test_run_trace_replay_materializes_kv_bytes_from_aic_model(monkeypatch, tmp_path): - kv_cache = importlib.import_module("dynamo.mocker.utils.kv_cache") + kv_cache = importlib.import_module("dingo.mocker.utils.kv_cache") def fake_compute_kv_bytes_per_token(model_path, kv_cache_dtype="auto"): return 1 if model_path == "test/model" else None diff --git a/lib/bindings/python/tests/replay/test_replay_trtllm_rejection.py b/lib/bindings/python/tests/replay/test_replay_trtllm_rejection.py index 109ca9f66181..b04eec5a5c19 100644 --- a/lib/bindings/python/tests/replay/test_replay_trtllm_rejection.py +++ b/lib/bindings/python/tests/replay/test_replay_trtllm_rejection.py @@ -20,8 +20,8 @@ import pytest -from dynamo.mocker import MockEngineArgs -from dynamo.replay import run_trace_replay +from dingo.mocker import MockEngineArgs +from dingo.replay import run_trace_replay pytestmark = [ pytest.mark.gpu_0, diff --git a/lib/bindings/python/tests/test_deprecated_enable_nats.py b/lib/bindings/python/tests/test_deprecated_enable_nats.py index a14c71c3963d..88a8a61dcdb0 100644 --- a/lib/bindings/python/tests/test_deprecated_enable_nats.py +++ b/lib/bindings/python/tests/test_deprecated_enable_nats.py @@ -21,7 +21,7 @@ import pytest from dynamo._core import DistributedRuntime -from dynamo.common.utils.runtime import create_runtime +from dingo.common.utils.runtime import create_runtime from dynamo.runtime import dynamo_worker pytestmark = [ @@ -212,7 +212,7 @@ def test_create_runtime_accepts_use_kv_events_kwarg(): assert param.default is None -@patch("dynamo.common.utils.runtime.DistributedRuntime") +@patch("dingo.common.utils.runtime.DistributedRuntime") def test_create_runtime_use_kv_events_true_emits_warning(mock_runtime_cls): """create_runtime(use_kv_events=True) should emit a DeprecationWarning. @@ -241,7 +241,7 @@ async def _run(): asyncio.run(_run()) -@patch("dynamo.common.utils.runtime.DistributedRuntime") +@patch("dingo.common.utils.runtime.DistributedRuntime") def test_create_runtime_use_kv_events_false_emits_warning(mock_runtime_cls): """create_runtime(use_kv_events=False) should also emit a DeprecationWarning. @@ -267,7 +267,7 @@ async def _run(): asyncio.run(_run()) -@patch("dynamo.common.utils.runtime.DistributedRuntime") +@patch("dingo.common.utils.runtime.DistributedRuntime") def test_create_runtime_no_use_kv_events_no_warning(mock_runtime_cls): """Omitting use_kv_events should not emit a DeprecationWarning. diff --git a/lib/bindings/python/tests/test_engine_perf_model.py b/lib/bindings/python/tests/test_engine_perf_model.py index 012a0aa1c884..aa1b01c8942a 100644 --- a/lib/bindings/python/tests/test_engine_perf_model.py +++ b/lib/bindings/python/tests/test_engine_perf_model.py @@ -5,7 +5,7 @@ import pytest -from dynamo.common.forward_pass_metrics import ( +from dingo.common.forward_pass_metrics import ( ForwardPassMetrics, QueuedRequestMetrics, ScheduledRequestMetrics, @@ -19,7 +19,7 @@ ] try: - from dynamo.mocker import ( + from dingo.mocker import ( EngineCapacityRequest, EnginePerfLimits, RustEnginePerfModel, diff --git a/lib/bindings/python/tests/test_http_server.py b/lib/bindings/python/tests/test_http_server.py index e048622625a1..a0367d0ae735 100644 --- a/lib/bindings/python/tests/test_http_server.py +++ b/lib/bindings/python/tests/test_http_server.py @@ -32,7 +32,7 @@ class _StatusLikeError(Exception): - """Mimics dynamo.common.http.HttpStatusError's .status + .message shape.""" + """Mimics dingo.common.http.HttpStatusError's .status + .message shape.""" def __init__(self, status: int, message: str): super().__init__(f"HTTP {status}: {message}") diff --git a/lib/llm/src/preprocessor/media/loader.rs b/lib/llm/src/preprocessor/media/loader.rs index 36613454f682..297f4728b103 100644 --- a/lib/llm/src/preprocessor/media/loader.rs +++ b/lib/llm/src/preprocessor/media/loader.rs @@ -32,7 +32,7 @@ const MAX_REDIRECTS: usize = 3; // (reserved). Link-local 169.254/16 covers the AWS / OpenStack metadata IP. // // Keep this list in sync with the Python counterpart -// (components/src/dynamo/common/multimodal/url_validator.py::_BLOCKED_IP_NETWORKS). +// (dingo/common/multimodal/url_validator.py::_BLOCKED_IP_NETWORKS). static BLOCKED_IP_NETWORKS: LazyLock> = LazyLock::new(|| { [ "0.0.0.0/8", @@ -67,7 +67,7 @@ static BLOCKED_IP_NETWORKS: LazyLock> = LazyLock::new(|| { // internal-service names to attacker IPs. Match is case-insensitive. // // Keep this list in sync with the Python counterpart -// (components/src/dynamo/common/multimodal/url_validator.py::_BLOCKED_HOSTS). +// (dingo/common/multimodal/url_validator.py::_BLOCKED_HOSTS). static BLOCKED_HOSTS: LazyLock> = LazyLock::new(|| { [ "localhost", diff --git a/lib/llm/src/protocols/common/preprocessor.rs b/lib/llm/src/protocols/common/preprocessor.rs index 0afad99a50b6..4c1b3ba7b81d 100644 --- a/lib/llm/src/protocols/common/preprocessor.rs +++ b/lib/llm/src/protocols/common/preprocessor.rs @@ -292,7 +292,7 @@ pub struct PreprocessedRequest { /// cross-worker coordination (KV transfer, bootstrap handshake, /// `require_prefill_result`) and run local-only. The wire-format key /// is `_HEALTH_CHECK` so the canary payload built by - /// `dynamo.common.backend.health_check.build_health_check_payload` + /// `dingo.common.backend.health_check.build_health_check_payload` /// (and the legacy `HealthCheckPayload` base class) round-trips through /// this field. Skipped from serialization when false so normal traffic /// doesn't carry the marker. diff --git a/lib/mocker/README.md b/lib/mocker/README.md index 47db36dd3602..9ec05a256999 100644 --- a/lib/mocker/README.md +++ b/lib/mocker/README.md @@ -49,4 +49,4 @@ embedding it directly as a standalone Rust dependency. - DynoSim runs guide: [../../docs/dynosim/runs.md](../../docs/dynosim/runs.md) - Python component README: - [../../components/src/dynamo/mocker/README.md](../../components/src/dynamo/mocker/README.md) + [../../dingo/mocker/README.md](../../dingo/mocker/README.md) diff --git a/lib/mocker/src/common/protocols.rs b/lib/mocker/src/common/protocols.rs index ff82a5b24706..733aaaac9919 100644 --- a/lib/mocker/src/common/protocols.rs +++ b/lib/mocker/src/common/protocols.rs @@ -117,7 +117,7 @@ impl KvEventPublishers { } /// Per-iteration forward pass snapshot, mirroring the Python `ForwardPassMetrics` -/// schema in `components/src/dynamo/common/forward_pass_metrics.py`. +/// schema in `dingo/common/forward_pass_metrics.py`. /// /// Produced by the scheduler core after each `execute_pass_internal()` call. /// Runtime publishers may either stamp identity at serialization time or fill diff --git a/lib/mocker/src/scheduler/kv_event_sink.rs b/lib/mocker/src/scheduler/kv_event_sink.rs index db9d8b9a0d76..40825fc2dcac 100644 --- a/lib/mocker/src/scheduler/kv_event_sink.rs +++ b/lib/mocker/src/scheduler/kv_event_sink.rs @@ -81,7 +81,7 @@ pub(crate) struct DeferredKvPublish { pub(crate) storage_tier: StorageTier, } -/// Captures raw KV publishes for the live `python -m dynamo.mocker` and online +/// Captures raw KV publishes for the live `python -m dingo.mocker` and online /// replay paths. /// /// Unlike `CapturedRouterEventBuffer`, this keeps `block_token_ids` so delayed diff --git a/lib/runtime/src/metrics.rs b/lib/runtime/src/metrics.rs index 46e7fbb2a759..ed05ff5be873 100644 --- a/lib/runtime/src/metrics.rs +++ b/lib/runtime/src/metrics.rs @@ -229,7 +229,7 @@ pub fn create_metric( // Auto-label injection: Always add dynamo_namespace, dynamo_component, dynamo_endpoint labels // based on the hierarchy. Label constants defined in prometheus_names.rs labels module. // - // Python counterpart: components/src/dynamo/common/utils/prometheus.py register_engine_metrics_callback() + // Python counterpart: dingo/common/utils/prometheus.py register_engine_metrics_callback() // Validate that user-provided labels don't conflict with auto-generated labels for (key, _) in labels { diff --git a/lib/runtime/src/metrics/prometheus_names.rs b/lib/runtime/src/metrics/prometheus_names.rs index 8e9e5ebd35f3..02f5b55c7589 100644 --- a/lib/runtime/src/metrics/prometheus_names.rs +++ b/lib/runtime/src/metrics/prometheus_names.rs @@ -107,7 +107,7 @@ pub mod name_prefix { /// /// These labels are auto-injected into metrics by the hierarchy system: /// - Rust: lib/runtime/src/metrics.rs create_metric() function -/// - Python: components/src/dynamo/common/utils/prometheus.py register_engine_metrics_callback() +/// - Python: dingo/common/utils/prometheus.py register_engine_metrics_callback() /// /// Python codegen: These constants are exported to lib/bindings/python/src/dynamo/prometheus_names.py pub mod labels { diff --git a/pyproject.toml b/pyproject.toml index 31d14741d51f..180677c57120 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "zstandard>=0.23.0,<1.0", "pyzmq>=26.0.0", # typing.Self is 3.11+; declared minimum is 3.10 (see requires-python), - # so back-port Self via typing_extensions in dynamo.common.configuration. + # so back-port Self via typing_extensions in dingo.common.configuration. "typing_extensions>=4.10.0", ] @@ -355,7 +355,7 @@ ignore_errors = true [[tool.mypy.overrides]] # Manual planner helpers are operational scripts, not typed library surfaces. -module = ["dynamo.planner.manual.*"] +module = ["dingo.planner.manual.*"] ignore_errors = true [[tool.mypy.overrides]] @@ -480,7 +480,7 @@ ignore_missing_imports = true [[tool.mypy.overrides]] # Profiler module was never previously type-checked and has many # union-attr / attr-defined issues. Skip errors for now. -module = ["dynamo.profiler.*"] +module = ["dingo.profiler.*"] ignore_errors = true [tool.sphinx] diff --git a/recipes/CONTRIBUTING.md b/recipes/CONTRIBUTING.md deleted file mode 100644 index 90e5da54f970..000000000000 --- a/recipes/CONTRIBUTING.md +++ /dev/null @@ -1,23 +0,0 @@ -# Recipes Contributing Guide - -When adding new model recipes, ensure they follow the standard structure: -```text -/ -├── model-cache/ -│ ├── model-cache.yaml -│ └── model-download.yaml -├── / -│ └── / -│ ├── deploy.yaml -│ └── perf.yaml (optional) -└── README.md (optional) -``` - -## Validation -The `run.sh` script expects this exact directory structure and will validate that the directories and files exist before deployment: -- Model directory exists in `recipes//` -- Framework is one of the supported frameworks (vllm, sglang, trtllm) -- Framework directory exists in `recipes///` -- Deployment directory exists in `recipes////` -- Required files (`deploy.yaml`) exist in the deployment directory -- If present, performance benchmarks (`perf.yaml`) will be automatically executed \ No newline at end of file diff --git a/recipes/README.md b/recipes/README.md deleted file mode 100644 index 41fcc03247f6..000000000000 --- a/recipes/README.md +++ /dev/null @@ -1,356 +0,0 @@ - - -# Dynamo Production-Ready Recipes - -Production-tested Kubernetes deployment recipes for LLM inference using NVIDIA Dynamo. - -> **Prerequisites:** This guide assumes you have already installed the Dynamo Kubernetes Platform. -> If not, follow the **[Kubernetes Deployment Guide](../docs/kubernetes/README.md)** first. - -## Available Recipes - -### Feature Comparison Recipes - -These recipes compare Dynamo performance features with benchmark results, each including both baseline and optimized deployment configurations: - -| Model | Framework | Configuration | GPUs | Features | -|-------|-----------|---------------|------|----------| -| **[Qwen3-32B](qwen3-32b/)** | vLLM | Disagg + KV-Router | 16x H200 | **Disaggregated Serving + KV-Aware Routing** — benchmark comparison with real-world Mooncake traces | -| **[DeepSeek-V3.2-NVFP4](deepseek-v32-fp4/)** | TensorRT-LLM | Agg + Disagg WideEP | 32x GB200 | **Disaggregated Serving + KV-Aware Routing** — benchmark comparison with Mooncake-based synthetic coding trace | -| **[Qwen3-VL-30B-A3B-FP8](qwen3-vl-30b/)** | vLLM | Agg + Embedding Cache | 1x GB200 | **Multimodal Embedding Cache** — benchmark comparison showing +16% throughput, -28% TTFT | - -### Aggregated & Disaggregated Recipes - -These recipes demonstrate aggregated or disaggregated serving: - -**GAIE Column**: Indicates whether the recipe includes integration with the [Gateway API Inference Extension (GAIE)](../deploy/inference-gateway/README.md) — a Kubernetes SIG project that extends the Gateway API for AI inference workloads, providing load balancing, model routing, and request management. - -| Model | Framework | Mode | GPUs | Deployment | Benchmark | Notes | GAIE | -|-------|-----------|------|------|------------|-----------|-------|------| -| **[Llama-3-70B](llama-3-70b/vllm/agg/)** | vLLM | Aggregated | 4x H100/H200 | ✅ | ✅ | FP8 dynamic quantization | ✅ | -| **[Llama-3-70B](llama-3-70b/vllm/disagg-single-node/)** | vLLM | Disagg (Single-Node) | 8x H100/H200 | ✅ | ✅ | Prefill + Decode separation | ❌ | -| **[Llama-3-70B](llama-3-70b/vllm/disagg-multi-node/)** | vLLM | Disagg (Multi-Node) | 16x H100/H200 | ✅ | ✅ | 2 nodes, 8 GPUs each | ❌ | -| **[Qwen3-32B-FP8](qwen3-32b-fp8/trtllm/agg/)** | TensorRT-LLM | Aggregated | 2x H100/H200/A100 | ✅ | ✅ | FP8 quantization | ❌ | -| **[Qwen3-32B-FP8](qwen3-32b-fp8/trtllm/disagg/)** | TensorRT-LLM | Disaggregated | 8x H100/H200/A100 | ✅ | ✅ | Prefill + Decode separation | ❌ | -| **[Qwen3-32B-FP8](qwen3-32b-fp8/vllm/disagg/)** | vLLM | Disagg (Single-Node) | 8x A100 | ✅ | ✅ | 2× TP2 prefill + 1× TP4 decode, NixlConnector KV transfer | ❌ | -| **[Qwen3-235B-A22B-FP8](qwen3-235b-a22b-fp8/trtllm/agg/hopper/)** | TensorRT-LLM | Aggregated (Hopper) | 16x H100/H200 | ✅ | ✅ | MoE model, TP4×EP4 | ❌ | -| **[Qwen3-235B-A22B-FP8](qwen3-235b-a22b-fp8/trtllm/agg/blackwell/)** | TensorRT-LLM | Aggregated (Blackwell) | 16x B100/B200 | ✅ | ✅ | MoE model, TP4×EP4, DEEPGEMM backend | ❌ | -| **[Qwen3-235B-A22B-FP8](qwen3-235b-a22b-fp8/trtllm/disagg/hopper/)** | TensorRT-LLM | Disaggregated (Hopper) | 16x H100/H200 | ✅ | ✅ | MoE model, Prefill + Decode | ❌ | -| **[Qwen3-235B-A22B-FP8](qwen3-235b-a22b-fp8/trtllm/disagg/blackwell/)** | TensorRT-LLM | Disaggregated (Blackwell) | 16x B100/B200 | ✅ | ✅ | MoE model, Prefill + Decode, DEEPGEMM backend | ❌ | -| **[GPT-OSS-120B](gpt-oss-120b/trtllm/agg/)** | TensorRT-LLM | Aggregated | 4x GB200 | ✅ | ✅ | Blackwell only, WideEP | ❌ | -| **[GPT-OSS-120B](gpt-oss-120b/trtllm/disagg/)** | TensorRT-LLM | Disaggregated | 5x Blackwell (GB200/B200) | ✅ | ✅ | Prefill/Decode split | ❌ | -| **[GLM-5-NVFP4](glm-5-nvfp4/sglang/disagg/)** | SGLang | Disagg Prefill/Decode | 20x GB200 | ✅ | ✅ | NVFP4, EAGLE speculative decoding, TP16 decode + TP4 prefill, stable SGLang runtime image | ❌ | -| **[DeepSeek-R1](deepseek-r1/sglang/disagg-8gpu/)** | SGLang | Disagg WideEP | 16x H200 | ✅ | ❌ | TP=8, single-node. Use `model-download-sglang.yaml` | ❌ | -| **[DeepSeek-R1](deepseek-r1/sglang/disagg-16gpu/)** | SGLang | Disagg WideEP | 32x H200 | ✅ | ❌ | TP=16, multi-node. Use `model-download-sglang.yaml` | ❌ | -| **[DeepSeek-R1](deepseek-r1/trtllm/disagg/wide_ep/gb200/)** | TensorRT-LLM | Disagg WideEP (GB200) | 36x GB200 | ✅ | ✅ | Multi-node: 8 decode + 1 prefill nodes | ❌ | -| **[DeepSeek-R1](deepseek-r1/)** | vLLM | Disagg DEP16 | 32x H200 | ✅ | ❌ | Multi-node, data-expert parallel | ❌ | -| **[Kimi-K2.5](kimi-k2.5/trtllm/disagg-eagle-kv-router/)** | TensorRT-LLM | Disaggregated | 24x GB200 | ✅ | ✅ | DEP4 prefill + TEP4 decode, TRTLLM-native KV host offload | ❌ | -| **[Kimi-K2.6](kimi-k2.6/vllm/)** | vLLM | Aggregated | 4x B200 / 8x H200 | ✅ | ✅ | MoE, NVFP4+FP8 KV (B200) / INT4 (H200), TP4/TP8, EAGLE3 MLA spec decode, LMCache CPU offload; text+image, chat + agentic profiles | ❌ | -| **[Nemotron-3-Super](nemotron-3-super/vllm/)** | vLLM | Aggregated | 4x B200 / 4x H200 | ✅ | ✅ | ~120B hybrid Mamba/Attention/MoE (~12B active), NVFP4 (B200) / FP8 (H200) + FP8 KV, TP4+EP, MTP, KV-aware routing; chat + agentic profiles | ❌ | -| **[Nemotron-3-Ultra](nemotron-3-ultra/vllm/)** | vLLM | Agg + Disagg | 4x B200 / 8x H200 | ✅ | ✅ | ~550B hybrid Mamba/Attention/MoE (~55B active), NVFP4 + FP8, TP4 (B200) / TP8 (H200) + EP, MTP, KV-aware routing; chat + agentic, plus 1P1D disagg on B200 | ❌ | - -**Legend:** -- **Deployment**: ✅ = Complete `deploy.yaml` manifest available -- **Benchmark**: ✅ = Includes `perf.yaml` for running AIPerf benchmarks - -### Functional Recipes (Not Yet Benchmarked) - -These recipes demonstrate functional deployments with Dynamo features, but have not yet been performance-tuned or paired with benchmark manifests. - -| Model | Framework | Mode | GPUs | Deployment | Notes | -|-------|-----------|-------|------|------------|-------| -| **[Nemotron-3-Super-FP8](nemotron-3-super-fp8/vllm/agg/)** | vLLM | Aggregated | 4x H100/H200 | ✅ | TP=4, KV-aware routing | -| **[Nemotron-3-Super-FP8](nemotron-3-super-fp8/sglang/agg/)** | SGLang | Aggregated | 4x H100/H200 | ✅ | TP=4, KV-aware routing, 1.0+ | -| **[Nemotron-3-Super-FP8](nemotron-3-super-fp8/trtllm/disagg/)** | TensorRT-LLM | Disaggregated | 4x H100/H200 | ✅ | TP=2 prefill/decode split, UCX KV transfer | -| **[Nemotron-3-Super-FP8](nemotron-3-super-fp8/sglang/disagg/)** | SGLang | Disaggregated | 4x H100/H200 | ✅ | TP=2 prefill/decode split, nixl KV transfer, 1.0+ | - -### Experimental Recipes - -These recipes are under active development and may require additional setup steps (e.g., container patching). They are functional but not yet fully validated for production use. - -| Model | Framework | Mode | GPUs | Deployment | Notes | -|-------|-----------|------|------|------------|-------| -| **[GLM-5-NVFP4 (EFA)](glm-5-nvfp4/sglang/disagg/efa/)** | SGLang | Disagg Prefill/Decode over AWS EFA | 20x GB200 | ✅ | KV transfer over AWS EFA via NIXL LIBFABRIC instead of UCX. Patched libfabric baked into image. Requires [custom container build](glm-5-nvfp4/sglang/disagg/efa/Dockerfile.efa). | -| **[Nemotron-3-Nano-Omni-NVFP4](nemotron-3-nano-omni/vllm/agg/)** | vLLM | Aggregated | 1x GPU | ✅ | Multimodal text/image/video/audio serving. Requires [custom container build](nemotron-3-nano-omni/). | -| **[DeepSeek-V4-Flash](deepseek-v4/deepseek-v4-flash/vllm/agg_b200/)** | vLLM | Aggregated | 4x B200 | ✅ | Text only — MoE model (284B / 13B active), DP=4 + EP, FP8 KV cache, reasoning + tool calling. Requires [custom container build](deepseek-v4/container/). | -| **[DeepSeek-V4-Flash](deepseek-v4/deepseek-v4-flash/vllm/agg_gb200/)** | vLLM | Aggregated | 4x GB200 | ✅ | Text only — MoE model (284B / 13B active), TP=4 + EP, `deep_gemm_mega_moe`, FP8 KV cache, reasoning + tool calling (single NVL4 tray). Requires [custom container build](deepseek-v4/container/). | -| **[DeepSeek-V4-Flash](deepseek-v4/deepseek-v4-flash/sglang/agg/)** | SGLang | Aggregated | 4x B200 | ✅ | Text only — MoE model (284B / 13B active), TP=4, MXFP4 MoE via FlashInfer, EAGLE MTP (3 steps / 4 draft tokens), reasoning + tool calling. Prebuilt image available; optional [custom container build](deepseek-v4/container/). | -| **[DeepSeek-V4-Pro](deepseek-v4/deepseek-v4-pro/vllm/agg/b200/)** | vLLM | Aggregated | 8x B200 | ✅ | Text only — MoE model (1.6T / 49B active, 1M context), TP=8 + EP, FP4+FP8 mixed checkpoint, FP8 KV cache, CSA+HCA attention, tool calling. Thinking modes unstable on Day-0 — run with `thinking: false`. Requires [custom container build](deepseek-v4/container/). | -| **[DeepSeek-V4-Pro](deepseek-v4/deepseek-v4-pro/vllm/agg/gb200/)** | vLLM | Aggregated | 8x GB200 (2 NVL4 trays) | ✅ | Text only — same model as B200 agg; TP=8 + EP cross-node via NVLink72 (MNNVL) + ComputeDomain. Requires [custom container build](deepseek-v4/container/). | -| **[DeepSeek-V4-Pro](deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/)** | vLLM | Disaggregated | 16x GB200 (4 NVL4 trays) | ✅ | Text only — DP=8 + EP per worker, 1P + 1D, NVLink72 (MNNVL) + ComputeDomain. Requires [custom container build](deepseek-v4/container/). | -| **[DeepSeek-V4-Pro](deepseek-v4/deepseek-v4-pro/sglang/agg/)** | SGLang | Aggregated | 8x B200 | ✅ | Text only — MoE model (1.6T / 49B active, 1M context), TP=8, MXFP4 MoE via FlashInfer, EAGLE MTP (3 steps / 4 draft tokens), reasoning + tool calling. Prebuilt image available (shared with [DeepSeek-V4-Flash](deepseek-v4/deepseek-v4-flash/sglang/agg/)). | - -## Recipe Structure - -Each complete recipe follows this standard structure: - -``` -/ -├── README.md (optional) # Model-specific deployment notes -├── model-cache/ -│ ├── model-cache.yaml # PersistentVolumeClaim for model storage -│ └── model-download.yaml # Job to download model from HuggingFace -└── / # vllm, sglang, or trtllm - └── / # agg, disagg, disagg-single-node, etc. - ├── deploy.yaml # Complete DynamoGraphDeployment manifest - └── perf.yaml (optional) # AIPerf benchmark job -``` - -## Quick Start - -### Prerequisites - -**1. Dynamo Platform Installed** - -The recipes require the Dynamo Kubernetes Platform to be installed. Follow the installation guide: - -- **[Kubernetes Deployment Guide](../docs/kubernetes/README.md)** - Quickstart (~10 minutes) -- **[Detailed Installation Guide](../docs/kubernetes/installation-guide.md)** - Advanced options - -**2. GPU Cluster Requirements** - -Ensure your cluster has: -- GPU nodes matching recipe requirements (see table above) -- GPU operator installed -- Appropriate GPU drivers and container runtime - -**3. HuggingFace Access** - -Configure authentication to download models: - -```bash -export NAMESPACE=your-namespace -kubectl create namespace ${NAMESPACE} - -# Create HuggingFace token secret -kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="your-token-here" \ - -n ${NAMESPACE} -``` - -**4. Storage Configuration** - -Update the `storageClassName` in `/model-cache/model-cache.yaml` to match your cluster: - -```bash -# Find your storage class name -kubectl get storageclass - -# Edit the model-cache.yaml file and update: -# spec: -# storageClassName: "your-actual-storage-class" -``` - -### Deploy a Recipe - -**Step 1: Download Model** - -```bash -cd recipes -# Update storageClassName in model-cache.yaml first! -kubectl apply -f /model-cache/ -n ${NAMESPACE} - -# Wait for download to complete (may take 10-60 minutes depending on model size) -kubectl wait --for=condition=Complete job/model-download -n ${NAMESPACE} --timeout=6000s - -# Monitor progress -kubectl logs -f job/model-download -n ${NAMESPACE} -``` - -**Step 2: Deploy Service** - -Update the image in `///deploy.yaml`. - -```bash -kubectl apply -f ///deploy.yaml -n ${NAMESPACE} - -# Check deployment status -kubectl get dynamographdeployment -n ${NAMESPACE} - -# Check pod status -kubectl get pods -n ${NAMESPACE} - -# Wait for pods to be ready -kubectl wait --for=condition=ready pod -l nvidia.com/dynamo-graph-deployment-name= -n ${NAMESPACE} --timeout=600s -``` - -**Step 3: Test Deployment** - -```bash -# Port forward to access the service locally -kubectl port-forward svc/-frontend 8000:8000 -n ${NAMESPACE} - -# In another terminal, test the endpoint -curl http://localhost:8000/v1/models - -# Send a test request -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 50 - }' -``` - -**Step 4: Run Benchmark (Optional)** - -```bash -# Only if perf.yaml exists in the recipe directory -kubectl apply -f ///perf.yaml -n ${NAMESPACE} - -# Monitor benchmark progress -kubectl logs -f job/ -n ${NAMESPACE} - -# View results after completion -kubectl logs job/ -n ${NAMESPACE} | tail -50 -``` - - -## Example Deployments - -### Llama-3-70B with vLLM (Aggregated) - -```bash -export NAMESPACE=dynamo-demo -kubectl create namespace ${NAMESPACE} - -# Create HF token secret -kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="your-token" \ - -n ${NAMESPACE} - -# Deploy -cd recipes -kubectl apply -f llama-3-70b/model-cache/ -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/model-download -n ${NAMESPACE} --timeout=6000s -kubectl apply -f llama-3-70b/vllm/agg/deploy.yaml -n ${NAMESPACE} - -# Test -kubectl port-forward svc/llama3-70b-agg-frontend 8000:8000 -n ${NAMESPACE} -``` - -### Inference Gateway (GAIE) Integration (Optional) - -For Llama-3-70B with vLLM (Aggregated), an example of integration with the Inference Gateway is provided. - -First, deploy the Dynamo Graph per instructions above. - -Then follow [Deploy Inference Gateway Section 2](../deploy/inference-gateway/README.md#2-deploy-inference-gateway) to install GAIE. - -Update the containers.epp.image in the deployment file, i.e. llama-3-70b/vllm/agg/gaie/k8s-manifests/epp/deployment.yaml. It should match the release tag and be in the format `nvcr.io/nvidia/ai-dynamo/frontend:` e.g. `nvcr.io/nvidia/ai-dynamo/frontend:0.9.0` -The recipe assumes you are using Kubernetes discovery backend and sets the `DYN_DISCOVERY_BACKEND` env variable in the epp deployment. If you want to use etcd enable the lines below and remove the DYN_DISCOVERY_BACKEND env var. -```bash -- name: ETCD_ENDPOINTS - value: "dynamo-platform-etcd.$(PLATFORM_NAMESPACE):2379" # update dynamo-platform to appropriate namespace -``` - -```bash -export DEPLOY_PATH=llama-3-70b/vllm/agg/ -# DEPLOY_PATH=/// -kubectl apply -R -f "$DEPLOY_PATH/gaie/k8s-manifests" -n "$NAMESPACE" -``` - -### DeepSeek-R1 on GB200 (Multi-node) - -See [deepseek-r1/trtllm/disagg/wide_ep/gb200/deploy.yaml](deepseek-r1/trtllm/disagg/wide_ep/gb200/deploy.yaml) for the complete multi-node WideEP configuration. - -## Customization - -Each `deploy.yaml` contains: -- **ConfigMap**: Engine-specific configuration (embedded in the manifest) -- **DynamoGraphDeployment**: Kubernetes resource definitions -- **Resource limits**: GPU count, memory, CPU requests/limits -- **Image references**: Container images with version tags - -### Key Customization Points - -**Model Configuration:** -```yaml -# In deploy.yaml under worker args: -args: - - python3 -m dingo.vllm --model --served-model-name -``` - -**GPU Resources:** -```yaml -resources: - limits: - gpu: "4" # Adjust based on your requirements - requests: - gpu: "4" -``` - -**Scaling:** -```yaml -services: - VllmDecodeWorker: - replicas: 2 # Scale to multiple workers -``` - -**Router Mode:** -```yaml -# In Frontend args: -args: - - python3 -m dingo.frontend --router-mode kv --http-port 8000 -# Options: round-robin, kv (KV-aware routing) -``` - -**Container Images:** -```yaml -image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:x.y.z -# Update version tag as needed -``` - -## Troubleshooting - -### Common Issues - -**Pods stuck in Pending:** -- Check GPU availability: `kubectl describe node ` -- Verify storage class exists: `kubectl get storageclass` -- Check resource requests vs. available resources - -**Model download fails:** -- Verify HuggingFace token is correct -- Check network connectivity from cluster -- Review job logs: `kubectl logs job/model-download -n ${NAMESPACE}` - -**Workers fail to start:** -- Check GPU compatibility (driver version, CUDA version) -- Verify image pull secrets if using private registries -- Review pod logs: `kubectl logs -n ${NAMESPACE}` - -**For more troubleshooting:** -- [Kubernetes Deployment Guide](../docs/kubernetes/README.md#troubleshooting) -- [Observability Documentation](../docs/kubernetes/observability/) - -## Related Documentation - -- **[Kubernetes Deployment Guide](../docs/kubernetes/README.md)** - Platform installation and concepts -- **[API Reference](../docs/kubernetes/api-reference.md)** - DynamoGraphDeployment CRD specification -- **[vLLM Backend Guide](../docs/backends/vllm/README.md)** - vLLM-specific features -- **[SGLang Backend Guide](../docs/backends/sglang/README.md)** - SGLang-specific features -- **[TensorRT-LLM Backend Guide](../docs/backends/trtllm/README.md)** - TensorRT-LLM features -- **[Observability](../docs/kubernetes/observability/)** - Monitoring and logging -- **[Benchmarking Guide](../docs/benchmarks/benchmarking.md)** - Performance testing - -## Contributing - -We welcome contributions of new recipes! See [CONTRIBUTING.md](CONTRIBUTING.md) for: -- Recipe submission guidelines -- Required components checklist -- Testing and validation requirements -- Documentation standards - -### Recipe Quality Standards - -A production-ready recipe must include: -- ✅ Complete `deploy.yaml` with DynamoGraphDeployment -- ✅ Model cache PVC and download job -- ✅ Benchmark recipe (`perf.yaml`) for performance testing -- ✅ Verification on target hardware -- ✅ Documentation of GPU requirements diff --git a/recipes/deepseek-r1/README.md b/recipes/deepseek-r1/README.md deleted file mode 100644 index 055fd1d012d1..000000000000 --- a/recipes/deepseek-r1/README.md +++ /dev/null @@ -1,103 +0,0 @@ -# DeepSeek-R1 Recipes - -Production-ready deployments for **DeepSeek-R1** (671B MoE) across multiple backends and hardware configurations. - -## Available Configurations - -| Configuration | GPUs | Backend | Mode | Description | -|--------------|------|---------|------|-------------| -| [**sglang/disagg-8gpu**](sglang/disagg-8gpu/) | 16x H200 | SGLang | Disaggregated WideEP | TP=8 per worker, single-node | -| [**sglang/disagg-16gpu**](sglang/disagg-16gpu/) | 32x H200 | SGLang | Disaggregated WideEP | TP=16 per worker, multi-node | -| [**trtllm/disagg/wide_ep/gb200**](trtllm/disagg/wide_ep/gb200/) | 36x GB200 | TensorRT-LLM | Disaggregated WideEP | 8 decode + 1 prefill nodes | -| [**vllm/disagg**](vllm/disagg/) | 32x H200 | vLLM | Disaggregated DEP16 | Multi-node, data-expert parallel | - -## Prerequisites - -1. **Dynamo Platform installed** — See [Kubernetes Deployment Guide](../../docs/kubernetes/README.md) -2. **GPU cluster** with H200 or GB200 GPUs matching the configuration requirements -3. **HuggingFace token** with access to DeepSeek models -4. **High-bandwidth networking** — InfiniBand or RoCE recommended for multi-node deployments - -## Quick Start - -```bash -# Set namespace -export NAMESPACE=dynamo-demo -kubectl create namespace ${NAMESPACE} - -# Create HuggingFace token secret -kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="your-token-here" \ - -n ${NAMESPACE} - -# Download model (update storageClassName in model-cache.yaml first!) -# For SGLang deployments: -kubectl apply -f model-cache/model-cache.yaml -n ${NAMESPACE} -kubectl apply -f model-cache/model-download-sglang.yaml -n ${NAMESPACE} - -# For vLLM/TRT-LLM deployments: -kubectl apply -f model-cache/model-cache.yaml -n ${NAMESPACE} -kubectl apply -f model-cache/model-download.yaml -n ${NAMESPACE} - -# Wait for download (this is a large model - may take 1+ hours) -# For SGLang: kubectl wait --for=condition=Complete job/model-download-sglang ... -# For vLLM/TRT-LLM: kubectl wait --for=condition=Complete job/model-download ... -kubectl wait --for=condition=Complete job/model-download-sglang -n ${NAMESPACE} --timeout=7200s - -# Deploy (choose one configuration) -kubectl apply -f sglang/disagg-8gpu/deploy.yaml -n ${NAMESPACE} -``` - -## Test the Deployment - -```bash -# Port-forward the frontend (service name varies by deployment) -kubectl port-forward svc/sgl-dsr1-8gpu-frontend 8000:8000 -n ${NAMESPACE} - -# Send a test request -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "deepseek-ai/DeepSeek-R1", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100 - }' -``` - -## Model Details - -- **Model**: `deepseek-ai/DeepSeek-R1` -- **Architecture**: 671B parameter Mixture-of-Experts (MoE) -- **Active parameters**: ~37B per token -- **Recommended**: FP8 quantization for production deployments - -## Hardware Requirements - -DeepSeek-R1 is a very large model requiring significant GPU memory: - -| Configuration | Min GPU Memory | Recommended | -|--------------|----------------|-------------| -| 16x H200 (SGLang TP=8) | 1.1TB total | H200 SXM (141GB each) | -| 32x H200 (SGLang TP=16, vLLM) | 2.2TB total | H200 SXM (141GB each) | -| 36x GB200 (TRT-LLM) | ~2.5TB total | GB200 NVL72 | - -## Notes - -- **Model download time**: DeepSeek-R1 is ~1.3TB; expect 1-2 hours for download -- **NCCL errors**: Usually indicate OOM. Reduce `--mem-fraction-static` in worker args -- **Multi-node**: Requires InfiniBand/IBGDA enabled. See [vLLM EP docs](https://docs.vllm.ai/en/latest/serving/expert_parallel_deployment/) -- **Storage class**: Update `storageClassName` in `model-cache/model-cache.yaml` before deploying - -## Backend-Specific Notes - -### SGLang -- Uses WideEP (Wide Expert Parallel) for efficient MoE inference -- See [sglang/README.md](sglang/README.md) for SGLang-specific configuration - -### TensorRT-LLM -- Requires FP4 quantized checkpoint -- GB200-specific optimizations - -### vLLM -- Uses DEP (Data-Expert Parallel) with hybrid load balancing -- See [vllm/disagg/README.md](vllm/disagg/README.md) for detailed setup diff --git a/recipes/deepseek-r1/model-cache/model-cache.yaml b/recipes/deepseek-r1/model-cache/model-cache.yaml deleted file mode 100644 index 89522ed1d79b..000000000000 --- a/recipes/deepseek-r1/model-cache/model-cache.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 1500Gi - storageClassName: "your-storage-class-name" \ No newline at end of file diff --git a/recipes/deepseek-r1/model-cache/model-download-sglang.yaml b/recipes/deepseek-r1/model-cache/model-download-sglang.yaml deleted file mode 100644 index d62236d903e5..000000000000 --- a/recipes/deepseek-r1/model-cache/model-download-sglang.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - tolerations: [] - containers: - - name: model-download - image: python:3.10-slim - command: ["sh", "-c"] - env: - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - - name: HF_HOME - value: /opt/model-cache - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.11.0 - hf download deepseek-ai/DeepSeek-R1 - volumeMounts: - - name: model-cache - mountPath: /opt/model-cache - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/deepseek-r1/model-cache/model-download.yaml b/recipes/deepseek-r1/model-cache/model-download.yaml deleted file mode 100644 index 36da7209ea96..000000000000 --- a/recipes/deepseek-r1/model-cache/model-download.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - tolerations: [] - containers: - - name: model-download - image: python:3.10-slim - command: ["sh", "-c"] - env: - # Uses up to 64 GB RAM for XET buffers; set "0" on low-memory nodes (8 GB cap): https://huggingface.co/docs/hub/en/xet/using-xet-storage#download-buffers - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - # Optional: create with: kubectl create secret generic hf-token-secret --from-literal=HF_TOKEN= -n - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: HF_TOKEN - optional: true - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.11.0 - hf download nvidia/DeepSeek-R1-FP4 --local-dir /model-cache/deepseek-r1-fp4 - hf download deepseek-ai/DeepSeek-R1 --local-dir /model-cache/deepseek-r1 - resources: - requests: - cpu: "2" - memory: "64Gi" - limits: - cpu: "8" - memory: "64Gi" - volumeMounts: - - name: model-cache - mountPath: /model-cache - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/deepseek-r1/sglang/README.md b/recipes/deepseek-r1/sglang/README.md deleted file mode 100644 index 3562cc1a6d3f..000000000000 --- a/recipes/deepseek-r1/sglang/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# DeepSeek R1 SGLang Recipe - -This recipe is for running DeepSeek R1 with SGLang in disaggregated mode. It is based on the WideEP recipe from the SGLang team. - -## Container - -Build the container using the `build.sh` script: - -```bash -./container/build.sh --framework SGLANG -``` - -Dynamo commits after `1b3eed4b6a0e735d4ecec6681f4c0b89f2112167` (Sep 18, 2025) are required. - -## Hardware - -The two deployment recipes are for 16x H200 (disagg-8gpu) and 32x H200 (disagg-16gpu). The folder names refer to GPUs per worker type (8 or 16), with separate prefill and decode workers each using that many GPUs. It should also work for other GPU SKUs. Change the TP and EP size accordingly to match the GPU capacity. - -If you see NCCL errors when sending requests to the engines, it is usually caused by OOM error. Try to reduce `--mem-fraction-static` in both prefill and decode engines. - diff --git a/recipes/deepseek-r1/sglang/deepep.json b/recipes/deepseek-r1/sglang/deepep.json deleted file mode 100644 index 99541df6439d..000000000000 --- a/recipes/deepseek-r1/sglang/deepep.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "normal_dispatch": { - "num_sms": 24, - "num_max_nvl_chunked_send_tokens": 12, - "num_max_nvl_chunked_recv_tokens": 512, - "num_max_rdma_chunked_send_tokens": 8, - "num_max_rdma_chunked_recv_tokens": 128 - }, - "normal_combine": { - "num_sms": 24, - "num_max_nvl_chunked_send_tokens": 1, - "num_max_nvl_chunked_recv_tokens": 512, - "num_max_rdma_chunked_send_tokens": 8, - "num_max_rdma_chunked_recv_tokens": 128 - } -} \ No newline at end of file diff --git a/recipes/deepseek-r1/sglang/disagg-16gpu/deploy.yaml b/recipes/deepseek-r1/sglang/disagg-16gpu/deploy.yaml deleted file mode 100644 index 468224e0097a..000000000000 --- a/recipes/deepseek-r1/sglang/disagg-16gpu/deploy.yaml +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: sgl-dsr1-16gpu -spec: - envs: - - name: HF_HOME - value: /opt/model - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /opt/model - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.1 - decode: - componentType: worker - subComponentType: decode - replicas: 1 - multinode: - nodeCount: 2 - resources: - limits: - gpu: "8" - volumeMounts: - - name: model-cache - mountPoint: /opt/model - sharedMemory: - size: 80Gi - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.1 - workingDir: /sgl-workspace/dynamo - command: - - python3 - - -m - - dingo.sglang - args: - - --model-path - - deepseek-ai/DeepSeek-R1 - - --served-model-name - - deepseek-ai/DeepSeek-R1 - - --tp - - "16" - - --dp - - "16" - - --enable-dp-attention - - --ep-size - - "16" - - --trust-remote-code - - --skip-tokenizer-init - - --disaggregation-mode - - decode - - --disaggregation-bootstrap-port - - "30001" - - --mem-fraction-static - - "0.75" - - --host - - 0.0.0.0 - - --prefill-round-robin-balance - - --watchdog-timeout - - "3600" - prefill: - componentType: worker - subComponentType: prefill - replicas: 1 - multinode: - nodeCount: 2 - resources: - limits: - gpu: "8" - volumeMounts: - - name: model-cache - mountPoint: /opt/model - sharedMemory: - size: 80Gi - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.1 - workingDir: /sgl-workspace/dynamo - command: - - python3 - - -m - - dingo.sglang - args: - - --model-path - - deepseek-ai/DeepSeek-R1 - - --served-model-name - - deepseek-ai/DeepSeek-R1 - - --tp - - "16" - - --ep-size - - "16" - - --trust-remote-code - - --skip-tokenizer-init - - --disaggregation-mode - - prefill - - --disaggregation-bootstrap-port - - "30001" - - --mem-fraction-static - - "0.75" - - --host - - 0.0.0.0 - - --load-balance-method - - round_robin - - --watchdog-timeout - - "3600" \ No newline at end of file diff --git a/recipes/deepseek-r1/sglang/disagg-8gpu/deploy.yaml b/recipes/deepseek-r1/sglang/disagg-8gpu/deploy.yaml deleted file mode 100644 index 41113a3af556..000000000000 --- a/recipes/deepseek-r1/sglang/disagg-8gpu/deploy.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: sgl-dsr1-8gpu -spec: - envs: - - name: HF_HOME - value: /opt/model - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /opt/model - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.1 - decode: - componentType: worker - subComponentType: decode - replicas: 1 - resources: - limits: - gpu: "8" - volumeMounts: - - name: model-cache - mountPoint: /opt/model - sharedMemory: - size: 80Gi - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.1 - workingDir: /workspace - command: - - python3 - - -m - - dingo.sglang - args: - - --model-path - - deepseek-ai/DeepSeek-R1 - - --served-model-name - - deepseek-ai/DeepSeek-R1 - - --tp - - "8" - - --dp - - "8" - - --enable-dp-attention - - --ep-size - - "8" - - --trust-remote-code - - --disaggregation-mode - - decode - - --disaggregation-bootstrap-port - - "30001" - - --mem-fraction-static - - "0.75" - - --host - - 0.0.0.0 - - --prefill-round-robin-balance - - --watchdog-timeout - - "3600" - prefill: - componentType: worker - subComponentType: prefill - replicas: 1 - resources: - limits: - gpu: "8" - volumeMounts: - - name: model-cache - mountPoint: /opt/model - sharedMemory: - size: 80Gi - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.1 - workingDir: /workspace - command: - - python3 - - -m - - dingo.sglang - args: - - --model-path - - deepseek-ai/DeepSeek-R1 - - --served-model-name - - deepseek-ai/DeepSeek-R1 - - --tp - - "8" - - --ep-size - - "8" - - --trust-remote-code - - --disaggregation-mode - - prefill - - --disaggregation-bootstrap-port - - "30001" - - --mem-fraction-static - - "0.75" - - --host - - 0.0.0.0 - - --load-balance-method - - round_robin - - --watchdog-timeout - - "3600" \ No newline at end of file diff --git a/recipes/deepseek-r1/trtllm/disagg/wide_ep/gb200/deploy.yaml b/recipes/deepseek-r1/trtllm/disagg/wide_ep/gb200/deploy.yaml deleted file mode 100644 index d40ec35a63ff..000000000000 --- a/recipes/deepseek-r1/trtllm/disagg/wide_ep/gb200/deploy.yaml +++ /dev/null @@ -1,263 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - - -# Instructions: -# 1. First, create the model cache PersistentVolumeClaim: -# kubectl apply -f model-cache.yaml -n -# 2. Download the model to the model cache: -# kubectl apply -f model-download.yaml -n -# 3. Once the above steps are complete, deploy the prefill and decode workers via this yaml: -# kubectl apply -f deploy.yaml -n -# 4. To benchmark the service, run: -# kubectl apply -f perf.yaml -n -# -# NOTE (empty /v1/models): If the frontend returns "data": [] from /v1/models, check frontend logs. -# The frontend discovery watcher may treat the workers' --model-path (a local path like -# /model-cache/deepseek-r1-fp4) as a HuggingFace model ID and fail with 404. Mounting the -# model-cache on the Frontend with HF_HOME allows the frontend to resolve local paths when -# the runtime supports it. Otherwise use a HuggingFace model ID for --model-path and HF_HOME -# on workers (with model downloaded in HF cache layout). - -# ConfigMap for prefill engine configuration -# This configuration sets up a DEP 4 prefill worker -apiVersion: v1 -kind: ConfigMap -metadata: - name: prefill-config -data: - prefill_config.yaml: | - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 1227 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - cuda_graph_config: null - print_iter_log: true - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: DEFAULT ---- - -# ConfigMap for decode engine configuration -# This configuration sets up a DEP 32 decode worker -apiVersion: v1 -kind: ConfigMap -metadata: - name: decode-config -data: - decode_config_dep32.yaml: | - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - enable_attention_dp: true - pipeline_parallel_size: 1 - max_batch_size: 32 - max_num_tokens: 32 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 384 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - dtype: fp8 - moe_config: - backend: WIDEEP - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: DEFAULT - stream_interval: 20 ---- - -# NOTE: The numNodes value should equal the total number of nodes across prefill and decode -# as specified in their respective sections below (prefill.multinode.nodeCount + decode.multinode.nodeCount). -# For autoscaling deployments, the compute domain will automatically adjust as needed. -apiVersion: resource.nvidia.com/v1beta1 -kind: ComputeDomain -metadata: - name: trtllm-test-compute-domain -spec: - numNodes: 9 - channel: - resourceClaimTemplate: - name: trtllm-test-compute-domain-channel ---- - -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: trtllm-disagg-multinode -spec: - pvcs: - - name: model-cache - create: false - envs: - - name: NCCL_MNNVL_ENABLE - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: TLLM_LOG_LEVEL - value: "info" - - name: TRTLLM_MOE_ENABLE_ALLTOALL_WITHOUT_ALLGATHER - value: "1" - - name: TRTLLM_ENABLE_PDL - value: "1" - backendFramework: trtllm - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /model-cache - extraPodSpec: - tolerations: [] - affinity: {} - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - env: - - name: HF_HOME - value: /model-cache - args: - - | - python3 -m dingo.frontend --http-port 8000 - command: - - /bin/sh - - -c - prefill: - componentType: worker - subComponentType: prefill - replicas: 1 - # NOTE: Prefill uses 1 node (no multinode section = single node) - # and contributes to ComputeDomain.numNodes (see above) - volumeMounts: - - name: model-cache - mountPoint: /model-cache - sharedMemory: - size: 800Gi - resources: - requests: - cpu: "130" - memory: "850Gi" - limits: - cpu: "130" - memory: "850Gi" - gpu: "4" - claims: - - name: compute-domain-channel - extraPodSpec: - tolerations: [] - affinity: {} - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - workingDir: /workspace/components/backends/trtllm - # NOTE: If your PVCs (Persistent Volume Claims) are really slow, - # you might need to increase 'failureThreshold' below to allow more time for startup - startupProbe: - httpGet: - path: /live - port: 9090 - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 600 - volumeMounts: - - name: prefill-config-volume - mountPath: /config - command: - - /bin/sh - - -c - args: - - >- - python3 -m dynamo.trtllm - --model-path /model-cache/deepseek-r1-fp4 - --served-model-name deepseek-ai/DeepSeek-R1 - --extra-engine-args /config/prefill_config.yaml - --disaggregation-mode prefill - resourceClaims: - - name: compute-domain-channel - resourceClaimTemplateName: trtllm-test-compute-domain-channel - volumes: - - name: prefill-config-volume - configMap: - name: prefill-config - decode: - componentType: worker - subComponentType: decode - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /model-cache - multinode: - # NOTE: This nodeCount contributes to ComputeDomain.numNodes (see above) - nodeCount: 8 - sharedMemory: - size: 800Gi - resources: - requests: - cpu: "130" - memory: "850Gi" - limits: - cpu: "130" - memory: "850Gi" - gpu: "4" - claims: - - name: compute-domain-channel - extraPodSpec: - tolerations: [] - affinity: {} - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - workingDir: /workspace/components/backends/trtllm - # NOTE: If your PVCs (Persistent Volume Claims) are really slow, - # you might need to increase 'failureThreshold' below to allow more time for startup - startupProbe: - httpGet: - path: /live - port: 9090 - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 600 - volumeMounts: - - name: decode-config-volume - mountPath: /config - command: - - /bin/sh - - -c - args: - - >- - python3 -m dynamo.trtllm - --model-path /model-cache/deepseek-r1-fp4 - --served-model-name deepseek-ai/DeepSeek-R1 - --extra-engine-args /config/decode_config_dep32.yaml - --disaggregation-mode decode - resourceClaims: - - name: compute-domain-channel - resourceClaimTemplateName: trtllm-test-compute-domain-channel - volumes: - - name: decode-config-volume - configMap: - name: decode-config diff --git a/recipes/deepseek-r1/trtllm/disagg/wide_ep/gb200/perf.yaml b/recipes/deepseek-r1/trtllm/disagg/wide_ep/gb200/perf.yaml deleted file mode 100644 index 47cfbfd87b3f..000000000000 --- a/recipes/deepseek-r1/trtllm/disagg/wide_ep/gb200/perf.yaml +++ /dev/null @@ -1,153 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: deepseek-r1-bench -spec: - backoffLimit: 1 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: deepseek-r1-bench - spec: - tolerations: [] - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: nvidia.com/dynamo-graph-deployment-name - operator: In - values: - - trtllm-disagg-multinode - topologyKey: kubernetes.io/hostname - containers: - - command: - - /bin/sh - - -c - - | - apt-get update && apt-get install -y curl jq procps git && apt-get clean - pip install aiperf==0.10.0; - echo "aiperf installation completed"; - sysctl -w net.ipv4.ip_local_port_range="1024 65000" - cat /proc/sys/net/ipv4/ip_local_port_range - export COLUMNS=200 - EPOCH=$(date +%s) - ## utility functions -- can be moved to a bash script / configmap - wait_for_model_ready() { - echo "Waiting for model '$TARGET_MODEL' at $ENDPOINT/v1/models (checking every 5s)..." - while ! curl -s "http://$ENDPOINT/v1/models" | jq -e --arg model "$TARGET_MODEL" '.data[]? | select(.id == $model)' >/dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready yet, sleeping 5s before checking again http://$ENDPOINT/v1/models" - sleep 5 - done - echo "✅ Model '$TARGET_MODEL' is now available!" - echo "Model '$TARGET_MODEL' is now available!" - curl -s "http://$ENDPOINT/v1/models" | jq . - } - run_perf() { - local concurrency=$1 - local isl=$2 - local osl=$3 - key=concurrency_${concurrency} - export ARTIFACT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/${key}" - mkdir -p "$ARTIFACT_DIR" - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - aiperf profile --artifact-dir $ARTIFACT_DIR \ - --model $TARGET_MODEL \ - --tokenizer /model-cache/deepseek-r1-fp4 \ - --endpoint-type chat \ - --endpoint /v1/chat/completions \ - --streaming \ - --url http://$ENDPOINT \ - --synthetic-input-tokens-mean $isl \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean $osl \ - --output-tokens-stddev 0 \ - --extra-inputs "max_tokens:$osl" \ - --extra-inputs "min_tokens:$osl" \ - --extra-inputs "ignore_eos:true" \ - --extra-inputs "repetition_penalty:1.0" \ - --extra-inputs "temperature: 0.0" \ - --concurrency $concurrency \ - --request-count $((10*concurrency)) \ - --warmup-request-count $concurrency \ - --num-dataset-entries 12800 \ - --random-seed 100 \ - --workers-max 252 \ - -H 'Authorization: Bearer NOT USED' \ - -H 'Accept: text/event-stream'\ - --record-processors 32 \ - --ui simple - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - ls -la $ARTIFACT_DIR - } - #### Actual execution #### - wait_for_model_ready - mkdir -p "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - # Calculate total concurrency based on per-GPU concurrency and GPU count - TOTAL_CONCURRENCY=$((CONCURRENCY_PER_GPU * DEPLOYMENT_GPU_COUNT)) - echo "Calculated total concurrency: $TOTAL_CONCURRENCY (${CONCURRENCY_PER_GPU} per GPU × ${DEPLOYMENT_GPU_COUNT} GPUs)" - # Write input_config.json - cat > "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/input_config.json" < - -### DeepSeek-R1 with vLLM — Disaggregated on 32x Hopper - -This recipe deploys DeepSeek-R1 using vLLM in a disaggregated prefill/decode setup across four Hopper nodes (32 GPUs total: 16 for prefill, 16 for decode). - -- Model cache PVC + download job: `recipes/deepseek-r1/model-cache/` -- Deployment manifest: `recipes/deepseek-r1/vllm/disagg/deploy_hopper_16gpu.yaml` - -### 0) Prerequisites: Install the platform - -Follow the Kubernetes deployment guide to install the Dynamo platform and prerequisites (CRDs/operator, etc.): -- `docs/kubernetes/README.md` - -Ensure you have a GPU-enabled cluster with sufficient capacity (32x H100/H200 "Hopper" across 4 nodes), and that the NVIDIA GPU Operator is healthy. - -### 1) Set namespace - -```bash -export NAMESPACE=dynamo-system -kubectl create namespace ${NAMESPACE} || true -``` - -### 2) Apply Hugging Face secret - -Edit your HF token into the provided secret and apply: - -```bash -# Option A: Apply YAML (edit the file to set your token) -kubectl apply -f ../../hf_hub_secret/hf_hub_secret.yaml -n ${NAMESPACE} - -# Option B: Create directly -# kubectl create secret generic hf-token-secret \ -# --from-literal=HF_TOKEN="" \ -# -n ${NAMESPACE} -``` - -### 3) Provision model cache and download models - -Update `storageClassName` in `recipes/deepseek-r1/model-cache/model-cache.yaml` to match your cluster, then apply: - -```bash -# PVC for model cache -# Ensure storageClassName in model-cache.yaml matches an available StorageClass on your cluster -kubectl apply -f ../../../deepseek-r1/model-cache/model-cache.yaml -n ${NAMESPACE} - -# Download DeepSeek-R1 weights into the cache -kubectl apply -f ../../../deepseek-r1/model-cache/model-download.yaml -n ${NAMESPACE} - -# Wait for download job to finish -kubectl wait --for=condition=Complete job/model-download -n ${NAMESPACE} --timeout=6000s -``` - -This will populate: -- `/model-cache/deepseek-r1` -- `/model-cache/deepseek-r1-fp4` - -### 4) Deploy vLLM (Disaggregated, 16-way Data-Expert Parallel) - -Apply the multi-node disaggregated deployment: - -```bash -kubectl apply -f ./deploy_hopper_16gpu.yaml -n ${NAMESPACE} -``` - -The manifest runs separate prefill and decode workers across multiple nodes, each mounting the shared model cache, with settings tuned for Hopper GPUs. - -Test the deployment locally by port-forwarding and sending a request: - -```bash -# Port-forward the frontend Service to localhost:8000 (replace with the actual Service name) -kubectl port-forward svc/vllm-dsr1-frontend 8000:8000 -n ${NAMESPACE} & -``` - -```bash -curl -sS http://localhost:8000/v1/chat/completions \ - -H 'Content-Type: application/json' \ - -H 'Authorization: Bearer dummy' \ - -d '{ - "model": "deepseek-ai/DeepSeek-R1", - "messages": [{"role":"user","content":"Say hello!"}], - "max_tokens": 64 - }' -``` - - - -### Notes -- For more details on expert parallel and advanced deployment configurations, refer to [vLLM Expert Parallel Deployment Documentation](https://docs.vllm.ai/en/latest/serving/expert_parallel_deployment/). -- If your cluster/network requires specific interfaces, adjust environment variables (e.g., `NCCL_SOCKET_IFNAME`) in the manifest accordingly. -- If your storage class differs, update `storageClassName` before applying the PVC. -- **If you want to run multinode deployments, IBGDA (InfiniBand GPU Direct Async) must be enabled on your nodes.** To enable IBGDA, you can follow this configuration script: [configure_system_drivers.sh](https://github.com/vllm-project/vllm/blob/v0.11.2/tools/ep_kernels/configure_system_drivers.sh). The script configures NVIDIA driver parameters and requires a system reboot to take effect. -- `VLLM_MOE_DP_CHUNK_SIZE` can be tuned further. The value 384 was chosen to be largest possible that still can be deployed on 16 H200s. This value should be greater than per rank concurrency. - - diff --git a/recipes/deepseek-r1/vllm/disagg/deploy_hopper_16gpu.yaml b/recipes/deepseek-r1/vllm/disagg/deploy_hopper_16gpu.yaml deleted file mode 100644 index 33f120d14f15..000000000000 --- a/recipes/deepseek-r1/vllm/disagg/deploy_hopper_16gpu.yaml +++ /dev/null @@ -1,171 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: vllm-dsr1 -spec: - backendFramework: vllm - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /model-cache - extraPodSpec: - mainContainer: - startupProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 10 - timeoutSeconds: 1800 - failureThreshold: 60 - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - decode: - componentType: worker - subComponentType: decode - replicas: 1 - multinode: - nodeCount: 2 - resources: - limits: - gpu: "8" - custom: - rdma/ib: "8" - volumeMounts: - - name: model-cache - mountPoint: /model-cache - sharedMemory: - size: 80Gi - extraPodSpec: - mainContainer: - startupProbe: - httpGet: - path: /health - port: 9090 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 600 - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - workingDir: /workspace/dynamo - env: - - name: VLLM_USE_DEEP_GEMM - value: "1" - - name: VLLM_MOE_DP_CHUNK_SIZE - value: "384" - - name: VLLM_SKIP_P2P_CHECK - value: "1" - - name: VLLM_RANDOMIZE_DP_DUMMY_INPUTS - value: "1" - - name: NVIDIA_GDRCOPY - value: enabled - - name: GLOO_SOCKET_IFNAME - value: eth0 - command: - - python3 - - -m - - dingo.vllm - args: - - --model - - /model-cache/deepseek-r1 - - --served-model-name - - deepseek-ai/DeepSeek-R1 - - --all2all-backend - - deepep_low_latency - - --data-parallel-hybrid-lb - - --tensor-parallel-size - - "1" - - --data-parallel-size - - "16" - - --enable-expert-parallel - - --max-model-len - - "16384" - - --enable-dbo - - --dbo-decode-token-threshold - - "32" - - --async-scheduling - - --enable-eplb - - --eplb-config - - '{"window_size":"1000","step_interval":"3000","num_redundant_experts":"32","log_balancedness":"False"}' - - --max-num-seqs - - "512" - - --compilation_config - - '{"pass_config":{"fuse_norm_quant":true,"eliminate_noops":true},"cudagraph_mode":"FULL_DECODE_ONLY"}' - - --kv-transfer-config - - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - prefill: - componentType: worker - subComponentType: prefill - replicas: 1 - multinode: - nodeCount: 2 - resources: - limits: - gpu: "8" - custom: - rdma/ib: "8" - volumeMounts: - - name: model-cache - mountPoint: /model-cache - sharedMemory: - size: 80Gi - extraPodSpec: - mainContainer: - startupProbe: - httpGet: - path: /health - port: 9090 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 600 - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - workingDir: /workspace/dynamo - env: - - name: VLLM_USE_DEEP_GEMM - value: "1" - - name: VLLM_SKIP_P2P_CHECK - value: "1" - - name: VLLM_RANDOMIZE_DP_DUMMY_INPUTS - value: "1" - - name: NVIDIA_GDRCOPY - value: enabled - - name: GLOO_SOCKET_IFNAME - value: eth0 - command: - - python3 - - -m - - dingo.vllm - args: - - --model - - /model-cache/deepseek-r1 - - --disaggregation-mode - - prefill - - --kv-transfer-config - - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - - --served-model-name - - deepseek-ai/DeepSeek-R1 - - --all2all-backend - - deepep_high_throughput - - --data-parallel-hybrid-lb - - --tensor-parallel-size - - "1" - - --data-parallel-size - - "16" - - --enable-expert-parallel - - --max-model-len - - "16384" - - --enable-dbo - - --dbo-decode-token-threshold - - "32" - - --async-scheduling - - --enable-eplb - - --eplb-config - - '{"window_size":"1000","step_interval":"3000","num_redundant_experts":"32","log_balancedness":"False"}' - - --max-num-seqs - - "512" diff --git a/recipes/deepseek-v32-fp4/README.md b/recipes/deepseek-v32-fp4/README.md deleted file mode 100644 index 1db601ab7bf2..000000000000 --- a/recipes/deepseek-v32-fp4/README.md +++ /dev/null @@ -1,200 +0,0 @@ -# DeepSeek V3.2 NVFP4: Aggregated Round Robin vs Disaggregated KV Routing with WideEP - -This **GB200 NVL72** recipe for DeepSeek V3.2 demonstrates the performance difference between **aggregated (round-robin) routing** and **disaggregated (KV-aware) routing + WideEP** on a synthetic trace dataset adapted from the [Mooncake FAST25 paper](https://github.com/kvcache-ai/Mooncake). - -## Results - -https://github.com/user-attachments/assets/fcdb703c-7c1a-4109-a7ca-54196fcef885 - -## Experiment Overview - -We compare two deployment modes on **32x GB200 GPUs across 8 nodes**: - -| Mode | Routing | Configuration | -|------|---------|---------------| -| **Aggregated** | Round-robin | 4x DEP8 workers | -| **Disaggregated** | KV-aware | 2x prefill + 2x decode w/ WideEP (DEP8) | - -## Dataset: Mooncake-based Synthetic Coding Trace - -The benchmark uses a trace which simulates coding workloads. We synthesize the trace by increasing the input sequence length and prefix reuse rate of the original [Mooncake conversation trace](https://github.com/kvcache-ai/Mooncake/blob/main/FAST25-release/traces/conversation_trace.jsonl). - -To reproduce our benchmark, run Dynamo's [prefix data generator tool](https://github.com/ai-dynamo/dynamo/tree/main/benchmarks/prefix_data_generator) on the Mooncake `conversation_trace.jsonl`: -```bash -datagen synthesize \ - --input-file conversation_trace.jsonl \ - --prefix-len-multiplier 16 \ - --prompt-len-multiplier 10 \ - --max-isl 110000 \ - --num-requests 10000 -# synthesizes `conversation_trace_synth_16.00x1+10.00_speedup1_maxisl110000.jsonl` -``` - -The ISL/OSL/cache hit statistics of our trace is below. - -
-Dataset statistics: Mooncake-based Synthetic Trace - -``` -============================================================ - DATASET ANALYSIS: Mooncake-based Synthetic Trace - ============================================================ - OVERVIEW - ---------------------------------------- - Total Requests: 10,000 - Unique Hash Blocks: 430,838 - Total Hash Blocks: 770,934 - INPUT SEQUENCE LENGTH (ISL) - ---------------------------------------- - Average: 39,186 tokens - Maximum: 109,459 tokens - Minimum: 12,801 tokens - OUTPUT SEQUENCE LENGTH (OSL) - ---------------------------------------- - Average: 344 tokens - Maximum: 2,000 tokens - Minimum: 1 tokens - KV CACHE / PREFIX REUSE - ---------------------------------------- - Block-level Hit Rate: 44.1% - Token-level Hit Rate: 44.0% - Avg Context (shared): 22,400 tokens/req - Avg Unique Prompt: 16,786 tokens/req - Shared Prefix Ratio: 57.2% - ============================================================ - - Summary: - • ~44% KV cache hit rate (block/token level) based on hash_id overlap across requests - • ~57% of input tokens come from shared context prefixes - • Long-context workload: avg 39K input tokens, up to 109K max -``` - -
- - -## Prerequisites - -1. **Dynamo Platform installed** - See [Kubernetes Deployment Guide](../../docs/kubernetes/README.md) -2. **32x GB200 GPUs** across 8 nodes -3. **HuggingFace token** configured: - ```bash - export NAMESPACE=your-namespace - kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="your-token" \ - -n ${NAMESPACE} - ``` - -## Quick Start - -### 1. Create Storage - -> **Note:** Edit `model-cache/model-cache.yaml` first and update `storageClassName` to match your cluster (run `kubectl get storageclass` to find available options). - -```bash -kubectl apply -f model-cache/model-cache.yaml -n ${NAMESPACE} -``` - -### 2. Configure K8 Benchmarking Environment -For multinode kubernetes deployments, your cluster may require a ComputeDomain to exist in your namespace such that the DRA scheduler can co-locate worker pods on MNNVL-connected nodes. (Otherwise, internode GPU peer memory access would fail.) -```bash -kubectl apply -f model-cache/compute-domain.yaml -n ${NAMESPACE} -``` -Make sure to apply any name modifications to this file to the deployment yamls, under `extraPodSpec.resourceClaims` and `mainContainer.resources.claims`. - - -### 3. Setup Model and Data -We use NVIDIA's official NVFP4-quantized checkpoint ([Huggingface](https://huggingface.co/nvidia/DeepSeek-V3.2-NVFP4)). Copy it into the PVC storage: - -```bash -kubectl apply -f model-cache/model-download.yaml -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/model-download -n ${NAMESPACE} --timeout=600s -``` -Similarly, copy the trace file for the benchmark into the PVC: -```bash -# conversation_trace_synth_16.00x1+10.00_speedup1_maxisl110000.jsonl in our case -kubectl cp your-namespace/:/model-cache/traces/ -``` - -### 4. Deploy & Benchmark - -**Option A: Aggregated (Round-Robin Baseline)** - -```bash -# Deploy -kubectl apply -f trtllm/agg-round-robin/deploy.yaml -n ${NAMESPACE} - -# Wait for ready -kubectl wait --for=condition=ready pod -l nvidia.com/dynamo-graph-deployment-name=agg-round-robin-dsv32-nvfp4 \ - -n ${NAMESPACE} --timeout=1200s - -# Run benchmark -kubectl apply -f trtllm/agg-round-robin/perf.yaml -n ${NAMESPACE} -``` - -**Option B: Disaggregated (KV-Aware Routing)** - -```bash -# Deploy -kubectl apply -f trtllm/disagg-kv-router/deploy.yaml -n ${NAMESPACE} - -# Wait for ready -kubectl wait --for=condition=ready pod -l nvidia.com/dynamo-graph-deployment-name=disagg-kv-dsv32-nvfp4 \ - -n ${NAMESPACE} --timeout=1200s - -# Run benchmark -kubectl apply -f trtllm/disagg-kv-router/perf.yaml -n ${NAMESPACE} -``` - -### 4. Monitor Benchmark Progress - -The benchmark runs inside a tmux session for easy monitoring: - -```bash -# Find the benchmark pod -kubectl get pods -n ${NAMESPACE} | grep benchmark - -# Attach to the tmux session to see intermediate results -kubectl exec -it -n ${NAMESPACE} -- tmux a -t benchmark - -# Detach from tmux: Ctrl+B, then D -``` - -### 5. View Results - -Results are saved to the `perf-cache` PVC: - -```bash -# Check artifact directory -kubectl exec -it -n ${NAMESPACE} -- ls -la /perf-cache/artifacts/ - -# Copy results to local machine -kubectl cp ${NAMESPACE}/:/perf-cache/artifacts ./benchmark-results -``` - -## Expected Results - -Since the benchmark uses `--fixed-schedule` (replaying requests at their original timestamps), **throughput metrics are fixed by the trace**—latency metrics are what we're comparing: - -| Metric | Why It Matters | -|--------|----------------| -| **TTFT** (Time to First Token) | KV-aware routing reduces prefill compute via prefix cache hits | -| **ITL** (Inter-Token Latency) | Disaggregated serving isolates decode from prefill interference | -| **Total Request Latency** | Combined benefit of both optimizations | - -For production contexts, we can further evaluate the deployments with **goodput**, i.e. the rate of requests which satisfy a predetermined service level agreement (SLA). For our experiments, we set the SLA as TTFT=20s and ITL=50ms. - -## Cleanup - -```bash -# Delete benchmark pods -kubectl delete job agg-round-robin-dsv32-nvfp4-bench disagg-kv-dsv32-nvfp4-bench -n ${NAMESPACE} - -# Delete deployments -kubectl delete dynamographdeployment agg-round-robin-dsv32-nvfp4 -n ${NAMESPACE} -kubectl delete dynamographdeployment disagg-kv-dsv32-nvfp4 -n ${NAMESPACE} -``` - -## References - -- [Mooncake: A KVCache-centric Disaggregated Architecture for LLM Serving](https://github.com/kvcache-ai/Mooncake) - FAST25 paper and trace data -- [Optimizing DeepSeek-V3.2 on NVIDIA Blackwell GPUs](https://nvidia.github.io/TensorRT-LLM/blogs/tech_blog/blog15_Optimizing_DeepSeek_V32_on_NVIDIA_Blackwell_GPUs.html) - TRTLLM tech blog on available optimizations for DSV3.2 on GB200 diff --git a/recipes/deepseek-v32-fp4/model-cache/compute-domain.yaml b/recipes/deepseek-v32-fp4/model-cache/compute-domain.yaml deleted file mode 100644 index 8f45db8b1f3d..000000000000 --- a/recipes/deepseek-v32-fp4/model-cache/compute-domain.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: resource.nvidia.com/v1beta1 -kind: ComputeDomain -metadata: - name: your-compute-domain - namespace: your-namespace -spec: - # 0 = on-demand allocation (nodes assigned when pods request them via resourceClaims). - numNodes: 0 - channel: - resourceClaimTemplate: - name: your-compute-domain-channel \ No newline at end of file diff --git a/recipes/deepseek-v32-fp4/model-cache/model-cache.yaml b/recipes/deepseek-v32-fp4/model-cache/model-cache.yaml deleted file mode 100644 index 82fd38a9a3fd..000000000000 --- a/recipes/deepseek-v32-fp4/model-cache/model-cache.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 400Gi - storageClassName: "your-storage-class-name" \ No newline at end of file diff --git a/recipes/deepseek-v32-fp4/model-cache/model-download.yaml b/recipes/deepseek-v32-fp4/model-cache/model-download.yaml deleted file mode 100644 index 85ead5fa74ef..000000000000 --- a/recipes/deepseek-v32-fp4/model-cache/model-download.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - seccompProfile: - type: RuntimeDefault - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: MODEL_NAME - value: nvidia/DeepSeek-V3.2-NVFP4 - - name: HF_HOME - value: /model-store - # Uses up to 64 GB RAM for XET buffers; set "0" on low-memory nodes (8 GB cap): https://huggingface.co/docs/hub/en/xet/using-xet-storage#download-buffers - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.11.0 - hf download $MODEL_NAME - resources: - requests: - cpu: "2" - memory: "64Gi" - limits: - cpu: "8" - memory: "64Gi" - volumeMounts: - - name: model-cache - mountPath: /model-store - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/deepseek-v32-fp4/trtllm/agg-round-robin/deploy.yaml b/recipes/deepseek-v32-fp4/trtllm/agg-round-robin/deploy.yaml deleted file mode 100644 index ed13d379738c..000000000000 --- a/recipes/deepseek-v32-fp4/trtllm/agg-round-robin/deploy.yaml +++ /dev/null @@ -1,172 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: agg-round-robin-dsv32-nvfp4 -spec: - services: - Frontend: - componentType: frontend - extraPodSpec: - containers: null - mainContainer: - command: - - python3 - args: - - -m - - dingo.frontend - - --router-mode - - round-robin - - --router-reset-states - - --request-plane - - nats - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - name: "" - resources: {} - # tolerations: # uncomment to populate any tolerations for the gpu nodes - replicas: 1 - agg: - componentType: worker - envFromSecret: hf-token-secret - extraPodSpec: - containers: null - mainContainer: - args: - - --model-path - - nvidia/DeepSeek-V3.2-NVFP4 - - --served-model-name - - nvidia/DeepSeek-V3.2-NVFP4 - - --extra-engine-args - - /config/aggregated.yaml - - --publish-events-and-metrics - - --request-plane - - nats - - --kv-block-size - - "64" - command: - - python3 - - -m - - dynamo.trtllm - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: TRITON_CACHE_DIR - value: /model-cache/.triton-cache - - name: NCCL_DEBUG - value: INFO - - name: NCCL_MNNVL_ENABLE - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: NVIDIA_GDRCOPY - value: "1" - - name: UCX_CUDA_IPC_ENABLE_MNNVL - value: "1" - - name: NCCL_SOCKET_IFNAME - value: eth0 - - name: GLOO_SOCKET_IFNAME - value: eth0 - - name: NCCL_STORE_TIMEOUT - value: "7200" - - name: TRTLLM_MOE_ENABLE_ALLTOALL_WITHOUT_ALLGATHER - value: "1" - - name: TRTLLM_ENABLE_PDL - value: "1" - - name: TRTLLM_SERVER_DISABLE_GC - value: "1" - - name: TRTLLM_WORKER_DISABLE_GC - value: "1" - - name: NCCL_GRAPH_MIXING_SUPPORT - value: "0" - - name: TRTLLM_FORCE_COMM_METHOD - value: NVLINK_TWO_SIDED - - name: ENABLE_CONFIGURABLE_MOE - value: "1" - - name: TLLM_LOG_LEVEL - value: "INFO" - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - name: "" - resources: {} - securityContext: - runAsUser: 0 - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 5 - volumeMounts: - - mountPath: /model-cache - name: model-cache - - mountPath: /config - name: trtllm-config - readOnly: true - workingDir: /workspace/ - nodeSelector: - kubernetes.io/arch: arm64 - #tolerations: : # uncomment to populate any tolerations for the gpu nodes - resourceClaims: - - name: compute-domain-channel - resourceClaimTemplateName: your-compute-domain-channel - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: trtllm-config - configMap: - name: dsv32-trtllm-config - multinode: - nodeCount: 2 - replicas: 4 - resources: - limits: - gpu: "4" - claims: - - name: compute-domain-channel ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: dsv32-trtllm-config -data: - aggregated.yaml: | - allreduce_strategy: MNNVL - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 120000 - max_num_tokens: 8192 - enable_chunked_prefill: true - disable_overlap_scheduler: true - cuda_graph_config: - max_batch_size: 8 - enable_padding: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - tokens_per_block: 64 - max_batch_size: 8 - max_seq_len: 121000 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 8 - print_iter_log: true - stream_interval: 10 - tensor_parallel_size: 8 \ No newline at end of file diff --git a/recipes/deepseek-v32-fp4/trtllm/agg-round-robin/perf.yaml b/recipes/deepseek-v32-fp4/trtllm/agg-round-robin/perf.yaml deleted file mode 100644 index 47779d9d40fa..000000000000 --- a/recipes/deepseek-v32-fp4/trtllm/agg-round-robin/perf.yaml +++ /dev/null @@ -1,204 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# AIPerf trace-replay benchmark for DeepSeek-V3.2 NVFP4 (agg). -# -# Replays requests from a Mooncake-format trace file at their original timestamps -# using aiperf --custom-dataset-type mooncake_trace --fixed-schedule. -# -# Prerequisites: -# - DGD deployed and in "normal" or "successful" state -# - model-cache PVC exists in your namespace -# - Trace file copied to PVC: /model-cache/traces/.jsonl -# -# Results: /model-cache/perf/_/ -# -apiVersion: batch/v1 -kind: Job -metadata: - name: agg-round-robin-dsv32-nvfp4-bench - namespace: your-namespace -spec: - backoffLimit: 1 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: agg-round-robin-dsv32-nvfp4-bench - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: nvidia.com/dynamo-graph-deployment-name - operator: In - values: - - agg-round-robin-dsv32-nvfp4 - topologyKey: kubernetes.io/hostname - containers: - - command: - - /bin/bash - - -c - - | - set -euo pipefail - ulimit -n 600000 - echo "File descriptor limit set to: $(ulimit -n)" - echo 2097152 > /proc/sys/fs/inotify/max_user_watches 2>/dev/null || true - echo 1024 > /proc/sys/fs/inotify/max_user_instances 2>/dev/null || true - apt-get update && apt-get install -y curl jq procps git && apt-get clean - # pip install git+https://github.com/ai-dynamo/aiperf.git - # Pin transformers==4.57.6 (verified to load the deepseek_v32 tokenizer). - # The trtllm-runtime base ships transformers 4.55.0; aiperf 0.6.0's - # `transformers>=4.56.0` floor would otherwise upgrade to 5.x, which lacks - # native support for model_type=deepseek_v32 (huggingface/transformers#41251) - # and fails AutoTokenizer.from_pretrained() before reading tokenizer.json. - # aiperf surfaces this as "Failed to load tokenizer". DYN-2878. - pip install "aiperf==0.10.0" "transformers==4.57.6" - echo "aiperf installation completed" - sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true - export COLUMNS=200 - EPOCH=$(date +%s) - - wait_for_model_ready() { - echo "Waiting for model '$TARGET_MODEL' at $ENDPOINT/v1/models (checking every 5s)..." - while ! curl -sf "http://$ENDPOINT/v1/models" | jq -e --arg model "$TARGET_MODEL" '.data[]? | select(.id == $model)' >/dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready yet, sleeping 5s..." - sleep 5 - done - echo "Model '$TARGET_MODEL' is now available!" - curl -s "http://$ENDPOINT/v1/models" | jq . - } - - wait_for_model_ready - mkdir -p "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - - # Validate trace file - if [ ! -f "${TRACE_FILE}" ]; then - echo "ERROR: Trace file not found: ${TRACE_FILE}" - echo "Copy trace to PVC first: kubectl cp your-namespace/:/model-cache/traces/" - exit 1 - fi - TRACE_LINES=$(wc -l < "${TRACE_FILE}") - echo "Trace contains ${TRACE_LINES} requests" - - printf '{"deployment":"agg-round-robin-dsv32-nvfp4","model":"%s","trace_file":"%s","trace_requests":%d,"ttft_threshold_ms":%s,"itl_threshold_ms":%s,"endpoint":"%s"}\n' \ - "nvidia/DeepSeek-V3.2-NVFP4" "${TRACE_FILE}" "${TRACE_LINES}" "${TTFT_THRESHOLD_MS}" "${ITL_THRESHOLD_MS}" "${ENDPOINT}" \ - > "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/input_config.json" - - TRACE_BASE_NAME="$(basename "${TRACE_FILE}" .jsonl)" - export ARTIFACT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/${TRACE_BASE_NAME}" - mkdir -p "$ARTIFACT_DIR" - - # Server metrics args - SERVER_METRICS_ARGS=() - if [ -n "${AIPERF_SERVER_METRICS_URLS:-}" ]; then - IFS=',' read -r -a server_metrics_urls <<< "${AIPERF_SERVER_METRICS_URLS}" - if [ ${#server_metrics_urls[@]} -gt 0 ]; then - SERVER_METRICS_ARGS+=(--server-metrics "${server_metrics_urls[@]}") - fi - fi - - echo "==============================================" - echo "Trace Replay Benchmark (aiperf)" - echo "==============================================" - echo "Endpoint: http://${ENDPOINT}" - echo "Model: nvidia/DeepSeek-V3.2-NVFP4" - echo "Trace file: Mooncake-based Synthetic Coding Trace" - echo "TTFT Threshold: ${TTFT_THRESHOLD_MS}ms" - echo "ITL Threshold: ${ITL_THRESHOLD_MS}ms" - echo "Artifact dir: ${ARTIFACT_DIR}" - echo "==============================================" - - echo "" - echo "Running warmup benchmark..." - set +e - aiperf profile \ - -m "nvidia/DeepSeek-V3.2-NVFP4" \ - --tokenizer "nvidia/DeepSeek-V3.2-NVFP4" \ - --url "http://${ENDPOINT}" \ - --streaming \ - --ui dashboard \ - --synthetic-input-tokens-mean 10000 \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean 200 \ - --output-tokens-stddev 0 \ - --extra-inputs "max_tokens:200" \ - --extra-inputs "min_tokens:200" \ - --extra-inputs "ignore_eos:true" \ - --concurrency 4 \ - --request-count 10 - echo "Warmup complete" - - # Trace replay - echo "" - echo "$(date '+%Y-%m-%d %H:%M:%S') - Starting trace replay benchmark" - aiperf profile \ - -m "nvidia/DeepSeek-V3.2-NVFP4" \ - --tokenizer "nvidia/DeepSeek-V3.2-NVFP4" \ - --input-file "${TRACE_FILE}" \ - --custom-dataset-type mooncake_trace \ - --fixed-schedule \ - --url "http://${ENDPOINT}" \ - --streaming \ - --random-seed 42 \ - --ui dashboard \ - --artifact-dir "${ARTIFACT_DIR}" \ - --workers-max 200 \ - --request-timeout-seconds 1000 \ - --profile-export-level records \ - --record-processors 8 \ - "${SERVER_METRICS_ARGS[@]}" \ - --goodput "time_to_first_token:${TTFT_THRESHOLD_MS} inter_token_latency:${ITL_THRESHOLD_MS}" - - BENCH_EXIT_CODE=$? - echo "" - echo "$(date '+%Y-%m-%d %H:%M:%S') - Benchmark complete (exit code: ${BENCH_EXIT_CODE})" - echo "Results: ${ARTIFACT_DIR}" - ls -la "${ARTIFACT_DIR}" 2>/dev/null || true - echo "Benchmark complete!" - exit $BENCH_EXIT_CODE - set -e - env: - - name: TARGET_MODEL - value: nvidia/DeepSeek-V3.2-NVFP4 - - name: ENDPOINT - value: agg-round-robin-dsv32-nvfp4-frontend:8000 - - name: TRACE_FILE - value: /model-cache/traces/conversation_trace_synth_16.00x1+10.00_speedup1_maxisl110000.jsonl - - name: TTFT_THRESHOLD_MS - value: "20000" - - name: ITL_THRESHOLD_MS - value: "50" - - name: AIPERF_HTTP_CONNECTION_LIMIT - value: "200" - - name: AIPERF_HTTP_SO_RCVTIMEO - value: "120" - - name: AIPERF_SERVER_METRICS_URLS - value: "http://agg-round-robin-dsv32-nvfp4-dec-0-dec-wkr:9090/metrics,http://agg-round-robin-dsv32-nvfp4-prefill-0:9090/metrics" - - name: JOB_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.labels['job-name'] - - name: ROOT_ARTIFACT_DIR - value: /model-cache/perf - - name: HF_HOME - value: /model-cache - - name: PYTHONUNBUFFERED - value: "1" - image: python:3.12-slim - imagePullPolicy: IfNotPresent - name: perf - securityContext: - privileged: true - volumeMounts: - - name: model-cache - mountPath: /model-cache - workingDir: /workspace - restartPolicy: Never - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/deepseek-v32-fp4/trtllm/disagg-kv-router/deploy.yaml b/recipes/deepseek-v32-fp4/trtllm/disagg-kv-router/deploy.yaml deleted file mode 100644 index efb236364430..000000000000 --- a/recipes/deepseek-v32-fp4/trtllm/disagg-kv-router/deploy.yaml +++ /dev/null @@ -1,318 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: disagg-kv-dsv32-nvfp4 -spec: - services: - Frontend: - componentType: frontend - extraPodSpec: - containers: null - mainContainer: - command: - - python3 - args: - - -m - - dingo.frontend - - --router-mode - - kv - - --router-reset-states - - --request-plane - - nats - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - name: "" - resources: {} - nodeSelector: - kubernetes.io/arch: arm64 - replicas: 1 - prefill: - componentType: worker - subComponentType: prefill - envFromSecret: hf-token-secret - extraPodSpec: - containers: null - mainContainer: - args: - - --model-path - - nvidia/DeepSeek-V3.2-NVFP4 - - --served-model-name - - nvidia/DeepSeek-V3.2-NVFP4 - - --extra-engine-args - - /config/prefill.yaml - - --disaggregation-mode - - prefill - - --publish-events-and-metrics - - --request-plane - - nats - - --kv-block-size - - "64" - command: - - python3 - - -m - - dynamo.trtllm - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: TRITON_CACHE_DIR - value: /model-cache/.triton-cache - - name: NCCL_DEBUG - value: INFO - - name: NCCL_MNNVL_ENABLE - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: NVIDIA_GDRCOPY - value: "1" - - name: UCX_CUDA_IPC_ENABLE_MNNVL - value: "1" - - name: NCCL_SOCKET_IFNAME - value: eth0 - - name: GLOO_SOCKET_IFNAME - value: eth0 - - name: NCCL_STORE_TIMEOUT - value: "7200" - - name: TRTLLM_MOE_ENABLE_ALLTOALL_WITHOUT_ALLGATHER - value: "1" - - name: TRTLLM_ENABLE_PDL - value: "1" - - name: TRTLLM_SERVER_DISABLE_GC - value: "1" - - name: TRTLLM_WORKER_DISABLE_GC - value: "1" - - name: NCCL_GRAPH_MIXING_SUPPORT - value: "0" - - name: TLLM_LOG_LEVEL - value: "INFO" - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - name: "" - resources: {} - securityContext: - runAsUser: 0 - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 5 - volumeMounts: - - mountPath: /model-cache - name: model-cache - - mountPath: /config - name: trtllm-config - readOnly: true - workingDir: /workspace/ - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - effect: NoSchedule - key: dedicated - operator: Equal - value: user-workload - - effect: NoExecute - key: dedicated - operator: Equal - value: user-workload - - effect: NoSchedule - key: dedicated - operator: Equal - value: system-workload - - effect: NoExecute - key: dedicated - operator: Equal - value: system-workload - resourceClaims: - - name: compute-domain-channel - resourceClaimTemplateName: your-compute-domain-channel - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: trtllm-config - configMap: - name: dsv32-trtllm-config - multinode: - nodeCount: 2 - replicas: 2 - resources: - limits: - gpu: "4" - claims: - - name: compute-domain-channel - dec: - componentType: worker - subComponentType: decode - envFromSecret: hf-token-secret - extraPodSpec: - containers: null - mainContainer: - args: - - --model-path - - nvidia/DeepSeek-V3.2-NVFP4 - - --served-model-name - - nvidia/DeepSeek-V3.2-NVFP4 - - --extra-engine-args - - /config/decode.yaml - - --disaggregation-mode - - decode - - --publish-events-and-metrics - - --request-plane - - nats - - --kv-block-size - - "64" - command: - - python3 - - -m - - dynamo.trtllm - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: TRITON_CACHE_DIR - value: /model-cache/.triton-cache - - name: NCCL_DEBUG - value: INFO - - name: NCCL_MNNVL_ENABLE - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: NVIDIA_GDRCOPY - value: "1" - - name: UCX_CUDA_IPC_ENABLE_MNNVL - value: "1" - - name: NCCL_SOCKET_IFNAME - value: eth0 - - name: GLOO_SOCKET_IFNAME - value: eth0 - - name: NCCL_STORE_TIMEOUT - value: "7200" - - name: TRTLLM_MOE_ENABLE_ALLTOALL_WITHOUT_ALLGATHER - value: "1" - - name: TRTLLM_ENABLE_PDL - value: "1" - - name: TRTLLM_SERVER_DISABLE_GC - value: "1" - - name: TRTLLM_WORKER_DISABLE_GC - value: "1" - - name: ENROOT_ALLOW_DEV - value: "yes" - - name: NCCL_GRAPH_MIXING_SUPPORT - value: "0" - - name: TRTLLM_FORCE_COMM_METHOD - value: NVLINK_TWO_SIDED - - name: ENABLE_CONFIGURABLE_MOE - value: "1" - - name: TLLM_LOG_LEVEL - value: "INFO" - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - name: "" - resources: {} - securityContext: - runAsUser: 0 - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 5 - volumeMounts: - - mountPath: /model-cache - name: model-cache - - mountPath: /config - name: trtllm-config - readOnly: true - workingDir: /workspace/ - nodeSelector: - kubernetes.io/arch: arm64 - resourceClaims: - - name: compute-domain-channel - resourceClaimTemplateName: your-compute-domain-channel - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: trtllm-config - configMap: - name: dsv32-trtllm-config - multinode: - nodeCount: 2 - replicas: 2 - resources: - limits: - gpu: "4" - claims: - - name: compute-domain-channel ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: dsv32-trtllm-config -data: - prefill.yaml: | - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 120000 - max_num_tokens: 8192 - enable_chunked_prefill: true - cuda_graph_config: - max_batch_size: 32 - enable_padding: true - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - free_gpu_memory_fraction: 0.9 - tokens_per_block: 64 - max_batch_size: 32 - max_seq_len: 121000 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 8 - print_iter_log: true - tensor_parallel_size: 8 - decode.yaml: | - allreduce_strategy: MNNVL - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 120000 - max_num_tokens: 8192 - cuda_graph_config: - max_batch_size: 8 - enable_padding: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - tokens_per_block: 64 - max_batch_size: 8 - max_seq_len: 121000 - moe_config: - backend: WIDEEP - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 8 - print_iter_log: true - stream_interval: 10 - tensor_parallel_size: 8 \ No newline at end of file diff --git a/recipes/deepseek-v32-fp4/trtllm/disagg-kv-router/perf.yaml b/recipes/deepseek-v32-fp4/trtllm/disagg-kv-router/perf.yaml deleted file mode 100644 index dfe703a68052..000000000000 --- a/recipes/deepseek-v32-fp4/trtllm/disagg-kv-router/perf.yaml +++ /dev/null @@ -1,203 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# AIPerf trace-replay benchmark for DeepSeek-V3.2 NVFP4 (disagg). -# -# Replays requests from a Mooncake-format trace file at their original timestamps -# using aiperf --custom-dataset-type mooncake_trace --fixed-schedule. -# -# Prerequisites: -# - DGD deployed and in "normal" or "successful" state -# - model-cache PVC exists in your namespace -# - Trace file copied to PVC: /model-cache/traces/.jsonl -# -# Results: /model-cache/perf/_/ -# -apiVersion: batch/v1 -kind: Job -metadata: - name: disagg-kv-dsv32-nvfp4-bench - namespace: your-namespace -spec: - backoffLimit: 1 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: disagg-kv-dsv32-nvfp4-bench - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: nvidia.com/dynamo-graph-deployment-name - operator: In - values: - - disagg-kv-dsv32-nvfp4 - topologyKey: kubernetes.io/hostname - containers: - - command: - - /bin/bash - - -c - - | - set -euo pipefail - ulimit -n 600000 - echo "File descriptor limit set to: $(ulimit -n)" - echo 2097152 > /proc/sys/fs/inotify/max_user_watches 2>/dev/null || true - echo 1024 > /proc/sys/fs/inotify/max_user_instances 2>/dev/null || true - apt-get update && apt-get install -y curl jq procps git && apt-get clean - # pip install git+https://github.com/ai-dynamo/aiperf.git - # Pin transformers==4.57.6 (verified to load the deepseek_v32 tokenizer). - # The trtllm-runtime base ships transformers 4.55.0; aiperf 0.6.0's - # `transformers>=4.56.0` floor would otherwise upgrade to 5.x, which lacks - # native support for model_type=deepseek_v32 (huggingface/transformers#41251) - # and fails AutoTokenizer.from_pretrained() before reading tokenizer.json. - # aiperf surfaces this as "Failed to load tokenizer". DYN-2878. - pip install "aiperf==0.10.0" "transformers==4.57.6" - echo "aiperf installation completed" - sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true - export COLUMNS=200 - EPOCH=$(date +%s) - - wait_for_model_ready() { - echo "Waiting for model '$TARGET_MODEL' at $ENDPOINT/v1/models (checking every 5s)..." - while ! curl -sf "http://$ENDPOINT/v1/models" | jq -e --arg model "$TARGET_MODEL" '.data[]? | select(.id == $model)' >/dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready yet, sleeping 5s..." - sleep 5 - done - echo "Model '$TARGET_MODEL' is now available!" - curl -s "http://$ENDPOINT/v1/models" | jq . - } - - wait_for_model_ready - mkdir -p "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - - # Validate trace file - if [ ! -f "${TRACE_FILE}" ]; then - echo "ERROR: Trace file not found: ${TRACE_FILE}" - echo "Copy trace to PVC first: kubectl cp your-namespace/:/model-cache/traces/" - exit 1 - fi - TRACE_LINES=$(wc -l < "${TRACE_FILE}") - echo "Trace contains ${TRACE_LINES} requests" - - printf '{"deployment":"disagg-kv-dsv32-nvfp4","model":"%s","trace_file":"%s","trace_requests":%d,"ttft_threshold_ms":%s,"itl_threshold_ms":%s,"endpoint":"%s"}\n' \ - "nvidia/DeepSeek-V3.2-NVFP4" "${TRACE_FILE}" "${TRACE_LINES}" "${TTFT_THRESHOLD_MS}" "${ITL_THRESHOLD_MS}" "${ENDPOINT}" \ - > "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/input_config.json" - - TRACE_BASE_NAME="$(basename "${TRACE_FILE}" .jsonl)" - export ARTIFACT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/${TRACE_BASE_NAME}" - mkdir -p "$ARTIFACT_DIR" - - # Server metrics args - SERVER_METRICS_ARGS=() - if [ -n "${AIPERF_SERVER_METRICS_URLS:-}" ]; then - IFS=',' read -r -a server_metrics_urls <<< "${AIPERF_SERVER_METRICS_URLS}" - if [ ${#server_metrics_urls[@]} -gt 0 ]; then - SERVER_METRICS_ARGS+=(--server-metrics "${server_metrics_urls[@]}") - fi - fi - - echo "==============================================" - echo "Trace Replay Benchmark (aiperf)" - echo "==============================================" - echo "Endpoint: http://${ENDPOINT}" - echo "Model: nvidia/DeepSeek-V3.2-NVFP4" - echo "Trace file: Mooncake-based Synthetic Coding Trace" - echo "TTFT Threshold: ${TTFT_THRESHOLD_MS}ms" - echo "ITL Threshold: ${ITL_THRESHOLD_MS}ms" - echo "Artifact dir: ${ARTIFACT_DIR}" - echo "==============================================" - - echo "" - echo "Running warmup benchmark..." - aiperf profile \ - -m "nvidia/DeepSeek-V3.2-NVFP4" \ - --tokenizer "nvidia/DeepSeek-V3.2-NVFP4" \ - --url "http://${ENDPOINT}" \ - --streaming \ - --ui dashboard \ - --synthetic-input-tokens-mean 10000 \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean 200 \ - --output-tokens-stddev 0 \ - --extra-inputs "max_tokens:200" \ - --extra-inputs "min_tokens:200" \ - --extra-inputs "ignore_eos:true" \ - --concurrency 4 \ - --request-count 10 - echo "Warmup complete" - - # Trace replay - echo "" - echo "$(date '+%Y-%m-%d %H:%M:%S') - Starting trace replay benchmark" - aiperf profile \ - -m "nvidia/DeepSeek-V3.2-NVFP4" \ - --tokenizer "nvidia/DeepSeek-V3.2-NVFP4" \ - --input-file "${TRACE_FILE}" \ - --custom-dataset-type mooncake_trace \ - --fixed-schedule \ - --url "http://${ENDPOINT}" \ - --streaming \ - --random-seed 42 \ - --ui dashboard \ - --artifact-dir "${ARTIFACT_DIR}" \ - --workers-max 200 \ - --request-timeout-seconds 1000 \ - --profile-export-level records \ - --record-processors 8 \ - "${SERVER_METRICS_ARGS[@]}" \ - --goodput "time_to_first_token:${TTFT_THRESHOLD_MS} inter_token_latency:${ITL_THRESHOLD_MS}" - - BENCH_EXIT_CODE=$? - echo "" - echo "$(date '+%Y-%m-%d %H:%M:%S') - Benchmark complete (exit code: ${BENCH_EXIT_CODE})" - echo "Results: ${ARTIFACT_DIR}" - ls -la "${ARTIFACT_DIR}" 2>/dev/null || true - echo "Benchmark complete!" - set -e - exit $BENCH_EXIT_CODE - env: - - name: TARGET_MODEL - value: nvidia/DeepSeek-V3.2-NVFP4 - - name: ENDPOINT - value: disagg-kv-dsv32-nvfp4-frontend:8000 - - name: TRACE_FILE - value: /model-cache/traces/conversation_trace_synth_16.00x1+10.00_speedup1_maxisl110000.jsonl - - name: TTFT_THRESHOLD_MS - value: "20000" - - name: ITL_THRESHOLD_MS - value: "50" - - name: AIPERF_HTTP_CONNECTION_LIMIT - value: "200" - - name: AIPERF_HTTP_SO_RCVTIMEO - value: "120" - - name: AIPERF_SERVER_METRICS_URLS - value: "http://disagg-kv-dsv32-nvfp4-dec-0-dec-wkr:9090/metrics,http://disagg-kv-dsv32-nvfp4-prefill-0:9090/metrics" - - name: JOB_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.labels['job-name'] - - name: ROOT_ARTIFACT_DIR - value: /model-cache/perf - - name: HF_HOME - value: /model-cache - - name: PYTHONUNBUFFERED - value: "1" - image: python:3.12-slim - imagePullPolicy: IfNotPresent - name: perf - securityContext: - privileged: true - volumeMounts: - - name: model-cache - mountPath: /model-cache - workingDir: /workspace - restartPolicy: Never - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/deepseek-v4/container/README.md b/recipes/deepseek-v4/container/README.md deleted file mode 100644 index cb1f5a896d33..000000000000 --- a/recipes/deepseek-v4/container/README.md +++ /dev/null @@ -1,72 +0,0 @@ - - -# DeepSeek-V4 Reference Containers - -Shared reference Dockerfiles for the DeepSeek-V4 family — used by both [`deepseek-v4-flash`](../deepseek-v4-flash/) and [`deepseek-v4-pro`](../deepseek-v4-pro/). Nothing in either image is recipe-specific; the model is selected at runtime via `--model-path` (SGLang). - -| Backend | Dockerfile | Base image | Build flow | -|---------|-----------|-----------|------------| -| SGLang (B200) | [`sglang/Dockerfile.dsv4.sglang.b200`](sglang/Dockerfile.dsv4.sglang.b200) | `lmsysorg/sglang:deepseek-v4-blackwell` (digest-pinned, amd64) | Two-stage; Dynamo runtime image as donor | -| SGLang (GB200) | [`sglang/Dockerfile.dsv4.sglang.gb200`](sglang/Dockerfile.dsv4.sglang.gb200) | `lmsysorg/sglang:deepseek-v4-grace-blackwell` (digest-pinned, arm64) | Two-stage; Dynamo runtime image as donor | - -NVIDIA also publishes the prebuilt images for vLLM and SGLang which manifests pull directly: -- `nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3` (multi-arch) -- `nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda13-dev.3` (arm64 only) -- `nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda12-dev.3` (amd64 only) - -The `cudaXY` suffix encodes the CUDA major version baked into the image, not the hardware target. - -> **Optional:** users may also build the standard Dynamo vLLM runtime image via `container/render.py`. See [`/container/README.md`](../../../container/README.md). - -## SGLang (`sglang/Dockerfile.dsv4.sglang.b200`) - -Two-stage build: a Dynamo SGLang runtime image as the donor (for nats / etcd / UCX / NIXL and the Dynamo wheels + Python source), layered onto the upstream SGLang dsv4 base. - -### Step 1 — Build the Dynamo SGLang runtime - -From the **repo root**: - -```bash -container/render.py --framework sglang --target runtime --output-short-filename -docker build -t dynamo:latest-sglang-runtime -f container/rendered.Dockerfile . -``` - -This produces the local tag `dynamo:latest-sglang-runtime`, which Step 2 expects as `DYNAMO_SRC_IMAGE`. The donor must contain the V4 tool/reasoning parsers and the SGLang routed_experts fix; the build asserts on this with a post-install `assert 'deepseek_v4' in get_tool_parser_names()`. - -See [`/container/README.md`](../../../container/README.md) for runtime-image build details and alternative tags. - -### Step 2 — Build the dsv4 overlay - -Still from the **repo root**: - -```bash -docker build \ - -f recipes/deepseek-v4/container/sglang/Dockerfile.dsv4.sglang.b200 \ - -t /sglang-dsv4: \ - . -``` - -The Dockerfile takes nothing from the build context (everything comes from `FROM` / `COPY --from=`), so any context directory works. - -### Build args - -| Arg | Default | Purpose | -|-----|---------|---------| -| `DYNAMO_SRC_IMAGE` | `dynamo:latest-sglang-runtime` | Source for nats / etcd / UCX / NIXL and the V4-aware Dynamo wheels. Default matches Step 1; override with a published Dynamo SGLang runtime tag for reproducible builds without rebuilding locally. | -| `DSV4_BASE_IMAGE` | `lmsysorg/sglang:deepseek-v4-blackwell@sha256:da2acdc8...` | The DeepSeek-V4 SGLang base. Digest-pinned for byte-stable rebuilds. | - -### Wire into a recipe - -Push: - -```bash -docker push /sglang-dsv4: -``` - -Set the `image:` field (Frontend + decode worker) in the recipe's SGLang manifest, then follow the recipe's Quick Start: - -- Flash → [`../deepseek-v4-flash/sglang/agg/deploy.yaml`](../deepseek-v4-flash/sglang/agg/deploy.yaml) — see [Quick Start](../deepseek-v4-flash/README.md#quick-start). -- Pro → [`../deepseek-v4-pro/sglang/agg/deploy.yaml`](../deepseek-v4-pro/sglang/agg/deploy.yaml) — see [Quick Start](../deepseek-v4-pro/README.md#quick-start). diff --git a/recipes/deepseek-v4/container/sglang/Dockerfile.dsv4.sglang.b200 b/recipes/deepseek-v4/container/sglang/Dockerfile.dsv4.sglang.b200 deleted file mode 100644 index 441d2eb42169..000000000000 --- a/recipes/deepseek-v4/container/sglang/Dockerfile.dsv4.sglang.b200 +++ /dev/null @@ -1,49 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -ARG DYNAMO_SRC_IMAGE=dynamo:latest-sglang-runtime -# based on https://github.com/sgl-project/sglang/pull/23600/changes/6d0858d2a82e18b657c8140c329a336ab3673ff8 -ARG DSV4_BASE_IMAGE=lmsysorg/sglang:deepseek-v4-blackwell@sha256:940f7652dd16b04e82ecc57c4dcb6a3ae7d7283bc38db1ce3c4aa6ae3685c4aa - -FROM ${DYNAMO_SRC_IMAGE} AS dynamo_src - -FROM ${DSV4_BASE_IMAGE} - -ENV DEBIAN_FRONTEND=noninteractive - -RUN pip uninstall -y black termplotlib \ - && apt-get update \ - && apt-get purge -y libx264-164 libx265-199 libde265-0 libavcodec60 libxvidcore4 \ - && apt list --upgradable 2>/dev/null | tail -n +2 | grep 'noble' | awk -F/ '{print $1}' | xargs -r apt-get install -y --only-upgrade \ - && rm -rf /var/lib/apt/lists/* - -RUN python3 -m pip install --no-cache-dir --break-system-packages "distro>=1.7,<2" - -COPY --from=dynamo_src /usr/bin/nats-server /usr/bin/nats-server -COPY --from=dynamo_src /usr/local/bin/etcd /usr/local/bin/etcd -ENV PATH=/usr/local/bin/etcd:${PATH} - -COPY --from=dynamo_src /usr/lib/x86_64-linux-gnu/ucx /usr/lib/x86_64-linux-gnu/ucx -COPY --from=dynamo_src /usr/lib/x86_64-linux-gnu/libuc*.so* /usr/lib/x86_64-linux-gnu/ - -COPY --from=dynamo_src /usr/local/lib/python3.12/dist-packages/nixl* /usr/local/lib/python3.12/dist-packages/ - -COPY --from=dynamo_src /opt/dynamo/wheelhouse /opt/dynamo/wheelhouse -RUN pip install --no-cache-dir --force-reinstall --no-deps \ - /opt/dynamo/wheelhouse/ai_dynamo_runtime*.whl \ - /opt/dynamo/wheelhouse/ai_dynamo-*.whl && \ - python3 -c "from dynamo._core import get_tool_parser_names; assert 'deepseek_v4' in get_tool_parser_names(), 'V4 parser missing!'; print('V4 parser verified')" - -COPY --from=dynamo_src /workspace/components/src/dynamo /workspace/components/src/dynamo -COPY --from=dynamo_src /workspace/ATTRIBUTIONS-Python.md /workspace/ATTRIBUTIONS-Rust.md /workspace/ATTRIBUTIONS.md /workspace/ - -# Avoid /workspace/sglang shadowing the dynamo Python package. -ENV PYTHONPATH=/workspace/sglang/python:/workspace/components/src - -ENV SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 \ - SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 - -WORKDIR /workspace - -ENTRYPOINT [] -CMD ["bash"] diff --git a/recipes/deepseek-v4/container/sglang/Dockerfile.dsv4.sglang.gb200 b/recipes/deepseek-v4/container/sglang/Dockerfile.dsv4.sglang.gb200 deleted file mode 100644 index ec4e2db0bac8..000000000000 --- a/recipes/deepseek-v4/container/sglang/Dockerfile.dsv4.sglang.gb200 +++ /dev/null @@ -1,48 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -ARG DYNAMO_SRC_IMAGE=dynamo:latest-sglang-runtime -# based on https://github.com/sgl-project/sglang/pull/23600/changes/1b497c7a0c5a2951ae86f21f4cfebe4678bbc7e4 -ARG DSV4_BASE_IMAGE=lmsysorg/sglang:deepseek-v4-grace-blackwell@sha256:e0a0ebb70572a70d2a90759de96482e7f482d501283c48c3393b1e62198051ef - -FROM ${DYNAMO_SRC_IMAGE} AS dynamo_src - -FROM ${DSV4_BASE_IMAGE} - -ENV DEBIAN_FRONTEND=noninteractive - -RUN pip uninstall -y black termplotlib \ - && apt-get update \ - && apt list --upgradable 2>/dev/null | tail -n +2 | grep 'noble' | awk -F/ '{print $1}' | xargs -r apt-get install -y --only-upgrade \ - && rm -rf /var/lib/apt/lists/* - -RUN python3 -m pip install --no-cache-dir --break-system-packages "distro>=1.7,<2" - -COPY --from=dynamo_src /usr/bin/nats-server /usr/bin/nats-server -COPY --from=dynamo_src /usr/local/bin/etcd /usr/local/bin/etcd -ENV PATH=/usr/local/bin/etcd:${PATH} - -COPY --from=dynamo_src /usr/lib/aarch64-linux-gnu/ucx /usr/lib/aarch64-linux-gnu/ucx -COPY --from=dynamo_src /usr/lib/aarch64-linux-gnu/libuc*.so* /usr/lib/aarch64-linux-gnu/ - -COPY --from=dynamo_src /usr/local/lib/python3.12/dist-packages/nixl* /usr/local/lib/python3.12/dist-packages/ - -COPY --from=dynamo_src /opt/dynamo/wheelhouse /opt/dynamo/wheelhouse -RUN pip install --no-cache-dir --force-reinstall --no-deps \ - /opt/dynamo/wheelhouse/ai_dynamo_runtime*.whl \ - /opt/dynamo/wheelhouse/ai_dynamo-*.whl && \ - python3 -c "from dynamo._core import get_tool_parser_names; assert 'deepseek_v4' in get_tool_parser_names(), 'V4 parser missing!'; print('V4 parser verified')" - -COPY --from=dynamo_src /workspace/components/src/dynamo /workspace/components/src/dynamo -COPY --from=dynamo_src /workspace/ATTRIBUTIONS-Python.md /workspace/ATTRIBUTIONS-Rust.md /workspace/ATTRIBUTIONS.md /workspace/ - -# Avoid /workspace/sglang shadowing the dynamo Python package. -ENV PYTHONPATH=/workspace/sglang/python:/workspace/components/src - -ENV SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 \ - SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 - -WORKDIR /workspace - -ENTRYPOINT [] -CMD ["bash"] diff --git a/recipes/deepseek-v4/deepseek-v4-flash/README.md b/recipes/deepseek-v4/deepseek-v4-flash/README.md deleted file mode 100644 index 84b0869ccbb7..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-flash/README.md +++ /dev/null @@ -1,262 +0,0 @@ - - -# DeepSeek-V4-Flash Recipe - -Aggregated-serving recipes for **DeepSeek-V4-Flash** on Dynamo. Two backends (**vLLM** and **SGLang**) and two hardware targets (**B200** and **GB200**) are documented side by side. All four variants are single-replica decode-only deployments using 4 GPUs. - -| Variant | Backend | Hardware | Manifest | Topology | Container | -|---------|---------|----------|----------|----------|-----------| -| **vllm-agg-b200** | vLLM | 4x B200 | [`vllm/agg_b200/deploy.yaml`](vllm/agg_b200/deploy.yaml) | DP=4 + Expert Parallel, TP=1 | Prebuilt NGC image (`...1.2.0-deepseek-v4-cuda13-dev.3`, multi-arch) | -| **vllm-agg-gb200** | vLLM | 4x GB200 | [`vllm/agg_gb200/deploy.yaml`](vllm/agg_gb200/deploy.yaml) | TP=4 + Expert Parallel, `deep_gemm_mega_moe` | Prebuilt NGC image (`...1.2.0-deepseek-v4-cuda13-dev.3`, multi-arch) | -| **sglang-agg** | SGLang | 4x B200 | [`sglang/agg/deploy.yaml`](sglang/agg/deploy.yaml) | TP=4, MXFP4 MoE via FlashInfer, EAGLE MTP 3/4 | Prebuilt NGC image (`...1.2.0-deepseek-v4-cuda12-dev.3`); optional [custom build](../container/) | -| **sglang-agg-gb200** | SGLang | 4x GB200 | [`sglang/agg-gb200/deploy.yaml`](sglang/agg-gb200/deploy.yaml) | TP=4, MXFP4 MoE via FlashInfer, EAGLE MTP 3/4 | Prebuilt NGC image (`...1.2.0-deepseek-v4-cuda13-dev.3`, arm64) | - -The B200 variants fill 4 of 8 GPUs on a B200 node; the GB200 variants fill all 4 GPUs of a single GB200 NVL4 tray. - -Status: **Experimental** (Day-0). Modality: text only. - -## Prerequisites - -1. **Dynamo Platform installed** — see the [Kubernetes Deployment Guide](../../../docs/kubernetes/README.md). -2. **GPU cluster.** At least 4 GPUs of the matching arch available on one node: - - **B200 variants**: 4 B200 GPUs (x86_64). - - **GB200 variants**: 4 GB200 GPUs (single NVL4 tray, arm64). Nodes must be labeled `nvidia.com/gpu.product=NVIDIA-GB200` and tainted `kubernetes.io/arch=arm64:NoSchedule` (the manifests carry the matching `nodeSelector` + `toleration`). -3. **HuggingFace token** with access to `deepseek-ai/DeepSeek-V4-Flash`. - -## Quick Start - -Common setup (run once — applies to all variants): - -```bash -export NAMESPACE=dynamo-demo -kubectl create namespace ${NAMESPACE} - -# HuggingFace token secret (consumed by the download Job and, as a convenience, by the worker) -kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="your-token-here" \ - -n ${NAMESPACE} - -# Download model into the model-cache PVC. -# Edit model-cache/model-cache.yaml and set storageClassName to a RWX class in your cluster. -# The PVC requests 400Gi; DeepSeek-V4-Flash is ~160GB on disk (46 safetensors shards, -# FP4+FP8 mixed) and typically takes 30-60 min to download on first apply. -kubectl apply -f model-cache/model-cache.yaml -n ${NAMESPACE} -kubectl apply -f model-cache/model-download.yaml -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/model-download -n ${NAMESPACE} --timeout=7200s -``` - -### Deploy — vLLM B200 (`vllm-agg-b200`) - -```bash -kubectl apply -f vllm/agg_b200/deploy.yaml -n ${NAMESPACE} - -# First launch of the decode worker takes up to ~60 minutes (weight load + -# FlashInfer autotune + cudagraph warmup). The startup probe is sized for this. -kubectl wait --for=condition=Ready pod \ - -l nvidia.com/dynamo-graph-deployment-name=dsv4-flash-agg \ - -n ${NAMESPACE} --timeout=3600s -``` - -### Deploy — vLLM GB200 (`vllm-agg-gb200`) - -```bash -kubectl apply -f vllm/agg_gb200/deploy.yaml -n ${NAMESPACE} - -# First launch ~60 minutes; the manifest's startup probe allows for it. -kubectl wait --for=condition=Ready pod \ - -l nvidia.com/dynamo-graph-deployment-name=dsv4-flash-agg \ - -n ${NAMESPACE} --timeout=3600s -``` - -### Deploy — SGLang B200 (`sglang-agg`) - -```bash -kubectl apply -f sglang/agg/deploy.yaml -n ${NAMESPACE} - -# First launch of the decode worker takes up to ~60 minutes (weight load + -# DeepGEMM warmup + cudagraph warmup). The startup probe is sized for this. -kubectl wait --for=condition=Ready pod \ - -l nvidia.com/dynamo-graph-deployment-name=sglang-dsv4-flash \ - -n ${NAMESPACE} --timeout=3600s -``` - -### Deploy — SGLang GB200 (`sglang-agg-gb200`) - -```bash -kubectl apply -f sglang/agg-gb200/deploy.yaml -n ${NAMESPACE} - -kubectl wait --for=condition=Ready pod \ - -l nvidia.com/dynamo-graph-deployment-name=sglang-dsv4-flash \ - -n ${NAMESPACE} --timeout=3600s -``` - -## Test the Deployment - -Port-forward the variant you deployed: - -```bash -# vLLM -kubectl port-forward svc/dsv4-flash-agg-frontend 8000:8000 -n ${NAMESPACE} - -# SGLang -kubectl port-forward svc/sglang-dsv4-flash-frontend 8000:8000 -n ${NAMESPACE} -``` - -Either way the request shape is the same — same model name, same OpenAI-compatible endpoints: - -```bash -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "deepseek-ai/DeepSeek-V4-Flash", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100 - }' -``` - -## Recipe Details - -### vLLM B200 (`vllm/agg_b200/deploy.yaml`) - -| Flag | Purpose | -|------|---------| -| `--tokenizer-mode deepseek_v4` | Selects the DeepSeek-V4 tokenizer | -| `--dyn-reasoning-parser deepseek_v4` | Extracts chain-of-thought into `message.reasoning_content` | -| `--dyn-tool-call-parser deepseek_v4` | Emits OpenAI-compatible structured `tool_calls` | -| `--attention-config '{"use_fp4_indexer_cache":true}'` | Blackwell FP4 indexer cache for CSA+HCA attention | -| `--kv-cache-dtype fp8` + `--block-size 256` | FP8 KV cache; block size matches the upstream recipe | -| `--tensor-parallel-size 1 --data-parallel-size 4 --enable-expert-parallel` | DP=4 + EP across the 4 GPUs (TP=1) | -| `--compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'` | Single-node DEP compilation config from the upstream recipe | -| `--no-enable-flashinfer-autotune` | Skip per-shape FlashInfer autotuning at startup; required on dsv4 for correct accuracy | -| `--max-num-seqs 256` | Concurrency cap | - -### vLLM GB200 (`vllm/agg_gb200/deploy.yaml`) - -Same OpenAI-renderer wiring as the B200 variant; differences below come from the [upstream vLLM GB200 recipe](https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Flash?features=tool_calling,reasoning&hardware=gb200) for V4-Flash. - -| Flag / env | Purpose | -|---|---| -| `--tensor-parallel-size 4 --enable-expert-parallel` | **TP=4 + EP** across the 4 GPUs of the NVL4 tray (DP dropped — the GB200 tray's intra-tray NVLink makes TP attractive for this size class) | -| `--moe-backend deep_gemm_mega_moe` | DeepGEMM "mega MoE" kernel — the optimized FP8 MoE path for V4 expert routing on Blackwell | -| `--no-enable-flashinfer-autotune` | Skip per-shape FlashInfer autotuning at startup; required on dsv4 for correct accuracy | -| `NCCL_NVLS_ENABLE=1`, `NCCL_P2P_LEVEL=NVL`, `VLLM_USE_NCCL_SYMM_MEM=1` | Enable NVLink Sharp (NVLS) multicast for one-shot all-reduce on the tray | - -### SGLang B200 (`sglang/agg/deploy.yaml`) - -| Flag | Purpose | -|------|---------| -| `--dyn-reasoning-parser deepseek_v4` | Extracts chain-of-thought into `message.reasoning_content` | -| `--dyn-tool-call-parser deepseek_v4` | Emits OpenAI-compatible structured `tool_calls` | -| `--trust-remote-code` | Required for the V4 architecture's custom modeling code | -| `--tp 4` | Tensor-parallel across the 4 GPUs of one node | -| `--moe-runner-backend flashinfer_mxfp4` | MXFP4 MoE kernel via FlashInfer for the V4 expert weights | -| `--speculative-algo EAGLE` + `--speculative-num-steps 3` + `--speculative-eagle-topk 1` + `--speculative-num-draft-tokens 4` | EAGLE MTP speculative decoding (3 draft steps, top-1 over the EAGLE head, 4 draft tokens per step) | -| `--chunked-prefill-size 4096` | Chunk long prompts at 4k tokens for steady-state decode interleaving | -| `--disable-flashinfer-autotune` | Skip per-shape autotuning at startup; the dsv4 base ships pre-tuned defaults | - -## Model Details - -| | | -|---|---| -| **Model** | `deepseek-ai/DeepSeek-V4-Flash` (MoE, 284B total / 13B active) | -| **Checkpoint** | Mixed FP4 (expert weights) + FP8 (attention, norm, router) | -| **Attention** | Hybrid CSA + HCA with Blackwell FP4 indexer cache | - -Recipe-level (per-variant) settings: - -| | vLLM B200 (`vllm-agg-b200`) | vLLM GB200 (`vllm-agg-gb200`) | SGLang B200 (`sglang-agg`) | SGLang GB200 (`sglang-agg-gb200`) | -|---|---|---|---|---| -| **Backend image** | Prebuilt `nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3` (multi-arch) | Prebuilt `nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3` (multi-arch) | Prebuilt `nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda12-dev.3` | Prebuilt `nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda13-dev.3` | -| **Parallelism** | DP=4 + Expert Parallel, TP=1 | TP=4 + Expert Parallel | TP=4 | TP=4 | -| **MoE backend** | vLLM's V4 expert kernel (FP4) | DeepGEMM mega MoE | FlashInfer MXFP4 | FlashInfer MXFP4 | -| **KV cache** | FP8, block size 256 | FP8, block size 256 | engine default | engine default | -| **Speculative decoding** | — | — | EAGLE MTP (3 steps / 4 draft tokens) | EAGLE MTP (3 steps / 4 draft tokens) | - -## Verifying Reasoning - -Same flow on both variants — same model, same `--dyn-reasoning-parser deepseek_v4`: - -```bash -curl -s http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "deepseek-ai/DeepSeek-V4-Flash", - "messages": [{"role": "user", "content": "What is 2+2? Answer briefly."}], - "max_tokens": 200 - }' | python3 -m json.tool -``` - -Expected: - -- `choices[0].message.reasoning_content` contains the model's chain-of-thought. -- `choices[0].message.content` contains only the final answer. -- No raw `` tags in either field. - -If `reasoning_content` is `null` and `` appears in `content`, the reasoning parser isn't wired up — confirm `--dyn-reasoning-parser deepseek_v4` is on the worker command. - -## Verifying Tool Calling - -Same flow on both variants: - -```bash -curl -s http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "deepseek-ai/DeepSeek-V4-Flash", - "messages": [{"role": "user", "content": "What is the weather in San Francisco?"}], - "tools": [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "City name"} - }, - "required": ["location"] - } - } - }], - "max_tokens": 300 - }' | python3 -m json.tool -``` - -Expected: - -- `choices[0].message.tool_calls` is a structured array with `function.name`, `function.arguments`, and `id`. -- `choices[0].finish_reason` is `"tool_calls"`. -- `choices[0].message.reasoning_content` may contain the model's reasoning about tool selection. - -If `tool_calls` is missing and raw tool-call markers appear in `content`, confirm `--dyn-tool-call-parser deepseek_v4` is on the worker command. - -## Notes - -### Common - -- **Storage class.** Update `storageClassName` in `model-cache/model-cache.yaml` to a RWX class that can serve the PVC to Frontend and worker pods. -- **Model size.** `deepseek-ai/DeepSeek-V4-Flash` is ~160 GB on disk (46 safetensors shards in FP4+FP8 mixed form). The 400Gi PVC leaves headroom for HF cache metadata and one alternate revision. -- **Parser flags.** Use the Dynamo variants on the worker (`--dyn-reasoning-parser`, `--dyn-tool-call-parser`). Each engine's native `--reasoning-parser` / `--tool-call-parser` are engine-side and do not feed the Dynamo OpenAI renderer. -- **Offline model cache.** Both workers run with `HF_HUB_OFFLINE=1` so the engine reads cached weights from the PVC and never contacts the HF Hub at startup. The HF token secret is mounted defensively; it isn't required at runtime once the download Job has completed. -- **First launch is slow.** Decode workers load weights and warm CUDA graphs / DeepGEMM kernels on first launch; the manifests' startup probes allow up to ~60 min (`failureThreshold: 360` at `periodSeconds: 10`). - -### vLLM-specific - -- **Prebuilt images.** Both `vllm/agg_b200/deploy.yaml` and `vllm/agg_gb200/deploy.yaml` reference `nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3` (multi-arch). To rebuild from source (custom Dynamo branch, different vLLM base, etc.), see [`/container/README.md`](../../../container/README.md). -- **Engine-ready timeout.** `VLLM_ENGINE_READY_TIMEOUT_S=3600` matches the startup probe budget on both variants. -- **DP stability (B200 only).** `VLLM_RANDOMIZE_DP_DUMMY_INPUTS=1` and `VLLM_SKIP_P2P_CHECK=1` mirror the DeepSeek-R1 vLLM recipe and stabilize DP dummy inputs. The GB200 variant uses TP (no DP), so `VLLM_RANDOMIZE_DP_DUMMY_INPUTS` is not set. -- **FlashInfer autotune.** `--no-enable-flashinfer-autotune` skips per-shape FlashInfer autotuning at startup and is set on both vLLM variants. Required on dsv4: the autotuner currently produces tunings that regress GSM8k accuracy. Skipping it also shortens first-launch warmup. -- **FlashInfer TRT-LLM allreduce on GB200.** You may see a non-fatal startup warning `Failed to initialize FlashInfer Allreduce norm fusion workspace ... Flashinfer allreduce-norm fusion will be disabled`. vLLM falls back to a non-fused allreduce + RMSNorm; correctness is unaffected. To enable the fused kernel, set the compilation pass: `--compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"pass_config":{"fuse_allreduce_rms":true}}'`. - -### SGLang-specific - -- **Prebuilt images.** `sglang/agg/deploy.yaml` references `nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda12-dev.3` and `sglang/agg-gb200/deploy.yaml` references the arm64 sibling `nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda13-dev.3`. To rebuild either (custom Dynamo branch, different SGLang base, etc.), see [`recipes/deepseek-v4/container/README.md`](../container/README.md). -- **DeepGEMM / FlashInfer warmup.** `SGLANG_JIT_DEEPGEMM_PRECOMPILE=0` + `SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1` skip the slow precompile and use the fast warmup path. `--disable-flashinfer-autotune` skips per-shape FlashInfer autotuning at startup; the dsv4 base ships pre-tuned defaults. -- **NCCL / Gloo.** `NCCL_CUMEM_ENABLE=1` is set for V4 NCCL collectives on Blackwell. `GLOO_SOCKET_IFNAME=eth0` pins Gloo to the standard pod interface. - -## Sibling Recipe - -[DeepSeek-V4-Pro](../deepseek-v4-pro/) is the larger sibling (1.6T / 49B active, 1M context, 8x B200) and shares the same dsv4 vLLM and SGLang container images. diff --git a/recipes/deepseek-v4/deepseek-v4-flash/model-cache/model-cache.yaml b/recipes/deepseek-v4/deepseek-v4-flash/model-cache/model-cache.yaml deleted file mode 100644 index a0e03a1e7967..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-flash/model-cache/model-cache.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 400Gi - storageClassName: "your-storage-class-name" diff --git a/recipes/deepseek-v4/deepseek-v4-flash/model-cache/model-download.yaml b/recipes/deepseek-v4/deepseek-v4-flash/model-cache/model-download.yaml deleted file mode 100644 index 8b8740405d3c..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-flash/model-cache/model-download.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: MODEL_NAME - value: deepseek-ai/DeepSeek-V4-Flash - - name: HF_HOME - value: /model-store - # Uses up to 64 GB RAM for XET buffers; set "0" on low-memory nodes (8 GB cap): https://huggingface.co/docs/hub/en/xet/using-xet-storage#download-buffers - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.11.0 - hf download $MODEL_NAME - resources: - requests: - cpu: "2" - memory: "64Gi" - limits: - cpu: "8" - memory: "64Gi" - volumeMounts: - - name: model-cache - mountPath: /model-store - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/deepseek-v4/deepseek-v4-flash/sglang/agg-gb200/deploy.yaml b/recipes/deepseek-v4/deepseek-v4-flash/sglang/agg-gb200/deploy.yaml deleted file mode 100644 index c2c2211e454d..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-flash/sglang/agg-gb200/deploy.yaml +++ /dev/null @@ -1,132 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSeek-V4-Flash SGLang DynamoGraphDeployment — GB200 (arm64) -# Aggregated mode, GB200x4 TP4, MXFP4 MoE, EAGLE MTP 3/4 -# Validated on dynamo-aws-dev-01 (p6e-gb200.36xlarge, EFA) -# -# Deploy: -# kubectl apply -f deploy.yaml -n -# -# Test: -# kubectl port-forward -n svc/sglang-dsv4-flash-frontend 8000:8000 -# curl http://localhost:8000/v1/models -# curl http://localhost:8000/v1/chat/completions -H 'Content-Type: application/json' \ -# -d '{"model":"deepseek-ai/DeepSeek-V4-Flash","messages":[{"role":"user","content":"Hello"}]}' - -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: sglang-dsv4-flash -spec: - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: shared-model-cache - mountPoint: /models - extraPodSpec: - nodeSelector: - kubernetes.io/arch: arm64 - tolerations: - - operator: Exists - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - imagePullPolicy: Always - env: - - name: HF_HOME - value: /models - - name: HF_HUB_OFFLINE - value: "1" - securityContext: - runAsUser: 0 - runAsGroup: 0 - startupProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - - decode: - componentType: worker - subComponentType: decode - replicas: 1 - resources: - limits: - gpu: "4" - volumeMounts: - - name: shared-model-cache - mountPoint: /models - sharedMemory: - size: 200Gi - extraPodSpec: - tolerations: - - operator: Exists - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - imagePullPolicy: Always - workingDir: /workspace - command: - - python3 - - -m - - dingo.sglang - args: - - --model-path - - deepseek-ai/DeepSeek-V4-Flash - - --served-model-name - - deepseek-ai/DeepSeek-V4-Flash - - --trust-remote-code - - --tp - - "4" - - --moe-runner-backend - - flashinfer_mxfp4 - - --speculative-algo - - EAGLE - - --speculative-num-steps - - "3" - - --speculative-eagle-topk - - "1" - - --speculative-num-draft-tokens - - "4" - - --chunked-prefill-size - - "4096" - - --disable-flashinfer-autotune - - --dyn-tool-call-parser - - deepseek_v4 - - --dyn-reasoning-parser - - deepseek_v4 - env: - - name: HF_HOME - value: /models - - name: HF_HUB_OFFLINE - value: "1" - - name: CUDA_VISIBLE_DEVICES - value: "0,1,2,3" - - name: SGLANG_JIT_DEEPGEMM_PRECOMPILE - value: "0" - - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: GLOO_SOCKET_IFNAME - value: eth0 - resources: - limits: - vpc.amazonaws.com/efa: "4" - securityContext: - runAsUser: 0 - runAsGroup: 0 - startupProbe: - httpGet: - path: /health - port: 9090 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - - pvcs: - - name: shared-model-cache - create: false diff --git a/recipes/deepseek-v4/deepseek-v4-flash/sglang/agg/deploy.yaml b/recipes/deepseek-v4/deepseek-v4-flash/sglang/agg/deploy.yaml deleted file mode 100644 index a2a24d93b976..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-flash/sglang/agg/deploy.yaml +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSeek-V4-Flash SGLang DynamoGraphDeployment -# Aggregated mode (no P/D disagg), B200x4 TP4, MXFP4 MoE, EAGLE MTP 3/4 -# -# Deploy: -# kubectl apply -f deploy.yaml -n -# -# Test: -# kubectl port-forward -n svc/sglang-dsv4-flash-frontend 8000:8000 -# curl http://localhost:8000/v1/models -# curl http://localhost:8000/v1/chat/completions -H 'Content-Type: application/json' \ -# -d '{"model":"deepseek-ai/DeepSeek-V4-Flash","messages":[{"role":"user","content":"Hello"}]}' - -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: sglang-dsv4-flash -spec: - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: shared-model-cache - mountPoint: /models - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda12-dev.3 - imagePullPolicy: Always - env: - - name: HF_HOME - value: /models - - name: HF_HUB_OFFLINE - value: "1" - startupProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - - decode: - componentType: worker - subComponentType: decode - replicas: 1 - resources: - limits: - gpu: "4" - volumeMounts: - - name: shared-model-cache - mountPoint: /models - sharedMemory: - size: 200Gi - extraPodSpec: - nodeSelector: - nvidia.com/gpu.product: NVIDIA-B200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda12-dev.3 - imagePullPolicy: Always - workingDir: /workspace - command: - - python3 - - -m - - dingo.sglang - args: - - --model-path - - deepseek-ai/DeepSeek-V4-Flash - - --served-model-name - - deepseek-ai/DeepSeek-V4-Flash - - --trust-remote-code - - --tp - - "4" - - --moe-runner-backend - - flashinfer_mxfp4 - - --speculative-algo - - EAGLE - - --speculative-num-steps - - "3" - - --speculative-eagle-topk - - "1" - - --speculative-num-draft-tokens - - "4" - - --chunked-prefill-size - - "4096" - - --disable-flashinfer-autotune - - --dyn-tool-call-parser - - deepseek_v4 - - --dyn-reasoning-parser - - deepseek_v4 - env: - - name: HF_HOME - value: /models - - name: HF_HUB_OFFLINE - value: "1" - - name: CUDA_VISIBLE_DEVICES - value: "0,1,2,3,4,5,6,7" - - name: SGLANG_JIT_DEEPGEMM_PRECOMPILE - value: "0" - - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: GLOO_SOCKET_IFNAME - value: eth0 - startupProbe: - httpGet: - path: /health - port: 9090 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - - pvcs: - - name: shared-model-cache - create: false diff --git a/recipes/deepseek-v4/deepseek-v4-flash/vllm/agg_b200/deploy.yaml b/recipes/deepseek-v4/deepseek-v4-flash/vllm/agg_b200/deploy.yaml deleted file mode 100644 index 491a46ed2625..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-flash/vllm/agg_b200/deploy.yaml +++ /dev/null @@ -1,108 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# DynamoGraphDeployment for deepseek-ai/DeepSeek-V4-Flash on vLLM, -# aggregated serving (no prefill/decode disaggregation). -# -# Upstream vLLM recipe: -# https://github.com/vllm-project/recipes/blob/main/models/deepseek-ai/DeepSeek-V4-Flash.yaml -# -# Shape: 1 replica x 4 B200 GPUs, DP=4 + Expert Parallel, TP=1. -# Tested on 4 of 8 GPUs per B200 node. -# -# Weights: served from the `model-cache` PVC populated by -# `../../model-cache/model-download.yaml`. -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: dsv4-flash-agg -spec: - backendFramework: vllm - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /opt/models - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - workingDir: /workspace/examples/backends/vllm - env: - - name: HF_HOME - value: /opt/models - - name: HF_HUB_OFFLINE - value: "1" - VllmDecodeWorker: - componentType: worker - subComponentType: decode - envFromSecret: hf-token-secret - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 200Gi - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - workingDir: /workspace/examples/backends/vllm - # Up to ~60 min for first launch: weight load + FlashInfer autotune + - # cudagraph warmup. periodSeconds * failureThreshold = 10 * 360 = 3600s. - startupProbe: - httpGet: - path: /health - port: 9090 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - env: - - name: SERVED_MODEL_NAME - value: deepseek-ai/DeepSeek-V4-Flash - - name: MODEL_PATH - value: deepseek-ai/DeepSeek-V4-Flash - - name: HF_HOME - value: /opt/models - # Read weights from the PVC only; do not hit the HF Hub at startup. - - name: HF_HUB_OFFLINE - value: "1" - # Give the engine room to finish first-launch init. - - name: VLLM_ENGINE_READY_TIMEOUT_S - value: "3600" - # Stabilize DP dummy inputs (matches the DeepSeek-R1 vLLM recipe). - - name: VLLM_RANDOMIZE_DP_DUMMY_INPUTS - value: "1" - - name: VLLM_SKIP_P2P_CHECK - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - command: - - /bin/sh - - -c - args: - - | - python3 -m dingo.vllm \ - --model "${MODEL_PATH}" \ - --served-model-name "${SERVED_MODEL_NAME}" \ - --trust-remote-code \ - --kv-cache-dtype fp8 \ - --block-size 256 \ - --tensor-parallel-size 1 \ - --data-parallel-size 4 \ - --enable-expert-parallel \ - --tokenizer-mode deepseek_v4 \ - --dyn-reasoning-parser deepseek_v4 \ - --dyn-tool-call-parser deepseek_v4 \ - --attention-config '{"use_fp4_indexer_cache":true}' \ - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ - --no-enable-flashinfer-autotune \ - --max-num-seqs 256 - replicas: 1 - resources: - limits: - gpu: "4" - requests: - gpu: "4" diff --git a/recipes/deepseek-v4/deepseek-v4-flash/vllm/agg_gb200/deploy.yaml b/recipes/deepseek-v4/deepseek-v4-flash/vllm/agg_gb200/deploy.yaml deleted file mode 100644 index 7cecad3ef927..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-flash/vllm/agg_gb200/deploy.yaml +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# DeepSeek-V4-Flash on vLLM, aggregated, GB200 (TP=4 + EP, single NVL4 tray). -# Weights: pre-staged on the `model-cache` PVC by `../../model-cache/model-download.yaml`. -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: dsv4-flash-agg -spec: - backendFramework: vllm - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /opt/models - extraPodSpec: - nodeSelector: - kubernetes.io/arch: arm64 - nvidia.com/gpu.product: NVIDIA-GB200 - tolerations: - - key: kubernetes.io/arch - operator: Equal - value: arm64 - effect: NoSchedule - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - workingDir: /workspace/examples/backends/vllm - env: - - name: HF_HOME - value: /opt/models - - name: HF_HUB_OFFLINE - value: "1" - VllmDecodeWorker: - componentType: worker - subComponentType: decode - envFromSecret: hf-token-secret - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 200Gi - extraPodSpec: - nodeSelector: - kubernetes.io/arch: arm64 - nvidia.com/gpu.product: NVIDIA-GB200 - tolerations: - - key: kubernetes.io/arch - operator: Equal - value: arm64 - effect: NoSchedule - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - workingDir: /workspace/examples/backends/vllm - # ~60 min budget for first launch (10 * 360 = 3600s). - startupProbe: - httpGet: - path: /health - port: 9090 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - env: - - name: SERVED_MODEL_NAME - value: deepseek-ai/DeepSeek-V4-Flash - - name: MODEL_PATH - value: deepseek-ai/DeepSeek-V4-Flash - - name: HF_HOME - value: /opt/models - # Set to "0" for a one-shot cold download into the PVC. - - name: HF_HUB_OFFLINE - value: "1" - - name: VLLM_ENGINE_READY_TIMEOUT_S - value: "3600" - - name: VLLM_USE_NCCL_SYMM_MEM - value: "1" - - name: VLLM_SKIP_P2P_CHECK - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: NCCL_P2P_LEVEL - value: NVL - command: - - /bin/sh - - -c - args: - - | - exec python3 -m dingo.vllm \ - --model "${MODEL_PATH}" \ - --served-model-name "${SERVED_MODEL_NAME}" \ - --trust-remote-code \ - --kv-cache-dtype fp8 \ - --block-size 256 \ - --tensor-parallel-size 4 \ - --enable-expert-parallel \ - --tokenizer-mode deepseek_v4 \ - --dyn-reasoning-parser deepseek_v4 \ - --dyn-tool-call-parser deepseek_v4 \ - --attention-config '{"use_fp4_indexer_cache":true}' \ - --moe-backend deep_gemm_mega_moe \ - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ - --no-enable-flashinfer-autotune \ - --max-num-seqs 256 - replicas: 1 - resources: - limits: - gpu: "4" - requests: - gpu: "4" diff --git a/recipes/deepseek-v4/deepseek-v4-pro/README.md b/recipes/deepseek-v4/deepseek-v4-pro/README.md deleted file mode 100644 index f6bbc95fbc72..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-pro/README.md +++ /dev/null @@ -1,305 +0,0 @@ - - -# DeepSeek-V4-Pro Recipe - -Recipes for **DeepSeek-V4-Pro** on Dynamo across two backends (**vLLM**, **SGLang**) and two hardware targets (**B200**, **GB200**). Single-node aggregated serving fills 8 GPUs of a B200 box; on GB200 the model exceeds a single 4-GPU NVL4 tray, so V4-Pro spans two GB200 trays via NVLink72 (MNNVL) — either **aggregated with TP=8 cross-node** or **disaggregated prefill/decode**. - -| Variant | Backend | Hardware | Manifest | Topology | Container | -|---------|---------|----------|----------|----------|-----------| -| **vllm-agg-b200** | vLLM | 1 node, 8x B200 | [`vllm/agg/b200/deploy.yaml`](vllm/agg/b200/deploy.yaml) | TP=8 + Expert Parallel (single node) | Prebuilt NGC image (`...1.2.0-deepseek-v4-cuda13-dev.3`, multi-arch) | -| **vllm-agg-gb200** | vLLM | 2 nodes, 4x GB200 each (8 total) | [`vllm/agg/gb200/deploy.yaml`](vllm/agg/gb200/deploy.yaml) | TP=8 + Expert Parallel cross-node, MNNVL via ComputeDomain (NVLink72) | Prebuilt NGC image (`...1.2.0-deepseek-v4-cuda13-dev.3`, multi-arch) | -| **vllm-disagg-gb200** | vLLM | 2 nodes, 4x GB200 each (8 prefill + 8 decode = 16 total) | [`vllm/disagg/gb200/deploy.yaml`](vllm/disagg/gb200/deploy.yaml) | 1P + 1D, DP=8 + Expert Parallel per worker, MNNVL via ComputeDomain (NVLink72) | Prebuilt NGC image (`...1.2.0-deepseek-v4-cuda13-dev.3`, multi-arch) | -| **sglang-agg** | SGLang | 1 node, 8x B200 | [`sglang/agg/deploy.yaml`](sglang/agg/deploy.yaml) | TP=8, MXFP4 MoE via FlashInfer, EAGLE MTP 3/4 | Prebuilt NGC image; optional [custom build](../container/) | - -A perf-benchmark Job for the GB200 disagg variant is provided alongside the deploy: - -| Perf Job | Notes | -|---|---| -| [`vllm/disagg/gb200/perf.yaml`](vllm/disagg/gb200/perf.yaml) | Runs `aiperf profile` against `dsv4-pro-disagg-frontend:8000` with an 8K-input / 1K-output concurrency sweep (256 / 512 / 1024). Override `CONCURRENCIES` in the Job env for a smaller smoke run. | - -Status: **Experimental** (Day-0). Modality: text only. - -## Prerequisites - -1. **Dynamo Platform installed** — see the [Kubernetes Deployment Guide](../../../docs/kubernetes/README.md). -2. **GPU cluster.** Hardware depends on the variant: - - **B200 variants** (`vllm-agg-b200`, `sglang-agg`): 8 B200 GPUs available on a single node (x86_64). TP=8 fills the box. - - **GB200 variants** (`vllm-agg-gb200`, `vllm-disagg-gb200`): **2 GB200 nodes**, each with 4 GPUs (single NVL4 tray each), connected to the **same NVLink72 clique**. Nodes must be labeled `nvidia.com/gpu.product=NVIDIA-GB200` and tainted `kubernetes.io/arch=arm64:NoSchedule`. The cluster must have the **DRA / ComputeDomain controller** installed (verify with `kubectl get crd | grep computedomain`); each manifest's `ComputeDomain` CR + `resourceClaims` are how the operator co-locates the worker pod set on the same NVLink72 fabric (the agg variant places 2 pods, the disagg variant places 4). -3. **HuggingFace token** with access to `deepseek-ai/DeepSeek-V4-Pro`. - -## Quick Start - -Common setup (run once — applies to both variants): - -```bash -export NAMESPACE=dynamo-demo -kubectl create namespace ${NAMESPACE} - -# HuggingFace token secret (consumed by the download Job and, as a convenience, by the worker) -kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="your-token-here" \ - -n ${NAMESPACE} - -# Download model into the model-cache PVC. -# Edit model-cache/model-cache.yaml and set storageClassName to a RWX class in your cluster. -# The PVC requests 1500Gi; DeepSeek-V4-Pro is ~865 GB on disk (64 safetensors shards, -# FP4+FP8 mixed) and typically takes 1.5-3 hours to download on first apply. -kubectl apply -f model-cache/model-cache.yaml -n ${NAMESPACE} -kubectl apply -f model-cache/model-download.yaml -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/model-download -n ${NAMESPACE} --timeout=14400s -``` - -### Deploy — vLLM B200 (`vllm-agg-b200`) - -```bash -kubectl apply -f vllm/agg/b200/deploy.yaml -n ${NAMESPACE} - -# First launch of the decode worker takes up to ~90 minutes (TP=8 weight load + -# FlashInfer autotune + cudagraph warmup). The startup probe is sized for this. -kubectl wait --for=condition=Ready pod \ - -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-agg \ - -n ${NAMESPACE} --timeout=5400s -``` - -### Deploy — vLLM GB200 agg (`vllm-agg-gb200`) - -```bash -kubectl apply -f vllm/agg/gb200/deploy.yaml -n ${NAMESPACE} - -# First launch of the decode worker takes up to ~90 minutes (TP=8 weight load -# + NCCL bring-up over MNNVL + cudagraph capture across 2 nodes). -kubectl wait --for=condition=Ready pod \ - -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-agg \ - -n ${NAMESPACE} --timeout=5400s -``` - -### Deploy — vLLM GB200 disagg (`vllm-disagg-gb200`) - -```bash -kubectl apply -f vllm/disagg/gb200/deploy.yaml -n ${NAMESPACE} - -# First launch of each leader takes up to ~90 minutes (DP=8 weight load + -# NIXL/UCX setup + NCCL bring-up over MNNVL + cudagraph capture). -kubectl wait --for=condition=Ready pod \ - -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-disagg \ - -n ${NAMESPACE} --timeout=5400s - -# Optional: run the perf benchmark Job (8K input / 1K output sweep at c=256/512/1024) -# kubectl apply -f vllm/disagg/gb200/perf.yaml -n ${NAMESPACE} -``` - -### Deploy — SGLang (`sglang-agg`) - -```bash -kubectl apply -f sglang/agg/deploy.yaml -n ${NAMESPACE} - -# First launch of the decode worker takes up to ~60 minutes (TP=8 weight load + -# DeepGEMM warmup + cudagraph warmup). The startup probe is sized for this. -kubectl wait --for=condition=Ready pod \ - -l nvidia.com/dynamo-graph-deployment-name=sglang-dsv4-pro \ - -n ${NAMESPACE} --timeout=3600s -``` - -## Test the Deployment - -Port-forward the variant you deployed: - -```bash -# vLLM B200 agg or GB200 agg (same DGD/service name — only one of these -# variants can be deployed in a given namespace at a time) -kubectl port-forward svc/dsv4-pro-agg-frontend 8000:8000 -n ${NAMESPACE} - -# vLLM GB200 disagg -kubectl port-forward svc/dsv4-pro-disagg-frontend 8000:8000 -n ${NAMESPACE} - -# SGLang -kubectl port-forward svc/sglang-dsv4-pro-frontend 8000:8000 -n ${NAMESPACE} -``` - -Either way the request shape is the same. Send `thinking: false` on the vLLM variant per the Day-0 caveat above: - -```bash -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "deepseek-ai/DeepSeek-V4-Pro", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - "chat_template_kwargs": {"thinking": false} - }' -``` - -## Recipe Details - -### vLLM B200 agg (`vllm/agg/b200/deploy.yaml`) - -| Flag | Purpose | -|------|---------| -| `--tokenizer-mode deepseek_v4` | Selects the DeepSeek-V4 tokenizer | -| `--dyn-reasoning-parser deepseek_v4` | Extracts chain-of-thought into `message.reasoning_content` | -| `--dyn-tool-call-parser deepseek_v4` | Emits OpenAI-compatible structured `tool_calls` | -| `--attention-config '{"use_fp4_indexer_cache":true}'` | Blackwell FP4 indexer cache for CSA+HCA attention | -| `--kv-cache-dtype fp8` + `--block-size 256` | FP8 KV cache; block size matches the upstream recipe | -| `--tensor-parallel-size 8 --enable-expert-parallel` | TP=8 across 8 GPUs of one node, with EP enabled for the MoE experts | -| `--compilation-config '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}'` | Conservative cudagraph mode appropriate for the larger Pro model (matches upstream V4-Pro example) | -| `--no-enable-flashinfer-autotune` | Skip per-shape FlashInfer autotuning at startup; required on dsv4 for correct accuracy | -| `--max-num-seqs 256` | Concurrency cap | - -### vLLM GB200 agg (`vllm/agg/gb200/deploy.yaml`) - -V4-Pro at ~865 GB on disk does not fit a single GB200 NVL4 tray (~768 GB HBM across 4 GPUs), so the GB200 agg recipe stretches one tensor-parallel group across **two** trays — the cross-node TP all-reduce / all-gather flows over NVLink72 (MNNVL), not RoCE. The two pods are co-located on the same NVLink72 clique by the DRA `ComputeDomain` controller. - -| Flag / env | Purpose | -|---|---| -| `--tensor-parallel-size 8 --enable-expert-parallel` | TP=8 + EP across 2 nodes (4 GPUs/node × 2 nodes) — no DP. | -| `--compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"pass_config":{"fuse_allreduce_rms":false}}'` | FULL_AND_PIECEWISE cudagraph + all custom ops; `fuse_allreduce_rms:false` avoids a non-fatal FlashInfer trtllm allreduce-norm workspace warning at startup. | -| `--attention-config '{"use_fp4_indexer_cache":true}'` + `--moe-backend deep_gemm_mega_moe` | Blackwell FP4 indexer cache + DeepGEMM "mega MoE" kernel — same kernels as the B200 agg variant. | -| `--no-enable-flashinfer-autotune` | Skip per-shape FlashInfer autotuning at startup; required on dsv4 for correct accuracy | -| `NCCL_MNNVL_ENABLE=1`, `UCX_CUDA_IPC_ENABLE_MNNVL=y`, `UCX_TLS=cuda_copy,cuda_ipc,tcp`, `NCCL_NVLS_ENABLE=1`, `NCCL_P2P_LEVEL=NVL` | Enable cross-node NVLink72 / MNNVL fabric. Required because the TP=8 process group spans 2 nodes. | -| `ComputeDomain` CR + `resourceClaimTemplate` (top of manifest) | DRA primitive that asks the scheduler to allocate an MNNVL channel on demand and co-locate the 2-pod set on the same NVLink72 clique. | -| (no `--data-parallel-rpc-port`) | TP-only — torch.distributed master binds `MASTER_PORT` (29500) for the cross-node rendezvous, which also satisfies the operator's `wait-for-leader-mp` TCP probe. | - -### vLLM GB200 disagg (`vllm/disagg/gb200/deploy.yaml`) - -V4-Pro at ~865 GB on disk does not fit a single GB200 NVL4 tray (~768 GB HBM across 4 GPUs), so the GB200 recipe is the **disaggregated** prefill/decode shape: one prefill replica spanning 2 GB200 nodes (DP=8 + EP) and one decode replica spanning 2 GB200 nodes (DP=8 + EP), all four pods placed on the same NVLink72 clique by the DRA `ComputeDomain` controller. - -| Flag / env | Purpose | -|---|---| -| `--data-parallel-size 8 --enable-expert-parallel --tensor-parallel-size 1` | DP=8 + EP across 2 nodes per worker (4 GPUs/node × 2 nodes) — TP=1 | -| `--data-parallel-rpc-port 29500` | Binds vLLM's DP coordinator on `:29500`. The dynamo operator's `wait-for-leader-mp` init container does a TCP probe to `:29500` and blocks worker startup until that port accepts; pinning the DP coord port to 29500 makes the real RPC server satisfy the probe (cleaner than parking a placeholder listener). | -| `--disaggregation-mode prefill` (prefill only) + `--kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'` | Prefill writes KV blocks via NIXL; decode reads them. NIXL's UCX active-messages control plane goes over TCP (`UCX_TLS=cuda_ipc,cuda_copy,tcp`) while bulk KV flows over MNNVL. | -| `NCCL_MNNVL_ENABLE=1`, `UCX_CUDA_IPC_ENABLE_MNNVL=y`, `NCCL_NVLS_ENABLE=1`, `NCCL_P2P_LEVEL=NVL` | Enable cross-node NVLink72 / MNNVL fabric. Required because the prefill and decode workers each span 2 nodes. | -| `ComputeDomain` CR + `resourceClaimTemplate` (top of manifest) | DRA primitive that asks the scheduler to allocate an MNNVL channel on demand and co-locate the 4-pod set on the same NVLink72 clique. Without it, NCCL bring-up across pods fails — TCP-only fallback is not viable for DP=8 cross-pod all-reduce. | -| `--compilation-config '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}'` (decode), `--enforce-eager` (prefill) | Conservative compile/graph config — matches the B200 agg variant's V4-Pro tuning. | -| `--no-enable-flashinfer-autotune` (prefill + decode) | Skip per-shape FlashInfer autotuning at startup; required on dsv4 for correct accuracy | -| `--max-model-len 9280`, `--max-num-seqs 16` (prefill) / `128` (decode) | Capped to the 8K-input / 1K-output benchmark shape. | - -### SGLang (`sglang/agg/deploy.yaml`) - -| Flag | Purpose | -|------|---------| -| `--dyn-reasoning-parser deepseek_v4` | Extracts chain-of-thought into `message.reasoning_content` | -| `--dyn-tool-call-parser deepseek_v4` | Emits OpenAI-compatible structured `tool_calls` | -| `--trust-remote-code` | Required for the V4 architecture's custom modeling code | -| `--tp 8` | Tensor-parallel across all 8 GPUs of one node | -| `--moe-runner-backend flashinfer_mxfp4` | MXFP4 MoE kernel via FlashInfer for the V4 expert weights | -| `--speculative-algo EAGLE` + `--speculative-num-steps 3` + `--speculative-eagle-topk 1` + `--speculative-num-draft-tokens 4` | EAGLE MTP speculative decoding (3 draft steps, top-1 over the EAGLE head, 4 draft tokens per step) | -| `--chunked-prefill-size 4096` | Chunk long prompts at 4k tokens for steady-state decode interleaving | -| `--disable-flashinfer-autotune` | Skip per-shape autotuning at startup; the dsv4 base ships pre-tuned defaults | - -### Why TP=8 (not DP=4 like Flash)? - -DeepSeek-V4-Pro is ~5.5x larger than Flash on disk (~865 GB vs. ~160 GB). With FP4+FP8 mixed weights it does not fit in 4 ranks at typical batch shapes, so the upstream tested shape for Pro is **TP=8** on both backends — across all 8 GPUs of one B200 node, or across two GB200 NVL4 trays connected by NVLink72. On vLLM, Expert Parallel is layered on top of TP — TP shards the dense (attention/router/norm) weights, EP shards the experts. On SGLang, the MXFP4 MoE backend handles the expert sharding internally under the same TP=8 process group. - -## Model Details - -Sourced from the [`deepseek-ai/DeepSeek-V4-Pro` model card](https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro) (preview release): - -| | | -|---|---| -| **Model** | `deepseek-ai/DeepSeek-V4-Pro` (MoE, 1.6T total / 49B active per token) | -| **Context length** | 1M tokens | -| **Checkpoint** | Mixed precision — MoE expert weights in FP4; most other parameters in FP8 | -| **Attention** | Hybrid Compressed Sparse Attention (CSA) + Heavily Compressed Attention (HCA). The vLLM variant enables the Blackwell FP4 indexer cache via `--attention-config '{"use_fp4_indexer_cache":true}'` | -| **Residual path** | Manifold-Constrained Hyper-Connections (mHC) | -| **Reasoning modes** | Three effort levels exposed via `chat_template_kwargs`: `{}` (Non-think), `{"thinking":true,"reasoning_effort":"high"}` (Think High), `{"thinking":true,"reasoning_effort":"max"}` (Think Max — needs `--max-model-len >= 393216`) | -| **Long-context efficiency** | Per the model card, ~27% of the per-token inference FLOPs and ~10% of the KV cache vs. DeepSeek-V3.2 at 1M context | -| **License** | MIT | - -Recipe-level (per-variant) settings: - -| | vLLM (`vllm-agg`) | SGLang (`sglang-agg`) | -|---|---|---| -| **Backend image** | Prebuilt `nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3` (multi-arch) | Prebuilt `nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda12-dev.3` | -| **Parallelism** | TP=8, Expert Parallel enabled | TP=8 | -| **MoE backend** | vLLM's V4 expert kernel (FP4) | FlashInfer MXFP4 | -| **KV cache** | FP8, block size 256 | engine default | -| **Speculative decoding** | — | EAGLE MTP (3 steps / 4 draft tokens) | - -## Verifying Reasoning - -Same flow on both variants — same model, same `--dyn-reasoning-parser deepseek_v4`. On the vLLM variant, omit `chat_template_kwargs.thinking` (or set it to `false`) per the Day-0 caveat: - -```bash -curl -s http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "deepseek-ai/DeepSeek-V4-Pro", - "messages": [{"role": "user", "content": "What is 2+2? Answer briefly."}], - "max_tokens": 200 - }' | python3 -m json.tool -``` - -Expected: - -- `choices[0].message.reasoning_content` contains the model's chain-of-thought. -- `choices[0].message.content` contains only the final answer. -- No raw `` tags in either field. - -If `reasoning_content` is `null` and `` appears in `content`, the reasoning parser isn't wired up — confirm `--dyn-reasoning-parser deepseek_v4` is on the worker command. - -## Verifying Tool Calling - -Same flow on both variants: - -```bash -curl -s http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "deepseek-ai/DeepSeek-V4-Pro", - "messages": [{"role": "user", "content": "What is the weather in San Francisco?"}], - "tools": [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "City name"} - }, - "required": ["location"] - } - } - }], - "max_tokens": 300 - }' | python3 -m json.tool -``` - -Expected: - -- `choices[0].message.tool_calls` is a structured array with `function.name`, `function.arguments`, and `id`. -- `choices[0].finish_reason` is `"tool_calls"`. -- `choices[0].message.reasoning_content` may contain the model's reasoning about tool selection. - -If `tool_calls` is missing and raw tool-call markers appear in `content`, confirm `--dyn-tool-call-parser deepseek_v4` is on the worker command. - -## Notes - -### Common - -- **Storage class.** Update `storageClassName` in `model-cache/model-cache.yaml` to a RWX class that can serve the PVC to Frontend and worker pods. -- **Model size.** `deepseek-ai/DeepSeek-V4-Pro` is ~865 GB on disk (64 safetensors shards in FP4+FP8 mixed form). The 1500Gi PVC leaves ~1.7x headroom for HF cache metadata and one alternate revision. -- **Parser flags.** Use the Dynamo variants on the worker (`--dyn-reasoning-parser`, `--dyn-tool-call-parser`). Each engine's native `--reasoning-parser` / `--tool-call-parser` are engine-side and do not feed the Dynamo OpenAI renderer. -- **Offline model cache.** Both workers run with `HF_HUB_OFFLINE=1` so the engine reads cached weights from the PVC and never contacts the HF Hub at startup. The HF token secret is mounted defensively; it isn't required at runtime once the download Job has completed. -- **First launch is slow.** Decode workers load weights across 8 TP ranks and warm CUDA graphs / DeepGEMM kernels on first launch; the manifests' startup probes allow ~60–90 min before failing readiness. - -### vLLM-specific - -- **Prebuilt images.** All three vLLM manifests (`vllm/agg/b200/`, `vllm/agg/gb200/`, `vllm/disagg/gb200/`) reference `nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3` (multi-arch). To rebuild from source (custom Dynamo branch, different vLLM base, etc.), see [`/container/README.md`](../../../container/README.md). -- **Engine-ready timeout.** `VLLM_ENGINE_READY_TIMEOUT_S=5400` is set to match the startup probe budget (`failureThreshold: 540` at `periodSeconds: 10`). -- **FlashInfer autotune.** `--no-enable-flashinfer-autotune` skips per-shape FlashInfer autotuning at startup and is set on every vLLM worker (prefill and decode). Required on dsv4: the autotuner currently produces tunings that regress GSM8k accuracy. Skipping it also shortens first-launch warmup. -- **GB200: agg vs. disagg.** Both spread V4-Pro across two GB200 NVL4 trays via MNNVL/ComputeDomain. The agg variant runs one TP=8 group across both nodes (lower-latency, simpler topology, 2 pods); the disagg variant runs separate prefill and decode DP=8 workers (higher steady-state throughput at high concurrency, 4 pods). Use the agg variant for general-purpose serving and the disagg variant when prefill/decode separation pays off for the workload. - -### SGLang-specific - -- **Prebuilt image.** `sglang/agg/deploy.yaml` already references the public NGC tag `nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda12-dev.3`. To rebuild (custom Dynamo branch, different SGLang base, etc.), see [`recipes/deepseek-v4/container/README.md`](../container/README.md). -- **DeepGEMM / FlashInfer warmup.** `SGLANG_JIT_DEEPGEMM_PRECOMPILE=0` + `SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1` skip the slow precompile and use the fast warmup path. `--disable-flashinfer-autotune` skips per-shape FlashInfer autotuning at startup; the dsv4 base ships pre-tuned defaults. -- **NCCL / Gloo.** `NCCL_CUMEM_ENABLE=1` is set for V4 NCCL collectives on Blackwell. `GLOO_SOCKET_IFNAME=eth0` pins Gloo to the standard pod interface. - -## Sibling Recipe - -[DeepSeek-V4-Flash](../deepseek-v4-flash/) is the smaller sibling (284B / 13B active, 4x B200) and shares the same dsv4 vLLM and SGLang container images. diff --git a/recipes/deepseek-v4/deepseek-v4-pro/model-cache/model-cache.yaml b/recipes/deepseek-v4/deepseek-v4-pro/model-cache/model-cache.yaml deleted file mode 100644 index 4c5d7fdc33a7..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-pro/model-cache/model-cache.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 1500Gi - storageClassName: "your-storage-class-name" diff --git a/recipes/deepseek-v4/deepseek-v4-pro/model-cache/model-download.yaml b/recipes/deepseek-v4/deepseek-v4-pro/model-cache/model-download.yaml deleted file mode 100644 index 30fb6d076ff3..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-pro/model-cache/model-download.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: MODEL_NAME - value: deepseek-ai/DeepSeek-V4-Pro - - name: HF_HOME - value: /model-store - # Uses up to 64 GB RAM for XET buffers; set "0" on low-memory nodes (8 GB cap): https://huggingface.co/docs/hub/en/xet/using-xet-storage#download-buffers - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.11.0 - hf download $MODEL_NAME - resources: - requests: - cpu: "2" - memory: "64Gi" - limits: - cpu: "8" - memory: "64Gi" - volumeMounts: - - name: model-cache - mountPath: /model-store - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/deepseek-v4/deepseek-v4-pro/sglang/agg-gb200/deploy.yaml b/recipes/deepseek-v4/deepseek-v4-pro/sglang/agg-gb200/deploy.yaml deleted file mode 100644 index 7a893917777f..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-pro/sglang/agg-gb200/deploy.yaml +++ /dev/null @@ -1,126 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# DeepSeek-V4-Pro SGLang Aggregated Serving on GB200 -# -# Topology: 2 nodes x 4 GPUs = TP8 (NCCL over Socket) -# -# EAGLE disabled: crashes on long context (>18k) and tool calling on TP8 - -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: sglang-dsv4-pro-agg - namespace: -spec: - envs: - - name: HF_HOME - value: /models - - name: HF_HUB_OFFLINE - value: "1" - - name: SGLANG_JIT_DEEPGEMM_PRECOMPILE - value: "0" - - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: GLOO_SOCKET_IFNAME - value: eth0 - pvcs: - - name: shared-model-cache - create: false - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: shared-model-cache - mountPoint: /models - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - imagePullPolicy: Always - securityContext: - runAsUser: 0 - runAsGroup: 0 - startupProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - - decode: - componentType: worker - subComponentType: decode - replicas: 1 - multinode: - nodeCount: 2 - resources: - limits: - gpu: "4" - volumeMounts: - - name: shared-model-cache - mountPoint: /models - sharedMemory: - size: 200Gi - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - imagePullPolicy: Always - workingDir: /workspace - command: - - python3 - - -m - - dingo.sglang - args: - - --model-path - - deepseek-ai/DeepSeek-V4-Pro - - --served-model-name - - deepseek-ai/DeepSeek-V4-Pro - - --trust-remote-code - - --tp - - "8" - - --moe-runner-backend - - flashinfer_mxfp4 - - --chunked-prefill-size - - "4096" - - --disable-flashinfer-autotune - - --dyn-tool-call-parser - - deepseek_v4 - - --dyn-reasoning-parser - - deepseek_v4 - - --watchdog-timeout - - "3600" - env: - # TODO: Remove LD_PRELOAD + NCCL Socket overrides once the image - # bundles a working NCCL for multi-node. Current image ships - # pynccl 2.27.7 which lacks Socket transport; LD_PRELOAD forces - # system NCCL 2.28.9. The NCCL_NET=Socket block is required for - # multi-node TP over TCP (no RDMA/NVLink between nodes). - - name: LD_PRELOAD - value: /usr/lib/aarch64-linux-gnu/libnccl.so.2 - - name: NCCL_NET - value: Socket - - name: NCCL_SOCKET_IFNAME - value: eth0 - - name: NCCL_P2P_DISABLE - value: "1" - - name: NCCL_SHM_DISABLE - value: "1" - - name: NCCL_IB_DISABLE - value: "1" - - name: NCCL_DEBUG - value: WARN - resources: {} - securityContext: - runAsUser: 0 - runAsGroup: 0 - startupProbe: - httpGet: - path: /health - port: 9090 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 540 diff --git a/recipes/deepseek-v4/deepseek-v4-pro/sglang/agg/deploy.yaml b/recipes/deepseek-v4/deepseek-v4-pro/sglang/agg/deploy.yaml deleted file mode 100644 index 0f61c6106a3d..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-pro/sglang/agg/deploy.yaml +++ /dev/null @@ -1,108 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSeek-V4-Pro SGLang DynamoGraphDeployment -# Aggregated mode, B200x8 TP8, MXFP4 MoE -# EAGLE disabled: crashes on long context (>18k tokens) and tool calling on B200 TP8 -# due to insufficient GPU memory headroom (4.23 GB/GPU with EAGLE vs 7.75 GB without) - -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: sglang-dsv4-pro -spec: - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: shared-model-cache - mountPoint: /models - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda12-dev.3 - imagePullPolicy: Always - env: - - name: HF_HOME - value: /models - - name: HF_HUB_OFFLINE - value: "1" - startupProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - - decode: - componentType: worker - subComponentType: decode - replicas: 1 - resources: - limits: - gpu: "8" - volumeMounts: - - name: shared-model-cache - mountPoint: /models - sharedMemory: - size: 200Gi - extraPodSpec: - nodeSelector: - nvidia.com/gpu.product: NVIDIA-B200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda12-dev.3 - imagePullPolicy: Always - workingDir: /workspace - command: - - python3 - - -m - - dingo.sglang - args: - - --model-path - - deepseek-ai/DeepSeek-V4-Pro - - --served-model-name - - deepseek-ai/DeepSeek-V4-Pro - - --trust-remote-code - - --tp - - "8" - - --moe-runner-backend - - flashinfer_mxfp4 - - --chunked-prefill-size - - "4096" - - --disable-flashinfer-autotune - - --dyn-tool-call-parser - - deepseek_v4 - - --dyn-reasoning-parser - - deepseek_v4 - env: - - name: HF_HOME - value: /models - - name: HF_HUB_OFFLINE - value: "1" - - name: CUDA_VISIBLE_DEVICES - value: "0,1,2,3,4,5,6,7" - - name: SGLANG_JIT_DEEPGEMM_PRECOMPILE - value: "0" - - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: GLOO_SOCKET_IFNAME - value: eth0 - startupProbe: - httpGet: - path: /health - port: 9090 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - - pvcs: - - name: shared-model-cache - create: false diff --git a/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-b200/README.md b/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-b200/README.md deleted file mode 100644 index 001a6b0af91a..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-b200/README.md +++ /dev/null @@ -1,74 +0,0 @@ - - -# DeepSeek-V4-Pro — Disaggregated Prefill/Decode on B200 - -Serves [deepseek-ai/DeepSeek-V4-Pro](https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro) using SGLang with disaggregated prefill/decode via Dynamo on B200 nodes with InfiniBand RDMA. - -## Topology - -| Role | Nodes | GPUs/node | Total GPUs | Parallelism | -|---------|-------|-----------|------------|-------------| -| Decode | 1 | 8 | 8 | TP8 | -| Prefill | 1 | 8 | 8 | TP8 | - -## Prerequisites - -- 2× B200 nodes with InfiniBand and `rdma/ib` device plugin -- Dynamo operator + dynamo-platform HelmRelease installed -- Shared RWX PVC for model weights (`shared-model-cache`) -- Hugging Face token secret: - ```bash - kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN= -n - ``` - -## Quick Start - -```bash -# 0. Replace in deploy.yaml with your namespace - -# 1. Deploy -kubectl apply -f deploy.yaml - -# 2. Test -kubectl port-forward svc/dsv4-pro-disagg-frontend 8000:8000 & -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model":"deepseek-ai/DeepSeek-V4-Pro","messages":[{"role":"user","content":"Hello!"}],"max_tokens":128}' -``` - -## InfiniBand RDMA Configuration - -The `deploy.yaml` includes InfiniBand-specific UCX configuration validated on nscale B200 (ConnectX-8): - -| Variable | Value | Purpose | -|----------|-------|---------| -| `UCX_TLS` | `rc_x,rc,cuda_copy,cuda_ipc` | Accelerated RC transport for IB RDMA | -| `UCX_NET_DEVICES` | `mlx5_0:1` | Single IB device (avoids bonded device issues) | -| `UCX_IB_ADDR_TYPE` | `eth` | Ethernet addressing — required for cross-pod IB on K8s | -| `UCX_RNDV_SCHEME` | `get_zcopy` | Zero-copy RDMA GET for KV transfers | -| `UCX_RNDV_THRESH` | `0` | Use rendezvous for all message sizes | -| `rdma/ib` | `8` (1 per GPU) | RDMA device plugin injects IB devices | - -### Why `UCX_IB_ADDR_TYPE=eth`? - -Without this setting, UCX uses LID-based InfiniBand addressing which does not route correctly between Kubernetes pods on different nodes. Setting `eth` switches to GID/Ethernet-style addressing that works across the pod network. This is the most common issue when bringing up NIXL disagg on InfiniBand clusters. - -### Bonded IB Device Gotcha - -Some clusters expose `mlx5_bond_0` with LID=0. Setting `UCX_NET_DEVICES=mlx5_0:1` avoids this device. If your cluster has a different IB device naming scheme, check with `ibv_devinfo`. - -## Key Configuration Notes - -- **EAGLE disabled** — Causes OOM on TP8 (4.23 GB free with EAGLE vs 7.75 without) -- **`mem-fraction-static=0.82`** — Higher than GB200 recipe (0.75) since B200 single-node TP avoids multi-node weight shuffle overhead -- **Security** — `IPC_LOCK` + `SYS_RESOURCE` capabilities required for RDMA memory pinning -- **Cold start ~20 min** — FP4 weight shuffle + CUDA graph capture - -## Related - -- [Disagg Communication Guide](../../../../../docs/kubernetes/disagg-communication-guide.md) — Full RDMA transport reference -- [GB200 Disagg Recipe](../disagg-gb200/) — Multi-node disagg on GB200 with ComputeDomain diff --git a/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-b200/deploy.yaml b/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-b200/deploy.yaml deleted file mode 100644 index b3ab83102c66..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-b200/deploy.yaml +++ /dev/null @@ -1,259 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# DeepSeek-V4-Pro SGLang Disaggregated P/D on B200 (InfiniBand) -# -# Topology: 2 nodes, single-node TP8 each -# prefill: 1 node × 8 GPUs = TP8 -# decode: 1 node × 8 GPUs = TP8 -# -# Prerequisites: -# - dynamo-platform HelmRelease applied -# - Model weights cached on shared-model-cache PVC -# - hf-token-secret created: kubectl create secret generic hf-token-secret \ -# --from-literal=HF_TOKEN= -n -# -# RDMA: Disaggregated KV transfer requires InfiniBand RDMA for NIXL. -# See README.md for cluster-specific details. -# -# EAGLE disabled: crashes on long context (>18k) and tool calling on TP8 -# due to insufficient GPU memory headroom (4.23 GB/GPU with vs 7.75 without) - -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: dsv4-pro-disagg - namespace: -spec: - backendFramework: sglang - envs: - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: HF_TOKEN - - name: HF_HOME - value: /models - - name: HF_HUB_OFFLINE - value: "1" - - name: GLOO_SOCKET_IFNAME - value: eth0 - - name: NCCL_SOCKET_IFNAME - value: eth0 - - pvcs: - - name: shared-model-cache - create: false - - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: shared-model-cache - mountPoint: /models - extraPodSpec: - mainContainer: - command: - - python3 - args: - - -m - - dingo.frontend - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda12-dev.2 - imagePullPolicy: Always - startupProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - - decode: - componentType: worker - subComponentType: decode - replicas: 1 - resources: - limits: - gpu: "8" - custom: - rdma/ib: "8" - volumeMounts: - - name: shared-model-cache - mountPoint: /models - sharedMemory: - size: 200Gi - extraPodSpec: - nodeSelector: - nvidia.com/gpu.product: NVIDIA-B200 - tolerations: - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - mainContainer: - command: - - bash - - -c - args: - - >- - ulimit -l unlimited && ulimit -n 1048576 && - exec python3 -m dingo.sglang - --model-path deepseek-ai/DeepSeek-V4-Pro - --served-model-name deepseek-ai/DeepSeek-V4-Pro - --trust-remote-code - --tp 8 - --moe-runner-backend flashinfer_mxfp4 - --chunked-prefill-size 4096 - --disable-flashinfer-autotune - --dyn-tool-call-parser deepseek_v4 - --dyn-reasoning-parser deepseek_v4 - --mem-fraction-static 0.82 - --host 0.0.0.0 - --disaggregation-mode decode - --disaggregation-transfer-backend nixl - --disaggregation-bootstrap-port 30001 - --prefill-round-robin-balance - --watchdog-timeout 3600 - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: SGLANG_JIT_DEEPGEMM_PRECOMPILE - value: "0" - - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP - value: "1" - # --- UCX (IB RDMA transport) --- - - name: UCX_TLS - value: "rc_x,rc,cuda_copy,cuda_ipc" - - name: UCX_NET_DEVICES - value: "mlx5_0:1" - - name: UCX_IB_ADDR_TYPE - value: "eth" - - name: UCX_RNDV_SCHEME - value: "get_zcopy" - - name: UCX_RNDV_THRESH - value: "0" - # --- NCCL --- - - name: NCCL_IB_DISABLE - value: "0" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_STORE_TIMEOUT - value: "7200" - # --- NIXL --- - - name: NIXL_LOG_LEVEL - value: INFO - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda12-dev.2 - imagePullPolicy: Always - securityContext: - runAsUser: 0 - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 5 - workingDir: /workspace/ - - prefill: - componentType: worker - subComponentType: prefill - replicas: 1 - resources: - limits: - gpu: "8" - custom: - rdma/ib: "8" - volumeMounts: - - name: shared-model-cache - mountPoint: /models - sharedMemory: - size: 200Gi - extraPodSpec: - nodeSelector: - nvidia.com/gpu.product: NVIDIA-B200 - tolerations: - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - mainContainer: - command: - - bash - - -c - args: - - >- - ulimit -l unlimited && ulimit -n 1048576 && - exec python3 -m dingo.sglang - --model-path deepseek-ai/DeepSeek-V4-Pro - --served-model-name deepseek-ai/DeepSeek-V4-Pro - --trust-remote-code - --tp 8 - --moe-runner-backend flashinfer_mxfp4 - --chunked-prefill-size 4096 - --disable-flashinfer-autotune - --dyn-tool-call-parser deepseek_v4 - --dyn-reasoning-parser deepseek_v4 - --mem-fraction-static 0.82 - --host 0.0.0.0 - --disaggregation-mode prefill - --disaggregation-transfer-backend nixl - --disaggregation-bootstrap-port 30001 - --load-balance-method round_robin - --watchdog-timeout 3600 - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: SGLANG_JIT_DEEPGEMM_PRECOMPILE - value: "0" - - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP - value: "1" - # --- UCX (IB RDMA transport) --- - - name: UCX_TLS - value: "rc_x,rc,cuda_copy,cuda_ipc" - - name: UCX_NET_DEVICES - value: "mlx5_0:1" - - name: UCX_IB_ADDR_TYPE - value: "eth" - - name: UCX_RNDV_SCHEME - value: "get_zcopy" - - name: UCX_RNDV_THRESH - value: "0" - # --- NCCL --- - - name: NCCL_IB_DISABLE - value: "0" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_STORE_TIMEOUT - value: "7200" - # --- NIXL --- - - name: NIXL_LOG_LEVEL - value: INFO - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda12-dev.2 - imagePullPolicy: Always - securityContext: - runAsUser: 0 - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 5 - workingDir: /workspace/ diff --git a/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-gb200/README.md b/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-gb200/README.md deleted file mode 100644 index 86cfb2a136bf..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-gb200/README.md +++ /dev/null @@ -1,98 +0,0 @@ - - -# DeepSeek-V4-Pro — Disaggregated Prefill/Decode on GB200 - -Serves [deepseek-ai/DeepSeek-V4-Pro](https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro) using SGLang with disaggregated prefill/decode via Dynamo on GB200 nodes. - -## Topology - -| Role | Nodes | GPUs/node | Total GPUs | Parallelism | -|---------|-------|-----------|------------|-------------| -| Decode | 2 | 4 | 8 | TP8 | -| Prefill | 2 | 4 | 8 | TP8 | - -## Prerequisites - -- 4× GB200 nodes with RDMA networking -- Dynamo operator + dynamo-platform HelmRelease installed -- ComputeDomain DRA driver (`compute-domain-default-channel.nvidia.com` device class) -- Shared RWX PVC for model weights - -## Quick Start - -```bash -# 1. Deploy -kubectl apply -f deploy.yaml - -# 2. Test -kubectl port-forward svc/dsv4-pro-disagg-frontend 8000:8000 & -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model":"deepseek-ai/DeepSeek-V4-Pro","messages":[{"role":"user","content":"Hello!"}],"max_tokens":128}' -``` - -## Cluster-Specific Configuration - -The base `deploy.yaml` is generic. Each cluster type requires additional configuration for RDMA networking (needed for NIXL KV cache transfer between prefill and decode). - -### GKE (GCP GB200) - -**RDMA annotations** — Add to both decode and prefill services: - -```yaml -# In spec.services.decode and spec.services.prefill: -annotations: - networking.gke.io/default-interface: eth0 - networking.gke.io/interfaces: | - [ - {"interfaceName":"eth0","network":"default"}, - {"interfaceName":"rdma0","network":"rdma-0"}, - {"interfaceName":"rdma1","network":"rdma-1"}, - {"interfaceName":"rdma2","network":"rdma-2"}, - {"interfaceName":"rdma3","network":"rdma-3"} - ] -resources: - limits: - custom: - networking.gke.io.networks/rdma-0: "1" - networking.gke.io.networks/rdma-1: "1" - networking.gke.io.networks/rdma-2: "1" - networking.gke.io.networks/rdma-3: "1" -``` - -**NATS workaround** — GCP clusters may isolate `system-cpu` and `customer-gpu` pod networks. If the dynamo-system NATS is unreachable, deploy a namespace-local NATS on a `customer-cpu` node and override `NATS_SERVER` in `spec.envs`. - -**Tolerations** — GCP GPU nodes have taints. Add to all services: - -```yaml -tolerations: - - key: dedicated - operator: Equal - value: user-workload - effect: NoExecute - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - - key: kubernetes.io/arch - operator: Equal - value: arm64 - effect: NoSchedule -``` - -### AWS EFA - -See [Disagg Communication Guide](../../../../../docs/kubernetes/disagg-communication-guide.md#aws-efa-configuration) for EFA-specific NIXL configuration with libfabric. - -### InfiniBand - -For clusters with native IB (e.g., DGX), add `rdma/ib` resource limits and ensure UCX UD transport is supported on your ConnectX firmware. - -## Key Configuration Notes - -- **EAGLE disabled** — Causes OOM on TP8 (4.23 GB free with EAGLE vs 7.75 without) -- **`mem-fraction-static=0.75`** — Lower than default to prevent OOM during FP4 MxFP4 weight shuffle -- **`cuda-graph-max-bs=128`** — Limits CUDA graph capture memory -- **Cold start ~45 min** — FP4 weight shuffle dominates startup time diff --git a/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-gb200/deploy.yaml b/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-gb200/deploy.yaml deleted file mode 100644 index 95d6656eb53b..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-gb200/deploy.yaml +++ /dev/null @@ -1,289 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# DeepSeek-V4-Pro SGLang Disaggregated P/D on GB200 -# -# Topology: 4 nodes total (ComputeDomain) -# prefill: 2 nodes × 4 GPUs = TP8 -# decode: 2 nodes × 4 GPUs = TP8 -# -# Prerequisites: -# - dynamo-platform HelmRelease applied -# - Model weights cached on shared-model-cache PVC -# - hf-token-secret created: kubectl create secret generic hf-token-secret \ -# --from-literal=HF_TOKEN= -n -# -# RDMA: Disaggregated KV transfer requires RDMA networking for NIXL. -# See README.md for cluster-specific RDMA configuration (GKE, EFA, IB). -# -# EAGLE disabled: crashes on long context (>18k) and tool calling on TP8 -# due to insufficient GPU memory headroom (4.23 GB/GPU with vs 7.75 without) - -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: dsv4-pro-disagg - namespace: -spec: - backendFramework: sglang - envs: - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: HF_TOKEN - - name: HF_HOME - value: /models - - name: HF_HUB_OFFLINE - value: "1" - - name: GLOO_SOCKET_IFNAME - value: eth0 - - name: NCCL_SOCKET_IFNAME - value: eth0 - - pvcs: - - name: shared-model-cache - create: false - - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: shared-model-cache - mountPoint: /models - extraPodSpec: - mainContainer: - command: - - python3 - args: - - -m - - dingo.frontend - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - imagePullPolicy: Always - startupProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - - decode: - componentType: worker - subComponentType: decode - replicas: 1 - multinode: - nodeCount: 2 - resources: - limits: - gpu: "4" - claims: - - name: compute-domain-channel - volumeMounts: - - name: shared-model-cache - mountPoint: /models - sharedMemory: - size: 200Gi - extraPodSpec: - resourceClaims: - - name: compute-domain-channel - resourceClaimTemplateName: dsv4-pro-compute-domain-channel - mainContainer: - command: - - bash - - -c - args: - - >- - ulimit -l unlimited && ulimit -n 1048576 && - exec python3 -m dingo.sglang - --model-path deepseek-ai/DeepSeek-V4-Pro - --served-model-name deepseek-ai/DeepSeek-V4-Pro - --trust-remote-code - --tp 8 - --moe-runner-backend flashinfer_mxfp4 - --chunked-prefill-size 4096 - --disable-flashinfer-autotune - --dyn-tool-call-parser deepseek_v4 - --dyn-reasoning-parser deepseek_v4 - --mem-fraction-static 0.75 - --cuda-graph-max-bs 128 - --host 0.0.0.0 - --disaggregation-mode decode - --disaggregation-transfer-backend nixl - --disaggregation-bootstrap-port 30001 - --prefill-round-robin-balance - --watchdog-timeout 3600 - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: SGLANG_JIT_DEEPGEMM_PRECOMPILE - value: "0" - - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP - value: "1" - - name: NCCL_IB_DISABLE - value: "0" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: NVIDIA_GDRCOPY - value: "1" - - name: NCCL_STORE_TIMEOUT - value: "7200" - - name: UCX_TLS - value: "cuda_ipc,cuda_copy,rc" - - name: UCX_IB_GID_INDEX - value: "3" - - name: UCX_RC_TIMEOUT - value: "600s" - - name: UCX_KEEPALIVE_INTERVAL - value: "300s" - - name: NIXL_LOG_LEVEL - value: INFO - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - imagePullPolicy: Always - securityContext: - capabilities: - add: - - IPC_LOCK - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 5 - workingDir: /workspace/ - - prefill: - componentType: worker - subComponentType: prefill - replicas: 1 - multinode: - nodeCount: 2 - resources: - limits: - gpu: "4" - claims: - - name: compute-domain-channel - volumeMounts: - - name: shared-model-cache - mountPoint: /models - sharedMemory: - size: 200Gi - extraPodSpec: - resourceClaims: - - name: compute-domain-channel - resourceClaimTemplateName: dsv4-pro-compute-domain-channel - mainContainer: - command: - - bash - - -c - args: - - >- - ulimit -l unlimited && ulimit -n 1048576 && - exec python3 -m dingo.sglang - --model-path deepseek-ai/DeepSeek-V4-Pro - --served-model-name deepseek-ai/DeepSeek-V4-Pro - --trust-remote-code - --tp 8 - --moe-runner-backend flashinfer_mxfp4 - --chunked-prefill-size 4096 - --disable-flashinfer-autotune - --dyn-tool-call-parser deepseek_v4 - --dyn-reasoning-parser deepseek_v4 - --mem-fraction-static 0.75 - --cuda-graph-max-bs 128 - --host 0.0.0.0 - --disaggregation-mode prefill - --disaggregation-transfer-backend nixl - --disaggregation-bootstrap-port 30001 - --load-balance-method round_robin - --watchdog-timeout 3600 - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: SGLANG_JIT_DEEPGEMM_PRECOMPILE - value: "0" - - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP - value: "1" - - name: NCCL_IB_DISABLE - value: "0" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: NVIDIA_GDRCOPY - value: "1" - - name: NCCL_STORE_TIMEOUT - value: "7200" - - name: UCX_TLS - value: "cuda_ipc,cuda_copy,rc" - - name: UCX_IB_GID_INDEX - value: "3" - - name: UCX_RC_TIMEOUT - value: "600s" - - name: UCX_KEEPALIVE_INTERVAL - value: "300s" - - name: NIXL_LOG_LEVEL - value: INFO - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - imagePullPolicy: Always - securityContext: - capabilities: - add: - - IPC_LOCK - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 5 - workingDir: /workspace/ ---- -apiVersion: resource.nvidia.com/v1beta1 -kind: ComputeDomain -metadata: - name: dsv4-pro-compute-domain - namespace: -spec: - channel: - allocationMode: Single - resourceClaimTemplate: - name: dsv4-pro-compute-domain-channel - numNodes: 4 ---- -apiVersion: resource.k8s.io/v1 -kind: ResourceClaimTemplate -metadata: - name: dsv4-pro-compute-domain-channel - namespace: -spec: - spec: - devices: - config: - - opaque: - driver: compute-domain.nvidia.com - parameters: - allocationMode: Single - apiVersion: resource.nvidia.com/v1beta1 - kind: ComputeDomainChannelConfig - requests: - - channel - requests: - - exactly: - allocationMode: ExactCount - count: 1 - deviceClassName: compute-domain-default-channel.nvidia.com - name: channel diff --git a/recipes/deepseek-v4/deepseek-v4-pro/vllm/agg/b200/deploy.yaml b/recipes/deepseek-v4/deepseek-v4-pro/vllm/agg/b200/deploy.yaml deleted file mode 100644 index eb8b0a2464a0..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-pro/vllm/agg/b200/deploy.yaml +++ /dev/null @@ -1,112 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# DynamoGraphDeployment for deepseek-ai/DeepSeek-V4-Pro on vLLM, -# aggregated serving (no prefill/decode disaggregation). -# -# Upstream reference command: -# docker run --gpus all ... vllm/vllm-openai:deepseekv4-cu130 \ -# deepseek-ai/DeepSeek-V4-Pro \ -# --trust-remote-code --kv-cache-dtype fp8 --block-size 256 \ -# --enable-expert-parallel --tensor-parallel-size 8 \ -# --compilation-config '{"mode": 0, "cudagraph_mode": "FULL_DECODE_ONLY"}' \ -# --attention_config.use_fp4_indexer_cache=True \ -# --tokenizer-mode deepseek_v4 \ -# --tool-call-parser deepseek_v4 --enable-auto-tool-choice \ -# --reasoning-parser deepseek_v4 -# -# Shape: 1 replica x 8 GPUs, TP=8 + Expert Parallel. Fills a single 8-GPU node. -# Weights: served from the `model-cache` PVC populated by -# `../../model-cache/model-download.yaml`. -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: dsv4-pro-agg -spec: - backendFramework: vllm - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /opt/models - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - workingDir: /workspace/examples/backends/vllm - env: - - name: HF_HOME - value: /opt/models - - name: HF_HUB_OFFLINE - value: "1" - VllmDecodeWorker: - componentType: worker - subComponentType: decode - envFromSecret: hf-token-secret - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 200Gi - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - workingDir: /workspace/examples/backends/vllm - # DeepSeek-V4-Pro (1.6T params) is large; first launch loads weights - # over TP=8 ranks plus FlashInfer autotune + cudagraph warmup. Allow - # ~90 min: periodSeconds * failureThreshold = 10 * 540 = 5400s. - startupProbe: - httpGet: - path: /health - port: 9090 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 540 - env: - - name: SERVED_MODEL_NAME - value: deepseek-ai/DeepSeek-V4-Pro - - name: MODEL_PATH - value: deepseek-ai/DeepSeek-V4-Pro - - name: HF_HOME - value: /opt/models - # Read weights from the PVC only; do not hit the HF Hub at startup. - - name: HF_HUB_OFFLINE - value: "1" - # Give the engine room to finish first-launch init. - - name: VLLM_ENGINE_READY_TIMEOUT_S - value: "5400" - # Stabilize TP/EP all-reduces and skip the IPC P2P probe. - - name: VLLM_SKIP_P2P_CHECK - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - command: - - /bin/sh - - -c - args: - - | - python3 -m dingo.vllm \ - --model "${MODEL_PATH}" \ - --served-model-name "${SERVED_MODEL_NAME}" \ - --trust-remote-code \ - --kv-cache-dtype fp8 \ - --block-size 256 \ - --tensor-parallel-size 8 \ - --enable-expert-parallel \ - --tokenizer-mode deepseek_v4 \ - --dyn-reasoning-parser deepseek_v4 \ - --dyn-tool-call-parser deepseek_v4 \ - --attention-config '{"use_fp4_indexer_cache":true}' \ - --compilation-config '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}' \ - --no-enable-flashinfer-autotune \ - --max-num-seqs 256 - replicas: 1 - resources: - limits: - gpu: "8" - requests: - gpu: "8" diff --git a/recipes/deepseek-v4/deepseek-v4-pro/vllm/agg/gb200/deploy.yaml b/recipes/deepseek-v4/deepseek-v4-pro/vllm/agg/gb200/deploy.yaml deleted file mode 100644 index 148b44bcdb33..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-pro/vllm/agg/gb200/deploy.yaml +++ /dev/null @@ -1,155 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# DynamoGraphDeployment for deepseek-ai/DeepSeek-V4-Pro on vLLM, aggregated. -# -# Shape: 1 decode replica spanning 2 GB200 nodes with 4 GPUs per node -# (TP=8 + EP, no DP). Total: 2 nodes x 4 GPUs = 8 GPUs. -# -# DSV4-Pro is too large for a single 4-GPU NVL4 tray, so the tensor-parallel -# group spans both nodes. Cross-node TP all-reduce/all-gather flows over -# NVLink72 (MNNVL) via ComputeDomain, not RoCE/IB. -# -# Prerequisites: DRA / ComputeDomain controller installed cluster-wide. -# Weights: served from the `model-cache` PVC (see ../../model-cache/model-download.yaml). ---- -apiVersion: resource.nvidia.com/v1beta1 -kind: ComputeDomain -metadata: - name: dsv4-pro-agg-compute-domain -spec: - numNodes: 0 - channel: - resourceClaimTemplate: - name: dsv4-pro-agg-compute-domain-channel ---- -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: dsv4-pro-agg -spec: - backendFramework: vllm - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 40Gi - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - workingDir: /workspace/examples/backends/vllm - env: - - name: HF_HOME - value: /opt/models - - name: HF_HUB_OFFLINE - value: "1" - VllmDecodeWorker: - componentType: worker - subComponentType: decode - envFromSecret: hf-token-secret - replicas: 1 - multinode: - nodeCount: 2 - resources: - limits: - gpu: "4" - requests: - gpu: "4" - claims: - - name: compute-domain-channel - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 200Gi - extraPodSpec: - resourceClaims: - - name: compute-domain-channel - resourceClaimTemplateName: dsv4-pro-agg-compute-domain-channel - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - workingDir: /workspace/examples/backends/vllm - # ~90 min budget for first launch (weights + autotune + cudagraph): 10 * 540 = 5400s. - startupProbe: - httpGet: - path: /health - port: 9090 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 540 - env: - - name: SERVED_MODEL_NAME - value: deepseek-ai/DeepSeek-V4-Pro - - name: MODEL_PATH - value: deepseek-ai/DeepSeek-V4-Pro - - name: HF_HOME - value: /opt/models - - name: HF_HUB_OFFLINE - value: "1" - - name: VLLM_ENGINE_READY_TIMEOUT_S - value: "5400" - - name: TILELANG_CLEANUP_TEMP_FILES - value: "1" - - name: VLLM_SERVER_DEV_MODE - value: "1" - - name: VLLM_USE_NCCL_SYMM_MEM - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_MNNVL_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: NCCL_P2P_LEVEL - value: NVL - - name: NCCL_STORE_TIMEOUT - value: "7200" - - name: NCCL_SOCKET_IFNAME - value: eth0 - - name: NVIDIA_GDRCOPY - value: "1" - - name: GLOO_SOCKET_IFNAME - value: eth0 - - name: UCX_MEMTYPE_CACHE - value: "n" - - name: UCX_TLS - value: cuda_copy,cuda_ipc,tcp - - name: UCX_CUDA_IPC_ENABLE_MNNVL - value: "y" - command: - - /bin/sh - - -c - args: - - | - # No DP coordinator; torch.distributed master binds MASTER_PORT (29500) - # for cross-node TP rendezvous, satisfying the operator's - # wait-for-leader-mp probe. - # mode:3 + pass_config.fuse_allreduce_rms:true enables FlashInfer's - # TRT-LLM fused allreduce + RMSNorm kernel for the cross-node TP - # group. If FlashInfer fails to allocate the workspace at startup - # vLLM logs a non-fatal warning and falls back to the non-fused path. - exec python3 -m dingo.vllm \ - --model "${MODEL_PATH}" \ - --served-model-name "${SERVED_MODEL_NAME}" \ - --trust-remote-code \ - --kv-cache-dtype fp8 \ - --block-size 256 \ - --tensor-parallel-size 8 \ - --pipeline-parallel-size 1 \ - --enable-expert-parallel \ - --max-num-seqs 256 \ - --gpu-memory-utilization 0.9 \ - --tokenizer-mode deepseek_v4 \ - --dyn-reasoning-parser deepseek_v4 \ - --dyn-tool-call-parser deepseek_v4 \ - --attention-config '{"use_fp4_indexer_cache":true}' \ - --moe-backend deep_gemm_mega_moe \ - --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"pass_config":{"fuse_allreduce_rms":true}}' \ - --no-enable-flashinfer-autotune diff --git a/recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/deploy.yaml b/recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/deploy.yaml deleted file mode 100644 index 0b94e66c88b5..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/deploy.yaml +++ /dev/null @@ -1,267 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# DynamoGraphDeployment for deepseek-ai/DeepSeek-V4-Pro on vLLM, -# disaggregated prefill/decode serving. -# -# Shape: 1 prefill replica + 1 decode replica, each spanning 2 GB200 nodes -# with 4 GPUs per node (DP=8, TP=1). Total: 4 nodes x 4 GPUs = 16 GPUs. -# -# Cross-node fabric: NVLink72 (MNNVL) via ComputeDomain. UCX active-messages -# control plane goes over TCP (eth0); bulk KV flows over MNNVL/cuda_ipc. -# -# Prerequisites: DRA / ComputeDomain controller installed cluster-wide. -# Weights: served from the `model-cache` PVC (see ../../model-cache/model-download.yaml). ---- -apiVersion: resource.nvidia.com/v1beta1 -kind: ComputeDomain -metadata: - name: dsv4-pro-compute-domain -spec: - numNodes: 0 - channel: - resourceClaimTemplate: - name: dsv4-pro-compute-domain-channel ---- -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: dsv4-pro-disagg -spec: - backendFramework: vllm - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 40Gi - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - workingDir: /workspace/examples/backends/vllm - env: - - name: HF_HOME - value: /opt/models - VllmPrefillWorker: - componentType: worker - subComponentType: prefill - envFromSecret: hf-token-secret - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 200Gi - extraPodSpec: - resourceClaims: - - name: compute-domain-channel - resourceClaimTemplateName: dsv4-pro-compute-domain-channel - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - workingDir: /workspace/examples/backends/vllm - # ~90 min budget for first launch (weights + autotune + cudagraph): 10 * 540 = 5400s. - startupProbe: - httpGet: - path: /health - port: 9090 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 540 - env: - - name: SERVED_MODEL_NAME - value: deepseek-ai/DeepSeek-V4-Pro - - name: MODEL_PATH - value: deepseek-ai/DeepSeek-V4-Pro - - name: HF_HOME - value: /opt/models - - name: VLLM_ENGINE_READY_TIMEOUT_S - value: "5400" - - name: TILELANG_CLEANUP_TEMP_FILES - value: "1" - - name: VLLM_SPARSE_INDEXER_MAX_LOGITS_MB - value: "1024" - - name: VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE - value: "2048" - - name: VLLM_SERVER_DEV_MODE - value: "1" - - name: VLLM_USE_NCCL_SYMM_MEM - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_MNNVL_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: NCCL_P2P_LEVEL - value: NVL - - name: NCCL_STORE_TIMEOUT - value: "7200" - - name: NCCL_SOCKET_IFNAME - value: eth0 - - name: NVIDIA_GDRCOPY - value: "1" - - name: GLOO_SOCKET_IFNAME - value: eth0 - # UCX_TLS must include an AM-capable transport (tcp); cuda_ipc/cuda_copy - # are memory-type-only and cannot carry NIXL's UCX control plane. - - name: UCX_MEMTYPE_CACHE - value: "n" - - name: UCX_TLS - value: cuda_copy,cuda_ipc,tcp - - name: UCX_CUDA_IPC_ENABLE_MNNVL - value: "y" - command: - - /bin/sh - - -c - args: - - | - # Bind vLLM's DP coordinator on :29500 so the operator's wait-for-leader-mp - # init container's TCP probe is satisfied by the real RPC server. - exec python3 -m dingo.vllm \ - --model "${MODEL_PATH}" \ - --served-model-name "${SERVED_MODEL_NAME}" \ - --trust-remote-code \ - --kv-cache-dtype fp8 \ - --block-size 256 \ - --tensor-parallel-size 1 \ - --pipeline-parallel-size 1 \ - --data-parallel-size 8 \ - --data-parallel-rpc-port 29500 \ - --disaggregation-mode prefill \ - --enable-expert-parallel \ - --enforce-eager \ - --max-model-len 9280 \ - --max-num-seqs 16 \ - --max-num-batched-tokens 32768 \ - --no-enable-prefix-caching \ - --no-enable-flashinfer-autotune \ - --no-async-scheduling \ - --gpu-memory-utilization 0.8 \ - --no-disable-hybrid-kv-cache-manager \ - --enable-sleep-mode \ - --numa-bind \ - --offload-group-size 3 \ - --offload-num-in-group 1 \ - --offload-prefetch-step 2 \ - --tokenizer-mode deepseek_v4 \ - --dyn-reasoning-parser deepseek_v4 \ - --dyn-tool-call-parser deepseek_v4 \ - --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - replicas: 1 - multinode: - nodeCount: 2 - resources: - limits: - gpu: "4" - requests: - gpu: "4" - claims: - - name: compute-domain-channel - VllmDecodeWorker: - componentType: worker - subComponentType: decode - envFromSecret: hf-token-secret - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 200Gi - extraPodSpec: - resourceClaims: - - name: compute-domain-channel - resourceClaimTemplateName: dsv4-pro-compute-domain-channel - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3 - workingDir: /workspace/examples/backends/vllm - startupProbe: - httpGet: - path: /health - port: 9090 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 540 - env: - - name: SERVED_MODEL_NAME - value: deepseek-ai/DeepSeek-V4-Pro - - name: MODEL_PATH - value: deepseek-ai/DeepSeek-V4-Pro - - name: HF_HOME - value: /opt/models - - name: VLLM_ENGINE_READY_TIMEOUT_S - value: "5400" - - name: TILELANG_CLEANUP_TEMP_FILES - value: "1" - - name: VLLM_SERVER_DEV_MODE - value: "1" - - name: VLLM_USE_NCCL_SYMM_MEM - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_MNNVL_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: NCCL_P2P_LEVEL - value: NVL - - name: NCCL_STORE_TIMEOUT - value: "7200" - - name: NCCL_SOCKET_IFNAME - value: eth0 - - name: NVIDIA_GDRCOPY - value: "1" - - name: GLOO_SOCKET_IFNAME - value: eth0 - - name: UCX_MEMTYPE_CACHE - value: "n" - - name: UCX_TLS - value: cuda_copy,cuda_ipc,tcp - - name: UCX_CUDA_IPC_ENABLE_MNNVL - value: "y" - command: - - /bin/sh - - -c - args: - - | - # See VllmPrefillWorker for the :29500 / wait-for-leader-mp rationale. - exec python3 -m dingo.vllm \ - --model "${MODEL_PATH}" \ - --served-model-name "${SERVED_MODEL_NAME}" \ - --trust-remote-code \ - --kv-cache-dtype fp8 \ - --block-size 256 \ - --tensor-parallel-size 1 \ - --pipeline-parallel-size 1 \ - --data-parallel-size 8 \ - --data-parallel-rpc-port 29500 \ - --disaggregation-mode decode \ - --enable-expert-parallel \ - --max-model-len 9280 \ - --max-num-seqs 128 \ - --max-cudagraph-capture-size 128 \ - --max-num-batched-tokens 128 \ - --no-enable-prefix-caching \ - --no-enable-flashinfer-autotune \ - --gpu-memory-utilization 0.9 \ - --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' \ - --stream-interval 50 \ - --no-disable-hybrid-kv-cache-manager \ - --enable-sleep-mode \ - --tokenizer-mode deepseek_v4 \ - --dyn-reasoning-parser deepseek_v4 \ - --dyn-tool-call-parser deepseek_v4 \ - --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - replicas: 1 - multinode: - nodeCount: 2 - resources: - limits: - gpu: "4" - requests: - gpu: "4" - claims: - - name: compute-domain-channel diff --git a/recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml b/recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml deleted file mode 100644 index 40265d685b81..000000000000 --- a/recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml +++ /dev/null @@ -1,143 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Perf Job for the dsv4-pro-disagg DGD: 8K input / 1K output, concurrency -# sweep 256x512x1024 -# -# Topology assumed (set by the matching deploy.yaml): -# 1 prefill (DP=8 across 2 GB200 nodes) + 1 decode (DP=8 across 2 GB200 nodes) -# => 16 decode GPUs, c=512 -# -# Deploy first: kubectl -n apply -f deploy.yaml -# Then run perf: kubectl -n apply -f perf.yaml -apiVersion: batch/v1 -kind: Job -metadata: - name: dsv4-pro-disagg-bench -spec: - backoffLimit: 1 - completions: 1 - parallelism: 1 - template: - spec: - restartPolicy: Never - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - seccompProfile: - type: RuntimeDefault - containers: - - name: bench - image: python:3.12-slim - securityContext: - allowPrivilegeEscalation: false - imagePullPolicy: IfNotPresent - env: - - name: TARGET_MODEL - value: deepseek-ai/DeepSeek-V4-Pro - - name: ENDPOINT - value: dsv4-pro-disagg-frontend:8000 - - name: ISL - value: "8192" - - name: OSL - value: "1024" - # Concurrency sweep — each value runs as its own aiperf profile under a - # per-concurrency artifact subdir. Override to a single value for a quick - # smoke run, e.g. CONCURRENCIES: "512". - - name: CONCURRENCIES - value: "256 512 1024" - - name: ARTIFACT_ROOT - value: /opt/models/perf/dsv4-pro-disagg-isl8k-osl1k - - name: TOKENIZER - value: deepseek-ai/DeepSeek-V4-Pro - - name: HF_HOME - value: /opt/models - - name: PYTHONUNBUFFERED - value: "1" - - name: COLUMNS - value: "200" - # HOME defaults to "/" for uid 1000 with no /etc/passwd entry; pip --user - # then tries to write /.local (read-only). Point HOME at /tmp (1777 sticky) - # so ~/.local resolves under a writable path. - - name: HOME - value: /tmp - - name: PATH - value: /tmp/.local/bin:/usr/local/bin:/usr/bin:/bin - envFrom: - - secretRef: - name: hf-token-secret - command: - - /bin/sh - - -c - - | - set -eu - # No apt-get: cluster runs us as non-root, and we don't actually need - # curl/jq — Python stdlib handles the readiness probe. Install aiperf - # into HOME=/tmp/.local via --user so it works under runAsNonRoot. - # NOTE: aiperf>=0.8.0 depends on crick, which has no prebuilt arm64 wheel - # and requires gcc/libc6-dev to build from source. This pod runs non-root - # with no apt-get access, so this install will fail on arm64 nodes. - # Ensure the cluster scheduler places this pod on an x86_64 node. - pip install --user -q aiperf==0.10.0 transformers==4.57.3 tokenizers==0.22.2 - - echo "Waiting for model at http://$ENDPOINT/v1/models..." - python3 - <<'PY' - import json, os, time, urllib.error, urllib.request - endpoint, target = os.environ["ENDPOINT"], os.environ["TARGET_MODEL"] - while True: - try: - with urllib.request.urlopen(f"http://{endpoint}/v1/models", timeout=5) as r: - data = json.load(r) - if any(m.get("id") == target for m in data.get("data", [])): - break - except (urllib.error.URLError, ValueError, KeyError): - pass - print(f"[{time.strftime('%H:%M:%S')}] not ready, retrying in 10s...", flush=True) - time.sleep(10) - print("Model ready!", flush=True) - PY - - for c in $CONCURRENCIES; do - ARTIFACT_DIR="$ARTIFACT_ROOT/c${c}" - mkdir -p "$ARTIFACT_DIR" - echo "" - echo "================================================================" - echo " concurrency=$c isl=$ISL osl=$OSL artifact=$ARTIFACT_DIR" - echo "================================================================" - - aiperf profile \ - --artifact-dir "$ARTIFACT_DIR" \ - --model "$TARGET_MODEL" \ - --tokenizer "$TOKENIZER" \ - --tokenizer-trust-remote-code \ - --endpoint-type chat \ - --endpoint /v1/chat/completions \ - --streaming \ - --url "http://$ENDPOINT" \ - --synthetic-input-tokens-mean $ISL \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean $OSL \ - --output-tokens-stddev 0 \ - --extra-inputs "max_tokens:$OSL" \ - --extra-inputs "min_tokens:$OSL" \ - --extra-inputs "ignore_eos:true" \ - --concurrency $c \ - --request-count $((c * 3)) \ - --warmup-request-count 16 \ - --num-dataset-entries 12800 \ - --random-seed 100 \ - --ui simple - done - - echo "" - echo "All sweeps complete. Results under $ARTIFACT_ROOT:" - ls -la "$ARTIFACT_ROOT" - volumeMounts: - - name: model-cache - mountPath: /opt/models - workingDir: /workspace - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/gaie_checks.sh b/recipes/gaie_checks.sh deleted file mode 100755 index 1c7018c469ac..000000000000 --- a/recipes/gaie_checks.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -Eeuo pipefail - -# ===== Namespace ensure ===== -# The documented GAIE flow sets NAMESPACE explicitly; the fallback matches the installer. -: "${NAMESPACE:=default}" -: "${GATEWAY_CONTROLLER_NAMESPACE:=${AGW_NAMESPACE:-agentgateway-system}}" - -ok() { printf "✅ %s\n" "$*"; } -fail(){ printf "❌ %s\n" "$*" >&2; exit 1; } -info(){ printf "ℹ️ %s\n" "$*"; } - -need() { command -v "$1" >/dev/null 2>&1 || fail "'$1' is required"; } - -need kubectl - -# ===== Pre-flight checks ===== -command -v helm >/dev/null 2>&1 || { echo "ERROR: helm not found"; exit 1; } -command -v kubectl >/dev/null 2>&1 || { echo "ERROR: kubectl not found"; exit 1; } - -GATEWAY_CRDS=( - gateways.gateway.networking.k8s.io - gatewayclasses.gateway.networking.k8s.io - httproutes.gateway.networking.k8s.io - referencegrants.gateway.networking.k8s.io -) -info "Checking Gateway API CRDs…" -for c in "${GATEWAY_CRDS[@]}"; do - kubectl get crd "$c" >/dev/null 2>&1 || fail "Missing CRD: $c (run step a)" - kubectl wait --for=condition=Established "crd/$c" --timeout=60s >/dev/null || fail "CRD not Established: $c" -done -ok "Gateway API CRDs present & Established" - -GAIE_CRDS=( - inferencemodelrewrites.inference.networking.x-k8s.io - inferenceobjectives.inference.networking.x-k8s.io - inferencepoolimports.inference.networking.x-k8s.io - inferencepools.inference.networking.k8s.io - inferencepools.inference.networking.x-k8s.io -) - -info "Checking GAIE (Inference Extension) CRDs…" -for c in "${GAIE_CRDS[@]}"; do - kubectl get crd "$c" >/dev/null 2>&1 || fail "Missing CRD: $c (run step b install of inference extension)" - kubectl wait --for=condition=Established "crd/$c" --timeout=60s >/dev/null || fail "CRD not Established: $c" -done -ok "GAIE CRDs present & Established" - -info "Checking agentgateway controller in namespace '$GATEWAY_CONTROLLER_NAMESPACE'…" -# namespace must exist -kubectl get ns "$GATEWAY_CONTROLLER_NAMESPACE" >/dev/null 2>&1 || fail "Namespace '$GATEWAY_CONTROLLER_NAMESPACE' not found (run step c Helm installs)" - -# pods should be running -if ! kubectl get pods -n "$GATEWAY_CONTROLLER_NAMESPACE" -l app.kubernetes.io/name=agentgateway >/dev/null 2>&1; then - # fallback label (charts sometimes label differently) - PODS=$(kubectl get pods -n "$GATEWAY_CONTROLLER_NAMESPACE" -o name | grep -E 'agentgateway|gateway' || true) - [[ -z "${PODS:-}" ]] && fail "agentgateway pods not found in '$GATEWAY_CONTROLLER_NAMESPACE'" -else - PODS=$(kubectl get pods -n "$GATEWAY_CONTROLLER_NAMESPACE" -l app.kubernetes.io/name=agentgateway -o name) -fi -for p in $PODS; do - kubectl wait -n "$GATEWAY_CONTROLLER_NAMESPACE" --for=condition=Ready "$p" --timeout=180s >/dev/null || fail "Pod not Ready: $p" -done -ok "agentgateway controller pods Ready" - -kubectl get gateway.gateway.networking.k8s.io inference-gateway -n "$NAMESPACE" >/dev/null 2>&1 || fail "Gateway 'inference-gateway' not found in $NAMESPACE (apply step d manifest)" - -ok "GAIE is installed and the gateway is up in namespace '$NAMESPACE'." diff --git a/recipes/glm-5-nvfp4/README.md b/recipes/glm-5-nvfp4/README.md deleted file mode 100644 index 70a137e541e8..000000000000 --- a/recipes/glm-5-nvfp4/README.md +++ /dev/null @@ -1,164 +0,0 @@ - - -# GLM-5 NVFP4 — Disaggregated Prefill/Decode on GB200 - -Serves [nvidia/GLM-5-NVFP4](https://huggingface.co/nvidia/GLM-5-NVFP4) using SGLang with -disaggregated prefill/decode and EAGLE speculative decoding via Dynamo on GB200 nodes. - -For GB200 on AWS, please see the [efa variant](sglang/disagg/efa/README.md). - -## Topology - -| Role | Nodes | GPUs/node | Total GPUs | Parallelism | -|---------|-------|-----------|------------|--------------------| -| Decode | 4 | 4 | 16 | TP16 / DP16 / EP16 | -| Prefill | 1 | 4 | 4 | TP4 | - -## Prerequisites - -- 5 4xGB200 nodes in an NVL36 or NVL72 domain -- A Kubernetes cluster with the [Dynamo Operator](../../docs/kubernetes/README.md) installed -- DRA / ComputeDomain support for MNNVL placement -- Access to `nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.1.1-cuda13` -- Shared RWX PVC for model weights and FlashInfer JIT artifacts - -The manifest uses standard NVIDIA GPU Feature Discovery labels to select GB200 -nodes and includes common GPU/ARM tolerations. If your cluster uses different -labels, taints, or storage classes, update `nodeSelector`, `tolerations`, and -`storageClassName` before deploying. - -## Step 1: Use the Published Runtime Image - -This recipe uses the stable Dynamo SGLang runtime image for GB200: -`nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.1.1-cuda13`. -The worker containers run as root because FlashInfer's bundled cubin package creates -TRTLLM MoE symlinks inside its installed package directory during startup. - -## Step 2: Download the Model - -Create the PVC, HuggingFace token secret, and download the model weights: - -```bash -export NAMESPACE= -kubectl create namespace ${NAMESPACE} - -# Edit model-cache.yaml first and set storageClassName to a RWX storage class. -kubectl apply -f recipes/glm-5-nvfp4/model-cache/model-cache.yaml -n ${NAMESPACE} - -kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN= \ - -n ${NAMESPACE} - -kubectl apply -f recipes/glm-5-nvfp4/model-cache/model-download.yaml -n ${NAMESPACE} -kubectl wait --for=condition=complete job/model-download -n ${NAMESPACE} --timeout=3600s -``` - -If your cluster already provides a shared RWX cache PVC, skip -`model-cache/model-cache.yaml` and update `claimName: model-cache` in -`model-cache/model-download.yaml`, `sglang/disagg/deploy.yaml`, and -`sglang/disagg/perf.yaml` to that PVC's name. Keep the mount path as -`/model-store` so the workers, model download job, and benchmark job share the -same HuggingFace cache and JIT artifacts. - -## Step 3: Deploy - -Edit `sglang/disagg/deploy.yaml` and replace the namespace placeholder: - -- `` — the value of `${NAMESPACE}` - -```bash -kubectl apply -f recipes/glm-5-nvfp4/sglang/disagg/deploy.yaml -kubectl wait --for=condition=Ready pod \ - -l nvidia.com/dynamo-graph-deployment-name=glm5-sglang \ - -n ${NAMESPACE} --timeout=7200s -``` - -Monitor startup. First cold starts can take up to about an hour while the stable -runtime loads weights and JIT-compiles FlashInfer/DeepGEMM kernels: - -```bash -kubectl get pods -n ${NAMESPACE} -l app.kubernetes.io/part-of=glm5-sglang -w -``` - -## Step 4: Test - -```bash -kubectl port-forward svc/glm5-sglang-frontend 8000:8000 -n ${NAMESPACE} & -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model":"nvidia/GLM-5-NVFP4","messages":[{"role":"user","content":"Hello!"}],"max_tokens":128}' -``` - -## Step 5: Benchmark (optional) - -Edit `sglang/disagg/perf.yaml` to replace the namespace placeholder with -`${NAMESPACE}`, then run: - -```bash -kubectl apply -f recipes/glm-5-nvfp4/sglang/disagg/perf.yaml -kubectl logs -f -l job-name=glm5-disagg-bench -n ${NAMESPACE} -``` - -Default benchmark: ISL=1000, OSL=8192, concurrency=512 (32/GPU). - -The benchmark pod runs as a non-root user and installs AIPerf into `/tmp/.local`. -It pins Transformers v5 because `nvidia/GLM-5-NVFP4` declares -`tokenizer_class=TokenizersBackend`, which older Transformers v4 releases do not -load. - -## Key Configuration Notes - -### Speculative Decoding (EAGLE MTP) -Two env vars enable working speculative decoding (~85-95% accept rate): - -- `SGLANG_ENABLE_SPEC_V2=1` — uses EAGLEWorkerV2 with overlap scheduler -- `SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE=1` — quantizes BF16 MTP layer to FP8 at load - time, matching the base model's compute path - -The MTP layer weights in `nvidia/GLM-5-NVFP4` are BF16 (split across shards 271-274) -and are fully indexed in the checkpoint's `model.safetensors.index.json`. - -### KV Cache -Uses `--kv-cache-dtype fp8_e4m3` (NSA backend auto-selects this on SM100/GB200). -Saves ~50% KV memory vs BF16. - -### FlashInfer JIT Cache -The stable runtime includes FlashInfer but not a prebuilt `flashinfer-jit-cache` -wheel. The recipe sets `FLASHINFER_WORKSPACE_BASE=/model-store` so first-run JIT -artifacts are written to the shared model PVC and reused by later pod starts. - -### NIXL / UCX -NIXL uses UCX for disaggregated KV transfer. The recipe sets -`UCX_TLS=cuda_copy,cuda_ipc,tcp` so CUDA IPC/copy can handle GPU memory movement -while TCP provides UCX active-message control traffic. - -### Discovery -Uses Kubernetes service discovery. Worker registration is tied to pod lifetime -via Kubernetes EndpointSlices, preventing TTL expiry issues under high load. - -### Recovery and Rollouts -The decode side is one TP16 rank group spread across four GB200 nodes. Treat -single decode-pod replacement as disruptive, and validate full-group recovery in -your cluster before relying on individual decode pod restarts for production -rollouts. In validation, cold deploy plus frontend and prefill replacement -recovered successfully, while deleting one decode worker restarted the decode -leader and the graph stayed NotReady during repeated rank-group reinitialization -attempts. - -## Performance (ISL=1k, OSL=8k, concurrency=512) - -Reference AIPerf run: 1,536 requests, 0 errors, 747.87s benchmark duration. -This is a concurrency-burst benchmark, so TTFT includes queueing under 512 -concurrent users. - -| Metric | Value | -|--------|-------| -| Output throughput | 16,824 tokens/sec | -| Request throughput | 2.05 requests/sec | -| TTFT p50 | 15,423ms | -| ITL avg | 23.31ms/token | -| Tokens/user/sec avg | 43.39 | -| Request errors | 0 | \ No newline at end of file diff --git a/recipes/glm-5-nvfp4/model-cache/model-cache.yaml b/recipes/glm-5-nvfp4/model-cache/model-cache.yaml deleted file mode 100644 index 244c1f5eda51..000000000000 --- a/recipes/glm-5-nvfp4/model-cache/model-cache.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 400Gi - storageClassName: "your-storage-class-name" diff --git a/recipes/glm-5-nvfp4/model-cache/model-download.yaml b/recipes/glm-5-nvfp4/model-cache/model-download.yaml deleted file mode 100644 index 35b716ed7ad6..000000000000 --- a/recipes/glm-5-nvfp4/model-cache/model-download.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - securityContext: - capabilities: - drop: ["ALL"] - seccompProfile: - type: RuntimeDefault - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: MODEL_NAME - value: nvidia/GLM-5-NVFP4 - - name: HF_HOME - value: /model-store - # Uses up to 64 GB RAM for XET buffers; set "0" on low-memory nodes (8 GB cap): https://huggingface.co/docs/hub/en/xet/using-xet-storage#download-buffers - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.11.0 - hf download $MODEL_NAME - resources: - requests: - cpu: "2" - memory: "64Gi" - limits: - cpu: "8" - memory: "64Gi" - volumeMounts: - - name: model-cache - mountPath: /model-store - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/glm-5-nvfp4/sglang/disagg/deploy.yaml b/recipes/glm-5-nvfp4/sglang/disagg/deploy.yaml deleted file mode 100644 index ab3fdec8eae4..000000000000 --- a/recipes/glm-5-nvfp4/sglang/disagg/deploy.yaml +++ /dev/null @@ -1,332 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# GLM-5 (NVFP4) disaggregated prefill/decode on GB200 via SGLang + Dynamo -# -# Prerequisites: -# - dynamo-platform HelmRelease applied -# - Model weights cached in a model-cache PVC mounted at /model-store -# - Dynamo SGLang runtime image nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.1.1-cuda13 -# - HF_TOKEN secret created: kubectl create secret generic hf-token-secret \ -# --from-literal=HF_TOKEN= -n -# -# Topology: 5 nodes total -# decode: 4 nodes x 4 GPUs = TP16 / DP16 / EP16 (EAGLE speculative decode) -# prefill: 1 node x 4 GPUs = TP4 -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: glm5-sglang - namespace: -spec: - backendFramework: sglang - envs: - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: HF_TOKEN - - name: HF_HOME - value: /model-store - - services: - Frontend: - componentType: frontend - replicas: 1 - extraPodSpec: - nodeSelector: - nvidia.com/gpu.product: NVIDIA-GB200 - tolerations: - - key: "nvidia.com/gpu" - operator: "Exists" - effect: "NoSchedule" - - key: "kubernetes.io/arch" - operator: "Equal" - value: "arm64" - effect: "NoSchedule" - mainContainer: - command: - - python3 - args: - - -m - - dingo.frontend - - --enforce-disagg - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.1.1-cuda13 - name: "" - resources: {} - volumeMounts: - - mountPath: /model-store - name: model-cache - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - decode: - componentType: worker - subComponentType: decode - replicas: 1 - multinode: - nodeCount: 4 - resources: - limits: - gpu: "4" - claims: - - name: compute-domain-channel - extraPodSpec: - nodeSelector: - nvidia.com/gpu.product: NVIDIA-GB200 - tolerations: - - key: "nvidia.com/gpu" - operator: "Exists" - effect: "NoSchedule" - - key: "kubernetes.io/arch" - operator: "Equal" - value: "arm64" - effect: "NoSchedule" - mainContainer: - command: - - bash - - -c - args: - - >- - ulimit -l unlimited && ulimit -n 1048576 && - exec python3 -m dingo.sglang - --model-path nvidia/GLM-5-NVFP4 - --served-model-name nvidia/GLM-5-NVFP4 - --trust-remote-code - --quantization modelopt_fp4 - --kv-cache-dtype fp8_e4m3 - --tensor-parallel-size 16 - --data-parallel-size 16 - --expert-parallel-size 16 - --enable-dp-attention - --enable-dp-lm-head - --max-running-requests 2048 - --cuda-graph-max-bs 128 - --mem-fraction-static 0.72 - --chunked-prefill-size 32768 - --max-prefill-tokens 32768 - --enable-flashinfer-allreduce-fusion - --attention-backend nsa - --nsa-decode-backend trtllm - --nsa-prefill-backend trtllm - --moe-runner-backend flashinfer_trtllm - --speculative-algorithm EAGLE - --speculative-num-steps 2 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 3 - --stream-interval 50 - --incremental-streaming-output - --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 32}' - --host 0.0.0.0 - --disaggregation-mode decode - --disaggregation-transfer-backend nixl - --disaggregation-bootstrap-port 30001 - --prefill-round-robin-balance - --watchdog-timeout 3600 - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: FLASHINFER_DISABLE_VERSION_CHECK - value: "1" - - name: FLASHINFER_WORKSPACE_BASE - value: /model-store - - name: SGLANG_ENABLE_JIT_DEEPGEMM - value: "1" - - name: SGLANG_ENABLE_SPEC_V2 - value: "1" - - name: SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE - value: "1" - - name: SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK - value: "1024" - - name: NCCL_IB_DISABLE - value: "0" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: NVIDIA_GDRCOPY - value: "1" - - name: NCCL_STORE_TIMEOUT - value: "7200" - - name: NCCL_GRAPH_MIXING_SUPPORT - value: "0" - # NIXL's UCX control plane requires an active-message transport. - # CUDA IPC/copy handle GPU memory movement; TCP carries control traffic. - - name: UCX_MEMTYPE_CACHE - value: "n" - - name: UCX_TLS - value: "cuda_copy,cuda_ipc,tcp" - - name: UCX_CUDA_IPC_ENABLE_MNNVL - value: "y" - - name: UCX_LOG_LEVEL - value: error - - name: NIXL_LOG_LEVEL - value: ERROR - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.1.1-cuda13 - name: "" - resources: {} - securityContext: - # FlashInfer's packaged cubins create package-local symlinks during - # TRTLLM MoE startup. - runAsUser: 0 - capabilities: - add: - - IPC_LOCK - startupProbe: - failureThreshold: 120 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 5 - volumeMounts: - - mountPath: /model-store - name: model-cache - workingDir: /workspace/ - resourceClaims: - - name: compute-domain-channel - resourceClaimTemplateName: glm5-compute-domain-channel - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - prefill: - componentType: worker - subComponentType: prefill - replicas: 1 - resources: - limits: - gpu: "4" - extraPodSpec: - nodeSelector: - nvidia.com/gpu.product: NVIDIA-GB200 - tolerations: - - key: "nvidia.com/gpu" - operator: "Exists" - effect: "NoSchedule" - - key: "kubernetes.io/arch" - operator: "Equal" - value: "arm64" - effect: "NoSchedule" - mainContainer: - command: - - bash - - -c - args: - - >- - ulimit -l unlimited && ulimit -n 1048576 && - exec python3 -m dingo.sglang - --model-path nvidia/GLM-5-NVFP4 - --served-model-name nvidia/GLM-5-NVFP4 - --trust-remote-code - --quantization modelopt_fp4 - --kv-cache-dtype fp8_e4m3 - --tensor-parallel-size 4 - --data-parallel-size 1 - --expert-parallel-size 1 - --enable-dp-lm-head - --max-running-requests 2048 - --cuda-graph-max-bs 128 - --mem-fraction-static 0.72 - --chunked-prefill-size 32768 - --max-prefill-tokens 32768 - --enable-flashinfer-allreduce-fusion - --attention-backend nsa - --nsa-decode-backend trtllm - --nsa-prefill-backend trtllm - --moe-runner-backend flashinfer_trtllm - --stream-interval 50 - --incremental-streaming-output - --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 32}' - --host 0.0.0.0 - --disaggregation-mode prefill - --disaggregation-transfer-backend nixl - --disaggregation-bootstrap-port 30001 - --load-balance-method round_robin - --watchdog-timeout 3600 - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: FLASHINFER_DISABLE_VERSION_CHECK - value: "1" - - name: FLASHINFER_WORKSPACE_BASE - value: /model-store - - name: SGLANG_ENABLE_JIT_DEEPGEMM - value: "1" - - name: SGLANG_ENABLE_SPEC_V2 - value: "1" - - name: SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE - value: "1" - - name: SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK - value: "1024" - - name: NCCL_IB_DISABLE - value: "0" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: NVIDIA_GDRCOPY - value: "1" - - name: NCCL_STORE_TIMEOUT - value: "7200" - # NIXL's UCX control plane requires an active-message transport. - # CUDA IPC/copy handle GPU memory movement; TCP carries control traffic. - - name: UCX_MEMTYPE_CACHE - value: "n" - - name: UCX_TLS - value: "cuda_copy,cuda_ipc,tcp" - - name: UCX_CUDA_IPC_ENABLE_MNNVL - value: "y" - - name: UCX_LOG_LEVEL - value: error - - name: NIXL_LOG_LEVEL - value: ERROR - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.1.1-cuda13 - name: "" - resources: {} - securityContext: - # FlashInfer's packaged cubins create package-local symlinks during - # TRTLLM MoE startup. - runAsUser: 0 - capabilities: - add: - - IPC_LOCK - startupProbe: - failureThreshold: 120 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 5 - volumeMounts: - - mountPath: /model-store - name: model-cache - workingDir: /workspace/ - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache ---- -# ComputeDomain enables MNNVL scheduling for the 5-node decode+prefill group. -apiVersion: resource.nvidia.com/v1beta1 -kind: ComputeDomain -metadata: - name: glm5-compute-domain - namespace: -spec: - channel: - allocationMode: Single - resourceClaimTemplate: - name: glm5-compute-domain-channel - numNodes: 5 diff --git a/recipes/glm-5-nvfp4/sglang/disagg/efa/Dockerfile.efa b/recipes/glm-5-nvfp4/sglang/disagg/efa/Dockerfile.efa deleted file mode 100644 index a820215539b6..000000000000 --- a/recipes/glm-5-nvfp4/sglang/disagg/efa/Dockerfile.efa +++ /dev/null @@ -1,249 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# EFA-enabled image for GLM-5-NVFP4 disaggregated serving on AWS p6e-gb200. -# Differences vs ./Dockerfile (the non-EFA variant): -# 1. Installs AWS EFA installer (libfabric headers + provider + aws-ofi-nccl) -# 2. Rebuilds libfabric from aws/libfabric@v2.4.0amzn1.0 with the -# efa_mr_is_cuda patch (ofiwg/libfabric#12019) applied to prov/efa/src/efa_mr.c -# 3. Builds GDRCopy + UCX with --with-cuda --with-verbs --with-efa --with-gdrcopy -# 4. Builds aws-ofi-nccl against the patched libfabric -# -# Without the patch, NIXL's fi_mr_reg on CUDA VRAM returns EFAULT on GB200 -# (arm64 + 64K-page kernel) and disagg KV transfer crashes at startup. -# The patch is a no-op on H100/amd64 (the legacy ibv_reg_mr path handles CUDA -# natively there), so it's safe to bake regardless of target hardware. -# -# References: -# - libfabric PR: https://github.com/ofiwg/libfabric/pull/12019 -# - SGLang patch: https://github.com/sgl-project/sglang/pull/17146 -ARG BASE_IMAGE="lmsysorg/sglang:v0.5.10.post1-cu130" -ARG ARCH=arm64 -ARG DYNAMO_COMMIT="main" -ARG CARGO_BUILD_JOBS="16" -ARG EFA_INSTALLER_VERSION="1.47.0" -ARG LIBFABRIC_BRANCH="v2.4.0amzn1.0" -ARG GDRCOPY_VERSION="v2.5.1" -ARG UCX_VERSION="v1.20.0" -ARG AWS_OFI_NCCL_VERSION="1.18.0" - -# ======== Stage 1: Builder (compiles Rust, builds Dynamo wheel) ======== -FROM ${BASE_IMAGE} AS builder - -ARG DYNAMO_COMMIT -ARG CARGO_BUILD_JOBS -ENV CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS} - -WORKDIR /build - -RUN set -eux; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - ca-certificates curl build-essential pkg-config libclang-dev protobuf-compiler git; \ - rm -rf /var/lib/apt/lists/*; \ - curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable - -ENV CARGO_HOME=/root/.cargo \ - RUSTUP_HOME=/root/.rustup -ENV PATH="${CARGO_HOME}/bin:${PATH}" - -RUN cargo install maturin --locked - -RUN git clone https://github.com/ai-dynamo/dynamo.git /build/dynamo && \ - cd /build/dynamo && git checkout ${DYNAMO_COMMIT} - -RUN --mount=type=cache,target=/root/.cargo/registry,sharing=shared \ - --mount=type=cache,target=/root/.cargo/git,sharing=shared \ - cd /build/dynamo/lib/bindings/python && \ - maturin build --release && \ - mkdir -p /build/dist && \ - cp target/wheels/*.whl /build/dist/ - -# ======== Stage 2: EFA stack builder ======== -FROM ${BASE_IMAGE} AS efa-builder - -ARG EFA_INSTALLER_VERSION -ARG LIBFABRIC_BRANCH -ARG GDRCOPY_VERSION -ARG UCX_VERSION -ARG AWS_OFI_NCCL_VERSION - -ENV DEBIAN_FRONTEND=noninteractive - -# Build dependencies for libfabric, GDRCopy, UCX, aws-ofi-nccl, plus the -# packages the AWS EFA installer expects (pciutils/environment-modules/tcl -# live in `universe`, which the NVIDIA CUDA base images strip from -# /etc/apt/sources.list — re-enable it first). -RUN set -eux; \ - if [ -f /etc/apt/sources.list ]; then \ - sed -i 's/^# *deb \(.*\) universe/deb \1 universe/' /etc/apt/sources.list || true; \ - fi; \ - apt-get update; \ - apt-get install -y --no-install-recommends software-properties-common; \ - add-apt-repository -y universe; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - ca-certificates curl git build-essential pkg-config \ - autoconf automake libtool \ - libnl-3-dev libnl-route-3-dev \ - check libsubunit-dev \ - flex bison \ - kmod \ - pciutils environment-modules tcl \ - libhwloc-dev libnuma-dev \ - libibverbs-dev librdmacm-dev rdma-core ibverbs-providers; \ - rm -rf /var/lib/apt/lists/* - -# 1. AWS EFA installer — provides headers, stock libfabric, aws-ofi-nccl source. -# We skip kmod (host already has efa kernel module) and limit-conf (handled -# via ulimit in pod spec). Pre-update apt so the installer's internal -# apt-get calls find packages from the universe pocket we just enabled. -RUN set -eux; \ - apt-get update; \ - cd /tmp; \ - curl -fsSL -O https://efa-installer.amazonaws.com/aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz; \ - tar xzf aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz; \ - cd aws-efa-installer; \ - ./efa_installer.sh -y --skip-kmod --skip-limit-conf --no-verify; \ - rm -rf /tmp/aws-efa-installer*; \ - rm -rf /var/lib/apt/lists/* - -# 2. Build aws/libfabric@v2.4.0amzn1.0 with the efa_mr_is_cuda patch -# (ofiwg/libfabric#12019). The sed targets the same line in both the AWS -# fork and upstream, so it's idempotent across libfabric versions. -# -# Critical: use aws/libfabric (NOT ofiwg/libfabric upstream). The AWS fork -# carries downstream EFA tuning that yields ~40% better TTFT on disagg -# workloads. -RUN set -eux; \ - git clone --depth 1 --branch ${LIBFABRIC_BRANCH} \ - https://github.com/aws/libfabric.git /tmp/libfabric; \ - cd /tmp/libfabric; \ - sed -i \ - 's/efa_mr_is_neuron(efa_mr) || efa_mr_is_rocr(efa_mr)/efa_mr_is_neuron(efa_mr) || efa_mr_is_rocr(efa_mr) || efa_mr_is_cuda(efa_mr)/' \ - prov/efa/src/efa_mr.c; \ - grep -F "efa_mr_is_cuda(efa_mr)" prov/efa/src/efa_mr.c; \ - ./autogen.sh; \ - CPPFLAGS="-I/usr/local/cuda/include" \ - LDFLAGS="-L/usr/local/cuda/lib64 -L/usr/local/cuda/lib64/stubs -Wl,-rpath,/usr/local/cuda/lib64" \ - ./configure --prefix=/opt/amazon/efa \ - --disable-verbs --disable-psm3 --disable-opx --disable-usnic \ - --enable-efa --with-cuda=/usr/local/cuda --enable-cuda-dlopen; \ - make -j"$(nproc)"; \ - make install; \ - rm -rf /tmp/libfabric - -# 3. GDRCopy (required for GPU-Direct buffer registration on EFA). -RUN set -eux; \ - git clone --depth 1 --branch ${GDRCOPY_VERSION} \ - https://github.com/NVIDIA/gdrcopy.git /tmp/gdrcopy; \ - cd /tmp/gdrcopy; \ - make CUDA=/usr/local/cuda lib install; \ - rm -rf /tmp/gdrcopy - -# 4. UCX with CUDA + verbs + EFA + GDRCopy. Without --with-cuda the NIXL UCX -# backend falls back to TCP (~10 Gbps). -RUN set -eux; \ - git clone --depth 1 --branch ${UCX_VERSION} \ - https://github.com/openucx/ucx.git /tmp/ucx; \ - cd /tmp/ucx; \ - ./autogen.sh; \ - ./configure --prefix=/usr/local/ucx \ - --with-cuda=/usr/local/cuda --with-verbs \ - --with-efa --with-gdrcopy --enable-mt --enable-shared; \ - make -j"$(nproc)"; \ - make install; \ - rm -rf /tmp/ucx - -# 5. aws-ofi-nccl, built against the patched libfabric. ABI mismatch between -# aws-ofi-nccl and libfabric silently halves bandwidth — they must be -# built together. -RUN set -eux; \ - git clone --depth 1 --branch v${AWS_OFI_NCCL_VERSION} \ - https://github.com/aws/aws-ofi-nccl.git /tmp/aws-ofi-nccl; \ - cd /tmp/aws-ofi-nccl; \ - ./autogen.sh; \ - ./configure --prefix=/opt/aws-ofi-nccl \ - --with-libfabric=/opt/amazon/efa \ - --with-cuda=/usr/local/cuda \ - --with-mpi=no; \ - make -j"$(nproc)"; \ - make install; \ - rm -rf /tmp/aws-ofi-nccl - -# ======== Stage 3: Final image ======== -FROM ${BASE_IMAGE} -ARG ARCH=arm64 -ARG EFA_INSTALLER_VERSION - -ENV DEBIAN_FRONTEND=noninteractive - -# Run the AWS EFA installer in the final stage too — this brings the matching -# rdma-core / libefa.so.1 ABI (EFA_1.4+) into /usr/lib/aarch64-linux-gnu so -# the patched libfabric we overlay next can load successfully. Pattern is -# borrowed from the customer's working Dockerfile. -RUN set -eux; \ - if [ -f /etc/apt/sources.list ]; then \ - sed -i 's/^# *deb \(.*\) universe/deb \1 universe/' /etc/apt/sources.list || true; \ - fi; \ - apt-get update; \ - apt-get install -y --no-install-recommends software-properties-common; \ - add-apt-repository -y universe; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - ca-certificates curl \ - pciutils environment-modules tcl \ - libhwloc-dev libnuma-dev \ - libibverbs-dev librdmacm-dev rdma-core ibverbs-providers \ - libnl-3-200 libnl-route-3-200 autoconf automake libtool; \ - cd /tmp; \ - curl -fsSL -O https://efa-installer.amazonaws.com/aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz; \ - tar xzf aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz; \ - cd aws-efa-installer; \ - ./efa_installer.sh -y --skip-kmod --skip-limit-conf --no-verify; \ - rm -rf /tmp/aws-efa-installer*; \ - rm -rf /var/lib/apt/lists/*; \ - rm -f /lib/aarch64-linux-gnu/libfabric.so* \ - /usr/lib/aarch64-linux-gnu/libfabric.so* 2>/dev/null || true - -# Overlay ibfabric (+ matched aws-ofi-nccl, UCX, GDRCopy) onto -# the EFA-installer base. This keeps the new libefa from the apt-managed -# rdma-core while replacing libfabric.so.1 with the CUDA-aware build. -COPY --from=efa-builder /opt/amazon/efa/lib /opt/amazon/efa/lib -COPY --from=efa-builder /opt/amazon/efa/include /opt/amazon/efa/include -COPY --from=efa-builder /opt/amazon/efa/share /opt/amazon/efa/share -COPY --from=efa-builder /opt/aws-ofi-nccl /opt/aws-ofi-nccl -COPY --from=efa-builder /usr/local/ucx /usr/local/ucx -COPY --from=efa-builder /usr/local/lib/libgdrapi.so* /usr/local/lib/ -COPY --from=efa-builder /usr/local/include/gdrapi.h /usr/local/include/ - -RUN set -eux; \ - echo "/opt/amazon/efa/lib" > /etc/ld.so.conf.d/efa.conf; \ - echo "/opt/amazon/efa/lib64" >> /etc/ld.so.conf.d/efa.conf; \ - echo "/opt/aws-ofi-nccl/lib" > /etc/ld.so.conf.d/aws-ofi-nccl.conf; \ - echo "/usr/local/ucx/lib" > /etc/ld.so.conf.d/ucx.conf; \ - echo "/usr/local/ucx/lib/ucx" >> /etc/ld.so.conf.d/ucx.conf; \ - ldconfig - -# Path order: /opt/amazon/efa first so libfabric is the patched build, not -# the stock one the base image may still have under /lib. -ENV PATH="/opt/amazon/efa/bin:/usr/local/ucx/bin:${PATH}" \ - LD_LIBRARY_PATH="/opt/amazon/efa/lib:/opt/amazon/efa/lib64:/opt/aws-ofi-nccl/lib:/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/usr/local/cuda/lib64:/usr/local/lib:${LD_LIBRARY_PATH}" - -# Verify the libfabric patch is in the installed libfabric. Doesn't probe runtime -# behavior (that happens during sglang init) — just confirms the patch text -# is present in the libfabric source bundled into share/ and that fi_info -# can at least invoke without missing-symbol errors. -RUN /opt/amazon/efa/bin/fi_info --version | grep -i libfabric - -# Install flashinfer with matching jit-cache (same as non-EFA Dockerfile). -RUN pip install "flashinfer-python==0.6.7" "flashinfer-cubin==0.6.7" && \ - pip install "flashinfer-jit-cache==0.6.7" --index-url https://flashinfer.ai/whl/cu130 - -# Install dynamo from source (includes _compat.py which handles sglang API changes). -COPY --from=builder /build/dynamo /sgl-workspace/dynamo -COPY --from=builder /build/dist/*.whl /tmp/ -RUN pip install --no-cache-dir /tmp/*.whl && \ - cd /sgl-workspace/dynamo && \ - pip install --no-cache-dir -e ".[sglang]" --no-deps && \ - rm -f /tmp/*.whl diff --git a/recipes/glm-5-nvfp4/sglang/disagg/efa/README.md b/recipes/glm-5-nvfp4/sglang/disagg/efa/README.md deleted file mode 100644 index ff789fceb4ca..000000000000 --- a/recipes/glm-5-nvfp4/sglang/disagg/efa/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# GLM-5 NVFP4 — Disaggregated Prefill/Decode on GB200 over AWS EFA - -Serves [nvidia/GLM-5-NVFP4](https://huggingface.co/nvidia/GLM-5-NVFP4) using SGLang with disaggregated prefill/decode, EAGLE speculative decoding, and KV cache transfer over AWS Elastic Fabric Adapter (EFA) RDMA via NIXL's LIBFABRIC backend. - -Compared to the non-EFA recipe (UCX/RoCE), this variant: - -- Carries inter-node KV transfer over EFA via NIXL LIBFABRIC backend. -- Bakes [ofiwg/libfabric#12019](https://github.com/ofiwg/libfabric/pull/12019) into the image so `fi_mr_reg` on CUDA VRAM succeeds on GB200's 64K-page arm64 kernel. -- Runs containers `privileged: true` so `fi_mr_reg` can pin VRAM for RDMA. -- Sets `SGLANG_DISAGGREGATION_NIXL_BACKEND=LIBFABRIC` — without this, SGLang defaults to UCX, which silently falls back to TCP on kernel ≥ 6.8. - -Please see the [Dynamo on EFA](../../../../../docs/kubernetes/cloud-providers/eks/efa.md) for more on EFA. - -## Topology - -Identical to the non-EFA recipe. - - -| Role | Nodes | GPUs/node | EFA NICs/node | Total GPUs | Parallelism | -| ------- | ----- | --------- | ------------- | ---------- | ------------------ | -| Decode | 4 | 4 | 4 | 16 | TP16 / DP16 / EP16 | -| Prefill | 1 | 4 | 4 | 4 | TP4 | - - -## Prerequisites - -- A Kubernetes cluster with **5 × p6e-gb200.36xlarge** nodes (or equivalent GB200 in an MNNVL domain) — each node provides 4× GB200 + 4× EFA NICs. -- AWS EFA driver ≥ 3.0.0g on the nodes (default on modern AWS EKS AMIs). -- Kernel ≥ 5.12 (modern AWS EKS AMIs ship kernel 6.14 — fine). On older kernels, the host also needs the `efa_nv_peermem` module loaded; on ≥ 5.12 the dmabuf path is the default and peermem is not required. -- A Kubernetes cluster with the Dynamo Operator installed. -- The NVIDIA `ComputeDomain` operator (for the MNNVL ResourceClaim used here). -- Shared NFS PVC for model weights (same as the non-EFA recipe). - -The libfabric is built into the image — no cluster-side DaemonSet is required. - -## Step 1: Build the Container - -```bash -docker buildx build \ - --platform linux/arm64 \ - --build-arg ARCH=arm64 \ - -t /sglang-dynamo-glm5-efa:latest \ - -f recipes/glm-5-nvfp4/sglang/disagg/efa/Dockerfile.efa \ - --push . -``` - -Quick sanity check on the resulting image: - -```bash -docker run --rm /sglang-dynamo-glm5-efa:latest \ - bash -c ' - /opt/amazon/efa/bin/fi_info --version && - ldd /opt/amazon/efa/lib/libfabric.so.1 | grep -i cuda && - grep -F "efa_mr_is_cuda(efa_mr)" \ - /opt/amazon/efa/share/doc/libfabric/efa_mr.c.diff 2>/dev/null || - echo "(patch verified at build time via Dockerfile RUN grep)"' -``` - -## Step 2: Download the Model - -Same as the non-EFA recipe: - -```bash -kubectl apply -f recipes/glm-5-nvfp4/model-cache/model-cache.yaml - -kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN= - -kubectl apply -f recipes/glm-5-nvfp4/model-cache/model-download.yaml -kubectl wait --for=condition=complete job/model-download --timeout=3600s -``` - -## Step 3: Deploy - -Edit `sglang/disagg/efa/deploy.yaml` and replace all `` values: - -- `` — your Kubernetes namespace -- `/sglang-dynamo-glm5-efa:latest` — your built container image - -```bash -kubectl apply -f recipes/glm-5-nvfp4/sglang/disagg/efa/deploy.yaml -``` - -Monitor startup: - -```bash -kubectl get pods -n -l app.kubernetes.io/part-of=glm5-sglang-efa -w -``` - -## Step 4: Verify EFA Engaged - -Three quick checks confirm the LIBFABRIC backend is actually carrying KV traffic (not silent TCP fallback): - -```bash -NS= -POD=$(kubectl -n $NS get pods -l nvidia.com/dynamo-component=decode \ - -o jsonpath='{.items[0].metadata.name}') - -# 1. NIXL selected LIBFABRIC at startup. -kubectl -n $NS logs $POD | grep -i 'NIXL.*backend.*LIBFABRIC' | head -1 -# Expected: "NIXL INFO Backend LIBFABRIC was instantiated" or similar. -# WRONG: "NIXL agent uses UCX backend". - -# 2. libplugin_LIBFABRIC.so is actually executing (not just dlopen'd). -kubectl -n $NS exec $POD -- bash -c ' - grep libplugin_LIBFABRIC /proc/$(pgrep -f sglang | head -1)/maps | - grep "r-xp"' | head -3 -# Expected: at least one line ending in "r-xp" (executable mapping). - -# 3. NIXL transfer metrics show no failures. -kubectl -n $NS exec $POD -- curl -s localhost:19090/metrics | - grep -E 'nixl_(bytes_transferred|num_failed)' -# Expected: nixl_num_failed_transfers_total stays 0; bytes_transferred grows. -``` - -If check 1 shows UCX, the most likely cause is `SGLANG_DISAGGREGATION_NIXL_BACKEND=LIBFABRIC` not being applied (verify the env block in the running pod). If check 2 is empty, `LD_LIBRARY_PATH` ordering is wrong — `/opt/amazon/efa/lib` must come before any other libfabric on the path. - -## Step 5: Test - -```bash -kubectl port-forward svc/glm5-sglang-efa-frontend 8000:8000 -n & -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model":"nvidia/GLM-5-NVFP4","messages":[{"role":"user","content":"Hello!"}],"max_tokens":128}' -``` - -## Step 6: Benchmark - -Edit `sglang/disagg/efa/perf.yaml` to set your namespace and PVC name, then run: - -```bash -kubectl apply -f recipes/glm-5-nvfp4/sglang/disagg/efa/perf.yaml -kubectl logs -f -l job-name=glm5-disagg-efa-bench -n -``` - -Workload shape is identical to the non-EFA `../perf.yaml`: ISL=1000, OSL=8192, concurrency=512 (32 req/GPU × 16 decode GPUs). - -## Performance (ISL=1k, OSL=8k, concurrency=512) - -Measured on 5 × p6e-gb200.36xlarge, EFA driver 3.0.0g, kernel 6.14.0-1018-aws-64k. Benchmark workload identical to the non-EFA recipe's `perf.yaml` (1,536 requests at concurrency 512 = 32 req/GPU × 16 decode GPUs). - - -| Metric | EFA (this recipe) | Non-EFA (UCX/RoCE) baseline | -| -------------------------- | ----------------- | --------------------------- | -| Output token throughput | **19,131 tok/s** | ~19,000 tok/s | -| Total token throughput | 21,468 tok/s | — | -| TTFT p50 | **621 ms** | ~850 ms | -| TTFT avg | 2,786 ms | — | -| ITL avg | **24.5 ms/token** | ~24 ms/token | -| Output tokens / user / sec | **41.0** | ~41 | -| Request count | 1,536 | — | -| Benchmark duration | 657 s | — | - - -Baseline numbers reproduced from [../README.md](../../../README.md). EFA achieves parity on all four published metrics. TTFT p50 is lower (better) on EFA, though TTFT avg is higher with a long tail (p99 ≈ 12.5 s). - -## Long-context performance (ISL=20k, OSL=2k, concurrency=64) - -The benefit of the LIBFABRIC backend grows with per-request KV cache size. Running the same recipe with `SGLANG_DISAGGREGATION_NIXL_BACKEND=LIBFABRIC` set vs. unset (NIXL defaults to UCX) on the same 5 × p6e-gb200.36xlarge hardware: - - -| Metric | LIBFABRIC (this recipe) | UCX (default) | -| ----------------------- | ----------------------- | ------------- | -| Output token throughput | **2,023 tok/s** | 1,454 tok/s | -| TTFT p50 | **20,273 ms** | 46,369 ms | -| TTFT avg | **20,452 ms** | 41,746 ms | -| ITL avg | 17.56 ms | 17.20 ms | -| Benchmark duration | **194 s** | 270 s | -| Request count | 192 | 192 | - - -With 20× larger per-request KV cache, LIBFABRIC is **39% higher throughput** and **56% lower TTFT p50** than UCX. ITL is essentially unchanged because steady-state decoding never touches the KV transfer path — only the prefill→decode KV hand-off does. - - -## References - -- [ofiwg/libfabric#12019](https://github.com/ofiwg/libfabric/pull/12019) — `efa_mr_is_cuda` patch -- [README.md](../../../README.md) — non-EFA variant of this recipe diff --git a/recipes/glm-5-nvfp4/sglang/disagg/efa/deploy.yaml b/recipes/glm-5-nvfp4/sglang/disagg/efa/deploy.yaml deleted file mode 100644 index d2f2ebc9c0b5..000000000000 --- a/recipes/glm-5-nvfp4/sglang/disagg/efa/deploy.yaml +++ /dev/null @@ -1,421 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# GLM-5 (NVFP4) disaggregated prefill/decode on GB200 via SGLang + Dynamo + EFA. -# -# Differences vs ../deploy.yaml (the non-EFA variant): -# * Disagg KV transfer rides AWS EFA RDMA instead of UCX RoCE. -# * NIXL backend is forced to LIBFABRIC (SGLANG_DISAGGREGATION_NIXL_BACKEND). -# Without this, SGLang defaults to UCX, which silently falls back to TCP -# on kernel >= 6.8 (~10 Gbps, ~10x worse TTFT). -# * Containers run privileged so fi_mr_reg can pin GPU VRAM for RDMA. -# IPC_LOCK alone fails on every EFA NIC. -# * Each pod requests vpc.amazonaws.com/efa: matching the GPU count. -# Prerequisites: -# - dynamo-platform HelmRelease applied -# - Cluster nodes: p6e-gb200.36xlarge (arm64, 4 GB200 + 4 EFA NICs per node) -# - Model weights at /models/nvidia-GLM-5-NVFP4 on shared NFS PVC -# - Container built from recipes/glm-5-nvfp4/sglang/disagg/efa/Dockerfile.efa -# - HF_TOKEN secret: kubectl create secret generic hf-token-secret \ -# --from-literal=HF_TOKEN= -n -# -# Topology: 5 nodes total -# decode: 4 nodes x 4 GPUs = TP16 / DP16 / EP16 (EAGLE speculative decode) -# prefill: 1 node x 4 GPUs = TP4 (matches non-EFA recipe) -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: glm5-sglang-efa - namespace: -spec: - backendFramework: sglang - envs: - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: HF_TOKEN - - name: HF_HOME - value: /model-store - - services: - Frontend: - componentType: frontend - replicas: 1 - extraPodSpec: - nodeSelector: - nvidia.com/gpu.product: NVIDIA-GB200 - tolerations: - - key: "nvidia.com/gpu" - operator: "Exists" - effect: "NoSchedule" - - key: "kubernetes.io/arch" - operator: "Equal" - value: "arm64" - effect: "NoSchedule" - mainContainer: - command: - - python3 - args: - - -m - - dingo.frontend - - --enforce-disagg - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: /sglang-dynamo-glm5-efa:latest - volumeMounts: - - mountPath: /model-store - name: model-cache - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - decode: - componentType: worker - subComponentType: decode - replicas: 1 - multinode: - nodeCount: 4 - resources: - limits: - gpu: "4" - custom: - # 1 EFA NIC per GPU on p6e-gb200.36xlarge. - vpc.amazonaws.com/efa: "4" - claims: - - name: compute-domain-channel - extraPodSpec: - hostIPC: true - nodeSelector: - nvidia.com/gpu.product: NVIDIA-GB200 - tolerations: - - key: "nvidia.com/gpu" - operator: "Exists" - effect: "NoSchedule" - - key: "kubernetes.io/arch" - operator: "Equal" - value: "arm64" - effect: "NoSchedule" - mainContainer: - command: - - bash - - -c - args: - - >- - ulimit -l unlimited && ulimit -n 1048576 && - exec python3 -m dingo.sglang - --model-path nvidia/GLM-5-NVFP4 - --served-model-name nvidia/GLM-5-NVFP4 - --trust-remote-code - --quantization modelopt_fp4 - --kv-cache-dtype fp8_e4m3 - --tensor-parallel-size 16 - --data-parallel-size 16 - --expert-parallel-size 16 - --enable-dp-attention - --enable-dp-lm-head - --max-running-requests 2048 - --cuda-graph-max-bs 128 - --mem-fraction-static 0.72 - --chunked-prefill-size 32768 - --max-prefill-tokens 32768 - --enable-flashinfer-allreduce-fusion - --attention-backend nsa - --nsa-decode-backend trtllm - --nsa-prefill-backend trtllm - --moe-runner-backend flashinfer_trtllm - --speculative-algorithm EAGLE - --speculative-num-steps 2 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 3 - --stream-interval 50 - --incremental-streaming-output - --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 32}' - --host 0.0.0.0 - --disaggregation-mode decode - --disaggregation-transfer-backend nixl - --disaggregation-bootstrap-port 30001 - --prefill-round-robin-balance - --watchdog-timeout 3600 - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - # === Force NIXL onto LIBFABRIC (otherwise UCX -> TCP fallback) === - - name: SGLANG_DISAGGREGATION_NIXL_BACKEND - value: LIBFABRIC - # === EFA provider configuration === - - name: FI_PROVIDER - value: efa - - name: FI_EFA_USE_DEVICE_RDMA - value: "1" - # MUST be 0. With shm transfer enabled, EFA's GPU buffers silently - # corrupt under load. - - name: FI_EFA_ENABLE_SHM_TRANSFER - value: "0" - - name: FI_EFA_ENABLE_SHM - value: "0" - - name: FI_EFA_FORK_SAFE - value: "0" - # FI_EFA_USE_HUGE_PAGE=0: hugepages aren't allocated to this pod - # (would require additional cpu/memory limits per K8s admission). - # The performance delta on these workloads is <5%. - - name: FI_EFA_USE_HUGE_PAGE - value: "0" - - name: FI_EFA_MR_MAX_CACHED_COUNT - value: "524288" - - name: FI_EFA_MR_MAX_CACHED_SIZE - value: "0" - - name: FI_HMEM - value: cuda - - name: FI_HMEM_CUDA_ENABLE_XFER - value: "1" - - name: FI_LOG_LEVEL - value: warn - # === Path order: /opt/amazon/efa first so the patched libfabric - # is the one loaded at runtime. NIXL plugin path includes - # lib/aarch64-linux-gnu/ because debian multiarch puts arm64 - # plugins there, not lib64/. - - name: PATH - value: /opt/dynamo/venv/bin:/opt/amazon/efa/bin:/usr/local/bin:/usr/bin:/bin - - name: LD_LIBRARY_PATH - value: /opt/amazon/efa/lib:/opt/amazon/efa/lib64:/opt/aws-ofi-nccl/lib:/opt/nvidia/nvda_nixl/lib64:/opt/nvidia/nvda_nixl/lib64/plugins:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/usr/local/cuda/lib64:/usr/local/lib - # === NIXL telemetry (use =y, _ENABLED=1 is a no-op) === - - name: NIXL_TELEMETRY_ENABLE - value: "y" - - name: NIXL_TELEMETRY_EXPORTER - value: prometheus - - name: NIXL_TELEMETRY_PROMETHEUS_PORT - value: "19090" - # === SGLang scheduler tuning for disagg === - - name: SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS - value: "64" - # === SGLang feature flags (unchanged from non-EFA recipe) === - - name: FLASHINFER_DISABLE_VERSION_CHECK - value: "1" - - name: SGLANG_ENABLE_JIT_DEEPGEMM - value: "1" - - name: SGLANG_ENABLE_SPEC_V2 - value: "1" - - name: SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE - value: "1" - - name: SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK - value: "1024" - # === NCCL — collectives can still ride NCCL/NVLS within the - # MNNVL clique; EFA is for inter-node KV transfer only. - - name: NCCL_IB_DISABLE - value: "0" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: NVIDIA_GDRCOPY - value: "1" - - name: NCCL_STORE_TIMEOUT - value: "7200" - - name: NCCL_GRAPH_MIXING_SUPPORT - value: "0" - - name: NIXL_LOG_LEVEL - value: ERROR - image: /sglang-dynamo-glm5-efa:latest - # REQUIRED. NIXL fi_mr_reg on VRAM needs broader caps than IPC_LOCK - # alone provides on EFA. IPC_LOCK is also added (belt-and-suspenders). - securityContext: - privileged: true - capabilities: - add: - - IPC_LOCK - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 5 - volumeMounts: - - mountPath: /model-store - name: model-cache - - mountPath: /dev/shm - name: shm - workingDir: /workspace/ - resourceClaims: - - name: compute-domain-channel - resourceClaimTemplateName: glm5-efa-compute-domain-channel - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: shm - emptyDir: - medium: Memory - sizeLimit: 80Gi - - prefill: - componentType: worker - subComponentType: prefill - replicas: 1 - resources: - limits: - gpu: "4" - custom: - vpc.amazonaws.com/efa: "4" - extraPodSpec: - hostIPC: true - nodeSelector: - nvidia.com/gpu.product: NVIDIA-GB200 - tolerations: - - key: "nvidia.com/gpu" - operator: "Exists" - effect: "NoSchedule" - - key: "kubernetes.io/arch" - operator: "Equal" - value: "arm64" - effect: "NoSchedule" - mainContainer: - command: - - bash - - -c - args: - - >- - ulimit -l unlimited && ulimit -n 1048576 && - exec python3 -m dingo.sglang - --model-path nvidia/GLM-5-NVFP4 - --served-model-name nvidia/GLM-5-NVFP4 - --trust-remote-code - --quantization modelopt_fp4 - --kv-cache-dtype fp8_e4m3 - --tensor-parallel-size 4 - --data-parallel-size 1 - --expert-parallel-size 1 - --enable-dp-lm-head - --max-running-requests 2048 - --cuda-graph-max-bs 128 - --mem-fraction-static 0.72 - --chunked-prefill-size 32768 - --max-prefill-tokens 32768 - --enable-flashinfer-allreduce-fusion - --attention-backend nsa - --nsa-decode-backend trtllm - --nsa-prefill-backend trtllm - --moe-runner-backend flashinfer_trtllm - --stream-interval 50 - --incremental-streaming-output - --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 32}' - --host 0.0.0.0 - --disaggregation-mode prefill - --disaggregation-transfer-backend nixl - --disaggregation-bootstrap-port 30001 - --load-balance-method round_robin - --watchdog-timeout 3600 - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: SGLANG_DISAGGREGATION_NIXL_BACKEND - value: LIBFABRIC - - name: FI_PROVIDER - value: efa - - name: FI_EFA_USE_DEVICE_RDMA - value: "1" - - name: FI_EFA_ENABLE_SHM_TRANSFER - value: "0" - - name: FI_EFA_ENABLE_SHM - value: "0" - - name: FI_EFA_FORK_SAFE - value: "0" - # FI_EFA_USE_HUGE_PAGE=0: hugepages aren't allocated to this pod - # (would require additional cpu/memory limits per K8s admission). - # The performance delta on these workloads is <5%. - - name: FI_EFA_USE_HUGE_PAGE - value: "0" - - name: FI_EFA_MR_MAX_CACHED_COUNT - value: "524288" - - name: FI_EFA_MR_MAX_CACHED_SIZE - value: "0" - - name: FI_HMEM - value: cuda - - name: FI_HMEM_CUDA_ENABLE_XFER - value: "1" - - name: FI_LOG_LEVEL - value: warn - - name: PATH - value: /opt/dynamo/venv/bin:/opt/amazon/efa/bin:/usr/local/bin:/usr/bin:/bin - - name: LD_LIBRARY_PATH - value: /opt/amazon/efa/lib:/opt/amazon/efa/lib64:/opt/aws-ofi-nccl/lib:/opt/nvidia/nvda_nixl/lib64:/opt/nvidia/nvda_nixl/lib64/plugins:/opt/nvidia/nvda_nixl/lib/aarch64-linux-gnu:/usr/local/ucx/lib:/usr/local/ucx/lib/ucx:/usr/local/cuda/lib64:/usr/local/lib - - name: NIXL_TELEMETRY_ENABLE - value: "y" - - name: NIXL_TELEMETRY_EXPORTER - value: prometheus - - name: NIXL_TELEMETRY_PROMETHEUS_PORT - value: "19090" - - name: SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS - value: "64" - - name: FLASHINFER_DISABLE_VERSION_CHECK - value: "1" - - name: SGLANG_ENABLE_JIT_DEEPGEMM - value: "1" - - name: SGLANG_ENABLE_SPEC_V2 - value: "1" - - name: SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE - value: "1" - - name: SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK - value: "1024" - - name: NCCL_IB_DISABLE - value: "0" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: NVIDIA_GDRCOPY - value: "1" - - name: NCCL_STORE_TIMEOUT - value: "7200" - - name: NIXL_LOG_LEVEL - value: ERROR - image: /sglang-dynamo-glm5-efa:latest - securityContext: - privileged: true - capabilities: - add: - - IPC_LOCK - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 5 - volumeMounts: - - mountPath: /model-store - name: model-cache - - mountPath: /dev/shm - name: shm - workingDir: /workspace/ - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: shm - emptyDir: - medium: Memory - sizeLimit: 80Gi ---- -# ComputeDomain enables MNNVL scheduling for the 5-node decode+prefill group. -# This is independent of EFA — collectives within an MNNVL clique still use -# NVLink72, while EFA carries the disagg KV traffic between cliques. -apiVersion: resource.nvidia.com/v1beta1 -kind: ComputeDomain -metadata: - name: glm5-efa-compute-domain - namespace: -spec: - channel: - allocationMode: Single - resourceClaimTemplate: - name: glm5-efa-compute-domain-channel - numNodes: 5 diff --git a/recipes/glm-5-nvfp4/sglang/disagg/efa/perf.yaml b/recipes/glm-5-nvfp4/sglang/disagg/efa/perf.yaml deleted file mode 100644 index 4bad655b460c..000000000000 --- a/recipes/glm-5-nvfp4/sglang/disagg/efa/perf.yaml +++ /dev/null @@ -1,133 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# aiperf benchmark job for the EFA disagg recipe. -# Workload shape (ISL=1k, OSL=8k, concurrency=512) is the same as the non-EFA -# recipe's perf.yaml so EFA vs non-EFA numbers can be compared directly. -apiVersion: batch/v1 -kind: Job -metadata: - name: glm5-disagg-efa-bench - namespace: -spec: - backoffLimit: 1 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: glm5-disagg-efa-bench - spec: - restartPolicy: Never - imagePullSecrets: - - name: nvcrimagepullsecret - containers: - - name: perf - image: python:3.12-slim - imagePullPolicy: IfNotPresent - securityContext: - privileged: false - allowPrivilegeEscalation: false - env: - - name: TARGET_MODEL - value: nvidia/GLM-5-NVFP4 - - name: ENDPOINT - value: glm5-sglang-efa-frontend:8000 - - name: ISL - value: "1000" - - name: OSL - value: "8192" - # 32 req/GPU * 16 decode GPUs (4 nodes x 4 GPUs) = 512 - - name: CONCURRENCY - value: "512" - - name: ARTIFACT_DIR - value: /model-store/perf/glm5-disagg-efa-isl1k-osl8k - - name: TOKENIZER - value: nvidia/GLM-5-NVFP4 - - name: HF_HOME - value: /model-store - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: HF_TOKEN - - name: PYTHONUNBUFFERED - value: "1" - - name: COLUMNS - value: "200" - command: - - /bin/sh - - -c - - | - set -eu - apt-get update && apt-get install -y --no-install-recommends curl jq git && apt-get clean - # torch is needed for the GLM-5 tokenizer's auto_map; install CPU-only - # to save image size and download time (~200MB vs ~2.5GB for the GPU - # build). Install separately because pytorch's wheel index does not - # mirror aiperf / transformers. - pip install torch --index-url https://download.pytorch.org/whl/cpu - # GLM-5 NVFP4 tokenizer uses the `TokenizersBackend` auto-tokenizer - # class introduced in transformers 5.x — pin to match the runtime - # image (see `pip show transformers` inside the sglang decode pod). - # NOTE: aiperf>=0.8.0 depends on crick (no prebuilt arm64 wheel); on arm64 - # nodes the apt-get step above must include gcc and libc6-dev so pip can - # compile crick from source. - pip install aiperf==0.10.0 transformers==5.3.0 tokenizers==0.22.2 - - # Pre-save tokenizer to a flat directory once, then point aiperf at - # that path. Avoids each of aiperf's N record-processor subprocesses - # racing to resolve trust_remote_code from the HF cache concurrently - # (which is what fails with "Failed to load tokenizer" errors). - TOKENIZER_LOCAL_DIR="/tmp/glm5-tokenizer" - python -c " - from transformers import AutoTokenizer - tok = AutoTokenizer.from_pretrained('$TOKENIZER', trust_remote_code=True) - tok.save_pretrained('$TOKENIZER_LOCAL_DIR') - print('Saved tokenizer to $TOKENIZER_LOCAL_DIR, vocab=', len(tok)) - " - - echo "Waiting for model '$TARGET_MODEL' at http://$ENDPOINT/v1/models..." - deadline=$(( $(date +%s) + 1800 )) - while ! curl -sf "http://$ENDPOINT/v1/models" | jq -e --arg m "$TARGET_MODEL" '.data[]? | select(.id == $m)' >/dev/null 2>&1; do - if [ "$(date +%s)" -ge "$deadline" ]; then - echo "Timed out after 30m waiting for model '$TARGET_MODEL' at http://$ENDPOINT/v1/models" - exit 1 - fi - echo "[$(date '+%H:%M:%S')] not ready, retrying in 10s..." - sleep 10 - done - echo "Model ready." - - mkdir -p "$ARTIFACT_DIR" - aiperf profile \ - --artifact-dir "$ARTIFACT_DIR" \ - --model "$TARGET_MODEL" \ - --tokenizer "$TOKENIZER_LOCAL_DIR" \ - --tokenizer-trust-remote-code \ - --endpoint-type chat \ - --endpoint /v1/chat/completions \ - --streaming \ - --url "http://$ENDPOINT" \ - --synthetic-input-tokens-mean $ISL \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean $OSL \ - --output-tokens-stddev 0 \ - --extra-inputs "max_tokens:$OSL" \ - --extra-inputs "min_tokens:$OSL" \ - --extra-inputs "ignore_eos:true" \ - --concurrency $CONCURRENCY \ - --request-count $((CONCURRENCY * 3)) \ - --warmup-request-count 16 \ - --num-dataset-entries 12800 \ - --random-seed 100 \ - --ui simple - echo "Done. Results in $ARTIFACT_DIR" - ls -la "$ARTIFACT_DIR" - volumeMounts: - - name: model-cache - mountPath: /model-store - workingDir: /workspace - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/glm-5-nvfp4/sglang/disagg/perf.yaml b/recipes/glm-5-nvfp4/sglang/disagg/perf.yaml deleted file mode 100644 index c77a03cf5fd6..000000000000 --- a/recipes/glm-5-nvfp4/sglang/disagg/perf.yaml +++ /dev/null @@ -1,128 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: glm5-disagg-bench - namespace: -spec: - backoffLimit: 1 - completions: 1 - parallelism: 1 - template: - spec: - restartPolicy: Never - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - seccompProfile: - type: RuntimeDefault - containers: - - name: bench - image: python:3.12-slim - securityContext: - allowPrivilegeEscalation: false - imagePullPolicy: IfNotPresent - env: - - name: TARGET_MODEL - value: nvidia/GLM-5-NVFP4 - - name: ENDPOINT - value: glm5-sglang-frontend:8000 - - name: ISL - value: "1000" - - name: OSL - value: "8192" - # Concurrency = CONCURRENCY_PER_GPU * num_decode_gpus - # Default: 32 req/GPU * 16 GPUs (4 nodes x 4 GPUs) = 512 - - name: CONCURRENCY - value: "512" - - name: ARTIFACT_DIR - value: /model-store/perf/glm5-disagg-isl1k-osl8k - - name: TOKENIZER - value: nvidia/GLM-5-NVFP4 - - name: HF_HOME - value: /model-store - - name: PYTHONUNBUFFERED - value: "1" - - name: COLUMNS - value: "200" - # HOME defaults to "/" for uid 1000 with no /etc/passwd entry; pip --user - # then tries to write /.local (read-only). Point HOME at /tmp (1777 sticky) - # so ~/.local resolves under a writable path. - - name: HOME - value: /tmp - - name: PATH - value: /tmp/.local/bin:/usr/local/bin:/usr/bin:/bin - envFrom: - - secretRef: - name: hf-token-secret - command: - - /bin/sh - - -c - - | - set -eu - # No apt-get: cluster benchmark pods run as non-root, and Python - # stdlib is enough for the readiness probe. - # GLM-5's tokenizer_config.json uses tokenizer_class=TokenizersBackend, - # which is supported by Transformers v5. - # NOTE: aiperf>=0.8.0 depends on crick, which has no prebuilt arm64 wheel - # and requires gcc/libc6-dev to build from source. This pod runs non-root - # with no apt-get access, so this install will fail on arm64 nodes. - # Ensure the cluster scheduler places this pod on an x86_64 node. - pip install --user -q aiperf==0.10.0 transformers==5.8.0 tokenizers==0.22.2 - - echo "Waiting for model at http://$ENDPOINT/v1/models..." - python3 - <<'PY' - import json, os, time, urllib.error, urllib.request - - endpoint = os.environ["ENDPOINT"] - target = os.environ["TARGET_MODEL"] - while True: - try: - with urllib.request.urlopen(f"http://{endpoint}/v1/models", timeout=5) as r: - data = json.load(r) - if any(model.get("id") == target for model in data.get("data", [])): - break - except (urllib.error.URLError, ValueError, KeyError): - pass - print(f"[{time.strftime('%H:%M:%S')}] not ready, retrying in 10s...", flush=True) - time.sleep(10) - print("Model ready!", flush=True) - PY - - mkdir -p "$ARTIFACT_DIR" - - aiperf profile \ - --artifact-dir "$ARTIFACT_DIR" \ - --model "$TARGET_MODEL" \ - --tokenizer "$TOKENIZER" \ - --tokenizer-trust-remote-code \ - --endpoint-type chat \ - --endpoint /v1/chat/completions \ - --streaming \ - --url "http://$ENDPOINT" \ - --synthetic-input-tokens-mean $ISL \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean $OSL \ - --output-tokens-stddev 0 \ - --extra-inputs "max_tokens:$OSL" \ - --extra-inputs "min_tokens:$OSL" \ - --extra-inputs "ignore_eos:true" \ - --concurrency $CONCURRENCY \ - --request-count $((CONCURRENCY * 3)) \ - --warmup-request-count 16 \ - --num-dataset-entries 12800 \ - --random-seed 100 \ - --ui simple - - echo "Done. Results in $ARTIFACT_DIR" - ls -la "$ARTIFACT_DIR" - volumeMounts: - - name: model-cache - mountPath: /model-store - workingDir: /workspace - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/gpt-oss-120b/README.md b/recipes/gpt-oss-120b/README.md deleted file mode 100644 index e0ec0837fb6a..000000000000 --- a/recipes/gpt-oss-120b/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# GPT-OSS-120B Recipes - -Production-ready deployment for **GPT-OSS-120B** using TensorRT-LLM on Blackwell (GB200) hardware. - -## Available Configurations - -| Configuration | GPUs | Mode | Description | -|--------------|------|------|-------------| -| [**trtllm/agg**](trtllm/agg/) | 4x GB200 | Aggregated | WideEP, ARM64 | -| [**trtllm/disagg**](trtllm/disagg/) | 5x Blackwell (GB200/B200) | Disaggregated | Prefill/Decode split | - -## Prerequisites - -1. **Dynamo Platform installed** — See [Kubernetes Deployment Guide](../../docs/kubernetes/README.md) -2. **GPU cluster** with GB200 (Blackwell) GPUs -3. **HuggingFace token** with access to the model - -## Quick Start - -```bash -# Set namespace -export NAMESPACE=dynamo-demo -kubectl create namespace ${NAMESPACE} - -# Create HuggingFace token secret -kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="your-token-here" \ - -n ${NAMESPACE} - -# Download model (update storageClassName in model-cache/model-cache.yaml first!) -kubectl apply -f model-cache/ -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/model-download -n ${NAMESPACE} --timeout=3600s - -# Deploy -kubectl apply -f trtllm/agg/deploy.yaml -n ${NAMESPACE} -``` - -## Test the Deployment - -```bash -# Port-forward the frontend -kubectl port-forward svc/gpt-oss-agg-frontend 8000:8000 -n ${NAMESPACE} - -# Send a test request -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "openai/gpt-oss-120b", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 50 - }' -``` - -## Notes - -- Update `storageClassName` in `model-cache/model-cache.yaml` before deploying -- This recipe requires ARM64 (GB200) nodes — it will not run on x86 Hopper/Ampere hardware -- Update the container image tag in `deploy.yaml` to match your Dynamo release version diff --git a/recipes/gpt-oss-120b/model-cache/model-cache.yaml b/recipes/gpt-oss-120b/model-cache/model-cache.yaml deleted file mode 100644 index 9dfc23d5b454..000000000000 --- a/recipes/gpt-oss-120b/model-cache/model-cache.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 100Gi - storageClassName: "your-storage-class-name" \ No newline at end of file diff --git a/recipes/gpt-oss-120b/model-cache/model-download.yaml b/recipes/gpt-oss-120b/model-cache/model-download.yaml deleted file mode 100644 index 7d8fcf8ff716..000000000000 --- a/recipes/gpt-oss-120b/model-cache/model-download.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: MODEL_NAME - value: openai/gpt-oss-120b - - name: HF_HOME - value: /model-store - # Uses up to 64 GB RAM for XET buffers; set "0" on low-memory nodes (8 GB cap): https://huggingface.co/docs/hub/en/xet/using-xet-storage#download-buffers - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - - name: MODEL_REVISION - value: b5c939de8f754692c1647ca79fbf85e8c1e70f8a - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.11.0 - hf download $MODEL_NAME --revision $MODEL_REVISION --exclude "original/*" --exclude "metal/*" - resources: - requests: - cpu: "2" - memory: "64Gi" - limits: - cpu: "8" - memory: "64Gi" - volumeMounts: - - name: model-cache - mountPath: /model-store - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/gpt-oss-120b/trtllm/agg/deploy.yaml b/recipes/gpt-oss-120b/trtllm/agg/deploy.yaml deleted file mode 100644 index dba39e0adcf5..000000000000 --- a/recipes/gpt-oss-120b/trtllm/agg/deploy.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: ConfigMap -metadata: - name: llm-config -data: - config.yaml: | - enable_attention_dp: true - cuda_graph_config: - max_batch_size: 800 - enable_padding: true - kv_cache_config: - enable_block_reuse: false - stream_interval: 20 - moe_config: - backend: CUTLASS ---- -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: gpt-oss-agg -spec: - backendFramework: trtllm - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - extraPodSpec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: nvidia.com/dynamo-graph-deployment-name - operator: In - values: - - gpt-oss-agg-frontend - topologyKey: kubernetes.io/hostname - mainContainer: - args: - - python3 -m dingo.frontend --router-mode round-robin --http-port 8000 - command: - - /bin/sh - - -c - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - replicas: 1 - TrtllmWorker: - componentType: worker - envFromSecret: hf-token-secret - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 80Gi - extraPodSpec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.present - operator: In - values: - - "true" - mainContainer: - args: - - | - python3 -m dynamo.trtllm \ - --model-path "${MODEL_PATH}" \ - --served-model-name "openai/gpt-oss-120b" \ - --extra-engine-args "${ENGINE_ARGS}" \ - --tensor-parallel-size 4 \ - --expert-parallel-size 4 \ - --max-batch-size 800 \ - --free-gpu-memory-fraction 0.9 - command: - - /bin/sh - - -c - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - env: - - name: TRTLLM_ENABLE_PDL - value: "1" - - name: TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL - value: "True" - - name: SERVED_MODEL_NAME - value: "openai/gpt-oss-120b" - - name: ENGINE_ARGS - value: "/opt/dynamo/configs/config.yaml" - - name: MODEL_PATH - value: "openai/gpt-oss-120b" - - name: HF_HOME - value: /opt/models - volumeMounts: - - mountPath: /opt/dynamo/configs - name: llm-config - readOnly: true - workingDir: /workspace/examples/backends/trtllm - volumes: - - configMap: - name: llm-config - name: llm-config - replicas: 1 - resources: - limits: - gpu: "4" - requests: - gpu: "4" \ No newline at end of file diff --git a/recipes/gpt-oss-120b/trtllm/agg/perf.yaml b/recipes/gpt-oss-120b/trtllm/agg/perf.yaml deleted file mode 100644 index c83f883acb2e..000000000000 --- a/recipes/gpt-oss-120b/trtllm/agg/perf.yaml +++ /dev/null @@ -1,152 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: gpt-oss-120b-bench -spec: - backoffLimit: 1 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: gpt-oss-120b-bench - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: nvidia.com/dynamo-graph-deployment-name - operator: In - values: - - gpt-oss-agg - topologyKey: kubernetes.io/hostname - containers: - - command: - - /bin/sh - - -c - - | - apt-get update && apt-get install -y curl jq procps git && apt-get clean - pip install "aiperf==0.10.0"; - echo "aiperf installation completed"; - sysctl -w net.ipv4.ip_local_port_range="1024 65000" - cat /proc/sys/net/ipv4/ip_local_port_range - export COLUMNS=200 - EPOCH=$(date +%s) - ## utility functions -- can be moved to a bash script / configmap - wait_for_model_ready() { - echo "Waiting for model '$TARGET_MODEL' at $ENDPOINT/v1/models (checking every 5s)..." - while ! curl -s "http://$ENDPOINT/v1/models" | jq -e --arg model "$TARGET_MODEL" '.data[]? | select(.id == $model)' >/dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready yet, sleeping 5s before checking again http://$ENDPOINT/v1/models" - sleep 5 - done - echo "✅ Model '$TARGET_MODEL' is now available!" - echo "Model '$TARGET_MODEL' is now available!" - curl -s "http://$ENDPOINT/v1/models" | jq . - } - run_perf() { - local concurrency=$1 - local isl=$2 - local osl=$3 - key=concurrency_${concurrency} - export ARTIFACT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/${key}" - mkdir -p "$ARTIFACT_DIR" - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - aiperf profile --artifact-dir $ARTIFACT_DIR \ - --model $TARGET_MODEL \ - --tokenizer /model-cache/hub/models--openai--gpt-oss-120b/snapshots/b5c939de8f754692c1647ca79fbf85e8c1e70f8a \ - --endpoint-type chat \ - --endpoint /v1/chat/completions \ - --streaming \ - --url http://$ENDPOINT \ - --synthetic-input-tokens-mean $isl \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean $osl \ - --output-tokens-stddev 0 \ - --extra-inputs "max_tokens:$osl" \ - --extra-inputs "min_tokens:$osl" \ - --extra-inputs "ignore_eos:true" \ - --extra-inputs "repetition_penalty:1.0" \ - --extra-inputs "temperature: 0.0" \ - --concurrency $concurrency \ - --request-count $((10*concurrency)) \ - --warmup-request-count $concurrency \ - --num-dataset-entries 12800 \ - --random-seed 100 \ - --workers-max 252 \ - -H 'Authorization: Bearer NOT USED' \ - -H 'Accept: text/event-stream'\ - --record-processors 32 \ - --ui simple - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - ls -la $ARTIFACT_DIR - } - #### Actual execution #### - wait_for_model_ready - mkdir -p "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - # Calculate total concurrency based on per-GPU concurrency and GPU count - TOTAL_CONCURRENCY=$((CONCURRENCY_PER_GPU * DEPLOYMENT_GPU_COUNT)) - echo "Calculated total concurrency: $TOTAL_CONCURRENCY (${CONCURRENCY_PER_GPU} per GPU × ${DEPLOYMENT_GPU_COUNT} GPUs)" - # Write input_config.json - cat > "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/input_config.json" < - -# GPT-OSS-120B Disaggregated Prefill/Decode - -Serves [openai/gpt-oss-120b](https://huggingface.co/openai/gpt-oss-120b) using TensorRT-LLM with -disaggregated prefill/decode via Dynamo on GB200 nodes. - -## Topology - -| Role | Nodes | GPUs/node | Total GPUs | Parallelism | -|---------|-------|-----------|------------|-------------| -| Prefill | 1 | 1 | 1 | TP1 | -| Decode | 1 | 4 | 4 | TP4 | - -## Prerequisites - -1. **Dynamo Platform installed** — See [Kubernetes Deployment Guide](../../../../docs/kubernetes/README.md) -2. **Blackwell GPU nodes** (GB200 or B200) -3. **HuggingFace token** with access to the model - -## Deploy - -Follow the [top-level Quick Start](../../README.md) to set up the namespace, HuggingFace -token secret, and model download. Then: - -```bash -kubectl apply -f trtllm/disagg/deploy.yaml -n ${NAMESPACE} -``` - -Monitor startup (model loading takes ~15–30 minutes depending on storage speed): - -```bash -kubectl get pods -n ${NAMESPACE} -l app.kubernetes.io/part-of=gpt-oss-disagg -w -``` - -## Test - -```bash -kubectl port-forward svc/gpt-oss-disagg-frontend 8000:8000 -n ${NAMESPACE} & -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model":"openai/gpt-oss-120b","messages":[{"role":"user","content":"Hello!"}],"max_tokens":50}' -``` - -## Benchmark (optional) - -Edit `perf.yaml` to set your namespace and PVC, then run: - -```bash -kubectl apply -f trtllm/disagg/perf.yaml -n ${NAMESPACE} -kubectl logs -f -l job-name=gpt-oss-120b-disagg-bench -n ${NAMESPACE} -``` - -## Key Configuration Notes - -### Engine Configs - -The `deploy.yaml` includes a ConfigMap with separate engine configurations for -prefill and decode workers. Key differences: - -- **Prefill**: TP1, `max_batch_size=64`, `free_gpu_memory_fraction=0.8`, overlap scheduler disabled -- **Decode**: TP4, `max_batch_size=1280`, `free_gpu_memory_fraction=0.85`, overlap scheduler enabled - -### KV Transfer - -Uses UCX-based cache transceiver (`max_tokens_in_buffer=9216`) for KV cache -transfer between prefill and decode workers. - -### Quantization - -Uses `W4A8_MXFP4_MXFP8` quantization via the `OVERRIDE_QUANT_ALGO` environment variable. diff --git a/recipes/gpt-oss-120b/trtllm/disagg/deploy.yaml b/recipes/gpt-oss-120b/trtllm/disagg/deploy.yaml deleted file mode 100644 index d8e825e81c9d..000000000000 --- a/recipes/gpt-oss-120b/trtllm/disagg/deploy.yaml +++ /dev/null @@ -1,224 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: ConfigMap -metadata: - name: llm-config -data: - prefill.yaml: | - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 9216 - cuda_graph_config: - enable_padding: true - max_batch_size: 30 - disable_overlap_scheduler: true - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - max_batch_size: 64 - max_num_tokens: 20000 - max_seq_len: 9000 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 1 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 20 - tensor_parallel_size: 1 - trust_remote_code: true - decode.yaml: | - allreduce_strategy: AUTO - attention_dp_config: - enable_balance: true - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 9216 - cuda_graph_config: - enable_padding: true - max_batch_size: 1280 - disable_overlap_scheduler: false - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - max_batch_size: 1280 - max_num_tokens: 20000 - max_seq_len: 11000 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 1 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 20 - tensor_parallel_size: 4 - trust_remote_code: true - ---- -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: gpt-oss-disagg -spec: - backendFramework: trtllm - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - dynamoNamespace: gpt-oss-disagg - extraPodSpec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: nvidia.com/dynamo-graph-deployment-name - operator: In - values: - - gpt-oss-disagg-frontend - topologyKey: kubernetes.io/hostname - mainContainer: - args: - - python3 -m dingo.frontend --router-mode round-robin --http-port 8000 - command: - - /bin/sh - - -c - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - replicas: 1 - TrtllmPrefillWorker: - componentType: main - dynamoNamespace: gpt-oss-disagg - envFromSecret: hf-token-secret - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 80Gi - extraPodSpec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.present - operator: In - values: - - "true" - mainContainer: - args: - - | - python3 -m dynamo.trtllm \ - --model-path "${MODEL_PATH}" \ - --served-model-name "openai/gpt-oss-120b" \ - --extra-engine-args "${ENGINE_ARGS}" \ - --disaggregation-mode prefill - command: - - /bin/sh - - -c - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - env: - - name: TRTLLM_ENABLE_PDL - value: "1" - - name: TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL - value: "True" - - name: OVERRIDE_QUANT_ALGO - value: "W4A8_MXFP4_MXFP8" - - name: NCCL_GRAPH_REGISTER - value: "0" - - name: OMPI_MCA_coll_ucc_enable - value: "0" - - name: SERVED_MODEL_NAME - value: "openai/gpt-oss-120b" - - name: ENGINE_ARGS - value: "/opt/dynamo/configs/prefill.yaml" - - name: MODEL_PATH - value: "openai/gpt-oss-120b" - - name: HF_HOME - value: /opt/models - volumeMounts: - - mountPath: /opt/dynamo/configs - name: llm-config - readOnly: true - workingDir: /workspace/examples/backends/trtllm - volumes: - - configMap: - name: llm-config - name: llm-config - replicas: 1 - resources: - limits: - gpu: "1" - requests: - gpu: "1" - TrtllmDecodeWorker: - componentType: main - dynamoNamespace: gpt-oss-disagg - envFromSecret: hf-token-secret - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 80Gi - extraPodSpec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.present - operator: In - values: - - "true" - mainContainer: - args: - - | - python3 -m dynamo.trtllm \ - --model-path "${MODEL_PATH}" \ - --served-model-name "openai/gpt-oss-120b" \ - --extra-engine-args "${ENGINE_ARGS}" \ - --disaggregation-mode decode - command: - - /bin/sh - - -c - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - env: - - name: TRTLLM_ENABLE_PDL - value: "1" - - name: TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL - value: "True" - - name: OVERRIDE_QUANT_ALGO - value: "W4A8_MXFP4_MXFP8" - - name: NCCL_GRAPH_REGISTER - value: "0" - - name: OMPI_MCA_coll_ucc_enable - value: "0" - - name: SERVED_MODEL_NAME - value: "openai/gpt-oss-120b" - - name: ENGINE_ARGS - value: "/opt/dynamo/configs/decode.yaml" - - name: MODEL_PATH - value: "openai/gpt-oss-120b" - - name: HF_HOME - value: /opt/models - volumeMounts: - - mountPath: /opt/dynamo/configs - name: llm-config - readOnly: true - workingDir: /workspace/examples/backends/trtllm - volumes: - - configMap: - name: llm-config - name: llm-config - replicas: 1 - resources: - limits: - gpu: "4" - requests: - gpu: "4" \ No newline at end of file diff --git a/recipes/gpt-oss-120b/trtllm/disagg/perf.yaml b/recipes/gpt-oss-120b/trtllm/disagg/perf.yaml deleted file mode 100644 index e07b69bc7ec5..000000000000 --- a/recipes/gpt-oss-120b/trtllm/disagg/perf.yaml +++ /dev/null @@ -1,152 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: gpt-oss-120b-disagg-bench -spec: - backoffLimit: 1 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: gpt-oss-120b-disagg-bench - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: nvidia.com/dynamo-graph-deployment-name - operator: In - values: - - gpt-oss-disagg - topologyKey: kubernetes.io/hostname - containers: - - command: - - /bin/sh - - -c - - | - apt-get update && apt-get install -y curl jq procps git && apt-get clean - pip install "aiperf==0.10.0"; - echo "aiperf installation completed"; - sysctl -w net.ipv4.ip_local_port_range="1024 65000" - cat /proc/sys/net/ipv4/ip_local_port_range - export COLUMNS=200 - EPOCH=$(date +%s) - ## utility functions -- can be moved to a bash script / configmap - wait_for_model_ready() { - echo "Waiting for model '$TARGET_MODEL' at $ENDPOINT/v1/models (checking every 5s)..." - while ! curl -s "http://$ENDPOINT/v1/models" | jq -e --arg model "$TARGET_MODEL" '.data[]? | select(.id == $model)' >/dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready yet, sleeping 5s before checking again http://$ENDPOINT/v1/models" - sleep 5 - done - echo "✅ Model '$TARGET_MODEL' is now available!" - echo "Model '$TARGET_MODEL' is now available!" - curl -s "http://$ENDPOINT/v1/models" | jq . - } - run_perf() { - local concurrency=$1 - local isl=$2 - local osl=$3 - key=concurrency_${concurrency} - export ARTIFACT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/${key}" - mkdir -p "$ARTIFACT_DIR" - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - aiperf profile --artifact-dir $ARTIFACT_DIR \ - --model $TARGET_MODEL \ - --tokenizer /model-cache/hub/models--openai--gpt-oss-120b/snapshots/b5c939de8f754692c1647ca79fbf85e8c1e70f8a \ - --endpoint-type chat \ - --endpoint /v1/chat/completions \ - --streaming \ - --url http://$ENDPOINT \ - --synthetic-input-tokens-mean $isl \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean $osl \ - --output-tokens-stddev 0 \ - --extra-inputs "max_tokens:$osl" \ - --extra-inputs "min_tokens:$osl" \ - --extra-inputs "ignore_eos:true" \ - --extra-inputs "repetition_penalty:1.0" \ - --extra-inputs "temperature: 0.0" \ - --concurrency $concurrency \ - --request-count $((10*concurrency)) \ - --warmup-request-count $concurrency \ - --num-dataset-entries 12800 \ - --random-seed 100 \ - --workers-max 252 \ - -H 'Authorization: Bearer NOT USED' \ - -H 'Accept: text/event-stream'\ - --record-processors 32 \ - --ui simple - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - ls -la $ARTIFACT_DIR - } - #### Actual execution #### - wait_for_model_ready - mkdir -p "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - # Calculate total concurrency based on per-GPU concurrency and GPU count - TOTAL_CONCURRENCY=$((CONCURRENCY_PER_GPU * DEPLOYMENT_GPU_COUNT)) - echo "Calculated total concurrency: $TOTAL_CONCURRENCY (${CONCURRENCY_PER_GPU} per GPU × ${DEPLOYMENT_GPU_COUNT} GPUs)" - # Write input_config.json - cat > "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/input_config.json" <" \ No newline at end of file diff --git a/recipes/kimi-k2.5/README.md b/recipes/kimi-k2.5/README.md deleted file mode 100644 index 5b3e7b89b116..000000000000 --- a/recipes/kimi-k2.5/README.md +++ /dev/null @@ -1,129 +0,0 @@ -# Kimi-K2.5 NVFP4 Recipes: Agentic Coding Workload - -Deployment recipes for **Kimi-K2.5-NVFP4** on an agentic coding workload. - -## Configurations - -We compare four Dynamo + TRTLLM deployment modes on **6x GB200 (24 GPU)**: - -| Mode | Routing | Speculative | KV Cache Offloading? | Concurrency | GPUs | -|------|---------|-------------|----------------------|-------------|------| -| **Disaggregated** | KV-aware | Eagle3 | ✅ (TRTLLM-native offload) | 32 | 3x DEP4 prefill + 3x TEP4 decode | -| **Aggregated** | KV-aware | Eagle3 | ❌ | 24 | 3x TEP=8 | -| **Aggregated** | Round-robin | Eagle3 | ❌ | 24 | 3x TEP=8 | -| **Aggregated** | Round-robin | ❌ | ❌ | 8 | 3x TEP=8 | - - -## Results - -throughput-interactivity-pareto - - -The disaggregated configuration with KV-aware routing, Eagle decoding, and KV offloading achieves the best system throughput and interactivity. - -## Dataset: Agentic Coding Workflow - -We benchmark on a Mooncake-style trace that emulates a long-context, KV-reuse-heavy agentic coding workload (~200k-token context window, multi-turn sessions with restart-splits and a layered prefix-cache model) with multiple concurrent sessions. - -The exact trace used for this recipe can be generated by following the instructions from the [dataset home in the AIPerf repository](https://github.com/ai-dynamo/aiperf/blob/1ecc2eac988eedc0e3a79b4c2d1063bfc295a014/src/aiperf/dataset/agentic_code_gen/datasets/1k_sessions_200k_ctx/manifest.json). - -## Prerequisites - -1. **Dynamo Platform installed** — see [Kubernetes Deployment Guide](../../docs/kubernetes/README.md). -2. **6x GB200 nodes (24 GPUs total)** with MNNVL across all nodes, plus ≥1 additional CPU/spare node if possible. -3. **HuggingFace token** with access to `nvidia/Kimi-K2.5-NVFP4` and `nvidia/Kimi-K2.5-Thinking-Eagle3`: - ```bash - export NAMESPACE=your-namespace - kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="your-token" \ - -n ${NAMESPACE} - ``` -4. A Dynamo TRT-LLM runtime image (i.e. v1.1.1~) that supports Kimi-K2.5 + Eagle3. The manifests ship with `nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:my-tag` as a placeholder — update the `image:` fields in each `deploy.yaml` before applying. - -## Quick Start - -### 1. Create Storage - -> **Note:** Edit `model-cache/model-cache.yaml` first and update `storageClassName` to match your cluster (`kubectl get storageclass`). - -```bash -kubectl apply -f model-cache/model-cache.yaml -n ${NAMESPACE} -``` - -### 2. (Multi-node) Apply ComputeDomain - -The multinode aggregated variants require a ComputeDomain CR so the DRA scheduler can co-locate worker pods across MNNVL-connected nodes: - -```bash -kubectl apply -f model-cache/compute-domain.yaml -n ${NAMESPACE} -``` - -If you rename the CD, mirror the change in every `deploy.yaml` under `extraPodSpec.resourceClaims` and `resources.claims`. - -### 3. Download model + Eagle3 head - -```bash -kubectl apply -f model-cache/model-download.yaml -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/model-download -n ${NAMESPACE} --timeout=3600s -``` - -### 4. Copy the trace file into the PVC - -```bash -# Use any pod that mounts the model-cache PVC at /model-cache as a helper. -# `kubectl cp` does not create missing parent directories, so mkdir first. -kubectl exec -n ${NAMESPACE} -- mkdir -p /model-cache/traces/agent_trace_data -kubectl cp ${NAMESPACE}/:/model-cache/traces/agent_trace_data/dataset.jsonl -``` - -### 5. Deploy & benchmark each variant - -```bash -# Pick one variant per run (resources are sized for the full 24 GPUs). -kubectl apply -f trtllm/disagg-eagle-kv-router/deploy.yaml -n ${NAMESPACE} -# kubectl apply -f trtllm/agg-eagle-kv-router/deploy.yaml -n ${NAMESPACE} -# kubectl apply -f trtllm/agg-eagle-round-robin/deploy.yaml -n ${NAMESPACE} -# kubectl apply -f trtllm/agg-round-robin/deploy.yaml -n ${NAMESPACE} - -# Wait for ready (replace DGD name for each variant) -kubectl wait --for=condition=ready pod \ - -l nvidia.com/dynamo-graph-deployment-name=kimi-k25-disagg-kv-eagle \ - -n ${NAMESPACE} --timeout=1800s - -# Run benchmark (one perf.yaml per deploy.yaml), 1 hour duration. -kubectl apply -f trtllm/disagg-eagle-kv-router/perf.yaml -n ${NAMESPACE} -``` - -## Expected metrics - -| Variant | tok/s/user (avg) | tok/s/GPU | -|---|---|---| -| Disagg + Eagle + KV routing + offload | ~130 | ~5,400 | -| Agg + Eagle + KV routing | ~85 | ~4,400 | -| Agg + Eagle + Round-robin | ~95 | ~4,000 | -| Agg + Round-robin (no Eagle) | ~105 | ~1,700 | - -(Consistent with the pareto plot at the top) - - -## Cleanup - -```bash -# Delete benchmark jobs -kubectl delete job -l app=kimi-k25-disagg-kv-eagle-bench -n ${NAMESPACE} -# (repeat for any other variants) - -# Delete deployments -kubectl delete dynamographdeployment \ - kimi-k25-disagg-kv-eagle \ - kimi-k25-agg-kv-eagle \ - kimi-k25-agg-rr-eagle \ - kimi-k25-agg-rr \ - -n ${NAMESPACE} 2>/dev/null || true -``` - -## References - -- [Mooncake: A KVCache-centric Disaggregated Architecture for LLM Serving](https://github.com/kvcache-ai/Mooncake) — FAST25 paper and trace. -- [Kimi-K2.5 model card](https://huggingface.co/nvidia/Kimi-K2.5-NVFP4) -- [Eagle3 speculative-decoding head model card](https://huggingface.co/nvidia/Kimi-K2.5-Thinking-Eagle3) diff --git a/recipes/kimi-k2.5/model-cache/compute-domain.yaml b/recipes/kimi-k2.5/model-cache/compute-domain.yaml deleted file mode 100644 index b0c20739a831..000000000000 --- a/recipes/kimi-k2.5/model-cache/compute-domain.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: resource.nvidia.com/v1beta1 -kind: ComputeDomain -metadata: - name: your-compute-domain -spec: - # 0 = on-demand allocation (nodes assigned when pods request them via resourceClaims). - numNodes: 0 - channel: - resourceClaimTemplate: - name: your-compute-domain-channel diff --git a/recipes/kimi-k2.5/model-cache/model-cache.yaml b/recipes/kimi-k2.5/model-cache/model-cache.yaml deleted file mode 100644 index d8581dc5ff8b..000000000000 --- a/recipes/kimi-k2.5/model-cache/model-cache.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 800Gi - storageClassName: "your-storage-class-name" diff --git a/recipes/kimi-k2.5/model-cache/model-download.yaml b/recipes/kimi-k2.5/model-cache/model-download.yaml deleted file mode 100644 index 8b1b66834fdf..000000000000 --- a/recipes/kimi-k2.5/model-cache/model-download.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Downloads Kimi-K2.5-NVFP4 and the Eagle3 speculative-decoding head into the -# shared model-cache PVC. Both checkpoints are pulled from HuggingFace. -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - seccompProfile: - type: RuntimeDefault - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: MODEL_NAME - value: nvidia/Kimi-K2.5-NVFP4 - - name: EAGLE_MODEL_NAME - value: nvidia/Kimi-K2.5-Thinking-Eagle3 - - name: HF_HOME - value: /model-cache - # Uses up to 64 GB RAM for XET buffers; set "0" on low-memory nodes (8 GB cap): https://huggingface.co/docs/hub/en/xet/using-xet-storage#download-buffers - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.11.0 - hf download $MODEL_NAME - hf download $EAGLE_MODEL_NAME - resources: - requests: - cpu: "2" - memory: "64Gi" - limits: - cpu: "8" - memory: "64Gi" - volumeMounts: - - name: model-cache - mountPath: /model-cache - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/kimi-k2.5/trtllm/agg-eagle-kv-router/deploy.yaml b/recipes/kimi-k2.5/trtllm/agg-eagle-kv-router/deploy.yaml deleted file mode 100644 index dae9df793412..000000000000 --- a/recipes/kimi-k2.5/trtllm/agg-eagle-kv-router/deploy.yaml +++ /dev/null @@ -1,187 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Aggregated Kimi-K2.5-NVFP4 with Eagle3 speculative decoding and KV-aware routing. -# -# Topology (6x GB200 nodes, 24 GPUs total): -# - 3x aggregated workers (TP=8 / EP=8), each spanning 2 nodes (4 GPUs/pod). -# -# Routing: frontend uses KV-aware routing with prefix-hash awareness. -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: kimi-k25-agg-kv-eagle -spec: - envs: - - name: NCCL_MNNVL_ENABLE - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: TLLM_LOG_LEVEL - value: "info" - services: - Frontend: - componentType: frontend - extraPodSpec: - containers: null - mainContainer: - command: - - python3 - args: - - -m - - dingo.frontend - - --router-mode - - kv - - --router-reset-states - - --request-plane - - nats - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime: - name: "" - resources: {} - nodeSelector: - kubernetes.io/arch: arm64 - # tolerations: # uncomment to populate any tolerations for the gpu nodes - replicas: 1 - - agg: - componentType: worker - envFromSecret: hf-token-secret - extraPodSpec: - containers: null - mainContainer: - args: - - --model-path - - nvidia/Kimi-K2.5-NVFP4 - - --served-model-name - - nvidia/Kimi-K2.5-NVFP4 - - --extra-engine-args - - /config/agg.yaml - - --publish-events-and-metrics - - --request-plane - - nats - - --kv-block-size - - "32" - - --dyn-tool-call-parser - - kimi_k2 - - --dyn-reasoning-parser - - kimi_k25 - command: - - python3 - - -m - - dynamo.trtllm - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: HF_HUB_OFFLINE - value: "1" - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: NCCL_SOCKET_IFNAME - value: eth0 - - name: GLOO_SOCKET_IFNAME - value: eth0 - - name: NCCL_GRAPH_MIXING_SUPPORT - value: "0" - - name: TRTLLM_ENABLE_PDL - value: "1" - # Uncomment the below to reproduce benchmark results in README exactly. - # - name: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS - # value: "2" - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime: - name: "" - resources: {} - securityContext: - runAsUser: 0 - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 20 - volumeMounts: - - mountPath: /model-cache - name: model-cache - - mountPath: /config - name: trtllm-config - readOnly: true - workingDir: /workspace/ - nodeSelector: - kubernetes.io/arch: arm64 - # tolerations: # uncomment to populate any tolerations for the gpu nodes - resourceClaims: - - name: compute-domain-channel - resourceClaimTemplateName: your-compute-domain-channel - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: trtllm-config - configMap: - name: kimi-k25-agg-kv-eagle-config - multinode: - nodeCount: 2 - replicas: 3 - resources: - limits: - gpu: "4" - claims: - - name: compute-domain-channel ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: kimi-k25-agg-kv-eagle-config -data: - agg.yaml: | - allreduce_strategy: MNNVL - max_num_tokens: 8192 - enable_chunked_prefill: true - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - enable_padding: true - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - free_gpu_memory_fraction: 0.75 - tokens_per_block: 32 - max_batch_size: 128 - moe_config: - backend: CUTLASS - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cuda_core - num_postprocess_workers: 8 - print_iter_log: true - stream_interval: 10 - tensor_parallel_size: 8 - trust_remote_code: true - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - allow_advanced_sampling: true - speculative_model_dir: nvidia/Kimi-K2.5-Thinking-Eagle3 diff --git a/recipes/kimi-k2.5/trtllm/agg-eagle-kv-router/perf.yaml b/recipes/kimi-k2.5/trtllm/agg-eagle-kv-router/perf.yaml deleted file mode 100644 index 75e212598540..000000000000 --- a/recipes/kimi-k2.5/trtllm/agg-eagle-kv-router/perf.yaml +++ /dev/null @@ -1,195 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# AIPerf trace-replay benchmark for Kimi-K2.5-NVFP4 aggregated KV-router deployment. -# -# Replays requests from a Mooncake-format trace file using -# aiperf --custom-dataset-type mooncake_trace. -# -# Prerequisites: -# - DGD `kimi-k25-agg-kv-eagle` is ready -# - model-cache PVC exists in the namespace -# - Trace file copied to /model-cache/traces/.jsonl -# -# Results: /model-cache/perf/_/ -apiVersion: batch/v1 -kind: Job -metadata: - name: kimi-k25-agg-kv-eagle-bench -spec: - backoffLimit: 1 - activeDeadlineSeconds: 7200 # 2h hard cap on the Job - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: kimi-k25-agg-kv-eagle-bench - spec: - # Requires an untainted CPU/spare node so the bench pod doesn't share a node with - # GPU workers. If no spare node is available on your setup, remove the affinity block below. - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: nvidia.com/dynamo-graph-deployment-name - operator: In - values: - - kimi-k25-agg-kv-eagle - topologyKey: kubernetes.io/hostname - # tolerations: # uncomment to populate any tolerations for the gpu nodes - containers: - - name: perf - image: python:3.12-slim - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -c - - | - set -euo pipefail - ulimit -n 600000 - apt-get update && apt-get install -y curl jq procps git && apt-get clean - pip install "aiperf==0.10.0" protobuf "transformers==4.57.3" - sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true - export COLUMNS=200 - EPOCH=$(date +%s) - - wait_for_model_ready() { - local max_attempts=${WAIT_FOR_MODEL_MAX_ATTEMPTS:-720} # 720 * 5s = 1h default - local attempt=0 - echo "Waiting for model '$TARGET_MODEL' at $ENDPOINT/v1/models (max ${max_attempts} attempts) ..." - while ! curl -sf "http://$ENDPOINT/v1/models" | jq -e --arg m "$TARGET_MODEL" '.data[]? | select(.id == $m)' >/dev/null 2>&1; do - attempt=$((attempt + 1)) - if [ "$attempt" -ge "$max_attempts" ]; then - echo "ERROR: model '$TARGET_MODEL' did not become ready after ${max_attempts} attempts. Aborting." - exit 1 - fi - echo "[$(date '+%H:%M:%S')] not ready (attempt ${attempt}/${max_attempts}), sleeping 5s" - sleep 5 - done - curl -s "http://$ENDPOINT/v1/models" | jq . - } - - if [ ! -f "${TRACE_FILE}" ]; then - echo "ERROR: trace file not found at ${TRACE_FILE}" - echo "Copy it onto the PVC via a helper pod, e.g.:" - echo " kubectl cp /:${TRACE_FILE}" - exit 1 - fi - - wait_for_model_ready - - ROOT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - mkdir -p "$ROOT_DIR" - - echo "==============================================" - echo "Trace Replay Benchmark (aiperf)" - echo "==============================================" - echo "Endpoint: http://${ENDPOINT}" - echo "Model: ${TARGET_MODEL}" - echo "Trace file: ${TRACE_FILE}" - echo "Concurrencies: ${CONCURRENCIES}" - echo "TTFT threshold: ${TTFT_THRESHOLD_MS}ms" - echo "ITL threshold: ${ITL_THRESHOLD_MS}ms" - echo "Artifact root: ${ROOT_DIR}" - echo "==============================================" - - # Warmup - WARMUP_DIR="${ROOT_DIR}/warmup" - mkdir -p "$WARMUP_DIR" - aiperf profile \ - -m "${TARGET_MODEL}" \ - --tokenizer "${TARGET_MODEL}" \ - --tokenizer-trust-remote-code \ - --url "http://${ENDPOINT}" \ - --streaming \ - --ui dashboard \ - --extra-inputs ignore_eos:true \ - --concurrency 1 \ - --request-count 5 \ - --artifact-dir "${WARMUP_DIR}" - echo "Warmup complete" - - MODEL_BASE="${TARGET_MODEL##*/}" - TS=$(date +'%Y%m%d_%H%M%S') - - IFS=',' read -r -a CONCURRENCY_LIST <<< "${CONCURRENCIES}" - for C in "${CONCURRENCY_LIST[@]}"; do - echo "" - echo "=== Concurrency=${C} ===" - RUN_DIR="${ROOT_DIR}/${MODEL_BASE}_trace_c${C}_${TS}" - mkdir -p "$RUN_DIR" - - aiperf profile \ - -m "${TARGET_MODEL}" \ - --tokenizer "${TARGET_MODEL}" \ - --tokenizer-trust-remote-code \ - --input-file "${TRACE_FILE}" \ - --custom-dataset-type mooncake_trace \ - --url "http://${ENDPOINT}" \ - --streaming \ - --extra-inputs ignore_eos:true \ - --concurrency "${C}" \ - --random-seed 42 \ - --ui dashboard \ - --artifact-dir "${RUN_DIR}" \ - --goodput "time_to_first_token:${TTFT_THRESHOLD_MS} inter_token_latency:${ITL_THRESHOLD_MS}" \ - --benchmark-duration "${BENCHMARK_DURATION}" \ - --benchmark-grace-period 60 \ - --request-timeout-seconds 1200 \ - --workers-max 200 \ - --record-processors 8 \ - --profile-export-level raw \ - --concurrency-ramp-duration 60 \ - --export-http-trace - - echo "Concurrency ${C} complete; artifacts in ${RUN_DIR}" - ls -la "${RUN_DIR}" || true - done - - echo "" - echo "All concurrencies done. Root: ${ROOT_DIR}" - env: - - name: TARGET_MODEL - value: nvidia/Kimi-K2.5-NVFP4 - - name: ENDPOINT - value: kimi-k25-agg-kv-eagle-frontend:8000 - - name: TRACE_FILE - value: /model-cache/traces/agent_trace_data/dataset.jsonl - - name: CONCURRENCIES - value: "24" - - name: BENCHMARK_DURATION - value: "3600" - - name: TTFT_THRESHOLD_MS - value: "5000" - - name: ITL_THRESHOLD_MS - value: "10" - - name: AIPERF_HTTP_CONNECTION_LIMIT - value: "200" - - name: AIPERF_HTTP_SO_RCVTIMEO - value: "120" - - name: AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT - value: "3600" - - name: AIPERF_DATASET_CONFIGURATION_TIMEOUT - value: "3600" - - name: JOB_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.labels['job-name'] - - name: ROOT_ARTIFACT_DIR - value: /model-cache/perf - - name: HF_HOME - value: /model-cache - - name: PYTHONUNBUFFERED - value: "1" - volumeMounts: - - name: model-cache - mountPath: /model-cache - restartPolicy: Never - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/kimi-k2.5/trtllm/agg-eagle-round-robin/deploy.yaml b/recipes/kimi-k2.5/trtllm/agg-eagle-round-robin/deploy.yaml deleted file mode 100644 index f721d634b078..000000000000 --- a/recipes/kimi-k2.5/trtllm/agg-eagle-round-robin/deploy.yaml +++ /dev/null @@ -1,186 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Aggregated Kimi-K2.5-NVFP4 with Eagle3 speculative decoding and round-robin routing. -# -# Topology (6x GB200 nodes, 24 GPUs total): -# - 3x aggregated workers (TP=8 / EP=8), each spanning 2 nodes (4 GPUs/pod). -# -# Routing: frontend uses round-robin routing across replicas. -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: kimi-k25-agg-rr-eagle -spec: - envs: - - name: NCCL_MNNVL_ENABLE - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: TLLM_LOG_LEVEL - value: "info" - services: - Frontend: - componentType: frontend - extraPodSpec: - containers: null - mainContainer: - command: - - python3 - args: - - -m - - dingo.frontend - - --router-mode - - round-robin - - --request-plane - - nats - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime: - name: "" - resources: {} - nodeSelector: - kubernetes.io/arch: arm64 - # tolerations: # uncomment to populate any tolerations for the gpu nodes - replicas: 1 - - agg: - componentType: worker - envFromSecret: hf-token-secret - extraPodSpec: - containers: null - mainContainer: - args: - - --model-path - - nvidia/Kimi-K2.5-NVFP4 - - --served-model-name - - nvidia/Kimi-K2.5-NVFP4 - - --extra-engine-args - - /config/agg.yaml - - --publish-events-and-metrics - - --request-plane - - nats - - --kv-block-size - - "32" - - --dyn-tool-call-parser - - kimi_k2 - - --dyn-reasoning-parser - - kimi_k25 - command: - - python3 - - -m - - dynamo.trtllm - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: HF_HUB_OFFLINE - value: "1" - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: NCCL_SOCKET_IFNAME - value: eth0 - - name: GLOO_SOCKET_IFNAME - value: eth0 - - name: NCCL_GRAPH_MIXING_SUPPORT - value: "0" - - name: TRTLLM_ENABLE_PDL - value: "1" - # Uncomment the below to reproduce benchmark results in README exactly. - # - name: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS - # value: "2" - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime: - name: "" - resources: {} - securityContext: - runAsUser: 0 - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 20 - volumeMounts: - - mountPath: /model-cache - name: model-cache - - mountPath: /config - name: trtllm-config - readOnly: true - workingDir: /workspace/ - nodeSelector: - kubernetes.io/arch: arm64 - # tolerations: # uncomment to populate any tolerations for the gpu nodes - resourceClaims: - - name: compute-domain-channel - resourceClaimTemplateName: your-compute-domain-channel - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: trtllm-config - configMap: - name: kimi-k25-agg-rr-eagle-config - multinode: - nodeCount: 2 - replicas: 3 - resources: - limits: - gpu: "4" - claims: - - name: compute-domain-channel ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: kimi-k25-agg-rr-eagle-config -data: - agg.yaml: | - allreduce_strategy: MNNVL - max_num_tokens: 8192 - enable_chunked_prefill: true - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - enable_padding: true - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - free_gpu_memory_fraction: 0.75 - tokens_per_block: 32 - max_batch_size: 128 - moe_config: - backend: CUTLASS - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cuda_core - num_postprocess_workers: 8 - print_iter_log: true - stream_interval: 10 - tensor_parallel_size: 8 - trust_remote_code: true - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - allow_advanced_sampling: true - speculative_model_dir: nvidia/Kimi-K2.5-Thinking-Eagle3 diff --git a/recipes/kimi-k2.5/trtllm/agg-eagle-round-robin/perf.yaml b/recipes/kimi-k2.5/trtllm/agg-eagle-round-robin/perf.yaml deleted file mode 100644 index 6063cc2f6dc6..000000000000 --- a/recipes/kimi-k2.5/trtllm/agg-eagle-round-robin/perf.yaml +++ /dev/null @@ -1,195 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# AIPerf trace-replay benchmark for Kimi-K2.5-NVFP4 aggregated round-robin (eagle) deployment. -# -# Replays requests from a Mooncake-format trace file using -# aiperf --custom-dataset-type mooncake_trace. -# -# Prerequisites: -# - DGD `kimi-k25-agg-rr-eagle` is ready -# - model-cache PVC exists in the namespace -# - Trace file copied to /model-cache/traces/.jsonl -# -# Results: /model-cache/perf/_/ -apiVersion: batch/v1 -kind: Job -metadata: - name: kimi-k25-agg-rr-eagle-bench -spec: - backoffLimit: 1 - activeDeadlineSeconds: 7200 # 2h hard cap on the Job - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: kimi-k25-agg-rr-eagle-bench - spec: - # Requires an untainted CPU/spare node so the bench pod doesn't share a node with - # GPU workers. If no spare node is available on your setup, remove the affinity block below. - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: nvidia.com/dynamo-graph-deployment-name - operator: In - values: - - kimi-k25-agg-rr-eagle - topologyKey: kubernetes.io/hostname - # tolerations: # uncomment to populate any tolerations for the gpu nodes - containers: - - name: perf - image: python:3.12-slim - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -c - - | - set -euo pipefail - ulimit -n 600000 - apt-get update && apt-get install -y curl jq procps git && apt-get clean - pip install "aiperf==0.10.0" protobuf "transformers==4.57.3" - sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true - export COLUMNS=200 - EPOCH=$(date +%s) - - wait_for_model_ready() { - local max_attempts=${WAIT_FOR_MODEL_MAX_ATTEMPTS:-720} # 720 * 5s = 1h default - local attempt=0 - echo "Waiting for model '$TARGET_MODEL' at $ENDPOINT/v1/models (max ${max_attempts} attempts) ..." - while ! curl -sf "http://$ENDPOINT/v1/models" | jq -e --arg m "$TARGET_MODEL" '.data[]? | select(.id == $m)' >/dev/null 2>&1; do - attempt=$((attempt + 1)) - if [ "$attempt" -ge "$max_attempts" ]; then - echo "ERROR: model '$TARGET_MODEL' did not become ready after ${max_attempts} attempts. Aborting." - exit 1 - fi - echo "[$(date '+%H:%M:%S')] not ready (attempt ${attempt}/${max_attempts}), sleeping 5s" - sleep 5 - done - curl -s "http://$ENDPOINT/v1/models" | jq . - } - - if [ ! -f "${TRACE_FILE}" ]; then - echo "ERROR: trace file not found at ${TRACE_FILE}" - echo "Copy it onto the PVC via a helper pod, e.g.:" - echo " kubectl cp /:${TRACE_FILE}" - exit 1 - fi - - wait_for_model_ready - - ROOT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - mkdir -p "$ROOT_DIR" - - echo "==============================================" - echo "Trace Replay Benchmark (aiperf)" - echo "==============================================" - echo "Endpoint: http://${ENDPOINT}" - echo "Model: ${TARGET_MODEL}" - echo "Trace file: ${TRACE_FILE}" - echo "Concurrencies: ${CONCURRENCIES}" - echo "TTFT threshold: ${TTFT_THRESHOLD_MS}ms" - echo "ITL threshold: ${ITL_THRESHOLD_MS}ms" - echo "Artifact root: ${ROOT_DIR}" - echo "==============================================" - - # Warmup - WARMUP_DIR="${ROOT_DIR}/warmup" - mkdir -p "$WARMUP_DIR" - aiperf profile \ - -m "${TARGET_MODEL}" \ - --tokenizer "${TARGET_MODEL}" \ - --tokenizer-trust-remote-code \ - --url "http://${ENDPOINT}" \ - --streaming \ - --ui dashboard \ - --extra-inputs ignore_eos:true \ - --concurrency 1 \ - --request-count 5 \ - --artifact-dir "${WARMUP_DIR}" - echo "Warmup complete" - - MODEL_BASE="${TARGET_MODEL##*/}" - TS=$(date +'%Y%m%d_%H%M%S') - - IFS=',' read -r -a CONCURRENCY_LIST <<< "${CONCURRENCIES}" - for C in "${CONCURRENCY_LIST[@]}"; do - echo "" - echo "=== Concurrency=${C} ===" - RUN_DIR="${ROOT_DIR}/${MODEL_BASE}_trace_c${C}_${TS}" - mkdir -p "$RUN_DIR" - - aiperf profile \ - -m "${TARGET_MODEL}" \ - --tokenizer "${TARGET_MODEL}" \ - --tokenizer-trust-remote-code \ - --input-file "${TRACE_FILE}" \ - --custom-dataset-type mooncake_trace \ - --url "http://${ENDPOINT}" \ - --streaming \ - --extra-inputs ignore_eos:true \ - --concurrency "${C}" \ - --random-seed 42 \ - --ui dashboard \ - --artifact-dir "${RUN_DIR}" \ - --goodput "time_to_first_token:${TTFT_THRESHOLD_MS} inter_token_latency:${ITL_THRESHOLD_MS}" \ - --benchmark-duration "${BENCHMARK_DURATION}" \ - --benchmark-grace-period 60 \ - --request-timeout-seconds 1200 \ - --workers-max 200 \ - --record-processors 8 \ - --profile-export-level raw \ - --concurrency-ramp-duration 60 \ - --export-http-trace - - echo "Concurrency ${C} complete; artifacts in ${RUN_DIR}" - ls -la "${RUN_DIR}" || true - done - - echo "" - echo "All concurrencies done. Root: ${ROOT_DIR}" - env: - - name: TARGET_MODEL - value: nvidia/Kimi-K2.5-NVFP4 - - name: ENDPOINT - value: kimi-k25-agg-rr-eagle-frontend:8000 - - name: TRACE_FILE - value: /model-cache/traces/agent_trace_data/dataset.jsonl - - name: CONCURRENCIES - value: "24" - - name: BENCHMARK_DURATION - value: "3600" - - name: TTFT_THRESHOLD_MS - value: "5000" - - name: ITL_THRESHOLD_MS - value: "10" - - name: AIPERF_HTTP_CONNECTION_LIMIT - value: "200" - - name: AIPERF_HTTP_SO_RCVTIMEO - value: "120" - - name: AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT - value: "3600" - - name: AIPERF_DATASET_CONFIGURATION_TIMEOUT - value: "3600" - - name: JOB_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.labels['job-name'] - - name: ROOT_ARTIFACT_DIR - value: /model-cache/perf - - name: HF_HOME - value: /model-cache - - name: PYTHONUNBUFFERED - value: "1" - volumeMounts: - - name: model-cache - mountPath: /model-cache - restartPolicy: Never - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/kimi-k2.5/trtllm/agg-round-robin/deploy.yaml b/recipes/kimi-k2.5/trtllm/agg-round-robin/deploy.yaml deleted file mode 100644 index 11e942de4844..000000000000 --- a/recipes/kimi-k2.5/trtllm/agg-round-robin/deploy.yaml +++ /dev/null @@ -1,178 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Aggregated Kimi-K2.5-NVFP4 with round-robin routing (no speculative decoding). -# -# Topology (6x GB200 nodes, 24 GPUs total): -# - 3x aggregated workers (TP=8 / EP=8), each spanning 2 nodes (4 GPUs/pod). -# -# Routing: frontend uses round-robin routing across replicas. -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: kimi-k25-agg-rr -spec: - envs: - - name: NCCL_MNNVL_ENABLE - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: TLLM_LOG_LEVEL - value: "info" - services: - Frontend: - componentType: frontend - extraPodSpec: - containers: null - mainContainer: - command: - - python3 - args: - - -m - - dingo.frontend - - --router-mode - - round-robin - - --request-plane - - nats - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime: - name: "" - resources: {} - nodeSelector: - kubernetes.io/arch: arm64 - # tolerations: # uncomment to populate any tolerations for the gpu nodes - replicas: 1 - - agg: - componentType: worker - envFromSecret: hf-token-secret - extraPodSpec: - containers: null - mainContainer: - args: - - --model-path - - nvidia/Kimi-K2.5-NVFP4 - - --served-model-name - - nvidia/Kimi-K2.5-NVFP4 - - --extra-engine-args - - /config/agg.yaml - - --publish-events-and-metrics - - --request-plane - - nats - - --kv-block-size - - "32" - - --dyn-tool-call-parser - - kimi_k2 - - --dyn-reasoning-parser - - kimi_k25 - command: - - python3 - - -m - - dynamo.trtllm - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: HF_HUB_OFFLINE - value: "1" - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: NCCL_SOCKET_IFNAME - value: eth0 - - name: GLOO_SOCKET_IFNAME - value: eth0 - - name: NCCL_GRAPH_MIXING_SUPPORT - value: "0" - - name: TRTLLM_ENABLE_PDL - value: "1" - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime: - name: "" - resources: {} - securityContext: - runAsUser: 0 - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 20 - volumeMounts: - - mountPath: /model-cache - name: model-cache - - mountPath: /config - name: trtllm-config - readOnly: true - workingDir: /workspace/ - nodeSelector: - kubernetes.io/arch: arm64 - # tolerations: # uncomment to populate any tolerations for the gpu nodes - resourceClaims: - - name: compute-domain-channel - resourceClaimTemplateName: your-compute-domain-channel - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: trtllm-config - configMap: - name: kimi-k25-agg-rr-config - multinode: - nodeCount: 2 - replicas: 3 - resources: - limits: - gpu: "4" - claims: - - name: compute-domain-channel ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: kimi-k25-agg-rr-config -data: - agg.yaml: | - allreduce_strategy: MNNVL - max_num_tokens: 8192 - enable_chunked_prefill: true - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - enable_padding: true - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - free_gpu_memory_fraction: 0.75 - tokens_per_block: 32 - max_batch_size: 128 - moe_config: - backend: CUTLASS - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cuda_core - num_postprocess_workers: 8 - print_iter_log: true - stream_interval: 10 - tensor_parallel_size: 8 - trust_remote_code: true diff --git a/recipes/kimi-k2.5/trtllm/agg-round-robin/perf.yaml b/recipes/kimi-k2.5/trtllm/agg-round-robin/perf.yaml deleted file mode 100644 index f6db2d1a6028..000000000000 --- a/recipes/kimi-k2.5/trtllm/agg-round-robin/perf.yaml +++ /dev/null @@ -1,195 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# AIPerf trace-replay benchmark for Kimi-K2.5-NVFP4 aggregated round-robin (no-eagle) deployment. -# -# Replays requests from a Mooncake-format trace file using -# aiperf --custom-dataset-type mooncake_trace. -# -# Prerequisites: -# - DGD `kimi-k25-agg-rr` is ready -# - model-cache PVC exists in the namespace -# - Trace file copied to /model-cache/traces/.jsonl -# -# Results: /model-cache/perf/_/ -apiVersion: batch/v1 -kind: Job -metadata: - name: kimi-k25-agg-rr-bench -spec: - backoffLimit: 1 - activeDeadlineSeconds: 7200 # 2h hard cap on the Job - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: kimi-k25-agg-rr-bench - spec: - # Requires an untainted CPU/spare node so the bench pod doesn't share a node with - # GPU workers. If no spare node is available on your setup, remove the affinity block below. - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: nvidia.com/dynamo-graph-deployment-name - operator: In - values: - - kimi-k25-agg-rr - topologyKey: kubernetes.io/hostname - # tolerations: # uncomment to populate any tolerations for the gpu nodes - containers: - - name: perf - image: python:3.12-slim - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -c - - | - set -euo pipefail - ulimit -n 600000 - apt-get update && apt-get install -y curl jq procps git && apt-get clean - pip install "aiperf==0.10.0" protobuf "transformers==4.57.3" - sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true - export COLUMNS=200 - EPOCH=$(date +%s) - - wait_for_model_ready() { - local max_attempts=${WAIT_FOR_MODEL_MAX_ATTEMPTS:-720} # 720 * 5s = 1h default - local attempt=0 - echo "Waiting for model '$TARGET_MODEL' at $ENDPOINT/v1/models (max ${max_attempts} attempts) ..." - while ! curl -sf "http://$ENDPOINT/v1/models" | jq -e --arg m "$TARGET_MODEL" '.data[]? | select(.id == $m)' >/dev/null 2>&1; do - attempt=$((attempt + 1)) - if [ "$attempt" -ge "$max_attempts" ]; then - echo "ERROR: model '$TARGET_MODEL' did not become ready after ${max_attempts} attempts. Aborting." - exit 1 - fi - echo "[$(date '+%H:%M:%S')] not ready (attempt ${attempt}/${max_attempts}), sleeping 5s" - sleep 5 - done - curl -s "http://$ENDPOINT/v1/models" | jq . - } - - if [ ! -f "${TRACE_FILE}" ]; then - echo "ERROR: trace file not found at ${TRACE_FILE}" - echo "Copy it onto the PVC via a helper pod, e.g.:" - echo " kubectl cp /:${TRACE_FILE}" - exit 1 - fi - - wait_for_model_ready - - ROOT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - mkdir -p "$ROOT_DIR" - - echo "==============================================" - echo "Trace Replay Benchmark (aiperf)" - echo "==============================================" - echo "Endpoint: http://${ENDPOINT}" - echo "Model: ${TARGET_MODEL}" - echo "Trace file: ${TRACE_FILE}" - echo "Concurrencies: ${CONCURRENCIES}" - echo "TTFT threshold: ${TTFT_THRESHOLD_MS}ms" - echo "ITL threshold: ${ITL_THRESHOLD_MS}ms" - echo "Artifact root: ${ROOT_DIR}" - echo "==============================================" - - # Warmup - WARMUP_DIR="${ROOT_DIR}/warmup" - mkdir -p "$WARMUP_DIR" - aiperf profile \ - -m "${TARGET_MODEL}" \ - --tokenizer "${TARGET_MODEL}" \ - --tokenizer-trust-remote-code \ - --url "http://${ENDPOINT}" \ - --streaming \ - --ui dashboard \ - --extra-inputs ignore_eos:true \ - --concurrency 1 \ - --request-count 5 \ - --artifact-dir "${WARMUP_DIR}" - echo "Warmup complete" - - MODEL_BASE="${TARGET_MODEL##*/}" - TS=$(date +'%Y%m%d_%H%M%S') - - IFS=',' read -r -a CONCURRENCY_LIST <<< "${CONCURRENCIES}" - for C in "${CONCURRENCY_LIST[@]}"; do - echo "" - echo "=== Concurrency=${C} ===" - RUN_DIR="${ROOT_DIR}/${MODEL_BASE}_trace_c${C}_${TS}" - mkdir -p "$RUN_DIR" - - aiperf profile \ - -m "${TARGET_MODEL}" \ - --tokenizer "${TARGET_MODEL}" \ - --tokenizer-trust-remote-code \ - --input-file "${TRACE_FILE}" \ - --custom-dataset-type mooncake_trace \ - --url "http://${ENDPOINT}" \ - --streaming \ - --extra-inputs ignore_eos:true \ - --concurrency "${C}" \ - --random-seed 42 \ - --ui dashboard \ - --artifact-dir "${RUN_DIR}" \ - --goodput "time_to_first_token:${TTFT_THRESHOLD_MS} inter_token_latency:${ITL_THRESHOLD_MS}" \ - --benchmark-duration "${BENCHMARK_DURATION}" \ - --benchmark-grace-period 60 \ - --request-timeout-seconds 1200 \ - --workers-max 200 \ - --record-processors 8 \ - --profile-export-level raw \ - --concurrency-ramp-duration 60 \ - --export-http-trace - - echo "Concurrency ${C} complete; artifacts in ${RUN_DIR}" - ls -la "${RUN_DIR}" || true - done - - echo "" - echo "All concurrencies done. Root: ${ROOT_DIR}" - env: - - name: TARGET_MODEL - value: nvidia/Kimi-K2.5-NVFP4 - - name: ENDPOINT - value: kimi-k25-agg-rr-frontend:8000 - - name: TRACE_FILE - value: /model-cache/traces/agent_trace_data/dataset.jsonl - - name: CONCURRENCIES - value: "8" - - name: BENCHMARK_DURATION - value: "3600" - - name: TTFT_THRESHOLD_MS - value: "5000" - - name: ITL_THRESHOLD_MS - value: "10" - - name: AIPERF_HTTP_CONNECTION_LIMIT - value: "200" - - name: AIPERF_HTTP_SO_RCVTIMEO - value: "120" - - name: AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT - value: "3600" - - name: AIPERF_DATASET_CONFIGURATION_TIMEOUT - value: "3600" - - name: JOB_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.labels['job-name'] - - name: ROOT_ARTIFACT_DIR - value: /model-cache/perf - - name: HF_HOME - value: /model-cache - - name: PYTHONUNBUFFERED - value: "1" - volumeMounts: - - name: model-cache - mountPath: /model-cache - restartPolicy: Never - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/kimi-k2.5/trtllm/disagg-eagle-kv-router/deploy.yaml b/recipes/kimi-k2.5/trtllm/disagg-eagle-kv-router/deploy.yaml deleted file mode 100644 index 1e13dbf95e6b..000000000000 --- a/recipes/kimi-k2.5/trtllm/disagg-eagle-kv-router/deploy.yaml +++ /dev/null @@ -1,328 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Disaggregated Kimi-K2.5-NVFP4 with Eagle3 speculative decoding, KV-aware routing, -# and host-memory KV offloading on prefill workers. -# -# Topology (6x GB200 nodes, 24 GPUs total): -# - 3x prefill workers (TP=4 / EP=4, single-node each) -# - 3x decode workers (TP=4 / EP=4, single-node each) -# -# Routing: frontend uses KV-aware routing with prefix-hash awareness to land -# requests on whichever prefill worker already holds matching KV blocks. -# -# Prefill workers offload spilled KV blocks to host memory (host_cache_size + -# secondary_offload_min_priority in prefill.yaml). -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: kimi-k25-disagg-kv-eagle -spec: - envs: - - name: NCCL_MNNVL_ENABLE - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: TLLM_LOG_LEVEL - value: "info" - - name: TRTLLM_ENABLE_PDL - value: "1" - services: - Frontend: - componentType: frontend - extraPodSpec: - containers: null - mainContainer: - command: - - python3 - args: - - -m - - dingo.frontend - - --router-mode - - kv - - --router-reset-states - - --request-plane - - nats - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime: - name: "" - resources: {} - nodeSelector: - kubernetes.io/arch: arm64 - # tolerations: # uncomment to populate any tolerations for the gpu nodes - replicas: 1 - - prefill: - componentType: worker - subComponentType: prefill - envFromSecret: hf-token-secret - extraPodSpec: - containers: null - mainContainer: - args: - - --model-path - - nvidia/Kimi-K2.5-NVFP4 - - --served-model-name - - nvidia/Kimi-K2.5-NVFP4 - - --extra-engine-args - - /config/prefill.yaml - - --disaggregation-mode - - prefill - - --publish-events-and-metrics - - --request-plane - - nats - - --kv-block-size - - "32" - - --dyn-tool-call-parser - - kimi_k2 - - --dyn-reasoning-parser - - kimi_k25 - command: - - python3 - - -m - - dynamo.trtllm - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: HF_HUB_OFFLINE - value: "1" - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: NCCL_GRAPH_MIXING_SUPPORT - value: "0" - # KV transfer (UCX zcopy on; no staging buffer required) - - name: TRTLLM_TRY_ZCOPY_FOR_KVCACHE_TRANSFER - value: "1" - - name: TRTLLM_KVCACHE_TRANSFER_BUFFER_SIZE - value: "0B" - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime: - name: "" - resources: {} - securityContext: - runAsUser: 0 - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 20 - volumeMounts: - - mountPath: /model-cache - name: model-cache - - mountPath: /config - name: trtllm-config - readOnly: true - workingDir: /workspace/ - nodeSelector: - kubernetes.io/arch: arm64 - # tolerations: # uncomment to populate any tolerations for the gpu nodes - resourceClaims: - - name: compute-domain-channel - resourceClaimTemplateName: your-compute-domain-channel - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: trtllm-config - configMap: - name: kimi-k25-disagg-kv-eagle-config - replicas: 3 - resources: - limits: - gpu: "4" - claims: - - name: compute-domain-channel - - decode: - componentType: worker - subComponentType: decode - envFromSecret: hf-token-secret - extraPodSpec: - containers: null - mainContainer: - args: - - --model-path - - nvidia/Kimi-K2.5-NVFP4 - - --served-model-name - - nvidia/Kimi-K2.5-NVFP4 - - --extra-engine-args - - /config/decode.yaml - - --disaggregation-mode - - decode - - --publish-events-and-metrics - - --request-plane - - nats - - --kv-block-size - - "32" - - --dyn-tool-call-parser - - kimi_k2 - - --dyn-reasoning-parser - - kimi_k25 - command: - - python3 - - -m - - dynamo.trtllm - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: HF_HUB_OFFLINE - value: "1" - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: NCCL_SOCKET_IFNAME - value: eth0 - - name: GLOO_SOCKET_IFNAME - value: eth0 - - name: NCCL_GRAPH_MIXING_SUPPORT - value: "0" - - name: ENABLE_CONFIGURABLE_MOE - value: "1" - # Uncomment the below to reproduce benchmark results in README exactly. - # - name: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS - # value: "2" - # KV transfer (UCX zcopy on; no staging buffer required) - - name: TRTLLM_TRY_ZCOPY_FOR_KVCACHE_TRANSFER - value: "1" - - name: TRTLLM_KVCACHE_TRANSFER_BUFFER_SIZE - value: "0B" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: NVIDIA_GDRCOPY - value: "1" - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime: - name: "" - resources: {} - securityContext: - runAsUser: 0 - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 20 - volumeMounts: - - mountPath: /model-cache - name: model-cache - - mountPath: /config - name: trtllm-config - readOnly: true - workingDir: /workspace/ - nodeSelector: - kubernetes.io/arch: arm64 - # tolerations: # uncomment to populate any tolerations for the gpu nodes - resourceClaims: - - name: compute-domain-channel - resourceClaimTemplateName: your-compute-domain-channel - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: trtllm-config - configMap: - name: kimi-k25-disagg-kv-eagle-config - replicas: 3 - resources: - limits: - gpu: "4" - claims: - - name: compute-domain-channel ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: kimi-k25-disagg-kv-eagle-config -data: - prefill.yaml: | - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 0 - kv_transfer_sender_future_timeout_ms: 30000 - allreduce_strategy: MNNVL - max_batch_size: 4 - max_num_tokens: 8192 - enable_chunked_prefill: true - max_seq_len: 200000 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 100 - trust_remote_code: true - cuda_graph_config: null - disable_overlap_scheduler: true - moe_config: - backend: TRTLLM - kv_cache_config: - enable_block_reuse: true - free_gpu_memory_fraction: 0.6 - host_cache_size: 107374182400 # 100 GiB host-memory KV offload - secondary_offload_min_priority: 35 - dtype: fp8 - tokens_per_block: 32 - speculative_config: - decoding_type: Eagle - max_draft_len: 3 - allow_advanced_sampling: true - speculative_model_dir: nvidia/Kimi-K2.5-Thinking-Eagle3 - decode.yaml: | - allreduce_strategy: MNNVL - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 0 - kv_transfer_sender_future_timeout_ms: 30000 - max_num_tokens: 640 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - enable_padding: true - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - tokens_per_block: 32 - max_batch_size: 128 - moe_config: - backend: CUTLASS - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cuda_core - num_postprocess_workers: 8 - print_iter_log: true - stream_interval: 10 - tensor_parallel_size: 4 - trust_remote_code: true - speculative_config: - decoding_type: Eagle - max_draft_len: 3 - allow_advanced_sampling: true - speculative_model_dir: nvidia/Kimi-K2.5-Thinking-Eagle3 diff --git a/recipes/kimi-k2.5/trtllm/disagg-eagle-kv-router/perf.yaml b/recipes/kimi-k2.5/trtllm/disagg-eagle-kv-router/perf.yaml deleted file mode 100644 index 6300f0a1a07c..000000000000 --- a/recipes/kimi-k2.5/trtllm/disagg-eagle-kv-router/perf.yaml +++ /dev/null @@ -1,195 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# AIPerf trace-replay benchmark for Kimi-K2.5-NVFP4 disaggregated KV-router deployment. -# -# Replays requests from a Mooncake-format trace file using -# aiperf --custom-dataset-type mooncake_trace. -# -# Prerequisites: -# - DGD `kimi-k25-disagg-kv-eagle` is ready -# - model-cache PVC exists in the namespace -# - Trace file copied to /model-cache/traces/.jsonl -# -# Results: /model-cache/perf/_/ -apiVersion: batch/v1 -kind: Job -metadata: - name: kimi-k25-disagg-kv-eagle-bench -spec: - backoffLimit: 1 - activeDeadlineSeconds: 7200 # 2h hard cap on the Job - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: kimi-k25-disagg-kv-eagle-bench - spec: - # Requires an untainted CPU/spare node so the bench pod doesn't share a node with - # GPU workers. If no spare node is available on your setup, remove the affinity block below. - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: nvidia.com/dynamo-graph-deployment-name - operator: In - values: - - kimi-k25-disagg-kv-eagle - topologyKey: kubernetes.io/hostname - # tolerations: # uncomment to populate any tolerations for the gpu nodes - containers: - - name: perf - image: python:3.12-slim - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -c - - | - set -euo pipefail - ulimit -n 600000 - apt-get update && apt-get install -y curl jq procps git && apt-get clean - pip install "aiperf==0.10.0" protobuf "transformers==4.57.3" - sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true - export COLUMNS=200 - EPOCH=$(date +%s) - - wait_for_model_ready() { - local max_attempts=${WAIT_FOR_MODEL_MAX_ATTEMPTS:-720} # 720 * 5s = 1h default - local attempt=0 - echo "Waiting for model '$TARGET_MODEL' at $ENDPOINT/v1/models (max ${max_attempts} attempts) ..." - while ! curl -sf "http://$ENDPOINT/v1/models" | jq -e --arg m "$TARGET_MODEL" '.data[]? | select(.id == $m)' >/dev/null 2>&1; do - attempt=$((attempt + 1)) - if [ "$attempt" -ge "$max_attempts" ]; then - echo "ERROR: model '$TARGET_MODEL' did not become ready after ${max_attempts} attempts. Aborting." - exit 1 - fi - echo "[$(date '+%H:%M:%S')] not ready (attempt ${attempt}/${max_attempts}), sleeping 5s" - sleep 5 - done - curl -s "http://$ENDPOINT/v1/models" | jq . - } - - if [ ! -f "${TRACE_FILE}" ]; then - echo "ERROR: trace file not found at ${TRACE_FILE}" - echo "Copy it onto the PVC via a helper pod, e.g.:" - echo " kubectl cp /:${TRACE_FILE}" - exit 1 - fi - - wait_for_model_ready - - ROOT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - mkdir -p "$ROOT_DIR" - - echo "==============================================" - echo "Trace Replay Benchmark (aiperf)" - echo "==============================================" - echo "Endpoint: http://${ENDPOINT}" - echo "Model: ${TARGET_MODEL}" - echo "Trace file: ${TRACE_FILE}" - echo "Concurrencies: ${CONCURRENCIES}" - echo "TTFT threshold: ${TTFT_THRESHOLD_MS}ms" - echo "ITL threshold: ${ITL_THRESHOLD_MS}ms" - echo "Artifact root: ${ROOT_DIR}" - echo "==============================================" - - # Warmup - WARMUP_DIR="${ROOT_DIR}/warmup" - mkdir -p "$WARMUP_DIR" - aiperf profile \ - -m "${TARGET_MODEL}" \ - --tokenizer "${TARGET_MODEL}" \ - --tokenizer-trust-remote-code \ - --url "http://${ENDPOINT}" \ - --streaming \ - --ui dashboard \ - --extra-inputs ignore_eos:true \ - --concurrency 1 \ - --request-count 5 \ - --artifact-dir "${WARMUP_DIR}" - echo "Warmup complete" - - MODEL_BASE="${TARGET_MODEL##*/}" - TS=$(date +'%Y%m%d_%H%M%S') - - IFS=',' read -r -a CONCURRENCY_LIST <<< "${CONCURRENCIES}" - for C in "${CONCURRENCY_LIST[@]}"; do - echo "" - echo "=== Concurrency=${C} ===" - RUN_DIR="${ROOT_DIR}/${MODEL_BASE}_trace_c${C}_${TS}" - mkdir -p "$RUN_DIR" - - aiperf profile \ - -m "${TARGET_MODEL}" \ - --tokenizer "${TARGET_MODEL}" \ - --tokenizer-trust-remote-code \ - --input-file "${TRACE_FILE}" \ - --custom-dataset-type mooncake_trace \ - --url "http://${ENDPOINT}" \ - --streaming \ - --extra-inputs ignore_eos:true \ - --concurrency "${C}" \ - --random-seed 42 \ - --ui dashboard \ - --artifact-dir "${RUN_DIR}" \ - --goodput "time_to_first_token:${TTFT_THRESHOLD_MS} inter_token_latency:${ITL_THRESHOLD_MS}" \ - --benchmark-duration "${BENCHMARK_DURATION}" \ - --benchmark-grace-period 60 \ - --request-timeout-seconds 1200 \ - --workers-max 200 \ - --record-processors 8 \ - --profile-export-level raw \ - --concurrency-ramp-duration 60 \ - --export-http-trace - - echo "Concurrency ${C} complete; artifacts in ${RUN_DIR}" - ls -la "${RUN_DIR}" || true - done - - echo "" - echo "All concurrencies done. Root: ${ROOT_DIR}" - env: - - name: TARGET_MODEL - value: nvidia/Kimi-K2.5-NVFP4 - - name: ENDPOINT - value: kimi-k25-disagg-kv-eagle-frontend:8000 - - name: TRACE_FILE - value: /model-cache/traces/agent_trace_data/dataset.jsonl - - name: CONCURRENCIES - value: "32" - - name: BENCHMARK_DURATION - value: "3600" - - name: TTFT_THRESHOLD_MS - value: "5000" - - name: ITL_THRESHOLD_MS - value: "10" - - name: AIPERF_HTTP_CONNECTION_LIMIT - value: "200" - - name: AIPERF_HTTP_SO_RCVTIMEO - value: "120" - - name: AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT - value: "3600" - - name: AIPERF_DATASET_CONFIGURATION_TIMEOUT - value: "3600" - - name: JOB_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.labels['job-name'] - - name: ROOT_ARTIFACT_DIR - value: /model-cache/perf - - name: HF_HOME - value: /model-cache - - name: PYTHONUNBUFFERED - value: "1" - volumeMounts: - - name: model-cache - mountPath: /model-cache - restartPolicy: Never - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/kimi-k2.6/README.md b/recipes/kimi-k2.6/README.md deleted file mode 100644 index 06fb0f6a40e7..000000000000 --- a/recipes/kimi-k2.6/README.md +++ /dev/null @@ -1,113 +0,0 @@ -# Kimi-K2.6 Recipes - -Recipes for **moonshotai/Kimi-K2.6**. - -## Configurations - -Dynamo + vLLM deployment profiles across two GPU SKUs and two target workloads: - -| | B200 chat | H200 chat | B200 agentic | H200 agentic | -|--------------------------|---------------------------------------|---------------------------------------|--------------------------------------|--------------------------------------| -| **GPU** (per worker) | 4x B200 | 8x H200 | 4x B200 | 8x H200 | -| **Mode** | aggregated | aggregated | aggregated | aggregated | -| **Framework** | vLLM 0.21.0 | vLLM 0.21.0 | vLLM 0.21.0 | vLLM 0.21.0 | -| **Precision** | NVFP4 + FP8 KV | INT4 | NVFP4 + FP8 KV | INT4 | -| **Parallelism** | TP4 | TP8 | TP4 | TP8 | -| **MoE backend** | FLASHINFER_TRTLLM | MARLIN | FLASHINFER_TRTLLM | MARLIN | -| **Attention backend** | FLASH_ATTN_MLA | FLASH_ATTN_MLA | FLASH_ATTN_MLA | FLASH_ATTN_MLA | -| **AllReduce backend** | NCCL symmetric memory | NCCL | NCCL symmetric memory | NCCL | -| **All2All backend** | N/A | N/A | N/A | N/A | -| **Routing** | KV-aware | KV-aware | KV-aware | KV-aware | -| **Speculative decoding** | EAGLE3 MLA (DL=3, SpeedBench AL=2.49) | EAGLE3 MLA (DL=3, SpeedBench AL=2.49) | EAGLE3 MLA (DL=3, SpeedBench AL=2.49) | EAGLE3 MLA (DL=3, SpeedBench AL=2.49) | -| **KV cache offloading** | LMCache CPU | LMCache CPU | LMCache CPU | LMCache CPU | - - -## Supported features - -- Modalities: Text + Image -- Reasoning -- Tool calling - - -## Prerequisites - -1. **Dynamo Platform installed** — see [Kubernetes Deployment Guide](../../docs/kubernetes/README.md). -2. **HuggingFace token** with access to `nvidia/Kimi-K2.6-NVFP4`, `moonshotai/Kimi-K2.6` and `lightseekorg/kimi-k2.6-eagle3-mla`: - ```bash - export NAMESPACE=your-namespace - kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="your-token" \ - -n ${NAMESPACE} - ``` - - -## Quick Start - -### 1. Create namespace - -``` -export NAMESPACE=your-namespace -kubectl create namespace ${NAMESPACE} -``` - -### 2. Create Storage - -> **Note:** Edit `model-cache/model-cache.yaml` first and update `storageClassName` to match your cluster (`kubectl get storageclass`). - -```bash -kubectl apply -f model-cache/model-cache.yaml -n ${NAMESPACE} -``` - -### 3. Download model + EAGLE3 head - -> **Note:** Edit `model-cache/model-download.yaml` first and remove the `hf download` lines that do not apply to your deployment (For H200, remove the NVFP4 download, for B200, remove the native INT4 download). - -```bash -kubectl apply -f model-cache/model-download.yaml -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/model-download -n ${NAMESPACE} --timeout=3600s -``` - -### 4. Deploy the DGD - -Deploy the target DGD: - -```bash -SKU=b200 # or h200 -USECASE=chat # or agentic - -kubectl apply -f vllm/agg-${SKU}-${USECASE}/deploy.yaml -n ${NAMESPACE} -``` - - -### 5. Benchmark - -See [`perf/README.md`](perf/README.md) for the full benchmark workflow — trace staging on the PVC, running the AIPerf trace-replay Job ([`perf/perf.yaml`](perf/perf.yaml)), running a concurrency sweep, and fetching artifacts. - - -## Optimization targets - -Recipes are optimized for the following configurations, at the target user interactivity: - -| Workload | Median ISL | Median OSL | KV cache hit rate | User output tok/s | -|------------------------|------------|------------|----------------|------------| -| Chat | 1k | 1k | 70% | 50 | -| Agentic | 64k | 400 | 90% | 50 | - - -Modified Mooncake traces are provided to showcase the value of KV-aware routing and CPU offloading, see [perf/README.md](./perf/README.md) for details. - - -## Performance results - -| Recipe | SKU | # of worker replicas | Concurrency | User output tok/s | System output tok/s/gpu | -|------------------------|------------|------------|----------------|------------|------------| -| Chat (15% subset) | B200 | 4 | 48 | 49.86 | 107.8 | -| Agentic (15% subset) | B200 | 4 | 64 | 55.50 | 166.5 | -| Chat (15% subset) | H200 | 4 | 32 | 54.86 | 38.7 | -| Agentic (15% subset) | H200 | 4 | 48 | 56.06 | 66.5 | - - -## Known issues - -1. Dynamo's KV cache router does not support all LMCache KV events, so routing can be sub-optimal -2. Some 400 HTTP errors from the workers on invalid inputs can be raised as 500 errors through the frontend diff --git a/recipes/kimi-k2.6/model-cache/model-cache.yaml b/recipes/kimi-k2.6/model-cache/model-cache.yaml deleted file mode 100644 index f59bed152f1c..000000000000 --- a/recipes/kimi-k2.6/model-cache/model-cache.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 1200Gi - # Edit this - storageClassName: "your-storage-class-name" \ No newline at end of file diff --git a/recipes/kimi-k2.6/model-cache/model-download.yaml b/recipes/kimi-k2.6/model-cache/model-download.yaml deleted file mode 100644 index e1d7f8abc794..000000000000 --- a/recipes/kimi-k2.6/model-cache/model-download.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - seccompProfile: - type: RuntimeDefault - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: HF_HOME - value: /model-cache - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - # Edit this, only pull the checkpoints you need, NVFP4 or native INT4 - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.16.4 - hf download nvidia/Kimi-K2.6-NVFP4 - hf download moonshotai/Kimi-K2.6 - hf download lightseekorg/kimi-k2.6-eagle3-mla - volumeMounts: - - name: model-cache - mountPath: /model-cache - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache \ No newline at end of file diff --git a/recipes/kimi-k2.6/perf/README.md b/recipes/kimi-k2.6/perf/README.md deleted file mode 100644 index 7557fa8cf451..000000000000 --- a/recipes/kimi-k2.6/perf/README.md +++ /dev/null @@ -1,155 +0,0 @@ -# Kimi-K2.6 Benchmark Recipe - -A single [AIPerf](https://github.com/ai-dynamo/aiperf) trace-replay Job — [perf.yaml](perf.yaml) — covers every Kimi-K2.6 DGD variant (B200/H200 × chat/agent traces). The benchmark is identical across variants; only `ENDPOINT` and `TRACE_FILE` need to change. - -The Job waits for `GET /v1/models` on the DGD frontend to return `moonshotai/Kimi-K2.6` (up to ~1h by default), runs a short warmup, then replays the configured trace at a single `CONCURRENCY` value and writes raw artifacts to the shared `model-cache` PVC. - -The bench pod is **co-located with the DGD frontend** (`podAffinity` on the frontend's host) so client → server traffic stays on a single node. - -## Targeting a variant - -Edit the `env` block in [perf.yaml](perf.yaml): - - -| Variant target | `ENDPOINT` | `TRACE_FILE` (chat / agent) | -| ------------------------ | ----------------------------------------------- | ----------------------------------------------------------------------- | -| B200 agg, chat workload | `kimi-k26-agg-b200-chat-frontend:8000` | `/model-cache/traces/8k_1k_70kv_chat_new_noschedule.jsonl` | -| B200 agg, agent workload | `kimi-k26-agg-b200-agentic-frontend:8000` | `/model-cache/traces/64k_400_90kv_agent_new_noschedule.jsonl` | -| H200 agg, chat workload | `kimi-k26-agg-h200-chat-frontend:8000` | `/model-cache/traces/8k_1k_70kv_chat_new_noschedule.jsonl` | -| H200 agg, agent workload | `kimi-k26-agg-h200-agentic-frontend:8000` | `/model-cache/traces/64k_400_90kv_agent_new_noschedule.jsonl` | - - -The chat- and agent-tuned DGDs both serve `moonshotai/Kimi-K2.6`, so either trace can be replayed against either DGD by swapping `TRACE_FILE`. - -If you run more than one benchmark in the same namespace, also update `metadata.name` / `labels.app` so jobs and artifact directories stay distinct. - -## Dataset - -The benchmark replays a [Mooncake-format](https://github.com/kvcache-ai/Mooncake) trace via `aiperf --custom-dataset-type mooncake_trace`. Each JSONL line describes one request (`input_length`, `output_length`, `hash_ids`). - -Trace flavours expected on the PVC: - -- **Chat** — `/model-cache/traces/8k_1k_70kv_chat_new_noschedule.jsonl` -- **Agent** — `/model-cache/traces/64k_400_90kv_agent_new_noschedule.jsonl` - -For shorter runs (smoke tests, faster iteration), point `TRACE_FILE` at a smaller variant of the same trace rather than capping run time. We typically stage 15% and 30% subsets alongside the full dataset, e.g.: - -``` -/model-cache/traces/.jsonl # full -/model-cache/traces/_short_30perc.jsonl # ~30% subset -/model-cache/traces/_short_15perc.jsonl # ~15% subset -``` - -The Dynamo Kimi-K2.5 recipe is the closest reference for trace handling — see [its README](https://github.com/ai-dynamo/dynamo/blob/main/recipes/kimi-k2.5/README.md#dataset-agentic-coding-workflow) and [perf.yaml](https://github.com/ai-dynamo/dynamo/blob/main/recipes/kimi-k2.5/trtllm/agg-eagle-kv-router/perf.yaml). - -## Workflow - -```bash -export NAMESPACE=your-namespace -``` - -### 1. Deploy the DGD - -See instructions in the [README](../README.md). - -Before deploying, do these adjustments to the DGD: -- Change the `SPECULATIVE_CONFIG` variable to point to `key: speculative-config-synthetic`. -- Modify the worker `replicas` to match your desired target - - -### 2. Stage the trace on the PVC - -Spin up a short-lived helper pod that mounts `model-cache`, then `kubectl cp` the traces in: - -```bash -kubectl run pvc-helper -n ${NAMESPACE} \ - --image=busybox:1.36 --restart=Never \ - --overrides='{"spec":{"containers":[{"name":"helper","image":"busybox:1.36","command":["sleep","3600"],"volumeMounts":[{"name":"model-cache","mountPath":"/model-cache"}]}],"volumes":[{"name":"model-cache","persistentVolumeClaim":{"claimName":"model-cache"}}]}}' \ - --command -- sleep 3600 - -kubectl cp ./traces ${NAMESPACE}/pvc-helper:/model-cache/ -``` - -Keep `pvc-helper` around for fetching artifacts later, or `kubectl delete pod pvc-helper -n ${NAMESPACE}` once you're done staging. - -### 3. Run the benchmark - -```bash -kubectl apply -f perf.yaml -n ${NAMESPACE} - -# Stream logs -kubectl logs -n ${NAMESPACE} -l job-name=kimi-k26-bench -f - -# Wait for completion (2h hard cap on the Job) -kubectl wait --for=condition=Complete \ - job/kimi-k26-bench \ - -n ${NAMESPACE} --timeout=7200s -``` - -### 4. Fetch artifacts - -```bash -kubectl cp ${NAMESPACE}/pvc-helper:/model-cache/perf/_kimi-k26-bench ./results -``` - -### 5. Cleanup - -```bash -kubectl delete job kimi-k26-bench -n ${NAMESPACE} -kubectl delete pod pvc-helper -n ${NAMESPACE} # if you kept it around -``` - -## Running a concurrency sweep - -`perf.yaml` runs a **single** `CONCURRENCY` value. To measure multiple concurrencies you must clear server state between runs — otherwise residual KV cache / prefix-cache hits from the previous run skew results. - -For each concurrency value you want to measure: - -```bash -# 1. Delete the previous bench job -kubectl delete job kimi-k26-bench -n ${NAMESPACE} --ignore-not-found - -# 2. Delete the DGD worker pods to drop KV / prefix-cache state. -# DGDs are Grove-managed (DGD → PodCliqueSet → PodClique → Pod), not Deployments, -# so `kubectl rollout restart deployment` won't match anything — delete the pods -# directly and let Grove recreate them. -DGD=kimi-k26-agg-b200-chat -kubectl delete pods -n ${NAMESPACE} \ - -l nvidia.com/dynamo-graph-deployment-name=${DGD},nvidia.com/dynamo-component-type=worker -kubectl wait --for=condition=Ready pod -n ${NAMESPACE} \ - -l nvidia.com/dynamo-graph-deployment-name=${DGD},nvidia.com/dynamo-component-type=worker \ - --timeout=900s - -# 3. Bump CONCURRENCY in perf.yaml (or use kubectl create -f perf.yaml \ -# with `--dry-run=client -o yaml | yq '.spec.template.spec.containers[0].env[] |= ...'`) -kubectl apply -f perf.yaml -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/kimi-k26-bench -n ${NAMESPACE} --timeout=7200s -``` - -(The bench Job's `wait_for_model_ready` loop handles the worker restart window — it re-polls `/v1/models` until the frontend reports ready again.) - -## Tunable environment variables - -Edit the `env` block on the `Job` to adjust: - - -| Variable | Default | Notes | -| -------------- | --------------------------------------------------- | ------------------------------------------------------------------------------- | -| `ENDPOINT` | `kimi-k26-agg-b200-chat-frontend:8000` | DGD frontend service:port — change per variant | -| `TRACE_FILE` | `/model-cache/traces/8k_1k_70kv_chat_new_noschedule.jsonl` | Swap to agent or to a smaller subset (`...short_15perc.jsonl`) for shorter runs | -| `CONCURRENCY` | `24` | Single value — see [Running a concurrency sweep](#running-a-concurrency-sweep) | -| `TARGET_MODEL` | `moonshotai/Kimi-K2.6` | Must match `--served-model-name` on the DGD frontend | - - -## Artifacts - -Results are written to: - -``` -/model-cache/perf/_/ - warmup/ - Kimi-K2.6_trace_c_/ - profile_export.json - inputs.json - ... -``` diff --git a/recipes/kimi-k2.6/perf/perf.yaml b/recipes/kimi-k2.6/perf/perf.yaml deleted file mode 100644 index 73d1ed17b45e..000000000000 --- a/recipes/kimi-k2.6/perf/perf.yaml +++ /dev/null @@ -1,183 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# AIPerf trace-replay benchmark for Kimi-K2.6 DGDs. -# -# One Job covers every variant (B200/H200 × chat/agent traces) — the -# benchmark itself is identical regardless of GPU SKU. To target a specific -# DGD, edit the env values below (and `metadata.name` / `labels.app` if you -# run multiple jobs in the same namespace). -# -# Runs a single concurrency value. To sweep concurrencies, restart the DGD -# workers between runs so KV cache and prefix-cache state are reset — see -# README.md "Running a concurrency sweep". -# -# Prerequisites: -# - Target DGD is Ready (see ../README.md) -# - model-cache PVC mounted in the namespace and contains TRACE_FILE -# -# Results: /model-cache/perf/_/ -apiVersion: batch/v1 -kind: Job -metadata: - name: kimi-k26-bench -spec: - backoffLimit: 1 - activeDeadlineSeconds: 7200 # 2h hard cap on the Job - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: kimi-k26-bench - spec: - # Co-locate the bench pod with the DGD frontend to minimise client/server - # network overhead. Update `values:` to the DGD you are benchmarking. - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: nvidia.com/dynamo-component-type - operator: In - values: - - frontend - - key: nvidia.com/dynamo-graph-deployment-name - operator: In - values: - - kimi-k26-agg-b200-chat - - kimi-k26-agg-b200-agentic - - kimi-k26-agg-h200-chat - - kimi-k26-agg-h200-agentic - topologyKey: kubernetes.io/hostname - # tolerations: # uncomment to populate any tolerations for the gpu nodes - containers: - - name: perf - image: python:3.12-slim - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -c - - | - set -euo pipefail - ulimit -n 600000 - apt-get update && apt-get install -y curl jq procps git && apt-get clean - pip install "aiperf==0.10.0" "protobuf==7.35.0" "transformers==4.57.3" "tiktoken==0.13.0" - sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true - export COLUMNS=200 - EPOCH=$(date +%s) - wait_for_model_ready() { - local max_attempts=${WAIT_FOR_MODEL_MAX_ATTEMPTS:-720} # 720 * 5s = 1h default - local attempt=0 - echo "Waiting for model '$TARGET_MODEL' at $ENDPOINT/v1/models (max ${max_attempts} attempts) ..." - while ! curl -sf "http://$ENDPOINT/v1/models" | jq -e --arg m "$TARGET_MODEL" '.data[]? | select(.id == $m)' >/dev/null 2>&1; do - attempt=$((attempt + 1)) - if [ "$attempt" -ge "$max_attempts" ]; then - echo "ERROR: model '$TARGET_MODEL' did not become ready after ${max_attempts} attempts. Aborting." - exit 1 - fi - echo "[$(date '+%H:%M:%S')] not ready (attempt ${attempt}/${max_attempts}), sleeping 5s" - sleep 5 - done - curl -s "http://$ENDPOINT/v1/models" | jq . - } - if [ ! -f "${TRACE_FILE}" ]; then - echo "ERROR: trace file not found at ${TRACE_FILE}" - echo "Copy it onto the PVC via a helper pod, e.g.:" - echo " kubectl cp /:${TRACE_FILE}" - exit 1 - fi - wait_for_model_ready - ROOT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - mkdir -p "$ROOT_DIR" - echo "==============================================" - echo "Trace Replay Benchmark (aiperf)" - echo "==============================================" - echo "Endpoint: http://${ENDPOINT}" - echo "Model: ${TARGET_MODEL}" - echo "Trace file: ${TRACE_FILE}" - echo "Concurrency: ${CONCURRENCY}" - echo "Artifact root:${ROOT_DIR}" - echo "==============================================" - # Warmup - WARMUP_DIR="${ROOT_DIR}/warmup" - mkdir -p "$WARMUP_DIR" - aiperf profile \ - -m "${TARGET_MODEL}" \ - --tokenizer "${TARGET_MODEL}" \ - --tokenizer-trust-remote-code \ - --url "http://${ENDPOINT}" \ - --streaming \ - --ui simple \ - --isl 8000 \ - --osl 1000 \ - --concurrency 1 \ - --request-count 5 \ - --artifact-dir "${WARMUP_DIR}" - echo "Warmup complete" - MODEL_BASE="${TARGET_MODEL##*/}" - TS=$(date +'%Y%m%d_%H%M%S') - RUN_DIR="${ROOT_DIR}/${MODEL_BASE}_trace_c${CONCURRENCY}_${TS}" - mkdir -p "$RUN_DIR" - aiperf profile \ - -m "${TARGET_MODEL}" \ - --tokenizer "${TARGET_MODEL}" \ - --tokenizer-trust-remote-code \ - --input-file "${TRACE_FILE}" \ - --custom-dataset-type mooncake_trace \ - --num-requests $(wc -l < "${TRACE_FILE}") \ - --prompt-input-tokens-block-size 512 \ - --url "http://${ENDPOINT}" \ - --streaming \ - --use-server-token-count \ - --extra-inputs ignore_eos:true \ - --concurrency "${CONCURRENCY}" \ - --random-seed 42 \ - --ui simple \ - --artifact-dir "${RUN_DIR}" \ - --request-timeout-seconds 1200 \ - --workers-max "${CONCURRENCY}" \ - --export-http-trace - echo "Concurrency ${CONCURRENCY} complete; artifacts in ${RUN_DIR}" - ls -la "${RUN_DIR}" || true - echo "" - echo "Done. Root: ${ROOT_DIR}" - env: - # --- Edit these to target the DGD + trace you want to benchmark --- - - name: ENDPOINT - value: kimi-k26-agg-b200-chat-frontend:8000 - - name: TRACE_FILE - value: /model-cache/traces/8k_1k_70kv_chat_new_noschedule.jsonl # use /model-cache/traces/64k_400_90kv_agent_new_noschedule.jsonl for agent - - name: CONCURRENCY - value: "24" - # --- The remainder is shared across all variants --- - - name: TARGET_MODEL - value: moonshotai/Kimi-K2.6 - - name: AIPERF_HTTP_CONNECTION_LIMIT - value: "200" - - name: AIPERF_HTTP_SO_RCVTIMEO - value: "120" - - name: AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT - value: "3600" - - name: AIPERF_DATASET_CONFIGURATION_TIMEOUT - value: "3600" - - name: JOB_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.labels['job-name'] - - name: ROOT_ARTIFACT_DIR - value: /model-cache/perf - - name: HF_HOME - value: /model-cache - - name: PYTHONUNBUFFERED - value: "1" - volumeMounts: - - name: model-cache - mountPath: /model-cache - restartPolicy: Never - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/kimi-k2.6/perf/traces/.gitattributes b/recipes/kimi-k2.6/perf/traces/.gitattributes deleted file mode 100644 index 817db20a164b..000000000000 --- a/recipes/kimi-k2.6/perf/traces/.gitattributes +++ /dev/null @@ -1,6 +0,0 @@ -64k_400_90kv_agent_new_noschedule.jsonl filter=lfs diff=lfs merge=lfs -text -64k_400_90kv_agent_new_noschedule_short_15perc.jsonl filter=lfs diff=lfs merge=lfs -text -64k_400_90kv_agent_new_noschedule_short_30perc.jsonl filter=lfs diff=lfs merge=lfs -text -8k_1k_70kv_chat_new_noschedule.jsonl filter=lfs diff=lfs merge=lfs -text -8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl filter=lfs diff=lfs merge=lfs -text -8k_1k_70kv_chat_new_noschedule_short_30perc.jsonl filter=lfs diff=lfs merge=lfs -text diff --git a/recipes/kimi-k2.6/perf/traces/64k_400_90kv_agent_new_noschedule.jsonl b/recipes/kimi-k2.6/perf/traces/64k_400_90kv_agent_new_noschedule.jsonl deleted file mode 100644 index aea03789f3d9..000000000000 --- a/recipes/kimi-k2.6/perf/traces/64k_400_90kv_agent_new_noschedule.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:aa0a64efcad575a01949e46e28dded094f20733db8708e0bc91b88ce9981fed6 -size 17623982 diff --git a/recipes/kimi-k2.6/perf/traces/64k_400_90kv_agent_new_noschedule_short_15perc.jsonl b/recipes/kimi-k2.6/perf/traces/64k_400_90kv_agent_new_noschedule_short_15perc.jsonl deleted file mode 100644 index 3fcfa9915c9e..000000000000 --- a/recipes/kimi-k2.6/perf/traces/64k_400_90kv_agent_new_noschedule_short_15perc.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f20d3f2bc83dd1306cda659fbe34e7c4d85ca5497626c98bc0b1c4d2211379d0 -size 2722326 diff --git a/recipes/kimi-k2.6/perf/traces/64k_400_90kv_agent_new_noschedule_short_30perc.jsonl b/recipes/kimi-k2.6/perf/traces/64k_400_90kv_agent_new_noschedule_short_30perc.jsonl deleted file mode 100644 index 1f5891b41524..000000000000 --- a/recipes/kimi-k2.6/perf/traces/64k_400_90kv_agent_new_noschedule_short_30perc.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:dae61f92fee05038f1dcb76474dba83c1b3c174f3d187589c7d0a16a7012eb79 -size 5401776 diff --git a/recipes/kimi-k2.6/perf/traces/8k_1k_70kv_chat_new_noschedule.jsonl b/recipes/kimi-k2.6/perf/traces/8k_1k_70kv_chat_new_noschedule.jsonl deleted file mode 100644 index a1ab5ad24a89..000000000000 --- a/recipes/kimi-k2.6/perf/traces/8k_1k_70kv_chat_new_noschedule.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5f369eb75ce639ad8b05cc209bb534bfedd627e9f7b923de32888155b4c9085a -size 5648304 diff --git a/recipes/kimi-k2.6/perf/traces/8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl b/recipes/kimi-k2.6/perf/traces/8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl deleted file mode 100644 index 1eea1bd8955c..000000000000 --- a/recipes/kimi-k2.6/perf/traces/8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b1221bca72b69f842897f339624306a84857f1b55ea0d866525f94d9ceb9b871 -size 894687 diff --git a/recipes/kimi-k2.6/perf/traces/8k_1k_70kv_chat_new_noschedule_short_30perc.jsonl b/recipes/kimi-k2.6/perf/traces/8k_1k_70kv_chat_new_noschedule_short_30perc.jsonl deleted file mode 100644 index 0ef335a203b3..000000000000 --- a/recipes/kimi-k2.6/perf/traces/8k_1k_70kv_chat_new_noschedule_short_30perc.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6de2a977b40dba8735a70aa5f9ab0d84e32530d7cf1fc9d24efc631c341f9c17 -size 1790696 diff --git a/recipes/kimi-k2.6/vllm/agg-b200-agentic/deploy.yaml b/recipes/kimi-k2.6/vllm/agg-b200-agentic/deploy.yaml deleted file mode 100644 index 22fdff8cfd8a..000000000000 --- a/recipes/kimi-k2.6/vllm/agg-b200-agentic/deploy.yaml +++ /dev/null @@ -1,210 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: v1 -kind: ConfigMap -metadata: - name: kimi-k26-agg-b200-agentic-specdec-config - labels: - app.kubernetes.io/name: kimi-k26-agg-b200-agentic - app.kubernetes.io/part-of: dynamo -data: - attention-config: |- - { - "backend": "FLASH_ATTN_MLA", - "mla_prefill_backend": "FLASH_ATTN_MLA", - "use_prefill_query_quantization": true - } - speculative-config-synthetic: |- - { - "method": "eagle3", - "model": "lightseekorg/kimi-k2.6-eagle3-mla", - "num_speculative_tokens": 3, - "rejection_sample_method": "synthetic", - "synthetic_acceptance_length": 2.49 - } - speculative-config: |- - { - "method": "eagle3", - "model": "lightseekorg/kimi-k2.6-eagle3-mla", - "num_speculative_tokens": 3 - } - kv-transfer-config: |- - { - "kv_connector": "LMCacheConnectorV1", - "kv_role": "kv_both", - "kv_connector_extra_config": { - "lmcache.enable_kv_events": false, - "lmcache.local_cpu": true, - "lmcache.max_local_cpu_size": 500, - "lmcache.pre_caching_hash_algorithm": "sha256_cbor", - "lmcache.extra_config": { - "save_only_first_rank": true - } - } - } - compilation-config: |- - { - "pass_config": { - "fuse_allreduce_rms": true, - "fuse_attn_quant": true - }, - "max_cudagraph_capture_size": 512 - } ---- -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: kimi-k26-agg-b200-agentic - labels: - app.kubernetes.io/name: kimi-k26-agg-b200-agentic - app.kubernetes.io/part-of: dynamo -spec: - backendFramework: vllm - components: - - name: Frontend - type: frontend - podTemplate: - spec: - containers: - - name: main - command: - - python3 - args: - - -m - - dingo.frontend - - --trust-remote-code - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: DYN_ROUTER_MODE - value: kv - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-kimi-k2.6-dev.1 - resources: {} - replicas: 1 - - - name: agg - type: worker - podTemplate: - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.product - operator: In - values: - - NVIDIA-B200 - containers: - - name: main - args: - - --model=nvidia/Kimi-K2.6-NVFP4 - - --served-model-name=moonshotai/Kimi-K2.6 - - --trust-remote-code - - --max-model-len=262144 - - --kv-cache-dtype=fp8 - - --gpu-memory-utilization=0.95 - - --enable-flashinfer-autotune - - --moe-backend=flashinfer_trtllm - - --mm-encoder-tp-mode=data - - --tensor-parallel-size=4 - # - --enable-expert-parallel - # - --all2all-backend deepep_low_latency - # - --all2all-backend flashinfer_all2allv - - --attention-config=$(ATTENTION_CONFIG) - - --compilation-config=$(COMPILATION_CONFIG) - - --speculative-config=$(SPECULATIVE_CONFIG) - - --kv-transfer-config=$(KV_TRANSFER_CONFIG) - - --enable-multimodal - - --dyn-tool-call-parser - - kimi_k2 - - --dyn-reasoning-parser - - kimi_k25 - # - --no-async-scheduling - command: - - python3 - - -m - - dingo.vllm - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: VLLM_CONFIG_ROOT - value: /tmp/vllm-config - - name: VLLM_CACHE_ROOT - value: /tmp/vllm-cache - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: ATTENTION_CONFIG - valueFrom: - configMapKeyRef: - name: kimi-k26-agg-b200-agentic-specdec-config - key: attention-config - - name: SPECULATIVE_CONFIG - valueFrom: - configMapKeyRef: - name: kimi-k26-agg-b200-agentic-specdec-config - key: speculative-config # speculative-config-synthetic for benchmarking - - name: KV_TRANSFER_CONFIG - valueFrom: - configMapKeyRef: - name: kimi-k26-agg-b200-agentic-specdec-config - key: kv-transfer-config - - name: COMPILATION_CONFIG - valueFrom: - configMapKeyRef: - name: kimi-k26-agg-b200-agentic-specdec-config - key: compilation-config - - name: NCCL_IB_DISABLE - value: "1" - # consistent LMCache hashes across ranks - - name: PYTHONHASHSEED - value: "42" - # NCCL symmetric AR backend with SHARP - - name: VLLM_USE_NCCL_SYMM_MEM - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - envFrom: - - secretRef: - name: hf-token-secret - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-kimi-k2.6-dev.1 - resources: - requests: - nvidia.com/gpu: "4" - memory: "750Gi" - limits: - nvidia.com/gpu: "4" - securityContext: - runAsUser: 0 - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 20 - volumeMounts: - - mountPath: /model-cache - name: model-cache - workingDir: /workspace/ - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - replicas: 1 diff --git a/recipes/kimi-k2.6/vllm/agg-b200-chat/deploy.yaml b/recipes/kimi-k2.6/vllm/agg-b200-chat/deploy.yaml deleted file mode 100644 index 28cc17ee58d6..000000000000 --- a/recipes/kimi-k2.6/vllm/agg-b200-chat/deploy.yaml +++ /dev/null @@ -1,210 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: v1 -kind: ConfigMap -metadata: - name: kimi-k26-agg-b200-chat-specdec-config - labels: - app.kubernetes.io/name: kimi-k26-agg-b200-chat - app.kubernetes.io/part-of: dynamo -data: - attention-config: |- - { - "backend": "FLASH_ATTN_MLA", - "mla_prefill_backend": "FLASH_ATTN_MLA", - "use_prefill_query_quantization": true - } - speculative-config-synthetic: |- - { - "method": "eagle3", - "model": "lightseekorg/kimi-k2.6-eagle3-mla", - "num_speculative_tokens": 3, - "rejection_sample_method": "synthetic", - "synthetic_acceptance_length": 2.49 - } - speculative-config: |- - { - "method": "eagle3", - "model": "lightseekorg/kimi-k2.6-eagle3-mla", - "num_speculative_tokens": 3 - } - kv-transfer-config: |- - { - "kv_connector": "LMCacheConnectorV1", - "kv_role": "kv_both", - "kv_connector_extra_config": { - "lmcache.enable_kv_events": false, - "lmcache.local_cpu": true, - "lmcache.max_local_cpu_size": 500, - "lmcache.pre_caching_hash_algorithm": "sha256_cbor", - "lmcache.extra_config": { - "save_only_first_rank": true - } - } - } - compilation-config: |- - { - "pass_config": { - "fuse_allreduce_rms": true, - "fuse_attn_quant": true - }, - "max_cudagraph_capture_size": 512 - } ---- -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: kimi-k26-agg-b200-chat - labels: - app.kubernetes.io/name: kimi-k26-agg-b200-chat - app.kubernetes.io/part-of: dynamo -spec: - backendFramework: vllm - components: - - name: Frontend - type: frontend - podTemplate: - spec: - containers: - - name: main - command: - - python3 - args: - - -m - - dingo.frontend - - --trust-remote-code - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: DYN_ROUTER_MODE - value: kv - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-kimi-k2.6-dev.1 - resources: {} - replicas: 1 - - - name: agg - type: worker - podTemplate: - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.product - operator: In - values: - - NVIDIA-B200 - containers: - - name: main - args: - - --model=nvidia/Kimi-K2.6-NVFP4 - - --served-model-name=moonshotai/Kimi-K2.6 - - --trust-remote-code - - --max-model-len=262144 - - --kv-cache-dtype=fp8 - - --gpu-memory-utilization=0.95 - - --enable-flashinfer-autotune - - --moe-backend=flashinfer_trtllm - - --mm-encoder-tp-mode=data - - --tensor-parallel-size=4 - # - --enable-expert-parallel - # - --all2all-backend deepep_low_latency - # - --all2all-backend flashinfer_all2allv - - --attention-config=$(ATTENTION_CONFIG) - - --compilation-config=$(COMPILATION_CONFIG) - - --speculative-config=$(SPECULATIVE_CONFIG) - - --kv-transfer-config=$(KV_TRANSFER_CONFIG) - - --enable-multimodal - - --dyn-tool-call-parser - - kimi_k2 - - --dyn-reasoning-parser - - kimi_k25 - # - --no-async-scheduling - command: - - python3 - - -m - - dingo.vllm - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: VLLM_CONFIG_ROOT - value: /tmp/vllm-config - - name: VLLM_CACHE_ROOT - value: /tmp/vllm-cache - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: ATTENTION_CONFIG - valueFrom: - configMapKeyRef: - name: kimi-k26-agg-b200-chat-specdec-config - key: attention-config - - name: SPECULATIVE_CONFIG - valueFrom: - configMapKeyRef: - name: kimi-k26-agg-b200-chat-specdec-config - key: speculative-config # speculative-config-synthetic for benchmarking - - name: KV_TRANSFER_CONFIG - valueFrom: - configMapKeyRef: - name: kimi-k26-agg-b200-chat-specdec-config - key: kv-transfer-config - - name: COMPILATION_CONFIG - valueFrom: - configMapKeyRef: - name: kimi-k26-agg-b200-chat-specdec-config - key: compilation-config - - name: NCCL_IB_DISABLE - value: "1" - # consistent LMCache hashes across ranks - - name: PYTHONHASHSEED - value: "42" - # NCCL symmetric AR backend with SHARP - - name: VLLM_USE_NCCL_SYMM_MEM - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - envFrom: - - secretRef: - name: hf-token-secret - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-kimi-k2.6-dev.1 - resources: - requests: - nvidia.com/gpu: "4" - memory: "750Gi" - limits: - nvidia.com/gpu: "4" - securityContext: - runAsUser: 0 - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 20 - volumeMounts: - - mountPath: /model-cache - name: model-cache - workingDir: /workspace/ - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - replicas: 1 diff --git a/recipes/kimi-k2.6/vllm/agg-h200-agentic/deploy.yaml b/recipes/kimi-k2.6/vllm/agg-h200-agentic/deploy.yaml deleted file mode 100644 index 43946e20ea69..000000000000 --- a/recipes/kimi-k2.6/vllm/agg-h200-agentic/deploy.yaml +++ /dev/null @@ -1,193 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: v1 -kind: ConfigMap -metadata: - name: kimi-k26-agg-h200-agentic-specdec-config - labels: - app.kubernetes.io/name: kimi-k26-agg-h200-agentic - app.kubernetes.io/part-of: dynamo -data: - attention-config: |- - { - } - speculative-config-synthetic: |- - { - "method": "eagle3", - "model": "lightseekorg/kimi-k2.6-eagle3-mla", - "num_speculative_tokens": 3, - "rejection_sample_method": "synthetic", - "synthetic_acceptance_length": 2.49 - } - speculative-config: |- - { - "method": "eagle3", - "model": "lightseekorg/kimi-k2.6-eagle3-mla", - "num_speculative_tokens": 3 - } - kv-transfer-config: |- - { - "kv_connector": "LMCacheConnectorV1", - "kv_role": "kv_both", - "kv_connector_extra_config": { - "lmcache.enable_kv_events": false, - "lmcache.local_cpu": true, - "lmcache.max_local_cpu_size": 500, - "lmcache.pre_caching_hash_algorithm": "sha256_cbor", - "lmcache.extra_config": { - "save_only_first_rank": true - } - } - } - compilation-config: |- - { - "pass_config": { - "fuse_allreduce_rms": true, - "fuse_attn_quant": true - }, - "max_cudagraph_capture_size": 512 - } ---- -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: kimi-k26-agg-h200-agentic - labels: - app.kubernetes.io/name: kimi-k26-agg-h200-agentic - app.kubernetes.io/part-of: dynamo -spec: - backendFramework: vllm - components: - - name: Frontend - type: frontend - podTemplate: - spec: - containers: - - name: main - command: - - python3 - args: - - -m - - dingo.frontend - - --trust-remote-code - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: DYN_ROUTER_MODE - value: kv - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-kimi-k2.6-dev.1 - resources: {} - replicas: 1 - - - name: agg - type: worker - podTemplate: - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.product - operator: In - values: - - NVIDIA-H200 - containers: - - name: main - args: - - --model=moonshotai/Kimi-K2.6 - - --served-model-name=moonshotai/Kimi-K2.6 - - --trust-remote-code - - --max-model-len=262144 - - --gpu-memory-utilization=0.85 - - --mm-encoder-tp-mode=data - - --tensor-parallel-size=8 - - --attention-config=$(ATTENTION_CONFIG) - - --compilation-config=$(COMPILATION_CONFIG) - - --speculative-config=$(SPECULATIVE_CONFIG) - - --kv-transfer-config=$(KV_TRANSFER_CONFIG) - - --enable-multimodal - - --dyn-tool-call-parser - - kimi_k2 - - --dyn-reasoning-parser - - kimi_k25 - # - --no-async-scheduling - command: - - python3 - - -m - - dingo.vllm - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: VLLM_CONFIG_ROOT - value: /tmp/vllm-config - - name: VLLM_CACHE_ROOT - value: /tmp/vllm-cache - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: ATTENTION_CONFIG - valueFrom: - configMapKeyRef: - name: kimi-k26-agg-h200-agentic-specdec-config - key: attention-config - - name: SPECULATIVE_CONFIG - valueFrom: - configMapKeyRef: - name: kimi-k26-agg-h200-agentic-specdec-config - key: speculative-config # speculative-config-synthetic for benchmarking - - name: KV_TRANSFER_CONFIG - valueFrom: - configMapKeyRef: - name: kimi-k26-agg-h200-agentic-specdec-config - key: kv-transfer-config - - name: COMPILATION_CONFIG - valueFrom: - configMapKeyRef: - name: kimi-k26-agg-h200-agentic-specdec-config - key: compilation-config - - name: NCCL_IB_DISABLE - value: "1" - - name: PYTHONHASHSEED - value: "42" - envFrom: - - secretRef: - name: hf-token-secret - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-kimi-k2.6-dev.1 - resources: - requests: - nvidia.com/gpu: "8" - memory: "750Gi" - limits: - nvidia.com/gpu: "8" - securityContext: - runAsUser: 0 - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 20 - volumeMounts: - - mountPath: /model-cache - name: model-cache - workingDir: /workspace/ - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - replicas: 1 diff --git a/recipes/kimi-k2.6/vllm/agg-h200-chat/deploy.yaml b/recipes/kimi-k2.6/vllm/agg-h200-chat/deploy.yaml deleted file mode 100644 index 0da14cb9b87d..000000000000 --- a/recipes/kimi-k2.6/vllm/agg-h200-chat/deploy.yaml +++ /dev/null @@ -1,193 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: v1 -kind: ConfigMap -metadata: - name: kimi-k26-agg-h200-chat-specdec-config - labels: - app.kubernetes.io/name: kimi-k26-agg-h200-chat - app.kubernetes.io/part-of: dynamo -data: - attention-config: |- - { - } - speculative-config-synthetic: |- - { - "method": "eagle3", - "model": "lightseekorg/kimi-k2.6-eagle3-mla", - "num_speculative_tokens": 3, - "rejection_sample_method": "synthetic", - "synthetic_acceptance_length": 2.49 - } - speculative-config: |- - { - "method": "eagle3", - "model": "lightseekorg/kimi-k2.6-eagle3-mla", - "num_speculative_tokens": 3 - } - kv-transfer-config: |- - { - "kv_connector": "LMCacheConnectorV1", - "kv_role": "kv_both", - "kv_connector_extra_config": { - "lmcache.enable_kv_events": false, - "lmcache.local_cpu": true, - "lmcache.max_local_cpu_size": 500, - "lmcache.pre_caching_hash_algorithm": "sha256_cbor", - "lmcache.extra_config": { - "save_only_first_rank": true - } - } - } - compilation-config: |- - { - "pass_config": { - "fuse_allreduce_rms": true, - "fuse_attn_quant": true - }, - "max_cudagraph_capture_size": 512 - } ---- -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: kimi-k26-agg-h200-chat - labels: - app.kubernetes.io/name: kimi-k26-agg-h200-chat - app.kubernetes.io/part-of: dynamo -spec: - backendFramework: vllm - components: - - name: Frontend - type: frontend - podTemplate: - spec: - containers: - - name: main - command: - - python3 - args: - - -m - - dingo.frontend - - --trust-remote-code - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: DYN_ROUTER_MODE - value: kv - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-kimi-k2.6-dev.1 - resources: {} - replicas: 1 - - - name: agg - type: worker - podTemplate: - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.product - operator: In - values: - - NVIDIA-H200 - containers: - - name: main - args: - - --model=moonshotai/Kimi-K2.6 - - --served-model-name=moonshotai/Kimi-K2.6 - - --trust-remote-code - - --max-model-len=262144 - - --gpu-memory-utilization=0.85 - - --mm-encoder-tp-mode=data - - --tensor-parallel-size=8 - - --attention-config=$(ATTENTION_CONFIG) - - --compilation-config=$(COMPILATION_CONFIG) - - --speculative-config=$(SPECULATIVE_CONFIG) - - --kv-transfer-config=$(KV_TRANSFER_CONFIG) - - --enable-multimodal - - --dyn-tool-call-parser - - kimi_k2 - - --dyn-reasoning-parser - - kimi_k25 - # - --no-async-scheduling - command: - - python3 - - -m - - dingo.vllm - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: VLLM_CONFIG_ROOT - value: /tmp/vllm-config - - name: VLLM_CACHE_ROOT - value: /tmp/vllm-cache - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: ATTENTION_CONFIG - valueFrom: - configMapKeyRef: - name: kimi-k26-agg-h200-chat-specdec-config - key: attention-config - - name: SPECULATIVE_CONFIG - valueFrom: - configMapKeyRef: - name: kimi-k26-agg-h200-chat-specdec-config - key: speculative-config # speculative-config-synthetic for benchmarking - - name: KV_TRANSFER_CONFIG - valueFrom: - configMapKeyRef: - name: kimi-k26-agg-h200-chat-specdec-config - key: kv-transfer-config - - name: COMPILATION_CONFIG - valueFrom: - configMapKeyRef: - name: kimi-k26-agg-h200-chat-specdec-config - key: compilation-config - - name: NCCL_IB_DISABLE - value: "1" - - name: PYTHONHASHSEED - value: "42" - envFrom: - - secretRef: - name: hf-token-secret - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-kimi-k2.6-dev.1 - resources: - requests: - nvidia.com/gpu: "8" - memory: "750Gi" - limits: - nvidia.com/gpu: "8" - securityContext: - runAsUser: 0 - startupProbe: - failureThreshold: 60 - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 20 - volumeMounts: - - mountPath: /model-cache - name: model-cache - workingDir: /workspace/ - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - replicas: 1 diff --git a/recipes/llama-3-70b/README.md b/recipes/llama-3-70b/README.md deleted file mode 100644 index 354f38d0f44c..000000000000 --- a/recipes/llama-3-70b/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# Llama-3.3-70B Recipes - -Production-ready deployments for **Llama-3.3-70B-Instruct** using vLLM with FP8 dynamic quantization. - -## Available Configurations - -| Configuration | GPUs | Mode | Description | -|--------------|------|------|-------------| -| [**vllm/agg**](vllm/agg/) | 4x H100/H200 | Aggregated | Single-node, TP4 | -| [**vllm/disagg-single-node**](vllm/disagg-single-node/) | 8x H100/H200 | Disaggregated | Prefill/decode separation on one node | -| [**vllm/disagg-multi-node**](vllm/disagg-multi-node/) | 16x H100/H200 | Disaggregated | 2 nodes, 8 GPUs each | - -## Prerequisites - -1. **Dynamo Platform installed** — See [Kubernetes Deployment Guide](../../docs/kubernetes/README.md) -2. **GPU cluster** with H100 or H200 GPUs matching the configuration requirements -3. **HuggingFace token** with access to Llama models - -## Quick Start - -```bash -# Set namespace -export NAMESPACE=dynamo-demo -kubectl create namespace ${NAMESPACE} - -# Create HuggingFace token secret -kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="your-token-here" \ - -n ${NAMESPACE} - -# Download model (update storageClassName in model-cache.yaml first!) -kubectl apply -f model-cache/ -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/model-download -n ${NAMESPACE} --timeout=3600s - -# Deploy (choose one configuration) -kubectl apply -f vllm/agg/deploy.yaml -n ${NAMESPACE} -# OR: kubectl apply -f vllm/disagg-single-node/deploy.yaml -n ${NAMESPACE} -# OR: kubectl apply -f vllm/disagg-multi-node/deploy.yaml -n ${NAMESPACE} -``` - -## Test the Deployment - -```bash -# Port-forward the frontend -kubectl port-forward svc/llama3-70b-agg-frontend 8000:8000 -n ${NAMESPACE} - -# Send a test request -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 50 - }' -``` - -## Model Details - -- **Model**: `RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic` -- **Quantization**: FP8 dynamic (applied at runtime) -- **Context length**: Default model context - -## Notes - -- Update `storageClassName` in `model-cache/model-cache.yaml` to match your cluster before deploying -- Model download takes approximately 15-30 minutes depending on network speed -- For GAIE (Gateway API Inference Extension) integration, `kubectl apply` the files from the corresponding subfolder i.e. [vllm/agg/gaie/](vllm/agg/gaie/) diff --git a/recipes/llama-3-70b/model-cache/model-cache.yaml b/recipes/llama-3-70b/model-cache/model-cache.yaml deleted file mode 100644 index 9dfc23d5b454..000000000000 --- a/recipes/llama-3-70b/model-cache/model-cache.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 100Gi - storageClassName: "your-storage-class-name" \ No newline at end of file diff --git a/recipes/llama-3-70b/model-cache/model-download.yaml b/recipes/llama-3-70b/model-cache/model-download.yaml deleted file mode 100644 index 8bc95238603e..000000000000 --- a/recipes/llama-3-70b/model-cache/model-download.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: MODEL_NAME - value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic" - - name: HF_HOME - value: /model-store - # Uses up to 64 GB RAM for XET buffers; set "0" on low-memory nodes (8 GB cap): https://huggingface.co/docs/hub/en/xet/using-xet-storage#download-buffers - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - - name: MODEL_REVISION - value: ddb4128556dfcff99e0c41aee159ea6c3e655dcd - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.11.0 - hf download $MODEL_NAME --revision $MODEL_REVISION - resources: - requests: - cpu: "2" - memory: "64Gi" - limits: - cpu: "8" - memory: "64Gi" - volumeMounts: - - name: model-cache - mountPath: /model-store - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/llama-3-70b/vllm/agg/deploy.yaml b/recipes/llama-3-70b/vllm/agg/deploy.yaml deleted file mode 100644 index a1678657593a..000000000000 --- a/recipes/llama-3-70b/vllm/agg/deploy.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: llama3-70b-agg -spec: - backendFramework: vllm - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - volumeMounts: - - name: model-cache - mountPoint: /opt/models - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - workingDir: /workspace/examples/backends/vllm - envs: - - name: HF_HOME - value: /opt/models - replicas: 1 - VllmPrefillWorker: - componentType: worker - envFromSecret: hf-token-secret - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 20Gi - extraPodSpec: - mainContainer: - env: - - name: SERVED_MODEL_NAME - value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic" - - name: MODEL_PATH - value: "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic" - - name: HF_HOME - value: /opt/models - args: - - "python3 -m dingo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 4 --data-parallel-size 1 --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128" - command: - - /bin/sh - - -c - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - workingDir: /workspace/examples/backends/vllm - replicas: 1 - resources: - limits: - gpu: "4" - requests: - gpu: "4" \ No newline at end of file diff --git a/recipes/llama-3-70b/vllm/agg/gaie/deploy.yaml b/recipes/llama-3-70b/vllm/agg/gaie/deploy.yaml deleted file mode 100644 index 06b7d77519b1..000000000000 --- a/recipes/llama-3-70b/vllm/agg/gaie/deploy.yaml +++ /dev/null @@ -1,112 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: llama3-70b-agg -spec: - backendFramework: vllm - components: - - name: Epp - type: epp - replicas: 1 - eppConfig: - # This config uses the same disagg-profile-handler as disaggregated deployments. - # The handler's graceful degradation feature makes this possible: - # - With no "prefill" profile defined, it runs only the "decode" profile. - # - The decode scorer receives isDisaggregated=false, so the Dynamo KV router - # uses full overlap scoring (overlap_score_credit=1.0) for aggregated mode. - # - If prefill workers were added later (and a prefill profile configured), - # the same handler would automatically switch to disaggregated routing. - config: - plugins: - - type: disagg-profile-handler - - name: decode-filter - type: label-filter - # allowsNoLabel: true lets pods without decode role labels pass through. - parameters: - label: nvidia.com/dynamo-component-type - validValues: - - decode - allowsNoLabel: true - - name: picker - type: max-score-picker - - name: dyn-decode - type: dyn-decode-scorer - # Only a "decode" profile — no "prefill" profile means pure aggregated mode. - schedulingProfiles: - - name: decode - plugins: - - pluginRef: decode-filter - weight: 1 - - pluginRef: dyn-decode - weight: 1 - - pluginRef: picker - weight: 1 - podTemplate: - spec: - containers: - - name: main - env: - - name: DYN_KV_CACHE_BLOCK_SIZE - value: "128" - - name: DYN_MODEL_NAME - value: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic - - name: DYN_DECODE_FALLBACK - value: "true" - envFrom: - - secretRef: - name: hf-token-secret - image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.2.1 - - name: VllmDecodeWorker - type: decode - replicas: 1 - sharedMemorySize: 20Gi - frontendSidecar: sidecar-frontend - podTemplate: - spec: - containers: - - name: main - args: - - python3 -m dingo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME - --tensor-parallel-size 4 --data-parallel-size 1 --gpu-memory-utilization - 0.90 --no-enable-prefix-caching --block-size 128 - command: - - /bin/sh - - -c - env: - - name: SERVED_MODEL_NAME - value: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic - - name: MODEL_PATH - value: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic - - name: HF_HOME - value: /opt/models - - name: DYN_STORE_KV - value: mem - envFrom: - - secretRef: - name: hf-token-secret - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - resources: - limits: - nvidia.com/gpu: "4" - requests: - nvidia.com/gpu: "4" - volumeMounts: - - mountPath: /opt/models - name: model-cache - workingDir: /workspace/examples/backends/vllm - - name: sidecar-frontend - args: - - -m - - dingo.frontend - - --router-mode - - direct - envFrom: - - secretRef: - name: hf-token-secret - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/llama-3-70b/vllm/agg/gaie/http-route.yaml b/recipes/llama-3-70b/vllm/agg/gaie/http-route.yaml deleted file mode 100644 index 2fb77f52eed7..000000000000 --- a/recipes/llama-3-70b/vllm/agg/gaie/http-route.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: llama3-70b-agg-route -spec: - hostnames: - - llama3-70b-agg.example.com - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: inference-gateway - rules: - - backendRefs: - - group: inference.networking.k8s.io - kind: InferencePool - name: llama3-70b-agg-pool - port: 8000 - weight: 1 - matches: - - path: - type: PathPrefix - value: / - timeouts: - request: 300s diff --git a/recipes/llama-3-70b/vllm/agg/perf.yaml b/recipes/llama-3-70b/vllm/agg/perf.yaml deleted file mode 100644 index 974eca17e543..000000000000 --- a/recipes/llama-3-70b/vllm/agg/perf.yaml +++ /dev/null @@ -1,144 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: llama3-70b-agg-perf -spec: - backoffLimit: 1 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: llama3-70b-agg-perf - spec: - restartPolicy: Never - containers: - - command: - - /bin/sh - - -c - - | - apt-get update && apt-get install -y curl jq procps git && apt-get clean - pip install "aiperf==0.10.0"; - echo "aiperf installation completed"; - sysctl -w net.ipv4.ip_local_port_range="1024 65000" - cat /proc/sys/net/ipv4/ip_local_port_range - export COLUMNS=200 - EPOCH=$(date +%s) - ## utility functions -- can be moved to a bash script / configmap - wait_for_model_ready() { - echo "Waiting for model '$TARGET_MODEL' at $ENDPOINT/v1/models (checking every 5s)..." - while ! curl -s "http://$ENDPOINT/v1/models" | jq -e --arg model "$TARGET_MODEL" '.data[]? | select(.id == $model)' >/dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready yet, sleeping 5s before checking again http://$ENDPOINT/v1/models" - sleep 5 - done - echo "✅ Model '$TARGET_MODEL' is now available!" - echo "Model '$TARGET_MODEL' is now available!" - curl -s "http://$ENDPOINT/v1/models" | jq . - } - run_perf() { - local concurrency=$1 - local isl=$2 - local osl=$3 - local max_threads=${concurrency} - key=concurrency_${concurrency} - export ARTIFACT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/${key}" - mkdir -p "$ARTIFACT_DIR" - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - aiperf profile --artifact-dir $ARTIFACT_DIR \ - --model $TARGET_MODEL \ - --tokenizer /root/.cache/huggingface/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd \ - --endpoint-type chat \ - --endpoint /v1/chat/completions \ - --streaming \ - --url http://$ENDPOINT \ - --synthetic-input-tokens-mean $isl \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean $osl \ - --output-tokens-stddev 0 \ - --extra-inputs max_tokens:$osl \ - --extra-inputs min_tokens:$osl \ - --extra-inputs ignore_eos:true \ - --extra-inputs repetition_penalty:1.0 \ - --extra-inputs temperature:0.0 \ - --concurrency $concurrency \ - --request-count $((10*concurrency)) \ - --warmup-request-count $concurrency \ - --num-dataset-entries 12800 \ - --random-seed 100 \ - --workers-max $max_threads \ - -H 'Authorization: Bearer NOT USED' \ - -H 'Accept: text/event-stream'\ - --record-processors 32 \ - --ui simple - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - ls -la $ARTIFACT_DIR - } - #### Actual execution #### - wait_for_model_ready - mkdir -p "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - # Calculate total concurrency based on per-GPU concurrency and GPU count - TOTAL_CONCURRENCY=$((CONCURRENCY_PER_GPU * DEPLOYMENT_GPU_COUNT)) - echo "Calculated total concurrency: $TOTAL_CONCURRENCY (${CONCURRENCY_PER_GPU} per GPU × ${DEPLOYMENT_GPU_COUNT} GPUs)" - # Write input_config.json - cat > "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/input_config.json" </dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready yet, sleeping 5s before checking again http://$ENDPOINT/v1/models" - sleep 5 - done - echo "✅ Model '$TARGET_MODEL' is now available!" - echo "Model '$TARGET_MODEL' is now available!" - curl -s "http://$ENDPOINT/v1/models" | jq . - } - run_perf() { - local concurrency=$1 - local isl=$2 - local osl=$3 - local max_threads=${concurrency} - key=concurrency_${concurrency} - export ARTIFACT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/${key}" - mkdir -p "$ARTIFACT_DIR" - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - aiperf profile --artifact-dir $ARTIFACT_DIR \ - --model $TARGET_MODEL \ - --tokenizer /root/.cache/huggingface/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd \ - --endpoint-type chat \ - --endpoint /v1/chat/completions \ - --streaming \ - --url http://$ENDPOINT \ - --synthetic-input-tokens-mean $isl \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean $osl \ - --output-tokens-stddev 0 \ - --extra-inputs max_tokens:$osl \ - --extra-inputs min_tokens:$osl \ - --extra-inputs ignore_eos:true \ - --extra-inputs repetition_penalty:1.0 \ - --extra-inputs temperature:0.0 \ - --concurrency $concurrency \ - --request-count $((10*concurrency)) \ - --warmup-request-count $concurrency \ - --num-dataset-entries 12800 \ - --random-seed 100 \ - --workers-max $max_threads \ - -H 'Authorization: Bearer NOT USED' \ - -H 'Accept: text/event-stream'\ - --record-processors 32 \ - --ui simple - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - ls -la $ARTIFACT_DIR - } - #### Actual execution #### - wait_for_model_ready - mkdir -p "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - # Calculate total concurrency based on per-GPU concurrency and GPU count - TOTAL_CONCURRENCY=$((CONCURRENCY_PER_GPU * DEPLOYMENT_GPU_COUNT)) - echo "Calculated total concurrency: $TOTAL_CONCURRENCY (${CONCURRENCY_PER_GPU} per GPU × ${DEPLOYMENT_GPU_COUNT} GPUs)" - # Write input_config.json - cat > "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/input_config.json" </dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready yet, sleeping 5s before checking again http://$ENDPOINT/v1/models" - sleep 5 - done - echo "✅ Model '$TARGET_MODEL' is now available!" - echo "Model '$TARGET_MODEL' is now available!" - curl -s "http://$ENDPOINT/v1/models" | jq . - } - run_perf() { - local concurrency=$1 - local isl=$2 - local osl=$3 - local max_threads=${concurrency} - key=concurrency_${concurrency} - export ARTIFACT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/${key}" - mkdir -p "$ARTIFACT_DIR" - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - aiperf profile --artifact-dir $ARTIFACT_DIR \ - --model $TARGET_MODEL \ - --tokenizer /root/.cache/huggingface/hub/models--RedHatAI--Llama-3.3-70B-Instruct-FP8-dynamic/snapshots/ddb4128556dfcff99e0c41aee159ea6c3e655dcd \ - --endpoint-type chat \ - --endpoint /v1/chat/completions \ - --streaming \ - --url http://$ENDPOINT \ - --synthetic-input-tokens-mean $isl \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean $osl \ - --output-tokens-stddev 0 \ - --extra-inputs max_tokens:$osl \ - --extra-inputs min_tokens:$osl \ - --extra-inputs ignore_eos:true \ - --extra-inputs repetition_penalty:1.0 \ - --extra-inputs temperature:0.0 \ - --concurrency $concurrency \ - --request-count $((10*concurrency)) \ - --warmup-request-count $concurrency \ - --num-dataset-entries 12800 \ - --random-seed 100 \ - --workers-max $max_threads \ - -H 'Authorization: Bearer NOT USED' \ - -H 'Accept: text/event-stream'\ - --record-processors 32 \ - --ui simple - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - ls -la $ARTIFACT_DIR - } - #### Actual execution #### - wait_for_model_ready - mkdir -p "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - # Calculate total concurrency based on per-GPU concurrency and GPU count - TOTAL_CONCURRENCY=$((CONCURRENCY_PER_GPU * DEPLOYMENT_GPU_COUNT)) - echo "Calculated total concurrency: $TOTAL_CONCURRENCY (${CONCURRENCY_PER_GPU} per GPU × ${DEPLOYMENT_GPU_COUNT} GPUs)" - # Write input_config.json - cat > "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/input_config.json" </nemotron-omni-vllm:latest \ -# -f recipes/nemotron-3-nano-omni/Dockerfile \ -# recipes/nemotron-3-nano-omni -# -# Override defaults with --build-arg, e.g.: -# --build-arg BASE_IMAGE=vllm/vllm-openai:v0.20.0 -# --build-arg DYNAMO_VERSION=1.2.0.dev20260427 - -ARG BASE_IMAGE="vllm/vllm-openai:v0.20.0" -ARG DYNAMO_VERSION="1.2.0.dev20260427" - -FROM ${BASE_IMAGE} -USER root - -ARG DYNAMO_VERSION - -# ai-dynamo Python package (dingo.frontend, dingo.vllm, ...) installed with -# --no-deps so that: -# - the base image's vLLM 0.20 / torch / cuda stack is preserved (otherwise -# pip would try to re-install vllm[flashinfer,otel,runai]==0.20.0 which is -# declared by ai-dynamo[vllm]), and -# - we skip `nixl` (KV transport, only used for disagg) and `ray` (multi-node -# orchestration), neither of which an aggregated single-GPU recipe needs. -RUN pip install --no-cache-dir --no-deps \ - --extra-index-url https://pypi.nvidia.com \ - ai-dynamo==${DYNAMO_VERSION} - -# ai-dynamo's declared core dependencies, plus the Rust _core extension wheel -# (`ai-dynamo-runtime`) that the Python package imports natively. `kubernetes`, -# `pydantic`, and `pydantic-settings` are explicitly pinned because the latest -# unpinned versions (kubernetes 35.x, pydantic 2.13.x) violate ai-dynamo's -# declared compatibility ranges. -RUN pip install --no-cache-dir \ - --extra-index-url https://pypi.nvidia.com \ - ai-dynamo-runtime==${DYNAMO_VERSION} \ - "kubernetes<33.0.0,>=32.0.1" \ - "pydantic<2.13" "pydantic-settings<2.13.0" \ - msgpack msgspec prometheus-client pyzmq transformers - -# ai-dynamo[vllm] extras minus nixl + ray (handled above), plus the -# multimodal-video Python deps (av/ftfy/nvtx/sentencepiece) that the Nemotron -# Nano Omni model needs at runtime but which aren't pulled in by any extra. -RUN pip install --no-cache-dir \ - blake3 librosa soundfile uvloop \ - av ftfy nvtx sentencepiece - -# vllm/vllm-openai's default ENTRYPOINT runs `vllm serve`; reset it so the -# image behaves as a plain dynamo runtime image. -ENTRYPOINT ["/bin/bash"] diff --git a/recipes/nemotron-3-nano-omni/README.md b/recipes/nemotron-3-nano-omni/README.md deleted file mode 100644 index 48b91b064c14..000000000000 --- a/recipes/nemotron-3-nano-omni/README.md +++ /dev/null @@ -1,185 +0,0 @@ - - -# Nemotron 3 Nano Omni NVFP4 - -Serves [nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4](https://huggingface.co/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4) -using vLLM with an aggregated Dynamo deployment. - -This recipe builds a custom container that layers the `ai-dynamo` wheel -(from ) onto an upstream vLLM image — no -source build, no Rust toolchain. - -## Topology - -| Role | Replicas | GPUs/replica | Notes | -|------|----------|--------------|-------| -| Frontend | 1 | 0 | Dynamo frontend with prefix-hash KV routing | -| vLLM worker | 1 | 1 | Text, image, video, and audio inputs | - -## Prerequisites - -- A Kubernetes cluster with the [Dynamo Operator](../../docs/kubernetes/README.md) installed -- One NVIDIA GPU per worker replica -- Shared PVC storage for the Hugging Face model cache -- Hugging Face access to `nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4` - -## Step 1: Build the Container - -```bash -docker build \ - -t /nemotron-omni-vllm:latest \ - -f recipes/nemotron-3-nano-omni/Dockerfile \ - recipes/nemotron-3-nano-omni -docker push /nemotron-omni-vllm:latest -``` - -Useful build args: - -- `BASE_IMAGE=` — pin to a different vLLM base (default `vllm/vllm-openai:v0.20.0`). -- `DYNAMO_VERSION=` — pin to a specific `ai-dynamo` release or nightly from . Default tracks the latest tested nightly. Make sure the chosen wheel's `vllm` dependency matches `BASE_IMAGE`. - -## Step 2: Download the Model - -Create the PVC, Hugging Face token secret, and download the model weights: - -```bash -export NAMESPACE= - -# Create the namespace if it does not already exist. -kubectl create namespace ${NAMESPACE} --dry-run=client -o yaml | kubectl apply -f - - -# First edit storageClassName in model-cache.yaml for your cluster. -kubectl apply -f recipes/nemotron-3-nano-omni/model-cache/model-cache.yaml -n ${NAMESPACE} - -kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN= \ - -n ${NAMESPACE} - -kubectl apply -f recipes/nemotron-3-nano-omni/model-cache/model-download.yaml -n ${NAMESPACE} -kubectl wait --for=condition=complete job/model-download -n ${NAMESPACE} --timeout=3600s -``` - -## Step 3: Deploy - -Edit `vllm/agg/deploy.yaml` and replace all `` values: - -- `/nemotron-omni-vllm:latest` - your built container image - -If your registry is private, add the appropriate `imagePullSecrets` to the -deployment. - -```bash -kubectl apply -f recipes/nemotron-3-nano-omni/vllm/agg/deploy.yaml -n ${NAMESPACE} -``` - -Monitor startup: - -```bash -kubectl get pods -n ${NAMESPACE} -l nvidia.com/dynamo-graph-deployment-name=nemotron-omni-vllm-agg -w -``` - -## Step 4: Test - -```bash -kubectl port-forward svc/nemotron-omni-vllm-agg-frontend 8000:8000 -n ${NAMESPACE} -``` - -In another terminal, send a minimal text request: - -```bash -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 128 - }' -``` - -To exercise the multimodal path, attach an image: - -```bash -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", - "messages": [{ - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/inpaint.png"}}, - {"type": "text", "text": "Describe what is in this image."} - ] - }], - "max_tokens": 256 - }' -``` - -…or an audio clip: - -```bash -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", - "messages": [{ - "role": "user", - "content": [ - {"type": "audio_url", "audio_url": {"url": "https://raw.githubusercontent.com/yuekaizhang/Triton-ASR-Client/main/datasets/mini_en/wav/1221-135766-0002.wav"}}, - {"type": "text", "text": "Transcribe this audio clip."} - ] - }], - "max_tokens": 256 - }' -``` - -## Key Configuration Notes - -- `--enable-multimodal` enables image, video, and audio inputs. -- `--media-io-kwargs '{"video": {"num_frames": 512, "fps": 1}}'` samples long - videos at one frame per second, capped at 512 frames. -- `--dyn-tool-call-parser nemotron_nano` and - `--dyn-reasoning-parser nemotron_nano` enable Nemotron Nano tool-call and - reasoning parsing. -- The frontend uses `--router-mode kv --no-kv-events`, which approximates - KV-aware routing with prefix hashing without requiring backend KV events. - -## Optional: Run without NATS - -The Dynamo runtime defaults to NATS for the event plane and connects to a -NATS server if `NATS_SERVER` is set in the environment (the operator -auto-injects this on most clusters). On clusters without NATS — or where -you'd rather avoid the dependency — you can run on TCP request plane + ZMQ -event plane only. Add to both Frontend and VllmWorker: - -```yaml -mainContainer: - env: - - name: DYN_EVENT_PLANE - value: zmq - command: ["/bin/bash", "-lc"] - args: - # Operator-injected NATS_SERVER takes effect even when set to ""; we have - # to actually unset it before the runtime reads env. - - >- - unset NATS_SERVER && - exec python3 -m dingo.frontend ... # or dingo.vllm -``` - -The request plane defaults to TCP already, so no further flags are needed. - -## File Layout - -```text -recipes/nemotron-3-nano-omni/ - README.md - Dockerfile - model-cache/ - model-cache.yaml - model-download.yaml - vllm/ - agg/ - deploy.yaml -``` \ No newline at end of file diff --git a/recipes/nemotron-3-nano-omni/model-cache/model-cache.yaml b/recipes/nemotron-3-nano-omni/model-cache/model-cache.yaml deleted file mode 100644 index 5d6e2b6e998b..000000000000 --- a/recipes/nemotron-3-nano-omni/model-cache/model-cache.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 250Gi - storageClassName: "your-storage-class-name" diff --git a/recipes/nemotron-3-nano-omni/model-cache/model-download.yaml b/recipes/nemotron-3-nano-omni/model-cache/model-download.yaml deleted file mode 100644 index 5b11cd1f7a8d..000000000000 --- a/recipes/nemotron-3-nano-omni/model-cache/model-download.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - seccompProfile: - type: RuntimeDefault - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: MODEL_NAME - value: nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4 - - name: HF_HOME - value: /model-store - # Uses up to 64 GB RAM for XET buffers; set "0" on low-memory nodes (8 GB cap): https://huggingface.co/docs/hub/en/xet/using-xet-storage#download-buffers - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.11.0 - hf download "$MODEL_NAME" - resources: - requests: - cpu: "2" - memory: "64Gi" - limits: - cpu: "8" - memory: "64Gi" - volumeMounts: - - name: model-cache - mountPath: /model-store - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/nemotron-3-nano-omni/vllm/agg/deploy.yaml b/recipes/nemotron-3-nano-omni/vllm/agg/deploy.yaml deleted file mode 100644 index 0d63d91cff3f..000000000000 --- a/recipes/nemotron-3-nano-omni/vllm/agg/deploy.yaml +++ /dev/null @@ -1,102 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# Nemotron Nano Omni aggregated vLLM deployment. -# -# Prerequisites: -# - Dynamo Platform installed -# - Model weights downloaded into the model-cache PVC -# - Container built from recipes/nemotron-3-nano-omni/Dockerfile -# - HF_TOKEN secret created: -# kubectl create secret generic hf-token-secret \ -# --from-literal=HF_TOKEN= -n -# -# Replace image references before applying: -# /nemotron-omni-vllm:latest -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: nemotron-omni-vllm-agg -spec: - backendFramework: vllm - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - envFromSecret: hf-token-secret - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /model-store - extraPodSpec: - mainContainer: - image: /nemotron-omni-vllm:latest - imagePullPolicy: IfNotPresent - startupProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 10 - timeoutSeconds: 1800 - failureThreshold: 60 - env: - - name: HF_HOME - value: /model-store - command: - - /bin/bash - - -lc - args: - - >- - exec python3 -m dingo.frontend - --router-mode kv - --no-kv-events - --http-port 8000 - - VllmWorker: - componentType: worker - envFromSecret: hf-token-secret - replicas: 1 - resources: - limits: - gpu: "1" - requests: - gpu: "1" - volumeMounts: - - name: model-cache - mountPoint: /model-store - sharedMemory: - size: 16Gi - extraPodSpec: - mainContainer: - image: /nemotron-omni-vllm:latest - imagePullPolicy: IfNotPresent - startupProbe: - httpGet: - path: /health - port: 9090 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 120 - env: - - name: HF_HOME - value: /model-store - # Match the --media-io-kwargs num_frames so dynamo's multimodal - # preprocessor and vLLM agree on the video frame ceiling. - - name: DYN_MM_VIDEO_NUM_FRAMES - value: "512" - command: - - /bin/bash - - -lc - args: - - >- - exec python3 -m dingo.vllm - --model nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4 - --served-model-name nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4 - --max-model-len 131072 - --enable-multimodal - --media-io-kwargs '{"video": {"num_frames": 512, "fps": 1}}' - --trust-remote-code - --video-pruning-rate 0.5 - --dyn-tool-call-parser nemotron_nano - --dyn-reasoning-parser nemotron_nano diff --git a/recipes/nemotron-3-super-fp8/README.md b/recipes/nemotron-3-super-fp8/README.md deleted file mode 100644 index 5711d001fcea..000000000000 --- a/recipes/nemotron-3-super-fp8/README.md +++ /dev/null @@ -1,164 +0,0 @@ - - -# Nemotron-3-Super FP8 Recipes - -Functional deployments for **nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8** (~124B hybrid Mamba/Attention/MoE) across multiple backends. - -These recipes target **Dynamo 1.0**. See [Dynamo 0.9.1 Compatibility](#dynamo-091-compatibility) for notes on running with older containers. - -## Available Configurations - -| Configuration | GPUs | Backend | Mode | Description | -|--------------|------|---------|------|-------------| -| [**vllm/agg**](vllm/agg/) | 4x H100/H200 | vLLM | Aggregated | TP=4, KV-aware routing | -| [**sglang/agg**](sglang/agg/) | 4x H100/H200 | SGLang | Aggregated | TP=4, KV-aware routing (not working on 0.9.1) | -| [**trtllm/disagg**](trtllm/disagg/) | 4x H100/H200 | TensorRT-LLM | Disaggregated | TP=2 P/D split, UCX KV transfer | -| [**sglang/disagg**](sglang/disagg/) | 4x H100/H200 | SGLang | Disaggregated | TP=2 P/D split, nixl KV transfer (not working on 0.9.1) | - -## Prerequisites - -1. **Dynamo Platform installed** -- See [Kubernetes Deployment Guide](../../docs/kubernetes/README.md) -2. **GPU cluster** with 4x H100 80GB (or H200) GPUs -3. **HuggingFace token** with access to NVIDIA models - -## Quick Start - -```bash -# Set namespace -export NAMESPACE=dynamo-demo -kubectl create namespace ${NAMESPACE} - -# Create HuggingFace token secret -kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="your-token-here" \ - -n ${NAMESPACE} - -# Download model (update storageClassName in model-cache.yaml first!) -kubectl apply -f model-cache/ -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/model-download -n ${NAMESPACE} --timeout=3600s - -# Deploy (choose one configuration) -kubectl apply -f vllm/agg/deploy.yaml -n ${NAMESPACE} -# OR: kubectl apply -f trtllm/disagg/deploy.yaml -n ${NAMESPACE} -# OR: kubectl apply -f sglang/agg/deploy.yaml -n ${NAMESPACE} -# OR: kubectl apply -f sglang/disagg/deploy.yaml -n ${NAMESPACE} -``` - -## Test the Deployment - -```bash -# Port-forward the frontend -# If deployed vllm/agg: -kubectl port-forward svc/nemotron-super-fp8-vllm-agg-frontend 8000:8000 -n ${NAMESPACE} -# If deployed trtllm/disagg: -# kubectl port-forward svc/nemotron-super-fp8-trtllm-disagg-frontend 8000:8000 -n ${NAMESPACE} -# If deployed sglang/agg: -# kubectl port-forward svc/nemotron-super-fp8-sglang-agg-frontend 8000:8000 -n ${NAMESPACE} -# If deployed sglang/disagg: -# kubectl port-forward svc/nemotron-super-fp8-sglang-disagg-frontend 8000:8000 -n ${NAMESPACE} - -# Basic chat (with reasoning) -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100 - }' - -# Tool calling -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8", - "messages": [{"role": "user", "content": "What is the weather in SF?"}], - "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}}}], - "max_tokens": 256 - }' - -# Disable thinking (only works with nemotron_nano reasoning parser in 1.0+) -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8", - "messages": [{"role": "user", "content": "What is 2+2?"}], - "chat_template_kwargs": {"enable_thinking": false}, - "max_tokens": 64 - }' -``` - -## Model Details - -- **Model**: `nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8` -- **Architecture**: Nemotron-H (hybrid Mamba/Attention/MoE, 88 layers) -- **Parameters**: ~124B total (~119B FP8, ~4.7B BF16) -- **Quantization**: ModelOpt FP8 (F8_E4M3) with FP8 KV cache - -## Parser Configuration - -All recipes include tool call and reasoning parsers: - -- `--dyn-reasoning-parser nemotron_nano` -- Extracts `...` into `reasoning_content`. Correctly handles both `enable_thinking: true` and `enable_thinking: false`. -- `--dyn-tool-call-parser nemotron_nano` -- Parses `` into structured `tool_calls`. - -To disable reasoning at request time, pass `"chat_template_kwargs": {"enable_thinking": false}`. The model also supports `"chat_template_kwargs": {"low_effort": true}` for lighter-weight reasoning. - -## Routing - -- **vLLM** and **SGLang** recipes use **approximate KV-aware routing** (`--router-mode kv --no-kv-events` on the frontend). The frontend uses prefix hashing to route requests to workers most likely to have relevant KV cache blocks, which helps workloads with shared system prompts or multi-turn conversations. -- The **TensorRT-LLM** disaggregated recipe uses **round-robin routing**. Nemotron-H on TRT-LLM still requires `enable_block_reuse: false`, so KV overlap routing does not provide a real cache-reuse benefit here and only adds misleading overlap bookkeeping. - -Approximate (hash-based) routing is used for the vLLM and SGLang variants because hybrid Mamba+Attention models do not yet have a reliable KV-event path in these recipes (`--kv-events-config` for vLLM/SGLang, `--publish-events-and-metrics` for TRT-LLM). - -## Backend-Specific Notes - -### vLLM -- No connector flags needed in 1.0 (default is no connector) -- Requires `--is-decode-worker` to skip KV event publisher setup -- Requires `--mamba-cache-mode align` to work around [vllm#34865](https://github.com/vllm-project/vllm/issues/34865): prefix caching with the default `mamba_cache_mode="all"` produces NaN logprobs and garbage tokens for Nemotron-H. Fixed in vLLM 0.17.0 ([vllm#34874](https://github.com/vllm-project/vllm/pull/34874)); the 1.0 container ships vLLM 0.16.0, so the workaround is needed. -- **Attention backend**: On Hopper the default (`FLASH_ATTN`) is safe. On Blackwell, vLLM defaults to FlashInfer, which has a [stale NaN bug](https://github.com/vllm-project/vllm/issues/35138) with hybrid Mamba models ([vllm#35219](https://github.com/vllm-project/vllm/pull/35219)). For Blackwell, specify `--attention-backend FLASH_ATTN` or `--attention-backend TRITON_ATTN` to avoid the issue. -- Sets `VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm` to avoid a [hang during CUDA graph capture](https://github.com/vllm-project/vllm/issues/35772) with TP>1. This is the [new default](https://github.com/vllm-project/vllm/pull/35793) in later vLLM versions but must be set explicitly in 0.16.0. - - -### TensorRT-LLM -- Uses PyTorch backend (`backend: pytorch` in engine config) -- Block reuse is still not supported for Nemotron-H / Mamba hybrid cache. Set `enable_block_reuse: false` explicitly in all TRT-LLM Nemotron configs. If the field is omitted, current TRT-LLM builds may still start only because the Nemotron model class silently applies a model default of `enable_block_reuse: false`; block reuse is not actually active. -- The TRT-LLM disaggregated recipe uses `--router-mode round-robin` rather than KV routing. With block reuse disabled, KV-overlap scoring does not correspond to a real runtime win for Nemotron-H. -- **Disaggregated mode** requires `cache_transceiver_config: backend: UCX`. NIXL and MOONCAKE backends do not support hybrid models with Mamba SSM state — only UCX (or MPI) can transfer both attention KV cache and Mamba conv/SSM state between workers. - -### SGLang -- Requires sglang >= v0.5.9 (1.0 ships v0.5.9; 0.9.1 ships v0.5.8 which has blocking bugs) -- **Disaggregated mode works** with nixl KV transfer (TP=2 per worker, 2 GPUs each). Mooncake (`--disaggregation-transfer-backend mooncake`) is also supported as an alternative transfer backend. -- Known issue: prefill warmup logs `Prefill warmup failed: 'SamplingParams' object is not subscriptable` -- non-blocking, does not affect functionality - -## Dynamo 0.9.1 Compatibility - -These recipes target Dynamo v1.0.0. To run on v0.9.1 containers, the following changes are needed: - -### vLLM (`vllm-runtime:0.9.1`) -- Change image tags from `:1.1.1` to `:0.9.1` -- **Add** `--connector none` to worker args (required in 0.9.1 to disable nixl KV connector; rejected in 1.0) -- Change `--dyn-reasoning-parser` from `nemotron_nano` to `deepseek_r1` (nemotron_nano reasoning parser is broken in 0.9.1) -- `enable_thinking: false` will **not work** with `deepseek_r1` parser (response content goes to `reasoning_content`, `content` is null) -- `--mamba-cache-mode align` is still needed (0.9.1 ships vLLM 0.14.1, also affected by [vllm#34865](https://github.com/vllm-project/vllm/issues/34865)) - -### TensorRT-LLM (`tensorrtllm-runtime:0.9.1`) -- Change image tags from `:1.1.1` to `:0.9.1` -- Change `--dyn-reasoning-parser` from `nemotron_nano` to `deepseek_r1` -- Same `enable_thinking: false` caveat as vLLM above -- Keep `enable_block_reuse: false` in `kv_cache_config` in the ConfigMap. This is still the effective setting for Nemotron-H on current TRT-LLM builds; omitting the field can appear to work only because TRT-LLM silently applies the same model default later. - -### SGLang (`sglang-runtime:0.9.1`) -- **Not supported.** The bundled sglang v0.5.8 has two blocking bugs: - 1. FP8 quantization bug (`ModelOptFp8LinearMethod.create_weights()` signature mismatch) - 2. Config format mismatch (`hybrid_override_pattern` vs `layers_block_type`) -- Both are fixed in sglang v0.5.9 but the 0.9.1 container ships v0.5.8 - -## Notes - -- **Disaggregated mode**: Supported with TRT-LLM via UCX (`trtllm/disagg`) and SGLang via nixl or mooncake (`sglang/disagg`). Not supported with vLLM due to hybrid KV cache incompatibilities. TRT-LLM disagg requires UCX because NIXL/MOONCAKE cannot transfer Mamba SSM state. -- **Storage class**: Update `storageClassName` in `model-cache/model-cache.yaml` before deploying. -- **Model size**: ~240GB download; expect 30-60 minutes depending on bandwidth. diff --git a/recipes/nemotron-3-super-fp8/model-cache/model-cache.yaml b/recipes/nemotron-3-super-fp8/model-cache/model-cache.yaml deleted file mode 100644 index b7e25d2d559b..000000000000 --- a/recipes/nemotron-3-super-fp8/model-cache/model-cache.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 300Gi - storageClassName: "your-storage-class-name" diff --git a/recipes/nemotron-3-super-fp8/model-cache/model-download.yaml b/recipes/nemotron-3-super-fp8/model-cache/model-download.yaml deleted file mode 100644 index 37f4b0bd3326..000000000000 --- a/recipes/nemotron-3-super-fp8/model-cache/model-download.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: MODEL_NAME - value: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - - name: HF_HOME - value: /model-store - # Uses up to 64 GB RAM for XET buffers; set "0" on low-memory nodes (8 GB cap): https://huggingface.co/docs/hub/en/xet/using-xet-storage#download-buffers - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.11.0 - hf download $MODEL_NAME - resources: - requests: - cpu: "2" - memory: "64Gi" - limits: - cpu: "8" - memory: "64Gi" - volumeMounts: - - name: model-cache - mountPath: /model-store - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/nemotron-3-super-fp8/sglang/agg/deploy.yaml b/recipes/nemotron-3-super-fp8/sglang/agg/deploy.yaml deleted file mode 100644 index d0a9944a6678..000000000000 --- a/recipes/nemotron-3-super-fp8/sglang/agg/deploy.yaml +++ /dev/null @@ -1,79 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# NOTE: This recipe requires dynamo 1.0+ with sglang >= v0.5.9. -# -# NOT working on dynamo 0.9.1 (sglang v0.5.8) due to two blocking bugs: -# -# 1. FP8 quantization bug: ModelOptFp8LinearMethod.create_weights() missing -# input_size/output_size parameters, causing TypeError on model load. -# Fixed in sglang v0.5.9 (commit 0ff24159a5). -# -# 2. Config format mismatch: sglang expects hybrid_override_pattern (string) -# but the model provides layers_block_type (list). Workaround: patch the -# model's config.json to add a hybrid_override_pattern field. -# -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: nemotron-super-fp8-sglang-agg -spec: - backendFramework: sglang - envs: - - name: HF_HOME - value: /opt/models - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /opt/models - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.1 - # Approximate KV-aware routing: uses prefix hashing to route - # requests to workers likely to have relevant KV cache, without - # requiring KV events from the backend. - command: - - /bin/sh - - -c - args: - - python3 -m dingo.frontend --router-mode kv --no-kv-events --http-port 8000 - SglangWorker: - componentType: worker - envFromSecret: hf-token-secret - replicas: 1 - resources: - limits: - gpu: "4" - requests: - gpu: "4" - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 16Gi - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.1 - workingDir: /workspace - command: - - python3 - - -m - - dingo.sglang - args: - - --model-path - - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - - --served-model-name - - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - - --tp - - "4" - - --trust-remote-code - - --dyn-tool-call-parser - - nemotron_nano - - --dyn-reasoning-parser - - nemotron_nano diff --git a/recipes/nemotron-3-super-fp8/sglang/disagg/deploy.yaml b/recipes/nemotron-3-super-fp8/sglang/disagg/deploy.yaml deleted file mode 100644 index 2484972aa3b5..000000000000 --- a/recipes/nemotron-3-super-fp8/sglang/disagg/deploy.yaml +++ /dev/null @@ -1,128 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Disaggregated SGLang deployment: prefill/decode split with nixl KV transfer. -# Tested with dynamo 1.0 (SGLang 0.5.9). -# -# Uses TP=2 per worker (prefill: 2 GPUs, decode: 2 GPUs) for a total of 4 GPUs. -# KV cache is transferred between workers via nixl (GPU-direct). -# -# NOT working on dynamo 0.9.1 — same blocking bugs as sglang/agg. -# -# Known issue: Prefill warmup logs a non-blocking warning: -# "Prefill warmup failed: 'SamplingParams' object is not subscriptable" -# This does not affect functionality. -# -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: nemotron-super-fp8-sglang-disagg -spec: - backendFramework: sglang - envs: - - name: HF_HOME - value: /opt/models - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /opt/models - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.1 - command: - - /bin/sh - - -c - args: - - python3 -m dingo.frontend --router-mode kv --no-kv-events --http-port 8000 - prefill: - componentType: worker - subComponentType: prefill - envFromSecret: hf-token-secret - replicas: 1 - resources: - limits: - gpu: "2" - requests: - gpu: "2" - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 16Gi - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.1 - workingDir: /workspace - command: - - python3 - - -m - - dingo.sglang - args: - - --model-path - - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - - --served-model-name - - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - - --tp - - "2" - - --trust-remote-code - - --disaggregation-mode - - prefill - - --disaggregation-bootstrap-port - - "12345" - - --disaggregation-transfer-backend - - nixl - - --host - - 0.0.0.0 - - --dyn-tool-call-parser - - nemotron_nano - - --dyn-reasoning-parser - - nemotron_nano - decode: - componentType: worker - subComponentType: decode - envFromSecret: hf-token-secret - replicas: 1 - resources: - limits: - gpu: "2" - requests: - gpu: "2" - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 16Gi - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.1 - workingDir: /workspace - command: - - python3 - - -m - - dingo.sglang - args: - - --model-path - - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - - --served-model-name - - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - - --tp - - "2" - - --trust-remote-code - - --disaggregation-mode - - decode - - --disaggregation-bootstrap-port - - "12345" - - --disaggregation-transfer-backend - - nixl - - --host - - 0.0.0.0 - - --dyn-tool-call-parser - - nemotron_nano - - --dyn-reasoning-parser - - nemotron_nano diff --git a/recipes/nemotron-3-super-fp8/trtllm/disagg/deploy.yaml b/recipes/nemotron-3-super-fp8/trtllm/disagg/deploy.yaml deleted file mode 100644 index 2e2e3915f9e5..000000000000 --- a/recipes/nemotron-3-super-fp8/trtllm/disagg/deploy.yaml +++ /dev/null @@ -1,215 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Disaggregated TRT-LLM deployment: prefill/decode split with UCX KV transfer. -# Tested with Dynamo v1.0.0 (TRT-LLM PyTorch backend). -# -# Uses TP=2 per worker (prefill: 2 GPUs, decode: 2 GPUs) for a total of 4 GPUs. -# KV cache (attention + Mamba SSM state) is transferred between workers via UCX. -# -# IMPORTANT: Must use UCX backend, not NIXL. NIXL and MOONCAKE backends do not -# support hybrid models with Mamba SSM state: -# ValueError: NIXL or MOONCAKE backend does not support hybrid models with -# RNN (Mamba) states. Please use UCX or MPI backend for cache transfer with -# hybrid models. -# -# Dynamo 0.9.1 compatibility notes: -# - Change image tags from :1.1.1 to :0.9.1 -# - Change --dyn-reasoning-parser from nemotron_nano to deepseek_r1 -# - With deepseek_r1 parser, enable_thinking: false will not work correctly -# - Keep enable_block_reuse: false in both kv_cache_config blocks. Current -# TRT-LLM builds still disable block reuse for Nemotron-H / Mamba hybrid cache. -# -apiVersion: v1 -kind: ConfigMap -metadata: - name: nemotron-super-prefill-config -data: - config.yaml: | - backend: pytorch - tensor_parallel_size: 2 - moe_expert_parallel_size: 1 - enable_attention_dp: false - enable_chunked_prefill: true - max_batch_size: 16 - max_num_tokens: 8192 - trust_remote_code: true - kv_cache_config: - free_gpu_memory_fraction: 0.85 - # Nemotron-H uses a Mamba hybrid cache. Block reuse is still unsupported, - # and explicit true still trips: - # "mamba hybrid cache requires block reuse to be disabled" - # Keep this explicit instead of relying on TRT-LLM's silent model default. - enable_block_reuse: false - moe_config: - backend: TRTLLM - cache_transceiver_config: - # UCX is required for hybrid Mamba+Attention models. - # NIXL/MOONCAKE do not support Mamba SSM state transfer. - backend: UCX - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - disable_overlap_scheduler: true ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: nemotron-super-decode-config -data: - config.yaml: | - backend: pytorch - tensor_parallel_size: 2 - moe_expert_parallel_size: 1 - enable_attention_dp: false - enable_chunked_prefill: true - max_batch_size: 16 - max_num_tokens: 8192 - trust_remote_code: true - kv_cache_config: - free_gpu_memory_fraction: 0.85 - # Nemotron-H uses a Mamba hybrid cache. Block reuse is still unsupported, - # and explicit true still trips: - # "mamba hybrid cache requires block reuse to be disabled" - # Keep this explicit instead of relying on TRT-LLM's silent model default. - enable_block_reuse: false - moe_config: - backend: TRTLLM - cache_transceiver_config: - # UCX is required for hybrid Mamba+Attention models. - # NIXL/MOONCAKE do not support Mamba SSM state transfer. - backend: UCX - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - disable_overlap_scheduler: false ---- -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: nemotron-super-fp8-trtllm-disagg -spec: - backendFramework: trtllm - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - replicas: 1 - extraPodSpec: - mainContainer: - # Round-robin routing is the simplest correct choice here. - # Nemotron-H on TRT-LLM has block reuse disabled, so KV-overlap - # routing does not provide a real cache reuse benefit. - args: - - python3 -m dingo.frontend --router-mode round-robin --http-port 8000 - command: - - /bin/sh - - -c - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - TrtllmPrefillWorker: - componentType: worker - subComponentType: prefill - envFromSecret: hf-token-secret - replicas: 1 - resources: - limits: - gpu: "2" - requests: - gpu: "2" - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 16Gi - extraPodSpec: - mainContainer: - startupProbe: - httpGet: - path: /health - port: 9090 - periodSeconds: 10 - timeoutSeconds: 10 - # TRT-LLM startup is slow (~7 min) due to CUDA graph compilation - failureThreshold: 600 - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - env: - - name: HF_HOME - value: "/opt/models" - - name: ENGINE_ARGS - value: "/opt/dynamo/configs/config.yaml" - - name: MODEL_PATH - value: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" - command: - - /bin/sh - - -c - args: - - | - python3 -m dynamo.trtllm \ - --model-path "${MODEL_PATH}" \ - --served-model-name "${MODEL_PATH}" \ - --extra-engine-args "${ENGINE_ARGS}" \ - --disaggregation-mode prefill \ - --dyn-tool-call-parser nemotron_nano \ - --dyn-reasoning-parser nemotron_nano - volumeMounts: - - mountPath: /opt/dynamo/configs - name: nemotron-super-prefill-config - readOnly: true - volumes: - - configMap: - name: nemotron-super-prefill-config - name: nemotron-super-prefill-config - TrtllmDecodeWorker: - componentType: worker - subComponentType: decode - envFromSecret: hf-token-secret - replicas: 1 - resources: - limits: - gpu: "2" - requests: - gpu: "2" - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 16Gi - extraPodSpec: - mainContainer: - startupProbe: - httpGet: - path: /health - port: 9090 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 600 - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - env: - - name: HF_HOME - value: "/opt/models" - - name: ENGINE_ARGS - value: "/opt/dynamo/configs/config.yaml" - - name: MODEL_PATH - value: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" - command: - - /bin/sh - - -c - args: - - | - python3 -m dynamo.trtllm \ - --model-path "${MODEL_PATH}" \ - --served-model-name "${MODEL_PATH}" \ - --extra-engine-args "${ENGINE_ARGS}" \ - --disaggregation-mode decode \ - --dyn-tool-call-parser nemotron_nano \ - --dyn-reasoning-parser nemotron_nano - volumeMounts: - - mountPath: /opt/dynamo/configs - name: nemotron-super-decode-config - readOnly: true - volumes: - - configMap: - name: nemotron-super-decode-config - name: nemotron-super-decode-config diff --git a/recipes/nemotron-3-super-fp8/vllm/agg/deploy.yaml b/recipes/nemotron-3-super-fp8/vllm/agg/deploy.yaml deleted file mode 100644 index a023ee9c80ed..000000000000 --- a/recipes/nemotron-3-super-fp8/vllm/agg/deploy.yaml +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Tested with Dynamo v1.0.0 (vLLM 0.16.0). -# -# Dynamo 0.9.1 compatibility notes: -# - Change image tags from :1.1.1 to :0.9.1 -# - Add `--connector none` to args (required in 0.9.1, rejected in 1.0) -# - Change --dyn-reasoning-parser from nemotron_nano to deepseek_r1 -# (nemotron_nano reasoning parser is broken in 0.9.1) -# - With deepseek_r1 parser, enable_thinking: false will not work correctly -# -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: nemotron-super-fp8-vllm-agg -spec: - backendFramework: vllm - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /opt/models - extraPodSpec: - mainContainer: - startupProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 10 - timeoutSeconds: 1800 - failureThreshold: 60 - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - # Approximate KV-aware routing: uses prefix hashing to route - # requests to workers likely to have relevant KV cache, without - # requiring KV events from the backend. - command: - - /bin/sh - - -c - args: - - python3 -m dingo.frontend --router-mode kv --no-kv-events --http-port 8000 - VllmWorker: - componentType: worker - envFromSecret: hf-token-secret - replicas: 1 - resources: - limits: - gpu: "4" - requests: - gpu: "4" - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 16Gi - extraPodSpec: - mainContainer: - startupProbe: - httpGet: - path: /health - port: 9090 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 120 - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - env: - - name: HF_HOME - value: "/opt/models" - # Workaround for vllm/vllm#35772: FlashInfer allreduce can hang during - # CUDA graph capture with TP>1. Fixed in vllm/vllm#35793 (default changed - # to trtllm). The 1.0 container ships vLLM 0.16.0, so set explicitly. - - name: VLLM_FLASHINFER_ALLREDUCE_BACKEND - value: "trtllm" - command: - - python3 - - -m - - dingo.vllm - args: - - --model - - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - - --served-model-name - - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - - --tensor-parallel-size - - "4" - - --trust-remote-code - # On Blackwell, vLLM defaults to FlashInfer, which has a stale NaN bug - # with hybrid Mamba models (vllm/vllm#35138). FlashInfer's multiply-by-zero - # masking doesn't clear NaN from stale Mamba fp32 blocks reused by attention - # layers, causing progressive accuracy degradation. - # On Hopper, the default (FLASH_ATTN) is safe and this can be omitted. - # On Blackwell, use FLASH_ATTN or TRITON_ATTN to avoid the bug: - # --attention-backend FLASH_ATTN - # or: - # --attention-backend TRITON_ATTN - # Workaround for vllm/vllm#34865: prefix caching with mamba_cache_mode="all" - # (the default for Nemotron-H) produces NaN logprobs and garbage tokens. - # Fixed in vLLM 0.17.0 (vllm/vllm#34874). Use "align" until then. - - --mamba-cache-mode - - align - # --connector none is no longer needed in 1.0 (default is no connector). - # In 0.9.1, you must add: --connector none - # - # --is-decode-worker also automatically disables KV event publishing, - # which pairs with --no-kv-events on the frontend for approximate routing. - - --is-decode-worker - - --dyn-tool-call-parser - - nemotron_nano - # nemotron_nano reasoning parser handles both enable_thinking: true and false. - # In 0.9.1, use deepseek_r1 instead (nemotron_nano reasoning parser is broken), - # but note that enable_thinking: false will not work with deepseek_r1. - - --dyn-reasoning-parser - - nemotron_nano diff --git a/recipes/nemotron-3-super/README.md b/recipes/nemotron-3-super/README.md deleted file mode 100644 index a2b9ab7b3eff..000000000000 --- a/recipes/nemotron-3-super/README.md +++ /dev/null @@ -1,149 +0,0 @@ - - -# Nemotron-3-Super Recipes - -Recipes for **nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4** (B200) and **nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8** (H200) — a ~120B hybrid Mamba/Attention/MoE model (~12B active). - -We ship Dynamo + vLLM deployment profiles across two GPU SKUs and two serving modes. - -## Configurations - -| | B200 chat | H200 chat | B200 agentic | H200 agentic | -|--------------------------|------------------------------|-------------------------------|------------------------------|------------------------------| -| **GPU** (per worker) | 4× B200 | 4× H200 | 4× B200 | 4× H200 | -| **Mode** | aggregated | aggregated | aggregated | aggregated | -| **Framework** | vLLM 0.21.0 | vLLM 0.21.0 | vLLM 0.21.0 | vLLM 0.21.0 | -| **Precision** | NVFP4 + FP8 KV | FP8 + FP8 KV | NVFP4 + FP8 KV | FP8 + FP8 KV | -| **Parallelism** | TP4 + EP | TP4 + EP | TP4 + EP | TP4 + EP | -| **MoE backend** | FLASHINFER_TRTLLM | FLASHINFER_CUTLASS (default) | FLASHINFER_TRTLLM | FLASHINFER_CUTLASS (default) | -| **Attention backend** | FLASH_ATTN | FLASH_ATTN (default) | FLASH_ATTN | FLASH_ATTN (default) | -| **AllReduce backend** | FlashInfer TRTLLM | FlashInfer TRTLLM (default) | FlashInfer TRTLLM | FlashInfer TRTLLM (default) | -| **All2All backend** | DeepEP high-throughput | FlashInfer NVLink one-sided | DeepEP low-latency | DeepEP high-throughput | -| **Routing** | KV-aware | KV-aware | KV-aware | KV-aware | -| **Speculative decoding** | MTP (DL=3) | MTP (DL=3) | MTP (DL=3) | MTP (DL=3) | - -## Supported features - -- Text-only chat -- Reasoning (`enable_thinking: true|false` via `chat_template_kwargs`) -- Tool calling -- Function calling with JSON arguments - -## Prerequisites - -1. **Dynamo Platform installed** on the target cluster (DGD CRDs registered with `nvidia.com/v1beta1` served). -2. **Namespace labeled for KAI**: - ```bash - export NAMESPACE=your-namespace - kubectl create namespace ${NAMESPACE} - kubectl label namespace ${NAMESPACE} kai.scheduler/enabled=true - ``` - Without this label, pods sit `SchedulingGated` indefinitely because KAI's `pod-grouper` filters by namespace label. -3. **HuggingFace token secret** with access to `nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4` (B200) and/or `nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8` (H200): - ```bash - kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="$HF_TOKEN" \ - -n ${NAMESPACE} - ``` - -## Quick Start - -### 1. Create storage - -> **Note:** edit `model-cache/model-cache.yaml` first and set `storageClassName` to match your cluster (`kubectl get storageclass`). - -```bash -kubectl apply -f model-cache/model-cache.yaml -n ${NAMESPACE} -``` - -### 2. Download model - -Two Jobs share the same PVC. Each pulls into the default HF cache layout (`HF_HOME=/model-cache`, files end up at `/model-cache/hub/models--nvidia--/snapshots//`). Apply only the one your target SKU needs (or both — note PVC sizing). - -```bash -# B200 — NVFP4 checkpoint (~80 GB, ~80 s on Vast) -kubectl apply -f model-cache/model-download.yaml -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/model-download -n ${NAMESPACE} --timeout=1800s - -# H200 — FP8 checkpoint (~120 GB) -kubectl apply -f model-cache/model-download-fp8.yaml -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/model-download-fp8 -n ${NAMESPACE} --timeout=3600s -``` - -> **Note:** running both Jobs lands ~200 GB on the PVC, which is the default size in `model-cache.yaml`. Bump `storage:` in that file before downloading both. - -### 3. Deploy the DGD - -Pick the use-case variant and apply: - -```bash -SKU=b200 # or h200 -USECASE=chat # or agentic - -kubectl apply -f vllm/agg-${SKU}-${USECASE}/deploy.yaml -n ${NAMESPACE} -kubectl get dgd nemotron-3-super-${SKU}-${USECASE} -n ${NAMESPACE} -w -``` - -Two worker replicas, 4× B200/H200 each (half a node). First-time boot per worker ≈ 6–9 min (image pull + vLLM engine init + Inductor + CUDA graph capture up to size 512). - -### 4. Smoke test - -```bash -kubectl port-forward svc/nemotron-3-super-${SKU}-${USECASE}-frontend 8000:8000 -n ${NAMESPACE} - -# B200 uses the NVFP4 model id; H200 uses the FP8 model id -MODEL_ID=nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 # or -FP8 for H200 - -curl http://localhost:8000/v1/models -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d "{\"model\":\"${MODEL_ID}\", - \"messages\":[{\"role\":\"user\",\"content\":\"Hello!\"}], - \"max_tokens\":64, - \"chat_template_kwargs\":{\"enable_thinking\":false}}" -``` - -### 5. Benchmark - -See [`perf/README.md`](perf/README.md) for the full benchmark workflow — staging Mooncake-format traces on the PVC, running the AIPerf trace-replay Job ([`perf/perf.yaml`](perf/perf.yaml)), running a concurrency sweep, and fetching artifacts. - -## Performance results - -| Recipe | SKU | # of worker replicas | Concurrency | User output tok/s | System output tok/s/gpu | -|----------------------|------|----------------------|-------------|-------------------|-------------------------| -| Chat (15% subset) | B200 | 2 | 128 | 61.25 | 844.5 | -| Agentic (15% subset) | B200 | 2 | 192 | 63.16 | 1388.4 | -| Chat (15% subset) | H200 | 2 | 64 | 56.07 | 404.6 | -| Agentic (15% subset) | H200 | 2 | 128 | 62.94 | 851.0 | - -## Spec-dec toggle (B200) - -B200 chat and agentic ship with **MTP spec-dec ON by default** — DL=3, `moe_backend=triton`, stripped `compilation-config`, and `MAX_NUM_BATCHED_TOKENS=65536`. - -To turn MTP off, remove the `- --speculative-config=$(SPECULATIVE_CONFIG)` line from worker args. With the extra memory headroom freed up, you can optionally bump `MAX_NUM_BATCHED_TOKENS` to `"131072"` and switch the `COMPILATION_CONFIG` env to the `compilation-config-fused` ConfigMap key (which enables `use_inductor_graph_partition` + `fuse_allreduce_rms` + `fuse_attn_quant`) for better throughput. - -## Known issues - -1. Some 400 HTTP errors raised by the workers on invalid inputs are surfaced as **500** errors through the Dynamo frontend (the proxy does not always preserve the worker's original status code). - -## File layout - -```text -recipes/nemotron-3-super/ - README.md - model-cache/ - model-cache.yaml # PVC (RWX, 200Gi on storageClass vast) - model-download.yaml # Job: hf download NVFP4 checkpoint (B200) - model-download-fp8.yaml # Job: hf download FP8 checkpoint (H200) - vllm/ - agg-b200-chat/deploy.yaml # DGD: B200×4, NVFP4, DeepEP high-throughput - agg-b200-agentic/deploy.yaml # DGD: B200×4, NVFP4, DeepEP low-latency - agg-h200-chat/deploy.yaml # DGD: H200×4, FP8, FlashInfer NVLink one-sided, MTP spec-dec - agg-h200-agentic/deploy.yaml # DGD: H200×4, FP8, DeepEP high-throughput, MTP spec-dec - perf/ - README.md # benchmark workflow - perf.yaml # AIPerf trace-replay Job -``` diff --git a/recipes/nemotron-3-super/model-cache/model-cache.yaml b/recipes/nemotron-3-super/model-cache/model-cache.yaml deleted file mode 100644 index 4da10818df7c..000000000000 --- a/recipes/nemotron-3-super/model-cache/model-cache.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Shared model cache for the Nemotron-3-Super checkpoints -# (NVFP4 for B200, FP8 for H200). -# Weights are stored in the default HF cache layout (HF_HOME=/model-cache); -# workers keep --model as the HF ID and resolve weights from this cache. -# -# Edit storageClassName to match your cluster: -# kubectl get storageclass -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 200Gi - storageClassName: "vast" diff --git a/recipes/nemotron-3-super/model-cache/model-download-fp8.yaml b/recipes/nemotron-3-super/model-cache/model-download-fp8.yaml deleted file mode 100644 index 3f90c0205cd6..000000000000 --- a/recipes/nemotron-3-super/model-cache/model-download-fp8.yaml +++ /dev/null @@ -1,51 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Downloads the FP8 Nemotron-3-Super checkpoint into the PVC using the -# default HF cache layout (HF_HOME=/model-cache, files end up at -# /model-cache/hub/models--nvidia--NVIDIA-Nemotron-3-Super-120B-A12B-FP8/snapshots//). -# -# Sibling of model-download.yaml — that one is the NVFP4 checkpoint for B200; -# this one is the FP8 checkpoint for H200. Apply whichever your target SKU -# needs (or both, but verify the PVC has ~200 GiB free for both). -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download-fp8 -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download-fp8 - spec: - restartPolicy: Never - containers: - - name: model-download-fp8 - image: python:3.10-slim - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: MODEL_NAME - value: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - - name: HF_HOME - value: /model-cache - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.16.4 - hf download "$MODEL_NAME" - ls /model-cache/hub/models--*/snapshots/*/ | head - volumeMounts: - - name: model-cache - mountPath: /model-cache - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/nemotron-3-super/model-cache/model-download.yaml b/recipes/nemotron-3-super/model-cache/model-download.yaml deleted file mode 100644 index 91aaa778150b..000000000000 --- a/recipes/nemotron-3-super/model-cache/model-download.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Downloads the NVFP4 Nemotron-3-Super checkpoint into the PVC using the -# default HF cache layout (HF_HOME=/model-cache, files end up at -# /model-cache/hub/models--nvidia--NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4/snapshots//). -# -# Reason for HF cache layout (instead of --local-dir): the worker keeps --model as an -# HF ID so the Dynamo frontend's discovery watcher can fetch model metadata -# from HuggingFace, while the actual weights are still resolved from the -# local cache (no runtime network pull). Using --local-dir instead would -# break frontend model registration (it issues hub::from_hf(<--model value>) -# which 404s when --model is a filesystem path). -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: MODEL_NAME - value: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 - - name: HF_HOME - value: /model-cache - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.16.4 - hf download "$MODEL_NAME" - ls /model-cache/hub/models--*/snapshots/*/ | head - volumeMounts: - - name: model-cache - mountPath: /model-cache - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/nemotron-3-super/perf/README.md b/recipes/nemotron-3-super/perf/README.md deleted file mode 100644 index cb0843498496..000000000000 --- a/recipes/nemotron-3-super/perf/README.md +++ /dev/null @@ -1,150 +0,0 @@ - - -# Nemotron-3-Super Benchmark Recipe - -A single [AIPerf](https://github.com/ai-dynamo/aiperf) trace-replay Job — [perf.yaml](perf.yaml) — covers every Nemotron-3-Super DGD variant (B200/H200 × chat/agent traces). The benchmark is identical across variants; only `ENDPOINT`, `TRACE_FILE`, and `TARGET_MODEL` need to change. - -The Job waits for `GET /v1/models` on the DGD frontend to return the configured `TARGET_MODEL` (up to ~1h by default), runs a short warmup, then replays the configured trace at a single `CONCURRENCY` value and writes raw artifacts to the shared `model-cache` PVC. - -The bench pod is **co-located with the DGD frontend** (`podAffinity` on the frontend's host) so client → server traffic stays on a single node. - -## Targeting a variant - -Edit the `env` block in [perf.yaml](perf.yaml): - -| Variant target | `ENDPOINT` | `TARGET_MODEL` | `TRACE_FILE` (chat / agent) | -| ------------------------ | ----------------------------------------------------- | --------------------------------------------------------- | ----------------------------------------------------------------------- | -| B200 agg, chat workload | `nemotron-3-super-b200-chat-frontend:8000` | `nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4` | `/model-cache/traces/8k_1k_70kv_chat_new_noschedule.jsonl` | -| B200 agg, agent workload | `nemotron-3-super-b200-agentic-frontend:8000` | `nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4` | `/model-cache/traces/64k_400_90kv_agent_new_noschedule.jsonl` | -| H200 agg, chat workload | `nemotron-3-super-h200-chat-frontend:8000` | `nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8` | `/model-cache/traces/8k_1k_70kv_chat_new_noschedule.jsonl` | -| H200 agg, agent workload | `nemotron-3-super-h200-agentic-frontend:8000` | `nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8` | `/model-cache/traces/64k_400_90kv_agent_new_noschedule.jsonl` | - -Both DGDs of a given SKU serve the same `--served-model-name`, so either trace can be replayed against either DGD by swapping `TRACE_FILE`. `TARGET_MODEL` only changes between B200 (NVFP4) and H200 (FP8). - -If you run more than one benchmark in the same namespace, also update `metadata.name` / `labels.app` so jobs and artifact directories stay distinct. - -## Dataset - -The benchmark replays a [Mooncake-format](https://github.com/kvcache-ai/Mooncake) trace via `aiperf --custom-dataset-type mooncake_trace`. Each JSONL line describes one request (`input_length`, `output_length`, `hash_ids`). - -Trace flavours expected on the PVC: - -- **Chat** — `/model-cache/traces/8k_1k_70kv_chat_new_noschedule.jsonl` -- **Agent** — `/model-cache/traces/64k_400_90kv_agent_new_noschedule.jsonl` - -For shorter runs (smoke tests, faster iteration), point `TRACE_FILE` at a smaller variant of the same trace rather than capping run time. Typical staging: - -``` -/model-cache/traces/.jsonl # full -/model-cache/traces/_short_30perc.jsonl # ~30% subset -/model-cache/traces/_short_15perc.jsonl # ~15% subset -``` - -These are workload-shape traces (not model-specific). Stage your own Mooncake-format JSONLs at the path you set in `TRACE_FILE`. - -## Workflow - -```bash -export NAMESPACE=your-namespace -``` - -### 1. Deploy the DGD - -See instructions in the [recipe README](../README.md). - -Before deploying, do these adjustments to the DGD: -- (H200 only) Change the `SPECULATIVE_CONFIG` env to point to `key: speculative-config-synthetic` if you want a fixed-AL synthetic MTP run. -- Modify the worker `replicas` to match your desired target. - -### 2. Stage the trace on the PVC - -Spin up a short-lived helper pod that mounts `model-cache`, then `kubectl cp` the traces in: - -```bash -kubectl run pvc-helper -n ${NAMESPACE} \ - --image=busybox:1.36 --restart=Never \ - --overrides='{"spec":{"containers":[{"name":"helper","image":"busybox:1.36","command":["sleep","3600"],"volumeMounts":[{"name":"model-cache","mountPath":"/model-cache"}]}],"volumes":[{"name":"model-cache","persistentVolumeClaim":{"claimName":"model-cache"}}]}}' \ - --command -- sleep 3600 - -kubectl cp ./traces ${NAMESPACE}/pvc-helper:/model-cache/ -``` - -Keep `pvc-helper` around for fetching artifacts later, or `kubectl delete pod pvc-helper -n ${NAMESPACE}` once you're done staging. - -### 3. Run the benchmark - -```bash -kubectl apply -f perf.yaml -n ${NAMESPACE} - -# Stream logs -kubectl logs -n ${NAMESPACE} -l job-name=nemotron-3-super-bench -f - -# Wait for completion (2h hard cap on the Job) -kubectl wait --for=condition=Complete \ - job/nemotron-3-super-bench \ - -n ${NAMESPACE} --timeout=7200s -``` - -### 4. Fetch artifacts - -```bash -kubectl cp ${NAMESPACE}/pvc-helper:/model-cache/perf/_nemotron-3-super-bench ./results -``` - -### 5. Cleanup - -```bash -kubectl delete job nemotron-3-super-bench -n ${NAMESPACE} -kubectl delete pod pvc-helper -n ${NAMESPACE} # if you kept it around -``` - -## Running a concurrency sweep - -`perf.yaml` runs a **single** `CONCURRENCY` value. To measure multiple concurrencies you must clear server state between runs — otherwise residual KV cache / prefix-cache hits from the previous run skew results. - -For each concurrency value you want to measure: - -```bash -# 1. Delete the previous bench job -kubectl delete job nemotron-3-super-bench -n ${NAMESPACE} --ignore-not-found - -# 2. Drop KV / prefix-cache by deleting the worker pods; Grove respawns them -# (Dynamo workers are PodClique pods, not k8s Deployments — `kubectl rollout -# restart deployment ...` is a silent no-op against the Grove resource chain.) -DGD=nemotron-3-super-b200-chat # or any of the four variants -kubectl delete pods -n ${NAMESPACE} \ - -l nvidia.com/dynamo-graph-deployment-name=${DGD},nvidia.com/dynamo-component-type=worker - -# 3. Bump CONCURRENCY in perf.yaml, then re-apply -kubectl apply -f perf.yaml -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/nemotron-3-super-bench -n ${NAMESPACE} --timeout=7200s -``` - -(The bench Job's `wait_for_model_ready` loop handles the worker restart window — it re-polls `/v1/models` until the frontend reports ready again.) - -## Tunable environment variables - -Edit the `env` block on the `Job` to adjust: - -| Variable | Default | Notes | -| -------------- | ---------------------------------------------------------------- | ------------------------------------------------------------------------------- | -| `ENDPOINT` | `nemotron-3-super-b200-chat-frontend:8000` | DGD frontend service:port — change per variant | -| `TRACE_FILE` | `/model-cache/traces/8k_1k_70kv_chat_new_noschedule.jsonl` | Swap to agent or to a smaller subset (`...short_15perc.jsonl`) for shorter runs | -| `CONCURRENCY` | `24` | Single value — see [Running a concurrency sweep](#running-a-concurrency-sweep) | -| `TARGET_MODEL` | `nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4` | Must match `--served-model-name` on the DGD frontend (NVFP4 for B200, FP8 for H200) | - -## Artifacts - -Results are written to: - -``` -/model-cache/perf/_/ - warmup/ - NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4_trace_c_/ - profile_export.json - inputs.json - ... -``` diff --git a/recipes/nemotron-3-super/perf/perf.yaml b/recipes/nemotron-3-super/perf/perf.yaml deleted file mode 100644 index ef808d3abd47..000000000000 --- a/recipes/nemotron-3-super/perf/perf.yaml +++ /dev/null @@ -1,187 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# AIPerf trace-replay benchmark for Nemotron-3-Super DGDs. -# -# One Job covers every variant (B200/H200 × chat/agent traces) — the -# benchmark itself is identical regardless of GPU SKU. To target a specific -# DGD, edit the env values below (and `metadata.name` / `labels.app` if you -# run multiple jobs in the same namespace). -# -# Runs a single concurrency value. To sweep concurrencies, restart the DGD -# workers between runs so KV cache and prefix-cache state are reset — see -# README.md "Running a concurrency sweep". -# -# Prerequisites: -# - Target DGD is Ready (see ../README.md) -# - model-cache PVC mounted in the namespace and contains TRACE_FILE -# - TARGET_MODEL matches the served-model-name on the DGD (NVFP4 for B200, -# FP8 for H200) -# -# Results: /model-cache/perf/_/ -apiVersion: batch/v1 -kind: Job -metadata: - name: nemotron-3-super-bench -spec: - backoffLimit: 1 - activeDeadlineSeconds: 7200 # 2h hard cap on the Job - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: nemotron-3-super-bench - spec: - # Co-locate the bench pod with the DGD frontend to minimise client/server - # network overhead. - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: nvidia.com/dynamo-component-type - operator: In - values: - - frontend - - key: nvidia.com/dynamo-graph-deployment-name - operator: In - values: - - nemotron-3-super-b200-chat - - nemotron-3-super-b200-agentic - - nemotron-3-super-h200-chat - - nemotron-3-super-h200-agentic - topologyKey: kubernetes.io/hostname - # tolerations: # uncomment to populate any tolerations for the gpu nodes - containers: - - name: perf - image: python:3.12-slim - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -c - - | - set -euo pipefail - ulimit -n 600000 - apt-get update && apt-get install -y curl jq procps git && apt-get clean - pip install "aiperf==0.10.0" "protobuf==7.35.0" "transformers==4.57.3" "tiktoken==0.13.0" - sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true - export COLUMNS=200 - EPOCH=$(date +%s) - wait_for_model_ready() { - local max_attempts=${WAIT_FOR_MODEL_MAX_ATTEMPTS:-720} # 720 * 5s = 1h default - local attempt=0 - echo "Waiting for model '$TARGET_MODEL' at $ENDPOINT/v1/models (max ${max_attempts} attempts) ..." - while ! curl -sf "http://$ENDPOINT/v1/models" | jq -e --arg m "$TARGET_MODEL" '.data[]? | select(.id == $m)' >/dev/null 2>&1; do - attempt=$((attempt + 1)) - if [ "$attempt" -ge "$max_attempts" ]; then - echo "ERROR: model '$TARGET_MODEL' did not become ready after ${max_attempts} attempts. Aborting." - exit 1 - fi - echo "[$(date '+%H:%M:%S')] not ready (attempt ${attempt}/${max_attempts}), sleeping 5s" - sleep 5 - done - curl -s "http://$ENDPOINT/v1/models" | jq . - } - if [ ! -f "${TRACE_FILE}" ]; then - echo "ERROR: trace file not found at ${TRACE_FILE}" - echo "Copy it onto the PVC via a helper pod, e.g.:" - echo " kubectl cp /:${TRACE_FILE}" - exit 1 - fi - wait_for_model_ready - ROOT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - mkdir -p "$ROOT_DIR" - echo "==============================================" - echo "Trace Replay Benchmark (aiperf)" - echo "==============================================" - echo "Endpoint: http://${ENDPOINT}" - echo "Model: ${TARGET_MODEL}" - echo "Trace file: ${TRACE_FILE}" - echo "Concurrency: ${CONCURRENCY}" - echo "Artifact root:${ROOT_DIR}" - echo "==============================================" - # Warmup - WARMUP_DIR="${ROOT_DIR}/warmup" - mkdir -p "$WARMUP_DIR" - aiperf profile \ - -m "${TARGET_MODEL}" \ - --tokenizer "${TARGET_MODEL}" \ - --tokenizer-trust-remote-code \ - --url "http://${ENDPOINT}" \ - --streaming \ - --ui simple \ - --isl 8000 \ - --osl 1000 \ - --concurrency 1 \ - --request-count 5 \ - --artifact-dir "${WARMUP_DIR}" - echo "Warmup complete" - MODEL_BASE="${TARGET_MODEL##*/}" - TS=$(date +'%Y%m%d_%H%M%S') - RUN_DIR="${ROOT_DIR}/${MODEL_BASE}_trace_c${CONCURRENCY}_${TS}" - mkdir -p "$RUN_DIR" - aiperf profile \ - -m "${TARGET_MODEL}" \ - --tokenizer "${TARGET_MODEL}" \ - --tokenizer-trust-remote-code \ - --input-file "${TRACE_FILE}" \ - --custom-dataset-type mooncake_trace \ - --num-requests $(wc -l < "${TRACE_FILE}") \ - --prompt-input-tokens-block-size 512 \ - --url "http://${ENDPOINT}" \ - --streaming \ - --use-server-token-count \ - --extra-inputs ignore_eos:true \ - --concurrency "${CONCURRENCY}" \ - --random-seed 42 \ - --ui simple \ - --artifact-dir "${RUN_DIR}" \ - --request-timeout-seconds 1200 \ - --workers-max "${CONCURRENCY}" \ - --export-http-trace - echo "Concurrency ${CONCURRENCY} complete; artifacts in ${RUN_DIR}" - ls -la "${RUN_DIR}" || true - echo "" - echo "Done. Root: ${ROOT_DIR}" - env: - # --- Edit these to target the DGD + trace you want to benchmark --- - - name: ENDPOINT - value: nemotron-3-super-b200-chat-frontend:8000 - - name: TRACE_FILE - value: /model-cache/traces/8k_1k_70kv_chat_new_noschedule.jsonl # use /model-cache/traces/64k_400_90kv_agent_new_noschedule.jsonl for agent - - name: CONCURRENCY - value: "24" - # B200 deploys serve the NVFP4 model id; H200 deploys serve FP8 — swap - # TARGET_MODEL to match the DGD's --served-model-name. - - name: TARGET_MODEL - value: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 # H200: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - # --- The remainder is shared across all variants --- - - name: AIPERF_HTTP_CONNECTION_LIMIT - value: "200" - - name: AIPERF_HTTP_SO_RCVTIMEO - value: "120" - - name: AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT - value: "3600" - - name: AIPERF_DATASET_CONFIGURATION_TIMEOUT - value: "3600" - - name: JOB_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.labels['job-name'] - - name: ROOT_ARTIFACT_DIR - value: /model-cache/perf - - name: HF_HOME - value: /model-cache - - name: PYTHONUNBUFFERED - value: "1" - volumeMounts: - - name: model-cache - mountPath: /model-cache - restartPolicy: Never - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/nemotron-3-super/perf/traces/.gitattributes b/recipes/nemotron-3-super/perf/traces/.gitattributes deleted file mode 100644 index 817db20a164b..000000000000 --- a/recipes/nemotron-3-super/perf/traces/.gitattributes +++ /dev/null @@ -1,6 +0,0 @@ -64k_400_90kv_agent_new_noschedule.jsonl filter=lfs diff=lfs merge=lfs -text -64k_400_90kv_agent_new_noschedule_short_15perc.jsonl filter=lfs diff=lfs merge=lfs -text -64k_400_90kv_agent_new_noschedule_short_30perc.jsonl filter=lfs diff=lfs merge=lfs -text -8k_1k_70kv_chat_new_noschedule.jsonl filter=lfs diff=lfs merge=lfs -text -8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl filter=lfs diff=lfs merge=lfs -text -8k_1k_70kv_chat_new_noschedule_short_30perc.jsonl filter=lfs diff=lfs merge=lfs -text diff --git a/recipes/nemotron-3-super/perf/traces/64k_400_90kv_agent_new_noschedule.jsonl b/recipes/nemotron-3-super/perf/traces/64k_400_90kv_agent_new_noschedule.jsonl deleted file mode 100644 index aea03789f3d9..000000000000 --- a/recipes/nemotron-3-super/perf/traces/64k_400_90kv_agent_new_noschedule.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:aa0a64efcad575a01949e46e28dded094f20733db8708e0bc91b88ce9981fed6 -size 17623982 diff --git a/recipes/nemotron-3-super/perf/traces/64k_400_90kv_agent_new_noschedule_short_15perc.jsonl b/recipes/nemotron-3-super/perf/traces/64k_400_90kv_agent_new_noschedule_short_15perc.jsonl deleted file mode 100644 index 3fcfa9915c9e..000000000000 --- a/recipes/nemotron-3-super/perf/traces/64k_400_90kv_agent_new_noschedule_short_15perc.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f20d3f2bc83dd1306cda659fbe34e7c4d85ca5497626c98bc0b1c4d2211379d0 -size 2722326 diff --git a/recipes/nemotron-3-super/perf/traces/64k_400_90kv_agent_new_noschedule_short_30perc.jsonl b/recipes/nemotron-3-super/perf/traces/64k_400_90kv_agent_new_noschedule_short_30perc.jsonl deleted file mode 100644 index 1f5891b41524..000000000000 --- a/recipes/nemotron-3-super/perf/traces/64k_400_90kv_agent_new_noschedule_short_30perc.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:dae61f92fee05038f1dcb76474dba83c1b3c174f3d187589c7d0a16a7012eb79 -size 5401776 diff --git a/recipes/nemotron-3-super/perf/traces/8k_1k_70kv_chat_new_noschedule.jsonl b/recipes/nemotron-3-super/perf/traces/8k_1k_70kv_chat_new_noschedule.jsonl deleted file mode 100644 index a1ab5ad24a89..000000000000 --- a/recipes/nemotron-3-super/perf/traces/8k_1k_70kv_chat_new_noschedule.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5f369eb75ce639ad8b05cc209bb534bfedd627e9f7b923de32888155b4c9085a -size 5648304 diff --git a/recipes/nemotron-3-super/perf/traces/8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl b/recipes/nemotron-3-super/perf/traces/8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl deleted file mode 100644 index 1eea1bd8955c..000000000000 --- a/recipes/nemotron-3-super/perf/traces/8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b1221bca72b69f842897f339624306a84857f1b55ea0d866525f94d9ceb9b871 -size 894687 diff --git a/recipes/nemotron-3-super/perf/traces/8k_1k_70kv_chat_new_noschedule_short_30perc.jsonl b/recipes/nemotron-3-super/perf/traces/8k_1k_70kv_chat_new_noschedule_short_30perc.jsonl deleted file mode 100644 index 0ef335a203b3..000000000000 --- a/recipes/nemotron-3-super/perf/traces/8k_1k_70kv_chat_new_noschedule_short_30perc.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6de2a977b40dba8735a70aa5f9ab0d84e32530d7cf1fc9d24efc631c341f9c17 -size 1790696 diff --git a/recipes/nemotron-3-super/vllm/agg-b200-agentic/deploy.yaml b/recipes/nemotron-3-super/vllm/agg-b200-agentic/deploy.yaml deleted file mode 100644 index 90986244218e..000000000000 --- a/recipes/nemotron-3-super/vllm/agg-b200-agentic/deploy.yaml +++ /dev/null @@ -1,221 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Aggregated NVFP4 deployment of NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 on -# B200x4 (TP=4) tuned for the agentic-coding workload: DeepEP low-latency -# all-to-all and FP8 KV cache (fp8_e4m3) to minimise ITL/TTFT spikes -# during short-burst tool/code edits. -# -# Cluster requirements: see ../agg-b200-chat/deploy.yaml. -apiVersion: v1 -kind: ConfigMap -metadata: - name: nemotron-3-super-b200-agentic-config - labels: - app.kubernetes.io/name: nemotron-3-super-b200-agentic - app.kubernetes.io/part-of: dynamo -data: - # Stripped compilation passes for the spec-dec path (no inductor graph - # partitioning, no fuse passes — they don't compose with the MTP draft - # model's compile graph yet). This is what worker env COMPILATION_CONFIG - # points at by default. - compilation-config: |- - { - "max_cudagraph_capture_size": 512 - } - # Fused compilation passes — for non-spec-dec runs only. Point - # COMPILATION_CONFIG at this key (configMapKeyRef.key) when running - # without --speculative-config. - compilation-config-fused: |- - { - "use_inductor_graph_partition": true, - "pass_config": { - "fuse_allreduce_rms": true, - "fuse_attn_quant": true - }, - "max_cudagraph_capture_size": 512 - } - # MTP speculative decoding. moe_backend=triton on B200 because the - # default FlashInfer-TRTLLM MoE path isn't supported in the spec-dec - # draft model loop yet. - speculative-config: |- - { - "method": "mtp", - "num_speculative_tokens": 3, - "moe_backend": "triton" - } - # Benchmarking-only: synthetic rejection sampling with a fixed AL. - # Switch SPECULATIVE_CONFIG env to this key to bench MTP at a known - # acceptance length without running the actual draft head. - speculative-config-synthetic: |- - { - "method": "mtp", - "num_speculative_tokens": 3, - "moe_backend": "triton", - "rejection_sample_method": "synthetic", - "synthetic_acceptance_length": 2.82 - } ---- -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: nemotron-3-super-b200-agentic - labels: - app.kubernetes.io/name: nemotron-3-super-b200-agentic - app.kubernetes.io/part-of: dynamo -spec: - backendFramework: vllm - components: - - name: Frontend - type: frontend - replicas: 1 - podTemplate: - spec: - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-super-dev.1 - command: - - python3 - args: - - -m - - dingo.frontend - - --trust-remote-code - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: DYN_ROUTER_MODE - value: kv - resources: {} - startupProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 10 - timeoutSeconds: 60 - failureThreshold: 60 - - - name: agg - type: worker - replicas: 2 - podTemplate: - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.product - operator: In - values: - - NVIDIA-B200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: dshm - emptyDir: - medium: Memory - sizeLimit: 16Gi - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-super-dev.1 - command: - - python3 - - -m - - dingo.vllm - args: - - --model - - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 - - --served-model-name - - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 - - --trust-remote-code - - --tensor-parallel-size - - "4" - - --enable-expert-parallel - - --moe-backend - - flashinfer_trtllm - - --enable-flashinfer-autotune - - --enable-prefix-caching - - --enable-chunked-prefill - - --max-model-len - - "131072" - - --max-num-batched-tokens=$(MAX_NUM_BATCHED_TOKENS) - - --all2all-backend - - deepep_low_latency - - --kv-cache-dtype - - fp8_e4m3 - - --mamba-ssm-cache-dtype - - float16 - - --compilation-config=$(COMPILATION_CONFIG) - - --speculative-config=$(SPECULATIVE_CONFIG) - - --dyn-tool-call-parser - - nemotron_nano - - --dyn-reasoning-parser - - nemotron_nano - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: VLLM_CONFIG_ROOT - value: /tmp/vllm-config - - name: VLLM_CACHE_ROOT - value: /tmp/vllm-cache - - name: COMPILATION_CONFIG - valueFrom: - configMapKeyRef: - name: nemotron-3-super-b200-agentic-config - key: compilation-config - - name: SPECULATIVE_CONFIG - valueFrom: - configMapKeyRef: - name: nemotron-3-super-b200-agentic-config - key: speculative-config - # Tracks the spec-dec mode: 65536 when --speculative-config is - # active, 131072 when running without spec-dec. Update this - # value (and remove --speculative-config from args) to disable. - - name: MAX_NUM_BATCHED_TOKENS - value: "65536" - - name: VLLM_FLASHINFER_ALLREDUCE_BACKEND - value: "trtllm" - - name: NCCL_IB_DISABLE - value: "1" - - name: PYTHONHASHSEED - value: "42" - envFrom: - - secretRef: - name: hf-token-secret - resources: - requests: - nvidia.com/gpu: "4" - memory: "256Gi" - limits: - nvidia.com/gpu: "4" - securityContext: - runAsUser: 0 - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 30 - timeoutSeconds: 20 - failureThreshold: 60 - volumeMounts: - - name: model-cache - mountPath: /model-cache - - name: dshm - mountPath: /dev/shm - workingDir: /workspace/ diff --git a/recipes/nemotron-3-super/vllm/agg-b200-chat/deploy.yaml b/recipes/nemotron-3-super/vllm/agg-b200-chat/deploy.yaml deleted file mode 100644 index 0677ec222cd7..000000000000 --- a/recipes/nemotron-3-super/vllm/agg-b200-chat/deploy.yaml +++ /dev/null @@ -1,223 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Aggregated NVFP4 deployment of NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 on -# B200x4 (TP=4) with Expert Parallel + DeepEP high-throughput all-to-all. -# -# Cluster requirements: -# - Namespace must carry label kai.scheduler/enabled=true so KAI's -# pod-grouper picks up the pods. -# - hf-token-secret (Opaque, key HF_TOKEN) must exist in the namespace. -# - storageClass with ReadWriteMany support (vast on Dynamo Nscale dev). -apiVersion: v1 -kind: ConfigMap -metadata: - name: nemotron-3-super-b200-chat-config - labels: - app.kubernetes.io/name: nemotron-3-super-b200-chat - app.kubernetes.io/part-of: dynamo -data: - # Stripped compilation passes for the spec-dec path (no inductor graph - # partitioning, no fuse passes — they don't compose with the MTP draft - # model's compile graph yet). This is what worker env COMPILATION_CONFIG - # points at by default. - compilation-config: |- - { - "max_cudagraph_capture_size": 512 - } - # Fused compilation passes — for non-spec-dec runs only. Point - # COMPILATION_CONFIG at this key (configMapKeyRef.key) when running - # without --speculative-config. - compilation-config-fused: |- - { - "use_inductor_graph_partition": true, - "pass_config": { - "fuse_allreduce_rms": true, - "fuse_attn_quant": true - }, - "max_cudagraph_capture_size": 512 - } - # MTP speculative decoding. moe_backend=triton on B200 because the - # default FlashInfer-TRTLLM MoE path isn't supported in the spec-dec - # draft model loop yet. - speculative-config: |- - { - "method": "mtp", - "num_speculative_tokens": 3, - "moe_backend": "triton" - } - # Benchmarking-only: synthetic rejection sampling with a fixed AL. - # Switch SPECULATIVE_CONFIG env to this key to bench MTP at a known - # acceptance length without running the actual draft head. - speculative-config-synthetic: |- - { - "method": "mtp", - "num_speculative_tokens": 3, - "moe_backend": "triton", - "rejection_sample_method": "synthetic", - "synthetic_acceptance_length": 2.82 - } ---- -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: nemotron-3-super-b200-chat - labels: - app.kubernetes.io/name: nemotron-3-super-b200-chat - app.kubernetes.io/part-of: dynamo -spec: - backendFramework: vllm - components: - - name: Frontend - type: frontend - replicas: 1 - podTemplate: - spec: - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-super-dev.1 - command: - - python3 - args: - - -m - - dingo.frontend - - --trust-remote-code - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: DYN_ROUTER_MODE - value: kv - resources: {} - startupProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 10 - timeoutSeconds: 60 - failureThreshold: 60 - - - name: agg - type: worker - replicas: 2 - podTemplate: - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.product - operator: In - values: - - NVIDIA-B200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: dshm - emptyDir: - medium: Memory - sizeLimit: 16Gi - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-super-dev.1 - command: - - python3 - - -m - - dingo.vllm - args: - - --model - - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 - - --served-model-name - - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 - - --trust-remote-code - - --tensor-parallel-size - - "4" - - --enable-expert-parallel - - --moe-backend - - flashinfer_trtllm - - --enable-flashinfer-autotune - - --enable-prefix-caching - - --enable-chunked-prefill - - --max-model-len - - "131072" - - --max-num-batched-tokens=$(MAX_NUM_BATCHED_TOKENS) - - --all2all-backend - - deepep_high_throughput - - --kv-cache-dtype - - fp8_e4m3 - - --mamba-ssm-cache-dtype - - float16 - - --compilation-config=$(COMPILATION_CONFIG) - - --speculative-config=$(SPECULATIVE_CONFIG) - - --dyn-tool-call-parser - - nemotron_nano - - --dyn-reasoning-parser - - nemotron_nano - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: VLLM_CONFIG_ROOT - value: /tmp/vllm-config - - name: VLLM_CACHE_ROOT - value: /tmp/vllm-cache - - name: COMPILATION_CONFIG - valueFrom: - configMapKeyRef: - name: nemotron-3-super-b200-chat-config - key: compilation-config - - name: SPECULATIVE_CONFIG - valueFrom: - configMapKeyRef: - name: nemotron-3-super-b200-chat-config - key: speculative-config - # Tracks the spec-dec mode: 65536 when --speculative-config is - # active, 131072 when running without spec-dec. Update this - # value (and remove --speculative-config from args) to disable. - - name: MAX_NUM_BATCHED_TOKENS - value: "65536" - - name: VLLM_FLASHINFER_ALLREDUCE_BACKEND - value: "trtllm" - - name: NCCL_IB_DISABLE - value: "1" - - name: PYTHONHASHSEED - value: "42" - envFrom: - - secretRef: - name: hf-token-secret - resources: - requests: - nvidia.com/gpu: "4" - memory: "256Gi" - limits: - nvidia.com/gpu: "4" - securityContext: - runAsUser: 0 - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 30 - timeoutSeconds: 20 - failureThreshold: 60 - volumeMounts: - - name: model-cache - mountPath: /model-cache - - name: dshm - mountPath: /dev/shm - workingDir: /workspace/ diff --git a/recipes/nemotron-3-super/vllm/agg-h200-agentic/deploy.yaml b/recipes/nemotron-3-super/vllm/agg-h200-agentic/deploy.yaml deleted file mode 100644 index 156f834079aa..000000000000 --- a/recipes/nemotron-3-super/vllm/agg-h200-agentic/deploy.yaml +++ /dev/null @@ -1,197 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Aggregated FP8 deployment of NVIDIA-Nemotron-3-Super-120B-A12B-FP8 on -# H200x4 (TP=4) with Expert Parallel + DeepEP high-throughput all-to-all. -# Differs from the chat variant (../agg-h200-chat/deploy.yaml), -# which uses the FlashInfer NVLink one-sided all-to-all backend; see that -# file for the rationale on dropped Blackwell-specific NCCL env. -apiVersion: v1 -kind: ConfigMap -metadata: - name: nemotron-3-super-h200-agentic-config - labels: - app.kubernetes.io/name: nemotron-3-super-h200-agentic - app.kubernetes.io/part-of: dynamo -data: - compilation-config: |- - { - "use_inductor_graph_partition": true, - "pass_config": { - "fuse_allreduce_rms": true, - "fuse_attn_quant": true - }, - "max_cudagraph_capture_size": 512 - } - speculative-config: |- - { - "method": "mtp", - "num_speculative_tokens": 3 - } - # Benchmarking-only: synthetic rejection sampling with a fixed AL - # (matches the speculative-config-synthetic pattern in Alexandre's k26 - # recipes). Switch SPECULATIVE_CONFIG env to this key to bench MTP - # at a known acceptance length without running the actual draft head. - speculative-config-synthetic: |- - { - "method": "mtp", - "num_speculative_tokens": 3, - "rejection_sample_method": "synthetic", - "synthetic_acceptance_length": 2.82 - } ---- -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: nemotron-3-super-h200-agentic - labels: - app.kubernetes.io/name: nemotron-3-super-h200-agentic - app.kubernetes.io/part-of: dynamo -spec: - backendFramework: vllm - components: - - name: Frontend - type: frontend - replicas: 1 - podTemplate: - spec: - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-super-dev.1 - command: - - python3 - args: - - -m - - dingo.frontend - - --trust-remote-code - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: DYN_ROUTER_MODE - value: kv - resources: {} - startupProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 10 - timeoutSeconds: 60 - failureThreshold: 60 - - - name: agg - type: worker - replicas: 2 - podTemplate: - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.product - operator: In - values: - - NVIDIA-H200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: dshm - emptyDir: - medium: Memory - sizeLimit: 16Gi - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-super-dev.1 - command: - - python3 - - -m - - dingo.vllm - args: - - --model - - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - - --served-model-name - - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - - --trust-remote-code - - --tensor-parallel-size - - "4" - - --enable-expert-parallel - - --enable-flashinfer-autotune - - --enable-prefix-caching - - --enable-chunked-prefill - - --max-model-len - - "131072" - - --max-num-batched-tokens - - "16384" - - --all2all-backend - - deepep_high_throughput - - --kv-cache-dtype - - fp8_e4m3 - - --mamba-ssm-cache-dtype - - float16 - - --compilation-config=$(COMPILATION_CONFIG) - - --speculative-config=$(SPECULATIVE_CONFIG) - - --dyn-tool-call-parser - - nemotron_nano - - --dyn-reasoning-parser - - nemotron_nano - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: VLLM_CONFIG_ROOT - value: /tmp/vllm-config - - name: VLLM_CACHE_ROOT - value: /tmp/vllm-cache - - name: COMPILATION_CONFIG - valueFrom: - configMapKeyRef: - name: nemotron-3-super-h200-agentic-config - key: compilation-config - - name: SPECULATIVE_CONFIG - valueFrom: - configMapKeyRef: - name: nemotron-3-super-h200-agentic-config - key: speculative-config - - name: NCCL_IB_DISABLE - value: "1" - - name: PYTHONHASHSEED - value: "42" - envFrom: - - secretRef: - name: hf-token-secret - resources: - requests: - nvidia.com/gpu: "4" - memory: "256Gi" - limits: - nvidia.com/gpu: "4" - securityContext: - runAsUser: 0 - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 30 - timeoutSeconds: 20 - failureThreshold: 60 - volumeMounts: - - name: model-cache - mountPath: /model-cache - - name: dshm - mountPath: /dev/shm - workingDir: /workspace/ diff --git a/recipes/nemotron-3-super/vllm/agg-h200-chat/deploy.yaml b/recipes/nemotron-3-super/vllm/agg-h200-chat/deploy.yaml deleted file mode 100644 index 1d5768b36484..000000000000 --- a/recipes/nemotron-3-super/vllm/agg-h200-chat/deploy.yaml +++ /dev/null @@ -1,198 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Aggregated FP8 deployment of NVIDIA-Nemotron-3-Super-120B-A12B-FP8 on -# H200x4 (TP=4) with Expert Parallel + FlashInfer NVLink one-sided all-to-all. -# Hopper variant — drops Blackwell-specific NCCL SHARP / symmetric-memory -# env vars and the FlashInfer NVFP4 MoE backend (those are B200-only). -# -# Cluster requirements (same as B200): see ../agg-b200-chat/deploy.yaml. -apiVersion: v1 -kind: ConfigMap -metadata: - name: nemotron-3-super-h200-chat-config - labels: - app.kubernetes.io/name: nemotron-3-super-h200-chat - app.kubernetes.io/part-of: dynamo -data: - compilation-config: |- - { - "use_inductor_graph_partition": true, - "pass_config": { - "fuse_allreduce_rms": true, - "fuse_attn_quant": true - }, - "max_cudagraph_capture_size": 512 - } - speculative-config: |- - { - "method": "mtp", - "num_speculative_tokens": 3 - } - # Benchmarking-only: synthetic rejection sampling with a fixed AL - # (matches the speculative-config-synthetic pattern in Alexandre's k26 - # recipes). Switch SPECULATIVE_CONFIG env to this key to bench MTP - # at a known acceptance length without running the actual draft head. - speculative-config-synthetic: |- - { - "method": "mtp", - "num_speculative_tokens": 3, - "rejection_sample_method": "synthetic", - "synthetic_acceptance_length": 2.82 - } ---- -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: nemotron-3-super-h200-chat - labels: - app.kubernetes.io/name: nemotron-3-super-h200-chat - app.kubernetes.io/part-of: dynamo -spec: - backendFramework: vllm - components: - - name: Frontend - type: frontend - replicas: 1 - podTemplate: - spec: - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-super-dev.1 - command: - - python3 - args: - - -m - - dingo.frontend - - --trust-remote-code - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: DYN_ROUTER_MODE - value: kv - resources: {} - startupProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 10 - timeoutSeconds: 60 - failureThreshold: 60 - - - name: agg - type: worker - replicas: 2 - podTemplate: - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.product - operator: In - values: - - NVIDIA-H200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: dshm - emptyDir: - medium: Memory - sizeLimit: 16Gi - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-super-dev.1 - command: - - python3 - - -m - - dingo.vllm - args: - - --model - - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - - --served-model-name - - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - - --trust-remote-code - - --tensor-parallel-size - - "4" - - --enable-expert-parallel - - --enable-flashinfer-autotune - - --enable-prefix-caching - - --enable-chunked-prefill - - --max-model-len - - "131072" - - --max-num-batched-tokens - - "16384" - - --all2all-backend - - flashinfer_nvlink_one_sided - - --kv-cache-dtype - - fp8_e4m3 - - --mamba-ssm-cache-dtype - - float16 - - --compilation-config=$(COMPILATION_CONFIG) - - --speculative-config=$(SPECULATIVE_CONFIG) - - --dyn-tool-call-parser - - nemotron_nano - - --dyn-reasoning-parser - - nemotron_nano - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: VLLM_CONFIG_ROOT - value: /tmp/vllm-config - - name: VLLM_CACHE_ROOT - value: /tmp/vllm-cache - - name: COMPILATION_CONFIG - valueFrom: - configMapKeyRef: - name: nemotron-3-super-h200-chat-config - key: compilation-config - - name: SPECULATIVE_CONFIG - valueFrom: - configMapKeyRef: - name: nemotron-3-super-h200-chat-config - key: speculative-config - - name: NCCL_IB_DISABLE - value: "1" - - name: PYTHONHASHSEED - value: "42" - envFrom: - - secretRef: - name: hf-token-secret - resources: - requests: - nvidia.com/gpu: "4" - memory: "256Gi" - limits: - nvidia.com/gpu: "4" - securityContext: - runAsUser: 0 - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 30 - timeoutSeconds: 20 - failureThreshold: 60 - volumeMounts: - - name: model-cache - mountPath: /model-cache - - name: dshm - mountPath: /dev/shm - workingDir: /workspace/ diff --git a/recipes/nemotron-3-ultra/README.md b/recipes/nemotron-3-ultra/README.md deleted file mode 100644 index 15bc187ccbc7..000000000000 --- a/recipes/nemotron-3-ultra/README.md +++ /dev/null @@ -1,203 +0,0 @@ - - -# Nemotron-3-Ultra Recipes - -Recipes for **nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4** — a ~550B hybrid Mamba/Attention/MoE model (~55B active). - -We ship Dynamo + vLLM deployment profiles across B200 and H200, with aggregated and disaggregated serving modes. - -Runtime image: - -```text -nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 -``` - -The recipes pin `VLLM_DISABLED_KERNELS=FlashInferFP8ScaledMMLinearKernel` and pass `--no-enable-flashinfer-autotune` on vLLM workers. These settings select the non-FlashInfer FP8 linear kernel path used for the B200 benchmark rows and avoid the measured vLLM 0.22 FlashInfer FP8 regression. - -## Configurations - -| | B200 chat | H200 chat | B200 agentic | H200 agentic | B200 disaggregated agentic | -|--------------------------|-----------|-----------|--------------|--------------|-----------------------------| -| **GPU** | 4× B200 | 8× H200 | 4× B200 | 8× H200 | 4× B200 prefill + 4× B200 decode | -| **Mode** | aggregated | aggregated | aggregated | aggregated | disaggregated | -| **Framework** | Dynamo + vLLM | Dynamo + vLLM | Dynamo + vLLM | Dynamo + vLLM | Dynamo + vLLM | -| **Precision** | NVFP4 + FP8 | NVFP4 + FP8 | NVFP4 + FP8 | NVFP4 + FP8 | NVFP4 + FP8 | -| **Parallelism** | TP4 + EP | TP8 + EP | TP4 + EP | TP8 + EP | TP4 prefill + TP4 decode | -| **Routing** | KV-aware | KV-aware | KV-aware | KV-aware | KV-aware + P/D transfer | -| **Speculative decoding** | MTP, 1 token | MTP, 1 token | MTP, 1 token | MTP, 1 token | no MTP | -| **Max model length** | 262144 | 262144 | 262144 | 262144 | 262144 | -| **Max sequences** | 64 | 16 | 24 | 32 | 32 | -| **Max batched tokens** | 32768 | 32768 | 32768 | 32768 | 32768 | -| **Block size** | 64 | 64 | 64 | 64 | 64 | -| **Reference concurrency** | 18 | 10 | 20 | 8 | 32 | -| **Manifest** | `vllm/agg-b200-chat-mtp/deploy.yaml` | `vllm/agg-h200-chat-mtp/deploy.yaml` | `vllm/agg-b200-agentic-mtp/deploy.yaml` | `vllm/agg-h200-agentic-mtp/deploy.yaml` | `vllm/disagg-b200-agentic/deploy.yaml` | - -Aggregated no-MTP fallback manifests are also included under `vllm/agg-*-nomtp/deploy.yaml`. - -## Supported Features - -- Text-only chat -- Reasoning control through `chat_template_kwargs` -- Tool calling with `qwen3_coder` -- Ultra reasoning parser support through the model-local `ultra_v3_reasoning_parser.py` -- Raw Moontrace replay through AIPerf - -## Prerequisites - -1. **Dynamo Platform installed** on the target cluster with DGD CRDs served. -2. **NGC image pull secret** named `nvcr-secret`. -3. **Hugging Face token secret** named `hf-token-secret` when using the model download Job: - ```bash - kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="$HF_TOKEN" \ - -n ${NAMESPACE} - ``` -4. A `shared-model-cache` PVC containing the tokenizer-patched Ultra model view, or permission to create and populate it with the manifests in `model-cache/`. - -## Quick Start - -```bash -export NAMESPACE=your-namespace -``` - -### 1. Create or Validate Model Cache - -If the namespace does not already provide `shared-model-cache`, edit the storage class in `model-cache/model-cache.yaml`, then create and populate the PVC: - -```bash -kubectl apply -f model-cache/model-cache.yaml -n ${NAMESPACE} -kubectl apply -f model-cache/model-download.yaml -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/nemotron-ultra-model-download -n ${NAMESPACE} --timeout=12h -``` - -Validate the patched model view before deploying a server: - -```bash -kubectl apply -f model-cache/model-validate.yaml -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/nemotron-ultra-model-validate -n ${NAMESPACE} --timeout=30m -``` - -### 2. Deploy the DGD - -Pick the SKU, use-case, and speculative decoding mode for an aggregated recipe: - -```bash -SKU=b200 # or h200 -USECASE=chat # or agentic -SPEC=mtp # or nomtp - -kubectl apply -f vllm/agg-${SKU}-${USECASE}-${SPEC}/deploy.yaml -n ${NAMESPACE} -``` - -The DGD name includes the `-nomtp` suffix only for no-MTP recipes: - -```bash -DGD=ultra-agg-${SKU}-${USECASE}-${SPEC} -kubectl get dgd ${DGD} -n ${NAMESPACE} -w -``` - -Disaggregated recipes are currently agentic b200, no-MTP only. - -```bash -kubectl apply -f vllm/disagg-b200-agentic/deploy.yaml -n ${NAMESPACE} -``` - -### 3. Smoke Test - -```bash -kubectl port-forward svc/${DGD}-frontend 8000:8000 -n ${NAMESPACE} - -MODEL_ID=nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - -curl http://localhost:8000/v1/models -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d "{\"model\":\"${MODEL_ID}\", - \"messages\":[{\"role\":\"user\",\"content\":\"Hello!\"}], - \"max_tokens\":64, - \"chat_template_kwargs\":{\"enable_thinking\":false,\"force_nonempty_content\":true}}" -``` - -### 4. Benchmark - -See [`perf/README.md`](perf/README.md) for the full benchmark workflow — staging Moontrace-format traces on the PVC, running the AIPerf trace-replay Job ([`perf/perf.yaml`](perf/perf.yaml)), running a concurrency sweep, and fetching artifacts. - -## Benchmark Results - -B200 rows use 15% raw Moontrace replay with `raw_direct_no_filter` trace semantics. H200 rows use 300-sample replay evidence. All rows should be treated together with their matching recipe, image, trace, and server-shape artifacts. - -The B200 rows below point at the actual recipe manifests in this tree. `User output tok/s` is Gen TPS/user p50 from AIPerf; `System output tok/s/GPU` is TPS/GPU. - -| Recipe | GPU | Topology | Workload | MTP | Concurrency | User output tok/s | System output tok/s/GPU | -|--------|-----|----------|----------|-----|-------------|-------------------|-------------------------| -| `vllm/agg-b200-agentic-mtp/deploy.yaml` | B200 | AGG | agentic | yes | 20 | 80.6 | 310.8 | -| `vllm/agg-b200-agentic-nomtp/deploy.yaml` | B200 | AGG | agentic | no | 8 | 99.5 | 175.9 | -| `vllm/agg-b200-chat-mtp/deploy.yaml` | B200 | AGG | chat | yes | 18 | 52.0 | 201.4 | -| `vllm/agg-b200-chat-nomtp/deploy.yaml` | B200 | AGG | chat | no | 16 | 51.0 | 181.3 | -| `vllm/disagg-b200-agentic/deploy.yaml` | B200 | 1P1D | agentic | no | 32 | 61.6 | 231.1 | -| `vllm/agg-h200-agentic-mtp/deploy.yaml` | H200 | AGG | agentic | yes | 8 | 53.2 | 27.4 | -| `vllm/agg-h200-agentic-nomtp/deploy.yaml` | H200 | AGG | agentic | no | 8 | 52.3 | 26.5 | -| `vllm/agg-h200-chat-mtp/deploy.yaml` | H200 | AGG | chat | yes | 10 | 58.7 | 46.8 | -| `vllm/agg-h200-chat-nomtp/deploy.yaml` | H200 | AGG | chat | no | 8 | 54.2 | 43.0 | - - -## Reasoning Controls - -Ultra no-thinking request control: - -```json -{ - "chat_template_kwargs": { - "enable_thinking": false, - "force_nonempty_content": true - } -} -``` - -Ultra reasoning budget request control: - -```json -{ - "nvext": { - "max_thinking_tokens": 10 - } -} -``` - -Do not send `force_nonempty_content` as a top-level request parameter. - -## Known Issues - -1. Optional OpenAI/vLLM/NIM API fields are shared Dynamo API compatibility gaps, not Ultra recipe-specific failures. -2. Top-level reasoning controls such as `include_reasoning`, `thinking_token_budget`, `reasoning_effort`, and `usage.reasoning_tokens` are part of that shared API compatibility work. Use the Ultra-specific `chat_template_kwargs` and `nvext` controls above as the current model-specific workaround. -3. Do not remove `VLLM_DISABLED_KERNELS=FlashInferFP8ScaledMMLinearKernel` or `--no-enable-flashinfer-autotune` from the vLLM worker commands unless rerunning the benchmark qualification. These are part of the performance recipe. -4. Raw Moontrace replay may contain over-context or pathological long-generation rows. Do not drop those rows silently; preserve them as HTTP/error evidence or classify the run accordingly. - -## File Layout - -```text -recipes/nemotron-3-ultra/ - README.md - model-cache/ - README.md - model-cache.yaml # PVC - model-download.yaml # Job: populate patched Ultra model view - model-validate.yaml # Job: validate model/tokenizer/parser files - vllm/ - agg-b200-chat-mtp/deploy.yaml - agg-b200-chat-nomtp/deploy.yaml - agg-b200-agentic-mtp/deploy.yaml - agg-b200-agentic-nomtp/deploy.yaml - agg-h200-chat-mtp/deploy.yaml - agg-h200-chat-nomtp/deploy.yaml - agg-h200-agentic-mtp/deploy.yaml - agg-h200-agentic-nomtp/deploy.yaml - disagg-b200-agentic/deploy.yaml - perf/ - README.md # benchmark workflow - perf.yaml # AIPerf trace-replay Job - traces/ # 15%, 30%, and full Moontrace JSONL assets -``` diff --git a/recipes/nemotron-3-ultra/model-cache/README.md b/recipes/nemotron-3-ultra/model-cache/README.md deleted file mode 100644 index 80f227e0cd8a..000000000000 --- a/recipes/nemotron-3-ultra/model-cache/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Nemotron Ultra Model Cache - -## Required Model View - -Expected worker path (all B200 + H200 recipes share this): - -```text -/opt/models/patched/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 -``` - -Source repo and revision: - -```text -nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 -https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 -revision: main, or set `MODEL_REVISION` to the release-pinned commit when provided -``` - -The DGD recipes expose the same public model ID as the served model name: - -```text -nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 -``` - -Minimum files validated before vLLM startup: - -```text -config.json -tokenizer.json -tokenizer_config.json -generation_config.json -ultra_v3_reasoning_parser.py -``` - -## Manifests - -| Manifest | Purpose | -|---|---| -| `model-cache.yaml` | Namespace-local PVC contract for fresh namespaces. Edit `storageClassName` for the target cluster before apply. Do not apply over an existing platform-managed `shared-model-cache` PVC because PVC storage class and capacity are immutable. | -| `model-download.yaml` | No-GPU model population job. Downloads the pinned revision into the exact patched model path. | -| `model-validate.yaml` | No-GPU validation job. Fails with an actionable message if the model view is missing. | - -## Usage - -Then dry-run and apply in the target namespace. If the namespace already has a platform-managed `shared-model-cache`, skip `model-cache.yaml`; server-side dry-run against the existing PVC may warn about immutable field changes even though the platform cache itself is valid. - -```bash -kubectl -n "${NAMESPACE}" apply --dry-run=server -f model-cache.yaml -kubectl -n "${NAMESPACE}" apply --dry-run=server -f model-download.yaml -kubectl -n "${NAMESPACE}" apply --dry-run=server -f model-validate.yaml - -kubectl -n "${NAMESPACE}" apply -f model-cache.yaml -kubectl -n "${NAMESPACE}" apply -f model-download.yaml -kubectl -n "${NAMESPACE}" wait --for=condition=Complete job/nemotron-ultra-model-download --timeout=12h -kubectl -n "${NAMESPACE}" apply -f model-validate.yaml -kubectl -n "${NAMESPACE}" wait --for=condition=Complete job/nemotron-ultra-model-validate --timeout=30m -``` - -If the platform already provides `shared-model-cache`, skip `model-cache.yaml` and `model-download.yaml`, then run `model-validate.yaml` before applying a server recipe. diff --git a/recipes/nemotron-3-ultra/model-cache/model-cache.yaml b/recipes/nemotron-3-ultra/model-cache/model-cache.yaml deleted file mode 100644 index 4a9d07619ec1..000000000000 --- a/recipes/nemotron-3-ultra/model-cache/model-cache.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: shared-model-cache - labels: - app.kubernetes.io/name: nemotron-ultra-model-cache - app.kubernetes.io/part-of: nemotron-ultra -spec: - accessModes: - - ReadWriteMany - storageClassName: your-storage-class-name - resources: - requests: - storage: 1200Gi diff --git a/recipes/nemotron-3-ultra/model-cache/model-download.yaml b/recipes/nemotron-3-ultra/model-cache/model-download.yaml deleted file mode 100644 index 4145e8fd7a40..000000000000 --- a/recipes/nemotron-3-ultra/model-cache/model-download.yaml +++ /dev/null @@ -1,63 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: nemotron-ultra-model-download - labels: - app.kubernetes.io/name: nemotron-ultra-model-download - app.kubernetes.io/part-of: nemotron-ultra -spec: - backoffLimit: 1 - template: - metadata: - labels: - app.kubernetes.io/name: nemotron-ultra-model-download - app.kubernetes.io/part-of: nemotron-ultra - spec: - restartPolicy: Never - containers: - - name: download - image: python:3.10-slim - imagePullPolicy: IfNotPresent - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - python3 -m pip install --no-cache-dir -U huggingface_hub - export HF_HOME=/opt/models - python3 - <<'PY' - import os - from huggingface_hub import snapshot_download - - model_repo = "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4" - model_revision = os.environ.get("MODEL_REVISION", "main") - - model_path = "/opt/models/patched/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4" - os.makedirs(model_path, exist_ok=True) - snapshot_download( - repo_id=model_repo, - revision=model_revision, - local_dir=model_path, - ) - print(f"MODEL_DOWNLOAD_SNAPSHOT_DONE repo={model_repo} path={model_path}") - PY - MODEL_PATH="/opt/models/patched/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4" - test -r "${MODEL_PATH}/config.json" - test -r "${MODEL_PATH}/tokenizer.json" - test -r "${MODEL_PATH}/tokenizer_config.json" - test -r "${MODEL_PATH}/generation_config.json" - test -r "${MODEL_PATH}/ultra_v3_reasoning_parser.py" - echo "MODEL_DOWNLOAD_PASS path=${MODEL_PATH}" - envFrom: - - secretRef: - name: hf-token-secret - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache diff --git a/recipes/nemotron-3-ultra/model-cache/model-validate.yaml b/recipes/nemotron-3-ultra/model-cache/model-validate.yaml deleted file mode 100644 index 4a2064856786..000000000000 --- a/recipes/nemotron-3-ultra/model-cache/model-validate.yaml +++ /dev/null @@ -1,68 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: nemotron-ultra-model-validate - labels: - app.kubernetes.io/name: nemotron-ultra-model-validate - app.kubernetes.io/part-of: nemotron-ultra -spec: - backoffLimit: 0 - template: - metadata: - labels: - app.kubernetes.io/name: nemotron-ultra-model-validate - app.kubernetes.io/part-of: nemotron-ultra - spec: - restartPolicy: Never - containers: - - name: validate - image: python:3.10-slim - imagePullPolicy: IfNotPresent - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - required=( - config.json - tokenizer.json - tokenizer_config.json - generation_config.json - ultra_v3_reasoning_parser.py - ) - MODEL_PATH="/opt/models/patched/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4" - if [ ! -d "${MODEL_PATH}" ]; then - echo "MODEL_VALIDATE_FAIL class=model_path_missing path=${MODEL_PATH}" - echo "Run model-download.yaml or bind a platform shared-model-cache PVC containing the patched Ultra model view." - exit 42 - fi - for rel in "${required[@]}"; do - if [ ! -r "${MODEL_PATH}/${rel}" ]; then - echo "MODEL_VALIDATE_FAIL class=model_required_file_missing path=${MODEL_PATH}/${rel}" - echo "Run model-download.yaml or fix the platform shared-model-cache contents." - exit 43 - fi - done - python3 - <<'PY' - import json - from pathlib import Path - model_path = Path("/opt/models/patched/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4") - config = json.loads((model_path / "config.json").read_text()) - print( - "MODEL_VALIDATE_PASS", - "hf_model=nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", - "model_type=" + str(config.get("model_type")), - "path=" + str(model_path), - ) - PY - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache diff --git a/recipes/nemotron-3-ultra/perf/README.md b/recipes/nemotron-3-ultra/perf/README.md deleted file mode 100644 index 4c0903a70c27..000000000000 --- a/recipes/nemotron-3-ultra/perf/README.md +++ /dev/null @@ -1,167 +0,0 @@ - - -# Nemotron-3-Ultra Benchmark Recipe - -A single AIPerf trace-replay Job — [perf.yaml](perf.yaml) — covers the Nemotron-3-Ultra DGD variants. The benchmark is identical across variants; change `ENDPOINT`, `TRACE_FILE`, `CONCURRENCY`, and `TARGET_MODEL` in the Job env block to target a specific server. - -The Job waits for `GET /v1/models` on the DGD frontend to return `TARGET_MODEL`, runs a short warmup, then replays the configured Mooncake-format trace at one `CONCURRENCY` value. Artifacts are written to the shared model-cache PVC under `/opt/models/perf`. - -Benchmark rows assume the vLLM DGD manifests in this recipe, including the CUDA13 image and worker runtime settings: - -```text -image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 -VLLM_DISABLED_KERNELS=FlashInferFP8ScaledMMLinearKernel ---no-enable-flashinfer-autotune -``` - -The bench pod is co-located with the DGD frontend through pod affinity on the frontend host. If you run more than one benchmark in the same namespace, also update `metadata.name` and `labels.app` so Jobs and artifact directories stay distinct. - -## Targeting a Variant - -Edit the `env` block in [perf.yaml](perf.yaml): - -| Variant target | `ENDPOINT` | `TARGET_MODEL` | `TRACE_FILE` | Typical `CONCURRENCY` | -|---|---|---|---|---:| -| B200 AGG chat MTP | `ultra-agg-b200-chat-mtp-frontend:8000` | `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` | `/opt/models/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl` | 18 | -| B200 AGG chat no-MTP | `ultra-agg-b200-chat-nomtp-frontend:8000` | `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` | `/opt/models/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl` | 16 | -| B200 AGG agentic MTP | `ultra-agg-b200-agentic-mtp-frontend:8000` | `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` | `/opt/models/traces/nim_turbo_64k_400_90kv_agent_new_noschedule_short_15perc.jsonl` | 20 | -| B200 AGG agentic no-MTP | `ultra-agg-b200-agentic-nomtp-frontend:8000` | `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` | `/opt/models/traces/nim_turbo_64k_400_90kv_agent_new_noschedule_short_15perc.jsonl` | 8 | -| B200 1P1D agentic no-MTP | `ultra-disagg-b200-1p1d-agentic-nomtp-frontend:8000` | `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` | `/opt/models/traces/nim_turbo_64k_400_90kv_agent_new_noschedule_short_15perc.jsonl` | 32 | -| H200 AGG chat MTP | `ultra-agg-h200-chat-mtp-frontend:8000` | `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` | `/opt/models/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl` | 10 | -| H200 AGG chat no-MTP | `ultra-agg-h200-chat-nomtp-frontend:8000` | `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` | `/opt/models/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl` | 8 | -| H200 AGG agentic MTP | `ultra-agg-h200-agentic-mtp-frontend:8000` | `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` | `/opt/models/traces/nim_turbo_64k_400_90kv_agent_new_noschedule_short_15perc.jsonl` | 8 | -| H200 AGG agentic no-MTP | `ultra-agg-h200-agentic-nomtp-frontend:8000` | `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` | `/opt/models/traces/nim_turbo_64k_400_90kv_agent_new_noschedule_short_15perc.jsonl` | 8 | - -The default Job is configured for the B200 AGG chat MTP 15% trace at concurrency 18. For other release-style benchmark rows, use the trace/concurrency pair that matches the recipe row being reported. - -## Dataset - -The benchmark replays a Mooncake-format trace through `aiperf --custom-dataset-type mooncake_trace`. Each JSONL line describes one request with fields such as `input_length`, `output_length`, and `hash_ids`. - -Trace files included in this recipe: - -| Trace | Rows | -|---|---:| -| `traces/nim_turbo_8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl` | 1805 | -| `traces/nim_turbo_8k_1k_70kv_chat_new_noschedule_short_30perc.jsonl` | 3609 | -| `traces/nim_turbo_8k_1k_70kv_chat_new_noschedule.jsonl` | 12031 | -| `traces/nim_turbo_64k_400_90kv_agent_new_noschedule_short_15perc.jsonl` | 3541 | -| `traces/nim_turbo_64k_400_90kv_agent_new_noschedule_short_30perc.jsonl` | 7082 | -| `traces/nim_turbo_64k_400_90kv_agent_new_noschedule.jsonl` | 23608 | - -The 15% and 30% traces are prefix slices, not random samples, so they preserve trace order and cache warmup behavior. - -## Replay Policy - -The intended replay policy is raw direct Moontrace replay: - -- No context-length filtering. -- No OSL clipping. -- No synthetic output-length substitution. -- No `--export-http-trace`. -- No `bad_words` guard. -- `ignore_eos:true` is the only extra input. -- HTTP400 and no-content rows remain benchmark failure evidence. - -## Workflow - -```bash -export NAMESPACE=your-namespace -``` - -### 1. Deploy the DGD - -See instructions in the [recipe README](../README.md). - -### 2. Stage the Traces on the PVC - -Spin up a short-lived helper pod that mounts `shared-model-cache` at `/opt/models`, then copy the bundled traces in: - -```bash -kubectl run pvc-helper -n ${NAMESPACE} \ - --image=busybox:1.36 --restart=Never \ - --overrides='{"spec":{"containers":[{"name":"helper","image":"busybox:1.36","command":["sleep","3600"],"volumeMounts":[{"name":"model-cache","mountPath":"/opt/models"}]}],"volumes":[{"name":"model-cache","persistentVolumeClaim":{"claimName":"shared-model-cache"}}]}}' \ - --command -- sleep 3600 - -kubectl exec -n ${NAMESPACE} pvc-helper -- mkdir -p /opt/models/traces -kubectl cp traces/. ${NAMESPACE}/pvc-helper:/opt/models/traces/ -``` - -Keep `pvc-helper` around for fetching artifacts later, or delete it once staging is complete. - -### 3. Run the Benchmark - -```bash -kubectl apply -f perf.yaml -n ${NAMESPACE} - -kubectl logs -n ${NAMESPACE} -l job-name=ultra-bench -f - -kubectl wait --for=condition=Complete \ - job/ultra-bench \ - -n ${NAMESPACE} --timeout=7200s -``` - -### 4. Fetch Artifacts - -```bash -kubectl cp ${NAMESPACE}/pvc-helper:/opt/models/perf/_ultra-bench ./results -``` - -### 5. Cleanup - -```bash -kubectl delete job ultra-bench -n ${NAMESPACE} -kubectl delete pod pvc-helper -n ${NAMESPACE} -``` - -## Running a Concurrency Sweep - -`perf.yaml` runs a single `CONCURRENCY` value. To measure multiple concurrencies, clear server state between runs so residual KV cache and prefix-cache hits from the previous run do not skew results. - -For each concurrency value: - -```bash -kubectl delete job ultra-bench -n ${NAMESPACE} --ignore-not-found - -DGD=ultra-agg-b200-chat-mtp -kubectl delete pods -n ${NAMESPACE} \ - -l nvidia.com/dynamo-graph-deployment-name=${DGD},nvidia.com/dynamo-component-type=worker - -kubectl apply -f perf.yaml -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/ultra-bench -n ${NAMESPACE} --timeout=7200s -``` - -The Job's `wait_for_model_ready` loop handles the worker restart window by polling `/v1/models` until the frontend reports the target model. - -## Tunable Environment Variables - -Edit the `env` block on the Job: - -| Variable | Default | Notes | -|---|---|---| -| `ENDPOINT` | `ultra-agg-b200-chat-mtp-frontend:8000` | DGD frontend service:port | -| `TRACE_FILE` | `/opt/models/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl` | Swap to chat or agentic 15%, 30%, or full trace | -| `CONCURRENCY` | `18` | Single value; see [Running a Concurrency Sweep](#running-a-concurrency-sweep) | -| `TARGET_MODEL` | `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` | Must match the served model name on the DGD frontend | -| `TOKENIZER_PATH` | `/opt/models/patched/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` | Used by AIPerf tokenization | -| `AIPERF_VERSION` | `0.10.0` | aiperf version used for benchmarking | -| `ROOT_ARTIFACT_DIR` | `/opt/models/perf` | Shared PVC artifact root | - -## Artifacts - -Results are written to: - -```text -/opt/models/perf/_/ - trace_replay_manifest.json - warmup/ - NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4_trace_c_/ - profile_export.json - inputs.json - ... -``` - -For release evidence, preserve the AIPerf output directory, Job logs, DGD manifest, image digest, trace SHA, server-shape evidence, and model-cache validation proof. diff --git a/recipes/nemotron-3-ultra/perf/perf.yaml b/recipes/nemotron-3-ultra/perf/perf.yaml deleted file mode 100644 index 45420627d345..000000000000 --- a/recipes/nemotron-3-ultra/perf/perf.yaml +++ /dev/null @@ -1,211 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# AIPerf raw trace-replay benchmark for Nemotron Ultra DGDs. -# -# One Job covers the Ultra DGD variants. To target a specific DGD, edit the -# env values below and, when running multiple jobs in one namespace, also edit -# metadata.name and labels.app. -# -# Trace policy: raw_direct_no_filter. Do not drop over-context rows, clip OSL, -# synthesize output lengths, or export HTTP traces. HTTP400/no-content rows are -# benchmark failure evidence. -# -# Results: /opt/models/perf/_/ -apiVersion: batch/v1 -kind: Job -metadata: - name: ultra-bench -spec: - backoffLimit: 1 - activeDeadlineSeconds: 7200 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: ultra-bench - spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: nvidia.com/dynamo-component-type - operator: In - values: - - frontend - - key: nvidia.com/dynamo-graph-deployment-name - operator: In - values: - - ultra-agg-b200-chat-mtp - - ultra-agg-b200-agentic-mtp - - ultra-agg-b200-chat-nomtp - - ultra-agg-b200-agentic-nomtp - - ultra-agg-h200-chat-mtp - - ultra-agg-h200-agentic-mtp - - ultra-agg-h200-chat-nomtp - - ultra-agg-h200-agentic-nomtp - - ultra-disagg-b200-1p1d-agentic-nomtp - topologyKey: kubernetes.io/hostname - containers: - - name: perf - image: python:3.12-slim - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -c - - | - set -euo pipefail - ulimit -n 600000 - apt-get update && apt-get install -y curl jq procps git coreutils && apt-get clean - pip install "aiperf==${AIPERF_VERSION}" protobuf "transformers==4.57.3" "tiktoken==0.13.0" - sysctl -w net.ipv4.ip_local_port_range="1024 65000" 2>/dev/null || true - export COLUMNS=200 - EPOCH=$(date +%s) - - wait_for_model_ready() { - local max_attempts=${WAIT_FOR_MODEL_MAX_ATTEMPTS:-720} - local attempt=0 - echo "Waiting for model '${TARGET_MODEL}' at ${ENDPOINT}/v1/models (max ${max_attempts} attempts) ..." - while ! curl -sf "http://${ENDPOINT}/v1/models" | jq -e --arg m "${TARGET_MODEL}" '.data[]? | select(.id == $m)' >/dev/null 2>&1; do - attempt=$((attempt + 1)) - if [ "${attempt}" -ge "${max_attempts}" ]; then - echo "ERROR: model '${TARGET_MODEL}' did not become ready after ${max_attempts} attempts." - exit 1 - fi - echo "[$(date '+%H:%M:%S')] not ready (attempt ${attempt}/${max_attempts}), sleeping 5s" - sleep 5 - done - curl -s "http://${ENDPOINT}/v1/models" | jq . - } - - if [ ! -f "${TRACE_FILE}" ]; then - echo "ERROR: trace file not found at ${TRACE_FILE}" - exit 1 - fi - if [ ! -f "${TOKENIZER_PATH}/tokenizer.json" ]; then - echo "ERROR: tokenizer not found at ${TOKENIZER_PATH}/tokenizer.json" - exit 1 - fi - - REQUEST_COUNT=$(wc -l < "${TRACE_FILE}" | tr -d ' ') - if [ "${REQUEST_COUNT}" -le 0 ]; then - echo "ERROR: trace file is empty: ${TRACE_FILE}" - exit 1 - fi - - wait_for_model_ready - ROOT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - mkdir -p "${ROOT_DIR}" - jq -n \ - --arg trace_file "${TRACE_FILE}" \ - --arg request_count "${REQUEST_COUNT}" \ - --arg concurrency "${CONCURRENCY}" \ - --arg target_model "${TARGET_MODEL}" \ - --arg endpoint "http://${ENDPOINT}" \ - '{ - trace_file: $trace_file, - request_count: ($request_count | tonumber), - concurrency: ($concurrency | tonumber), - target_model: $target_model, - endpoint: $endpoint, - context_filtering: false, - osl_clipping: false, - output_length_substitution: false, - export_http_trace: false - }' | tee "${ROOT_DIR}/trace_replay_manifest.json" - - echo "==============================================" - echo "Trace Replay Benchmark (AIPerf)" - echo "==============================================" - echo "Endpoint: http://${ENDPOINT}" - echo "Model: ${TARGET_MODEL}" - echo "Trace file: ${TRACE_FILE}" - echo "Request count: ${REQUEST_COUNT}" - echo "Concurrency: ${CONCURRENCY}" - echo "Artifact root: ${ROOT_DIR}" - echo "==============================================" - - WARMUP_DIR="${ROOT_DIR}/warmup" - mkdir -p "${WARMUP_DIR}" - aiperf profile \ - -m "${TARGET_MODEL}" \ - --tokenizer "${TOKENIZER_PATH}" \ - --tokenizer-trust-remote-code \ - --url "http://${ENDPOINT}" \ - --streaming \ - --ui simple \ - --isl 8000 \ - --osl 1000 \ - --concurrency 1 \ - --request-count 5 \ - --artifact-dir "${WARMUP_DIR}" - echo "Warmup complete" - - MODEL_BASE="${TARGET_MODEL##*/}" - TS=$(date +'%Y%m%d_%H%M%S') - RUN_DIR="${ROOT_DIR}/${MODEL_BASE}_trace_c${CONCURRENCY}_${TS}" - mkdir -p "${RUN_DIR}" - aiperf profile \ - -m "${TARGET_MODEL}" \ - --tokenizer "${TOKENIZER_PATH}" \ - --tokenizer-trust-remote-code \ - --input-file "${TRACE_FILE}" \ - --custom-dataset-type mooncake_trace \ - --num-requests $(wc -l < "${TRACE_FILE}") \ - --prompt-input-tokens-block-size 512 \ - --url "http://${ENDPOINT}" \ - --streaming \ - --use-server-token-count \ - --extra-inputs ignore_eos:true \ - --concurrency "${CONCURRENCY}" \ - --random-seed 42 \ - --ui simple \ - --artifact-dir "${RUN_DIR}" \ - --request-timeout-seconds 1200 \ - --workers-max "${CONCURRENCY}" - echo "Concurrency ${CONCURRENCY} complete; artifacts in ${RUN_DIR}" - ls -la "${RUN_DIR}" || true - echo "Done. Root: ${ROOT_DIR}" - env: - - name: ENDPOINT - value: ultra-agg-b200-chat-mtp-frontend:8000 - - name: TRACE_FILE - value: /opt/models/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl - - name: CONCURRENCY - value: "18" - - name: TARGET_MODEL - value: nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: TOKENIZER_PATH - value: /opt/models/patched/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: AIPERF_HTTP_CONNECTION_LIMIT - value: "200" - - name: AIPERF_VERSION - value: "0.10.0" - - name: AIPERF_HTTP_SO_RCVTIMEO - value: "120" - - name: AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT - value: "3600" - - name: AIPERF_DATASET_CONFIGURATION_TIMEOUT - value: "3600" - - name: JOB_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.labels['job-name'] - - name: ROOT_ARTIFACT_DIR - value: /opt/models/perf - - name: HF_HOME - value: /opt/models - - name: PYTHONUNBUFFERED - value: "1" - volumeMounts: - - name: model-cache - mountPath: /opt/models - restartPolicy: Never - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: shared-model-cache diff --git a/recipes/nemotron-3-ultra/perf/traces/.gitattributes b/recipes/nemotron-3-ultra/perf/traces/.gitattributes deleted file mode 100644 index 8c9c4f1c7835..000000000000 --- a/recipes/nemotron-3-ultra/perf/traces/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -*.jsonl filter=lfs diff=lfs merge=lfs -text diff --git a/recipes/nemotron-3-ultra/perf/traces/nim_turbo_64k_400_90kv_agent_new_noschedule.jsonl b/recipes/nemotron-3-ultra/perf/traces/nim_turbo_64k_400_90kv_agent_new_noschedule.jsonl deleted file mode 100644 index aea03789f3d9..000000000000 --- a/recipes/nemotron-3-ultra/perf/traces/nim_turbo_64k_400_90kv_agent_new_noschedule.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:aa0a64efcad575a01949e46e28dded094f20733db8708e0bc91b88ce9981fed6 -size 17623982 diff --git a/recipes/nemotron-3-ultra/perf/traces/nim_turbo_64k_400_90kv_agent_new_noschedule_short_15perc.jsonl b/recipes/nemotron-3-ultra/perf/traces/nim_turbo_64k_400_90kv_agent_new_noschedule_short_15perc.jsonl deleted file mode 100644 index 3fcfa9915c9e..000000000000 --- a/recipes/nemotron-3-ultra/perf/traces/nim_turbo_64k_400_90kv_agent_new_noschedule_short_15perc.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f20d3f2bc83dd1306cda659fbe34e7c4d85ca5497626c98bc0b1c4d2211379d0 -size 2722326 diff --git a/recipes/nemotron-3-ultra/perf/traces/nim_turbo_64k_400_90kv_agent_new_noschedule_short_30perc.jsonl b/recipes/nemotron-3-ultra/perf/traces/nim_turbo_64k_400_90kv_agent_new_noschedule_short_30perc.jsonl deleted file mode 100644 index 1f5891b41524..000000000000 --- a/recipes/nemotron-3-ultra/perf/traces/nim_turbo_64k_400_90kv_agent_new_noschedule_short_30perc.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:dae61f92fee05038f1dcb76474dba83c1b3c174f3d187589c7d0a16a7012eb79 -size 5401776 diff --git a/recipes/nemotron-3-ultra/perf/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule.jsonl b/recipes/nemotron-3-ultra/perf/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule.jsonl deleted file mode 100644 index a1ab5ad24a89..000000000000 --- a/recipes/nemotron-3-ultra/perf/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5f369eb75ce639ad8b05cc209bb534bfedd627e9f7b923de32888155b4c9085a -size 5648304 diff --git a/recipes/nemotron-3-ultra/perf/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl b/recipes/nemotron-3-ultra/perf/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl deleted file mode 100644 index 1eea1bd8955c..000000000000 --- a/recipes/nemotron-3-ultra/perf/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b1221bca72b69f842897f339624306a84857f1b55ea0d866525f94d9ceb9b871 -size 894687 diff --git a/recipes/nemotron-3-ultra/perf/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule_short_30perc.jsonl b/recipes/nemotron-3-ultra/perf/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule_short_30perc.jsonl deleted file mode 100644 index 0ef335a203b3..000000000000 --- a/recipes/nemotron-3-ultra/perf/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule_short_30perc.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6de2a977b40dba8735a70aa5f9ab0d84e32530d7cf1fc9d24efc631c341f9c17 -size 1790696 diff --git a/recipes/nemotron-3-ultra/vllm/agg-b200-agentic-mtp/deploy.yaml b/recipes/nemotron-3-ultra/vllm/agg-b200-agentic-mtp/deploy.yaml deleted file mode 100644 index f065eaadc7b6..000000000000 --- a/recipes/nemotron-3-ultra/vllm/agg-b200-agentic-mtp/deploy.yaml +++ /dev/null @@ -1,254 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: ultra-agg-b200-agentic-mtp - labels: - app.kubernetes.io/name: ultra-agg-b200-agentic-mtp - app.kubernetes.io/part-of: nemotron-ultra - nemotron-ultra.nvidia.com/backend: vllm - nemotron-ultra.nvidia.com/topology: agg1-tp4-mtp1 - nemotron-ultra.nvidia.com/workload: agentic -spec: - backendFramework: vllm - env: - - name: HF_HOME - value: /opt/models - - name: HF_HUB_CACHE - value: /opt/models/hub - - name: HF_HUB_OFFLINE - value: "1" - - name: TRANSFORMERS_OFFLINE - value: "1" - - name: SERVED_MODEL_NAME - value: nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: MODEL_PATH - value: /opt/models/patched/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: LOG_DIR - value: /artifacts/logs - - name: HOME - value: /tmp - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: XDG_CACHE_HOME - value: /tmp/cache - - name: TORCH_EXTENSIONS_DIR - value: /tmp/torch_extensions - - name: TRITON_CACHE_DIR - value: /tmp/triton - - name: VLLM_CACHE_ROOT - value: /tmp/vllm - - name: PYTHONHASHSEED - value: "0" - - name: VLLM_LOGGING_LEVEL - value: INFO - - name: VLLM_WORKER_MULTIPROC_METHOD - value: spawn - - name: VLLM_ALLREDUCE_USE_SYMM_MEM - value: "0" - - name: VLLM_DISABLED_KERNELS - value: FlashInferFP8ScaledMMLinearKernel - - name: VLLM_SSM_CONV_STATE_LAYOUT - value: DS - - name: VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE - value: "1" - - name: DYN_VLLM_APPEND_PREFILL_OUTPUT_TOKENS - value: "0" - - name: NCCL_IB_DISABLE - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: DYN_LOG - value: info,dynamo_kv_router=debug,dynamo_llm::kv_router=debug - components: - - name: Frontend - type: frontend - replicas: 1 - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: frontend - nemotron-ultra.nvidia.com/workload: agentic - spec: - imagePullSecrets: - - name: nvcr-secret - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - mkdir -p "${LOG_DIR:-/tmp/nemotron-ultra}/status" - frontend_args=( - --router-mode kv - --router-kv-events - --kv-cache-block-size 64 - --router-reset-states - --http-host 0.0.0.0 - --http-port 8000 - ) - printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dingo.frontend "${frontend_args[@]}" - ports: - - name: http - containerPort: 8000 - startupProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - readinessProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 3 - failureThreshold: 3 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - emptyDir: {} - - name: runtime-tmp - emptyDir: {} - - name: VllmWorker - type: worker - replicas: 1 - sharedMemorySize: 64Gi - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: aggregate-worker - nemotron-ultra.nvidia.com/workload: agentic - spec: - imagePullSecrets: - - name: nvcr-secret - runtimeClassName: nvidia - nodeSelector: - nvidia.com/gpu.product: NVIDIA-B200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - test -r "${MODEL_PATH}/config.json" || { echo "MODEL_PREFLIGHT_FAIL class=model_config_missing path=${MODEL_PATH}/config.json"; exit 42; } - test -r "${MODEL_PATH}/ultra_v3_reasoning_parser.py" || { echo "MODEL_PREFLIGHT_FAIL class=reasoning_parser_missing path=${MODEL_PATH}/ultra_v3_reasoning_parser.py"; exit 43; } - echo "PRESTART_GPU_GUARD_BEGIN role=agg workload=agentic ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - nvidia-smi -L - nvidia-smi --query-gpu=index,uuid,memory.total,memory.used,memory.free,utilization.gpu --format=csv,noheader,nounits - visible_count="$(nvidia-smi -L | grep -c '^GPU ')" - if [ "${visible_count}" -ne 4 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_visibility_count_mismatch role=agg workload=agentic visible_count=${visible_count}" - exit 96 - fi - mem_rows="$(nvidia-smi --query-gpu=index,uuid,memory.used,utilization.gpu --format=csv,noheader,nounits)" - echo "${mem_rows}" - dirty_count="$(printf '%s\n' "${mem_rows}" | awk -F',' '{gsub(/ /,"",$3); if (($3+0)>1024) c++} END{print c+0}')" - if [ "${dirty_count}" -ne 0 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_memory_dirty role=agg workload=agentic dirty_count=${dirty_count}" - exit 97 - fi - compute_apps="$(nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_memory --format=csv,noheader,nounits || true)" - if [ -n "${compute_apps}" ]; then - printf '%s\n' "${compute_apps}" - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_compute_apps_visible role=agg workload=agentic" - exit 98 - fi - echo "PRESTART_GPU_GUARD_PASS role=agg workload=agentic" - ulimit -l unlimited - exec python3 -m dingo.vllm \ - --model "${MODEL_PATH}" \ - --served-model-name "${SERVED_MODEL_NAME}" \ - --tensor-parallel-size 4 \ - --trust-remote-code \ - --max-model-len 262144 \ - --max-num-seqs 24 \ - --max-num-batched-tokens 32768 \ - --gpu-memory-utilization 0.9 \ - --no-enable-flashinfer-autotune \ - --block-size 64 \ - --enable-expert-parallel \ - --mamba-cache-mode align \ - --enable-prefix-caching \ - --spec-method nemotron_h_mtp \ - --spec-tokens 1 \ - --dyn-tool-call-parser qwen3_coder \ - --dyn-reasoning-parser nemotron3 \ - --reasoning-parser-plugin "${MODEL_PATH}/ultra_v3_reasoning_parser.py" \ - --reasoning-parser nemotron_v3 \ - --no-disable-hybrid-kv-cache-manager - envFrom: - - secretRef: - name: hf-token-secret - resources: - requests: - nvidia.com/gpu: "4" - memory: 750Gi - ephemeral-storage: 20Gi - limits: - nvidia.com/gpu: "4" - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - runAsUser: 0 - runAsGroup: 0 - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 20 - failureThreshold: 30 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - - name: flashinfer-cubins - mountPath: /usr/local/lib/python3.12/dist-packages/flashinfer_cubin/cubins/flashinfer - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - emptyDir: {} - - name: runtime-tmp - emptyDir: {} - - name: flashinfer-cubins - emptyDir: - medium: Memory diff --git a/recipes/nemotron-3-ultra/vllm/agg-b200-agentic-nomtp/deploy.yaml b/recipes/nemotron-3-ultra/vllm/agg-b200-agentic-nomtp/deploy.yaml deleted file mode 100644 index a0d3e0fbbf65..000000000000 --- a/recipes/nemotron-3-ultra/vllm/agg-b200-agentic-nomtp/deploy.yaml +++ /dev/null @@ -1,252 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: ultra-agg-b200-agentic-nomtp - labels: - app.kubernetes.io/name: ultra-agg-b200-agentic-nomtp - app.kubernetes.io/part-of: nemotron-ultra - nemotron-ultra.nvidia.com/backend: vllm - nemotron-ultra.nvidia.com/topology: agg1-tp4-nomtp - nemotron-ultra.nvidia.com/workload: agentic -spec: - backendFramework: vllm - env: - - name: HF_HOME - value: /opt/models - - name: HF_HUB_CACHE - value: /opt/models/hub - - name: HF_HUB_OFFLINE - value: "1" - - name: TRANSFORMERS_OFFLINE - value: "1" - - name: SERVED_MODEL_NAME - value: nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: MODEL_PATH - value: /opt/models/patched/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: LOG_DIR - value: /artifacts/logs - - name: HOME - value: /tmp - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: XDG_CACHE_HOME - value: /tmp/cache - - name: TORCH_EXTENSIONS_DIR - value: /tmp/torch_extensions - - name: TRITON_CACHE_DIR - value: /tmp/triton - - name: VLLM_CACHE_ROOT - value: /tmp/vllm - - name: PYTHONHASHSEED - value: "0" - - name: VLLM_LOGGING_LEVEL - value: INFO - - name: VLLM_WORKER_MULTIPROC_METHOD - value: spawn - - name: VLLM_ALLREDUCE_USE_SYMM_MEM - value: "0" - - name: VLLM_DISABLED_KERNELS - value: FlashInferFP8ScaledMMLinearKernel - - name: VLLM_SSM_CONV_STATE_LAYOUT - value: DS - - name: VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE - value: "1" - - name: DYN_VLLM_APPEND_PREFILL_OUTPUT_TOKENS - value: "0" - - name: NCCL_IB_DISABLE - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: DYN_LOG - value: info,dynamo_kv_router=debug,dynamo_llm::kv_router=debug - components: - - name: Frontend - type: frontend - replicas: 1 - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: frontend - nemotron-ultra.nvidia.com/workload: agentic - spec: - imagePullSecrets: - - name: nvcr-secret - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - mkdir -p "${LOG_DIR:-/tmp/nemotron-ultra}/status" - frontend_args=( - --router-mode kv - --router-kv-events - --kv-cache-block-size 64 - --router-reset-states - --http-host 0.0.0.0 - --http-port 8000 - ) - printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dingo.frontend "${frontend_args[@]}" - ports: - - name: http - containerPort: 8000 - startupProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - readinessProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 3 - failureThreshold: 3 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - emptyDir: {} - - name: runtime-tmp - emptyDir: {} - - name: VllmWorker - type: worker - replicas: 1 - sharedMemorySize: 64Gi - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: aggregate-worker - nemotron-ultra.nvidia.com/workload: agentic - spec: - imagePullSecrets: - - name: nvcr-secret - runtimeClassName: nvidia - nodeSelector: - nvidia.com/gpu.product: NVIDIA-B200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - test -r "${MODEL_PATH}/config.json" || { echo "MODEL_PREFLIGHT_FAIL class=model_config_missing path=${MODEL_PATH}/config.json"; exit 42; } - test -r "${MODEL_PATH}/ultra_v3_reasoning_parser.py" || { echo "MODEL_PREFLIGHT_FAIL class=reasoning_parser_missing path=${MODEL_PATH}/ultra_v3_reasoning_parser.py"; exit 43; } - echo "PRESTART_GPU_GUARD_BEGIN role=agg workload=agentic ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - nvidia-smi -L - nvidia-smi --query-gpu=index,uuid,memory.total,memory.used,memory.free,utilization.gpu --format=csv,noheader,nounits - visible_count="$(nvidia-smi -L | grep -c '^GPU ')" - if [ "${visible_count}" -ne 4 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_visibility_count_mismatch role=agg workload=agentic visible_count=${visible_count}" - exit 96 - fi - mem_rows="$(nvidia-smi --query-gpu=index,uuid,memory.used,utilization.gpu --format=csv,noheader,nounits)" - echo "${mem_rows}" - dirty_count="$(printf '%s\n' "${mem_rows}" | awk -F',' '{gsub(/ /,"",$3); if (($3+0)>1024) c++} END{print c+0}')" - if [ "${dirty_count}" -ne 0 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_memory_dirty role=agg workload=agentic dirty_count=${dirty_count}" - exit 97 - fi - compute_apps="$(nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_memory --format=csv,noheader,nounits || true)" - if [ -n "${compute_apps}" ]; then - printf '%s\n' "${compute_apps}" - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_compute_apps_visible role=agg workload=agentic" - exit 98 - fi - echo "PRESTART_GPU_GUARD_PASS role=agg workload=agentic" - ulimit -l unlimited - exec python3 -m dingo.vllm \ - --model "${MODEL_PATH}" \ - --served-model-name "${SERVED_MODEL_NAME}" \ - --tensor-parallel-size 4 \ - --trust-remote-code \ - --max-model-len 262144 \ - --max-num-seqs 32 \ - --max-num-batched-tokens 65536 \ - --gpu-memory-utilization 0.9 \ - --no-enable-flashinfer-autotune \ - --block-size 64 \ - --enable-expert-parallel \ - --mamba-cache-mode align \ - --enable-prefix-caching \ - --dyn-tool-call-parser qwen3_coder \ - --dyn-reasoning-parser nemotron3 \ - --reasoning-parser-plugin "${MODEL_PATH}/ultra_v3_reasoning_parser.py" \ - --reasoning-parser nemotron_v3 \ - --no-disable-hybrid-kv-cache-manager - envFrom: - - secretRef: - name: hf-token-secret - resources: - requests: - nvidia.com/gpu: "4" - memory: 750Gi - ephemeral-storage: 20Gi - limits: - nvidia.com/gpu: "4" - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - runAsUser: 0 - runAsGroup: 0 - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 20 - failureThreshold: 30 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - - name: flashinfer-cubins - mountPath: /usr/local/lib/python3.12/dist-packages/flashinfer_cubin/cubins/flashinfer - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - emptyDir: {} - - name: runtime-tmp - emptyDir: {} - - name: flashinfer-cubins - emptyDir: - medium: Memory diff --git a/recipes/nemotron-3-ultra/vllm/agg-b200-chat-mtp/deploy.yaml b/recipes/nemotron-3-ultra/vllm/agg-b200-chat-mtp/deploy.yaml deleted file mode 100644 index 5d8bd352a5f3..000000000000 --- a/recipes/nemotron-3-ultra/vllm/agg-b200-chat-mtp/deploy.yaml +++ /dev/null @@ -1,254 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: ultra-agg-b200-chat-mtp - labels: - app.kubernetes.io/name: ultra-agg-b200-chat-mtp - app.kubernetes.io/part-of: nemotron-ultra - nemotron-ultra.nvidia.com/backend: vllm - nemotron-ultra.nvidia.com/topology: agg1-tp4-mtp1 - nemotron-ultra.nvidia.com/workload: chat -spec: - backendFramework: vllm - env: - - name: HF_HOME - value: /opt/models - - name: HF_HUB_CACHE - value: /opt/models/hub - - name: HF_HUB_OFFLINE - value: "1" - - name: TRANSFORMERS_OFFLINE - value: "1" - - name: SERVED_MODEL_NAME - value: nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: MODEL_PATH - value: /opt/models/patched/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: LOG_DIR - value: /artifacts/logs - - name: HOME - value: /tmp - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: XDG_CACHE_HOME - value: /tmp/cache - - name: TORCH_EXTENSIONS_DIR - value: /tmp/torch_extensions - - name: TRITON_CACHE_DIR - value: /tmp/triton - - name: VLLM_CACHE_ROOT - value: /tmp/vllm - - name: PYTHONHASHSEED - value: "0" - - name: VLLM_LOGGING_LEVEL - value: INFO - - name: VLLM_WORKER_MULTIPROC_METHOD - value: spawn - - name: VLLM_ALLREDUCE_USE_SYMM_MEM - value: "0" - - name: VLLM_DISABLED_KERNELS - value: FlashInferFP8ScaledMMLinearKernel - - name: VLLM_SSM_CONV_STATE_LAYOUT - value: DS - - name: VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE - value: "1" - - name: DYN_VLLM_APPEND_PREFILL_OUTPUT_TOKENS - value: "0" - - name: NCCL_IB_DISABLE - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: DYN_LOG - value: info,dynamo_kv_router=debug,dynamo_llm::kv_router=debug - components: - - name: Frontend - type: frontend - replicas: 1 - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: frontend - nemotron-ultra.nvidia.com/workload: chat - spec: - imagePullSecrets: - - name: nvcr-secret - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - mkdir -p "${LOG_DIR:-/tmp/nemotron-ultra}/status" - frontend_args=( - --router-mode kv - --router-kv-events - --kv-cache-block-size 64 - --router-reset-states - --http-host 0.0.0.0 - --http-port 8000 - ) - printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dingo.frontend "${frontend_args[@]}" - ports: - - name: http - containerPort: 8000 - startupProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - readinessProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 3 - failureThreshold: 3 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - emptyDir: {} - - name: runtime-tmp - emptyDir: {} - - name: VllmWorker - type: worker - replicas: 1 - sharedMemorySize: 64Gi - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: aggregate-worker - nemotron-ultra.nvidia.com/workload: chat - spec: - imagePullSecrets: - - name: nvcr-secret - runtimeClassName: nvidia - nodeSelector: - nvidia.com/gpu.product: NVIDIA-B200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - test -r "${MODEL_PATH}/config.json" || { echo "MODEL_PREFLIGHT_FAIL class=model_config_missing path=${MODEL_PATH}/config.json"; exit 42; } - test -r "${MODEL_PATH}/ultra_v3_reasoning_parser.py" || { echo "MODEL_PREFLIGHT_FAIL class=reasoning_parser_missing path=${MODEL_PATH}/ultra_v3_reasoning_parser.py"; exit 43; } - echo "PRESTART_GPU_GUARD_BEGIN role=agg workload=chat ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - nvidia-smi -L - nvidia-smi --query-gpu=index,uuid,memory.total,memory.used,memory.free,utilization.gpu --format=csv,noheader,nounits - visible_count="$(nvidia-smi -L | grep -c '^GPU ')" - if [ "${visible_count}" -ne 4 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_visibility_count_mismatch role=agg workload=chat visible_count=${visible_count}" - exit 96 - fi - mem_rows="$(nvidia-smi --query-gpu=index,uuid,memory.used,utilization.gpu --format=csv,noheader,nounits)" - echo "${mem_rows}" - dirty_count="$(printf '%s\n' "${mem_rows}" | awk -F',' '{gsub(/ /,"",$3); if (($3+0)>1024) c++} END{print c+0}')" - if [ "${dirty_count}" -ne 0 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_memory_dirty role=agg workload=chat dirty_count=${dirty_count}" - exit 97 - fi - compute_apps="$(nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_memory --format=csv,noheader,nounits || true)" - if [ -n "${compute_apps}" ]; then - printf '%s\n' "${compute_apps}" - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_compute_apps_visible role=agg workload=chat" - exit 98 - fi - echo "PRESTART_GPU_GUARD_PASS role=agg workload=chat" - ulimit -l unlimited - exec python3 -m dingo.vllm \ - --model "${MODEL_PATH}" \ - --served-model-name "${SERVED_MODEL_NAME}" \ - --tensor-parallel-size 4 \ - --trust-remote-code \ - --max-model-len 262144 \ - --max-num-seqs 64 \ - --max-num-batched-tokens 32768 \ - --gpu-memory-utilization 0.9 \ - --no-enable-flashinfer-autotune \ - --block-size 64 \ - --enable-expert-parallel \ - --mamba-cache-mode align \ - --enable-prefix-caching \ - --spec-method nemotron_h_mtp \ - --spec-tokens 1 \ - --dyn-tool-call-parser qwen3_coder \ - --dyn-reasoning-parser nemotron3 \ - --reasoning-parser-plugin "${MODEL_PATH}/ultra_v3_reasoning_parser.py" \ - --reasoning-parser nemotron_v3 \ - --no-disable-hybrid-kv-cache-manager - envFrom: - - secretRef: - name: hf-token-secret - resources: - requests: - nvidia.com/gpu: "4" - memory: 750Gi - ephemeral-storage: 20Gi - limits: - nvidia.com/gpu: "4" - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - runAsUser: 0 - runAsGroup: 0 - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 20 - failureThreshold: 30 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - - name: flashinfer-cubins - mountPath: /usr/local/lib/python3.12/dist-packages/flashinfer_cubin/cubins/flashinfer - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - emptyDir: {} - - name: runtime-tmp - emptyDir: {} - - name: flashinfer-cubins - emptyDir: - medium: Memory diff --git a/recipes/nemotron-3-ultra/vllm/agg-b200-chat-nomtp/deploy.yaml b/recipes/nemotron-3-ultra/vllm/agg-b200-chat-nomtp/deploy.yaml deleted file mode 100644 index c9bd5932e05e..000000000000 --- a/recipes/nemotron-3-ultra/vllm/agg-b200-chat-nomtp/deploy.yaml +++ /dev/null @@ -1,252 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: ultra-agg-b200-chat-nomtp - labels: - app.kubernetes.io/name: ultra-agg-b200-chat-nomtp - app.kubernetes.io/part-of: nemotron-ultra - nemotron-ultra.nvidia.com/backend: vllm - nemotron-ultra.nvidia.com/topology: agg1-tp4-nomtp - nemotron-ultra.nvidia.com/workload: chat -spec: - backendFramework: vllm - env: - - name: HF_HOME - value: /opt/models - - name: HF_HUB_CACHE - value: /opt/models/hub - - name: HF_HUB_OFFLINE - value: "1" - - name: TRANSFORMERS_OFFLINE - value: "1" - - name: SERVED_MODEL_NAME - value: nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: MODEL_PATH - value: /opt/models/patched/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: LOG_DIR - value: /artifacts/logs - - name: HOME - value: /tmp - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: XDG_CACHE_HOME - value: /tmp/cache - - name: TORCH_EXTENSIONS_DIR - value: /tmp/torch_extensions - - name: TRITON_CACHE_DIR - value: /tmp/triton - - name: VLLM_CACHE_ROOT - value: /tmp/vllm - - name: PYTHONHASHSEED - value: "0" - - name: VLLM_LOGGING_LEVEL - value: INFO - - name: VLLM_WORKER_MULTIPROC_METHOD - value: spawn - - name: VLLM_ALLREDUCE_USE_SYMM_MEM - value: "0" - - name: VLLM_DISABLED_KERNELS - value: FlashInferFP8ScaledMMLinearKernel - - name: VLLM_SSM_CONV_STATE_LAYOUT - value: DS - - name: VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE - value: "1" - - name: DYN_VLLM_APPEND_PREFILL_OUTPUT_TOKENS - value: "0" - - name: NCCL_IB_DISABLE - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: DYN_LOG - value: info,dynamo_kv_router=debug,dynamo_llm::kv_router=debug - components: - - name: Frontend - type: frontend - replicas: 1 - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: frontend - nemotron-ultra.nvidia.com/workload: chat - spec: - imagePullSecrets: - - name: nvcr-secret - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - mkdir -p "${LOG_DIR:-/tmp/nemotron-ultra}/status" - frontend_args=( - --router-mode kv - --router-kv-events - --kv-cache-block-size 32 - --router-reset-states - --http-host 0.0.0.0 - --http-port 8000 - ) - printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dingo.frontend "${frontend_args[@]}" - ports: - - name: http - containerPort: 8000 - startupProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - readinessProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 3 - failureThreshold: 3 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - emptyDir: {} - - name: runtime-tmp - emptyDir: {} - - name: VllmWorker - type: worker - replicas: 1 - sharedMemorySize: 64Gi - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: aggregate-worker - nemotron-ultra.nvidia.com/workload: chat - spec: - imagePullSecrets: - - name: nvcr-secret - runtimeClassName: nvidia - nodeSelector: - nvidia.com/gpu.product: NVIDIA-B200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - test -r "${MODEL_PATH}/config.json" || { echo "MODEL_PREFLIGHT_FAIL class=model_config_missing path=${MODEL_PATH}/config.json"; exit 42; } - test -r "${MODEL_PATH}/ultra_v3_reasoning_parser.py" || { echo "MODEL_PREFLIGHT_FAIL class=reasoning_parser_missing path=${MODEL_PATH}/ultra_v3_reasoning_parser.py"; exit 43; } - echo "PRESTART_GPU_GUARD_BEGIN role=agg workload=chat ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - nvidia-smi -L - nvidia-smi --query-gpu=index,uuid,memory.total,memory.used,memory.free,utilization.gpu --format=csv,noheader,nounits - visible_count="$(nvidia-smi -L | grep -c '^GPU ')" - if [ "${visible_count}" -ne 4 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_visibility_count_mismatch role=agg workload=chat visible_count=${visible_count}" - exit 96 - fi - mem_rows="$(nvidia-smi --query-gpu=index,uuid,memory.used,utilization.gpu --format=csv,noheader,nounits)" - echo "${mem_rows}" - dirty_count="$(printf '%s\n' "${mem_rows}" | awk -F',' '{gsub(/ /,"",$3); if (($3+0)>1024) c++} END{print c+0}')" - if [ "${dirty_count}" -ne 0 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_memory_dirty role=agg workload=chat dirty_count=${dirty_count}" - exit 97 - fi - compute_apps="$(nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_memory --format=csv,noheader,nounits || true)" - if [ -n "${compute_apps}" ]; then - printf '%s\n' "${compute_apps}" - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_compute_apps_visible role=agg workload=chat" - exit 98 - fi - echo "PRESTART_GPU_GUARD_PASS role=agg workload=chat" - ulimit -l unlimited - exec python3 -m dingo.vllm \ - --model "${MODEL_PATH}" \ - --served-model-name "${SERVED_MODEL_NAME}" \ - --tensor-parallel-size 4 \ - --trust-remote-code \ - --max-model-len 262144 \ - --max-num-seqs 40 \ - --max-num-batched-tokens 32768 \ - --gpu-memory-utilization 0.9 \ - --no-enable-flashinfer-autotune \ - --block-size 32 \ - --enable-expert-parallel \ - --mamba-cache-mode align \ - --enable-prefix-caching \ - --dyn-tool-call-parser qwen3_coder \ - --dyn-reasoning-parser nemotron3 \ - --reasoning-parser-plugin "${MODEL_PATH}/ultra_v3_reasoning_parser.py" \ - --reasoning-parser nemotron_v3 \ - --no-disable-hybrid-kv-cache-manager - envFrom: - - secretRef: - name: hf-token-secret - resources: - requests: - nvidia.com/gpu: "4" - memory: 750Gi - ephemeral-storage: 20Gi - limits: - nvidia.com/gpu: "4" - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - runAsUser: 0 - runAsGroup: 0 - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 20 - failureThreshold: 30 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - - name: flashinfer-cubins - mountPath: /usr/local/lib/python3.12/dist-packages/flashinfer_cubin/cubins/flashinfer - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - emptyDir: {} - - name: runtime-tmp - emptyDir: {} - - name: flashinfer-cubins - emptyDir: - medium: Memory diff --git a/recipes/nemotron-3-ultra/vllm/agg-h200-agentic-mtp/deploy.yaml b/recipes/nemotron-3-ultra/vllm/agg-h200-agentic-mtp/deploy.yaml deleted file mode 100644 index 985af9c6440c..000000000000 --- a/recipes/nemotron-3-ultra/vllm/agg-h200-agentic-mtp/deploy.yaml +++ /dev/null @@ -1,254 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: ultra-agg-h200-agentic-mtp - labels: - app.kubernetes.io/name: ultra-agg-h200-agentic-mtp - app.kubernetes.io/part-of: nemotron-ultra - nemotron-ultra.nvidia.com/backend: vllm - nemotron-ultra.nvidia.com/topology: agg1-tp8 - nemotron-ultra.nvidia.com/workload: agentic -spec: - backendFramework: vllm - env: - - name: HF_HOME - value: /opt/models - - name: HF_HUB_CACHE - value: /opt/models/hub - - name: HF_HUB_OFFLINE - value: "1" - - name: TRANSFORMERS_OFFLINE - value: "1" - - name: SERVED_MODEL_NAME - value: nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: MODEL_PATH - value: /opt/models/patched/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: LOG_DIR - value: /artifacts/logs - - name: HOME - value: /tmp - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: XDG_CACHE_HOME - value: /tmp/cache - - name: TORCH_EXTENSIONS_DIR - value: /tmp/torch_extensions - - name: TRITON_CACHE_DIR - value: /tmp/triton - - name: VLLM_CACHE_ROOT - value: /tmp/vllm - - name: PYTHONHASHSEED - value: "0" - - name: VLLM_LOGGING_LEVEL - value: INFO - - name: VLLM_WORKER_MULTIPROC_METHOD - value: spawn - - name: VLLM_ALLREDUCE_USE_SYMM_MEM - value: "0" - - name: VLLM_DISABLED_KERNELS - value: FlashInferFP8ScaledMMLinearKernel - - name: VLLM_SSM_CONV_STATE_LAYOUT - value: DS - - name: VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE - value: "1" - - name: DYN_VLLM_APPEND_PREFILL_OUTPUT_TOKENS - value: "0" - - name: NCCL_IB_DISABLE - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: DYN_LOG - value: info,dynamo_kv_router=debug,dynamo_llm::kv_router=debug - components: - - name: Frontend - type: frontend - replicas: 1 - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: frontend - nemotron-ultra.nvidia.com/workload: agentic - spec: - imagePullSecrets: - - name: nvcr-secret - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - mkdir -p "${LOG_DIR:-/tmp/nemotron-ultra}/status" - frontend_args=( - --router-mode kv - --router-kv-events - --kv-cache-block-size 64 - --router-reset-states - --http-host 0.0.0.0 - --http-port 8000 - ) - printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dingo.frontend "${frontend_args[@]}" - ports: - - name: http - containerPort: 8000 - startupProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - readinessProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 3 - failureThreshold: 3 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - emptyDir: {} - - name: runtime-tmp - emptyDir: {} - - name: VllmWorker - type: worker - replicas: 1 - sharedMemorySize: 64Gi - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: aggregate-worker - nemotron-ultra.nvidia.com/workload: agentic - spec: - imagePullSecrets: - - name: nvcr-secret - runtimeClassName: nvidia - nodeSelector: - nvidia.com/gpu.product: NVIDIA-H200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - test -r "${MODEL_PATH}/config.json" || { echo "MODEL_PREFLIGHT_FAIL class=model_config_missing path=${MODEL_PATH}/config.json"; exit 42; } - test -r "${MODEL_PATH}/ultra_v3_reasoning_parser.py" || { echo "MODEL_PREFLIGHT_FAIL class=reasoning_parser_missing path=${MODEL_PATH}/ultra_v3_reasoning_parser.py"; exit 43; } - echo "PRESTART_GPU_GUARD_BEGIN role=agg workload=agentic ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - nvidia-smi -L - nvidia-smi --query-gpu=index,uuid,memory.total,memory.used,memory.free,utilization.gpu --format=csv,noheader,nounits - visible_count="$(nvidia-smi -L | grep -c '^GPU ')" - if [ "${visible_count}" -ne 8 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_visibility_count_mismatch role=agg workload=agentic visible_count=${visible_count}" - exit 96 - fi - mem_rows="$(nvidia-smi --query-gpu=index,uuid,memory.used,utilization.gpu --format=csv,noheader,nounits)" - echo "${mem_rows}" - dirty_count="$(printf '%s\n' "${mem_rows}" | awk -F',' '{gsub(/ /,"",$3); if (($3+0)>1024) c++} END{print c+0}')" - if [ "${dirty_count}" -ne 0 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_memory_dirty role=agg workload=agentic dirty_count=${dirty_count}" - exit 97 - fi - compute_apps="$(nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_memory --format=csv,noheader,nounits || true)" - if [ -n "${compute_apps}" ]; then - printf '%s\n' "${compute_apps}" - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_compute_apps_visible role=agg workload=agentic" - exit 98 - fi - echo "PRESTART_GPU_GUARD_PASS role=agg workload=agentic" - ulimit -l unlimited - exec python3 -m dingo.vllm \ - --model "${MODEL_PATH}" \ - --served-model-name "${SERVED_MODEL_NAME}" \ - --tensor-parallel-size 8 \ - --trust-remote-code \ - --max-model-len 262144 \ - --max-num-seqs 32 \ - --max-num-batched-tokens 32768 \ - --gpu-memory-utilization 0.85 \ - --no-enable-flashinfer-autotune \ - --block-size 64 \ - --enable-expert-parallel \ - --mamba-cache-mode align \ - --enable-prefix-caching \ - --spec-method nemotron_h_mtp \ - --spec-tokens 1 \ - --dyn-tool-call-parser qwen3_coder \ - --dyn-reasoning-parser nemotron3 \ - --reasoning-parser-plugin "${MODEL_PATH}/ultra_v3_reasoning_parser.py" \ - --reasoning-parser nemotron_v3 \ - --no-disable-hybrid-kv-cache-manager - envFrom: - - secretRef: - name: hf-token-secret - resources: - requests: - nvidia.com/gpu: "8" - memory: 750Gi - ephemeral-storage: 20Gi - limits: - nvidia.com/gpu: "8" - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - runAsUser: 0 - runAsGroup: 0 - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 20 - failureThreshold: 30 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - - name: flashinfer-cubins - mountPath: /usr/local/lib/python3.12/dist-packages/flashinfer_cubin/cubins/flashinfer - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - emptyDir: {} - - name: runtime-tmp - emptyDir: {} - - name: flashinfer-cubins - emptyDir: - medium: Memory diff --git a/recipes/nemotron-3-ultra/vllm/agg-h200-agentic-nomtp/deploy.yaml b/recipes/nemotron-3-ultra/vllm/agg-h200-agentic-nomtp/deploy.yaml deleted file mode 100644 index bc3ef5ea54fc..000000000000 --- a/recipes/nemotron-3-ultra/vllm/agg-h200-agentic-nomtp/deploy.yaml +++ /dev/null @@ -1,252 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: ultra-agg-h200-agentic-nomtp - labels: - app.kubernetes.io/name: ultra-agg-h200-agentic-nomtp - app.kubernetes.io/part-of: nemotron-ultra - nemotron-ultra.nvidia.com/backend: vllm - nemotron-ultra.nvidia.com/topology: agg1-tp8-nomtp - nemotron-ultra.nvidia.com/workload: agentic -spec: - backendFramework: vllm - env: - - name: HF_HOME - value: /opt/models - - name: HF_HUB_CACHE - value: /opt/models/hub - - name: HF_HUB_OFFLINE - value: "1" - - name: TRANSFORMERS_OFFLINE - value: "1" - - name: SERVED_MODEL_NAME - value: nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: MODEL_PATH - value: /opt/models/patched/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: LOG_DIR - value: /artifacts/logs - - name: HOME - value: /tmp - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: XDG_CACHE_HOME - value: /tmp/cache - - name: TORCH_EXTENSIONS_DIR - value: /tmp/torch_extensions - - name: TRITON_CACHE_DIR - value: /tmp/triton - - name: VLLM_CACHE_ROOT - value: /tmp/vllm - - name: PYTHONHASHSEED - value: "0" - - name: VLLM_LOGGING_LEVEL - value: INFO - - name: VLLM_WORKER_MULTIPROC_METHOD - value: spawn - - name: VLLM_ALLREDUCE_USE_SYMM_MEM - value: "0" - - name: VLLM_DISABLED_KERNELS - value: FlashInferFP8ScaledMMLinearKernel - - name: VLLM_SSM_CONV_STATE_LAYOUT - value: DS - - name: VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE - value: "1" - - name: DYN_VLLM_APPEND_PREFILL_OUTPUT_TOKENS - value: "0" - - name: NCCL_IB_DISABLE - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: DYN_LOG - value: info,dynamo_kv_router=debug,dynamo_llm::kv_router=debug - components: - - name: Frontend - type: frontend - replicas: 1 - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: frontend - nemotron-ultra.nvidia.com/workload: agentic - spec: - imagePullSecrets: - - name: nvcr-secret - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - mkdir -p "${LOG_DIR:-/tmp/nemotron-ultra}/status" - frontend_args=( - --router-mode kv - --router-kv-events - --kv-cache-block-size 64 - --router-reset-states - --http-host 0.0.0.0 - --http-port 8000 - ) - printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dingo.frontend "${frontend_args[@]}" - ports: - - name: http - containerPort: 8000 - startupProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - readinessProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 3 - failureThreshold: 3 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - emptyDir: {} - - name: runtime-tmp - emptyDir: {} - - name: VllmWorker - type: worker - replicas: 1 - sharedMemorySize: 64Gi - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: aggregate-worker - nemotron-ultra.nvidia.com/workload: agentic - spec: - imagePullSecrets: - - name: nvcr-secret - runtimeClassName: nvidia - nodeSelector: - nvidia.com/gpu.product: NVIDIA-H200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - test -r "${MODEL_PATH}/config.json" || { echo "MODEL_PREFLIGHT_FAIL class=model_config_missing path=${MODEL_PATH}/config.json"; exit 42; } - test -r "${MODEL_PATH}/ultra_v3_reasoning_parser.py" || { echo "MODEL_PREFLIGHT_FAIL class=reasoning_parser_missing path=${MODEL_PATH}/ultra_v3_reasoning_parser.py"; exit 43; } - echo "PRESTART_GPU_GUARD_BEGIN role=agg workload=agentic ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - nvidia-smi -L - nvidia-smi --query-gpu=index,uuid,memory.total,memory.used,memory.free,utilization.gpu --format=csv,noheader,nounits - visible_count="$(nvidia-smi -L | grep -c '^GPU ')" - if [ "${visible_count}" -ne 8 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_visibility_count_mismatch role=agg workload=agentic visible_count=${visible_count}" - exit 96 - fi - mem_rows="$(nvidia-smi --query-gpu=index,uuid,memory.used,utilization.gpu --format=csv,noheader,nounits)" - echo "${mem_rows}" - dirty_count="$(printf '%s\n' "${mem_rows}" | awk -F',' '{gsub(/ /,"",$3); if (($3+0)>1024) c++} END{print c+0}')" - if [ "${dirty_count}" -ne 0 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_memory_dirty role=agg workload=agentic dirty_count=${dirty_count}" - exit 97 - fi - compute_apps="$(nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_memory --format=csv,noheader,nounits || true)" - if [ -n "${compute_apps}" ]; then - printf '%s\n' "${compute_apps}" - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_compute_apps_visible role=agg workload=agentic" - exit 98 - fi - echo "PRESTART_GPU_GUARD_PASS role=agg workload=agentic" - ulimit -l unlimited - exec python3 -m dingo.vllm \ - --model "${MODEL_PATH}" \ - --served-model-name "${SERVED_MODEL_NAME}" \ - --tensor-parallel-size 8 \ - --trust-remote-code \ - --max-model-len 262144 \ - --max-num-seqs 32 \ - --max-num-batched-tokens 32768 \ - --gpu-memory-utilization 0.85 \ - --no-enable-flashinfer-autotune \ - --block-size 64 \ - --enable-expert-parallel \ - --mamba-cache-mode align \ - --enable-prefix-caching \ - --dyn-tool-call-parser qwen3_coder \ - --dyn-reasoning-parser nemotron3 \ - --reasoning-parser-plugin "${MODEL_PATH}/ultra_v3_reasoning_parser.py" \ - --reasoning-parser nemotron_v3 \ - --no-disable-hybrid-kv-cache-manager - envFrom: - - secretRef: - name: hf-token-secret - resources: - requests: - nvidia.com/gpu: "8" - memory: 750Gi - ephemeral-storage: 20Gi - limits: - nvidia.com/gpu: "8" - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - runAsUser: 0 - runAsGroup: 0 - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 20 - failureThreshold: 30 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - - name: flashinfer-cubins - mountPath: /usr/local/lib/python3.12/dist-packages/flashinfer_cubin/cubins/flashinfer - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - emptyDir: {} - - name: runtime-tmp - emptyDir: {} - - name: flashinfer-cubins - emptyDir: - medium: Memory diff --git a/recipes/nemotron-3-ultra/vllm/agg-h200-chat-mtp/deploy.yaml b/recipes/nemotron-3-ultra/vllm/agg-h200-chat-mtp/deploy.yaml deleted file mode 100644 index 8f7303a889f6..000000000000 --- a/recipes/nemotron-3-ultra/vllm/agg-h200-chat-mtp/deploy.yaml +++ /dev/null @@ -1,254 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: ultra-agg-h200-chat-mtp - labels: - app.kubernetes.io/name: ultra-agg-h200-chat-mtp - app.kubernetes.io/part-of: nemotron-ultra - nemotron-ultra.nvidia.com/backend: vllm - nemotron-ultra.nvidia.com/topology: agg1-tp8 - nemotron-ultra.nvidia.com/workload: chat -spec: - backendFramework: vllm - env: - - name: HF_HOME - value: /opt/models - - name: HF_HUB_CACHE - value: /opt/models/hub - - name: HF_HUB_OFFLINE - value: "1" - - name: TRANSFORMERS_OFFLINE - value: "1" - - name: SERVED_MODEL_NAME - value: nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: MODEL_PATH - value: /opt/models/patched/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: LOG_DIR - value: /artifacts/logs - - name: HOME - value: /tmp - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: XDG_CACHE_HOME - value: /tmp/cache - - name: TORCH_EXTENSIONS_DIR - value: /tmp/torch_extensions - - name: TRITON_CACHE_DIR - value: /tmp/triton - - name: VLLM_CACHE_ROOT - value: /tmp/vllm - - name: PYTHONHASHSEED - value: "0" - - name: VLLM_LOGGING_LEVEL - value: INFO - - name: VLLM_WORKER_MULTIPROC_METHOD - value: spawn - - name: VLLM_ALLREDUCE_USE_SYMM_MEM - value: "0" - - name: VLLM_DISABLED_KERNELS - value: FlashInferFP8ScaledMMLinearKernel - - name: VLLM_SSM_CONV_STATE_LAYOUT - value: DS - - name: VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE - value: "1" - - name: DYN_VLLM_APPEND_PREFILL_OUTPUT_TOKENS - value: "0" - - name: NCCL_IB_DISABLE - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: DYN_LOG - value: info,dynamo_kv_router=debug,dynamo_llm::kv_router=debug - components: - - name: Frontend - type: frontend - replicas: 1 - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: frontend - nemotron-ultra.nvidia.com/workload: chat - spec: - imagePullSecrets: - - name: nvcr-secret - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - mkdir -p "${LOG_DIR:-/tmp/nemotron-ultra}/status" - frontend_args=( - --router-mode kv - --router-kv-events - --kv-cache-block-size 64 - --router-reset-states - --http-host 0.0.0.0 - --http-port 8000 - ) - printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dingo.frontend "${frontend_args[@]}" - ports: - - name: http - containerPort: 8000 - startupProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - readinessProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 3 - failureThreshold: 3 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - emptyDir: {} - - name: runtime-tmp - emptyDir: {} - - name: VllmWorker - type: worker - replicas: 1 - sharedMemorySize: 64Gi - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: aggregate-worker - nemotron-ultra.nvidia.com/workload: chat - spec: - imagePullSecrets: - - name: nvcr-secret - runtimeClassName: nvidia - nodeSelector: - nvidia.com/gpu.product: NVIDIA-H200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - test -r "${MODEL_PATH}/config.json" || { echo "MODEL_PREFLIGHT_FAIL class=model_config_missing path=${MODEL_PATH}/config.json"; exit 42; } - test -r "${MODEL_PATH}/ultra_v3_reasoning_parser.py" || { echo "MODEL_PREFLIGHT_FAIL class=reasoning_parser_missing path=${MODEL_PATH}/ultra_v3_reasoning_parser.py"; exit 43; } - echo "PRESTART_GPU_GUARD_BEGIN role=agg workload=chat ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - nvidia-smi -L - nvidia-smi --query-gpu=index,uuid,memory.total,memory.used,memory.free,utilization.gpu --format=csv,noheader,nounits - visible_count="$(nvidia-smi -L | grep -c '^GPU ')" - if [ "${visible_count}" -ne 8 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_visibility_count_mismatch role=agg workload=chat visible_count=${visible_count}" - exit 96 - fi - mem_rows="$(nvidia-smi --query-gpu=index,uuid,memory.used,utilization.gpu --format=csv,noheader,nounits)" - echo "${mem_rows}" - dirty_count="$(printf '%s\n' "${mem_rows}" | awk -F',' '{gsub(/ /,"",$3); if (($3+0)>1024) c++} END{print c+0}')" - if [ "${dirty_count}" -ne 0 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_memory_dirty role=agg workload=chat dirty_count=${dirty_count}" - exit 97 - fi - compute_apps="$(nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_memory --format=csv,noheader,nounits || true)" - if [ -n "${compute_apps}" ]; then - printf '%s\n' "${compute_apps}" - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_compute_apps_visible role=agg workload=chat" - exit 98 - fi - echo "PRESTART_GPU_GUARD_PASS role=agg workload=chat" - ulimit -l unlimited - exec python3 -m dingo.vllm \ - --model "${MODEL_PATH}" \ - --served-model-name "${SERVED_MODEL_NAME}" \ - --tensor-parallel-size 8 \ - --trust-remote-code \ - --max-model-len 262144 \ - --max-num-seqs 16 \ - --max-num-batched-tokens 32768 \ - --gpu-memory-utilization 0.85 \ - --no-enable-flashinfer-autotune \ - --block-size 64 \ - --enable-expert-parallel \ - --mamba-cache-mode align \ - --enable-prefix-caching \ - --spec-method nemotron_h_mtp \ - --spec-tokens 1 \ - --dyn-tool-call-parser qwen3_coder \ - --dyn-reasoning-parser nemotron3 \ - --reasoning-parser-plugin "${MODEL_PATH}/ultra_v3_reasoning_parser.py" \ - --reasoning-parser nemotron_v3 \ - --no-disable-hybrid-kv-cache-manager - envFrom: - - secretRef: - name: hf-token-secret - resources: - requests: - nvidia.com/gpu: "8" - memory: 750Gi - ephemeral-storage: 20Gi - limits: - nvidia.com/gpu: "8" - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - runAsUser: 0 - runAsGroup: 0 - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 20 - failureThreshold: 30 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - - name: flashinfer-cubins - mountPath: /usr/local/lib/python3.12/dist-packages/flashinfer_cubin/cubins/flashinfer - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - emptyDir: {} - - name: runtime-tmp - emptyDir: {} - - name: flashinfer-cubins - emptyDir: - medium: Memory diff --git a/recipes/nemotron-3-ultra/vllm/agg-h200-chat-nomtp/deploy.yaml b/recipes/nemotron-3-ultra/vllm/agg-h200-chat-nomtp/deploy.yaml deleted file mode 100644 index 170e739e9794..000000000000 --- a/recipes/nemotron-3-ultra/vllm/agg-h200-chat-nomtp/deploy.yaml +++ /dev/null @@ -1,252 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: ultra-agg-h200-chat-nomtp - labels: - app.kubernetes.io/name: ultra-agg-h200-chat-nomtp - app.kubernetes.io/part-of: nemotron-ultra - nemotron-ultra.nvidia.com/backend: vllm - nemotron-ultra.nvidia.com/topology: agg1-tp8-nomtp - nemotron-ultra.nvidia.com/workload: chat -spec: - backendFramework: vllm - env: - - name: HF_HOME - value: /opt/models - - name: HF_HUB_CACHE - value: /opt/models/hub - - name: HF_HUB_OFFLINE - value: "1" - - name: TRANSFORMERS_OFFLINE - value: "1" - - name: SERVED_MODEL_NAME - value: nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: MODEL_PATH - value: /opt/models/patched/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: LOG_DIR - value: /artifacts/logs - - name: HOME - value: /tmp - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: XDG_CACHE_HOME - value: /tmp/cache - - name: TORCH_EXTENSIONS_DIR - value: /tmp/torch_extensions - - name: TRITON_CACHE_DIR - value: /tmp/triton - - name: VLLM_CACHE_ROOT - value: /tmp/vllm - - name: PYTHONHASHSEED - value: "0" - - name: VLLM_LOGGING_LEVEL - value: INFO - - name: VLLM_WORKER_MULTIPROC_METHOD - value: spawn - - name: VLLM_ALLREDUCE_USE_SYMM_MEM - value: "0" - - name: VLLM_DISABLED_KERNELS - value: FlashInferFP8ScaledMMLinearKernel - - name: VLLM_SSM_CONV_STATE_LAYOUT - value: DS - - name: VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE - value: "1" - - name: DYN_VLLM_APPEND_PREFILL_OUTPUT_TOKENS - value: "0" - - name: NCCL_IB_DISABLE - value: "1" - - name: NCCL_CUMEM_ENABLE - value: "1" - - name: NCCL_NVLS_ENABLE - value: "1" - - name: DYN_LOG - value: info,dynamo_kv_router=debug,dynamo_llm::kv_router=debug - components: - - name: Frontend - type: frontend - replicas: 1 - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: frontend - nemotron-ultra.nvidia.com/workload: chat - spec: - imagePullSecrets: - - name: nvcr-secret - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - mkdir -p "${LOG_DIR:-/tmp/nemotron-ultra}/status" - frontend_args=( - --router-mode kv - --router-kv-events - --kv-cache-block-size 64 - --router-reset-states - --http-host 0.0.0.0 - --http-port 8000 - ) - printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dingo.frontend "${frontend_args[@]}" - ports: - - name: http - containerPort: 8000 - startupProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - readinessProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 3 - failureThreshold: 3 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - emptyDir: {} - - name: runtime-tmp - emptyDir: {} - - name: VllmWorker - type: worker - replicas: 1 - sharedMemorySize: 64Gi - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: aggregate-worker - nemotron-ultra.nvidia.com/workload: chat - spec: - imagePullSecrets: - - name: nvcr-secret - runtimeClassName: nvidia - nodeSelector: - nvidia.com/gpu.product: NVIDIA-H200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - test -r "${MODEL_PATH}/config.json" || { echo "MODEL_PREFLIGHT_FAIL class=model_config_missing path=${MODEL_PATH}/config.json"; exit 42; } - test -r "${MODEL_PATH}/ultra_v3_reasoning_parser.py" || { echo "MODEL_PREFLIGHT_FAIL class=reasoning_parser_missing path=${MODEL_PATH}/ultra_v3_reasoning_parser.py"; exit 43; } - echo "PRESTART_GPU_GUARD_BEGIN role=agg workload=chat ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - nvidia-smi -L - nvidia-smi --query-gpu=index,uuid,memory.total,memory.used,memory.free,utilization.gpu --format=csv,noheader,nounits - visible_count="$(nvidia-smi -L | grep -c '^GPU ')" - if [ "${visible_count}" -ne 8 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_visibility_count_mismatch role=agg workload=chat visible_count=${visible_count}" - exit 96 - fi - mem_rows="$(nvidia-smi --query-gpu=index,uuid,memory.used,utilization.gpu --format=csv,noheader,nounits)" - echo "${mem_rows}" - dirty_count="$(printf '%s\n' "${mem_rows}" | awk -F',' '{gsub(/ /,"",$3); if (($3+0)>1024) c++} END{print c+0}')" - if [ "${dirty_count}" -ne 0 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_memory_dirty role=agg workload=chat dirty_count=${dirty_count}" - exit 97 - fi - compute_apps="$(nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_memory --format=csv,noheader,nounits || true)" - if [ -n "${compute_apps}" ]; then - printf '%s\n' "${compute_apps}" - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_compute_apps_visible role=agg workload=chat" - exit 98 - fi - echo "PRESTART_GPU_GUARD_PASS role=agg workload=chat" - ulimit -l unlimited - exec python3 -m dingo.vllm \ - --model "${MODEL_PATH}" \ - --served-model-name "${SERVED_MODEL_NAME}" \ - --tensor-parallel-size 8 \ - --trust-remote-code \ - --max-model-len 262144 \ - --max-num-seqs 32 \ - --max-num-batched-tokens 32768 \ - --gpu-memory-utilization 0.85 \ - --no-enable-flashinfer-autotune \ - --block-size 64 \ - --enable-expert-parallel \ - --mamba-cache-mode align \ - --enable-prefix-caching \ - --dyn-tool-call-parser qwen3_coder \ - --dyn-reasoning-parser nemotron3 \ - --reasoning-parser-plugin "${MODEL_PATH}/ultra_v3_reasoning_parser.py" \ - --reasoning-parser nemotron_v3 \ - --no-disable-hybrid-kv-cache-manager - envFrom: - - secretRef: - name: hf-token-secret - resources: - requests: - nvidia.com/gpu: "8" - memory: 750Gi - ephemeral-storage: 20Gi - limits: - nvidia.com/gpu: "8" - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - runAsUser: 0 - runAsGroup: 0 - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 60 - timeoutSeconds: 20 - failureThreshold: 30 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - - name: flashinfer-cubins - mountPath: /usr/local/lib/python3.12/dist-packages/flashinfer_cubin/cubins/flashinfer - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - emptyDir: {} - - name: runtime-tmp - emptyDir: {} - - name: flashinfer-cubins - emptyDir: - medium: Memory diff --git a/recipes/nemotron-3-ultra/vllm/disagg-b200-agentic/deploy.yaml b/recipes/nemotron-3-ultra/vllm/disagg-b200-agentic/deploy.yaml deleted file mode 100644 index 0d5d4cdaf9dc..000000000000 --- a/recipes/nemotron-3-ultra/vllm/disagg-b200-agentic/deploy.yaml +++ /dev/null @@ -1,441 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: ultra-disagg-b200-1p1d-agentic-nomtp - labels: - app.kubernetes.io/name: ultra-disagg-b200-1p1d-agentic-nomtp - app.kubernetes.io/part-of: nemotron-ultra - nemotron-ultra.nvidia.com/backend: vllm - nemotron-ultra.nvidia.com/topology: 1p1d-tp4-tp4 - nemotron-ultra.nvidia.com/workload: agentic -spec: - backendFramework: vllm - env: - - name: HF_HOME - value: /opt/models - - name: HF_HUB_CACHE - value: /opt/models/hub - - name: HF_HUB_OFFLINE - value: "1" - - name: TRANSFORMERS_OFFLINE - value: "1" - - name: SERVED_MODEL_NAME - value: nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: MODEL_PATH - value: /opt/models/patched/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - name: LOG_DIR - value: /artifacts/logs - - name: HOME - value: /tmp - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: XDG_CACHE_HOME - value: /tmp/cache - - name: TORCH_EXTENSIONS_DIR - value: /tmp/torch_extensions - - name: TRITON_CACHE_DIR - value: /tmp/triton - - name: VLLM_CACHE_ROOT - value: /tmp/vllm - - name: PYTHONHASHSEED - value: "0" - - name: VLLM_LOGGING_LEVEL - value: INFO - - name: VLLM_WORKER_MULTIPROC_METHOD - value: spawn - - name: VLLM_ALLREDUCE_USE_SYMM_MEM - value: "0" - - name: VLLM_DISABLED_KERNELS - value: FlashInferFP8ScaledMMLinearKernel - - name: VLLM_SSM_CONV_STATE_LAYOUT - value: DS - - name: VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE - value: "1" - - name: DYN_VLLM_APPEND_PREFILL_OUTPUT_TOKENS - value: "0" - - name: DYN_LOG - value: info,dynamo_kv_router=debug,dynamo_llm::kv_router=debug - components: - - name: Frontend - type: frontend - replicas: 1 - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: frontend - nemotron-ultra.nvidia.com/topology: 1p1d-tp4-tp4 - nemotron-ultra.nvidia.com/recipe: ultra-disagg-b200-1p1d-agentic-nomtp - spec: - imagePullSecrets: - - name: nvcr-secret - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - mkdir -p "${LOG_DIR:-/tmp/nemotron-ultra}/status" - frontend_args=( - --router-mode kv - --router-kv-events - --kv-cache-block-size 64 - --router-reset-states - --http-host 0.0.0.0 - --http-port 8000 - ) - printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dingo.frontend "${frontend_args[@]}" - ports: - - name: http - containerPort: 8000 - startupProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 360 - readinessProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - timeoutSeconds: 3 - failureThreshold: 3 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - persistentVolumeClaim: - claimName: nemotron-ultra-aiperf-artifacts - - name: runtime-tmp - emptyDir: {} - - name: VllmPrefillWorker - type: prefill - replicas: 1 - sharedMemorySize: 64Gi - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: prefill - nemotron-ultra.nvidia.com/topology: 1p1d-tp4-tp4 - nemotron-ultra.nvidia.com/recipe: ultra-disagg-b200-1p1d-agentic-nomtp - spec: - imagePullSecrets: - - name: nvcr-secret - runtimeClassName: nvidia - nodeSelector: - nvidia.com/gpu.product: NVIDIA-B200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - test -r "${MODEL_PATH}/config.json" || { echo "MODEL_PREFLIGHT_FAIL class=model_config_missing path=${MODEL_PATH}/config.json"; exit 42; } - test -r "${MODEL_PATH}/ultra_v3_reasoning_parser.py" || { echo "MODEL_PREFLIGHT_FAIL class=reasoning_parser_missing path=${MODEL_PATH}/ultra_v3_reasoning_parser.py"; exit 43; } - echo "PRESTART_GPU_GUARD_BEGIN role=prefill ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - nvidia-smi -L - nvidia-smi --query-gpu=index,uuid,memory.total,memory.used,memory.free,utilization.gpu --format=csv,noheader,nounits - visible_count="$(nvidia-smi -L | grep -c '^GPU ')" - if [ "${visible_count}" -ne 4 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_visibility_count_mismatch role=prefill visible_count=${visible_count}" - exit 96 - fi - mem_rows="$(nvidia-smi --query-gpu=index,uuid,memory.used,utilization.gpu --format=csv,noheader,nounits)" - echo "${mem_rows}" - dirty_count="$(printf '%s\n' "${mem_rows}" | awk -F',' '{gsub(/ /,"",$3); if (($3+0)>1024) c++} END{print c+0}')" - if [ "${dirty_count}" -ne 0 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_memory_dirty role=prefill dirty_count=${dirty_count}" - exit 97 - fi - compute_apps="$(nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_memory --format=csv,noheader,nounits || true)" - if [ -n "${compute_apps}" ]; then - printf '%s\n' "${compute_apps}" - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_compute_apps_visible role=prefill" - exit 98 - fi - ls -l /dev/infiniband || true - echo "PRESTART_GPU_GUARD_PASS role=prefill" - export ULTRA_NIXL_XFER_DEBUG_JSONL="/artifacts/runs/nemotron-ultra-1p1d/native/prefill/${HOSTNAME}.jsonl" - mkdir -p "$(dirname "${ULTRA_NIXL_XFER_DEBUG_JSONL}")" - echo "ULTRA_NIXL_XFER_DEBUG_JSONL=${ULTRA_NIXL_XFER_DEBUG_JSONL} schema=${ULTRA_NIXL_XFER_DEBUG_SCHEMA}" - ulimit -l unlimited - exec python3 -m dingo.vllm \ - --model "${MODEL_PATH}" \ - --served-model-name "${SERVED_MODEL_NAME}" \ - --tensor-parallel-size 4 \ - --trust-remote-code \ - --max-model-len 262144 \ - --max-num-seqs 32 \ - --max-num-batched-tokens 32768 \ - --gpu-memory-utilization 0.9 \ - --no-enable-flashinfer-autotune \ - --block-size 64 \ - --enable-expert-parallel \ - --mamba-cache-mode align \ - --enable-prefix-caching \ - --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \ - --dyn-tool-call-parser qwen3_coder \ - --dyn-reasoning-parser nemotron3 \ - --reasoning-parser-plugin "${MODEL_PATH}/ultra_v3_reasoning_parser.py" \ - --reasoning-parser nemotron_v3 \ - --no-disable-hybrid-kv-cache-manager \ - --disaggregation-mode prefill \ - --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5571","enable_kv_cache_events":true}' - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: UCX_TLS - value: rc_x,rc,cuda_copy,cuda_ipc - - name: UCX_NET_DEVICES - value: mlx5_0:1 - - name: UCX_IB_ADDR_TYPE - value: eth - - name: UCX_RNDV_SCHEME - value: get_zcopy - - name: UCX_RNDV_THRESH - value: "0" - - name: UCX_RC_TIMEOUT - value: 600s - - name: UCX_KEEPALIVE_INTERVAL - value: 300s - - name: NCCL_IB_DISABLE - value: "0" - - name: NCCL_SOCKET_IFNAME - value: eth0 - - name: GLOO_SOCKET_IFNAME - value: eth0 - - name: NCCL_STORE_TIMEOUT - value: "7200" - - name: NIXL_LOG_LEVEL - value: INFO - - name: ULTRA_NIXL_XFER_DEBUG_SCHEMA - value: ULTRA_NIXL_NATIVE_XFER_DEBUG_V1 - - name: ULTRA_NIXL_XFER_DEBUG_ACTION_ID - value: TURBO_NEMOTRON_ULTRA_1P1D_B200 - - name: ULTRA_NIXL_XFER_DEBUG_ROLE - value: prefill - - name: ULTRA_NIXL_XFER_DEBUG_RUN_DIR - value: nemotron-ultra-1p1d - resources: - requests: - nvidia.com/gpu: "4" - rdma/ib: "4" - memory: 750Gi - ephemeral-storage: 20Gi - limits: - nvidia.com/gpu: "4" - rdma/ib: "4" - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - runAsUser: 0 - runAsGroup: 0 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - - name: flashinfer-cubins - mountPath: /usr/local/lib/python3.12/dist-packages/flashinfer_cubin/cubins/flashinfer - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - persistentVolumeClaim: - claimName: nemotron-ultra-aiperf-artifacts - - name: runtime-tmp - emptyDir: {} - - name: flashinfer-cubins - emptyDir: - medium: Memory - - name: VllmDecodeWorker - type: decode - replicas: 1 - sharedMemorySize: 64Gi - podTemplate: - metadata: - labels: - nemotron-ultra.nvidia.com/role: decode - nemotron-ultra.nvidia.com/topology: 1p1d-tp4-tp4 - nemotron-ultra.nvidia.com/recipe: ultra-disagg-b200-1p1d-agentic-nomtp - spec: - imagePullSecrets: - - name: nvcr-secret - runtimeClassName: nvidia - nodeSelector: - nvidia.com/gpu.product: NVIDIA-B200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-nemotron-ultra-dev.1 - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -lc - args: - - | - set -Eeuo pipefail - test -r "${MODEL_PATH}/config.json" || { echo "MODEL_PREFLIGHT_FAIL class=model_config_missing path=${MODEL_PATH}/config.json"; exit 42; } - test -r "${MODEL_PATH}/ultra_v3_reasoning_parser.py" || { echo "MODEL_PREFLIGHT_FAIL class=reasoning_parser_missing path=${MODEL_PATH}/ultra_v3_reasoning_parser.py"; exit 43; } - echo "PRESTART_GPU_GUARD_BEGIN role=decode ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - nvidia-smi -L - nvidia-smi --query-gpu=index,uuid,memory.total,memory.used,memory.free,utilization.gpu --format=csv,noheader,nounits - visible_count="$(nvidia-smi -L | grep -c '^GPU ')" - if [ "${visible_count}" -ne 4 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_visibility_count_mismatch role=decode visible_count=${visible_count}" - exit 96 - fi - mem_rows="$(nvidia-smi --query-gpu=index,uuid,memory.used,utilization.gpu --format=csv,noheader,nounits)" - echo "${mem_rows}" - dirty_count="$(printf '%s\n' "${mem_rows}" | awk -F',' '{gsub(/ /,"",$3); if (($3+0)>1024) c++} END{print c+0}')" - if [ "${dirty_count}" -ne 0 ]; then - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_memory_dirty role=decode dirty_count=${dirty_count}" - exit 97 - fi - compute_apps="$(nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_memory --format=csv,noheader,nounits || true)" - if [ -n "${compute_apps}" ]; then - printf '%s\n' "${compute_apps}" - echo "PRESTART_GPU_GUARD_FAIL class=prestart_gpu_compute_apps_visible role=decode" - exit 98 - fi - ls -l /dev/infiniband || true - echo "PRESTART_GPU_GUARD_PASS role=decode" - export ULTRA_NIXL_XFER_DEBUG_JSONL="/artifacts/runs/nemotron-ultra-1p1d/native/decode/${HOSTNAME}.jsonl" - mkdir -p "$(dirname "${ULTRA_NIXL_XFER_DEBUG_JSONL}")" - echo "ULTRA_NIXL_XFER_DEBUG_JSONL=${ULTRA_NIXL_XFER_DEBUG_JSONL} schema=${ULTRA_NIXL_XFER_DEBUG_SCHEMA}" - ulimit -l unlimited - exec python3 -m dingo.vllm \ - --model "${MODEL_PATH}" \ - --served-model-name "${SERVED_MODEL_NAME}" \ - --tensor-parallel-size 4 \ - --trust-remote-code \ - --max-model-len 262144 \ - --max-num-seqs 32 \ - --max-num-batched-tokens 32768 \ - --gpu-memory-utilization 0.9 \ - --no-enable-flashinfer-autotune \ - --block-size 64 \ - --enable-expert-parallel \ - --mamba-cache-mode align \ - --enable-prefix-caching \ - --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \ - --dyn-tool-call-parser qwen3_coder \ - --dyn-reasoning-parser nemotron3 \ - --reasoning-parser-plugin "${MODEL_PATH}/ultra_v3_reasoning_parser.py" \ - --reasoning-parser nemotron_v3 \ - --no-disable-hybrid-kv-cache-manager \ - --disaggregation-mode decode - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: UCX_TLS - value: rc_x,rc,cuda_copy,cuda_ipc - - name: UCX_NET_DEVICES - value: mlx5_0:1 - - name: UCX_IB_ADDR_TYPE - value: eth - - name: UCX_RNDV_SCHEME - value: get_zcopy - - name: UCX_RNDV_THRESH - value: "0" - - name: UCX_RC_TIMEOUT - value: 600s - - name: UCX_KEEPALIVE_INTERVAL - value: 300s - - name: NCCL_IB_DISABLE - value: "0" - - name: NCCL_SOCKET_IFNAME - value: eth0 - - name: GLOO_SOCKET_IFNAME - value: eth0 - - name: NCCL_STORE_TIMEOUT - value: "7200" - - name: NIXL_LOG_LEVEL - value: INFO - - name: ULTRA_NIXL_XFER_DEBUG_SCHEMA - value: ULTRA_NIXL_NATIVE_XFER_DEBUG_V1 - - name: ULTRA_NIXL_XFER_DEBUG_ACTION_ID - value: TURBO_NEMOTRON_ULTRA_1P1D_B200 - - name: ULTRA_NIXL_XFER_DEBUG_ROLE - value: decode - - name: ULTRA_NIXL_XFER_DEBUG_RUN_DIR - value: nemotron-ultra-1p1d - resources: - requests: - nvidia.com/gpu: "4" - rdma/ib: "4" - memory: 750Gi - ephemeral-storage: 20Gi - limits: - nvidia.com/gpu: "4" - rdma/ib: "4" - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - runAsUser: 0 - runAsGroup: 0 - volumeMounts: - - name: shared-model-cache - mountPath: /opt/models - readOnly: true - - name: artifact-root - mountPath: /artifacts - - name: runtime-tmp - mountPath: /tmp - - name: flashinfer-cubins - mountPath: /usr/local/lib/python3.12/dist-packages/flashinfer_cubin/cubins/flashinfer - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: artifact-root - persistentVolumeClaim: - claimName: nemotron-ultra-aiperf-artifacts - - name: runtime-tmp - emptyDir: {} - - name: flashinfer-cubins - emptyDir: - medium: Memory diff --git a/recipes/qwen3-0.6b/model-cache/model-cache.yaml b/recipes/qwen3-0.6b/model-cache/model-cache.yaml deleted file mode 100644 index f9fdc737f905..000000000000 --- a/recipes/qwen3-0.6b/model-cache/model-cache.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 10Gi - storageClassName: "your-storage-class-name" diff --git a/recipes/qwen3-0.6b/model-cache/model-download.yaml b/recipes/qwen3-0.6b/model-cache/model-download.yaml deleted file mode 100644 index 9a5a50a288eb..000000000000 --- a/recipes/qwen3-0.6b/model-cache/model-download.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: MODEL_NAME - value: "Qwen/Qwen3-0.6B" - - name: HF_HOME - value: /model-store - - name: HF_HUB_ENABLE_HF_TRANSFER - value: "1" - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub hf_transfer - hf download $MODEL_NAME - volumeMounts: - - name: model-cache - mountPath: /model-store - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/qwen3-0.6b/vllm/agg/gaie/deploy.yaml b/recipes/qwen3-0.6b/vllm/agg/gaie/deploy.yaml deleted file mode 100644 index c7e1af8424e9..000000000000 --- a/recipes/qwen3-0.6b/vllm/agg/gaie/deploy.yaml +++ /dev/null @@ -1,88 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: qwen3-0-6b-agg -spec: - backendFramework: vllm - pvcs: - - name: model-cache - create: false - services: - Epp: - componentType: epp - replicas: 1 - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.2.1 - env: - - name: DYN_KV_CACHE_BLOCK_SIZE - value: "16" - - name: DYN_MODEL_NAME - value: "Qwen/Qwen3-0.6B" - - name: DYN_DECODE_FALLBACK - value: "true" - eppConfig: - config: - plugins: - - type: disagg-profile-handler - - name: decode-filter - type: label-filter - parameters: - label: "nvidia.com/dynamo-sub-component-type" - validValues: - - "decode" - allowsNoLabel: true - - name: picker - type: max-score-picker - - name: dyn-decode - type: dyn-decode-scorer - schedulingProfiles: - - name: decode - plugins: - - pluginRef: decode-filter - weight: 1 - - pluginRef: dyn-decode - weight: 1 - - pluginRef: picker - weight: 1 - VllmDecodeWorker: - componentType: worker - envFromSecret: hf-token-secret - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 2Gi - frontendSidecar: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - args: - - -m - - dingo.frontend - - --router-mode - - direct - extraPodSpec: - mainContainer: - env: - - name: SERVED_MODEL_NAME - value: "Qwen/Qwen3-0.6B" - - name: MODEL_PATH - value: "Qwen/Qwen3-0.6B" - - name: HF_HOME - value: /opt/models - - name: DYN_STORE_KV - value: "mem" - args: - - "python3 -m dingo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 1 --data-parallel-size 1 --gpu-memory-utilization 0.90 --enable-prefix-caching --kv-events-config '{\"enable_kv_cache_events\":true}' --block-size 16" - command: - - /bin/sh - - -c - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - workingDir: /workspace/examples/backends/vllm - replicas: 2 - resources: - limits: - gpu: "1" - requests: - gpu: "1" diff --git a/recipes/qwen3-0.6b/vllm/agg/gaie/dr.yaml b/recipes/qwen3-0.6b/vllm/agg/gaie/dr.yaml deleted file mode 100644 index 3a2f074cda81..000000000000 --- a/recipes/qwen3-0.6b/vllm/agg/gaie/dr.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# DestinationRule for Qwen3-0.6B aggregated GAIE deployment with Istio -apiVersion: networking.istio.io/v1 -kind: DestinationRule -metadata: - name: qwen3-0-6b-agg-epp -spec: - # Short name resolves in the DestinationRule's own namespace — apply this - # manifest into the same namespace as the EPP service. - host: qwen3-0-6b-agg-epp - trafficPolicy: - tls: - insecureSkipVerify: true - mode: SIMPLE diff --git a/recipes/qwen3-0.6b/vllm/agg/gaie/httproute.yaml b/recipes/qwen3-0.6b/vllm/agg/gaie/httproute.yaml deleted file mode 100644 index a810406c0ba8..000000000000 --- a/recipes/qwen3-0.6b/vllm/agg/gaie/httproute.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# HTTPRoute for Qwen3-0.6B aggregated GAIE deployment -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: qwen3-0-6b-agg -spec: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: inference-gateway - rules: - - matches: - - headers: - - name: X-Gateway-Model-Name - type: Exact - value: Qwen/Qwen3-0.6B - path: - type: PathPrefix - value: / - backendRefs: - - group: inference.networking.k8s.io - kind: InferencePool - name: qwen3-0-6b-agg-pool - port: 8000 - weight: 1 - timeouts: - request: 300s diff --git a/recipes/qwen3-235b-a22b-fp8/README.md b/recipes/qwen3-235b-a22b-fp8/README.md deleted file mode 100644 index bdf8cf4975e0..000000000000 --- a/recipes/qwen3-235b-a22b-fp8/README.md +++ /dev/null @@ -1,88 +0,0 @@ -# Qwen3-235B-A22B-FP8 Recipes - -Production-ready deployments for **Qwen3-235B-A22B** (MoE model with 22B active parameters) using TensorRT-LLM. - -## Available Configurations - -| Configuration | GPUs | Hardware | Mode | Description | -|--------------|------|----------|------|-------------| -| [**trtllm/agg/hopper**](trtllm/agg/hopper/) | 16x GPU | H100/H200 | Aggregated | TP4, EP4, KV-aware routing | -| [**trtllm/agg/blackwell**](trtllm/agg/blackwell/) | 16x GPU | B100/B200 | Aggregated | TP4, EP4, KV-aware routing, DEEPGEMM | -| [**trtllm/disagg/hopper**](trtllm/disagg/hopper/) | 16x GPU | H100/H200 | Disaggregated | Prefill/decode separation | -| [**trtllm/disagg/blackwell**](trtllm/disagg/blackwell/) | 16x GPU | B100/B200 | Disaggregated | Prefill/decode separation, DEEPGEMM | - -## Prerequisites - -1. **Dynamo Platform installed** — See [Kubernetes Deployment Guide](../../docs/kubernetes/README.md) -2. **GPU cluster** with H100/H200 (Hopper) or B100/B200 (Blackwell) GPUs — see [Hardware Requirements](#hardware-requirements) -3. **HuggingFace token** with access to Qwen models - -## Quick Start - -```bash -# Set namespace -export NAMESPACE=dynamo-demo -kubectl create namespace ${NAMESPACE} - -# Create HuggingFace token secret -kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="your-token-here" \ - -n ${NAMESPACE} - -# Download model (update storageClassName in model-cache.yaml first!) -kubectl apply -f model-cache/ -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/model-download -n ${NAMESPACE} --timeout=3600s - -# Deploy — choose the variant matching your hardware: -kubectl apply -f trtllm/agg/hopper/deploy.yaml -n ${NAMESPACE} # H100/H200 -# OR: kubectl apply -f trtllm/agg/blackwell/deploy.yaml -n ${NAMESPACE} # B100/B200 -# OR: kubectl apply -f trtllm/disagg/hopper/deploy.yaml -n ${NAMESPACE} # H100/H200 -# OR: kubectl apply -f trtllm/disagg/blackwell/deploy.yaml -n ${NAMESPACE} # B100/B200 -``` - -## Test the Deployment - -```bash -# Port-forward the frontend -kubectl port-forward svc/qwen3-235b-a22b-agg-frontend 8000:8000 -n ${NAMESPACE} - -# Send a test request -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "Qwen/Qwen3-235B-A22B-FP8", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 50 - }' -``` - -## Model Details - -- **Model**: `Qwen/Qwen3-235B-A22B-FP8` -- **Architecture**: 235B parameter Mixture-of-Experts (MoE) -- **Active parameters**: ~22B per token -- **Backend**: TensorRT-LLM (PyTorch backend) -- **Parallelism**: TP4 × EP4 (Expert Parallel) - -## Hardware Requirements - -This recipe has separate variants for Hopper and Blackwell because the two architectures require different MoE backend configurations with TRT-LLM 1.3.x: - -- **Hopper (H100/H200, SM90)**: uses the default MoE backend — `trtllm/{agg,disagg}/hopper/` -- **Blackwell (B100/B200, SM100+)**: requires `moe_config.backend: DEEPGEMM` — `trtllm/{agg,disagg}/blackwell/` - -The difference: the default CUTLASS MoE backend in TRT-LLM 1.3.x falls through to a Hopper-specific JIT path when running on SM100, causing a crash. DEEPGEMM is the required workaround on Blackwell for this version. DEEPGEMM in turn crashes on Hopper due to a scale-factor dtype mismatch. Hence two separate variants. - -| Configuration | GPUs | Min GPU VRAM (Total) | -|--------------|------|----------------------| -| Aggregated (Hopper) | 16x H100/H200 | ~1.3TB | -| Aggregated (Blackwell) | 16x B100/B200 | ~1.3TB | -| Disaggregated (Hopper) | 16x H100/H200 | ~1.3TB | -| Disaggregated (Blackwell) | 16x B100/B200 | ~1.3TB | - -## Notes - -- Update `storageClassName` in `model-cache/model-cache.yaml` before deploying -- Model download may take 30-60 minutes -- Uses KV-aware routing for efficient cache utilization -- Chunked prefill enabled for aggregated mode (disabled for disaggregated) diff --git a/recipes/qwen3-235b-a22b-fp8/model-cache/model-cache.yaml b/recipes/qwen3-235b-a22b-fp8/model-cache/model-cache.yaml deleted file mode 100644 index b7e25d2d559b..000000000000 --- a/recipes/qwen3-235b-a22b-fp8/model-cache/model-cache.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 300Gi - storageClassName: "your-storage-class-name" diff --git a/recipes/qwen3-235b-a22b-fp8/model-cache/model-download.yaml b/recipes/qwen3-235b-a22b-fp8/model-cache/model-download.yaml deleted file mode 100644 index 5dfe37a29841..000000000000 --- a/recipes/qwen3-235b-a22b-fp8/model-cache/model-download.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: MODEL_NAME - value: Qwen/Qwen3-235B-A22B-FP8 - - name: HF_HOME - value: /model-store - # Uses up to 64 GB RAM for XET buffers; set "0" on low-memory nodes (8 GB cap): https://huggingface.co/docs/hub/en/xet/using-xet-storage#download-buffers - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - - name: MODEL_REVISION - value: 39eb2b067ea6b8e3e1dd97d3cd0c7ffeaf3e1a35 - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.11.0 - hf download $MODEL_NAME --revision $MODEL_REVISION - resources: - requests: - cpu: "2" - memory: "64Gi" - limits: - cpu: "8" - memory: "64Gi" - volumeMounts: - - name: model-cache - mountPath: /model-store - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/qwen3-235b-a22b-fp8/trtllm/agg/blackwell/deploy.yaml b/recipes/qwen3-235b-a22b-fp8/trtllm/agg/blackwell/deploy.yaml deleted file mode 100644 index 170a63cdda8e..000000000000 --- a/recipes/qwen3-235b-a22b-fp8/trtllm/agg/blackwell/deploy.yaml +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: ConfigMap -metadata: - name: agg-config -data: - agg.yaml: | - backend: pytorch - trust_remote_code: true - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - moe_tensor_parallel_size: 1 - enable_attention_dp: false - enable_chunked_prefill: true - kv_cache_config: - enable_block_reuse: true - free_gpu_memory_fraction: 0.8 - dtype: auto - cache_transceiver_config: - backend: DEFAULT - cuda_graph_config: - enable_padding: true - max_batch_size: 128 - disable_overlap_scheduler: false - print_iter_log: false - # Required on Blackwell (SM100+): default CUTLASS backend falls through to a - # Hopper-specific JIT path on SM100 with TRT-LLM 1.3.x. DEEPGEMM is the workaround. - moe_config: - backend: DEEPGEMM - max_num_tokens: 8192 ---- -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: qwen3-235b-a22b-agg -spec: - backendFramework: trtllm - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - replicas: 1 - extraPodSpec: - tolerations: [] - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: nvidia.com/dynamo-graph-deployment-name - operator: In - values: - - qwen3-235b-a22b-agg-frontend - topologyKey: kubernetes.io/hostname - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - args: - - python3 -m dingo.frontend --router-mode kv --http-port 8000 - command: - - /bin/sh - - -c - TrtllmWorker: - componentType: worker - envFromSecret: hf-token-secret - sharedMemory: - size: 256Gi - extraPodSpec: - tolerations: [] - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.present - operator: In - values: - - "true" - mainContainer: - env: - - name: MODEL_PATH - value: Qwen/Qwen3-235B-A22B-FP8 - - name: HF_HOME - value: /mnt/model-cache - - name: ENGINE_ARGS - value: /engine_configs/agg.yaml - command: - - /bin/sh - - -c - args: - - | - python3 -m dynamo.trtllm \ - --model-path "${MODEL_PATH}" \ - --served-model-name "Qwen/Qwen3-235B-A22B-FP8" \ - --max-batch-size 128 \ - --max-num-tokens 8192 \ - --max-seq-len 8192 \ - --extra-engine-args "${ENGINE_ARGS}" - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 - workingDir: /workspace/components/backends/trtllm - volumeMounts: - - name: agg-config - mountPath: /engine_configs - - name: model-cache - mountPath: /mnt/model-cache - volumes: - - name: agg-config - configMap: - name: agg-config - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - replicas: 4 - resources: - limits: - gpu: "4" - requests: - gpu: "4" diff --git a/recipes/qwen3-235b-a22b-fp8/trtllm/agg/blackwell/perf.yaml b/recipes/qwen3-235b-a22b-fp8/trtllm/agg/blackwell/perf.yaml deleted file mode 100644 index bb151fe56564..000000000000 --- a/recipes/qwen3-235b-a22b-fp8/trtllm/agg/blackwell/perf.yaml +++ /dev/null @@ -1,152 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: qwen3-235b-a22b-bench -spec: - backoffLimit: 1 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: qwen3-235b-a22b-bench - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: nvidia.com/dynamo-graph-deployment-name - operator: In - values: - - qwen3-235b-a22b-agg - topologyKey: kubernetes.io/hostname - containers: - - command: - - /bin/sh - - -c - - | - apt-get update && apt-get install -y curl jq procps git && apt-get clean - pip install "aiperf==0.10.0"; - echo "aiperf installation completed"; - sysctl -w net.ipv4.ip_local_port_range="1024 65000" - cat /proc/sys/net/ipv4/ip_local_port_range - export COLUMNS=200 - EPOCH=$(date +%s) - ## utility functions -- can be moved to a bash script / configmap - wait_for_model_ready() { - echo "Waiting for model '$TARGET_MODEL' at $ENDPOINT/v1/models (checking every 5s)..." - while ! curl -s "http://$ENDPOINT/v1/models" | jq -e --arg model "$TARGET_MODEL" '.data[]? | select(.id == $model)' >/dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready yet, sleeping 5s before checking again http://$ENDPOINT/v1/models" - sleep 5 - done - echo "✅ Model '$TARGET_MODEL' is now available!" - echo "Model '$TARGET_MODEL' is now available!" - curl -s "http://$ENDPOINT/v1/models" | jq . - } - run_perf() { - local concurrency=$1 - local isl=$2 - local osl=$3 - key=concurrency_${concurrency} - export ARTIFACT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/${key}" - mkdir -p "$ARTIFACT_DIR" - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - aiperf profile --artifact-dir $ARTIFACT_DIR \ - --model $TARGET_MODEL \ - --tokenizer $TARGET_MODEL \ - --endpoint-type chat \ - --endpoint /v1/chat/completions \ - --streaming \ - --url http://$ENDPOINT \ - --synthetic-input-tokens-mean $isl \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean $osl \ - --output-tokens-stddev 0 \ - --extra-inputs "max_tokens:$osl" \ - --extra-inputs "min_tokens:$osl" \ - --extra-inputs "ignore_eos:true" \ - --extra-inputs "repetition_penalty:1.0" \ - --extra-inputs "temperature: 0.0" \ - --concurrency $concurrency \ - --request-count $((10*concurrency)) \ - --warmup-request-count $concurrency \ - --num-dataset-entries 12800 \ - --random-seed 100 \ - --workers-max 252 \ - -H 'Authorization: Bearer NOT USED' \ - -H 'Accept: text/event-stream'\ - --record-processors 32 \ - --ui simple - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - ls -la $ARTIFACT_DIR - } - #### Actual execution #### - wait_for_model_ready - mkdir -p "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - # Calculate total concurrency based on per-GPU concurrency and GPU count - TOTAL_CONCURRENCY=$((CONCURRENCY_PER_GPU * DEPLOYMENT_GPU_COUNT)) - echo "Calculated total concurrency: $TOTAL_CONCURRENCY (${CONCURRENCY_PER_GPU} per GPU × ${DEPLOYMENT_GPU_COUNT} GPUs)" - # Write input_config.json - cat > "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/input_config.json" </dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready yet, sleeping 5s before checking again http://$ENDPOINT/v1/models" - sleep 5 - done - echo "✅ Model '$TARGET_MODEL' is now available!" - echo "Model '$TARGET_MODEL' is now available!" - curl -s "http://$ENDPOINT/v1/models" | jq . - } - run_perf() { - local concurrency=$1 - local isl=$2 - local osl=$3 - key=concurrency_${concurrency} - export ARTIFACT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/${key}" - mkdir -p "$ARTIFACT_DIR" - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - aiperf profile --artifact-dir $ARTIFACT_DIR \ - --model $TARGET_MODEL \ - --tokenizer $TARGET_MODEL \ - --endpoint-type chat \ - --endpoint /v1/chat/completions \ - --streaming \ - --url http://$ENDPOINT \ - --synthetic-input-tokens-mean $isl \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean $osl \ - --output-tokens-stddev 0 \ - --extra-inputs "max_tokens:$osl" \ - --extra-inputs "min_tokens:$osl" \ - --extra-inputs "ignore_eos:true" \ - --extra-inputs "repetition_penalty:1.0" \ - --extra-inputs "temperature: 0.0" \ - --concurrency $concurrency \ - --request-count $((10*concurrency)) \ - --warmup-request-count $concurrency \ - --num-dataset-entries 12800 \ - --random-seed 100 \ - --workers-max 252 \ - -H 'Authorization: Bearer NOT USED' \ - -H 'Accept: text/event-stream'\ - --record-processors 32 \ - --ui simple - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - ls -la $ARTIFACT_DIR - } - #### Actual execution #### - wait_for_model_ready - mkdir -p "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - # Calculate total concurrency based on per-GPU concurrency and GPU count - TOTAL_CONCURRENCY=$((CONCURRENCY_PER_GPU * DEPLOYMENT_GPU_COUNT)) - echo "Calculated total concurrency: $TOTAL_CONCURRENCY (${CONCURRENCY_PER_GPU} per GPU × ${DEPLOYMENT_GPU_COUNT} GPUs)" - # Write input_config.json - cat > "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/input_config.json" </dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready yet, sleeping 5s before checking again http://$ENDPOINT/v1/models" - sleep 5 - done - echo "✅ Model '$TARGET_MODEL' is now available!" - echo "Model '$TARGET_MODEL' is now available!" - curl -s "http://$ENDPOINT/v1/models" | jq . - } - run_perf() { - local concurrency=$1 - local isl=$2 - local osl=$3 - key=concurrency_${concurrency} - export ARTIFACT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/${key}" - mkdir -p "$ARTIFACT_DIR" - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - aiperf profile --artifact-dir $ARTIFACT_DIR \ - --model $TARGET_MODEL \ - --tokenizer $TARGET_MODEL \ - --endpoint-type chat \ - --endpoint /v1/chat/completions \ - --streaming \ - --url http://$ENDPOINT \ - --synthetic-input-tokens-mean $isl \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean $osl \ - --output-tokens-stddev 0 \ - --extra-inputs "max_tokens:$osl" \ - --extra-inputs "min_tokens:$osl" \ - --extra-inputs "ignore_eos:true" \ - --extra-inputs "repetition_penalty:1.0" \ - --extra-inputs "temperature: 0.0" \ - --concurrency $concurrency \ - --request-count $((10*concurrency)) \ - --warmup-request-count $concurrency \ - --num-dataset-entries 12800 \ - --random-seed 100 \ - --workers-max 252 \ - -H 'Authorization: Bearer NOT USED' \ - -H 'Accept: text/event-stream'\ - --record-processors 32 \ - --ui simple - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - ls -la $ARTIFACT_DIR - } - #### Actual execution #### - wait_for_model_ready - mkdir -p "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - # Calculate total concurrency based on per-GPU concurrency and GPU count - TOTAL_CONCURRENCY=$((CONCURRENCY_PER_GPU * DEPLOYMENT_GPU_COUNT)) - echo "Calculated total concurrency: $TOTAL_CONCURRENCY (${CONCURRENCY_PER_GPU} per GPU × ${DEPLOYMENT_GPU_COUNT} GPUs)" - # Write input_config.json - cat > "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/input_config.json" </dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready yet, sleeping 5s before checking again http://$ENDPOINT/v1/models" - sleep 5 - done - echo "✅ Model '$TARGET_MODEL' is now available!" - echo "Model '$TARGET_MODEL' is now available!" - curl -s "http://$ENDPOINT/v1/models" | jq . - } - run_perf() { - local concurrency=$1 - local isl=$2 - local osl=$3 - key=concurrency_${concurrency} - export ARTIFACT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/${key}" - mkdir -p "$ARTIFACT_DIR" - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - aiperf profile --artifact-dir $ARTIFACT_DIR \ - --model $TARGET_MODEL \ - --tokenizer $TARGET_MODEL \ - --endpoint-type chat \ - --endpoint /v1/chat/completions \ - --streaming \ - --url http://$ENDPOINT \ - --synthetic-input-tokens-mean $isl \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean $osl \ - --output-tokens-stddev 0 \ - --extra-inputs "max_tokens:$osl" \ - --extra-inputs "min_tokens:$osl" \ - --extra-inputs "ignore_eos:true" \ - --extra-inputs "repetition_penalty:1.0" \ - --extra-inputs "temperature: 0.0" \ - --concurrency $concurrency \ - --request-count $((10*concurrency)) \ - --warmup-request-count $concurrency \ - --num-dataset-entries 12800 \ - --random-seed 100 \ - --workers-max 252 \ - -H 'Authorization: Bearer NOT USED' \ - -H 'Accept: text/event-stream'\ - --record-processors 32 \ - --ui simple - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - ls -la $ARTIFACT_DIR - } - #### Actual execution #### - wait_for_model_ready - mkdir -p "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - # Calculate total concurrency based on per-GPU concurrency and GPU count - TOTAL_CONCURRENCY=$((CONCURRENCY_PER_GPU * DEPLOYMENT_GPU_COUNT)) - echo "Calculated total concurrency: $TOTAL_CONCURRENCY (${CONCURRENCY_PER_GPU} per GPU × ${DEPLOYMENT_GPU_COUNT} GPUs)" - # Write input_config.json - cat > "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/input_config.json" </dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready yet, sleeping 5s before checking again http://$ENDPOINT/v1/models" - sleep 5 - done - echo "✅ Model '$TARGET_MODEL' is now available!" - echo "Model '$TARGET_MODEL' is now available!" - curl -s "http://$ENDPOINT/v1/models" | jq . - } - run_perf() { - local concurrency=$1 - local isl=$2 - local osl=$3 - key=concurrency_${concurrency} - export ARTIFACT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/${key}" - mkdir -p "$ARTIFACT_DIR" - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - aiperf profile --artifact-dir $ARTIFACT_DIR \ - --model $TARGET_MODEL \ - --tokenizer $TARGET_MODEL \ - --endpoint-type chat \ - --endpoint /v1/chat/completions \ - --streaming \ - --url http://$ENDPOINT \ - --synthetic-input-tokens-mean $isl \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean $osl \ - --output-tokens-stddev 0 \ - --extra-inputs "max_tokens:$osl" \ - --extra-inputs "min_tokens:$osl" \ - --extra-inputs "ignore_eos:true" \ - --extra-inputs "repetition_penalty:1.0" \ - --extra-inputs "temperature: 0.0" \ - --concurrency $concurrency \ - --request-count $((10*concurrency)) \ - --warmup-request-count $concurrency \ - --num-dataset-entries 12800 \ - --random-seed 100 \ - --workers-max 252 \ - -H 'Authorization: Bearer NOT USED' \ - -H 'Accept: text/event-stream'\ - --record-processors 32 \ - --ui simple - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - ls -la $ARTIFACT_DIR - } - #### Actual execution #### - wait_for_model_ready - mkdir -p "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - # Calculate total concurrency based on per-GPU concurrency and GPU count - TOTAL_CONCURRENCY=$((CONCURRENCY_PER_GPU * DEPLOYMENT_GPU_COUNT)) - echo "Calculated total concurrency: $TOTAL_CONCURRENCY (${CONCURRENCY_PER_GPU} per GPU × ${DEPLOYMENT_GPU_COUNT} GPUs)" - # Write input_config.json - cat > "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/input_config.json" </dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready yet, sleeping 5s before checking again http://$ENDPOINT/v1/models" - sleep 5 - done - echo "✅ Model '$TARGET_MODEL' is now available!" - echo "Model '$TARGET_MODEL' is now available!" - curl -s "http://$ENDPOINT/v1/models" | jq . - } - run_perf() { - local concurrency=$1 - local isl=$2 - local osl=$3 - key=concurrency_${concurrency} - export ARTIFACT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/${key}" - mkdir -p "$ARTIFACT_DIR" - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - aiperf profile --artifact-dir $ARTIFACT_DIR \ - --model $TARGET_MODEL \ - --tokenizer $TARGET_MODEL \ - --endpoint-type chat \ - --endpoint /v1/chat/completions \ - --streaming \ - --url http://$ENDPOINT \ - --synthetic-input-tokens-mean $isl \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean $osl \ - --output-tokens-stddev 0 \ - --extra-inputs "max_tokens:$osl" \ - --extra-inputs "min_tokens:$osl" \ - --extra-inputs "ignore_eos:true" \ - --extra-inputs "repetition_penalty:1.0" \ - --extra-inputs "temperature: 0.0" \ - --concurrency $concurrency \ - --request-count $((10*concurrency)) \ - --warmup-request-count $concurrency \ - --num-dataset-entries 12800 \ - --random-seed 100 \ - --workers-max 252 \ - -H 'Authorization: Bearer NOT USED' \ - -H 'Accept: text/event-stream'\ - --record-processors 32 \ - --ui simple - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - ls -la $ARTIFACT_DIR - } - #### Actual execution #### - wait_for_model_ready - mkdir -p "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - # Calculate total concurrency based on per-GPU concurrency and GPU count - TOTAL_CONCURRENCY=$((CONCURRENCY_PER_GPU * DEPLOYMENT_GPU_COUNT)) - echo "Calculated total concurrency: $TOTAL_CONCURRENCY (${CONCURRENCY_PER_GPU} per GPU × ${DEPLOYMENT_GPU_COUNT} GPUs)" - # Write input_config.json - cat > "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/input_config.json" </dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready yet, sleeping 5s before checking again http://$ENDPOINT/v1/models" - sleep 5 - done - echo "✅ Model '$TARGET_MODEL' is now available!" - echo "Model '$TARGET_MODEL' is now available!" - curl -s "http://$ENDPOINT/v1/models" | jq . - } - run_perf() { - local concurrency=$1 - local isl=$2 - local osl=$3 - key=concurrency_${concurrency} - export ARTIFACT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/${key}" - mkdir -p "$ARTIFACT_DIR" - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - aiperf profile --artifact-dir $ARTIFACT_DIR \ - --model $TARGET_MODEL \ - --tokenizer $TARGET_MODEL \ - --endpoint-type chat \ - --endpoint /v1/chat/completions \ - --streaming \ - --url http://$ENDPOINT \ - --synthetic-input-tokens-mean $isl \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean $osl \ - --output-tokens-stddev 0 \ - --extra-inputs "max_tokens:$osl" \ - --extra-inputs "min_tokens:$osl" \ - --extra-inputs "ignore_eos:true" \ - --extra-inputs "repetition_penalty:1.0" \ - --extra-inputs "temperature: 0.0" \ - --concurrency $concurrency \ - --request-count $((10*concurrency)) \ - --warmup-request-count $concurrency \ - --num-dataset-entries 12800 \ - --random-seed 100 \ - --workers-max $concurrency \ - -H 'Authorization: Bearer NOT USED' \ - -H 'Accept: text/event-stream'\ - --record-processors 32 \ - --ui simple - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - ls -la $ARTIFACT_DIR - } - #### Actual execution #### - wait_for_model_ready - mkdir -p "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - # Calculate total concurrency based on per-GPU concurrency and GPU count - TOTAL_CONCURRENCY=$((CONCURRENCY_PER_GPU * DEPLOYMENT_GPU_COUNT)) - echo "Calculated total concurrency: $TOTAL_CONCURRENCY (${CONCURRENCY_PER_GPU} per GPU × ${DEPLOYMENT_GPU_COUNT} GPUs)" - # Write input_config.json - cat > "${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/input_config.json" < **Note:** Edit `model-cache/cache.yaml` first and update `storageClassName` to match your cluster (run `kubectl get storageclass` to find available options). - -```bash -kubectl apply -f model-cache/cache.yaml -n ${NAMESPACE} -``` - -### 2. Download Model - -```bash -kubectl apply -f model-cache/model-download.yaml -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/model-download -n ${NAMESPACE} --timeout=600s -``` - -### 3. Deploy & Benchmark - -**Option A: Aggregated (Round-Robin Baseline)** - -```bash -# Deploy -kubectl apply -f vllm/agg-round-robin/deploy.yaml -n ${NAMESPACE} - -# Wait for ready -kubectl wait --for=condition=ready pod -l nvidia.com/dynamo-graph-deployment-name=agg-8xtp2 \ - -n ${NAMESPACE} --timeout=1200s - -# Run benchmark -kubectl apply -f vllm/agg-round-robin/perf.yaml -n ${NAMESPACE} -``` - -**Option B: Disaggregated (KV-Aware Routing)** - -```bash -# Deploy -kubectl apply -f vllm/disagg-kv-router/deploy.yaml -n ${NAMESPACE} - -# Wait for ready -kubectl wait --for=condition=ready pod -l nvidia.com/dynamo-graph-deployment-name=disagg-router-6p-2d \ - -n ${NAMESPACE} --timeout=1200s - -# Run benchmark -kubectl apply -f vllm/disagg-kv-router/perf.yaml -n ${NAMESPACE} -``` - -### 4. Monitor Benchmark Progress - -The benchmark runs inside a tmux session for easy monitoring: - -```bash -# Find the benchmark pod -kubectl get pods -n ${NAMESPACE} | grep benchmark - -# Attach to the tmux session to see intermediate results -kubectl exec -it -n ${NAMESPACE} -- tmux a -t benchmark - -# Detach from tmux: Ctrl+B, then D -``` - -### 5. View Results - -Results are saved to the `perf-cache` PVC: - -```bash -# Check artifact directory -kubectl exec -it -n ${NAMESPACE} -- ls -la /perf-cache/artifacts/ - -# Copy results to local machine -kubectl cp ${NAMESPACE}/:/perf-cache/artifacts ./benchmark-results -``` - -## Expected Results - -Since the benchmark uses `--fixed-schedule` (replaying requests at their original timestamps), **throughput metrics are fixed by the trace**—latency metrics are what we're comparing: - -| Metric | Why It Matters | -|--------|----------------| -| **TTFT** (Time to First Token) | KV-aware routing reduces prefill compute via prefix cache hits | -| **ITL** (Inter-Token Latency) | Disaggregated serving isolates decode from prefill interference | -| **Total Request Latency** | Combined benefit of both optimizations | - -**Why disaggregated + KV-aware routing helps this workload:** - -1. **KV-aware routing** leverages the 36% cache efficiency to route requests to workers that already have relevant KV cache blocks, reducing redundant prefill computation and lowering TTFT. - -2. **Disaggregated serving** separates prefill and decode workers. With long input sequences (avg 12K tokens) and short outputs (avg 343 tokens), dedicated decode workers avoid "prefill injection"—where a new long-context request interrupts ongoing decode operations, causing ITL spikes. - -## Cleanup - -```bash -# Delete benchmark pods -kubectl delete pod -l app=benchmark -n ${NAMESPACE} - -# Delete deployments -kubectl delete dynamographdeployment agg-8xtp2 -n ${NAMESPACE} -kubectl delete dynamographdeployment disagg-router-6p-2d -n ${NAMESPACE} -``` - -## References - -- [Mooncake: A KVCache-centric Disaggregated Architecture for LLM Serving](https://github.com/kvcache-ai/Mooncake) - FAST25 paper and trace data diff --git a/recipes/qwen3-32b/model-cache/cache.yaml b/recipes/qwen3-32b/model-cache/cache.yaml deleted file mode 100644 index b296767fb153..000000000000 --- a/recipes/qwen3-32b/model-cache/cache.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 100Gi - storageClassName: "your-storage-class-name" ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: compilation-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 50Gi - storageClassName: "your-storage-class-name" ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: perf-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 50Gi - storageClassName: "your-storage-class-name" diff --git a/recipes/qwen3-32b/model-cache/model-download.yaml b/recipes/qwen3-32b/model-cache/model-download.yaml deleted file mode 100644 index d603a9be7458..000000000000 --- a/recipes/qwen3-32b/model-cache/model-download.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: MODEL_NAME - value: "Qwen/Qwen3-32B" - - name: HF_HOME - value: /home/dynamo/.cache/huggingface - # Uses up to 64 GB RAM for XET buffers; set "0" on low-memory nodes (8 GB cap): https://huggingface.co/docs/hub/en/xet/using-xet-storage#download-buffers - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - - name: MODEL_REVISION - value: 9216db5781bf21249d130ec9da846c4624c16137 - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.11.0 - hf download $MODEL_NAME --revision $MODEL_REVISION - resources: - requests: - cpu: "2" - memory: "64Gi" - limits: - cpu: "8" - memory: "64Gi" - volumeMounts: - - name: model-cache - mountPath: /home/dynamo/.cache/huggingface - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/qwen3-32b/vllm/agg-kvbm/README.md b/recipes/qwen3-32b/vllm/agg-kvbm/README.md deleted file mode 100644 index 9b769fd9b9d9..000000000000 --- a/recipes/qwen3-32b/vllm/agg-kvbm/README.md +++ /dev/null @@ -1,102 +0,0 @@ -# Qwen3-32B: Aggregated + KVBM (single GPU) - -Single-GPU aggregated deployment of `Qwen/Qwen3-32B` with the KV Block Manager -(KVBM) enabled. KVBM offloads cold KV cache blocks to host memory so the -effective cache footprint extends beyond GPU HBM, which improves prefix-reuse -hit rate on long or repeated prompts without adding GPUs. - -## Hardware - -- **1x NVIDIA H200 (141 GB) or B200 (192 GB)**. Qwen3-32B in BF16 is ~64 GB of - weights plus KV cache and activations, so 80 GB H100 leaves very little room - and is likely to OOM under real load. If you only have H100 80 GB, see - `../../../qwen3-32b-fp8/` for the FP8 variant. -- **≥ ~150 GiB of host memory on the node**. `DYN_KVBM_CPU_CACHE_GB=100` is - pinned as page-locked host memory for KVBM's G2 tier. The worker declares - `resources.requests.memory: 150Gi` and `resources.limits.memory: 200Gi` - (100 GiB pinned KV pool + ~50 GiB headroom for Python, weight-loader - working memory, and CUDA/NCCL buffers). If you raise `DYN_KVBM_CPU_CACHE_GB`, - scale these up by roughly the same delta. - -## Prerequisites - -Same as the sibling recipes in this directory: - -1. **Dynamo Platform installed** — see the [Kubernetes Deployment Guide](../../../../docs/kubernetes/README.md). -2. **Pre-existing `model-cache` and `compilation-cache` PVCs** — see - [`../../model-cache/cache.yaml`](../../model-cache/cache.yaml) and - [`../../model-cache/model-download.yaml`](../../model-cache/model-download.yaml). -3. **HuggingFace token Secret** named `hf-token-secret` in your namespace. - -```bash -export NAMESPACE=your-namespace -kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="your-token" \ - -n ${NAMESPACE} -``` - -## Deploy - -```bash -kubectl apply -f deploy.yaml -n ${NAMESPACE} - -kubectl wait --for=condition=ready pod \ - -l nvidia.com/dynamo-graph-deployment-name=agg-kvbm-qwen3-32b \ - -n ${NAMESPACE} --timeout=1200s -``` - -## Verify - -```bash -kubectl port-forward svc/agg-kvbm-qwen3-32b-frontend 8000:8000 -n ${NAMESPACE} - -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "Qwen/Qwen3-32B", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 64 - }' -``` - -## KVBM configuration - -The connector is selected through the worker's `--kv-transfer-config`: - -```json -{"kv_connector":"DynamoConnector","kv_role":"kv_both","kv_connector_module_path":"kvbm.vllm_integration.connector"} -``` - -Worker env var set by this recipe: - -| Variable | Default in this recipe | Description | -|---|---|---| -| `DYN_KVBM_CPU_CACHE_GB` | `100` | CPU memory reserved for offloaded KV blocks. Raise for longer contexts or higher reuse; if you change this, also bump `resources.requests.memory` / `limits.memory` on the worker by roughly the same delta. | - -### (Optional) Prometheus metrics - -Metrics are **off** by default. To expose them, add the following to the -worker's `env` and `mainContainer.ports` in `deploy.yaml`: - -```yaml -env: - - name: DYN_KVBM_METRICS - value: "true" - - name: DYN_KVBM_METRICS_PORT - value: "6880" -ports: - - name: kvbm - containerPort: 6880 -``` - -Once enabled, scrape `:6880/metrics` for counters like -`kvbm_offload_blocks_d2h`, `kvbm_onboard_blocks_h2d`, `kvbm_matched_tokens`, -`kvbm_host_cache_hit_rate`, plus per-route transfer counters. If you run the -Prometheus Operator, add a `PodMonitor` selecting pods with -`nvidia.com/dynamo-component-type: worker` and port `kvbm`. - -## Cleanup - -```bash -kubectl delete dynamographdeployment agg-kvbm-qwen3-32b -n ${NAMESPACE} -``` diff --git a/recipes/qwen3-32b/vllm/agg-kvbm/deploy.yaml b/recipes/qwen3-32b/vllm/agg-kvbm/deploy.yaml deleted file mode 100644 index 6673b600349b..000000000000 --- a/recipes/qwen3-32b/vllm/agg-kvbm/deploy.yaml +++ /dev/null @@ -1,68 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: agg-kvbm-qwen3-32b -spec: - pvcs: - - create: false - name: model-cache - - create: false - name: compilation-cache - services: - Frontend: - componentType: frontend - envs: - - name: HF_HOME - value: /home/dynamo/.cache/huggingface - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - workingDir: /workspace - command: - - python3 - - -m - - dingo.frontend - args: - - --router-reset-states - replicas: 1 - subComponentType: null - VllmDecodeWorker: - componentType: worker - envFromSecret: hf-token-secret - volumeMounts: - - name: model-cache - mountPoint: /home/dynamo/.cache/huggingface - - name: compilation-cache - mountPoint: /home/dynamo/.cache/vllm - useAsCompilationCache: true - extraPodSpec: - mainContainer: - args: - - --model - - Qwen/Qwen3-32B - - --kv-transfer-config - - '{"kv_connector":"DynamoConnector","kv_role":"kv_both","kv_connector_module_path":"kvbm.vllm_integration.connector"}' - command: - - python3 - - -m - - dingo.vllm - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - env: - - name: DYN_HEALTH_CHECK_ENABLED - value: "false" - - name: HF_HOME - value: /home/dynamo/.cache/huggingface - # NOTE: change this to tune the CPU cache size for your system - - name: DYN_KVBM_CPU_CACHE_GB - value: "100" - workingDir: /workspace - replicas: 1 - resources: - limits: - gpu: "1" - memory: "200Gi" - requests: - gpu: "1" - memory: "150Gi" diff --git a/recipes/qwen3-32b/vllm/agg-round-robin/deploy.yaml b/recipes/qwen3-32b/vllm/agg-round-robin/deploy.yaml deleted file mode 100644 index f062699a57e9..000000000000 --- a/recipes/qwen3-32b/vllm/agg-round-robin/deploy.yaml +++ /dev/null @@ -1,81 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: agg-8xtp2 -spec: - pvcs: - - create: false - name: model-cache - - create: false - name: compilation-cache - services: - Frontend: - componentType: frontend - envs: - - name: HF_HOME - value: /home/dynamo/.cache/huggingface - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - workingDir: /workspace - command: - - python3 - - -m - - dingo.frontend - args: - - --router-reset-states - replicas: 1 - resources: - requests: - cpu: "8" - limits: - cpu: "8" - subComponentType: null - VllmDecodeWorker: - componentType: worker - envFromSecret: hf-token-secret - volumeMounts: - - name: model-cache - mountPoint: /home/dynamo/.cache/huggingface - - name: compilation-cache - mountPoint: /home/dynamo/.cache/vllm - useAsCompilationCache: true - extraPodSpec: - mainContainer: - args: - - --model - - Qwen/Qwen3-32B - - --tensor-parallel-size - - '2' - - --no-enable-log-requests - - --gpu-memory-utilization - - '0.90' - - --async-scheduling - - --block-size - - '64' - - --hf-overrides - - '{"rope_scaling":{"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":32768},"max_position_embeddings":131072}' - - --max-model-len - - '131072' - command: - - python3 - - -m - - dingo.vllm - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - env: - - name: DYN_HEALTH_CHECK_ENABLED - value: "false" - - name: HF_HOME - value: /home/dynamo/.cache/huggingface - workingDir: /workspace - replicas: 8 - resources: - limits: - gpu: '2' - custom: - rdma/ib: "2" - requests: - gpu: '2' - subComponentType: decode diff --git a/recipes/qwen3-32b/vllm/agg-round-robin/perf.yaml b/recipes/qwen3-32b/vllm/agg-round-robin/perf.yaml deleted file mode 100644 index c2086c812cc7..000000000000 --- a/recipes/qwen3-32b/vllm/agg-round-robin/perf.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: Pod -metadata: - name: agg-8xtp2-benchmark - labels: - app: benchmark -spec: - containers: - - name: python - image: python:3.11 - command: - - /bin/bash - - -lc - - | - # Setup - ulimit -n 1048576 - ulimit -u 65536 - apt update && apt install tmux wget curl jq -y - - # Install benchmarking tool - pip install aiperf==0.10.0 - - # Wait for model to be ready - echo "Waiting for model '${MODEL_NAME}' at http://${FRONTEND}:8000/v1/models..." - until curl -s "http://${FRONTEND}:8000/v1/models" | jq -e --arg model "${MODEL_NAME}" '.data[]? | select(.id == $model)' >/dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready, retrying in 5s..." - sleep 5 - done - echo "Model '${MODEL_NAME}' is ready!" - - # Download Mooncake conversation trace dataset if not already present - mkdir -p ${BASE_DIR}/traces - mkdir -p ${BASE_DIR}/artifacts - if [ ! -f ${BASE_DIR}/traces/conversation_trace.jsonl ]; then - wget -qO ${BASE_DIR}/traces/conversation_trace.jsonl https://raw.githubusercontent.com/kvcache-ai/Mooncake/main/FAST25-release/traces/conversation_trace.jsonl - fi - - # Setup Paths and Endpoints - export INPUT_FILE="${BASE_DIR}/traces/conversation_trace.jsonl" - export MODEL_BASE_NAME="${MODEL_NAME##*/}" - export FRONTEND_LIB="${FRONTEND%-frontend}" - export ARTIFACT_DIR="${BASE_DIR}/artifacts/${MODEL_BASE_NAME}_${FRONTEND_LIB}" - mkdir -p "${ARTIFACT_DIR}" - - # Run Benchmark so its easy to attach and watch - tmux new-session -d -s benchmark -c "${ARTIFACT_DIR}" - tmux send-keys -t benchmark "aiperf profile -m ${MODEL_NAME} --input-file ${INPUT_FILE} --custom-dataset-type mooncake_trace --fixed-schedule --url http://${FRONTEND}:8000 --streaming --artifact-dir ${ARTIFACT_DIR} --goodput \"time_to_first_token:2000 inter_token_latency:25\"" C-m - sleep 7200 - env: - - name: MODEL_NAME - value: Qwen/Qwen3-32B - - name: FRONTEND - value: agg-8xtp2-frontend - - name: BASE_DIR - value: /perf-cache - resources: - requests: - cpu: "8" - memory: 16Gi - limits: - cpu: "16" - memory: 32Gi - volumeMounts: - - name: model-cache - mountPath: /home/dynamo/.cache/huggingface - - name: perf-cache - mountPath: /perf-cache - workingDir: /workspace - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: perf-cache - persistentVolumeClaim: - claimName: perf-cache - restartPolicy: Never diff --git a/recipes/qwen3-32b/vllm/disagg-kv-router/deploy.yaml b/recipes/qwen3-32b/vllm/disagg-kv-router/deploy.yaml deleted file mode 100644 index 9877fa6ee669..000000000000 --- a/recipes/qwen3-32b/vllm/disagg-kv-router/deploy.yaml +++ /dev/null @@ -1,145 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: disagg-router-6p-2d -spec: - pvcs: - - create: false - name: model-cache - - create: false - name: compilation-cache - services: - Frontend: - componentType: frontend - envs: - - name: HF_HOME - value: /home/dynamo/.cache/huggingface - extraPodSpec: - mainContainer: - args: - - --router-mode - - kv - - --router-reset-states - command: - - python - - -m - - dingo.frontend - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - workingDir: /workspace - replicas: 1 - resources: - requests: - cpu: "8" - limits: - cpu: "8" - subComponentType: null - VllmDecodeWorker: - componentType: worker - envFromSecret: hf-token-secret - extraPodSpec: - mainContainer: - args: - - --model - - Qwen/Qwen3-32B - - --disaggregation-mode - - decode - - --kv-transfer-config - - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - - --tensor-parallel-size - - '2' - - --no-enable-log-requests - - --gpu-memory-utilization - - '0.90' - - --no-enable-prefix-caching - - --async-scheduling - - --block-size - - '64' - - --hf-overrides - - '{"rope_scaling":{"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":32768},"max_position_embeddings":131072}' - - --max-model-len - - '131072' - command: - - python3 - - -m - - dingo.vllm - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - workingDir: /workspace - env: - - name: DYN_HEALTH_CHECK_ENABLED - value: "false" - - name: HF_HOME - value: /home/dynamo/.cache/huggingface - replicas: 2 - resources: - limits: - gpu: '2' - custom: - rdma/ib: "2" - requests: - gpu: '2' - subComponentType: decode - volumeMounts: - - name: model-cache - mountPoint: /home/dynamo/.cache/huggingface - - name: compilation-cache - mountPoint: /home/dynamo/.cache/vllm - useAsCompilationCache: true - VllmPrefillWorker: - componentType: worker - envFromSecret: hf-token-secret - extraPodMetadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9400" - prometheus.io/path: "/metrics" - extraPodSpec: - mainContainer: - args: - - --model - - Qwen/Qwen3-32B - - --disaggregation-mode - - prefill - - --kv-transfer-config - - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - - --tensor-parallel-size - - '2' - - --no-enable-log-requests - - --gpu-memory-utilization - - '0.90' - - --async-scheduling - - --block-size - - '64' - - --hf-overrides - - '{"rope_scaling":{"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":32768},"max_position_embeddings":131072}' - - --max-model-len - - '131072' - - --kv-events-config - - '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - command: - - python3 - - -m - - dingo.vllm - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - env: - - name: DYN_HEALTH_CHECK_ENABLED - value: "false" - - name: HF_HOME - value: /home/dynamo/.cache/huggingface - workingDir: /workspace - replicas: 6 - resources: - limits: - gpu: '2' - custom: - rdma/ib: "2" - requests: - gpu: '2' - subComponentType: prefill - volumeMounts: - - name: model-cache - mountPoint: /home/dynamo/.cache/huggingface - - name: compilation-cache - mountPoint: /home/dynamo/.cache/vllm - useAsCompilationCache: true diff --git a/recipes/qwen3-32b/vllm/disagg-kv-router/perf.yaml b/recipes/qwen3-32b/vllm/disagg-kv-router/perf.yaml deleted file mode 100644 index 52f73097c560..000000000000 --- a/recipes/qwen3-32b/vllm/disagg-kv-router/perf.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: Pod -metadata: - name: disagg-router-6p-2d-benchmark - labels: - app: benchmark -spec: - containers: - - name: python - image: python:3.11 - command: - - /bin/bash - - -lc - - | - # Setup - ulimit -n 1048576 - ulimit -u 65536 - apt update && apt install tmux wget curl jq -y - - # Install benchmarking tool - pip install aiperf==0.10.0 - - # Wait for model to be ready - echo "Waiting for model '${MODEL_NAME}' at http://${FRONTEND}:8000/v1/models..." - until curl -s "http://${FRONTEND}:8000/v1/models" | jq -e --arg model "${MODEL_NAME}" '.data[]? | select(.id == $model)' >/dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready, retrying in 5s..." - sleep 5 - done - echo "Model '${MODEL_NAME}' is ready!" - - # Download Mooncake conversation trace dataset if not already present - mkdir -p ${BASE_DIR}/traces - mkdir -p ${BASE_DIR}/artifacts - if [ ! -f ${BASE_DIR}/traces/conversation_trace.jsonl ]; then - wget -qO ${BASE_DIR}/traces/conversation_trace.jsonl https://raw.githubusercontent.com/kvcache-ai/Mooncake/main/FAST25-release/traces/conversation_trace.jsonl - fi - - # Setup Paths and Endpoints - export INPUT_FILE="${BASE_DIR}/traces/conversation_trace.jsonl" - export MODEL_BASE_NAME="${MODEL_NAME##*/}" - export FRONTEND_LIB="${FRONTEND%-frontend}" - export ARTIFACT_DIR="${BASE_DIR}/artifacts/${MODEL_BASE_NAME}_${FRONTEND_LIB}" - mkdir -p "${ARTIFACT_DIR}" - - # Run Benchmark so its easy to attach and watch - tmux new-session -d -s benchmark -c "${ARTIFACT_DIR}" - tmux send-keys -t benchmark "aiperf profile -m ${MODEL_NAME} --input-file ${INPUT_FILE} --custom-dataset-type mooncake_trace --fixed-schedule --url http://${FRONTEND}:8000 --streaming --artifact-dir ${ARTIFACT_DIR} --goodput \"time_to_first_token:2000 inter_token_latency:25\"" C-m - sleep 7200 - env: - - name: MODEL_NAME - value: Qwen/Qwen3-32B - - name: FRONTEND - value: disagg-router-6p-2d-frontend - - name: BASE_DIR - value: /perf-cache - resources: - requests: - cpu: "8" - memory: 16Gi - limits: - cpu: "16" - memory: 32Gi - volumeMounts: - - name: model-cache - mountPath: /home/dynamo/.cache/huggingface - - name: perf-cache - mountPath: /perf-cache - workingDir: /workspace - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: perf-cache - persistentVolumeClaim: - claimName: perf-cache - restartPolicy: Never diff --git a/recipes/qwen3-vl-30b/README.md b/recipes/qwen3-vl-30b/README.md deleted file mode 100644 index f04234f50a2d..000000000000 --- a/recipes/qwen3-vl-30b/README.md +++ /dev/null @@ -1,127 +0,0 @@ -# Qwen3-VL-30B-A3B-Instruct-FP8: Aggregated Embedding Cache On vs Off Comparison - -This recipe demonstrates the performance difference when embedding cache is enabled for multi-modal payloads. It includes guidance on creating an artificial dataset with user-defined image re-use, and production-ready deployments for `Qwen/Qwen3-VL-30B-A3B-Instruct-FP8`. - -## Results - -| Metric | Cache ON | Cache OFF | Delta | -|----------------------|---------:|----------:|-------:| -| Output TPS (tok/s) | 3575.6 | 3072.3 | +16.4% | -| TTFT avg (ms) | 526.0 | 727.5 | -27.7% | -| TTFT p50 (ms) | 356.8 | 510.8 | -30.1% | -| ITL avg (ms) | 14.1 | 15.5 | -8.8% | -| Req Latency avg (ms) | 2630.0 | 3035.7 | -13.4% | - -**Enabling embedding cache on `Qwen3-VL-30B-A3B-Instruct-FP8` shows an average improvement of +16% throughput, -28% TTFT, and -13% request latency on a single aggregated replica of GB200 using the vLLM backend** - -## Pre-requisites - -To reproduce the results in the table, the following is required: - -1. **Dynamo Platform installed** - See [Kubernetes Deployment Guide](../../docs/kubernetes/README.md) -2. **GB200** -3. **HuggingFace token** configured: - ```bash - export NAMESPACE=your-namespace - kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="your-token" \ - -n ${NAMESPACE} - ``` - -## Dataset Generation - -`data-gen/generate-datasets-job.yaml` creates a dataset of synthetic text + image data with 80% image overlap. The script does this by manipulating the "total slots" and "image pool". - -Total number of slots is calculated as `num_requests*images/request`, representing how many total images the benchmark will iterate through. The image pool is how many images the benchmark can choose from to attach to a request. - -The `data-gen/generate-datasets-job.yaml` script creates a dataset of 1000 requests, 1 image per request, and an image pool of 200. Each request will pick an image from this pool without replacement, and loop back through the image pool after it has been exhausted. Thus, the first 200 out of 1000 requests will contain unique images, while the remaining 800 out of 1000 requests will have been seen already by the inference engine. Refer to jsonl [documentation](https://github.com/ai-dynamo/dynamo/tree/main/benchmarks/multimodal/jsonl) for more details on data generation. - -Each dataset is hardcoded to have 400 tokens of user-input text. - -To generate the dataset, run: - -```bash -kubectl apply -f data-gen/generate-datasets-job.yaml -n ${NAMESPACE} -``` - -## Notes - -1. Exact cache hit rates cannot be explicitly controlled via dataset due to potential LRU embedding cache eviction policies; however, decreasing the image pool relative to the number of requests allows for proportionally higher probabilities of seeing duplicate images and cache hits. Increasing the embedding cache capacity also allows for higher cache hit rate because it will evict less. - -**2. Agg embedding cache uses vLLM's native `ec_both` ECConnector role, supported in vLLM 0.17+. No patches required. See [multimodal-vllm.md](https://github.com/ai-dynamo/dynamo/blob/main/docs/features/multimodal/multimodal-vllm.md#embedding-cache) for more details.** - -3. Replace placeholders in `*.yaml` before running: - - `storageClassName: "your-storage-class-name"` in `model-cache/model-cache.yaml` - - `image: ` in all `vllm/*/deploy.yaml` files - - `NAMESPACE=your-namespace` and `HF_TOKEN="your-token"` in the setup commands -## Directory setup - -This recipe has three top-level components: `model-cache/` for PVC/model prep, `data-gen/` for dataset creation, and `vllm/agg-embedding-cache/` for deployment and benchmarking with [AIPerf](https://github.com/ai-dynamo/aiperf). - -```text -qwen3-vl-30b/ -├── data-gen/ -│ └── generate-datasets-job.yaml -├── model-cache/ -│ ├── model-cache.yaml -│ └── model-download.yaml -└── vllm/ - └── agg-embedding-cache/ - ├── deploy.yaml - ├── perf.yaml - └── run-benchmark.sh -``` - -The `deploy.yaml` script has `DYN_MULTIMODAL_EMBEDDING_CACHE_GB=10` by default, which represents an embedding cache **on** configuration. To toggle it off, set the env variable to 0. - -Similarly, each `perf.yaml` exposes a `CACHE_MODE` env variable to control where AIPerf dumps its results. Set it to either `cache_on` or `cache_off` depending on your deployment. - -## Quick Start - -### 1. Set Namespace and Create Storage - -```bash -export NAMESPACE=your-namespace -kubectl apply -f model-cache/model-cache.yaml -n ${NAMESPACE} -kubectl get pvc -n ${NAMESPACE} -``` - -### 2. Download Model and Generate Datasets - -```bash -kubectl apply -f model-cache/model-download.yaml -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/model-download -n ${NAMESPACE} --timeout=3600s - -kubectl apply -f data-gen/generate-datasets-job.yaml -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/qwen3-vl-30b-generate-datasets -n ${NAMESPACE} --timeout=3600s -kubectl logs job/qwen3-vl-30b-generate-datasets -n ${NAMESPACE} -``` - -### 3. Deploy and Benchmark (`agg-embedding-cache`) - -```bash -# deploy.yaml defaults to cache ON (DYN_MULTIMODAL_EMBEDDING_CACHE_GB=10) -kubectl apply -f vllm/agg-embedding-cache/deploy.yaml -n ${NAMESPACE} -kubectl wait --for=condition=Ready dynamographdeployment/qwen3-vl-agg -n ${NAMESPACE} --timeout=900s - -kubectl apply -f vllm/agg-embedding-cache/perf.yaml -n ${NAMESPACE} -kubectl wait --for=condition=Ready pod/qwen3-vl-agg-benchmark -n ${NAMESPACE} --timeout=300s -``` - -Optional: to run cache OFF, change `DYN_MULTIMODAL_EMBEDDING_CACHE_GB` to `0` in `vllm/agg-embedding-cache/deploy.yaml` and set `CACHE_MODE=cache_off` in `vllm/agg-embedding-cache/perf.yaml` before applying. - -### 4. Monitor Benchmark Progress - -```bash -kubectl get pods -n ${NAMESPACE} -l app=benchmark - -# Follow benchmark logs in real time -kubectl logs -f qwen3-vl-agg-benchmark -n ${NAMESPACE} - -# Wait for completion -kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/qwen3-vl-agg-benchmark -n ${NAMESPACE} --timeout=7200s -``` - -Wait for `Run complete. Artifacts in /perf-cache/artifacts/qwen3_vl_30b_embedding_cache/agg/`. - -`vllm/agg-embedding-cache/run-benchmark.sh` is also provided as a helper to launch cache-on/cache-off runs. \ No newline at end of file diff --git a/recipes/qwen3-vl-30b/data-gen/generate-datasets-job.yaml b/recipes/qwen3-vl-30b/data-gen/generate-datasets-job.yaml deleted file mode 100644 index 0973515b91fe..000000000000 --- a/recipes/qwen3-vl-30b/data-gen/generate-datasets-job.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: qwen3-vl-30b-generate-datasets -spec: - backoffLimit: 1 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: generate-datasets - spec: - restartPolicy: Never - securityContext: - runAsUser: 0 - runAsGroup: 0 - fsGroup: 0 - containers: - - name: generate-datasets - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - imagePullPolicy: IfNotPresent - command: - - /bin/bash - - -lc - - | - set -euo pipefail - - GENERATOR_MAIN="/workspace/benchmarks/multimodal/jsonl/main.py" - OUTPUT_DIR="/perf-cache/datasets" - - if [[ ! -f "${GENERATOR_MAIN}" ]]; then - echo "Generator not found at ${GENERATOR_MAIN}" - exit 1 - fi - - mkdir -p "${OUTPUT_DIR}" - - python3 "${GENERATOR_MAIN}" \ - -n 1000 \ - --images-per-request 1 \ - --images-pool 200 \ - --user-text-tokens 400 \ - --image-mode http \ - --image-dir /perf-cache/images \ - -o "${OUTPUT_DIR}/qwen3_vl_1000req_1img_pool200.jsonl" - - echo "Dataset generation complete in ${OUTPUT_DIR}" - volumeMounts: - - name: perf-cache - mountPath: /perf-cache - volumes: - - name: perf-cache - persistentVolumeClaim: - claimName: perf-cache diff --git a/recipes/qwen3-vl-30b/model-cache/model-cache.yaml b/recipes/qwen3-vl-30b/model-cache/model-cache.yaml deleted file mode 100644 index 68c9124b8bbd..000000000000 --- a/recipes/qwen3-vl-30b/model-cache/model-cache.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 100Gi - storageClassName: "your-storage-class-name" ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: compilation-cache -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: "your-storage-class-name" ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: perf-cache -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Gi - storageClassName: "your-storage-class-name" diff --git a/recipes/qwen3-vl-30b/model-cache/model-download.yaml b/recipes/qwen3-vl-30b/model-cache/model-download.yaml deleted file mode 100644 index 0918d8615671..000000000000 --- a/recipes/qwen3-vl-30b/model-cache/model-download.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: MODEL_NAME - value: "Qwen/Qwen3-VL-30B-A3B-Instruct-FP8" # Remove FP8 for BF16 variant - - name: HF_HOME - value: /home/dynamo/.cache/huggingface - # Uses up to 64 GB RAM for XET buffers; set "0" on low-memory nodes (8 GB cap): https://huggingface.co/docs/hub/en/xet/using-xet-storage#download-buffers - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - - name: MODEL_REVISION - value: "main" - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.11.0 - hf download "$MODEL_NAME" --revision "$MODEL_REVISION" - resources: - requests: - cpu: "2" - memory: "64Gi" - limits: - cpu: "8" - memory: "64Gi" - volumeMounts: - - name: model-cache - mountPath: /home/dynamo/.cache/huggingface - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/qwen3-vl-30b/vllm/agg-embedding-cache/deploy.yaml b/recipes/qwen3-vl-30b/vllm/agg-embedding-cache/deploy.yaml deleted file mode 100644 index 79bb37fceece..000000000000 --- a/recipes/qwen3-vl-30b/vllm/agg-embedding-cache/deploy.yaml +++ /dev/null @@ -1,81 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: qwen3-vl-agg -spec: - pvcs: - - create: false - name: model-cache - - create: false - name: compilation-cache - services: - Frontend: - componentType: frontend - envs: - - name: HF_HOME - value: /home/dynamo/.cache/huggingface - - name: DYN_REQUEST_PLANE - value: tcp - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - imagePullPolicy: IfNotPresent - workingDir: /workspace - replicas: 1 - resources: - requests: - cpu: "1" - limits: - cpu: "1" - subComponentType: null - - VllmWorker: - componentType: worker - envFromSecret: hf-token-secret - extraPodSpec: - mainContainer: - command: - - /bin/bash - - -lc - args: - - | - set -euo pipefail - python3 -m dingo.vllm \ - --model Qwen/Qwen3-VL-30B-A3B-Instruct-FP8 \ - --enable-multimodal \ - --tensor-parallel-size 1 \ - --gpu-memory-utilization 0.85 \ - --max-model-len 16384 \ - --no-enable-log-requests \ - --enable-prefix-caching \ - --multimodal-embedding-cache-capacity-gb "${DYN_MULTIMODAL_EMBEDDING_CACHE_GB}" - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - imagePullPolicy: IfNotPresent - env: - - name: HF_HOME - value: /home/dynamo/.cache/huggingface - - name: DYN_REQUEST_PLANE - value: tcp - - name: DYN_VLLM_EMBEDDING_TRANSFER_MODE - value: nixl-write - - name: DYN_MULTIMODAL_EMBEDDING_CACHE_GB - value: "10" - # Permit the COCO http:// image URLs in the multimodal benchmark dataset. - - name: DYN_MM_ALLOW_INTERNAL - value: "1" - workingDir: /workspace - replicas: 1 - resources: - limits: - gpu: "1" - requests: - gpu: "1" - subComponentType: null - volumeMounts: - - name: model-cache - mountPoint: /home/dynamo/.cache/huggingface - - name: compilation-cache - mountPoint: /home/dynamo/.cache/vllm - useAsCompilationCache: true diff --git a/recipes/qwen3-vl-30b/vllm/agg-embedding-cache/perf.yaml b/recipes/qwen3-vl-30b/vllm/agg-embedding-cache/perf.yaml deleted file mode 100644 index 275054f57c93..000000000000 --- a/recipes/qwen3-vl-30b/vllm/agg-embedding-cache/perf.yaml +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: Pod -metadata: - name: qwen3-vl-agg-benchmark - labels: - app: benchmark -spec: - containers: - - name: benchmark - image: python:3.11 - command: - - /bin/bash - - -lc - - | - set -euo pipefail - ulimit -n 1048576 - ulimit -u 65536 - - apt update && apt install -y tmux curl jq - pip install aiperf==0.10.0 - - echo "Waiting for model '${MODEL_NAME}' at http://${FRONTEND}:8000/v1/models..." - until curl -s "http://${FRONTEND}:8000/v1/models" | jq -e --arg model "${MODEL_NAME}" '.data[]? | select(.id == $model)' >/dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] Model not ready, retrying in 5s..." - sleep 5 - done - echo "Model '${MODEL_NAME}' is ready." - - INPUT_FILE="${DATASET_DIR}/qwen3_vl_1000req_1img_pool200.jsonl" - if [ ! -f "${INPUT_FILE}" ]; then - echo "Dataset not found: ${INPUT_FILE}" - exit 1 - fi - - RUN_DIR="${ARTIFACT_BASE_DIR}/${CACHE_MODE}" - mkdir -p "${RUN_DIR}" - - echo "Running benchmark ..." - aiperf profile \ - --model "${MODEL_NAME}" \ - --input-file "${INPUT_FILE}" \ - --custom-dataset-type single_turn \ - --url "http://${FRONTEND}:8000" \ - --streaming \ - --ui-type none \ - --request-count "${REQUEST_COUNT}" \ - --concurrency "${CONCURRENCY}" \ - --request-rate-mode constant \ - --warmup-request-count "${WARMUP_REQUEST_COUNT}" \ - --artifact-dir "${RUN_DIR}" \ - --extra-inputs "max_tokens:${MAX_TOKENS}" \ - --extra-inputs "min_tokens:${MAX_TOKENS}" \ - --extra-inputs "ignore_eos:true" - - echo "Run complete. Artifacts in ${RUN_DIR}" - sleep 3600 - env: - - name: MODEL_NAME - value: Qwen/Qwen3-VL-30B-A3B-Instruct-FP8 - - name: FRONTEND - value: qwen3-vl-agg-frontend - - name: CACHE_MODE - value: cache_on - - name: MAX_TOKENS - value: "150" - - name: REQUEST_COUNT - value: "1000" - - name: CONCURRENCY - value: "64" - - name: WARMUP_REQUEST_COUNT - value: "3" - - name: DATASET_DIR - value: /perf-cache/datasets - - name: ARTIFACT_BASE_DIR - value: /perf-cache/artifacts/qwen3_vl_30b_embedding_cache/agg - resources: - requests: - cpu: "8" - memory: 16Gi - limits: - cpu: "16" - memory: 32Gi - volumeMounts: - - name: perf-cache - mountPath: /perf-cache - volumes: - - name: perf-cache - persistentVolumeClaim: - claimName: perf-cache - restartPolicy: Never diff --git a/recipes/qwen3-vl-30b/vllm/agg-embedding-cache/run-benchmark.sh b/recipes/qwen3-vl-30b/vllm/agg-embedding-cache/run-benchmark.sh deleted file mode 100755 index aa0d8b0fd6d3..000000000000 --- a/recipes/qwen3-vl-30b/vllm/agg-embedding-cache/run-benchmark.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Usage: -# ./run-benchmark.sh on # benchmark with embedding cache ON (10GB) -# ./run-benchmark.sh off # benchmark with embedding cache OFF -# -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -NAMESPACE="${NAMESPACE:-dynamo}" - -if [[ $# -ne 1 ]] || [[ "$1" != "on" && "$1" != "off" ]]; then - echo "Usage: $0 " - exit 1 -fi - -MODE="$1" - -if [[ "${MODE}" == "on" ]]; then - CACHE_GB="10" - CACHE_MODE="cache_on" -else - CACHE_GB="0" - CACHE_MODE="cache_off" -fi - -echo "==> Embedding cache: ${MODE} (${CACHE_GB}GB)" - -# Patch deploy.yaml: set DYN_MULTIMODAL_EMBEDDING_CACHE_GB value -awk -v cache_gb="${CACHE_GB}" ' - /name: DYN_MULTIMODAL_EMBEDDING_CACHE_GB/ { print; getline; print " value: \"" cache_gb "\""; next } - { print } -' "${SCRIPT_DIR}/deploy.yaml" | \ - kubectl apply -f - -n "${NAMESPACE}" - -echo "==> Waiting for worker to be ready..." -kubectl wait --for=condition=Ready \ - dynamographdeployment/qwen3-vl-agg \ - -n "${NAMESPACE}" --timeout=600s - -# Delete old benchmark pod if exists -kubectl delete pod qwen3-vl-agg-benchmark \ - -n "${NAMESPACE}" --ignore-not-found - -# Patch perf.yaml: replace CACHE_MODE value -sed 's/value: cache_o[nf]*/value: '"${CACHE_MODE}"'/' \ - "${SCRIPT_DIR}/perf.yaml" | \ - kubectl apply -f - -n "${NAMESPACE}" - -echo "==> Benchmark pod launched (cache ${MODE})" -echo " Monitor with: kubectl logs -f qwen3-vl-agg-benchmark -n ${NAMESPACE}" \ No newline at end of file diff --git a/recipes/qwen3-vl-32b-fp8/README.md b/recipes/qwen3-vl-32b-fp8/README.md deleted file mode 100644 index 2b195d30d8ef..000000000000 --- a/recipes/qwen3-vl-32b-fp8/README.md +++ /dev/null @@ -1,122 +0,0 @@ - - -# Qwen3-VL-32B-Instruct-FP8 Recipes - -Production-ready deployments for **Qwen/Qwen3-VL-32B-Instruct-FP8**, a 32B vision-language model with FP8 quantization and multimodal (image) support. - -## Available Configurations - -| Configuration | GPUs | Mode | Description | -|--------------|------|------|-------------| -| [**vllm/agg**](vllm/agg/) | 1x H100/H200 | Aggregated | Single GPU, vision + decode combined | -| [**vllm/hetero_hardware_disagg**](vllm/hetero_hardware_disagg/) | 1x Intel XPU + 1x NVIDIA GPU | Disaggregated | Encode on XPU, decode on GPU with embedding transfer via RDMA | - -## Prerequisites - -1. **Dynamo Platform installed** — See [Kubernetes Deployment Guide](../../docs/kubernetes/README.md) -2. **GPU cluster** matching the configuration requirements: - - **Aggregated**: 1x NVIDIA H100/H200 - - **Disaggregated**: 1x Intel XPU (encode) + 1x NVIDIA GPU (decode) with RDMA connectivity -3. **HuggingFace token** with access to Qwen models -4. **DRA (Dynamic Resource Allocation)** configured for disaggregated mode (ResourceClaimTemplates provided) - -## Quick Start - -```bash -# Set namespace -export NAMESPACE=dynamo-demo -kubectl create namespace ${NAMESPACE} - -# Create HuggingFace token secret -kubectl create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="your-token-here" \ - -n ${NAMESPACE} - -# Download model (update storageClassName in model-cache.yaml first!) -kubectl apply -f model-cache/ -n ${NAMESPACE} -kubectl wait --for=condition=Complete job/model-download -n ${NAMESPACE} --timeout=3600s - -# Deploy (choose one configuration) -kubectl apply -f vllm/agg/deploy.yaml -n ${NAMESPACE} -# OR for disaggregated (apply resource claim templates first): -# kubectl apply -f vllm/hetero_hardware_disagg/intel_xpu_rdma_template.yaml -n ${NAMESPACE} -# kubectl apply -f vllm/hetero_hardware_disagg/nvidia_gpu_rdma_template.yaml -n ${NAMESPACE} -# kubectl apply -f vllm/hetero_hardware_disagg/deploy.yaml -n ${NAMESPACE} -``` - -## Test the Deployment - -```bash -# Port-forward the frontend -kubectl port-forward svc/qwen3-vl-32b-fp8-vllm-agg-frontend 8000:8000 -n ${NAMESPACE} -# For disaggregated: -kubectl port-forward svc/qwen3-vl-32b-fp8-vllm-disagg-frontend 8000:8000 -n ${NAMESPACE} - -# Send a text-only request -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "Qwen/Qwen3-VL-32B-Instruct-FP8", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 50 - }' - -# Send a multimodal (image) request -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "Qwen/Qwen3-VL-32B-Instruct-FP8", - "messages": [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/4/47/PNG_transparency_demonstration_1.png/300px-PNG_transparency_demonstration_1.png"}}, - {"type": "text", "text": "Describe this image in detail."} - ] - } - ], - "max_tokens": 256 - }' -``` - -## Run Performance Benchmarks - -```bash -# Benchmark (aggregated and disaggregated share the same template) -# Example for aggregated: -./benchmark/run-benchmark.sh --config agg -n ${NAMESPACE} - -# Example for disaggregated: -./benchmark/run-benchmark.sh --config disagg -n ${NAMESPACE} -``` - -## Model Details - -- **Model**: `Qwen/Qwen3-VL-32B-Instruct-FP8` -- **Parameters**: 32B -- **Quantization**: FP8 (pre-quantized weights) -- **Modalities**: Text + Vision (image understanding) -- **KV Cache**: FP8 dtype for memory efficiency -- **Context length**: Default model context - -## Architecture: Disaggregated Mode - -The disaggregated configuration separates vision encoding from text decoding: - -```text -Client → Frontend → EncodeWorker (Intel XPU) ──NIXL/RDMA──→ DecodeWorker (NVIDIA GPU) → Response -``` - -- **EncodeWorker**: Processes images and generates vision embeddings on Intel XPU -- **DecodeWorker**: Runs autoregressive text generation on NVIDIA GPU -- **KV Transfer**: NIXL connector over RDMA for low-latency embedding transfer between devices - -## Notes - -- Update `storageClassName` in `model-cache/model-cache.yaml` to match your cluster before deploying -- Model download takes approximately 15-30 minutes depending on network speed -- The disaggregated mode requires RDMA-capable network interfaces between encode and decode nodes -- `ResourceClaimTemplate` files in `vllm/hetero_hardware_disagg/` must be applied before the hetero_hardware_disagg deployment diff --git a/recipes/qwen3-vl-32b-fp8/benchmark/perf.yaml b/recipes/qwen3-vl-32b-fp8/benchmark/perf.yaml deleted file mode 100644 index d23862a147bb..000000000000 --- a/recipes/qwen3-vl-32b-fp8/benchmark/perf.yaml +++ /dev/null @@ -1,201 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Shared aiperf benchmark Job template — one file, both configs (agg / disagg). -# run-benchmark.sh exports per-config knobs via envsubst: -# ${BENCH_NAME} — Job + label name (qwen3-vl-32b-fp8-agg-perf / ...-disagg-perf) -# ${BENCH_ENDPOINT} — Service:port the bench hits (e.g. qwen3-vl-32b-fp8-vllm-agg-frontend:8000) -# -apiVersion: batch/v1 -kind: Job -metadata: - name: ${BENCH_NAME} -spec: - backoffLimit: 1 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: ${BENCH_NAME} - spec: - restartPolicy: Never - containers: - - command: - - /bin/sh - - -c - - | - set -e - apt-get update && apt-get install -y curl jq procps git && apt-get clean - - pip install aiperf==0.10.0 - echo "aiperf installation completed" - - export COLUMNS=200 - EPOCH=$(date +%s) - - # ── Wait for model to be ready ────────────────────────────────────── - wait_for_model_ready() { - local timeout=600 - local start=$(date +%s) - echo "Waiting for model '$TARGET_MODEL' at http://$ENDPOINT/v1/models (timeout: ${timeout}s)..." - while ! curl -s "http://$ENDPOINT/v1/models" \ - | jq -e --arg model "$TARGET_MODEL" '.data[]? | select(.id == $model)' >/dev/null 2>&1; do - if [ $(( $(date +%s) - start )) -ge $timeout ]; then - echo "ERROR: Timed out waiting for model '$TARGET_MODEL' after ${timeout}s" - exit 1 - fi - echo "[$(date '+%H:%M:%S')] Not ready, retrying in 5s..." - sleep 5 - done - echo "Model '$TARGET_MODEL' is ready" - curl -s "http://$ENDPOINT/v1/models" | jq . - } - - # ── Core benchmark function ───────────────────────────────────────── - # Args: - run_perf() { - local qps=$1 - local concurrency=$2 - local req_count=$3 - local key="qps_${qps}_isl_128_osl_256_con_${concurrency}_req_${req_count}" - local ARTIFACT_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}/${key}" - mkdir -p "$ARTIFACT_DIR" - echo "-------------------------------------------------------------" - echo "Running: QPS=$qps ISL=128 OSL=256 Concurrency=$concurrency Requests=$req_count images=20 res=854x480" - echo "ARTIFACT_DIR: $ARTIFACT_DIR" - - aiperf profile \ - --artifact-dir "$ARTIFACT_DIR" \ - --model "$TARGET_MODEL" \ - --tokenizer "$TOKENIZER_PATH" \ - --tokenizer-trust-remote-code \ - --endpoint-type chat \ - --streaming \ - --url "http://$ENDPOINT" \ - --synthetic-input-tokens-mean 128 \ - --synthetic-input-tokens-stddev 0 \ - --output-tokens-mean 256 \ - --output-tokens-stddev 0 \ - --image-width-mean 854 \ - --image-width-stddev 0 \ - --image-height-mean 480 \ - --image-height-stddev 0 \ - --image-batch-size 20 \ - --request-rate "$qps" \ - --request-count "$req_count" \ - --num-dataset-entries "$req_count" \ - --concurrency "$concurrency" \ - --request-timeout-seconds 600 \ - --random-seed 0 \ - --extra-inputs ignore_eos:true \ - --extra-inputs min_tokens:256 \ - --extra-inputs max_tokens:256 \ - --use-server-token-count \ - --ui simple \ - 2>&1 | tee "${ARTIFACT_DIR}/aiperf_qps${qps}_con${concurrency}.txt" - - echo "Done: $key" - ls -la "$ARTIFACT_DIR" - } - - # ── Main execution ────────────────────────────────────────────────── - wait_for_model_ready - - RUN_DIR="${ROOT_ARTIFACT_DIR}/${EPOCH}_${JOB_NAME}" - mkdir -p "$RUN_DIR" - - cat > "$RUN_DIR/input_config.json" <32, 0.25->64, rest->128", - "tokenizer": "$TOKENIZER_PATH" - } - EOF - - # ── QPS / num-prompts / Concurrency grouped sweep ─────────────────── - QPS_GROUP="0.1 0.25 0.5 0.75 1 1.25" - PROMPTS_GROUP="32 64 128 256 256 256" - - # Convert to indexed arrays - i=0 - for qps in $QPS_GROUP; do - i=$((i+1)) - eval "QPS_$i=$qps" - done - total=$i - - i=0 - for np in $PROMPTS_GROUP; do - i=$((i+1)) - eval "NP_$i=$np" - done - - echo "===============================================================" - echo "Running grouped sweep: $total configurations" - echo "QPS: $QPS_GROUP" - echo "Num prompts: $PROMPTS_GROUP" - echo "ISL=128, OSL=256 (fixed)" - echo "Concurrency: 32(qps=0.1), 64(qps=0.25), 128(rest)" - echo "===============================================================" - - i=1 - while [ $i -le $total ]; do - eval "qps=\$QPS_$i" - eval "req_count=\$NP_$i" - - # Set concurrency based on QPS - case "$qps" in - 0.1) concurrency=32 ;; - 0.25) concurrency=64 ;; - *) concurrency=128 ;; - esac - - run_perf "$qps" "$concurrency" "$req_count" - i=$((i+1)) - done - - echo "" - echo "All grouped sweep runs complete." - echo "Results in: $RUN_DIR" - env: - - name: TARGET_MODEL - value: Qwen/Qwen3-VL-32B-Instruct-FP8 - - name: TOKENIZER_PATH - value: /root/.cache/huggingface/Qwen/Qwen3-VL-32B-Instruct-FP8 - - name: ENDPOINT - value: ${BENCH_ENDPOINT} - - name: AIPERF_HTTP_CONNECTION_LIMIT - value: "200" - - name: JOB_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.labels['job-name'] - - name: ROOT_ARTIFACT_DIR - value: /root/.cache/huggingface/perf - - name: HF_HOME - value: /root/.cache/huggingface - - name: PYTHONUNBUFFERED - value: "1" - image: python:3.12-slim - imagePullPolicy: IfNotPresent - name: perf - volumeMounts: - - name: model-cache - mountPath: /root/.cache/huggingface - workingDir: /workspace - imagePullSecrets: - - name: nvcrimagepullsecret - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/qwen3-vl-32b-fp8/benchmark/run-benchmark.sh b/recipes/qwen3-vl-32b-fp8/benchmark/run-benchmark.sh deleted file mode 100755 index 011c7e938c5e..000000000000 --- a/recipes/qwen3-vl-32b-fp8/benchmark/run-benchmark.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Per-config benchmark driver for Qwen3-VL-32B-Instruct-FP8 recipe. -# Applies the shared benchmark/perf.yaml template with config-specific -# env vars via envsubst, then submits the Job. -# -# Usage: -# ./run-benchmark.sh --config agg # benchmark aggregated deployment -# ./run-benchmark.sh --config disagg # benchmark disaggregated deployment -# ./run-benchmark.sh --config agg --dry-run # render YAML without applying -# -set -euo pipefail - -CONFIG="" -DRY_RUN=0 -NAMESPACE="" - -while [[ $# -gt 0 ]]; do - case "$1" in - --config|-c) CONFIG="$2"; shift 2 ;; - --dry-run) DRY_RUN=1; shift ;; - -n|--namespace) NAMESPACE="$2"; shift 2 ;; - -h|--help) - echo "Usage: $0 --config {agg|disagg} [-n ] [--dry-run]" - exit 0 ;; - *) echo "Unknown arg: $1" >&2; exit 2 ;; - esac -done - -if [[ -z "$CONFIG" ]]; then - echo "ERROR: --config {agg|disagg} is required" >&2 - exit 2 -fi - -# ── Per-config knobs ────────────────────────────────────────────────────────── -case "$CONFIG" in - agg) - export BENCH_NAME="qwen3-vl-32b-fp8-agg-perf" - export BENCH_ENDPOINT="qwen3-vl-32b-fp8-vllm-agg-frontend:8000" - ;; - disagg) - export BENCH_NAME="qwen3-vl-32b-fp8-disagg-perf" - export BENCH_ENDPOINT="qwen3-vl-32b-fp8-vllm-disagg-frontend:8000" - ;; - *) - echo "ERROR: unknown config '$CONFIG'. Use 'agg' or 'disagg'." >&2 - exit 2 - ;; -esac - -# ── Locate the shared template ──────────────────────────────────────────────── -HERE="$(cd "$(dirname "$0")" && pwd)" -TEMPLATE="$HERE/perf.yaml" - -if [[ ! -f "$TEMPLATE" ]]; then - echo "ERROR: template not found: $TEMPLATE" >&2 - exit 1 -fi - -# ── Render via envsubst ─────────────────────────────────────────────────────── -RENDERED=$(envsubst '${BENCH_NAME} ${BENCH_ENDPOINT}' < "$TEMPLATE") - -if [[ "$DRY_RUN" == "1" ]]; then - echo "$RENDERED" - exit 0 -fi - -# ── Apply ───────────────────────────────────────────────────────────────────── -NS_ARGS="" -if [[ -n "$NAMESPACE" ]]; then - NS_ARGS="-n $NAMESPACE" -fi - -echo "[run-benchmark] config=$CONFIG name=$BENCH_NAME endpoint=$BENCH_ENDPOINT" -echo "$RENDERED" | kubectl apply ${NS_ARGS} -f - -echo "[run-benchmark] Job '$BENCH_NAME' submitted." -echo "[run-benchmark] Monitor: kubectl ${NS_ARGS} logs -f job/$BENCH_NAME" diff --git a/recipes/qwen3-vl-32b-fp8/model-cache/model-cache.yaml b/recipes/qwen3-vl-32b-fp8/model-cache/model-cache.yaml deleted file mode 100644 index f57dcb80ebed..000000000000 --- a/recipes/qwen3-vl-32b-fp8/model-cache/model-cache.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 100Gi - storageClassName: "your-storage-class-name" diff --git a/recipes/qwen3-vl-32b-fp8/model-cache/model-download.yaml b/recipes/qwen3-vl-32b-fp8/model-cache/model-download.yaml deleted file mode 100644 index 4e9c850b802d..000000000000 --- a/recipes/qwen3-vl-32b-fp8/model-cache/model-download.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: MODEL_NAME - value: Qwen/Qwen3-VL-32B-Instruct-FP8 - - name: HF_HOME - value: /model-store - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - - name: MODEL_REVISION - value: 4bf2c2f39c37c0fede78bede4056e1f18cdf8109 - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.11.0 - hf download $MODEL_NAME --revision $MODEL_REVISION - volumeMounts: - - name: model-cache - mountPath: /model-store - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - diff --git a/recipes/qwen3-vl-32b-fp8/vllm/agg/deploy.yaml b/recipes/qwen3-vl-32b-fp8/vllm/agg/deploy.yaml deleted file mode 100644 index 94fabff335a3..000000000000 --- a/recipes/qwen3-vl-32b-fp8/vllm/agg/deploy.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Aggregated deployment for Qwen3-VL-32B-Instruct-FP8. -# Single GPU decode worker with multimodal (vision) support. -# -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: qwen3-vl-32b-fp8-vllm-agg -spec: - backendFramework: vllm - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /opt/models - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0 - workingDir: /workspace/examples/multimodal - envs: - - name: HF_HOME - value: /opt/models - - name: DYN_STORE_KV - value: "mem" - - name: DYN_REQUEST_PLANE - value: "tcp" - - name: DYN_TCP_MAX_MESSAGE_SIZE - value: "67108864" - VllmWorker: - componentType: worker - envFromSecret: hf-token-secret - replicas: 1 - resources: - limits: - gpu: "1" - requests: - gpu: "1" - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 80Gi - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0 - workingDir: /workspace/examples/multimodal - env: - - name: SERVED_MODEL_NAME - value: "Qwen/Qwen3-VL-32B-Instruct-FP8" - - name: MODEL_PATH - value: "Qwen/Qwen3-VL-32B-Instruct-FP8" - - name: HF_HOME - value: /opt/models - - name: DYN_STORE_KV - value: "mem" - - name: DYN_REQUEST_PLANE - value: "tcp" - - name: DYN_TCP_MAX_MESSAGE_SIZE - value: "67108864" - command: - - /bin/sh - - -c - args: - - >- - python3 -m dingo.vllm \ - --model $MODEL_PATH \ - --served-model-name $SERVED_MODEL_NAME \ - --enable-multimodal \ - --enable-mm-embeds \ - --max-num-seqs 40 \ - --tensor-parallel-size 1 \ - --kv-cache-dtype fp8 \ - --gpu-memory-utilization 0.95 \ - --no-enable-prefix-caching - diff --git a/recipes/qwen3-vl-32b-fp8/vllm/agg/resourceclaimtemplate.yaml b/recipes/qwen3-vl-32b-fp8/vllm/agg/resourceclaimtemplate.yaml deleted file mode 100644 index 7f4ab0f96248..000000000000 --- a/recipes/qwen3-vl-32b-fp8/vllm/agg/resourceclaimtemplate.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: resource.k8s.io/v1 -kind: ResourceClaimTemplate -metadata: - name: nvidia-1-gpu -spec: - spec: - devices: - requests: - - name: gpu - exactly: - deviceClassName: gpu.nvidia.com - count: 1 - diff --git a/recipes/qwen3-vl-32b-fp8/vllm/hetero_hardware_disagg/deploy.yaml b/recipes/qwen3-vl-32b-fp8/vllm/hetero_hardware_disagg/deploy.yaml deleted file mode 100644 index e2e33255a66f..000000000000 --- a/recipes/qwen3-vl-32b-fp8/vllm/hetero_hardware_disagg/deploy.yaml +++ /dev/null @@ -1,188 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Disaggregated deployment for Qwen3-VL-32B-Instruct-FP8. -# Separates multimodal encode (prefill) from decode using NIXL KV transfer. -# Encode worker runs on Intel XPU with RDMA; decode worker runs on NVIDIA GPU with RDMA. -# -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: qwen3-vl-32b-fp8-vllm-disagg -spec: - backendFramework: vllm - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /opt/models - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0 - workingDir: /workspace/examples/multimodal - envs: - - name: HF_HOME - value: /opt/models - - name: DYN_STORE_KV - value: "mem" - - name: DYN_REQUEST_PLANE - value: "tcp" - - name: DYN_TCP_MAX_MESSAGE_SIZE - value: "67108864" - EncodeWorker: - componentType: worker - envFromSecret: hf-token-secret - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 80Gi - extraPodSpec: - resourceClaims: - - name: intel-gpu-rdma-resource - resourceClaimTemplateName: intel-xpu-rdma-template - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime-xpu:1.2.0 - workingDir: /workspace/examples/multimodal - securityContext: - runAsUser: 0 - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - - NET_RAW - env: - - name: DYN_STORE_KV - value: "mem" - - name: DYN_REQUEST_PLANE - value: "tcp" - - name: DYN_TCP_MAX_MESSAGE_SIZE - value: "67108864" - - name: SERVED_MODEL_NAME - value: "Qwen/Qwen3-VL-32B-Instruct-FP8" - - name: MODEL_PATH - value: "Qwen/Qwen3-VL-32B-Instruct-FP8" - - name: HF_HOME - value: /opt/models - - name: DYN_VLLM_EMBEDDING_TRANSFER_MODE - value: "nixl-read" - - name: ENABLE_ENCODER_CACHE - value: "0" - - name: VISION_ENCODE_SERIALIZE - value: "1" - - name: VLLM_NIXL_SIDE_CHANNEL_PORT - value: "20098" - - name: VLLM_NIXL_SIDE_CHANNEL_HOST - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: UCX_TLS - value: "ib,rc,ud,rc_verbs,ud_verbs,ze_copy" - - name: UCX_IB_ROCE_REACHABILITY_MODE - value: "all" - - name: UCX_MEMTYPE_CACHE - value: "0" - command: - - /bin/sh - - -c - args: - - >- - ulimit -l unlimited - TRANSFER_LOCAL=0 \ - python -m dingo.vllm \ - --model $MODEL_PATH \ - --served-model-name $SERVED_MODEL_NAME \ - --enable-multimodal \ - --multimodal-encode-worker \ - --enable-mm-embeds \ - --dtype bfloat16 \ - --enforce-eager \ - --block-size 64 \ - --gpu-memory-utilization 0.7 \ - --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_buffer_device":"xpu","kv_connector_extra_config":{"enforce_handshake_compat":false}}' - resources: - claims: - - name: intel-gpu-rdma-resource - VllmDecodeWorker: - componentType: worker - envFromSecret: hf-token-secret - replicas: 1 - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 80Gi - extraPodSpec: - resourceClaims: - - name: gpu-rdma-resource - resourceClaimTemplateName: nvidia-gpu-rdma-template - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.0 - workingDir: /workspace/examples/multimodal - securityContext: - runAsUser: 0 - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - - NET_RAW - env: - - name: DYN_STORE_KV - value: "mem" - - name: DYN_REQUEST_PLANE - value: "tcp" - - name: DYN_TCP_MAX_MESSAGE_SIZE - value: "67108864" - - name: SERVED_MODEL_NAME - value: "Qwen/Qwen3-VL-32B-Instruct-FP8" - - name: MODEL_PATH - value: "Qwen/Qwen3-VL-32B-Instruct-FP8" - - name: HF_HOME - value: /opt/models - - name: DYN_VLLM_EMBEDDING_TRANSFER_MODE - value: "nixl-read" - - name: DYN_VLLM_KV_EVENT_PORT - value: "20080" - - name: ENABLE_ENCODER_CACHE - value: "0" - - name: VLLM_NIXL_SIDE_CHANNEL_PORT - value: "20098" - - name: VLLM_NIXL_SIDE_CHANNEL_HOST - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: UCX_TLS - value: "ib,rc,ud,rc_verbs,ud_verbs,cuda_copy" - - name: UCX_IB_ROCE_REACHABILITY_MODE - value: "all" - - name: UCX_MEMTYPE_CACHE - value: "0" - command: - - /bin/sh - - -c - args: - - >- - ulimit -l unlimited - TRANSFER_LOCAL=0 CUDA_VISIBLE_DEVICES=0 \ - python -m dingo.vllm \ - --model $MODEL_PATH \ - --served-model-name $SERVED_MODEL_NAME \ - --enable-multimodal \ - --multimodal-worker \ - --enable-mm-embeds \ - --route-to-encoder \ - --mm-prompt-template "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>assistant\n" \ - --dtype bfloat16 \ - --max-num-seqs 40 \ - --kv-cache-dtype fp8 \ - --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false}}' - resources: - claims: - - name: gpu-rdma-resource - diff --git a/recipes/qwen3-vl-32b-fp8/vllm/hetero_hardware_disagg/intel_xpu_rdma_template.yaml b/recipes/qwen3-vl-32b-fp8/vllm/hetero_hardware_disagg/intel_xpu_rdma_template.yaml deleted file mode 100644 index 5fbc2bbe0eb8..000000000000 --- a/recipes/qwen3-vl-32b-fp8/vllm/hetero_hardware_disagg/intel_xpu_rdma_template.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: resource.k8s.io/v1 -kind: ResourceClaimTemplate -metadata: - name: intel-xpu-rdma-template -spec: - spec: - devices: - requests: - - name: gpu - exactly: - deviceClassName: gpu.intel.com - count: 1 - - name: rdma-net-interface - exactly: - deviceClassName: rdma-dranet - count: 1 diff --git a/recipes/qwen3-vl-32b-fp8/vllm/hetero_hardware_disagg/nvidia_gpu_rdma_template.yaml b/recipes/qwen3-vl-32b-fp8/vllm/hetero_hardware_disagg/nvidia_gpu_rdma_template.yaml deleted file mode 100644 index 3a7442261f6c..000000000000 --- a/recipes/qwen3-vl-32b-fp8/vllm/hetero_hardware_disagg/nvidia_gpu_rdma_template.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: resource.k8s.io/v1 -kind: ResourceClaimTemplate -metadata: - name: nvidia-gpu-rdma-template -spec: - spec: - devices: - requests: - - name: gpu - exactly: - deviceClassName: gpu.nvidia.com - count: 1 - - name: rdma-net-interface - exactly: - deviceClassName: rdma-dranet - count: 1 diff --git a/recipes/qwen3.6-35b-a3b/model-cache/model-cache.yaml b/recipes/qwen3.6-35b-a3b/model-cache/model-cache.yaml deleted file mode 100644 index f57dcb80ebed..000000000000 --- a/recipes/qwen3.6-35b-a3b/model-cache/model-cache.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: model-cache -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 100Gi - storageClassName: "your-storage-class-name" diff --git a/recipes/qwen3.6-35b-a3b/model-cache/model-download.yaml b/recipes/qwen3.6-35b-a3b/model-cache/model-download.yaml deleted file mode 100644 index a3cc5d1c2600..000000000000 --- a/recipes/qwen3.6-35b-a3b/model-cache/model-download.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: batch/v1 -kind: Job -metadata: - name: model-download -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - command: ["sh", "-c"] - envFrom: - - secretRef: - name: hf-token-secret - env: - - name: MODEL_NAME - value: "Qwen/Qwen3.6-35B-A3B" - - name: HF_HOME - value: /model-store - - name: HF_HUB_ENABLE_HF_TRANSFER - value: "1" - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub hf_transfer - hf download $MODEL_NAME - volumeMounts: - - name: model-cache - mountPath: /model-store - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache diff --git a/recipes/qwen3.6-35b-a3b/vllm/agg/deploy.yaml b/recipes/qwen3.6-35b-a3b/vllm/agg/deploy.yaml deleted file mode 100644 index e6d27e239b72..000000000000 --- a/recipes/qwen3.6-35b-a3b/vllm/agg/deploy.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: qwen3-6-35b-a3b-agg -spec: - backendFramework: vllm - pvcs: - - name: model-cache - create: false - services: - Frontend: - componentType: frontend - volumeMounts: - - name: model-cache - mountPoint: /opt/models - extraPodSpec: - mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - workingDir: /workspace/examples/backends/vllm - envs: - - name: HF_HOME - value: /opt/models - replicas: 1 - VllmWorker: - componentType: worker - envFromSecret: hf-token-secret - volumeMounts: - - name: model-cache - mountPoint: /opt/models - sharedMemory: - size: 20Gi - extraPodSpec: - mainContainer: - env: - - name: SERVED_MODEL_NAME - value: "Qwen/Qwen3.6-35B-A3B" - - name: MODEL_PATH - value: "Qwen/Qwen3.6-35B-A3B" - - name: HF_HOME - value: /opt/models - args: - - "python3 -m dingo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 1 --data-parallel-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --enforce-eager --block-size 128" - command: - - /bin/sh - - -c - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - workingDir: /workspace/examples/backends/vllm - replicas: 1 - resources: - limits: - gpu: "1" - requests: - gpu: "1" diff --git a/recipes/qwen3.6-35b/README.md b/recipes/qwen3.6-35b/README.md deleted file mode 100644 index 8e6c17a8ebdf..000000000000 --- a/recipes/qwen3.6-35b/README.md +++ /dev/null @@ -1,199 +0,0 @@ -# Qwen3.6-35B-A3B-FP8 — 3-way `vllm serve` vs Dynamo benchmark - -K8s recipe for benchmarking -[`Qwen/Qwen3.6-35B-A3B-FP8`](https://huggingface.co/Qwen/Qwen3.6-35B-A3B-FP8) -across three configs on the same single-GPU hardware target: - -| Config | Stack | Multimodal | Frontend-decoding | Embedding cache | -|----------------|---------------|------------|-------------------|-----------------| -| `vllm-serve` | vanilla vLLM | n/a | n/a | n/a | -| `dynamo-fd` | Dynamo + vLLM | on | on | off | -| `dynamo-fd-ec` | Dynamo + vLLM | on | on | 8 GiB | - -All three configs share one hardware target (H100 or GB200) chosen at -deploy time via `--hw {h100,gb200}`. See [Hardware targets](#hardware-targets) below. - -## Pre-requisites - -1. Kubectl context pointing at a cluster with the right GPUs. -2. A namespace you have write access to (`$NAMESPACE` below). -3. A `shared-model-cache` PVC in that namespace (RWX). If your cluster - pre-provisions it (common on platform-managed AWS / FSx clusters), - you don't need to do anything. Otherwise see - [Storage: shared-model-cache](#storage-shared-model-cache). -4. **Fill in your hostname** in `hw/h100.env` or `hw/gb200.env` — - replace the `` placeholder. See - [Hardware targets](#hardware-targets) for the lookup command. -5. `envsubst` on the laptop driving the recipe (Ubuntu: - `apt install gettext-base`; macOS: `brew install gettext`). -6. **HuggingFace token: not required.** `Qwen/Qwen3.6-35B-A3B-FP8` is - public (`gated: false`), so neither the download Job nor `vllm serve` - needs one. To swap in a gated model, uncomment the `hf-token-secret` - blocks in `model-cache/model-download.yaml` + `deploy/.yaml` and create: - ```bash - kubectl -n "$NAMESPACE" create secret generic hf-token-secret \ - --from-literal=HF_TOKEN="$HF_TOKEN" - ``` - -## Quick start - -```bash -export NAMESPACE= -export HW=gb200 # or h100 - -# Run all three configs sequentially (prep + deploy + bench + retrieve + clean -# per config). Artifacts land under -# ~/workspace/dynamo-tmp/logs//qwen36-fp8-${HW}/{vllm-serve,dynamo-fd,dynamo-fd-ec}/. -./run-all-benchmarks.sh -n ${NAMESPACE} --hw ${HW} -``` - -Each config's `profile_export_aiperf.json` is retrieved into the matching -sub-directory; throughput / TTFT / ITL numbers can be read directly from -that file. - -Or step-by-step for a single config: - -```bash -./run-benchmark.sh -n ${NAMESPACE} --hw ${HW} --config vllm-serve -./run-benchmark.sh -n ${NAMESPACE} --hw ${HW} --config dynamo-fd -./run-benchmark.sh -n ${NAMESPACE} --hw ${HW} --config dynamo-fd-ec -``` - -`run-benchmark.sh` accepts `--step {pvc|download|dataset|deploy|bench|retrieve|clean}` for granular control. `pvc`, `download`, and `dataset` are config-agnostic (any `--config` works to run them once). - -## Directory layout - -```text -qwen3.6-35b/ -├── README.md -├── run-benchmark.sh # Unified driver — branches on --config/--hw -├── run-all-benchmarks.sh # Sequential 3-config orchestrator -├── perf.yaml # Single shared aiperf bench Pod template -├── data-gen-job.yaml # Sliding-window jsonl generator Job -├── hw/ # Per-cluster user state — edit hostname here -│ ├── h100.env -│ └── gb200.env -├── model-cache/ # Model-caching subsystem -│ └── model-download.yaml -└── deploy/ # 3 deploy targets — grouped because >1 sibling - ├── vllm-serve.yaml # Plain Deployment + Service (baseline) - ├── dynamo-fd.yaml # DynamoGraphDeployment, frontend-decoding ON - └── dynamo-fd-ec.yaml # DynamoGraphDeployment, FD + embedding cache -``` - -Layout rule: **singletons flatten to root** (`perf.yaml`, `data-gen-job.yaml`); **dirs hold ≥2 files** (`hw/`, `deploy/`); **`model-cache/` is the exception** — a role bucket kept for future model-caching siblings. - -The three deploy targets share one `perf.yaml` because the only deltas -across them (pod name, frontend service, run-label) are exported as -`${BENCH_POD}` / `${BENCH_FRONTEND}` / `${BENCH_RUN_LABEL}` by -`run-benchmark.sh` and resolved via `envsubst` at apply time. - -## Hardware targets - -`hw/h100.env` and `hw/gb200.env` are sibling to the three config -directories and shared across all three. Each file exports three vars -the YAML templates substitute via `envsubst`: - -- `VLLM_IMAGE` — `nvcr.io/nvidia/ai-dynamo/vllm-runtime:` (multi-arch - manifest, same tag works on amd64 / arm64). -- `HW_NODE_SELECTOR` — JSON-flow nodeSelector (currently - `{"kubernetes.io/hostname":"…"}` for both targets). -- `HW_TOLERATIONS` — JSON-flow toleration array. H100 has `[]`; GB200 - carries the `kubernetes.io/arch=arm64:NoSchedule` toleration. - -**Before first use**: edit `hw/h100.env` and `hw/gb200.env` and replace -the `` placeholders with `kubernetes.io/hostname` -values from your cluster: - -```bash -# H100 -kubectl get nodes -L nvidia.com/gpu.product | awk '/H100/' -# GB200 -kubectl get nodes -L kubernetes.io/arch -L nvidia.com/gpu.product \ - | awk '/arm64/ && /GB200/' -``` - -Adding a new hardware target later is a one-file change in `hw/`. - -## Storage: shared-model-cache - -The recipe expects a single PVC named `shared-model-cache` (RWX) in -the target namespace — typically backed by FSx Lustre on AWS or any -RWX storage class on your cluster. It's mounted at three locations: - -| Mount in pod | subPath | What lives there | -|--------------|---------|------------------| -| `/home/dynamo/.cache/huggingface` | — (root) | Shared HF Hub cache (anything else in the namespace re-uses it) | -| `/home/dynamo/.cache/vllm` | `qwen36-bench/vllm-cache` | vllm cudagraph compilation cache | -| `/perf-cache` | `qwen36-bench/perf-cache` | Generated dataset + aiperf artifacts | - -The per-recipe subPath prefix `qwen36-bench/` keeps this recipe's -private state from colliding with future recipes (e.g. -`qwen3vl30b-bench/`). - -The HF cache is mounted at the root so any model already cached in -the namespace is reused. The Qwen3.6-35B-A3B-FP8 download lands in -the standard `hub/models--Qwen--Qwen3.6-35B-A3B-FP8/` directory. - -If your cluster doesn't pre-provision `shared-model-cache`, create it -out-of-band before running the recipe, picking an RWX storage class -(e.g. `dgxc-enterprise-file` on dgxc, FSx Lustre on AWS): - -```yaml -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: shared-model-cache -spec: - accessModes: [ReadWriteMany] - resources: - requests: - storage: 200Gi - storageClassName: -``` - -Prefer RWX/Retain (e.g. FSx Lustre) over RWO/Delete (e.g. EBS) — -RWO EBS volumes get pinned to whichever AZ the first-consumer pod -schedules into, leaving the GPU pod unschedulable if your GPU -nodes live in a different AZ. - -## aiperf install - -We install `aiperf==0.10.0` from PyPI. This release includes -[PR 824](https://github.com/ai-dynamo/aiperf/pull/824) -(`feat(dataset): add session_id to single-turn for causal ordering`), -which makes `single_turn` mode honor `session_id` ordering so -prefix-cache hits across the 8 turns of a given user are real. - -The version is pinned in `perf.yaml` and applies identically across -all three configs. Bump the `aiperf==` pin there to roll forward. - -## Naming & ownership - -All resources carry a `qwen36-` prefix (per-model) and these labels: - -```yaml -labels: - app.kubernetes.io/name: qwen3.6-35b - app.kubernetes.io/managed-by: dynamo-recipe -``` - -So in a shared namespace you can find this recipe's resources via: - -```bash -kubectl -n "$NAMESPACE" get pvc,deploy,job,pod \ - -l app.kubernetes.io/name=qwen3.6-35b -``` - -## Notes - -- Dataset is sliding-window with `window=5`, `turns=8`, `users=30`, - `image_size=2400x1080`, `user_text_tokens=8000`. Yields 240 requests - (`users × turns`) over 12 unique images per user - (`window + turns - 1`). Base64-inlined. -- Each jsonl row carries `session_id=user_`. With aiperf PR 824, the - `single_turn` dataset type honors session ordering so the 8 turns of - any one user are sent in causal order, letting prefix-cache hits land. -- The vllm command in `deploy.yaml` uses `--mm-processor-cache-gb 30` - and `--max-model-len 32768` to handle the 5-image multimodal context - (mirrors the 397B sweep yaml's settings adapted for 1 GPU). diff --git a/recipes/qwen3.6-35b/data-gen-job.yaml b/recipes/qwen3.6-35b/data-gen-job.yaml deleted file mode 100644 index 77913e1100ce..000000000000 --- a/recipes/qwen3.6-35b/data-gen-job.yaml +++ /dev/null @@ -1,102 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Generate 30u_8t_5w_8000word_base64.jsonl into /perf-cache/datasets/. -# Sliding-window: 30 users × 8 turns × window=5 = 240 requests over 12 -# unique images per user (window + turns - 1 = 5 + 8 - 1 = 12), so 360 -# total images; 8000 text tokens per request, 2400x1080 base64-inlined. -# Each row has session_id=user_ so aiperf can issue same-user turns -# in causal order (requires aiperf PR 824). -# -# The 8-turn variant is the standard sliding-window EC test bed: from -# turn 2 onwards each turn reuses 4-of-5 images from the previous turn's -# window, exercising multimodal-processor-cache and (future) embedding- -# cache hit rates. The 3-turn variant barely warms either cache. -apiVersion: batch/v1 -kind: Job -metadata: - name: qwen36-generate-datasets - labels: - app.kubernetes.io/name: qwen3.6-35b - app.kubernetes.io/component: data-gen - app.kubernetes.io/managed-by: dynamo-recipe -spec: - backoffLimit: 1 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app.kubernetes.io/name: qwen3.6-35b - app.kubernetes.io/component: data-gen - spec: - restartPolicy: Never - securityContext: - runAsUser: 0 - runAsGroup: 0 - fsGroup: 0 - containers: - - name: generate-datasets - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 - imagePullPolicy: IfNotPresent - # No privilege escalation, drop all Linux caps, default seccomp. - # runAsUser:0 is set at the pod level for PVC fsGroup writes — - # this container security context complements it without removing - # the root requirement. - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - env: - # nvcr.io/.../vllm-runtime:1.2.1 was cut from a release branch - # that predates dynamo PR 8201 (sliding-window strategy), so the - # baked /workspace/benchmarks/multimodal/jsonl/main.py is the - # older single-turn-only version. Pull a pinned main SHA to get - # the sliding-window subcommand. Bump DYNAMO_REF when you want - # newer generator fixes. - - name: DYNAMO_REF - value: "e2f63d01e07b6deadf4c0912269564503846d204" - command: - - /bin/bash - - -lc - - | - set -euo pipefail - GEN_SRC=/tmp/dynamo-jsonl - rm -rf "$GEN_SRC" - # Shallow fetch a specific SHA (or ref). `git clone --branch` - # doesn't accept SHAs, so use init + fetch + checkout - # — works for both SHAs and branch names. - git init -q "$GEN_SRC" - git -C "$GEN_SRC" remote add origin https://github.com/ai-dynamo/dynamo.git - git -C "$GEN_SRC" fetch --depth=1 origin "$DYNAMO_REF" - git -C "$GEN_SRC" checkout -q FETCH_HEAD - cd "$GEN_SRC/benchmarks/multimodal/jsonl" - echo "[gen] using dynamo @ $(git -C $GEN_SRC rev-parse --short HEAD)" - python3 main.py --help | head -3 - OUTPUT_DIR="/perf-cache/datasets" - IMAGE_DIR="/perf-cache/images/2400x1080" - mkdir -p "${OUTPUT_DIR}" "${IMAGE_DIR}" - python3 main.py sliding-window \ - --num-users 30 \ - --turns-per-user 8 \ - --window-size 5 \ - --image-size 2400 1080 \ - --image-mode base64 \ - --image-dir "${IMAGE_DIR}" \ - --user-text-tokens 8000 \ - -o "${OUTPUT_DIR}/30u_8t_5w_8000word_base64.jsonl" - ls -lh "${OUTPUT_DIR}/30u_8t_5w_8000word_base64.jsonl" - echo "Dataset generation complete in ${OUTPUT_DIR}" - volumeMounts: - - name: shared-model-cache - mountPath: /perf-cache - # Per-recipe subdir so different recipes (qwen36, qwen3-vl-30b…) - # don't collide on /perf-cache/datasets and /perf-cache/images. - subPath: qwen36-bench/perf-cache - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache diff --git a/recipes/qwen3.6-35b/deploy/dynamo-fd-ec.yaml b/recipes/qwen3.6-35b/deploy/dynamo-fd-ec.yaml deleted file mode 100644 index 4f4326afbf26..000000000000 --- a/recipes/qwen3.6-35b/deploy/dynamo-fd-ec.yaml +++ /dev/null @@ -1,138 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Aggregated Dynamo on a single H100, with multimodal frontend-decoding -# and embedding-cache enabled. Modeled on -# recipes/qwen3-vl-30b/vllm/agg-embedding-cache/deploy.yaml but -# adapted for Qwen3.6-35B-A3B-FP8 (TP=1, max-model-len 32768) and the -# PR 8235 `dynamo-fd-ec` config: -# - --enable-multimodal : MM request handling on -# - --frontend-decoding : Frontend (Rust) decodes images; worker receives embeddings -# - --multimodal-embedding-cache-capacity-gb 8 : 8 GiB EC per worker (DRAM-backed post-encoder cache) -# - --mm-processor-cache-gb 30 : 30 GiB HF processor cache (pre-encoder) -# - --dyn-tool-call-parser / --dyn-reasoning-parser : qwen3 tool/reasoning routing -# -# DGD's operator auto-creates a Service named `qwen36-dynamo-fd-ec-frontend` -# on port 8000 which perf.yaml targets. -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: qwen36-dynamo-fd-ec - labels: - app.kubernetes.io/name: qwen3.6-35b - app.kubernetes.io/component: dynamo-fd-ec - app.kubernetes.io/managed-by: dynamo-recipe -spec: - pvcs: - - create: false - name: shared-model-cache - services: - Frontend: - componentType: frontend - envs: - - name: HF_HOME - value: /home/dynamo/.cache/huggingface - - name: DYN_REQUEST_PLANE - value: tcp - - name: DYN_HTTP_BODY_LIMIT_MB - value: "200" - extraPodSpec: - # Hostname pin + toleration + image come from hw/${HW}.env - # (shared across vllm-serve, dynamo-fd, dynamo-fd-ec). Both - # Frontend and VllmWorker get the same nodeSelector so the - # gang scheduler places both on that exact node. This - # supersedes inter-pod podAffinity (which never worked under - # kai-scheduler gang scheduling — see git log). - # If the pinned host has no free GPU at apply time, both pods - # stay Pending — edit hw/${HW}.env to retarget. - nodeSelector: ${HW_NODE_SELECTOR} - tolerations: ${HW_TOLERATIONS} - mainContainer: - image: ${VLLM_IMAGE} - imagePullPolicy: IfNotPresent - workingDir: /workspace - # Frontend-decoding does CPU-side image decode + base64 parsing, - # so we give it more CPU than the qwen3-vl-30b recipe's cpu=1. - replicas: 1 - resources: - requests: - cpu: "4" - memory: 16Gi - limits: - cpu: "8" - memory: 32Gi - subComponentType: null - - VllmWorker: - componentType: worker - extraPodSpec: - # Same hostname pin as Frontend — keeps the NIXL embedding - # transfer a same-node memcpy. - nodeSelector: ${HW_NODE_SELECTOR} - tolerations: ${HW_TOLERATIONS} - mainContainer: - command: - - /bin/bash - - -lc - args: - - | - set -euo pipefail - ulimit -l unlimited - python3 -m dingo.vllm \ - --model "${MODEL_NAME}" \ - --enable-multimodal \ - --frontend-decoding \ - --multimodal-embedding-cache-capacity-gb "${DYN_MULTIMODAL_EMBEDDING_CACHE_GB}" \ - --mm-processor-cache-gb 30 \ - --dyn-tool-call-parser qwen3_coder \ - --dyn-reasoning-parser qwen3 \ - --tensor-parallel-size 1 \ - --gpu-memory-utilization 0.85 \ - --max-model-len 32768 \ - --enable-prefix-caching \ - --no-enable-log-requests - image: ${VLLM_IMAGE} - imagePullPolicy: IfNotPresent - env: - - name: MODEL_NAME - value: "Qwen/Qwen3.6-35B-A3B-FP8" - - name: HF_HOME - value: /home/dynamo/.cache/huggingface - - name: HF_HUB_OFFLINE - value: "1" - - name: DYN_REQUEST_PLANE - value: tcp - # nixl-write is the EC connector's transfer mode for the - # post-encoder embeddings between workers (only matters at - # TP>1, but harmless at TP=1). - - name: DYN_VLLM_EMBEDDING_TRANSFER_MODE - value: nixl-write - # 8 GiB embedding cache per worker (= 8 GiB total at TP=1). - - name: DYN_MULTIMODAL_EMBEDDING_CACHE_GB - value: "8" - # Permit http:// image URLs (our jsonl uses local PNG paths, - # not URLs — keeping this on doesn't hurt and future-proofs). - - name: DYN_MM_ALLOW_INTERNAL - value: "1" - workingDir: /workspace - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - replicas: 1 - resources: - limits: - gpu: "1" - requests: - gpu: "1" - subComponentType: null - volumeMounts: - - name: shared-model-cache - mountPoint: /home/dynamo/.cache/huggingface - # Same shared FSx HF cache as vllm-serve recipe. No subPath → - # benefit from already-downloaded weights. Compilation cache - # is NOT persisted here (single PVC, single mount point in - # DGD's volumeMounts grammar); we eat a ~1 min torch.compile + - # DeepGEMM JIT penalty per pod restart. Acceptable for bench - # runs that stay up for the entire sweep. diff --git a/recipes/qwen3.6-35b/deploy/dynamo-fd.yaml b/recipes/qwen3.6-35b/deploy/dynamo-fd.yaml deleted file mode 100644 index 22d4af13f567..000000000000 --- a/recipes/qwen3.6-35b/deploy/dynamo-fd.yaml +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Aggregated Dynamo on a single H100 with **frontend-decoding ON**, -# **embedding cache OFF** — isolates the frontend-decoding effect from -# the EC effect when compared against: -# - deploy/vllm-serve.yaml : baseline (no Dynamo features) -# - deploy/dynamo-fd-ec.yaml : Dynamo + frontend-decoding + EC -# -# Per-config flag diff: -# fd fd-ec vllm-serve -# processor cache 30 GB 30 GB 30 GB (same → fair comparison) -# --frontend-decoding yes yes n/a -# --multimodal-embedding-cache-capacity-gb -- 8 GB n/a -# -# (PR 8235's 397B yaml uses 60 GB processor cache on fd and 30 GB on -# fd-ec — different sizes. Vision-sweep skill guidance is to match -# processor cache across configs so the only flag that differs between -# fd and fd-ec is the EC capacity. We follow vision-sweep.) -apiVersion: nvidia.com/v1alpha1 -kind: DynamoGraphDeployment -metadata: - name: qwen36-dynamo-fd - labels: - app.kubernetes.io/name: qwen3.6-35b - app.kubernetes.io/component: dynamo-fd - app.kubernetes.io/managed-by: dynamo-recipe -spec: - pvcs: - - create: false - name: shared-model-cache - services: - Frontend: - componentType: frontend - envs: - - name: HF_HOME - value: /home/dynamo/.cache/huggingface - - name: DYN_REQUEST_PLANE - value: tcp - - name: DYN_HTTP_BODY_LIMIT_MB - value: "200" - extraPodSpec: - # Hostname pin + image come from hw/${HW}.env (shared with - # vllm-serve, dynamo-fd-ec). See deploy/dynamo-fd-ec.yaml for - # full rationale on why a hostname pin (not podAffinity). - nodeSelector: ${HW_NODE_SELECTOR} - tolerations: ${HW_TOLERATIONS} - mainContainer: - image: ${VLLM_IMAGE} - imagePullPolicy: IfNotPresent - workingDir: /workspace - replicas: 1 - resources: - requests: - cpu: "4" - memory: 16Gi - limits: - cpu: "8" - memory: 32Gi - subComponentType: null - - VllmWorker: - componentType: worker - extraPodSpec: - nodeSelector: ${HW_NODE_SELECTOR} - tolerations: ${HW_TOLERATIONS} - mainContainer: - command: - - /bin/bash - - -lc - args: - - | - set -euo pipefail - ulimit -l unlimited - python3 -m dingo.vllm \ - --model "${MODEL_NAME}" \ - --enable-multimodal \ - --frontend-decoding \ - --mm-processor-cache-gb 30 \ - --dyn-tool-call-parser qwen3_coder \ - --dyn-reasoning-parser qwen3 \ - --tensor-parallel-size 1 \ - --gpu-memory-utilization 0.85 \ - --max-model-len 32768 \ - --enable-prefix-caching \ - --no-enable-log-requests - image: ${VLLM_IMAGE} - imagePullPolicy: IfNotPresent - env: - - name: MODEL_NAME - value: "Qwen/Qwen3.6-35B-A3B-FP8" - - name: HF_HOME - value: /home/dynamo/.cache/huggingface - - name: HF_HUB_OFFLINE - value: "1" - - name: DYN_REQUEST_PLANE - value: tcp - # DYN_VLLM_EMBEDDING_TRANSFER_MODE and DYN_MULTIMODAL_EMBEDDING_CACHE_GB - # intentionally omitted — no embedding cache in this config. - - name: DYN_MM_ALLOW_INTERNAL - value: "1" - workingDir: /workspace - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - replicas: 1 - resources: - limits: - gpu: "1" - requests: - gpu: "1" - subComponentType: null - volumeMounts: - - name: shared-model-cache - mountPoint: /home/dynamo/.cache/huggingface diff --git a/recipes/qwen3.6-35b/deploy/vllm-serve.yaml b/recipes/qwen3.6-35b/deploy/vllm-serve.yaml deleted file mode 100644 index 474180d99d90..000000000000 --- a/recipes/qwen3.6-35b/deploy/vllm-serve.yaml +++ /dev/null @@ -1,151 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# vllm serve baseline — no Dynamo components. Mounts the namespace's -# `shared-model-cache` PVC (RWX, e.g. FSx Lustre) at HF_HOME so the HF -# Hub cache is shared across all configs in the namespace, AND at a -# subPath for the per-model vllm compilation cache. ---- -apiVersion: v1 -kind: Service -metadata: - name: qwen36-vllm-serve - labels: - app.kubernetes.io/name: qwen3.6-35b - app.kubernetes.io/component: vllm-serve - app.kubernetes.io/managed-by: dynamo-recipe -spec: - type: ClusterIP - selector: - app.kubernetes.io/name: qwen3.6-35b - app.kubernetes.io/component: vllm-serve - ports: - - name: http - port: 8000 - targetPort: 8000 - protocol: TCP ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: qwen36-vllm-serve - labels: - app.kubernetes.io/name: qwen3.6-35b - app.kubernetes.io/component: vllm-serve - app.kubernetes.io/managed-by: dynamo-recipe -spec: - replicas: 1 - # First-time bootstrap = image pull (~3 min) + model load from FSx (~4 min) - # + torch.compile (~1 min) + cudagraph capture for 102 sizes (~5 min) ≈ 13 - # min. K8s default of 600s declares the rollout failed at 10 min even - # though the pod is healthy and still booting. Bump to 30 min. - progressDeadlineSeconds: 1800 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: qwen3.6-35b - app.kubernetes.io/component: vllm-serve - template: - metadata: - labels: - app.kubernetes.io/name: qwen3.6-35b - app.kubernetes.io/component: vllm-serve - app.kubernetes.io/managed-by: dynamo-recipe - spec: - restartPolicy: Always - # Hostname pin + image come from hw/${HW}.env (sibling to this - # recipe's vllm-serve/ dir). All three configs (vllm-serve, - # dynamo-fd, dynamo-fd-ec) share that file so they land on the - # same node and benchmark on identical hardware. envsubst in - # run-benchmark.sh resolves these placeholders at apply time. - nodeSelector: ${HW_NODE_SELECTOR} - tolerations: ${HW_TOLERATIONS} - securityContext: - runAsUser: 0 - containers: - - name: vllm - image: ${VLLM_IMAGE} - imagePullPolicy: IfNotPresent - workingDir: /workspace - command: - - /bin/bash - - -lc - - | - set -euo pipefail - ulimit -l unlimited - exec vllm serve "${MODEL_NAME}" \ - --host 0.0.0.0 \ - --port 8000 \ - --tensor-parallel-size 1 \ - --gpu-memory-utilization 0.85 \ - --max-model-len 32768 \ - --mm-processor-cache-gb 30 \ - --enable-prefix-caching \ - --enable-auto-tool-choice \ - --tool-call-parser qwen3_coder \ - --reasoning-parser qwen3 \ - --no-enable-log-requests - env: - - name: MODEL_NAME - value: "Qwen/Qwen3.6-35B-A3B-FP8" - - name: HF_HOME - value: /home/dynamo/.cache/huggingface - - name: VLLM_CACHE_ROOT - value: /home/dynamo/.cache/vllm - - name: HF_HUB_OFFLINE - value: "1" - # Qwen3.6-35B-A3B-FP8 is a public model — vllm serve doesn't - # need an HF token to load it. Plus HF_HUB_OFFLINE=1 above - # short-circuits any hub lookups at runtime. Uncomment when - # swapping in a gated model. - # envFrom: - # - secretRef: - # name: hf-token-secret - # optional: true - ports: - - name: http - containerPort: 8000 - protocol: TCP - readinessProbe: - httpGet: - path: /v1/models - port: 8000 - initialDelaySeconds: 60 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 60 - resources: - requests: - cpu: "8" - memory: 64Gi - nvidia.com/gpu: "1" - limits: - cpu: "16" - memory: 128Gi - nvidia.com/gpu: "1" - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - volumeMounts: - - name: shared-model-cache - mountPath: /home/dynamo/.cache/huggingface - # No subPath — share the HF cache root with the namespace - # so already-downloaded models (Qwen3-30B-A3B, etc.) are - # available, and our Qwen3.6 download appears under the - # standard `hub/models--Qwen--Qwen3.6-...` layout. - - name: shared-model-cache - mountPath: /home/dynamo/.cache/vllm - subPath: qwen36-bench/vllm-cache - - name: shm - mountPath: /dev/shm - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: shm - emptyDir: - medium: Memory - sizeLimit: 40Gi diff --git a/recipes/qwen3.6-35b/hw/gb200.env b/recipes/qwen3.6-35b/hw/gb200.env deleted file mode 100644 index e04ddbc3734d..000000000000 --- a/recipes/qwen3.6-35b/hw/gb200.env +++ /dev/null @@ -1,22 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Hardware target: GB200 (ARM64 / Blackwell, single NVL4 tray). -# Shared by all three configs (vllm-serve, dynamo-fd, dynamo-fd-ec). -# -# BEFORE FIRST USE: replace below with the -# kubernetes.io/hostname of the target GB200 node. Find candidates via: -# -# kubectl get nodes -L kubernetes.io/arch -L nvidia.com/gpu.product \ -# | awk '/arm64/ && /GB200/' -# -# Same multi-arch tag as h100.env — nvcr.io vllm-runtime ships amd64 + -# arm64 under one manifest. If GB200 needs a different build, change -# the tag here (does not affect H100). -export VLLM_IMAGE="nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1" - -export HW_NODE_SELECTOR='{"kubernetes.io/hostname":""}' - -# GB200 nodes are tainted kubernetes.io/arch=arm64:NoSchedule — the -# hostname pin alone is not enough; the toleration is required. -export HW_TOLERATIONS='[{"key":"kubernetes.io/arch","operator":"Equal","value":"arm64","effect":"NoSchedule"}]' diff --git a/recipes/qwen3.6-35b/hw/h100.env b/recipes/qwen3.6-35b/hw/h100.env deleted file mode 100644 index 0b24ecc67da6..000000000000 --- a/recipes/qwen3.6-35b/hw/h100.env +++ /dev/null @@ -1,22 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Hardware target: H100. -# Shared by all three configs (vllm-serve, dynamo-fd, dynamo-fd-ec) — -# pinning to one specific hostname so they all benchmark on identical -# hardware. -# -# BEFORE FIRST USE: replace with the -# kubernetes.io/hostname of the target H100 node. Find candidates via: -# -# kubectl get nodes -L nvidia.com/gpu.product | awk '/H100/' - -# Multi-arch manifest — same tag works on amd64 (H100) and arm64 (GB200). -export VLLM_IMAGE="nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1" - -# JSON-flow value (single line) so envsubst can drop it inline into the -# YAML without indentation gymnastics. -export HW_NODE_SELECTOR='{"kubernetes.io/hostname":""}' - -# H100 nodes carry no special taint — empty toleration array. -export HW_TOLERATIONS='[]' diff --git a/recipes/qwen3.6-35b/model-cache/model-download.yaml b/recipes/qwen3.6-35b/model-cache/model-download.yaml deleted file mode 100644 index 860b0d24ba03..000000000000 --- a/recipes/qwen3.6-35b/model-cache/model-download.yaml +++ /dev/null @@ -1,67 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Mounts shared-model-cache at the standard HF_HOME path so the -# downloaded weights land at /home/dynamo/.cache/huggingface/hub/ -# models--Qwen--Qwen3.6-35B-A3B-FP8/, sharing the HF Hub cache layout -# with anything else in the namespace (other downloads in the same -# namespace are reused — no double download). -apiVersion: batch/v1 -kind: Job -metadata: - name: qwen36-model-download - labels: - app.kubernetes.io/name: qwen3.6-35b - app.kubernetes.io/component: model-download - app.kubernetes.io/managed-by: dynamo-recipe -spec: - backoffLimit: 3 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app.kubernetes.io/name: qwen3.6-35b - app.kubernetes.io/component: model-download - spec: - restartPolicy: Never - containers: - - name: model-download - image: python:3.10-slim - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - command: ["sh", "-c"] - env: - - name: MODEL_NAME - value: "Qwen/Qwen3.6-35B-A3B-FP8" - - name: HF_HOME - value: /home/dynamo/.cache/huggingface - # Uses up to 64 GB RAM for XET buffers; set "0" on low-memory nodes (8 GB cap): https://huggingface.co/docs/hub/en/xet/using-xet-storage#download-buffers - - name: HF_XET_HIGH_PERFORMANCE - value: "1" - - name: MODEL_REVISION - value: "main" - args: - - | - set -eux - pip install --no-cache-dir huggingface_hub==1.11.0 - hf download "$MODEL_NAME" --revision "$MODEL_REVISION" - resources: - requests: - cpu: "2" - memory: "64Gi" - limits: - cpu: "8" - memory: "64Gi" - volumeMounts: - - name: shared-model-cache - mountPath: /home/dynamo/.cache/huggingface - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache diff --git a/recipes/qwen3.6-35b/perf.yaml b/recipes/qwen3.6-35b/perf.yaml deleted file mode 100644 index 7ed7bca8bf8b..000000000000 --- a/recipes/qwen3.6-35b/perf.yaml +++ /dev/null @@ -1,134 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Shared aiperf benchmark pod template for all 3 configs (vllm-serve, -# dynamo-fd, dynamo-fd-ec). run-benchmark.sh exports the per-config -# values (${BENCH_POD}, ${BENCH_FRONTEND}, ${BENCH_RUN_LABEL}) plus -# the per-hardware ones (${HW_NODE_SELECTOR}, ${HW_TOLERATIONS}) and -# pipes this file through `envsubst` before `kubectl apply`. -# -apiVersion: v1 -kind: Pod -metadata: - name: ${BENCH_POD} - labels: - app.kubernetes.io/name: qwen3.6-35b - app.kubernetes.io/component: benchmark - app.kubernetes.io/managed-by: dynamo-recipe -spec: - restartPolicy: Never - # Hostname pin + toleration come from hw/${HW}.env — same target - # as the deploy target (vllm-serve Deployment, or DGD Frontend + - # Worker). That same-node guarantee is what we actually rely on; - # we deliberately do NOT add a podAffinity rule: - # * vllm-serve vs DGD pods carry different label schemas, so a - # single rule wouldn't cover all three configs. - # * podAffinity would be a no-op anyway given the hostname pin. - nodeSelector: ${HW_NODE_SELECTOR} - tolerations: ${HW_TOLERATIONS} - containers: - - name: benchmark - image: python:3.11 - command: - - /bin/bash - - -lc - - | - set -euo pipefail - ulimit -n 1048576 - ulimit -u 65536 - - apt update && apt install -y --no-install-recommends curl jq git - - pip install --no-cache-dir "aiperf==0.10.0" - python -c "import aiperf; print('aiperf', aiperf.__version__, 'OK')" - - echo "Waiting for model '${MODEL_NAME}' at http://${FRONTEND}:8000/v1/models ..." - until curl -s "http://${FRONTEND}:8000/v1/models" \ - | jq -e --arg m "${MODEL_NAME}" '.data[]? | select(.id == $m)' >/dev/null 2>&1; do - echo "[$(date '+%H:%M:%S')] not ready, sleeping 10s" - sleep 10 - done - echo "Model '${MODEL_NAME}' is ready." - - INPUT_FILE="${DATASET_DIR}/30u_8t_5w_8000word_base64.jsonl" - if [[ ! -f "${INPUT_FILE}" ]]; then - echo "Dataset not found: ${INPUT_FILE}" >&2 - exit 1 - fi - - RUN_DIR="${ARTIFACT_BASE_DIR}/${RUN_LABEL}" - mkdir -p "${RUN_DIR}" - echo "Running benchmark; artifacts → ${RUN_DIR}" - - # single_turn (NOT multi_turn): each jsonl row is one request. - # PR 824 makes single_turn respect session_id, so the 8 turns - # of user_ are issued in causal order (no out-of-order - # interleave across user_'s turns). - aiperf profile \ - --model "${MODEL_NAME}" \ - --input-file "${INPUT_FILE}" \ - --custom-dataset-type single_turn \ - --url "http://${FRONTEND}:8000" \ - --streaming \ - --ui-type none \ - --request-count "${REQUEST_COUNT}" \ - --concurrency "${CONCURRENCY}" \ - --request-rate-mode constant \ - --warmup-request-count "${WARMUP_REQUEST_COUNT}" \ - --artifact-dir "${RUN_DIR}" \ - --extra-inputs "max_tokens:${MAX_TOKENS}" \ - --extra-inputs "min_tokens:${MAX_TOKENS}" \ - --extra-inputs "ignore_eos:true" - - # aiperf writes inputs.json containing every prompt + every - # base64 image actually sent — ~4-12 GB for our multimodal - # datasets, vs ~30 MB of actual metric outputs. It's - # regenerable from the input jsonl + aiperf args, so we drop - # it by default. Set KEEP_INPUTS_JSON=1 to retain it for - # individual-request debugging. - if [[ "${KEEP_INPUTS_JSON:-}" != "1" ]]; then - rm -fv "${RUN_DIR}/inputs.json" || true - else - echo "KEEP_INPUTS_JSON=1 → retaining $(du -sh ${RUN_DIR}/inputs.json 2>/dev/null | cut -f1) inputs.json" - fi - - echo "Run complete. Artifacts in ${RUN_DIR}" - sleep 3600 - env: - - name: MODEL_NAME - value: Qwen/Qwen3.6-35B-A3B-FP8 - - name: FRONTEND - value: ${BENCH_FRONTEND} - - name: RUN_LABEL - value: ${BENCH_RUN_LABEL} - - name: MAX_TOKENS - value: "1024" - - name: REQUEST_COUNT - value: "240" # 30 users × 8 turns - - name: CONCURRENCY - value: "30" - - name: WARMUP_REQUEST_COUNT - value: "2" - - name: DATASET_DIR - value: /perf-cache/datasets - - name: ARTIFACT_BASE_DIR - value: /perf-cache/artifacts/qwen36_fp8 - - name: AIPERF_DATASET_CONFIGURATION_TIMEOUT - value: "1800" - - name: AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT - value: "1800" - resources: - requests: - cpu: "8" - memory: 16Gi - limits: - cpu: "16" - memory: 32Gi - volumeMounts: - - name: shared-model-cache - mountPath: /perf-cache - subPath: qwen36-bench/perf-cache - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache diff --git a/recipes/qwen3.6-35b/run-all-benchmarks.sh b/recipes/qwen3.6-35b/run-all-benchmarks.sh deleted file mode 100755 index 3d08a6a737de..000000000000 --- a/recipes/qwen3.6-35b/run-all-benchmarks.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Sequentially run all three configs (vllm-serve, dynamo-fd, dynamo-fd-ec) -# against the same hardware target and collect aiperf artifacts side-by-side -# under one dated directory ready for the 3-way comparison. -# -# Order per config: deploy → bench → retrieve → clean. Cleaning between -# configs frees the GPU for the next deployment so they share one node. -# -# Usage: -# ./run-all-benchmarks.sh -n # H100 -# ./run-all-benchmarks.sh -n --hw gb200 # GB200 -# ./run-all-benchmarks.sh -n --hw gb200 --skip-prep # already PVC/download/dataset-ready -# -# Prep step (PVC + model download + data gen) runs once before the first -# deploy unless --skip-prep is passed. -set -euo pipefail - -NAMESPACE="" -HW="h100" -SKIP_PREP="0" - -while [[ $# -gt 0 ]]; do - case "$1" in - -n|--namespace) NAMESPACE="$2"; shift 2 ;; - --hw) HW="$2"; shift 2 ;; - --skip-prep) SKIP_PREP="1"; shift ;; - -h|--help) - grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; - *) echo "unknown arg: $1" >&2; exit 2 ;; - esac -done -if [[ -z "$NAMESPACE" ]]; then - echo "ERROR: -n required" >&2; exit 2 -fi - -HERE="$(cd "$(dirname "$0")" && pwd)" -DRIVER="$HERE/run-benchmark.sh" -CONFIGS=(vllm-serve dynamo-fd dynamo-fd-ec) - -TS_DIR="$(date +%m-%d)" -# Override the summary root via $BENCHMARK_RESULTS_DIR — also picked up by -# run-benchmark.sh's retrieve(), so the two paths stay in sync. -BASE_DIR="${BENCHMARK_RESULTS_DIR:-$HOME/workspace/dynamo-tmp/logs}" -SUMMARY_DIR="$BASE_DIR/${TS_DIR}/qwen36-fp8-${HW}" -mkdir -p "$SUMMARY_DIR" -RUN_LOG="$SUMMARY_DIR/run-all-benchmarks.log" -echo "[run-all] hw=$HW namespace=$NAMESPACE" | tee -a "$RUN_LOG" -echo "[run-all] summary dir: $SUMMARY_DIR" | tee -a "$RUN_LOG" - -# Prep is config-agnostic; pick any config to source so config.env loads cleanly. -if [[ "$SKIP_PREP" != "1" ]]; then - for step in pvc download dataset; do - echo "[prep] $step" | tee -a "$RUN_LOG" - "$DRIVER" -n "$NAMESPACE" --hw "$HW" --config vllm-serve --step "$step" 2>&1 | tee -a "$RUN_LOG" - done -fi - -for cfg in "${CONFIGS[@]}"; do - echo "" | tee -a "$RUN_LOG" - echo "========== [$cfg] ==========" | tee -a "$RUN_LOG" - for step in deploy bench retrieve clean; do - "$DRIVER" -n "$NAMESPACE" --hw "$HW" --config "$cfg" --step "$step" 2>&1 | tee -a "$RUN_LOG" - done -done - -echo "" | tee -a "$RUN_LOG" -echo "[run-all] all three configs done." | tee -a "$RUN_LOG" -echo "[run-all] results: $SUMMARY_DIR/{vllm-serve,dynamo-fd,dynamo-fd-ec}/" | tee -a "$RUN_LOG" -echo "[run-all] each config's profile_export_aiperf.json holds the headline metrics." | tee -a "$RUN_LOG" diff --git a/recipes/qwen3.6-35b/run-benchmark.sh b/recipes/qwen3.6-35b/run-benchmark.sh deleted file mode 100755 index d3a4c3fba194..000000000000 --- a/recipes/qwen3.6-35b/run-benchmark.sh +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Unified driver for the Qwen3.6-35B-A3B-FP8 3-way benchmark. -# Idempotent — re-running steps that already completed is a no-op. -# -# Two axes: -# --hw → sources hw/.env (VLLM_IMAGE, HW_NODE_SELECTOR, HW_TOLERATIONS) -# --config → resolves to DEPLOY_KIND, DEPLOY_NAME, BENCH_POD inline (see CONFIGS table below) -# -# Usage: -# ./run-benchmark.sh -n --hw h100 --config vllm-serve -# ./run-benchmark.sh -n --hw gb200 --config dynamo-fd-ec --step deploy -# -# Steps: pvc | download | dataset | deploy | bench | retrieve | clean | all -# pvc/download/dataset are config-agnostic (idempotent prep). -# deploy/bench/retrieve/clean are config-specific. -set -euo pipefail - -NAMESPACE="" -STEP="all" -HW="h100" -CONFIG="" - -while [[ $# -gt 0 ]]; do - case "$1" in - -n|--namespace) NAMESPACE="$2"; shift 2 ;; - --step) STEP="$2"; shift 2 ;; - --hw) HW="$2"; shift 2 ;; - --config) CONFIG="$2"; shift 2 ;; - -h|--help) - grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; - *) echo "unknown arg: $1" >&2; exit 2 ;; - esac -done -if [[ -z "$NAMESPACE" ]]; then - echo "ERROR: -n required" >&2; exit 2 -fi -HERE="$(cd "$(dirname "$0")" && pwd)" - -# Per-config metadata. Keep this list in sync with the sibling config dirs. -# DEPLOY_KIND branches deploy() + clean(): -# deployment → kubectl rollout status + delete Deployment+Service -# dgd → kubectl wait on operator-stamped DGD pod labels + delete DGD -# BENCH_POD — name of the aiperf Pod for this config. -# BENCH_FRONTEND — service name the bench Pod hits at $FRONTEND:8000. -# vllm-serve: a plain Service; DGDs: `-frontend` -# created automatically by the dynamo operator. -# BENCH_RUN_LABEL — sub-directory written under /perf-cache/artifacts/ -# so the 3 configs' aiperf artifacts don't collide. -case "$CONFIG" in - vllm-serve) - DEPLOY_KIND="deployment" - DEPLOY_NAME="qwen36-vllm-serve" - BENCH_POD="qwen36-bench" - BENCH_FRONTEND="qwen36-vllm-serve" - BENCH_RUN_LABEL="vllm-serve" - ;; - dynamo-fd) - DEPLOY_KIND="dgd" - DEPLOY_NAME="qwen36-dynamo-fd" - BENCH_POD="qwen36-fd-bench" - BENCH_FRONTEND="qwen36-dynamo-fd-frontend" - BENCH_RUN_LABEL="dynamo-fd" - ;; - dynamo-fd-ec) - DEPLOY_KIND="dgd" - DEPLOY_NAME="qwen36-dynamo-fd-ec" - BENCH_POD="qwen36-fd-ec-bench" - BENCH_FRONTEND="qwen36-dynamo-fd-ec-frontend" - BENCH_RUN_LABEL="dynamo-fd-ec" - ;; - "") - echo "ERROR: --config required" >&2 - echo "Available: vllm-serve dynamo-fd dynamo-fd-ec" >&2 - exit 2 ;; - *) - echo "ERROR: unknown config: $CONFIG" >&2 - echo "Available: vllm-serve dynamo-fd dynamo-fd-ec" >&2 - exit 2 ;; -esac -export BENCH_POD BENCH_FRONTEND BENCH_RUN_LABEL - -HW_ENV="$HERE/hw/${HW}.env" -if [[ ! -f "$HW_ENV" ]]; then - echo "ERROR: hardware env file not found: $HW_ENV" >&2 - echo "Available: $(ls "$HERE/hw/" 2>/dev/null | tr '\n' ' ')" >&2 - exit 2 -fi -DEPLOY_TPL="$HERE/deploy/${CONFIG}.yaml" - -if ! command -v envsubst >/dev/null 2>&1; then - echo "ERROR: envsubst missing. Install gettext-base (apt) or gettext (brew)." >&2 - exit 2 -fi - -# shellcheck disable=SC1090 -set -a; . "$HW_ENV"; set +a -echo "[hw] $HW → image=$VLLM_IMAGE node=$HW_NODE_SELECTOR" -echo "[config] $CONFIG → kind=$DEPLOY_KIND deploy=$DEPLOY_NAME bench-pod=$BENCH_POD" - -K="kubectl -n $NAMESPACE" -# Limit envsubst to our own template vars so embedded ${MODEL_NAME} / -# ${KEEP_INPUTS_JSON:-} shell vars inside perf.yaml's inline bash stay -# literal. $BENCH_* drive the shared perf.yaml; $VLLM_IMAGE / $HW_* -# drive deploy.yaml + perf.yaml. -TPL_VARS='$VLLM_IMAGE $HW_NODE_SELECTOR $HW_TOLERATIONS $BENCH_POD $BENCH_FRONTEND $BENCH_RUN_LABEL' -APPLY_TPL() { envsubst "$TPL_VARS" <"$1" | $K apply -f -; } - -# ---------------- config-agnostic prep ---------------- - -pvc() { - # `shared-model-cache` is expected to be pre-provisioned in the namespace - # (RWX, e.g. FSx Lustre). If your cluster doesn't pre-provision it, create - # the PVC out-of-band — see README.md → "Storage: shared-model-cache". - if ! $K get pvc shared-model-cache >/dev/null 2>&1; then - echo "[pvc] ERROR: PVC 'shared-model-cache' not found in namespace '$NAMESPACE'" >&2 - echo "[pvc] See README.md → 'Storage: shared-model-cache' for provisioning guidance." >&2 - exit 1 - fi - $K get pvc shared-model-cache -} - -download() { - if $K get job qwen36-model-download >/dev/null 2>&1; then - if [[ "$($K get job qwen36-model-download -o jsonpath='{.status.succeeded}')" == "1" ]]; then - echo "[download] already complete" - return - fi - echo "[download] previous job present but not Complete — deleting and re-applying" - $K delete job qwen36-model-download - fi - $K apply -f "$HERE/model-cache/model-download.yaml" - $K wait --for=condition=Complete job/qwen36-model-download --timeout=3600s -} - -dataset() { - if $K get job qwen36-generate-datasets >/dev/null 2>&1; then - if [[ "$($K get job qwen36-generate-datasets -o jsonpath='{.status.succeeded}')" == "1" ]]; then - echo "[dataset] already complete" - return - fi - $K delete job qwen36-generate-datasets - fi - $K apply -f "$HERE/data-gen-job.yaml" - $K wait --for=condition=Complete job/qwen36-generate-datasets --timeout=1800s - $K logs job/qwen36-generate-datasets | tail -20 -} - -# ---------------- config-specific lifecycle ---------------- - -deploy() { - APPLY_TPL "$DEPLOY_TPL" - case "$DEPLOY_KIND" in - deployment) - $K rollout status "deploy/$DEPLOY_NAME" --timeout=900s - ;; - dgd) - local sel_fe="nvidia.com/dynamo-graph-deployment-name=$DEPLOY_NAME,nvidia.com/dynamo-component-type=frontend" - local sel_wk="nvidia.com/dynamo-graph-deployment-name=$DEPLOY_NAME,nvidia.com/dynamo-component-type=worker" - echo "[deploy] waiting for DGD Frontend pod ..." - $K wait --for=condition=Ready pod -l "$sel_fe" --timeout=900s - echo "[deploy] waiting for VllmWorker pod ..." - $K wait --for=condition=Ready pod -l "$sel_wk" --timeout=1500s - ;; - *) - echo "ERROR: unknown DEPLOY_KIND=$DEPLOY_KIND" >&2; exit 2 ;; - esac -} - -bench() { - $K delete pod "$BENCH_POD" --ignore-not-found - APPLY_TPL "$HERE/perf.yaml" - $K wait --for=condition=Ready "pod/$BENCH_POD" --timeout=300s - echo "[bench] streaming logs — Ctrl-C to detach (the run continues in pod)" - $K logs -f "$BENCH_POD" || true -} - -retrieve() { - # Override the destination root via $BENCHMARK_RESULTS_DIR if your - # workspace layout differs from the default. - local base="${BENCHMARK_RESULTS_DIR:-$HOME/workspace/dynamo-tmp/logs}" - local dest="$base/$(date +%m-%d)/qwen36-fp8-${HW}/${CONFIG}" - mkdir -p "$dest" - $K exec "$BENCH_POD" -- \ - tar c --exclude='inputs.json' -C /perf-cache artifacts \ - | tar x -C "$dest" - echo "[retrieve] landed at $dest" - find "$dest" -name 'profile_export_aiperf.json' -print -} - -clean() { - $K delete pod "$BENCH_POD" --ignore-not-found - case "$DEPLOY_KIND" in - deployment) - $K delete deploy "$DEPLOY_NAME" --ignore-not-found - $K delete service "$DEPLOY_NAME" --ignore-not-found - ;; - dgd) - $K delete dynamographdeployment "$DEPLOY_NAME" --ignore-not-found - ;; - esac - # Note: PVCs intentionally NOT deleted — that would force model re-download. - # To wipe everything: - # kubectl -n $NS delete pvc shared-model-cache -} - -all() { - pvc - download - dataset - deploy - bench - retrieve -} - -case "$STEP" in - pvc|download|dataset|deploy|bench|retrieve|clean|all) "$STEP" ;; - *) echo "unknown step: $STEP" >&2; exit 2 ;; -esac diff --git a/scripts/dco_check.py b/scripts/dco_check.py deleted file mode 100755 index c0f628e49623..000000000000 --- a/scripts/dco_check.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import re -import sys - - -def commit_message_has_signoff(message): - """ - Check if the commit message has a Signed-off-by line. - - Args: - message (str): The commit message. - - Returns: - bool: True if the message is valid, False otherwise. - """ - for line in message.splitlines(): - if re.match(r"^Signed-off-by: .+ <.+>$", line): - return True - return False - - -def main(): - if len(sys.argv) != 2: - print("Usage: python dco_check.py ") - sys.exit(1) - - # Read the commit message from the file passed as an argument by Git - with open(sys.argv[1], "r") as file: - message = file.read().strip() - - # Validate the commit message - if not commit_message_has_signoff(message): - print( - "The commit message does not contain a Signed-off-by line. Please review CONTRIBUTING.md for more details." - ) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/tests/README.md b/tests/README.md index 1f396a463b40..4e13935b3984 100644 --- a/tests/README.md +++ b/tests/README.md @@ -13,7 +13,7 @@ All tests run inside containers. See the [Container Development Guide](../contai Each area can have one or more of the following types of tests: 1. **Unit** -- Exercises a single function, class, or module in isolation. No external services, no GPU. Each test typically runs in milliseconds; all unit tests combined may take <5 minutes. -2. **Integration** -- Wires multiple components together using **mock engines** (`dynamo.mocker`) and **real infrastructure** (ETCD for service discovery, NATS for messaging, if enabled). Validates that the router, planner, frontend gRPC, and similar subsystems work together without launching a real inference engine. No GPU required. Each test typically runs in seconds; all integration tests combined may take <30 minutes. +2. **Integration** -- Wires multiple components together using **mock engines** (`dingo.mocker`) and **real infrastructure** (ETCD for service discovery, NATS for messaging, if enabled). Validates that the router, planner, frontend gRPC, and similar subsystems work together without launching a real inference engine. No GPU required. Each test typically runs in seconds; all integration tests combined may take <30 minutes. 3. **End-to-End (E2E)** -- Starts a **real inference engine** (vLLM, SGLang, or TRT-LLM), sends requests through the frontend, and validates responses. Requires GPU. Each test typically runs in minutes; the full E2E suite may take several hours. It is absolutely important to be mindful of how long a test you write takes. Slow tests have a compounding cost: they burn GPU-hours in CI (GPUs are expensive and shared), they discourage engineers from running suites locally (so bugs slip through to CI), and they slow down the entire team's development velocity. A test suite that takes too long becomes a test suite that nobody runs. When adding or modifying tests, include a per-test time estimate in your PR description -- CI GPU resources are limited and these estimates help the team schedule tests across pre-merge, nightly, and weekly pipelines. @@ -92,10 +92,10 @@ dynamo/ | KVBM Integration | KV block manager integration | `tests/kvbm_integration/` | | GPU Memory Service | GPU Memory Service E2E | `tests/gpu_memory_service/` | | Router | Router E2E with backends | `tests/router/` | -| Planner | Planner unit + integration tests | `components/src/dynamo/planner/tests/` | +| Planner | Planner unit + integration tests | `dingo/planner/tests/` | | Frontend | Frontend HTTP/gRPC tests | `tests/frontend/` | -| Profiler | Profiler unit + integration tests | `components/src/dynamo/profiler/tests/` | -| Global Planner | Global planner unit tests | `components/src/dynamo/global_planner/tests/` | +| Profiler | Profiler unit + integration tests | `dingo/profiler/tests/` | +| Global Planner | Global planner unit tests | `dingo/global_planner/tests/` | | Fault Tolerance | Chaos, migration, cancellation | `tests/fault_tolerance/` | | Deployment | Deployment validation | `tests/deploy/` | | Benchmark | Performance/load | `benchmarks/` | @@ -548,7 +548,7 @@ Tests must be deterministic. A flaky test -- one that sometimes passes and somet - `tests/frontend/test_frontend_api_surface_compliance.py:_retry_network_op` -- sync, network-only exception list. - `tests/router/helper.py:send_request_with_retry` -- async, status-code-driven (aiohttp). - `tests/utils/managed_deployment.py` -- sync connect retry with 1.5x backoff. - - `components/src/dynamo/planner/connectors/remote_client.py` -- sync exponential backoff (`2**attempt`). + - `dingo/planner/connectors/remote_client.py` -- sync exponential backoff (`2**attempt`). 3. **If retry is not enough either**, quarantine the test to prevent it from blocking other developers: - `@pytest.mark.skip(reason="Flaky: ")` -- disables the test entirely. Use when the test provides no signal in its current state. - `@pytest.mark.xfail(reason="Flaky: ", strict=False)` -- runs the test but does not fail the suite. Use when you still want visibility into pass/fail rates while you investigate. diff --git a/tests/fault_tolerance/test_unified_canary.py b/tests/fault_tolerance/test_unified_canary.py index b3f9c0a52e12..125f4c4f6b13 100644 --- a/tests/fault_tolerance/test_unified_canary.py +++ b/tests/fault_tolerance/test_unified_canary.py @@ -13,7 +13,7 @@ import pytest import requests -from dynamo.common.utils.paths import WORKSPACE_DIR +from dingo.common.utils.paths import WORKSPACE_DIR from tests.utils.engine_process import EngineConfig, EngineProcess CANARY_READY_BUDGET_S = 60 diff --git a/tests/frontend/conftest.py b/tests/frontend/conftest.py index c0915e4b1c5c..db4f25f407f9 100644 --- a/tests/frontend/conftest.py +++ b/tests/frontend/conftest.py @@ -230,7 +230,7 @@ def start_services_with_grpc( class MockerWorkerProcess(ManagedProcess): """Shared mocker worker process for frontend tests. - Uses dynamo.mocker with configurable model and speedup ratio. + Uses dingo.mocker with configurable model and speedup ratio. Can be used by any frontend test that needs a fast mock backend. """ @@ -252,7 +252,7 @@ def __init__( command = [ "python3", "-m", - "dynamo.mocker", + "dingo.mocker", "--model-path", model, "--speedup-ratio", @@ -286,7 +286,7 @@ def __init__( display_output=True, terminate_all_matching_process_names=False, stragglers=["VLLM::EngineCore"], - straggler_commands=["-m dynamo.mocker"], + straggler_commands=["-m dingo.mocker"], log_dir=log_dir, ) @@ -339,7 +339,7 @@ def start_services_with_mocker( class SampleUnifiedWorkerProcess(ManagedProcess): - """Unified-backend sample worker (`dynamo.common.backend.sample_main`). + """Unified-backend sample worker (`dingo.common.backend.sample_main`). CPU-only Python reference engine that exercises the unified backend's `Worker.run()` path — the same code path real backends (vllm/trtllm/ @@ -370,7 +370,7 @@ def __init__( command = [ "python3", "-m", - "dynamo.common.backend.sample_main", + "dingo.common.backend.sample_main", "--model-name", model_name, "--component", @@ -405,7 +405,7 @@ def __init__( timeout=120, display_output=True, terminate_all_matching_process_names=False, - straggler_commands=["-m dynamo.common.backend.sample_main"], + straggler_commands=["-m dingo.common.backend.sample_main"], log_dir=log_dir, ) diff --git a/tests/frontend/http_status_propagation_worker.py b/tests/frontend/http_status_propagation_worker.py index 6e2d088bbe67..99849a46c50d 100644 --- a/tests/frontend/http_status_propagation_worker.py +++ b/tests/frontend/http_status_propagation_worker.py @@ -23,7 +23,7 @@ class _StatusLikeError(Exception): """Duck-typed `.status` + `.message` — same shape as - `dynamo.common.http.HttpStatusError`.""" + `dingo.common.http.HttpStatusError`.""" def __init__(self, status: int, message: str): super().__init__(f"HTTP {status}: {message}") diff --git a/tests/router/counter_worker.py b/tests/router/counter_worker.py index 363b3f0e0949..7e66f405b7f1 100644 --- a/tests/router/counter_worker.py +++ b/tests/router/counter_worker.py @@ -18,16 +18,16 @@ import asyncio import os -from dynamo.common.configuration.arg_group import ArgGroup -from dynamo.common.configuration.groups.kv_router_args import ( +from dingo.common.configuration.arg_group import ArgGroup +from dingo.common.configuration.groups.kv_router_args import ( KvRouterArgGroup, KvRouterConfigBase, ) -from dynamo.common.configuration.groups.router_args import ( +from dingo.common.configuration.groups.router_args import ( RouterArgGroup, RouterConfigBase, ) -from dynamo.common.configuration.utils import add_argument +from dingo.common.configuration.utils import add_argument request_count = 0 diff --git a/tests/router/mocker_process.py b/tests/router/mocker_process.py index 2007ea0634a8..d4d135819ebc 100644 --- a/tests/router/mocker_process.py +++ b/tests/router/mocker_process.py @@ -48,7 +48,7 @@ def _build_mocker_command( command = [ sys.executable, "-m", - "dynamo.mocker", + "dingo.mocker", "--model-path", MODEL_NAME, "--endpoint", diff --git a/tests/router/test_router_e2e_with_mockers.py b/tests/router/test_router_e2e_with_mockers.py index c848bade8a5f..004af00c9467 100644 --- a/tests/router/test_router_e2e_with_mockers.py +++ b/tests/router/test_router_e2e_with_mockers.py @@ -100,7 +100,7 @@ def _pin_nats_event_plane_for_mocker(request, monkeypatch): ) PLANNER_PROFILE_DATA_DIR = ( Path(__file__).resolve().parents[2] - / "components/src/dynamo/planner/tests/data/profiling_results/H200_TP1P_TP1D" + / "dingo/planner/tests/data/profiling_results/H200_TP1P_TP1D" ) ROUTER_AIC_CONFIG = { "aic_backend": "vllm", diff --git a/tests/router/test_router_e2e_with_unified.py b/tests/router/test_router_e2e_with_unified.py index b3846948c85f..889f4fda6458 100644 --- a/tests/router/test_router_e2e_with_unified.py +++ b/tests/router/test_router_e2e_with_unified.py @@ -3,7 +3,7 @@ """End-to-end router tests against the **unified-backend** entrypoints (``python -m dingo.vllm.unified_main`` etc.) introduced under the -``dynamo.common.backend`` abstraction. +``dingo.common.backend`` abstraction. Each backend's existing legacy entrypoint already has e2e router tests in ``test_router_e2e_with_{vllm,sglang}.py``. This file mirrors a diff --git a/tests/serve/common.py b/tests/serve/common.py index ceb66536bc14..855328db0974 100644 --- a/tests/serve/common.py +++ b/tests/serve/common.py @@ -15,7 +15,7 @@ import pytest -from dynamo.common.utils.paths import WORKSPACE_DIR +from dingo.common.utils.paths import WORKSPACE_DIR from tests.conftest import ServicePorts from tests.utils.client import send_request from tests.utils.constants import DefaultPort diff --git a/tests/serve/conftest.py b/tests/serve/conftest.py index 3c8b963976c6..bdbe8bfc2f8f 100644 --- a/tests/serve/conftest.py +++ b/tests/serve/conftest.py @@ -7,7 +7,7 @@ import pytest from pytest_httpserver import HTTPServer -from dynamo.common.utils.paths import WORKSPACE_DIR +from dingo.common.utils.paths import WORKSPACE_DIR from tests.serve.lora_utils import MinioLoraConfig, MinioService from tests.utils.port_utils import allocate_port, deallocate_port diff --git a/tests/serve/launch/template_verifier.py b/tests/serve/launch/template_verifier.py index 00863ddf3f5d..8d641f010441 100644 --- a/tests/serve/launch/template_verifier.py +++ b/tests/serve/launch/template_verifier.py @@ -8,7 +8,7 @@ import uvloop from transformers import AutoTokenizer -from dynamo.common.utils.paths import WORKSPACE_DIR +from dingo.common.utils.paths import WORKSPACE_DIR from dynamo.llm import ModelInput, ModelType, WorkerType, register_model from dynamo.runtime import DistributedRuntime, dynamo_worker diff --git a/tests/serve/test_prometheus_exposition_format_injection.py b/tests/serve/test_prometheus_exposition_format_injection.py index a07c7c28acba..0070e92f3b02 100644 --- a/tests/serve/test_prometheus_exposition_format_injection.py +++ b/tests/serve/test_prometheus_exposition_format_injection.py @@ -12,7 +12,7 @@ from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram from dynamo import prometheus_names -from dynamo.common.utils.prometheus import get_prometheus_expfmt +from dingo.common.utils.prometheus import get_prometheus_expfmt pytestmark = [pytest.mark.unit, pytest.mark.pre_merge, pytest.mark.gpu_0] diff --git a/tests/utils/managed_deployment.py b/tests/utils/managed_deployment.py index 9c830ebce8e3..23726fc2ae0d 100644 --- a/tests/utils/managed_deployment.py +++ b/tests/utils/managed_deployment.py @@ -21,7 +21,7 @@ def _get_workspace_dir() -> str: - """Get workspace directory without depending on dynamo.common package. + """Get workspace directory without depending on dingo.common package. This allows tests to run without requiring dynamo package to be installed. """ diff --git a/tests/utils/multimodal.py b/tests/utils/multimodal.py index 3c09215ac114..d8c1dedfb4ce 100644 --- a/tests/utils/multimodal.py +++ b/tests/utils/multimodal.py @@ -9,7 +9,7 @@ import pytest -from dynamo.common.utils.paths import WORKSPACE_DIR +from dingo.common.utils.paths import WORKSPACE_DIR from tests.serve.conftest import MULTIMODAL_IMG_URL, get_multimodal_test_image_bytes from tests.utils.engine_process import EngineConfig from tests.utils.payload_builder import chat_payload