From 1be4cf7744e8258f50eabbdf2c8efb275f1c4c42 Mon Sep 17 00:00:00 2001 From: xguo01 <13910754971@163.com> Date: Sat, 4 Jul 2026 02:09:12 +0800 Subject: [PATCH 1/7] [feat][frontend] Splite frontend. --- README.md | 2 +- README.zh-CN.md | 2 +- benchmarks/frontend/README.md | 2 +- benchmarks/frontend/dgd/templates/mocker.yaml | 2 +- .../dgd/templates/vllm-gpt-oss-20b.yaml | 2 +- benchmarks/frontend/dgd/templates/vllm.yaml | 2 +- benchmarks/frontend/scripts/bpf/README.md | 2 +- .../frontend/scripts/flamegraph/README.md | 2 +- benchmarks/frontend/scripts/run_perf.sh | 8 ++-- benchmarks/frontend/scripts/scaling-test.md | 2 +- .../embedding_cache/trtllm_e_pd.sh | 4 +- benchmarks/router/README.md | 10 ++-- components/README.md | 2 +- .../configuration/groups/kv_router_args.py | 2 +- .../configuration/groups/router_args.py | 2 +- .../configuration/test_kv_router_args.py | 2 +- .../manual/perf_test_configs/agg_8b.yaml | 2 +- .../perf_test_configs/disagg_8b_2p2d.yaml | 2 +- .../perf_test_configs/disagg_8b_3p1d.yaml | 2 +- .../perf_test_configs/disagg_8b_planner.yaml | 2 +- .../perf_test_configs/disagg_8b_tp2.yaml | 2 +- .../manual/scaling/disagg_planner_load.yaml | 2 +- .../utils/config_modifiers/protocol.py | 2 +- components/src/dynamo/router/CLAUDE.md | 6 +-- components/src/dynamo/router/README.md | 2 +- components/src/dynamo/router/args.py | 2 +- .../src/dynamo/thunderagent_router/README.md | 2 +- .../thunderagent_router/run_minimax_8xh100.sh | 2 +- .../vllm/tests/test_vllm_renderer_api.py | 46 +++++++++---------- container/README.md | 8 ++-- container/launch_message/frontend.txt | 2 +- container/launch_message/runtime.txt | 2 +- container/templates/frontend.Dockerfile | 1 + container/templates/sglang_runtime.Dockerfile | 2 +- container/templates/trtllm_runtime.Dockerfile | 2 +- container/templates/vllm_runtime.Dockerfile | 2 +- ...nvidia.com_dynamocomponentdeployments.yaml | 4 +- .../nvidia.com_dynamographdeployments.yaml | 4 +- .../dynamocomponentdeployment_types.go | 4 +- .../dynamographdeployment_conversion_test.go | 4 +- ...nvidia.com_dynamocomponentdeployments.yaml | 4 +- .../nvidia.com_dynamographdeployments.yaml | 4 +- .../internal/dynamo/backend_vllm_test.go | 4 +- .../internal/dynamo/component_frontend.go | 2 +- deploy/operator/internal/dynamo/graph_test.go | 8 ++-- deploy/operator/samples/dgd-gms-failover.yaml | 2 +- .../samples/v1beta1/dgd-gms-failover.yaml | 2 +- dev/observability/prometheus.yml | 2 +- dev/sanity_check.py | 12 +++-- dingo/__init__.py | 3 ++ .../src/dynamo => dingo}/frontend/CLAUDE.md | 2 +- .../src/dynamo => dingo}/frontend/README.md | 0 .../src/dynamo => dingo}/frontend/__init__.py | 0 .../src/dynamo => dingo}/frontend/__main__.py | 2 +- .../frontend/frontend_args.py | 0 .../src/dynamo => dingo}/frontend/main.py | 6 +-- .../src/dynamo => dingo}/frontend/prepost.py | 0 .../frontend/sglang_prepost.py | 0 .../frontend/sglang_processor.py | 2 +- .../frontend/tests/FRONTEND_CASES.md | 8 ++-- .../frontend/tests/_routed_engine_fakes.py | 0 .../frontend/tests/test_frontend_utils.py | 2 +- .../frontend/tests/test_multimodal_utils.py | 2 +- .../tests/test_sglang_multimodal_prepost.py | 2 +- .../tests/test_sglang_processor_api.py | 2 +- .../test_sglang_processor_metrics_unit.py | 6 +-- .../tests/test_sglang_processor_unit.py | 10 ++-- .../frontend/tests/test_sglang_tool_calls.py | 2 +- .../tests/test_vllm_processor_unit.py | 12 ++--- .../src/dynamo => dingo}/frontend/utils.py | 0 .../frontend/vllm_processor.py | 2 +- docs/agents/agent-harnesses.md | 2 +- docs/agents/thunderagent-router.md | 2 +- docs/backends/sglang/sglang-chat-processor.md | 10 ++-- docs/backends/sglang/sglang-hicache.md | 6 +-- docs/backends/sglang/sglang-observability.md | 4 +- .../backends/trtllm/trtllm-dp-rank-routing.md | 2 +- docs/backends/trtllm/trtllm-examples.md | 4 +- docs/backends/trtllm/trtllm-gpt-oss.md | 2 +- docs/backends/trtllm/trtllm-observability.md | 2 +- docs/backends/vllm/README.md | 2 +- docs/backends/vllm/vllm-chat-processor.md | 6 +-- docs/components/frontend/README.md | 6 +-- docs/components/frontend/Tokenizer.md | 4 +- docs/components/frontend/configuration.md | 2 +- docs/components/frontend/frontend-guide.md | 6 +-- docs/components/kvbm/kvbm-guide.md | 6 +-- docs/components/router/README.md | 2 +- docs/components/router/README.zh-CN.md | 2 +- docs/components/router/router-concepts.md | 2 +- .../components/router/router-configuration.md | 4 +- .../router/router-disaggregated-serving.md | 2 +- docs/components/router/router-examples.md | 2 +- docs/components/router/router-guide.md | 20 ++++---- docs/components/router/router-operations.md | 14 +++--- docs/components/router/standalone-indexer.md | 2 +- docs/contribution-guide.md | 2 +- docs/contribution-guide.zh-CN.md | 2 +- docs/design-docs/event-plane.md | 6 +-- docs/design-docs/request-plane.md | 4 +- docs/development/unified-backends.md | 4 +- .../agentic-inference/agentic-harnesses.md | 2 +- docs/documentation-style-guide.md | 2 +- docs/fault-tolerance/graceful-shutdown.md | 2 +- docs/fault-tolerance/request-rejection.md | 4 +- docs/features/diffusion/fastvideo.md | 4 +- docs/features/tokenizer/README.md | 8 ++-- docs/getting-started/building-from-source.md | 2 +- .../building-from-source.zh-CN.md | 2 +- docs/getting-started/local-installation.md | 6 +-- .../local-installation.zh-CN.md | 6 +-- docs/getting-started/quickstart.mdx | 2 +- docs/getting-started/quickstart.zh-CN.mdx | 2 +- docs/integrations/flexkv-integration.md | 6 +-- docs/kubernetes/api-reference.md | 2 +- docs/kubernetes/cloud-providers/ecs/ecs.md | 2 +- .../deployment/create-deployment.md | 2 +- docs/kubernetes/gateway-api/README.mdx | 2 +- docs/kubernetes/gateway-api/reference.mdx | 2 +- docs/kubernetes/topology-aware-scheduling.md | 6 +-- docs/observability/health-checks.md | 2 +- docs/observability/logging.md | 4 +- docs/observability/metrics.md | 6 +-- docs/observability/prometheus-grafana.md | 2 +- docs/reasoning/README.md | 2 +- docs/templates/component-readme.md | 4 +- docs/tool-calling/README.md | 2 +- docs/tool-calling/README.zh-CN.md | 2 +- docs/tool-calling/engine-fallback.md | 4 +- docs/tool-calling/parser-configuration.md | 8 ++-- docs/tool-calling/structural-tag.md | 2 +- dynamo.code-workspace | 2 +- examples/backends/sample/launch/agg.sh | 2 +- .../backends/sample/launch/agg_diffusion.sh | 2 +- examples/backends/sample/launch/disagg.sh | 4 +- examples/backends/sglang/launch/_test_agg.sh | 4 +- examples/backends/sglang/launch/agg.sh | 4 +- examples/backends/sglang/launch/agg_agent.sh | 2 +- examples/backends/sglang/launch/agg_embed.sh | 4 +- .../sglang/launch/agg_multimodal_router.sh | 2 +- examples/backends/sglang/launch/agg_router.sh | 4 +- examples/backends/sglang/launch/agg_vision.sh | 6 +-- .../backends/sglang/launch/diffusion_llada.sh | 2 +- examples/backends/sglang/launch/disagg.sh | 4 +- .../backends/sglang/launch/disagg_router.sh | 4 +- .../backends/sglang/launch/disagg_same_gpu.sh | 4 +- .../backends/sglang/launch/image_diffusion.sh | 2 +- .../backends/sglang/launch/lora/agg_lora.sh | 2 +- .../sglang/launch/multimodal_disagg.sh | 4 +- .../backends/sglang/launch/multimodal_epd.sh | 4 +- .../sglang/launch/text-to-video-diffusion.sh | 2 +- .../sglang/test_sglang_expert_info.py | 2 +- .../backends/sglang/test_sglang_profile.py | 2 +- .../backends/tritonserver/launch/identity.sh | 2 +- examples/backends/trtllm/deploy/README.md | 2 +- .../trtllm/deploy/disagg-multinode.yaml | 2 +- .../trtllm/deploy/disagg_planner.yaml | 2 +- .../deploy/v1beta1/disagg-multinode.yaml | 2 +- .../trtllm/deploy/v1beta1/disagg_planner.yaml | 2 +- examples/backends/trtllm/launch/agg.sh | 4 +- .../trtllm/launch/agg_image_diffusion.sh | 2 +- .../backends/trtllm/launch/agg_metrics.sh | 4 +- .../backends/trtllm/launch/agg_multimodal.sh | 4 +- .../trtllm/launch/agg_multimodal_router.sh | 2 +- examples/backends/trtllm/launch/agg_router.sh | 4 +- .../trtllm/launch/agg_router_approx.sh | 2 +- .../trtllm/launch/agg_video_diffusion.sh | 2 +- examples/backends/trtllm/launch/disagg.sh | 4 +- .../backends/trtllm/launch/disagg_e_pd.sh | 4 +- .../trtllm/launch/disagg_multimodal.sh | 4 +- .../backends/trtllm/launch/disagg_router.sh | 4 +- .../backends/trtllm/launch/disagg_same_gpu.sh | 4 +- examples/backends/trtllm/launch/epd_disagg.sh | 4 +- .../epd_multimodal_image_and_embeddings.sh | 4 +- .../backends/trtllm/launch/gpt_oss_disagg.sh | 4 +- .../trtllm/mm_router_worker/README.md | 2 +- .../trtllm/mm_router_worker/launch.sh | 2 +- .../vllm/deploy/agg_router_kv_approx.yaml | 2 +- .../vllm/deploy/disagg-multinode.yaml | 2 +- examples/backends/vllm/deploy/gaie/agg.yaml | 2 +- .../backends/vllm/deploy/gaie/disagg.yaml | 4 +- .../backends/vllm/deploy/gms-failover.yaml | 2 +- .../deploy/v1beta1/agg_router_kv_approx.yaml | 2 +- .../vllm/deploy/v1beta1/disagg-multinode.yaml | 2 +- .../vllm/deploy/v1beta1/gms-failover.yaml | 2 +- examples/backends/vllm/launch/agg.sh | 4 +- examples/backends/vllm/launch/agg_embed.sh | 4 +- .../vllm/launch/agg_embed_multiworker.sh | 2 +- examples/backends/vllm/launch/agg_flexkv.sh | 4 +- .../backends/vllm/launch/agg_flexkv_router.sh | 4 +- examples/backends/vllm/launch/agg_kvbm.sh | 4 +- .../backends/vllm/launch/agg_kvbm_router.sh | 4 +- examples/backends/vllm/launch/agg_lmcache.sh | 2 +- .../backends/vllm/launch/agg_lmcache_mp.sh | 2 +- .../vllm/launch/agg_lmcache_multiproc.sh | 2 +- .../backends/vllm/launch/agg_multimodal.sh | 4 +- .../vllm/launch/agg_multimodal_router.sh | 2 +- .../agg_multimodal_router_chat_processor.sh | 2 +- .../backends/vllm/launch/agg_omni_audio.sh | 2 +- examples/backends/vllm/launch/agg_omni_i2v.sh | 2 +- .../backends/vllm/launch/agg_omni_image.sh | 2 +- .../backends/vllm/launch/agg_omni_realtime.sh | 2 +- .../backends/vllm/launch/agg_omni_video.sh | 2 +- .../vllm/launch/agg_request_planes.sh | 2 +- examples/backends/vllm/launch/agg_router.sh | 4 +- .../backends/vllm/launch/agg_router_approx.sh | 2 +- .../vllm/launch/agg_router_replicas.sh | 4 +- .../backends/vllm/launch/agg_spec_decoding.sh | 2 +- examples/backends/vllm/launch/agg_tracing.sh | 4 +- examples/backends/vllm/launch/dep.sh | 4 +- examples/backends/vllm/launch/disagg.sh | 4 +- .../backends/vllm/launch/disagg_flexkv.sh | 4 +- examples/backends/vllm/launch/disagg_kvbm.sh | 4 +- .../backends/vllm/launch/disagg_kvbm_2p2d.sh | 4 +- .../vllm/launch/disagg_kvbm_router.sh | 4 +- .../backends/vllm/launch/disagg_lmcache.sh | 4 +- .../vllm/launch/disagg_multimodal_e_pd.sh | 2 +- .../vllm/launch/disagg_multimodal_epd.sh | 4 +- .../vllm/launch/disagg_multimodal_p_d.sh | 2 +- .../vllm/launch/disagg_omni_glm_image.sh | 2 +- .../backends/vllm/launch/disagg_router.sh | 4 +- .../vllm/launch/disagg_router_gaudi.sh | 2 +- .../backends/vllm/launch/disagg_same_gpu.sh | 4 +- .../backends/vllm/launch/disagg_tracing.sh | 4 +- examples/backends/vllm/launch/dsr1_dep.sh | 4 +- .../backends/vllm/launch/lora/agg_lora.sh | 4 +- .../vllm/launch/lora/agg_lora_router.sh | 4 +- .../vllm/launch/lora/multimodal/lora_agg.sh | 2 +- .../launch/lora/xpu/agg_lora_router_xpu.sh | 4 +- .../vllm/launch/lora/xpu/agg_lora_xpu.sh | 4 +- .../backends/vllm/launch/multi_node_tp.sh | 2 +- .../vllm/launch/realtime_omni_client.py | 4 +- .../launch/xpu/agg_lmcache_multiproc_xpu.sh | 2 +- .../vllm/launch/xpu/agg_lmcache_xpu.sh | 2 +- ...gg_multimodal_router_chat_processor_xpu.sh | 2 +- .../launch/xpu/agg_multimodal_router_xpu.sh | 2 +- .../vllm/launch/xpu/agg_multimodal_xpu.sh | 4 +- .../vllm/launch/xpu/agg_request_planes_xpu.sh | 2 +- .../vllm/launch/xpu/agg_router_approx_xpu.sh | 2 +- .../vllm/launch/xpu/agg_router_xpu.sh | 4 +- examples/backends/vllm/launch/xpu/agg_xpu.sh | 4 +- .../launch/xpu/disagg_multimodal_epd_xpu.sh | 4 +- .../vllm/launch/xpu/disagg_router_xpu_gdr.sh | 2 +- examples/common/launch_utils.sh | 2 +- .../ECS/task_definition_frontend.json | 2 +- .../EKS/manifests/vllm/disagg-p5.yaml | 2 +- .../EKS/manifests/vllm/disagg.yaml | 2 +- .../EKS/manifests/vllm/v1beta1/disagg-p5.yaml | 2 +- .../EKS/manifests/vllm/v1beta1/disagg.yaml | 2 +- examples/diffusers/deploy/agg.yaml | 2 +- .../diffusers/deploy/agg_user_workload.yaml | 2 +- examples/diffusers/deploy/v1beta1/agg.yaml | 2 +- .../deploy/v1beta1/agg_user_workload.yaml | 2 +- examples/diffusers/local/docker-compose.yml | 2 +- examples/diffusers/local/run_local.sh | 2 +- .../global-planner-gpu-budget.yaml | 4 +- .../global-planner-mocker-test.yaml | 2 +- .../global-planner-vllm-test.yaml | 2 +- .../v1beta1/global-planner-gpu-budget.yaml | 4 +- .../v1beta1/global-planner-mocker-test.yaml | 2 +- .../v1beta1/global-planner-vllm-test.yaml | 2 +- hatch_build.py | 11 +++-- .../examples/mocker/Dockerfile.frontend | 5 +- lib/bench/README.md | 2 +- lib/llm/tests/http_namespace_integration.rs | 2 +- lib/runtime/examples/system_metrics/README.md | 2 +- pyproject.toml | 5 +- recipes/README.md | 2 +- .../trtllm/disagg/wide_ep/gb200/deploy.yaml | 2 +- .../trtllm/agg-round-robin/deploy.yaml | 2 +- .../trtllm/disagg-kv-router/deploy.yaml | 2 +- .../sglang/disagg-b200/deploy.yaml | 2 +- .../sglang/disagg-gb200/deploy.yaml | 2 +- recipes/glm-5-nvfp4/sglang/disagg/deploy.yaml | 2 +- .../glm-5-nvfp4/sglang/disagg/efa/deploy.yaml | 2 +- recipes/gpt-oss-120b/trtllm/agg/deploy.yaml | 2 +- .../gpt-oss-120b/trtllm/disagg/deploy.yaml | 2 +- .../kimi-k2.5/tokenspeed/agg/nvidia/README.md | 2 +- .../tokenspeed/agg/nvidia/deploy.yaml | 2 +- .../trtllm/agg-eagle-kv-router/deploy.yaml | 2 +- .../trtllm/agg-eagle-round-robin/deploy.yaml | 2 +- .../trtllm/agg-round-robin/deploy.yaml | 2 +- .../trtllm/disagg-eagle-kv-router/deploy.yaml | 2 +- .../vllm/agg-b200-agentic/deploy.yaml | 2 +- .../kimi-k2.6/vllm/agg-b200-chat/deploy.yaml | 2 +- .../vllm/agg-h200-agentic/deploy.yaml | 2 +- .../kimi-k2.6/vllm/agg-h200-chat/deploy.yaml | 2 +- recipes/llama-3-70b/vllm/agg/gaie/deploy.yaml | 2 +- .../vllm/disagg-single-node/gaie/deploy.yaml | 4 +- recipes/nemotron-3-nano-omni/Dockerfile | 2 +- recipes/nemotron-3-nano-omni/README.md | 2 +- .../nemotron-3-nano-omni/vllm/agg/deploy.yaml | 2 +- .../sglang/agg/deploy.yaml | 2 +- .../sglang/disagg/deploy.yaml | 2 +- .../trtllm/disagg/deploy.yaml | 2 +- .../nemotron-3-super-fp8/vllm/agg/deploy.yaml | 2 +- .../vllm/agg-b200-agentic/deploy.yaml | 2 +- .../vllm/agg-b200-chat/deploy.yaml | 2 +- .../vllm/agg-h200-agentic/deploy.yaml | 2 +- .../vllm/agg-h200-chat/deploy.yaml | 2 +- .../vllm/agg-b200-agentic-mtp/deploy.yaml | 4 +- .../vllm/agg-b200-agentic-nomtp/deploy.yaml | 4 +- .../vllm/agg-b200-chat-mtp/deploy.yaml | 4 +- .../vllm/agg-b200-chat-nomtp/deploy.yaml | 4 +- .../vllm/agg-h200-agentic-mtp/deploy.yaml | 4 +- .../vllm/agg-h200-agentic-nomtp/deploy.yaml | 4 +- .../vllm/agg-h200-chat-mtp/deploy.yaml | 4 +- .../vllm/agg-h200-chat-nomtp/deploy.yaml | 4 +- .../vllm/disagg-b200-agentic/deploy.yaml | 4 +- recipes/qwen3-0.6b/vllm/agg/gaie/deploy.yaml | 2 +- .../trtllm/agg/blackwell/deploy.yaml | 2 +- .../trtllm/agg/hopper/deploy.yaml | 2 +- .../trtllm/disagg/blackwell/deploy.yaml | 2 +- .../trtllm/disagg/hopper/deploy.yaml | 2 +- recipes/qwen3-32b-fp8/trtllm/agg/deploy.yaml | 2 +- .../qwen3-32b-fp8/trtllm/disagg/deploy.yaml | 2 +- recipes/qwen3-32b/vllm/agg-kvbm/deploy.yaml | 2 +- .../vllm/agg-round-robin/deploy.yaml | 2 +- .../vllm/disagg-kv-router/deploy.yaml | 2 +- tests/conftest.py | 2 +- tests/fault_tolerance/deploy/README.md | 2 +- tests/fault_tolerance/deploy/scenarios.py | 2 +- tests/fault_tolerance/etcd_ha/utils.py | 2 +- tests/frontend/realtime_echo_worker.py | 2 +- tests/frontend/test_prepost.py | 2 +- tests/frontend/test_prepost_mistral.py | 2 +- tests/frontend/test_realtime_omni_bridge.py | 2 +- tests/frontend/test_realtime_python_bridge.py | 2 +- tests/frontend/test_tool_calling_sglang.py | 8 ++-- .../test_consolidator_router_e2e.py | 4 +- .../test_determinism_disagg.py | 2 +- tests/lmcache/deploy-baseline-dynamo-disag.sh | 2 +- tests/lmcache/deploy-baseline-dynamo.sh | 2 +- .../deploy-lmcache_enabled-dynamo-disag.sh | 2 +- .../lmcache/deploy-lmcache_enabled-dynamo.sh | 2 +- tests/mm_router/test_mm_router_e2e.py | 4 +- ...test_router_rust_mm_frontend_decode_e2e.py | 4 +- .../test_router_rust_mm_router_e2e.py | 4 +- tests/mm_router/test_vllm_mm_router_e2e.py | 4 +- tests/router/helper.py | 2 +- tests/router/router_process.py | 4 +- tests/router/test_router_e2e_with_sglang.py | 2 +- tests/router/test_router_e2e_with_trtllm.py | 2 +- tests/router/test_router_e2e_with_vllm.py | 2 +- .../serve/launch/agg_raw_embeddings_llava.sh | 2 +- tests/serve/launch/multi_node_tp_headless.sh | 2 +- tests/serve/launch/template_verifier.sh | 4 +- tests/utils/managed_process.py | 8 ++-- 348 files changed, 561 insertions(+), 546 deletions(-) create mode 100644 dingo/__init__.py rename {components/src/dynamo => dingo}/frontend/CLAUDE.md (91%) rename {components/src/dynamo => dingo}/frontend/README.md (100%) rename {components/src/dynamo => dingo}/frontend/__init__.py (100%) rename {components/src/dynamo => dingo}/frontend/__main__.py (82%) rename {components/src/dynamo => dingo}/frontend/frontend_args.py (100%) rename {components/src/dynamo => dingo}/frontend/main.py (98%) rename {components/src/dynamo => dingo}/frontend/prepost.py (100%) rename {components/src/dynamo => dingo}/frontend/sglang_prepost.py (100%) rename {components/src/dynamo => dingo}/frontend/sglang_processor.py (99%) rename {components/src/dynamo => dingo}/frontend/tests/FRONTEND_CASES.md (94%) rename {components/src/dynamo => dingo}/frontend/tests/_routed_engine_fakes.py (100%) rename {components/src/dynamo => dingo}/frontend/tests/test_frontend_utils.py (98%) rename {components/src/dynamo => dingo}/frontend/tests/test_multimodal_utils.py (98%) rename {components/src/dynamo => dingo}/frontend/tests/test_sglang_multimodal_prepost.py (98%) rename {components/src/dynamo => dingo}/frontend/tests/test_sglang_processor_api.py (99%) rename {components/src/dynamo => dingo}/frontend/tests/test_sglang_processor_metrics_unit.py (97%) rename {components/src/dynamo => dingo}/frontend/tests/test_sglang_processor_unit.py (99%) rename {components/src/dynamo => dingo}/frontend/tests/test_sglang_tool_calls.py (99%) rename {components/src/dynamo => dingo}/frontend/tests/test_vllm_processor_unit.py (97%) rename {components/src/dynamo => dingo}/frontend/utils.py (100%) rename {components/src/dynamo => dingo}/frontend/vllm_processor.py (99%) diff --git a/README.md b/README.md index be9711fa4754..3579921f1f77 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ the Gateway API setup, supported features, and configuration. docker run --gpus all --network host --rm -it nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.1 # Inside the container — start frontend and worker -python3 -m dynamo.frontend --http-port 8000 --discovery-backend file > /dev/null 2>&1 & +python3 -m dingo.frontend --http-port 8000 --discovery-backend file > /dev/null 2>&1 & python3 -m dynamo.sglang --model-path Qwen/Qwen3-0.6B --discovery-backend file & # Send a request diff --git a/README.zh-CN.md b/README.zh-CN.md index 66718e7d6355..1e6491a02507 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -116,7 +116,7 @@ limitations under the License. docker run --gpus all --network host --rm -it nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.1 # 在容器内启动 frontend 和 worker -python3 -m dynamo.frontend --http-port 8000 --discovery-backend file > /dev/null 2>&1 & +python3 -m dingo.frontend --http-port 8000 --discovery-backend file > /dev/null 2>&1 & python3 -m dynamo.sglang --model-path Qwen/Qwen3-0.6B --discovery-backend file & # 发送请求 diff --git a/benchmarks/frontend/README.md b/benchmarks/frontend/README.md index f658e7722759..d0a2d1a18f6c 100644 --- a/benchmarks/frontend/README.md +++ b/benchmarks/frontend/README.md @@ -43,7 +43,7 @@ Local mode starts a mocker backend and frontend process on the current machine, **Prerequisites:** -- `dynamo.mocker` and `dynamo.frontend` installed (from the Dynamo repo) +- `dynamo.mocker` and `dingo.frontend` installed (from the Dynamo repo) - `aiperf` installed and on `$PATH` - A HuggingFace model accessible locally (default: `Qwen/Qwen3-0.6B`) diff --git a/benchmarks/frontend/dgd/templates/mocker.yaml b/benchmarks/frontend/dgd/templates/mocker.yaml index ab142a08df43..8cacdddd3951 100644 --- a/benchmarks/frontend/dgd/templates/mocker.yaml +++ b/benchmarks/frontend/dgd/templates/mocker.yaml @@ -39,7 +39,7 @@ ${FRONTEND_IMAGE_PULL_SECRETS_BLOCK} - /bin/sh - -c args: - - python3 -m dynamo.frontend --router-mode ${ROUTER_MODE} --http-port ${FRONTEND_PORT} + - python3 -m dingo.frontend --router-mode ${ROUTER_MODE} --http-port ${FRONTEND_PORT} env: - name: DYN_TOKENIZER value: "${DYN_TOKENIZER}" diff --git a/benchmarks/frontend/dgd/templates/vllm-gpt-oss-20b.yaml b/benchmarks/frontend/dgd/templates/vllm-gpt-oss-20b.yaml index 199c6c27f612..1f64048204ae 100644 --- a/benchmarks/frontend/dgd/templates/vllm-gpt-oss-20b.yaml +++ b/benchmarks/frontend/dgd/templates/vllm-gpt-oss-20b.yaml @@ -55,7 +55,7 @@ ${FRONTEND_IMAGE_PULL_SECRETS_BLOCK} - /bin/sh - -c args: - - python3 -m dynamo.frontend --router-mode ${ROUTER_MODE} --http-port ${FRONTEND_PORT} + - python3 -m dingo.frontend --router-mode ${ROUTER_MODE} --http-port ${FRONTEND_PORT} env: - name: DYN_TOKENIZER value: "${DYN_TOKENIZER}" diff --git a/benchmarks/frontend/dgd/templates/vllm.yaml b/benchmarks/frontend/dgd/templates/vllm.yaml index 5fc9c59a343d..bf861256dba1 100644 --- a/benchmarks/frontend/dgd/templates/vllm.yaml +++ b/benchmarks/frontend/dgd/templates/vllm.yaml @@ -38,7 +38,7 @@ ${FRONTEND_IMAGE_PULL_SECRETS_BLOCK} - /bin/sh - -c args: - - python3 -m dynamo.frontend --router-mode ${ROUTER_MODE} --http-port ${FRONTEND_PORT} + - python3 -m dingo.frontend --router-mode ${ROUTER_MODE} --http-port ${FRONTEND_PORT} env: - name: DYN_TOKENIZER value: "${DYN_TOKENIZER}" diff --git a/benchmarks/frontend/scripts/bpf/README.md b/benchmarks/frontend/scripts/bpf/README.md index 5164d2ec9c72..22c5b41ec144 100644 --- a/benchmarks/frontend/scripts/bpf/README.md +++ b/benchmarks/frontend/scripts/bpf/README.md @@ -28,7 +28,7 @@ After granting capabilities, bpftrace runs **without sudo**. ```bash # Get the frontend PID from a running capture -FRONTEND_PID=$(pgrep -f "dynamo.frontend" | head -1) +FRONTEND_PID=$(pgrep -f "dingo.frontend" | head -1) # Run a single script ./run.sh --pid $FRONTEND_PID offcputime diff --git a/benchmarks/frontend/scripts/flamegraph/README.md b/benchmarks/frontend/scripts/flamegraph/README.md index 9514df3efded..cb14fce89b71 100644 --- a/benchmarks/frontend/scripts/flamegraph/README.md +++ b/benchmarks/frontend/scripts/flamegraph/README.md @@ -16,7 +16,7 @@ Scripts for generating CPU, off-CPU, and differential flame graph SVGs from the ```bash # Get the frontend PID from a running capture -FRONTEND_PID=$(pgrep -f "dynamo.frontend" | head -1) +FRONTEND_PID=$(pgrep -f "dingo.frontend" | head -1) # CPU flame graph (30s sample) ./cpu_flamegraph.sh --pid $FRONTEND_PID --duration 30 diff --git a/benchmarks/frontend/scripts/run_perf.sh b/benchmarks/frontend/scripts/run_perf.sh index 8d4cdc5105a1..cb22ffde9a63 100755 --- a/benchmarks/frontend/scripts/run_perf.sh +++ b/benchmarks/frontend/scripts/run_perf.sh @@ -15,7 +15,7 @@ # Output: artifacts/obs_YYYYMMDD_HHMMSS/ with subdirs for each data source. # # Prerequisites: -# - dynamo.mocker and dynamo.frontend installed +# - dynamo.mocker and dingo.frontend installed # - aiperf installed # - Optional: nsys, perf, bpftrace, flamegraph tools (auto-detected) @@ -462,7 +462,7 @@ if [[ "$HAS_NSYS" == true ]]; then --cpuctxsw=none \ --output="${OUTPUT_DIR}/nsys/frontend" \ --force-overwrite=true \ - python -m dynamo.frontend \ + python -m dingo.frontend \ > "$OUTPUT_DIR/logs/frontend.log" 2>&1 & NSYS_WRAPPER_PID=$! ALL_PIDS+=($NSYS_WRAPPER_PID) @@ -471,7 +471,7 @@ if [[ "$HAS_NSYS" == true ]]; then FRONTEND_PID="" for _try in $(seq 1 30); do sleep 1 - _child=$(pgrep -P "$NSYS_WRAPPER_PID" -f "python.*dynamo.frontend" 2>/dev/null | head -1 || true) + _child=$(pgrep -P "$NSYS_WRAPPER_PID" -f "python.*dingo.frontend" 2>/dev/null | head -1 || true) if [[ -n "$_child" ]]; then FRONTEND_PID="$_child" break @@ -485,7 +485,7 @@ if [[ "$HAS_NSYS" == true ]]; then echo " nsys wrapper PID: $NSYS_WRAPPER_PID" echo " Frontend PID: $FRONTEND_PID" else - env "${FRONTEND_ENV[@]}" python -m dynamo.frontend \ + env "${FRONTEND_ENV[@]}" python -m dingo.frontend \ > "$OUTPUT_DIR/logs/frontend.log" 2>&1 & FRONTEND_PID=$! ALL_PIDS+=($FRONTEND_PID) diff --git a/benchmarks/frontend/scripts/scaling-test.md b/benchmarks/frontend/scripts/scaling-test.md index 6e551d1d5281..1baa461b94b1 100644 --- a/benchmarks/frontend/scripts/scaling-test.md +++ b/benchmarks/frontend/scripts/scaling-test.md @@ -126,7 +126,7 @@ spec: mainContainer: image: command: ["/bin/sh", "-c"] - args: ["python3 -m dynamo.frontend --router-mode round-robin --http-port 8000"] + args: ["python3 -m dingo.frontend --router-mode round-robin --http-port 8000"] env: - name: DYN_TOKENIZER value: "default" diff --git a/benchmarks/multimodal/sweep/experiments/embedding_cache/trtllm_e_pd.sh b/benchmarks/multimodal/sweep/experiments/embedding_cache/trtllm_e_pd.sh index 3f254ee899c8..a16f8fe7b0f2 100755 --- a/benchmarks/multimodal/sweep/experiments/embedding_cache/trtllm_e_pd.sh +++ b/benchmarks/multimodal/sweep/experiments/embedding_cache/trtllm_e_pd.sh @@ -36,8 +36,8 @@ cleanup() { trap cleanup EXIT INT TERM # run frontend -# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) -python3 -m dynamo.frontend & +# dingo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) +python3 -m dingo.frontend & DYNAMO_PID=$! # run encode worker (vision encoder on GPU 0) diff --git a/benchmarks/router/README.md b/benchmarks/router/README.md index 3fd4e7636f88..1b3779aabddd 100644 --- a/benchmarks/router/README.md +++ b/benchmarks/router/README.md @@ -125,7 +125,7 @@ python -m dynamo.mocker --model-path "$MODEL" \ # Terminal 3: Frontend with KV router # --model-path must be the on-disk snapshot directory MODEL_PATH=$(find ~/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots -mindepth 1 -maxdepth 1 -type d | head -1) -python -m dynamo.frontend --namespace "$NAMESPACE" \ +python -m dingo.frontend --namespace "$NAMESPACE" \ --model-name "$MODEL" --model-path "$MODEL_PATH" \ --router-mode kv --http-port 8000 --kv-cache-block-size 16 ``` @@ -148,7 +148,7 @@ In a **new terminal**, launch the Dynamo router using the Python CLI: # Explicitly set NATS server for KV event publishing export NATS_SERVER="${NATS_SERVER:-nats://localhost:4222}" -python -m dynamo.frontend \ +python -m dingo.frontend \ --router-mode kv \ --http-port 8000 ``` @@ -163,7 +163,7 @@ and is not needed for these benchmark commands. To see all available router arguments, run: ```bash -python -m dynamo.frontend --help +python -m dingo.frontend --help ``` For detailed explanations of router arguments (especially KV cache routing parameters), see the [Router Guide](../../docs/components/router/router-guide.md). @@ -172,7 +172,7 @@ For detailed explanations of router arguments (especially KV cache routing param > If you're unsure whether your backend engines correctly emit KV events for certain models (e.g., hybrid models like gpt-oss or nemotron nano 2), use the `--no-router-kv-events` flag to disable KV event tracking and use approximate KV indexing instead: > > ```bash -> python -m dynamo.frontend \ +> python -m dingo.frontend \ > --router-mode kv \ > --http-port 8000 \ > --no-router-kv-events @@ -298,7 +298,7 @@ The router queue is enabled by default, but the default threshold is conservativ ```bash # Launch the router with a sensitive priority queue threshold. -python -m dynamo.frontend \ +python -m dingo.frontend \ --router-mode kv \ --router-queue-threshold 0.0 ``` diff --git a/components/README.md b/components/README.md index 786ff584d3bf..a066e4e3f767 100644 --- a/components/README.md +++ b/components/README.md @@ -32,7 +32,7 @@ Dynamo supports multiple inference engines, each with their own deployment confi Each engine provides launch and deploy scripts for different deployment patterns in the [examples](../examples/backends/) folder. -### [Frontend](src/dynamo/frontend/) +### [Frontend](../dingo/frontend/) The frontend component provides the HTTP API layer and request processing: diff --git a/components/src/dynamo/common/configuration/groups/kv_router_args.py b/components/src/dynamo/common/configuration/groups/kv_router_args.py index 4a37d437716b..42cdaec4b429 100644 --- a/components/src/dynamo/common/configuration/groups/kv_router_args.py +++ b/components/src/dynamo/common/configuration/groups/kv_router_args.py @@ -4,7 +4,7 @@ """Shared KV router configuration ArgGroup. Defines the shared KvRouterConfig parameters once so that both -``dynamo.frontend`` and ``dynamo.router`` can reuse them without duplication. +``dingo.frontend`` and ``dynamo.router`` can reuse them without duplication. Field names on ``KvRouterConfigBase`` match the ``KvRouterConfig`` Python constructor kwargs 1:1, so ``kv_router_kwargs()`` returns a dict that can be unpacked directly into ``KvRouterConfig(**config.kv_router_kwargs())``. diff --git a/components/src/dynamo/common/configuration/groups/router_args.py b/components/src/dynamo/common/configuration/groups/router_args.py index 1c9b334bcd94..51ab63c15a16 100644 --- a/components/src/dynamo/common/configuration/groups/router_args.py +++ b/components/src/dynamo/common/configuration/groups/router_args.py @@ -4,7 +4,7 @@ """Shared router configuration ArgGroup. Defines the router configuration parameters once so that both -``dynamo.frontend`` and other components can reuse them without duplication. +``dingo.frontend`` and other components can reuse them without duplication. Field names on ``RouterConfigBase`` match the ``RouterConfig`` Python constructor kwargs 1:1 (for the non-positional args), so ``router_kwargs()`` returns a dict that can be unpacked into diff --git a/components/src/dynamo/common/tests/configuration/test_kv_router_args.py b/components/src/dynamo/common/tests/configuration/test_kv_router_args.py index 7937afcdbe52..aa25de62dd83 100644 --- a/components/src/dynamo/common/tests/configuration/test_kv_router_args.py +++ b/components/src/dynamo/common/tests/configuration/test_kv_router_args.py @@ -14,7 +14,7 @@ KvRouterArgGroup, KvRouterConfigBase, ) -from dynamo.frontend.frontend_args import FrontendArgGroup, FrontendConfig +from dingo.frontend.frontend_args import FrontendArgGroup, FrontendConfig pytestmark = [pytest.mark.pre_merge, pytest.mark.unit, pytest.mark.gpu_0] diff --git a/components/src/dynamo/planner/tests/manual/perf_test_configs/agg_8b.yaml b/components/src/dynamo/planner/tests/manual/perf_test_configs/agg_8b.yaml index af3ea6395cac..c85b336f0095 100644 --- a/components/src/dynamo/planner/tests/manual/perf_test_configs/agg_8b.yaml +++ b/components/src/dynamo/planner/tests/manual/perf_test_configs/agg_8b.yaml @@ -43,7 +43,7 @@ spec: - /bin/sh - -c args: - - "python3 -m dynamo.frontend --http-port 8000" + - "python3 -m dingo.frontend --http-port 8000" VllmDecodeWorker: envFromSecret: hf-token-secret componentType: worker diff --git a/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_2p2d.yaml b/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_2p2d.yaml index 77cbbe97271c..3b2f36e2ba49 100644 --- a/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_2p2d.yaml +++ b/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_2p2d.yaml @@ -43,7 +43,7 @@ spec: - /bin/sh - -c args: - - "python3 -m dynamo.frontend --http-port 8000 --kv-cache-block-size 128" + - "python3 -m dingo.frontend --http-port 8000 --kv-cache-block-size 128" VllmDecodeWorker: envFromSecret: hf-token-secret componentType: worker diff --git a/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_3p1d.yaml b/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_3p1d.yaml index 9c522d92c689..d8b064e0ee71 100644 --- a/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_3p1d.yaml +++ b/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_3p1d.yaml @@ -43,7 +43,7 @@ spec: - /bin/sh - -c args: - - "python3 -m dynamo.frontend --http-port 8000 --kv-cache-block-size 128" + - "python3 -m dingo.frontend --http-port 8000 --kv-cache-block-size 128" VllmDecodeWorker: envFromSecret: hf-token-secret componentType: worker diff --git a/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_planner.yaml b/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_planner.yaml index 96b544262cc6..984d665f122a 100644 --- a/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_planner.yaml +++ b/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_planner.yaml @@ -46,7 +46,7 @@ spec: - /bin/sh - -c args: - - "python3 -m dynamo.frontend --http-port 8000 --kv-cache-block-size 128 --router-mode kv --router-kv-overlap-score-credit 0.0 --router-temperature 0.0 --no-router-kv-events" + - "python3 -m dingo.frontend --http-port 8000 --kv-cache-block-size 128 --router-mode kv --router-kv-overlap-score-credit 0.0 --router-temperature 0.0 --no-router-kv-events" Planner: envFromSecret: hf-token-secret componentType: planner diff --git a/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_tp2.yaml b/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_tp2.yaml index 775196fb7c48..0cb41cd7c996 100644 --- a/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_tp2.yaml +++ b/components/src/dynamo/planner/tests/manual/perf_test_configs/disagg_8b_tp2.yaml @@ -43,7 +43,7 @@ spec: - /bin/sh - -c args: - - "python3 -m dynamo.frontend --http-port 8000 --kv-cache-block-size 128" + - "python3 -m dingo.frontend --http-port 8000 --kv-cache-block-size 128" VllmDecodeWorker: envFromSecret: hf-token-secret componentType: worker diff --git a/components/src/dynamo/planner/tests/manual/scaling/disagg_planner_load.yaml b/components/src/dynamo/planner/tests/manual/scaling/disagg_planner_load.yaml index 5fdb664c8e93..b74b3bccb686 100644 --- a/components/src/dynamo/planner/tests/manual/scaling/disagg_planner_load.yaml +++ b/components/src/dynamo/planner/tests/manual/scaling/disagg_planner_load.yaml @@ -18,7 +18,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - kv Planner: diff --git a/components/src/dynamo/profiler/utils/config_modifiers/protocol.py b/components/src/dynamo/profiler/utils/config_modifiers/protocol.py index 78c827f45d2b..5e14d28a94e9 100644 --- a/components/src/dynamo/profiler/utils/config_modifiers/protocol.py +++ b/components/src/dynamo/profiler/utils/config_modifiers/protocol.py @@ -333,7 +333,7 @@ def _update_frontend_cli( # If operator defaults are being used (no command/args), we must provide full CLI. if not c.command and not c.args: - c.command = ["python3", "-m", "dynamo.frontend"] + c.command = ["python3", "-m", "dingo.frontend"] c.args = [] def _patch(tokens: list[str]) -> list[str]: diff --git a/components/src/dynamo/router/CLAUDE.md b/components/src/dynamo/router/CLAUDE.md index e1a3d8980012..66151cc1df13 100644 --- a/components/src/dynamo/router/CLAUDE.md +++ b/components/src/dynamo/router/CLAUDE.md @@ -11,7 +11,7 @@ logic but do not share the same serialization, RPC, or process boundaries. 1. **Integrated Rust frontend** ``` - dynamo.frontend + dingo.frontend -> Rust OpenAIPreprocessor -> in-process Rust KvPushRouter / KvRouter -> worker @@ -21,10 +21,10 @@ logic but do not share the same serialization, RPC, or process boundaries. `--router-mode kv` uses an in-process Rust router. There is no router RPC hop. -2. **Python chat processor inside `dynamo.frontend`** +2. **Python chat processor inside `dingo.frontend`** ``` - dynamo.frontend + dingo.frontend -> Python VllmProcessor or SglangProcessor -> PyO3 RoutedEngine -> in-process Rust KvPushRouter / KvRouter diff --git a/components/src/dynamo/router/README.md b/components/src/dynamo/router/README.md index a5a18954c448..ca9f28e76a10 100644 --- a/components/src/dynamo/router/README.md +++ b/components/src/dynamo/router/README.md @@ -51,7 +51,7 @@ For an integrated frontend disaggregated example, see [`examples/backends/vllm/l ```bash # Start frontend router for decode workers -python -m dynamo.frontend \ +python -m dingo.frontend \ --router-mode kv \ --http-port 8000 \ --router-kv-overlap-score-credit 0 # Pure load balancing for decode diff --git a/components/src/dynamo/router/args.py b/components/src/dynamo/router/args.py index 603334c3c3e8..83eb808b53d3 100644 --- a/components/src/dynamo/router/args.py +++ b/components/src/dynamo/router/args.py @@ -105,7 +105,7 @@ def add_arguments(self, parser) -> None: dest="serve_indexer", ) - # KV router options (shared with dynamo.frontend) + # KV router options (shared with dingo.frontend) KvRouterArgGroup().add_arguments(parser) AicPerfArgGroup().add_arguments(parser) diff --git a/components/src/dynamo/thunderagent_router/README.md b/components/src/dynamo/thunderagent_router/README.md index 7cf2d24f36aa..4aafb3503fee 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/components/src/dynamo/thunderagent_router/README.md @@ -53,7 +53,7 @@ python -m dynamo.thunderagent_router \ # 3. Start the frontend (any router mode -- the frontend just needs to find # a model handler, which our service registered) -python -m dynamo.frontend --router-mode round-robin --router-reset-states +python -m dingo.frontend --router-mode round-robin --router-reset-states ``` The control-loop knobs (`--pause-threshold`, `--pause-target`, diff --git a/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh b/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh index 10b0e90a36a5..fe8e06120a4d 100755 --- a/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh +++ b/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh @@ -39,7 +39,7 @@ python -m dynamo.thunderagent_router \ --router-block-size "$BLOCK_SIZE" & # Frontend (round-robin; the router owns scheduling and registered the model). -python -m dynamo.frontend \ +python -m dingo.frontend \ --http-port "$HTTP_PORT" \ --router-mode round-robin \ --router-reset-states & diff --git a/components/src/dynamo/vllm/tests/test_vllm_renderer_api.py b/components/src/dynamo/vllm/tests/test_vllm_renderer_api.py index 6d47b97dc056..04609c505245 100755 --- a/components/src/dynamo/vllm/tests/test_vllm_renderer_api.py +++ b/components/src/dynamo/vllm/tests/test_vllm_renderer_api.py @@ -6,8 +6,8 @@ Unit tests to verify vLLM renderer API compatibility. These tests lock the vLLM API surface used by: -- components/src/dynamo/frontend/vllm_processor.py -- components/src/dynamo/frontend/prepost.py +- dingo/frontend/vllm_processor.py +- dingo/frontend/prepost.py If vLLM changes these APIs, these tests should fail early during upgrade. """ @@ -90,8 +90,8 @@ def test_chat_completion_request_has_required_fields(self): assert not missing, ( "ChatCompletionRequest fields changed!\n" f"Missing required fields: {missing}\n" - "Update components/src/dynamo/frontend/prepost.py and " - "components/src/dynamo/frontend/vllm_processor.py to match new vLLM API." + "Update dingo/frontend/prepost.py and " + "dingo/frontend/vllm_processor.py to match new vLLM API." ) def test_chat_completion_request_model_validate_contract(self): @@ -130,7 +130,7 @@ def test_delta_tool_call_fields(self): assert not missing, ( "DeltaToolCall fields changed!\n" f"Missing required fields: {missing}\n" - "Update tool-call merge logic in components/src/dynamo/frontend/prepost.py" + "Update tool-call merge logic in dingo/frontend/prepost.py" ) def test_delta_message_fields(self): @@ -152,7 +152,7 @@ def test_delta_message_fields(self): "DeltaMessage fields changed!\n" f"Missing required fields: {missing}\n" "Update streaming post-processing logic in " - "components/src/dynamo/frontend/prepost.py" + "dingo/frontend/prepost.py" ) def test_sampling_params_annotations_and_output_kind(self): @@ -195,7 +195,7 @@ def test_sampling_params_annotations_and_output_kind(self): "SamplingParams annotations changed!\n" f"Missing required fields: {missing}\n" "Update sampling field projection in " - "components/src/dynamo/frontend/vllm_processor.py" + "dingo/frontend/vllm_processor.py" ) assert RequestOutputKind.DELTA.name == "DELTA" @@ -226,7 +226,7 @@ def test_tokens_prompt_supports_optional_keys_used_by_frontend(self): "TokensPrompt shape changed!\n" f"Missing keys: {missing}\n" "Update prompt construction in " - "components/src/dynamo/frontend/vllm_processor.py" + "dingo/frontend/vllm_processor.py" ) def test_input_processor_method_signatures(self): @@ -260,7 +260,7 @@ def test_input_processor_method_signatures(self): assert list(assign_request_id_sig.parameters) == ["request"], ( "InputProcessor.assign_request_id signature changed; " "update request-id assignment in " - "components/src/dynamo/frontend/vllm_processor.py" + "dingo/frontend/vllm_processor.py" ) def test_input_processor_attributes(self): @@ -275,13 +275,13 @@ def test_input_processor_attributes(self): assert "self.renderer" in init_source, ( "InputProcessor.__init__ no longer initializes 'renderer'; " "update preprocess_chat_request call in " - "components/src/dynamo/frontend/vllm_processor.py" + "dingo/frontend/vllm_processor.py" ) assert hasattr(InputProcessor, "get_tokenizer"), ( "InputProcessor no longer has 'get_tokenizer' method; " "update EngineFactory in " - "components/src/dynamo/frontend/vllm_processor.py" + "dingo/frontend/vllm_processor.py" ) assert callable( getattr(InputProcessor, "get_tokenizer") @@ -300,7 +300,7 @@ def test_output_processor_api_shape(self): init_params = list(init_sig.parameters) assert init_params[:3] == ["self", "tokenizer", "log_stats"], ( "OutputProcessor.__init__ signature changed; " - "update construction in components/src/dynamo/frontend/vllm_processor.py" + "update construction in dingo/frontend/vllm_processor.py" ) assert "stream_interval" in init_params @@ -318,7 +318,7 @@ def test_output_processor_api_shape(self): process_outputs_params = list(process_outputs_sig.parameters) assert process_outputs_params[:2] == ["self", "engine_core_outputs"], ( "OutputProcessor.process_outputs signature changed; " - "update processing in components/src/dynamo/frontend/vllm_processor.py" + "update processing in dingo/frontend/vllm_processor.py" ) abort_requests_sig = inspect.signature(OutputProcessor.abort_requests) @@ -348,7 +348,7 @@ def test_output_processor_request_states_attribute(self): init_source = inspect.getsource(OutputProcessor.__init__) assert "request_states" in init_source, ( "OutputProcessor.__init__ no longer initializes 'request_states'; " - "update cleanup logic in components/src/dynamo/frontend/vllm_processor.py" + "update cleanup logic in dingo/frontend/vllm_processor.py" ) def test_engine_core_struct_contract(self): @@ -397,7 +397,7 @@ def test_engine_core_struct_contract(self): "EngineCoreRequest fields changed!\n" f"Expected variants: {valid_request_fields}\n" f"Actual: {actual_request_fields}\n" - "Update request construction in components/src/dynamo/frontend/vllm_processor.py" + "Update request construction in dingo/frontend/vllm_processor.py" ) base_output_fields = ( @@ -454,7 +454,7 @@ def test_engine_core_struct_contract(self): "EngineCoreOutput fields changed!\n" f"Expected variants: {valid_output_fields}\n" f"Actual: {actual_output_fields}\n" - "Update output mapping in components/src/dynamo/frontend/vllm_processor.py" + "Update output mapping in dingo/frontend/vllm_processor.py" ) req_config = getattr(EngineCoreRequest, "__struct_config__", None) @@ -523,11 +523,11 @@ def test_tool_and_reasoning_parser_manager_contract(self): assert list(tool_parser_sig.parameters) == ["name"], ( "ToolParserManager.get_tool_parser signature changed; " - "update EngineFactory in components/src/dynamo/frontend/vllm_processor.py" + "update EngineFactory in dingo/frontend/vllm_processor.py" ) assert list(reasoning_parser_sig.parameters) == ["name"], ( "ReasoningParserManager.get_reasoning_parser signature changed; " - "update EngineFactory in components/src/dynamo/frontend/vllm_processor.py" + "update EngineFactory in dingo/frontend/vllm_processor.py" ) def test_tool_parser_method_signatures(self): @@ -540,7 +540,7 @@ def test_tool_parser_method_signatures(self): assert hasattr(ToolParser, "adjust_request"), ( "ToolParser no longer has 'adjust_request'; " "update preprocess_chat_request in " - "components/src/dynamo/frontend/prepost.py" + "dingo/frontend/prepost.py" ) adjust_sig = inspect.signature(ToolParser.adjust_request) adjust_params = list(adjust_sig.parameters) @@ -552,7 +552,7 @@ def test_tool_parser_method_signatures(self): assert hasattr(ToolParser, "extract_tool_calls_streaming"), ( "ToolParser no longer has 'extract_tool_calls_streaming'; " "update StreamingPostProcessor in " - "components/src/dynamo/frontend/prepost.py" + "dingo/frontend/prepost.py" ) extract_sig = inspect.signature(ToolParser.extract_tool_calls_streaming) extract_params = list(extract_sig.parameters) @@ -582,7 +582,7 @@ def test_reasoning_parser_method_signatures(self): assert hasattr(ReasoningParser, "extract_reasoning_streaming"), ( "ReasoningParser no longer has 'extract_reasoning_streaming'; " "update StreamingPostProcessor in " - "components/src/dynamo/frontend/prepost.py" + "dingo/frontend/prepost.py" ) extract_sig = inspect.signature(ReasoningParser.extract_reasoning_streaming) extract_params = list(extract_sig.parameters) @@ -603,7 +603,7 @@ def test_reasoning_parser_method_signatures(self): assert hasattr(ReasoningParser, "is_reasoning_end_streaming"), ( "ReasoningParser no longer has 'is_reasoning_end_streaming'; " "update StreamingPostProcessor in " - "components/src/dynamo/frontend/prepost.py" + "dingo/frontend/prepost.py" ) end_sig = inspect.signature(ReasoningParser.is_reasoning_end_streaming) end_params = list(end_sig.parameters) @@ -615,7 +615,7 @@ def test_reasoning_parser_method_signatures(self): assert hasattr(ReasoningParser, "extract_reasoning"), ( "ReasoningParser no longer has 'extract_reasoning'; " "update StreamingPostProcessor in " - "components/src/dynamo/frontend/prepost.py" + "dingo/frontend/prepost.py" ) batch_sig = inspect.signature(ReasoningParser.extract_reasoning) batch_params = list(batch_sig.parameters) diff --git a/container/README.md b/container/README.md index e172352ea998..ce8f3e1567e1 100644 --- a/container/README.md +++ b/container/README.md @@ -456,7 +456,7 @@ dev/sanity_check.py # 4. Run inference (requires both frontend and backend) # Start frontend -python -m dynamo.frontend & +python -m dingo.frontend & # Start backend (vLLM example) python -m dynamo.vllm --model Qwen/Qwen3-0.6B --gpu-memory-utilization 0.20 & @@ -485,7 +485,7 @@ sudo chown -R dynamo:0 /opt/miniforge3/envs/sglang cargo build --locked --features dynamo-llm/block-manager --workspace # 3a. ai_dynamo_runtime (Rust bindings: dynamo._core) cd lib/bindings/python && maturin develop --uv && cd - -# 3b. ai-dynamo (Python namespace packages: dynamo.frontend, dynamo.sglang, ...) +# 3b. ai-dynamo (Python namespace packages: dingo.frontend, dynamo.sglang, ...) uv pip install --no-deps -e /workspace # 3c. NIXL python bindings (C++ libs are already baked in at /opt/intel/intel_nixl; # local-dev intentionally skips installing the wheel into the env) @@ -499,7 +499,7 @@ nats-server -js & etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://0.0.0.0:2379 --data-dir /tmp/etcd & # 6. Run inference (frontend + SGLang XPU backend) -python -m dynamo.frontend & +python -m dingo.frontend & python -m dynamo.sglang --model Qwen/Qwen3-0.6B --mem-fraction-static 0.20 & ``` @@ -553,7 +553,7 @@ dev/sanity_check.py --runtime-check-only python -m pytest tests/ # 7. (Optional) Start frontend and backend for interactive testing -python -m dynamo.frontend & +python -m dingo.frontend & # Start worker backend (choose one framework): # vLLM diff --git a/container/launch_message/frontend.txt b/container/launch_message/frontend.txt index 9ee67814ca02..9727c27e1799 100644 --- a/container/launch_message/frontend.txt +++ b/container/launch_message/frontend.txt @@ -61,6 +61,6 @@ Start mocker with custom configuration: --enable-prefix-caching Start frontend server: -> python -m dynamo.frontend --http-port 8000 +> python -m dingo.frontend --http-port 8000 diff --git a/container/launch_message/runtime.txt b/container/launch_message/runtime.txt index 4989705c5c7a..ac8d6999225d 100644 --- a/container/launch_message/runtime.txt +++ b/container/launch_message/runtime.txt @@ -38,7 +38,7 @@ This is a minimum runtime container for interacting with Dynamo via our CLI tools. Try the following to begin interacting with a model: -> python -m dynamo.frontend [--http-port 8000] +> python -m dingo.frontend [--http-port 8000] > python -m dynamo.{vllm,sglang,trtllm} --model Qwen/Qwen2.5-3B-Instruct To run more complete deployment examples, instances of etcd and nats need to be diff --git a/container/templates/frontend.Dockerfile b/container/templates/frontend.Dockerfile index 5c13c0664c62..c647cb3be3c1 100644 --- a/container/templates/frontend.Dockerfile +++ b/container/templates/frontend.Dockerfile @@ -84,6 +84,7 @@ COPY --chown=dynamo: benchmarks /workspace/benchmarks COPY --chown=dynamo: deploy /workspace/deploy COPY --chown=dynamo: dev /workspace/dev COPY --chown=dynamo: components/ /workspace/components/ +COPY --chown=dynamo: dingo/ /workspace/dingo/ COPY --chown=dynamo: recipes/ /workspace/recipes/ # Copy attribution files with correct ownership COPY --chown=dynamo: ATTRIBUTION* LICENSE /workspace/ diff --git a/container/templates/sglang_runtime.Dockerfile b/container/templates/sglang_runtime.Dockerfile index 02c3094659aa..8ec51462d0db 100644 --- a/container/templates/sglang_runtime.Dockerfile +++ b/container/templates/sglang_runtime.Dockerfile @@ -162,7 +162,7 @@ COPY --chmod=775 --chown=dynamo:0 examples /workspace/examples COPY --chmod=775 --chown=dynamo:0 deploy /workspace/deploy COPY --chmod=775 --chown=dynamo:0 dev /workspace/dev COPY --chmod=775 --chown=dynamo:0 components/src/dynamo/common /workspace/components/src/dynamo/common -COPY --chmod=775 --chown=dynamo:0 components/src/dynamo/frontend /workspace/components/src/dynamo/frontend +COPY --chmod=775 --chown=dynamo:0 dingo/frontend /workspace/dingo/frontend COPY --chmod=775 --chown=dynamo:0 components/src/dynamo/sglang /workspace/components/src/dynamo/sglang COPY --chmod=775 --chown=dynamo:0 components/src/dynamo/mocker /workspace/components/src/dynamo/mocker COPY --chmod=775 --chown=dynamo:0 recipes/ /workspace/recipes/ diff --git a/container/templates/trtllm_runtime.Dockerfile b/container/templates/trtllm_runtime.Dockerfile index 0e252f63c53c..9407adb57bb2 100644 --- a/container/templates/trtllm_runtime.Dockerfile +++ b/container/templates/trtllm_runtime.Dockerfile @@ -14,7 +14,7 @@ COPY --chmod=775 examples /workspace_src/examples COPY --chmod=775 deploy /workspace_src/deploy COPY --chmod=775 dev /workspace_src/dev COPY --chmod=775 components/src/dynamo/common /workspace_src/components/src/dynamo/common -COPY --chmod=775 components/src/dynamo/frontend /workspace_src/components/src/dynamo/frontend +COPY --chmod=775 dingo/frontend /workspace_src/dingo/frontend COPY --chmod=775 components/src/dynamo/trtllm /workspace_src/components/src/dynamo/trtllm COPY --chmod=775 components/src/dynamo/mocker /workspace_src/components/src/dynamo/mocker COPY --chmod=775 lib /workspace_src/lib diff --git a/container/templates/vllm_runtime.Dockerfile b/container/templates/vllm_runtime.Dockerfile index 2b922fa4e8f5..79cb7a27cef4 100644 --- a/container/templates/vllm_runtime.Dockerfile +++ b/container/templates/vllm_runtime.Dockerfile @@ -284,7 +284,7 @@ COPY --chmod=775 --chown=dynamo:0 tests /workspace/tests COPY --chmod=775 --chown=dynamo:0 examples /workspace/examples COPY --chmod=775 --chown=dynamo:0 dev /workspace/dev COPY --chmod=775 --chown=dynamo:0 components/src/dynamo/common /workspace/components/src/dynamo/common -COPY --chmod=775 --chown=dynamo:0 components/src/dynamo/frontend /workspace/components/src/dynamo/frontend +COPY --chmod=775 --chown=dynamo:0 dingo/frontend /workspace/dingo/frontend COPY --chmod=775 --chown=dynamo:0 components/src/dynamo/vllm /workspace/components/src/dynamo/vllm COPY --chown=dynamo:0 lib /workspace/lib diff --git a/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamocomponentdeployments.yaml b/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamocomponentdeployments.yaml index 64e4d1678d92..3adc156ad391 100644 --- a/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamocomponentdeployments.yaml @@ -10734,8 +10734,8 @@ spec: args: description: |- Args overrides the default frontend arguments. When specified, these replace - the default ["-m", "dynamo.frontend"] entirely. - For example, ["-m", "dynamo.frontend", "--router-mode", "direct"] for GAIE deployments. + the default ["-m", "dingo.frontend"] entirely. + For example, ["-m", "dingo.frontend", "--router-mode", "direct"] for GAIE deployments. items: type: string type: array diff --git a/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeployments.yaml b/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeployments.yaml index 37ca5ce0518a..3e88f9d82fa8 100644 --- a/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeployments.yaml +++ b/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeployments.yaml @@ -11034,8 +11034,8 @@ spec: args: description: |- Args overrides the default frontend arguments. When specified, these replace - the default ["-m", "dynamo.frontend"] entirely. - For example, ["-m", "dynamo.frontend", "--router-mode", "direct"] for GAIE deployments. + the default ["-m", "dingo.frontend"] entirely. + For example, ["-m", "dingo.frontend", "--router-mode", "direct"] for GAIE deployments. items: type: string type: array diff --git a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go index 5de3e6aa04c2..6b329e3bdd11 100644 --- a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go @@ -473,8 +473,8 @@ type FrontendSidecarSpec struct { Image string `json:"image"` // Args overrides the default frontend arguments. When specified, these replace - // the default ["-m", "dynamo.frontend"] entirely. - // For example, ["-m", "dynamo.frontend", "--router-mode", "direct"] for GAIE deployments. + // the default ["-m", "dingo.frontend"] entirely. + // For example, ["-m", "dingo.frontend", "--router-mode", "direct"] for GAIE deployments. // +optional Args []string `json:"args,omitempty"` diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go index cf786fa9ac20..0786151a8275 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go @@ -1091,7 +1091,7 @@ func TestDGD_RoundTrip_FrontendSidecar(t *testing.T) { { Name: "sidecar-frontend", Image: "dynamo-frontend:latest", - Args: []string{"-m", "dynamo.frontend"}, + Args: []string{"-m", "dingo.frontend"}, }, }, }, @@ -1515,7 +1515,7 @@ func TestDGD_FromV1alpha1_FrontendSidecarFullRoundTrip(t *testing.T) { ComponentType: "epp", FrontendSidecar: &FrontendSidecarSpec{ Image: "dynamo-frontend:1.2.3", - Args: []string{"-m", "dynamo.frontend", "--router-mode", "direct"}, + Args: []string{"-m", "dingo.frontend", "--router-mode", "direct"}, EnvFromSecret: &secret, Envs: []corev1.EnvVar{ {Name: "FRONTEND_FLAG", Value: annotationTrue}, diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index 64e4d1678d92..3adc156ad391 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -10734,8 +10734,8 @@ spec: args: description: |- Args overrides the default frontend arguments. When specified, these replace - the default ["-m", "dynamo.frontend"] entirely. - For example, ["-m", "dynamo.frontend", "--router-mode", "direct"] for GAIE deployments. + the default ["-m", "dingo.frontend"] entirely. + For example, ["-m", "dingo.frontend", "--router-mode", "direct"] for GAIE deployments. items: type: string type: array diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 37ca5ce0518a..3e88f9d82fa8 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -11034,8 +11034,8 @@ spec: args: description: |- Args overrides the default frontend arguments. When specified, these replace - the default ["-m", "dynamo.frontend"] entirely. - For example, ["-m", "dynamo.frontend", "--router-mode", "direct"] for GAIE deployments. + the default ["-m", "dingo.frontend"] entirely. + For example, ["-m", "dingo.frontend", "--router-mode", "direct"] for GAIE deployments. items: type: string type: array diff --git a/deploy/operator/internal/dynamo/backend_vllm_test.go b/deploy/operator/internal/dynamo/backend_vllm_test.go index 853b531ef385..151245a0dbcd 100644 --- a/deploy/operator/internal/dynamo/backend_vllm_test.go +++ b/deploy/operator/internal/dynamo/backend_vllm_test.go @@ -120,7 +120,7 @@ func TestVLLMBackend_UpdateContainer(t *testing.T) { role: RoleMain, component: &v1alpha1.DynamoComponentDeploymentSharedSpec{}, multinodeDeployer: &GroveMultinodeDeployer{}, - initialContainer: &corev1.Container{Args: []string{"python3", "-m", "dynamo.frontend"}}, + initialContainer: &corev1.Container{Args: []string{"python3", "-m", "dingo.frontend"}}, gpuCount: 0, expectNotModified: true, }, @@ -564,7 +564,7 @@ func TestUpdateVLLMMultinodeArgs(t *testing.T) { name: "main role does not modify args", role: RoleMain, multinodeDeployer: &GroveMultinodeDeployer{}, - initialContainer: &corev1.Container{Args: []string{"python3", "-m", "dynamo.frontend"}}, + initialContainer: &corev1.Container{Args: []string{"python3", "-m", "dingo.frontend"}}, gpuCount: 0, annotations: nil, expectNotModified: true, diff --git a/deploy/operator/internal/dynamo/component_frontend.go b/deploy/operator/internal/dynamo/component_frontend.go index eb553575f298..701305268467 100644 --- a/deploy/operator/internal/dynamo/component_frontend.go +++ b/deploy/operator/internal/dynamo/component_frontend.go @@ -28,7 +28,7 @@ func (f *FrontendDefaults) GetBaseContainer(context ComponentContext) (corev1.Co // Set default command and args container.Command = []string{"python3"} - container.Args = []string{"-m", "dynamo.frontend"} + container.Args = []string{"-m", "dingo.frontend"} // Add HTTP port container.Ports = []corev1.ContainerPort{ diff --git a/deploy/operator/internal/dynamo/graph_test.go b/deploy/operator/internal/dynamo/graph_test.go index 4ffeb27101f6..b23c0c0854b6 100644 --- a/deploy/operator/internal/dynamo/graph_test.go +++ b/deploy/operator/internal/dynamo/graph_test.go @@ -5761,7 +5761,7 @@ func TestGenerateBasePodSpec_Frontend(t *testing.T) { // Check command and args wantCommand := []string{"python3"} - wantArgs := []string{"-m", "dynamo.frontend"} + wantArgs := []string{"-m", "dingo.frontend"} if !reflect.DeepEqual(podSpec.Containers[0].Command, wantCommand) { t.Errorf("GenerateBasePodSpec() command = %v, want %v", podSpec.Containers[0].Command, wantCommand) @@ -8842,7 +8842,7 @@ func TestGenerateBasePodSpec_FrontendSidecar(t *testing.T) { ComponentType: commonconsts.ComponentTypeWorker, FrontendSidecar: &v1alpha1.FrontendSidecarSpec{ Image: "my-frontend:latest", - Args: []string{"-m", "dynamo.frontend", "--router-mode", "direct"}, + Args: []string{"-m", "dingo.frontend", "--router-mode", "direct"}, }, }, parentDGDName: "test-dgd", @@ -8850,7 +8850,7 @@ func TestGenerateBasePodSpec_FrontendSidecar(t *testing.T) { wantSidecarCount: 2, wantSidecarName: commonconsts.FrontendSidecarContainerName, wantSidecarImage: "my-frontend:latest", - wantSidecarArgs: []string{"-m", "dynamo.frontend", "--router-mode", "direct"}, + wantSidecarArgs: []string{"-m", "dingo.frontend", "--router-mode", "direct"}, wantSidecarEnvVars: map[string]string{ "DYN_NAMESPACE": "test-ns-test-dgd", "DYN_COMPONENT": commonconsts.ComponentTypeFrontend, @@ -8876,7 +8876,7 @@ func TestGenerateBasePodSpec_FrontendSidecar(t *testing.T) { wantSidecarCount: 2, wantSidecarName: commonconsts.FrontendSidecarContainerName, wantSidecarImage: "my-frontend:latest", - wantSidecarArgs: []string{"-m", "dynamo.frontend"}, + wantSidecarArgs: []string{"-m", "dingo.frontend"}, wantSidecarEnvFrom: 1, wantSidecarProbes: true, wantSidecarPorts: true, diff --git a/deploy/operator/samples/dgd-gms-failover.yaml b/deploy/operator/samples/dgd-gms-failover.yaml index 0b4052a7333e..e520b7d88c65 100644 --- a/deploy/operator/samples/dgd-gms-failover.yaml +++ b/deploy/operator/samples/dgd-gms-failover.yaml @@ -88,4 +88,4 @@ spec: extraPodSpec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest - # command: ["python3", "-m", "dynamo.frontend"] + # command: ["python3", "-m", "dingo.frontend"] diff --git a/deploy/operator/samples/v1beta1/dgd-gms-failover.yaml b/deploy/operator/samples/v1beta1/dgd-gms-failover.yaml index d0ad609cd9e9..e098ec874a13 100644 --- a/deploy/operator/samples/v1beta1/dgd-gms-failover.yaml +++ b/deploy/operator/samples/v1beta1/dgd-gms-failover.yaml @@ -115,7 +115,7 @@ spec: - envFrom: - secretRef: name: hf-token-secret - # command: ["python3", "-m", "dynamo.frontend"] + # command: ["python3", "-m", "dingo.frontend"] image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest name: main replicas: 1 diff --git a/dev/observability/prometheus.yml b/dev/observability/prometheus.yml index aa97610bfbee..fef7b9aeed91 100644 --- a/dev/observability/prometheus.yml +++ b/dev/observability/prometheus.yml @@ -35,7 +35,7 @@ scrape_configs: # This is a demo service that needs to be launched manually # Note that you may need to disable the firewall on your host. On Ubuntu: sudo ufw allow 8000/tcp - # You can also force the port, if the default is different: python -m dynamo.frontend --http-port 8000 + # You can also force the port, if the default is different: python -m dingo.frontend --http-port 8000 - job_name: 'dynamo-frontend' scrape_interval: 10s static_configs: diff --git a/dev/sanity_check.py b/dev/sanity_check.py index d4e846233576..1e5a54458248 100755 --- a/dev/sanity_check.py +++ b/dev/sanity_check.py @@ -86,7 +86,7 @@ │ ├─ ✅ dynamo.llm $HOME/dynamo/lib/bindings/python/src/dynamo/llm/__init__.py │ └─ ✅ dynamo.runtime $HOME/dynamo/lib/bindings/python/src/dynamo/runtime/__init__.py └─ ✅ Framework components: ai-dynamo 0.5.0 - ├─ ✅ dynamo.frontend $HOME/dynamo/components/src/dynamo/frontend/__init__.py + ├─ ✅ dingo.frontend $HOME/dynamo/dingo/frontend/__init__.py ├─ ✅ dynamo.llama_cpp $HOME/dynamo/components/src/dynamo/llama_cpp/__init__.py ├─ ✅ dynamo.sglang $HOME/dynamo/components/src/dynamo/sglang/__init__.py ├─ ✅ dynamo.trtllm $HOME/dynamo/components/src/dynamo/trtllm/__init__.py @@ -3093,7 +3093,7 @@ def __init__( if self.runtime_check and not components: # Try common framework components even if not discovered components = [ - "dynamo.frontend", + "dingo.frontend", "dynamo.vllm", "dynamo.sglang", "dynamo.trtllm", @@ -3168,7 +3168,7 @@ def _discover_framework_components(self, workspace_dir: Optional[str]) -> list: Returns: List of framework component module names - Example: ['dynamo.frontend', 'dynamo.planner', 'dynamo.vllm', 'dynamo.sglang'] + Example: ['dingo.frontend', 'dynamo.planner', 'dynamo.vllm', 'dynamo.sglang'] Note: Scans components/src/dynamo/... directory for modules with __init__.py files. """ @@ -3177,6 +3177,12 @@ def _discover_framework_components(self, workspace_dir: Optional[str]) -> list: if not workspace_dir: return components + dingo_frontend_path = os.path.join( + workspace_dir, "dingo", "frontend", "__init__.py" + ) + if os.path.exists(dingo_frontend_path): + components.append("dingo.frontend") + # Scan the components/src/dynamo/... Python directory for __init__.py files components_path = os.path.join(workspace_dir, "components", "src", "dynamo") if os.path.exists(components_path): diff --git a/dingo/__init__.py b/dingo/__init__.py new file mode 100644 index 000000000000..a4d636226b9a --- /dev/null +++ b/dingo/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + diff --git a/components/src/dynamo/frontend/CLAUDE.md b/dingo/frontend/CLAUDE.md similarity index 91% rename from components/src/dynamo/frontend/CLAUDE.md rename to dingo/frontend/CLAUDE.md index a5a09975298e..119b2070eefe 100644 --- a/components/src/dynamo/frontend/CLAUDE.md +++ b/dingo/frontend/CLAUDE.md @@ -1,6 +1,6 @@ # Frontend Configuration Boundary -`dynamo.frontend` parses CLI arguments and environment defaults into +`dingo.frontend` parses CLI arguments and environment defaults into `FrontendConfig`. After that point, treat `FrontendConfig` as the source of truth for frontend-owned settings. diff --git a/components/src/dynamo/frontend/README.md b/dingo/frontend/README.md similarity index 100% rename from components/src/dynamo/frontend/README.md rename to dingo/frontend/README.md diff --git a/components/src/dynamo/frontend/__init__.py b/dingo/frontend/__init__.py similarity index 100% rename from components/src/dynamo/frontend/__init__.py rename to dingo/frontend/__init__.py diff --git a/components/src/dynamo/frontend/__main__.py b/dingo/frontend/__main__.py similarity index 82% rename from components/src/dynamo/frontend/__main__.py rename to dingo/frontend/__main__.py index 15c0cf3ef64f..1264dd246b4a 100644 --- a/components/src/dynamo/frontend/__main__.py +++ b/dingo/frontend/__main__.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from dynamo.frontend.main import main +from dingo.frontend.main import main if __name__ == "__main__": main() diff --git a/components/src/dynamo/frontend/frontend_args.py b/dingo/frontend/frontend_args.py similarity index 100% rename from components/src/dynamo/frontend/frontend_args.py rename to dingo/frontend/frontend_args.py diff --git a/components/src/dynamo/frontend/main.py b/dingo/frontend/main.py similarity index 98% rename from components/src/dynamo/frontend/main.py rename to dingo/frontend/main.py index 106dd1314890..ce28d47e09b9 100644 --- a/components/src/dynamo/frontend/main.py +++ b/dingo/frontend/main.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Usage: `python -m dynamo.frontend [args]` +# Usage: `python -m dingo.frontend [args]` # # Start a frontend node. This runs: # - OpenAI HTTP server. @@ -13,7 +13,7 @@ # Pass `--interactive` or `-i` for text chat instead of HTTP server. # # For TLS: -# - python -m dynamo.frontend --http-port 8443 --tls-cert-path cert.pem --tls-key-path key.pem +# - python -m dingo.frontend --http-port 8443 --tls-cert-path cert.pem --tls-key-path key.pem # import argparse @@ -171,7 +171,7 @@ async def async_main(): # The system status server port is a worker concern. # # Serve tests set DYN_SYSTEM_PORT for the worker, but aggregated launch scripts - # start `dynamo.frontend` first. If the frontend inherits DYN_SYSTEM_PORT, it can + # start `dingo.frontend` first. If the frontend inherits DYN_SYSTEM_PORT, it can # bind that port before the worker, causing port conflicts and/or scraping the # wrong metrics endpoint. os.environ.pop("DYN_SYSTEM_PORT", None) diff --git a/components/src/dynamo/frontend/prepost.py b/dingo/frontend/prepost.py similarity index 100% rename from components/src/dynamo/frontend/prepost.py rename to dingo/frontend/prepost.py diff --git a/components/src/dynamo/frontend/sglang_prepost.py b/dingo/frontend/sglang_prepost.py similarity index 100% rename from components/src/dynamo/frontend/sglang_prepost.py rename to dingo/frontend/sglang_prepost.py diff --git a/components/src/dynamo/frontend/sglang_processor.py b/dingo/frontend/sglang_processor.py similarity index 99% rename from components/src/dynamo/frontend/sglang_processor.py rename to dingo/frontend/sglang_processor.py index b3e3f00c255c..8adf623a798c 100644 --- a/components/src/dynamo/frontend/sglang_processor.py +++ b/dingo/frontend/sglang_processor.py @@ -19,7 +19,7 @@ from sglang.srt.utils.hf_transformers_utils import get_tokenizer from dynamo._internal import ModelDeploymentCard -from dynamo.frontend.frontend_args import FrontendConfig +from dingo.frontend.frontend_args import FrontendConfig from dynamo.llm import ModelCardInstanceId, PythonAsyncEngine, RoutedEngine from dynamo.llm.exceptions import InvalidArgument, Unknown diff --git a/components/src/dynamo/frontend/tests/FRONTEND_CASES.md b/dingo/frontend/tests/FRONTEND_CASES.md similarity index 94% rename from components/src/dynamo/frontend/tests/FRONTEND_CASES.md rename to dingo/frontend/tests/FRONTEND_CASES.md index d2cea5e00269..8cfbbba7d3f2 100644 --- a/components/src/dynamo/frontend/tests/FRONTEND_CASES.md +++ b/dingo/frontend/tests/FRONTEND_CASES.md @@ -1,9 +1,9 @@ # Frontend Chat-Processor Test Cases Reference taxonomy for unit testing the **frontend chat-processor layer** — -the code under `components/src/dynamo/frontend/` that sits between the +the code under `dingo/frontend/` that sits between the OpenAI-shaped HTTP request and the backend engine. Tests for this layer -live under `components/src/dynamo/frontend/tests/`. +live under `dingo/frontend/tests/`. This is the **frontend** companion to `lib/parsers/TOOLCALLING_CASES.md`. The two taxonomies cover different surfaces: @@ -13,7 +13,7 @@ two taxonomies cover different surfaces: | `lib/parsers/TOOLCALLING_CASES.md` | Tool-call parser behavior on **model output** | `TOOLCALLING.batch.*`, `TOOLCALLING.stream.*`, `TOOLCALLING.fmt.*`, `TOOLCALLING.xml.*`, `TOOLCALLING.harmony.*` | | `lib/parsers/REASONING_CASES.md` | Reasoning parser behavior on **model output** | `REASONING.batch.*`, `REASONING.stream.*` | | `lib/parsers/PIPELINE_CASES.md` | Pipeline-boundary contracts (parser output independence from upstream metadata) | `PIPELINE.*` | -| `components/src/dynamo/frontend/tests/FRONTEND_CASES.md` | Chat-processor layer: request preprocessing, output assembly, error surface, worker plumbing | `FRONTEND.*` | +| `dingo/frontend/tests/FRONTEND_CASES.md` | Chat-processor layer: request preprocessing, output assembly, error surface, worker plumbing | `FRONTEND.*` | Backends covered by this taxonomy: **vllm** (`prepost.py` + `vllm_processor.py`) and **sglang** (`sglang_prepost.py` + `sglang_processor.py`). trtllm has its @@ -48,7 +48,7 @@ class TestUtilities: def test_normalize_prompt_token_ids(self): ... # FRONTEND.6 ``` -`grep -r 'FRONTEND.1' components/src/dynamo/frontend/tests/` returns every +`grep -r 'FRONTEND.1' dingo/frontend/tests/` returns every chat-template-preprocessing test across vllm + sglang in one shot. --- diff --git a/components/src/dynamo/frontend/tests/_routed_engine_fakes.py b/dingo/frontend/tests/_routed_engine_fakes.py similarity index 100% rename from components/src/dynamo/frontend/tests/_routed_engine_fakes.py rename to dingo/frontend/tests/_routed_engine_fakes.py diff --git a/components/src/dynamo/frontend/tests/test_frontend_utils.py b/dingo/frontend/tests/test_frontend_utils.py similarity index 98% rename from components/src/dynamo/frontend/tests/test_frontend_utils.py rename to dingo/frontend/tests/test_frontend_utils.py index bd46d8969b82..89b196f94699 100644 --- a/components/src/dynamo/frontend/tests/test_frontend_utils.py +++ b/dingo/frontend/tests/test_frontend_utils.py @@ -5,7 +5,7 @@ import pytest -from dynamo.frontend.utils import ( +from dingo.frontend.utils import ( handle_engine_error, make_backend_error, make_internal_error, diff --git a/components/src/dynamo/frontend/tests/test_multimodal_utils.py b/dingo/frontend/tests/test_multimodal_utils.py similarity index 98% rename from components/src/dynamo/frontend/tests/test_multimodal_utils.py rename to dingo/frontend/tests/test_multimodal_utils.py index 9946275a3a23..37d89b0831b2 100644 --- a/components/src/dynamo/frontend/tests/test_multimodal_utils.py +++ b/dingo/frontend/tests/test_multimodal_utils.py @@ -3,7 +3,7 @@ import pytest -from dynamo.frontend.utils import extract_mm_urls +from dingo.frontend.utils import extract_mm_urls pytestmark = [ pytest.mark.unit, diff --git a/components/src/dynamo/frontend/tests/test_sglang_multimodal_prepost.py b/dingo/frontend/tests/test_sglang_multimodal_prepost.py similarity index 98% rename from components/src/dynamo/frontend/tests/test_sglang_multimodal_prepost.py rename to dingo/frontend/tests/test_sglang_multimodal_prepost.py index 7e2d7fbfd3b1..16cb5bf94853 100644 --- a/components/src/dynamo/frontend/tests/test_sglang_multimodal_prepost.py +++ b/dingo/frontend/tests/test_sglang_multimodal_prepost.py @@ -10,7 +10,7 @@ import pytest -from dynamo.frontend.sglang_prepost import ( +from dingo.frontend.sglang_prepost import ( _normalize_messages_for_template, preprocess_chat_request, ) diff --git a/components/src/dynamo/frontend/tests/test_sglang_processor_api.py b/dingo/frontend/tests/test_sglang_processor_api.py similarity index 99% rename from components/src/dynamo/frontend/tests/test_sglang_processor_api.py rename to dingo/frontend/tests/test_sglang_processor_api.py index 8b6ff23b874b..f1df2eec0583 100644 --- a/components/src/dynamo/frontend/tests/test_sglang_processor_api.py +++ b/dingo/frontend/tests/test_sglang_processor_api.py @@ -229,7 +229,7 @@ def test_sglang_tool_model_dump(): def test_preprocess_result_picklability(): """Verify SglangPreprocessWorkerResult survives pickle round-trip.""" - from dynamo.frontend.sglang_processor import SglangPreprocessWorkerResult + from dingo.frontend.sglang_processor import SglangPreprocessWorkerResult result = SglangPreprocessWorkerResult( prompt_token_ids=[1, 2, 3], diff --git a/components/src/dynamo/frontend/tests/test_sglang_processor_metrics_unit.py b/dingo/frontend/tests/test_sglang_processor_metrics_unit.py similarity index 97% rename from components/src/dynamo/frontend/tests/test_sglang_processor_metrics_unit.py rename to dingo/frontend/tests/test_sglang_processor_metrics_unit.py index 40a26451c7c2..86809a66157a 100644 --- a/components/src/dynamo/frontend/tests/test_sglang_processor_metrics_unit.py +++ b/dingo/frontend/tests/test_sglang_processor_metrics_unit.py @@ -148,7 +148,7 @@ def _load_processor_module(module_stubs): _install_module(install_module, "dynamo._internal", ModelDeploymentCard=object) _install_module( install_module, - "dynamo.frontend.frontend_args", + "dingo.frontend.frontend_args", FrontendConfig=object, ) _install_module( @@ -164,8 +164,8 @@ def _load_processor_module(module_stubs): InvalidArgument=type("InvalidArgument", (Exception,), {}), Unknown=type("Unknown", (Exception,), {}), ) - remove_module("dynamo.frontend.sglang_processor") - return importlib.import_module("dynamo.frontend.sglang_processor") + remove_module("dingo.frontend.sglang_processor") + return importlib.import_module("dingo.frontend.sglang_processor") def test_stream_emits_llm_metrics_annotation(module_stubs): diff --git a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py b/dingo/frontend/tests/test_sglang_processor_unit.py similarity index 99% rename from components/src/dynamo/frontend/tests/test_sglang_processor_unit.py rename to dingo/frontend/tests/test_sglang_processor_unit.py index 86f5f8fdcc5b..f75b0a194fe8 100644 --- a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py +++ b/dingo/frontend/tests/test_sglang_processor_unit.py @@ -20,9 +20,9 @@ from sglang.srt.function_call.json_array_parser import JsonArrayParser from sglang.srt.utils.hf_transformers_utils import get_tokenizer -import dynamo.frontend.sglang_prepost as sglang_prepost_module -import dynamo.frontend.sglang_processor as sglang_processor_module -from dynamo.frontend.sglang_prepost import ( +import dingo.frontend.sglang_prepost as sglang_prepost_module +import dingo.frontend.sglang_processor as sglang_processor_module +from dingo.frontend.sglang_prepost import ( SglangPreprocessResult, SglangStreamingPostProcessor, _flatten_message_content, @@ -36,7 +36,7 @@ preprocess_chat_request, resolve_request_force_reasoning, ) -from dynamo.frontend.sglang_processor import ( +from dingo.frontend.sglang_processor import ( SglangPreprocessWorkerResult, SglangProcessor, _build_dynamo_preproc, @@ -46,7 +46,7 @@ _runtime_config_parser_name, _tokenizer_eos_token_ids, ) -from dynamo.frontend.utils import ( +from dingo.frontend.utils import ( PreprocessError, nvext_extra_field_requested, random_call_id, diff --git a/components/src/dynamo/frontend/tests/test_sglang_tool_calls.py b/dingo/frontend/tests/test_sglang_tool_calls.py similarity index 99% rename from components/src/dynamo/frontend/tests/test_sglang_tool_calls.py rename to dingo/frontend/tests/test_sglang_tool_calls.py index ab249db3faf8..f0105940527c 100644 --- a/components/src/dynamo/frontend/tests/test_sglang_tool_calls.py +++ b/dingo/frontend/tests/test_sglang_tool_calls.py @@ -19,7 +19,7 @@ from sglang.srt.parser.reasoning_parser import ReasoningParser from sglang.srt.utils.hf_transformers_utils import get_tokenizer -from dynamo.frontend.sglang_prepost import SglangStreamingPostProcessor +from dingo.frontend.sglang_prepost import SglangStreamingPostProcessor # Needs sglang packages (gpu_1 container), but does not allocate GPU VRAM. pytestmark = [ diff --git a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py b/dingo/frontend/tests/test_vllm_processor_unit.py similarity index 97% rename from components/src/dynamo/frontend/tests/test_vllm_processor_unit.py rename to dingo/frontend/tests/test_vllm_processor_unit.py index 84db2b94e85f..7abfac51ccbb 100644 --- a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py +++ b/dingo/frontend/tests/test_vllm_processor_unit.py @@ -15,7 +15,7 @@ from transformers import AutoTokenizer from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser -from dynamo.frontend.prepost import _prepare_request +from dingo.frontend.prepost import _prepare_request # Needs vllm packages (gpu_1 container), but does not allocate GPU VRAM. pytestmark = [ @@ -128,7 +128,7 @@ def test_no_tools_in_request(self, tokenizer): class TestReasoningParserMetadata: def test_no_reasoning_parser_returns_none(self): - from dynamo.frontend.vllm_processor import _build_reasoning_parser_metadata + from dingo.frontend.vllm_processor import _build_reasoning_parser_metadata assert _build_reasoning_parser_metadata( None, @@ -139,7 +139,7 @@ def test_no_reasoning_parser_returns_none(self): ) == (None, None) def test_include_reasoning_false_marks_reasoning_ended(self): - from dynamo.frontend.vllm_processor import _build_reasoning_parser_metadata + from dingo.frontend.vllm_processor import _build_reasoning_parser_metadata class ParserShouldNotBeBuilt: def __init__(self, *args, **kwargs): @@ -157,7 +157,7 @@ def __init__(self, *args, **kwargs): assert parser_kwargs == {"chat_template_kwargs": {"reasoning_effort": "low"}} def test_parser_receives_chat_template_kwargs(self): - from dynamo.frontend.vllm_processor import _build_reasoning_parser_metadata + from dingo.frontend.vllm_processor import _build_reasoning_parser_metadata class FakeReasoningParser: def __init__(self, tokenizer, *, chat_template_kwargs): @@ -180,7 +180,7 @@ def is_reasoning_end(self, prompt_token_ids): assert parser_kwargs == {"chat_template_kwargs": {"reasoning_effort": "high"}} def test_kv_router_copies_reasoning_metadata_to_extra_args(self): - from dynamo.frontend.vllm_processor import _inject_routing_metadata + from dingo.frontend.vllm_processor import _inject_routing_metadata kv_kwargs = {"extra_args": {"mm_hashes": [123]}} _inject_routing_metadata( @@ -235,7 +235,7 @@ def process_output(self, output): @pytest.fixture def vllm_processor_module(monkeypatch): - import dynamo.frontend.vllm_processor as module + import dingo.frontend.vllm_processor as module class FakeEngineCoreOutput: __struct_fields__ = () diff --git a/components/src/dynamo/frontend/utils.py b/dingo/frontend/utils.py similarity index 100% rename from components/src/dynamo/frontend/utils.py rename to dingo/frontend/utils.py diff --git a/components/src/dynamo/frontend/vllm_processor.py b/dingo/frontend/vllm_processor.py similarity index 99% rename from components/src/dynamo/frontend/vllm_processor.py rename to dingo/frontend/vllm_processor.py index 777e7600a120..6f01a4a6f0bc 100644 --- a/components/src/dynamo/frontend/vllm_processor.py +++ b/dingo/frontend/vllm_processor.py @@ -35,7 +35,7 @@ ) from dynamo.common.multimodal.routing_utils import build_mm_routing_info_from_features from dynamo.common.utils import nvtx_utils as _nvtx -from dynamo.frontend.frontend_args import FrontendConfig +from dingo.frontend.frontend_args import FrontendConfig from dynamo.llm import ModelCardInstanceId, PythonAsyncEngine, RoutedEngine from .prepost import StreamingPostProcessor, preprocess_chat_request diff --git a/docs/agents/agent-harnesses.md b/docs/agents/agent-harnesses.md index cfa9ad6a6673..ed151d57d5a7 100644 --- a/docs/agents/agent-harnesses.md +++ b/docs/agents/agent-harnesses.md @@ -37,7 +37,7 @@ Codex sends a `session-id` header that Dynamo maps to `session_id`. ## Claude Code -Claude Code uses Anthropic-compatible Messages API. The local launcher above starts `dynamo.frontend` with `--enable-anthropic-api`; for other deployments, pass that flag when starting the frontend. Then set: +Claude Code uses Anthropic-compatible Messages API. The local launcher above starts `dingo.frontend` with `--enable-anthropic-api`; for other deployments, pass that flag when starting the frontend. Then set: ```bash export ANTHROPIC_BASE_URL=http://localhost:8000 diff --git a/docs/agents/thunderagent-router.md b/docs/agents/thunderagent-router.md index 5b3ed562d83e..f10f2a0303f0 100644 --- a/docs/agents/thunderagent-router.md +++ b/docs/agents/thunderagent-router.md @@ -70,7 +70,7 @@ All `KvRouter` flags from `dynamo.router` (`--router-temperature`, `--use-kv-eve ```text ┌─────────────────────────────────────────────────────────────┐ -│ dynamo.frontend (HTTP + auth + tracing sink) │ +│ dingo.frontend (HTTP + auth + tracing sink) │ └────────────────────┬────────────────────────────────────────┘ │ chat completions, with session headers ▼ diff --git a/docs/backends/sglang/sglang-chat-processor.md b/docs/backends/sglang/sglang-chat-processor.md index 02330b227954..921ea01437df 100644 --- a/docs/backends/sglang/sglang-chat-processor.md +++ b/docs/backends/sglang/sglang-chat-processor.md @@ -23,7 +23,7 @@ If the parser you need is missing from the Rust preprocessor, consider [opening ```bash # Frontend with SGLang processor, tool calling, and reasoning -python -m dynamo.frontend \ +python -m dingo.frontend \ --router-mode kv \ --dyn-chat-processor sglang \ --tool-call-parser hermes \ @@ -58,7 +58,7 @@ These arguments are passed to the **frontend** (not the worker) when using `--dy The processor supports all SGLang tool call formats. Pass `--tool-call-parser` on the frontend: ```bash -python -m dynamo.frontend \ +python -m dingo.frontend \ --dyn-chat-processor sglang \ --tool-call-parser hermes ``` @@ -116,7 +116,7 @@ Response: For models that produce chain-of-thought reasoning (e.g., Qwen3, DeepSeek-R1), pass `--reasoning-parser`: ```bash -python -m dynamo.frontend \ +python -m dingo.frontend \ --dyn-chat-processor sglang \ --reasoning-parser qwen3 ``` @@ -130,11 +130,11 @@ The parser separates think tag content into the `reasoning_content` field and re ```diff # Before (deprecated) - python -m dynamo.sglang --model-path --use-sglang-tokenizer -- python -m dynamo.frontend +- python -m dingo.frontend # After python -m dynamo.sglang --model-path -+ python -m dynamo.frontend --dyn-chat-processor sglang ++ python -m dingo.frontend --dyn-chat-processor sglang ``` Key differences: diff --git a/docs/backends/sglang/sglang-hicache.md b/docs/backends/sglang/sglang-hicache.md index 944bc5c15844..a59c98cb0b9b 100644 --- a/docs/backends/sglang/sglang-hicache.md +++ b/docs/backends/sglang/sglang-hicache.md @@ -39,7 +39,7 @@ python -m dynamo.sglang \ Then start the frontend: ```bash -python -m dynamo.frontend --http-port 8000 +python -m dingo.frontend --http-port 8000 ``` @@ -149,7 +149,7 @@ You also need: ## Setup > [!WARNING] -> **Known limitation in 1.2.0.** With both `--enable-metrics` and `--disable-piecewise-cuda-graph` set on the SGLang worker, the process can crash on the first KV-cache write due to a race in the upstream `mooncake-transfer-engine` thread pool. The recipe below omits these flags; per-process metrics scraping via the `dynamo.frontend` is unaffected. The mooncake-side fix is being tracked upstream. +> **Known limitation in 1.2.0.** With both `--enable-metrics` and `--disable-piecewise-cuda-graph` set on the SGLang worker, the process can crash on the first KV-cache write due to a race in the upstream `mooncake-transfer-engine` thread pool. The recipe below omits these flags; per-process metrics scraping via the `dingo.frontend` is unaffected. The mooncake-side fix is being tracked upstream. **SGLang worker** — HiCache with Mooncake storage: @@ -170,7 +170,7 @@ Launch additional workers on other GPUs / hosts with the same Mooncake config so **Dynamo frontend** — enable tier-aware routing: ```bash -python -m dynamo.frontend \ +python -m dingo.frontend \ --http-port 8000 \ --router-mode kv \ --shared-cache-type hicache \ diff --git a/docs/backends/sglang/sglang-observability.md b/docs/backends/sglang/sglang-observability.md index f3873fe55a24..310f3007caea 100644 --- a/docs/backends/sglang/sglang-observability.md +++ b/docs/backends/sglang/sglang-observability.md @@ -37,7 +37,7 @@ Launch a frontend and SGLang backend to test metrics: ```bash # Start frontend (default port 8000, override with --http-port or DYN_HTTP_PORT env var) -$ python -m dynamo.frontend +$ python -m dingo.frontend # Enable system metrics server on port 8081 $ DYN_SYSTEM_PORT=8081 python -m dynamo.sglang --model --enable-metrics @@ -308,7 +308,7 @@ export OTEL_EXPORT_ENABLED=true export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4317 # Frontend -OTEL_SERVICE_NAME=dynamo-frontend python -m dynamo.frontend & +OTEL_SERVICE_NAME=dynamo-frontend python -m dingo.frontend & # SGLang worker with tracing OTEL_SERVICE_NAME=dynamo-worker-sglang \ diff --git a/docs/backends/trtllm/trtllm-dp-rank-routing.md b/docs/backends/trtllm/trtllm-dp-rank-routing.md index ed1578916161..23ffcc962eed 100644 --- a/docs/backends/trtllm/trtllm-dp-rank-routing.md +++ b/docs/backends/trtllm/trtllm-dp-rank-routing.md @@ -27,7 +27,7 @@ CUDA_VISIBLE_DEVICES=0,1 python3 -m dynamo.trtllm \ --publish-events-and-metrics # Frontend with KV routing -python3 -m dynamo.frontend --router-mode kv +python3 -m dingo.frontend --router-mode kv ``` The `--enable-attention-dp` flag sets `attention_dp_size = tensor_parallel_size` and configures Dynamo to publish KV events per DP rank. The router automatically creates routing targets for each `(worker_id, dp_rank)` combination. diff --git a/docs/backends/trtllm/trtllm-examples.md b/docs/backends/trtllm/trtllm-examples.md index 819f79043deb..c1f02af541c0 100644 --- a/docs/backends/trtllm/trtllm-examples.md +++ b/docs/backends/trtllm/trtllm-examples.md @@ -30,7 +30,7 @@ docker compose -f dev/docker-compose.yml up -d -Each launch script runs the frontend and worker(s) in a single terminal. You can run each command separately in different terminals for testing. Each shell script simply runs `python3 -m dynamo.frontend ` to start up the ingress and `python3 -m dynamo.trtllm ` to start up the workers. +Each launch script runs the frontend and worker(s) in a single terminal. You can run each command separately in different terminals for testing. Each shell script simply runs `python3 -m dingo.frontend ` to start up the ingress and `python3 -m dynamo.trtllm ` to start up the workers. For detailed information about KV-aware routing behavior, see [Routing Concepts](../../components/router/router-concepts.md). For deployment modes, see the [Router Guide](../../components/router/router-guide.md). @@ -110,5 +110,5 @@ For complete Kubernetes deployment instructions, configurations, and troubleshoo See the [client](../sglang/README.md#testing-the-deployment) section to learn how to send requests to the deployment. -To send a request to a multi-node deployment, target the node which is running `python3 -m dynamo.frontend `. +To send a request to a multi-node deployment, target the node which is running `python3 -m dingo.frontend `. diff --git a/docs/backends/trtllm/trtllm-gpt-oss.md b/docs/backends/trtllm/trtllm-gpt-oss.md index d39f9b83a259..e08d89fa9233 100644 --- a/docs/backends/trtllm/trtllm-gpt-oss.md +++ b/docs/backends/trtllm/trtllm-gpt-oss.md @@ -140,7 +140,7 @@ cd /workspace/examples/backends/trtllm 1. **Start frontend**: ```bash # Start frontend with round-robin routing -python3 -m dynamo.frontend --router-mode round-robin --http-port 8000 & +python3 -m dingo.frontend --router-mode round-robin --http-port 8000 & ``` 2. **Launch prefill worker**: diff --git a/docs/backends/trtllm/trtllm-observability.md b/docs/backends/trtllm/trtllm-observability.md index bf58fa9615b2..3787a69d99e5 100644 --- a/docs/backends/trtllm/trtllm-observability.md +++ b/docs/backends/trtllm/trtllm-observability.md @@ -41,7 +41,7 @@ Launch a frontend and TensorRT-LLM backend to test metrics: ```bash # Start frontend (default port 8000, override with --http-port or DYN_HTTP_PORT env var) -$ python -m dynamo.frontend +$ python -m dingo.frontend # Enable system metrics server on port 8081 and enable metrics collection $ DYN_SYSTEM_PORT=8081 python -m dynamo.trtllm --model --publish-events-and-metrics diff --git a/docs/backends/vllm/README.md b/docs/backends/vllm/README.md index 76349a8669af..6492629e1c46 100644 --- a/docs/backends/vllm/README.md +++ b/docs/backends/vllm/README.md @@ -111,7 +111,7 @@ frontend, then launch the Rust worker in another terminal: ```bash docker compose -f dev/docker-compose.yml up -d -python -m dynamo.frontend --http-port 8000 +python -m dingo.frontend --http-port 8000 ``` ```bash diff --git a/docs/backends/vllm/vllm-chat-processor.md b/docs/backends/vllm/vllm-chat-processor.md index ffa7d46fa976..120b3534a220 100644 --- a/docs/backends/vllm/vllm-chat-processor.md +++ b/docs/backends/vllm/vllm-chat-processor.md @@ -23,7 +23,7 @@ If the parser you need is missing from the Rust preprocessor, consider [opening ```bash # Frontend with vLLM processor, tool calling, and reasoning -python -m dynamo.frontend \ +python -m dingo.frontend \ --router-mode kv \ --dyn-chat-processor vllm \ --enable-auto-tool-choice \ @@ -59,7 +59,7 @@ These arguments are passed to the **frontend** (not the worker) when using `--dy The processor supports all vLLM tool call formats. Pass `--tool-call-parser` (and typically `--enable-auto-tool-choice`) on the frontend: ```bash -python -m dynamo.frontend \ +python -m dingo.frontend \ --dyn-chat-processor vllm \ --enable-auto-tool-choice \ --tool-call-parser hermes @@ -118,7 +118,7 @@ Response: For models that produce chain-of-thought reasoning (e.g., Qwen3, DeepSeek-R1), pass `--reasoning-parser`: ```bash -python -m dynamo.frontend \ +python -m dingo.frontend \ --dyn-chat-processor vllm \ --reasoning-parser qwen3 ``` diff --git a/docs/components/frontend/README.md b/docs/components/frontend/README.md index 7554d5bb26ae..1601758cea36 100644 --- a/docs/components/frontend/README.md +++ b/docs/components/frontend/README.md @@ -39,7 +39,7 @@ The Dynamo Frontend is the API gateway for serving LLM inference requests. It pr ### HTTP Frontend ```bash -python -m dynamo.frontend --http-port 8000 +python -m dingo.frontend --http-port 8000 ``` This starts an OpenAI-compatible HTTP server with integrated pre/post processing and routing. Backends are auto-discovered when they call `register_model`. @@ -53,7 +53,7 @@ If the model is not available on Hugging Face, such as a private or customized m ### KServe gRPC Frontend ```bash -python -m dynamo.frontend --kserve-grpc-server +python -m dingo.frontend --kserve-grpc-server ``` See the [Frontend Guide](frontend-guide.md) for KServe-specific configuration and message formats. @@ -75,7 +75,7 @@ spec: command: - python - -m - - dynamo.frontend + - dingo.frontend - --http-port - "8000" ``` diff --git a/docs/components/frontend/Tokenizer.md b/docs/components/frontend/Tokenizer.md index fb3bd0b7b4e2..dd18366d6de5 100644 --- a/docs/components/frontend/Tokenizer.md +++ b/docs/components/frontend/Tokenizer.md @@ -39,11 +39,11 @@ Set the backend with a CLI flag or environment variable. The CLI flag takes prec ```bash # CLI flag -python -m dynamo.frontend --tokenizer fastokens +python -m dingo.frontend --tokenizer fastokens # Environment variable export DYN_TOKENIZER=fastokens -python -m dynamo.frontend +python -m dingo.frontend ``` ## Dynamo Frontend Behavior diff --git a/docs/components/frontend/configuration.md b/docs/components/frontend/configuration.md index 93b167ea5e4c..a48a8141c8cf 100644 --- a/docs/components/frontend/configuration.md +++ b/docs/components/frontend/configuration.md @@ -5,7 +5,7 @@ title: Frontend Configuration Reference subtitle: Complete reference for all frontend CLI arguments, environment variables, and HTTP endpoints --- -This page documents all configuration options for the Dynamo Frontend (`python -m dynamo.frontend`). +This page documents all configuration options for the Dynamo Frontend (`python -m dingo.frontend`). Every CLI argument has a corresponding environment variable. CLI arguments take precedence over environment variables. diff --git a/docs/components/frontend/frontend-guide.md b/docs/components/frontend/frontend-guide.md index 64ee6a8c306a..c83354a5e738 100644 --- a/docs/components/frontend/frontend-guide.md +++ b/docs/components/frontend/frontend-guide.md @@ -27,7 +27,7 @@ This documentation assumes readers are familiar with the usage of KServe v2 API To start the KServe frontend, run the below command: ```bash -python -m dynamo.frontend --kserve-grpc-server +python -m dingo.frontend --kserve-grpc-server ``` ## gRPC Performance Tuning @@ -45,7 +45,7 @@ The gRPC server supports optional HTTP/2 flow control tuning via environment var # For 128 concurrent 15k-token requests export DYN_GRPC_INITIAL_CONNECTION_WINDOW_SIZE=16777216 # 16MB export DYN_GRPC_INITIAL_STREAM_WINDOW_SIZE=1048576 # 1MB -python -m dynamo.frontend --kserve-grpc-server +python -m dingo.frontend --kserve-grpc-server ``` If these variables are not set, the server uses tonic's default values. @@ -145,7 +145,7 @@ The frontend may be started via Python binding, this is useful when integrating The frontend includes an integrated router for request distribution. Configure routing mode: ```bash -python -m dynamo.frontend --router-mode kv --http-port 8000 +python -m dingo.frontend --router-mode kv --http-port 8000 ``` See [Router Documentation](../router/README.md) for routing configuration details. diff --git a/docs/components/kvbm/kvbm-guide.md b/docs/components/kvbm/kvbm-guide.md index d81c12abecd1..22228e8ae8fc 100644 --- a/docs/components/kvbm/kvbm-guide.md +++ b/docs/components/kvbm/kvbm-guide.md @@ -155,7 +155,7 @@ kv_connector_config: EOF # Start dynamo frontend -python3 -m dynamo.frontend --http-port 8000 & +python3 -m dingo.frontend --http-port 8000 & # Serve the model with KVBM python3 -m dynamo.trtllm \ @@ -200,7 +200,7 @@ python -m dynamo.sglang \ --hicache-storage-backend nixl # In a separate terminal, start the frontend -python -m dynamo.frontend --http-port 8000 +python -m dingo.frontend --http-port 8000 # Send a test request curl localhost:8000/v1/chat/completions \ @@ -531,7 +531,7 @@ container/run.sh --image dynamo:latest-vllm-local-dev -it --mount-workspace --us export PATH=/opt/nvidia/nsight-systems/2025.5.1/bin:$PATH # example usage of nsys: delay 30 seconds and then capture 60 seconds -python -m dynamo.frontend & +python -m dingo.frontend & DYN_KVBM_CPU_CACHE_GB=10 \ nsys profile -o /tmp/kvbm-nsys --trace-fork-before-exec=true --cuda-graph-trace=node --delay 30 --duration 60 \ diff --git a/docs/components/router/README.md b/docs/components/router/README.md index 24739d4a3919..cc62750192ac 100644 --- a/docs/components/router/README.md +++ b/docs/components/router/README.md @@ -16,7 +16,7 @@ The Dynamo KV Router intelligently routes requests by evaluating their computati To launch the Dynamo frontend with the KV Router: ```bash -python -m dynamo.frontend --router-mode kv --http-port 8000 +python -m dingo.frontend --router-mode kv --http-port 8000 ``` For Kubernetes, set `DYN_ROUTER_MODE=kv` on the Frontend service. For event-driven KV state, configure backend workers to publish KV cache events using the backend-specific flags described in [Router Operations](router-operations.md#additional-notes). Use `--no-router-kv-events` only when you want approximate cache-state prediction. diff --git a/docs/components/router/README.zh-CN.md b/docs/components/router/README.zh-CN.md index 1034628a3c9d..5c15f3857b88 100644 --- a/docs/components/router/README.zh-CN.md +++ b/docs/components/router/README.zh-CN.md @@ -15,7 +15,7 @@ Dynamo KV Router 通过评估不同 worker 上的计算成本来智能地路由 我们可以通过 Dynamo frontend 使用 KV Router： ```bash -python -m dynamo.frontend --router-mode kv --http-port 8000 +python -m dingo.frontend --router-mode kv --http-port 8000 ``` 对于 Kubernetes，请在 Frontend service 上设置 `DYN_ROUTER_MODE=kv`。对于事件驱动的 KV 状态，请使用 [Router Operations](router-operations.md#additional-notes) 中描述的后端专用 flag，配置 backend worker 发布 KV cache 事件。仅当你希望使用近似的 cache 状态预测时，才使用 `--no-router-kv-events`。 diff --git a/docs/components/router/router-concepts.md b/docs/components/router/router-concepts.md index f805290822df..0338601daa9d 100644 --- a/docs/components/router/router-concepts.md +++ b/docs/components/router/router-concepts.md @@ -70,7 +70,7 @@ physical class to dispatch before worker scoring runs. To enable KV cache-aware routing, start the frontend node like this: ```bash -python -m dynamo.frontend --router-mode kv +python -m dingo.frontend --router-mode kv ``` When KV blocks are created or removed, the engine notifies the Dynamo router, which then identifies the worker with the best matching blocks and routes traffic accordingly. diff --git a/docs/components/router/router-configuration.md b/docs/components/router/router-configuration.md index 17be76ca6bb6..4f8332411b50 100644 --- a/docs/components/router/router-configuration.md +++ b/docs/components/router/router-configuration.md @@ -82,7 +82,7 @@ compute cache state for classification. See the tested [sample policy](../../../examples/router/policy-class-queues.yaml). ```bash -python -m dynamo.frontend \ +python -m dingo.frontend \ --router-mode kv \ --router-policy-config examples/router/policy-class-queues.yaml ``` @@ -156,7 +156,7 @@ Use `--router-prefill-load-model aic` when you want prompt-side load tracking to Enable it on the frontend like this: ```bash -python -m dynamo.frontend \ +python -m dingo.frontend \ --router-mode kv \ --router-prefill-load-model aic \ --aic-backend vllm \ diff --git a/docs/components/router/router-disaggregated-serving.md b/docs/components/router/router-disaggregated-serving.md index 96328723393f..4d28f793c4a7 100644 --- a/docs/components/router/router-disaggregated-serving.md +++ b/docs/components/router/router-disaggregated-serving.md @@ -64,7 +64,7 @@ await prefill_endpoint.serve_endpoint(prefill_handler.generate) ``` >[!Note] -> The automatic disaggregated routing setup described here is currently supported by the integrated `dynamo.frontend` path. It is not provided as a single turnkey mode by the standalone Python router (`python -m dynamo.router`). If you build this topology with standalone routers, you must launch and connect the prefill and decode routing stages yourself and handle request handoff, including the `disaggregated_params` returned by prefill. For an advanced reference, see the [Global Router](https://github.com/ai-dynamo/dynamo/tree/main/components/src/dynamo/global_router), which composes local prefill and decode router pools explicitly. +> The automatic disaggregated routing setup described here is currently supported by the integrated `dingo.frontend` path. It is not provided as a single turnkey mode by the standalone Python router (`python -m dynamo.router`). If you build this topology with standalone routers, you must launch and connect the prefill and decode routing stages yourself and handle request handoff, including the `disaggregated_params` returned by prefill. For an advanced reference, see the [Global Router](https://github.com/ai-dynamo/dynamo/tree/main/components/src/dynamo/global_router), which composes local prefill and decode router pools explicitly. ## Request Flow diff --git a/docs/components/router/router-examples.md b/docs/components/router/router-examples.md index 10bb4c72160e..4067490a775a 100644 --- a/docs/components/router/router-examples.md +++ b/docs/components/router/router-examples.md @@ -158,7 +158,7 @@ extraPodSpec: - /bin/sh - -c args: - - "python3 -m dynamo.frontend --router-mode kv --router-temperature 0.5 --http-port 8000" + - "python3 -m dingo.frontend --router-mode kv --router-temperature 0.5 --http-port 8000" ``` **Recommendation:** Use environment variables for easier configuration management and consistency with Dynamo's K8s patterns. diff --git a/docs/components/router/router-guide.md b/docs/components/router/router-guide.md index 0f593fe6bcd1..1394c464a73f 100644 --- a/docs/components/router/router-guide.md +++ b/docs/components/router/router-guide.md @@ -19,7 +19,7 @@ The router can be deployed using [Python / CLI](#python--cli-deployment), [Kuber To launch the Dynamo frontend with the KV Router: ```bash -python -m dynamo.frontend --router-mode kv --http-port 8000 +python -m dingo.frontend --router-mode kv --http-port 8000 ``` This command: @@ -47,7 +47,7 @@ Backend workers register themselves using the `register_model` API. For accurate | `--serve-indexer` | `false` | Serve the Dynamo-native remote indexer from this frontend/router on the worker component | | `--use-remote-indexer` | `false` | Query the worker component's served remote indexer instead of maintaining a local overlap indexer | -For all available options: `python -m dynamo.frontend --help` +For all available options: `python -m dingo.frontend --help` For detailed configuration options and tuning parameters, see [Configuration and Tuning](router-configuration.md). For candidate eligibility rules, see [Router Filtering](router-filtering.md). For how the router models prefill and decode load in the cost function, see [Routing Concepts](router-concepts.md#active-load-modeling). @@ -102,7 +102,7 @@ You can also run the KV router as a standalone service (without the Dynamo front | Deployment | Process | Metrics Port | Use Case | |------------|---------|--------------|----------| -| **Frontend-embedded** | `python -m dynamo.frontend --router-mode kv` | Frontend HTTP port (default 8000) | Standard deployment; router runs inside the frontend process | +| **Frontend-embedded** | `python -m dingo.frontend --router-mode kv` | Frontend HTTP port (default 8000) | Standard deployment; router runs inside the frontend process | | **Standalone** | `python -m dynamo.router` | `DYN_SYSTEM_PORT` (if set) | Multi-tier architectures, advanced disaggregated prefill routing, custom pipelines | The standalone router does not include the HTTP frontend (no `/v1/chat/completions` endpoint). It exposes only the `RouterRequestMetrics` via the system status server. See the [Standalone Router README](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/router/README.md). @@ -113,13 +113,13 @@ The Dynamo router can be deployed in several configurations. The table below sho | Mode | Command | Routing Logic | KV Events | Topology | Use Case | |------|---------|---------------|-----------|----------|----------| -| **Frontend + Round-Robin** | `python -m dynamo.frontend --router-mode round-robin` | Cycles through workers | None | Aggregated | Simplest baseline; no KV awareness | -| **Frontend + Random** | `python -m dynamo.frontend --router-mode random` | Random worker selection | None | Aggregated | Stateless load balancing | -| **Frontend + KV (Aggregated)** | `python -m dynamo.frontend --router-mode kv` | KV cache overlap + load | NATS Core / JetStream / ZMQ / Approx | Aggregated | Production single-pool serving with cache reuse | -| **Frontend + KV (Disaggregated)** | `python -m dynamo.frontend --router-mode kv` with prefill + decode workers | KV cache overlap + load | NATS Core / JetStream / ZMQ / Approx | Disaggregated (prefill + decode pools) | Separate prefill/decode for large-scale serving | -| **Frontend + Least-Loaded** | `python -m dynamo.frontend --router-mode least-loaded` | Fewest active connections | None | Aggregated or disaggregated fallback | Simple load-aware balancing without KV awareness | -| **Frontend + Device-Aware Weighted** | `python -m dynamo.frontend --router-mode device-aware-weighted` | Device-aware budget + least-loaded within selected device group | None | Aggregated or disaggregated fallback | Heterogeneous fleet balancing (CPU/non-CPU); degenerates to least-loaded when only one device class is present | -| **Frontend + Direct** | `python -m dynamo.frontend --router-mode direct` | Worker ID from request hints | None | Aggregated | External orchestrator (e.g., EPP/GAIE) selects workers | +| **Frontend + Round-Robin** | `python -m dingo.frontend --router-mode round-robin` | Cycles through workers | None | Aggregated | Simplest baseline; no KV awareness | +| **Frontend + Random** | `python -m dingo.frontend --router-mode random` | Random worker selection | None | Aggregated | Stateless load balancing | +| **Frontend + KV (Aggregated)** | `python -m dingo.frontend --router-mode kv` | KV cache overlap + load | NATS Core / JetStream / ZMQ / Approx | Aggregated | Production single-pool serving with cache reuse | +| **Frontend + KV (Disaggregated)** | `python -m dingo.frontend --router-mode kv` with prefill + decode workers | KV cache overlap + load | NATS Core / JetStream / ZMQ / Approx | Disaggregated (prefill + decode pools) | Separate prefill/decode for large-scale serving | +| **Frontend + Least-Loaded** | `python -m dingo.frontend --router-mode least-loaded` | Fewest active connections | None | Aggregated or disaggregated fallback | Simple load-aware balancing without KV awareness | +| **Frontend + Device-Aware Weighted** | `python -m dingo.frontend --router-mode device-aware-weighted` | Device-aware budget + least-loaded within selected device group | None | Aggregated or disaggregated fallback | Heterogeneous fleet balancing (CPU/non-CPU); degenerates to least-loaded when only one device class is present | +| **Frontend + Direct** | `python -m dingo.frontend --router-mode direct` | Worker ID from request hints | None | Aggregated | External orchestrator (e.g., EPP/GAIE) selects workers | | **Standalone Router** | `python -m dynamo.router` | KV cache overlap + load | NATS Core / JetStream / ZMQ | Any | Routing without the HTTP frontend (multi-tier, custom pipelines) | ### Routing Modes (`--router-mode`) diff --git a/docs/components/router/router-operations.md b/docs/components/router/router-operations.md index 96fd666a6048..7a75da486d98 100644 --- a/docs/components/router/router-operations.md +++ b/docs/components/router/router-operations.md @@ -9,7 +9,7 @@ This page covers day-2 operational topics for router deployments. For flags and ## Serving Multiple Router Replicas -For improved fault tolerance, you can launch multiple frontend-plus-router replicas. If multiple `dynamo.frontend` processes share the same host or network namespace, give each instance a different HTTP port. In Kubernetes or on separate hosts, replicas can usually reuse the same container port. Alternatively, you can deploy the router separately as the standalone `python -m dynamo.router` service. +For improved fault tolerance, you can launch multiple frontend-plus-router replicas. If multiple `dingo.frontend` processes share the same host or network namespace, give each instance a different HTTP port. In Kubernetes or on separate hosts, replicas can usually reuse the same container port. Alternatively, you can deploy the router separately as the standalone `python -m dynamo.router` service. ## Router State Management @@ -49,7 +49,7 @@ JetStream mode requires `--router-durable-kv-events` on both frontend and worker - You can launch a third router replica even if the first two are down, and it will recover the full prefix state. ```bash -python -m dynamo.frontend \ +python -m dingo.frontend \ --router-mode kv \ --http-port 8002 \ --router-durable-kv-events @@ -77,17 +77,17 @@ There are two operating modes for active blocks: ```bash # Router replica 1 -python -m dynamo.frontend --router-mode kv --http-port 8000 --router-replica-sync +python -m dingo.frontend --router-mode kv --http-port 8000 --router-replica-sync # Router replica 2 -python -m dynamo.frontend --router-mode kv --http-port 8001 --router-replica-sync +python -m dingo.frontend --router-mode kv --http-port 8001 --router-replica-sync ``` With replica sync enabled, a new router still starts with zero active-block knowledge, but it converges through live request handling and active-sequence events from other replicas. Without it, each replica keeps an isolated active-block view, which can lead to suboptimal load balancing. ## Dynamo-Native Remote Indexer -For Dynamo-native deployments, the remote indexer is served by `dynamo.frontend` or `dynamo.router`, not by `dynamo.indexer`. +For Dynamo-native deployments, the remote indexer is served by `dingo.frontend` or `dynamo.router`, not by `dynamo.indexer`. - Use `--serve-indexer` on router or frontend replicas that should expose `kv_indexer_query` from the worker component. - Use `--use-remote-indexer` on consumer routers or frontends that should query that served endpoint instead of maintaining a local overlap indexer. @@ -97,10 +97,10 @@ Frontend example: ```bash # Serving anchors -python -m dynamo.frontend --router-mode kv --serve-indexer +python -m dingo.frontend --router-mode kv --serve-indexer # Consumer frontend -python -m dynamo.frontend --router-mode kv --use-remote-indexer +python -m dingo.frontend --router-mode kv --use-remote-indexer ``` The served service is request-plane only. Each serving router or frontend keeps its normal local KV event ingestion, gap detection, and worker-query recovery path; remote consumers only issue hash-based overlap queries. diff --git a/docs/components/router/standalone-indexer.md b/docs/components/router/standalone-indexer.md index 9b2be07a97c7..28c03c2bc18a 100644 --- a/docs/components/router/standalone-indexer.md +++ b/docs/components/router/standalone-indexer.md @@ -16,7 +16,7 @@ The standalone KV indexer (`python -m dynamo.indexer`) is a lightweight service This is distinct from the [Standalone Router](../../../components/src/dynamo/router/README.md), which is a full routing service. The standalone indexer provides only the indexing and query layer without routing logic. -For Dynamo-native remote indexing, use `--serve-indexer` on `dynamo.frontend` or `dynamo.router` and `--use-remote-indexer` on consumers instead. That request-plane service reuses the router's existing event ingestion and recovery machinery; it is not implemented by `dynamo.indexer`. +For Dynamo-native remote indexing, use `--serve-indexer` on `dingo.frontend` or `dynamo.router` and `--use-remote-indexer` on consumers instead. That request-plane service reuses the router's existing event ingestion and recovery machinery; it is not implemented by `dynamo.indexer`. The HTTP API follows the [Mooncake KV Indexer RFC](https://github.com/kvcache-ai/Mooncake/issues/1403) conventions. diff --git a/docs/contribution-guide.md b/docs/contribution-guide.md index d0c464a08f2d..b4145c41d3c7 100644 --- a/docs/contribution-guide.md +++ b/docs/contribution-guide.md @@ -180,7 +180,7 @@ uv pip install -e . #### 8. Verify the Build ```bash -python3 -m dynamo.frontend --help +python3 -m dingo.frontend --help ``` > [!TIP] diff --git a/docs/contribution-guide.zh-CN.md b/docs/contribution-guide.zh-CN.md index 535d62c7164d..06a4872b20c9 100644 --- a/docs/contribution-guide.zh-CN.md +++ b/docs/contribution-guide.zh-CN.md @@ -181,7 +181,7 @@ uv pip install -e . #### 8. 验证构建 ```bash -python3 -m dynamo.frontend --help +python3 -m dingo.frontend --help ``` > [!TIP] diff --git a/docs/design-docs/event-plane.md b/docs/design-docs/event-plane.md index 0fb0b815422d..fca41956f407 100644 --- a/docs/design-docs/event-plane.md +++ b/docs/design-docs/event-plane.md @@ -78,7 +78,7 @@ python3 -m dynamo.vllm --model Qwen/Qwen3-0.6B \ --kv-events-config '{"publisher":"nats","topic":"kv-events","enable_kv_cache_events":true}' # Start frontend -- it subscribes to events from NATS automatically -python3 -m dynamo.frontend --router-mode kv +python3 -m dingo.frontend --router-mode kv ``` ## ZMQ Transport @@ -99,7 +99,7 @@ python3 -m dynamo.vllm --model Qwen/Qwen3-0.6B \ --kv-events-config '{"publisher":"zmq","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' # Start frontend -- discovers workers and connects directly -python3 -m dynamo.frontend --router-mode kv +python3 -m dingo.frontend --router-mode kv ``` ## Disabling the Event Plane @@ -107,7 +107,7 @@ python3 -m dynamo.frontend --router-mode kv If you do not need KV-aware routing, you can disable the event plane entirely: ```bash -python3 -m dynamo.frontend --router-mode kv --no-router-kv-events +python3 -m dingo.frontend --router-mode kv --no-router-kv-events ``` With `--no-router-kv-events`: diff --git a/docs/design-docs/request-plane.md b/docs/design-docs/request-plane.md index 87e81f44f4f9..b6425d285aba 100644 --- a/docs/design-docs/request-plane.md +++ b/docs/design-docs/request-plane.md @@ -76,7 +76,7 @@ export DYN_TCP_RPC_HOST=0.0.0.0 # Default host # export DYN_TCP_RPC_PORT=9999 # Optional: specify a fixed port # Run your Dynamo service -DYN_REQUEST_PLANE=tcp python -m dynamo.frontend --http-port=8000 & +DYN_REQUEST_PLANE=tcp python -m dingo.frontend --http-port=8000 & DYN_REQUEST_PLANE=tcp python -m dynamo.vllm --model Qwen/Qwen3-0.6B ``` @@ -112,7 +112,7 @@ NATS provides durable jetstream messaging for request plane and can be used for export DYN_REQUEST_PLANE=nats # Run your Dynamo service -DYN_REQUEST_PLANE=nats python -m dynamo.frontend --http-port=8000 & +DYN_REQUEST_PLANE=nats python -m dingo.frontend --http-port=8000 & DYN_REQUEST_PLANE=nats python -m dynamo.vllm --model Qwen/Qwen3-0.6B ``` diff --git a/docs/development/unified-backends.md b/docs/development/unified-backends.md index b23f1f004c93..e734a9e1b57f 100644 --- a/docs/development/unified-backends.md +++ b/docs/development/unified-backends.md @@ -700,7 +700,7 @@ pip install -e . my-backend --model-name Qwen/Qwen3-0.6B --namespace dynamo # In another shell, start the Dynamo frontend: -python -m dynamo.frontend --http-port 8000 +python -m dingo.frontend --http-port 8000 ``` Then send a request: @@ -1545,7 +1545,7 @@ cargo build --release --endpoint generate # In another shell, start the Python frontend from the dynamo repo: -python -m dynamo.frontend --http-port 8000 +python -m dingo.frontend --http-port 8000 ``` Then send a request: diff --git a/docs/digest/agentic-inference/agentic-harnesses.md b/docs/digest/agentic-inference/agentic-harnesses.md index 6002e236ed37..89967d3485a5 100644 --- a/docs/digest/agentic-inference/agentic-harnesses.md +++ b/docs/digest/agentic-inference/agentic-harnesses.md @@ -29,7 +29,7 @@ To reproduce our results, configure the frontend with the Anthropic-compatible A Putting all of this together: ```bash -python -m dynamo.frontend \ +python -m dingo.frontend \ --http-port 8000 \ --enable-anthropic-api \ --strip-anthropic-preamble \ diff --git a/docs/documentation-style-guide.md b/docs/documentation-style-guide.md index 9a5eadca9cd9..a8191daa717d 100644 --- a/docs/documentation-style-guide.md +++ b/docs/documentation-style-guide.md @@ -144,7 +144,7 @@ Much of this prose is now drafted by agents. Edit it so it does not read that wa paths, and env vars (`DYN_*`) in backticks. ```bash -python3 -m dynamo.frontend --router-mode kv +python3 -m dingo.frontend --router-mode kv ``` - **Diagrams** use ` ```mermaid ` blocks. **Images** live under `docs/assets/img/` with descriptive diff --git a/docs/fault-tolerance/graceful-shutdown.md b/docs/fault-tolerance/graceful-shutdown.md index 62dde6ed7df3..3fbaccc6b358 100644 --- a/docs/fault-tolerance/graceful-shutdown.md +++ b/docs/fault-tolerance/graceful-shutdown.md @@ -240,7 +240,7 @@ Match `terminationGracePeriodSeconds` to your expected request completion time: Enable migration at the frontend to allow request recovery when workers shut down: ```bash -python3 -m dynamo.frontend ... --migration-limit 3 # Allow up to 3 migration attempts +python3 -m dingo.frontend ... --migration-limit 3 # Allow up to 3 migration attempts ``` This allows the frontend to automatically retry disconnected streams on healthy workers. diff --git a/docs/fault-tolerance/request-rejection.md b/docs/fault-tolerance/request-rejection.md index 419181e5a1a7..c37db207a35b 100644 --- a/docs/fault-tolerance/request-rejection.md +++ b/docs/fault-tolerance/request-rejection.md @@ -46,7 +46,7 @@ When all workers exceed their configured busy thresholds, new requests receive a Configure busy thresholds when starting the frontend. `--admission-control token-capacity` is required to activate the thresholds; the default (`none`) leaves them disabled. For decode-block rejection, start the frontend in KV router mode so the worker load metrics path is active. ```bash -python -m dynamo.frontend \ +python -m dingo.frontend \ --admission-control token-capacity \ --router-mode kv \ --active-decode-blocks-threshold 0.85 \ @@ -292,7 +292,7 @@ To disable request rejection entirely: ```bash # Simply don't set the threshold arguments -python -m dynamo.frontend +python -m dingo.frontend ``` Without thresholds configured, all requests are accepted regardless of worker load. diff --git a/docs/features/diffusion/fastvideo.md b/docs/features/diffusion/fastvideo.md index 22d40adcf968..1cbbfcff4563 100644 --- a/docs/features/diffusion/fastvideo.md +++ b/docs/features/diffusion/fastvideo.md @@ -106,7 +106,7 @@ Environment variables: | `NUM_GPUS` | `1` | Number of GPUs | | `HTTP_PORT` | `8000` | Frontend HTTP port | | `WORKER_EXTRA_ARGS` | — | Extra flags for `worker.py` (for example, `--enable-optimizations --attention-backend FLASH_ATTN`) | -| `FRONTEND_EXTRA_ARGS` | — | Extra flags for `dynamo.frontend` | +| `FRONTEND_EXTRA_ARGS` | — | Extra flags for `dingo.frontend` | Example: @@ -119,7 +119,7 @@ WORKER_EXTRA_ARGS="--enable-optimizations --attention-backend FLASH_ATTN" \ ``` > [!NOTE] -> `--enable-optimizations` and `--attention-backend` are `worker.py` flags, not `dynamo.frontend` flags, so pass them through `WORKER_EXTRA_ARGS` when you want a non-default worker configuration. +> `--enable-optimizations` and `--attention-backend` are `worker.py` flags, not `dingo.frontend` flags, so pass them through `WORKER_EXTRA_ARGS` when you want a non-default worker configuration. The script writes logs to: diff --git a/docs/features/tokenizer/README.md b/docs/features/tokenizer/README.md index c6206445cc46..f65531dae805 100644 --- a/docs/features/tokenizer/README.md +++ b/docs/features/tokenizer/README.md @@ -66,17 +66,17 @@ Enable `fastokens` on the frontend with either the CLI flag or the environment v ```bash # CLI flag -python -m dynamo.frontend --tokenizer fastokens +python -m dingo.frontend --tokenizer fastokens # Environment variable export DYN_TOKENIZER=fastokens -python -m dynamo.frontend +python -m dingo.frontend ``` To return to the default HuggingFace tokenizer backend, omit the flag or set `DYN_TOKENIZER=default`. ```bash -python -m dynamo.frontend --tokenizer default +python -m dingo.frontend --tokenizer default ``` No client changes are required. Request payloads, OpenAI-compatible API behavior, and streamed responses remain the same. @@ -148,7 +148,7 @@ See the [frontend benchmarking guide](https://github.com/ai-dynamo/dynamo/tree/m ## Troubleshooting **I enabled `fastokens`, but the logs do not show `Using fastokens tokenizer backend`.** -Make sure the setting is applied to the frontend process, not only to the backend worker. For local launches, pass `--tokenizer fastokens` to `python -m dynamo.frontend` or set `DYN_TOKENIZER=fastokens` before starting the frontend. For benchmark DGD templates, use `DYN_TOKENIZER=fastokens`; the sweep runner maps `--tokenizers fastokens` to that value and restarts the frontend pod. +Make sure the setting is applied to the frontend process, not only to the backend worker. For local launches, pass `--tokenizer fastokens` to `python -m dingo.frontend` or set `DYN_TOKENIZER=fastokens` before starting the frontend. For benchmark DGD templates, use `DYN_TOKENIZER=fastokens`; the sweep runner maps `--tokenizers fastokens` to that value and restarts the frontend pod. **The frontend logs `Failed to load fastokens, falling back to HuggingFace`.** The model's tokenizer file uses a feature that `fastokens` does not support, or it is not a BPE `tokenizer.json` path. Dynamo has already fallen back to HuggingFace and should keep serving traffic. Check the tokenizer format, compare against the [tested models list](https://github.com/crusoecloud/fastokens#tested-models), and use `--tokenizer default` if you want to avoid the warning. diff --git a/docs/getting-started/building-from-source.md b/docs/getting-started/building-from-source.md index d1eb84427294..d7302ab4d9b3 100644 --- a/docs/getting-started/building-from-source.md +++ b/docs/getting-started/building-from-source.md @@ -88,7 +88,7 @@ uv pip install -e . ## 8. Verify the Build ```bash -python3 -m dynamo.frontend --help +python3 -m dingo.frontend --help ``` You should see the frontend command help output. diff --git a/docs/getting-started/building-from-source.zh-CN.md b/docs/getting-started/building-from-source.zh-CN.md index 0721d4aac20c..fec30cf672a6 100644 --- a/docs/getting-started/building-from-source.zh-CN.md +++ b/docs/getting-started/building-from-source.zh-CN.md @@ -89,7 +89,7 @@ uv pip install -e . ## 8. 验证构建 ```bash -python3 -m dynamo.frontend --help +python3 -m dingo.frontend --help ``` 你应该会看到 frontend 命令的帮助输出。 diff --git a/docs/getting-started/local-installation.md b/docs/getting-started/local-installation.md index 4358bfc45681..aa14da0dc861 100644 --- a/docs/getting-started/local-installation.md +++ b/docs/getting-started/local-installation.md @@ -117,7 +117,7 @@ This guide uses `--discovery-backend file`. For etcd setup, see [Service Discove Verify the CLI is installed and callable: ```bash -python3 -m dynamo.frontend --help +python3 -m dingo.frontend --help ``` If you cloned the repository, you can run additional system checks: @@ -130,14 +130,14 @@ python3 dev/sanity_check.py ```bash # Start the OpenAI compatible frontend (default port is 8000) -python3 -m dynamo.frontend --discovery-backend file +python3 -m dingo.frontend --discovery-backend file ``` To run in a single terminal (useful in containers), append `> logfile.log 2>&1 &` to run processes in background: ```bash -python3 -m dynamo.frontend --discovery-backend file > dynamo.frontend.log 2>&1 & +python3 -m dingo.frontend --discovery-backend file > dingo.frontend.log 2>&1 & ``` ### Start a Worker diff --git a/docs/getting-started/local-installation.zh-CN.md b/docs/getting-started/local-installation.zh-CN.md index 829ce3c36054..2f097693b1d4 100644 --- a/docs/getting-started/local-installation.zh-CN.md +++ b/docs/getting-started/local-installation.zh-CN.md @@ -117,7 +117,7 @@ Dynamo 组件通过共享后端相互发现。可使用两个选项： 验证 CLI 已安装并可调用： ```bash -python3 -m dynamo.frontend --help +python3 -m dingo.frontend --help ``` 如果你克隆了仓库，可以运行其他系统检查： @@ -130,14 +130,14 @@ python3 dev/sanity_check.py ```bash # Start the OpenAI compatible frontend (default port is 8000) -python3 -m dynamo.frontend --discovery-backend file +python3 -m dingo.frontend --discovery-backend file ``` 如需在单个终端中运行（在容器中很有用），追加 `> logfile.log 2>&1 &` 以在后台运行进程： ```bash -python3 -m dynamo.frontend --discovery-backend file > dynamo.frontend.log 2>&1 & +python3 -m dingo.frontend --discovery-backend file > dingo.frontend.log 2>&1 & ``` ### 启动 Worker diff --git a/docs/getting-started/quickstart.mdx b/docs/getting-started/quickstart.mdx index 00ab4f29cc0a..ef34ab058f7f 100644 --- a/docs/getting-started/quickstart.mdx +++ b/docs/getting-started/quickstart.mdx @@ -65,7 +65,7 @@ For container versions and tags, see [Release Artifacts](../reference/release-ar In your container, start the OpenAI-compatible frontend on port 8000: ```bash -python3 -m dynamo.frontend --discovery-backend file +python3 -m dingo.frontend --discovery-backend file ``` diff --git a/docs/getting-started/quickstart.zh-CN.mdx b/docs/getting-started/quickstart.zh-CN.mdx index ae757d8601db..27f72ad94544 100644 --- a/docs/getting-started/quickstart.zh-CN.mdx +++ b/docs/getting-started/quickstart.zh-CN.mdx @@ -65,7 +65,7 @@ Dynamo 与后端无关 — 每种安装路径都适用于 **SGLang**、**TensorR 在容器中，在端口 8000 上启动 OpenAI 兼容前端： ```bash -python3 -m dynamo.frontend --discovery-backend file +python3 -m dingo.frontend --discovery-backend file ``` diff --git a/docs/integrations/flexkv-integration.md b/docs/integrations/flexkv-integration.md index b72c584e7525..9ffc71c2da3b 100644 --- a/docs/integrations/flexkv-integration.md +++ b/docs/integrations/flexkv-integration.md @@ -52,7 +52,7 @@ python -m dynamo.vllm --model Qwen/Qwen3-0.6B --kv-transfer-config '{"kv_connect ```bash # Terminal 1: Start frontend -python -m dynamo.frontend & +python -m dingo.frontend & # Terminal 2: Start vLLM worker with FlexKV DYNAMO_USE_FLEXKV=1 \ @@ -66,7 +66,7 @@ For multi-worker deployments with KV-aware routing to maximize cache reuse: ```bash # Terminal 1: Start frontend with KV router -python -m dynamo.frontend \ +python -m dingo.frontend \ --router-mode kv \ --router-reset-states & @@ -108,7 +108,7 @@ FlexKV can be used with disaggregated prefill/decode serving. The prefill worker ```bash # Terminal 1: Start frontend -python -m dynamo.frontend & +python -m dingo.frontend & # Terminal 2: Decode worker (without FlexKV) CUDA_VISIBLE_DEVICES=0 python -m dynamo.vllm --model Qwen/Qwen3-0.6B \ diff --git a/docs/kubernetes/api-reference.md b/docs/kubernetes/api-reference.md index b7baffdc898f..ba6e53c75b37 100644 --- a/docs/kubernetes/api-reference.md +++ b/docs/kubernetes/api-reference.md @@ -908,7 +908,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `image` _string_ | Image is the container image for the frontend sidecar. | | Required: \{\}
| -| `args` _string array_ | Args overrides the default frontend arguments. When specified, these replace
the default ["-m", "dynamo.frontend"] entirely.
For example, ["-m", "dynamo.frontend", "--router-mode", "direct"] for GAIE deployments. | | Optional: \{\}
| +| `args` _string array_ | Args overrides the default frontend arguments. When specified, these replace
the default ["-m", "dingo.frontend"] entirely.
For example, ["-m", "dingo.frontend", "--router-mode", "direct"] for GAIE deployments. | | Optional: \{\}
| | `envFromSecret` _string_ | EnvFromSecret references a Secret whose key/value pairs will be exposed as
environment variables in the frontend sidecar container. | | Optional: \{\}
| | `envs` _[EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#envvar-v1-core) array_ | Envs defines additional environment variables for the frontend sidecar.
These are merged with (and can override) the auto-generated Dynamo env vars. | | Optional: \{\}
| diff --git a/docs/kubernetes/cloud-providers/ecs/ecs.md b/docs/kubernetes/cloud-providers/ecs/ecs.md index 71c2ab4339e2..f2282c1ee2be 100644 --- a/docs/kubernetes/cloud-providers/ecs/ecs.md +++ b/docs/kubernetes/cloud-providers/ecs/ecs.md @@ -92,7 +92,7 @@ Please follow steps below to create this task |ETCD_ENDPOINTS|Value|http://IP_ADDRESS:2379| |NATS_SERVER|Value|nats://IP_ADDRESS:4222| - Docker configuration -Add `sh,-c` in **Entry point** and `cd examples/backends/vllm && python -m dynamo.frontend --router-mode kv & python3 -m dynamo.vllm --model Qwen/Qwen3-0.6B --enforce-eager` in **Command** +Add `sh,-c` in **Entry point** and `cd examples/backends/vllm && python -m dingo.frontend --router-mode kv & python3 -m dynamo.vllm --model Qwen/Qwen3-0.6B --enforce-eager` in **Command** 2. Dynamo vLLM PrefillWorker Task Create the PrefillWorker task same as the frontend worker, except for following changes diff --git a/docs/kubernetes/deployment/create-deployment.md b/docs/kubernetes/deployment/create-deployment.md index b7de9f9c0bcf..f3bac30ad766 100644 --- a/docs/kubernetes/deployment/create-deployment.md +++ b/docs/kubernetes/deployment/create-deployment.md @@ -147,7 +147,7 @@ Here's a template structure based on the examples: Consult the corresponding sh file. Each of the python commands to launch a component will go into your yaml spec under the `extraPodSpec: -> mainContainer: -> args:` -The front end is launched with "python3 -m dynamo.frontend [--http-port 8000] [--router-mode kv]" +The front end is launched with "python3 -m dingo.frontend [--http-port 8000] [--router-mode kv]" Each worker will launch `python -m dynamo.YOUR_INFERENCE_BACKEND --model YOUR_MODEL --your-flags `command. diff --git a/docs/kubernetes/gateway-api/README.mdx b/docs/kubernetes/gateway-api/README.mdx index cb7d457425a0..98ef435d0337 100644 --- a/docs/kubernetes/gateway-api/README.mdx +++ b/docs/kubernetes/gateway-api/README.mdx @@ -150,7 +150,7 @@ podTemplate: - name: sidecar-frontend args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - direct ``` diff --git a/docs/kubernetes/gateway-api/reference.mdx b/docs/kubernetes/gateway-api/reference.mdx index 417e931aee21..d0c3ae99f735 100644 --- a/docs/kubernetes/gateway-api/reference.mdx +++ b/docs/kubernetes/gateway-api/reference.mdx @@ -76,7 +76,7 @@ podTemplate: - name: sidecar-frontend args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - direct ``` diff --git a/docs/kubernetes/topology-aware-scheduling.md b/docs/kubernetes/topology-aware-scheduling.md index edb8ae7f9686..186d98552245 100644 --- a/docs/kubernetes/topology-aware-scheduling.md +++ b/docs/kubernetes/topology-aware-scheduling.md @@ -73,7 +73,7 @@ spec: image: my-image command: ["/bin/sh", "-c"] args: - - python3 -m dynamo.frontend + - python3 -m dingo.frontend ``` ### Example 2: Service-Level Constraint Only @@ -114,7 +114,7 @@ spec: image: my-image command: ["/bin/sh", "-c"] args: - - python3 -m dynamo.frontend + - python3 -m dingo.frontend ``` ### Example 3: Mixed (Deployment-Level Default + Per-Service Override) @@ -157,7 +157,7 @@ spec: image: my-image command: ["/bin/sh", "-c"] args: - - python3 -m dynamo.frontend + - python3 -m dingo.frontend ``` ## Hierarchy Rules diff --git a/docs/observability/health-checks.md b/docs/observability/health-checks.md index 958f07849a5f..98c810ee9cce 100644 --- a/docs/observability/health-checks.md +++ b/docs/observability/health-checks.md @@ -30,7 +30,7 @@ Enable health checks and query endpoints: ```bash # Start your Dynamo components (default port 8000, override with --http-port or DYN_HTTP_PORT env var) -python -m dynamo.frontend & +python -m dingo.frontend & # Enable system status server on port 8081 DYN_SYSTEM_PORT=8081 python -m dynamo.vllm --model Qwen/Qwen3-0.6B --enforce-eager & diff --git a/docs/observability/logging.md b/docs/observability/logging.md index 1c3d0ce6a226..9c18a2f1bd9f 100644 --- a/docs/observability/logging.md +++ b/docs/observability/logging.md @@ -62,7 +62,7 @@ export DYN_LOGGING_JSONL=true export DYN_LOG=debug # Start your Dynamo components (default port 8000, override with --http-port or DYN_HTTP_PORT env var) -python -m dynamo.frontend & +python -m dingo.frontend & python -m dynamo.vllm --model Qwen/Qwen3-0.6B --enforce-eager & ``` @@ -133,7 +133,7 @@ export DYN_LOGGING_JSONL=true export DYN_LOG=debug # Set to debug to see detailed trace logs # Start your Dynamo components (e.g., frontend and worker) (default port 8000, override with --http-port or DYN_HTTP_PORT env var) -python -m dynamo.frontend & +python -m dingo.frontend & python -m dynamo.vllm --model Qwen/Qwen3-0.6B --enforce-eager & ``` diff --git a/docs/observability/metrics.md b/docs/observability/metrics.md index 3047b0bb2246..78d070819f0b 100644 --- a/docs/observability/metrics.md +++ b/docs/observability/metrics.md @@ -36,7 +36,7 @@ Launch a frontend and vLLM backend to test metrics: ```bash # Start frontend (default port 8000, override with --http-port or DYN_HTTP_PORT env var) -$ python -m dynamo.frontend +$ python -m dingo.frontend # Enable backend worker's system metrics on port 8081 $ DYN_SYSTEM_PORT=8081 python -m dynamo.vllm --model Qwen/Qwen3-0.6B \ @@ -167,7 +167,7 @@ Backend `dynamo_component_*` series carry two groups of labels: the ones the Dyn #### Component Names -Values you will see in the `dynamo_component` label on `dynamo_component_*` series. The HTTP frontend (`python -m dynamo.frontend`) is **not** in this list — it exposes its own `dynamo_frontend_*` metric family, not `dynamo_component_*`. +Values you will see in the `dynamo_component` label on `dynamo_component_*` series. The HTTP frontend (`python -m dingo.frontend`) is **not** in this list — it exposes its own `dynamo_frontend_*` metric family, not `dynamo_component_*`. | Value | Meaning | |-------|---------| @@ -215,7 +215,7 @@ Some components expose additional metrics specific to their functionality: **Important:** The frontend and backend workers are separate components that expose metrics on different ports. See [Backend Component Metrics](#backend-component-metrics) for backend metrics. -The Dynamo HTTP Frontend (`python -m dynamo.frontend`) exposes `dynamo_frontend_*` metrics on port 8000 by default (configurable via `--http-port` or `DYN_HTTP_PORT`) at the `/metrics` endpoint. Most metrics include `model` labels containing the model name: +The Dynamo HTTP Frontend (`python -m dingo.frontend`) exposes `dynamo_frontend_*` metrics on port 8000 by default (configurable via `--http-port` or `DYN_HTTP_PORT`) at the `/metrics` endpoint. Most metrics include `model` labels containing the model name: - `dynamo_frontend_active_requests`: Number of requests currently being handled by the frontend, from HTTP handler entry until the response stream completes (gauge). This is the top-level in-flight count with no stage breakdown. - `dynamo_frontend_stage_requests`: Number of requests currently in a given frontend pipeline stage (gauge, labels: `stage`, `phase`). See [Stage and phase labels](#stage-and-phase-labels) below. diff --git a/docs/observability/prometheus-grafana.md b/docs/observability/prometheus-grafana.md index c845a3949173..bc305757d5c3 100644 --- a/docs/observability/prometheus-grafana.md +++ b/docs/observability/prometheus-grafana.md @@ -37,7 +37,7 @@ Start frontend and worker (a simple single GPU example): ```bash # Start frontend (default port 8000, override with --http-port or DYN_HTTP_PORT env var) -python -m dynamo.frontend & +python -m dingo.frontend & # Start vLLM worker with metrics enabled on port 8081 DYN_SYSTEM_PORT=8081 python -m dynamo.vllm --model Qwen/Qwen3-0.6B --enforce-eager diff --git a/docs/reasoning/README.md b/docs/reasoning/README.md index 953740954f56..dcf924bff6be 100644 --- a/docs/reasoning/README.md +++ b/docs/reasoning/README.md @@ -85,7 +85,7 @@ Reasoning parsing happens before tool call parsing. If a model emits both reason python -m dynamo.sglang --model Qwen/Qwen3.5-4B --dyn-tool-call-parser qwen3_coder --dyn-reasoning-parser qwen3 # launch frontend worker -python -m dynamo.frontend +python -m dingo.frontend ``` ### Reasoning Request Example diff --git a/docs/templates/component-readme.md b/docs/templates/component-readme.md index cde711dcae07..de556898cbb0 100644 --- a/docs/templates/component-readme.md +++ b/docs/templates/component-readme.md @@ -25,7 +25,7 @@ title: Component README ```bash # Add minimal usage example from existing docs # Example pattern (from Router): -# python -m dynamo.frontend --router-mode kv --http-port 8000 +# python -m dingo.frontend --router-mode kv --http-port 8000 ``` ### Kubernetes @@ -52,7 +52,7 @@ title: Component README ### Usage ```bash -python -m dynamo.frontend --router-mode kv --http-port 8000 +python -m dingo.frontend --router-mode kv --http-port 8000 ``` ### Kubernetes diff --git a/docs/tool-calling/README.md b/docs/tool-calling/README.md index f131f6859611..9a00065bc38f 100644 --- a/docs/tool-calling/README.md +++ b/docs/tool-calling/README.md @@ -87,7 +87,7 @@ parser exists for this format. python -m dynamo.sglang --model Qwen/Qwen3.5-4B --dyn-tool-call-parser qwen3_coder --dyn-reasoning-parser qwen3 # launch frontend worker -python -m dynamo.frontend +python -m dingo.frontend ``` ### Tool Calling Request Example diff --git a/docs/tool-calling/README.zh-CN.md b/docs/tool-calling/README.zh-CN.md index 7f30e63a4432..6a377c3036b4 100644 --- a/docs/tool-calling/README.zh-CN.md +++ b/docs/tool-calling/README.zh-CN.md @@ -74,7 +74,7 @@ python -m dynamo. --help python -m dynamo.sglang --model Qwen/Qwen3.5-4B --dyn-tool-call-parser qwen3_coder --dyn-reasoning-parser qwen3 # 启动 frontend -python -m dynamo.frontend +python -m dingo.frontend ``` ### 工具调用请求示例 diff --git a/docs/tool-calling/engine-fallback.md b/docs/tool-calling/engine-fallback.md index ae91e087deef..d2d6e29effe9 100644 --- a/docs/tool-calling/engine-fallback.md +++ b/docs/tool-calling/engine-fallback.md @@ -25,11 +25,11 @@ These are distinct from the Dynamo-native `--dyn-tool-call-parser` / `--dyn-reas ```bash # vLLM chat processor — frontend carries the parser flags, then launch the worker: -python -m dynamo.frontend --dyn-chat-processor vllm --tool-call-parser hermes --reasoning-parser qwen3 +python -m dingo.frontend --dyn-chat-processor vllm --tool-call-parser hermes --reasoning-parser qwen3 python -m dynamo.vllm --model Qwen/Qwen3-0.6B # SGLang chat processor -python -m dynamo.frontend --dyn-chat-processor sglang --tool-call-parser qwen25 --reasoning-parser qwen3 +python -m dingo.frontend --dyn-chat-processor sglang --tool-call-parser qwen25 --reasoning-parser qwen3 python -m dynamo.sglang --model Qwen/Qwen3-0.6B ``` diff --git a/docs/tool-calling/parser-configuration.md b/docs/tool-calling/parser-configuration.md index 2c75c44d54b7..9086e51b3ffc 100644 --- a/docs/tool-calling/parser-configuration.md +++ b/docs/tool-calling/parser-configuration.md @@ -44,8 +44,8 @@ Default (Dynamo-native) — the common case. The same `--dyn-*` flags work on ev ```bash # Frontend — chat processor defaults to `dynamo`, so these two are identical: -python -m dynamo.frontend -python -m dynamo.frontend --dyn-chat-processor dynamo +python -m dingo.frontend +python -m dingo.frontend --dyn-chat-processor dynamo # Worker selects the Dynamo parsers — same flags on vLLM, SGLang, or TRT-LLM: python -m dynamo.vllm --model Qwen/Qwen3-0.6B \ @@ -60,11 +60,11 @@ Engine fallback — only when Dynamo lacks a parser for your model. Supported on ```bash # vLLM chat processor — frontend carries the parser flags, then launch the worker: -python -m dynamo.frontend --dyn-chat-processor vllm --tool-call-parser hermes --reasoning-parser qwen3 +python -m dingo.frontend --dyn-chat-processor vllm --tool-call-parser hermes --reasoning-parser qwen3 python -m dynamo.vllm --model Qwen/Qwen3-0.6B # SGLang chat processor -python -m dynamo.frontend --dyn-chat-processor sglang --tool-call-parser qwen25 --reasoning-parser qwen3 +python -m dingo.frontend --dyn-chat-processor sglang --tool-call-parser qwen25 --reasoning-parser qwen3 python -m dynamo.sglang --model Qwen/Qwen3-0.6B ``` diff --git a/docs/tool-calling/structural-tag.md b/docs/tool-calling/structural-tag.md index 347de271d1d2..3627b9941a30 100644 --- a/docs/tool-calling/structural-tag.md +++ b/docs/tool-calling/structural-tag.md @@ -39,7 +39,7 @@ python -m dynamo.sglang \ --dyn-enable-structural-tag # Launch frontend -python -m dynamo.frontend +python -m dingo.frontend ``` Eligible tool-calling requests will now use xgrammar structural tags for guided diff --git a/dynamo.code-workspace b/dynamo.code-workspace index 2c0b63bcd5d8..2467642ef37b 100644 --- a/dynamo.code-workspace +++ b/dynamo.code-workspace @@ -29,7 +29,7 @@ "program": "${workspaceFolder}/.venv/bin/python", "args": [ "-m", - "dynamo.frontend", + "dingo.frontend", ], "initCommands": [ "settings set target.disable-aslr false" diff --git a/examples/backends/sample/launch/agg.sh b/examples/backends/sample/launch/agg.sh index 7e3aaeae1f9d..1768c0dddf2a 100755 --- a/examples/backends/sample/launch/agg.sh +++ b/examples/backends/sample/launch/agg.sh @@ -41,7 +41,7 @@ HTTP_PORT="${DYN_HTTP_PORT:-8000}" print_launch_banner "Launching Sample Aggregated Serving" "$MODEL_NAME" "$HTTP_PORT" # run frontend -python3 -m dynamo.frontend & +python3 -m dingo.frontend & # run sample worker python3 -m dynamo.common.backend.sample_main \ diff --git a/examples/backends/sample/launch/agg_diffusion.sh b/examples/backends/sample/launch/agg_diffusion.sh index 7b6158a7ee83..aaf61d0a7fa3 100755 --- a/examples/backends/sample/launch/agg_diffusion.sh +++ b/examples/backends/sample/launch/agg_diffusion.sh @@ -43,7 +43,7 @@ HTTP_PORT="${DYN_HTTP_PORT:-8000}" print_launch_banner "Launching Sample Diffusion Aggregated Serving" "$MODEL_NAME" "$HTTP_PORT" # run frontend -python3 -m dynamo.frontend & +python3 -m dingo.frontend & # run sample diffusion worker (serves /v1/images/generations) python3 -m dynamo.common.backend.sample_diffusion_main \ diff --git a/examples/backends/sample/launch/disagg.sh b/examples/backends/sample/launch/disagg.sh index 439e70c61db5..de81f6d1deef 100755 --- a/examples/backends/sample/launch/disagg.sh +++ b/examples/backends/sample/launch/disagg.sh @@ -4,7 +4,7 @@ # # Disaggregated serving with the sample (echo) backend — GPU-free smoke test. # -# Spawns dynamo.frontend plus one prefill worker and one decode worker, both +# Spawns dingo.frontend plus one prefill worker and one decode worker, both # backed by the sample engine. Prefill workers register as WorkerType::Prefill # (the unified Rust Worker overrides registration based on # WorkerConfig.disaggregation_mode); the frontend's PrefillRouter forwards @@ -51,7 +51,7 @@ HTTP_PORT="${DYN_HTTP_PORT:-8000}" print_launch_banner "Launching Sample Disaggregated Serving (CPU-only)" "$MODEL_NAME" "$HTTP_PORT" # run frontend -python3 -m dynamo.frontend & +python3 -m dingo.frontend & # Per-worker DYN_SYSTEM_PORT so parallel CI runs don't collide on the metrics # port. Mirrors examples/backends/vllm/launch/disagg.sh. diff --git a/examples/backends/sglang/launch/_test_agg.sh b/examples/backends/sglang/launch/_test_agg.sh index 4113e9d2b618..8bc39642448c 100755 --- a/examples/backends/sglang/launch/_test_agg.sh +++ b/examples/backends/sglang/launch/_test_agg.sh @@ -98,7 +98,7 @@ print_launch_banner "Launching Test-Aggregated + KV Router (2 GPUs, full observa cat < 1. Please make sure that the prefill instance is launched with `--load-balance-method round_robin` and `--prefill-round-robin-balance` is set for decode server. diff --git a/examples/backends/sglang/launch/disagg_router.sh b/examples/backends/sglang/launch/disagg_router.sh index 35857190e86c..e1baa72b3290 100755 --- a/examples/backends/sglang/launch/disagg_router.sh +++ b/examples/backends/sglang/launch/disagg_router.sh @@ -53,9 +53,9 @@ print_launch_banner "Launching Disaggregated + KV Routing (4 GPUs)" "$MODEL" "$H # Start frontend with KV routing # The frontend will automatically detect prefill workers and activate an internal prefill router # No standalone prefill router needed - the frontend handles prefill routing internally -# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) +# dingo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) OTEL_SERVICE_NAME=dynamo-frontend \ -python3 -m dynamo.frontend \ +python3 -m dingo.frontend \ --router-mode kv \ --router-reset-states & diff --git a/examples/backends/sglang/launch/disagg_same_gpu.sh b/examples/backends/sglang/launch/disagg_same_gpu.sh index 8d76d15f18de..3d818c31fd37 100755 --- a/examples/backends/sglang/launch/disagg_same_gpu.sh +++ b/examples/backends/sglang/launch/disagg_same_gpu.sh @@ -89,7 +89,7 @@ print_launch_banner "Launching Disaggregated (same GPU)" "$MODEL" "$HTTP_PORT" \ "Workers: 2 (prefill + decode, fraction is per worker)" # run ingress -# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) +# dingo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) # Set DYN_CHAT_PROCESSOR=sglang to exercise the Python pre/post processor instead of Rust. FRONTEND_ARGS=() if [[ -n "${DYN_CHAT_PROCESSOR:-}" ]]; then @@ -98,7 +98,7 @@ fi if [[ -n "${DYN_ROUTER_MODE:-}" ]]; then FRONTEND_ARGS+=(--router-mode "$DYN_ROUTER_MODE") fi -python3 -m dynamo.frontend "${FRONTEND_ARGS[@]}" & +python3 -m dingo.frontend "${FRONTEND_ARGS[@]}" & # NOTE: Each worker picks a random NCCL port (get_free_port) for torch.distributed. # This has a TOCTOU race — the port can be grabbed before init_process_group binds it, diff --git a/examples/backends/sglang/launch/image_diffusion.sh b/examples/backends/sglang/launch/image_diffusion.sh index 0a70521535f8..7347b69ee432 100755 --- a/examples/backends/sglang/launch/image_diffusion.sh +++ b/examples/backends/sglang/launch/image_diffusion.sh @@ -93,7 +93,7 @@ fi # Launch frontend echo "Starting Dynamo Frontend on port $HTTP_PORT..." -python3 -m dynamo.frontend \ +python3 -m dingo.frontend \ --http-port "$HTTP_PORT" & sleep 2 diff --git a/examples/backends/sglang/launch/lora/agg_lora.sh b/examples/backends/sglang/launch/lora/agg_lora.sh index 665e07b6eeee..0555b7ea0b53 100755 --- a/examples/backends/sglang/launch/lora/agg_lora.sh +++ b/examples/backends/sglang/launch/lora/agg_lora.sh @@ -46,7 +46,7 @@ echo " -d '{\"model\": \"${LORA_NAME}\", \"messages\": [{\"role\": \"user\", echo "==========================================" # Frontend -python3 -m dynamo.frontend & +python3 -m dingo.frontend & # Worker DYN_SYSTEM_ENABLED=true DYN_SYSTEM_PORT=${SYSTEM_PORT} \ diff --git a/examples/backends/sglang/launch/multimodal_disagg.sh b/examples/backends/sglang/launch/multimodal_disagg.sh index a700fc26f88f..3d62d79a55ca 100755 --- a/examples/backends/sglang/launch/multimodal_disagg.sh +++ b/examples/backends/sglang/launch/multimodal_disagg.sh @@ -124,8 +124,8 @@ HTTP_PORT="${DYN_HTTP_PORT:-8000}" print_launch_banner --multimodal "Launching Disaggregated Multimodal E/P/D" "$MODEL_NAME" "$HTTP_PORT" # run ingress -# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) -python3 -m dynamo.frontend & +# dingo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) +python3 -m dingo.frontend & # run SGLang multimodal encode worker (frontend-facing: encodes images, routes to worker) echo "Starting encode worker on GPU $DYN_ENCODE_WORKER_GPU (GPU mem: $DYN_ENCODE_GPU_MEM)..." diff --git a/examples/backends/sglang/launch/multimodal_epd.sh b/examples/backends/sglang/launch/multimodal_epd.sh index 0ca17635d161..77266a4cbfc9 100755 --- a/examples/backends/sglang/launch/multimodal_epd.sh +++ b/examples/backends/sglang/launch/multimodal_epd.sh @@ -141,8 +141,8 @@ fi print_launch_banner --multimodal "Launching Multimodal E/PD ($GPU_LABEL)" "$MODEL_NAME" "$HTTP_PORT" # run ingress -# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) -python3 -m dynamo.frontend & +# dingo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) +python3 -m dingo.frontend & # run SGLang multimodal encode worker (frontend-facing: encodes images, routes to worker) echo "Starting encode worker on GPU $DYN_ENCODE_WORKER_GPU..." diff --git a/examples/backends/sglang/launch/text-to-video-diffusion.sh b/examples/backends/sglang/launch/text-to-video-diffusion.sh index 30ec9ef3b7e1..b0b204ff8648 100755 --- a/examples/backends/sglang/launch/text-to-video-diffusion.sh +++ b/examples/backends/sglang/launch/text-to-video-diffusion.sh @@ -117,7 +117,7 @@ CURL # Launch frontend echo "Starting Dynamo Frontend on port $HTTP_PORT..." -python3 -m dynamo.frontend \ +python3 -m dingo.frontend \ --http-port "$HTTP_PORT" & sleep 2 diff --git a/examples/backends/sglang/test_sglang_expert_info.py b/examples/backends/sglang/test_sglang_expert_info.py index c10cc900d11a..20dc976773fd 100644 --- a/examples/backends/sglang/test_sglang_expert_info.py +++ b/examples/backends/sglang/test_sglang_expert_info.py @@ -47,7 +47,7 @@ def start_frontend(): os.makedirs(LOG_DIR, exist_ok=True) log = open(f"{LOG_DIR}/frontend.log", "w") - cmd = [sys.executable, "-m", "dynamo.frontend", "--http-port", str(FRONTEND_PORT)] + cmd = [sys.executable, "-m", "dingo.frontend", "--http-port", str(FRONTEND_PORT)] print(f" Command: {' '.join(cmd)}") print(f" Logs: {LOG_DIR}/frontend.log") process = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT) diff --git a/examples/backends/sglang/test_sglang_profile.py b/examples/backends/sglang/test_sglang_profile.py index 923744c1f088..e361a8eb5bf7 100644 --- a/examples/backends/sglang/test_sglang_profile.py +++ b/examples/backends/sglang/test_sglang_profile.py @@ -45,7 +45,7 @@ def start_frontend(): cmd = [ "python", "-m", - "dynamo.frontend", + "dingo.frontend", "--http-port", str(PORT), ] diff --git a/examples/backends/tritonserver/launch/identity.sh b/examples/backends/tritonserver/launch/identity.sh index 0bda8f6a4e65..ab126a4ed415 100755 --- a/examples/backends/tritonserver/launch/identity.sh +++ b/examples/backends/tritonserver/launch/identity.sh @@ -112,7 +112,7 @@ export DYN_DISCOVERY_BACKEND="$DISCOVERY_BACKEND" # Run frontend in background # --kserve-grpc-server enables the KServe gRPC endpoint for tensor models echo "Starting Dynamo frontend..." -python3 -m dynamo.frontend --kserve-grpc-server --discovery-backend "$DISCOVERY_BACKEND" & +python3 -m dingo.frontend --kserve-grpc-server --discovery-backend "$DISCOVERY_BACKEND" & FRONTEND_PID=$! # Give frontend time to start diff --git a/examples/backends/trtllm/deploy/README.md b/examples/backends/trtllm/deploy/README.md index 8582a69b1580..6b85e92d3e2e 100644 --- a/examples/backends/trtllm/deploy/README.md +++ b/examples/backends/trtllm/deploy/README.md @@ -227,7 +227,7 @@ TensorRT-LLM workers are configured through command-line arguments in the deploy Send a test request to verify your deployment. See the [client section](../../../../docs/backends/vllm/README.md#client) for detailed instructions. -**Note:** For multi-node deployments, target the node running `python3 -m dynamo.frontend `. +**Note:** For multi-node deployments, target the node running `python3 -m dingo.frontend `. ## Model Configuration diff --git a/examples/backends/trtllm/deploy/disagg-multinode.yaml b/examples/backends/trtllm/deploy/disagg-multinode.yaml index 27e1ca83437f..e70ec50c015b 100644 --- a/examples/backends/trtllm/deploy/disagg-multinode.yaml +++ b/examples/backends/trtllm/deploy/disagg-multinode.yaml @@ -102,7 +102,7 @@ spec: command: - python3 - -m - - dynamo.frontend + - dingo.frontend args: - --http-port - "8000" diff --git a/examples/backends/trtllm/deploy/disagg_planner.yaml b/examples/backends/trtllm/deploy/disagg_planner.yaml index ebecb54a6c4d..3923497970e4 100644 --- a/examples/backends/trtllm/deploy/disagg_planner.yaml +++ b/examples/backends/trtllm/deploy/disagg_planner.yaml @@ -18,7 +18,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --http-port - "8000" - --kv-cache-block-size diff --git a/examples/backends/trtllm/deploy/v1beta1/disagg-multinode.yaml b/examples/backends/trtllm/deploy/v1beta1/disagg-multinode.yaml index ab87088d40cd..a3876c5e921d 100644 --- a/examples/backends/trtllm/deploy/v1beta1/disagg-multinode.yaml +++ b/examples/backends/trtllm/deploy/v1beta1/disagg-multinode.yaml @@ -91,7 +91,7 @@ spec: command: - python3 - -m - - dynamo.frontend + - dingo.frontend image: my-registry/tensorrtllm-runtime:my-tag name: main volumeMounts: diff --git a/examples/backends/trtllm/deploy/v1beta1/disagg_planner.yaml b/examples/backends/trtllm/deploy/v1beta1/disagg_planner.yaml index e3abbd989f9f..eef8ff03c89e 100644 --- a/examples/backends/trtllm/deploy/v1beta1/disagg_planner.yaml +++ b/examples/backends/trtllm/deploy/v1beta1/disagg_planner.yaml @@ -13,7 +13,7 @@ spec: containers: - args: - -m - - dynamo.frontend + - dingo.frontend - --http-port - "8000" - --kv-cache-block-size diff --git a/examples/backends/trtllm/launch/agg.sh b/examples/backends/trtllm/launch/agg.sh index d76472f9377d..f93de114df16 100755 --- a/examples/backends/trtllm/launch/agg.sh +++ b/examples/backends/trtllm/launch/agg.sh @@ -71,9 +71,9 @@ HTTP_PORT="${DYN_HTTP_PORT:-8000}" print_launch_banner "Launching Aggregated Serving" "$MODEL_PATH" "$HTTP_PORT" # run frontend -# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) +# dingo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) OTEL_SERVICE_NAME=dynamo-frontend \ -python3 -m dynamo.frontend & +python3 -m dingo.frontend & # run worker # Additional command line args can be passed diff --git a/examples/backends/trtllm/launch/agg_image_diffusion.sh b/examples/backends/trtllm/launch/agg_image_diffusion.sh index 18cb63b2c6f3..a41aa7a599ee 100755 --- a/examples/backends/trtllm/launch/agg_image_diffusion.sh +++ b/examples/backends/trtllm/launch/agg_image_diffusion.sh @@ -62,7 +62,7 @@ print_curl_footer <&1 | sed "s/^/[frontend] /" & diff --git a/examples/backends/vllm/deploy/agg_router_kv_approx.yaml b/examples/backends/vllm/deploy/agg_router_kv_approx.yaml index a0b24f147bb4..5d43e47889c3 100644 --- a/examples/backends/vllm/deploy/agg_router_kv_approx.yaml +++ b/examples/backends/vllm/deploy/agg_router_kv_approx.yaml @@ -21,7 +21,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - kv - --no-kv-events diff --git a/examples/backends/vllm/deploy/disagg-multinode.yaml b/examples/backends/vllm/deploy/disagg-multinode.yaml index 73cc7b8a478f..9da100978793 100644 --- a/examples/backends/vllm/deploy/disagg-multinode.yaml +++ b/examples/backends/vllm/deploy/disagg-multinode.yaml @@ -17,7 +17,7 @@ spec: command: - python3 - -m - - dynamo.frontend + - dingo.frontend args: - --http-port - "8000" diff --git a/examples/backends/vllm/deploy/gaie/agg.yaml b/examples/backends/vllm/deploy/gaie/agg.yaml index 751b3a3732f4..2a8a2e407334 100644 --- a/examples/backends/vllm/deploy/gaie/agg.yaml +++ b/examples/backends/vllm/deploy/gaie/agg.yaml @@ -85,7 +85,7 @@ spec: - name: sidecar-frontend args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - direct envFrom: diff --git a/examples/backends/vllm/deploy/gaie/disagg.yaml b/examples/backends/vllm/deploy/gaie/disagg.yaml index d9871eb0ea30..053da1f3064d 100644 --- a/examples/backends/vllm/deploy/gaie/disagg.yaml +++ b/examples/backends/vllm/deploy/gaie/disagg.yaml @@ -157,7 +157,7 @@ spec: - name: sidecar-frontend args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - direct envFrom: @@ -215,7 +215,7 @@ spec: - name: sidecar-frontend args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - direct envFrom: diff --git a/examples/backends/vllm/deploy/gms-failover.yaml b/examples/backends/vllm/deploy/gms-failover.yaml index 133901214425..7e833dde2caa 100644 --- a/examples/backends/vllm/deploy/gms-failover.yaml +++ b/examples/backends/vllm/deploy/gms-failover.yaml @@ -92,4 +92,4 @@ spec: extraPodSpec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest - # command: ["python3", "-m", "dynamo.frontend"] + # command: ["python3", "-m", "dingo.frontend"] diff --git a/examples/backends/vllm/deploy/v1beta1/agg_router_kv_approx.yaml b/examples/backends/vllm/deploy/v1beta1/agg_router_kv_approx.yaml index f56f177ad37e..585d53786a73 100644 --- a/examples/backends/vllm/deploy/v1beta1/agg_router_kv_approx.yaml +++ b/examples/backends/vllm/deploy/v1beta1/agg_router_kv_approx.yaml @@ -17,7 +17,7 @@ spec: containers: - args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - kv - --no-kv-events diff --git a/examples/backends/vllm/deploy/v1beta1/disagg-multinode.yaml b/examples/backends/vllm/deploy/v1beta1/disagg-multinode.yaml index 6f06c77e6cfe..e0fe7fdd57e3 100644 --- a/examples/backends/vllm/deploy/v1beta1/disagg-multinode.yaml +++ b/examples/backends/vllm/deploy/v1beta1/disagg-multinode.yaml @@ -17,7 +17,7 @@ spec: command: - python3 - -m - - dynamo.frontend + - dingo.frontend image: my-registry/vllm-runtime:my-tag name: main workingDir: /workspace/examples/backends/vllm diff --git a/examples/backends/vllm/deploy/v1beta1/gms-failover.yaml b/examples/backends/vllm/deploy/v1beta1/gms-failover.yaml index 1ac1813be645..34d8ae3c59b9 100644 --- a/examples/backends/vllm/deploy/v1beta1/gms-failover.yaml +++ b/examples/backends/vllm/deploy/v1beta1/gms-failover.yaml @@ -113,7 +113,7 @@ spec: - envFrom: - secretRef: name: hf-token-secret - # command: ["python3", "-m", "dynamo.frontend"] + # command: ["python3", "-m", "dingo.frontend"] image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest name: main replicas: 1 diff --git a/examples/backends/vllm/launch/agg.sh b/examples/backends/vllm/launch/agg.sh index fa92895dac4e..fd4b9683a8e8 100755 --- a/examples/backends/vllm/launch/agg.sh +++ b/examples/backends/vllm/launch/agg.sh @@ -57,8 +57,8 @@ HTTP_PORT="${DYN_HTTP_PORT:-8000}" print_launch_banner "Launching Aggregated Serving (1 GPU)" "$MODEL" "$HTTP_PORT" # run ingress -# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) -python -m dynamo.frontend & +# dingo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) +python -m dingo.frontend & # run worker # --enforce-eager is added for quick deployment. for production use, need to remove this flag diff --git a/examples/backends/vllm/launch/agg_embed.sh b/examples/backends/vllm/launch/agg_embed.sh index fee4be87ee4a..3254e72ffa90 100755 --- a/examples/backends/vllm/launch/agg_embed.sh +++ b/examples/backends/vllm/launch/agg_embed.sh @@ -58,8 +58,8 @@ print_curl_footer </dev/null 2>&1 || { exit 1 } -python -m dynamo.frontend & +python -m dingo.frontend & DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT:-8081} \ python -m dynamo.vllm --model "$MODEL" --enforce-eager \ diff --git a/examples/backends/vllm/launch/agg_lmcache_multiproc.sh b/examples/backends/vllm/launch/agg_lmcache_multiproc.sh index e41767be2cc6..76ead293d5ba 100755 --- a/examples/backends/vllm/launch/agg_lmcache_multiproc.sh +++ b/examples/backends/vllm/launch/agg_lmcache_multiproc.sh @@ -35,7 +35,7 @@ GPU_MEM_ARGS=$(build_vllm_gpu_mem_args) HTTP_PORT="${DYN_HTTP_PORT:-8000}" print_launch_banner "Launching Aggregated + LMCache + Multiproc (1 GPU)" "$MODEL" "$HTTP_PORT" -python -m dynamo.frontend & +python -m dingo.frontend & DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT:-8081} \ PROMETHEUS_MULTIPROC_DIR="$PROMETHEUS_MULTIPROC_DIR" \ diff --git a/examples/backends/vllm/launch/agg_multimodal.sh b/examples/backends/vllm/launch/agg_multimodal.sh index 81c6836961ea..fe892f0e8421 100755 --- a/examples/backends/vllm/launch/agg_multimodal.sh +++ b/examples/backends/vllm/launch/agg_multimodal.sh @@ -84,8 +84,8 @@ print_curl_footer <&1 | tee $LOG_DIR/dsr1_dep_ingress.log & + DYN_LOG=debug python -m dingo.frontend --router-mode kv 2>&1 | tee $LOG_DIR/dsr1_dep_ingress.log & fi mkdir -p $LOG_DIR diff --git a/examples/backends/vllm/launch/lora/agg_lora.sh b/examples/backends/vllm/launch/lora/agg_lora.sh index 17a2dbed6952..7feeae8b494e 100755 --- a/examples/backends/vllm/launch/lora/agg_lora.sh +++ b/examples/backends/vllm/launch/lora/agg_lora.sh @@ -56,8 +56,8 @@ echo "" echo "==========================================" # run ingress -# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var. -python -m dynamo.frontend & +# dingo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var. +python -m dingo.frontend & # ---- Tunable (override via env vars) ---- MAX_MODEL_LEN="${MAX_MODEL_LEN:-4096}" diff --git a/examples/backends/vllm/launch/lora/agg_lora_router.sh b/examples/backends/vllm/launch/lora/agg_lora_router.sh index c19981a7e1dc..29b334237649 100755 --- a/examples/backends/vllm/launch/lora/agg_lora_router.sh +++ b/examples/backends/vllm/launch/lora/agg_lora_router.sh @@ -57,8 +57,8 @@ echo "" echo "==========================================" # run frontend + KV router -# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) -python -m dynamo.frontend \ +# dingo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) +python -m dingo.frontend \ --router-mode kv \ --router-reset-states & diff --git a/examples/backends/vllm/launch/lora/multimodal/lora_agg.sh b/examples/backends/vllm/launch/lora/multimodal/lora_agg.sh index 0efcf006d62f..67315472f55a 100755 --- a/examples/backends/vllm/launch/lora/multimodal/lora_agg.sh +++ b/examples/backends/vllm/launch/lora/multimodal/lora_agg.sh @@ -121,7 +121,7 @@ esac echo "" echo "Starting frontend..." -python -m dynamo.frontend & +python -m dingo.frontend & FRONTEND_PID=$! # Wait for frontend to become ready diff --git a/examples/backends/vllm/launch/lora/xpu/agg_lora_router_xpu.sh b/examples/backends/vllm/launch/lora/xpu/agg_lora_router_xpu.sh index ba551ec3e2db..c22f8009b561 100755 --- a/examples/backends/vllm/launch/lora/xpu/agg_lora_router_xpu.sh +++ b/examples/backends/vllm/launch/lora/xpu/agg_lora_router_xpu.sh @@ -59,8 +59,8 @@ echo "" echo "==========================================" # run frontend + KV router -# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) -python -m dynamo.frontend \ +# dingo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) +python -m dingo.frontend \ --router-mode kv \ --router-reset-states & diff --git a/examples/backends/vllm/launch/lora/xpu/agg_lora_xpu.sh b/examples/backends/vllm/launch/lora/xpu/agg_lora_xpu.sh index af27c3657fd6..e32f10dab9d0 100755 --- a/examples/backends/vllm/launch/lora/xpu/agg_lora_xpu.sh +++ b/examples/backends/vllm/launch/lora/xpu/agg_lora_xpu.sh @@ -59,8 +59,8 @@ echo "" echo "==========================================" # run ingress -# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var. -python -m dynamo.frontend & +# dingo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var. +python -m dingo.frontend & # ---- Tunable (override via env vars) ---- MAX_MODEL_LEN="${MAX_MODEL_LEN:-4096}" diff --git a/examples/backends/vllm/launch/multi_node_tp.sh b/examples/backends/vllm/launch/multi_node_tp.sh index 04bf6a456149..72584b6c78c7 100755 --- a/examples/backends/vllm/launch/multi_node_tp.sh +++ b/examples/backends/vllm/launch/multi_node_tp.sh @@ -53,7 +53,7 @@ done if [[ "${ROLE}" == "head" ]]; then echo "Starting Dynamo frontend..." - python3 -m dynamo.frontend & + python3 -m dingo.frontend & echo "Starting dynamo.vllm head node (TP=${TP}, nnodes=${NNODES}, node-rank=0)..." python3 -m dynamo.vllm \ diff --git a/examples/backends/vllm/launch/realtime_omni_client.py b/examples/backends/vllm/launch/realtime_omni_client.py index 405fd1ba14a7..f68596c87f9d 100644 --- a/examples/backends/vllm/launch/realtime_omni_client.py +++ b/examples/backends/vllm/launch/realtime_omni_client.py @@ -3,7 +3,7 @@ """Client for the Dynamo vLLM-Omni realtime endpoint (``/v1/realtime``). -Streams an audio file to a launched ``dynamo.frontend`` over a WebSocket using +Streams an audio file to a launched ``dingo.frontend`` over a WebSocket using OpenAI Realtime client events, prints every server event as it arrives, and writes the synthesized audio into an output folder: each returned ``response.output_audio.delta`` as ``chunk_NNNN.wav`` plus the concatenated @@ -18,7 +18,7 @@ ``--input-audio`` is omitted, a sample clip is fetched from the vLLM-Omni GitHub repo (``tests/assets/qwen3_tts/clone_2.wav``). -python -m dynamo.frontend --http-port 8000 +python -m dingo.frontend --http-port 8000 python -m dynamo.vllm.omni --realtime --model Qwen/Qwen3-Omni-30B-A3B-Instruct Usage (omit --input-audio to fetch the sample clip from GitHub): diff --git a/examples/backends/vllm/launch/xpu/agg_lmcache_multiproc_xpu.sh b/examples/backends/vllm/launch/xpu/agg_lmcache_multiproc_xpu.sh index 365b6aecbfcd..1eb096243ff7 100755 --- a/examples/backends/vllm/launch/xpu/agg_lmcache_multiproc_xpu.sh +++ b/examples/backends/vllm/launch/xpu/agg_lmcache_multiproc_xpu.sh @@ -42,7 +42,7 @@ GPU_MEM_ARGS=$(build_vllm_gpu_mem_args) HTTP_PORT="${DYN_HTTP_PORT:-8000}" print_launch_banner "Launching Aggregated + LMCache + Multiproc (1 GPU)" "$MODEL" "$HTTP_PORT" -python -m dynamo.frontend & +python -m dingo.frontend & DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT:-8081} \ PROMETHEUS_MULTIPROC_DIR="$PROMETHEUS_MULTIPROC_DIR" \ diff --git a/examples/backends/vllm/launch/xpu/agg_lmcache_xpu.sh b/examples/backends/vllm/launch/xpu/agg_lmcache_xpu.sh index 5e4a1fb575e0..0ef3646b96e6 100755 --- a/examples/backends/vllm/launch/xpu/agg_lmcache_xpu.sh +++ b/examples/backends/vllm/launch/xpu/agg_lmcache_xpu.sh @@ -32,7 +32,7 @@ GPU_MEM_ARGS=$(build_vllm_gpu_mem_args) HTTP_PORT="${DYN_HTTP_PORT:-8000}" print_launch_banner "Launching Aggregated Serving + LMCache (1 GPU)" "$MODEL" "$HTTP_PORT" -python -m dynamo.frontend & +python -m dingo.frontend & DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT:-8081} \ python -m dynamo.vllm --model "$MODEL" --enforce-eager \ diff --git a/examples/backends/vllm/launch/xpu/agg_multimodal_router_chat_processor_xpu.sh b/examples/backends/vllm/launch/xpu/agg_multimodal_router_chat_processor_xpu.sh index 7b9416291e32..b9e1e115d452 100755 --- a/examples/backends/vllm/launch/xpu/agg_multimodal_router_chat_processor_xpu.sh +++ b/examples/backends/vllm/launch/xpu/agg_multimodal_router_chat_processor_xpu.sh @@ -273,7 +273,7 @@ for f in $(seq 1 "${NUM_FRONTENDS}"); do env "${COMMON_ENV[@]}" \ "DYN_LOG=debug" \ "DYN_SYSTEM_PORT=${FE_SYSTEM_PORT}" \ - python -m dynamo.frontend \ + python -m dingo.frontend \ --http-port "${FE_HTTP_PORT}" \ --dyn-chat-processor vllm \ --router-mode kv \ diff --git a/examples/backends/vllm/launch/xpu/agg_multimodal_router_xpu.sh b/examples/backends/vllm/launch/xpu/agg_multimodal_router_xpu.sh index f40c3a06c7f2..858975843456 100755 --- a/examples/backends/vllm/launch/xpu/agg_multimodal_router_xpu.sh +++ b/examples/backends/vllm/launch/xpu/agg_multimodal_router_xpu.sh @@ -203,7 +203,7 @@ done echo "=== Starting frontend (KV router, lightseek MM exact routing) ===" env "${COMMON_ENV[@]}" \ "DYN_LOG=${DYN_LOG_VAL}" \ -python -m dynamo.frontend \ +python -m dingo.frontend \ --http-port "${HTTP_PORT}" \ --router-mode kv \ --kv-cache-block-size "${BLOCK_SIZE}" & diff --git a/examples/backends/vllm/launch/xpu/agg_multimodal_xpu.sh b/examples/backends/vllm/launch/xpu/agg_multimodal_xpu.sh index 1dc83d277d28..c2f96e50f48a 100755 --- a/examples/backends/vllm/launch/xpu/agg_multimodal_xpu.sh +++ b/examples/backends/vllm/launch/xpu/agg_multimodal_xpu.sh @@ -58,8 +58,8 @@ print_launch_banner --multimodal "Launching Aggregated Multimodal Serving" "$MOD export DYN_REQUEST_PLANE=tcp # Start frontend with Rust OpenAIPreprocessor -# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) -python -m dynamo.frontend & +# dingo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) +python -m dingo.frontend & # ---- Per-model defaults ---- MAX_MODEL_LEN="${MAX_MODEL_LEN:-4096}" diff --git a/examples/backends/vllm/launch/xpu/agg_request_planes_xpu.sh b/examples/backends/vllm/launch/xpu/agg_request_planes_xpu.sh index 1248a6bfeb94..aba9200148ee 100755 --- a/examples/backends/vllm/launch/xpu/agg_request_planes_xpu.sh +++ b/examples/backends/vllm/launch/xpu/agg_request_planes_xpu.sh @@ -57,7 +57,7 @@ GPU_MEM_ARGS=$(build_vllm_gpu_mem_args) HTTP_PORT="${DYN_HTTP_PORT:-8000}" print_launch_banner "Launching Aggregated Serving + Request Planes (1 GPU)" "$MODEL" "$HTTP_PORT" -python -m dynamo.frontend & +python -m dingo.frontend & DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT:-8081} \ DYN_HEALTH_CHECK_ENABLED=true \ diff --git a/examples/backends/vllm/launch/xpu/agg_router_approx_xpu.sh b/examples/backends/vllm/launch/xpu/agg_router_approx_xpu.sh index a8769f1d2c88..0791bd0a709b 100755 --- a/examples/backends/vllm/launch/xpu/agg_router_approx_xpu.sh +++ b/examples/backends/vllm/launch/xpu/agg_router_approx_xpu.sh @@ -17,7 +17,7 @@ HTTP_PORT="${DYN_HTTP_PORT:-8000}" print_launch_banner "Launching Aggregated + Approximate KV Routing (2 GPUs)" "$MODEL" "$HTTP_PORT" # run frontend with KV router (--router-mode kv) in approximate mode (--no-kv-events) -python -m dynamo.frontend \ +python -m dingo.frontend \ --router-mode kv \ --no-kv-events & diff --git a/examples/backends/vllm/launch/xpu/agg_router_xpu.sh b/examples/backends/vllm/launch/xpu/agg_router_xpu.sh index c717a98f5205..91a20bdf64a0 100755 --- a/examples/backends/vllm/launch/xpu/agg_router_xpu.sh +++ b/examples/backends/vllm/launch/xpu/agg_router_xpu.sh @@ -21,8 +21,8 @@ HTTP_PORT="${DYN_HTTP_PORT:-8000}" print_launch_banner "Launching Aggregated + KV Routing (2 GPUs)" "$MODEL" "$HTTP_PORT" # run frontend + KV router -# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) -python -m dynamo.frontend \ +# dingo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) +python -m dingo.frontend \ --router-mode kv \ --router-reset-states & diff --git a/examples/backends/vllm/launch/xpu/agg_xpu.sh b/examples/backends/vllm/launch/xpu/agg_xpu.sh index a897b006cf63..ac52e944103a 100755 --- a/examples/backends/vllm/launch/xpu/agg_xpu.sh +++ b/examples/backends/vllm/launch/xpu/agg_xpu.sh @@ -46,8 +46,8 @@ HTTP_PORT="${DYN_HTTP_PORT:-8000}" print_launch_banner "Launching Aggregated Serving (1 GPU)" "$MODEL" "$HTTP_PORT" # run ingress -# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) -python -m dynamo.frontend & +# dingo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) +python -m dingo.frontend & # run worker # --enforce-eager is added for quick deployment. for production use, need to remove this flag diff --git a/examples/backends/vllm/launch/xpu/disagg_multimodal_epd_xpu.sh b/examples/backends/vllm/launch/xpu/disagg_multimodal_epd_xpu.sh index b6f4f6206137..b8c13c6dc2f6 100755 --- a/examples/backends/vllm/launch/xpu/disagg_multimodal_epd_xpu.sh +++ b/examples/backends/vllm/launch/xpu/disagg_multimodal_epd_xpu.sh @@ -81,8 +81,8 @@ print_launch_banner --multimodal "Launching Disaggregated Multimodal E/P/D ($GPU # Start frontend (no router mode) echo "Starting frontend..." -# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) -python -m dynamo.frontend & +# dingo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) +python -m dingo.frontend & EXTRA_ARGS="" PD_EXTRA_ARGS="" diff --git a/examples/backends/vllm/launch/xpu/disagg_router_xpu_gdr.sh b/examples/backends/vllm/launch/xpu/disagg_router_xpu_gdr.sh index 1c4519022bfa..6029cf11fde1 100755 --- a/examples/backends/vllm/launch/xpu/disagg_router_xpu_gdr.sh +++ b/examples/backends/vllm/launch/xpu/disagg_router_xpu_gdr.sh @@ -23,7 +23,7 @@ export UCX_TLS=ib,rc,ze_copy # Start frontend with KV routing # The frontend will automatically detect prefill workers and activate an internal prefill router # edit --router-mode to random / round-robin / kv -python -m dynamo.frontend \ +python -m dingo.frontend \ --router-mode kv \ --http-port 8000 \ --router-reset-states & diff --git a/examples/common/launch_utils.sh b/examples/common/launch_utils.sh index 5241f40cf29d..6cd06409841c 100755 --- a/examples/common/launch_utils.sh +++ b/examples/common/launch_utils.sh @@ -61,7 +61,7 @@ EXAMPLE_PROMPT_VISUAL="A golden retriever riding a skateboard through a neon-lit # from child processes (OOM, Python exceptions, etc.) still propagate. # # Usage: -# python -m dynamo.frontend & +# python -m dingo.frontend & # python -m dynamo.vllm --model "$MODEL" & # wait_any_exit diff --git a/examples/deployments/ECS/task_definition_frontend.json b/examples/deployments/ECS/task_definition_frontend.json index 2d8e517a2de1..2afbc1f23191 100644 --- a/examples/deployments/ECS/task_definition_frontend.json +++ b/examples/deployments/ECS/task_definition_frontend.json @@ -23,7 +23,7 @@ "-c" ], "command": [ - "cd examples/backends/vllm && python -m dynamo.frontend --router-mode kv & python3 -m dynamo.vllm --model Qwen/Qwen3-0.6B --enforce-eager" + "cd examples/backends/vllm && python -m dingo.frontend --router-mode kv & python3 -m dynamo.vllm --model Qwen/Qwen3-0.6B --enforce-eager" ], "environment": [ { diff --git a/examples/deployments/EKS/manifests/vllm/disagg-p5.yaml b/examples/deployments/EKS/manifests/vllm/disagg-p5.yaml index e4e61cc6e713..eb6ec76ad8ee 100644 --- a/examples/deployments/EKS/manifests/vllm/disagg-p5.yaml +++ b/examples/deployments/EKS/manifests/vllm/disagg-p5.yaml @@ -41,7 +41,7 @@ spec: - /bin/bash - -c - | - exec python3 -m dynamo.frontend --router-mode kv --router-reset-states + exec python3 -m dingo.frontend --router-mode kv --router-reset-states VllmDecodeWorker: envFromSecret: hf-token-secret componentType: worker diff --git a/examples/deployments/EKS/manifests/vllm/disagg.yaml b/examples/deployments/EKS/manifests/vllm/disagg.yaml index b3f40bc5b4b5..deace14d198d 100644 --- a/examples/deployments/EKS/manifests/vllm/disagg.yaml +++ b/examples/deployments/EKS/manifests/vllm/disagg.yaml @@ -41,7 +41,7 @@ spec: - /bin/bash - -c - | - exec python3 -m dynamo.frontend --router-mode kv --router-reset-states + exec python3 -m dingo.frontend --router-mode kv --router-reset-states VllmDecodeWorker: envFromSecret: hf-token-secret componentType: worker diff --git a/examples/deployments/EKS/manifests/vllm/v1beta1/disagg-p5.yaml b/examples/deployments/EKS/manifests/vllm/v1beta1/disagg-p5.yaml index af83f66b70a5..88172381de6e 100644 --- a/examples/deployments/EKS/manifests/vllm/v1beta1/disagg-p5.yaml +++ b/examples/deployments/EKS/manifests/vllm/v1beta1/disagg-p5.yaml @@ -22,7 +22,7 @@ spec: - /bin/bash - -c - | - exec python3 -m dynamo.frontend --router-mode kv --router-reset-states + exec python3 -m dingo.frontend --router-mode kv --router-reset-states env: - name: HF_HOME value: /home/dynamo/.cache/huggingface diff --git a/examples/deployments/EKS/manifests/vllm/v1beta1/disagg.yaml b/examples/deployments/EKS/manifests/vllm/v1beta1/disagg.yaml index 39d36d363eef..a8ea6a63861e 100644 --- a/examples/deployments/EKS/manifests/vllm/v1beta1/disagg.yaml +++ b/examples/deployments/EKS/manifests/vllm/v1beta1/disagg.yaml @@ -22,7 +22,7 @@ spec: - /bin/bash - -c - | - exec python3 -m dynamo.frontend --router-mode kv --router-reset-states + exec python3 -m dingo.frontend --router-mode kv --router-reset-states env: - name: HF_HOME value: /home/dynamo/.cache/huggingface diff --git a/examples/diffusers/deploy/agg.yaml b/examples/diffusers/deploy/agg.yaml index 7003302f292e..0b9a5cdb1e77 100644 --- a/examples/diffusers/deploy/agg.yaml +++ b/examples/diffusers/deploy/agg.yaml @@ -24,7 +24,7 @@ spec: command: - python - -m - - dynamo.frontend + - dingo.frontend args: - --http-port - "8000" diff --git a/examples/diffusers/deploy/agg_user_workload.yaml b/examples/diffusers/deploy/agg_user_workload.yaml index a4dc7caa074e..3b6706170444 100644 --- a/examples/diffusers/deploy/agg_user_workload.yaml +++ b/examples/diffusers/deploy/agg_user_workload.yaml @@ -35,7 +35,7 @@ spec: command: - python - -m - - dynamo.frontend + - dingo.frontend args: - --http-port - "8000" diff --git a/examples/diffusers/deploy/v1beta1/agg.yaml b/examples/diffusers/deploy/v1beta1/agg.yaml index 2130edd5de3c..4cad2941f54d 100644 --- a/examples/diffusers/deploy/v1beta1/agg.yaml +++ b/examples/diffusers/deploy/v1beta1/agg.yaml @@ -54,7 +54,7 @@ spec: command: - python - -m - - dynamo.frontend + - dingo.frontend env: - name: DYN_DISCOVERY_BACKEND value: kubernetes diff --git a/examples/diffusers/deploy/v1beta1/agg_user_workload.yaml b/examples/diffusers/deploy/v1beta1/agg_user_workload.yaml index 1479805ce238..cad78757d0d1 100644 --- a/examples/diffusers/deploy/v1beta1/agg_user_workload.yaml +++ b/examples/diffusers/deploy/v1beta1/agg_user_workload.yaml @@ -65,7 +65,7 @@ spec: command: - python - -m - - dynamo.frontend + - dingo.frontend env: - name: DYN_DISCOVERY_BACKEND value: kubernetes diff --git a/examples/diffusers/local/docker-compose.yml b/examples/diffusers/local/docker-compose.yml index b82b2d29e036..218177f54850 100644 --- a/examples/diffusers/local/docker-compose.yml +++ b/examples/diffusers/local/docker-compose.yml @@ -42,7 +42,7 @@ services: image: dynamo-fastvideo-diffusers:latest restart: on-failure command: > - python -m dynamo.frontend + python -m dingo.frontend --http-port 8000 --discovery-backend file environment: diff --git a/examples/diffusers/local/run_local.sh b/examples/diffusers/local/run_local.sh index 976f907744dc..fcf7224b422f 100755 --- a/examples/diffusers/local/run_local.sh +++ b/examples/diffusers/local/run_local.sh @@ -35,7 +35,7 @@ if [[ -n "${WORKER_EXTRA_ARGS}" ]]; then worker_cmd+=("${worker_extra[@]}") fi -frontend_cmd=("${PYTHON_BIN}" -m dynamo.frontend --http-port "${HTTP_PORT}" --discovery-backend file) +frontend_cmd=("${PYTHON_BIN}" -m dingo.frontend --http-port "${HTTP_PORT}" --discovery-backend file) if [[ -n "${FRONTEND_EXTRA_ARGS}" ]]; then # shellcheck disable=SC2206 frontend_extra=( ${FRONTEND_EXTRA_ARGS} ) diff --git a/examples/global_planner/global-planner-gpu-budget.yaml b/examples/global_planner/global-planner-gpu-budget.yaml index e5440523e5ec..9f4040ca0d6a 100644 --- a/examples/global_planner/global-planner-gpu-budget.yaml +++ b/examples/global_planner/global-planner-gpu-budget.yaml @@ -92,7 +92,7 @@ spec: command: - python3 - -m - - dynamo.frontend + - dingo.frontend args: - --model-name - ${MODEL_A} @@ -187,7 +187,7 @@ spec: command: - python3 - -m - - dynamo.frontend + - dingo.frontend args: - --model-name - ${MODEL_B} diff --git a/examples/global_planner/global-planner-mocker-test.yaml b/examples/global_planner/global-planner-mocker-test.yaml index 2783e1bd4906..1c6711af52f3 100644 --- a/examples/global_planner/global-planner-mocker-test.yaml +++ b/examples/global_planner/global-planner-mocker-test.yaml @@ -62,7 +62,7 @@ spec: command: - python3 - -m - - dynamo.frontend + - dingo.frontend args: - --router-mode - round-robin diff --git a/examples/global_planner/global-planner-vllm-test.yaml b/examples/global_planner/global-planner-vllm-test.yaml index 87a8daf2b9c6..0b3d65df420e 100644 --- a/examples/global_planner/global-planner-vllm-test.yaml +++ b/examples/global_planner/global-planner-vllm-test.yaml @@ -95,7 +95,7 @@ spec: command: - python3 - -m - - dynamo.frontend + - dingo.frontend args: - --router-mode - round-robin diff --git a/examples/global_planner/v1beta1/global-planner-gpu-budget.yaml b/examples/global_planner/v1beta1/global-planner-gpu-budget.yaml index bf126a3e583b..21b8d748dc85 100644 --- a/examples/global_planner/v1beta1/global-planner-gpu-budget.yaml +++ b/examples/global_planner/v1beta1/global-planner-gpu-budget.yaml @@ -93,7 +93,7 @@ spec: command: - python3 - -m - - dynamo.frontend + - dingo.frontend image: ${DYNAMO_IMAGE} name: main workingDir: /workspace @@ -194,7 +194,7 @@ spec: command: - python3 - -m - - dynamo.frontend + - dingo.frontend image: ${DYNAMO_IMAGE} name: main workingDir: /workspace diff --git a/examples/global_planner/v1beta1/global-planner-mocker-test.yaml b/examples/global_planner/v1beta1/global-planner-mocker-test.yaml index 0078c33d4392..c14ad05d09fd 100644 --- a/examples/global_planner/v1beta1/global-planner-mocker-test.yaml +++ b/examples/global_planner/v1beta1/global-planner-mocker-test.yaml @@ -64,7 +64,7 @@ spec: command: - python3 - -m - - dynamo.frontend + - dingo.frontend image: ${DYNAMO_IMAGE} name: main workingDir: /workspace diff --git a/examples/global_planner/v1beta1/global-planner-vllm-test.yaml b/examples/global_planner/v1beta1/global-planner-vllm-test.yaml index 248309df4916..84c06974a788 100644 --- a/examples/global_planner/v1beta1/global-planner-vllm-test.yaml +++ b/examples/global_planner/v1beta1/global-planner-vllm-test.yaml @@ -99,7 +99,7 @@ spec: command: - python3 - -m - - dynamo.frontend + - dingo.frontend image: ${DYNAMO_IMAGE} name: main workingDir: /workspace diff --git a/hatch_build.py b/hatch_build.py index d5e155aae71a..f9b99da0e7db 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -9,12 +9,11 @@ def get_components(): """ - Scan the components/src/dynamo directory to get the list of available components. + Scan component package directories to get the list of available components. Returns full paths to component directories and RuntimeError if no components are found. """ - components_dir = os.path.join( - os.path.dirname(__file__), "components", "src", "dynamo" - ) + root = os.path.dirname(__file__) + components_dir = os.path.join(root, "components", "src", "dynamo") if not os.path.exists(components_dir): raise RuntimeError(f"Components directory not found: {components_dir}") @@ -28,6 +27,10 @@ def get_components(): if not components: raise RuntimeError(f"No components found in directory: {components_dir}") + dingo_frontend = os.path.join(root, "dingo", "frontend") + if os.path.isdir(dingo_frontend): + components.append(dingo_frontend) + return components diff --git a/lib/backend-common/examples/mocker/Dockerfile.frontend b/lib/backend-common/examples/mocker/Dockerfile.frontend index 740b3b066444..c4e29eb66e75 100644 --- a/lib/backend-common/examples/mocker/Dockerfile.frontend +++ b/lib/backend-common/examples/mocker/Dockerfile.frontend @@ -5,7 +5,7 @@ # Minimal Dynamo frontend image built from source. # # Build context MUST be the dynamo workspace root (see docker-compose.yml). -# Produces an image that runs `python -m dynamo.frontend`. +# Produces an image that runs `python -m dingo.frontend`. FROM python:3.12-slim-bookworm AS builder @@ -52,6 +52,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY --from=builder /tmp/wheels /tmp/wheels COPY --from=builder /build/pyproject.toml /build/hatch_build.py /build/README.md /build/LICENSE /app/ COPY --from=builder /build/components /app/components +COPY --from=builder /build/dingo /app/dingo WORKDIR /app @@ -59,4 +60,4 @@ RUN pip install --no-cache-dir /tmp/wheels/*.whl \ && pip install --no-cache-dir . \ && rm -rf /tmp/wheels -ENTRYPOINT ["python", "-m", "dynamo.frontend"] +ENTRYPOINT ["python", "-m", "dingo.frontend"] diff --git a/lib/bench/README.md b/lib/bench/README.md index 9edf4d4518c1..c9df414b9d7e 100644 --- a/lib/bench/README.md +++ b/lib/bench/README.md @@ -39,7 +39,7 @@ python -m dynamo.vllm \ --model deepseek-ai/DeepSeek-R1-Distill-Llama-8B # Terminal 2 — frontend with KV router -python -m dynamo.frontend \ +python -m dingo.frontend \ --router-mode kv \ --http-port 8000 ``` diff --git a/lib/llm/tests/http_namespace_integration.rs b/lib/llm/tests/http_namespace_integration.rs index b06ebb172e96..7152bf0a4e1c 100644 --- a/lib/llm/tests/http_namespace_integration.rs +++ b/lib/llm/tests/http_namespace_integration.rs @@ -27,7 +27,7 @@ fn test_endpoint_id_namespace_extraction() { "backend", "generate", ), - ("dynamo.frontend.http", "dynamo", "frontend", "http"), + ("dingo.frontend.http", "dynamo", "frontend", "http"), ( "tensorrt-llm.backend.inference", "tensorrt-llm", diff --git a/lib/runtime/examples/system_metrics/README.md b/lib/runtime/examples/system_metrics/README.md index bc47cce0cdb5..00fd02e37007 100644 --- a/lib/runtime/examples/system_metrics/README.md +++ b/lib/runtime/examples/system_metrics/README.md @@ -187,7 +187,7 @@ The server will start an system status server on the specified port (8081 in thi To Run an actual LLM frontend + server (aggregated example), launch both of them. By default, the frontend listens to port 8000. ``` -python -m dynamo.frontend & +python -m dingo.frontend & DYN_SYSTEM_PORT=8081 python -m dynamo.vllm --model Qwen/Qwen3-0.6B --enforce-eager --no-enable-prefix-caching & ``` diff --git a/pyproject.toml b/pyproject.toml index 49050df7a111..03f00cbb93dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,6 +117,7 @@ path = "hatch_build.py" [tool.hatch.build.targets.wheel] packages = [ "components/src/dynamo", + "dingo", ] [tool.hatch.metadata] @@ -150,7 +151,7 @@ line_length = 88 balanced_wrapping = true indent = " " skip = ["build"] -known_first_party = ["dynamo", "deploy"] +known_first_party = ["dynamo", "dingo", "deploy"] # isort may confuse what is 1st or 3rd library. e.g. # when dynamo/vllm/omni/xx.py import vllm, local isort may treat this `vllm` as first # party heuristically. This causes local sort differs from GitHub sort and pre-commit @@ -330,7 +331,7 @@ indent-width = 4 # Configure it so that developers can use "go-to-definition", "hover types" and other # features. [tool.basedpyright] -extraPaths = ["components/src", "lib/bindings/python/src"] +extraPaths = ["components/src", "lib/bindings/python/src", "."] # This is for external dependencies. venvPath = "." venv = ".venv" diff --git a/recipes/README.md b/recipes/README.md index 305d11bc736d..b9c1d7f4fb9e 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -296,7 +296,7 @@ services: ```yaml # In Frontend args: args: - - python3 -m dynamo.frontend --router-mode kv --http-port 8000 + - python3 -m dingo.frontend --router-mode kv --http-port 8000 # Options: round-robin, kv (KV-aware routing) ``` diff --git a/recipes/deepseek-r1/trtllm/disagg/wide_ep/gb200/deploy.yaml b/recipes/deepseek-r1/trtllm/disagg/wide_ep/gb200/deploy.yaml index 257adfc9ebe8..d40ec35a63ff 100644 --- a/recipes/deepseek-r1/trtllm/disagg/wide_ep/gb200/deploy.yaml +++ b/recipes/deepseek-r1/trtllm/disagg/wide_ep/gb200/deploy.yaml @@ -142,7 +142,7 @@ spec: value: /model-cache args: - | - python3 -m dynamo.frontend --http-port 8000 + python3 -m dingo.frontend --http-port 8000 command: - /bin/sh - -c diff --git a/recipes/deepseek-v32-fp4/trtllm/agg-round-robin/deploy.yaml b/recipes/deepseek-v32-fp4/trtllm/agg-round-robin/deploy.yaml index 2c2ae9e3f806..ed13d379738c 100644 --- a/recipes/deepseek-v32-fp4/trtllm/agg-round-robin/deploy.yaml +++ b/recipes/deepseek-v32-fp4/trtllm/agg-round-robin/deploy.yaml @@ -16,7 +16,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - round-robin - --router-reset-states diff --git a/recipes/deepseek-v32-fp4/trtllm/disagg-kv-router/deploy.yaml b/recipes/deepseek-v32-fp4/trtllm/disagg-kv-router/deploy.yaml index 4f4a38ec3c97..efb236364430 100644 --- a/recipes/deepseek-v32-fp4/trtllm/disagg-kv-router/deploy.yaml +++ b/recipes/deepseek-v32-fp4/trtllm/disagg-kv-router/deploy.yaml @@ -16,7 +16,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - kv - --router-reset-states diff --git a/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-b200/deploy.yaml b/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-b200/deploy.yaml index 92bb19733973..1bf0915030fa 100644 --- a/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-b200/deploy.yaml +++ b/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-b200/deploy.yaml @@ -58,7 +58,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend env: - name: POD_UID valueFrom: diff --git a/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-gb200/deploy.yaml b/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-gb200/deploy.yaml index 28d3808e9ed0..997c5b5618e3 100644 --- a/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-gb200/deploy.yaml +++ b/recipes/deepseek-v4/deepseek-v4-pro/sglang/disagg-gb200/deploy.yaml @@ -58,7 +58,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend env: - name: POD_UID valueFrom: diff --git a/recipes/glm-5-nvfp4/sglang/disagg/deploy.yaml b/recipes/glm-5-nvfp4/sglang/disagg/deploy.yaml index 9ea6e12452a8..4fb06fe8858f 100644 --- a/recipes/glm-5-nvfp4/sglang/disagg/deploy.yaml +++ b/recipes/glm-5-nvfp4/sglang/disagg/deploy.yaml @@ -48,7 +48,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --enforce-disagg env: - name: POD_UID diff --git a/recipes/glm-5-nvfp4/sglang/disagg/efa/deploy.yaml b/recipes/glm-5-nvfp4/sglang/disagg/efa/deploy.yaml index fe1ab6341e75..540fab282127 100644 --- a/recipes/glm-5-nvfp4/sglang/disagg/efa/deploy.yaml +++ b/recipes/glm-5-nvfp4/sglang/disagg/efa/deploy.yaml @@ -57,7 +57,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --enforce-disagg env: - name: POD_UID diff --git a/recipes/gpt-oss-120b/trtllm/agg/deploy.yaml b/recipes/gpt-oss-120b/trtllm/agg/deploy.yaml index 4077d0c82676..dba39e0adcf5 100644 --- a/recipes/gpt-oss-120b/trtllm/agg/deploy.yaml +++ b/recipes/gpt-oss-120b/trtllm/agg/deploy.yaml @@ -41,7 +41,7 @@ spec: topologyKey: kubernetes.io/hostname mainContainer: args: - - python3 -m dynamo.frontend --router-mode round-robin --http-port 8000 + - python3 -m dingo.frontend --router-mode round-robin --http-port 8000 command: - /bin/sh - -c diff --git a/recipes/gpt-oss-120b/trtllm/disagg/deploy.yaml b/recipes/gpt-oss-120b/trtllm/disagg/deploy.yaml index 55216b8e189b..d8e825e81c9d 100644 --- a/recipes/gpt-oss-120b/trtllm/disagg/deploy.yaml +++ b/recipes/gpt-oss-120b/trtllm/disagg/deploy.yaml @@ -86,7 +86,7 @@ spec: topologyKey: kubernetes.io/hostname mainContainer: args: - - python3 -m dynamo.frontend --router-mode round-robin --http-port 8000 + - python3 -m dingo.frontend --router-mode round-robin --http-port 8000 command: - /bin/sh - -c diff --git a/recipes/kimi-k2.5/tokenspeed/agg/nvidia/README.md b/recipes/kimi-k2.5/tokenspeed/agg/nvidia/README.md index e45b07fd6ba4..84f569af39a7 100644 --- a/recipes/kimi-k2.5/tokenspeed/agg/nvidia/README.md +++ b/recipes/kimi-k2.5/tokenspeed/agg/nvidia/README.md @@ -131,7 +131,7 @@ This creates four Deployments + three Services: |---|---|---| | `kimi-k25-tokenspeed-etcd` (Deployment + Service) | Discovery backend | `gcr.io/etcd-development/etcd:v3.6.7` | | `kimi-k25-tokenspeed-nats` (Deployment + Service) | Event plane (JetStream) | `nats:2.12.4` | -| `kimi-k25-tokenspeed-frontend` (Deployment) | `dynamo.frontend` in KV-router mode on port 8000 | your locally-built `dynamo-tokenspeed` | +| `kimi-k25-tokenspeed-frontend` (Deployment) | `dingo.frontend` in KV-router mode on port 8000 | your locally-built `dynamo-tokenspeed` | | `kimi-k25-tokenspeed-agg-frontend` (Service) | Stable name for port-forward; selects the frontend Deployment | — | | `kimi-k25-tokenspeed-worker` (Deployment) | `dynamo.tokenspeed` against `nvidia/Kimi-K2.5-NVFP4`, TP=4 + EP=4, NVFP4 weights, FP8 KV cache, MLA attention via `trtllm_mla`, MoE via `flashinfer_trtllm`, with `kimi_k25` reasoning + `kimi_k2` tool-call parsers | your locally-built `dynamo-tokenspeed` | diff --git a/recipes/kimi-k2.5/tokenspeed/agg/nvidia/deploy.yaml b/recipes/kimi-k2.5/tokenspeed/agg/nvidia/deploy.yaml index 1a82fd912b73..114219b4512e 100644 --- a/recipes/kimi-k2.5/tokenspeed/agg/nvidia/deploy.yaml +++ b/recipes/kimi-k2.5/tokenspeed/agg/nvidia/deploy.yaml @@ -162,7 +162,7 @@ spec: - /bin/sh - -c args: - - python3 -m dynamo.frontend --router-mode kv --http-port 8000 + - python3 -m dingo.frontend --router-mode kv --http-port 8000 env: - name: ETCD_ENDPOINTS value: http://kimi-k25-tokenspeed-etcd:2379 diff --git a/recipes/kimi-k2.5/trtllm/agg-eagle-kv-router/deploy.yaml b/recipes/kimi-k2.5/trtllm/agg-eagle-kv-router/deploy.yaml index 827623aac835..dae9df793412 100644 --- a/recipes/kimi-k2.5/trtllm/agg-eagle-kv-router/deploy.yaml +++ b/recipes/kimi-k2.5/trtllm/agg-eagle-kv-router/deploy.yaml @@ -29,7 +29,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - kv - --router-reset-states diff --git a/recipes/kimi-k2.5/trtllm/agg-eagle-round-robin/deploy.yaml b/recipes/kimi-k2.5/trtllm/agg-eagle-round-robin/deploy.yaml index 72556d6b6f4b..f721d634b078 100644 --- a/recipes/kimi-k2.5/trtllm/agg-eagle-round-robin/deploy.yaml +++ b/recipes/kimi-k2.5/trtllm/agg-eagle-round-robin/deploy.yaml @@ -29,7 +29,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - round-robin - --request-plane diff --git a/recipes/kimi-k2.5/trtllm/agg-round-robin/deploy.yaml b/recipes/kimi-k2.5/trtllm/agg-round-robin/deploy.yaml index afd5a946d72f..11e942de4844 100644 --- a/recipes/kimi-k2.5/trtllm/agg-round-robin/deploy.yaml +++ b/recipes/kimi-k2.5/trtllm/agg-round-robin/deploy.yaml @@ -29,7 +29,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - round-robin - --request-plane diff --git a/recipes/kimi-k2.5/trtllm/disagg-eagle-kv-router/deploy.yaml b/recipes/kimi-k2.5/trtllm/disagg-eagle-kv-router/deploy.yaml index 71914976003a..1e13dbf95e6b 100644 --- a/recipes/kimi-k2.5/trtllm/disagg-eagle-kv-router/deploy.yaml +++ b/recipes/kimi-k2.5/trtllm/disagg-eagle-kv-router/deploy.yaml @@ -37,7 +37,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - kv - --router-reset-states diff --git a/recipes/kimi-k2.6/vllm/agg-b200-agentic/deploy.yaml b/recipes/kimi-k2.6/vllm/agg-b200-agentic/deploy.yaml index 6d67f49c5726..408ca50223a3 100644 --- a/recipes/kimi-k2.6/vllm/agg-b200-agentic/deploy.yaml +++ b/recipes/kimi-k2.6/vllm/agg-b200-agentic/deploy.yaml @@ -72,7 +72,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --trust-remote-code env: - name: POD_UID diff --git a/recipes/kimi-k2.6/vllm/agg-b200-chat/deploy.yaml b/recipes/kimi-k2.6/vllm/agg-b200-chat/deploy.yaml index e3d1811b5eee..bbcf5359b656 100644 --- a/recipes/kimi-k2.6/vllm/agg-b200-chat/deploy.yaml +++ b/recipes/kimi-k2.6/vllm/agg-b200-chat/deploy.yaml @@ -72,7 +72,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --trust-remote-code env: - name: POD_UID diff --git a/recipes/kimi-k2.6/vllm/agg-h200-agentic/deploy.yaml b/recipes/kimi-k2.6/vllm/agg-h200-agentic/deploy.yaml index b2ee1f4935f2..8e70529b983e 100644 --- a/recipes/kimi-k2.6/vllm/agg-h200-agentic/deploy.yaml +++ b/recipes/kimi-k2.6/vllm/agg-h200-agentic/deploy.yaml @@ -69,7 +69,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --trust-remote-code env: - name: POD_UID diff --git a/recipes/kimi-k2.6/vllm/agg-h200-chat/deploy.yaml b/recipes/kimi-k2.6/vllm/agg-h200-chat/deploy.yaml index 52d04ee6e715..62b94dfd0b7a 100644 --- a/recipes/kimi-k2.6/vllm/agg-h200-chat/deploy.yaml +++ b/recipes/kimi-k2.6/vllm/agg-h200-chat/deploy.yaml @@ -69,7 +69,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --trust-remote-code env: - name: POD_UID diff --git a/recipes/llama-3-70b/vllm/agg/gaie/deploy.yaml b/recipes/llama-3-70b/vllm/agg/gaie/deploy.yaml index d6ceb481b51c..d5ef54743472 100644 --- a/recipes/llama-3-70b/vllm/agg/gaie/deploy.yaml +++ b/recipes/llama-3-70b/vllm/agg/gaie/deploy.yaml @@ -99,7 +99,7 @@ spec: - name: sidecar-frontend args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - direct envFrom: diff --git a/recipes/llama-3-70b/vllm/disagg-single-node/gaie/deploy.yaml b/recipes/llama-3-70b/vllm/disagg-single-node/gaie/deploy.yaml index e1f8340b5185..b28ff7785ed1 100644 --- a/recipes/llama-3-70b/vllm/disagg-single-node/gaie/deploy.yaml +++ b/recipes/llama-3-70b/vllm/disagg-single-node/gaie/deploy.yaml @@ -119,7 +119,7 @@ spec: - name: sidecar-frontend args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - direct envFrom: @@ -186,7 +186,7 @@ spec: - name: sidecar-frontend args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - direct envFrom: diff --git a/recipes/nemotron-3-nano-omni/Dockerfile b/recipes/nemotron-3-nano-omni/Dockerfile index 1d13afe628ff..3b459c5e956d 100644 --- a/recipes/nemotron-3-nano-omni/Dockerfile +++ b/recipes/nemotron-3-nano-omni/Dockerfile @@ -25,7 +25,7 @@ USER root ARG DYNAMO_VERSION -# ai-dynamo Python package (dynamo.frontend, dynamo.vllm, ...) installed with +# ai-dynamo Python package (dingo.frontend, dynamo.vllm, ...) installed with # --no-deps so that: # - the base image's vLLM 0.20 / torch / cuda stack is preserved (otherwise # pip would try to re-install vllm[flashinfer,otel,runai]==0.20.0 which is diff --git a/recipes/nemotron-3-nano-omni/README.md b/recipes/nemotron-3-nano-omni/README.md index a041a4e42e9d..129c02ace08b 100644 --- a/recipes/nemotron-3-nano-omni/README.md +++ b/recipes/nemotron-3-nano-omni/README.md @@ -165,7 +165,7 @@ mainContainer: # to actually unset it before the runtime reads env. - >- unset NATS_SERVER && - exec python3 -m dynamo.frontend ... # or dynamo.vllm + exec python3 -m dingo.frontend ... # or dynamo.vllm ``` The request plane defaults to TCP already, so no further flags are needed. diff --git a/recipes/nemotron-3-nano-omni/vllm/agg/deploy.yaml b/recipes/nemotron-3-nano-omni/vllm/agg/deploy.yaml index 1c03c392f811..09d78d34e072 100644 --- a/recipes/nemotron-3-nano-omni/vllm/agg/deploy.yaml +++ b/recipes/nemotron-3-nano-omni/vllm/agg/deploy.yaml @@ -48,7 +48,7 @@ spec: - -lc args: - >- - exec python3 -m dynamo.frontend + exec python3 -m dingo.frontend --router-mode kv --no-kv-events --http-port 8000 diff --git a/recipes/nemotron-3-super-fp8/sglang/agg/deploy.yaml b/recipes/nemotron-3-super-fp8/sglang/agg/deploy.yaml index 4c0ad208044a..599d5844f69d 100644 --- a/recipes/nemotron-3-super-fp8/sglang/agg/deploy.yaml +++ b/recipes/nemotron-3-super-fp8/sglang/agg/deploy.yaml @@ -42,7 +42,7 @@ spec: - /bin/sh - -c args: - - python3 -m dynamo.frontend --router-mode kv --no-kv-events --http-port 8000 + - python3 -m dingo.frontend --router-mode kv --no-kv-events --http-port 8000 SglangWorker: componentType: worker envFromSecret: hf-token-secret diff --git a/recipes/nemotron-3-super-fp8/sglang/disagg/deploy.yaml b/recipes/nemotron-3-super-fp8/sglang/disagg/deploy.yaml index 1749ea061716..7bb620726aea 100644 --- a/recipes/nemotron-3-super-fp8/sglang/disagg/deploy.yaml +++ b/recipes/nemotron-3-super-fp8/sglang/disagg/deploy.yaml @@ -39,7 +39,7 @@ spec: - /bin/sh - -c args: - - python3 -m dynamo.frontend --router-mode kv --no-kv-events --http-port 8000 + - python3 -m dingo.frontend --router-mode kv --no-kv-events --http-port 8000 prefill: componentType: worker subComponentType: prefill diff --git a/recipes/nemotron-3-super-fp8/trtllm/disagg/deploy.yaml b/recipes/nemotron-3-super-fp8/trtllm/disagg/deploy.yaml index ae963d795bc1..2e2e3915f9e5 100644 --- a/recipes/nemotron-3-super-fp8/trtllm/disagg/deploy.yaml +++ b/recipes/nemotron-3-super-fp8/trtllm/disagg/deploy.yaml @@ -103,7 +103,7 @@ spec: # Nemotron-H on TRT-LLM has block reuse disabled, so KV-overlap # routing does not provide a real cache reuse benefit. args: - - python3 -m dynamo.frontend --router-mode round-robin --http-port 8000 + - python3 -m dingo.frontend --router-mode round-robin --http-port 8000 command: - /bin/sh - -c diff --git a/recipes/nemotron-3-super-fp8/vllm/agg/deploy.yaml b/recipes/nemotron-3-super-fp8/vllm/agg/deploy.yaml index 0d37f85565dc..8bc5fa4ccb58 100644 --- a/recipes/nemotron-3-super-fp8/vllm/agg/deploy.yaml +++ b/recipes/nemotron-3-super-fp8/vllm/agg/deploy.yaml @@ -43,7 +43,7 @@ spec: - /bin/sh - -c args: - - python3 -m dynamo.frontend --router-mode kv --no-kv-events --http-port 8000 + - python3 -m dingo.frontend --router-mode kv --no-kv-events --http-port 8000 VllmWorker: componentType: worker envFromSecret: hf-token-secret diff --git a/recipes/nemotron-3-super/vllm/agg-b200-agentic/deploy.yaml b/recipes/nemotron-3-super/vllm/agg-b200-agentic/deploy.yaml index e08f8c66f23d..b4a899c4c1d4 100644 --- a/recipes/nemotron-3-super/vllm/agg-b200-agentic/deploy.yaml +++ b/recipes/nemotron-3-super/vllm/agg-b200-agentic/deploy.yaml @@ -78,7 +78,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --trust-remote-code env: - name: POD_UID diff --git a/recipes/nemotron-3-super/vllm/agg-b200-chat/deploy.yaml b/recipes/nemotron-3-super/vllm/agg-b200-chat/deploy.yaml index cb369fd7e43e..cb1257ed6f17 100644 --- a/recipes/nemotron-3-super/vllm/agg-b200-chat/deploy.yaml +++ b/recipes/nemotron-3-super/vllm/agg-b200-chat/deploy.yaml @@ -80,7 +80,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --trust-remote-code env: - name: POD_UID diff --git a/recipes/nemotron-3-super/vllm/agg-h200-agentic/deploy.yaml b/recipes/nemotron-3-super/vllm/agg-h200-agentic/deploy.yaml index 7a9e42568719..568f7bc8e620 100644 --- a/recipes/nemotron-3-super/vllm/agg-h200-agentic/deploy.yaml +++ b/recipes/nemotron-3-super/vllm/agg-h200-agentic/deploy.yaml @@ -62,7 +62,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --trust-remote-code env: - name: POD_UID diff --git a/recipes/nemotron-3-super/vllm/agg-h200-chat/deploy.yaml b/recipes/nemotron-3-super/vllm/agg-h200-chat/deploy.yaml index da5c036e79b5..c740bd128722 100644 --- a/recipes/nemotron-3-super/vllm/agg-h200-chat/deploy.yaml +++ b/recipes/nemotron-3-super/vllm/agg-h200-chat/deploy.yaml @@ -63,7 +63,7 @@ spec: - python3 args: - -m - - dynamo.frontend + - dingo.frontend - --trust-remote-code env: - name: POD_UID diff --git a/recipes/nemotron-3-ultra/vllm/agg-b200-agentic-mtp/deploy.yaml b/recipes/nemotron-3-ultra/vllm/agg-b200-agentic-mtp/deploy.yaml index bb89ff4d4454..5a40706c7177 100644 --- a/recipes/nemotron-3-ultra/vllm/agg-b200-agentic-mtp/deploy.yaml +++ b/recipes/nemotron-3-ultra/vllm/agg-b200-agentic-mtp/deploy.yaml @@ -94,9 +94,9 @@ spec: --http-host 0.0.0.0 --http-port 8000 ) - printf '%q ' python3 -m dynamo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" + printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dynamo.frontend "${frontend_args[@]}" + exec python3 -m dingo.frontend "${frontend_args[@]}" ports: - name: http containerPort: 8000 diff --git a/recipes/nemotron-3-ultra/vllm/agg-b200-agentic-nomtp/deploy.yaml b/recipes/nemotron-3-ultra/vllm/agg-b200-agentic-nomtp/deploy.yaml index dd3dec5d7350..827f5939136f 100644 --- a/recipes/nemotron-3-ultra/vllm/agg-b200-agentic-nomtp/deploy.yaml +++ b/recipes/nemotron-3-ultra/vllm/agg-b200-agentic-nomtp/deploy.yaml @@ -94,9 +94,9 @@ spec: --http-host 0.0.0.0 --http-port 8000 ) - printf '%q ' python3 -m dynamo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" + printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dynamo.frontend "${frontend_args[@]}" + exec python3 -m dingo.frontend "${frontend_args[@]}" ports: - name: http containerPort: 8000 diff --git a/recipes/nemotron-3-ultra/vllm/agg-b200-chat-mtp/deploy.yaml b/recipes/nemotron-3-ultra/vllm/agg-b200-chat-mtp/deploy.yaml index 1a84bae5a294..cdbf1c65eaa1 100644 --- a/recipes/nemotron-3-ultra/vllm/agg-b200-chat-mtp/deploy.yaml +++ b/recipes/nemotron-3-ultra/vllm/agg-b200-chat-mtp/deploy.yaml @@ -94,9 +94,9 @@ spec: --http-host 0.0.0.0 --http-port 8000 ) - printf '%q ' python3 -m dynamo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" + printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dynamo.frontend "${frontend_args[@]}" + exec python3 -m dingo.frontend "${frontend_args[@]}" ports: - name: http containerPort: 8000 diff --git a/recipes/nemotron-3-ultra/vllm/agg-b200-chat-nomtp/deploy.yaml b/recipes/nemotron-3-ultra/vllm/agg-b200-chat-nomtp/deploy.yaml index 89eb99e720db..3e1102c935c2 100644 --- a/recipes/nemotron-3-ultra/vllm/agg-b200-chat-nomtp/deploy.yaml +++ b/recipes/nemotron-3-ultra/vllm/agg-b200-chat-nomtp/deploy.yaml @@ -94,9 +94,9 @@ spec: --http-host 0.0.0.0 --http-port 8000 ) - printf '%q ' python3 -m dynamo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" + printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dynamo.frontend "${frontend_args[@]}" + exec python3 -m dingo.frontend "${frontend_args[@]}" ports: - name: http containerPort: 8000 diff --git a/recipes/nemotron-3-ultra/vllm/agg-h200-agentic-mtp/deploy.yaml b/recipes/nemotron-3-ultra/vllm/agg-h200-agentic-mtp/deploy.yaml index 9712579adf92..4e98321ddbe6 100644 --- a/recipes/nemotron-3-ultra/vllm/agg-h200-agentic-mtp/deploy.yaml +++ b/recipes/nemotron-3-ultra/vllm/agg-h200-agentic-mtp/deploy.yaml @@ -94,9 +94,9 @@ spec: --http-host 0.0.0.0 --http-port 8000 ) - printf '%q ' python3 -m dynamo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" + printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dynamo.frontend "${frontend_args[@]}" + exec python3 -m dingo.frontend "${frontend_args[@]}" ports: - name: http containerPort: 8000 diff --git a/recipes/nemotron-3-ultra/vllm/agg-h200-agentic-nomtp/deploy.yaml b/recipes/nemotron-3-ultra/vllm/agg-h200-agentic-nomtp/deploy.yaml index 38136fff71ad..9f8b7b33dc9e 100644 --- a/recipes/nemotron-3-ultra/vllm/agg-h200-agentic-nomtp/deploy.yaml +++ b/recipes/nemotron-3-ultra/vllm/agg-h200-agentic-nomtp/deploy.yaml @@ -94,9 +94,9 @@ spec: --http-host 0.0.0.0 --http-port 8000 ) - printf '%q ' python3 -m dynamo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" + printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dynamo.frontend "${frontend_args[@]}" + exec python3 -m dingo.frontend "${frontend_args[@]}" ports: - name: http containerPort: 8000 diff --git a/recipes/nemotron-3-ultra/vllm/agg-h200-chat-mtp/deploy.yaml b/recipes/nemotron-3-ultra/vllm/agg-h200-chat-mtp/deploy.yaml index cf5899fc3de8..8829da74afce 100644 --- a/recipes/nemotron-3-ultra/vllm/agg-h200-chat-mtp/deploy.yaml +++ b/recipes/nemotron-3-ultra/vllm/agg-h200-chat-mtp/deploy.yaml @@ -94,9 +94,9 @@ spec: --http-host 0.0.0.0 --http-port 8000 ) - printf '%q ' python3 -m dynamo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" + printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dynamo.frontend "${frontend_args[@]}" + exec python3 -m dingo.frontend "${frontend_args[@]}" ports: - name: http containerPort: 8000 diff --git a/recipes/nemotron-3-ultra/vllm/agg-h200-chat-nomtp/deploy.yaml b/recipes/nemotron-3-ultra/vllm/agg-h200-chat-nomtp/deploy.yaml index f5f60bafa926..89737bb95c1b 100644 --- a/recipes/nemotron-3-ultra/vllm/agg-h200-chat-nomtp/deploy.yaml +++ b/recipes/nemotron-3-ultra/vllm/agg-h200-chat-nomtp/deploy.yaml @@ -94,9 +94,9 @@ spec: --http-host 0.0.0.0 --http-port 8000 ) - printf '%q ' python3 -m dynamo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" + printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dynamo.frontend "${frontend_args[@]}" + exec python3 -m dingo.frontend "${frontend_args[@]}" ports: - name: http containerPort: 8000 diff --git a/recipes/nemotron-3-ultra/vllm/disagg-b200-agentic/deploy.yaml b/recipes/nemotron-3-ultra/vllm/disagg-b200-agentic/deploy.yaml index 7a9e4e1c6bcf..30116a0c2155 100644 --- a/recipes/nemotron-3-ultra/vllm/disagg-b200-agentic/deploy.yaml +++ b/recipes/nemotron-3-ultra/vllm/disagg-b200-agentic/deploy.yaml @@ -89,9 +89,9 @@ spec: --http-host 0.0.0.0 --http-port 8000 ) - printf '%q ' python3 -m dynamo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" + printf '%q ' python3 -m dingo.frontend "${frontend_args[@]}" >"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" printf '\n' >>"${LOG_DIR:-/tmp/nemotron-ultra}/status/vllm_frontend_command.txt" - exec python3 -m dynamo.frontend "${frontend_args[@]}" + exec python3 -m dingo.frontend "${frontend_args[@]}" ports: - name: http containerPort: 8000 diff --git a/recipes/qwen3-0.6b/vllm/agg/gaie/deploy.yaml b/recipes/qwen3-0.6b/vllm/agg/gaie/deploy.yaml index bfa98166cc67..3e1b1fffd754 100644 --- a/recipes/qwen3-0.6b/vllm/agg/gaie/deploy.yaml +++ b/recipes/qwen3-0.6b/vllm/agg/gaie/deploy.yaml @@ -59,7 +59,7 @@ spec: image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 args: - -m - - dynamo.frontend + - dingo.frontend - --router-mode - direct extraPodSpec: diff --git a/recipes/qwen3-235b-a22b-fp8/trtllm/agg/blackwell/deploy.yaml b/recipes/qwen3-235b-a22b-fp8/trtllm/agg/blackwell/deploy.yaml index b6d64922b461..170a63cdda8e 100644 --- a/recipes/qwen3-235b-a22b-fp8/trtllm/agg/blackwell/deploy.yaml +++ b/recipes/qwen3-235b-a22b-fp8/trtllm/agg/blackwell/deploy.yaml @@ -58,7 +58,7 @@ spec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 args: - - python3 -m dynamo.frontend --router-mode kv --http-port 8000 + - python3 -m dingo.frontend --router-mode kv --http-port 8000 command: - /bin/sh - -c diff --git a/recipes/qwen3-235b-a22b-fp8/trtllm/agg/hopper/deploy.yaml b/recipes/qwen3-235b-a22b-fp8/trtllm/agg/hopper/deploy.yaml index 357c85947649..1973438bf740 100644 --- a/recipes/qwen3-235b-a22b-fp8/trtllm/agg/hopper/deploy.yaml +++ b/recipes/qwen3-235b-a22b-fp8/trtllm/agg/hopper/deploy.yaml @@ -53,7 +53,7 @@ spec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 args: - - python3 -m dynamo.frontend --router-mode kv --http-port 8000 + - python3 -m dingo.frontend --router-mode kv --http-port 8000 command: - /bin/sh - -c diff --git a/recipes/qwen3-235b-a22b-fp8/trtllm/disagg/blackwell/deploy.yaml b/recipes/qwen3-235b-a22b-fp8/trtllm/disagg/blackwell/deploy.yaml index fe872a4463bd..7a6a09b07290 100644 --- a/recipes/qwen3-235b-a22b-fp8/trtllm/disagg/blackwell/deploy.yaml +++ b/recipes/qwen3-235b-a22b-fp8/trtllm/disagg/blackwell/deploy.yaml @@ -88,7 +88,7 @@ spec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 args: - - python3 -m dynamo.frontend --router-mode kv --http-port 8000 + - python3 -m dingo.frontend --router-mode kv --http-port 8000 command: - /bin/sh - -c diff --git a/recipes/qwen3-235b-a22b-fp8/trtllm/disagg/hopper/deploy.yaml b/recipes/qwen3-235b-a22b-fp8/trtllm/disagg/hopper/deploy.yaml index 1e375e20940f..b830021744bc 100644 --- a/recipes/qwen3-235b-a22b-fp8/trtllm/disagg/hopper/deploy.yaml +++ b/recipes/qwen3-235b-a22b-fp8/trtllm/disagg/hopper/deploy.yaml @@ -78,7 +78,7 @@ spec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.1 args: - - python3 -m dynamo.frontend --router-mode kv --http-port 8000 + - python3 -m dingo.frontend --router-mode kv --http-port 8000 command: - /bin/sh - -c diff --git a/recipes/qwen3-32b-fp8/trtllm/agg/deploy.yaml b/recipes/qwen3-32b-fp8/trtllm/agg/deploy.yaml index 272cb6b15e9b..33efc912f0b4 100644 --- a/recipes/qwen3-32b-fp8/trtllm/agg/deploy.yaml +++ b/recipes/qwen3-32b-fp8/trtllm/agg/deploy.yaml @@ -57,7 +57,7 @@ spec: topologyKey: kubernetes.io/hostname mainContainer: args: - - python3 -m dynamo.frontend --router-mode round-robin --http-port 8000 + - python3 -m dingo.frontend --router-mode round-robin --http-port 8000 command: - /bin/sh - -c diff --git a/recipes/qwen3-32b-fp8/trtllm/disagg/deploy.yaml b/recipes/qwen3-32b-fp8/trtllm/disagg/deploy.yaml index 3c51c9092720..9351260dad9b 100644 --- a/recipes/qwen3-32b-fp8/trtllm/disagg/deploy.yaml +++ b/recipes/qwen3-32b-fp8/trtllm/disagg/deploy.yaml @@ -214,7 +214,7 @@ spec: topologyKey: kubernetes.io/hostname mainContainer: args: - - python3 -m dynamo.frontend --router-mode round-robin --http-port 8000 + - python3 -m dingo.frontend --router-mode round-robin --http-port 8000 command: - /bin/sh - -c diff --git a/recipes/qwen3-32b/vllm/agg-kvbm/deploy.yaml b/recipes/qwen3-32b/vllm/agg-kvbm/deploy.yaml index 72dc7dd9800c..bcb041dc7465 100644 --- a/recipes/qwen3-32b/vllm/agg-kvbm/deploy.yaml +++ b/recipes/qwen3-32b/vllm/agg-kvbm/deploy.yaml @@ -23,7 +23,7 @@ spec: command: - python3 - -m - - dynamo.frontend + - dingo.frontend args: - --router-reset-states replicas: 1 diff --git a/recipes/qwen3-32b/vllm/agg-round-robin/deploy.yaml b/recipes/qwen3-32b/vllm/agg-round-robin/deploy.yaml index e1cb766c6404..cf32d9093f6b 100644 --- a/recipes/qwen3-32b/vllm/agg-round-robin/deploy.yaml +++ b/recipes/qwen3-32b/vllm/agg-round-robin/deploy.yaml @@ -23,7 +23,7 @@ spec: command: - python3 - -m - - dynamo.frontend + - dingo.frontend args: - --router-reset-states replicas: 1 diff --git a/recipes/qwen3-32b/vllm/disagg-kv-router/deploy.yaml b/recipes/qwen3-32b/vllm/disagg-kv-router/deploy.yaml index 43bdf232e426..ad78e1dee2f2 100644 --- a/recipes/qwen3-32b/vllm/disagg-kv-router/deploy.yaml +++ b/recipes/qwen3-32b/vllm/disagg-kv-router/deploy.yaml @@ -25,7 +25,7 @@ spec: command: - python - -m - - dynamo.frontend + - dingo.frontend image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.1 workingDir: /workspace replicas: 1 diff --git a/tests/conftest.py b/tests/conftest.py index d402a06426ba..c45c33db9358 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1274,7 +1274,7 @@ def num_system_ports(request) -> int: def dynamo_dynamic_ports(num_system_ports) -> Generator[ServicePorts, None, None]: """Allocate per-test ports for Dynamo deployments. - - frontend_port: OpenAI-compatible HTTP/gRPC ingress (dynamo.frontend) + - frontend_port: OpenAI-compatible HTTP/gRPC ingress (dingo.frontend) - system_ports: List of worker metrics/system ports (configurable count via num_system_ports) - kv_event_port: ZMQ port for vLLM KV event publishing (avoids collisions under xdist) """ diff --git a/tests/fault_tolerance/deploy/README.md b/tests/fault_tolerance/deploy/README.md index 4dc2cd7b59bc..e86794795fe9 100644 --- a/tests/fault_tolerance/deploy/README.md +++ b/tests/fault_tolerance/deploy/README.md @@ -106,7 +106,7 @@ The following failure types are defined in `scenarios.py`: | Failure Name | Description | Injection Method | Applicable Backends | |-------------------------------|----------------------------------------------------|-------------------------------|---------------------| | `none` | No failure injection (baseline). | N/A | All | -| `frontend` | Terminate frontend process. | `SIGINT` to `dynamo.frontend` | All | +| `frontend` | Terminate frontend process. | `SIGINT` to `dingo.frontend` | All | | `frontend_pod` | Delete frontend pod. | Kubernetes API pod deletion | All | | `decode_worker` | Terminate decode worker process. | `SIGKILL` to `dynamo.` | All | | `decode_worker_pod` | Delete decode worker pod. | Kubernetes API pod deletion | All | diff --git a/tests/fault_tolerance/deploy/scenarios.py b/tests/fault_tolerance/deploy/scenarios.py index 84da773d941e..363938c50450 100644 --- a/tests/fault_tolerance/deploy/scenarios.py +++ b/tests/fault_tolerance/deploy/scenarios.py @@ -942,7 +942,7 @@ def _create_backend_failures(backend, deploy_type="disagg"): failures = { "frontend": [ TerminateProcessFailure( - 30, ["Frontend"], "SIGINT", process_name="dynamo.frontend" + 30, ["Frontend"], "SIGINT", process_name="dingo.frontend" ) ], "frontend_pod": [DeletePodFailure(30, ["Frontend"])], diff --git a/tests/fault_tolerance/etcd_ha/utils.py b/tests/fault_tolerance/etcd_ha/utils.py index c4d0fc00247a..904173c08427 100644 --- a/tests/fault_tolerance/etcd_ha/utils.py +++ b/tests/fault_tolerance/etcd_ha/utils.py @@ -38,7 +38,7 @@ def __init__( "ETCD_ENDPOINTS": ",".join(etcd_endpoints), } # terminate_all_matching_process_names=False is required here: the frontend - # is launched as `python -m dynamo.frontend`, so its _command_name is the + # is launched as `python -m dingo.frontend`, so its _command_name is the # generic "python". With True, ManagedProcess.__enter__ would SIGTERM/SIGKILL # every "python" process on the host on startup — including this test's own # etcd cluster and vLLM worker (and sibling framework jobs). That orphans diff --git a/tests/frontend/realtime_echo_worker.py b/tests/frontend/realtime_echo_worker.py index 1ff8cd7e78e6..4542bbec6800 100644 --- a/tests/frontend/realtime_echo_worker.py +++ b/tests/frontend/realtime_echo_worker.py @@ -5,7 +5,7 @@ Registers a ``ModelType.Realtime`` + ``ModelInput.Text`` model and serves a Python ``async def generate(request_stream, context)`` bidirectional engine -via ``serve_bidirectional_endpoint``. A launched ``dynamo.frontend`` discovers +via ``serve_bidirectional_endpoint``. A launched ``dingo.frontend`` discovers this worker via the configured discovery backend (the e2e uses the file backend, ``DYN_FILE_KV``) and installs a typed realtime PushRouter to it. """ diff --git a/tests/frontend/test_prepost.py b/tests/frontend/test_prepost.py index 4f34c34a718b..8329514d80ee 100644 --- a/tests/frontend/test_prepost.py +++ b/tests/frontend/test_prepost.py @@ -25,7 +25,7 @@ from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser - from dynamo.frontend.prepost import StreamingPostProcessor + from dingo.frontend.prepost import StreamingPostProcessor else: # Fake some types so that `pre-commit` passes class CompletionOutput: diff --git a/tests/frontend/test_prepost_mistral.py b/tests/frontend/test_prepost_mistral.py index 75213182b05c..3903ec976179 100644 --- a/tests/frontend/test_prepost_mistral.py +++ b/tests/frontend/test_prepost_mistral.py @@ -26,7 +26,7 @@ from vllm.tokenizers.mistral import MistralTokenizer from vllm.tool_parsers.mistral_tool_parser import MistralToolParser - from dynamo.frontend.prepost import StreamingPostProcessor + from dingo.frontend.prepost import StreamingPostProcessor else: # Fake some types so that `pre-commit` passes class MistralTokenizer: diff --git a/tests/frontend/test_realtime_omni_bridge.py b/tests/frontend/test_realtime_omni_bridge.py index a58abfeb841a..0062f15500dd 100644 --- a/tests/frontend/test_realtime_omni_bridge.py +++ b/tests/frontend/test_realtime_omni_bridge.py @@ -4,7 +4,7 @@ """ End-to-end realtime WebSocket test for the vLLM-Omni realtime bridge. -A launched ``dynamo.frontend`` discovers a mock-Omni realtime worker (the real +A launched ``dingo.frontend`` discovers a mock-Omni realtime worker (the real ``RealtimeOmniHandler`` backed by a fake AsyncOmni that echoes audio) and installs a typed realtime PushRouter to it. A WebSocket client connects to ``/v1/realtime``, drives OpenAI Realtime client events, and asserts the diff --git a/tests/frontend/test_realtime_python_bridge.py b/tests/frontend/test_realtime_python_bridge.py index 786e9fba9416..80cec50ac7f9 100644 --- a/tests/frontend/test_realtime_python_bridge.py +++ b/tests/frontend/test_realtime_python_bridge.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """ -End-to-end realtime WebSocket test through a launched ``dynamo.frontend`` and +End-to-end realtime WebSocket test through a launched ``dingo.frontend`` and a separately launched Python bidirectional worker. The frontend discovers the worker's ``ModelType.Realtime`` MDC and installs a diff --git a/tests/frontend/test_tool_calling_sglang.py b/tests/frontend/test_tool_calling_sglang.py index 4d9190b03e87..61c391c63e12 100644 --- a/tests/frontend/test_tool_calling_sglang.py +++ b/tests/frontend/test_tool_calling_sglang.py @@ -76,7 +76,7 @@ def _prepare_log_dir(request, suffix: str) -> str: # switches will not touch unrelated processes in the system. _SGLANG_PROCESS_PATTERNS: tuple[str, ...] = ( "-m dynamo.sglang", - "-m dynamo.frontend", + "-m dingo.frontend", "SGLANG:EngineCore", "sglang::scheduler", ) @@ -136,7 +136,7 @@ def _cleanup_sglang_stragglers(timeout: float = 10.0) -> None: # frontend (``--dyn-chat-processor sglang``) with parsers declared as # frontend flags. Worker is a plain ``dynamo.sglang`` engine. # -# * ``rust_parsers`` — plain Rust frontend (``dynamo.frontend``) with no +# * ``rust_parsers`` — plain Rust frontend (``dingo.frontend``) with no # chat processor or parser flags. Parsers are declared on the worker # (``--dyn-reasoning-parser`` / ``--dyn-tool-call-parser``) and # propagated to the frontend via the model runtime config registered at @@ -207,7 +207,7 @@ def __init__(self, request, *, frontend_port: int, topology: str): command = [ "python3", "-m", - "dynamo.frontend", + "dingo.frontend", "--http-port", str(frontend_port), "--router-mode", @@ -233,7 +233,7 @@ def __init__(self, request, *, frontend_port: int, topology: str): timeout=240, display_output=True, terminate_all_matching_process_names=False, - straggler_commands=["-m dynamo.frontend"], + straggler_commands=["-m dingo.frontend"], log_dir=_prepare_log_dir(request, f"frontend-{topology}"), ) diff --git a/tests/kvbm_integration/test_consolidator_router_e2e.py b/tests/kvbm_integration/test_consolidator_router_e2e.py index d91482b2ad80..0d23ac6db1a4 100755 --- a/tests/kvbm_integration/test_consolidator_router_e2e.py +++ b/tests/kvbm_integration/test_consolidator_router_e2e.py @@ -281,7 +281,7 @@ def frontend_server(test_directory, runtime_services): command = [ "python", "-m", - "dynamo.frontend", + "dingo.frontend", "--http-port", str(FRONTEND_PORT), "--router-mode", @@ -734,7 +734,7 @@ def test_remove_deduplication_across_sources( frontend_command = [ "python", "-m", - "dynamo.frontend", + "dingo.frontend", "--http-port", str(FRONTEND_PORT), "--router-mode", diff --git a/tests/kvbm_integration/test_determinism_disagg.py b/tests/kvbm_integration/test_determinism_disagg.py index cce2878f5d08..087b023fd290 100755 --- a/tests/kvbm_integration/test_determinism_disagg.py +++ b/tests/kvbm_integration/test_determinism_disagg.py @@ -132,7 +132,7 @@ def _set_up_dynamo_config(self, router_mode: str = "round-robin"): self.dynamo_frontend_cmd = [ "python3", "-m", - "dynamo.frontend", + "dingo.frontend", "--router-mode", router_mode, "--http-port", diff --git a/tests/lmcache/deploy-baseline-dynamo-disag.sh b/tests/lmcache/deploy-baseline-dynamo-disag.sh index 9deba6f3ba14..3880c76e5d39 100755 --- a/tests/lmcache/deploy-baseline-dynamo-disag.sh +++ b/tests/lmcache/deploy-baseline-dynamo-disag.sh @@ -26,7 +26,7 @@ echo " Port: 8000" echo " Mode: Disaggregated (prefill + decode workers)" echo "🔧 Starting dynamo disaggregated serving without LMCache..." -python -m dynamo.frontend & +python -m dingo.frontend & CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.vllm --model $MODEL_URL --disaggregation-mode decode & diff --git a/tests/lmcache/deploy-baseline-dynamo.sh b/tests/lmcache/deploy-baseline-dynamo.sh index b20406bbd17a..ff6ebcdd61ae 100755 --- a/tests/lmcache/deploy-baseline-dynamo.sh +++ b/tests/lmcache/deploy-baseline-dynamo.sh @@ -24,5 +24,5 @@ echo " Model: $MODEL_URL" echo " Port: 8000" echo "🔧 Starting dynamo worker without LMCache..." -python -m dynamo.frontend & +python -m dingo.frontend & python3 -m dynamo.vllm --model $MODEL_URL diff --git a/tests/lmcache/deploy-lmcache_enabled-dynamo-disag.sh b/tests/lmcache/deploy-lmcache_enabled-dynamo-disag.sh index 2f606fa87ed2..ed6383820112 100755 --- a/tests/lmcache/deploy-lmcache_enabled-dynamo-disag.sh +++ b/tests/lmcache/deploy-lmcache_enabled-dynamo-disag.sh @@ -27,7 +27,7 @@ echo " Mode: Disaggregated (prefill + decode workers) + LMCache" echo " !! Remember to kill the old dynamo processes otherwise the port will be busy !!" echo "🔧 Starting dynamo disaggregated serving with LMCache enabled..." -python -m dynamo.frontend & +python -m dingo.frontend & CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.vllm --model $MODEL_URL --disaggregation-mode decode & diff --git a/tests/lmcache/deploy-lmcache_enabled-dynamo.sh b/tests/lmcache/deploy-lmcache_enabled-dynamo.sh index 298b7efdcbdf..5a3b3e5f4bb5 100755 --- a/tests/lmcache/deploy-lmcache_enabled-dynamo.sh +++ b/tests/lmcache/deploy-lmcache_enabled-dynamo.sh @@ -25,6 +25,6 @@ echo " Port: 8000" echo " !! Remmber to kill the old dynamo processes other wise the port will be busy !! " echo "🔧 Starting dynamo worker with LMCache enabled..." -python -m dynamo.frontend & +python -m dingo.frontend & python3 -m dynamo.vllm --model $MODEL_URL --kv-transfer-config '{"kv_connector":"LMCacheConnectorV1","kv_role":"kv_both"}' diff --git a/tests/mm_router/test_mm_router_e2e.py b/tests/mm_router/test_mm_router_e2e.py index a8b4855bd01d..3acec4dbefb1 100644 --- a/tests/mm_router/test_mm_router_e2e.py +++ b/tests/mm_router/test_mm_router_e2e.py @@ -176,7 +176,7 @@ def __init__(self, request, *, frontend_port: int): command=[ "python3", "-m", - "dynamo.frontend", + "dingo.frontend", "--http-port", str(frontend_port), "--router-mode", @@ -187,7 +187,7 @@ def __init__(self, request, *, frontend_port: int): (f"http://localhost:{frontend_port}/v1/models", check_models_api) ], timeout=240, - straggler_commands=["-m dynamo.frontend"], + straggler_commands=["-m dingo.frontend"], log_dir=_prepare_log_dir(request, "trtllm-mm-frontend"), **_COMMON_PROCESS_KWARGS, ) diff --git a/tests/mm_router/test_router_rust_mm_frontend_decode_e2e.py b/tests/mm_router/test_router_rust_mm_frontend_decode_e2e.py index e7f01f302d31..47feed461568 100644 --- a/tests/mm_router/test_router_rust_mm_frontend_decode_e2e.py +++ b/tests/mm_router/test_router_rust_mm_frontend_decode_e2e.py @@ -150,7 +150,7 @@ def __init__(self, request, *, frontend_port: int): command=[ "python3", "-m", - "dynamo.frontend", + "dingo.frontend", "--http-port", str(frontend_port), "--router-mode", @@ -165,7 +165,7 @@ def __init__(self, request, *, frontend_port: int): (f"http://localhost:{frontend_port}/v1/models", check_models_api) ], timeout=240, - straggler_commands=["-m dynamo.frontend"], + straggler_commands=["-m dingo.frontend"], log_dir=_prepare_log_dir(request, "frontend-fed"), **_COMMON_PROCESS_KWARGS, ) diff --git a/tests/mm_router/test_router_rust_mm_router_e2e.py b/tests/mm_router/test_router_rust_mm_router_e2e.py index 2a460d6b965f..c4a9badfb121 100644 --- a/tests/mm_router/test_router_rust_mm_router_e2e.py +++ b/tests/mm_router/test_router_rust_mm_router_e2e.py @@ -151,7 +151,7 @@ def __init__(self, request, *, frontend_port: int): command=[ "python3", "-m", - "dynamo.frontend", + "dingo.frontend", "--http-port", str(frontend_port), "--router-mode", @@ -166,7 +166,7 @@ def __init__(self, request, *, frontend_port: int): (f"http://localhost:{frontend_port}/v1/models", check_models_api) ], timeout=240, - straggler_commands=["-m dynamo.frontend"], + straggler_commands=["-m dingo.frontend"], log_dir=_prepare_log_dir(request, "router-rust-frontend"), **_COMMON_PROCESS_KWARGS, ) diff --git a/tests/mm_router/test_vllm_mm_router_e2e.py b/tests/mm_router/test_vllm_mm_router_e2e.py index c51e8fd588ba..55abfbbb92b3 100644 --- a/tests/mm_router/test_vllm_mm_router_e2e.py +++ b/tests/mm_router/test_vllm_mm_router_e2e.py @@ -164,7 +164,7 @@ def __init__(self, request, *, frontend_port: int, transfer_mode: str = "shm"): command=[ "python3", "-m", - "dynamo.frontend", + "dingo.frontend", "--http-port", str(frontend_port), "--dyn-chat-processor", @@ -181,7 +181,7 @@ def __init__(self, request, *, frontend_port: int, transfer_mode: str = "shm"): (f"http://localhost:{frontend_port}/v1/models", check_models_api) ], timeout=240, - straggler_commands=["-m dynamo.frontend"], + straggler_commands=["-m dingo.frontend"], log_dir=_prepare_log_dir(request, f"vllm-mm-frontend-{transfer_mode}"), **_COMMON_PROCESS_KWARGS, ) diff --git a/tests/router/helper.py b/tests/router/helper.py index 3f67848e46c1..543373854205 100644 --- a/tests/router/helper.py +++ b/tests/router/helper.py @@ -191,7 +191,7 @@ async def wait_for_frontend_ready( 2. Polls GET /v1/models until at least one model is registered. 3. Sends a test POST to /v1/chat/completions to verify the request pipeline is functional. - Use this when testing through the HTTP frontend server (dynamo.frontend). + Use this when testing through the HTTP frontend server (dingo.frontend). For direct Python API testing with KvRouter, use wait_for_workers_ready() instead. Args: diff --git a/tests/router/router_process.py b/tests/router/router_process.py index 988fbbf19886..42e5d63f626c 100644 --- a/tests/router/router_process.py +++ b/tests/router/router_process.py @@ -31,7 +31,7 @@ def __init__(self, request, port: int): class FrontendRouterProcess(ManagedProcess): - """Manages a dynamo.frontend process with configurable --router-mode. + """Manages a dingo.frontend process with configurable --router-mode. Supports all router modes (round-robin, random, kv, direct) and all KV-specific options (block size, thresholds, durable events, disagg). @@ -63,7 +63,7 @@ def __init__( command = [ sys.executable, "-m", - "dynamo.frontend", + "dingo.frontend", "--router-mode", router_mode, "--http-port", diff --git a/tests/router/test_router_e2e_with_sglang.py b/tests/router/test_router_e2e_with_sglang.py index d85538fea29e..57c1369675f6 100644 --- a/tests/router/test_router_e2e_with_sglang.py +++ b/tests/router/test_router_e2e_with_sglang.py @@ -57,7 +57,7 @@ class SGLangProcess(ManagedEngineProcessMixin): The key difference: dynamo.sglang automatically handles: - HTTP API serving - KV cache event publishing (ZMQ → NATS bridge) - - Integration with dynamo.frontend router + - Integration with dingo.frontend router """ def __init__( diff --git a/tests/router/test_router_e2e_with_trtllm.py b/tests/router/test_router_e2e_with_trtllm.py index cb0273754528..66c8211270af 100644 --- a/tests/router/test_router_e2e_with_trtllm.py +++ b/tests/router/test_router_e2e_with_trtllm.py @@ -64,7 +64,7 @@ class TRTLLMProcess(ManagedEngineProcessMixin): The key difference: dynamo.trtllm automatically handles: - HTTP API serving - KV cache event publishing - - Integration with dynamo.frontend router + - Integration with dingo.frontend router """ def __init__( diff --git a/tests/router/test_router_e2e_with_vllm.py b/tests/router/test_router_e2e_with_vllm.py index d8ac82d299ba..fe26b1eb6662 100644 --- a/tests/router/test_router_e2e_with_vllm.py +++ b/tests/router/test_router_e2e_with_vllm.py @@ -82,7 +82,7 @@ class VLLMProcess(ManagedEngineProcessMixin): The key difference: dynamo.vllm automatically handles: - HTTP API serving - KV cache event publishing (ZMQ → NATS bridge) - - Integration with dynamo.frontend router + - Integration with dingo.frontend router """ def __init__( diff --git a/tests/serve/launch/agg_raw_embeddings_llava.sh b/tests/serve/launch/agg_raw_embeddings_llava.sh index ca922890ff4f..3d93652fdcf5 100755 --- a/tests/serve/launch/agg_raw_embeddings_llava.sh +++ b/tests/serve/launch/agg_raw_embeddings_llava.sh @@ -186,7 +186,7 @@ echo " Encode worker → CUDA_VISIBLE_DEVICES=${ENCODE_CUDA_VISIBLE_DEVICES}" echo " PD worker → CUDA_VISIBLE_DEVICES=${PD_CUDA_VISIBLE_DEVICES}" # Frontend -python3 -m dynamo.frontend & +python3 -m dingo.frontend & # Encode worker (vision encoder on GPU 0) echo "[Phase 2] Starting Encode worker on GPU ${ENCODE_CUDA_VISIBLE_DEVICES} ..." diff --git a/tests/serve/launch/multi_node_tp_headless.sh b/tests/serve/launch/multi_node_tp_headless.sh index 6fed9b1316c0..67e9c720cf66 100755 --- a/tests/serve/launch/multi_node_tp_headless.sh +++ b/tests/serve/launch/multi_node_tp_headless.sh @@ -19,7 +19,7 @@ if [[ -n "$KV_BYTES" ]]; then fi echo "Starting Dynamo frontend..." -python3 -m dynamo.frontend & +python3 -m dingo.frontend & echo "Starting dynamo.vllm head node (TP=2, nnodes=2, node-rank=0, GPU 0)..." CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.vllm \ diff --git a/tests/serve/launch/template_verifier.sh b/tests/serve/launch/template_verifier.sh index 5a8550968384..b16766fc3d1b 100755 --- a/tests/serve/launch/template_verifier.sh +++ b/tests/serve/launch/template_verifier.sh @@ -17,8 +17,8 @@ cleanup() { trap cleanup EXIT INT TERM # run ingress -# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) -python3 -m dynamo.frontend & +# dingo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) +python3 -m dingo.frontend & FRONTEND_PID=$! # run the mock worker + template validation generate() diff --git a/tests/utils/managed_process.py b/tests/utils/managed_process.py index d2189f69323d..acb50ca6509b 100644 --- a/tests/utils/managed_process.py +++ b/tests/utils/managed_process.py @@ -933,14 +933,14 @@ def __init__( frontend_port = allocate_port(DefaultPort.FRONTEND.value) self._allocated_http_port = frontend_port - # If frontend_port is unset, dynamo.frontend defaults to DefaultPort.FRONTEND. + # If frontend_port is unset, dingo.frontend defaults to DefaultPort.FRONTEND. self.http_port = ( DefaultPort.FRONTEND.value if frontend_port is None else int(frontend_port) ) - command = ["python", "-m", "dynamo.frontend", "--router-mode", router_mode] + command = ["python", "-m", "dingo.frontend", "--router-mode", router_mode] - # dynamo.frontend defaults to 8000 when neither env nor flag is provided. + # dingo.frontend defaults to 8000 when neither env nor flag is provided. if frontend_port is not None: command.extend(["--http-port", str(frontend_port)]) # Migration limit is configured at the frontend level @@ -995,7 +995,7 @@ def main(): # Example: Safe for parallel tests with terminate_all_matching_process_names=False with ManagedProcess( - command=["python", "-m", "dynamo.frontend", "--port", "8000"], + command=["python", "-m", "dingo.frontend", "--port", "8000"], display_output=True, terminate_all_matching_process_names=False, # ✅ Safe - only kills what we launch health_check_ports=[8000], From c32b9df6c94b8ed80cb53a8884b1256f2e25f2b0 Mon Sep 17 00:00:00 2001 From: xguo01 <13910754971@163.com> Date: Sat, 4 Jul 2026 02:16:51 +0800 Subject: [PATCH 2/7] [feat][ai] Delete ai. --- .agents/skills/debug-session/SKILL.md | 166 ----- .agents/skills/dep-create/SKILL.md | 144 ----- .agents/skills/dep-status/SKILL.md | 80 --- .agents/skills/dep-update/SKILL.md | 96 --- .../dynamo-clone-hotpath-audit/SKILL.md | 223 ------- .../scripts/clone_inventory.py | 397 ------------ .agents/skills/dynamo-docs/SKILL.md | 407 ------------ .../skills/dynamo-frontend-benchmark/SKILL.md | 233 ------- .../scripts/analyze_folded.py | 105 ---- .../scripts/capture_offcpu.sh | 79 --- .../dynamo-frontend-benchmark/scripts/env.sh | 58 -- .../scripts/extract_throughput.py | 91 --- .../scripts/isolate.sh | 56 -- .../scripts/profile_oncpu.sh | 84 --- .../scripts/run_aiperf.sh | 43 -- .../scripts/smoke.sh | 38 -- .../scripts/start.sh | 159 ----- .../dynamo-frontend-benchmark/scripts/stop.sh | 48 -- .../scripts/unisolate.sh | 36 -- .../dynamo-interconnect-check/BENCHMARK.md | 64 -- .../skills/dynamo-interconnect-check/SKILL.md | 160 ----- .../evals/evals.json | 62 -- .../references/interconnect-env-vars.md | 67 -- .../scripts/check_interconnect.py | 356 ----------- .../dynamo-interconnect-check/skill-card.md | 49 -- .../dynamo-interconnect-check/skill.oms.sig | 1 - .../skills/dynamo-recipe-runner/BENCHMARK.md | 64 -- .agents/skills/dynamo-recipe-runner/SKILL.md | 213 ------- .../dynamo-recipe-runner/evals/evals.json | 66 -- .../references/k8s-recipe-workflow.md | 119 ---- .../scripts/recipe_tool.py | 325 ---------- .../skills/dynamo-recipe-runner/skill-card.md | 50 -- .../skills/dynamo-recipe-runner/skill.oms.sig | 1 - .../skills/dynamo-router-starter/BENCHMARK.md | 63 -- .agents/skills/dynamo-router-starter/SKILL.md | 174 ------ .../dynamo-router-starter/evals/evals.json | 63 -- .../references/router-modes.md | 58 -- .../scripts/check_router_health.py | 142 ----- .../dynamo-router-starter/skill-card.md | 49 -- .../dynamo-router-starter/skill.oms.sig | 1 - .../skills/dynamo-troubleshoot/BENCHMARK.md | 66 -- .agents/skills/dynamo-troubleshoot/SKILL.md | 165 ----- .../dynamo-troubleshoot/evals/evals.json | 64 -- .../references/failure-decision-tree.md | 170 ----- .../scripts/collect_dynamo_debug_bundle.py | 266 -------- .../skills/dynamo-troubleshoot/skill-card.md | 52 -- .../skills/dynamo-troubleshoot/skill.oms.sig | 1 - .agents/skills/gh-issue-bug/SKILL.md | 73 --- .agents/skills/graham-code-review/SKILL.md | 100 --- .agents/skills/pr-monitor/SKILL.md | 257 -------- .ai/bash-launch-guidelines.md | 309 --------- .ai/ci-guidelines.md | 47 -- .ai/linear-ticket-refs.md | 30 - .ai/pytest-guidelines.md | 587 ------------------ .ai/python-guidelines.md | 427 ------------- .ai/test-model-size-guardrails.md | 61 -- .github/filters.yaml | 3 - .github/workflows/nvskills-team-request.yml | 202 ------ .github/workflows/request-nvskills-ci.yml | 19 - docs/README.md | 9 - .../tests/test_nixl_connect_lazy_import.py | 3 +- skills | 1 - tests/conftest.py | 2 +- tests/utils/payloads.py | 3 +- 64 files changed, 3 insertions(+), 7604 deletions(-) delete mode 100644 .agents/skills/debug-session/SKILL.md delete mode 100644 .agents/skills/dep-create/SKILL.md delete mode 100644 .agents/skills/dep-status/SKILL.md delete mode 100644 .agents/skills/dep-update/SKILL.md delete mode 100644 .agents/skills/dynamo-clone-hotpath-audit/SKILL.md delete mode 100755 .agents/skills/dynamo-clone-hotpath-audit/scripts/clone_inventory.py delete mode 100644 .agents/skills/dynamo-docs/SKILL.md delete mode 100644 .agents/skills/dynamo-frontend-benchmark/SKILL.md delete mode 100755 .agents/skills/dynamo-frontend-benchmark/scripts/analyze_folded.py delete mode 100755 .agents/skills/dynamo-frontend-benchmark/scripts/capture_offcpu.sh delete mode 100755 .agents/skills/dynamo-frontend-benchmark/scripts/env.sh delete mode 100755 .agents/skills/dynamo-frontend-benchmark/scripts/extract_throughput.py delete mode 100755 .agents/skills/dynamo-frontend-benchmark/scripts/isolate.sh delete mode 100755 .agents/skills/dynamo-frontend-benchmark/scripts/profile_oncpu.sh delete mode 100755 .agents/skills/dynamo-frontend-benchmark/scripts/run_aiperf.sh delete mode 100755 .agents/skills/dynamo-frontend-benchmark/scripts/smoke.sh delete mode 100755 .agents/skills/dynamo-frontend-benchmark/scripts/start.sh delete mode 100755 .agents/skills/dynamo-frontend-benchmark/scripts/stop.sh delete mode 100755 .agents/skills/dynamo-frontend-benchmark/scripts/unisolate.sh delete mode 100644 .agents/skills/dynamo-interconnect-check/BENCHMARK.md delete mode 100644 .agents/skills/dynamo-interconnect-check/SKILL.md delete mode 100644 .agents/skills/dynamo-interconnect-check/evals/evals.json delete mode 100644 .agents/skills/dynamo-interconnect-check/references/interconnect-env-vars.md delete mode 100755 .agents/skills/dynamo-interconnect-check/scripts/check_interconnect.py delete mode 100644 .agents/skills/dynamo-interconnect-check/skill-card.md delete mode 100644 .agents/skills/dynamo-interconnect-check/skill.oms.sig delete mode 100644 .agents/skills/dynamo-recipe-runner/BENCHMARK.md delete mode 100644 .agents/skills/dynamo-recipe-runner/SKILL.md delete mode 100644 .agents/skills/dynamo-recipe-runner/evals/evals.json delete mode 100644 .agents/skills/dynamo-recipe-runner/references/k8s-recipe-workflow.md delete mode 100755 .agents/skills/dynamo-recipe-runner/scripts/recipe_tool.py delete mode 100644 .agents/skills/dynamo-recipe-runner/skill-card.md delete mode 100644 .agents/skills/dynamo-recipe-runner/skill.oms.sig delete mode 100644 .agents/skills/dynamo-router-starter/BENCHMARK.md delete mode 100644 .agents/skills/dynamo-router-starter/SKILL.md delete mode 100644 .agents/skills/dynamo-router-starter/evals/evals.json delete mode 100644 .agents/skills/dynamo-router-starter/references/router-modes.md delete mode 100755 .agents/skills/dynamo-router-starter/scripts/check_router_health.py delete mode 100644 .agents/skills/dynamo-router-starter/skill-card.md delete mode 100644 .agents/skills/dynamo-router-starter/skill.oms.sig delete mode 100644 .agents/skills/dynamo-troubleshoot/BENCHMARK.md delete mode 100644 .agents/skills/dynamo-troubleshoot/SKILL.md delete mode 100644 .agents/skills/dynamo-troubleshoot/evals/evals.json delete mode 100644 .agents/skills/dynamo-troubleshoot/references/failure-decision-tree.md delete mode 100755 .agents/skills/dynamo-troubleshoot/scripts/collect_dynamo_debug_bundle.py delete mode 100644 .agents/skills/dynamo-troubleshoot/skill-card.md delete mode 100644 .agents/skills/dynamo-troubleshoot/skill.oms.sig delete mode 100644 .agents/skills/gh-issue-bug/SKILL.md delete mode 100644 .agents/skills/graham-code-review/SKILL.md delete mode 100644 .agents/skills/pr-monitor/SKILL.md delete mode 100644 .ai/bash-launch-guidelines.md delete mode 100644 .ai/ci-guidelines.md delete mode 100644 .ai/linear-ticket-refs.md delete mode 100644 .ai/pytest-guidelines.md delete mode 100644 .ai/python-guidelines.md delete mode 100644 .ai/test-model-size-guardrails.md delete mode 100644 .github/workflows/nvskills-team-request.yml delete mode 100644 .github/workflows/request-nvskills-ci.yml delete mode 120000 skills diff --git a/.agents/skills/debug-session/SKILL.md b/.agents/skills/debug-session/SKILL.md deleted file mode 100644 index ced9bb545b15..000000000000 --- a/.agents/skills/debug-session/SKILL.md +++ /dev/null @@ -1,166 +0,0 @@ ---- -name: debug-session -description: Start a debugging session with worklog file -user-invocable: true -disable-model-invocation: true ---- - -# Start Debug Session - - - -Create a structured debugging session for an issue in the Dynamo ecosystem. - -## Step 1: Get the Bug Report - -Ask the user how they want to provide the bug: - -**Option A: Linear ticket** -- User provides ticket ID (e.g., "DYN-123") -- Fetch via Linear MCP tools -- Extract: title, description, reproduction steps - -**Option B: GitHub issue** -- User provides issue URL -- Fetch via `gh issue view ` -- Extract: title, description, reproduction steps - -**Option C: Paste** -- Ask user to paste the bug report directly -- Parse out the key details - -## Step 2: Discover Environment - -Gather environment information: - -!`nvidia-smi --query-gpu=name,count --format=csv,noheader 2>/dev/null || echo "No GPU detected"` - -!`uname -a` - -!`which python && python --version` - -This tells you: -- GPU type and count (L40s, H100s, etc.) -- OS/platform -- Python environment - -**Note**: The user's `~/.claude/CLAUDE.md` may have more details about their dev environment (paths, aliases, preferences). Check there for additional context. - -## Step 3: Create Worklog - -Create a worklog file to track the investigation: - -- Filename: `.md` in current directory -- Template: - -```markdown -# Debug: [Issue Title] - -**Date**: [today's date] -**Source**: [Linear ticket / GitHub issue / user report] -**Status**: investigating -**Environment**: [GPU type/count from nvidia-smi] - -## Problem -[Description of the issue] - -## Reproduction Steps -1. [Step to reproduce] -2. ... - -## Expected vs Actual -- **Expected**: -- **Actual**: - -## Investigation Log - -### [timestamp] -[Notes on what you tried/found] - -## Root Cause -[Fill in when found] - -## Fix -[Fill in when implemented] -``` - -## Step 4: Set Up Testing - -### Build Commands - -Rebuild Dynamo after making changes: -```bash -cd lib/bindings/python && maturin develop --uv && cd ../../.. && uv pip install -e . -``` - -If a framework change is required (sglang, vllm, trtllm), check the user's `~/.claude/CLAUDE.md` for rebuild instructions specific to that framework. - -### Running Examples - -Examples are located at: `/home/ubuntu/dynamo/examples/backends/` - -Available backends: -- `sglang/launch/` - SGLang backend examples -- `vllm/launch/` - vLLM backend examples -- `trtllm/launch/` - TensorRT-LLM backend examples - -Based on the bug report, determine which backend is relevant: -- If unclear, **ask the user** which backend/example to run -- Run the example in the background -- Wait for model to be ready - -### Verifying the Model is Up - -```bash -curl localhost:8000/v1/models -``` - -### Testing with a Request - -```bash -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "", - "messages": [{"role": "user", "content": "Hello"}], - "max_tokens": 50 - }' -``` - -## Step 5: Begin Investigation - -### Dynamo Infrastructure Debugging - -**KV cache and routing issues:** -- Check KV event logs in `lib/llm/src/block_manager/kv_consolidator/tracker.rs` -- Look at block manager state and consolidation behavior -- Inspect routing decisions in the KV-aware router - -**ZMQ / networking issues:** -- Check ZMQ socket configuration and endpoint bindings -- Look for connection timeouts or message drops -- Verify nats/etcd connectivity for service discovery - -**Multi-node / disaggregated issues:** -- Check prefill/decode worker assignment -- Verify DGD (disaggregated) status reporting -- Inspect inter-node communication via `nvidia-smi` on each node -- Check NCCL and GPU direct RDMA status - -**Process inspection:** -- `ps aux | grep dynamo` - check running processes -- `nvidia-smi` - GPU utilization and memory -- `ss -tlnp | grep 8000` - check port bindings -- `journalctl -u dynamo` - systemd logs if applicable - -### General Debugging Workflow - -1. **Reproduce first** - verify you can trigger the bug before attempting fixes -2. **Document as you go** - update the worklog with findings -3. **Minimal changes** - fix the bug, do not refactor surrounding code -4. **Verify the fix** - confirm the reproduction case now passes - -Performance-critical code - avoid unnecessary abstractions or comments. diff --git a/.agents/skills/dep-create/SKILL.md b/.agents/skills/dep-create/SKILL.md deleted file mode 100644 index 8488b2938e44..000000000000 --- a/.agents/skills/dep-create/SKILL.md +++ /dev/null @@ -1,144 +0,0 @@ ---- -name: dep-create -description: Create or update Dynamo Enhancement Proposals as GitHub issues, including lightweight DEPs, implementation plans, and retroactive DEPs for ai-dynamo/dynamo. ---- - -# Skill: Create a DEP as a GitHub Issue - - - -## Purpose - -Create a new Dynamo Enhancement Proposal (DEP) as a GitHub Issue on -`ai-dynamo/dynamo`. The issue number becomes the DEP number. Also -handles adding implementation plans and retroactive DEPs for existing -work. - -## When to Use - -When the user wants to propose a new feature, architecture change, or -process improvement via the issue-based DEP workflow. Also when adding -an implementation plan to an existing DEP, or filing a retroactive DEP -for work already merged. - -## Workflow - -### Create a New DEP - -1. **Ask for source material**: Prompt the user for a Google Doc, - Confluence page, or other NVIDIA-internal document that contains - the background, customer context, or detailed requirements. Read - it using the appropriate tool (gdocs, Confluence MCP, WebFetch). - Include the link in the issue's References section — the document - is only accessible to NVIDIA employees and serves as the record - for customer-specific context that cannot appear in the public - issue prose. - -2. **Gather required fields** from the user and source doc (prompt - if missing): - - **Summary**: One-paragraph description of the proposal - - **Motivation**: Why this change is needed - - **Proposal**: Detailed description of the proposed change - -3. **Determine the area label** based on proposal content. Area labels - are bare names (e.g., `frontend`, `router`, `backend-vllm`) that - correspond to CODEOWNERS teams. - -4. **Decide template**: full or lightweight. - Use lightweight if only Summary, Motivation, and Proposal are needed. - -5. **Create the issue** (full DEP): - -```bash -gh issue create \ - --repo ai-dynamo/dynamo \ - --title "DEP: " \ - --label "dep:draft" \ - --label "" \ - --body "$(cat <<'EOF' -## Summary - - -## Motivation - - -## Proposal - - -## Alternate Solutions - - -## Requirements - - -## References - -EOF -)" -``` - - **For lightweight DEP**, use: - -```bash -gh issue create \ - --repo ai-dynamo/dynamo \ - --title "DEP (light): " \ - --label "dep:draft" \ - --label "dep:lightweight" \ - --label "" \ - --body "$(cat <<'EOF' -## Summary - - -## Motivation - - -## Proposal - -EOF -)" -``` - -6. **Report** the created issue number and URL to the user. - -### Add an Implementation Plan - -1. **Read the DEP issue** and its discussion: - -```bash -gh issue view --repo ai-dynamo/dynamo -gh issue view --repo ai-dynamo/dynamo --comments -``` - -2. **Draft the plan** with phases, tasks, effort estimates, - dependencies, risks, and testing strategy. - -3. **Post as a comment**: - -```bash -gh issue comment --repo ai-dynamo/dynamo --body-file /tmp/plan.md -``` - -### Retroactive DEP - -For work already merged without a DEP, file with `dep:implementing` -or `dep:done` and reference the existing PRs. - -## Notes - -- The issue body IS the spec — treat it as a living document. -- `dep:draft` is applied automatically. PIC changes to - `dep:under-review` when ready. -- For lightweight DEPs, use `dep:lightweight` label and omit optional - sections. -- For plan revisions, post a new comment with a changelog at the top. - Do not edit the original — preserve the timeline. -- **Customer name stripping**: Before creating or updating a DEP, - scan the summary, motivation, proposal, and all other fields for - specific customer names, company names, or partner names. Replace - them with generic references (e.g., "a customer", "a cloud - partner", "an enterprise user"). DEPs are public — no customer - names should appear in issue bodies, comments, or plans. diff --git a/.agents/skills/dep-status/SKILL.md b/.agents/skills/dep-status/SKILL.md deleted file mode 100644 index b58ac70f8223..000000000000 --- a/.agents/skills/dep-status/SKILL.md +++ /dev/null @@ -1,80 +0,0 @@ ---- -name: dep-status -description: Check Dynamo Enhancement Proposal issue status, list DEPs by lifecycle state or area, and find related DEP issues in ai-dynamo/dynamo. ---- - -# Skill: Check DEP Status - - - -## Purpose - -List DEP issues with their current status, area, PIC, and approval -state. Find related DEPs for a given topic or component. - -## When to Use - -When the user wants to see the status of one or more DEPs, check what's -pending review, find DEPs related to a component, or get a triage -summary. - -## Workflow - -1. **List open DEP issues**: - -```bash -gh issue list --repo ai-dynamo/dynamo \ - --search 'label:"dep:draft","dep:under-review","dep:approved","dep:implementing"' \ - --json number,title,labels,assignees,createdAt,updatedAt -``` - -2. **Filter by area** (if requested): - -```bash -gh issue list --repo ai-dynamo/dynamo \ - --label "" \ - --json number,title,labels,assignees -``` - -3. **Filter by status** (if requested): - -```bash -gh issue list --repo ai-dynamo/dynamo \ - --label "dep:" \ - --json number,title,labels,assignees -``` - -4. **Format as a summary table**: - -```text -| # | Title | Status | Area | PIC | Updated | -|---|-------|--------|------|-----|---------| -| 42 | DEP: KV router scheduling | dep:under-review | router | @pic | 2026-03-28 | -``` - -5. **Find related DEPs** by searching issue titles and bodies: - -```bash -gh issue list --repo ai-dynamo/dynamo \ - --search 'DEP label:"dep:draft","dep:under-review","dep:approved","dep:implementing","dep:done"' \ - --json number,title,labels,state -``` - -6. **Include closed DEPs** if requested: - -```bash -gh issue list --repo ai-dynamo/dynamo \ - --state closed \ - --search 'label:"dep:done","dep:deferred","dep:rejected","dep:replaced"' \ - --json number,title,labels,assignees,closedAt -``` - -## Notes - -- For a full triage view, include both open and recently closed DEPs. -- Cross-reference with `dep:lightweight` label to distinguish full vs. - lightweight DEPs. -- Area labels are bare names (e.g., `frontend`, `router`) — no prefix. diff --git a/.agents/skills/dep-update/SKILL.md b/.agents/skills/dep-update/SKILL.md deleted file mode 100644 index 72efae221bb8..000000000000 --- a/.agents/skills/dep-update/SKILL.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -name: dep-update -description: Update Dynamo Enhancement Proposal lifecycle state in GitHub, including triage, PIC assignment, review, approval, and status label changes. ---- - -# Skill: Update DEP Lifecycle - - - -## Purpose - -Update DEP status through its lifecycle — triage, review, approve, -defer, or close. Covers the PIC workflow from initial assignment -through final approval. - -## When to Use - -When triaging DEP issues, reviewing a DEP as PIC or reviewer, -approving a DEP that is under review, or updating DEP status. - -## Workflow - -### Triage (assign PIC) - -1. **List unassigned DEPs**: - -```bash -gh issue list --repo ai-dynamo/dynamo \ - --label "dep:draft" \ - --json number,title,labels,assignees \ - --jq '.[] | select(.assignees | length == 0)' -``` - -2. **Assign PIC** based on the area label: - -```bash -gh issue edit --repo ai-dynamo/dynamo \ - --add-assignee "" -``` - -3. **Move to review** when the spec is ready: - -```bash -gh issue edit --repo ai-dynamo/dynamo \ - --remove-label "dep:draft" \ - --add-label "dep:under-review" -``` - -### Review - -1. **Read the DEP issue and discussion**: - -```bash -gh issue view --repo ai-dynamo/dynamo -gh issue view --repo ai-dynamo/dynamo --comments -``` - -2. **Post review feedback** as comments on the issue. - -3. **Request changes** or clarifications from the author. - -### Approve - -1. **Verify the issue is under review**: - -```bash -gh issue view --repo ai-dynamo/dynamo --json labels -``` - -2. **Post the approval comment**: - -```bash -gh issue comment --repo ai-dynamo/dynamo --body "/approve" -``` - -3. **If this is the PIC approving** (or all required reviewers have - approved), update the label: - -```bash -gh issue edit --repo ai-dynamo/dynamo \ - --remove-label "dep:under-review" \ - --add-label "dep:approved" -``` - -## Notes - -- For straightforward DEPs, the PIC's `/approve` is sufficient. -- For multi-reviewer DEPs, the PIC maintains a pinned approval - checklist and updates the label only when all required approvals are - collected. -- `/approve` comments are searchable for audit: - `gh search issues --repo ai-dynamo/dynamo "/approve" in:comments` -- Area labels are bare names (e.g., `frontend`, `router`) — no prefix. diff --git a/.agents/skills/dynamo-clone-hotpath-audit/SKILL.md b/.agents/skills/dynamo-clone-hotpath-audit/SKILL.md deleted file mode 100644 index 3a117fada55b..000000000000 --- a/.agents/skills/dynamo-clone-hotpath-audit/SKILL.md +++ /dev/null @@ -1,223 +0,0 @@ ---- -name: dynamo-clone-hotpath-audit -description: Audit Dynamo Rust hot-path `.clone()` calls, explain which clones are removable and why, and only apply clone-removal patches when explicitly requested. -license: Apache-2.0 -metadata: - author: NVIDIA - tags: - - dynamo - - rust - - performance - - code-review - - allocation - permissions: - - file_read - - file_write ---- - -# Dynamo Clone Hotpath Audit - - - -## Purpose - -Find `.clone()` calls in Dynamo Rust request, scheduling, KV, block-manager, and -runtime hot paths that can be removed without changing ownership semantics. This -is an audit-first workflow, not a blanket clone-removal tool. - -Default behavior is read-only audit. Do not edit files, commit, or open a PR -unless the user explicitly asks to fix, patch, apply, implement, or create an -MR/PR. A skill invocation such as `$dynamo-clone-hotpath-audit`, "audit", -"check", or "scan" is not patch permission. - -## Prerequisites - -- Rust source checkout of `ai-dynamo/dynamo`. -- Python 3.10+ for the inventory script. -- Ability to run targeted Rust validation commands for touched crates. -- Subagent tooling if available. If subagents are unavailable, run the same - roles as separate, explicit review passes and say they were not independent. - -## Instructions - -### 1. Build The Inventory - -Run the read-only scanner from the repository root: - -```bash -python3 .agents/skills/dynamo-clone-hotpath-audit/scripts/clone_inventory.py \ - --only-actionable \ - --limit 80 -``` - -Use narrower paths when the user names a subsystem: - -```bash -python3 .agents/skills/dynamo-clone-hotpath-audit/scripts/clone_inventory.py \ - --paths lib/kv-router/src/scheduling lib/llm/src/backend \ - --only-actionable -``` - -Treat the scanner as a triage aid. It ranks likely expensive clones but does not -prove removability. - -### 2. Split The Audit By Hot Path - -Prioritize in this order: - -1. per-request LLM paths: `lib/llm/src/backend.rs`, preprocessor, HTTP/gRPC - services, protocol conversion, migration -2. KV routing and scheduling: `lib/kv-router/src/scheduling`, - `lib/kv-router/src/sequences`, `lib/kv-router/src/indexer` -3. KV/block manager event paths: `lib/llm/src/block_manager`, - `lib/bindings/kvbm/src/block_manager`, `lib/kvbm-engine/src/offload` -4. runtime request/transport paths: `lib/runtime/src/component`, - `lib/runtime/src/pipeline`, `lib/runtime/src/transports` -5. tests, examples, debug paths, and benchmark setup only after hot paths - -### 3. Use Independent Review Roles - -For non-trivial audits, use subagents. Give each subagent only the inventory -slice and relevant source files, not your intended answer. - -Required roles: - -- candidate finder: identify high-value clones and classify cheap/required ones -- ownership refactorer: propose concrete borrow, move, `Arc`, or `mem::take` - changes -- correctness adversary: reject changes that alter sharing, extend lock - lifetimes, borrow across `.await`, break spawned task ownership, or make APIs - less clear -- validation planner: choose the smallest tests, clippy commands, or benchmarks - that prove the touched behavior - -If two roles disagree, keep the clone unless you can write down a precise -ownership proof and validation plan. - -### 4. Classify Every Candidate - -Use these buckets: - -- cheap required: `Arc`, sender, cancellation token, runtime handle, watch - receiver, metrics handle, or tracing span clone needed to share ownership -- semantic required: the original value must stay available for a later use, - retry, fan-out, or async task -- cold/test-only: outside production hot paths -- removable move: value is cloned immediately before its final ownership use -- removable borrow: callee does not need ownership and can accept `&T`, `&str`, - slice, or iterator -- structural refactor: requires changing data layout, e.g. `Vec` to - `Arc<[T]>`, or changing an API family - -Do not patch candidates in the first three buckets. - -### 5. Patch In Small Batches - -Only run this section when the user explicitly asks for fixes or a patch. If the -user only asked for an audit, stop after the report and list recommended patch -batches as follow-up work. - -Keep each patch batch narrow: one subsystem or one repeated pattern. Avoid a -single repository-wide clone cleanup PR unless the user explicitly asks for it. - -Preferred fixes: - -- move a value when the clone is immediately consumed and not used afterward -- pass `&T`, `&str`, `&[T]`, or an iterator when the callee only reads -- consume batches with `into_iter()` instead of indexing and cloning -- extract small `Copy` fields before moving a large event -- use `std::mem::take` only when leaving the source value empty is part of the - intended semantics -- use `Arc` only when shared ownership is semantically right, not just to avoid - borrow-checker work - -Do not: - -- remove clones that cross `tokio::spawn`, channel send, callback storage, or - task lifetime boundaries without proving ownership -- extend mutex/RwLock guard lifetimes to avoid a clone -- borrow across `.await` unless the borrow is local and compiler-verified -- trade a clear cheap clone for a confusing lifetime-heavy API -- make public APIs borrow data whose ownership was intentionally independent - -### 6. Required Finding Format - -For every proposed change, report: - -- `file:line` -- hot-path rationale -- cloned value and likely clone cost -- bucket and recommended change -- removal rationale: why this clone is unnecessary, not only why it is expensive -- required-clone check: why it is not cheap required or semantic required -- correctness proof: why the old and new ownership semantics match -- validation command - -Example: - -```text -lib/llm/src/preprocessor.rs:123 -Hot path: every embeddings request. -Cost: clones Vec before moving into spawn_blocking. -Bucket: removable move. -Change: move input_strs into the closure. -Removal rationale: the clone feeds the only owned consumer and the original is -not read after closure construction. -Required-clone check: no fan-out, retry, async task sharing, or later logging -uses the original value. -Proof: the closure receives the same owned Vec; all later behavior reads -from that moved value. -Validation: cargo test -p dynamo-llm preprocessor -``` - -### 7. Validate Before Finishing - -If patches were made, always run formatting and the narrowest relevant Rust -tests. For broad API changes, also run clippy or crate-level tests for each -touched crate. If no patches were made, do not run tests just to make the audit -look validated; report the inventory command and any read-only review checks. - -Return: - -- clone candidates reviewed -- candidates intentionally left alone and why -- patches made, or `none: audit-only` -- tests run and results, or `not run: no files changed` -- residual high-value candidates that should be separate PRs - -## Available Scripts - -| Script | Purpose | Arguments | -|---|---|---| -| `scripts/clone_inventory.py` | Rank Rust `.clone()` call sites by hot-path likelihood and removal potential | `--paths`, `--limit`, `--format`, `--only-actionable`, `--include-tests` | - -Invoke via the agentskills.io `run_script()` protocol: - -```python -run_script("scripts/clone_inventory.py", args=["--only-actionable", "--limit", "80"]) -``` - -## Output Contract - -Return a concise audit report or PR summary with: - -- scope and inventory command -- top findings with buckets -- per finding: removal rationale, required-clone check, correctness proof, and - validation command -- exact refactors made, or `none: audit-only` -- independent review role outcomes -- validation commands and status -- follow-up candidates excluded from the current audit or patch - -## Limitations - -- The scanner is heuristic and line-based. It can miss macro-expanded clones, - multi-line ownership patterns, or clones hidden behind helper methods. -- Some expensive-looking clones are required for async task ownership, fan-out, - retries, or API clarity. -- Performance impact is inferred unless backed by benchmarks or allocation - profiles. diff --git a/.agents/skills/dynamo-clone-hotpath-audit/scripts/clone_inventory.py b/.agents/skills/dynamo-clone-hotpath-audit/scripts/clone_inventory.py deleted file mode 100755 index ce955619f5a9..000000000000 --- a/.agents/skills/dynamo-clone-hotpath-audit/scripts/clone_inventory.py +++ /dev/null @@ -1,397 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Rank Rust .clone() call sites for Dynamo hot-path clone audits.""" - -from __future__ import annotations - -import argparse -import json -import re -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Iterable - -DEFAULT_PATHS = ("lib", "crates", "components") - -CLONE_RE = re.compile(r"(?:\bArc::clone\s*\(|\.clone\s*\()") - -TEST_PATH_MARKERS = ( - "/tests/", - "/benches/", - "/examples/", - "/bench/", - "_test.rs", - "tests.rs", -) - -HOT_PATH_RULES = ( - ("lib/llm/src/backend", 10, "llm request backend"), - ("lib/llm/src/preprocessor", 9, "llm request preprocessing"), - ("lib/llm/src/http", 8, "http request path"), - ("lib/llm/src/grpc", 8, "grpc request path"), - ("lib/llm/src/protocols", 7, "protocol conversion"), - ("lib/llm/src/migration", 7, "request retry/migration"), - ("lib/llm/src/kv_router", 8, "llm kv routing"), - ("lib/llm/src/block_manager", 8, "block manager path"), - ("lib/bindings/kvbm/src/block_manager", 8, "kvbm binding block manager"), - ("lib/kvbm-engine/src/offload", 8, "kvbm offload path"), - ("lib/kvbm-logical/src", 7, "kvbm logical manager"), - ("lib/kvbm-physical/src", 7, "kvbm physical manager"), - ("lib/kv-router/src/scheduling", 9, "kv-router scheduler"), - ("lib/kv-router/src/sequences", 8, "kv-router sequence tracker"), - ("lib/kv-router/src/indexer", 8, "kv-router indexer"), - ("lib/runtime/src/component", 7, "runtime component path"), - ("lib/runtime/src/pipeline", 7, "runtime pipeline path"), - ("lib/runtime/src/transports", 7, "runtime transport path"), - ("lib/runtime/src/storage", 6, "runtime storage path"), -) - -CHEAP_HINTS = ( - "Arc::clone", - "CancellationToken", - "cancel_token", - "shutdown_token", - "sender", - "receiver", - "_tx", - "_rx", - "watch::", - "Handle", - "span.clone", - "metrics", -) - -DEEP_VALUE_HINTS = ( - "tokens", - "token_ids", - "token_block", - "token_chunks", - "request", - "response", - "event", - "payload", - "metadata", - "blocks", - "block_hashes", - "scores", - "overlap", - "runtime_data", - "annotations", - "messages", - "content", - "schema", - "config", -) - -BOUNDARY_HINTS = ( - "tokio::spawn", - "spawn_blocking", - ".send(", - ".try_send(", - ".instrument(", - ".map_err(", - "async move", -) - - -@dataclass(frozen=True) -class CloneCandidate: - score: int - tier: str - category: str - path: str - line: int - hot_path: str - reasons: list[str] - text: str - - -def require_under_root(root: Path, path: Path) -> Path: - resolved_root = root.resolve() - resolved_path = path.resolve() - try: - resolved_path.relative_to(resolved_root) - except ValueError as exc: - msg = f"refusing to scan path outside repository root: {path}" - raise SystemExit(msg) from exc - return resolved_path - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--root", - default=".", - help="Repository root. Defaults to current working directory.", - ) - parser.add_argument( - "--paths", - nargs="+", - default=DEFAULT_PATHS, - help="Files or directories to scan, relative to --root.", - ) - parser.add_argument( - "--limit", - type=int, - default=120, - help="Maximum candidates to print. Use 0 for all.", - ) - parser.add_argument( - "--format", - choices=("markdown", "json"), - default="markdown", - help="Output format.", - ) - parser.add_argument( - "--only-actionable", - action="store_true", - help="Hide cheap shared-ownership and test-only clones.", - ) - parser.add_argument( - "--include-tests", - action="store_true", - help="Keep test, benchmark, and example clone sites in normal scoring.", - ) - return parser.parse_args() - - -def rust_files(root: Path, paths: Iterable[str]) -> list[Path]: - files: set[Path] = set() - for raw_path in paths: - path = require_under_root(root, root / raw_path) - if path.is_file() and path.suffix == ".rs": - files.add(path) - elif path.is_dir(): - files.update(p for p in path.rglob("*.rs") if p.is_file()) - return sorted(files) - - -def rel_path(root: Path, path: Path) -> str: - try: - return path.resolve().relative_to(root.resolve()).as_posix() - except ValueError: - return path.as_posix() - - -def is_test_path(path: str) -> bool: - normalized = f"/{path}" - return any(marker in normalized for marker in TEST_PATH_MARKERS) - - -def hot_path_score(path: str) -> tuple[int, str]: - for prefix, score, label in HOT_PATH_RULES: - if path.startswith(prefix): - return score, label - return 2, "not a known hot path" - - -def context_window(lines: list[str], index: int) -> str: - start = max(0, index - 5) - end = min(len(lines), index + 3) - return "\n".join(lines[start:end]) - - -def has_loop_context(window: str) -> bool: - return bool(re.search(r"^\s*(for|while|loop)\b", window, re.MULTILINE)) - - -def score_candidate( - path: str, line: str, window: str, include_tests: bool -) -> CloneCandidate: - base_score, hot_path = hot_path_score(path) - score = base_score - reasons = [hot_path] - - stripped = line.strip() - path_is_test = is_test_path(path) - is_cold = path_is_test and not include_tests - is_cheap = any(hint in stripped for hint in CHEAP_HINTS) - is_loop = has_loop_context(window) - is_clone_into_arc = "Arc::new(" in stripped and ".clone" in stripped - - if is_cold: - score -= 8 - reasons.append("test, benchmark, or example path") - - if is_cheap: - score -= 5 - reasons.append("looks like shared-handle or control-plane clone") - - if any(hint in stripped for hint in DEEP_VALUE_HINTS): - score += 3 - reasons.append("name suggests non-trivial owned data") - - if is_loop: - score += 4 - reasons.append("clone appears inside or near a loop") - - if is_clone_into_arc: - score += 5 - reasons.append("owned value may be cloned before Arc wrapping") - - if ".clone()" in stripped and any(hint in window for hint in BOUNDARY_HINTS): - score -= 2 - reasons.append("near async/task/channel boundary; ownership may be required") - - if re.search(r"let\s+\w+\s*=\s*\w+\.clone\(\)\s*;", stripped): - score += 2 - reasons.append("simple clone assignment; check whether final use can move") - - if ".clone()." in stripped or ( - ".clone()" in stripped and "unwrap_or_else" in stripped - ): - score += 1 - reasons.append("clone participates in expression chain") - - if is_cold: - category = "cold_or_test" - elif is_cheap: - category = "cheap_shared" - elif is_clone_into_arc: - category = "clone_into_arc" - elif is_loop: - category = "loop_clone" - elif score >= 8: - category = "suspicious_hotpath" - else: - category = "needs_review" - - tier = "high" if score >= 12 else "medium" if score >= 8 else "low" - return CloneCandidate( - score=score, - tier=tier, - category=category, - path=path, - line=0, - hot_path=hot_path, - reasons=reasons, - text=stripped, - ) - - -def scan_file(root: Path, path: Path, include_tests: bool) -> list[CloneCandidate]: - rel = rel_path(root, path) - text = path.read_text(encoding="utf-8", errors="replace") - lines = text.splitlines() - candidates: list[CloneCandidate] = [] - - for index, line in enumerate(lines): - stripped = line.strip() - if not stripped or stripped.startswith("//"): - continue - if not CLONE_RE.search(line): - continue - - candidate = score_candidate( - rel, line, context_window(lines, index), include_tests - ) - candidates.append( - CloneCandidate( - score=candidate.score, - tier=candidate.tier, - category=candidate.category, - path=rel, - line=index + 1, - hot_path=candidate.hot_path, - reasons=candidate.reasons, - text=candidate.text, - ) - ) - - return candidates - - -def scan(root: Path, paths: Iterable[str], include_tests: bool) -> list[CloneCandidate]: - candidates: list[CloneCandidate] = [] - for path in rust_files(root, paths): - candidates.extend(scan_file(root, path, include_tests)) - return sorted(candidates, key=lambda c: (-c.score, c.path, c.line)) - - -def filtered( - candidates: list[CloneCandidate], only_actionable: bool -) -> list[CloneCandidate]: - if not only_actionable: - return candidates - ignored = {"cheap_shared", "cold_or_test"} - return [candidate for candidate in candidates if candidate.category not in ignored] - - -def limited(candidates: list[CloneCandidate], limit: int) -> list[CloneCandidate]: - if limit <= 0: - return candidates - return candidates[:limit] - - -def summarize(candidates: list[CloneCandidate]) -> dict[str, object]: - by_tier: dict[str, int] = {} - by_category: dict[str, int] = {} - for candidate in candidates: - by_tier[candidate.tier] = by_tier.get(candidate.tier, 0) + 1 - by_category[candidate.category] = by_category.get(candidate.category, 0) + 1 - return { - "total": len(candidates), - "by_tier": dict(sorted(by_tier.items())), - "by_category": dict(sorted(by_category.items())), - } - - -def markdown_escape(value: str) -> str: - return value.replace("|", "\\|").replace("\n", " ") - - -def print_markdown( - candidates: list[CloneCandidate], all_candidates: list[CloneCandidate] -) -> None: - summary = summarize(all_candidates) - print("# Rust Clone Hotpath Inventory") - print() - print(f"Total candidates after filters: {summary['total']}") - print(f"Tier counts: `{summary['by_tier']}`") - print(f"Category counts: `{summary['by_category']}`") - print() - print("| Score | Tier | Category | Location | Reasons | Code |") - print("|---:|---|---|---|---|---|") - for candidate in candidates: - location = f"{candidate.path}:{candidate.line}" - reasons = "; ".join(candidate.reasons) - print( - "| " - f"{candidate.score} | " - f"{candidate.tier} | " - f"{candidate.category} | " - f"`{markdown_escape(location)}` | " - f"{markdown_escape(reasons)} | " - f"`{markdown_escape(candidate.text)}` |" - ) - - -def print_json( - candidates: list[CloneCandidate], all_candidates: list[CloneCandidate] -) -> None: - payload = { - "summary": summarize(all_candidates), - "candidates": [asdict(candidate) for candidate in candidates], - } - print(json.dumps(payload, indent=2, sort_keys=True)) - - -def main() -> int: - args = parse_args() - root = Path(args.root).resolve() - all_candidates = filtered( - scan(root, args.paths, args.include_tests), - only_actionable=args.only_actionable, - ) - candidates = limited(all_candidates, args.limit) - - if args.format == "json": - print_json(candidates, all_candidates) - else: - print_markdown(candidates, all_candidates) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.agents/skills/dynamo-docs/SKILL.md b/.agents/skills/dynamo-docs/SKILL.md deleted file mode 100644 index 5e86f1115689..000000000000 --- a/.agents/skills/dynamo-docs/SKILL.md +++ /dev/null @@ -1,407 +0,0 @@ ---- -name: dynamo-docs -description: Add, update, move, or remove content on the Dynamo Fern docs site — standard docs pages, catalog-driven recipe and feature-benchmark pages, examples, recipes, and translations — keeping everything in line with the documentation style guide. Use for any change under docs/, recipes/, or examples/ (new page, edit, section move, rename, removal, recipe/benchmark page, .zh-CN translation, version cut) and whenever content needs its frontmatter, headings, links, callouts, or terminology fixed. ---- - -# Dynamo Docs Maintenance - - - -Unified skill for adding, updating, moving, and removing content on the Dynamo Fern documentation -site, in line with the project's authoring guides. - -Two authoring guides govern this work; read whichever applies before writing: - -- [`docs/documentation-style-guide.md`](https://github.com/ai-dynamo/dynamo/blob/main/docs/documentation-style-guide.md) — the standard for **every** page: frontmatter, headings, prose, terminology, links, callouts. The must-fix subset is distilled in [Style Guide Is the Standard](#style-guide-is-the-standard) and [Content Rules](#content-rules) below. -- [`docs/recipes/_catalog/README.md`](https://github.com/ai-dynamo/dynamo/blob/main/docs/recipes/_catalog/README.md) — the standard for **recipe and feature-benchmark pages** (the catalog contract, the `.mdx` page blueprint, and the pure-CSS target picker). See [Add a Recipe or Feature Benchmark Page](#add-a-recipe-or-feature-benchmark-page). - -## Branch Rule - -**ALL edits happen on `main` (or a feature branch based on `main`).** -The `docs-website` branch is CI-managed and must **never** be edited by hand. - -## Style Guide Is the Standard - -Every page under `docs/` (and the READMEs under `examples/` and `recipes/`) follows the -[Documentation Style Guide](https://github.com/ai-dynamo/dynamo/blob/main/docs/documentation-style-guide.md) -(`docs/documentation-style-guide.md`). Read it before writing content. The docs bot enforces a -**must-fix** subset on every PR — get these right or the checks fail: - -- **SPDX header** on every file, copyright range `2025-2026`. Fern pages put the two `#` lines - *inside* the `---` frontmatter; plain READMEs use an HTML-comment block. -- **Frontmatter with at least one metadata key** (`title`/`subtitle`/`sidebar-title`) and **no body - `# H1`**. Fern renders the page H1 from the nav `page:` value, so a body `# H1` produces a - duplicate title — and a bare `#` SPDX line left in the body also renders as an H1. Start the body - at `##`. -- **A nav entry** in `docs/index.yml` for every new page — a page not in the nav is unreachable. -- **Links**: relative path *with extension* within `docs/` (`[Routing](router-concepts.md)`); - absolute `https://github.com/ai-dynamo/dynamo/blob/main/` URL for targets outside `docs/` - (examples, recipes, source; `/tree/main/` for a directory). No `../` path that escapes `docs/`, and - never a hardcoded `https://docs.nvidia.com/...` link to a page in this repo. Link text names the - destination, never "click here". -- **No internal or sensitive references**: NVBug/JIRA/Linear IDs, internal hostnames, secrets, - `TODO`/`FIXME`. - -Everything else in the style guide (page types, heading case, terminology, list and code-fence -formatting, the pre-merge checklist) is guidance — the high-value rules are distilled in -[Content Rules](#content-rules) below; apply them and deviate only with a reason. - -## Content Rules - -Apply these on every page so the result reads like a person wrote it and passes review without a -round-trip to the style guide. These are defaults; deviate with a reason. - -- **Page type (Diátaxis).** Each page serves one need — *tutorial* (`getting-started/`), *how-to* - (`backends//`, `kubernetes/`), *reference* (flags/APIs/config), or *explanation* - (`design-docs/`). Don't blend a how-to into a flag reference; split and cross-link. -- **Headings.** Title Case for short label / noun-phrase headings ("Routing Behavior"); sentence - case for full-phrase headings ("Choosing a checkpoint flow"). Be consistent within a page. No end - punctuation. Logical `##` → `###` hierarchy, no skipped levels. Renaming a heading breaks inbound - `#anchor` links — rename deliberately. -- **Terminology, exact casing.** Backends: **vLLM**, **SGLang**, **TensorRT-LLM** (or **TRT-LLM**) — - never "vllm", "Sglang", "TensorRT LLM". **NVIDIA Dynamo** on first mention, then **Dynamo**; **KV - router**, **NIXL**, **GPU**; **Kubernetes**, not "k8s", in prose. Expand acronyms on first use - ("Time To First Token (TTFT)"). Use one word per concept. -- **Inclusive terms.** "denylist"/"allowlist", not "blacklist"/"whitelist"; "primary"/"replica", not - "master"/"slave". -- **Cut marketing and bombast.** Remove "seamless, robust, powerful, blazing-fast, cutting-edge, - effortless, unlock, leverage, delve, comprehensive, rich ecosystem, world-class, game-changing". - Cut filler ("it's important to note", "simply", "just", "in order to") and difficulty words - ("easy", "easily"). Start sentences with a verb; active voice; present tense; second-person - imperative. Name the flag/default/command, not "configure the appropriate settings". Avoid the - em-dash-aside tic. -- **Procedures.** Condition before instruction ("To enable KV-aware routing, set `--router-mode - kv`", not the reverse). One action per numbered step. -- **Links.** Follow the must-fix Links rule in - [Style Guide Is the Standard](#style-guide-is-the-standard) (relative + extension inside `docs/`, - absolute GitHub URL outside, no `../` escape, no `docs.nvidia.com` self-link). -- **Code fences** always tag a language (`bash`, not `sh`); no `$`/`#` prompt prefixes; put output in - its own `text` block. Wrap flags, paths, and `DYN_*` env vars in backticks in prose. -- **Lifecycle.** Mark preview features **Experimental.** and legacy ones **Deprecated.** (with a - `> [!WARNING]`); note availability for new features ("Available since v0.X"). - -## Operations - -Pick your operation: - -- Standard `.md` doc page → [Add a Page](#add-a-page) -- Rendered recipe / feature-benchmark page (`.mdx` + catalog triple) → [Add a Recipe or Feature Benchmark Page](#add-a-recipe-or-feature-benchmark-page) -- Code under `examples/` or `recipes/` → [Add an Example or Recipe (code)](#add-an-example-or-recipe-code) -- Edit, move, or remove existing content → [Update a Page](#update-a-page), [Remove a Page](#remove-a-page) (recipes: [Move, defer, or remove a recipe](#move-defer-or-remove-a-recipe)) -- Chinese translation or version cut → [Translations and Versioned Navs](#translations-and-versioned-navs) - -### Add a Page - -1. **Choose placement from the live nav.** Open `docs/index.yml` and find the existing page closest - in topic to yours — your page joins **that** section, and its file goes in that sibling's - subdirectory under `docs/`. Page *type* narrows the field (tutorial → `getting-started/`, how-to → - `backends//` or `kubernetes/`, reference → flags/APIs/config, explanation → - `design-docs/`), but the nearest existing page is the tie-breaker — don't guess from the section - names in [Navigation](#navigation-tabs-and-sections), read the file. Note the section, the - subdirectory, a kebab-case `.md` filename, and the page title. -2. Create `docs//.md`. Frontmatter carries the SPDX header plus at least one - metadata key; the body starts at `##` with a short intro — **no body `# H1`**: - -```markdown ---- -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -title: -subtitle: ---- - -Short intro paragraph stating what the page covers. - -## -``` - -3. Add a nav entry in `docs/index.yml` under the section you chose in step 1 — a `- page:` in that - section's `contents:`, 2-space indent, `path:` relative to `docs/` (see - [Navigation](#navigation-tabs-and-sections) for the grammar): - -```yaml -- page: - path: /.md -``` - -### Update a Page - -1. Locate by file path, page title, or keyword search (`grep -rn` in `docs/`). -2. **Content only** -- edit the markdown file directly; keep it within the style guide. -3. **Title/label change** -- update the frontmatter (`title`/`sidebar-title`) and the `- page:` name - in `docs/index.yml`. -4. **Section move** -- `git mv` the file when the subdirectory changes, move the nav entry to the new - section, and update every incoming link. - -> [!IMPORTANT] -> A page's URL is `/`, where the page-name slug comes from the nav -> `page:` label. Moving a page to another section **or** renaming its label changes that URL. Add a -> **dev-scoped** redirect to the `redirects:` list in `fern/docs.yml`: `/dynamo/dev/` → -> `/dynamo/dev/`. Editing `docs/index.yml` regenerates only the `dev` nav, so do **not** redirect -> the unversioned (`/dynamo/`) or `/dynamo/latest/` forms — those serve **Latest**, a frozen -> release snapshot that `main` edits don't touch, and a redirect there would break a working URL. See -> [Redirects and the version model](#redirects-and-the-version-model). - -### Remove a Page - -1. Find incoming links: `grep -rn "" docs/`. -2. `git rm docs//.md`. -3. Remove the `- page:` block from `docs/index.yml`. If it was the last page in a section, remove the - whole `- section:` block. -4. Fix or remove every incoming link found in step 1, and add a `fern/docs.yml` redirect if the page - had a stable URL. - -### Add a Recipe or Feature Benchmark Page - -Recipe and feature-benchmark pages are **catalog-driven** and use `.mdx` (they embed a pure-CSS -target picker). Authoritative guide: -[`docs/recipes/_catalog/README.md`](https://github.com/ai-dynamo/dynamo/blob/main/docs/recipes/_catalog/README.md). -Each page is a triple — page + catalog entry + nav: - -1. **Write the `.mdx`** at `docs/recipes/.mdx` (or `docs/benchmarks/.mdx`). Frontmatter - carries SPDX + `title` + one-sentence `subtitle`; body starts with a short intro, then the target - picker — multi-target pages use the radio picker, single-target pages use the **static** form - (exact classes under [Target picker](#target-picker) below) — then the fixed section order: - `## Prerequisites` → `## Deploy` → `## Smoke Test` → `## Benchmark` → `## Expected Performance` - (omit if no numbers) → `## Compare All Targets` (multi-target only) → `## Related Feature - Benchmarks` → `## Notes` → `## Source`. **MDX rule:** blank line after `
` and before - `
`; keep code fences at column 0. -2. **Add a catalog entry** — one file at `docs/recipes/_catalog/recipes/.yaml` (or - `docs/benchmarks/_catalog/benchmarks/.yaml`), SPDX header, exactly one object. **Read the - sibling `schema.json` first for the exact field set** (`docs/recipes/_catalog/schema.json` for - recipes, `docs/benchmarks/_catalog/schema.json` for benchmarks — they are **different** schemas) — - each is `additionalProperties: false`, so an invented or misspelled key fails validation; don't - guess the shape. A **recipe** entry requires `id`, - `title`, `provider`, `model`, `status`, `targets`, `maintainer`, and each `targets[]` item - requires `id`, `recommended`, `hardware`, `runtime`, `topology`, `techniques`, `workload`, - `deploy`, `expected_performance`. Internal `id:` **must equal the filename**; active entries carry - `page:`, deferred ones carry `deferred_reason` and omit `page:`. Add the `` to the matching - `_catalog/index.yaml` (`recipes:` for active, `deferred_recipes:` for deferred — it controls - sidebar/landing order). -3. **Wire navigation** in `docs/index.yml`: a `- page:` under `- tab: recipes` for recipes, or under - the **Feature Benchmarks** section (`- tab: docs`) for benchmarks. Per-benchmark pages are usually - `hidden: true` (surfaced from the landing page). -4. **Patch `fern/main.css` only if** the page introduces a picker axis value not already supported - (`recipe-sku`: `b200`/`h200`/`h100`/`gb200`/`hopper`/`blackwell`; `recipe-usecase`: - `chat`/`agentic`; `recipe-variant`: `agg`/`disagg`/…). A value missing from CSS renders but - filters nothing. -5. **Add the landing card** in `docs/recipes/README.mdx` and update the model/target counts. -6. **Validate**: `python3 docs/recipes/_catalog/validate.py` (covers both catalogs), then `fern - check` and `fern docs broken-links`. - -#### Catalog entry shape - -`schema.json` is authoritative for the field set; this skeleton just anchors the **nested shapes and -enums** that are easy to get wrong (`model`/`hardware`/`runtime`/`workload`/`deploy`/ -`expected_performance` are **objects**, not scalars; `status` and `topology` are **enums**). Minimal -valid active entry: - -```yaml -id: llama-3-1-8b # == filename; pattern ^[a-z0-9][a-z0-9-]*$ -title: Llama 3.1 8B -provider: meta # landing-page filter key (meta, qwen, nvidia, …) -model: - name: Llama 3.1 8B - hf_id: Meta-Llama/Llama-3.1-8B - precision: BF16 -status: validated # enum: validated | experimental (NOT "active") -page: recipes/llama-3-1-8b.mdx # active only; deferred → omit page:, add deferred_reason: -maintainer: Jane Doe # or null (null is tracked as a gap) -targets: # >= 1 item - - id: vllm-agg-h100 - recommended: true # bool - hardware: { gpu: H100, count: 1 } - runtime: { framework: vllm } - topology: aggregated # enum: aggregated | disaggregated - techniques: [bf16] - workload: { type: chat } - deploy: { asset: recipes/llama-3-1-8b/vllm/agg/deploy.yaml } - expected_performance: { available: false } # add summary: when numbers exist -``` - -**Benchmarks use a different schema.** A `docs/benchmarks/_catalog/benchmarks/.yaml` entry -validates against `docs/benchmarks/_catalog/schema.json`, whose required set is `id`, `title`, `page`, -`claim`, `subtype` (enum: `ab-test`/`feature-stack`/`topology`/`provider-comparison`/`hands-on`), -`features`, `model`, `hardware`, `traffic`, `arms`, `results`, `maintainer` — **no** `provider`, -`status`, or `targets`. The skeleton above is recipe-only; read the benchmark schema for that shape. - -#### Target picker - -The picker is pure CSS under the `dynamo-*` namespace — **MDX uses `className`, not `class`**, and the -exact class names matter (a wrong class name, or a `class=`-spelled wrapper, renders but filters nothing). A -**multi-target** page renders `
` containing a -`dynamo-target-picker-title`, one `dynamo-target-picker-row` per dimension (a `dynamo-target-picker-dim` -label plus radio `` + `