diff --git a/content/blog/2026-09-02-modelplane-v0-4/index.mdx b/content/blog/2026-09-02-modelplane-v0-4/index.mdx new file mode 100644 index 0000000..095e1fe --- /dev/null +++ b/content/blog/2026-09-02-modelplane-v0-4/index.mdx @@ -0,0 +1,268 @@ +--- +title: "Modelplane v0.4: NVIDIA Dynamo and AI Cluster Runtime" +description: "Modelplane v0.4 composes NVIDIA's inference stack across a fleet: a new Dynamo serving stack, and cluster software built from NVIDIA AI Cluster Runtime." +date: "2026-09-02" +authors: + - name: "Nic Cope" + title: "Sr Principal Engineer, Upbound" + url: "https://github.com/negz" + avatar: "/authors/nic.jpg" + github: "https://github.com/negz" + linkedin: "https://www.linkedin.com/in/ncope/" + bio: "Nic is a Senior Principal Engineer at Upbound and a core maintainer of Modelplane and Crossplane. He's spent the last 15+ years working on cloud and infrastructure, including designing and building Crossplane from v0.1 through v2.0, and is now bringing that work to AI inference with Modelplane." +tags: ["release", "inference", "control-plane", "nvidia"] +cover: "/blog/modelplane-v0-4/cover.png" +draft: false +pinned: true +--- + +Modelplane v0.4 is out, and its two headline features both came out of work with +NVIDIA. A new `Dynamo` serving stack brings gang scheduling and peer-to-peer +weight transfer to a cluster. Modelplane also now takes the versions and +configuration for much of each cloud's serving stack from NVIDIA AI Cluster +Runtime, which validates them on real hardware. Modelplane composes NVIDIA's +stack rather than reimplementing it. + +We've also made Modelplane a little more efficient. It now only starts a +provider when you provision a cluster on that cloud. Here's what's new. + +## NVIDIA Dynamo, fleet-wide + +We designed the new `Dynamo` serving stack with NVIDIA's +[Dynamo](https://developer.nvidia.com/dynamo) team. You can now choose the +serving stack each Modelplane-managed cluster runs. +[Grove](https://github.com/ai-dynamo/grove) and the [KAI +Scheduler](https://github.com/NVIDIA/KAI-Scheduler) place a multi-node engine as +a gang, and [ModelExpress](https://github.com/ai-dynamo/modelexpress) moves +model weights GPU to GPU between replicas. + +The `ModelDeployment` an ML team writes is unchanged. The same manifest runs +on either stack. Which stack a cluster runs is a platform decision, made per +cluster, so a fleet can run both at once. + +## The cluster serving stack + +Modelplane operates a fleet. It provisions inference clusters and node pools, +schedules model replicas onto hardware that fits, and fronts the whole fleet +with an OpenAI-compatible endpoint. It isn't a serving layer itself. + +A serving stack owns what happens inside one cluster. It places a multi-node +engine's pods and gets the model's weights into GPU memory. Dynamo does both, +and it does things Modelplane's current "standard" stack doesn't, like gang +scheduling, P2P weight transfer, and keeping weights resident in GPU memory +across an engine crash. + +## Opting a cluster in + +```yaml title="dynamo-cluster.yaml" +apiVersion: modelplane.ai/v1alpha1 +kind: InferenceCluster +metadata: + name: eks-h200-us-east +spec: + # Standard (the default) or Dynamo. Immutable. + stack: Dynamo + cluster: + source: EKS + eks: + region: us-east-1 + nodePools: + - name: gpu + className: eks-h200-8x + nodeCount: 2 +``` + +On a `Dynamo` cluster Modelplane installs Grove, the KAI Scheduler, and the +ModelExpress server. On a `Standard` cluster it installs the +[LeaderWorkerSet](https://github.com/kubernetes-sigs/lws) controller. Everything +else about a cluster, from how it fronts requests to how it stages model +weights, is the same on both. + +The choice is immutable, so adoption is incremental. A platform team stands up a +Dynamo cluster next to the ones it already runs and moves deployments over +cluster by cluster. + +## Gang scheduling with Grove and the KAI Scheduler + +Gang scheduling makes the most of the GPU time you're paying for. A multi-node +engine is a gang. Its leader and workers are useless apart. Schedule those pods +one at a time and a gang can half-land, holding GPUs while serving nothing, +waiting for nodes that may not be free for a while. KAI places the whole gang or +none of it. + +Modelplane composes an engine onto whichever stack its cluster runs. A `Leader` +and `Worker` gang is a LeaderWorkerSet on `Standard`, and on `Dynamo` a Grove +`PodCliqueSet` with a leader clique and a worker clique, scheduled by KAI. + +So a serving stack has to run two pod specs with distinct commands, and give a +worker a way to find its leader. Grove does both. + +## Weight transfer with ModelExpress + +Loading weights is slow. Each replica reads the model from storage before it can +serve a token, and several replicas scaling up together compete for reads from +the same storage. ModelExpress makes that one read rather than one per replica. + +A `Dynamo` cluster runs a ModelExpress server. The server remembers and +advertises which replica holds a model in GPU memory. The first replica loads +from the cache volume and publishes itself as a source, and later replicas pull +the weights from a peer's GPU over RDMA, across a fast fabric like EFA on EKS. A +replica that finds no peer, or no fabric to reach one over, reads the cache +volume instead, so size and keep the cache for every replica on either stack. + +## The same manifest on either stack + +The ML team authors a `ModelDeployment` the same way they always have. A +`ModelDeployment` describes the inference engines Modelplane should run. It says +nothing about the stack underneath it. + +Here's a 480B model across two nodes, tensor-parallel within each node and +pipeline-parallel across them, that also opts into ModelExpress. +`$(MODELPLANE_LEADER_ADDRESS)` is the address the leader is reachable at, and it +resolves on both stacks: + +```yaml title="qwen3-coder.yaml" +apiVersion: modelplane.ai/v1alpha1 +kind: ModelDeployment +metadata: + name: qwen3-coder + namespace: ml-team +spec: + replicas: 1 + template: + spec: + modelCacheRef: + name: qwen3-coder + engines: + - name: qwen3-coder + members: + - role: Leader + nodeSelector: + devices: + # Eight GPUs per node, each with at least 120Gi of memory. + - name: gpu + count: 8 + selectors: + - cel: | + device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("120Gi")) >= 0 + template: + spec: + containers: + - name: engine + image: vllm/vllm-openai:v0.23.0 + command: + - /bin/sh + - -c + - >- + pip install --index-url https://pypi.nvidia.com modelexpress && + exec vllm serve Qwen/Qwen3-Coder-480B-A35B-Instruct + --served-model-name=qwen3-coder + --load-format modelexpress + --tensor-parallel-size=8 + --pipeline-parallel-size=2 + --distributed-executor-backend=mp + --nnodes=2 --node-rank=0 + --master-addr=$(MODELPLANE_LEADER_ADDRESS) + --max-model-len=32768 + --port=8000 + - role: Worker + worker: + nodes: 1 + # nodeSelector is the same as the leader's. Omitted for brevity. + template: + spec: + containers: + - name: engine + image: vllm/vllm-openai:v0.23.0 + command: + - /bin/sh + - -c + - >- + pip install --index-url https://pypi.nvidia.com modelexpress && + exec vllm serve Qwen/Qwen3-Coder-480B-A35B-Instruct + --served-model-name=qwen3-coder + --load-format modelexpress + --tensor-parallel-size=8 + --pipeline-parallel-size=2 + --distributed-executor-backend=mp + --nnodes=2 --node-rank=1 + --master-addr=$(MODELPLANE_LEADER_ADDRESS) + --headless + --max-model-len=32768 +``` + +The `modelCacheRef` names a `ModelCache`, which stages a model's weights once +per cluster on shared storage. The `--load-format modelexpress` flag configures +vLLM to attempt to load weights via ModelExpress. Run it on a `Standard` +cluster, where nothing runs a ModelExpress server, and the engine reads the +cached weights from a persistent volume just like it does today. + +`$(MODELPLANE_LEADER_ADDRESS)` resolves on Dynamo because the Dynamo team +changed Grove so it could. Grove injected its pod-discovery variables in an +order that left a pod template unable to derive a value from them. We hit it +while building the stack, and Stefan from the Dynamo team fixed it in +[grove#753](https://github.com/ai-dynamo/grove/pull/753). + +## Serving stacks, built with NVIDIA AI Cluster Runtime + +A serving stack is more than the engine that serves tokens. Under it sits a +layer of GPU-adjacent software: GPU and DRA drivers, GPU and network operators, +and a scheduler. All of it has to agree with the GPU, the node OS, and +the Kubernetes release beneath it. Those versions interact, and a combination +that's wrong for the hardware causes failures and slowdowns that are hard to +diagnose. Finding a known-good set for a cloud, GPU, and OS is real work. NVIDIA +used to do it on its own hardware, in-house. + +NVIDIA now publishes that work. [NVIDIA AI Cluster +Runtime](https://github.com/NVIDIA/aicr) (AICR) is an Apache 2.0 tool that +captures known-good combinations of drivers, operators, and configuration and +publishes them as version-locked recipes, one per cloud, accelerator, and OS, +validated on the hardware by NVIDIA and the AICR community. + +In v0.4 Modelplane builds each cloud's stack from a fixed list of components. +Much of it comes from AICR: the GPU drivers and operators, node tuning, and +cluster monitoring. Modelplane supplies the serving pieces on top: the gateway, +weight transfer, and the leader and worker controllers. + +A release now installs one stack per cloud, the same on every cluster, tested as +a whole. A cluster's spec no longer carries version fields: nothing to tune, and +nothing to get wrong. + +## Providers that start on demand + +A Modelplane control plane can provision clusters on any cloud it supports, but +a given one usually runs one or two. v0.4 stops it running every cloud's +providers regardless. A control plane now starts a cloud's providers only once +it has an `InferenceCluster` on that cloud, and leaves the rest idle. + +## What's next with NVIDIA + +We're not done building on NVIDIA's stack. Work with the Dynamo team, some +already in flight and some still ahead, would let Modelplane compose more of it. + +The Dynamo team is working on +[grove#755](https://github.com/ai-dynamo/grove/pull/755), which exposes a +scaling-group-wide pod index. That would let us alias a rank the way we alias +the leader address, so every pod of a gang could share one command. + +Further out, we plan to power the `Dynamo` stack with Dynamo's +`DynamoGraphDeployment` (DGD) custom resource, which drives Grove, KAI, +ModelExpress, and the GPU Memory Service (GMS). That depends on changes from the +Dynamo team: distinct leader and worker pod specs the operator won't rewrite +([dynamo#12696](https://github.com/ai-dynamo/dynamo/issues/12696)), and serving +from a stock engine image +([dynamo#10835](https://github.com/ai-dynamo/dynamo/issues/10835)). + +Every layer NVIDIA builds inside a cluster is a layer Modelplane can compose +across a fleet. + +## Try it + +The full release notes are on +[GitHub](https://github.com/modelplaneai/modelplane/releases/tag/v0.4.0). The +[getting-started guide](https://docs.modelplane.ai/getting-started/) covers +standing up a fleet, and +[how it works](https://docs.modelplane.ai/overview/how-it-works/) covers what a +serving stack installs. Modelplane is Apache 2.0 and moving fast at +[github.com/modelplaneai/modelplane](https://github.com/modelplaneai/modelplane), +and questions are welcome in [Slack](https://slack.modelplane.ai). diff --git a/public/blog/modelplane-v0-4/cover.png b/public/blog/modelplane-v0-4/cover.png new file mode 100644 index 0000000..483607f Binary files /dev/null and b/public/blog/modelplane-v0-4/cover.png differ diff --git a/src/lib/blog.ts b/src/lib/blog.ts index 16f7018..33f5403 100644 --- a/src/lib/blog.ts +++ b/src/lib/blog.ts @@ -50,6 +50,13 @@ function readPostFile(folder: string): { data: Record; content: return matter(raw) } +// Drop keys whose value is undefined. getStaticProps can't serialize undefined, +// and the author pages pass an author straight through as a prop, so an absent +// optional field has to be missing rather than explicitly undefined. +function withoutUndefined(o: T): T { + return Object.fromEntries(Object.entries(o).filter(([, v]) => v !== undefined)) as T +} + function normalizeAuthors(input: unknown): Author[] { if (!Array.isArray(input)) return [] return input @@ -57,7 +64,7 @@ function normalizeAuthors(input: unknown): Author[] { if (typeof a === 'string') return { name: a } if (a && typeof a === 'object' && typeof (a as Author).name === 'string') { const author = a as Author - return { + return withoutUndefined({ name: author.name, title: author.title, url: author.url, @@ -66,7 +73,7 @@ function normalizeAuthors(input: unknown): Author[] { github: author.github, linkedin: author.linkedin, x: author.x, - } + }) } return null }) @@ -221,14 +228,10 @@ export function getAllAuthors(): AuthorWithMeta[] { const slug = authorNameToSlug(a.name) const existing = bySlug.get(slug) if (existing) { - existing.count += 1 - existing.title ??= a.title - existing.url ??= a.url - existing.avatar ??= a.avatar - existing.bio ??= a.bio - existing.github ??= a.github - existing.linkedin ??= a.linkedin - existing.x ??= a.x + // The most recent post wins, and older posts fill its gaps. Merging by + // spread rather than field-by-field keeps a field that's absent from + // every post absent, instead of setting it to undefined. + bySlug.set(slug, { ...a, ...existing, count: existing.count + 1 }) } else { bySlug.set(slug, { ...a, slug, count: 1 }) }