Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions cmd/agent/internal/daemon/nodeoperator.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,10 @@ func hasDrift(applied, desired *provision.AgentConfig) bool {
return true
}

if applied.DisableNvidia != desired.DisableNvidia {
return true
}

if applied.Kubelet.ApiServer != desired.Kubelet.ApiServer {
return true
}
Expand Down
7 changes: 7 additions & 0 deletions cmd/agent/internal/daemon/nodeoperator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,13 @@ func Test_hasDrift_OciImageChange(t *testing.T) {
assert.True(t, hasDrift(applied, desired))
}

func Test_hasDrift_DisableNvidiaChange(t *testing.T) {
applied := baseConfig()
desired := baseConfig()
desired.DisableNvidia = true
assert.True(t, hasDrift(applied, desired))
}

func Test_hasDrift_LabelsChange(t *testing.T) {
applied := baseConfig()
desired := baseConfig()
Expand Down
6 changes: 6 additions & 0 deletions docs/content/reference/gpu/nvidia.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,12 @@ The unbounded-agent automatically detects NVIDIA GPUs on the host, forwards the
driver's userspace libraries into the nspawn container, generates a CDI
specification, and configures containerd so that GPU workloads can be scheduled on the node.

Set `DisableNvidia` to `true` in the agent config to disable NVIDIA GPU
discovery and runtime setup even when `/dev/nvidia*` devices are present on the
host. When disabled, the agent does not select the default NVIDIA OCI image,
bind-mount NVIDIA devices or libraries, generate the NVIDIA CDI spec, or
register the NVIDIA containerd runtime.

## Prerequisites

Before the agent can expose GPUs, the **host VM** must have:
Expand Down
4 changes: 4 additions & 0 deletions pkg/agent/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ type AgentConfig struct {
// "ghcr.io/org/repo:tag") used to bootstrap the machine rootfs.
// When empty the agent falls back to debootstrap.
OCIImage string `json:"OCIImage,omitempty"`

// DisableNvidia disables NVIDIA GPU discovery and runtime setup even when
// NVIDIA devices are present on the host.
DisableNvidia bool `json:"DisableNvidia,omitempty"`
}

// BackfillNodeName resolves and stores the Kubernetes Node name once. An
Expand Down
5 changes: 4 additions & 1 deletion pkg/agent/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,8 @@ func TestCRIConfig_JSONRoundTrip(t *testing.T) {
t.Parallel()

cfg := AgentConfig{
MachineName: "test",
MachineName: "test",
DisableNvidia: true,
CRI: CRIConfig{
Containerd: ContainerdConfig{Version: "2.1.0"},
Runc: RuncConfig{Version: "1.2.0"},
Expand All @@ -101,6 +102,7 @@ func TestCRIConfig_JSONRoundTrip(t *testing.T) {
assert.Equal(t, "2.1.0", decoded.CRI.Containerd.Version)
assert.Equal(t, "1.2.0", decoded.CRI.Runc.Version)
assert.Equal(t, "1.6.0", decoded.CNI.PluginVersion)
assert.True(t, decoded.DisableNvidia)
}

func TestCRIConfig_OmittedWhenEmpty(t *testing.T) {
Expand All @@ -124,6 +126,7 @@ func TestCRIConfig_OmittedWhenEmpty(t *testing.T) {

cni := parsed["CNI"].(map[string]interface{})
assert.NotContains(t, cni, "PluginVersion")
assert.NotContains(t, parsed, "DisableNvidia")
}

func TestAgentConfig_DeepCopy(t *testing.T) {
Expand Down
4 changes: 2 additions & 2 deletions pkg/agent/goalstates/containerd.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@ type Containerd struct {
}

// ResolveContainerd returns the containerd configuration goal state.
func ResolveContainerd() Containerd {
func ResolveContainerd(nvidia NvidiaHost) Containerd {
return Containerd{
SandboxImage: SandboxImage,
ContainerdBinPath: filepath.Join("/"+BinDir, "containerd"),
RuncBinaryPath: filepath.Join("/"+BinDir, "runc"),
CNIBinDir: CNIBinDir,
CNIConfDir: CNIConfigDir,
MetricsAddress: ContainerdMetricsAddress,
NvidiaRuntime: resolveNvidiaRuntime(),
NvidiaRuntime: resolveNvidiaRuntime(nvidia),
}
}
8 changes: 4 additions & 4 deletions pkg/agent/goalstates/nvidia.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,11 +113,11 @@ func ResolveNvidiaHost(arch string) (NvidiaHost, error) {
}

// resolveNvidiaRuntime returns the NVIDIA container runtime goal state.
// When GPU devices are present the runtime is enabled with default paths;
// otherwise it is disabled.
func resolveNvidiaRuntime() NvidiaRuntime {
// When GPU devices are present in the resolved host state the runtime is
// enabled with default paths; otherwise it is disabled.
func resolveNvidiaRuntime(nvidia NvidiaHost) NvidiaRuntime {
return NvidiaRuntime{
Enabled: len(discoverNVIDIADevices()) > 0,
Enabled: len(nvidia.GPUDevicePaths) > 0,
RuntimeClassName: NvidiaRuntimeClassName,
RuntimePath: NvidiaContainerRuntimePath,
DisableSetAsDefaultRuntime: false,
Expand Down
20 changes: 13 additions & 7 deletions pkg/agent/goalstates/resolve.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@
NodeStart *NodeStart
}

// ResolveMachine probes the host (kernel version, hostname, GPU hardware) and
// resolves the complete goal state for the named nspawn machine from an agent
// config.
// ResolveMachine probes the host (kernel version, hostname, GPU hardware unless
// disabled) and resolves the complete goal state for the named nspawn machine
// from an agent config.
func ResolveMachine(log *slog.Logger, cfg *config.AgentConfig, machineName string, downloads *DownloadOverrides) (*MachineGoalState, error) {
kernel, err := hostKernel()
if err != nil {
Expand All @@ -39,9 +39,15 @@
return nil, fmt.Errorf("get host hostname: %w", err)
}

nvidia, err := ResolveNvidiaHost(runtime.GOARCH)
if err != nil {
return nil, fmt.Errorf("resolve nvidia host: %w", err)
var nvidia NvidiaHost
if cfg.DisableNvidia {

Check failure on line 43 in pkg/agent/goalstates/resolve.go

View workflow job for this annotation

GitHub Actions / Lint

missing whitespace above this line (no shared variables above if) (wsl_v5)
log.Info("NVIDIA support disabled by agent config")
} else {
var err error
nvidia, err = ResolveNvidiaHost(runtime.GOARCH)

Check failure on line 47 in pkg/agent/goalstates/resolve.go

View workflow job for this annotation

GitHub Actions / Lint

missing whitespace above this line (invalid statement above assign) (wsl_v5)
if err != nil {
return nil, fmt.Errorf("resolve nvidia host: %w", err)
}
}

ociImage := ResolveOCIImage(log, cfg.OCIImage, len(nvidia.GPUDevicePaths) > 0)
Expand Down Expand Up @@ -95,7 +101,7 @@
KubeMachineName: cfg.MachineName,
NodeName: cfg.NodeName,
MachineDir: filepath.Join("/var/lib/machines", machineName),
Containerd: ResolveContainerd(),
Containerd: ResolveContainerd(nvidia),
Kubelet: kubelet,
Nvidia: nvidia,
}
Expand Down
43 changes: 43 additions & 0 deletions pkg/agent/goalstates/resolve_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package goalstates
import (
"log/slog"
"os"
"runtime"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -102,6 +103,44 @@ func TestResolveOCIImage_DefaultWithGPU(t *testing.T) {
assert.Equal(t, DefaultNvidiaOCImage, got)
}

func TestResolveContainerd_NvidiaRuntimeFollowsResolvedHost(t *testing.T) {
withoutGPU := ResolveContainerd(NvidiaHost{})
assert.False(t, withoutGPU.NvidiaRuntime.Enabled)

withGPU := ResolveContainerd(NvidiaHost{GPUDevicePaths: []string{"/dev/nvidia0"}})
assert.True(t, withGPU.NvidiaRuntime.Enabled)
}

func TestResolveMachine_DisableNvidia(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("ResolveMachine requires Linux host kernel discovery")
}

t.Setenv("AGENT_DISABLE_OCI_IMAGE", "")
t.Setenv("AGENT_OCI_IMAGE", "")

cfg := &config.AgentConfig{
MachineName: "machine-1",
DisableNvidia: true,
Cluster: config.AgentClusterConfig{
CaCertBase64: "Y2EtYnl0ZXM=",
},
Kubelet: config.AgentKubeletConfig{
ApiServer: "https://api.example.com",
},
}

got, err := ResolveMachine(discardLogger(), cfg, "kube1", nil)
require.NoError(t, err)

assert.Equal(t, DefaultOCIImage, got.RootFS.OCIImage)
assert.Empty(t, got.RootFS.Nvidia.GPUDevicePaths)
assert.Empty(t, got.RootFS.Nvidia.LibMappings)
assert.Empty(t, got.RootFS.Nvidia.LibDirMounts)
assert.Empty(t, got.NodeStart.Nvidia.GPUDevicePaths)
assert.False(t, got.NodeStart.Containerd.NvidiaRuntime.Enabled)
}

func TestResolveOCIImage_Priority(t *testing.T) {
// Verify the full priority chain: config > disable > env var > default.
log := discardLogger()
Expand Down Expand Up @@ -223,6 +262,10 @@ func TestResolveKubelet_InvalidNodeIPRejected(t *testing.T) {
}

func TestResolveMachine_UsesConfigNodeName(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("ResolveMachine requires Linux host kernel discovery")
}

cfg := &config.AgentConfig{
MachineName: "machine-1",
NodeName: "configured-node",
Expand Down
Loading