Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions docs/usages/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,12 @@ At least one join or Azure authentication method must be configured. `azure.boot
| `node.kubelet.clusterFQDN` | string | Kubernetes API server FQDN. Required for bootstrap token mode. | `example.hcp.canadacentral.azmk8s.io` |
| `node.kubelet.caCertData` | string | Base64-encoded cluster CA data. Required for bootstrap token mode. | `<base64-ca-data>` |
| `node.kubelet.nodeIP` | string | Optional node IP override for kubelet `--node-ip`. | `10.0.0.4` |
| `node.kubelet.imageCredentialProvider.configPath` | string | Optional absolute path inside the nspawn machine to a kubelet exec image credential provider configuration file or supported configuration directory. Must be set with `binDir`. | `/etc/kubernetes/credential-provider.yaml` |
| `node.kubelet.imageCredentialProvider.binDir` | string | Optional absolute path inside the nspawn machine containing exec image credential provider binaries. Must be set with `configPath`. | `/usr/local/lib/kubelet-credential-providers` |

Provider paths must be clean absolute machine paths without whitespace or systemd argument characters. Include the provider files in the OCI rootfs or expose them with read-only `bootstrap.additionalHostMounts`.

The image credential provider executes a plugin to obtain short-lived pull credentials; it does not place registry passwords or tokens in the FlexNode configuration. Do not store static registry credentials in this file or provider configuration.

## Component Versions

Expand Down Expand Up @@ -348,3 +354,36 @@ Use `bootstrap.additionalHostMounts` to expose host files or directories inside
```

Read-only entries render as systemd-nspawn `BindReadOnly=` directives; writable entries render as `Bind=` directives.

### Image Pull Credential Provider

Kubelet exec image credential providers can obtain short-lived registry credentials without storing tokens in the FlexNode config. The provider configuration and executable must exist inside the nspawn machine, either in the OCI rootfs or through host mounts:

```json
{
"bootstrap": {
"additionalHostMounts": [
{
"source": "/opt/aks-flex-node/credential-provider/config.yaml",
"target": "/etc/kubernetes/credential-provider.yaml",
"readOnly": true
},
{
"source": "/opt/aks-flex-node/credential-provider/bin",
"target": "/usr/local/lib/kubelet-credential-providers",
"readOnly": true
}
]
},
"node": {
"kubelet": {
"imageCredentialProvider": {
"configPath": "/etc/kubernetes/credential-provider.yaml",
"binDir": "/usr/local/lib/kubelet-credential-providers"
}
}
}
}
```

Provider binaries must be executable before the machine starts. Mount provider assets read-only unless the provider explicitly requires writable state.
12 changes: 6 additions & 6 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0
github.com/Azure/kubelogin v0.2.15
github.com/Azure/unbounded v0.2.2
github.com/Azure/unbounded v0.2.3-alpha.3
github.com/go-logr/logr v1.4.4
github.com/google/renameio/v2 v2.0.2
github.com/google/uuid v1.6.0
github.com/spf13/cobra v1.10.2
k8s.io/api v0.36.2
k8s.io/apimachinery v0.36.2
k8s.io/client-go v0.36.2
k8s.io/api v0.36.3
k8s.io/apimachinery v0.36.3
k8s.io/client-go v0.36.3
k8s.io/utils v0.0.0-20260319190234-28399d86e0b5
sigs.k8s.io/controller-runtime v0.24.1
)
Expand Down Expand Up @@ -101,12 +101,12 @@ require (
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiextensions-apiserver v0.36.0 // indirect
k8s.io/apiextensions-apiserver v0.36.3 // indirect
k8s.io/klog/v2 v2.140.0 // indirect
k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 // indirect
oras.land/oras-go/v2 v2.6.2 // indirect
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
sigs.k8s.io/randfill v1.0.0 // indirect
sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect
sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect
sigs.k8s.io/yaml v1.6.0 // indirect
)
24 changes: 12 additions & 12 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys=
github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls=
github.com/Azure/unbounded v0.2.2 h1:uu5hYj20UBbrSAlf4qnDiMTGaR0xxsYnkdbTWTdBHX8=
github.com/Azure/unbounded v0.2.2/go.mod h1:bZqzs6NIfXJqA8FRYSeajKJ0TYIqT8Xjfvfwu6OpHkw=
github.com/Azure/unbounded v0.2.3-alpha.3 h1:grxHFtQpulYHSFRusRTPZl//bPrHO9lifsJrjcFVrxo=
github.com/Azure/unbounded v0.2.3-alpha.3/go.mod h1:G+yWlIQB/KwOk+O7HPvcRLBW88qZUgSWt51qytBTgFE=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU=
Expand Down Expand Up @@ -348,14 +348,14 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
k8s.io/api v0.36.2 h1:TF6YDLIzKfccK7cq9YpTcGX8TJmEkHVRv78DM51fRYY=
k8s.io/api v0.36.2/go.mod h1:F4LbMO4brjZYh7yFkXWhynSvtB7YauxV4c+HHkNRGNg=
k8s.io/apiextensions-apiserver v0.36.0 h1:Wt7E8J+VBCbj4FjiBfDTK/neXDDjyJVJc7xfuOHImZ0=
k8s.io/apiextensions-apiserver v0.36.0/go.mod h1:kGDjH0msuiIB3tgsYRV0kS9GqpMYMUsQ3GHv7TApyug=
k8s.io/apimachinery v0.36.2 h1:0PE/W/WNy1UX61NLbXY5TMbJ6UwLL6E6lAPkYrKFxbQ=
k8s.io/apimachinery v0.36.2/go.mod h1:fvf/HOLXq9RId0rnDIbN1OEBvHXdQbLMM8nu0LcBUf4=
k8s.io/client-go v0.36.2 h1:bfgxmFKc9CgqsgX4xKLAAdmTQlWee7Ob/HlDOrJ5TBI=
k8s.io/client-go v0.36.2/go.mod h1:1vgO4OAlfPnoLcb+Rze2GF5rAr14w8qjrYMoyXJzQj0=
k8s.io/api v0.36.3 h1:NxB+05W2UGqXWFXcLO0RB5cnqnUPP5v5sVlaOH0Iz4w=
k8s.io/api v0.36.3/go.mod h1:JzLQKqRHC5+I8RVj/lS3lCg0mg6nWI9Fo/Sk3ElxHzg=
k8s.io/apiextensions-apiserver v0.36.3 h1:dPmOAPhwTtqb1bTxbFPsy18KHPhktQeO3WUPXunZIB0=
k8s.io/apiextensions-apiserver v0.36.3/go.mod h1:KTXFqgXiuw2pRoL+Wpmttqc+up9Xt/GohadPWeLLOa4=
k8s.io/apimachinery v0.36.3 h1:PkzMRBRG8joFD8EhCuQAtNPvJlxb82FwplP26HIzvAM=
k8s.io/apimachinery v0.36.3/go.mod h1:cTSjBWgPe/6CQyBKzY/hDIRWCQQQeK0mfLbml0UYFHE=
k8s.io/client-go v0.36.3 h1:M4JdVzXxYcZk4fGpfDdYnxSwhLKWCFoQsHW6t+z8Hfg=
k8s.io/client-go v0.36.3/go.mod h1:gcPwr0c87vjjG6HB6pWEqOeuYVoXSsREjzux2j6GF30=
k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc=
k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0=
k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 h1:Sztf7ESG9tAXRW/ACJZjrj5jhdOUqS2KFRQT+CTvu78=
Expand All @@ -370,7 +370,7 @@ sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5E
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8=
sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw=
sigs.k8s.io/structured-merge-diff/v6 v6.3.3/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
19 changes: 19 additions & 0 deletions pkg/config/adapter.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig {
NodeIP: cfg.Node.Kubelet.NodeIP,
Labels: cfg.Node.Labels,
RegisterWithTaints: cfg.Node.Taints,
Configuration: kubeletConfiguration(cfg),
},
CRI: agentconfig.CRIConfig{
Containerd: agentconfig.ContainerdConfig{
Expand All @@ -57,6 +58,13 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig {
},
}

if provider := cfg.Node.Kubelet.ImageCredentialProvider; provider != nil {
ac.Kubelet.ImageCredentialProvider = &agentconfig.ImageCredentialProvider{
ConfigPath: provider.ConfigPath,
BinDir: provider.BinDir,
}
}

if cfg.Bootstrap.OfflineArtifacts.Source != "" {
ac.OfflineArtifacts = &agentconfig.AgentOfflineArtifacts{
Source: cfg.Bootstrap.OfflineArtifacts.Source,
Expand Down Expand Up @@ -103,6 +111,17 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig {
return ac
}

func kubeletConfiguration(cfg *Config) map[string]any {
return map[string]any{
"maxPods": cfg.Node.MaxPods,
"imageGCHighThresholdPercent": cfg.Node.Kubelet.ImageGCHighThreshold,
"imageGCLowThresholdPercent": cfg.Node.Kubelet.ImageGCLowThreshold,
"logging": map[string]any{
"verbosity": cfg.Node.Kubelet.Verbosity,
},
}
}

// ResolveMachineGoalState converts FlexNode config to the shared agent config
// and resolves the nspawn machine goal state. Bootstrap and preflight both use
// this helper so preflight validates the same sources that bootstrap consumes.
Expand Down
43 changes: 38 additions & 5 deletions pkg/config/adapter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,20 @@ func TestToAgentConfig_BootstrapToken(t *testing.T) {
Components: ComponentsConfig{Kubernetes: "1.30.0"},
Networking: NetworkingConfig{DNSServiceIP: "10.0.0.10"},
Node: NodeConfig{
Labels: map[string]string{"env": "test"},
Taints: []string{"dedicated=infra:NoSchedule"},
MaxPods: 42,
Labels: map[string]string{"env": "test"},
Taints: []string{"dedicated=infra:NoSchedule"},
Kubelet: KubeletConfig{
ClusterFQDN: "api.example.com:6443",
CACertData: "dGVzdC1jYS1kYXRh",
NodeIP: "10.225.0.4",
Verbosity: 4,
ImageGCHighThreshold: 90,
ImageGCLowThreshold: 75,
ClusterFQDN: "api.example.com:6443",
CACertData: "dGVzdC1jYS1kYXRh",
NodeIP: "10.225.0.4",
ImageCredentialProvider: &ImageCredentialProviderConfig{
ConfigPath: "/etc/kubernetes/credential-provider.yaml",
BinDir: "/usr/local/lib/kubelet-credential-providers",
},
},
},
}
Expand Down Expand Up @@ -65,6 +73,31 @@ func TestToAgentConfig_BootstrapToken(t *testing.T) {
if len(ac.Kubelet.RegisterWithTaints) != 1 || ac.Kubelet.RegisterWithTaints[0] != "dedicated=infra:NoSchedule" {
t.Fatalf("Kubelet.RegisterWithTaints=%v, want [dedicated=infra:NoSchedule]", ac.Kubelet.RegisterWithTaints)
}
if got := ac.Kubelet.Configuration["maxPods"]; got != 42 {
t.Fatalf("Kubelet.Configuration.maxPods=%v, want 42", got)
}
if got := ac.Kubelet.Configuration["imageGCHighThresholdPercent"]; got != 90 {
t.Fatalf("Kubelet.Configuration.imageGCHighThresholdPercent=%v, want 90", got)
}
if got := ac.Kubelet.Configuration["imageGCLowThresholdPercent"]; got != 75 {
t.Fatalf("Kubelet.Configuration.imageGCLowThresholdPercent=%v, want 75", got)
}
logging, ok := ac.Kubelet.Configuration["logging"].(map[string]any)
if !ok {
t.Fatalf("Kubelet.Configuration.logging=%T, want map[string]any", ac.Kubelet.Configuration["logging"])
}
if logging["verbosity"] != 4 {
t.Fatalf("Kubelet.Configuration.logging=%v, want verbosity=4", logging)
}
if ac.Kubelet.ImageCredentialProvider == nil {
t.Fatal("Kubelet.ImageCredentialProvider=nil, want provider")
}
if ac.Kubelet.ImageCredentialProvider.ConfigPath != "/etc/kubernetes/credential-provider.yaml" {
t.Fatalf("Kubelet.ImageCredentialProvider.ConfigPath=%q", ac.Kubelet.ImageCredentialProvider.ConfigPath)
}
if ac.Kubelet.ImageCredentialProvider.BinDir != "/usr/local/lib/kubelet-credential-providers" {
t.Fatalf("Kubelet.ImageCredentialProvider.BinDir=%q", ac.Kubelet.ImageCredentialProvider.BinDir)
}
}

func TestToAgentConfig_NodeName(t *testing.T) {
Expand Down
49 changes: 49 additions & 0 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"encoding/json"
"encoding/pem"
"fmt"
"math"
"net/url"
"os"
"path/filepath"
Expand Down Expand Up @@ -251,6 +252,17 @@ type KubeletConfig struct {
ClusterFQDN string `json:"clusterFQDN,omitempty"` // Kubernetes API server FQDN from AKS RP bootstrap data
CACertData string `json:"caCertData"` // Base64-encoded CA certificate data
NodeIP string `json:"nodeIP"` // IP address to advertise as the node's primary IP (--node-ip kubelet flag)

// ImageCredentialProvider configures kubelet's exec image credential
// provider. The referenced paths are inside the nspawn machine.
ImageCredentialProvider *ImageCredentialProviderConfig `json:"imageCredentialProvider,omitempty"`
}

// ImageCredentialProviderConfig identifies the exec image credential provider
// configuration and binary directory inside the nspawn machine.
type ImageCredentialProviderConfig struct {
ConfigPath string `json:"configPath"`
BinDir string `json:"binDir"`
}

// NetworkingConfig is the AKS RP networking contract used by the agent at runtime.
Expand Down Expand Up @@ -882,6 +894,9 @@ func (c *Config) validate() error {
if err := c.Bootstrap.validate(); err != nil {
return err
}
if err := c.Node.validate(); err != nil {
return err
}

if err := c.validateAuthSettings(); err != nil {
return err
Expand All @@ -893,6 +908,40 @@ func (c *Config) validate() error {
return nil
}

func (c *NodeConfig) validate() error {
if c.MaxPods < 0 || int64(c.MaxPods) > math.MaxInt32 {
return fmt.Errorf("node.maxPods must be between 0 and %d, inclusive", math.MaxInt32)
}
return c.Kubelet.validate()
}

func (c *KubeletConfig) validate() error {
if c.Verbosity < 0 || int64(c.Verbosity) > math.MaxInt32 {
return fmt.Errorf("node.kubelet.verbosity must be between 0 and %d, inclusive", math.MaxInt32)
}
if c.ImageGCHighThreshold < 0 || c.ImageGCHighThreshold > 100 {
return fmt.Errorf("node.kubelet.imageGCHighThreshold must be between 0 and 100, inclusive")
}
if c.ImageGCLowThreshold < 0 || c.ImageGCLowThreshold > 100 {
return fmt.Errorf("node.kubelet.imageGCLowThreshold must be between 0 and 100, inclusive")
}
if c.ImageGCLowThreshold >= c.ImageGCHighThreshold {
return fmt.Errorf("node.kubelet.imageGCLowThreshold must be less than node.kubelet.imageGCHighThreshold")
}

kubelet := agentconfig.AgentKubeletConfig{}
if c.ImageCredentialProvider != nil {
kubelet.ImageCredentialProvider = &agentconfig.ImageCredentialProvider{
ConfigPath: c.ImageCredentialProvider.ConfigPath,
BinDir: c.ImageCredentialProvider.BinDir,
}
}
if err := kubelet.Validate(); err != nil {
return fmt.Errorf("invalid node.kubelet configuration: %w", err)
}
Comment thread
bcho marked this conversation as resolved.
return nil
}

func (c *Config) validateAuthSettings() error {
armAuthMethodCount := 0
for _, m := range []bool{c.IsARCEnabled(), c.IsSPConfigured(), c.IsMIConfigured()} {
Expand Down
14 changes: 14 additions & 0 deletions pkg/config/copy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@ func TestConfigDeepCopy_DoesNotSharePointersOrMaps(t *testing.T) {
Node: NodeConfig{
Labels: map[string]string{"l": "1"},
Taints: []string{"dedicated=infra:NoSchedule"},
Kubelet: KubeletConfig{
ImageCredentialProvider: &ImageCredentialProviderConfig{
ConfigPath: "/etc/kubernetes/credential-provider.yaml",
BinDir: "/usr/local/lib/kubelet-credential-providers",
},
},
},
}

Expand Down Expand Up @@ -47,6 +53,9 @@ func TestConfigDeepCopy_DoesNotSharePointersOrMaps(t *testing.T) {
if cfg.Components.Gantry == nil || copy.Components.Gantry == nil || cfg.Components.Gantry == copy.Components.Gantry {
t.Fatalf("Gantry pointer shared or nil")
}
if cfg.Node.Kubelet.ImageCredentialProvider == nil || copy.Node.Kubelet.ImageCredentialProvider == nil || cfg.Node.Kubelet.ImageCredentialProvider == copy.Node.Kubelet.ImageCredentialProvider {
t.Fatalf("ImageCredentialProvider pointer shared or nil")
}

// Maps should not be shared (validate via independent mutation behavior).
cfg.Azure.Arc.Tags["k"] = "orig"
Expand All @@ -67,6 +76,11 @@ func TestConfigDeepCopy_DoesNotSharePointersOrMaps(t *testing.T) {
t.Fatalf("Node.Labels shared; orig=%q, want %q", cfg.Node.Labels["l"], "orig")
}

copy.Node.Kubelet.ImageCredentialProvider.ConfigPath = "/etc/kubernetes/other.yaml"
if cfg.Node.Kubelet.ImageCredentialProvider.ConfigPath != "/etc/kubernetes/credential-provider.yaml" {
t.Fatal("ImageCredentialProvider shared between config copies")
}

// Taints slice should not be shared.
if len(copy.Node.Taints) != 1 || copy.Node.Taints[0] != "dedicated=infra:NoSchedule" {
t.Fatalf("Node.Taints copy=%v, want [dedicated=infra:NoSchedule]", copy.Node.Taints)
Expand Down
Loading
Loading