Refresh nspawn device config before machine startup

cmd/agent/internal/cmd/cmd.go

@@ -30,6 +30,8 @@ func Run() {
 		newCmdDaemon(cmdCtx),
 		newCmdReset(cmdCtx),
 		newCmdVersion(),
+		newCmdRegenerateConfig(cmdCtx),
+		newCmdRegenerateNSpawnConfig(cmdCtx),
 		newCmdRecordAgentUpgradeFailureSignal(),
 	)
 

cmd/agent/internal/cmd/nspawn_config.go

@@ -0,0 +1,145 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/signal"
+
+	"github.com/spf13/cobra"
+
+	"github.com/Azure/unbounded/internal/executil"
+	"github.com/Azure/unbounded/internal/provision"
+	"github.com/Azure/unbounded/pkg/agent/goalstates"
+	"github.com/Azure/unbounded/pkg/agent/phases"
+	"github.com/Azure/unbounded/pkg/agent/phases/rootfs"
+)
+
+func newCmdRegenerateConfig(cmdCtx *CommandContext) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:    "regenerate-config MACHINE_NAME",
+		Short:  "Regenerate host-side configuration for a machine",
+		Hidden: true,
+		Args:   cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt)
+			defer cancel()
+
+			cmdCtx.Setup()
+
+			return regenerateConfig(ctx, cmdCtx.Logger, args[0])
+		},
+	}
+
+	return cmd
+}
+
+func newCmdRegenerateNSpawnConfig(cmdCtx *CommandContext) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:    "regenerate-nspawn-config MACHINE_NAME",
+		Short:  "Regenerate host-side nspawn configuration for a machine",
+		Hidden: true,
+		Args:   cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt)
+			defer cancel()
+
+			cmdCtx.Setup()
+
+			return regenerateNSpawnConfig(ctx, cmdCtx.Logger, args[0])
+		},
+	}
+
+	return cmd
+}
+
+func regenerateConfig(ctx context.Context, log *slog.Logger, machineName string) error {
+	return regenerateNSpawnConfig(ctx, log, machineName)
+}
+
+func regenerateNSpawnConfig(ctx context.Context, log *slog.Logger, machineName string) error {
+	cfg, ok, err := loadAppliedConfigForMachine(log, machineName)
+	if err != nil {
+		return err
+	}
+
+	if !ok {
+		log.Info("applied config not found, skipping nspawn config regeneration", "machine", machineName)
+		return nil
+	}
+
+	rootFS, err := goalstates.ResolveNSpawnConfig(cfg, machineName)
+	if err != nil {
+		return fmt.Errorf("resolve nspawn config goal state: %w", err)
+	}
+
+	if err := phases.ExecuteTask(ctx, log, rootfs.EnsureNSpawnConfig(log, rootFS)); err != nil {
+		return fmt.Errorf("regenerate nspawn config for %s: %w", machineName, err)
+	}
+
+	// systemd loaded the nspawn service drop-in before starting this required
+	// oneshot unit. Reload the manager so the pending nspawn start observes the
+	// regenerated service properties, including path-specific DeviceAllow entries.
+	if err := executil.RunCmd(ctx, log, executil.Systemctl(), "daemon-reload"); err != nil {
+		return fmt.Errorf("reload systemd after regenerating config for %s: %w", machineName, err)
+	}
+
+	return nil
+}
+
+func loadAppliedConfigForMachine(log *slog.Logger, machineName string) (*provision.AgentConfig, bool, error) {
+	if machineName != goalstates.NSpawnMachineKube1 && machineName != goalstates.NSpawnMachineKube2 {
+		return nil, false, fmt.Errorf("unsupported nspawn machine %q", machineName)
+	}
+
+	return loadAppliedConfig(log, goalstates.AppliedConfigPath(machineName), goalstates.AppliedConfigChecksumPath(machineName))
+}
+
+func loadAppliedConfig(log *slog.Logger, path, checksumPath string) (*provision.AgentConfig, bool, error) {
+	data, err := os.ReadFile(path)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, false, nil
+	}
+
+	if err != nil {
+		return nil, false, fmt.Errorf("read applied config %s: %w", path, err)
+	}
+
+	if err := goalstates.VerifyChecksum(data, checksumPath); err != nil {
+		return nil, false, fmt.Errorf("verify applied config checksum for %s: %w", path, err)
+	}
+
+	if _, statErr := os.Stat(checksumPath); errors.Is(statErr, os.ErrNotExist) {
+		log.Warn(
+			"no checksum sidecar found, skipping integrity check",
+			"config_path", path,
+			"checksum_path", checksumPath,
+		)
+	}
+
+	var cfg provision.AgentConfig
+	if err := json.Unmarshal(data, &cfg); err != nil {
+		return nil, false, fmt.Errorf("decode applied config %s: %w", path, err)
+	}
+
+	source, err := provision.ResolveMachineName(&cfg)
+	if err != nil {
+		return nil, false, fmt.Errorf("resolve applied config machine name %s: %w", path, err)
+	}
+
+	if source != "config" {
+		log.Info("resolved unbounded MachineName", "name", cfg.MachineName, "source", source)
+	}
+
+	if err := cfg.BackfillNodeName(); err != nil {
+		return nil, false, fmt.Errorf("backfill applied config node name %s: %w", path, err)
+	}
+
+	return &cfg, true, nil
+}

cmd/agent/internal/cmd/nspawn_config_test.go

@@ -0,0 +1,69 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+package cmd
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/Azure/unbounded/internal/provision"
+	"github.com/Azure/unbounded/pkg/agent/goalstates"
+)
+
+func TestLoadAppliedConfig(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	configPath := filepath.Join(dir, "applied-config.json")
+	checksumPath := configPath + ".sha256"
+	want := provision.AgentConfig{
+		MachineName: "machine-1",
+		NodeName:    "node-1",
+	}
+
+	data, err := json.Marshal(&want)
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(configPath, data, 0o600))
+	require.NoError(t, os.WriteFile(checksumPath, []byte(goalstates.ComputeChecksum(data)+"\n"), 0o600))
+
+	got, ok, err := loadAppliedConfig(testLogger(), configPath, checksumPath)
+	require.NoError(t, err)
+	require.True(t, ok)
+	require.Equal(t, want.MachineName, got.MachineName)
+	require.Equal(t, want.NodeName, got.NodeName)
+}
+
+func TestLoadAppliedConfigMissing(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	got, ok, err := loadAppliedConfig(
+		testLogger(),
+		filepath.Join(dir, "missing.json"),
+		filepath.Join(dir, "missing.json.sha256"),
+	)
+
+	require.NoError(t, err)
+	require.False(t, ok)
+	require.Nil(t, got)
+}
+
+func TestLoadAppliedConfigChecksumMismatch(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	configPath := filepath.Join(dir, "applied-config.json")
+	checksumPath := configPath + ".sha256"
+	require.NoError(t, os.WriteFile(configPath, []byte(`{"MachineName":"machine-1"}`), 0o600))
+	require.NoError(t, os.WriteFile(checksumPath, []byte(goalstates.ComputeChecksum([]byte("different"))), 0o600))
+
+	got, ok, err := loadAppliedConfig(testLogger(), configPath, checksumPath)
+	require.ErrorIs(t, err, goalstates.ErrChecksumMismatch)
+	require.False(t, ok)
+	require.Nil(t, got)
+}

docs/content/reference/agent/nspawn.md

@@ -152,16 +152,20 @@ The agent also auto-mounts host storage and InfiniBand hardware:
   access. Sources are not created or required to exist during config
   validation.
 
-Device discovery runs once when the machine is provisioned. Disks or HCAs
-hot-plugged after the machine has started are not picked up until the machine
-is re-provisioned or soft-rebooted.
+Device discovery runs when the machine is provisioned and is refreshed by a
+host-side systemd hook before systemd starts the nspawn machine. Device mapping
+changes that occur while the host is offline are picked up on the next host
+boot before the machine starts. Disks or HCAs hot-plugged after the machine has
+started are not picked up until the machine is restarted, re-provisioned, or
+soft-rebooted.
 
-The configuration is written to two files on the host before the machine boots:
+The configuration is written to these files on the host before the machine boots:
 
 | File | Path |
 |---|---|
 | nspawn config | `/etc/systemd/nspawn/<MachineName>.nspawn` |
 | Service override | `/etc/systemd/system/systemd-nspawn@<MachineName>.service.d/override.conf` |
+| Config regeneration unit | `/etc/systemd/system/unbounded-agent-regenerate-config@<MachineName>.service` |
 
 ### Customization points
 
@@ -284,6 +288,7 @@ The container operates in the host's network namespace (`VirtualEthernet=no`):
 | `/var/lib/machines/<MachineName>` | Container rootfs directory. |
 | `/etc/systemd/nspawn/<MachineName>.nspawn` | nspawn configuration file. |
 | `/etc/systemd/system/systemd-nspawn@<MachineName>.service.d/override.conf` | Systemd service override. |
+| `/etc/systemd/system/unbounded-agent-regenerate-config@<MachineName>.service` | Host-side oneshot unit that regenerates host-side configuration before machine start. |
 | `/run/host-nvidia/<index>/` | (Inside container) Read-only bind-mount of host NVIDIA library directories. |
 
 ## See Also