diff --git a/cmd/agent/internal/cmd/cmd.go b/cmd/agent/internal/cmd/cmd.go index 075d1b60d..c3cd57e34 100644 --- a/cmd/agent/internal/cmd/cmd.go +++ b/cmd/agent/internal/cmd/cmd.go @@ -26,6 +26,7 @@ func Run() { root.AddCommand( newCmdStart(cmdCtx), + newCmdPreflight(cmdCtx), newCmdDaemon(cmdCtx), newCmdReset(cmdCtx), newCmdVersion(), diff --git a/cmd/agent/internal/cmd/preflight.go b/cmd/agent/internal/cmd/preflight.go new file mode 100644 index 000000000..6a15cccd1 --- /dev/null +++ b/cmd/agent/internal/cmd/preflight.go @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + + "github.com/spf13/cobra" + + "github.com/Azure/unbounded/internal/provision" + "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/phases/host" + "github.com/Azure/unbounded/pkg/agent/phases/nodestart" + "github.com/Azure/unbounded/pkg/agent/phases/rootfs" + "github.com/Azure/unbounded/pkg/agent/preflight" +) + +type preflightHandler struct { + cmdCtx *CommandContext + configPath string + ignorePreflightErrors []string + failOnWarnings bool + output string + writer io.Writer +} + +func newCmdPreflight(cmdCtx *CommandContext) *cobra.Command { + handler := &preflightHandler{cmdCtx: cmdCtx, writer: os.Stdout} + + cmd := &cobra.Command{ + Use: "preflight", + Short: "Run non-mutating preflight checks", + Long: "Run non-mutating preflight checks for the host and agent configuration before node bootstrap.", + RunE: func(cmd *cobra.Command, _ []string) error { + return handler.execute(cmd.Context()) + }, + } + + cmd.Flags().StringVar(&handler.configPath, "config", "", "Path to agent config file") + cmd.Flags().StringSliceVar( + &handler.ignorePreflightErrors, + "ignore-preflight-errors", + nil, + "Comma-separated preflight check names whose errors should be reported as warnings", + ) + cmd.Flags().BoolVar(&handler.failOnWarnings, "fail-on-warnings", false, "Fail when any preflight warning is returned") + cmd.Flags().StringVar(&handler.output, "output", "text", "Output format: text or json") + + return cmd +} + +func (h *preflightHandler) execute(ctx context.Context) error { + h.cmdCtx.Setup() + logger := h.cmdCtx.Logger + + if h.configPath != "" { + oldConfigPath := os.Getenv(configFileEnv) + defer os.Setenv(configFileEnv, oldConfigPath) //nolint:errcheck // best effort restore + + if err := os.Setenv(configFileEnv, h.configPath); err != nil { + return err + } + } + + cfg, err := loadConfig(logger) + if err != nil { + return err + } + + if err := cfg.Validate(); err != nil { + return fmt.Errorf("validate agent config: %w", err) + } + + goalState, err := goalstates.ResolveMachine( + logger, + &cfg.AgentConfig, + goalstates.NSpawnMachineKube1, + provision.ResolveDownloadOverrides(cfg.Downloads), + ) + if err != nil { + return fmt.Errorf("resolve goal state: %w", err) + } + + checks := preflight.Flatten( + host.Preflight(logger, cfg, goalState), + nodestart.Preflight(logger, cfg, goalState), + rootfs.Preflight(logger, cfg, goalState), + ) + + opts := preflight.Options{ + IgnoreErrors: h.ignorePreflightErrors, + FailOnWarnings: h.failOnWarnings, + } + report := preflight.Run(ctx, checks, opts) + + switch strings.ToLower(h.output) { + case "", "text": + if err := writePreflightText(h.writer, report); err != nil { + return err + } + case "json": + enc := json.NewEncoder(h.writer) + enc.SetIndent("", " ") + + if err := enc.Encode(report); err != nil { + return err + } + default: + return fmt.Errorf("unsupported output format %q", h.output) + } + + return report.Err(h.failOnWarnings) +} + +func writePreflightText(w io.Writer, report preflight.Report) error { + if _, err := fmt.Fprintln(w, "[preflight] Running unbounded-agent pre-flight checks"); err != nil { + return err + } + + var errors []preflight.Result + + for _, result := range report.Checks { + switch result.Severity { + case preflight.SeverityOK: + if err := writePreflightResult(w, "OK", result); err != nil { + return err + } + case preflight.SeverityError: + errors = append(errors, result) + case preflight.SeverityWarning: + if err := writePreflightResult(w, "WARNING", result); err != nil { + return err + } + } + } + + if len(errors) == 0 { + return nil + } + + if _, err := fmt.Fprintln(w, "[preflight] Some fatal errors occurred:"); err != nil { + return err + } + + for _, result := range errors { + if _, err := fmt.Fprintf(w, "\t[ERROR %s]: %s", result.Name, result.Message); err != nil { + return err + } + + if result.Target != "" { + if _, err := fmt.Fprintf(w, " (target: %s)", result.Target); err != nil { + return err + } + } + + if _, err := fmt.Fprintln(w); err != nil { + return err + } + } + + _, err := fmt.Fprintln( + w, + "[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`", + ) + + return err +} + +func writePreflightResult(w io.Writer, status string, result preflight.Result) error { + if _, err := fmt.Fprintf(w, "\t[%s %s]: %s", status, result.Name, result.Message); err != nil { + return err + } + + if result.Target != "" { + if _, err := fmt.Fprintf(w, " (target: %s)", result.Target); err != nil { + return err + } + } + + _, err := fmt.Fprintln(w) + + return err +} diff --git a/cmd/agent/internal/cmd/preflight_test.go b/cmd/agent/internal/cmd/preflight_test.go new file mode 100644 index 000000000..856b3884e --- /dev/null +++ b/cmd/agent/internal/cmd/preflight_test.go @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/internal/provision" +) + +func preflightConfig(apiServer string) provision.UnboundedAgentConfig { + cfg := sampleConfig() + cfg.Kubelet.ApiServer = apiServer + cfg.OCIImage = "registry.example.com/unbounded/rootfs:v1" + + return cfg +} + +func TestNewCmdPreflight(t *testing.T) { + cmd := newCmdPreflight(&CommandContext{LogFormat: "text"}) + + assert.Equal(t, "preflight", cmd.Use) + assert.NotNil(t, cmd.Flags().Lookup("config")) + assert.NotNil(t, cmd.Flags().Lookup("ignore-preflight-errors")) + assert.NotNil(t, cmd.Flags().Lookup("fail-on-warnings")) + assert.NotNil(t, cmd.Flags().Lookup("output")) +} + +func TestPreflightJSONOutput(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + + path := writeConfigFile(t, preflightConfig(srv.URL)) + + var out bytes.Buffer + + h := &preflightHandler{ + cmdCtx: &CommandContext{LogFormat: "text"}, + configPath: path, + ignorePreflightErrors: []string{"all"}, + output: "json", + writer: &out, + } + + require.NoError(t, h.execute(context.Background())) + + var report struct { + Status string `json:"status"` + Checks []struct { + Name string `json:"name"` + Ignored bool `json:"ignored"` + } `json:"checks"` + } + require.NoError(t, json.Unmarshal(out.Bytes(), &report)) + assert.Equal(t, "ok", report.Status) + assert.NotEmpty(t, report.Checks) +} + +func TestPreflightTextOutputError(t *testing.T) { + path := writeConfigFile(t, preflightConfig("https://127.0.0.1:1")) + + var out bytes.Buffer + + h := &preflightHandler{ + cmdCtx: &CommandContext{LogFormat: "text"}, + configPath: path, + output: "text", + writer: &out, + } + + err := h.execute(context.Background()) + require.Error(t, err) + assert.Contains(t, out.String(), "[ERROR api-server-reachable]") + assert.NotContains(t, out.String(), "127.0.0.1") +} + +func TestPreflightTextOutputIncludesOK(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + + path := writeConfigFile(t, preflightConfig(srv.URL)) + + var out bytes.Buffer + + h := &preflightHandler{ + cmdCtx: &CommandContext{LogFormat: "text"}, + configPath: path, + ignorePreflightErrors: []string{"all"}, + output: "text", + writer: &out, + } + + require.NoError(t, h.execute(context.Background())) + assert.Contains(t, out.String(), "[OK goal-state]") +} diff --git a/hack/agent/e2e-kind/e2e.py b/hack/agent/e2e-kind/e2e.py index 36223e9f5..6714d47e4 100755 --- a/hack/agent/e2e-kind/e2e.py +++ b/hack/agent/e2e-kind/e2e.py @@ -213,6 +213,10 @@ def scp_cmd(src: str, dst: str) -> subprocess.CompletedProcess[str]: return run(["scp", *SSH_OPTS, src, dst]) +def scp_from_vm(src: str, dst: Path) -> subprocess.CompletedProcess[str]: + return run(["scp", *SSH_OPTS, f"{SSH_TARGET}:{src}", str(dst)]) + + def kubectl(args: list[str], **kw: Any) -> subprocess.CompletedProcess[str]: return run([KUBECTL, *args], **kw) @@ -1639,6 +1643,21 @@ def _run_agent_inner(agent_url: str, node_config: NodeConfig) -> None: bootstrap_script_path.chmod(0o600) log(f"Bootstrap script written to {bootstrap_script_path}") + bootstrap_preflight_script = bootstrap_script.replace( + 'echo "Running unbounded-agent start..."\n"${AGENT_BIN}" start ${_START_ARGS}', + textwrap.dedent("""\ + echo "Running unbounded-agent preflight..." + "${AGENT_BIN}" preflight ${_START_ARGS} --output text | tee /tmp/unbounded-agent-preflight.txt + "${AGENT_BIN}" preflight ${_START_ARGS} --output json > /tmp/unbounded-agent-preflight.json + echo "Running unbounded-agent start..." + "${AGENT_BIN}" start ${_START_ARGS}"""), + ) + if bootstrap_preflight_script == bootstrap_script: + die("failed to inject unbounded-agent preflight into bootstrap script") + + bootstrap_script = bootstrap_preflight_script + bootstrap_script_path.write_text(bootstrap_script) + # Wait for cloud-init and verify connectivity log("Waiting for cloud-init to complete on VM...") subprocess.run(["ssh", *SSH_OPTS, SSH_TARGET, "sudo cloud-init status --wait"], @@ -1664,6 +1683,10 @@ def _run_agent_inner(agent_url: str, node_config: NodeConfig) -> None: f"sudo {env_prefix} /tmp/bootstrap.sh", ]) + log("Copying preflight reports from VM...") + scp_from_vm("/tmp/unbounded-agent-preflight.txt", VM_DIR / "unbounded-agent-preflight.txt") + scp_from_vm("/tmp/unbounded-agent-preflight.json", VM_DIR / "unbounded-agent-preflight.json") + # --------------------------------------------------------------------------- # wait-for-node @@ -2881,6 +2904,21 @@ def _collect_one_vm_logs(logs_dir: Path, vm_name: str, vm_ip: str, vm_dir: Path, ] ssh_target = f"{VM_SSH_USER}@{vm_ip}" + for name in ("unbounded-agent-preflight.txt", "unbounded-agent-preflight.json"): + src = vm_dir / name + if src.exists(): + shutil.copyfile(src, logs_dir / f"{prefix}{name}") + + for name in ("unbounded-agent-preflight.txt", "unbounded-agent-preflight.json"): + result = subprocess.run( + ["scp", *ssh_opts, f"{ssh_target}:/tmp/{name}", str(logs_dir / f"{prefix}{name}")], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if result.returncode == 0: + diag(f"Collected {name} from VM") + def ssh_log(name: str, command: str) -> None: _write_command_log(logs_dir / f"{prefix}{name}", ["ssh", *ssh_opts, ssh_target, command]) diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 56f025761..14a46c71d 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "maps" + "net/url" "os" "slices" "strings" @@ -111,6 +112,45 @@ func (a *AgentConfig) DeepCopy() *AgentConfig { return &out } +// Validate checks that required agent configuration fields are present and +// internally consistent. Kubelet auth is validated when present; callers that +// require a bootstrap credential should enforce that separately because some +// flows fill the credential later through attestation. +func (a *AgentConfig) Validate() error { + if a == nil { + return fmt.Errorf("agent config is nil") + } + + var errs []error + if strings.TrimSpace(a.MachineName) == "" { + errs = append(errs, fmt.Errorf("MachineName is required")) + } + + if nodeName := strings.TrimSpace(a.NodeName); nodeName == "" { + errs = append(errs, fmt.Errorf("NodeName is required")) + } else if !isValidNodeName(nodeName) { + errs = append(errs, fmt.Errorf("NodeName is not a valid Kubernetes node name")) + } + + if strings.TrimSpace(a.Cluster.ClusterDNS) == "" { + errs = append(errs, fmt.Errorf("Cluster.ClusterDNS is required")) + } + + apiServer := strings.TrimSpace(a.Kubelet.ApiServer) + if apiServer == "" { + errs = append(errs, fmt.Errorf("Kubelet.ApiServer is required")) + } else if u, err := url.Parse(apiServer); err != nil || u.Scheme == "" || u.Host == "" { + errs = append(errs, fmt.Errorf("Kubelet.ApiServer is invalid")) + } + + // Kubelet auth is intentionally not validated here. Some consumers provide + // credentials later through product-specific flows such as attestation, and + // that context is outside the shared AgentConfig. Callers that require a + // static bootstrap credential should validate Kubelet.Auth separately. + + return errors.Join(errs...) +} + // AgentClusterConfig holds the cluster-level values the agent needs to // join the Kubernetes control plane. type AgentClusterConfig struct { diff --git a/pkg/agent/config/config_test.go b/pkg/agent/config/config_test.go index f734050dd..a1c0f9077 100644 --- a/pkg/agent/config/config_test.go +++ b/pkg/agent/config/config_test.go @@ -103,6 +103,77 @@ func TestCRIConfig_JSONRoundTrip(t *testing.T) { assert.Equal(t, "1.6.0", decoded.CNI.PluginVersion) } +func TestAgentConfig_Validate(t *testing.T) { + t.Parallel() + + valid := func() *AgentConfig { + return &AgentConfig{ + MachineName: "machine-1", + NodeName: "node-1", + Cluster: AgentClusterConfig{ + CaCertBase64: "Y2E=", + ClusterDNS: "10.0.0.10", + Version: "1.34.0", + }, + Kubelet: AgentKubeletConfig{ + ApiServer: "https://api.example.com:443", + Auth: KubeletAuthInfo{ + BootstrapToken: "abc123.secret456", + }, + }, + } + } + + tests := []struct { + name string + mutate func(*AgentConfig) + wantErr string + }{ + { + name: "valid", + }, + { + name: "missing machine name", + mutate: func(cfg *AgentConfig) { + cfg.MachineName = "" + }, + wantErr: "MachineName", + }, + { + name: "invalid node name", + mutate: func(cfg *AgentConfig) { + cfg.NodeName = "Invalid_Node" + }, + wantErr: "NodeName", + }, + { + name: "missing auth allowed", + mutate: func(cfg *AgentConfig) { + cfg.Kubelet.Auth = KubeletAuthInfo{} + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + cfg := valid() + if tt.mutate != nil { + tt.mutate(cfg) + } + + err := cfg.Validate() + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + + require.NoError(t, err) + }) + } +} + func TestCRIConfig_OmittedWhenEmpty(t *testing.T) { t.Parallel() diff --git a/pkg/agent/internal/utilio/fs.go b/pkg/agent/internal/utilio/fs.go index b0cb349fc..eee35bf26 100644 --- a/pkg/agent/internal/utilio/fs.go +++ b/pkg/agent/internal/utilio/fs.go @@ -81,3 +81,20 @@ func UpdateSymlink(linkPath, targetPath string) error { return renameio.Symlink(targetPath, linkPath) } + +// ProbeWritableDir verifies that dir accepts file creation and removal without +// leaving durable state behind. +func ProbeWritableDir(dir string) error { + f, err := os.CreateTemp(dir, ".unbounded-probe-*") + if err != nil { + return err + } + + name := f.Name() + if err := f.Close(); err != nil { + os.Remove(name) //nolint:errcheck // best effort cleanup after close failure. + return err + } + + return os.Remove(name) +} diff --git a/pkg/agent/internal/utilio/fs_test.go b/pkg/agent/internal/utilio/fs_test.go index a826523df..87c6c52a2 100644 --- a/pkg/agent/internal/utilio/fs_test.go +++ b/pkg/agent/internal/utilio/fs_test.go @@ -145,3 +145,21 @@ func TestUpdateSymlink(t *testing.T) { t.Fatalf("second target = %q, want %q", target, secondTarget) } } + +func TestProbeWritableDir(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + if err := ProbeWritableDir(dir); err != nil { + t.Fatalf("probe writable dir: %v", err) + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read temp dir: %v", err) + } + + if len(entries) != 0 { + t.Fatalf("probe left entries behind: %v", entries) + } +} diff --git a/pkg/agent/phases/host/preflight_host.go b/pkg/agent/phases/host/preflight_host.go new file mode 100644 index 000000000..905bfceef --- /dev/null +++ b/pkg/agent/phases/host/preflight_host.go @@ -0,0 +1,382 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package host + +import ( + "bufio" + "context" + "io/fs" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + + "github.com/Azure/unbounded/internal/executil" + "github.com/Azure/unbounded/internal/provision" + "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/internal/utilio" + "github.com/Azure/unbounded/pkg/agent/preflight" +) + +const ( + checkIsPrivilegedUserName = "is-privileged-user" + checkHostPackagesName = "host-packages" + checkHostOSConfigurationName = "host-os-configuration" + checkNSpawnRuntimeName = "nspawn-runtime" + checkDockerActiveName = "docker-active" + checkSwapActiveName = "swap-active" + checkDiskSpaceName = "disk-space" + checkCgroupsName = "cgroups" + + minFreeDiskBytes = 8 * 1024 * 1024 * 1024 +) + +type hostCheckDeps struct { + lookupPath func(string) (string, error) + uid func() int + statfs func(string, *syscall.Statfs_t) error + readFile func(string) ([]byte, error) + stat func(string) (fs.FileInfo, error) + writeProbe func(string) error + outputCmd func(context.Context, *slog.Logger, string, ...string) (string, error) +} + +func defaultHostCheckDeps() hostCheckDeps { + return hostCheckDeps{ + lookupPath: exec.LookPath, + uid: os.Geteuid, + statfs: syscall.Statfs, + readFile: os.ReadFile, + stat: os.Stat, + writeProbe: utilio.ProbeWritableDir, + outputCmd: executil.OutputCmd, + } +} + +type simpleHostChecker struct { + name string + check func(context.Context) []preflight.Result +} + +func (c simpleHostChecker) Name() string { return c.name } + +func (c simpleHostChecker) Check(ctx context.Context) []preflight.Result { return c.check(ctx) } + +// Preflight returns the standard host environment checks required before +// provisioning an nspawn machine. +func Preflight(log *slog.Logger, _ *provision.UnboundedAgentConfig, _ *goalstates.MachineGoalState) []preflight.Checker { + return []preflight.Checker{ + CheckIsPrivilegedUser(log), + CheckHostPackages(log), + CheckHostOSConfiguration(log), + CheckNSpawnRuntime(log), + CheckDockerActive(log), + CheckSwapActive(log), + CheckDiskSpace(log), + CheckCgroups(log), + } +} + +// CheckIsPrivilegedUser verifies preflight is running as root. +func CheckIsPrivilegedUser(log *slog.Logger) preflight.Checker { + return checkIsPrivilegedUser(log, defaultHostCheckDeps()) +} + +func checkIsPrivilegedUser(log *slog.Logger, deps hostCheckDeps) preflight.Checker { + return simpleHostChecker{name: checkIsPrivilegedUserName, check: func(context.Context) []preflight.Result { + uid := deps.uid() + log.Debug("checking effective user", "uid", uid) + + if uid != 0 { + return preflight.ResultsError(checkIsPrivilegedUserName, "host user", "preflight must run as root") + } + + return preflight.ResultsOK(checkIsPrivilegedUserName, "host user", "preflight is running as root") + }} +} + +// CheckHostPackages verifies all required host packages are already installed. +func CheckHostPackages(log *slog.Logger) preflight.Checker { + return checkHostPackages(log, defaultHostCheckDeps()) +} + +func checkHostPackages(log *slog.Logger, deps hostCheckDeps) preflight.Checker { + return simpleHostChecker{name: checkHostPackagesName, check: func(ctx context.Context) []preflight.Result { + pm, err := detectHostPackageManager(deps.lookupPath) + if err != nil { + log.Debug("host package manager detection failed") + + return preflight.ResultsError( + checkHostPackagesName, + "host packages", + "supported host package manager is required: apt-get, tdnf, or dnf", + ) + } + + log.Debug( + "detected host package manager", + "packageManager", pm.name, + "requiredPackages", strings.Join(pm.requiredPackages, ","), + ) + + var missing []string + + for _, pkg := range pm.requiredPackages { + if !pm.installed(ctx, log, pkg) { + // TODO: when offline mode is configured, missing required host + // packages should be reported as an error because bootstrap cannot + // rely on package source access to remediate them. + missing = append(missing, pkg) + } + } + + if len(missing) > 0 { + log.Debug("required host packages are missing", "packages", strings.Join(missing, ",")) + + // TODO: when offline mode is configured, missing required host + // packages should be reported as an error because bootstrap cannot + // rely on package source access to remediate them. + return preflight.ResultsWarning( + checkHostPackagesName, + "host packages", + "required host packages are missing and may be installed by bootstrap: %s", + strings.Join(missing, ", "), + ) + } + + log.Debug("required host packages are installed") + + return preflight.ResultsOK(checkHostPackagesName, "host packages", "required host packages are installed") + }} +} + +// CheckHostOSConfiguration verifies host OS configuration paths are writable. +func CheckHostOSConfiguration(log *slog.Logger) preflight.Checker { + return checkHostOSConfiguration(log, defaultHostCheckDeps()) +} + +func checkHostOSConfiguration(log *slog.Logger, deps hostCheckDeps) preflight.Checker { + return simpleHostChecker{name: checkHostOSConfigurationName, check: func(context.Context) []preflight.Result { + var results []preflight.Result + + sysctlDir := filepath.Dir(hostSysctlPath) + log.Debug("checking host OS configuration path", "path", sysctlDir) + + if err := deps.writeProbe(sysctlDir); err != nil { + results = append(results, preflight.Error( + checkHostOSConfigurationName, + "host OS configuration", + "host OS configuration path is not writable: %s", + sysctlDir, + )) + } + + log.Debug("checking systemd unit directory", "path", goalstates.SystemdSystemDir) + + if err := deps.writeProbe(goalstates.SystemdSystemDir); err != nil { + results = append(results, preflight.Error( + checkHostOSConfigurationName, + "host OS configuration", + "systemd unit directory is not writable: %s", + goalstates.SystemdSystemDir, + )) + } + + if len(results) > 0 { + return results + } + + return preflight.ResultsOK( + checkHostOSConfigurationName, + "host OS configuration", + "host OS configuration can be applied", + ) + }} +} + +// CheckNSpawnRuntime verifies systemd-nspawn runtime tools are available. +func CheckNSpawnRuntime(log *slog.Logger) preflight.Checker { + return checkNSpawnRuntime(log, defaultHostCheckDeps()) +} + +func checkNSpawnRuntime(log *slog.Logger, deps hostCheckDeps) preflight.Checker { + return simpleHostChecker{name: checkNSpawnRuntimeName, check: func(context.Context) []preflight.Result { + var results []preflight.Result + + for _, binary := range []string{"systemctl", "machinectl", "systemd-nspawn"} { + log.Debug("checking nspawn runtime tool", "binary", binary) + + if _, err := deps.lookupPath(binary); err != nil { + // TODO: when offline mode is configured, missing nspawn runtime + // tools should be reported as an error because bootstrap cannot rely + // on package installation to remediate them. + results = append(results, preflight.Warning( + checkNSpawnRuntimeName, + "nspawn runtime", + "nspawn runtime tool is missing and may be installed by bootstrap: %s", + binary, + )) + } + } + + systemdRuntimePath := "/run/systemd/system" + log.Debug("checking systemd runtime path", "path", systemdRuntimePath) + + if _, err := deps.stat(systemdRuntimePath); err != nil { + results = append(results, preflight.Warning( + checkNSpawnRuntimeName, + "nspawn runtime", + "systemd runtime path is not currently available: %s", + systemdRuntimePath, + )) + } + + if len(results) > 0 { + return results + } + + return preflight.ResultsOK(checkNSpawnRuntimeName, "nspawn runtime", "nspawn runtime is available") + }} +} + +// CheckDockerActive warns when Docker is active. +func CheckDockerActive(log *slog.Logger) preflight.Checker { + return checkDockerActive(log, defaultHostCheckDeps()) +} + +func checkDockerActive(log *slog.Logger, deps hostCheckDeps) preflight.Checker { + return simpleHostChecker{name: checkDockerActiveName, check: func(ctx context.Context) []preflight.Result { + out, err := deps.outputCmd(ctx, log, "systemctl", "is-active", dockerServiceUnit) + log.Debug( + "checked Docker unit state", + "unit", dockerServiceUnit, + "state", strings.TrimSpace(out), + "error", err != nil, + ) + + if err == nil && strings.TrimSpace(out) == "active" { + return preflight.ResultsWarning( + checkDockerActiveName, + "docker service", + "Docker is active and bootstrap will disable it", + ) + } + + return preflight.ResultsOK(checkDockerActiveName, "docker service", "Docker is not active") + }} +} + +// CheckSwapActive warns when host swap is active. +func CheckSwapActive(log *slog.Logger) preflight.Checker { + return checkSwapActive(log, defaultHostCheckDeps()) +} + +func checkSwapActive(log *slog.Logger, deps hostCheckDeps) preflight.Checker { + return simpleHostChecker{name: checkSwapActiveName, check: func(context.Context) []preflight.Result { + active, err := swapActive(deps.readFile) + log.Debug("checked host swap state", "active", active, "error", err != nil) + + if err != nil { + return preflight.ResultsWarning( + checkSwapActiveName, + "host swap", + "swap state could not be determined from /proc/swaps", + ) + } + + if active { + return preflight.ResultsWarning(checkSwapActiveName, "host swap", "swap is enabled and bootstrap will disable it") + } + + return preflight.ResultsOK(checkSwapActiveName, "host swap", "swap is not active") + }} +} + +// CheckDiskSpace verifies enough free disk is available for bootstrap. +func CheckDiskSpace(log *slog.Logger) preflight.Checker { + return checkDiskSpace(log, defaultHostCheckDeps()) +} + +func checkDiskSpace(log *slog.Logger, deps hostCheckDeps) preflight.Checker { + return simpleHostChecker{name: checkDiskSpaceName, check: func(context.Context) []preflight.Result { + var stat syscall.Statfs_t + + diskPath := "/var/lib" + if err := deps.statfs(diskPath, &stat); err != nil { + log.Debug("failed to check disk space", "path", diskPath) + + return preflight.ResultsError( + checkDiskSpaceName, + "host disk", + "available disk space could not be determined for %s", + diskPath, + ) + } + + free := stat.Bavail * uint64(stat.Bsize) + log.Debug( + "checked disk space", + "path", diskPath, + "freeGiB", gib(free), + "requiredGiB", gib(minFreeDiskBytes), + ) + + if free < minFreeDiskBytes { + return preflight.ResultsError( + checkDiskSpaceName, + "host disk", + "available disk space is below the minimum for %s: current %.1f GiB, required %.1f GiB", + diskPath, + gib(free), + gib(minFreeDiskBytes), + ) + } + + return preflight.ResultsOK(checkDiskSpaceName, "host disk", "sufficient disk space is available") + }} +} + +// CheckCgroups verifies the host cgroup filesystem is available. +func CheckCgroups(log *slog.Logger) preflight.Checker { + return checkCgroups(log, defaultHostCheckDeps()) +} + +func checkCgroups(log *slog.Logger, deps hostCheckDeps) preflight.Checker { + return simpleHostChecker{name: checkCgroupsName, check: func(context.Context) []preflight.Result { + cgroupPath := "/sys/fs/cgroup" + log.Debug("checking cgroup filesystem", "path", cgroupPath) + + if _, err := deps.stat(cgroupPath); err != nil { + return preflight.ResultsError(checkCgroupsName, "host cgroups", "cgroup filesystem is required at %s", cgroupPath) + } + + return preflight.ResultsOK(checkCgroupsName, "host cgroups", "cgroup filesystem is available") + }} +} + +func gib(bytes uint64) float64 { + return float64(bytes) / (1024 * 1024 * 1024) +} + +func swapActive(readFile func(string) ([]byte, error)) (bool, error) { + data, err := readFile("/proc/swaps") + if err != nil { + return false, err + } + + scanner := bufio.NewScanner(strings.NewReader(string(data))) + for scanner.Scan() { + if strings.HasPrefix(scanner.Text(), "Filename") { + continue + } + + if strings.TrimSpace(scanner.Text()) != "" { + return true, nil + } + } + + return false, scanner.Err() +} diff --git a/pkg/agent/phases/host/preflight_host_test.go b/pkg/agent/phases/host/preflight_host_test.go new file mode 100644 index 000000000..6fa5127f1 --- /dev/null +++ b/pkg/agent/phases/host/preflight_host_test.go @@ -0,0 +1,179 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package host + +import ( + "context" + "errors" + "io/fs" + "log/slog" + "os/exec" + "syscall" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/Azure/unbounded/pkg/agent/preflight" +) + +func TestCheckIsPrivilegedUser(t *testing.T) { + results := checkIsPrivilegedUser(slog.New(slog.DiscardHandler), hostCheckDeps{uid: func() int { return 0 }}).Check(context.Background()) + assert.Equal(t, preflight.SeverityOK, results[0].Severity) + + results = checkIsPrivilegedUser(slog.New(slog.DiscardHandler), hostCheckDeps{uid: func() int { return 1000 }}).Check(context.Background()) + assert.Equal(t, preflight.SeverityError, results[0].Severity) +} + +func TestCheckHostPackagesMissingPackageManager(t *testing.T) { + deps := defaultHostCheckDeps() + deps.lookupPath = lookupPathWith(nil) + + results := checkHostPackages(slog.New(slog.DiscardHandler), deps).Check(context.Background()) + + assert.Equal(t, preflight.SeverityError, results[0].Severity) + assert.Contains(t, results[0].Message, "apt-get") +} + +func TestCheckHostPackagesListsMissingPackages(t *testing.T) { + deps := defaultHostCheckDeps() + deps.lookupPath = lookupPathWith(map[string]bool{"apt-get": true}) + + results := checkHostPackages(slog.New(slog.DiscardHandler), deps).Check(context.Background()) + + assert.Equal(t, preflight.SeverityWarning, results[0].Severity) + assert.Contains(t, results[0].Message, "systemd-container") +} + +func TestCheckHostOSConfiguration(t *testing.T) { + deps := defaultHostCheckDeps() + deps.writeProbe = func(string) error { return nil } + + results := checkHostOSConfiguration(slog.New(slog.DiscardHandler), deps).Check(context.Background()) + assert.Equal(t, preflight.SeverityOK, results[0].Severity) + + deps.writeProbe = func(string) error { return errors.New("denied") } + results = checkHostOSConfiguration(slog.New(slog.DiscardHandler), deps).Check(context.Background()) + assert.Len(t, results, 2) + assert.Equal(t, preflight.SeverityError, results[0].Severity) + assert.Contains(t, results[0].Message, "/etc/sysctl.d") + assert.Equal(t, preflight.SeverityError, results[1].Severity) + assert.Contains(t, results[1].Message, "systemd") +} + +func TestCheckNSpawnRuntime(t *testing.T) { + deps := defaultHostCheckDeps() + deps.lookupPath = lookupPathWith(map[string]bool{ + "systemctl": true, + "machinectl": true, + "systemd-nspawn": true, + }) + deps.stat = func(string) (fs.FileInfo, error) { return nil, nil } + + results := checkNSpawnRuntime(slog.New(slog.DiscardHandler), deps).Check(context.Background()) + assert.Equal(t, preflight.SeverityOK, results[0].Severity) + + deps.lookupPath = lookupPathWith(map[string]bool{"systemctl": true}) + deps.stat = statMissing() + results = checkNSpawnRuntime(slog.New(slog.DiscardHandler), deps).Check(context.Background()) + assert.Len(t, results, 3) + assert.Equal(t, preflight.SeverityWarning, results[0].Severity) + assert.Contains(t, results[0].Message, "machinectl") + assert.Equal(t, preflight.SeverityWarning, results[1].Severity) + assert.Contains(t, results[1].Message, "systemd-nspawn") + assert.Equal(t, preflight.SeverityWarning, results[2].Severity) + assert.Contains(t, results[2].Message, "/run/systemd/system") +} + +func TestCheckDockerActive(t *testing.T) { + deps := defaultHostCheckDeps() + deps.outputCmd = outputWith("active\n", nil) + + results := checkDockerActive(slog.New(slog.DiscardHandler), deps).Check(context.Background()) + assert.Equal(t, preflight.SeverityWarning, results[0].Severity) + + deps.outputCmd = outputWith("inactive\n", nil) + results = checkDockerActive(slog.New(slog.DiscardHandler), deps).Check(context.Background()) + assert.Equal(t, preflight.SeverityOK, results[0].Severity) +} + +func TestCheckSwapActive(t *testing.T) { + deps := defaultHostCheckDeps() + deps.readFile = readFileString("Filename\tType\tSize\tUsed\tPriority\n", nil) + + results := checkSwapActive(slog.New(slog.DiscardHandler), deps).Check(context.Background()) + assert.Equal(t, preflight.SeverityOK, results[0].Severity) + + deps.readFile = readFileString("Filename\tType\tSize\tUsed\tPriority\n/swapfile file 1024 0 -2\n", nil) + results = checkSwapActive(slog.New(slog.DiscardHandler), deps).Check(context.Background()) + assert.Equal(t, preflight.SeverityWarning, results[0].Severity) + + deps.readFile = readFileString("", errors.New("missing")) + results = checkSwapActive(slog.New(slog.DiscardHandler), deps).Check(context.Background()) + assert.Contains(t, results[0].Message, "/proc/swaps") +} + +func TestCheckDiskSpace(t *testing.T) { + deps := defaultHostCheckDeps() + deps.statfs = statfsWithFreeBytes(minFreeDiskBytes) + + results := checkDiskSpace(slog.New(slog.DiscardHandler), deps).Check(context.Background()) + assert.Equal(t, preflight.SeverityOK, results[0].Severity) + + deps.statfs = statfsWithFreeBytes(1) + results = checkDiskSpace(slog.New(slog.DiscardHandler), deps).Check(context.Background()) + assert.Equal(t, preflight.SeverityError, results[0].Severity) + assert.Contains(t, results[0].Message, "/var/lib") + assert.Contains(t, results[0].Message, "current 0.0 GiB") + assert.Contains(t, results[0].Message, "required 8.0 GiB") +} + +func TestCheckCgroups(t *testing.T) { + deps := defaultHostCheckDeps() + deps.stat = statExists() + + results := checkCgroups(slog.New(slog.DiscardHandler), deps).Check(context.Background()) + assert.Equal(t, preflight.SeverityOK, results[0].Severity) + + deps.stat = statMissing() + results = checkCgroups(slog.New(slog.DiscardHandler), deps).Check(context.Background()) + assert.Equal(t, preflight.SeverityError, results[0].Severity) + assert.Contains(t, results[0].Message, "/sys/fs/cgroup") +} + +func statfsWithFreeBytes(bytes uint64) func(string, *syscall.Statfs_t) error { + return func(_ string, stat *syscall.Statfs_t) error { + stat.Bsize = 1 + stat.Bavail = bytes + + return nil + } +} + +func statExists() func(string) (fs.FileInfo, error) { + return func(string) (fs.FileInfo, error) { return nil, nil } +} + +func statMissing() func(string) (fs.FileInfo, error) { + return func(string) (fs.FileInfo, error) { return nil, errors.New("missing") } +} + +func lookupPathWith(paths map[string]bool) func(string) (string, error) { + return func(name string) (string, error) { + if paths[name] { + return "/usr/bin/" + name, nil + } + + return "", exec.ErrNotFound + } +} + +func outputWith(value string, err error) func(context.Context, *slog.Logger, string, ...string) (string, error) { + return func(context.Context, *slog.Logger, string, ...string) (string, error) { + return value, err + } +} + +func readFileString(value string, err error) func(string) ([]byte, error) { + return func(string) ([]byte, error) { return []byte(value), err } +} diff --git a/pkg/agent/phases/nodestart/preflight_api_server.go b/pkg/agent/phases/nodestart/preflight_api_server.go new file mode 100644 index 000000000..0943bcd1d --- /dev/null +++ b/pkg/agent/phases/nodestart/preflight_api_server.go @@ -0,0 +1,123 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package nodestart + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "log/slog" + "net/http" + "net/url" + "strings" + "time" + + "github.com/Azure/unbounded/internal/provision" + "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/preflight" +) + +const checkAPIServerReachableName = "api-server-reachable" + +type apiServerReachableChecker struct { + log *slog.Logger + config *provision.UnboundedAgentConfig + httpClient *http.Client +} + +// Preflight returns the standard node-start checks that can run before the +// nspawn machine starts. +func Preflight(log *slog.Logger, cfg *provision.UnboundedAgentConfig, _ *goalstates.MachineGoalState) []preflight.Checker { + return []preflight.Checker{ + CheckAPIServerReachable(log, cfg), + } +} + +// CheckAPIServerReachable returns a non-mutating checker that validates the +// cluster credentials and configured Kubernetes API server reachability. The +// checker redacts the configured endpoint from result messages. +func CheckAPIServerReachable(log *slog.Logger, cfg *provision.UnboundedAgentConfig) preflight.Checker { + return apiServerReachableChecker{log: log, config: cfg} +} + +func (c apiServerReachableChecker) Name() string { return checkAPIServerReachableName } + +func (c apiServerReachableChecker) Check(ctx context.Context) []preflight.Result { + if c.config == nil { + return preflight.ResultsError(checkAPIServerReachableName, "cluster API server", "agent config is missing") + } + + if errs := c.validateClusterCredentials(); len(errs) > 0 { + return preflight.ResultsError(checkAPIServerReachableName, "cluster credentials", "%s", strings.Join(errs, "; ")) + } + + apiServer := c.config.Kubelet.ApiServer + if strings.TrimSpace(apiServer) == "" { + return preflight.ResultsError(checkAPIServerReachableName, "cluster API server", "API server is required") + } + + parsed, err := url.Parse(apiServer) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return preflight.ResultsError(checkAPIServerReachableName, "cluster API server", "API server endpoint is invalid") + } + + client := c.httpClient + if client == nil { + client = c.httpClientWithCA() + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(apiServer, "/")+"/readyz", http.NoBody) + if err != nil { + return preflight.ResultsError(checkAPIServerReachableName, "cluster API server", "API server request could not be created") + } + + resp, err := client.Do(req) + if err != nil { + return preflight.ResultsError(checkAPIServerReachableName, "cluster API server", "API server is not reachable") + } + defer resp.Body.Close() //nolint:errcheck // best effort close + + if resp.StatusCode >= http.StatusInternalServerError { + return preflight.ResultsError(checkAPIServerReachableName, "cluster API server", "API server returned status %d", resp.StatusCode) + } + + return preflight.ResultsOK(checkAPIServerReachableName, "cluster API server", "API server is reachable") +} + +func (c apiServerReachableChecker) validateClusterCredentials() []string { + var errs []string + if _, err := base64.StdEncoding.DecodeString(c.config.Cluster.CaCertBase64); err != nil { + errs = append(errs, "cluster CA data is invalid") + } + + if c.config.Attest == nil { + auth := c.config.Kubelet.Auth + if err := auth.Validate(); err != nil { + errs = append(errs, "bootstrap credential is invalid") + } + } + + return errs +} + +func (c apiServerReachableChecker) httpClientWithCA() *http.Client { + transport := &http.Transport{} + if defaultTransport, ok := http.DefaultTransport.(*http.Transport); ok { + transport = defaultTransport.Clone() + } + + caCertData, err := base64.StdEncoding.DecodeString(c.config.Cluster.CaCertBase64) + if err == nil && len(caCertData) > 0 { + pool, err := x509.SystemCertPool() + if err != nil { + pool = x509.NewCertPool() + } + + pool.AppendCertsFromPEM(caCertData) + transport.TLSClientConfig = &tls.Config{RootCAs: pool} //nolint:gosec // uses configured root CAs. + } + + return &http.Client{Timeout: 10 * time.Second, Transport: transport} +} diff --git a/pkg/agent/phases/nodestart/preflight_api_server_test.go b/pkg/agent/phases/nodestart/preflight_api_server_test.go new file mode 100644 index 000000000..eee7b5dd4 --- /dev/null +++ b/pkg/agent/phases/nodestart/preflight_api_server_test.go @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package nodestart + +import ( + "context" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/Azure/unbounded/internal/provision" + "github.com/Azure/unbounded/pkg/agent/preflight" +) + +func TestCheckAPIServerReachableOK(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/readyz", r.URL.Path) + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + + results := CheckAPIServerReachable(slog.New(slog.DiscardHandler), apiServerPreflightConfig(srv.URL)).Check(context.Background()) + + assert.Equal(t, preflight.ResultsOK(checkAPIServerReachableName, "cluster API server", "API server is reachable"), results) +} + +func TestCheckAPIServerReachableInvalidEndpoint(t *testing.T) { + results := CheckAPIServerReachable(slog.New(slog.DiscardHandler), apiServerPreflightConfig("://bad")).Check(context.Background()) + + assert.Equal(t, preflight.SeverityError, results[0].Severity) + assert.Equal(t, checkAPIServerReachableName, results[0].Name) + assert.Equal(t, "cluster API server", results[0].Target) + assert.Equal(t, "API server endpoint is invalid", results[0].Message) +} + +func TestCheckAPIServerReachableRequestFailureIsRedacted(t *testing.T) { + const endpoint = "https://127.0.0.1:1" + + results := CheckAPIServerReachable(slog.New(slog.DiscardHandler), apiServerPreflightConfig(endpoint)).Check(context.Background()) + + assert.Equal(t, preflight.SeverityError, results[0].Severity) + assert.Equal(t, "API server is not reachable", results[0].Message) + assert.NotContains(t, results[0].Message, endpoint) +} + +func TestCheckAPIServerReachableServerError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + t.Cleanup(srv.Close) + + results := CheckAPIServerReachable(slog.New(slog.DiscardHandler), apiServerPreflightConfig(srv.URL)).Check(context.Background()) + + assert.Equal(t, preflight.SeverityError, results[0].Severity) + assert.Equal(t, "API server returned status 500", results[0].Message) +} + +func TestCheckAPIServerReachableAllowsAttestation(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + + cfg := apiServerPreflightConfig(srv.URL) + cfg.Kubelet.Auth.BootstrapToken = "" + cfg.Attest = &provision.AgentAttestConfig{URL: "http://metalman.example.com:8880"} + + results := CheckAPIServerReachable(slog.New(slog.DiscardHandler), cfg).Check(context.Background()) + + assert.Equal(t, preflight.SeverityOK, results[0].Severity) +} + +func TestCheckAPIServerReachableRequiresAuthWhenNoAttestation(t *testing.T) { + cfg := apiServerPreflightConfig("https://api.example.com:443") + cfg.Kubelet.Auth.BootstrapToken = "" + + results := CheckAPIServerReachable(slog.New(slog.DiscardHandler), cfg).Check(context.Background()) + + assert.Equal(t, preflight.SeverityError, results[0].Severity) + assert.Equal(t, checkAPIServerReachableName, results[0].Name) + assert.Equal(t, "cluster credentials", results[0].Target) + assert.Equal(t, "bootstrap credential is invalid", results[0].Message) +} + +func TestCheckAPIServerReachableInvalidCA(t *testing.T) { + cfg := apiServerPreflightConfig("https://api.example.com:443") + cfg.Cluster.CaCertBase64 = "not-base64" + + results := CheckAPIServerReachable(slog.New(slog.DiscardHandler), cfg).Check(context.Background()) + + assert.Equal(t, preflight.SeverityError, results[0].Severity) + assert.Contains(t, results[0].Message, "cluster CA data is invalid") +} + +func apiServerPreflightConfig(apiServer string) *provision.UnboundedAgentConfig { + return &provision.UnboundedAgentConfig{ + AgentConfig: provision.AgentConfig{ + Cluster: provision.AgentClusterConfig{ + CaCertBase64: "Y2E=", + }, + Kubelet: provision.AgentKubeletConfig{ + ApiServer: apiServer, + Auth: provision.KubeletAuthInfo{ + BootstrapToken: "abc123.secret456", + }, + }, + }, + } +} diff --git a/pkg/agent/phases/rootfs/preflight_goal_state.go b/pkg/agent/phases/rootfs/preflight_goal_state.go new file mode 100644 index 000000000..1aac02391 --- /dev/null +++ b/pkg/agent/phases/rootfs/preflight_goal_state.go @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package rootfs + +import ( + "context" + "log/slog" + + "github.com/Azure/unbounded/internal/provision" + "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/preflight" +) + +const ( + checkGoalStateName = "goal-state" + checkOCIImageReachableName = "oci-image-reachable" +) + +type goalStateChecker struct { + log *slog.Logger + rootFS *goalstates.RootFS +} + +// Preflight returns the standard rootfs checks for a resolved machine goal state. +func Preflight(log *slog.Logger, _ *provision.UnboundedAgentConfig, goalState *goalstates.MachineGoalState) []preflight.Checker { + var rootFS *goalstates.RootFS + if goalState != nil { + rootFS = goalState.RootFS + } + + return []preflight.Checker{ + CheckGoalState(log, rootFS), + CheckNSpawnMachineProvisioning(log, rootFS), + } +} + +// CheckGoalState validates the agent config resolved into a machine goal state. +func CheckGoalState(log *slog.Logger, rootFS *goalstates.RootFS) preflight.Checker { + return goalStateChecker{log: log, rootFS: rootFS} +} + +func (c goalStateChecker) Name() string { return checkGoalStateName } + +func (c goalStateChecker) Check(context.Context) []preflight.Result { + if c.rootFS == nil { + return preflight.ResultsError(checkGoalStateName, "goal state", "goal state could not be resolved") + } + + if c.rootFS.OCIImage == "" { + // TODO: replace this with an OCI manifest reachability check that uses + // the same registry parsing and plain-HTTP handling as OCI rootfs + // provisioning, without pulling image layers. + return preflight.ResultsError( + checkOCIImageReachableName, + "rootfs image", + "OCI rootfs image is required but no image was selected", + ) + } + + return preflight.ResultsOK(checkGoalStateName, "goal state", "goal state resolved") +} diff --git a/pkg/agent/phases/rootfs/preflight_goal_state_test.go b/pkg/agent/phases/rootfs/preflight_goal_state_test.go new file mode 100644 index 000000000..455bffa04 --- /dev/null +++ b/pkg/agent/phases/rootfs/preflight_goal_state_test.go @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package rootfs + +import ( + "context" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/preflight" +) + +func validRootFSGoalState(t *testing.T) *goalstates.RootFS { + t.Helper() + + dir := t.TempDir() + + return &goalstates.RootFS{ + MachineDir: dir, + NSpawnConfigFile: dir + "/nspawn/kube1.nspawn", + ServiceOverrideFile: dir + "/systemd/system/systemd-nspawn@kube1.service.d/override.conf", + OCIImage: "registry.example.com/unbounded/rootfs:v1", + } +} + +func TestCheckGoalStateOK(t *testing.T) { + results := CheckGoalState(slog.New(slog.DiscardHandler), validRootFSGoalState(t)).Check(context.Background()) + + assert.Equal(t, []preflight.Result{preflight.OK(checkGoalStateName, "goal state", "goal state resolved")}, results) +} + +func TestCheckGoalStateResolveError(t *testing.T) { + results := CheckGoalState(slog.New(slog.DiscardHandler), nil).Check(context.Background()) + + assert.Equal(t, preflight.SeverityError, results[0].Severity) + assert.Equal(t, checkGoalStateName, results[0].Name) + assert.Equal(t, "goal state could not be resolved", results[0].Message) +} diff --git a/pkg/agent/phases/rootfs/preflight_local.go b/pkg/agent/phases/rootfs/preflight_local.go new file mode 100644 index 000000000..f558d6ca4 --- /dev/null +++ b/pkg/agent/phases/rootfs/preflight_local.go @@ -0,0 +1,199 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package rootfs + +import ( + "context" + "errors" + "io" + "io/fs" + "log/slog" + "os" + "path/filepath" + + "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/internal/utilio" + "github.com/Azure/unbounded/pkg/agent/preflight" +) + +const ( + checkNSpawnMachineProvisioningName = "nspawn-machine-provisioning" +) + +type rootFSCheckDeps struct { + stat func(string) (fs.FileInfo, error) + open func(string) (*os.File, error) + writeProbe func(string) error +} + +func defaultRootFSCheckDeps() rootFSCheckDeps { + return rootFSCheckDeps{ + stat: os.Stat, + open: os.Open, + writeProbe: utilio.ProbeWritableDir, + } +} + +type nspawnMachineProvisioningChecker struct { + log *slog.Logger + gs *goalstates.RootFS + deps rootFSCheckDeps +} + +// CheckNSpawnMachineProvisioning validates local host paths needed to provision +// and configure the nspawn machine rootfs. +func CheckNSpawnMachineProvisioning(log *slog.Logger, gs *goalstates.RootFS) preflight.Checker { + return checkNSpawnMachineProvisioning(log, gs, defaultRootFSCheckDeps()) +} + +func checkNSpawnMachineProvisioning(log *slog.Logger, gs *goalstates.RootFS, deps rootFSCheckDeps) preflight.Checker { + return nspawnMachineProvisioningChecker{log: log, gs: gs, deps: deps} +} + +func (c nspawnMachineProvisioningChecker) Name() string { + return checkNSpawnMachineProvisioningName +} + +func (c nspawnMachineProvisioningChecker) Check(context.Context) []preflight.Result { + if c.gs == nil || c.gs.MachineDir == "" { + return preflight.ResultsError( + checkNSpawnMachineProvisioningName, + "nspawn machine provisioning", + "machine directory is not configured", + ) + } + + results := c.checkMachineDir() + + if result := c.checkCreatableDir(filepath.Dir(c.gs.MachineDir), "rootfs parent directory"); result != nil { + results = append(results, result...) + } + + for _, path := range []string{ + filepath.Dir(c.gs.NSpawnConfigFile), + filepath.Dir(c.gs.ServiceOverrideFile), + } { + if result := c.checkCreatableDir(path, "nspawn provisioning path"); result != nil { + results = append(results, result...) + } + } + + if len(results) > 0 { + return results + } + + return preflight.ResultsOK( + checkNSpawnMachineProvisioningName, + "nspawn machine provisioning", + "nspawn machine provisioning paths are ready", + ) +} + +func (c nspawnMachineProvisioningChecker) checkMachineDir() []preflight.Result { + var results []preflight.Result + + machineDir := c.gs.MachineDir + c.log.Debug("checking machine directory", "path", machineDir) + + info, err := c.deps.stat(machineDir) + switch { + case errors.Is(err, os.ErrNotExist): + // Missing machine directory is fine if the parent/provisioning paths are writable. + return nil + case err != nil: + return preflight.ResultsError( + checkNSpawnMachineProvisioningName, + "nspawn machine provisioning", + "machine directory cannot be inspected: %s", + machineDir, + ) + case !info.IsDir(): + return preflight.ResultsError( + checkNSpawnMachineProvisioningName, + "nspawn machine provisioning", + "machine directory path is not a directory: %s", + machineDir, + ) + } + + // The nspawn machine root needs traversal permissions so dbus inside the + // container can operate correctly when reusing an existing rootfs. + if info.Mode().Perm()&0o055 != 0o055 { + results = append(results, preflight.Warning( + checkNSpawnMachineProvisioningName, + "nspawn machine provisioning", + "machine directory permissions are too restrictive: %s", + machineDir, + )) + } + + empty, err := isDirEmpty(c.deps.open, machineDir) + if err != nil { + return append(results, preflight.Error( + checkNSpawnMachineProvisioningName, + "nspawn machine provisioning", + "machine directory cannot be read: %s", + machineDir, + )) + } + + if !empty { + // A populated machine directory is expected during rejoin or reuse of an + // existing kube1/kube2 rootfs. Rootfs provisioning will skip bootstrap + // rather than overwrite it. + c.log.Debug("machine directory exists and is not empty", "path", machineDir) + } else { + c.log.Debug("machine directory exists and is empty", "path", machineDir) + } + + return results +} + +func (c nspawnMachineProvisioningChecker) checkCreatableDir(path, label string) []preflight.Result { + c.log.Debug("checking "+label, "path", path) + existing := nearestExistingParent(c.deps.stat, path) + + if err := c.deps.writeProbe(existing); err != nil { + return preflight.ResultsError( + checkNSpawnMachineProvisioningName, + "nspawn machine provisioning", + "%s cannot be created under: %s", + label, + existing, + ) + } + + return nil +} + +func nearestExistingParent(stat func(string) (fs.FileInfo, error), path string) string { + for { + if info, err := stat(path); err == nil && info.IsDir() { + return path + } + + parent := filepath.Dir(path) + if parent == path { + return path + } + + path = parent + } +} + +func isDirEmpty(open func(string) (*os.File, error), dir string) (bool, error) { + f, err := open(dir) + if err != nil { + return false, err + } + + defer f.Close() //nolint:errcheck // best effort close + + _, err = f.Readdirnames(1) + if errors.Is(err, io.EOF) { + return true, nil + } + + return false, err +} diff --git a/pkg/agent/phases/rootfs/preflight_local_test.go b/pkg/agent/phases/rootfs/preflight_local_test.go new file mode 100644 index 000000000..858dfbac2 --- /dev/null +++ b/pkg/agent/phases/rootfs/preflight_local_test.go @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package rootfs + +import ( + "context" + "errors" + "io/fs" + "log/slog" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/pkg/agent/preflight" +) + +func TestCheckNSpawnMachineProvisioningMissingMachineDir(t *testing.T) { + gs := validRootFSGoalState(t) + gs.MachineDir = filepath.Join(t.TempDir(), "missing") + deps := defaultRootFSCheckDeps() + deps.writeProbe = func(string) error { return nil } + + results := checkNSpawnMachineProvisioning(slog.New(slog.DiscardHandler), gs, deps).Check(context.Background()) + + assert.Equal(t, preflight.SeverityOK, results[0].Severity) +} + +func TestCheckNSpawnMachineProvisioningMissingParentDir(t *testing.T) { + base := t.TempDir() + gs := validRootFSGoalState(t) + gs.MachineDir = filepath.Join(base, "var", "lib", "machines", "kube1") + + results := CheckNSpawnMachineProvisioning(slog.New(slog.DiscardHandler), gs).Check(context.Background()) + + assert.Equal(t, preflight.SeverityOK, results[0].Severity) +} + +func TestCheckNSpawnMachineProvisioningNonEmptyMachineDir(t *testing.T) { + gs := validRootFSGoalState(t) + require.NoError(t, os.WriteFile(filepath.Join(gs.MachineDir, "file"), []byte("x"), 0o600)) + + deps := defaultRootFSCheckDeps() + deps.writeProbe = func(string) error { return nil } + + results := checkNSpawnMachineProvisioning(slog.New(slog.DiscardHandler), gs, deps).Check(context.Background()) + + assert.Equal(t, preflight.SeverityOK, results[0].Severity) +} + +func TestCheckNSpawnMachineProvisioningMachineDirFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "file") + require.NoError(t, os.WriteFile(path, []byte("x"), 0o600)) + gs := validRootFSGoalState(t) + gs.MachineDir = path + + results := CheckNSpawnMachineProvisioning(slog.New(slog.DiscardHandler), gs).Check(context.Background()) + + assert.Equal(t, preflight.SeverityError, results[0].Severity) +} + +func TestCheckNSpawnMachineProvisioningRestrictiveMachineDir(t *testing.T) { + gs := validRootFSGoalState(t) + require.NoError(t, os.Chmod(gs.MachineDir, 0o700)) + + results := CheckNSpawnMachineProvisioning(slog.New(slog.DiscardHandler), gs).Check(context.Background()) + + assert.Equal(t, preflight.SeverityWarning, results[0].Severity) + assert.Equal(t, "machine directory permissions are too restrictive: "+gs.MachineDir, results[0].Message) +} + +func TestCheckNSpawnMachineProvisioningCollectsWarningAndError(t *testing.T) { + gs := validRootFSGoalState(t) + deps := defaultRootFSCheckDeps() + deps.stat = func(string) (fs.FileInfo, error) { return fakeRestrictiveDirInfo{}, nil } + deps.open = func(string) (*os.File, error) { return nil, errors.New("denied") } + deps.writeProbe = func(string) error { return errors.New("denied") } + + results := checkNSpawnMachineProvisioning(slog.New(slog.DiscardHandler), gs, deps).Check(context.Background()) + + require.GreaterOrEqual(t, len(results), 2) + assert.Equal(t, preflight.SeverityWarning, results[0].Severity) + assert.Equal(t, preflight.SeverityError, results[1].Severity) + assert.Equal(t, "machine directory cannot be read: "+gs.MachineDir, results[1].Message) +} + +func TestCheckNSpawnMachineProvisioningWritablePaths(t *testing.T) { + gs := validRootFSGoalState(t) + deps := defaultRootFSCheckDeps() + deps.writeProbe = func(string) error { return nil } + + results := checkNSpawnMachineProvisioning(slog.New(slog.DiscardHandler), gs, deps).Check(context.Background()) + assert.Equal(t, preflight.SeverityOK, results[0].Severity) + + deps.writeProbe = func(string) error { return errors.New("denied") } + results = checkNSpawnMachineProvisioning(slog.New(slog.DiscardHandler), gs, deps).Check(context.Background()) + assert.Equal(t, preflight.SeverityError, results[0].Severity) +} + +func TestCheckNSpawnMachineProvisioningUnreadableMachineDir(t *testing.T) { + gs := validRootFSGoalState(t) + deps := defaultRootFSCheckDeps() + deps.stat = func(string) (fs.FileInfo, error) { return fakeDirInfo{}, nil } + deps.open = func(string) (*os.File, error) { return nil, errors.New("denied") } + + results := checkNSpawnMachineProvisioning(slog.New(slog.DiscardHandler), gs, deps).Check(context.Background()) + + assert.Equal(t, preflight.SeverityError, results[0].Severity) +} + +type fakeDirInfo struct{} + +func (fakeDirInfo) Name() string { return "dir" } +func (fakeDirInfo) Size() int64 { return 0 } +func (fakeDirInfo) Mode() fs.FileMode { return fs.ModeDir | 0o755 } +func (fakeDirInfo) ModTime() time.Time { return time.Time{} } +func (fakeDirInfo) IsDir() bool { return true } +func (fakeDirInfo) Sys() any { return nil } + +type fakeRestrictiveDirInfo struct{} + +func (fakeRestrictiveDirInfo) Name() string { return "dir" } +func (fakeRestrictiveDirInfo) Size() int64 { return 0 } +func (fakeRestrictiveDirInfo) Mode() fs.FileMode { return fs.ModeDir | 0o700 } +func (fakeRestrictiveDirInfo) ModTime() time.Time { return time.Time{} } +func (fakeRestrictiveDirInfo) IsDir() bool { return true } +func (fakeRestrictiveDirInfo) Sys() any { return nil } diff --git a/pkg/agent/preflight/preflight.go b/pkg/agent/preflight/preflight.go new file mode 100644 index 000000000..8221500e5 --- /dev/null +++ b/pkg/agent/preflight/preflight.go @@ -0,0 +1,230 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Package preflight provides reusable non-mutating checks for validating a host +// and agent configuration before bootstrap. +package preflight + +import ( + "context" + "fmt" + "slices" + "strings" + "sync" +) + +type Severity string + +const ( + // SeverityOK indicates the check completed successfully. + SeverityOK Severity = "ok" + // SeverityWarning indicates the check found a condition that bootstrap can + // usually remediate or that does not necessarily block bootstrap. + SeverityWarning Severity = "warning" + // SeverityError indicates the check found a fatal condition that should block + // bootstrap unless explicitly ignored. + SeverityError Severity = "error" +) + +// Checker is a non-mutating preflight validation unit. +type Checker interface { + Name() string + Check(ctx context.Context) []Result +} + +// Flatten returns one checker slice from ordered checker groups. +func Flatten(groups ...[]Checker) []Checker { + total := 0 + for _, group := range groups { + total += len(group) + } + + checks := make([]Checker, 0, total) + for _, group := range groups { + checks = append(checks, group...) + } + + return checks +} + +// Result describes one preflight check outcome. Message and Target must not +// include raw configured values such as URLs, tokens, image references, or file +// contents. +type Result struct { + Name string `json:"name"` + Target string `json:"target"` + Severity Severity `json:"severity"` + Message string `json:"message"` + Ignored bool `json:"ignored"` +} + +// Options controls preflight result handling. +type Options struct { + IgnoreErrors []string + FailOnWarnings bool +} + +// Report is the complete preflight output for both text and JSON consumers. +type Report struct { + Status string `json:"status"` + Checks []Result `json:"checks"` + Summary Summary `json:"summary"` +} + +// Summary contains aggregate result counts after ignore handling is applied. +type Summary struct { + OK int `json:"ok"` + Warnings int `json:"warnings"` + Errors int `json:"errors"` +} + +// Run executes all checks, applies ignore rules, and returns a complete report. +func Run(ctx context.Context, checks []Checker, opts Options) Report { + ignored := ignoreSet(opts.IgnoreErrors) + checkResults := make([][]Result, len(checks)) + + var wg sync.WaitGroup + + for i, check := range checks { + wg.Go(func() { + checkResults[i] = check.Check(ctx) + }) + } + + wg.Wait() + + results := make([]Result, 0, len(checks)) + for i, check := range checks { + for _, result := range checkResults[i] { + if result.Name == "" { + result.Name = check.Name() + } + + if result.Target == "" { + result.Target = result.Name + } + + if result.Severity == "" { + result.Severity = SeverityOK + } + + if result.Severity == SeverityError && ignored(result.Name) { + result.Severity = SeverityWarning + result.Ignored = true + } + + results = append(results, result) + } + } + + return buildReport(results, opts.FailOnWarnings) +} + +// OK returns a successful check result. +func OK(name, target, message string) Result { + return Result{Name: name, Target: target, Severity: SeverityOK, Message: message} +} + +// Warning returns a warning check result. When args are provided, message is +// formatted with fmt.Sprintf. +func Warning(name, target, message string, args ...any) Result { + return Result{Name: name, Target: target, Severity: SeverityWarning, Message: formatMessage(message, args...)} +} + +// Error returns a fatal check result. When args are provided, message is +// formatted with fmt.Sprintf. +func Error(name, target, message string, args ...any) Result { + return Result{Name: name, Target: target, Severity: SeverityError, Message: formatMessage(message, args...)} +} + +// Results returns a result slice for concise checker returns and test fixtures. +func Results(results ...Result) []Result { + return results +} + +// ResultsOK returns a single successful check result as a slice. +func ResultsOK(name, target, message string) []Result { + return Results(OK(name, target, message)) +} + +// ResultsWarning returns a single warning check result as a slice. When args +// are provided, message is formatted with fmt.Sprintf. +func ResultsWarning(name, target, message string, args ...any) []Result { + return Results(Warning(name, target, message, args...)) +} + +// ResultsError returns a single fatal check result as a slice. When args are +// provided, message is formatted with fmt.Sprintf. +func ResultsError(name, target, message string, args ...any) []Result { + return Results(Error(name, target, message, args...)) +} + +func formatMessage(message string, args ...any) string { + if len(args) == 0 { + return message + } + + return fmt.Sprintf(message, args...) +} + +// HasErrors reports whether any fatal errors remain after ignore handling. +func (r Report) HasErrors() bool { + return r.Summary.Errors > 0 +} + +// HasWarnings reports whether the report contains any warnings. +func (r Report) HasWarnings() bool { + return r.Summary.Warnings > 0 +} + +// Err converts the report status into a command error. +func (r Report) Err(failOnWarnings bool) error { + if r.HasErrors() { + return fmt.Errorf("preflight checks failed") + } + + if failOnWarnings && r.HasWarnings() { + return fmt.Errorf("preflight checks returned warnings") + } + + return nil +} + +func buildReport(results []Result, failOnWarnings bool) Report { + summary := Summary{} + + for _, result := range results { + switch result.Severity { + case SeverityError: + summary.Errors++ + case SeverityWarning: + summary.Warnings++ + default: + summary.OK++ + } + } + + status := "ok" + if summary.Errors > 0 || (failOnWarnings && summary.Warnings > 0) { + status = "failed" + } + + return Report{Status: status, Checks: results, Summary: summary} +} + +func ignoreSet(values []string) func(string) bool { + normalized := make([]string, 0, len(values)) + for _, value := range values { + for _, part := range strings.Split(value, ",") { + part = strings.ToLower(strings.TrimSpace(part)) + if part != "" { + normalized = append(normalized, part) + } + } + } + + return func(name string) bool { + name = strings.ToLower(strings.TrimSpace(name)) + return slices.Contains(normalized, "all") || slices.Contains(normalized, name) + } +} diff --git a/pkg/agent/preflight/preflight_test.go b/pkg/agent/preflight/preflight_test.go new file mode 100644 index 000000000..76dfbdd9f --- /dev/null +++ b/pkg/agent/preflight/preflight_test.go @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package preflight + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +type fakeChecker struct { + name string + results []Result +} + +func (f fakeChecker) Name() string { return f.name } + +func (f fakeChecker) Check(context.Context) []Result { return f.results } + +func TestRunIncludesAllResults(t *testing.T) { + report := Run(context.Background(), []Checker{ + fakeChecker{results: []Result{OK("agent-config", "agent config", "valid")}}, + fakeChecker{results: []Result{Warning("swap-active", "host swap", "enabled")}}, + fakeChecker{results: []Result{Error("host-packages", "host packages", "missing")}}, + }, Options{}) + + assert.Equal(t, "failed", report.Status) + assert.Equal(t, Summary{OK: 1, Warnings: 1, Errors: 1}, report.Summary) + assert.Len(t, report.Checks, 3) +} + +func TestFlatten(t *testing.T) { + first := fakeChecker{name: "first"} + second := fakeChecker{name: "second"} + third := fakeChecker{name: "third"} + + checks := Flatten([]Checker{first}, nil, []Checker{second, third}) + + assert.Equal(t, []Checker{first, second, third}, checks) +} + +func TestRunDowngradesIgnoredErrors(t *testing.T) { + report := Run(context.Background(), []Checker{ + fakeChecker{results: []Result{Error("host-packages", "host packages", "missing")}}, + }, Options{IgnoreErrors: []string{"host-packages"}}) + + assert.Equal(t, "ok", report.Status) + assert.Equal(t, Summary{Warnings: 1}, report.Summary) + assert.True(t, report.Checks[0].Ignored) + assert.Equal(t, SeverityWarning, report.Checks[0].Severity) +} + +func TestRunFailOnWarnings(t *testing.T) { + report := Run(context.Background(), []Checker{ + fakeChecker{results: []Result{Warning("swap-active", "host swap", "enabled")}}, + }, Options{FailOnWarnings: true}) + + assert.Equal(t, "failed", report.Status) + assert.Error(t, report.Err(true)) +} + +func TestRunIgnoreAll(t *testing.T) { + report := Run(context.Background(), []Checker{ + fakeChecker{results: []Result{Error("host-packages", "host packages", "missing")}}, + }, Options{IgnoreErrors: []string{"all"}}) + + assert.Equal(t, "ok", report.Status) + assert.True(t, report.Checks[0].Ignored) +} + +func TestFormattedWarningAndErrorMessages(t *testing.T) { + assert.Equal(t, "mode 700 is too restrictive", Warning("machine-dir", "machine directory", "mode %o is too restrictive", 0o700).Message) + assert.Equal(t, "status 500", Error("api-server", "cluster API server", "status %d", 500).Message) + assert.Equal(t, "path /var/lib/machines/kube1", ResultsWarning("machine-dir", "machine directory", "path %s", "/var/lib/machines/kube1")[0].Message) + assert.Equal(t, "path /var/lib/machines/kube1", ResultsError("machine-dir", "machine directory", "path %s", "/var/lib/machines/kube1")[0].Message) +} + +func TestRunPreservesInputOrderWhileRunningConcurrently(t *testing.T) { + release := make(chan struct{}) + started := make(chan string, 2) + + report := make(chan Report, 1) + + go func() { + report <- Run(context.Background(), []Checker{ + blockingChecker{name: "first", started: started, release: release}, + blockingChecker{name: "second", started: started, release: release}, + }, Options{}) + }() + + seen := map[string]bool{} + + for range 2 { + select { + case name := <-started: + seen[name] = true + case <-time.After(time.Second): + t.Fatal("checks did not start concurrently") + } + } + + assert.True(t, seen["first"]) + assert.True(t, seen["second"]) + + close(release) + + got := <-report + assert.Equal(t, "first", got.Checks[0].Name) + assert.Equal(t, "second", got.Checks[1].Name) +} + +type blockingChecker struct { + name string + started chan<- string + release <-chan struct{} +} + +func (b blockingChecker) Name() string { return b.name } + +func (b blockingChecker) Check(context.Context) []Result { + b.started <- b.name + + <-b.release + + return ResultsOK(b.name, b.name, b.name) +}