From 600c7faf0db43d2af9931d4e044906f6f58a7043 Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Sun, 1 Nov 2026 19:30:26 +0200 Subject: [PATCH 1/7] feat(nix): Add a flake that builds brig with its Linux runtime A host with nix had no route to a working sandbox: the nixpkgs brig package is the CLI alone, and the bundle wants a tarball under /var/lib/brig and systemd units it writes itself. The flake builds brig, urunc from the pinned commit and the guest assets, and wraps them with the same environment and config the bundle generates, so `nix build` plus one systemd unit boots a sandbox. Three choices are not obvious from the files. urunc is pinned to the commit pins.env records, not the one the README names, because brig asks for the hvi monitor on Linux and only that branch knows it. The shim's default OCI runtime is sed'd in the vendored go-runc, as the upstream Makefile does, because go-runc v1.0.0 makes it a const. virtiofsd and cloud-hypervisor come from the bundle tarball rather than nixpkgs, because urunc pivots into a monitor rootfs before it spawns them and the nixpkgs builds are dynamically linked. flake.lock is committed, pinning nixpkgs-unstable, because without it `nix build github:NOFireAI/brig-standalone-linux#brig-runtime` fails with "cannot write modified lock file". The README says to enable flakes, since the single-user installer leaves them off. Verified on Debian 12 with nested virtualization: doctor green, the ubuntu profile boots, guest DNS and apt work, the unit survives a reboot. Signed-off-by: Panagiotis Moustafellos --- flake.lock | 27 ++++++ flake.nix | 35 ++++++++ nix/README.md | 67 +++++++++++++++ nix/brig.nix | 86 +++++++++++++++++++ nix/guest.nix | 76 +++++++++++++++++ nix/runtime.nix | 218 ++++++++++++++++++++++++++++++++++++++++++++++++ nix/urunc.nix | 64 ++++++++++++++ 7 files changed, 573 insertions(+) create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 nix/README.md create mode 100644 nix/brig.nix create mode 100644 nix/guest.nix create mode 100644 nix/runtime.nix create mode 100644 nix/urunc.nix diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..83da613 --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1793582901, + "narHash": "sha256-WDValiY9se78X2Wh7aWWaq/c7HN29U555+OKbHEzYrw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "5c9de89ec55bff981cb8da1a4ee92b10f1438ffc", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..98f2360 --- /dev/null +++ b/flake.nix @@ -0,0 +1,35 @@ +{ + description = "Brig and the Linux microVM runtime it drives"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + + outputs = + { self, nixpkgs }: + let + systems = [ + "x86_64-linux" + "aarch64-linux" + ]; + forAll = f: nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system}); + in + { + packages = forAll ( + pkgs: + let + brig = pkgs.callPackage ./nix/brig.nix { }; + urunc = pkgs.callPackage ./nix/urunc.nix { }; + brig-guest = pkgs.callPackage ./nix/guest.nix { }; + brig-runtime = pkgs.callPackage ./nix/runtime.nix { inherit brig urunc brig-guest; }; + in + { + inherit + brig + urunc + brig-guest + brig-runtime + ; + default = brig-runtime; + } + ); + }; +} diff --git a/nix/README.md b/nix/README.md new file mode 100644 index 0000000..b1ac39b --- /dev/null +++ b/nix/README.md @@ -0,0 +1,67 @@ +# Brig with its Linux runtime, from Nix + +The flake at the root of this repository builds Brig together with the +runtime it drives on Linux, so one `nix build` gives a host everything a +sandbox needs. + +| Package | What it is | +| --- | --- | +| `brig` | `brig`, `brigd` and `brig-broker`, built from the Brig release. The same expression as `pkgs/by-name/br/brig` in nixpkgs. | +| `urunc` | `urunc` and `containerd-shim-urunc-v2`, built from the commit the bundle release pins. | +| `brig-guest` | The guest kernel, the initrd, the signed checksum files and the static monitors (`hvi`, `cloud-hypervisor`, `virtiofsd`), taken from the bundle release tarball. | +| `brig-runtime` | The default. `brig` wrapped with the environment the bundle's `brig-env.sh` sets, `brig-containerd` to run containerd with the bundle's config, and a systemd unit. containerd, nerdctl and the CNI plugins come from nixpkgs. | + +Only root drives this stack, as with the bundle's node-wide install. There +is no rootless path. + +## Install on a host with nix + +```sh +nix build github:NOFireAI/brig-standalone-linux#brig-runtime +sudo cp result/lib/systemd/system/brig-containerd.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now brig-containerd +sudo ./result/bin/brig doctor +sudo ./result/bin/brig run ubuntu ~/demo +``` + +`nix build` needs flakes on. A fresh single-user nix has them off; put +`experimental-features = nix-command flakes` in `~/.config/nix/nix.conf`. + +`nix profile install` on the same output puts `brig` on your PATH with its +shell completions. Run it under `sudo` with the full path, since `sudo` +resets PATH. + +The host needs `/dev/kvm` and the `vhost_vsock` module: + +```sh +sudo modprobe vhost_vsock +echo vhost_vsock | sudo tee /etc/modules-load.d/brig.conf +``` + +## Why some pieces are binaries + +Three things in `brig-guest` are not built here. The guest kernel comes from +`hull-assets`, whose repository is not public. The initrd is assembled by +`scripts/build-bundle.sh` from `urunit` and busybox. `hvi` is a static build +from `brig-sh/hull`. All three are taken from the bundle release tarball, +pinned by the sha256 its signed `checksums.txt` lists. + +`virtiofsd` and `cloud-hypervisor` are in nixpkgs, but the nixpkgs builds +are dynamically linked. `urunc` pivots into a monitor rootfs of its own +before it spawns them, and a dynamically linked binary cannot find its +interpreter in there. The bundle carries static builds, so those are used. + +## State and paths + +Same as the bundle's node-wide install: state under `/var/lib/brig/agent`, +the containerd socket at `/run/brig/containerd.sock`, the CNI bridge +`brig0` on `10.44.0.0/24`. + +## Tested + +Debian 12 on a GCP `n2-standard-4` with nested virtualization and +single-user nix, on 1 November 2026: `brig doctor` all `ok`, `brig run +ubuntu` booted, DNS and `apt-get update` worked in the guest, a file +written under `/work` appeared on the host, and the unit came back after a +reboot. diff --git a/nix/brig.nix b/nix/brig.nix new file mode 100644 index 0000000..0ca5457 --- /dev/null +++ b/nix/brig.nix @@ -0,0 +1,86 @@ +{ + lib, + stdenv, + buildGoModule, + fetchFromGitHub, + installShellFiles, + versionCheckHook, + nix-update-script, +}: + +buildGoModule (finalAttrs: { + pname = "brig"; + version = "0.5.0"; + + src = fetchFromGitHub { + owner = "brig-sh"; + repo = "brig"; + tag = "v${finalAttrs.version}"; + hash = "sha256-HPkr/fHTi8QHIR09PVTDMAPefdtV6K9kYXVLdtbPupY="; + }; + + vendorHash = "sha256-513twbhAd5XxpvGisWoCGwmIFtK0fypViOOBpMNjNEQ="; + + subPackages = [ + "cmd/brig" + "cmd/brigd" + "cmd/brig-broker" + ]; + + env.CGO_ENABLED = 0; + + # The binary reads the tag and commit the Go toolchain embeds from a git + # checkout. A tarball carries none, so the values are passed here, the way + # the upstream Makefile does for a worktree. Without them `brig version` + # prints "dev". + ldflags = [ + "-s" + "-w" + "-X github.com/brig-sh/brig/internal/buildinfo.gitCommit=8059e00a8a8741a94177aa19626886f63eb396e1" + "-X github.com/brig-sh/brig/internal/buildinfo.gitCommitTime=2026-10-31T12:08:40Z" + "-X github.com/brig-sh/brig/internal/buildinfo.gitDescribe=v${finalAttrs.version}-0-g8059e00" + ]; + + nativeBuildInputs = [ installShellFiles ]; + + # The tests are written for a source checkout: one of them expects a test + # binary built from a tarball to carry no commit. The stamped values above + # are for the shipped binaries only. + preCheck = '' + ldflags="-s -w" + ''; + + postInstall = lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' + installShellCompletion --cmd brig \ + --bash <($out/bin/brig completion bash) \ + --fish <($out/bin/brig completion fish) \ + --zsh <($out/bin/brig completion zsh) + ''; + + # On darwin the secret store tests write to the real login keychain. + doCheck = !stdenv.hostPlatform.isDarwin; + + nativeInstallCheckInputs = [ versionCheckHook ]; + versionCheckProgramArg = "version"; + doInstallCheck = true; + + passthru.updateScript = nix-update-script { }; + + meta = { + description = "Run coding agents in a microVM sandbox"; + longDescription = '' + Brig boots a coding agent in a microVM and hands it a project directory + and named credentials, nothing else. This package holds the brig CLI, + the brigd session daemon and the brig-broker credential broker. It does + not include a runtime: on Linux Brig drives nerdctl with the urunc shim, + on macOS the hull runtime. `brig doctor` reports what is missing, and + https://brig.sh/docs/install/ says how to install the runtime. + ''; + homepage = "https://brig.sh"; + changelog = "https://github.com/brig-sh/brig/releases/tag/v${finalAttrs.version}"; + license = lib.licenses.asl20; + mainProgram = "brig"; + maintainers = with lib.maintainers; [ pmoust ]; + platforms = lib.platforms.unix; + }; +}) diff --git a/nix/guest.nix b/nix/guest.nix new file mode 100644 index 0000000..b208f05 --- /dev/null +++ b/nix/guest.nix @@ -0,0 +1,76 @@ +{ + lib, + stdenvNoCC, + fetchurl, +}: + +# The guest kernel, the container-initrd and the monitors, taken from the +# Linux runtime bundle release. The kernel is built by hull-assets, whose +# repository is not public, the initrd is assembled by scripts/build-bundle.sh +# from urunit and busybox, and hvi is a static binary from brig-sh/hull. Until +# those build here, the bundle tarball is the source, pinned by the sha256 its +# signed checksums.txt lists. +# +# virtiofsd and cloud-hypervisor come from the same tarball although nixpkgs +# has both: urunc pivots into a monitor rootfs of its own before it spawns +# them, and a dynamically linked binary cannot find its interpreter in there. +# The bundle's builds are static. +# +# brig checks the kernel and initrd against share/guest/SHA256SUMS, and that +# file against the release's checksums.txt, whose cosign signature and +# certificate sit beside it, the way the bundle's installer leaves them. +let + version = "0.1.0-rc21"; + base = "https://github.com/NOFireAI/brig-standalone-linux/releases/download/v${version}"; + release = + name: hash: + fetchurl { + url = "${base}/${name}"; + sha256 = hash; + }; + checksums = release "checksums.txt" "441bc9aaded057ed301b9e7bec265c4d75d2c418f32fc75a45dd841736ce67fa"; + signature = release "checksums.txt.sig" "bcfb6a50348c2f3a760419c5e2ad290bbe816ee57f77c2653543d7c1b1e83dc6"; + certificate = release "checksums.txt.pem" "d2cd0ff6918724d50dbe22d915a3dd9ee5cd3b26a26d4b0d52cdbf950bf84744"; + arch = + stdenvNoCC.hostPlatform.go.GOARCH + or (if stdenvNoCC.hostPlatform.isAarch64 then "arm64" else "amd64"); + hashes = { + amd64 = "3a376bb1da931c876d512643a205fd6ccefc6a8f6f696bdc6c548a477ef3205f"; + arm64 = "bcf280444a78adca58c5e89336edc6c55893f3da6416bc9bc2a103ad0e4c9b36"; + }; +in +stdenvNoCC.mkDerivation { + pname = "brig-guest"; + inherit version; + + src = fetchurl { + url = "${base}/brig-standalone-v${version}-linux-${arch}.tar.gz"; + sha256 = hashes.${arch}; + }; + + dontBuild = true; + dontFixup = true; + + installPhase = '' + runHook preInstall + mkdir -p $out/share $out/bin + cp -r share/guest $out/share/guest + cp ${checksums} $out/share/guest/checksums.txt + cp ${signature} $out/share/guest/checksums.txt.sig + cp ${certificate} $out/share/guest/checksums.txt.pem + install -m 0755 bin/hvi bin/cloud-hypervisor $out/bin/ + install -m 0755 libexec/virtiofsd $out/bin/virtiofsd + runHook postInstall + ''; + + meta = { + description = "Guest kernel, initrd and monitors for Brig sandboxes on Linux"; + homepage = "https://github.com/NOFireAI/brig-standalone-linux"; + sourceProvenance = [ lib.sourceTypes.binaryNativeCode ]; + license = lib.licenses.asl20; + platforms = [ + "x86_64-linux" + "aarch64-linux" + ]; + }; +} diff --git a/nix/runtime.nix b/nix/runtime.nix new file mode 100644 index 0000000..e417fe9 --- /dev/null +++ b/nix/runtime.nix @@ -0,0 +1,218 @@ +{ + lib, + stdenv, + runCommand, + makeWrapper, + brig, + urunc, + brig-guest, + containerd, + nerdctl, + cni-plugins, + iptables, + nftables, + util-linux, + cosign, + oras, +}: + +# The Linux runtime around brig: containerd with the urunc shim, the CNI +# bridge, the monitors and the guest assets, wired the way the runtime bundle +# wires them. State lives under /var/lib/brig/agent and the socket under +# /run/brig, so a host that had the bundle and one that has this package +# look the same to brig. Only root drives this stack; there is no rootless +# path here. +let + stateDir = "/var/lib/brig/agent"; + runDir = "/run/brig"; + sock = "${runDir}/containerd.sock"; + goarch = if stdenv.hostPlatform.isAarch64 then "arm64" else "amd64"; + + etc = runCommand "brig-runtime-etc" { } '' + mkdir -p $out/etc/cni/net.d $out/etc/certs.d + + cat > $out/etc/containerd.toml < $out/etc/nerdctl.toml < $out/etc/urunc.toml < $out/etc/cni/net.d/10-brig-bridge.conflist < $out/bin/brig-containerd < $out/lib/systemd/system/brig-containerd.service < Date: Sun, 1 Nov 2026 20:00:01 +0200 Subject: [PATCH 2/7] feat(nix): Add a NixOS module and a VM test for the runtime On NixOS the Debian recipe, copying the unit from the store by hand, is the wrong shape. The module declares the unit, the kernel modules and the state directories, and puts brig on PATH, so services.brig.enable is the whole install. The VM test boots a NixOS machine from the module alone and checks the unit, the socket, /dev/vhost-vsock and brig doctor. The test machine has no network, so it pulls no image and boots no sandbox; a real boot is still a manual check on a host with KVM. Passed on a Debian host with nested virtualization. Signed-off-by: Panagiotis Moustafellos --- flake.nix | 15 ++++++++++++ nix/README.md | 28 +++++++++++++++++++++++ nix/module.nix | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++ nix/test.nix | 34 +++++++++++++++++++++++++++ 4 files changed, 139 insertions(+) create mode 100644 nix/module.nix create mode 100644 nix/test.nix diff --git a/flake.nix b/flake.nix index 98f2360..b8ecae2 100644 --- a/flake.nix +++ b/flake.nix @@ -31,5 +31,20 @@ default = brig-runtime; } ); + + nixosModules.default = import ./nix/module.nix; + + checks = forAll ( + pkgs: + let + brig-runtime = self.packages.${pkgs.system}.brig-runtime; + in + { + module = pkgs.callPackage ./nix/test.nix { + module = self.nixosModules.default; + inherit brig-runtime; + }; + } + ); }; } diff --git a/nix/README.md b/nix/README.md index b1ac39b..e09e021 100644 --- a/nix/README.md +++ b/nix/README.md @@ -39,6 +39,34 @@ sudo modprobe vhost_vsock echo vhost_vsock | sudo tee /etc/modules-load.d/brig.conf ``` +## NixOS + +The flake exports a module. With it, the unit, the kernel modules and the +directories are declared, and `brig` is on PATH: + +```nix +{ + inputs.brig-runtime.url = "github:NOFireAI/brig-standalone-linux"; + outputs = { nixpkgs, brig-runtime, ... }: { + nixosConfigurations.host = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ + brig-runtime.nixosModules.default + { + services.brig.enable = true; + services.brig.package = brig-runtime.packages.x86_64-linux.brig-runtime; + } + ]; + }; + }; +} +``` + +`nix flake check` runs a NixOS VM test of the module: the unit comes up, +the socket appears, `brig doctor` finds the runtime and the boot assets. +The test VM has no network, so it boots no sandbox. It needs KVM on the +host that runs it. + ## Why some pieces are binaries Three things in `brig-guest` are not built here. The guest kernel comes from diff --git a/nix/module.nix b/nix/module.nix new file mode 100644 index 0000000..3345973 --- /dev/null +++ b/nix/module.nix @@ -0,0 +1,62 @@ +# NixOS module: services.brig. Runs the containerd that brig drives, loads +# the kernel modules a sandbox needs and puts brig on PATH. The same layout +# as the bundle's node-wide install, so a host that had the bundle and one +# that has this module look the same to brig. Only root drives the stack. +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.services.brig; +in +{ + options.services.brig = { + enable = lib.mkEnableOption "the brig sandbox runtime"; + + package = lib.mkOption { + type = lib.types.package; + description = "The brig-runtime package to run and install."; + }; + }; + + config = lib.mkIf cfg.enable { + environment.systemPackages = [ cfg.package ]; + + boot.kernelModules = [ + "kvm" + "vhost_vsock" + ]; + + # containerd's state and the socket directory, created before the unit + # starts. brig-containerd creates them too; this keeps them across the + # unit's absence. + systemd.tmpfiles.rules = [ + "d /var/lib/brig/agent 0755 root root -" + "d /run/brig 0710 root root -" + ]; + + systemd.services.brig-containerd = { + description = "brig containerd"; + documentation = [ "https://github.com/brig-sh/brig" ]; + after = [ + "network.target" + "local-fs.target" + ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "notify"; + Delegate = true; + KillMode = "process"; + Restart = "always"; + RestartSec = 5; + ExecStart = "${cfg.package}/bin/brig-containerd"; + LimitNPROC = "infinity"; + LimitCORE = "infinity"; + TasksMax = "infinity"; + OOMScoreAdjust = -999; + }; + }; + }; +} diff --git a/nix/test.nix b/nix/test.nix new file mode 100644 index 0000000..69642a4 --- /dev/null +++ b/nix/test.nix @@ -0,0 +1,34 @@ +# NixOS VM test of the module. The test VM has no network, so no image can +# be pulled and no sandbox boots here. What it checks: the unit comes up, +# the socket appears, and `brig doctor` finds the runtime and the boot +# assets on a host built from the module alone. +{ + testers, + module, + brig-runtime, +}: +testers.nixosTest { + name = "brig-runtime"; + + nodes.machine = { + imports = [ module ]; + services.brig = { + enable = true; + package = brig-runtime; + }; + virtualisation.memorySize = 2048; + virtualisation.diskSize = 4096; + }; + + testScript = '' + machine.wait_for_unit("brig-containerd.service") + machine.wait_for_file("/run/brig/containerd.sock") + machine.succeed("test -c /dev/vhost-vsock") + out = machine.succeed("brig doctor || true") + print(out) + assert "ok runtime" in out, "doctor did not find nerdctl" + assert "ok boot" in out, "doctor did not find the boot assets" + assert "ok rootfs" in out, "doctor found no overlay root" + machine.succeed("brig version | grep -q 0.5.0") + ''; +} From e1b21b6081edb5f139df5242a15ed456234b1d3e Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Mon, 2 Nov 2026 12:50:31 +0200 Subject: [PATCH 3/7] fix(nix): Make the runtime boot a sandbox on a NixOS host The Debian run passed for reasons that do not hold on NixOS: root can write into a single-user /nix/store, and /usr/sbin and /lib exist. On a NixOS machine built from the module, a sandbox stopped three times before it booted. nerdctl writes a lock and one conflist per sandbox network into the CNI config directory, so it cannot be the store. It is now a state directory that brig-containerd seeds from the store on start. On the Debian test host the old package had in fact written those files into three store paths; the store was collected and the package rebuilt from scratch with the same output hash, and nothing is written to the store now. urunc builds the overlay-block rootfs with /usr/sbin/mke2fs by default. urunc.toml now names the nixpkgs mke2fs. urunc binds /lib, /lib64 and /usr/lib from the host into the monitor's rootfs. NixOS has no /lib or /usr/lib, and the monitors here are static. A patch skips a directory that is not there, as upstream already does for /lib64. The flake gains a demo NixOS machine with network, which is where this was found and where a sandbox now boots: image pulled, guest agent up, brig sh runs commands in the guest, a file written under /work appears on the host. The VM test under checks still boots no sandbox, since it has no network. The README records the runs on a fresh Debian 12 host and on the NixOS demo machine, and the two things that make a scripted first run hang: the telemetry consent prompt and brig run without -d. Signed-off-by: Panagiotis Moustafellos --- flake.nix | 35 +++++++++++++++++++++++++ nix/README.md | 45 +++++++++++++++++++++++++++++--- nix/runtime.nix | 15 +++++++++-- nix/urunc-optional-libdirs.patch | 27 +++++++++++++++++++ nix/urunc.nix | 6 +++++ 5 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 nix/urunc-optional-libdirs.patch diff --git a/flake.nix b/flake.nix index b8ecae2..a406d9c 100644 --- a/flake.nix +++ b/flake.nix @@ -34,6 +34,41 @@ nixosModules.default = import ./nix/module.nix; + # A NixOS machine with the module on, for a run with network: + # nix build .#nixosConfigurations.demo.config.system.build.vm + # then ./result/bin/run-demo-vm. The VM test under checks has no + # network, so this is where a sandbox boots on a NixOS host. + nixosConfigurations.demo = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ + self.nixosModules.default + ( + { pkgs, ... }: + { + services.brig.enable = true; + services.brig.package = self.packages.x86_64-linux.brig-runtime; + virtualisation.vmVariant.virtualisation = { + memorySize = 4096; + cores = 2; + diskSize = 8192; + graphics = false; + forwardPorts = [ + { + from = "host"; + host.port = 2222; + guest.port = 22; + } + ]; + }; + services.openssh.enable = true; + services.openssh.settings.PermitRootLogin = "yes"; + users.users.root.initialPassword = "demo"; + system.stateVersion = "25.11"; + } + ) + ]; + }; + checks = forAll ( pkgs: let diff --git a/nix/README.md b/nix/README.md index e09e021..e947496 100644 --- a/nix/README.md +++ b/nix/README.md @@ -67,6 +67,26 @@ the socket appears, `brig doctor` finds the runtime and the boot assets. The test VM has no network, so it boots no sandbox. It needs KVM on the host that runs it. +For a sandbox boot on NixOS with network, the flake has a demo machine: + +```sh +nix build .#nixosConfigurations.demo.config.system.build.vm +./result/bin/run-nixos-vm # root, password demo, ssh on port 2222 +``` + +Three things only showed up on NixOS, because a single-user nix store on +Debian is writable by root and `/usr/sbin` and `/lib` exist there: + +- nerdctl writes a lock and one conflist per sandbox network into the CNI + config directory, so that directory is `/var/lib/brig/agent/cni/net.d`, + seeded from the store when `brig-containerd` starts. +- urunc runs `/usr/sbin/mke2fs` by default to build the overlay-block + rootfs. `urunc.toml` points it at the nixpkgs `mke2fs`. +- urunc binds `/lib`, `/lib64` and `/usr/lib` into the monitor's rootfs for + a dynamically linked monitor. NixOS has no `/lib` or `/usr/lib`, and the + monitors here are static. `nix/urunc-optional-libdirs.patch` skips a + directory that is not there, as upstream already does for `/lib64`. + ## Why some pieces are binaries Three things in `brig-guest` are not built here. The guest kernel comes from @@ -89,7 +109,24 @@ the containerd socket at `/run/brig/containerd.sock`, the CNI bridge ## Tested Debian 12 on a GCP `n2-standard-4` with nested virtualization and -single-user nix, on 1 November 2026: `brig doctor` all `ok`, `brig run -ubuntu` booted, DNS and `apt-get update` worked in the guest, a file -written under `/work` appeared on the host, and the unit came back after a -reboot. +single-user nix, on 2 November 2026, following the steps above on a host +with nothing else on it: `nix build` from this repository took 2 minutes +(binaries from cache.nixos.org, `brig` and `urunc` built from source), +`brig doctor` all `ok`, a sandbox booted in 14 seconds on first pull, +`apt-get update` ran in the guest, a file written under `/work/demo` +appeared on the host, nothing was written under `/nix/store`, `nix store +verify` passed on the closure, and the unit, `vhost_vsock` and a new +sandbox came back after a reboot. + +NixOS 26.11, the demo machine above, run as a qemu VM on a Debian host +on 2 November 2026, so the sandbox was a third level of virtualization: +`brig doctor` all `ok`, `brig run ubuntu` pulled the image and booted, the +guest agent answered within seconds, `brig sh` ran commands in the guest in +about a second, DNS resolved, a file written under `/work` appeared on the +host. + +On a scripted host, run `brig telemetry off` or `on` once first. The +first `brig run` on a tty otherwise waits on the consent prompt, and +`brig run` without `-d` stays attached. + +Not tested: aarch64, a NixOS host on real hardware. diff --git a/nix/runtime.nix b/nix/runtime.nix index e417fe9..2d3461c 100644 --- a/nix/runtime.nix +++ b/nix/runtime.nix @@ -14,6 +14,7 @@ util-linux, cosign, oras, + e2fsprogs, }: # The Linux runtime around brig: containerd with the urunc shim, the CNI @@ -59,7 +60,7 @@ let namespace = "brig" snapshotter = "overlayfs" cni_path = "${cni-plugins}/bin" - cni_netconfpath = "$out/etc/cni/net.d" + cni_netconfpath = "${stateDir}/cni/net.d" data_root = "${stateDir}/nerdctl" cgroup_manager = "systemd" hosts_dir = ["$out/etc/certs.d"] @@ -90,6 +91,11 @@ let [extra_binaries.virtiofsd] path = "${brig-guest}/bin/virtiofsd" options = "--cache always --sandbox none" + + # urunc builds the overlay-block rootfs with mke2fs before it pivots, + # and defaults to /usr/sbin/mke2fs, which a NixOS host does not have. + [extra_binaries.mke2fs] + path = "${e2fsprogs}/bin/mke2fs" EOF cat > $out/etc/cni/net.d/10-brig-bridge.conflist <= 4 && monitorName[:4] == "qemu" { diff --git a/nix/urunc.nix b/nix/urunc.nix index fcaa441..ab79388 100644 --- a/nix/urunc.nix +++ b/nix/urunc.nix @@ -21,6 +21,12 @@ buildGoModule (finalAttrs: { vendorHash = "sha256-08PC8ethomYCZTBfP1PVCCXgXeyrSHHoozcVsWmqJUE="; + # urunc binds /lib, /lib64 and /usr/lib from the host into the monitor's + # rootfs for a dynamically linked monitor. NixOS has no /lib or /usr/lib, + # and the monitors here are static. The patch skips a directory that is + # not there, as upstream already does for /lib64. + patches = [ ./urunc-optional-libdirs.patch ]; + subPackages = [ "cmd/urunc" "cmd/containerd-shim-urunc-v2" From 7d50dbe0c84f93a1a634a17f41b302a3442b5341 Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Mon, 2 Nov 2026 18:35:14 +0200 Subject: [PATCH 4/7] fix(nix): Share the unit, wrap nerdctl and refresh the bridge config Review of the first round found four things. The module declared its own copy of brig-containerd.service, so it could drift from the one the package ships; it now takes the unit from the package through systemd.packages. nerdctl was a bare symlink, so a hand-run nerdctl talked to /run/containerd and not to brig's instance; it is wrapped with the same defaults as brig. The bridge conflist was copied into the state directory once, so a new package never updated it; it is copied on every start. pkgs.system is deprecated in favour of pkgs.stdenv.hostPlatform.system. The top-level README points at nix/README.md. Tested on Debian 12: the module VM test passes, a hand-run nerdctl ps shows the sandbox, an edit to the state conflist is gone after a restart, a sandbox boots and execs, nothing is written to the store. Signed-off-by: Panagiotis Moustafellos --- README.md | 3 +++ flake.nix | 2 +- nix/module.nix | 26 +++++--------------------- nix/runtime.nix | 18 +++++++++++++----- 4 files changed, 22 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index c07a43f..67df06d 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,9 @@ list is under [What it touches outside the base directories](#what-it-touches-ou ## Install +On a host with nix, or on NixOS, see [nix/README.md](nix/README.md): the +flake builds brig with this runtime, and a module declares the service. + ```console # curl -fsSL https://raw.githubusercontent.com/NOFireAI/brig-standalone-linux/main/install.sh | sh - ``` diff --git a/flake.nix b/flake.nix index a406d9c..6b78bcc 100644 --- a/flake.nix +++ b/flake.nix @@ -72,7 +72,7 @@ checks = forAll ( pkgs: let - brig-runtime = self.packages.${pkgs.system}.brig-runtime; + brig-runtime = self.packages.${pkgs.stdenv.hostPlatform.system}.brig-runtime; in { module = pkgs.callPackage ./nix/test.nix { diff --git a/nix/module.nix b/nix/module.nix index 3345973..77378b1 100644 --- a/nix/module.nix +++ b/nix/module.nix @@ -37,26 +37,10 @@ in "d /run/brig 0710 root root -" ]; - systemd.services.brig-containerd = { - description = "brig containerd"; - documentation = [ "https://github.com/brig-sh/brig" ]; - after = [ - "network.target" - "local-fs.target" - ]; - wantedBy = [ "multi-user.target" ]; - serviceConfig = { - Type = "notify"; - Delegate = true; - KillMode = "process"; - Restart = "always"; - RestartSec = 5; - ExecStart = "${cfg.package}/bin/brig-containerd"; - LimitNPROC = "infinity"; - LimitCORE = "infinity"; - TasksMax = "infinity"; - OOMScoreAdjust = -999; - }; - }; + # The unit comes from the package, so this module and a host that copies + # the file by hand run the same definition. NixOS ignores [Install] in a + # packaged unit, so the target is named here. + systemd.packages = [ cfg.package ]; + systemd.services.brig-containerd.wantedBy = [ "multi-user.target" ]; }; } diff --git a/nix/runtime.nix b/nix/runtime.nix index 2d3461c..95718e1 100644 --- a/nix/runtime.nix +++ b/nix/runtime.nix @@ -183,7 +183,15 @@ runCommand "brig-runtime" --prefix PATH : "${clientPath}" done ln -s ${brig}/bin/brig-broker $out/bin/brig-broker - ln -s ${nerdctl}/bin/nerdctl $out/bin/nerdctl + # nerdctl with the same defaults, so a hand-run `nerdctl ps` talks to + # brig's containerd and not to /run/containerd. + makeWrapper ${nerdctl}/bin/nerdctl $out/bin/nerdctl \ + --set-default CONTAINERD_ADDRESS "unix://${sock}" \ + --set-default CONTAINERD_NAMESPACE brig \ + --set-default CONTAINERD_SNAPSHOTTER overlayfs \ + --set-default NERDCTL_TOML "${etc}/etc/nerdctl.toml" \ + --set-default CNI_PATH "${cni-plugins}/bin" \ + --prefix PATH : "${clientPath}" cat > $out/bin/brig-containerd < Date: Mon, 2 Nov 2026 18:54:55 +0200 Subject: [PATCH 5/7] refactor(nix): Drop the demo NixOS machine The flake carried a whole NixOS configuration whose only job was to be booted by hand once. It is not something a user of this flake wants, and it broke `nix flake check`: a bare machine config has no root filesystem and no bootloader, so checking it failed the assertions before the module test ran. The README says how to add a VM variant to the configuration it already shows, which is the same thing in four lines on the reader's side. Signed-off-by: Panagiotis Moustafellos --- flake.nix | 35 ----------------------------------- nix/README.md | 21 ++++++++++++++++----- 2 files changed, 16 insertions(+), 40 deletions(-) diff --git a/flake.nix b/flake.nix index 6b78bcc..2e7f64a 100644 --- a/flake.nix +++ b/flake.nix @@ -34,41 +34,6 @@ nixosModules.default = import ./nix/module.nix; - # A NixOS machine with the module on, for a run with network: - # nix build .#nixosConfigurations.demo.config.system.build.vm - # then ./result/bin/run-demo-vm. The VM test under checks has no - # network, so this is where a sandbox boots on a NixOS host. - nixosConfigurations.demo = nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; - modules = [ - self.nixosModules.default - ( - { pkgs, ... }: - { - services.brig.enable = true; - services.brig.package = self.packages.x86_64-linux.brig-runtime; - virtualisation.vmVariant.virtualisation = { - memorySize = 4096; - cores = 2; - diskSize = 8192; - graphics = false; - forwardPorts = [ - { - from = "host"; - host.port = 2222; - guest.port = 22; - } - ]; - }; - services.openssh.enable = true; - services.openssh.settings.PermitRootLogin = "yes"; - users.users.root.initialPassword = "demo"; - system.stateVersion = "25.11"; - } - ) - ]; - }; - checks = forAll ( pkgs: let diff --git a/nix/README.md b/nix/README.md index e947496..6a7f0ab 100644 --- a/nix/README.md +++ b/nix/README.md @@ -67,11 +67,21 @@ the socket appears, `brig doctor` finds the runtime and the boot assets. The test VM has no network, so it boots no sandbox. It needs KVM on the host that runs it. -For a sandbox boot on NixOS with network, the flake has a demo machine: +A sandbox needs network, which the test VM has not. To boot one on NixOS, +give the configuration above a VM variant and build that: + +```nix +virtualisation.vmVariant.virtualisation = { + memorySize = 4096; + cores = 2; + diskSize = 8192; + graphics = false; +}; +``` ```sh -nix build .#nixosConfigurations.demo.config.system.build.vm -./result/bin/run-nixos-vm # root, password demo, ssh on port 2222 +nix build .#nixosConfigurations.host.config.system.build.vm +./result/bin/run-nixos-vm ``` Three things only showed up on NixOS, because a single-user nix store on @@ -118,8 +128,9 @@ appeared on the host, nothing was written under `/nix/store`, `nix store verify` passed on the closure, and the unit, `vhost_vsock` and a new sandbox came back after a reboot. -NixOS 26.11, the demo machine above, run as a qemu VM on a Debian host -on 2 November 2026, so the sandbox was a third level of virtualization: +NixOS 26.11, built from the module with user-mode network, run as a qemu +VM on a Debian host on 2 November 2026, so the sandbox was a third level +of virtualization: `brig doctor` all `ok`, `brig run ubuntu` pulled the image and booted, the guest agent answered within seconds, `brig sh` ran commands in the guest in about a second, DNS resolved, a file written under `/work` appeared on the From 73f7c38138b808e6f905d3045f3275d58d3c0a0f Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Mon, 2 Nov 2026 18:54:55 +0200 Subject: [PATCH 6/7] docs(nix): Note the version drift from the bundle The bundle pins containerd and nerdctl in build-bundle.sh; here they come from the nixpkgs that flake.lock pins, so the versions differ and move with nix flake update. Say which versions that is today, and that the -info flag error containerd logs against the shim is harmless. Also correct two lines that no longer hold: brig-containerd copies the bridge config on every start rather than seeding it once, and the module VM test hangs on single-user nix unless the build user can open /dev/kvm and nix.conf names the kvm system feature. Signed-off-by: Panagiotis Moustafellos --- nix/README.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/nix/README.md b/nix/README.md index 6a7f0ab..d0b798e 100644 --- a/nix/README.md +++ b/nix/README.md @@ -14,6 +14,13 @@ sandbox needs. Only root drives this stack, as with the bundle's node-wide install. There is no rootless path. +containerd, nerdctl and the CNI plugins come from the nixpkgs that +`flake.lock` pins, so their versions are not the ones +`scripts/build-bundle.sh` pins for the bundle. At this lock, nix gives +containerd and nerdctl 2.4.1 where the bundle carries 2.3.5, and `nix +flake update` moves them. containerd 2.4.1 logs a harmless `-info` flag +error against the 1.7-era shim. + ## Install on a host with nix ```sh @@ -65,7 +72,9 @@ directories are declared, and `brig` is on PATH: `nix flake check` runs a NixOS VM test of the module: the unit comes up, the socket appears, `brig doctor` finds the runtime and the boot assets. The test VM has no network, so it boots no sandbox. It needs KVM on the -host that runs it. +host that runs it, and on single-user nix it hangs with no output unless +the build user can open `/dev/kvm` and `~/.config/nix/nix.conf` has +`system-features = nixos-test kvm`. A sandbox needs network, which the test VM has not. To boot one on NixOS, give the configuration above a VM variant and build that: @@ -88,8 +97,9 @@ Three things only showed up on NixOS, because a single-user nix store on Debian is writable by root and `/usr/sbin` and `/lib` exist there: - nerdctl writes a lock and one conflist per sandbox network into the CNI - config directory, so that directory is `/var/lib/brig/agent/cni/net.d`, - seeded from the store when `brig-containerd` starts. + config directory, so that directory is `/var/lib/brig/agent/cni/net.d`. + `brig-containerd` copies the bridge config there from the store on every + start. - urunc runs `/usr/sbin/mke2fs` by default to build the overlay-block rootfs. `urunc.toml` points it at the nixpkgs `mke2fs`. - urunc binds `/lib`, `/lib64` and `/usr/lib` into the monitor's rootfs for From e045ee2392185a7167307b3192a72ad8f10c343d Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Mon, 2 Nov 2026 19:04:52 +0200 Subject: [PATCH 7/7] docs(nix): Give the NixOS example a stateVersion and say why rootless The NixOS configuration in the README had no system.stateVersion, which a real machine config needs, and the VM variant below it was checked with one. Keep the text and what was built the same. The rootless note said only that there is no rootless path. Say why that matters: a node-wide install reaches the host directories a sandbox shares as root, and point at the bundle's rootless mode. Signed-off-by: Panagiotis Moustafellos --- nix/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nix/README.md b/nix/README.md index d0b798e..8f90401 100644 --- a/nix/README.md +++ b/nix/README.md @@ -12,7 +12,10 @@ sandbox needs. | `brig-runtime` | The default. `brig` wrapped with the environment the bundle's `brig-env.sh` sets, `brig-containerd` to run containerd with the bundle's config, and a systemd unit. containerd, nerdctl and the CNI plugins come from nixpkgs. | Only root drives this stack, as with the bundle's node-wide install. There -is no rootless path. +is no rootless path here yet, and rootless is the safer shape: the host +directories a sandbox shares are then reached with a user's own +privileges and not root's. The bundle's `--rootless` mode and +[docs/rootless.md](../docs/rootless.md) describe it. containerd, nerdctl and the CNI plugins come from the nixpkgs that `flake.lock` pins, so their versions are not the ones @@ -62,6 +65,7 @@ directories are declared, and `brig` is on PATH: { services.brig.enable = true; services.brig.package = brig-runtime.packages.x86_64-linux.brig-runtime; + system.stateVersion = "25.11"; } ]; };