diff --git a/README.md b/README.md index c07a43f..67df06d 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,9 @@ list is under [What it touches outside the base directories](#what-it-touches-ou ## Install +On a host with nix, or on NixOS, see [nix/README.md](nix/README.md): the +flake builds brig with this runtime, and a module declares the service. + ```console # curl -fsSL https://raw.githubusercontent.com/NOFireAI/brig-standalone-linux/main/install.sh | sh - ``` diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..83da613 --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1793582901, + "narHash": "sha256-WDValiY9se78X2Wh7aWWaq/c7HN29U555+OKbHEzYrw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "5c9de89ec55bff981cb8da1a4ee92b10f1438ffc", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..2e7f64a --- /dev/null +++ b/flake.nix @@ -0,0 +1,50 @@ +{ + description = "Brig and the Linux microVM runtime it drives"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + + outputs = + { self, nixpkgs }: + let + systems = [ + "x86_64-linux" + "aarch64-linux" + ]; + forAll = f: nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system}); + in + { + packages = forAll ( + pkgs: + let + brig = pkgs.callPackage ./nix/brig.nix { }; + urunc = pkgs.callPackage ./nix/urunc.nix { }; + brig-guest = pkgs.callPackage ./nix/guest.nix { }; + brig-runtime = pkgs.callPackage ./nix/runtime.nix { inherit brig urunc brig-guest; }; + in + { + inherit + brig + urunc + brig-guest + brig-runtime + ; + default = brig-runtime; + } + ); + + nixosModules.default = import ./nix/module.nix; + + checks = forAll ( + pkgs: + let + brig-runtime = self.packages.${pkgs.stdenv.hostPlatform.system}.brig-runtime; + in + { + module = pkgs.callPackage ./nix/test.nix { + module = self.nixosModules.default; + inherit brig-runtime; + }; + } + ); + }; +} diff --git a/nix/README.md b/nix/README.md new file mode 100644 index 0000000..8f90401 --- /dev/null +++ b/nix/README.md @@ -0,0 +1,157 @@ +# Brig with its Linux runtime, from Nix + +The flake at the root of this repository builds Brig together with the +runtime it drives on Linux, so one `nix build` gives a host everything a +sandbox needs. + +| Package | What it is | +| --- | --- | +| `brig` | `brig`, `brigd` and `brig-broker`, built from the Brig release. The same expression as `pkgs/by-name/br/brig` in nixpkgs. | +| `urunc` | `urunc` and `containerd-shim-urunc-v2`, built from the commit the bundle release pins. | +| `brig-guest` | The guest kernel, the initrd, the signed checksum files and the static monitors (`hvi`, `cloud-hypervisor`, `virtiofsd`), taken from the bundle release tarball. | +| `brig-runtime` | The default. `brig` wrapped with the environment the bundle's `brig-env.sh` sets, `brig-containerd` to run containerd with the bundle's config, and a systemd unit. containerd, nerdctl and the CNI plugins come from nixpkgs. | + +Only root drives this stack, as with the bundle's node-wide install. There +is no rootless path here yet, and rootless is the safer shape: the host +directories a sandbox shares are then reached with a user's own +privileges and not root's. The bundle's `--rootless` mode and +[docs/rootless.md](../docs/rootless.md) describe it. + +containerd, nerdctl and the CNI plugins come from the nixpkgs that +`flake.lock` pins, so their versions are not the ones +`scripts/build-bundle.sh` pins for the bundle. At this lock, nix gives +containerd and nerdctl 2.4.1 where the bundle carries 2.3.5, and `nix +flake update` moves them. containerd 2.4.1 logs a harmless `-info` flag +error against the 1.7-era shim. + +## Install on a host with nix + +```sh +nix build github:NOFireAI/brig-standalone-linux#brig-runtime +sudo cp result/lib/systemd/system/brig-containerd.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now brig-containerd +sudo ./result/bin/brig doctor +sudo ./result/bin/brig run ubuntu ~/demo +``` + +`nix build` needs flakes on. A fresh single-user nix has them off; put +`experimental-features = nix-command flakes` in `~/.config/nix/nix.conf`. + +`nix profile install` on the same output puts `brig` on your PATH with its +shell completions. Run it under `sudo` with the full path, since `sudo` +resets PATH. + +The host needs `/dev/kvm` and the `vhost_vsock` module: + +```sh +sudo modprobe vhost_vsock +echo vhost_vsock | sudo tee /etc/modules-load.d/brig.conf +``` + +## NixOS + +The flake exports a module. With it, the unit, the kernel modules and the +directories are declared, and `brig` is on PATH: + +```nix +{ + inputs.brig-runtime.url = "github:NOFireAI/brig-standalone-linux"; + outputs = { nixpkgs, brig-runtime, ... }: { + nixosConfigurations.host = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ + brig-runtime.nixosModules.default + { + services.brig.enable = true; + services.brig.package = brig-runtime.packages.x86_64-linux.brig-runtime; + system.stateVersion = "25.11"; + } + ]; + }; + }; +} +``` + +`nix flake check` runs a NixOS VM test of the module: the unit comes up, +the socket appears, `brig doctor` finds the runtime and the boot assets. +The test VM has no network, so it boots no sandbox. It needs KVM on the +host that runs it, and on single-user nix it hangs with no output unless +the build user can open `/dev/kvm` and `~/.config/nix/nix.conf` has +`system-features = nixos-test kvm`. + +A sandbox needs network, which the test VM has not. To boot one on NixOS, +give the configuration above a VM variant and build that: + +```nix +virtualisation.vmVariant.virtualisation = { + memorySize = 4096; + cores = 2; + diskSize = 8192; + graphics = false; +}; +``` + +```sh +nix build .#nixosConfigurations.host.config.system.build.vm +./result/bin/run-nixos-vm +``` + +Three things only showed up on NixOS, because a single-user nix store on +Debian is writable by root and `/usr/sbin` and `/lib` exist there: + +- nerdctl writes a lock and one conflist per sandbox network into the CNI + config directory, so that directory is `/var/lib/brig/agent/cni/net.d`. + `brig-containerd` copies the bridge config there from the store on every + start. +- urunc runs `/usr/sbin/mke2fs` by default to build the overlay-block + rootfs. `urunc.toml` points it at the nixpkgs `mke2fs`. +- urunc binds `/lib`, `/lib64` and `/usr/lib` into the monitor's rootfs for + a dynamically linked monitor. NixOS has no `/lib` or `/usr/lib`, and the + monitors here are static. `nix/urunc-optional-libdirs.patch` skips a + directory that is not there, as upstream already does for `/lib64`. + +## Why some pieces are binaries + +Three things in `brig-guest` are not built here. The guest kernel comes from +`hull-assets`, whose repository is not public. The initrd is assembled by +`scripts/build-bundle.sh` from `urunit` and busybox. `hvi` is a static build +from `brig-sh/hull`. All three are taken from the bundle release tarball, +pinned by the sha256 its signed `checksums.txt` lists. + +`virtiofsd` and `cloud-hypervisor` are in nixpkgs, but the nixpkgs builds +are dynamically linked. `urunc` pivots into a monitor rootfs of its own +before it spawns them, and a dynamically linked binary cannot find its +interpreter in there. The bundle carries static builds, so those are used. + +## State and paths + +Same as the bundle's node-wide install: state under `/var/lib/brig/agent`, +the containerd socket at `/run/brig/containerd.sock`, the CNI bridge +`brig0` on `10.44.0.0/24`. + +## Tested + +Debian 12 on a GCP `n2-standard-4` with nested virtualization and +single-user nix, on 2 November 2026, following the steps above on a host +with nothing else on it: `nix build` from this repository took 2 minutes +(binaries from cache.nixos.org, `brig` and `urunc` built from source), +`brig doctor` all `ok`, a sandbox booted in 14 seconds on first pull, +`apt-get update` ran in the guest, a file written under `/work/demo` +appeared on the host, nothing was written under `/nix/store`, `nix store +verify` passed on the closure, and the unit, `vhost_vsock` and a new +sandbox came back after a reboot. + +NixOS 26.11, built from the module with user-mode network, run as a qemu +VM on a Debian host on 2 November 2026, so the sandbox was a third level +of virtualization: +`brig doctor` all `ok`, `brig run ubuntu` pulled the image and booted, the +guest agent answered within seconds, `brig sh` ran commands in the guest in +about a second, DNS resolved, a file written under `/work` appeared on the +host. + +On a scripted host, run `brig telemetry off` or `on` once first. The +first `brig run` on a tty otherwise waits on the consent prompt, and +`brig run` without `-d` stays attached. + +Not tested: aarch64, a NixOS host on real hardware. diff --git a/nix/brig.nix b/nix/brig.nix new file mode 100644 index 0000000..0ca5457 --- /dev/null +++ b/nix/brig.nix @@ -0,0 +1,86 @@ +{ + lib, + stdenv, + buildGoModule, + fetchFromGitHub, + installShellFiles, + versionCheckHook, + nix-update-script, +}: + +buildGoModule (finalAttrs: { + pname = "brig"; + version = "0.5.0"; + + src = fetchFromGitHub { + owner = "brig-sh"; + repo = "brig"; + tag = "v${finalAttrs.version}"; + hash = "sha256-HPkr/fHTi8QHIR09PVTDMAPefdtV6K9kYXVLdtbPupY="; + }; + + vendorHash = "sha256-513twbhAd5XxpvGisWoCGwmIFtK0fypViOOBpMNjNEQ="; + + subPackages = [ + "cmd/brig" + "cmd/brigd" + "cmd/brig-broker" + ]; + + env.CGO_ENABLED = 0; + + # The binary reads the tag and commit the Go toolchain embeds from a git + # checkout. A tarball carries none, so the values are passed here, the way + # the upstream Makefile does for a worktree. Without them `brig version` + # prints "dev". + ldflags = [ + "-s" + "-w" + "-X github.com/brig-sh/brig/internal/buildinfo.gitCommit=8059e00a8a8741a94177aa19626886f63eb396e1" + "-X github.com/brig-sh/brig/internal/buildinfo.gitCommitTime=2026-10-31T12:08:40Z" + "-X github.com/brig-sh/brig/internal/buildinfo.gitDescribe=v${finalAttrs.version}-0-g8059e00" + ]; + + nativeBuildInputs = [ installShellFiles ]; + + # The tests are written for a source checkout: one of them expects a test + # binary built from a tarball to carry no commit. The stamped values above + # are for the shipped binaries only. + preCheck = '' + ldflags="-s -w" + ''; + + postInstall = lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' + installShellCompletion --cmd brig \ + --bash <($out/bin/brig completion bash) \ + --fish <($out/bin/brig completion fish) \ + --zsh <($out/bin/brig completion zsh) + ''; + + # On darwin the secret store tests write to the real login keychain. + doCheck = !stdenv.hostPlatform.isDarwin; + + nativeInstallCheckInputs = [ versionCheckHook ]; + versionCheckProgramArg = "version"; + doInstallCheck = true; + + passthru.updateScript = nix-update-script { }; + + meta = { + description = "Run coding agents in a microVM sandbox"; + longDescription = '' + Brig boots a coding agent in a microVM and hands it a project directory + and named credentials, nothing else. This package holds the brig CLI, + the brigd session daemon and the brig-broker credential broker. It does + not include a runtime: on Linux Brig drives nerdctl with the urunc shim, + on macOS the hull runtime. `brig doctor` reports what is missing, and + https://brig.sh/docs/install/ says how to install the runtime. + ''; + homepage = "https://brig.sh"; + changelog = "https://github.com/brig-sh/brig/releases/tag/v${finalAttrs.version}"; + license = lib.licenses.asl20; + mainProgram = "brig"; + maintainers = with lib.maintainers; [ pmoust ]; + platforms = lib.platforms.unix; + }; +}) diff --git a/nix/guest.nix b/nix/guest.nix new file mode 100644 index 0000000..b208f05 --- /dev/null +++ b/nix/guest.nix @@ -0,0 +1,76 @@ +{ + lib, + stdenvNoCC, + fetchurl, +}: + +# The guest kernel, the container-initrd and the monitors, taken from the +# Linux runtime bundle release. The kernel is built by hull-assets, whose +# repository is not public, the initrd is assembled by scripts/build-bundle.sh +# from urunit and busybox, and hvi is a static binary from brig-sh/hull. Until +# those build here, the bundle tarball is the source, pinned by the sha256 its +# signed checksums.txt lists. +# +# virtiofsd and cloud-hypervisor come from the same tarball although nixpkgs +# has both: urunc pivots into a monitor rootfs of its own before it spawns +# them, and a dynamically linked binary cannot find its interpreter in there. +# The bundle's builds are static. +# +# brig checks the kernel and initrd against share/guest/SHA256SUMS, and that +# file against the release's checksums.txt, whose cosign signature and +# certificate sit beside it, the way the bundle's installer leaves them. +let + version = "0.1.0-rc21"; + base = "https://github.com/NOFireAI/brig-standalone-linux/releases/download/v${version}"; + release = + name: hash: + fetchurl { + url = "${base}/${name}"; + sha256 = hash; + }; + checksums = release "checksums.txt" "441bc9aaded057ed301b9e7bec265c4d75d2c418f32fc75a45dd841736ce67fa"; + signature = release "checksums.txt.sig" "bcfb6a50348c2f3a760419c5e2ad290bbe816ee57f77c2653543d7c1b1e83dc6"; + certificate = release "checksums.txt.pem" "d2cd0ff6918724d50dbe22d915a3dd9ee5cd3b26a26d4b0d52cdbf950bf84744"; + arch = + stdenvNoCC.hostPlatform.go.GOARCH + or (if stdenvNoCC.hostPlatform.isAarch64 then "arm64" else "amd64"); + hashes = { + amd64 = "3a376bb1da931c876d512643a205fd6ccefc6a8f6f696bdc6c548a477ef3205f"; + arm64 = "bcf280444a78adca58c5e89336edc6c55893f3da6416bc9bc2a103ad0e4c9b36"; + }; +in +stdenvNoCC.mkDerivation { + pname = "brig-guest"; + inherit version; + + src = fetchurl { + url = "${base}/brig-standalone-v${version}-linux-${arch}.tar.gz"; + sha256 = hashes.${arch}; + }; + + dontBuild = true; + dontFixup = true; + + installPhase = '' + runHook preInstall + mkdir -p $out/share $out/bin + cp -r share/guest $out/share/guest + cp ${checksums} $out/share/guest/checksums.txt + cp ${signature} $out/share/guest/checksums.txt.sig + cp ${certificate} $out/share/guest/checksums.txt.pem + install -m 0755 bin/hvi bin/cloud-hypervisor $out/bin/ + install -m 0755 libexec/virtiofsd $out/bin/virtiofsd + runHook postInstall + ''; + + meta = { + description = "Guest kernel, initrd and monitors for Brig sandboxes on Linux"; + homepage = "https://github.com/NOFireAI/brig-standalone-linux"; + sourceProvenance = [ lib.sourceTypes.binaryNativeCode ]; + license = lib.licenses.asl20; + platforms = [ + "x86_64-linux" + "aarch64-linux" + ]; + }; +} diff --git a/nix/module.nix b/nix/module.nix new file mode 100644 index 0000000..77378b1 --- /dev/null +++ b/nix/module.nix @@ -0,0 +1,46 @@ +# NixOS module: services.brig. Runs the containerd that brig drives, loads +# the kernel modules a sandbox needs and puts brig on PATH. The same layout +# as the bundle's node-wide install, so a host that had the bundle and one +# that has this module look the same to brig. Only root drives the stack. +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.services.brig; +in +{ + options.services.brig = { + enable = lib.mkEnableOption "the brig sandbox runtime"; + + package = lib.mkOption { + type = lib.types.package; + description = "The brig-runtime package to run and install."; + }; + }; + + config = lib.mkIf cfg.enable { + environment.systemPackages = [ cfg.package ]; + + boot.kernelModules = [ + "kvm" + "vhost_vsock" + ]; + + # containerd's state and the socket directory, created before the unit + # starts. brig-containerd creates them too; this keeps them across the + # unit's absence. + systemd.tmpfiles.rules = [ + "d /var/lib/brig/agent 0755 root root -" + "d /run/brig 0710 root root -" + ]; + + # The unit comes from the package, so this module and a host that copies + # the file by hand run the same definition. NixOS ignores [Install] in a + # packaged unit, so the target is named here. + systemd.packages = [ cfg.package ]; + systemd.services.brig-containerd.wantedBy = [ "multi-user.target" ]; + }; +} diff --git a/nix/runtime.nix b/nix/runtime.nix new file mode 100644 index 0000000..95718e1 --- /dev/null +++ b/nix/runtime.nix @@ -0,0 +1,237 @@ +{ + lib, + stdenv, + runCommand, + makeWrapper, + brig, + urunc, + brig-guest, + containerd, + nerdctl, + cni-plugins, + iptables, + nftables, + util-linux, + cosign, + oras, + e2fsprogs, +}: + +# The Linux runtime around brig: containerd with the urunc shim, the CNI +# bridge, the monitors and the guest assets, wired the way the runtime bundle +# wires them. State lives under /var/lib/brig/agent and the socket under +# /run/brig, so a host that had the bundle and one that has this package +# look the same to brig. Only root drives this stack; there is no rootless +# path here. +let + stateDir = "/var/lib/brig/agent"; + runDir = "/run/brig"; + sock = "${runDir}/containerd.sock"; + goarch = if stdenv.hostPlatform.isAarch64 then "arm64" else "amd64"; + + etc = runCommand "brig-runtime-etc" { } '' + mkdir -p $out/etc/cni/net.d $out/etc/certs.d + + cat > $out/etc/containerd.toml < $out/etc/nerdctl.toml < $out/etc/urunc.toml < $out/etc/cni/net.d/10-brig-bridge.conflist < $out/bin/brig-containerd < $out/lib/systemd/system/brig-containerd.service <= 4 && monitorName[:4] == "qemu" { diff --git a/nix/urunc.nix b/nix/urunc.nix new file mode 100644 index 0000000..ab79388 --- /dev/null +++ b/nix/urunc.nix @@ -0,0 +1,70 @@ +{ + lib, + buildGoModule, + fetchFromGitHub, +}: + +# urunc and its containerd shim, from the branch the Linux runtime bundle +# builds (feat/linux-hvi-v050). No urunc release reads the boot annotations +# Brig sets or knows the hvi monitor, so a release is not an option yet. The +# commit is the one the bundle release records in its pins.env as URUNC_REF. +buildGoModule (finalAttrs: { + pname = "urunc"; + version = "0-unstable-2026-10-29"; + + src = fetchFromGitHub { + owner = "urunc-dev"; + repo = "urunc"; + rev = "3c00b69ff9ac1b8bb260975cdc63d18a7043558b"; + hash = "sha256-NbH+6nQWLEgcI2VBmtA/50EHbMB8Q1k4RJk67miYXtU="; + }; + + vendorHash = "sha256-08PC8ethomYCZTBfP1PVCCXgXeyrSHHoozcVsWmqJUE="; + + # urunc binds /lib, /lib64 and /usr/lib from the host into the monitor's + # rootfs for a dynamically linked monitor. NixOS has no /lib or /usr/lib, + # and the monitors here are static. The patch skips a directory that is + # not there, as upstream already does for /lib64. + patches = [ ./urunc-optional-libdirs.patch ]; + + subPackages = [ + "cmd/urunc" + "cmd/containerd-shim-urunc-v2" + ]; + + # libcontainer's nsenter is C, so the urunc binary needs cgo. The shim + # does not, but one derivation builds both. + env.CGO_ENABLED = 1; + tags = [ + "netgo" + "osusergo" + ]; + + ldflags = [ + "-s" + "-w" + "-X main.version=${finalAttrs.version}" + ]; + + # The shim execs the OCI runtime that go-runc names, and the upstream + # Makefile seds that name from runc to urunc in the vendored source before + # it builds. go-runc v1.0.0 declares it as a const, so a linker flag cannot + # set it. The vendor directory is a copy by this point, with the store's + # read-only permissions. + postConfigure = '' + f=vendor/github.com/containerd/go-runc/runc.go + chmod u+w "$(dirname $f)" $f + grep -q 'DefaultCommand = "runc"' $f + sed -i 's/DefaultCommand = "runc"/DefaultCommand = "urunc"/' $f + ''; + + # The tests want KVM and a monitor. + doCheck = false; + + meta = { + description = "Container runtime that boots the container as a microVM"; + homepage = "https://github.com/urunc-dev/urunc"; + license = lib.licenses.asl20; + platforms = lib.platforms.linux; + }; +})