diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index 143429455..d7fdd441a 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31.11.1 - run: nix flake check -L + # NixOS VM test against headscale (nix/e2e.nix); not a check since it needs KVM. + - run: nix build -L .#e2e build: strategy: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5eee9fc8a..bd3caa869 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -174,6 +174,13 @@ Target-independent Rust checks (clippy, fmt, deny) and validation of the Nix str $ nix flake check ``` +End-to-end tests of the examples against a real [headscale](https://github.com/juanfont/headscale), +in NixOS VMs (see [`nix/e2e.nix`](nix/e2e.nix)), need Linux with KVM: + +```sh +$ nix build .#e2e -L +``` + ## Releases We aim for a roughly monthly release schedule. Releases are time-based, not feature-based. We will diff --git a/flake.lock b/flake.lock index 998284c15..fc003556e 100644 --- a/flake.lock +++ b/flake.lock @@ -15,6 +15,29 @@ "type": "github" } }, + "flake-checks": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": [ + "headscale", + "nixpkgs" + ], + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1790324098, + "narHash": "sha256-O35odpnkPBPhRWPixikS/jk+Sr0FaiXzjoOsQ/mZEHg=", + "owner": "kradalby", + "repo": "flake-checks", + "rev": "fccef887b41c9b91d8c31351630d6da7a547c5d4", + "type": "github" + }, + "original": { + "owner": "kradalby", + "repo": "flake-checks", + "type": "github" + } + }, "flake-parts": { "inputs": { "nixpkgs-lib": "nixpkgs-lib" @@ -33,18 +56,74 @@ "type": "github" } }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "headscale": { + "inputs": { + "flake-checks": "flake-checks", + "flake-utils": "flake-utils_2", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1791362013, + "narHash": "sha256-zJGZBinz3UEdg5TJH8L7RQ6V3bDB236Rucj5Ms+QIn0=", + "owner": "juanfont", + "repo": "headscale", + "rev": "3e00a42bd23a166bd192f5d58178b3413543efe2", + "type": "github" + }, + "original": { + "owner": "juanfont", + "repo": "headscale", + "type": "github" + } + }, "nixpkgs": { "locked": { - "lastModified": 1778430510, - "narHash": "sha256-Ti+ZBvW6yrWWAg2szExVTwCd4qOJ3KlVr1tFHfyfi8Q=", - "owner": "nixos", + "lastModified": 1790510107, + "narHash": "sha256-EVMNYv7hYDDD9TGVT/hIyTYgpiXA8y3m5xIEIxuGNU0=", + "owner": "NixOS", "repo": "nixpkgs", - "rev": "8fd9daa3db09ced9700431c5b7ad0e8ba199b575", + "rev": "3181085bfd08663b6b9e60bc7a8395c2aaa741bd", "type": "github" }, "original": { - "owner": "nixos", - "ref": "nixos-25.11", + "owner": "NixOS", + "ref": "nixpkgs-unstable", "repo": "nixpkgs", "type": "github" } @@ -64,11 +143,28 @@ "type": "github" } }, + "nixpkgs_2": { + "locked": { + "lastModified": 1778430510, + "narHash": "sha256-Ti+ZBvW6yrWWAg2szExVTwCd4qOJ3KlVr1tFHfyfi8Q=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "8fd9daa3db09ced9700431c5b7ad0e8ba199b575", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-25.11", + "repo": "nixpkgs", + "type": "github" + } + }, "root": { "inputs": { "crane": "crane", "flake-parts": "flake-parts", - "nixpkgs": "nixpkgs", + "headscale": "headscale", + "nixpkgs": "nixpkgs_2", "rust-advisory-db": "rust-advisory-db", "rust-overlay": "rust-overlay" } @@ -108,6 +204,58 @@ "repo": "rust-overlay", "type": "github" } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "headscale", + "flake-checks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1786901030, + "narHash": "sha256-WSFCsDSE5ffgD2MqzkM2CYjeFiKhRF/dJUN8uedb6YE=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "27b3b12a8e6375f28ebe122f07d230ca5459bbfa", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 206583957..231544a08 100644 --- a/flake.nix +++ b/flake.nix @@ -12,6 +12,10 @@ url = "github:rustsec/advisory-db"; flake = false; }; + + # Control server for the e2e VM test (nix/e2e.nix). Doesn't follow our nixpkgs: headscale + # builds with the Go its go.mod requires, which a stable nixpkgs may not have yet. + headscale.url = "github:juanfont/headscale"; }; description = "tailscale-rs: tailscale client in rust"; @@ -209,6 +213,15 @@ system = system; }; in makePackages crossPkgs); + } // lib.optionalAttrs (pkgs.stdenv.hostPlatform.isLinux && inputs.headscale.packages ? ${system}) { + # tailscale-rs against a real headscale in NixOS VMs. Needs KVM, so it's kept out of + # `checks`: `nix build .#e2e -L`. + e2e = import ./nix/e2e.nix { + pkgs = pkgs; + deps = packages.deps; + rustsrc = rustsrc; + headscale = inputs.headscale; + }; }; # Run these (as well as checking the flake for valid structure) with `nix flake check`. diff --git a/nix/e2e.nix b/nix/e2e.nix new file mode 100644 index 000000000..bdf54ff16 --- /dev/null +++ b/nix/e2e.nix @@ -0,0 +1,148 @@ +# End-to-end test: the tailscale-rs examples against a real headscale, using headscale's NixOS +# test kit (https://github.com/juanfont/headscale/blob/main/nix/README.md), with Go tailscaled +# peers on the other side. Needs Linux with KVM: +# +# nix build .#e2e -L +# nix build .#e2e.driverInteractive && ./result/bin/nixos-test-driver +{ + pkgs, + deps, + rustsrc, + headscale, +}: let + # The test-only features let the examples fetch /key over plain HTTP and accept the kit's + # self-signed (InsecureForTests) DERP. + examples = pkgs.craneLib.buildPackage (deps.passthru.buildDeps // { + pname = "tailscale-e2e-examples"; + version = "dev"; + + strictDeps = true; + src = rustsrc; + + cargoArtifacts = deps; + cargoExtraArgs = "--locked -p tailscale --examples --features axum,ts_control/insecure-keyfetch,ts_control/insecure-derp"; + # nextest already covers the tests. + doCheck = false; + }); + + policy = acls: pkgs.writeText "policy.json" (builtins.toJSON { acls = acls; }); + allowAll = policy [ { action = "accept"; src = [ "*" ]; dst = [ "*:*" ]; } ]; + httpOnly = policy [ { action = "accept"; src = [ "peer@" ]; dst = [ "rs@:80" ]; } ]; + + goPeer = { + imports = [ headscale.nixosModules.testkit-peer ]; + environment.systemPackages = with pkgs; [ curl jq netcat-openbsd ]; + }; + +in pkgs.testers.runNixOSTest { + name = "tailscale-rs-e2e"; + + nodes = { + headscale.imports = [ headscale.nixosModules.testkit ]; + + # Runs the examples: userspace netstack, no tailscaled or TUN. The direct UDP socket binds a + # random port. + rs.networking.firewall.enable = false; + + # tailscaled without a UDP socket: all traffic with rs crosses the kit's DERP. The NixOS + # module doesn't trust tailscale0, and rs's datagrams must get in. + peer = { + imports = [ goPeer ]; + systemd.services.tailscaled.environment.TS_DEBUG_ALWAYS_USE_DERP = "1"; + networking.firewall.trustedInterfaces = [ "tailscale0" ]; + }; + + # Same LAN as rs, with UDP, so disco can find a direct path. + direct.imports = [ goPeer ]; + }; + + testScript = '' + # Every command runs under pipefail: no `head`/`grep -m` cutting a producer short. Waits + # make one attempt per second, each also capped at T seconds. + T = 120 + + def run_example(host, example, args="", key=None): + """Start EXAMPLE on rs as systemd unit and tailnet host HOST, with its own key file.""" + auth = f"-E TS_AUTH_KEY={key}" if key else "" + rs.succeed( + f"systemd-run --collect --unit={host} -E NO_COLOR=1 -E TS_CONTROL_URL=http://headscale {auth}" + f" ${examples}/bin/{example} -H {host} -c /var/lib/rs/{host}.json {args}" + ) + + def tailnet_ip(host): + return peer.wait_until_succeeds(f"tailscale ip -4 {host}", timeout=T).strip() + + # The helpers below build shell checks for a Go peer to run. + + def http_ok(ip): + return f"curl -fsS -m 10 http://{ip}/index.html | grep -F 'tailscale-rs'" + + def peer_json(host, cond): + return f"tailscale status --json | jq -e '.Peer[] | select(.HostName == \"{host}\") | {cond}'" + + # Setup: two tailscale-rs nodes on rs (hosts "echo" and "http"), two Go peers. + start_all() + rs_key = headscale.succeed("hs-authkey rs").strip() + peer_key = headscale.succeed("hs-authkey peer").strip() + rs.wait_for_unit("multi-user.target") + run_example("echo", "tcp_echo", key=rs_key) + run_example("http", "axum", key=rs_key) + peer.succeed(f"hs-join {peer_key}") + direct.succeed(f"hs-join {peer_key}") + echo_ip, http_ip = tailnet_ip("echo"), tailnet_ip("http") + # `timeout`, not `nc -w`: an echo that never closes must fail. + tcp_echo_ok = f"echo hi | timeout 10 nc -N {echo_ip} 1234 | grep -x hi" + + # The subtests run in order and share these nodes. The policy test restores allow-all. + + with subtest("auth key + home DERP"): + # rs ignores pings over DERP, so read its home region from a Go peer's netmap. + peer.wait_until_succeeds(peer_json("echo", '.Relay == "headscale"'), timeout=T) + + with subtest("inbound TCP over DERP"): + peer.wait_until_succeeds(tcp_echo_ok, timeout=T) + peer.succeed("head -c 1M /dev/urandom > /tmp/blob") + peer.succeed(f"timeout 60 nc -N {echo_ip} 1234 < /tmp/blob | cmp - /tmp/blob") + + with subtest("HTTP via tailscale::axum"): + peer.wait_until_succeeds(http_ok(http_ip), timeout=T) + + with subtest("outbound UDP"): + peer_ip = peer.succeed("tailscale ip -4").strip() + run_example("ping", "peer_ping", f"-i 0.5 -p {peer_ip}:5678", key=rs_key) + peer.wait_until_succeeds( + "timeout 5 nc -d -u -l 5678 > /tmp/udp || true; grep -F hello /tmp/udp", timeout=T + ) + + with subtest("direct UDP path"): + # With the default --until-direct, a pong relayed over DERP doesn't count. + direct.wait_until_succeeds("tailscale ping -c 1 --timeout 3s echo", timeout=T) + direct.wait_until_succeeds(tcp_echo_ok, timeout=T) + direct.succeed(peer_json("echo", '.CurAddr != ""')) + + with subtest("restart reuses key file"): + rs.succeed("systemctl stop echo") + run_example("echo", "tcp_echo") # no auth key this time + # Proves the same node: a re-registered one would get a new IP, and an + # unregistered one no IP at all. + peer.wait_until_succeeds(tcp_echo_ok, timeout=T) + + with subtest("live packet filter"): + headscale.succeed("headscale policy set -f ${httpOnly}") + peer.wait_until_fails(tcp_echo_ok, timeout=T) + # :80 still passes while :1234 is dropped, so it's the filter, not an outage. + peer.wait_until_succeeds(http_ok(http_ip), timeout=T) + peer.fail(tcp_echo_ok) + headscale.succeed("headscale policy set -f ${allowAll}") + peer.wait_until_succeeds(tcp_echo_ok, timeout=T) + + with subtest("interactive login"): + run_example("login", "axum") # fresh key file, no auth key + # axum logs the auth URL from Device::is_authorized; it ends in headscale's auth ID. + auth_id = rs.wait_until_succeeds( + "journalctl -u login -o cat | grep -oE 'hskey-authreq-[[:xdigit:]]+'", timeout=T + ).split()[0] + headscale.succeed(f"headscale auth register --user rs --auth-id {auth_id}") + peer.wait_until_succeeds(http_ok(tailnet_ip("login")), timeout=T) + ''; +}