From c2e0543d983014476f4a8dba5351031af4f318a8 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Sun, 11 Oct 2026 16:57:38 -0300 Subject: [PATCH 1/5] openvmm: publish guest listeners to the host's private networks The portable profile rejected --network-ingress allow before launch, because its user-mode NAT had no way to reach a guest listener: every inbound connection needs a host socket, and Consomme bound only the ports that explicit --host-loopback-forward options named. MXC's network.ingress.default: allow means that clients on the host's private networks reach every port on which the workload listens, without a port list, while the internet never reaches it and host loopback stays a separate decision (#275). dd8def143 (nanvix/openvmm#133) implements that contract. With ingress allow, OpenVMM appends the host-owned virtnet_ingress=allow token, which callers may not supply, and the guest is to announce the TCP and UDP ports on which it listens in UDP datagrams to port 1021 of the IPv4 gateway, from a privileged source port so that only guest root can publish. For each announced port and family, OpenVMM listens on that port of every host address and relays only peers whose addresses are not globally reachable, lie outside the guest's own networks, and are not IPv6 link-local; it resets the TCP connections and drops the datagrams of other peers, so internet clients never reach the guest. The host itself, at loopback or one of its own addresses, is admitted only with explicit --host-loopback allow, which ingress allow now accepts without forwards. Replies to admitted peers pass whatever the egress policy, which admits a TCP segment other than an initial SYN, or a UDP datagram, only when the endpoint carries its flow. Ingress allow records the ingress action in canonical policy encoding version 5, leaving the digests of existing policies unchanged, and snapshot capture and restore reject it, because its listeners are live host attachments. The run guide gains a private-network ingress section, the usage reference describes --network-ingress and --host-loopback for run and sandbox, the cold-boot and machine ABI designs document the token, the announcement, and the policy, and the current limits and the implementation status describe what ingress admits and refuses. The directional-network-policy scenario no longer expects OpenVMM to reject ingress allow, and the validation design no longer lists that check. The guest does not announce its listeners until the next commit adds its announcer, so ingress allow publishes nothing yet. nvx.py verify passes with this pin. Part of #275. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/design/cold-boot.md | 6 +- doc/design/current-limits.md | 7 +- .../integrated-implementation-status.md | 2 +- doc/design/machine-and-device-abi.md | 55 ++++++++++++--- doc/design/validation.md | 4 +- doc/run.md | 67 +++++++++++++++++-- doc/usage.md | 8 +-- openvmm | 2 +- scripts/nvx_tools/microvm_tests.py | 23 ------- 9 files changed, 121 insertions(+), 53 deletions(-) diff --git a/doc/design/cold-boot.md b/doc/design/cold-boot.md index f2fca903..38e8372c 100644 --- a/doc/design/cold-boot.md +++ b/doc/design/cold-boot.md @@ -99,8 +99,8 @@ tokens. A fresh boot then appends host-owned tokens in this order: slot, boot console, sandbox blocks, and the control console followed by `nvx_control_tty=hvc2`; and 5. network bootstrap tokens, including gateway DNS only when the egress - policy permits it, which names the IPv4 gateway or else the IPv6 one, - followed by + policy permits it, which names the IPv4 gateway or else the IPv6 one, and + `virtnet_ingress=allow` last when ingress is allowed, followed by [filesystem bootstrap tokens](machine-and-device-abi.md#filesystem) for the attached HostFs export or aggregate. @@ -111,7 +111,7 @@ fails with `E_CMDLINE_CLOCK_TOKEN`. Callers may not supply `earlycon=`, `console=`, `virtio_mmio.device=`, `virtnet_ip=`, `virtnet_mask=`, `virtnet_gw=`, `virtnet_dns=`, `virtnet_ip6=`, -`virtnet_gw6=`, `virtfs_dir=`, `virtfs_tag=`, `virtfs_mode=`, +`virtnet_gw6=`, `virtnet_ingress=`, `virtfs_dir=`, `virtfs_tag=`, `virtfs_mode=`, `virtfs_aggregate=`, `nvx_snapshot_tier=`, `nr_cpus=`, `nvx_workload_uid=`, `nvx_workload_gid=`, or `nvx_lifecycle=` tokens. A machine with a control console also applies the diff --git a/doc/design/current-limits.md b/doc/design/current-limits.md index eb3762f7..0456c15b 100644 --- a/doc/design/current-limits.md +++ b/doc/design/current-limits.md @@ -20,9 +20,10 @@ The current ABI family intentionally does not provide: hard-link support; - sandbox-block, network, or control-console construction through the management RPC, or its restore of snapshots that contain them; -- host networking other than the in-process portable endpoint, unrestricted - ingress, generic bidirectional host-loopback connectivity, or live - host-loopback port forwards in snapshots; +- host networking other than the in-process portable endpoint, ingress from + globally reachable addresses, generic bidirectional host-loopback + connectivity without private-network ingress, or live host-loopback port + forwards or private-network ingress in snapshots; - a production control-session RPC protocol beyond the managed lifecycle subset; - replaceable sandbox configuration, a production OCI agent, or a public sandbox snapshot workflow; diff --git a/doc/design/integrated-implementation-status.md b/doc/design/integrated-implementation-status.md index 25852749..3521fb46 100644 --- a/doc/design/integrated-implementation-status.md +++ b/doc/design/integrated-implementation-status.md @@ -10,7 +10,7 @@ The implementation and remaining proposals separate as follows: | Snapshot and restore | Guest-requested version-5 publication, same-backend restore, exact saved inventory, private COW RAM, optional fresh RAM expansion, fresh/paired scratch, three sandbox tiers, input gating, single-use resume claims, a single-use restore readiness event, and management exclusion at the capture boundary. The management RPC drives the same capture and restore for blockless machines without a NIC or control console. Older supported manifests remain readable subject to their recorded capabilities. | | Resource activation | Opt-in CPU-prefix and 128-MiB-aligned RAM targets within immutable capacity. Explicit MSHV targets materialize only that VP prefix and cannot be saved again; KVM, WHP, and untargeted MSHV retain full VP capacity. | | Console | Boot virtio-console with private RX/TX state and reconnect policies; bounded portb/host-relay drain at process exit, with guest exit status propagated to management-RPC waiters. The separate control console has authenticated Linux and Windows endpoints, bounded framing with receive credits, per-instance epochs, broker saved state, and managed workload RPC. | -| Network | Static dual-stack identity, fixed transport, the portable in-process Consomme endpoint, directional egress `allow`/`deny`, fixed ingress `deny`, IPv4 and IPv6 L3/L4 egress rules, host-loopback deny or explicit port publishing, an exact proxy exception, and quiesced restore are implemented. Stateful replies remain available; unrestricted ingress is rejected before launch. Capture drains packet ownership instead of serializing arbitrary pending packets or host flow state. | +| Network | Static dual-stack identity, fixed transport, the portable in-process Consomme endpoint, directional egress `allow`/`deny`, ingress `deny` or private-network ingress that publishes every port on which the guest listens to clients that are not globally reachable, IPv4 and IPv6 L3/L4 egress rules, host-loopback deny or explicit port publishing, an exact proxy exception, and quiesced restore are implemented. Stateful replies remain available under both ingress actions; ingress from globally reachable addresses is refused, and snapshots reject private-network ingress. Capture drains packet ownership instead of serializing arbitrary pending packets or host flow state. | | Guest filesystem | Kernel features and shell bootstrap for EROFS over ext4 scratch, overlayfs, namespace/cgroup isolation, capability stripping, supervision, and low-level snapshot hooks. Not a complete OCI agent or public sandbox restore workflow. | | Host filesystem | Fixed no-DAX HostFs in a slot that command-line and management-RPC cold boots always expose and that stays dormant without an attachment, live attachment revalidation, server-side denied subtrees with alias-identity enforcement, allowed paths that expose subtrees of denied paths again through traverse-only directories, write narrowing to declared writable paths of a read-write export, guest-created symbolic links on read-write exports with exact targets that the host never follows, and optional caller ownership on Linux hosts, which performs each request as its guest caller's UID and GID with root squashed to the export owner and fails closed with `EPERM`. Provider-backed immutable filesystem generations remain outside the current profile. | | Workload policy and operations | Host-owned non-root workload identity, one-shot or managed lifecycle selection, bounded local outcome reports, and opt-in lifecycle profiling that does not change ordinary runs. | diff --git a/doc/design/machine-and-device-abi.md b/doc/design/machine-and-device-abi.md index 9537a246..d221d592 100644 --- a/doc/design/machine-and-device-abi.md +++ b/doc/design/machine-and-device-abi.md @@ -534,7 +534,9 @@ These are destination-based L3/L4 rules, not hostname or HTTP-content filters. They apply to every frame that the guest transmits, including replies on explicitly forwarded connections, but they are not an independent RX firewall or host-wide firewall. Responses arriving through the host sockets use the RX -path. +path. Ingress `allow` additionally admits replies on the flows that the +endpoint carries, as [Host-local access and ingress](#host-local-access-and-ingress) +describes. #### Host-local access and ingress @@ -557,9 +559,9 @@ separate backend service governed by the egress policy, not by the general host-loopback mapping. An outbound connection does not make guest services reachable from the host -or LAN. General `--network-ingress allow` is unsupported and rejected before -launch. Deliberate host-to-guest publishing instead requires explicit -localhost TCP or UDP port forwards, at most 64 of them. For example: +or LAN. Without ingress `allow`, which the paragraphs below describe, +deliberate host-to-guest publishing requires explicit localhost TCP or UDP +port forwards, at most 64 of them. For example: ```text --host-loopback allow @@ -570,11 +572,43 @@ This binds a real host listener on `127.0.0.1:8080` and forwards connections to guest port `80`; it does not bind a LAN-facing listener. Each forward requires explicit `--host-loopback allow`, and omitting `--host-loopback` keeps the default gateway mapping without publishing guest ports. The portable -NAT cannot offer generic bidirectional host-loopback connectivity, so an -explicit `--host-loopback allow` is accepted only with explicit -`--host-loopback-forward` attachments. For host-loopback peers, the backend -uses synthetic peer addresses so the guest routes replies through its NIC -rather than its own loopback, and reverse-translates those addresses on TX. +NAT cannot offer generic bidirectional host-loopback connectivity on its own, +so an explicit `--host-loopback allow` is accepted only with explicit +`--host-loopback-forward` attachments or with ingress `allow`. For +host-loopback peers, the backend uses synthetic peer addresses so the guest +routes replies through its NIC rather than its own loopback, and +reverse-translates those addresses on TX. + +Ingress `allow` publishes the guest's listeners to the host's private +networks. OpenVMM then appends the `virtnet_ingress=allow` token, which callers +may not supply, after the other network tokens, and the guest announces its +listeners in UDP datagrams to port 1021 of the IPv4 gateway from a source port +below 1024. An announcement is the magic `NVXL`, version 1, a flags byte whose +bit 0 marks omitted listeners, a big-endian 16-bit count of at most 64, and +that many four-byte entries in strictly increasing order of protocol and port: +the IP protocol number, 6 for TCP or 17 for UDP, a family mask with 1 for IPv4 +and 2 for IPv6, and the big-endian nonzero port. The endpoint consumes +announcements before the egress policy, which neither authorizes nor forwards +them, and each one replaces the published set. + +For each published port and family, the endpoint listens on that port of every +host address and relays only peers in the IPv4 private-use, shared, +link-local, benchmarking, and documentation ranges or the IPv6 unique local, +site-local, documentation, and benchmarking ranges, none of which is globally +reachable, except peers inside the guest's own subnets, which the guest would +take for its gateway or its neighbors. Peers at the host's own addresses, +including loopback, need an explicit host-loopback allow. + +Under ingress allow, the policy also admits ARP requests and Neighbor +Solicitations for the guest's subnet and Neighbor Solicitations for the +link-local addresses at which the gateway presents host peers. It admits a +frame that its rules deny when the frame continues a flow that the endpoint +carries: a TCP segment other than an initial SYN, or a UDP datagram from a +published port to a peer that recently sent one to it, which leaves through +the port's host listener. Ingress allow selects canonical policy encoding +version 5, which adds the ingress action, and leaves the encoding and digest +of ingress-deny policies unchanged. Snapshot capture and restore reject it, +and the TAP backend rejects ingress-allow policies. #### DNS @@ -622,7 +656,8 @@ digest. The snapshot records the IPv6 identity; a snapshot whose network predates it restores IPv4-only. Restore reconstructs a fresh endpoint and requires the profile and policy again; native sockets, DNS requests, and flow objects are never serialized. Explicit localhost forwards are process-local -attachments, and snapshot capture and restore reject them. +attachments, and so are the listeners that ingress `allow` publishes; snapshot +capture and restore reject both. Capture quiesces the endpoint, drains completion ownership, and requires the saved queues to contain no unrepresented RX or TX packets. It then saves the diff --git a/doc/design/validation.md b/doc/design/validation.md index 15a83f68..df7d27fa 100644 --- a/doc/design/validation.md +++ b/doc/design/validation.md @@ -38,8 +38,8 @@ across capture and a capture right after the guest swaps two files and two directories with `RENAME_EXCHANGE`, which Linux hosts perform and Windows hosts reject without changing either name. Directional network coverage verifies an egress request and response with ingress denied, denial of a host connection -to an active guest listener, complete egress denial, and pre-boot rejection of -unsupported ingress on KVM, MSHV, and WHP. L3/L4 coverage verifies TCP and UDP +to an active guest listener, and complete egress denial on KVM, MSHV, and WHP. +L3/L4 coverage verifies TCP and UDP port rules, overlapping deny precedence, default-deny behavior, and pre-boot rejection of malformed or incomplete rules. Protocol coverage verifies that TCP, UDP, and ICMP rules without a port, `any` with a port, and `any` alone diff --git a/doc/run.md b/doc/run.md index 8ae779c8..6b862a78 100644 --- a/doc/run.md +++ b/doc/run.md @@ -121,8 +121,9 @@ Direct OpenVMM launches accept generic directional network defaults: Egress defaults to `allow` and ingress defaults to `deny`. With ingress denied, responses to guest-initiated connections remain available, while new inbound -connections do not. The portable profile supports egress `allow` or `deny` but -rejects ingress `allow` before the workload starts. +connections do not. With ingress `allow`, clients on the host's private +networks reach the guest's listeners, as described in +[Private-network ingress](#private-network-ingress). The NIC is dual-stack. The guest's IPv6 address embeds its IPv4 address in the unique local prefix `fd00::/96`, with a prefix of 96 plus the IPv4 prefix, and @@ -334,10 +335,11 @@ same port and other host service ports remain blocked even with ordinary egress allowed. The portable profile does **not** support generic bidirectional host-loopback -allow. Explicit `--host-loopback allow` without any forward is rejected before -VM resources are opened. For deliberate port publishing, repeat -`--host-loopback-forward tcp:HOST_PORT:GUEST_PORT` or its UDP form to expose -only selected localhost ports toward the guest, with explicit +allow on its own. Explicit `--host-loopback allow` without any forward is +rejected before VM resources are opened unless ingress is `allow`, which then +also admits the host to the guest's listeners. For deliberate port publishing, +repeat `--host-loopback-forward tcp:HOST_PORT:GUEST_PORT` or its UDP form to +expose only selected localhost ports toward the guest, with explicit `--host-loopback allow`. These forwards do not satisfy a generic allow policy that provides no port list. Omitting `--host-loopback` preserves existing guest-to-host mapping without publishing guest ports. Guest-originated traffic @@ -378,6 +380,59 @@ example: --restore-ready-path /run/nvx/restore-ready.sock ``` +### Private-network ingress + +Ingress `allow` lets clients on the host's private networks reach every TCP +and UDP port on which the guest listens, without a port list: + +```bash +./bin/openvmm \ + --single-process \ + --machine microvm \ + --hypervisor kvm \ + --memory 128M \ + --kernel guest/vmlinux \ + --initrd guest/initramfs.cpio.gz \ + --net 10.0.0.2/24 \ + --network-profile portable \ + --network-egress deny \ + --network-ingress allow +``` + +OpenVMM appends `virtnet_ingress=allow` to the kernel command line, and the +guest's init then starts `nvx-ingress`, which reports the guest's listening +TCP sockets and its unconnected UDP sockets to the gateway, except those bound +to a loopback address or an IPv6 link-local one. It reports a change within +100 milliseconds, or about a second for a new UDP socket, and repeats the +report every 2 seconds. Only guest root can send the report, from a privileged +port, and the egress policy does not apply to it. For each reported port, +OpenVMM listens on the same port of every host address, of each family on which +the guest listens, and relays connections and datagrams to the guest. + +OpenVMM admits a client only when its address is not globally reachable, such +as one in `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `100.64.0.0/10`, +`169.254.0.0/16`, or `fc00::/7`. It resets the TCP connections and drops the +UDP datagrams of other clients, so internet clients never reach the guest. It +also refuses clients that the guest could not answer: those inside the guest's +own networks, such as `10.0.0.0/24` and `fd00::a00:0/120` for `10.0.0.2/24`, +which the guest would take for its gateway or its neighbors, and IPv6 +link-local clients, so choose a guest network that contains no client. The +host itself, at a loopback address or one of its own, counts as host loopback: +OpenVMM admits it only with explicit `--host-loopback allow`. Replies to +admitted clients pass whatever the egress policy, while the guest's own +connections remain subject to it. + +Publication is best effort. OpenVMM cannot publish a port that a host process +holds, or on Linux a privileged port that it may not bind; it logs a warning +and retries with the next report. An explicit `--host-loopback-forward` to the +same guest port keeps working beside the publication. The guest reports at +most 64 ports. A TCP socket listening on `::` counts for IPv4 too unless it is +IPv6-only, and a UDP socket on `::` unless the guest sets +`net.ipv6.bindv6only`. Clients on other hosts also need the host's firewall to +admit them to OpenVMM, for example an inbound rule for `openvmm.exe` on +Windows. Snapshot capture and restore reject ingress `allow`, because the +published listeners are live host attachments. + ## Migration from the retired profile `microvm` is now the only selector and launches the contract previously named diff --git a/doc/usage.md b/doc/usage.md index 92adec68..32683ec6 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -513,11 +513,11 @@ python3 scripts/nvx.py run | `--net IPV4/PREFIX` | none | Enable virtio-net with the static guest IPv4 address and prefix. The guest also gets the IPv6 address that embeds it in `fd00::/96`, such as `fd00::a00:2/120` for `10.0.0.2/24`. | | `--network-profile {portable}` | none | Select the required cross-platform network behavior contract; must be specified with `--net`. | | `--network-egress {allow,deny}` | `allow` | Set the default guest egress policy. | -| `--network-ingress {allow,deny}` | `deny` | Set the default host ingress policy. The portable profile currently supports only `deny`; `allow` is rejected before launch. | +| `--network-ingress {allow,deny}` | `deny` | Set the default host ingress policy. `allow` publishes every TCP and UDP port on which the guest listens to clients on the host's private networks, and to the host itself only with `--host-loopback allow`; see [Private-network ingress](run.md#private-network-ingress). Snapshot capture and restore reject `allow`. | | `--network-egress-allow CIDR[:PROTOCOL[:PORT[-PORT]]]` | none | Allow matching guest egress; repeat to add rules. `CIDR` is an IPv4 or IPv6 address or network, and a rule matches only its own family. `PROTOCOL` is `tcp`, `udp`, or `icmp` and matches every port when `PORT` is omitted; only `tcp` and `udp` take a `PORT` or an inclusive `PORT-PORT` range, such as `8000-8010`. | | `--network-egress-deny CIDR[:PROTOCOL[:PORT[-PORT]]]` | none | Deny matching guest egress; repeat to add rules. Deny rules take precedence. `PROTOCOL` and `PORT` select traffic as for `--network-egress-allow`. | | `--network-egress-policy-file PATH` | none | Load bounded IPv4 and IPv6 ranges and rule-local CIDR exclusions from JSON. Requires explicit `--network-egress`; cannot be mixed with explicit allow/deny rule flags. | -| `--host-loopback {allow,deny}` | existing mapping | Control guest access to host loopback services. | +| `--host-loopback {allow,deny}` | existing mapping | Control guest access to host loopback services. Explicit `allow` requires `--host-loopback-forward` or `--network-ingress allow`, whose published ports it then also opens to the host. | | `--network-proxy IPV4:TCP-PORT` | none | Allow one explicit host TCP proxy endpoint. | | `--host-loopback-forward PROTOCOL:HOST_PORT:GUEST_PORT` | none | Publish one TCP or UDP localhost port to the guest; repeat to add forwards. | | `--outcome-report PATH` | none | Write a bounded local JSON outcome report. | @@ -621,11 +621,11 @@ launches. | `--net IPV4/PREFIX` | none | Enable virtio-net with a static guest address. The guest also gets the IPv6 address that embeds it in `fd00::/96`. | | `--network-profile {portable}` | none | Select the required cross-platform network behavior contract; must be specified with `--net`. | | `--network-egress {allow,deny}` | `allow` | Set the default guest egress policy for `run` or `provision`. | -| `--network-ingress {allow,deny}` | `deny` | Set the host ingress policy for `run` or `provision`. The portable profile supports only `deny`. | +| `--network-ingress {allow,deny}` | `deny` | Set the host ingress policy for `run` or `provision`. `allow` publishes every TCP and UDP port on which the workload listens to clients on the host's private networks, and to the host itself only with `--host-loopback allow`; see [Private-network ingress](run.md#private-network-ingress). | | `--network-egress-allow CIDR[:PROTOCOL[:PORT[-PORT]]]` | none | Allow matching guest egress; repeat to add rules. Requires explicit `--network-egress`. `CIDR` is an IPv4 or IPv6 address or network, and a rule matches only its own family. `PROTOCOL` is `tcp`, `udp`, or `icmp` and matches every port when `PORT` is omitted; only `tcp` and `udp` take a `PORT` or an inclusive `PORT-PORT` range, such as `8000-8010`. | | `--network-egress-deny CIDR[:PROTOCOL[:PORT[-PORT]]]` | none | Deny matching guest egress; repeat to add rules. Requires explicit `--network-egress`; deny rules take precedence. `PROTOCOL` and `PORT` select traffic as for `--network-egress-allow`. | | `--network-egress-policy-file PATH` | none | Load bounded IPv4 and IPv6 ranges and rule-local CIDR exclusions for `run` or `provision`. Managed provision persists lowered rules, not this path. Requires explicit `--network-egress`; cannot be mixed with explicit allow/deny rule flags. | -| `--host-loopback {allow,deny}` | existing mapping | Control guest access to host loopback services for `run` or `provision`. | +| `--host-loopback {allow,deny}` | existing mapping | Control guest access to host loopback services for `run` or `provision`. Explicit `allow` requires `--host-loopback-forward` or `--network-ingress allow`, whose published ports it then also opens to the host. | | `--network-proxy IPV4:TCP-PORT` | none | Allow one explicit host TCP proxy endpoint; the IPv4 address must match the guest gateway. | | `--host-loopback-forward PROTOCOL:HOST_PORT:GUEST_PORT` | none | Publish one TCP or UDP localhost port to the guest; repeat to add forwards and set `--host-loopback allow`. | | `--outcome-report PATH` | none | Write a bounded local JSON outcome report for one-shot `run` or managed `exec`. | diff --git a/openvmm b/openvmm index 1feb1f4a..dd8def14 160000 --- a/openvmm +++ b/openvmm @@ -1 +1 @@ -Subproject commit 1feb1f4a39ee1b60445680741cad25b5016e886e +Subproject commit dd8def143ddcbbf174719d851cd5f28cd07489a7 diff --git a/scripts/nvx_tools/microvm_tests.py b/scripts/nvx_tools/microvm_tests.py index bd29853b..31d4492f 100644 --- a/scripts/nvx_tools/microvm_tests.py +++ b/scripts/nvx_tools/microvm_tests.py @@ -1841,29 +1841,6 @@ def serve_allowed_request() -> None: else: unexpected.close() raise RuntimeError("deny-all egress reached the host TCP listener") - - unsupported_command = _directional_network_command( - executable, - kernel, - initrd, - backend, - memory_mib, - egress="allow", - ingress="allow", - ) - with OpenvmmProcess( - unsupported_command, - output_dir / "directional-network-unsupported-ingress.log", - ) as process: - unsupported = process.wait(timeout) - if ( - unsupported.returncode == 0 - or b"--network-ingress allow is unsupported" not in unsupported.output - or BOOT_MARKER in unsupported.output - ): - raise RuntimeError( - "unsupported ingress policy was not rejected before boot" - ) finally: listener.close() server.join(timeout=NETWORK_SERVER_JOIN_TIMEOUT_SECONDS) From 9690ffe83073efadf1ae4cb479863e54bbe46e63 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 13 Oct 2026 08:05:27 -0300 Subject: [PATCH 2/5] doc: index the OpenVMM commit that the private-network ingress pin adds The upstream roadmap indexes the OpenVMM commits between its mapped tip and NVX's pin, which moves from 1feb1f4a3 to dd8def143. Record the new pin, extend the branch DAG to position 315, and add position 315, which lets --network-ingress allow publish the guest's announced listeners to clients on the host's private networks. Part of #275. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/openvmm-upstream-roadmap.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/openvmm-upstream-roadmap.md b/doc/openvmm-upstream-roadmap.md index dbade4c9..fc69c5f6 100644 --- a/doc/openvmm-upstream-roadmap.md +++ b/doc/openvmm-upstream-roadmap.md @@ -5,7 +5,7 @@ - OpenVMM fork base: `e1cdbd916d3aacb216309c8e7779b832546759a1`. - Mapped fork tip: `2728f33ea9a44d2d0fb956d9f01768ae2100fb62`, position 166. The graph, PR summary, and commit mapping below cover positions 001-166. -- NVX's pinned OpenVMM revision: `1feb1f4a39ee1b60445680741cad25b5016e886e`, 148 commits past the +- NVX's pinned OpenVMM revision: `dd8def143ddcbbf174719d851cd5f28cd07489a7`, 149 commits past the mapped tip on the same first-parent chain. These commits are not yet assigned to PRs; see [Commits after the mapped tip](#commits-after-the-mapped-tip). - Upstream main: `1fd455b19e7c69c72a2d375d0ed1cb35d901c688` (2026-10-01), 18 commits past @@ -199,7 +199,7 @@ e1cdbd916 -> PR18/PR23 split [162] -> PR18 fixup [163] -> PR25 fixup [164] -> PR27 [165-166] -> 2728f33ea (mapped tip) - -> unassigned [167-314] -> 1feb1f4a3 (NVX pin) + -> unassigned [167-315] -> dd8def143 (NVX pin) ``` For upstream review, PR01-PR08 and PR14 can be rebased independently onto current upstream main. @@ -215,7 +215,7 @@ can be submitted independently if preferred. ## Commits after the mapped tip -The NVX pin extends the chain past position 166 without merges. The 148 later commits are not +The NVX pin extends the chain past position 166 without merges. The 149 later commits are not yet classified or assigned to PRs. This table groups them by topic in chain order; ranges are inclusive, as in the [PR summary](#pr-summary). @@ -239,6 +239,7 @@ inclusive, as in the [PR summary](#pr-summary). | 310-311 | `9b2073ded..d43dd87d5` | 2 | Restore advances VM time and the RTC by the downtime plus the recorded interval from their stop at capture to the capture anchor | | 312 | `a5955855c` | 1 | Tests of `RENAME_EXCHANGE` on microVM virtio-fs shares: node names, the alias limits, and snapshots | | 313-314 | `5a7aad12c..1feb1f4a3` | 2 | Host paths with commas in `--mount`, whose mode follows the last comma | +| 315 | `dd8def143` | 1 | Private-network ingress: `--network-ingress allow` publishes the guest's announced listeners to clients on the host's private networks | Some of these commits change mapped work rather than add to it. Positions 255-259 remove the legacy microVM clock paths and change files that PR10 and PR17 also change. Position 261 makes From f429675c50429fc39e7d30d1646c4de71a96f161 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Sun, 11 Oct 2026 16:57:39 -0300 Subject: [PATCH 3/5] guest: announce the guest's listeners for private-network ingress The pinned OpenVMM publishes the guest's listeners to the host's private networks when the kernel command line carries virtnet_ingress=allow, but only the listeners that the guest announces to its gateway (#275). nvx-ingress, a static musl helper beside nvx-time and nvx-exit, reads the kernel's TCP and UDP socket tables every 100 milliseconds. A TCP socket in the LISTEN state is a listener, and so is an unconnected UDP socket that persists for a second, which skips short-lived client sockets, such as those of DNS queries. Each UDP socket settles on its own, identified by its inode in the socket tables, and stays announced while it exists, even once a server connects it to its client as busybox nc -u -l does; another socket that reuses a port, such as a client's, settles anew, and a port announces the families of its announced sockets. Sockets bound to loopback addresses, or to IPv6 link-local ones, are unreachable through the gateway. A TCP listener on :: also takes IPv4 connections unless it is IPv6-only, which the helper reads from the kernel's socket diagnostics (INET_DIAG_SKV6ONLY) on every scan; a UDP socket on ::, which the guest kernel's diagnostics do not cover, follows net.ipv6.bindv6only. It sends the canonical announcement to UDP port 1021 of the IPv4 gateway from the highest free port from 1023 down to 512, which only root can bind, whenever the set changes and at least every 2 seconds, and marks the listeners beyond the 64 that an announcement carries as omitted. The common init starts it in the background, as root and with standard input from /dev/null, after configuring the network when the token is present, and reports a network error when the helper or the IPv4 gateway is missing. The Alpine and Ubuntu builds compile it with the other musl helpers, and the Azure Linux Dockerfile compiles it in the same loop, which a unit test now keeps in step with the musl helper list. Its test subcommand runs numbered snapshots of the socket tables, net.ipv6.bindv6only, and a socket diagnostics dump through the same scans, and prints each announcement. scripts/test_guest_ingress.py, which validate-nvx now runs, compiles the helper on Linux and checks the announcement encoding against a golden vector, which listeners are reachable through the gateway, the IPv6-only classification and its fallback, malformed diagnostics, UDP settling, socket identity, and per-socket families, and the truncation to 64 listeners. The diagnostics query times out after a scan interval, so a dump that never ends cannot stall the scans. Part of #275. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/actions/validate-nvx/action.yml | 4 +- docker/Dockerfile | 2 +- guest/common/init | 11 + guest/common/nvx-ingress.c | 660 ++++++++++++++++++++++++ scripts/nvx_tools/build_constants.py | 9 +- scripts/test_guest_ingress.py | 416 +++++++++++++++ scripts/test_nvx_tools.py | 6 + 7 files changed, 1101 insertions(+), 7 deletions(-) create mode 100644 guest/common/nvx-ingress.c create mode 100644 scripts/test_guest_ingress.py diff --git a/.github/actions/validate-nvx/action.yml b/.github/actions/validate-nvx/action.yml index 835b4887..87d771c0 100644 --- a/.github/actions/validate-nvx/action.yml +++ b/.github/actions/validate-nvx/action.yml @@ -10,7 +10,7 @@ runs: run: | set -euo pipefail python3 -m compileall -q scripts - python3 -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py -v + python3 -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_guest_ingress.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py -v python3 scripts/test_adversarial.py -v python3 .github/skills/nvx-host-connect/scripts/test_hosts.py -v python3 scripts/nvx.py --help @@ -29,7 +29,7 @@ runs: $ErrorActionPreference = "Stop" python -m compileall -q scripts if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - python -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py -v + python -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_guest_ingress.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py -v if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python scripts\test_adversarial.py -v if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } diff --git a/docker/Dockerfile b/docker/Dockerfile index 2dea1185..e1328058 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -65,7 +65,7 @@ RUN set -eux; \ for helper in nvx-reseed nvx-mmio-write nvx-port-io nvx-console-pending nvx-managed-agent; do \ cc -static -Os -s -Wall -Wextra -Werror -o "/out/helpers/$helper" "/repo/guest/common/$helper.c"; \ done; \ - for helper in nvx-time nvx-exit; do \ + for helper in nvx-time nvx-exit nvx-ingress; do \ musl-gcc -static -Os -s -Wall -Wextra -Werror -o "/out/helpers/$helper" "/repo/guest/common/$helper.c"; \ done; \ musl-gcc -std=gnu11 -static -O2 -pthread -s -Wall -Wextra -Werror -o /out/helpers/nvx-time-probe \ diff --git a/guest/common/init b/guest/common/init index 787c9cef..5a24affa 100644 --- a/guest/common/init +++ b/guest/common/init @@ -248,6 +248,17 @@ if [ -n "$nip" ]; then IFS=$oldifs fi + # With private-network ingress, the gateway learns the workload's + # listeners from nvx-ingress and publishes them on the host. + ningress=$(sed -n 's/.*virtnet_ingress=\([^ ]*\).*/\1/p' /proc/cmdline) + if [ "$ningress" = allow ]; then + if [ -n "$ngw" ] && [ -x /sbin/nvx-ingress ]; then + /sbin/nvx-ingress "$ngw" /tmp/virtnet-probe.log 2>&1; then diff --git a/guest/common/nvx-ingress.c b/guest/common/nvx-ingress.c new file mode 100644 index 00000000..572abf21 --- /dev/null +++ b/guest/common/nvx-ingress.c @@ -0,0 +1,660 @@ +// nvx-ingress announces the guest's TCP and UDP listeners to the gateway of +// the portable network profile, which publishes them to the host's private +// networks. The init system starts it, as root, when the kernel command line +// carries virtnet_ingress=allow. +// +// Every 100 ms it reads the kernel's socket tables. A TCP socket in the LISTEN +// state is a listener, and so is an unconnected UDP socket that persists for a +// second, which skips short-lived client sockets, such as those of DNS queries. +// Each UDP socket settles on its own, identified by the kernel's inode for it, +// and stays announced while it exists, even once a server connects it to its +// client; another socket that reuses a port, such as a client's, settles anew, +// and a port announces the families of its announced sockets. Sockets bound to +// loopback addresses, or to IPv6 link-local ones, are unreachable through the +// gateway. A TCP listener on the unspecified IPv6 address also takes IPv4 +// connections unless the kernel's socket diagnostics report it IPv6-only; a UDP +// socket there, which the guest kernel's diagnostics do not cover, follows +// net.ipv6.bindv6only. It sends the canonical set of listeners to UDP port 1021 +// of the IPv4 gateway from a privileged port, which only root can bind, +// whenever the set changes and at least every 2 seconds. +// +// An announcement is the magic "NVXL", version 1, a flags byte whose bit 0 +// marks omitted listeners, a big-endian count, and that many entries in +// increasing order of protocol and port: the IP protocol number, a family +// mask (1 for IPv4, 2 for IPv6), and the big-endian port. +// +// "nvx-ingress test DIRECTORY" runs the numbered snapshots in DIRECTORY, from +// 0, as successive scans instead, and prints each scan's time and +// announcement in hexadecimal. A snapshot holds the socket tables under net/, +// optionally sys/net/ipv6/bindv6only, optionally a socket diagnostics dump in +// diag, and the scan time in milliseconds in time. + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ANNOUNCEMENT_PORT 1021 +#define FIRST_SOURCE_PORT 1023 +#define LAST_SOURCE_PORT 512 +#define MAX_ANNOUNCED 64 +#define MAX_TRACKED 1024 +#define SCAN_INTERVAL_MS 100 +#define REFRESH_INTERVAL_MS 2000 +#define UDP_SETTLE_MS 1000 +#define FAMILY_IPV4 1 +#define FAMILY_IPV6 2 +#define PROTOCOL_TCP 6 +#define PROTOCOL_UDP 17 +#define TCP_LISTEN 0x0a +#define UDP_CONNECTED 0x01 +#define UDP_UNCONNECTED 0x07 +#define HEADER_LENGTH 8 +#define ENTRY_LENGTH 4 + +// The kernel's netlink socket diagnostics ABI (linux/netlink.h, +// linux/sock_diag.h, and linux/inet_diag.h), whose headers musl lacks. +#define NETLINK_SOCK_DIAG_PROTOCOL 4 +#define NETLINK_REQUEST 0x01 +#define NETLINK_DUMP 0x300 +#define NETLINK_ERROR 2 +#define NETLINK_DONE 3 +#define SOCK_DIAG_BY_FAMILY 20 +#define INET_DIAG_SKV6ONLY 11 + +struct netlink_header { + uint32_t length; + uint16_t type; + uint16_t flags; + uint32_t sequence; + uint32_t pid; +}; + +struct netlink_address { + sa_family_t family; + unsigned short pad; + uint32_t pid; + uint32_t groups; +}; + +struct netlink_attribute { + uint16_t length; + uint16_t type; +}; + +struct inet_diag_socket_id { + uint16_t source_port; + uint16_t destination_port; + uint32_t source[4]; + uint32_t destination[4]; + uint32_t interface; + uint32_t cookie[2]; +}; + +struct inet_diag_request { + uint8_t family; + uint8_t protocol; + uint8_t extensions; + uint8_t pad; + uint32_t states; + struct inet_diag_socket_id id; +}; + +struct inet_diag_message { + uint8_t family; + uint8_t state; + uint8_t timer; + uint8_t retransmits; + struct inet_diag_socket_id id; + uint32_t expires; + uint32_t receive_queue; + uint32_t write_queue; + uint32_t uid; + uint32_t inode; +}; + +// Whether the IPv6 TCP listeners on the unspecified address at one port take +// IPv4 connections, as the kernel's socket diagnostics report them. +struct wildcard_listener { + uint16_t port; + uint8_t dual_stack; + uint8_t ipv6_only; +}; + +struct listener { + uint8_t protocol; + uint8_t families; + uint16_t port; +}; + +struct table { + struct listener entries[MAX_TRACKED]; + size_t count; +}; + +// A reachable UDP socket, which the kernel's inode identifies while it exists. +struct udp_socket { + uint16_t port; + uint8_t families; + uint8_t connected; + unsigned long inode; +}; + +// A UDP socket that is settling or, once it has persisted unconnected for +// UDP_SETTLE_MS, announced. +struct udp_state { + uint16_t port; + uint8_t families; + uint8_t announced; + unsigned long inode; + uint64_t first_seen_ms; +}; + +static struct table found; +static struct udp_socket udp_sockets[MAX_TRACKED]; +static size_t udp_socket_count; +static struct udp_state udp_states[MAX_TRACKED]; +static size_t udp_state_count; +static struct udp_state next_udp_states[MAX_TRACKED]; +static uint8_t payload[HEADER_LENGTH + MAX_ANNOUNCED * ENTRY_LENGTH]; +static uint8_t last_payload[sizeof(payload)]; +static size_t last_length; +static int ipv6_dual_stack = 1; +static struct wildcard_listener wildcard_tcp[MAX_TRACKED]; +static size_t wildcard_tcp_count; +static uint8_t diag_buffer[32768] __attribute__((aligned(8))); +// The root of the socket tables and sysctls, which test mode replaces. +static const char *proc_root = "/proc"; +static int test_mode; + +static uint64_t now_ms(void) +{ + struct timespec now; + clock_gettime(CLOCK_MONOTONIC, &now); + return (uint64_t)now.tv_sec * 1000 + (uint64_t)now.tv_nsec / 1000000; +} + +static void sleep_ms(unsigned int milliseconds) +{ + struct timespec delay = { + .tv_sec = milliseconds / 1000, + .tv_nsec = (long)(milliseconds % 1000) * 1000000, + }; + while (nanosleep(&delay, &delay) != 0 && errno == EINTR) { + } +} + +// Records a listener, merging the address families of one protocol and port. +static void add(struct table *table, uint8_t protocol, uint16_t port, uint8_t families) +{ + for (size_t index = 0; index < table->count; ++index) { + struct listener *entry = &table->entries[index]; + if (entry->protocol == protocol && entry->port == port) { + entry->families |= families; + return; + } + } + if (table->count < MAX_TRACKED) { + table->entries[table->count++] = (struct listener){ + .protocol = protocol, + .families = families, + .port = port, + }; + } +} + +// Returns the tracked state of the UDP socket with `inode` at `port`, if any. +static const struct udp_state *find_udp_state(uint16_t port, unsigned long inode) +{ + for (size_t index = 0; index < udp_state_count; ++index) { + if (udp_states[index].port == port && udp_states[index].inode == inode) { + return &udp_states[index]; + } + } + return NULL; +} + +static int parse_hex_u32(const char *text, uint32_t *value) +{ + char digits[9]; + char *end; + memcpy(digits, text, 8); + digits[8] = '\0'; + errno = 0; + unsigned long parsed = strtoul(digits, &end, 16); + if (errno != 0 || *end != '\0' || end != digits + 8) { + return 0; + } + *value = (uint32_t)parsed; + return 1; +} + +// Records the IPV6_V6ONLY state of an IPv6 TCP listener on the unspecified +// address from one socket diagnostics message. +static void note_wildcard_tcp(const uint8_t *message, size_t length) +{ + struct inet_diag_message diag; + if (length < sizeof(diag)) { + return; + } + memcpy(&diag, message, sizeof(diag)); + if (diag.family != AF_INET6 || diag.state != TCP_LISTEN) { + return; + } + for (int word = 0; word < 4; ++word) { + if (diag.id.source[word] != 0) { + return; + } + } + uint16_t port = ntohs(diag.id.source_port); + for (size_t offset = sizeof(diag); offset + sizeof(struct netlink_attribute) <= length;) { + struct netlink_attribute attribute; + memcpy(&attribute, message + offset, sizeof(attribute)); + if (attribute.length < sizeof(attribute) || attribute.length > length - offset) { + return; + } + if (attribute.type == INET_DIAG_SKV6ONLY && attribute.length > sizeof(attribute)) { + uint8_t ipv6_only = message[offset + sizeof(attribute)] != 0; + struct wildcard_listener *entry = NULL; + for (size_t index = 0; index < wildcard_tcp_count; ++index) { + if (wildcard_tcp[index].port == port) { + entry = &wildcard_tcp[index]; + } + } + if (entry == NULL && wildcard_tcp_count < MAX_TRACKED) { + entry = &wildcard_tcp[wildcard_tcp_count++]; + *entry = (struct wildcard_listener){.port = port}; + } + if (entry != NULL) { + entry->dual_stack |= (uint8_t)!ipv6_only; + entry->ipv6_only |= ipv6_only; + } + return; + } + offset += (attribute.length + 3u) & ~3u; + } +} + +// Records the IPv6 TCP listeners on the unspecified address of one part of a +// socket diagnostics dump, returning 0 once the dump ends or is malformed. +static int parse_wildcard_tcp(const uint8_t *buffer, size_t length) +{ + for (size_t offset = 0; offset + sizeof(struct netlink_header) <= length;) { + struct netlink_header header; + memcpy(&header, buffer + offset, sizeof(header)); + if (header.length < sizeof(header) || header.length > length - offset || + header.type == NETLINK_DONE || header.type == NETLINK_ERROR) { + return 0; + } + if (header.type == SOCK_DIAG_BY_FAMILY) { + note_wildcard_tcp(buffer + offset + sizeof(header), header.length - sizeof(header)); + } + offset += (header.length + 3u) & ~3u; + } + return 1; +} + +// Asks the kernel's socket diagnostics, or in test mode the snapshot's dump, +// which IPv6 TCP listeners on the unspecified address are IPv6-only. Without +// an answer, none is known. +static void query_wildcard_tcp(void) +{ + wildcard_tcp_count = 0; + if (test_mode) { + char path[4096]; + if (snprintf(path, sizeof(path), "%s/diag", proc_root) >= (int)sizeof(path)) { + return; + } + FILE *file = fopen(path, "rbe"); + if (file != NULL) { + size_t length = fread(diag_buffer, 1, sizeof(diag_buffer), file); + fclose(file); + parse_wildcard_tcp(diag_buffer, length); + } + return; + } + int fd = socket(AF_NETLINK, SOCK_DGRAM | SOCK_CLOEXEC, NETLINK_SOCK_DIAG_PROTOCOL); + if (fd < 0) { + return; + } + // A dump that never ends must not stall the scans. + struct timeval timeout = {.tv_usec = SCAN_INTERVAL_MS * 1000}; + struct { + struct netlink_header header; + struct inet_diag_request request; + } query = { + .header = + { + .length = sizeof(query), + .type = SOCK_DIAG_BY_FAMILY, + .flags = NETLINK_REQUEST | NETLINK_DUMP, + }, + .request = + { + .family = AF_INET6, + .protocol = IPPROTO_TCP, + .states = 1u << TCP_LISTEN, + }, + }; + struct netlink_address kernel = {.family = AF_NETLINK}; + if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)) != 0 || + sendto(fd, &query, sizeof(query), 0, (struct sockaddr *)&kernel, sizeof(kernel)) != + (ssize_t)sizeof(query)) { + close(fd); + return; + } + for (;;) { + ssize_t received = recv(fd, diag_buffer, sizeof(diag_buffer), 0); + if (received <= 0 || !parse_wildcard_tcp(diag_buffer, (size_t)received)) { + break; + } + } + close(fd); +} + +// Returns the families through which an IPv6 socket of `protocol` on the +// unspecified address at `port` is reachable. +static uint8_t wildcard_families(uint8_t protocol, uint16_t port) +{ + if (protocol == PROTOCOL_TCP) { + for (size_t index = 0; index < wildcard_tcp_count; ++index) { + const struct wildcard_listener *entry = &wildcard_tcp[index]; + if (entry->port == port && (entry->dual_stack || entry->ipv6_only)) { + return entry->dual_stack ? FAMILY_IPV4 | FAMILY_IPV6 : FAMILY_IPV6; + } + } + } + return ipv6_dual_stack ? FAMILY_IPV4 | FAMILY_IPV6 : FAMILY_IPV6; +} + +// Returns the families through which a socket of `protocol` at `port`, bound +// to the kernel's hexadecimal rendering of an address, is reachable from the +// gateway, or 0. The kernel prints each 32-bit word of the address as it lies +// in memory. +static uint8_t reachable_families(const char *address, int ipv6, uint8_t protocol, uint16_t port) +{ + if (!ipv6) { + struct in_addr ipv4; + if (strlen(address) != 8 || !parse_hex_u32(address, &ipv4.s_addr)) { + return 0; + } + return (ntohl(ipv4.s_addr) >> 24) == 127 ? 0 : FAMILY_IPV4; + } + struct in6_addr ipv6_address; + if (strlen(address) != 32) { + return 0; + } + for (int word = 0; word < 4; ++word) { + uint32_t value; + if (!parse_hex_u32(address + word * 8, &value)) { + return 0; + } + memcpy(&ipv6_address.s6_addr[word * 4], &value, sizeof(value)); + } + if (IN6_IS_ADDR_UNSPECIFIED(&ipv6_address)) { + return wildcard_families(protocol, port); + } + if (IN6_IS_ADDR_LOOPBACK(&ipv6_address) || IN6_IS_ADDR_LINKLOCAL(&ipv6_address)) { + return 0; + } + if (IN6_IS_ADDR_V4MAPPED(&ipv6_address)) { + return ipv6_address.s6_addr[12] == 127 ? 0 : FAMILY_IPV4; + } + return FAMILY_IPV6; +} + +// Adds the TCP listeners of one of the kernel's socket tables, `name` under +// net/, to `found`, and its reachable UDP sockets, connected or not, to +// `udp_sockets`. +static void scan(const char *name, uint8_t protocol, int ipv6, uint16_t own_port) +{ + char line[512]; + char path[4096]; + if (snprintf(path, sizeof(path), "%s/net/%s", proc_root, name) >= (int)sizeof(path)) { + return; + } + FILE *file = fopen(path, "re"); + if (file == NULL) { + return; + } + if (fgets(line, sizeof(line), file) == NULL) { + fclose(file); + return; + } + while (fgets(line, sizeof(line), file) != NULL) { + char local[64]; + char remote[64]; + unsigned int state; + unsigned long inode; + if (sscanf(line, " %*u: %63s %63s %x %*s %*s %*s %*u %*u %lu", local, remote, &state, + &inode) != 4) { + continue; + } + char *local_port = strrchr(local, ':'); + char *remote_port = strrchr(remote, ':'); + if (local_port == NULL || remote_port == NULL) { + continue; + } + *local_port++ = '\0'; + *remote_port++ = '\0'; + unsigned long port = strtoul(local_port, NULL, 16); + if (port == 0 || port > UINT16_MAX) { + continue; + } + uint8_t connected = 0; + if (protocol == PROTOCOL_TCP) { + if (state != TCP_LISTEN) { + continue; + } + } else if (state == UDP_CONNECTED) { + connected = 1; + } else if (state != UDP_UNCONNECTED || strtoul(remote_port, NULL, 16) != 0 || + strspn(remote, "0") != strlen(remote) || port == own_port) { + continue; + } + uint8_t families = reachable_families(local, ipv6, protocol, (uint16_t)port); + if (families == 0) { + continue; + } + if (protocol == PROTOCOL_TCP) { + add(&found, protocol, (uint16_t)port, families); + } else if (udp_socket_count < MAX_TRACKED) { + udp_sockets[udp_socket_count++] = (struct udp_socket){ + .port = (uint16_t)port, + .families = families, + .connected = connected, + .inode = inode, + }; + } + } + fclose(file); +} + +static int compare_listeners(const void *left, const void *right) +{ + const struct listener *a = left; + const struct listener *b = right; + if (a->protocol != b->protocol) { + return a->protocol < b->protocol ? -1 : 1; + } + return a->port < b->port ? -1 : a->port > b->port; +} + +// Scans the socket tables and encodes the announcement into `payload`, +// returning its length. +static size_t collect(uint16_t own_port, uint64_t now) +{ + found.count = 0; + udp_socket_count = 0; + query_wildcard_tcp(); + scan("tcp", PROTOCOL_TCP, 0, own_port); + scan("tcp6", PROTOCOL_TCP, 1, own_port); + scan("udp", PROTOCOL_UDP, 0, own_port); + scan("udp6", PROTOCOL_UDP, 1, own_port); + + // An unconnected UDP socket is announced once it persists for + // UDP_SETTLE_MS, and stays announced, even once a server connects it to a + // client, while it exists. Each port announces the families of its + // announced sockets. + size_t next_count = 0; + for (size_t index = 0; index < udp_socket_count; ++index) { + const struct udp_socket *socket = &udp_sockets[index]; + const struct udp_state *previous = find_udp_state(socket->port, socket->inode); + struct udp_state next = { + .port = socket->port, + .families = socket->families, + .inode = socket->inode, + .first_seen_ms = previous != NULL ? previous->first_seen_ms : now, + }; + if (previous != NULL && previous->announced) { + next.announced = 1; + } else if (socket->connected) { + continue; + } else { + next.announced = now - next.first_seen_ms >= UDP_SETTLE_MS; + } + next_udp_states[next_count++] = next; + if (next.announced) { + add(&found, PROTOCOL_UDP, next.port, next.families); + } + } + memcpy(udp_states, next_udp_states, next_count * sizeof(udp_states[0])); + udp_state_count = next_count; + size_t announced = found.count; + + qsort(found.entries, announced, sizeof(found.entries[0]), compare_listeners); + size_t count = announced < MAX_ANNOUNCED ? announced : MAX_ANNOUNCED; + memcpy(payload, "NVXL", 4); + payload[4] = 1; + payload[5] = announced > MAX_ANNOUNCED; + payload[6] = (uint8_t)(count >> 8); + payload[7] = (uint8_t)count; + for (size_t index = 0; index < count; ++index) { + uint8_t *entry = &payload[HEADER_LENGTH + index * ENTRY_LENGTH]; + entry[0] = found.entries[index].protocol; + entry[1] = found.entries[index].families; + entry[2] = (uint8_t)(found.entries[index].port >> 8); + entry[3] = (uint8_t)found.entries[index].port; + } + return HEADER_LENGTH + count * ENTRY_LENGTH; +} + +static void read_bindv6only(void) +{ + char path[4096]; + if (snprintf(path, sizeof(path), "%s/sys/net/ipv6/bindv6only", proc_root) >= + (int)sizeof(path)) { + return; + } + FILE *file = fopen(path, "re"); + if (file == NULL) { + return; + } + int value; + if (fscanf(file, "%d", &value) == 1) { + ipv6_dual_stack = value == 0; + } + fclose(file); +} + +// Runs the snapshots in `directory` as successive scans; see the top of the +// file. +static int run_test(const char *directory) +{ + static char root[4096]; + test_mode = 1; + for (unsigned int index = 0;; ++index) { + char path[4200]; + if (snprintf(root, sizeof(root), "%s/%u", directory, index) >= (int)sizeof(root) || + snprintf(path, sizeof(path), "%s/time", root) >= (int)sizeof(path)) { + return 1; + } + FILE *file = fopen(path, "re"); + if (file == NULL) { + return index == 0; + } + unsigned long long now; + int parsed = fscanf(file, "%llu", &now); + fclose(file); + if (parsed != 1) { + return 1; + } + proc_root = root; + ipv6_dual_stack = 1; + read_bindv6only(); + size_t length = collect(FIRST_SOURCE_PORT, now); + printf("%llu ", now); + for (size_t offset = 0; offset < length; ++offset) { + printf("%02x", payload[offset]); + } + printf("\n"); + } +} + +int main(int argc, char **argv) +{ + if (argc == 3 && strcmp(argv[1], "test") == 0) { + return run_test(argv[2]); + } + struct sockaddr_in gateway = { + .sin_family = AF_INET, + .sin_port = htons(ANNOUNCEMENT_PORT), + }; + if (argc != 2 || inet_pton(AF_INET, argv[1], &gateway.sin_addr) != 1) { + fprintf(stderr, "usage: nvx-ingress GATEWAY_IPV4\n"); + return 2; + } + signal(SIGINT, SIG_IGN); + signal(SIGHUP, SIG_IGN); + read_bindv6only(); + + int sock = socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0); + if (sock < 0) { + fprintf(stderr, "nvx-ingress: socket: %s\n", strerror(errno)); + return 1; + } + uint16_t own_port = 0; + for (int port = FIRST_SOURCE_PORT; port >= LAST_SOURCE_PORT && own_port == 0; --port) { + struct sockaddr_in source = { + .sin_family = AF_INET, + .sin_port = htons((uint16_t)port), + .sin_addr.s_addr = htonl(INADDR_ANY), + }; + if (bind(sock, (struct sockaddr *)&source, sizeof(source)) == 0) { + own_port = (uint16_t)port; + } else if (errno != EADDRINUSE) { + fprintf(stderr, "nvx-ingress: bind privileged port: %s\n", strerror(errno)); + return 1; + } + } + if (own_port == 0) { + fprintf(stderr, "nvx-ingress: no privileged source port is free\n"); + return 1; + } + + uint64_t last_sent = 0; + for (;;) { + uint64_t now = now_ms(); + size_t length = collect(own_port, now); + if (length != last_length || memcmp(payload, last_payload, length) != 0 || + now - last_sent >= REFRESH_INTERVAL_MS) { + if (sendto(sock, payload, length, 0, (struct sockaddr *)&gateway, + sizeof(gateway)) == (ssize_t)length) { + memcpy(last_payload, payload, length); + last_length = length; + last_sent = now; + } + } + sleep_ms(SCAN_INTERVAL_MS); + } +} diff --git a/scripts/nvx_tools/build_constants.py b/scripts/nvx_tools/build_constants.py index 35f34574..dca473bc 100644 --- a/scripts/nvx_tools/build_constants.py +++ b/scripts/nvx_tools/build_constants.py @@ -338,11 +338,12 @@ class InitramfsBuildConstants: "nvx-console-pending", "nvx-managed-agent", ) - # Time tools and nvx-exit are linked against musl: a static glibc binary - # adds about 700 KB to the initramfs, and every unpacked kilobyte delays - # cold boot. nvx-exit is also a restored guest's usual first exec. + # Time tools, nvx-exit, and nvx-ingress are linked against musl: a static + # glibc binary adds about 700 KB to the initramfs, and every unpacked + # kilobyte delays cold boot. nvx-exit is also a restored guest's usual + # first exec. MUSL_COMPILER: Final = "musl-gcc" - MUSL_STATIC_HELPERS: Final = ("nvx-time", "nvx-exit") + MUSL_STATIC_HELPERS: Final = ("nvx-time", "nvx-exit", "nvx-ingress") TIME_PROBE_NAME: Final = "nvx-time-probe" TIME_PROBE_CFLAGS: Final = ( "-std=gnu11", diff --git a/scripts/test_guest_ingress.py b/scripts/test_guest_ingress.py new file mode 100644 index 00000000..0ddd33d5 --- /dev/null +++ b/scripts/test_guest_ingress.py @@ -0,0 +1,416 @@ +#!/usr/bin/env python3 +"""Tests for the guest listener announcer, guest/common/nvx-ingress.c. + +The helper's test mode runs synthetic snapshots of the kernel's socket tables +and socket diagnostics through the scans that a guest runs, so these tests +check what it announces without a guest or privileges. +""" + +from __future__ import annotations + +import ipaddress +import shutil +import struct +import subprocess +import sys +import tempfile +import unittest +from dataclasses import dataclass +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +SOURCE = REPO_ROOT / "guest" / "common" / "nvx-ingress.c" + +TCP = 6 +UDP = 17 +IPV4 = 1 +IPV6 = 2 +BOTH = IPV4 | IPV6 +TCP_LISTEN = 0x0A +TCP_ESTABLISHED = 0x01 +UDP_CONNECTED = 0x01 +UDP_UNCONNECTED = 0x07 +UDP_SETTLE_MS = 1000 +# The helper's first source port, which its test mode excludes like its own. +OWN_PORT = 1023 +TABLE_HEADER = ( + " sl local_address rem_address st tx_queue rx_queue tr tm->when " + "retrnsmt uid timeout inode\n" +) +NETLINK_HEADER = struct.Struct("=IHHII") +SOCK_DIAG_BY_FAMILY = 20 +NETLINK_DONE = 3 +INET_DIAG_SKV6ONLY = 11 +AF_INET6 = 10 + + +def kernel_address(address: str) -> str: + """Render an address as the kernel's socket tables print it on x86.""" + packed = ipaddress.ip_address(address).packed + return "".join( + f"{int.from_bytes(packed[offset : offset + 4], 'little'):08X}" + for offset in range(0, len(packed), 4) + ) + + +@dataclass(frozen=True) +class Socket: + protocol: int + address: str + port: int + inode: int + state: int + peer: tuple[str, int] | None = None + + @property + def ipv6(self) -> bool: + return ipaddress.ip_address(self.address).version == 6 + + @property + def table(self) -> str: + return ("tcp" if self.protocol == TCP else "udp") + ("6" if self.ipv6 else "") + + def line(self, index: int) -> str: + remote, remote_port = self.peer or ("::" if self.ipv6 else "0.0.0.0", 0) + return ( + f"{index:4}: {kernel_address(self.address)}:{self.port:04X} " + f"{kernel_address(remote)}:{remote_port:04X} {self.state:02X} " + "00000000:00000000 00:00000000 00000000 0 0 " + f"{self.inode} 2 0000000000000000\n" + ) + + +def tcp_listener(address: str, port: int, inode: int = 1) -> Socket: + return Socket(TCP, address, port, inode, TCP_LISTEN) + + +def udp_socket( + address: str, port: int, inode: int, peer: tuple[str, int] | None = None +) -> Socket: + state = UDP_UNCONNECTED if peer is None else UDP_CONNECTED + return Socket(UDP, address, port, inode, state, peer) + + +def diag_listener(port: int, ipv6_only: bool, address: str = "::") -> bytes: + """Encode a socket diagnostics message for an IPv6 TCP listener.""" + message = struct.pack("=BBBB", AF_INET6, TCP_LISTEN, 0, 0) + message += struct.pack("!HH", port, 0) + message += ipaddress.IPv6Address(address).packed + bytes(16) + message += struct.pack("=III", 0, 0, 0) + struct.pack("=IIIII", 0, 0, 0, 0, 0) + message += struct.pack("=HH", 5, INET_DIAG_SKV6ONLY) + bytes( + (int(ipv6_only), 0, 0, 0) + ) + return ( + NETLINK_HEADER.pack( + NETLINK_HEADER.size + len(message), SOCK_DIAG_BY_FAMILY, 2, 1, 0 + ) + + message + ) + + +def diag_done() -> bytes: + return NETLINK_HEADER.pack(NETLINK_HEADER.size + 4, NETLINK_DONE, 2, 1, 0) + bytes( + 4 + ) + + +@dataclass(frozen=True) +class Snapshot: + time_ms: int + sockets: tuple[Socket, ...] = () + bindv6only: int | None = None + diag: bytes | None = None + + def write(self, root: Path) -> None: + (root / "net").mkdir(parents=True) + for table in ("tcp", "tcp6", "udp", "udp6"): + sockets = [socket for socket in self.sockets if socket.table == table] + (root / "net" / table).write_text( + TABLE_HEADER + + "".join(socket.line(index) for index, socket in enumerate(sockets)), + encoding="utf-8", + ) + if self.bindv6only is not None: + sysctl = root / "sys" / "net" / "ipv6" + sysctl.mkdir(parents=True) + (sysctl / "bindv6only").write_text(f"{self.bindv6only}\n", encoding="utf-8") + if self.diag is not None: + (root / "diag").write_bytes(self.diag) + (root / "time").write_text(f"{self.time_ms}\n", encoding="utf-8") + + +@dataclass(frozen=True) +class Announcement: + payload: bytes + truncated: bool + listeners: dict[tuple[int, int], int] + order: tuple[tuple[int, int], ...] + + @classmethod + def decode(cls, payload: bytes) -> Announcement: + magic, version, flags, count = struct.unpack("!4sBBH", payload[:8]) + if magic != b"NVXL" or version != 1 or len(payload) != 8 + 4 * count: + raise AssertionError(f"malformed announcement {payload.hex()}") + entries = [ + struct.unpack("!BBH", payload[8 + 4 * index : 12 + 4 * index]) + for index in range(count) + ] + return cls( + payload, + bool(flags & 1), + {(protocol, port): families for protocol, families, port in entries}, + tuple((protocol, port) for protocol, _families, port in entries), + ) + + +@unittest.skipUnless(sys.platform.startswith("linux"), "nvx-ingress builds on Linux") +class GuestIngressTests(unittest.TestCase): + binary: Path + directory: tempfile.TemporaryDirectory[str] + + @classmethod + def setUpClass(cls) -> None: + compiler = shutil.which("cc") or shutil.which("gcc") + if compiler is None: + raise unittest.SkipTest("a C compiler is unavailable") + cls.directory = tempfile.TemporaryDirectory() + cls.binary = Path(cls.directory.name) / "nvx-ingress" + subprocess.run( + [ + compiler, + "-O1", + "-Wall", + "-Wextra", + "-Werror", + "-o", + str(cls.binary), + str(SOURCE), + ], + check=True, + capture_output=True, + timeout=120, + ) + + @classmethod + def tearDownClass(cls) -> None: + cls.directory.cleanup() + + def announce(self, *snapshots: Snapshot) -> list[Announcement]: + with tempfile.TemporaryDirectory() as temporary: + for index, snapshot in enumerate(snapshots): + snapshot.write(Path(temporary) / str(index)) + result = subprocess.run( + [str(self.binary), "test", temporary], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + lines = result.stdout.splitlines() + self.assertEqual( + [int(line.split()[0]) for line in lines], + [snapshot.time_ms for snapshot in snapshots], + ) + return [Announcement.decode(bytes.fromhex(line.split()[1])) for line in lines] + + def test_announcement_golden_vector(self): + (announcement,) = self.announce( + Snapshot( + 0, + (tcp_listener("0.0.0.0", 8080), tcp_listener("fd00::a00:2", 8443)), + ) + ) + self.assertEqual(announcement.payload.hex(), "4e56584c0100000206011f90060220fb") + + def test_only_listeners_reachable_through_the_gateway_are_announced(self): + sockets = ( + tcp_listener("0.0.0.0", 8080), + tcp_listener("192.0.2.2", 8081), + tcp_listener("127.0.0.1", 8082), + tcp_listener("127.0.0.53", 8083), + tcp_listener("::", 8084), + tcp_listener("fd00::a00:2", 8085), + tcp_listener("::1", 8086), + tcp_listener("fe80::1", 8087), + tcp_listener("::ffff:127.0.0.1", 8088), + tcp_listener("::ffff:192.0.2.2", 8089), + # One port's IPv4 and IPv6 listeners merge their families. + tcp_listener("0.0.0.0", 8090), + tcp_listener("fd00::a00:2", 8090, inode=2), + # Established connections are not listeners. + Socket(TCP, "192.0.2.2", 8091, 3, TCP_ESTABLISHED, ("192.0.2.1", 40000)), + ) + (announcement,) = self.announce(Snapshot(0, sockets)) + self.assertEqual( + announcement.listeners, + { + (TCP, 8080): IPV4, + (TCP, 8081): IPV4, + (TCP, 8084): BOTH, + (TCP, 8085): IPV6, + (TCP, 8089): IPV4, + (TCP, 8090): BOTH, + }, + ) + self.assertFalse(announcement.truncated) + + def test_ipv6_only_tcp_listeners_follow_socket_diagnostics(self): + sockets = tuple( + tcp_listener("::", port, inode=port) for port in range(7001, 7006) + ) + diag = ( + diag_listener(7001, ipv6_only=True) + + diag_listener(7002, ipv6_only=False) + # A port that IPv6-only and dual-stack sockets share takes IPv4. + + diag_listener(7003, ipv6_only=True) + + diag_listener(7003, ipv6_only=False) + # Only listeners on the unspecified address are classified. + + diag_listener(7004, ipv6_only=True, address="fd00::a00:2") + + diag_done() + # Messages after the end of the dump are ignored. + + diag_listener(7005, ipv6_only=True) + ) + default, ipv6_only_default = self.announce( + Snapshot(0, sockets, diag=diag), + Snapshot(100, sockets, bindv6only=1, diag=diag_listener(7002, False)), + ) + self.assertEqual( + default.listeners, + { + (TCP, 7001): IPV6, + (TCP, 7002): BOTH, + (TCP, 7003): BOTH, + (TCP, 7004): BOTH, + (TCP, 7005): BOTH, + }, + ) + # Without diagnostics for a socket, net.ipv6.bindv6only decides. + self.assertEqual( + ipv6_only_default.listeners, + { + (TCP, 7001): IPV6, + (TCP, 7002): BOTH, + (TCP, 7003): IPV6, + (TCP, 7004): IPV6, + (TCP, 7005): IPV6, + }, + ) + + def test_malformed_socket_diagnostics_are_ignored(self): + sockets = (tcp_listener("::", 7101),) + valid = diag_listener(7101, ipv6_only=True) + short_attribute = bytearray(valid) + struct.pack_into("=H", short_attribute, NETLINK_HEADER.size + 72, 2) + long_attribute = bytearray(valid) + struct.pack_into("=H", long_attribute, NETLINK_HEADER.size + 72, 200) + for name, diag in ( + ("garbage", bytes(range(37))), + ("short header", NETLINK_HEADER.pack(8, SOCK_DIAG_BY_FAMILY, 0, 0, 0)), + ( + "overlong message", + NETLINK_HEADER.pack(4096, SOCK_DIAG_BY_FAMILY, 0, 0, 0), + ), + ("truncated message", valid[:40]), + ("short attribute", bytes(short_attribute)), + ("overlong attribute", bytes(long_attribute)), + ): + with self.subTest(name): + (announcement,) = self.announce(Snapshot(0, sockets, diag=diag)) + self.assertEqual(announcement.listeners, {(TCP, 7101): BOTH}) + + def test_udp_sockets_settle_before_they_are_announced(self): + sockets = ( + udp_socket("0.0.0.0", 7201, 10), + udp_socket("::", 7202, 11), + # The helper's own source port is never announced. + udp_socket("0.0.0.0", OWN_PORT, 12), + # Nor is a UDP socket bound to loopback. + udp_socket("127.0.0.1", 7203, 13), + ) + announcements = self.announce( + Snapshot(0, sockets), + Snapshot(UDP_SETTLE_MS - 1, sockets), + Snapshot(UDP_SETTLE_MS, sockets), + Snapshot(UDP_SETTLE_MS + 100, sockets, bindv6only=1), + ) + self.assertEqual(announcements[0].listeners, {}) + self.assertEqual(announcements[1].listeners, {}) + self.assertEqual( + announcements[2].listeners, {(UDP, 7201): IPV4, (UDP, 7202): BOTH} + ) + self.assertEqual( + announcements[3].listeners, {(UDP, 7201): IPV4, (UDP, 7202): IPV6} + ) + + def test_udp_ports_follow_the_identity_of_their_sockets(self): + server = udp_socket("0.0.0.0", 7301, 20) + replacement = udp_socket("0.0.0.0", 7301, 21) + connected = udp_socket("192.0.2.2", 7301, 21, ("198.51.100.7", 5353)) + client = udp_socket("192.0.2.2", 7301, 22, ("192.0.2.1", 53)) + timeline = ( + (0, (server,), None), + (UDP_SETTLE_MS, (server,), IPV4), + # Another socket on the port settles anew. + (UDP_SETTLE_MS + 100, (replacement,), None), + (2 * UDP_SETTLE_MS + 99, (replacement,), None), + (2 * UDP_SETTLE_MS + 100, (replacement,), IPV4), + # A server that connects its announced socket keeps the port. + (2 * UDP_SETTLE_MS + 200, (connected,), IPV4), + (5 * UDP_SETTLE_MS, (connected,), IPV4), + # A connected client that reuses the port never takes it over. + (5 * UDP_SETTLE_MS + 100, (client,), None), + (9 * UDP_SETTLE_MS, (client,), None), + ) + announcements = self.announce( + *(Snapshot(time_ms, sockets) for time_ms, sockets, _ in timeline) + ) + for (time_ms, _sockets, families), announcement in zip( + timeline, announcements, strict=True + ): + with self.subTest(time_ms=time_ms): + expected = {} if families is None else {(UDP, 7301): families} + self.assertEqual(announcement.listeners, expected) + + def test_udp_ports_announce_the_families_of_their_sockets(self): + ipv4 = udp_socket("0.0.0.0", 7401, 30) + ipv6 = udp_socket("::", 7401, 31) + ipv4_connected = udp_socket("192.0.2.2", 7401, 30, ("198.51.100.7", 4000)) + ipv6_connected = udp_socket("fd00::a00:2", 7401, 31, ("fd12::7", 4000)) + timeline = ( + (0, (ipv4, ipv6), None), + (UDP_SETTLE_MS, (ipv4, ipv6), BOTH), + (UDP_SETTLE_MS + 100, (ipv4_connected, ipv6), BOTH), + (UDP_SETTLE_MS + 200, (ipv4_connected, ipv6_connected), BOTH), + (UDP_SETTLE_MS + 300, (ipv6_connected,), IPV6), + (UDP_SETTLE_MS + 400, (), None), + ) + announcements = self.announce( + *( + Snapshot(time_ms, sockets, bindv6only=1) + for time_ms, sockets, _ in timeline + ) + ) + for (time_ms, _sockets, families), announcement in zip( + timeline, announcements, strict=True + ): + with self.subTest(time_ms=time_ms): + expected = {} if families is None else {(UDP, 7401): families} + self.assertEqual(announcement.listeners, expected) + + def test_announcements_carry_the_lowest_64_listeners(self): + sockets = tuple( + tcp_listener("0.0.0.0", port, inode=port) for port in range(2069, 1999, -1) + ) + (announcement,) = self.announce(Snapshot(0, sockets)) + self.assertTrue(announcement.truncated) + self.assertEqual( + announcement.order, tuple((TCP, port) for port in range(2000, 2064)) + ) + (complete,) = self.announce(Snapshot(0, sockets[-64:])) + self.assertFalse(complete.truncated) + self.assertEqual(len(complete.order), 64) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_nvx_tools.py b/scripts/test_nvx_tools.py index f034a7e7..2f283e8d 100644 --- a/scripts/test_nvx_tools.py +++ b/scripts/test_nvx_tools.py @@ -9298,6 +9298,12 @@ def find_tool(name: str) -> str: encoding="utf-8" ) self.assertIn(" musl-tools \\\n", dockerfile) + self.assertIn( + " for helper in " + + " ".join(InitramfsBuildConstants.MUSL_STATIC_HELPERS) + + "; do \\\n musl-gcc ", + dockerfile, + ) self.assertEqual(helpers[probe]["source_sha256"], common.sha256_file(source)) self.assertEqual( helpers[probe]["binary_sha256"], From f1178f9ff59c61aea9ba749e91fd7b5e8b61e2bb Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Sun, 11 Oct 2026 16:57:39 -0300 Subject: [PATCH 4/5] aci_edge_sandboxes: apply private-network ingress in OpenVMM The OpenVMM backend rejected network.ingress.default: allow with policy_validation before provisioning, because OpenVMM could not publish a workload's listeners (#275). The pinned OpenVMM and guest publish them, so the backend now reports the ingress_allow capability, and nvx-rs's network lowering, which the backend uses, passes the ingress default to OpenVMM as --network-ingress beside the egress posture, with host loopback still denied, instead of refusing ingress allow. A policy that denies egress without allow rules still attaches the network device when it allows ingress. hostLoopback: allow stays unsupported, so the fake OpenVMM test that covered the ingress rejection now checks that rejection instead, and the launch-arguments test checks the ingress arguments under denied egress. A unit test covers the OpenVMM options for ingress allow under either egress default. The new end-to-end test private_network_ingress_publishes_workload_listeners runs nc -l in a sandbox that denies egress and allows ingress, waits until the host holds the workload's port, requires the host's own connection to be refused before it reaches the workload, waits until the closed listener's port is withdrawn, and checks that ingress deny publishes nothing. The README's policy honor matrix and network rules, and the documentation of the backend and of nvx-rs's network posture, describe what ingress admits and refuses. Part of #275. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- aci_edge_sandboxes/README.md | 21 +++++-- aci_edge_sandboxes/src/openvmm/mod.rs | 11 +++- aci_edge_sandboxes/tests/openvmm_e2e.rs | 70 ++++++++++++++++++++++++ aci_edge_sandboxes/tests/openvmm_fake.rs | 24 +++++++- nvx-rs/src/host/network.rs | 55 ++++++++++++------- 5 files changed, 151 insertions(+), 30 deletions(-) diff --git a/aci_edge_sandboxes/README.md b/aci_edge_sandboxes/README.md index c96809c2..e12a535d 100644 --- a/aci_edge_sandboxes/README.md +++ b/aci_edge_sandboxes/README.md @@ -358,8 +358,9 @@ provisioned again to accept the change. behind the gateway's counterpart, such as `fd00::a00:1`. The guest names the gateway as its DNS server in `/etc/resolv.conf` when the policy allows TCP or UDP port 53 to it, and otherwise the gateway's IPv6 address when the policy allows DNS to that one. Choose a guest - network that contains no address the guest must reach. Ingress must be `deny`, and - host-loopback access must be `deny` unless ports are forwarded, as described next. + network that contains no address the guest must reach. Unlike with the + [OpenVMM backend](#network-rules), ingress must be `deny`, and host-loopback access must be + `deny` unless ports are forwarded, as described next. ### Native proxy and forwarded ports @@ -545,7 +546,7 @@ terminate the VM when the caller exits. | `filesystem.deniedPaths` | hidden inside mapped paths | n/a | | `network.egress.default` | applied (`allow` or `deny`) | n/a | | `network.egress.allow`, `deny` | applied; see [Network rules](#network-rules) | n/a | -| `network.ingress.default` | `deny` applied; `allow` rejected | n/a | +| `network.ingress.default` | applied (`allow` or `deny`); see [Network rules](#network-rules) | n/a | | `network.ingress.hostLoopback` | `deny` applied (also when omitted); `allow` rejected | n/a | | `microvm.provision.memoryMib` | applied | n/a | | `process.commandLine` | n/a | run as `/bin/sh -c `; at most 4096 bytes | @@ -733,6 +734,17 @@ otherwise the guest gets `10.0.0.2/24` behind the profile's NAT gateway `10.0.0.1`, which also serves DNS, and `fd00::a00:2/120` behind the gateway's IPv6 address `fd00::a00:1`. +With `network.ingress.default: allow`, OpenVMM listens on every host address at +each TCP and UDP port on which the workload listens, within about a second of +the workload starting to listen, and relays clients on the host's private +networks, whose addresses are not globally reachable, to the guest. It refuses +other clients, clients inside the guest network, which the guest would take for +its neighbors, IPv6 link-local clients, and the host itself, because host +loopback stays denied. Replies to admitted clients pass whatever the egress +policy. A port that a host process already holds is not published, and clients +on other hosts need the host's firewall to admit them to OpenVMM; see +[Private-network ingress](../doc/run.md#private-network-ingress). + ### Idempotence and concurrency - Every provision creates a new sandbox. @@ -826,7 +838,8 @@ fake-OpenVMM lifecycle test. discovered artifacts. - `nvx.py test-aci-edge-sandboxes` runs the ignored `openvmm_e2e` tests against a real hypervisor with the repository's kernel and Alpine initramfs: the lifecycle, - working directories, host path mapping, network rules, and exec environments. + working directories, host path mapping, network rules, private-network + ingress, and exec environments. CI runs them on Linux/KVM, Linux/MSHV, and Windows/WHP. diff --git a/aci_edge_sandboxes/src/openvmm/mod.rs b/aci_edge_sandboxes/src/openvmm/mod.rs index 61aa4418..a8881b34 100644 --- a/aci_edge_sandboxes/src/openvmm/mod.rs +++ b/aci_edge_sandboxes/src/openvmm/mod.rs @@ -35,7 +35,8 @@ //! | `filesystem.readonlyPaths`, `readwritePaths` | mapped at their [`nvx::guest_path`] | n/a | //! | `filesystem.deniedPaths` | hidden inside mapped paths | n/a | //! | `network.egress` | default and IPv4 and IPv6 rules applied | n/a | -//! | `network.ingress`, `hostLoopback` | `deny` only | n/a | +//! | `network.ingress.default` | `allow` and `deny` applied; see below | n/a | +//! | `network.ingress.hostLoopback` | `deny` only | n/a | //! | `microvm.provision.memoryMib` | applied | n/a | //! | `process.commandLine` | n/a | run as `/bin/sh -c `, at most 4096 bytes | //! | `process.cwd` | n/a | an absolute guest path of at most 4095 bytes; `/` when omitted | @@ -59,7 +60,12 @@ //! Egress rules are expanded //! into OpenVMM's IPv4 and IPv6 rules exactly; rules that would need more than 256 OpenVMM rules //! are rejected. Egress denied without allow rules, with ingress denied, -//! attaches no network device. Workloads run as the fixed non-root identity of +//! attaches no network device. With `network.ingress.default: allow`, OpenVMM listens on every +//! host address at the TCP and UDP ports where the workload listens, and admits clients on the +//! host's private networks, whose addresses are not globally reachable, but not clients inside +//! the guest network, IPv6 link-local clients, or the host itself, because host loopback stays +//! denied; replies to their connections pass whatever the egress +//! policy. Workloads run as the fixed non-root identity of //! [`OpenVmmConfig::workload_uid`] and [`OpenVmmConfig::workload_gid`] (see //! [`OpenVmmConfig::map_host_identity`] for Linux hosts) with no capabilities, read end-of-file on //! standard input, and may produce at most 1 MiB of combined output. @@ -395,6 +401,7 @@ impl Backend for OpenVmmBackend { capabilities.exec.max_output_bytes = Some(MAX_OUTPUT_BYTES as u64); capabilities.network.egress_allow = true; capabilities.network.egress_deny = true; + capabilities.network.ingress_allow = true; capabilities.network.ingress_deny = true; capabilities.network.host_loopback_deny = true; capabilities.network.egress_rules = true; diff --git a/aci_edge_sandboxes/tests/openvmm_e2e.rs b/aci_edge_sandboxes/tests/openvmm_e2e.rs index b42feee6..837494d6 100644 --- a/aci_edge_sandboxes/tests/openvmm_e2e.rs +++ b/aci_edge_sandboxes/tests/openvmm_e2e.rs @@ -1304,6 +1304,76 @@ fn ipv6_network_policies_are_enforced() { } } +/// Returns whether a host socket holds TCP `port` on every IPv4 address, as OpenVMM's listener +/// for a published guest port does. +fn published(port: u16) -> bool { + std::net::TcpListener::bind(("0.0.0.0", port)).is_err() +} + +fn wait_until(mut condition: impl FnMut() -> bool, description: &str) { + let deadline = Instant::now() + Duration::from_secs(30); + while !condition() { + assert!( + Instant::now() < deadline, + "timed out waiting until {description}" + ); + thread::sleep(Duration::from_millis(50)); + } +} + +#[test] +#[ignore = "requires a hypervisor and NVX guest artifacts; run scripts/nvx.py test-aci-edge-sandboxes"] +fn private_network_ingress_publishes_workload_listeners() { + use std::io::Read as _; + use std::io::Write as _; + + let (nvx, backend) = client("ingress"); + let port = std::net::TcpListener::bind(("0.0.0.0", 0)) + .unwrap() + .local_addr() + .unwrap() + .port(); + let listen = || ExecRequest::command_line(format!("timeout 10 nc -l -p {port}")); + let mut ingress = NetworkPolicy::deny_all(); + ingress.ingress.default = Access::Allow; + let sandbox = started( + &nvx, + &backend, + &ProvisionRequest::new().with_network(ingress), + ); + let listener = nvx.exec(id(&sandbox), &listen()).unwrap(); + wait_until(|| published(port), "the workload's port is published"); + // The host is not a private-network client: the published listener completes its + // connection and then resets it, so nothing reaches the workload. A marker that the + // workload printed would reveal a connection that reached it. + let mut stream = std::net::TcpStream::connect(("127.0.0.1", port)).unwrap(); + let _ = stream.write_all(b"NVX-HOST-PROBE\n"); + let _ = stream.shutdown(std::net::Shutdown::Write); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + let mut buffer = [0u8; 16]; + if let Ok(count) = stream.read(&mut buffer) { + assert_eq!(count, 0, "the host reached the workload"); + } + let output = listener.wait_with_output().unwrap(); + assert!(output.stdout.is_empty(), "{output:?}"); + wait_until(|| !published(port), "the closed listener is withdrawn"); + drop(sandbox); + + // With ingress denied, nothing is published. + let sandbox = started( + &nvx, + &backend, + &ProvisionRequest::new().with_network(NetworkPolicy::egress(Access::Allow)), + ); + let listener = nvx.exec(id(&sandbox), &listen()).unwrap(); + thread::sleep(Duration::from_secs(2)); + assert!(!published(port)); + listener.canceller().cancel().unwrap(); + let _ = listener.wait(); +} + #[test] #[ignore = "requires a hypervisor and NVX guest artifacts; run scripts/nvx.py test-aci-edge-sandboxes"] fn network_port_ranges_are_enforced() { diff --git a/aci_edge_sandboxes/tests/openvmm_fake.rs b/aci_edge_sandboxes/tests/openvmm_fake.rs index 0ecfc5d4..1ac358de 100644 --- a/aci_edge_sandboxes/tests/openvmm_fake.rs +++ b/aci_edge_sandboxes/tests/openvmm_fake.rs @@ -897,10 +897,12 @@ fn unsupported_requests_are_rejected_before_anything_runs() { .code(), ErrorCode::PolicyValidation ); - let mut ingress = NetworkPolicy::deny_all(); - ingress.ingress.default = Access::Allow; + // The backend publishes listeners to private networks, but never to the host itself. + let mut host_loopback = NetworkPolicy::deny_all(); + host_loopback.ingress.default = Access::Allow; + host_loopback.ingress.host_loopback = Some(Access::Allow); assert_eq!( - nvx.provision(&ProvisionRequest::new().with_network(ingress)) + nvx.provision(&ProvisionRequest::new().with_network(host_loopback)) .unwrap_err() .code(), ErrorCode::PolicyValidation @@ -1373,6 +1375,22 @@ fn launch_arguments_select_the_managed_alpine_guest() { ); nvx.stop(&sandbox_id).unwrap(); nvx.deprovision(&sandbox_id).unwrap(); + + // Ingress attaches the device under denied egress and keeps host loopback denied. + let mut ingress = NetworkPolicy::deny_all(); + ingress.ingress.default = Access::Allow; + let sandbox_id = nvx + .provision(&ProvisionRequest::new().with_network(ingress)) + .unwrap() + .sandbox_id; + nvx.start(&sandbox_id).unwrap(); + let arguments = fixture.launch_arguments(&sandbox_id); + assert!(has_pair(&arguments, "--net", "10.0.0.2/24")); + assert!(has_pair(&arguments, "--network-egress", "deny")); + assert!(has_pair(&arguments, "--network-ingress", "allow")); + assert!(has_pair(&arguments, "--host-loopback", "deny")); + nvx.stop(&sandbox_id).unwrap(); + nvx.deprovision(&sandbox_id).unwrap(); } #[test] diff --git a/nvx-rs/src/host/network.rs b/nvx-rs/src/host/network.rs index 84128ef4..d4eae8e0 100644 --- a/nvx-rs/src/host/network.rs +++ b/nvx-rs/src/host/network.rs @@ -157,7 +157,8 @@ impl EgressRules { /// /// A connection matching a `deny` rule is blocked. Otherwise a connection matching an `allow` /// rule is permitted, and any other connection follows the `egress` default. OpenVMM's portable -/// profile denies ingress and the guest's access to host loopback services. +/// profile denies the guest's access to host loopback services, and the host's own access to the +/// guest. #[derive(Debug, Clone, PartialEq, Eq)] pub struct NetworkPolicy { /// Default policy for guest-initiated connections. @@ -166,8 +167,11 @@ pub struct NetworkPolicy { pub allow: EgressRules, /// Destinations blocked despite an `allow` default or an `allow` rule. pub deny: EgressRules, - /// Default policy for host-initiated connections, which OpenVMM's portable profile can only - /// deny; [`openvmm_arguments`] refuses [`Access::Allow`]. + /// Default policy for connections that clients initiate to the guest. With [`Access::Allow`], + /// OpenVMM listens on every host address at the TCP and UDP ports where the guest listens, + /// and admits clients on the host's private networks, whose addresses are not globally + /// reachable, but not clients inside the guest network, IPv6 link-local clients, or the host + /// itself; replies to their connections pass whatever the egress policy. pub ingress: Access, } @@ -200,13 +204,13 @@ impl NetworkPolicy { /// Returns the OpenVMM options that enforce `policy` for a guest at `guest_network`. /// -/// Without a policy, or when egress is denied without allow rules, no network device is -/// attached. Otherwise the portable profile applies the egress posture with ingress and host -/// loopback denied. +/// Without a policy, or when egress is denied without allow rules and ingress is denied, no +/// network device is attached. Otherwise the portable profile applies the egress posture and the +/// ingress default with host loopback denied. /// -/// Fails with [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) when the policy allows -/// ingress, which the portable profile cannot express, when a rule cannot be expressed exactly, -/// or when the rules of one action expand to more than [`MAX_RULES`] OpenVMM rules. +/// Fails with [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) when a rule cannot be +/// expressed exactly, or when the rules of one action expand to more than [`MAX_RULES`] OpenVMM +/// rules. pub fn openvmm_arguments( policy: Option<&NetworkPolicy>, guest_network: &str, @@ -214,13 +218,10 @@ pub fn openvmm_arguments( let Some(policy) = policy else { return Ok(Vec::new()); }; - if policy.ingress != Access::Deny { - return Err(Error::unsupported( - "ingress must be denied; OpenVMM's portable network profile cannot admit \ - host-initiated connections", - )); - } - if policy.egress == Access::Deny && policy.allow.rules.is_empty() { + if policy.egress == Access::Deny + && policy.allow.rules.is_empty() + && policy.ingress == Access::Deny + { return Ok(Vec::new()); } let mut arguments: Vec = [ @@ -231,7 +232,7 @@ pub fn openvmm_arguments( "--network-egress", policy.egress.as_str(), "--network-ingress", - "deny", + policy.ingress.as_str(), "--host-loopback", "deny", ] @@ -403,13 +404,25 @@ mod tests { .unwrap() .is_empty() ); - // The portable profile denies ingress, so a policy that allows it is refused rather than - // tightened. + // Ingress needs the device even when egress is denied, and host loopback stays denied. for egress in [Access::Deny, Access::Allow] { let mut ingress = NetworkPolicy::new(egress); ingress.ingress = Access::Allow; - let error = openvmm_arguments(Some(&ingress), "10.0.0.2/24").unwrap_err(); - assert_eq!(error.kind(), crate::ErrorKind::Unsupported, "{error}"); + assert_eq!( + openvmm_arguments(Some(&ingress), "10.0.0.2/24").unwrap(), + [ + "--net", + "10.0.0.2/24", + "--network-profile", + "portable", + "--network-egress", + egress.as_str(), + "--network-ingress", + "allow", + "--host-loopback", + "deny", + ] + ); } } From 48a5daf10b2d13a6b4dad46b16c18a7e46e8bf72 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Sun, 11 Oct 2026 16:57:39 -0300 Subject: [PATCH 5/5] test-microvm: verify private-network ingress The new private-network-ingress scenario, part of the default suite, boots the guest three times with busybox nc TCP and UDP servers on one port and checks from the host what each policy lets through (#275): - With egress denied and ingress and host loopback allowed, the port is published for TCP and UDP on every host IPv4 address. The host exchanges a line with each server over 127.0.0.1, gets the UDP reply from the port that it reached, and sees the TCP port withdrawn once its server exits, while the guest still cannot connect to a host listener of its own. - With host loopback denied, the host's connections from loopback and from its own addresses are reset before they reach the guest, and its datagrams go unanswered. - With ingress denied, nothing is published, and the host's connection fails. The scenario then requires OpenVMM to reject a snapshot destination with ingress allowed before boot. The guest script checks that the virtnet_ingress token and the announcer match the policy before it starts its servers, and that each server received exactly what the policy admits. A single host has no private-network peer other than itself, so the pinned OpenVMM's unit tests classify the peers of other hosts. Unit tests run the guest script under a POSIX shell with stubbed servers and socket tables for each policy and failure, check the policies and the snapshot rejection of the scenario, and check the host's publication probe. The validation design describes the coverage. Fixes #275. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/design/validation.md | 10 +- .../private-network-ingress.sh.in | 92 +++++ scripts/nvx_tools/microvm_tests.py | 380 +++++++++++++++++- scripts/test_microvm_tests.py | 329 +++++++++++++++ 4 files changed, 808 insertions(+), 3 deletions(-) create mode 100644 scripts/nvx_tools/microvm_test_scripts/private-network-ingress.sh.in diff --git a/doc/design/validation.md b/doc/design/validation.md index df7d27fa..f2882087 100644 --- a/doc/design/validation.md +++ b/doc/design/validation.md @@ -39,7 +39,15 @@ directories with `RENAME_EXCHANGE`, which Linux hosts perform and Windows hosts reject without changing either name. Directional network coverage verifies an egress request and response with ingress denied, denial of a host connection to an active guest listener, and complete egress denial on KVM, MSHV, and WHP. -L3/L4 coverage verifies TCP and UDP +Private-network ingress coverage verifies, with egress denied, that the +guest's TCP and UDP listeners are published at their own ports on the host, +that the host reaches them and gets their replies when host loopback is +allowed, that a closed TCP listener is withdrawn, and that the guest still +cannot open connections of its own; that the host's connections from loopback +and from its own addresses are reset, and its datagrams go unanswered, when +host loopback is denied; that ingress denied publishes nothing; and that a +snapshot with private-network ingress is rejected before boot. Unit tests +classify the peers of other hosts. L3/L4 coverage verifies TCP and UDP port rules, overlapping deny precedence, default-deny behavior, and pre-boot rejection of malformed or incomplete rules. Protocol coverage verifies that TCP, UDP, and ICMP rules without a port, `any` with a port, and `any` alone diff --git a/scripts/nvx_tools/microvm_test_scripts/private-network-ingress.sh.in b/scripts/nvx_tools/microvm_test_scripts/private-network-ingress.sh.in new file mode 100644 index 00000000..76b22148 --- /dev/null +++ b/scripts/nvx_tools/microvm_test_scripts/private-network-ingress.sh.in @@ -0,0 +1,92 @@ +set -eu +ingress=@INGRESS@ +expect=@EXPECT@ +outbound_port=@OUTBOUND_PORT@ + +fail() { + code="$1" + echo "NVX-PRIVATE-INGRESS-FAIL code=$code" + nvx-exit "$code" + exit "$code" +} + +# Waits until the kernel lists a socket on the hexadecimal port in state $2 of +# the tables that follow. +wait_for_socket() { + port_hex=$(printf '%04X' "$1") + state=$2 + shift 2 + tries=0 + while ! awk -v port=":$port_hex" -v state="$state" \ + '$2 ~ port "$" && $4 == state { found = 1 } END { exit !found }' "$@"; do + [ "$tries" -lt 100 ] || return 1 + tries=$((tries + 1)) + sleep 0.05 + done +} + +if grep -q 'virtnet_ingress=allow' /proc/cmdline; then + [ "$ingress" = allow ] || fail 70 + pidof nvx-ingress >/dev/null || fail 71 +else + [ "$ingress" = deny ] || fail 72 + if pidof nvx-ingress >/dev/null; then + fail 73 + fi +fi + +rm -f /tmp/nvx-ingress-tcp /tmp/nvx-ingress-udp +# Each server answers its first client and keeps its sending side open, so it +# reads what the client sends. +{ + printf 'NVX-PRIVATE-INGRESS-TCP-REPLY\n' + sleep 20 +} | nc -l -p @TCP_PORT@ -w 20 >/tmp/nvx-ingress-tcp 2>/dev/null & +{ + printf 'NVX-PRIVATE-INGRESS-UDP-REPLY\n' + sleep 20 +} | nc -u -l -p @UDP_PORT@ -w 20 >/tmp/nvx-ingress-udp 2>/dev/null & +wait_for_socket @TCP_PORT@ 0A /proc/net/tcp /proc/net/tcp6 || fail 74 +wait_for_socket @UDP_PORT@ 07 /proc/net/udp /proc/net/udp6 || fail 75 +echo NVX-PRIVATE-INGRESS-LISTENING + +read -r release || fail 76 +[ "$release" = NVX-PRIVATE-INGRESS-CHECKED ] || fail 77 + +# The servers may write what they received after the host's release arrives. +wait_for_line() { + tries=0 + while ! grep -qx "$1" "$2" 2>/dev/null; do + [ "$tries" -lt 100 ] || return 1 + tries=$((tries + 1)) + sleep 0.05 + done +} + +case "$expect" in + reached) + wait_for_line NVX-PRIVATE-INGRESS-TCP-REQUEST /tmp/nvx-ingress-tcp || fail 78 + wait_for_line NVX-PRIVATE-INGRESS-UDP-REQUEST /tmp/nvx-ingress-udp || fail 79 + ;; + unreached) + [ ! -s /tmp/nvx-ingress-tcp ] || fail 80 + [ ! -s /tmp/nvx-ingress-udp ] || fail 81 + ;; + *) + fail 82 + ;; +esac + +if [ "$outbound_port" != 0 ]; then + # Egress stays denied for connections that the guest opens itself. A + # denied connection fails with 1; execution failures must not pass. + if printf 'NVX-PRIVATE-INGRESS-OUTBOUND\n' | + nc -w 2 @GATEWAY_IPV4@ "$outbound_port" >/dev/null 2>&1; then + fail 83 + else + [ "$?" -eq 1 ] || fail 84 + fi +fi + +echo @COMPLETION_MARKER@ +nvx-exit 0 diff --git a/scripts/nvx_tools/microvm_tests.py b/scripts/nvx_tools/microvm_tests.py index 31d4492f..b42bb381 100644 --- a/scripts/nvx_tools/microvm_tests.py +++ b/scripts/nvx_tools/microvm_tests.py @@ -19,8 +19,8 @@ import threading import time import uuid -from collections.abc import Sequence -from contextlib import ExitStack +from collections.abc import Callable, Sequence +from contextlib import ExitStack, suppress from dataclasses import dataclass from pathlib import Path from stat import S_ISUID @@ -112,6 +112,7 @@ "managed-lifecycle", "managed-exec-config", "network-snapshot", + "private-network-ingress", "restore-downtime", "restore-memory", "restore-processors", @@ -207,6 +208,15 @@ # IANA's dynamic port range, which is also Windows' default. The host-loopback # scenario draws the port of each TCP and UDP listener pair from it. HOST_LOOPBACK_PORT_RANGE = (49152, 65535) +PRIVATE_INGRESS_LISTENING_MARKER = b"NVX-PRIVATE-INGRESS-LISTENING" +PRIVATE_INGRESS_CHECKED = b"NVX-PRIVATE-INGRESS-CHECKED" +PRIVATE_INGRESS_REACHED_MARKER = b"NVX-PRIVATE-INGRESS-REACHED-OK" +PRIVATE_INGRESS_REFUSED_MARKER = b"NVX-PRIVATE-INGRESS-REFUSED-OK" +PRIVATE_INGRESS_UNPUBLISHED_MARKER = b"NVX-PRIVATE-INGRESS-UNPUBLISHED-OK" +PRIVATE_INGRESS_TCP_REQUEST = b"NVX-PRIVATE-INGRESS-TCP-REQUEST\n" +PRIVATE_INGRESS_TCP_REPLY = b"NVX-PRIVATE-INGRESS-TCP-REPLY\n" +PRIVATE_INGRESS_UDP_REQUEST = b"NVX-PRIVATE-INGRESS-UDP-REQUEST\n" +PRIVATE_INGRESS_UDP_REPLY = b"NVX-PRIVATE-INGRESS-UDP-REPLY\n" SANDBOX_BLOCK_SIZE = 8 * 1024 * 1024 SNAPSHOT_CORE_CONTINUED_MARKER = b"NVX-SNAPSHOT-CORE-CONTINUED" SNAPSHOT_CORE_COMPLETION_MARKER = b"NVX-SNAPSHOT-CORE-OK" @@ -3507,6 +3517,361 @@ def run_host_loopback_rejections( ) +def _private_ingress_port() -> int: + """Draw a port that the host can publish for both TCP and UDP.""" + last_error: OSError | None = None + for port in _host_loopback_port_candidates(): + with ExitStack() as sockets: + try: + for kind in (socket.SOCK_STREAM, socket.SOCK_DGRAM): + probe = socket.socket(socket.AF_INET, kind) + sockets.callback(probe.close) + probe.bind(("0.0.0.0", port)) + except OSError as error: + last_error = error + continue + return port + raise RuntimeError( + "failed to find a port free for TCP and UDP on every host address" + ) from last_error + + +def _publishes(port: int, kind: socket.SocketKind) -> bool: + """Return whether a host socket holds ``port`` on every IPv4 address. + + During the scenario, that socket is OpenVMM's listener for the guest port. + """ + probe = socket.socket(socket.AF_INET, kind) + try: + probe.bind(("0.0.0.0", port)) + except OSError: + return True + finally: + probe.close() + return False + + +def _wait_until(condition: Callable[[], bool], timeout: float, failure: str) -> None: + deadline = time.monotonic() + timeout + while not condition(): + if time.monotonic() >= deadline: + raise TimeoutError(failure) + time.sleep(0.05) + + +def _host_ipv4_addresses() -> list[str]: + """Return the host's own non-loopback IPv4 addresses that are known.""" + addresses: set[str] = set() + with suppress(OSError): + for info in socket.getaddrinfo(socket.gethostname(), None, socket.AF_INET): + addresses.add(str(info[4][0])) + with suppress(OSError), socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as route: + # Connecting a UDP socket sends nothing; it selects the source address + # of the default route. + route.connect(("192.0.2.1", 9)) + addresses.add(route.getsockname()[0]) + return sorted( + address + for address in addresses + if not ipaddress.ip_address(address).is_loopback + ) + + +def _udp_client(timeout: float) -> socket.socket: + client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + client.bind(("127.0.0.1", 0)) + client.settimeout(timeout) + return client + + +def _private_ingress_command( + executable: Path, + kernel: Path, + initrd: Path, + backend: str, + memory_mib: int, + *policy: str, +) -> list[str]: + command = workload_boot_command( + executable, + backend, + kernel, + initrd, + memory_mib, + "quiet loglevel=0", + network=DIRECTIONAL_NETWORK_CIDR, + ) + command.extend(policy) + return command + + +def _run_private_ingress_guest( + command: Sequence[str], + *, + ingress: str, + expect: str, + port: int, + outbound_port: int, + completion_marker: bytes, + timeout: float, + log_path: Path, + probe: Callable[[], None], +) -> None: + """Boot a guest that serves TCP and UDP on ``port``, run ``probe`` from the + host while it listens, and require the guest to observe ``expect``.""" + with OpenvmmProcess(command, log_path) as process: + process.wait_for(BOOT_MARKER, timeout) + _stage_script( + process, + "/tmp/nvx-private-network-ingress", + "NVX_PRIVATE_NETWORK_INGRESS", + _render_script( + "private-network-ingress.sh.in", + INGRESS=ingress, + EXPECT=expect, + TCP_PORT=str(port), + UDP_PORT=str(port), + GATEWAY_IPV4=DIRECTIONAL_NETWORK_GATEWAY_IPV4, + OUTBOUND_PORT=str(outbound_port), + COMPLETION_MARKER=completion_marker.decode(), + ), + ) + process.wait_for_line(PRIVATE_INGRESS_LISTENING_MARKER, timeout) + probe() + process.send_line(PRIVATE_INGRESS_CHECKED.decode()) + process.wait_for_line(completion_marker, timeout) + result = process.wait(timeout) + if result.returncode != 0: + raise RuntimeError( + f"private-network ingress guest exited with {result.returncode}" + ) + + +def run_private_network_ingress( + executable: Path, + kernel: Path, + initrd: Path, + backend: str, + *, + memory_mib: int, + timeout: float, + output_dir: Path, +) -> None: + """Verify that ``--network-ingress allow`` publishes the guest's listeners. + + A single host has no private-network peer other than itself, which the + published listeners refuse unless host loopback is allowed. The scenario + therefore exchanges TCP and UDP with the guest's servers from the host with + host loopback allowed and egress denied, which also requires the guest's + replies to pass; it requires the host's own connections to be refused with + host loopback denied, and nothing to be published with ingress denied. + Unit tests classify the peers of other hosts. + """ + port = _private_ingress_port() + outbound = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + outbound.bind(("127.0.0.1", 0)) + outbound.listen(1) + outbound_port = int(outbound.getsockname()[1]) + + def exchange() -> None: + _wait_until( + lambda: ( + _publishes(port, socket.SOCK_STREAM) + and _publishes(port, socket.SOCK_DGRAM) + ), + timeout, + f"guest port {port} was not published to the host", + ) + with socket.create_connection(("127.0.0.1", port), timeout=timeout) as client: + client.settimeout(timeout) + reply = b"" + while not reply.endswith(b"\n"): + chunk = client.recv(4096) + if not chunk: + break + reply += chunk + if reply != PRIVATE_INGRESS_TCP_REPLY: + raise RuntimeError(f"unexpected guest TCP reply: {reply!r}") + client.sendall(PRIVATE_INGRESS_TCP_REQUEST) + client.shutdown(socket.SHUT_WR) + with _udp_client(0.5) as client: + deadline = time.monotonic() + timeout + while True: + client.sendto(PRIVATE_INGRESS_UDP_REQUEST, ("127.0.0.1", port)) + try: + reply, source = client.recvfrom(4096) + break + except (TimeoutError, ConnectionResetError): + if time.monotonic() >= deadline: + raise + if reply != PRIVATE_INGRESS_UDP_REPLY or source != ("127.0.0.1", port): + raise RuntimeError( + f"unexpected guest UDP reply {reply!r} from {source}" + ) + # The guest's TCP server accepted its one client and closed, so its port + # is withdrawn, while the UDP server still uses its own. + _wait_until( + lambda: not _publishes(port, socket.SOCK_STREAM), + timeout, + f"closed guest TCP port {port} remained published", + ) + + try: + _run_private_ingress_guest( + _private_ingress_command( + executable, + kernel, + initrd, + backend, + memory_mib, + "--network-egress", + "deny", + "--network-ingress", + "allow", + "--host-loopback", + "allow", + ), + ingress="allow", + expect="reached", + port=port, + outbound_port=outbound_port, + completion_marker=PRIVATE_INGRESS_REACHED_MARKER, + timeout=timeout, + log_path=output_dir / "private-network-ingress-host-allowed.log", + probe=exchange, + ) + outbound.settimeout(NETWORK_NEGATIVE_OBSERVATION_TIMEOUT_SECONDS) + try: + unexpected, _ = outbound.accept() + except TimeoutError: + pass + else: + unexpected.close() + raise RuntimeError("denied egress opened a guest connection to the host") + finally: + outbound.close() + + port = _private_ingress_port() + + def refused() -> None: + _wait_until( + lambda: ( + _publishes(port, socket.SOCK_STREAM) + and _publishes(port, socket.SOCK_DGRAM) + ), + timeout, + f"guest port {port} was not published to the host", + ) + # The host completes the handshake with the published listener, which + # then resets connections from the host itself. + for address in ["127.0.0.1", *_host_ipv4_addresses()]: + with socket.create_connection((address, port), timeout=timeout) as client: + client.settimeout(timeout) + try: + data = client.recv(4096) + except (ConnectionResetError, ConnectionAbortedError): + data = b"" + if data: + raise RuntimeError( + f"a refused connection from {address} reached the guest: {data!r}" + ) + with _udp_client(NETWORK_NEGATIVE_OBSERVATION_TIMEOUT_SECONDS) as client: + for _ in range(3): + client.sendto(PRIVATE_INGRESS_UDP_REQUEST, ("127.0.0.1", port)) + try: + reply = client.recv(4096) + except (TimeoutError, ConnectionResetError): + continue + raise RuntimeError(f"a refused datagram drew a reply: {reply!r}") + + _run_private_ingress_guest( + _private_ingress_command( + executable, + kernel, + initrd, + backend, + memory_mib, + "--network-egress", + "deny", + "--network-ingress", + "allow", + "--host-loopback", + "deny", + ), + ingress="allow", + expect="unreached", + port=port, + outbound_port=0, + completion_marker=PRIVATE_INGRESS_REFUSED_MARKER, + timeout=timeout, + log_path=output_dir / "private-network-ingress-host-denied.log", + probe=refused, + ) + + port = _private_ingress_port() + + def unpublished() -> None: + # Longer than the guest needs to announce a UDP listener. + time.sleep(2) + if _publishes(port, socket.SOCK_STREAM) or _publishes(port, socket.SOCK_DGRAM): + raise RuntimeError("ingress deny published a guest port") + try: + with socket.create_connection(("127.0.0.1", port), timeout=timeout): + pass + except OSError: + return + raise RuntimeError("ingress deny accepted a connection toward the guest") + + _run_private_ingress_guest( + _private_ingress_command( + executable, + kernel, + initrd, + backend, + memory_mib, + "--network-egress", + "allow", + "--network-ingress", + "deny", + ), + ingress="deny", + expect="unreached", + port=port, + outbound_port=0, + completion_marker=PRIVATE_INGRESS_UNPUBLISHED_MARKER, + timeout=timeout, + log_path=output_dir / "private-network-ingress-denied.log", + probe=unpublished, + ) + + with tempfile.TemporaryDirectory(prefix="nvx-ingress-snapshot-") as temporary: + invalid = _private_ingress_command( + executable, + kernel, + initrd, + backend, + memory_mib, + "--network-ingress", + "allow", + "--snapshot-destination", + str(Path(temporary) / "snapshot"), + ) + with OpenvmmProcess( + invalid, + output_dir / "private-network-ingress-snapshot.log", + ) as process: + result = process.wait(timeout) + if ( + result.returncode == 0 + or b"microVM snapshots do not support --network-ingress allow" + not in result.output + or BOOT_MARKER in result.output + ): + raise RuntimeError( + "a snapshot with private-network ingress was not rejected before boot" + ) + + IO_REPARSE_TAG_LX_SYMLINK = 0xA000001D LX_SYMLINK_VERSION = 2 @@ -7162,6 +7527,17 @@ def run(args: argparse.Namespace) -> int: timeout=args.timeout, output_dir=output_dir, ) + if "private-network-ingress" in scenarios: + print(f"Running microVM private-network ingress on OpenVMM/{args.backend}") + run_private_network_ingress( + executable, + kernel, + initrd, + args.backend, + memory_mib=args.memory_mib, + timeout=args.timeout, + output_dir=output_dir, + ) if "denied-filesystem-paths" in scenarios: print(f"Running microVM denied filesystem paths on OpenVMM/{args.backend}") run_denied_filesystem_paths( diff --git a/scripts/test_microvm_tests.py b/scripts/test_microvm_tests.py index 119e1f06..cd176342 100644 --- a/scripts/test_microvm_tests.py +++ b/scripts/test_microvm_tests.py @@ -4,6 +4,7 @@ import argparse import ast import errno +import functools import inspect import io import ipaddress @@ -54,6 +55,11 @@ def _posix_shell() -> str | None: return shell +# The line that releases the private-network ingress guest after the host's +# probes. +PRIVATE_INGRESS_RELEASE = microvm_tests.PRIVATE_INGRESS_CHECKED + b"\n" + + # What `/usr/bin/env` prints in the emulated workload without a supplied environment # or working directory. WORKLOAD_DEFAULT_ENVIRONMENT = ( @@ -220,6 +226,11 @@ def measure(command: list[str], **kwargs: object) -> None: return MagicMock(side_effect=measure) +def _fresh_openvmm_command(*_args: object, **_kwargs: object) -> list[str]: + # A new list for every call, which the scenario extends with its policy. + return ["openvmm"] + + class MicrovmTestParserTests(unittest.TestCase): def test_parser_defaults_to_all_correctness_scenarios(self): args = nvx.parse_args(["test-microvm", "--backend", "mshv"]) @@ -2883,6 +2894,324 @@ def test_host_loopback_scenario_requires_observed_udp_control(self): self.assertNotIn("--host-loopback", guest.call_args.args[0]) process.assert_not_called() + def _run_private_ingress_script( + self, + *, + ingress: str, + expect: str, + cmdline: str, + ingress_running: bool, + tcp_request: bool, + udp_request: bool, + outbound_status: int = 1, + outbound_port: str = "0", + ) -> subprocess.CompletedProcess[str]: + shell = _posix_shell() + if shell is None: + self.skipTest("POSIX shell is unavailable") + header = " sl local_address rem_address st\n" + tcp6 = header + ( + f" 0: {'0' * 32}:1F90 {'0' * 32}:0000 0A 00000000:00000000 00:00000000\n" + ) + udp6 = header + ( + f" 0: {'0' * 32}:1F90 {'0' * 32}:0000 07 00000000:00000000 00:00000000\n" + ) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "proc" / "net").mkdir(parents=True) + (root / "proc" / "cmdline").write_text(cmdline + "\n", encoding="utf-8") + for name, table in ( + ("tcp", header), + ("tcp6", tcp6), + ("udp", header), + ("udp6", udp6), + ): + (root / "proc" / "net" / name).write_text(table, encoding="utf-8") + script = ( + microvm_tests._render_script( + "private-network-ingress.sh.in", + INGRESS=ingress, + EXPECT=expect, + TCP_PORT="8080", + UDP_PORT="8080", + GATEWAY_IPV4="192.0.2.1", + OUTBOUND_PORT=outbound_port, + COMPLETION_MARKER="NVX-TEST-OK", + ) + .replace("nvx-exit", "nvx_exit") + .replace("/proc/", f"{root.as_posix()}/proc/") + .replace("/tmp/nvx-ingress-", f"{root.as_posix()}/nvx-ingress-") + ) + tcp = "echo NVX-PRIVATE-INGRESS-TCP-REQUEST" if tcp_request else ":" + udp = "echo NVX-PRIVATE-INGRESS-UDP-REQUEST" if udp_request else ":" + # The stubbed servers finish at once; wait for their output before + # releasing the guest. + script = script.replace("read -r release", "wait; read -r release") + path = root / "private-network-ingress.sh" + path.write_text( + "sleep() { :; }\n" + f"pidof() {{ return {0 if ingress_running else 1}; }}\n" + 'nc() { case "$*" in\n' + f"'-l -p '*) {tcp} ;;\n" + f"'-u -l -p '*) {udp} ;;\n" + f"*) return {outbound_status} ;;\n" + "esac; }\n" + 'nvx_exit() { exit "$1"; }\n' + script, + encoding="utf-8", + newline="\n", + ) + result = subprocess.run( + [shell, path.as_posix()], + input=PRIVATE_INGRESS_RELEASE, + capture_output=True, + timeout=10, + check=False, + ) + return subprocess.CompletedProcess( + result.args, + result.returncode, + result.stdout.decode(errors="replace"), + result.stderr.decode(errors="replace"), + ) + + def test_private_network_ingress_script_checks_policy_and_traffic(self): + token = "console=hvc0 virtnet_ip=192.0.2.2 virtnet_ingress=allow" + plain = "console=hvc0 virtnet_ip=192.0.2.2" + for label, run, expected in ( + ( + "reached", + functools.partial( + self._run_private_ingress_script, + ingress="allow", + expect="reached", + cmdline=token, + ingress_running=True, + tcp_request=True, + udp_request=True, + outbound_port="4000", + ), + 0, + ), + ( + "unpublished", + functools.partial( + self._run_private_ingress_script, + ingress="deny", + expect="unreached", + cmdline=plain, + ingress_running=False, + tcp_request=False, + udp_request=False, + ), + 0, + ), + ( + "token-missing", + functools.partial( + self._run_private_ingress_script, + ingress="allow", + expect="reached", + cmdline=plain, + ingress_running=True, + tcp_request=True, + udp_request=True, + ), + 72, + ), + ( + "announcer-missing", + functools.partial( + self._run_private_ingress_script, + ingress="allow", + expect="reached", + cmdline=token, + ingress_running=False, + tcp_request=True, + udp_request=True, + ), + 71, + ), + ( + "announcer-unexpected", + functools.partial( + self._run_private_ingress_script, + ingress="deny", + expect="unreached", + cmdline=plain, + ingress_running=True, + tcp_request=False, + udp_request=False, + ), + 73, + ), + ( + "tcp-missed", + functools.partial( + self._run_private_ingress_script, + ingress="allow", + expect="reached", + cmdline=token, + ingress_running=True, + tcp_request=False, + udp_request=True, + ), + 78, + ), + ( + "udp-leaked", + functools.partial( + self._run_private_ingress_script, + ingress="allow", + expect="unreached", + cmdline=token, + ingress_running=True, + tcp_request=False, + udp_request=True, + ), + 81, + ), + ( + "outbound-opened", + functools.partial( + self._run_private_ingress_script, + ingress="allow", + expect="reached", + cmdline=token, + ingress_running=True, + tcp_request=True, + udp_request=True, + outbound_status=0, + outbound_port="4000", + ), + 83, + ), + ( + "outbound-failed-to-run", + functools.partial( + self._run_private_ingress_script, + ingress="allow", + expect="reached", + cmdline=token, + ingress_running=True, + tcp_request=True, + udp_request=True, + outbound_status=127, + outbound_port="4000", + ), + 84, + ), + ): + with self.subTest(label): + result = run() + output = result.stdout + result.stderr + self.assertEqual(result.returncode, expected, output) + # Policy mismatches fail before the servers start. + self.assertEqual( + "NVX-PRIVATE-INGRESS-LISTENING" in result.stdout, + expected not in (71, 72, 73), + output, + ) + if expected: + self.assertNotIn("NVX-TEST-OK", result.stdout) + else: + self.assertIn("NVX-TEST-OK", result.stdout) + + def test_private_network_ingress_scenario_covers_three_policies(self): + with tempfile.TemporaryDirectory() as temporary: + with ( + patch.object(microvm_tests, "_run_private_ingress_guest") as guest, + patch.object(microvm_tests, "OpenvmmProcess") as process, + patch.object( + microvm_tests, + "workload_boot_command", + side_effect=_fresh_openvmm_command, + ), + ): + process.return_value.__enter__.return_value.wait.return_value = ( + openvmm_process.OpenvmmProcessResult( + 2, b"microVM snapshots do not support --network-ingress allow" + ) + ) + microvm_tests.run_private_network_ingress( + Path("openvmm"), + Path("kernel"), + Path("initrd"), + "whp", + memory_mib=128, + timeout=5, + output_dir=Path(temporary), + ) + self.assertEqual(guest.call_count, 3) + policies = [ + ( + call.args[0][call.args[0].index("--network-egress") + 1], + call.args[0][call.args[0].index("--network-ingress") + 1], + call.args[0][call.args[0].index("--host-loopback") + 1] + if "--host-loopback" in call.args[0] + else None, + call.kwargs["ingress"], + call.kwargs["expect"], + call.kwargs["outbound_port"] != 0, + ) + for call in guest.call_args_list + ] + self.assertEqual( + policies, + [ + ("deny", "allow", "allow", "allow", "reached", True), + ("deny", "allow", "deny", "allow", "unreached", False), + ("allow", "deny", None, "deny", "unreached", False), + ], + ) + snapshot = process.call_args.args[0] + self.assertIn("--snapshot-destination", snapshot) + self.assertEqual(snapshot[snapshot.index("--network-ingress") + 1], "allow") + + def test_private_network_ingress_snapshot_must_be_rejected_before_boot(self): + diagnostic = b"microVM snapshots do not support --network-ingress allow" + for result in ( + openvmm_process.OpenvmmProcessResult(0, diagnostic), + openvmm_process.OpenvmmProcessResult(2, b"other failure"), + openvmm_process.OpenvmmProcessResult( + 2, diagnostic + b"\n" + microvm_tests.BOOT_MARKER + ), + ): + with ( + self.subTest(result=result), + tempfile.TemporaryDirectory() as temporary, + patch.object(microvm_tests, "_run_private_ingress_guest"), + patch.object(microvm_tests, "OpenvmmProcess") as process, + patch.object( + microvm_tests, + "workload_boot_command", + side_effect=_fresh_openvmm_command, + ), + ): + process.return_value.__enter__.return_value.wait.return_value = result + with self.assertRaisesRegex(RuntimeError, "before boot"): + microvm_tests.run_private_network_ingress( + Path("openvmm"), + Path("kernel"), + Path("initrd"), + "whp", + memory_mib=128, + timeout=5, + output_dir=Path(temporary), + ) + + def test_publication_probe_detects_a_held_port(self): + holder = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + holder.bind(("0.0.0.0", 0)) + port = int(holder.getsockname()[1]) + self.assertTrue(microvm_tests._publishes(port, socket.SOCK_STREAM)) + finally: + holder.close() + self.assertFalse(microvm_tests._publishes(port, socket.SOCK_STREAM)) + port = microvm_tests._private_ingress_port() + self.assertFalse(microvm_tests._publishes(port, socket.SOCK_STREAM)) + self.assertFalse(microvm_tests._publishes(port, socket.SOCK_DGRAM)) + def test_host_loopback_udp_probes_reject_failed_sender(self): shell = _posix_shell() if shell is None: