Feature/xtask/e2e wifi ap mode check

docs/TESTING.md

@@ -52,3 +52,19 @@ Any board or chip target that `cargo xtask` accepts works in place of
 
 Each `tests/*.rs` file is a separate flash and run. A new file needs a
 `[[test]]` entry with `harness = false` in `Cargo.toml`.
+
+# End-to-end tests
+
+End-to-end scenarios drive a board that already runs the firmware
+(`cargo xtask <board> run`) from the host: they reset it, read the `boot`
+and `net_up` JSON objects from its console, and act on them.
+
+```sh
+cargo xtask test e2e --board esp32c6-devkitc
+```
+
+`--port` picks the serial port when there are several, and `--interface` the
+host's wireless interface that joins the device's access point. Each scenario
+is a plain function in [`xtask/src/cmd/test/e2e.rs`](../xtask/src/cmd/test/e2e.rs),
+selected with `--scenario`; `wifi-ap`, the default, joins the access point the
+device hosts and checks its SSH port is reachable.

xtask/src/cmd/mod.rs

@@ -10,6 +10,7 @@ pub mod cargo;
 pub mod list;
 pub mod reset;
 pub mod size;
+pub mod test;
 
 /// The extra features every build has.
 pub const BENCH_FEATURES: &[&str] = &["mem-probe", "bench-loopback"];

xtask/src/cmd/test/e2e.rs

@@ -0,0 +1,251 @@
+// SPDX-FileCopyrightText: 2026 Julio Beltran Ortega <jubeormk1@gmail.com>
+// SPDX-FileCopyrightText: 2026 Roman Valls Guimera <brainstorm@nopcode.org>
+//
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+//! `xtask test e2e`
+//!
+//! Runs end-to-end scenarios against an already flashed board. A scenario is
+//! a plain function calling its steps in order, and the device state it needs
+//! comes from the `boot` and `net_up` JSON objects on the serial console.
+
+use crate::board::{self, Board};
+use crate::device::{self, Serial};
+use crate::host::AccessPoint;
+use crate::util::{retry, shell};
+use anyhow::{Context, Result, bail, ensure};
+use clap::{Args as ClapArgs, ValueEnum};
+use serde::Deserialize;
+use ssh_stamp::json::SCHEMA;
+use std::net::Ipv4Addr;
+use std::time::Duration;
+use xshell::cmd;
+
+/// How often the serial capture is checked for new events.
+const POLL_INTERVAL: Duration = Duration::from_millis(250);
+
+#[derive(ClapArgs)]
+pub struct Args {
+    /// Board to run the scenario on.
+    #[arg(long, value_parser = board::name_parser())]
+    board: &'static Board,
+    /// The serial port used to reset the board and read its console.
+    #[arg(long)]
+    port: Option<String>,
+    /// The wireless interface that joins the device's access point, needed on multi-NIC hosts.
+    #[arg(long)]
+    interface: Option<String>,
+    /// The scenario to run.
+    #[arg(long, value_enum, default_value_t = Scenario::WifiAp)]
+    scenario: Scenario,
+    /// Echo serial output while running.
+    #[arg(long)]
+    verbose: bool,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
+pub enum Scenario {
+    /// Reset the board, join the access point it hosts and reach its SSH port.
+    WifiAp,
+}
+
+pub fn run(args: &Args) -> Result<()> {
+    let port = Serial::resolve_port(args.port.as_deref())?;
+    let serial = Serial::open(&port, args.verbose)?;
+
+    let result = match args.scenario {
+        Scenario::WifiAp => wifi_ap(args, &port, &serial),
+    };
+
+    serial.report_health();
+    result
+}
+
+fn wifi_ap(args: &Args, port: &str, serial: &Serial) -> Result<()> {
+    let (boot, net_up) = reset_and_wait_for_network(args.board, port, serial)?;
+    ensure!(
+        net_up.role == "ap",
+        "expected the device to host an access point, but its role is `{}`",
+        net_up.role
+    );
+    eprintln!(
+        "=== device hosts {} at {} ===",
+        boot.wifi_ap.ssid, net_up.ip
+    );
+
+    let access_point = AccessPoint {
+        ssid: boot.wifi_ap.ssid,
+        psk: boot.wifi_ap.psk,
+    };
+    ensure!(
+        access_point.wait_for_reachable(&net_up.ip.to_string(), args.interface.as_deref()),
+        "{} did not become reachable",
+        net_up.ip
+    );
+
+    eprintln!("=== e2e {:?} passed ===", args.scenario);
+    Ok(())
+}
+
+/// Resets the board and waits for the `boot` and `net_up` events of the boot
+/// that follows, ignoring anything captured before the reset.
+fn reset_and_wait_for_network(board: &Board, port: &str, serial: &Serial) -> Result<(Boot, NetUp)> {
+    let skip = serial.current_capture().len();
+    let shell = shell()?;
+    let soc = board.soc;
+    cmd!(shell, "espflash reset --port {port} --chip {soc}")
+        .run()
+        .context("espflash reset failed")?;
+
+    retry(device::BOOT_TIMEOUT, POLL_INTERVAL, || {
+        let lines = serial.current_capture();
+        let events = events(lines.get(skip..).unwrap_or_default())?;
+        let boot = events.iter().find_map(|event| match event {
+            Event::Boot(boot) => Some(boot.clone()),
+            _ => None,
+        });
+        let net_up = events.iter().find_map(|event| match event {
+            Event::NetUp(net_up) => Some(net_up.clone()),
+            _ => None,
+        });
+        match (boot, net_up) {
+            (Some(boot), Some(net_up)) => Ok((boot, net_up)),
+            (None, _) => bail!("timed out waiting for the `boot` event"),
+            (_, None) => bail!("timed out waiting for the `net_up` event"),
+        }
+    })
+}
+
+/// The version every JSON object the firmware prints starts with, see `ssh_stamp::json`.
+#[derive(Debug, Deserialize)]
+struct Schema {
+    schema: u32,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(tag = "event", rename_all = "snake_case")]
+enum Event {
+    Boot(Boot),
+    NetUp(NetUp),
+    #[serde(other)]
+    Other,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+struct Boot {
+    wifi_ap: WifiAp,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+struct WifiAp {
+    ssid: String,
+    psk: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+struct NetUp {
+    role: String,
+    ip: Ipv4Addr,
+}
+
+/// Cuts the firmware's JSON objects out of the console `lines`. A top-level
+/// object always opens and closes with `{` and `}` alone on their lines, so
+/// everything else is console noise. Objects cut short, e.g. by a reset, are
+/// skipped.
+fn events(lines: &[String]) -> Result<Vec<Event>> {
+    let mut events = Vec::new();
+    let mut object: Option<String> = None;
+
+    for line in lines {
+        match line.as_str() {
+            "{" => object = Some(String::from("{")),
+            "}" => {
+                let Some(mut text) = object.take() else {
+                    continue;
+                };
+                text.push('}');
+                let Ok(Schema { schema }) = serde_json::from_str(&text) else {
+                    continue;
+                };
+                ensure!(
+                    schema == SCHEMA,
+                    "the firmware prints JSON schema {schema}, but xtask understands {SCHEMA}"
+                );
+                events.push(serde_json::from_str(&text).context("parsing a console event")?);
+            }
+            _ => {
+                if let Some(text) = object.as_mut() {
+                    text.push_str(line);
+                }
+            }
+        }
+    }
+
+    Ok(events)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn lines(text: &str) -> Vec<String> {
+        text.lines().map(str::to_string).collect()
+    }
+
+    #[test]
+    fn parses_console_events() {
+        let console = lines(
+            r#"ESP-ROM:esp32c6-20220919
+{
+  "schema": 1,
+  "event": "boot",
+  "ident": "SSH-2.0-ssh-stamp",
+  "wifi_ap": {
+    "ssid": "my \"ssid\"",
+    "psk": "test-password",
+    "band": "2.4GHz",
+    "channel": 1
+  },
+  "first_login": true
+}
+INFO - unrelated log line
+{
+  "schema": 1,
+  "event": "net_up",
+  "role": "ap",
+  "ssid": "my \"ssid\"",
+  "ip": "10.0.0.1"
+}
+{
+  "schema": 1,
+  "event": "log",
+  "msg": "ignored"
+}"#,
+        );
+        let events = events(&console).unwrap();
+        let [Event::Boot(boot), Event::NetUp(net_up), Event::Other] = events.as_slice() else {
+            panic!("unexpected events: {events:?}");
+        };
+        assert_eq!(boot.wifi_ap.ssid, r#"my "ssid""#);
+        assert_eq!(boot.wifi_ap.psk, "test-password");
+        assert_eq!(net_up.role, "ap");
+        assert_eq!(net_up.ip, Ipv4Addr::new(10, 0, 0, 1));
+    }
+
+    #[test]
+    fn skips_truncated_objects() {
+        let console = lines(
+            "{\n  \"schema\": 1,\n  \"event\": \"bo\n{\n  \"schema\": 1,\n  \"event\": \"net_up\",\n  \"role\": \"ap\",\n  \"ip\": \"192.168.4.1\"\n}",
+        );
+        assert!(matches!(
+            events(&console).unwrap().as_slice(),
+            [Event::NetUp(_)]
+        ));
+    }
+
+    #[test]
+    fn rejects_unknown_schema() {
+        let console = lines("{\n  \"schema\": 999,\n  \"event\": \"boot\"\n}");
+        assert!(events(&console).is_err());
+    }
+}

xtask/src/cmd/test/mod.rs

@@ -0,0 +1,30 @@
+// SPDX-FileCopyrightText: 2026 Roman Valls Guimera <brainstorm@nopcode.org>
+//
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+//! `xtask test`
+//!
+//! Test suites that need more than `cargo test`, such as real hardware.
+
+use anyhow::Result;
+use clap::{Args as ClapArgs, Subcommand};
+
+pub mod e2e;
+
+#[derive(ClapArgs)]
+pub struct Args {
+    #[command(subcommand)]
+    pub command: Command,
+}
+
+#[derive(Subcommand)]
+pub enum Command {
+    /// Run end-to-end integration tests on hardware.
+    E2e(e2e::Args),
+}
+
+pub fn run(args: &Args) -> Result<()> {
+    match &args.command {
+        Command::E2e(args) => e2e::run(args),
+    }
+}

xtask/src/main.rs

@@ -53,6 +53,8 @@ enum Command {
     Size(cmd::size::Args),
     /// Resets the storage on board, while keeping the firmware intact.
     Reset(cmd::reset::Args),
+    /// Run test suites that need more than `cargo test`, e.g. on hardware.
+    Test(cmd::test::Args),
     /// `<target> <cargo command> [args...]` forwarded to cargo.
     #[command(external_subcommand)]
     Cargo(Vec<String>),
@@ -65,6 +67,7 @@ fn main() -> Result<()> {
         Command::Bmf(args) => cmd::bmf::run(&args),
         Command::Size(args) => cmd::size::run(&args),
         Command::Reset(args) => cmd::reset::run(&args),
+        Command::Test(args) => cmd::test::run(&args),
         Command::Cargo(argv) => cmd::cargo::run(&argv),
     }
 }
@@ -101,6 +104,21 @@ mod tests {
             .map_err(Box::new)
     }
 
+    fn e2e_cmd(argv: &[&str]) -> Result<cmd::test::e2e::Args, Box<clap::Error>> {
+        Cli::try_parse_from(
+            ["xtask", "test", "e2e"]
+                .into_iter()
+                .chain(argv.iter().copied()),
+        )
+        .map(|cli| match cli.command {
+            Command::Test(cmd::test::Args {
+                command: cmd::test::Command::E2e(parsed),
+            }) => parsed,
+            _ => unreachable!(),
+        })
+        .map_err(Box::new)
+    }
+
     #[test]
     fn cli_is_consistent() {
         Cli::command().debug_assert();
@@ -217,4 +235,31 @@ mod tests {
             .is_err()
         );
     }
+
+    #[test]
+    fn e2e() {
+        assert!(e2e_cmd(&["--board", "esp32c6-devkitc"]).is_ok());
+        assert!(
+            e2e_cmd(&[
+                "--board",
+                "esp32c6-devkitc",
+                "--port",
+                "/dev/ttyUSB0",
+                "--interface",
+                "wlan0",
+                "--scenario",
+                "wifi-ap",
+                "--verbose"
+            ])
+            .is_ok()
+        );
+        assert!(e2e_cmd(&["--board", "esp32-fake-name"]).is_err());
+        assert!(e2e_cmd(&[]).is_err());
+        assert!(e2e_cmd(&["--board", "esp32c6-devkitc", "--scenario", "nope"]).is_err());
+        assert!(e2e_cmd(&["--board", "esp32c6-devkitc", "--host", "1.2.3.4"]).is_err());
+        assert!(
+            Cli::try_parse_from(["xtask", "e2e", "--board", "esp32c6-devkitc"])
+                .is_ok_and(|cli| matches!(cli.command, Command::Cargo(_)))
+        );
+    }
 }