From d9cb8bd18514cc48967506875d9e1179de9cff63 Mon Sep 17 00:00:00 2001 From: Julio Beltran Ortega Date: Mon, 14 Sep 2026 11:33:15 +0000 Subject: [PATCH 1/4] feat(xtask): adding e2e command for first integration flow The intention is to explore how to define multi-step flows that might require keeping a serial connection alive, parsing information and stablishing ssh connections. This PR is aimed at doing an integration test for the first set of actions. --- Cargo.lock | 1 + xtask/Cargo.toml | 1 + xtask/e2e/wifi_ap_mode.yml | 6 + xtask/src/cmd/e2e.rs | 263 +++++++++++++++++++++++++++++++++++++ xtask/src/cmd/mod.rs | 1 + xtask/src/host.rs | 50 +++---- xtask/src/main.rs | 34 +++++ 7 files changed, 333 insertions(+), 23 deletions(-) create mode 100644 xtask/e2e/wifi_ap_mode.yml create mode 100644 xtask/src/cmd/e2e.rs diff --git a/Cargo.lock b/Cargo.lock index a4f2ff9..a35c278 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4639,6 +4639,7 @@ dependencies = [ "quick-xml", "serde", "serde_json", + "serde_yaml", "serial2", "ssh-key", "ssh-stamp", diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 3d6b1f0..93ca680 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -31,6 +31,7 @@ object = { version = "0.37", default-features = false, features = [ ] } serde = { version = "1", features = ["derive"] } serde_json = "1" +serde_yaml = "0.9" serial2 = "0.2" strip-ansi-escapes = "0.2" xshell = "0.2" diff --git a/xtask/e2e/wifi_ap_mode.yml b/xtask/e2e/wifi_ap_mode.yml new file mode 100644 index 0000000..c364b7d --- /dev/null +++ b/xtask/e2e/wifi_ap_mode.yml @@ -0,0 +1,6 @@ +actions: + - restart_target_device + - parse_ssh_stamp_network_from_serial + - connect_to_ssid_using_psk + - verify_ip_obtained + - wait_for_reachable diff --git a/xtask/src/cmd/e2e.rs b/xtask/src/cmd/e2e.rs new file mode 100644 index 0000000..d389065 --- /dev/null +++ b/xtask/src/cmd/e2e.rs @@ -0,0 +1,263 @@ +// SPDX-FileCopyrightText: 2026 Julio Beltran Ortega +// +// SPDX-License-Identifier: GPL-3.0-or-later + +//! `xtask e2e` +//! +//! Runs e2e scenarios from YAML steps. + +use crate::board::{self, Board}; +use crate::device::Serial; +use crate::host::AccessPoint; +use crate::util::workspace_root; +use anyhow::{Context, Result, bail}; +use clap::Args as ClapArgs; +use serde::Deserialize; +use std::net::Ipv4Addr; +use std::path::PathBuf; +use std::time::{Duration, Instant}; +use strip_ansi_escapes::strip_str; +use xshell::{Shell, cmd}; + +const SERIAL_PARSE_TIMEOUT: Duration = Duration::from_secs(45); + +#[derive(Debug, Deserialize)] +struct Scenario { + actions: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "snake_case")] +enum Action { + RestartTargetDevice, + ParseSshStampNetworkFromSerial, + ConnectToSsidUsingPsk, + VerifyIpObtained, + WaitForReachable, +} + +#[derive(Debug, Clone)] +struct NetworkInfo { + ssid: String, + psk: String, + ip: Ipv4Addr, +} + +#[derive(Default)] +struct ContextState { + network: Option, +} + +#[derive(ClapArgs)] +pub struct Args { + /// Board to run end-to-end integration testing on. + #[arg(long, value_parser = board::name_parser())] + board: &'static Board, + /// The serial device used to program/reset and read logs (prg_serial). + #[arg(long)] + prg_serial: Option, + /// The wireless interface that joins the device's AP, needed on multi-NIC hosts. + #[arg(long)] + interface: Option, + /// YAML scenario file to execute. + #[arg(long)] + scenario: Option, + /// Echo serial output while running. + #[arg(long)] + verbose: bool, +} + +pub fn run(args: &Args) -> Result<()> { + let scenario_path = args.scenario.clone().unwrap_or_else(|| { + workspace_root() + .join("xtask") + .join("e2e") + .join("wifi_ap_mode.yml") + }); + let scenario = load_scenario(&scenario_path)?; + let port = Serial::resolve_port(args.prg_serial.as_deref())?; + + let serial = Serial::open(&port, args.verbose)?; + let mut state = ContextState::default(); + for action in scenario.actions { + match action { + Action::RestartTargetDevice => { + let shell = Shell::new()?; + let soc = args.board.soc; + cmd!(shell, "espflash reset --port {port} --chip {soc}") + .run() + .context("espflash reset failed")?; + } + Action::ParseSshStampNetworkFromSerial => { + let info = parse_network_from_serial(&serial, SERIAL_PARSE_TIMEOUT)?; + eprintln!("parsed SSID={} and IP={}", info.ssid, info.ip); + state.network = Some(info); + } + Action::ConnectToSsidUsingPsk => { + let network = state + .network + .as_ref() + .context("network info is not parsed yet")?; + AccessPoint { + ssid: network.ssid.clone(), + psk: network.psk.clone(), + } + .join(args.interface.as_deref())?; + } + Action::VerifyIpObtained => { + let network = state + .network + .as_ref() + .context("network info is not parsed yet")?; + verify_host_ip_obtained(args.interface.as_deref(), network.ip)?; + } + Action::WaitForReachable => { + let network = state + .network + .as_ref() + .context("network info is not parsed yet")?; + let access_point = AccessPoint { + ssid: network.ssid.clone(), + psk: network.psk.clone(), + }; + if !access_point + .wait_for_reachable(&network.ip.to_string(), args.interface.as_deref()) + { + bail!("{} did not become reachable", network.ip); + } + } + } + } + serial.report_health(); + Ok(()) +} + +fn load_scenario(path: &PathBuf) -> Result { + let raw = + std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?; + serde_yaml::from_str(&raw).with_context(|| format!("parsing {}", path.display())) +} + +fn parse_network_from_serial(serial: &Serial, timeout: Duration) -> Result { + let deadline = Instant::now() + timeout; + loop { + let lines = serial.current_capture(); + if let Some(parsed) = parse_network_lines(&lines) { + return Ok(parsed); + } + if Instant::now() >= deadline { + bail!("timed out waiting for SSID, PSK and IP in serial output"); + } + std::thread::sleep(Duration::from_millis(250)); + } +} + +fn parse_network_lines(lines: &[String]) -> Option { + let mut ssid: Option = None; + let mut psk: Option = None; + let mut ip: Option = None; + + for raw in lines { + let line = strip_str(raw); + if let Some(v) = line.split_once("WIFI SSID:").map(|(_, v)| v.trim()) { + if !v.is_empty() { + ssid = Some(v.to_string()); + } + } + if let Some(v) = line.split_once("WIFI PSK:").map(|(_, v)| v.trim()) { + if !v.is_empty() { + psk = Some(v.to_string()); + } + } + if let Some(v) = line.split_once(" with IP ").map(|(_, v)| v.trim()) { + let token = v.split_whitespace().next().unwrap_or_default(); + let token = token.split('/').next().unwrap_or_default(); + if let Ok(addr) = token.parse::() { + ip = Some(addr); + } + } + } + + Some(NetworkInfo { + ssid: ssid?, + psk: psk?, + ip: ip?, + }) +} + +fn verify_host_ip_obtained(interface: Option<&str>, target_ip: Ipv4Addr) -> Result<()> { + let shell = Shell::new()?; + if cfg!(target_os = "linux") { + let output = match interface { + Some(iface) => cmd!(shell, "ip -4 addr show dev {iface}").read(), + None => { + let ip = target_ip.to_string(); + cmd!(shell, "ip -4 route get {ip}").read() + } + } + .context("checking host IPv4 configuration")?; + if output.contains(" inet ") || output.contains(" src ") { + return Ok(()); + } + bail!("host has no IPv4 address for this network"); + } + + if cfg!(target_os = "macos") { + let iface = interface.context("`--interface` is required to verify host IP on macOS")?; + let output = cmd!(shell, "ipconfig getifaddr {iface}") + .read() + .context("checking host IPv4 configuration")?; + if output.trim().parse::().is_ok() { + return Ok(()); + } + bail!("host has no IPv4 address on interface {iface}"); + } + + if cfg!(windows) { + let output = match interface { + Some(name) => cmd!(shell, "netsh interface ipv4 show addresses name={name}").read(), + None => cmd!(shell, "ipconfig").read(), + } + .context("checking host IPv4 configuration")?; + if output.contains("IP Address") || output.contains("IPv4 Address") { + return Ok(()); + } + bail!("host has no IPv4 address for this network"); + } + + bail!("host IP verification is unsupported on this OS") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_serial_wifi_lines() { + let lines = vec![ + "\u{1b}[32mINFO - WIFI SSID: test-ssid\u{1b}[0m".to_string(), + "INFO - WIFI PSK: test-password".to_string(), + "INFO - Connect to the AP `test-ssid` with IP 192.168.4.1/24".to_string(), + ]; + let parsed = parse_network_lines(&lines).unwrap(); + assert_eq!(parsed.ssid, "test-ssid"); + assert_eq!(parsed.psk, "test-password"); + assert_eq!(parsed.ip, "192.168.4.1".parse::().unwrap()); + } + + #[test] + fn parses_wifi_ap_mode_scenario() { + let scenario = serde_yaml::from_str::( + r#" +actions: + - restart_target_device + - parse_ssh_stamp_network_from_serial + - connect_to_ssid_using_psk + - verify_ip_obtained + - wait_for_reachable +"#, + ) + .unwrap(); + assert_eq!(scenario.actions.len(), 5); + } +} diff --git a/xtask/src/cmd/mod.rs b/xtask/src/cmd/mod.rs index 5412aec..93b5603 100644 --- a/xtask/src/cmd/mod.rs +++ b/xtask/src/cmd/mod.rs @@ -7,6 +7,7 @@ pub mod bench; pub mod bmf; pub mod cargo; +pub mod e2e; pub mod list; pub mod reset; pub mod size; diff --git a/xtask/src/host.rs b/xtask/src/host.rs index ab1d7f0..0613406 100644 --- a/xtask/src/host.rs +++ b/xtask/src/host.rs @@ -54,36 +54,40 @@ impl AccessPoint { .context("networksetup -setairportnetwork"); } - let ifname = interface - .map(|interface| vec!["ifname".to_string(), interface.to_string()]) - .unwrap_or_default(); + if cfg!(target_os = "linux") { + let ifname = interface + .map(|interface| vec!["ifname".to_string(), interface.to_string()]) + .unwrap_or_default(); - let nmcli = cmd!( - shell, - "nmcli device wifi connect {ssid} password {psk} {ifname...}" - ) - .secret() - .quiet() - .run(); - - if let Err(err) = nmcli { - let Some(station) = interface else { - bail!( - "nmcli failed with `{err}`, it's possible to use iwctl but `--interface` must be specified" - ); - }; - - return cmd!( + let nmcli = cmd!( shell, - "iwctl --passphrase {psk} station {station} connect {ssid}" + "nmcli device wifi connect {ssid} password {psk} {ifname...}" ) .secret() .quiet() - .run() - .with_context(|| format!("iwctl fallback after nmcli failed with `{err}`")); + .run(); + + if let Err(err) = nmcli { + let Some(station) = interface else { + bail!( + "nmcli failed with `{err}`, it's possible to use iwctl but `--interface` must be specified" + ); + }; + + return cmd!( + shell, + "iwctl --passphrase {psk} station {station} connect {ssid}" + ) + .secret() + .quiet() + .run() + .with_context(|| format!("iwctl fallback after nmcli failed with `{err}`")); + } + + return Ok(()); } - Ok(()) + bail!("joining Wi-Fi access points is unsupported on this OS") } /// Imports a profile for `ssid` into the Windows WLAN storage. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index b80ddfb..c4e3553 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -49,6 +49,8 @@ enum Command { Bench(cmd::bench::Args), /// Convert benchmark results.json into Bencher Metric Format. Bmf(cmd::bmf::Args), + /// Run basic end-to-end integration testing on hardware. + E2e(cmd::e2e::Args), /// Determine the size of a firmware build. Size(cmd::size::Args), /// Resets the storage on board, while keeping the firmware intact. @@ -63,6 +65,7 @@ fn main() -> Result<()> { Command::List(args) => cmd::list::run(&args), Command::Bench(mut args) => cmd::bench::run(&mut args), Command::Bmf(args) => cmd::bmf::run(&args), + Command::E2e(args) => cmd::e2e::run(&args), Command::Size(args) => cmd::size::run(&args), Command::Reset(args) => cmd::reset::run(&args), Command::Cargo(argv) => cmd::cargo::run(&argv), @@ -101,6 +104,15 @@ mod tests { .map_err(Box::new) } + fn e2e_cmd(argv: &[&str]) -> Result> { + Cli::try_parse_from(["xtask", "e2e"].into_iter().chain(argv.iter().copied())) + .map(|cli| match cli.command { + Command::E2e(parsed) => parsed, + _ => unreachable!(), + }) + .map_err(Box::new) + } + #[test] fn cli_is_consistent() { Cli::command().debug_assert(); @@ -217,4 +229,26 @@ mod tests { .is_err() ); } + + #[test] + fn e2e() { + assert!(e2e_cmd(&["--board", "esp32c6-devkitc"]).is_ok()); + assert!( + e2e_cmd(&[ + "--board", + "esp32c6-devkitc", + "--prg-serial", + "/dev/ttyUSB0", + "--interface", + "wlan0", + "--scenario", + "xtask/e2e/wifi_ap_mode.yml", + "--verbose" + ]) + .is_ok() + ); + assert!(e2e_cmd(&["--board", "esp32-fake-name"]).is_err()); + assert!(e2e_cmd(&[]).is_err()); + assert!(e2e_cmd(&["--board", "esp32c6-devkitc", "--host", "1.2.3.4"]).is_err()); + } } From 185f1894ac6caf4617ad900eb424e599d7d78ac7 Mon Sep 17 00:00:00 2001 From: Julio Beltran Ortega Date: Tue, 15 Sep 2026 21:52:00 +0000 Subject: [PATCH 2/4] xtask e2e refactor Removing redundant action and refactoring the action match into smaller handlers for clarity. --- xtask/e2e/wifi_ap_mode.yml | 1 - xtask/src/cmd/e2e.rs | 139 +++++++++++++------------------------ 2 files changed, 49 insertions(+), 91 deletions(-) diff --git a/xtask/e2e/wifi_ap_mode.yml b/xtask/e2e/wifi_ap_mode.yml index c364b7d..459ffc3 100644 --- a/xtask/e2e/wifi_ap_mode.yml +++ b/xtask/e2e/wifi_ap_mode.yml @@ -2,5 +2,4 @@ actions: - restart_target_device - parse_ssh_stamp_network_from_serial - connect_to_ssid_using_psk - - verify_ip_obtained - wait_for_reachable diff --git a/xtask/src/cmd/e2e.rs b/xtask/src/cmd/e2e.rs index d389065..36da660 100644 --- a/xtask/src/cmd/e2e.rs +++ b/xtask/src/cmd/e2e.rs @@ -32,7 +32,6 @@ enum Action { RestartTargetDevice, ParseSshStampNetworkFromSerial, ConnectToSsidUsingPsk, - VerifyIpObtained, WaitForReachable, } @@ -78,60 +77,64 @@ pub fn run(args: &Args) -> Result<()> { let port = Serial::resolve_port(args.prg_serial.as_deref())?; let serial = Serial::open(&port, args.verbose)?; - let mut state = ContextState::default(); + let mut context = ContextState::default(); for action in scenario.actions { match action { - Action::RestartTargetDevice => { - let shell = Shell::new()?; - let soc = args.board.soc; - cmd!(shell, "espflash reset --port {port} --chip {soc}") - .run() - .context("espflash reset failed")?; - } + Action::RestartTargetDevice => restart_target_device(args, &port, &mut context)?, Action::ParseSshStampNetworkFromSerial => { - let info = parse_network_from_serial(&serial, SERIAL_PARSE_TIMEOUT)?; - eprintln!("parsed SSID={} and IP={}", info.ssid, info.ip); - state.network = Some(info); - } - Action::ConnectToSsidUsingPsk => { - let network = state - .network - .as_ref() - .context("network info is not parsed yet")?; - AccessPoint { - ssid: network.ssid.clone(), - psk: network.psk.clone(), - } - .join(args.interface.as_deref())?; - } - Action::VerifyIpObtained => { - let network = state - .network - .as_ref() - .context("network info is not parsed yet")?; - verify_host_ip_obtained(args.interface.as_deref(), network.ip)?; - } - Action::WaitForReachable => { - let network = state - .network - .as_ref() - .context("network info is not parsed yet")?; - let access_point = AccessPoint { - ssid: network.ssid.clone(), - psk: network.psk.clone(), - }; - if !access_point - .wait_for_reachable(&network.ip.to_string(), args.interface.as_deref()) - { - bail!("{} did not become reachable", network.ip); - } + parse_ssh_stamp_network_from_serial(&serial, &mut context)? } + Action::ConnectToSsidUsingPsk => connect_to_ssid_using_psk(args, &mut context)?, + Action::WaitForReachable => wait_for_reachable(args, &mut context)?, } } serial.report_health(); Ok(()) } +fn restart_target_device(args: &Args, port: &str, _context: &mut ContextState) -> Result<()> { + let shell = Shell::new()?; + let soc = args.board.soc; + cmd!(shell, "espflash reset --port {port} --chip {soc}") + .run() + .context("espflash reset failed") +} + +fn parse_ssh_stamp_network_from_serial(serial: &Serial, context: &mut ContextState) -> Result<()> { + let info = parse_network_from_serial(serial, SERIAL_PARSE_TIMEOUT)?; + eprintln!("parsed SSID={} and IP={}", info.ssid, info.ip); + context.network = Some(info); + Ok(()) +} + +fn connect_to_ssid_using_psk(args: &Args, context: &mut ContextState) -> Result<()> { + let network = context + .network + .as_ref() + .context("network info is not parsed yet")?; + AccessPoint { + ssid: network.ssid.clone(), + psk: network.psk.clone(), + } + .join(args.interface.as_deref()) +} + +fn wait_for_reachable(args: &Args, context: &mut ContextState) -> Result<()> { + let network = context + .network + .as_ref() + .context("network info is not parsed yet")?; + let access_point = AccessPoint { + ssid: network.ssid.clone(), + psk: network.psk.clone(), + }; + if access_point.wait_for_reachable(&network.ip.to_string(), args.interface.as_deref()) { + Ok(()) + } else { + bail!("{} did not become reachable", network.ip) + } +} + fn load_scenario(path: &PathBuf) -> Result { let raw = std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?; @@ -185,49 +188,6 @@ fn parse_network_lines(lines: &[String]) -> Option { }) } -fn verify_host_ip_obtained(interface: Option<&str>, target_ip: Ipv4Addr) -> Result<()> { - let shell = Shell::new()?; - if cfg!(target_os = "linux") { - let output = match interface { - Some(iface) => cmd!(shell, "ip -4 addr show dev {iface}").read(), - None => { - let ip = target_ip.to_string(); - cmd!(shell, "ip -4 route get {ip}").read() - } - } - .context("checking host IPv4 configuration")?; - if output.contains(" inet ") || output.contains(" src ") { - return Ok(()); - } - bail!("host has no IPv4 address for this network"); - } - - if cfg!(target_os = "macos") { - let iface = interface.context("`--interface` is required to verify host IP on macOS")?; - let output = cmd!(shell, "ipconfig getifaddr {iface}") - .read() - .context("checking host IPv4 configuration")?; - if output.trim().parse::().is_ok() { - return Ok(()); - } - bail!("host has no IPv4 address on interface {iface}"); - } - - if cfg!(windows) { - let output = match interface { - Some(name) => cmd!(shell, "netsh interface ipv4 show addresses name={name}").read(), - None => cmd!(shell, "ipconfig").read(), - } - .context("checking host IPv4 configuration")?; - if output.contains("IP Address") || output.contains("IPv4 Address") { - return Ok(()); - } - bail!("host has no IPv4 address for this network"); - } - - bail!("host IP verification is unsupported on this OS") -} - #[cfg(test)] mod tests { use super::*; @@ -253,11 +213,10 @@ actions: - restart_target_device - parse_ssh_stamp_network_from_serial - connect_to_ssid_using_psk - - verify_ip_obtained - wait_for_reachable "#, ) .unwrap(); - assert_eq!(scenario.actions.len(), 5); + assert_eq!(scenario.actions.len(), 4); } } From 95edcf866cbf578f6171660595d2a90ee43ef4f0 Mon Sep 17 00:00:00 2001 From: brainstorm Date: Wed, 21 Oct 2026 19:27:21 +0200 Subject: [PATCH 3/4] refactor(xtask): move e2e under `xtask test e2e` with scenarios in Rust Scenarios are now plain Rust functions selected with `--scenario` (defaulting to `wifi-ap`), instead of a YAML list of actions, which drops the serde_yaml dependency. The SSID, PSK and AP address come from the `boot` and `net_up` JSON objects the firmware prints since #123, rather than scraping log lines, so the AP address is no longer assumed. Objects from before the reset are ignored, and an unknown JSON schema version fails the run. Also align with the other commands: `--port` instead of `--prg-serial`, and no separate join step since `wait_for_reachable` already joins. The unrelated host.rs change is dropped. --- Cargo.lock | 1 - xtask/Cargo.toml | 1 - xtask/e2e/wifi_ap_mode.yml | 5 - xtask/src/cmd/e2e.rs | 222 -------------------------------- xtask/src/cmd/mod.rs | 2 +- xtask/src/cmd/test/e2e.rs | 251 +++++++++++++++++++++++++++++++++++++ xtask/src/cmd/test/mod.rs | 30 +++++ xtask/src/host.rs | 50 ++++---- xtask/src/main.rs | 35 ++++-- 9 files changed, 328 insertions(+), 269 deletions(-) delete mode 100644 xtask/e2e/wifi_ap_mode.yml delete mode 100644 xtask/src/cmd/e2e.rs create mode 100644 xtask/src/cmd/test/e2e.rs create mode 100644 xtask/src/cmd/test/mod.rs diff --git a/Cargo.lock b/Cargo.lock index a35c278..a4f2ff9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4639,7 +4639,6 @@ dependencies = [ "quick-xml", "serde", "serde_json", - "serde_yaml", "serial2", "ssh-key", "ssh-stamp", diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 93ca680..3d6b1f0 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -31,7 +31,6 @@ object = { version = "0.37", default-features = false, features = [ ] } serde = { version = "1", features = ["derive"] } serde_json = "1" -serde_yaml = "0.9" serial2 = "0.2" strip-ansi-escapes = "0.2" xshell = "0.2" diff --git a/xtask/e2e/wifi_ap_mode.yml b/xtask/e2e/wifi_ap_mode.yml deleted file mode 100644 index 459ffc3..0000000 --- a/xtask/e2e/wifi_ap_mode.yml +++ /dev/null @@ -1,5 +0,0 @@ -actions: - - restart_target_device - - parse_ssh_stamp_network_from_serial - - connect_to_ssid_using_psk - - wait_for_reachable diff --git a/xtask/src/cmd/e2e.rs b/xtask/src/cmd/e2e.rs deleted file mode 100644 index 36da660..0000000 --- a/xtask/src/cmd/e2e.rs +++ /dev/null @@ -1,222 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Julio Beltran Ortega -// -// SPDX-License-Identifier: GPL-3.0-or-later - -//! `xtask e2e` -//! -//! Runs e2e scenarios from YAML steps. - -use crate::board::{self, Board}; -use crate::device::Serial; -use crate::host::AccessPoint; -use crate::util::workspace_root; -use anyhow::{Context, Result, bail}; -use clap::Args as ClapArgs; -use serde::Deserialize; -use std::net::Ipv4Addr; -use std::path::PathBuf; -use std::time::{Duration, Instant}; -use strip_ansi_escapes::strip_str; -use xshell::{Shell, cmd}; - -const SERIAL_PARSE_TIMEOUT: Duration = Duration::from_secs(45); - -#[derive(Debug, Deserialize)] -struct Scenario { - actions: Vec, -} - -#[derive(Debug, Clone, Deserialize)] -#[serde(rename_all = "snake_case")] -enum Action { - RestartTargetDevice, - ParseSshStampNetworkFromSerial, - ConnectToSsidUsingPsk, - WaitForReachable, -} - -#[derive(Debug, Clone)] -struct NetworkInfo { - ssid: String, - psk: String, - ip: Ipv4Addr, -} - -#[derive(Default)] -struct ContextState { - network: Option, -} - -#[derive(ClapArgs)] -pub struct Args { - /// Board to run end-to-end integration testing on. - #[arg(long, value_parser = board::name_parser())] - board: &'static Board, - /// The serial device used to program/reset and read logs (prg_serial). - #[arg(long)] - prg_serial: Option, - /// The wireless interface that joins the device's AP, needed on multi-NIC hosts. - #[arg(long)] - interface: Option, - /// YAML scenario file to execute. - #[arg(long)] - scenario: Option, - /// Echo serial output while running. - #[arg(long)] - verbose: bool, -} - -pub fn run(args: &Args) -> Result<()> { - let scenario_path = args.scenario.clone().unwrap_or_else(|| { - workspace_root() - .join("xtask") - .join("e2e") - .join("wifi_ap_mode.yml") - }); - let scenario = load_scenario(&scenario_path)?; - let port = Serial::resolve_port(args.prg_serial.as_deref())?; - - let serial = Serial::open(&port, args.verbose)?; - let mut context = ContextState::default(); - for action in scenario.actions { - match action { - Action::RestartTargetDevice => restart_target_device(args, &port, &mut context)?, - Action::ParseSshStampNetworkFromSerial => { - parse_ssh_stamp_network_from_serial(&serial, &mut context)? - } - Action::ConnectToSsidUsingPsk => connect_to_ssid_using_psk(args, &mut context)?, - Action::WaitForReachable => wait_for_reachable(args, &mut context)?, - } - } - serial.report_health(); - Ok(()) -} - -fn restart_target_device(args: &Args, port: &str, _context: &mut ContextState) -> Result<()> { - let shell = Shell::new()?; - let soc = args.board.soc; - cmd!(shell, "espflash reset --port {port} --chip {soc}") - .run() - .context("espflash reset failed") -} - -fn parse_ssh_stamp_network_from_serial(serial: &Serial, context: &mut ContextState) -> Result<()> { - let info = parse_network_from_serial(serial, SERIAL_PARSE_TIMEOUT)?; - eprintln!("parsed SSID={} and IP={}", info.ssid, info.ip); - context.network = Some(info); - Ok(()) -} - -fn connect_to_ssid_using_psk(args: &Args, context: &mut ContextState) -> Result<()> { - let network = context - .network - .as_ref() - .context("network info is not parsed yet")?; - AccessPoint { - ssid: network.ssid.clone(), - psk: network.psk.clone(), - } - .join(args.interface.as_deref()) -} - -fn wait_for_reachable(args: &Args, context: &mut ContextState) -> Result<()> { - let network = context - .network - .as_ref() - .context("network info is not parsed yet")?; - let access_point = AccessPoint { - ssid: network.ssid.clone(), - psk: network.psk.clone(), - }; - if access_point.wait_for_reachable(&network.ip.to_string(), args.interface.as_deref()) { - Ok(()) - } else { - bail!("{} did not become reachable", network.ip) - } -} - -fn load_scenario(path: &PathBuf) -> Result { - let raw = - std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?; - serde_yaml::from_str(&raw).with_context(|| format!("parsing {}", path.display())) -} - -fn parse_network_from_serial(serial: &Serial, timeout: Duration) -> Result { - let deadline = Instant::now() + timeout; - loop { - let lines = serial.current_capture(); - if let Some(parsed) = parse_network_lines(&lines) { - return Ok(parsed); - } - if Instant::now() >= deadline { - bail!("timed out waiting for SSID, PSK and IP in serial output"); - } - std::thread::sleep(Duration::from_millis(250)); - } -} - -fn parse_network_lines(lines: &[String]) -> Option { - let mut ssid: Option = None; - let mut psk: Option = None; - let mut ip: Option = None; - - for raw in lines { - let line = strip_str(raw); - if let Some(v) = line.split_once("WIFI SSID:").map(|(_, v)| v.trim()) { - if !v.is_empty() { - ssid = Some(v.to_string()); - } - } - if let Some(v) = line.split_once("WIFI PSK:").map(|(_, v)| v.trim()) { - if !v.is_empty() { - psk = Some(v.to_string()); - } - } - if let Some(v) = line.split_once(" with IP ").map(|(_, v)| v.trim()) { - let token = v.split_whitespace().next().unwrap_or_default(); - let token = token.split('/').next().unwrap_or_default(); - if let Ok(addr) = token.parse::() { - ip = Some(addr); - } - } - } - - Some(NetworkInfo { - ssid: ssid?, - psk: psk?, - ip: ip?, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parses_serial_wifi_lines() { - let lines = vec![ - "\u{1b}[32mINFO - WIFI SSID: test-ssid\u{1b}[0m".to_string(), - "INFO - WIFI PSK: test-password".to_string(), - "INFO - Connect to the AP `test-ssid` with IP 192.168.4.1/24".to_string(), - ]; - let parsed = parse_network_lines(&lines).unwrap(); - assert_eq!(parsed.ssid, "test-ssid"); - assert_eq!(parsed.psk, "test-password"); - assert_eq!(parsed.ip, "192.168.4.1".parse::().unwrap()); - } - - #[test] - fn parses_wifi_ap_mode_scenario() { - let scenario = serde_yaml::from_str::( - r#" -actions: - - restart_target_device - - parse_ssh_stamp_network_from_serial - - connect_to_ssid_using_psk - - wait_for_reachable -"#, - ) - .unwrap(); - assert_eq!(scenario.actions.len(), 4); - } -} diff --git a/xtask/src/cmd/mod.rs b/xtask/src/cmd/mod.rs index 93b5603..9798672 100644 --- a/xtask/src/cmd/mod.rs +++ b/xtask/src/cmd/mod.rs @@ -7,10 +7,10 @@ pub mod bench; pub mod bmf; pub mod cargo; -pub mod e2e; pub mod list; pub mod reset; pub mod size; +pub mod test; /// The extra features every build has. pub const BENCH_FEATURES: &[&str] = &["mem-probe", "bench-loopback"]; diff --git a/xtask/src/cmd/test/e2e.rs b/xtask/src/cmd/test/e2e.rs new file mode 100644 index 0000000..bf5655b --- /dev/null +++ b/xtask/src/cmd/test/e2e.rs @@ -0,0 +1,251 @@ +// SPDX-FileCopyrightText: 2026 Julio Beltran Ortega +// SPDX-FileCopyrightText: 2026 Roman Valls Guimera +// +// SPDX-License-Identifier: GPL-3.0-or-later + +//! `xtask test e2e` +//! +//! Runs end-to-end scenarios against an already flashed board. A scenario is +//! a plain function calling its steps in order, and the device state it needs +//! comes from the `boot` and `net_up` JSON objects on the serial console. + +use crate::board::{self, Board}; +use crate::device::{self, Serial}; +use crate::host::AccessPoint; +use crate::util::{retry, shell}; +use anyhow::{Context, Result, bail, ensure}; +use clap::{Args as ClapArgs, ValueEnum}; +use serde::Deserialize; +use ssh_stamp::json::SCHEMA; +use std::net::Ipv4Addr; +use std::time::Duration; +use xshell::cmd; + +/// How often the serial capture is checked for new events. +const POLL_INTERVAL: Duration = Duration::from_millis(250); + +#[derive(ClapArgs)] +pub struct Args { + /// Board to run the scenario on. + #[arg(long, value_parser = board::name_parser())] + board: &'static Board, + /// The serial port used to reset the board and read its console. + #[arg(long)] + port: Option, + /// The wireless interface that joins the device's access point, needed on multi-NIC hosts. + #[arg(long)] + interface: Option, + /// The scenario to run. + #[arg(long, value_enum, default_value_t = Scenario::WifiAp)] + scenario: Scenario, + /// Echo serial output while running. + #[arg(long)] + verbose: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub enum Scenario { + /// Reset the board, join the access point it hosts and reach its SSH port. + WifiAp, +} + +pub fn run(args: &Args) -> Result<()> { + let port = Serial::resolve_port(args.port.as_deref())?; + let serial = Serial::open(&port, args.verbose)?; + + let result = match args.scenario { + Scenario::WifiAp => wifi_ap(args, &port, &serial), + }; + + serial.report_health(); + result +} + +fn wifi_ap(args: &Args, port: &str, serial: &Serial) -> Result<()> { + let (boot, net_up) = reset_and_wait_for_network(args.board, port, serial)?; + ensure!( + net_up.role == "ap", + "expected the device to host an access point, but its role is `{}`", + net_up.role + ); + eprintln!( + "=== device hosts {} at {} ===", + boot.wifi_ap.ssid, net_up.ip + ); + + let access_point = AccessPoint { + ssid: boot.wifi_ap.ssid, + psk: boot.wifi_ap.psk, + }; + ensure!( + access_point.wait_for_reachable(&net_up.ip.to_string(), args.interface.as_deref()), + "{} did not become reachable", + net_up.ip + ); + + eprintln!("=== e2e {:?} passed ===", args.scenario); + Ok(()) +} + +/// Resets the board and waits for the `boot` and `net_up` events of the boot +/// that follows, ignoring anything captured before the reset. +fn reset_and_wait_for_network(board: &Board, port: &str, serial: &Serial) -> Result<(Boot, NetUp)> { + let skip = serial.current_capture().len(); + let shell = shell()?; + let soc = board.soc; + cmd!(shell, "espflash reset --port {port} --chip {soc}") + .run() + .context("espflash reset failed")?; + + retry(device::BOOT_TIMEOUT, POLL_INTERVAL, || { + let lines = serial.current_capture(); + let events = events(lines.get(skip..).unwrap_or_default())?; + let boot = events.iter().find_map(|event| match event { + Event::Boot(boot) => Some(boot.clone()), + _ => None, + }); + let net_up = events.iter().find_map(|event| match event { + Event::NetUp(net_up) => Some(net_up.clone()), + _ => None, + }); + match (boot, net_up) { + (Some(boot), Some(net_up)) => Ok((boot, net_up)), + (None, _) => bail!("timed out waiting for the `boot` event"), + (_, None) => bail!("timed out waiting for the `net_up` event"), + } + }) +} + +/// The version every JSON object the firmware prints starts with, see `ssh_stamp::json`. +#[derive(Debug, Deserialize)] +struct Schema { + schema: u32, +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "event", rename_all = "snake_case")] +enum Event { + Boot(Boot), + NetUp(NetUp), + #[serde(other)] + Other, +} + +#[derive(Debug, Clone, Deserialize)] +struct Boot { + wifi_ap: WifiAp, +} + +#[derive(Debug, Clone, Deserialize)] +struct WifiAp { + ssid: String, + psk: String, +} + +#[derive(Debug, Clone, Deserialize)] +struct NetUp { + role: String, + ip: Ipv4Addr, +} + +/// Cuts the firmware's JSON objects out of the console `lines`. A top-level +/// object always opens and closes with `{` and `}` alone on their lines, so +/// everything else is console noise. Objects cut short, e.g. by a reset, are +/// skipped. +fn events(lines: &[String]) -> Result> { + let mut events = Vec::new(); + let mut object: Option = None; + + for line in lines { + match line.as_str() { + "{" => object = Some(String::from("{")), + "}" => { + let Some(mut text) = object.take() else { + continue; + }; + text.push('}'); + let Ok(Schema { schema }) = serde_json::from_str(&text) else { + continue; + }; + ensure!( + schema == SCHEMA, + "the firmware prints JSON schema {schema}, but xtask understands {SCHEMA}" + ); + events.push(serde_json::from_str(&text).context("parsing a console event")?); + } + _ => { + if let Some(text) = object.as_mut() { + text.push_str(line); + } + } + } + } + + Ok(events) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn lines(text: &str) -> Vec { + text.lines().map(str::to_string).collect() + } + + #[test] + fn parses_console_events() { + let console = lines( + r#"ESP-ROM:esp32c6-20220919 +{ + "schema": 1, + "event": "boot", + "ident": "SSH-2.0-ssh-stamp", + "wifi_ap": { + "ssid": "my \"ssid\"", + "psk": "test-password", + "band": "2.4GHz", + "channel": 1 + }, + "first_login": true +} +INFO - unrelated log line +{ + "schema": 1, + "event": "net_up", + "role": "ap", + "ssid": "my \"ssid\"", + "ip": "10.0.0.1" +} +{ + "schema": 1, + "event": "log", + "msg": "ignored" +}"#, + ); + let events = events(&console).unwrap(); + let [Event::Boot(boot), Event::NetUp(net_up), Event::Other] = events.as_slice() else { + panic!("unexpected events: {events:?}"); + }; + assert_eq!(boot.wifi_ap.ssid, r#"my "ssid""#); + assert_eq!(boot.wifi_ap.psk, "test-password"); + assert_eq!(net_up.role, "ap"); + assert_eq!(net_up.ip, Ipv4Addr::new(10, 0, 0, 1)); + } + + #[test] + fn skips_truncated_objects() { + let console = lines( + "{\n \"schema\": 1,\n \"event\": \"bo\n{\n \"schema\": 1,\n \"event\": \"net_up\",\n \"role\": \"ap\",\n \"ip\": \"192.168.4.1\"\n}", + ); + assert!(matches!( + events(&console).unwrap().as_slice(), + [Event::NetUp(_)] + )); + } + + #[test] + fn rejects_unknown_schema() { + let console = lines("{\n \"schema\": 999,\n \"event\": \"boot\"\n}"); + assert!(events(&console).is_err()); + } +} diff --git a/xtask/src/cmd/test/mod.rs b/xtask/src/cmd/test/mod.rs new file mode 100644 index 0000000..8e8ab5c --- /dev/null +++ b/xtask/src/cmd/test/mod.rs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2026 Roman Valls Guimera +// +// SPDX-License-Identifier: GPL-3.0-or-later + +//! `xtask test` +//! +//! Test suites that need more than `cargo test`, such as real hardware. + +use anyhow::Result; +use clap::{Args as ClapArgs, Subcommand}; + +pub mod e2e; + +#[derive(ClapArgs)] +pub struct Args { + #[command(subcommand)] + pub command: Command, +} + +#[derive(Subcommand)] +pub enum Command { + /// Run end-to-end integration tests on hardware. + E2e(e2e::Args), +} + +pub fn run(args: &Args) -> Result<()> { + match &args.command { + Command::E2e(args) => e2e::run(args), + } +} diff --git a/xtask/src/host.rs b/xtask/src/host.rs index 0613406..ab1d7f0 100644 --- a/xtask/src/host.rs +++ b/xtask/src/host.rs @@ -54,40 +54,36 @@ impl AccessPoint { .context("networksetup -setairportnetwork"); } - if cfg!(target_os = "linux") { - let ifname = interface - .map(|interface| vec!["ifname".to_string(), interface.to_string()]) - .unwrap_or_default(); + let ifname = interface + .map(|interface| vec!["ifname".to_string(), interface.to_string()]) + .unwrap_or_default(); - let nmcli = cmd!( + let nmcli = cmd!( + shell, + "nmcli device wifi connect {ssid} password {psk} {ifname...}" + ) + .secret() + .quiet() + .run(); + + if let Err(err) = nmcli { + let Some(station) = interface else { + bail!( + "nmcli failed with `{err}`, it's possible to use iwctl but `--interface` must be specified" + ); + }; + + return cmd!( shell, - "nmcli device wifi connect {ssid} password {psk} {ifname...}" + "iwctl --passphrase {psk} station {station} connect {ssid}" ) .secret() .quiet() - .run(); - - if let Err(err) = nmcli { - let Some(station) = interface else { - bail!( - "nmcli failed with `{err}`, it's possible to use iwctl but `--interface` must be specified" - ); - }; - - return cmd!( - shell, - "iwctl --passphrase {psk} station {station} connect {ssid}" - ) - .secret() - .quiet() - .run() - .with_context(|| format!("iwctl fallback after nmcli failed with `{err}`")); - } - - return Ok(()); + .run() + .with_context(|| format!("iwctl fallback after nmcli failed with `{err}`")); } - bail!("joining Wi-Fi access points is unsupported on this OS") + Ok(()) } /// Imports a profile for `ssid` into the Windows WLAN storage. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index c4e3553..ab0e4bc 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -49,12 +49,12 @@ enum Command { Bench(cmd::bench::Args), /// Convert benchmark results.json into Bencher Metric Format. Bmf(cmd::bmf::Args), - /// Run basic end-to-end integration testing on hardware. - E2e(cmd::e2e::Args), /// Determine the size of a firmware build. Size(cmd::size::Args), /// Resets the storage on board, while keeping the firmware intact. Reset(cmd::reset::Args), + /// Run test suites that need more than `cargo test`, e.g. on hardware. + Test(cmd::test::Args), /// ` [args...]` forwarded to cargo. #[command(external_subcommand)] Cargo(Vec), @@ -65,9 +65,9 @@ fn main() -> Result<()> { Command::List(args) => cmd::list::run(&args), Command::Bench(mut args) => cmd::bench::run(&mut args), Command::Bmf(args) => cmd::bmf::run(&args), - Command::E2e(args) => cmd::e2e::run(&args), Command::Size(args) => cmd::size::run(&args), Command::Reset(args) => cmd::reset::run(&args), + Command::Test(args) => cmd::test::run(&args), Command::Cargo(argv) => cmd::cargo::run(&argv), } } @@ -104,13 +104,19 @@ mod tests { .map_err(Box::new) } - fn e2e_cmd(argv: &[&str]) -> Result> { - Cli::try_parse_from(["xtask", "e2e"].into_iter().chain(argv.iter().copied())) - .map(|cli| match cli.command { - Command::E2e(parsed) => parsed, - _ => unreachable!(), - }) - .map_err(Box::new) + fn e2e_cmd(argv: &[&str]) -> Result> { + Cli::try_parse_from( + ["xtask", "test", "e2e"] + .into_iter() + .chain(argv.iter().copied()), + ) + .map(|cli| match cli.command { + Command::Test(cmd::test::Args { + command: cmd::test::Command::E2e(parsed), + }) => parsed, + _ => unreachable!(), + }) + .map_err(Box::new) } #[test] @@ -237,18 +243,23 @@ mod tests { e2e_cmd(&[ "--board", "esp32c6-devkitc", - "--prg-serial", + "--port", "/dev/ttyUSB0", "--interface", "wlan0", "--scenario", - "xtask/e2e/wifi_ap_mode.yml", + "wifi-ap", "--verbose" ]) .is_ok() ); assert!(e2e_cmd(&["--board", "esp32-fake-name"]).is_err()); assert!(e2e_cmd(&[]).is_err()); + assert!(e2e_cmd(&["--board", "esp32c6-devkitc", "--scenario", "nope"]).is_err()); assert!(e2e_cmd(&["--board", "esp32c6-devkitc", "--host", "1.2.3.4"]).is_err()); + assert!( + Cli::try_parse_from(["xtask", "e2e", "--board", "esp32c6-devkitc"]) + .is_ok_and(|cli| matches!(cli.command, Command::Cargo(_))) + ); } } From dd33c10ea9c8f1781010b4bcf127183be9890221 Mon Sep 17 00:00:00 2001 From: brainstorm Date: Wed, 21 Oct 2026 19:27:21 +0200 Subject: [PATCH 4/4] docs: document `cargo xtask test e2e` --- docs/TESTING.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/TESTING.md b/docs/TESTING.md index 66ffc97..d39400e 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -52,3 +52,19 @@ Any board or chip target that `cargo xtask` accepts works in place of Each `tests/*.rs` file is a separate flash and run. A new file needs a `[[test]]` entry with `harness = false` in `Cargo.toml`. + +# End-to-end tests + +End-to-end scenarios drive a board that already runs the firmware +(`cargo xtask run`) from the host: they reset it, read the `boot` +and `net_up` JSON objects from its console, and act on them. + +```sh +cargo xtask test e2e --board esp32c6-devkitc +``` + +`--port` picks the serial port when there are several, and `--interface` the +host's wireless interface that joins the device's access point. Each scenario +is a plain function in [`xtask/src/cmd/test/e2e.rs`](../xtask/src/cmd/test/e2e.rs), +selected with `--scenario`; `wifi-ap`, the default, joins the access point the +device hosts and checks its SSH port is reachable.