diff --git a/docs/TESTING.md b/docs/TESTING.md index 66ffc97..d39400e 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -52,3 +52,19 @@ Any board or chip target that `cargo xtask` accepts works in place of Each `tests/*.rs` file is a separate flash and run. A new file needs a `[[test]]` entry with `harness = false` in `Cargo.toml`. + +# End-to-end tests + +End-to-end scenarios drive a board that already runs the firmware +(`cargo xtask run`) from the host: they reset it, read the `boot` +and `net_up` JSON objects from its console, and act on them. + +```sh +cargo xtask test e2e --board esp32c6-devkitc +``` + +`--port` picks the serial port when there are several, and `--interface` the +host's wireless interface that joins the device's access point. Each scenario +is a plain function in [`xtask/src/cmd/test/e2e.rs`](../xtask/src/cmd/test/e2e.rs), +selected with `--scenario`; `wifi-ap`, the default, joins the access point the +device hosts and checks its SSH port is reachable. diff --git a/xtask/src/cmd/mod.rs b/xtask/src/cmd/mod.rs index 5412aec..9798672 100644 --- a/xtask/src/cmd/mod.rs +++ b/xtask/src/cmd/mod.rs @@ -10,6 +10,7 @@ pub mod cargo; pub mod list; pub mod reset; pub mod size; +pub mod test; /// The extra features every build has. pub const BENCH_FEATURES: &[&str] = &["mem-probe", "bench-loopback"]; diff --git a/xtask/src/cmd/test/e2e.rs b/xtask/src/cmd/test/e2e.rs new file mode 100644 index 0000000..bf5655b --- /dev/null +++ b/xtask/src/cmd/test/e2e.rs @@ -0,0 +1,251 @@ +// SPDX-FileCopyrightText: 2026 Julio Beltran Ortega +// SPDX-FileCopyrightText: 2026 Roman Valls Guimera +// +// SPDX-License-Identifier: GPL-3.0-or-later + +//! `xtask test e2e` +//! +//! Runs end-to-end scenarios against an already flashed board. A scenario is +//! a plain function calling its steps in order, and the device state it needs +//! comes from the `boot` and `net_up` JSON objects on the serial console. + +use crate::board::{self, Board}; +use crate::device::{self, Serial}; +use crate::host::AccessPoint; +use crate::util::{retry, shell}; +use anyhow::{Context, Result, bail, ensure}; +use clap::{Args as ClapArgs, ValueEnum}; +use serde::Deserialize; +use ssh_stamp::json::SCHEMA; +use std::net::Ipv4Addr; +use std::time::Duration; +use xshell::cmd; + +/// How often the serial capture is checked for new events. +const POLL_INTERVAL: Duration = Duration::from_millis(250); + +#[derive(ClapArgs)] +pub struct Args { + /// Board to run the scenario on. + #[arg(long, value_parser = board::name_parser())] + board: &'static Board, + /// The serial port used to reset the board and read its console. + #[arg(long)] + port: Option, + /// The wireless interface that joins the device's access point, needed on multi-NIC hosts. + #[arg(long)] + interface: Option, + /// The scenario to run. + #[arg(long, value_enum, default_value_t = Scenario::WifiAp)] + scenario: Scenario, + /// Echo serial output while running. + #[arg(long)] + verbose: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub enum Scenario { + /// Reset the board, join the access point it hosts and reach its SSH port. + WifiAp, +} + +pub fn run(args: &Args) -> Result<()> { + let port = Serial::resolve_port(args.port.as_deref())?; + let serial = Serial::open(&port, args.verbose)?; + + let result = match args.scenario { + Scenario::WifiAp => wifi_ap(args, &port, &serial), + }; + + serial.report_health(); + result +} + +fn wifi_ap(args: &Args, port: &str, serial: &Serial) -> Result<()> { + let (boot, net_up) = reset_and_wait_for_network(args.board, port, serial)?; + ensure!( + net_up.role == "ap", + "expected the device to host an access point, but its role is `{}`", + net_up.role + ); + eprintln!( + "=== device hosts {} at {} ===", + boot.wifi_ap.ssid, net_up.ip + ); + + let access_point = AccessPoint { + ssid: boot.wifi_ap.ssid, + psk: boot.wifi_ap.psk, + }; + ensure!( + access_point.wait_for_reachable(&net_up.ip.to_string(), args.interface.as_deref()), + "{} did not become reachable", + net_up.ip + ); + + eprintln!("=== e2e {:?} passed ===", args.scenario); + Ok(()) +} + +/// Resets the board and waits for the `boot` and `net_up` events of the boot +/// that follows, ignoring anything captured before the reset. +fn reset_and_wait_for_network(board: &Board, port: &str, serial: &Serial) -> Result<(Boot, NetUp)> { + let skip = serial.current_capture().len(); + let shell = shell()?; + let soc = board.soc; + cmd!(shell, "espflash reset --port {port} --chip {soc}") + .run() + .context("espflash reset failed")?; + + retry(device::BOOT_TIMEOUT, POLL_INTERVAL, || { + let lines = serial.current_capture(); + let events = events(lines.get(skip..).unwrap_or_default())?; + let boot = events.iter().find_map(|event| match event { + Event::Boot(boot) => Some(boot.clone()), + _ => None, + }); + let net_up = events.iter().find_map(|event| match event { + Event::NetUp(net_up) => Some(net_up.clone()), + _ => None, + }); + match (boot, net_up) { + (Some(boot), Some(net_up)) => Ok((boot, net_up)), + (None, _) => bail!("timed out waiting for the `boot` event"), + (_, None) => bail!("timed out waiting for the `net_up` event"), + } + }) +} + +/// The version every JSON object the firmware prints starts with, see `ssh_stamp::json`. +#[derive(Debug, Deserialize)] +struct Schema { + schema: u32, +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "event", rename_all = "snake_case")] +enum Event { + Boot(Boot), + NetUp(NetUp), + #[serde(other)] + Other, +} + +#[derive(Debug, Clone, Deserialize)] +struct Boot { + wifi_ap: WifiAp, +} + +#[derive(Debug, Clone, Deserialize)] +struct WifiAp { + ssid: String, + psk: String, +} + +#[derive(Debug, Clone, Deserialize)] +struct NetUp { + role: String, + ip: Ipv4Addr, +} + +/// Cuts the firmware's JSON objects out of the console `lines`. A top-level +/// object always opens and closes with `{` and `}` alone on their lines, so +/// everything else is console noise. Objects cut short, e.g. by a reset, are +/// skipped. +fn events(lines: &[String]) -> Result> { + let mut events = Vec::new(); + let mut object: Option = None; + + for line in lines { + match line.as_str() { + "{" => object = Some(String::from("{")), + "}" => { + let Some(mut text) = object.take() else { + continue; + }; + text.push('}'); + let Ok(Schema { schema }) = serde_json::from_str(&text) else { + continue; + }; + ensure!( + schema == SCHEMA, + "the firmware prints JSON schema {schema}, but xtask understands {SCHEMA}" + ); + events.push(serde_json::from_str(&text).context("parsing a console event")?); + } + _ => { + if let Some(text) = object.as_mut() { + text.push_str(line); + } + } + } + } + + Ok(events) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn lines(text: &str) -> Vec { + text.lines().map(str::to_string).collect() + } + + #[test] + fn parses_console_events() { + let console = lines( + r#"ESP-ROM:esp32c6-20220919 +{ + "schema": 1, + "event": "boot", + "ident": "SSH-2.0-ssh-stamp", + "wifi_ap": { + "ssid": "my \"ssid\"", + "psk": "test-password", + "band": "2.4GHz", + "channel": 1 + }, + "first_login": true +} +INFO - unrelated log line +{ + "schema": 1, + "event": "net_up", + "role": "ap", + "ssid": "my \"ssid\"", + "ip": "10.0.0.1" +} +{ + "schema": 1, + "event": "log", + "msg": "ignored" +}"#, + ); + let events = events(&console).unwrap(); + let [Event::Boot(boot), Event::NetUp(net_up), Event::Other] = events.as_slice() else { + panic!("unexpected events: {events:?}"); + }; + assert_eq!(boot.wifi_ap.ssid, r#"my "ssid""#); + assert_eq!(boot.wifi_ap.psk, "test-password"); + assert_eq!(net_up.role, "ap"); + assert_eq!(net_up.ip, Ipv4Addr::new(10, 0, 0, 1)); + } + + #[test] + fn skips_truncated_objects() { + let console = lines( + "{\n \"schema\": 1,\n \"event\": \"bo\n{\n \"schema\": 1,\n \"event\": \"net_up\",\n \"role\": \"ap\",\n \"ip\": \"192.168.4.1\"\n}", + ); + assert!(matches!( + events(&console).unwrap().as_slice(), + [Event::NetUp(_)] + )); + } + + #[test] + fn rejects_unknown_schema() { + let console = lines("{\n \"schema\": 999,\n \"event\": \"boot\"\n}"); + assert!(events(&console).is_err()); + } +} diff --git a/xtask/src/cmd/test/mod.rs b/xtask/src/cmd/test/mod.rs new file mode 100644 index 0000000..8e8ab5c --- /dev/null +++ b/xtask/src/cmd/test/mod.rs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2026 Roman Valls Guimera +// +// SPDX-License-Identifier: GPL-3.0-or-later + +//! `xtask test` +//! +//! Test suites that need more than `cargo test`, such as real hardware. + +use anyhow::Result; +use clap::{Args as ClapArgs, Subcommand}; + +pub mod e2e; + +#[derive(ClapArgs)] +pub struct Args { + #[command(subcommand)] + pub command: Command, +} + +#[derive(Subcommand)] +pub enum Command { + /// Run end-to-end integration tests on hardware. + E2e(e2e::Args), +} + +pub fn run(args: &Args) -> Result<()> { + match &args.command { + Command::E2e(args) => e2e::run(args), + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index b80ddfb..ab0e4bc 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -53,6 +53,8 @@ enum Command { Size(cmd::size::Args), /// Resets the storage on board, while keeping the firmware intact. Reset(cmd::reset::Args), + /// Run test suites that need more than `cargo test`, e.g. on hardware. + Test(cmd::test::Args), /// ` [args...]` forwarded to cargo. #[command(external_subcommand)] Cargo(Vec), @@ -65,6 +67,7 @@ fn main() -> Result<()> { Command::Bmf(args) => cmd::bmf::run(&args), Command::Size(args) => cmd::size::run(&args), Command::Reset(args) => cmd::reset::run(&args), + Command::Test(args) => cmd::test::run(&args), Command::Cargo(argv) => cmd::cargo::run(&argv), } } @@ -101,6 +104,21 @@ mod tests { .map_err(Box::new) } + fn e2e_cmd(argv: &[&str]) -> Result> { + Cli::try_parse_from( + ["xtask", "test", "e2e"] + .into_iter() + .chain(argv.iter().copied()), + ) + .map(|cli| match cli.command { + Command::Test(cmd::test::Args { + command: cmd::test::Command::E2e(parsed), + }) => parsed, + _ => unreachable!(), + }) + .map_err(Box::new) + } + #[test] fn cli_is_consistent() { Cli::command().debug_assert(); @@ -217,4 +235,31 @@ mod tests { .is_err() ); } + + #[test] + fn e2e() { + assert!(e2e_cmd(&["--board", "esp32c6-devkitc"]).is_ok()); + assert!( + e2e_cmd(&[ + "--board", + "esp32c6-devkitc", + "--port", + "/dev/ttyUSB0", + "--interface", + "wlan0", + "--scenario", + "wifi-ap", + "--verbose" + ]) + .is_ok() + ); + assert!(e2e_cmd(&["--board", "esp32-fake-name"]).is_err()); + assert!(e2e_cmd(&[]).is_err()); + assert!(e2e_cmd(&["--board", "esp32c6-devkitc", "--scenario", "nope"]).is_err()); + assert!(e2e_cmd(&["--board", "esp32c6-devkitc", "--host", "1.2.3.4"]).is_err()); + assert!( + Cli::try_parse_from(["xtask", "e2e", "--board", "esp32c6-devkitc"]) + .is_ok_and(|cli| matches!(cli.command, Command::Cargo(_))) + ); + } }