From 88f6db7b04d657e8efe873cfe2826f7431dec635 Mon Sep 17 00:00:00 2001 From: dragon Date: Tue, 13 Oct 2026 17:43:24 +0800 Subject: [PATCH 1/4] firmware: take a firmware image over any link, resumably (ADR-0016) xfer.begin/chunk/end/cancel over the VibeBuddy Protocol, so the same transfer runs over USB, the local network and the Relay (inside the Noise session off the cable). The box writes the OTA slot it isn't running from, erasing one sector ahead of the writes, acknowledges each chunk of up to 3072 bytes with the next offset it expects, and keeps the progress in memory: an xfer.begin for the same image carries on from there, over any way of carrying the link. On xfer.end it reads the slot back and checks the SHA-256 before it writes otadata, then restarts; the new image boots once and confirms itself, or the bootloader goes back (ADR-0013). ota.rs learns which slot is running, how to select the other one (in the sector that keeps the running entry, so there is something to go back to), and whether the last update was rolled back. The box answers hello with TRANSFER 1, and FIRMWARE ROLLED BACK after a rollback. Lines from a computer may now be 4608 bytes, so a chunk fits one; a sealed line on the local network 8192. Refs #138 Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + firmware-rs/core/Cargo.toml | 2 + firmware-rs/core/src/firmware.rs | 95 ++++++++- firmware-rs/core/src/lan.rs | 4 +- firmware-rs/core/src/lib.rs | 1 + firmware-rs/core/src/ota.rs | 94 +++++++++ firmware-rs/core/src/transfer.rs | 305 +++++++++++++++++++++++++++++ firmware-rs/core/tests/firmware.rs | 80 +++++++- firmware-rs/device/Cargo.lock | 1 + 9 files changed, 578 insertions(+), 5 deletions(-) create mode 100644 firmware-rs/core/src/transfer.rs diff --git a/Cargo.lock b/Cargo.lock index 9f10153..0eb4c03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5137,6 +5137,7 @@ dependencies = [ "libm", "rand_core 0.6.4", "serde_json", + "sha2", "tungstenite", "vibebuddy-protocol", ] diff --git a/firmware-rs/core/Cargo.toml b/firmware-rs/core/Cargo.toml index 60da2a1..4b289a1 100644 --- a/firmware-rs/core/Cargo.toml +++ b/firmware-rs/core/Cargo.toml @@ -19,6 +19,8 @@ libm = "0.2" # Noise sessions on links off the cable (noise.rs, ADR-0015): no std and no getrandom; the device supplies the randomness. clatter = { version = "2.3", default-features = false, features = ["use-25519", "use-chacha20poly1305", "use-sha"] } rand_core = { version = "0.6", default-features = false } +# Checking a firmware image written over the air (transfer.rs, ADR-0016). +sha2 = { version = "0.10", default-features = false } serde_json = { version = "1", default-features = false, features = ["alloc"] } vibebuddy-protocol = { path = "../../protocol", default-features = false } diff --git a/firmware-rs/core/src/firmware.rs b/firmware-rs/core/src/firmware.rs index 2d1be03..cdda9a5 100644 --- a/firmware-rs/core/src/firmware.rs +++ b/firmware-rs/core/src/firmware.rs @@ -27,8 +27,11 @@ use crate::noise::Sealer; use crate::link::{self, Link, LinkStore, PairError, Wifi}; use crate::look::Look; use crate::voices::{Pick, Voices, VoiceError}; +use crate::transfer::{self, Transfer}; +use crate::ota; -pub const MAX_LINE_BYTES: usize = 1024; +/// Longest line taken from a computer: a transfer chunk of 3072 bytes is about 4.2 KB as a line (ADR-0016). +pub const MAX_LINE_BYTES: usize = 4608; const LINE_BUFFER_BYTES: usize = MAX_LINE_BYTES + 2; /// With no message for longer than this, the link to the Mac counts as lost. const LINK_TIMEOUT_MS: u32 = 15000; @@ -171,6 +174,8 @@ pub struct Firmware { menu: Menu, /// voice.begin arrived and we are waiting for playback to stop before erasing the partition: total bytes and when the wait began. pending_voice_begin: Option<(u32, u32)>, + /// A firmware image coming in over the link (ADR-0016). + transfer: Transfer, /// The last task end that was spoken: whether it was a failure, and when. last_end_spoken: Option<(bool, u32)>, @@ -225,6 +230,20 @@ fn number(fields: &impl Fields, key: &str) -> Option { fields.field(key).and_then(Value::as_f64) } +/// Exactly `N` bytes from lowercase or uppercase hex. +fn decode_hex(text: &str) -> Option<[u8; N]> { + let digits = text.as_bytes(); + if digits.len() != N * 2 { + return None; + } + let mut out = [0u8; N]; + for (index, pair) in digits.chunks(2).enumerate() { + let value = |digit: u8| (digit as char).to_digit(16); + out[index] = (value(pair[0])? * 16 + value(pair[1])?) as u8; + } + Some(out) +} + fn string<'a>(fields: &'a impl Fields, key: &str) -> Option<&'a str> { fields.field(key).and_then(Value::as_str) } @@ -285,6 +304,7 @@ impl Firmware { muted: false, menu: Menu::new(), pending_voice_begin: None, + transfer: Transfer::new(), last_end_spoken: None, reported_tier: Tier::Alert, reported_lights_out: false, @@ -346,6 +366,7 @@ impl Firmware { Self::write_literal(board, "DISPLAY ERROR\n"); } Self::write_value_line(board, "FIRMWARE VERSION ", self.version.as_bytes()); + Self::announce_updates(board); self.announce_key(board); let voices_region = find_partition(board.flash(), "voices"); @@ -1300,6 +1321,7 @@ impl Firmware { } } _ if event.starts_with("voice.") => self.handle_voice_event(board, fields, event), + _ if event.starts_with("xfer.") => self.handle_transfer(board, fields, event), // Pairing is the computer at the other end of the cable telling the box who it is (ADR-0012). // Only over the cable: whoever is at the box decides who may link and which network it joins. "device.pair" | "device.unpair" | "device.pairs" | "device.wifi.set" | "device.wifi.forget" | "device.wifi.scan" | "device.relay.set" @@ -1493,10 +1515,81 @@ impl Firmware { board.restart(); } + /// `TRANSFER 1`: this firmware takes firmware images over any link (ADR-0016). `FIRMWARE ROLLED BACK`: the last + /// image sent that way didn't confirm itself, and the bootloader went back to this one; it stays until the next. + fn announce_updates(board: &mut B) { + let Some(otadata) = find_partition(board.flash(), "otadata") else { return }; + Self::write_literal(board, "TRANSFER 1\n"); + if ota::rolled_back(board.flash(), otadata).unwrap_or(false) { + Self::write_literal(board, "FIRMWARE ROLLED BACK\n"); + } + } + + /// `xfer.ready` or `xfer.ack` with the offset to send from, `xfer.done`, or `xfer.error` with its code; JSON lines, + /// since the computer's window is driven by them. + fn transfer_reply(board: &mut B, event: &str, offset: Option) { + let line = match offset { + Some(offset) => text!(80, "{{\"version\":1,\"event\":\"{}\",\"offset\":{}}}\n", event, offset), + None => text!(80, "{{\"version\":1,\"event\":\"{}\"}}\n", event), + }; + board.write(line.as_bytes()); + } + + fn transfer_error(board: &mut B, failure: transfer::Error) { + let line = text!(80, "{{\"version\":1,\"event\":\"xfer.error\",\"code\":\"{}\"}}\n", failure.code()); + board.write(line.as_bytes()); + } + + /// `xfer.*`: a firmware image over any link, in chunks, resumable (ADR-0016, [`crate::transfer`]). + fn handle_transfer(&mut self, board: &mut B, fields: &BTreeMap, event: &str) { + match event { + "xfer.begin" => { + let kind = string(fields, "kind").unwrap_or(""); + let sha256 = string(fields, "sha256").and_then(decode_hex::<32>); + let (Some(size), Some(sha256)) = (number(fields, "size"), sha256) else { + return Self::transfer_error(board, transfer::Error::BadMessage); + }; + match self.transfer.begin(board.flash(), kind, size as u32, sha256) { + Ok(offset) => Self::transfer_reply(board, "xfer.ready", Some(offset)), + Err(failure) => Self::transfer_error(board, failure), + } + } + "xfer.chunk" => { + let (Some(offset), Some(crc), Some(data)) = (number(fields, "offset"), number(fields, "crc"), string(fields, "data")) else { + self.transfer.cancel(); + return Self::transfer_error(board, transfer::Error::BadMessage); + }; + let mut bytes = alloc::vec![0u8; transfer::CHUNK_BYTES + 3]; + let Some(length) = crate::voice_pack::decode_base64(data.as_bytes(), &mut bytes) else { + self.transfer.cancel(); + return Self::transfer_error(board, transfer::Error::BadMessage); + }; + match self.transfer.chunk(board.flash(), offset as u32, &bytes[..length], crc as u32) { + Ok(next) => Self::transfer_reply(board, "xfer.ack", Some(next)), + Err(failure) => Self::transfer_error(board, failure), + } + } + "xfer.end" => match self.transfer.end(board.flash()) { + Ok(()) => { + Self::transfer_reply(board, "xfer.done", None); + // The new image boots once and confirms itself, or the bootloader goes back (ADR-0013). + board.restart(); + } + Err(failure) => Self::transfer_error(board, failure), + }, + "xfer.cancel" => { + self.transfer.cancel(); + Self::transfer_error(board, transfer::Error::Cancelled); + } + _ => Self::transfer_error(board, transfer::Error::BadMessage), + } + } + /// Reports the device's whole static state: firmware build and version, its key, mode, voice and volume. fn announce_state(&self, board: &mut B) { Self::write_value_line(board, "DISPLAY READY BUILD ", &self.build); Self::write_value_line(board, "FIRMWARE VERSION ", self.version.as_bytes()); + Self::announce_updates(board); self.announce_key(board); Self::write_value_line(board, "MODE ", self.display.mode().name().as_bytes()); Self::write_value_line(board, "VOICES ", self.voices.current_id().as_bytes()); diff --git a/firmware-rs/core/src/lan.rs b/firmware-rs/core/src/lan.rs index 3459e32..f402f61 100644 --- a/firmware-rs/core/src/lan.rs +++ b/firmware-rs/core/src/lan.rs @@ -28,8 +28,8 @@ pub const HANDSHAKE_MS: u32 = 10_000; /// How long after the last line over USB the cable still counts as in use; the same silence makes the /// firmware call the link lost. pub const USB_QUIET_MS: u32 = 15_000; -/// Longest line read from a connection: a sealed protocol line of 1024 bytes is 1388 characters. -const MAX_LINE_BYTES: usize = 2048; +/// Longest line read from a connection: a sealed protocol line of 4608 bytes is 6166 characters. +const MAX_LINE_BYTES: usize = 8192; /// A connection, numbered by the device layer. pub type Conn = u32; diff --git a/firmware-rs/core/src/lib.rs b/firmware-rs/core/src/lib.rs index 33acb46..2e0adec 100644 --- a/firmware-rs/core/src/lib.rs +++ b/firmware-rs/core/src/lib.rs @@ -31,6 +31,7 @@ pub mod ota; pub mod pomodoro; pub mod relay_box; pub mod storage; +pub mod transfer; pub mod voice_pack; pub mod voices; diff --git a/firmware-rs/core/src/ota.rs b/firmware-rs/core/src/ota.rs index 60c13c0..436a57b 100644 --- a/firmware-rs/core/src/ota.rs +++ b/firmware-rs/core/src/ota.rs @@ -15,6 +15,8 @@ const ENTRY_BYTES: usize = 32; const STATE_NEW: u32 = 0; const STATE_PENDING_VERIFY: u32 = 1; const STATE_VALID: u32 = 2; +const STATE_INVALID: u32 = 3; +const STATE_ABORTED: u32 = 4; const UNSET: u32 = 0xFFFF_FFFF; /// What confirming did. @@ -35,6 +37,67 @@ pub fn sequence_crc(sequence: u32) -> u32 { crc32(UNSET, &sequence.to_le_bytes()) } +/// The two otadata entries as (sector, sequence, state), for the ones whose CRC checks out. +fn entries(flash: &mut dyn Flash, otadata: Region) -> Result<[Option<(u32, u32)>; 2], FlashError> { + let mut out = [None; 2]; + for sector in 0..2u32 { + let mut entry = [0u8; ENTRY_BYTES]; + flash.read(otadata.offset + sector * SECTOR_BYTES, &mut entry)?; + let sequence = word(&entry, 0); + if sequence != UNSET && word(&entry, 28) == sequence_crc(sequence) { + out[sector as usize] = Some((sequence, word(&entry, 24))); + } + } + Ok(out) +} + +/// The entry the running image came from: the newest one the bootloader didn't give up on. None after a USB +/// flash, which leaves otadata blank and boots `ota_0`. +fn running_entry(entries: &[Option<(u32, u32)>; 2]) -> Option<(u32, u32)> { + (0..2u32) + .filter_map(|sector| entries[sector as usize].map(|(sequence, state)| (sector, sequence, state))) + .filter(|&(_, _, state)| !matches!(state, STATE_INVALID | STATE_ABORTED)) + .max_by_key(|&(_, sequence, _)| sequence) + .map(|(sector, sequence, _)| (sector, sequence)) +} + +/// ESP-IDF's rule: sequence n selects slot (n - 1) % 2. +fn slot_of(sequence: u32) -> usize { + (sequence.wrapping_sub(1) % 2) as usize +} + +/// Which slot, 0 or 1, the box is running from. +pub fn running_slot(flash: &mut dyn Flash, otadata: Region) -> Result { + Ok(running_entry(&entries(flash, otadata)?).map_or(0, |(_, sequence)| slot_of(sequence))) +} + +/// Makes `slot` boot next, as NEW: the bootloader runs it once as PENDING_VERIFY and goes back unless it +/// confirms itself. The entry goes in the sector the running image's entry isn't in, so that one is still there +/// to go back to; its sequence is the next above every entry that selects `slot`. +pub fn select(flash: &mut dyn Flash, otadata: Region, slot: usize) -> Result<(), FlashError> { + let entries = entries(flash, otadata)?; + let newest = entries.iter().flatten().map(|&(sequence, _)| sequence).max().unwrap_or(0); + let mut sequence = newest + 1; + if slot_of(sequence) != slot { + sequence += 1; + } + let sector = running_entry(&entries).map_or(0, |(sector, _)| 1 - sector); + let mut entry = [0xFFu8; ENTRY_BYTES]; + entry[0..4].copy_from_slice(&sequence.to_le_bytes()); + entry[24..28].copy_from_slice(&STATE_NEW.to_le_bytes()); + entry[28..32].copy_from_slice(&sequence_crc(sequence).to_le_bytes()); + let offset = otadata.offset + sector * SECTOR_BYTES; + flash.erase(offset, offset + SECTOR_BYTES)?; + flash.write(offset, &entry) +} + +/// The newest entry was given up on: the last update didn't confirm itself and the box went back. +pub fn rolled_back(flash: &mut dyn Flash, otadata: Region) -> Result { + let entries = entries(flash, otadata)?; + let newest = entries.iter().flatten().max_by_key(|&&(sequence, _)| sequence); + Ok(newest.is_some_and(|&(_, state)| matches!(state, STATE_INVALID | STATE_ABORTED))) +} + /// Marks the selected image VALID if the bootloader left it NEW or PENDING_VERIFY. pub fn confirm(flash: &mut dyn Flash, otadata: Region) -> Result { let mut current: Option<(u32, u32)> = None; @@ -122,6 +185,37 @@ mod tests { assert_eq!(confirm(&mut flash, OTADATA), Ok(Confirmed::Nothing)); } + #[test] + fn a_usb_flash_runs_ota_0_and_an_update_goes_to_ota_1() { + let mut flash = MemoryFlash::new(0x4000); + assert_eq!(running_slot(&mut flash, OTADATA), Ok(0)); + select(&mut flash, OTADATA, 1).unwrap(); + assert_eq!(word(&flash.bytes[0x2000..], 0), 2, "sequence 2 selects ota_1"); + assert_eq!(state(&flash, 0), STATE_NEW); + // The bootloader boots it; it confirms itself. + assert_eq!(confirm(&mut flash, OTADATA), Ok(Confirmed::Now)); + assert_eq!(running_slot(&mut flash, OTADATA), Ok(1)); + + // The next update goes back to ota_0, in the other sector, leaving the running entry alone. + select(&mut flash, OTADATA, 0).unwrap(); + assert_eq!(word(&flash.bytes[0x3000..], 0), 3); + assert_eq!((state(&flash, 0), state(&flash, 1)), (STATE_VALID, STATE_NEW)); + assert_eq!(rolled_back(&mut flash, OTADATA), Ok(false)); + } + + #[test] + fn a_rolled_back_update_is_noticed_and_the_next_one_keeps_the_running_entry() { + let mut flash = MemoryFlash::new(0x4000); + put(&mut flash, 0, &entry(1, STATE_VALID)); + put(&mut flash, 1, &entry(2, STATE_ABORTED)); + assert_eq!(rolled_back(&mut flash, OTADATA), Ok(true)); + assert_eq!(running_slot(&mut flash, OTADATA), Ok(0), "back on ota_0"); + select(&mut flash, OTADATA, 1).unwrap(); + assert_eq!(word(&flash.bytes[0x3000..], 0), 4, "above the aborted 2, and selecting ota_1"); + assert_eq!(state(&flash, 0), STATE_VALID, "the running entry stays to go back to"); + assert_eq!(rolled_back(&mut flash, OTADATA), Ok(false)); + } + #[test] fn a_corrupt_entry_is_ignored() { let mut flash = MemoryFlash::new(0x4000); diff --git a/firmware-rs/core/src/transfer.rs b/firmware-rs/core/src/transfer.rs new file mode 100644 index 0000000..e88c90f --- /dev/null +++ b/firmware-rs/core/src/transfer.rs @@ -0,0 +1,305 @@ +//! Receiving a firmware image over any link, in chunks, resumably (ADR-0016). The computer sends +//! `xfer.begin` with the image's size and SHA-256, then `xfer.chunk`s of at most [`CHUNK_BYTES`] with up to +//! four unacknowledged, then `xfer.end`. The box writes the OTA slot it isn't running from, erasing one sector +//! ahead of the writes, and acknowledges each chunk with the next offset it expects; a chunk at any other offset +//! is acknowledged with that offset again, since every way of carrying a link delivers in order and a chunk only +//! goes missing when the link drops. +//! +//! What has been written is kept in memory, so an `xfer.begin` for the same image, over any way of carrying the +//! link, carries on from there. On `xfer.end` the box reads the slot back and checks the length and SHA-256 before +//! it touches otadata, so a transfer that fails anywhere leaves the box running what it ran. + +use sha2::{Digest, Sha256}; + +use crate::ota; +use crate::storage::{Flash, Region, SECTOR_BYTES, find_partition}; +use crate::voice_pack::crc32; + +/// Most image bytes in one chunk: as base64 inside its line, about 4.2 KB. +pub const CHUNK_BYTES: usize = 3072; + +/// Why a transfer ended; the code goes to the computer as `xfer.error`'s `code`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Error { + /// A kind this firmware can't take, or a box without the OTA layout. + Unsupported, + TooBig, + Crc, + Sha256, + /// Not an ESP32-S3 app image. + BadImage, + Flash, + /// A chunk or an end with no transfer, or an end before the last chunk. + Sequence, + Cancelled, + /// A message without the fields it needs. + BadMessage, +} + +impl Error { + pub fn code(self) -> &'static str { + match self { + Self::Unsupported => "unsupported", + Self::TooBig => "too_big", + Self::Crc => "crc", + Self::Sha256 => "sha256", + Self::BadImage => "bad_image", + Self::Flash => "flash", + Self::Sequence => "sequence", + Self::Cancelled => "cancelled", + Self::BadMessage => "bad_message", + } + } +} + +struct Image { + size: u32, + sha256: [u8; 32], + slot: usize, + target: Region, + /// Bytes of the image written so far. + next: u32, + /// Flash erased up to here, absolute. + erased_until: u32, +} + +/// The transfer under way, if any. +#[derive(Default)] +pub struct Transfer { + image: Option, +} + +/// An ESP32-S3 app image starts with the magic 0xE9, and its extended header names the chip, 9 for the S3. +fn is_esp32s3_image(first: &[u8]) -> bool { + first.len() >= 14 && first[0] == 0xE9 && first[12..14] == [9, 0] +} + +impl Transfer { + pub fn new() -> Self { + Self::default() + } + + pub fn active(&self) -> bool { + self.image.is_some() + } + + /// How far along, 0 to 100. + pub fn percent(&self) -> Option { + self.image.as_ref().map(|image| (u64::from(image.next) * 100 / u64::from(image.size.max(1))) as u8) + } + + /// Starts a transfer, or resumes the same one; returns the offset to send from. + pub fn begin(&mut self, flash: &mut dyn Flash, kind: &str, size: u32, sha256: [u8; 32]) -> Result { + if let Some(image) = &self.image + && image.size == size + && image.sha256 == sha256 + && kind == "firmware" + { + return Ok(image.next); + } + self.image = None; + if kind != "firmware" { + return Err(Error::Unsupported); + } + let (Some(otadata), Some(ota_0), Some(ota_1)) = + (find_partition(flash, "otadata"), find_partition(flash, "ota_0"), find_partition(flash, "ota_1")) + else { + return Err(Error::Unsupported); + }; + let slot = 1 - ota::running_slot(flash, otadata).map_err(|_| Error::Flash)?; + let target = if slot == 0 { ota_0 } else { ota_1 }; + if size == 0 || size > target.size { + return Err(Error::TooBig); + } + self.image = Some(Image { size, sha256, slot, target, next: 0, erased_until: target.offset }); + Ok(0) + } + + /// One chunk at `offset`; returns the offset expected next. An error ends the transfer. + pub fn chunk(&mut self, flash: &mut dyn Flash, offset: u32, data: &[u8], crc: u32) -> Result { + let result = self.write(flash, offset, data, crc); + if result.is_err() { + self.image = None; + } + result + } + + fn write(&mut self, flash: &mut dyn Flash, offset: u32, data: &[u8], crc: u32) -> Result { + let image = self.image.as_mut().ok_or(Error::Sequence)?; + if offset != image.next { + return Ok(image.next); + } + if data.is_empty() || data.len() > CHUNK_BYTES || crc32(0, data) != crc { + return Err(if data.len() > CHUNK_BYTES { Error::TooBig } else { Error::Crc }); + } + if offset == 0 && !is_esp32s3_image(data) { + return Err(Error::BadImage); + } + let end = offset + data.len() as u32; + if end > image.size { + return Err(Error::TooBig); + } + let at = image.target.offset + offset; + // One sector at a time, as the writes reach it: the main loop stalls for each erase. + while image.erased_until < at + data.len() as u32 { + flash.erase(image.erased_until, image.erased_until + SECTOR_BYTES).map_err(|_| Error::Flash)?; + image.erased_until += SECTOR_BYTES; + } + // Flash writes go in whole words; the last chunk is padded with what erased flash holds anyway. + let mut padded = [0xFFu8; CHUNK_BYTES]; + padded[..data.len()].copy_from_slice(data); + flash.write(at, &padded[..data.len().next_multiple_of(4)]).map_err(|_| Error::Flash)?; + image.next = end; + Ok(end) + } + + /// The last chunk is in: checks the slot holds the image, then makes it boot next. The caller restarts. + pub fn end(&mut self, flash: &mut dyn Flash) -> Result<(), Error> { + let image = self.image.take().ok_or(Error::Sequence)?; + if image.next != image.size { + return Err(Error::Sequence); + } + let mut hasher = Sha256::new(); + let mut block = [0u8; SECTOR_BYTES as usize]; + let mut done = 0; + while done < image.size { + let length = (image.size - done).min(SECTOR_BYTES) as usize; + flash.read(image.target.offset + done, &mut block[..length.next_multiple_of(4)]).map_err(|_| Error::Flash)?; + hasher.update(&block[..length]); + done += length as u32; + } + if <[u8; 32]>::from(hasher.finalize()) != image.sha256 { + return Err(Error::Sha256); + } + let otadata = find_partition(flash, "otadata").ok_or(Error::Unsupported)?; + ota::select(flash, otadata, image.slot).map_err(|_| Error::Flash) + } + + pub fn cancel(&mut self) { + self.image = None; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::tests::MemoryFlash; + use alloc::vec::Vec; + + /// A 16 MB flash with this firmware's partition table and blank otadata, as a USB flash leaves it. + fn flash() -> MemoryFlash { + let mut flash = MemoryFlash::new(0xa14000); + let mut table = Vec::new(); + for (label, kind, subtype, offset, size) in [ + ("ota_0", 0u8, 0x10u8, 0x10000u32, 0x400000u32), + ("ota_1", 0, 0x11, 0x610000, 0x400000), + ("otadata", 1, 0x00, 0xa10000, 0x2000), + ] { + let mut entry = [0u8; 32]; + entry[0..2].copy_from_slice(&[0xAA, 0x50]); + entry[2] = kind; + entry[3] = subtype; + entry[4..8].copy_from_slice(&offset.to_le_bytes()); + entry[8..12].copy_from_slice(&size.to_le_bytes()); + entry[12..12 + label.len()].copy_from_slice(label.as_bytes()); + table.extend_from_slice(&entry); + } + flash.bytes[0x8000..0x8000 + table.len()].copy_from_slice(&table); + flash + } + + fn test_image(size: usize) -> Vec { + let mut image: Vec = (0..size).map(|index| (index * 7 + 3) as u8).collect(); + image[0] = 0xE9; + image[12..14].copy_from_slice(&[9, 0]); + image + } + + fn crc(data: &[u8]) -> u32 { + crc32(0, data) + } + + fn sha(data: &[u8]) -> [u8; 32] { + Sha256::digest(data).into() + } + + fn send(transfer: &mut Transfer, flash: &mut MemoryFlash, image: &[u8], from: u32) -> u32 { + let mut next = from; + while (next as usize) < image.len() { + let end = (next as usize + CHUNK_BYTES).min(image.len()); + let piece = &image[next as usize..end]; + next = transfer.chunk(flash, next, piece, crc(piece)).unwrap(); + } + next + } + + #[test] + fn an_image_lands_in_the_other_slot_and_boots_next() { + let mut flash = flash(); + let image = test_image(10_000); + let mut transfer = Transfer::new(); + assert_eq!(transfer.begin(&mut flash, "firmware", image.len() as u32, sha(&image)), Ok(0)); + assert_eq!(send(&mut transfer, &mut flash, &image, 0), 10_000); + assert_eq!(transfer.percent(), Some(100)); + transfer.end(&mut flash).unwrap(); + assert_eq!(&flash.bytes[0x610000..0x610000 + image.len()], &image[..], "ota_1, since a USB flash runs ota_0"); + let otadata = find_partition(&mut flash, "otadata").unwrap(); + assert_eq!(ota::running_slot(&mut flash, otadata), Ok(1), "selected; the bootloader will run it"); + assert!(!transfer.active()); + } + + #[test] + fn the_same_image_resumes_where_it_stopped() { + let mut flash = flash(); + let image = test_image(10_000); + let mut transfer = Transfer::new(); + transfer.begin(&mut flash, "firmware", image.len() as u32, sha(&image)).unwrap(); + let first = &image[..CHUNK_BYTES]; + assert_eq!(transfer.chunk(&mut flash, 0, first, crc(first)), Ok(CHUNK_BYTES as u32)); + // The link dropped and came back, perhaps another way: the same image carries on. + assert_eq!(transfer.begin(&mut flash, "firmware", image.len() as u32, sha(&image)), Ok(CHUNK_BYTES as u32)); + // A chunk sent twice, or one ahead of a lost one, gets the expected offset again. + assert_eq!(transfer.chunk(&mut flash, 0, first, crc(first)), Ok(CHUNK_BYTES as u32)); + let later = &image[2 * CHUNK_BYTES..3 * CHUNK_BYTES]; + assert_eq!(transfer.chunk(&mut flash, 2 * CHUNK_BYTES as u32, later, crc(later)), Ok(CHUNK_BYTES as u32)); + send(&mut transfer, &mut flash, &image, CHUNK_BYTES as u32); + assert_eq!(transfer.end(&mut flash), Ok(())); + + // Another image starts over. + let other = test_image(5_000); + assert_eq!(transfer.begin(&mut flash, "firmware", other.len() as u32, sha(&other)), Ok(0)); + } + + #[test] + fn a_bad_image_a_bad_chunk_or_a_wrong_hash_never_selects_the_slot() { + let mut flash = flash(); + let otadata = find_partition(&mut flash, "otadata").unwrap(); + let mut transfer = Transfer::new(); + + let mut not_ours = test_image(4_000); + not_ours[12] = 2; + transfer.begin(&mut flash, "firmware", 4_000, sha(¬_ours)).unwrap(); + assert_eq!(transfer.chunk(&mut flash, 0, ¬_ours[..CHUNK_BYTES], crc(¬_ours[..CHUNK_BYTES])), Err(Error::BadImage)); + assert!(!transfer.active()); + + let image = test_image(4_000); + transfer.begin(&mut flash, "firmware", 4_000, sha(&image)).unwrap(); + assert_eq!(transfer.chunk(&mut flash, 0, &image[..CHUNK_BYTES], 1), Err(Error::Crc)); + + transfer.begin(&mut flash, "firmware", 4_000, [0; 32]).unwrap(); + send(&mut transfer, &mut flash, &image, 0); + assert_eq!(transfer.end(&mut flash), Err(Error::Sha256)); + assert_eq!(ota::running_slot(&mut flash, otadata), Ok(0)); + assert_eq!(flash.bytes[0xa10000..0xa10004], [0xFF; 4], "otadata untouched"); + } + + #[test] + fn what_the_box_cannot_take_is_refused_at_begin() { + let mut flash = flash(); + let mut transfer = Transfer::new(); + assert_eq!(transfer.begin(&mut flash, "character", 100, [0; 32]), Err(Error::Unsupported)); + assert_eq!(transfer.begin(&mut flash, "firmware", 0x400001, [0; 32]), Err(Error::TooBig)); + assert_eq!(transfer.begin(&mut MemoryFlash::new(0x10000), "firmware", 100, [0; 32]), Err(Error::Unsupported), "no OTA layout"); + assert_eq!(transfer.end(&mut flash), Err(Error::Sequence)); + } +} diff --git a/firmware-rs/core/tests/firmware.rs b/firmware-rs/core/tests/firmware.rs index 8ec76d1..742a76a 100644 --- a/firmware-rs/core/tests/firmware.rs +++ b/firmware-rs/core/tests/firmware.rs @@ -237,6 +237,7 @@ fn boot_reports_like_the_c_firmware() { "TALLY LOADED 0 0S DAY 0", "DISPLAY READY BUILD abc1234 2026-09-26 10:00", "FIRMWARE VERSION 1.2.3", + "TRANSFER 1", "LINK NOISE", BOX_KEY, RELAY_URL, @@ -349,7 +350,7 @@ fn bad_lines_are_reported() { assert_eq!(send(&mut firmware, &mut board, r#"{"version":1,"event":""}"#), ["ERROR invalid_message"]); assert_eq!(send(&mut firmware, &mut board, r#"{"version":1,"event":"x","title":3}"#), ["ERROR invalid_message"]); assert_eq!(send(&mut firmware, &mut board, r#"{"version":2,"event":"x"}"#), ["ERROR unsupported_version"]); - let long = format!(r#"{{"version":1,"event":"x","title":"{}"}}"#, "a".repeat(1100)); + let long = format!(r#"{{"version":1,"event":"x","title":"{}"}}"#, "a".repeat(4700)); assert_eq!(send(&mut firmware, &mut board, &long), ["ERROR input_too_large"]); // A line ending in CRLF is accepted too. firmware.receive(&mut board, b"{\"version\":1,\"event\":\"device.identify\"}\r\n"); @@ -490,7 +491,7 @@ fn echo_reports_length_and_crc() { fn hello_repeats_the_static_state() { let (mut firmware, mut board, _) = booted(blank_flash()); let lines = send(&mut firmware, &mut board, r#"{"version":1,"event":"device.hello"}"#); - assert_eq!(lines, ["DISPLAY READY BUILD abc1234 2026-09-26 10:00", "FIRMWARE VERSION 1.2.3", "LINK NOISE", BOX_KEY, RELAY_URL, "MODE DUTY", "VOICES builtin", "VOLUME 65"]); + assert_eq!(lines, ["DISPLAY READY BUILD abc1234 2026-09-26 10:00", "FIRMWARE VERSION 1.2.3", "TRANSFER 1", "LINK NOISE", BOX_KEY, RELAY_URL, "MODE DUTY", "VOICES builtin", "VOLUME 65"]); } #[test] @@ -1241,3 +1242,78 @@ mod relay { assert!(!relay.has_peer()); } } + +mod transfer { + use super::*; + use serde_json::Value; + use sha2::{Digest, Sha256}; + use vibebuddy_firmware_core::transfer::CHUNK_BYTES; + use vibebuddy_firmware_core::voice_pack::crc32; + + fn base64(bytes: &[u8]) -> String { + const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::new(); + for chunk in bytes.chunks(3) { + let bits = chunk.iter().enumerate().fold(0u32, |acc, (index, &byte)| acc | u32::from(byte) << (16 - 8 * index)); + for index in 0..4 { + out.push(if index <= chunk.len() { ALPHABET[(bits >> (18 - 6 * index) & 63) as usize] as char } else { '=' }); + } + } + out + } + + fn reply(lines: &[String]) -> Value { + serde_json::from_str(lines.last().unwrap()).unwrap() + } + + #[test] + fn a_firmware_image_comes_over_the_link_and_the_box_restarts_into_it() { + let (mut firmware, mut board, _) = booted(blank_flash()); + let mut image: Vec = (0..10_000).map(|index| (index * 13 + 1) as u8).collect(); + image[0] = 0xE9; + image[12..14].copy_from_slice(&[9, 0]); + let sha: String = Sha256::digest(&image).iter().map(|byte| format!("{byte:02x}")).collect(); + let begin = format!(r#"{{"version":1,"event":"xfer.begin","kind":"firmware","size":{},"sha256":"{sha}"}}"#, image.len()); + assert_eq!(reply(&send(&mut firmware, &mut board, &begin)), serde_json::json!({"version": 1, "event": "xfer.ready", "offset": 0})); + for (index, piece) in image.chunks(CHUNK_BYTES).enumerate() { + let line = format!( + r#"{{"version":1,"event":"xfer.chunk","offset":{},"crc":{},"data":"{}"}}"#, + index * CHUNK_BYTES, + crc32(0, piece), + base64(piece) + ); + assert!(line.len() < 4608 || piece.len() < CHUNK_BYTES, "a full chunk fits a line"); + let ack = reply(&send(&mut firmware, &mut board, &line)); + assert_eq!(ack["offset"], (index * CHUNK_BYTES + piece.len()) as u64); + } + // Asked again, the same image resumes at its end. + assert_eq!(reply(&send(&mut firmware, &mut board, &begin))["offset"], image.len() as u64); + assert_eq!(reply(&send(&mut firmware, &mut board, r#"{"version":1,"event":"xfer.end"}"#))["event"], "xfer.done"); + assert_eq!(board.restarts, 1); + assert_eq!(&board.flash.bytes[0x610000..0x610000 + image.len()], &image[..]); + } + + #[test] + fn a_chunk_of_a_full_size_survives_the_line_limit() { + let (mut firmware, mut board, _) = booted(blank_flash()); + let mut image = vec![0xA5u8; CHUNK_BYTES * 2]; + image[0] = 0xE9; + image[12..14].copy_from_slice(&[9, 0]); + let sha: String = Sha256::digest(&image).iter().map(|byte| format!("{byte:02x}")).collect(); + send(&mut firmware, &mut board, &format!(r#"{{"version":1,"event":"xfer.begin","kind":"firmware","size":{},"sha256":"{sha}"}}"#, image.len())); + let piece = &image[..CHUNK_BYTES]; + let line = format!(r#"{{"version":1,"event":"xfer.chunk","offset":0,"crc":{},"data":"{}"}}"#, crc32(0, piece), base64(piece)); + assert_eq!(reply(&send(&mut firmware, &mut board, &line))["offset"], CHUNK_BYTES as u64); + } + + #[test] + fn a_bad_chunk_ends_the_transfer_with_its_reason() { + let (mut firmware, mut board, _) = booted(blank_flash()); + let begin = format!(r#"{{"version":1,"event":"xfer.begin","kind":"firmware","size":100,"sha256":"{}"}}"#, "00".repeat(32)); + send(&mut firmware, &mut board, &begin); + let line = format!(r#"{{"version":1,"event":"xfer.chunk","offset":0,"crc":1,"data":"{}"}}"#, base64(&[0xE9; 100])); + assert_eq!(reply(&send(&mut firmware, &mut board, &line)), serde_json::json!({"version": 1, "event": "xfer.error", "code": "crc"})); + assert_eq!(reply(&send(&mut firmware, &mut board, r#"{"version":1,"event":"xfer.end"}"#))["code"], "sequence"); + assert_eq!(board.restarts, 0); + } +} diff --git a/firmware-rs/device/Cargo.lock b/firmware-rs/device/Cargo.lock index bdff855..37107fb 100644 --- a/firmware-rs/device/Cargo.lock +++ b/firmware-rs/device/Cargo.lock @@ -3321,6 +3321,7 @@ dependencies = [ "libm", "rand_core 0.6.4", "serde_json", + "sha2 0.10.9", "vibebuddy-protocol", ] From 660bbc9cf0c739cba4d904ec6271cc185a866a19 Mon Sep 17 00:00:00 2001 From: dragon Date: Tue, 13 Oct 2026 17:51:16 +0800 Subject: [PATCH 2/4] Update firmware over any link, resumably (ADR-0016) The daemon sends the app image as xfer.chunk lines of 3072 bytes, four ahead of the box's acknowledgements, to any box that reports TRANSFER 1, on the cable as well as over the local network and the Relay. When the link drops or moves it waits up to three minutes for it to come back and asks again with xfer.begin; the box answers with where it got to. After xfer.done it waits for the box to report its build again, and reports a failure if the box says FIRMWARE ROLLED BACK. Older firmware keeps the ROM flasher, and still needs the cable. The protocol's line limit goes from 1024 to 4608 bytes so a chunk fits one line; only chunks come near it, and only firmware that reports TRANSFER 1 gets them. /v1/status reports device.transfer and device.rolled_back. The Mac and Linux apps keep their Update and Flash from file... buttons as they are; after a failed update they show the hold-K0-and-replug advice only for firmware that updates through the ROM. Docs: Firmware over the link in docs/protocol.md, the update flow in docs/app.md; resuming large writes is no longer undecided in docs/relay-protocol.md. Refs #138 Co-Authored-By: Claude Opus 5.5 --- app/Sources/VibeBuddy/SettingsView.swift | 5 +- app/Sources/VibeBuddyCore/Status.swift | 4 +- daemon/src/main.rs | 72 ++++++ daemon/src/status.rs | 17 ++ daemon/src/transfer.rs | 276 +++++++++++++++++++++++ daemon/src/voice_writer.rs | 2 +- desktop/src/main.rs | 15 +- desktop/src/status.rs | 2 + docs/app.md | 2 +- docs/protocol.md | 34 ++- docs/relay-protocol.md | 2 +- protocol/src/lib.rs | 4 +- 12 files changed, 419 insertions(+), 16 deletions(-) create mode 100644 daemon/src/transfer.rs diff --git a/app/Sources/VibeBuddy/SettingsView.swift b/app/Sources/VibeBuddy/SettingsView.swift index d7a22df..3578c02 100644 --- a/app/Sources/VibeBuddy/SettingsView.swift +++ b/app/Sources/VibeBuddy/SettingsView.swift @@ -575,7 +575,10 @@ struct DeviceView: View { if let operation = model.operation, operation.kind == .firmware { OperationRow(operation: operation) if operation.state == .failed { - Text("Before retrying, hold K0 on the box and replug the cable to put it in download mode.").font(.caption).foregroundStyle(.secondary) + // Firmware that updates over the link carries on from where it got to; download mode is for the rest. + if model.status?.device.transfer != true { + Text("Before retrying, hold K0 on the box and replug the cable to put it in download mode.").font(.caption).foregroundStyle(.secondary) + } Button("Retry") { model.updateFirmware() }.disabled(!connected) } } diff --git a/app/Sources/VibeBuddyCore/Status.swift b/app/Sources/VibeBuddyCore/Status.swift index 67e1144..db206cd 100644 --- a/app/Sources/VibeBuddyCore/Status.swift +++ b/app/Sources/VibeBuddyCore/Status.swift @@ -109,7 +109,9 @@ public struct DeviceState: Codable, Equatable { public var wifiError: String? /// What the box heard on its last scan, strongest first. Absent from older daemons. public var wifiSeen: [SeenNetwork]? - enum CodingKeys: String, CodingKey { case wifiNetwork = "wifi_network", wifiAddress = "wifi_address", wifiError = "wifi_error", wifiSeen = "wifi_seen", relay, busy, connected, port, bridge, network, mode, firmwareBuild = "firmware_build", firmwareVersion = "firmware_version", voice, volume, foreignFirmware = "foreign_firmware", pin, candidates, unsupportedBoard = "unsupported_board", boxKey = "box_key", computerKey = "computer_key", pairedComputers = "paired_computers" } + /// The firmware takes updates over any link (ADR-0016): no cable, no download mode. Absent from older daemons. + public var transfer: Bool? + enum CodingKeys: String, CodingKey { case transfer, wifiNetwork = "wifi_network", wifiAddress = "wifi_address", wifiError = "wifi_error", wifiSeen = "wifi_seen", relay, busy, connected, port, bridge, network, mode, firmwareBuild = "firmware_build", firmwareVersion = "firmware_version", voice, volume, foreignFirmware = "foreign_firmware", pin, candidates, unsupportedBoard = "unsupported_board", boxKey = "box_key", computerKey = "computer_key", pairedComputers = "paired_computers" } public init(connected: Bool, port: String? = nil, bridge: Bool = false, mode: String? = nil, firmwareBuild: String? = nil, firmwareVersion: String? = nil, voice: String? = nil, volume: Int? = nil, foreignFirmware: Bool? = nil) { self.connected = connected; self.port = port; self.bridge = bridge; self.mode = mode; self.firmwareBuild = firmwareBuild; self.firmwareVersion = firmwareVersion; self.voice = voice; self.volume = volume self.foreignFirmware = foreignFirmware diff --git a/daemon/src/main.rs b/daemon/src/main.rs index 2d6baf7..007cd3e 100644 --- a/daemon/src/main.rs +++ b/daemon/src/main.rs @@ -16,6 +16,7 @@ mod serial_transport; mod session_titles; mod source_opener; mod status; +mod transfer; mod updates; mod voice_writer; @@ -667,6 +668,20 @@ fn report_progress(state: &AppState, fraction: f32, message: Option) { }); } +/// Changes only what a running operation says, as when an update waits for the link. +fn report_message(state: &AppState, message: &str) { + let state = state.clone(); + let message = message.to_owned(); + tokio::spawn(async move { + if let Some(operation) = state.operation.lock().await.as_mut() + && operation.state == OperationState::Running + { + operation.message = message; + } + state.notify_status(); + }); +} + async fn post_identify(State(state): State) -> (StatusCode, Json) { post_event(State(state), Json(device_command("device.identify"))).await } @@ -932,6 +947,12 @@ async fn post_firmware( let (port, bridge) = { let device = state.device.lock().await; match &device.port { + // Firmware that takes images over the link gets one that way, over whichever it is: it can roll back, + // and it needs no cable (ADR-0016). The bootloader and partition table never change after the OTA layout. + Some(_) if device.connected && device.transfer => { + drop(device); + return update_over_the_link(state, segments).await; + } Some(_) if device.connected && device.network => { return ( StatusCode::CONFLICT, @@ -994,6 +1015,57 @@ async fn post_firmware( /// plugging in) stays there until it loses power, with a dark screen. Only the firmware's own `DISPLAY READY` proves it /// booted; until then the flash isn't done, and if it never comes the user has to replug the box. The replug wait has no /// deadline: a stale `replug` would keep the app's onboarding from ever showing the box as found, however late it boots. +/// How long a box updated over the link may take to come back: restart, Wi-Fi, and the daemon finding it again. +const OTA_BOOT_TIMEOUT: Duration = Duration::from_secs(180); + +/// Sends the app image over the link (ADR-0016), then waits for the box to come back on it. Success is the box +/// reporting its build again without `FIRMWARE ROLLED BACK`. +async fn update_over_the_link(state: AppState, segments: Vec) -> (StatusCode, Json) { + let Some(image) = segments.into_iter().find(|segment| segment.address == 0x10000).map(|segment| segment.data) else { + return (StatusCode::BAD_REQUEST, Json(ApiResponse { accepted: false, message: "no app image".to_owned() })); + }; + if let Some(refused) = begin_operation(&state, OperationKind::Firmware, "sending the firmware to the box".to_owned()).await { + return refused; + } + let task_state = state.clone(); + tokio::spawn(async move { + let bus = task_state.device_bus.subscribe(); + let progress_state = task_state.clone(); + let resuming_state = task_state.clone(); + let result = transfer::send_image( + task_state.transport.clone(), + bus, + image, + move |fraction| report_progress(&progress_state, fraction * 0.95, Some("sending the firmware to the box".to_owned())), + move || report_message(&resuming_state, "waiting for the link to come back"), + ) + .await; + let firmware = |state, progress, message: &str| Some(Operation { kind: OperationKind::Firmware, state, progress, message: message.to_owned() }); + if let Err(error) = result { + warn!(%error, "firmware update over the link failed"); + task_state.set_operation(firmware(OperationState::Failed, 0.0, &error)).await; + return; + } + info!("firmware sent over the link, waiting for the box to restart"); + // Forget the old build, so only what the box reports after its restart counts. + task_state.device.lock().await.firmware_build = None; + task_state.set_operation(firmware(OperationState::Running, 0.95, "waiting for the box to restart")).await; + if !wait_for_firmware_build(&task_state, OTA_BOOT_TIMEOUT).await { + task_state.set_operation(firmware(OperationState::Failed, 0.0, "the box didn't come back after the update")).await; + return; + } + // The box reports a rollback right after its build, in the same answer to hello. + tokio::time::sleep(Duration::from_secs(1)).await; + if task_state.device.lock().await.rolled_back { + warn!("the new firmware didn't confirm itself; the box went back"); + task_state.set_operation(firmware(OperationState::Failed, 0.0, "the update didn't take; the box went back to its previous firmware")).await; + } else { + task_state.set_operation(firmware(OperationState::Done, 1.0, "updated, box restarted")).await; + } + }); + (StatusCode::ACCEPTED, Json(ApiResponse { accepted: true, message: "sending the firmware to the box".to_owned() })) +} + async fn await_first_boot(state: &AppState, boot_timeout: Duration) { let firmware = |state, message: &str| Some(Operation { kind: OperationKind::Firmware, state, progress: 1.0, message: message.to_owned() }); state.set_operation(firmware(OperationState::Running, "waiting for the box to restart")).await; diff --git a/daemon/src/status.rs b/daemon/src/status.rs index 0f47c58..1d7fb7d 100644 --- a/daemon/src/status.rs +++ b/daemon/src/status.rs @@ -36,6 +36,10 @@ pub struct DeviceState { pub firmware_build: Option, /// The firmware version, such as `0.2.2`, which orders releases; firmware older than ADR-0010 reports none. pub firmware_version: Option, + /// The firmware takes firmware images over any link (`TRANSFER 1`, ADR-0016), so updating needs no cable. + pub transfer: bool, + /// The last image sent over the link didn't confirm itself, and the box went back (`FIRMWARE ROLLED BACK`). + pub rolled_back: bool, /// Id of the announcement voice the device is using; `builtin` means the built-in voice. pub voice: Option, /// Speaker volume (the codec's 20 to 100), stored on the device; the app is just a remote. @@ -120,6 +124,8 @@ impl DeviceState { self.foreign_firmware = false; self.firmware_build = None; self.firmware_version = None; + self.transfer = false; + self.rolled_back = false; self.mode = None; self.voice = None; self.volume = None; @@ -152,6 +158,8 @@ impl DeviceState { self.unsupported_board = None; self.firmware_build = None; self.firmware_version = None; + self.transfer = false; + self.rolled_back = false; self.mode = None; self.voice = None; self.volume = None; @@ -183,6 +191,13 @@ impl DeviceState { } else if let Some((_, build)) = line.split_once(BUILD_MARKER) { self.foreign_firmware = false; self.firmware_build = Some(build.trim().to_owned()); + // The box reports these after the build, each time it reports its state. + self.transfer = false; + self.rolled_back = false; + } else if line.trim() == "TRANSFER 1" { + self.transfer = true; + } else if line.trim() == "FIRMWARE ROLLED BACK" { + self.rolled_back = true; } else if let Some(key) = line.strip_prefix(BOX_KEY_PREFIX) { let key = key.trim(); // Another key on the same port: the box made itself a new identity and restarted without the @@ -403,6 +418,8 @@ mod tests { busy: None, mode: Some("pomodoro".to_owned()), firmware_build: Some("21a8360-dirty 2026-09-16 10:23".to_owned()), + transfer: false, + rolled_back: false, firmware_version: Some("0.2.2".to_owned()), voice: Some("builtin".to_owned()), volume: Some(65), diff --git a/daemon/src/transfer.rs b/daemon/src/transfer.rs new file mode 100644 index 0000000..3b1c1c4 --- /dev/null +++ b/daemon/src/transfer.rs @@ -0,0 +1,276 @@ +//! Sending a firmware image to the box over any link, in chunks, resumably (ADR-0016, docs/protocol.md +//! "Firmware over the link"). The image goes as `xfer.chunk` lines of at most [`CHUNK_BYTES`], up to [`WINDOW`] +//! of them unacknowledged; the box acknowledges each with the next offset it expects. When the link drops or +//! moves (the Relay to the cable, say), the daemon waits for it to come back and asks again with `xfer.begin`, +//! and the box answers with where it got to. + +use std::sync::Arc; +use std::time::Duration; + +use base64::Engine; +use sha2::{Digest, Sha256}; +use tokio::sync::broadcast; +use vibebuddy_protocol::Event; + +use crate::serial_transport::{DeviceMessage, Transport}; +use crate::voice_writer::send; + +/// Image bytes per chunk: as base64 inside its line about 4.2 KB, under the box's 4608. +pub const CHUNK_BYTES: usize = 3072; +/// Chunks sent ahead of the acknowledgements: enough to keep the Relay busy at 130-250 ms a round trip, while +/// staying under its 50 messages a second. +const WINDOW: usize = 4; +/// No answer to a chunk for this long means it went missing: ask where the box got to. +const STEP_TIMEOUT: Duration = Duration::from_secs(20); +/// The box reads 4 MB back to check it before it answers `xfer.end`. +const END_TIMEOUT: Duration = Duration::from_secs(60); +/// How long a dropped link may take to come back before the update gives up. +const RECONNECT_TIMEOUT: Duration = Duration::from_secs(180); +/// Times the transfer may start over from where the box got to. +const RESUMES: u32 = 10; + +/// Why sending stopped. +#[derive(Debug, PartialEq, Eq)] +enum Stop { + /// The link dropped or moved, or the box went quiet: resume from where it got to. + Resume, + /// The box refused, with its code; or something that resuming won't fix. + Fail(String), +} + +fn event(name: &str, fields: serde_json::Value) -> Event { + let mut event = Event::named(name); + if let serde_json::Value::Object(map) = fields { + event.extra = map.into_iter().collect(); + } + event +} + +pub fn begin_event(size: usize, sha256: &str) -> Event { + event("xfer.begin", serde_json::json!({"kind": "firmware", "size": size, "sha256": sha256})) +} + +pub fn chunk_event(offset: usize, piece: &[u8]) -> Event { + event( + "xfer.chunk", + serde_json::json!({ + "offset": offset, + "crc": crc32fast::hash(piece), + "data": base64::engine::general_purpose::STANDARD.encode(piece), + }), + ) +} + +/// The box's next answer about the transfer: `xfer.ready`/`xfer.ack` with an offset, `xfer.done`, or a stop. +async fn answer(bus: &mut broadcast::Receiver, timeout: Duration) -> Result<(String, Option), Stop> { + let deadline = tokio::time::Instant::now() + timeout; + loop { + let message = match tokio::time::timeout_at(deadline, bus.recv()).await { + Ok(Ok(message)) => message, + Ok(Err(broadcast::error::RecvError::Lagged(_))) => continue, + Ok(Err(broadcast::error::RecvError::Closed)) => return Err(Stop::Fail("device message channel closed".to_owned())), + Err(_) => return Err(Stop::Resume), + }; + match message { + DeviceMessage::Disconnected | DeviceMessage::Connected { .. } | DeviceMessage::ConnectedNetwork { .. } | DeviceMessage::ConnectedRelay { .. } => { + return Err(Stop::Resume); + } + DeviceMessage::Event(event) if event.event == "xfer.error" => { + let code = event.extra.get("code").and_then(|value| value.as_str()).unwrap_or("error"); + return Err(Stop::Fail(format!("the box refused the image: {code}"))); + } + DeviceMessage::Event(event) if event.event.starts_with("xfer.") => { + let offset = event.extra.get("offset").and_then(serde_json::Value::as_u64).map(|offset| offset as usize); + return Ok((event.event, offset)); + } + _ => {} + } + } +} + +/// Waits for the link to come back after it dropped; true once it has. +async fn reconnected(bus: &mut broadcast::Receiver) -> bool { + let deadline = tokio::time::Instant::now() + RECONNECT_TIMEOUT; + loop { + match tokio::time::timeout_at(deadline, bus.recv()).await { + Ok(Ok(DeviceMessage::Connected { .. } | DeviceMessage::ConnectedNetwork { .. } | DeviceMessage::ConnectedRelay { .. })) => return true, + Ok(Ok(_) | Err(broadcast::error::RecvError::Lagged(_))) => {} + Ok(Err(broadcast::error::RecvError::Closed)) | Err(_) => return false, + } + } +} + +/// Sends the image from wherever the box says it got to, until the box has it all. +async fn send_from_box( + transport: &Arc, + bus: &mut broadcast::Receiver, + image: &[u8], + sha256: &str, + progress: &impl Fn(f32), +) -> Result<(), Stop> { + send(transport, begin_event(image.len(), sha256)).await.map_err(|_| Stop::Resume)?; + let mut acked = loop { + if let (name, Some(offset)) = answer(bus, STEP_TIMEOUT).await? + && name == "xfer.ready" + { + break offset; + } + }; + let mut next = acked; + while acked < image.len() { + while next < image.len() && next < acked + WINDOW * CHUNK_BYTES { + let end = (next + CHUNK_BYTES).min(image.len()); + send(transport, chunk_event(next, &image[next..end])).await.map_err(|_| Stop::Resume)?; + next = end; + } + if let (name, Some(offset)) = answer(bus, STEP_TIMEOUT).await? + && name == "xfer.ack" + { + acked = acked.max(offset); + progress(acked as f32 / image.len() as f32); + } + } + Ok(()) +} + +/// Sends the whole image and has the box check it and make it boot next; the box then restarts. `progress` +/// receives 0 to 1 as the box acknowledges, and `resuming` is told when the transfer waits for the link. +pub async fn send_image( + transport: Arc, + mut bus: broadcast::Receiver, + image: Vec, + progress: impl Fn(f32), + resuming: impl Fn(), +) -> Result<(), String> { + let sha256: String = Sha256::digest(&image).iter().map(|byte| format!("{byte:02x}")).collect(); + let mut resumes = 0; + loop { + match send_from_box(&transport, &mut bus, &image, &sha256, &progress).await { + Ok(()) => { + send(&transport, Event::named("xfer.end")).await?; + match answer(&mut bus, END_TIMEOUT).await { + Ok((name, _)) if name == "xfer.done" => return Ok(()), + Ok((name, _)) => return Err(format!("unexpected {name} at the end")), + // The box keeps what it has: asking again carries on, and ends again. + Err(Stop::Resume) => {} + Err(Stop::Fail(why)) => return Err(why), + } + } + Err(Stop::Fail(why)) => return Err(why), + Err(Stop::Resume) => {} + } + resumes += 1; + if resumes > RESUMES { + return Err("the link kept dropping".to_owned()); + } + resuming(); + // A link that moved is already back; one that dropped gets a while to come back. + let connected = transport_connected(&mut bus).await; + if !connected && !reconnected(&mut bus).await { + return Err("the box didn't come back".to_owned()); + } + } +} + +/// Drains what the bus already holds; true if the latest word about the link is that it is up. +async fn transport_connected(bus: &mut broadcast::Receiver) -> bool { + let mut connected = None; + while let Ok(message) = bus.try_recv() { + match message { + DeviceMessage::Disconnected => connected = Some(false), + DeviceMessage::Connected { .. } | DeviceMessage::ConnectedNetwork { .. } | DeviceMessage::ConnectedRelay { .. } => connected = Some(true), + _ => {} + } + } + connected.unwrap_or(true) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + use crate::serial_transport::TransportError; + + /// Plays the box: takes chunks in order, acknowledges each with the next offset, and on xfer.begin + /// answers where it got to. `drop_after` chunks in, it drops the link once and loses what was in flight. + struct FakeBox { + bus: broadcast::Sender, + state: Mutex<(usize, Vec, Option)>, + } + + impl Transport for FakeBox { + fn send(&self, frame: Vec) -> Result<(), TransportError> { + let value: serde_json::Value = serde_json::from_slice(&frame).unwrap(); + let reply = |name: &str, offset: Option| { + let mut event = Event::named(name); + if let Some(offset) = offset { + event.extra.insert("offset".to_owned(), offset.into()); + } + let _ = self.bus.send(DeviceMessage::Event(event)); + }; + let mut state = self.state.lock().unwrap(); + let (next, written, drop_after) = &mut *state; + match value["event"].as_str().unwrap() { + "xfer.begin" => reply("xfer.ready", Some(*next)), + "xfer.chunk" => { + if *drop_after == Some(0) { + *drop_after = None; + let _ = self.bus.send(DeviceMessage::Disconnected); + let _ = self.bus.send(DeviceMessage::ConnectedRelay { url: "wss://relay".to_owned() }); + return Ok(()); + } + if let Some(count) = drop_after { + *count -= 1; + } + let offset = value["offset"].as_u64().unwrap() as usize; + if offset == *next { + let data = base64::engine::general_purpose::STANDARD.decode(value["data"].as_str().unwrap()).unwrap(); + assert_eq!(value["crc"].as_u64().unwrap() as u32, crc32fast::hash(&data)); + written.extend_from_slice(&data); + *next += data.len(); + } + reply("xfer.ack", Some(*next)); + } + "xfer.end" => reply("xfer.done", None), + other => panic!("unexpected {other}"), + } + Ok(()) + } + } + + async fn run(image: Vec, drop_after: Option) -> (Result<(), String>, Vec, usize) { + let (bus, receiver) = broadcast::channel(1024); + let fake = Arc::new(FakeBox { bus, state: Mutex::new((0, Vec::new(), drop_after)) }); + let resumed = Arc::new(Mutex::new(0)); + let counter = resumed.clone(); + let result = send_image(fake.clone(), receiver, image, |_| {}, move || *counter.lock().unwrap() += 1).await; + let written = std::mem::take(&mut fake.state.lock().unwrap().1); + let resumed = *resumed.lock().unwrap(); + (result, written, resumed) + } + + #[tokio::test] + async fn the_whole_image_arrives_in_order() { + let image: Vec = (0..20_000).map(|index| (index * 31) as u8).collect(); + let (result, written, resumed) = run(image.clone(), None).await; + assert_eq!(result, Ok(())); + assert_eq!(written, image); + assert_eq!(resumed, 0); + } + + #[tokio::test] + async fn a_dropped_link_resumes_where_the_box_got_to() { + let image: Vec = (0..30_000).map(|index| (index * 17) as u8).collect(); + let (result, written, resumed) = run(image.clone(), Some(3)).await; + assert_eq!(result, Ok(())); + assert_eq!(written, image, "nothing written twice, nothing missing"); + assert_eq!(resumed, 1); + } + + #[test] + fn a_full_chunk_fits_the_boxs_line_limit() { + let frame = chunk_event(4_000_000, &vec![0xAB; CHUNK_BYTES]).to_ndjson().unwrap(); + assert!(frame.len() <= 4608, "{} bytes", frame.len()); + } +} diff --git a/daemon/src/voice_writer.rs b/daemon/src/voice_writer.rs index cace9f5..54713a0 100644 --- a/daemon/src/voice_writer.rs +++ b/daemon/src/voice_writer.rs @@ -59,7 +59,7 @@ pub fn end_event() -> Event { event("voice.end", vec![]) } -async fn send(transport: &Arc, event: Event) -> Result<(), String> { +pub async fn send(transport: &Arc, event: Event) -> Result<(), String> { let frame = event.to_ndjson().map_err(|error| error.to_string())?; // A full queue just means the device hasn't caught up yet; wait a bit and push again. for _ in 0..200 { diff --git a/desktop/src/main.rs b/desktop/src/main.rs index b698325..4dfae39 100644 --- a/desktop/src/main.rs +++ b/desktop/src/main.rs @@ -1308,14 +1308,15 @@ impl App { (false, false, _) => None, }; let flashing = operation.filter(|operation| operation.kind == status::OperationKind::Firmware).map(|operation| { + // Firmware that updates over the link carries on from where it got to; download mode is for the rest. + let over_the_link = device.is_some_and(|device| device.transfer); let failed = (operation.state == status::OperationState::Failed).then(|| { - column![ - text(tr("Before retrying, hold K0 on the box and replug the cable to put it in download mode.", &[])) - .size(13), - button(text(tr("Retry", &[]))) - .on_press_maybe((online && downloaded).then_some(Message::FlashFirmware)), - ] - .spacing(8) + column![] + .push((!over_the_link).then(|| { + text(tr("Before retrying, hold K0 on the box and replug the cable to put it in download mode.", &[])).size(13) + })) + .push(button(text(tr("Retry", &[]))).on_press_maybe((online && downloaded).then_some(Message::FlashFirmware))) + .spacing(8) }); column![text(operation.summary()).size(13)].push(failed).spacing(8) }); diff --git a/desktop/src/status.rs b/desktop/src/status.rs index ee745d3..e52a058 100644 --- a/desktop/src/status.rs +++ b/desktop/src/status.rs @@ -154,6 +154,8 @@ pub struct Device { pub candidates: Vec, /// The box's key (ADR-0012); none for firmware that can't pair, or from an older daemon. pub box_key: Option, + /// The firmware takes updates over any link (ADR-0016): no cable, no download mode. + pub transfer: bool, /// This computer's key, to tell it apart in `paired_computers`. pub computer_key: Option, pub paired_computers: Vec, diff --git a/docs/app.md b/docs/app.md index a11324d..6738a8b 100644 --- a/docs/app.md +++ b/docs/app.md @@ -100,7 +100,7 @@ Firmware isn't bundled with the app; it is released on its own (`firmware-vX.Y.Z "Update to x" appears only when the box runs something older, a build without a version (firmware from before ADR-0010) or a `-dirty` build of the same version, and only once the download is on disk; a newer box is never downgraded. Nor is it offered to our firmware built for other hardware: a build that prints `BOARD x` for anything but `alientek-box` (the breadboard devkit says `goouuu-s3-spi`) gets a note instead, since released firmware is built for the box alone and starts neither screen, speaker nor buttons elsewhere. Released firmware prints no `BOARD` line, so the box itself is unaffected. One notification per new firmware version is posted while the box is connected (macOS only for now). When firmware is wanted but not on disk (update checks off, a failed download, or still downloading), the Device tab and onboarding say why and point at the releases page and "Flash from file…". -Flow: confirm → daemon releases the serial port → espflash flashes (automatically no-stub over the bridge) → box restarts → reconnect, with progress visible throughout. On failure it offers a retry and a fallback instruction to "hold K0 and replug the cable". Updates don't erase the `voices` and `nvs` partitions, so changing firmware keeps the voice and today's stats. Never updates automatically. +Flow, for firmware that reports `TRANSFER 1` (ADR-0016): confirm → the daemon sends the app image over the link, whichever it is (USB, Wi-Fi, the Relay), resuming where it got to if the link drops → the box checks it, restarts into it and confirms it → reconnect, with progress visible throughout. If the new firmware doesn't confirm itself, the box goes back to the old one and the update is reported as failed; retrying needs no cable. For older firmware and boxes not running ours: confirm → daemon releases the serial port → the ROM flasher writes (small blocks over the bridge) → box restarts → reconnect; on failure it offers a retry and a fallback instruction to "hold K0 and replug the cable", which the Apps show only for this path. Older firmware off the cable is told to plug in the cable. Updates don't erase the `voices` and `nvs` partitions, so changing firmware keeps the voice and today's stats. Never updates automatically. When the Mac can't see the box at all, onboarding's "Find the box" page and the Device tab give two steps in order. First the everyday causes: a charge-only cable, or not clicking Allow when macOS asks whether to let the accessory connect. Only then K0: holding it while plugging in starts the ROM bootloader instead of the firmware (K0 is the BOOT button on GPIO0), so the box shows up with a dark screen and no build number and the page offers to flash it. K0 is never suggested as a general fix, because a box in download mode does nothing until it is flashed. diff --git a/docs/protocol.md b/docs/protocol.md index cc5b880..a933221 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -7,7 +7,7 @@ VibeBuddy Protocol is the application protocol between `vibebuddyd` and a VibeBu - Each message is one JSON object terminated by a single LF (`\n`). - Receivers also accept CRLF (`\r\n`), but senders always use LF. - Blank lines are ignored. -- A single JSON message is at most 1024 bytes, excluding the line terminator. Anything longer is dropped as a whole line and answered with `ERROR input_too_large`. +- A single JSON message is at most 4608 bytes, excluding the line terminator, so a firmware chunk fits one (see Firmware over the link); firmware that doesn't report `TRANSFER 1` takes at most 1024. Anything longer is dropped as a whole line and answered with `ERROR input_too_large`. - v1 doesn't support JSON spanning multiple lines, JSON arrays, or multiple JSON objects on one line. ## Envelope @@ -123,6 +123,34 @@ The device's acknowledgments: `begin` first waits for any line currently playing to finish, then erases the needed range; `size` is the whole pack's byte count, including the header: 1024 bytes for a Character pack, 256 for a voice pack, told apart by the magic in the first chunk. Chunks must arrive consecutively by `seq` starting from 0, each with at most 672 bytes of raw data, so the whole line after base64 still fits within the 1024-byte limit. `crc` is the CRC32 of the chunk's raw bytes (the same as zlib's); the device checks it right after decoding and, on a mismatch, replies `voice.error` and aborts rather than waiting until the end. The header stays in device memory; on `end` the device reads the flash back to verify the payload CRC, writes the header only if that passes, then switches to the new Character. If any step fails it replies `voice.error` and abandons the session; the firmware then sees the partition as empty and announcements fall back to the built-in voice. Announcements during a write use the built-in voice. +## Firmware over the link + +Firmware that answers `device.hello` with `TRANSFER 1` (just after `FIRMWARE VERSION`) takes a firmware image over whatever carries the link: USB, the local network or the Relay, inside the Noise session off the cable (ADR-0016, `firmware-rs/core/src/transfer.rs` and `daemon/src/transfer.rs`). + +```json +{"version":1,"event":"xfer.begin","kind":"firmware","size":2820896,"sha256":"<64 hex digits>"} +{"version":1,"event":"xfer.chunk","offset":0,"crc":305419896,"data":""} +{"version":1,"event":"xfer.end"} +{"version":1,"event":"xfer.cancel"} +``` + +The box answers with JSON lines: + +```json +{"version":1,"event":"xfer.ready","offset":0} +{"version":1,"event":"xfer.ack","offset":3072} +{"version":1,"event":"xfer.done"} +{"version":1,"event":"xfer.error","code":"sha256"} +``` + +`size` and `sha256` are the app image's (`vibebuddy-fw.bin`; the bootloader and partition table never change once a box has the OTA layout). The box writes the slot it isn't running from, `ota_1` after a USB flash, erasing one 4 KB sector ahead of the writes, and checks that the first chunk starts an ESP32-S3 app image. `xfer.ready` and `xfer.ack` carry the offset the box expects next; the computer keeps at most four chunks unacknowledged. A chunk at any other offset changes nothing and is acknowledged with the expected offset again: every way of carrying the link delivers in order, so a chunk only goes missing when the link drops. `crc` is zlib's CRC32 of the chunk's raw bytes. The box keeps the image's size, hash and how far it got in memory, so an `xfer.begin` for the same image, over any way of carrying the link, answers with that offset and the transfer carries on; another image starts over, and a restart forgets. When the link drops or moves, `vibebuddyd` waits up to 3 minutes for it to come back and asks again. + +On `xfer.end` the box reads the slot back, checks the length and SHA-256, writes the next `otadata` entry selecting that slot in the state NEW (in the sector that doesn't hold the running image's entry, so that one is still there to go back to), answers `xfer.done` and restarts. Nothing touches `otadata` before then, so a transfer that fails anywhere leaves the box running what it ran. The new image boots once and confirms itself early in boot, or the bootloader goes back to the old one (ADR-0013); a box that went back reports `FIRMWARE ROLLED BACK` after `TRANSFER 1` until the next update. `vibebuddyd` counts the update done when the box reports its build again without it. + +`xfer.error` ends the transfer and the box forgets it. Codes: `unsupported` (another `kind`, or no OTA layout), `too_big`, `crc`, `sha256`, `bad_image`, `flash`, `sequence` (a chunk or an end with no transfer, or an end before the last chunk), `cancelled`, `bad_message`. + +`POST /v1/device/firmware` takes this path whenever the box reports `TRANSFER 1`, on the cable as well, and the ROM flasher otherwise. Character packs still go by `voice.*`. + ## Pairing A paired computer may later link to the box over the local network or through the Relay (ADR-0012, [`relay-protocol.md`](relay-protocol.md)). Pairing happens over USB only: the cable is the proof that whoever pairs is at the box. Keys are Ed25519 public keys, 32 bytes as unpadded base64url (43 characters). @@ -185,7 +213,7 @@ A box links to one computer at a time (ADR-0011). A paired computer gets the box On the computer's side, `vibebuddyd` remembers the box's key when it sees `BOX KEY` over USB (`box-key` in its state directory), and the address the box reports over USB once it is on Wi-Fi, `WIFI ADDRESS ` (`box-address`). It connects to port 7340 there; `VIBEBUDDY_LAN_ADDR=host:port` fixes the address instead, as for the simulator (`tools/simulate-device.py --tcp 7340`). -Once on Wi-Fi the box also answers mDNS (DNS-SD) for `_vibebuddy._tcp` on port 7340, as the instance `VibeBuddy ` on host `vibebuddy-.local`, with the box key in its TXT record: `key=`. When nothing answers at the remembered address, as after the router hands the box another one while it is off the cable, `vibebuddyd` connects to whichever advertised instance carries the remembered key, and remembers that address once the handshake succeeds. mDNS only finds the box; the handshake still checks its key, so an instance that claims the key without holding it gets nowhere. The daemon starts browsing the first time the remembered address doesn't answer and keeps listening from then on (Bonjour's port on macOS and Avahi's on Linux are shared, no system service needed). A fixed `VIBEBUDDY_LAN_ADDR` turns discovery off. While linked through the Relay, the 30 s check for the local network tries both the remembered address and any advertised one. It tries the network only while no box is on the cable, and checks the cable every 2 s while linked over the network. Moving between the two is not a drop: `/v1/status` goes from one connection to the other with no `connected: false` in between, and `device.network` says which it is. Flashing needs the cable. +Once on Wi-Fi the box also answers mDNS (DNS-SD) for `_vibebuddy._tcp` on port 7340, as the instance `VibeBuddy ` on host `vibebuddy-.local`, with the box key in its TXT record: `key=`. When nothing answers at the remembered address, as after the router hands the box another one while it is off the cable, `vibebuddyd` connects to whichever advertised instance carries the remembered key, and remembers that address once the handshake succeeds. mDNS only finds the box; the handshake still checks its key, so an instance that claims the key without holding it gets nowhere. The daemon starts browsing the first time the remembered address doesn't answer and keeps listening from then on (Bonjour's port on macOS and Avahi's on Linux are shared, no system service needed). A fixed `VIBEBUDDY_LAN_ADDR` turns discovery off. While linked through the Relay, the 30 s check for the local network tries both the remembered address and any advertised one. It tries the network only while no box is on the cable, and checks the cable every 2 s while linked over the network. Moving between the two is not a drop: `/v1/status` goes from one connection to the other with no `connected: false` in between, and `device.network` says which it is. Firmware that reports `TRANSFER 1` is updated over the network too (see Firmware over the link); older firmware needs the cable. USB stays in the clear: the cable is physical presence, as pairing already assumes. @@ -196,4 +224,4 @@ USB stays in the clear: the cable is physical presence, as pairing already assum | `ERROR invalid_json` | The line isn't complete, valid JSON | | `ERROR invalid_message` | The root isn't an object, or `version`, `event` or `title` is missing or misused | | `ERROR unsupported_version` | `version` isn't `1` | -| `ERROR input_too_large` | A line exceeds 1024 bytes | +| `ERROR input_too_large` | A line exceeds 4608 bytes (1024 for firmware without `TRANSFER 1`) | diff --git a/docs/relay-protocol.md b/docs/relay-protocol.md index 8ea846e..1708993 100644 --- a/docs/relay-protocol.md +++ b/docs/relay-protocol.md @@ -90,4 +90,4 @@ The simulator plays the box on the Relay with `tools/simulate-device.py --relay ## Not decided yet -Resuming large writes after a drop. Noticing a box that lost power without closing its socket: the Relay shows it online until the edge notices. +Noticing a box that lost power without closing its socket: the Relay shows it online until the edge notices. diff --git a/protocol/src/lib.rs b/protocol/src/lib.rs index a7c8109..f69477d 100644 --- a/protocol/src/lib.rs +++ b/protocol/src/lib.rs @@ -18,7 +18,9 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; pub const VERSION: i64 = 1; -pub const MAX_MESSAGE_BYTES: usize = 1024; +/// The longest line the box takes: a firmware chunk of 3072 bytes is about 4.2 KB as a line (ADR-0016). Firmware +/// before that took 1024; only those chunks come near it, and they go only to firmware that reports `TRANSFER 1`. +pub const MAX_MESSAGE_BYTES: usize = 4608; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Event { From b21c2e3e3ee247ad6dd2c3e9a02acb9662c8667c Mon Sep 17 00:00:00 2001 From: dragon Date: Tue, 13 Oct 2026 18:29:24 +0800 Subject: [PATCH 3/4] daemon: count an update done only once a rollback had time to show On a box over USB, an image that failed before confirming itself had already printed its build, so the daemon reported the update done while the bootloader was going back. The daemon now waits 8 s after the new build and reads the box's answer after that: a rollback restarts it within about 2 s and says FIRMWARE ROLLED BACK. Over the network the box is only reached once it has confirmed itself. Co-Authored-By: Claude Opus 5.5 --- daemon/src/main.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/daemon/src/main.rs b/daemon/src/main.rs index 007cd3e..2b51a72 100644 --- a/daemon/src/main.rs +++ b/daemon/src/main.rs @@ -1017,6 +1017,8 @@ async fn post_firmware( /// deadline: a stale `replug` would keep the app's onboarding from ever showing the box as found, however late it boots. /// How long a box updated over the link may take to come back: restart, Wi-Fi, and the daemon finding it again. const OTA_BOOT_TIMEOUT: Duration = Duration::from_secs(180); +/// Long enough for an image that fails before confirming itself to restart and be rolled back (seen on a box: about 2 s). +const OTA_SETTLE: Duration = Duration::from_secs(8); /// Sends the app image over the link (ADR-0016), then waits for the box to come back on it. Success is the box /// reporting its build again without `FIRMWARE ROLLED BACK`. @@ -1054,7 +1056,14 @@ async fn update_over_the_link(state: AppState, segments: Vec Date: Tue, 13 Oct 2026 19:08:55 +0800 Subject: [PATCH 4/4] Faster updates over the link On a box a 2.8 MB image took 38 s over USB, 115 s over the local network and about 4 minutes through the Relay. The box erased one 4 KB sector per chunk, about 690 erases; it read the network 256 bytes at a time, each piece an event for the main loop; and four chunks in flight were too few for the Relay's round trip. The box now erases what the image needs before it answers xfer.begin, one 64 KB block per main-loop pass, and only writes while chunks arrive. That also keeps erases, which stop interrupts, away from bytes arriving on the UART bridge. It reads the network 1 KB at a time. The daemon keeps up to 16 chunks in flight and sends at most 40 a second, under the Relay's 50 messages a second, and gives xfer.ready 60 s. Co-Authored-By: Claude Opus 5.5 --- daemon/src/transfer.rs | 16 +++-- docs/protocol.md | 2 +- firmware-rs/core/src/firmware.rs | 10 +++- firmware-rs/core/src/transfer.rs | 93 +++++++++++++++++++++++------- firmware-rs/core/tests/firmware.rs | 19 +++++- firmware-rs/device/src/net.rs | 3 +- 6 files changed, 111 insertions(+), 32 deletions(-) diff --git a/daemon/src/transfer.rs b/daemon/src/transfer.rs index 3b1c1c4..e40a607 100644 --- a/daemon/src/transfer.rs +++ b/daemon/src/transfer.rs @@ -17,9 +17,14 @@ use crate::voice_writer::send; /// Image bytes per chunk: as base64 inside its line about 4.2 KB, under the box's 4608. pub const CHUNK_BYTES: usize = 3072; -/// Chunks sent ahead of the acknowledgements: enough to keep the Relay busy at 130-250 ms a round trip, while -/// staying under its 50 messages a second. -const WINDOW: usize = 4; +/// Chunks sent ahead of the acknowledgements: enough to keep the Relay busy at 130-250 ms a round trip and the box +/// writing; [`PACE`] keeps the rate down. +const WINDOW: usize = 16; +/// At most one chunk this often, 40 a second: under the Relay's 50 messages a second for good, with room for the +/// heartbeat (docs/relay-protocol.md). About 120 KB/s, more than the box takes over any link. +const PACE: Duration = Duration::from_millis(25); +/// The box erases what the image needs before it answers `xfer.begin`: up to 4 MB, a block per main-loop pass. +const READY_TIMEOUT: Duration = Duration::from_secs(60); /// No answer to a chunk for this long means it went missing: ask where the box got to. const STEP_TIMEOUT: Duration = Duration::from_secs(20); /// The box reads 4 MB back to check it before it answers `xfer.end`. @@ -110,15 +115,18 @@ async fn send_from_box( ) -> Result<(), Stop> { send(transport, begin_event(image.len(), sha256)).await.map_err(|_| Stop::Resume)?; let mut acked = loop { - if let (name, Some(offset)) = answer(bus, STEP_TIMEOUT).await? + if let (name, Some(offset)) = answer(bus, READY_TIMEOUT).await? && name == "xfer.ready" { break offset; } }; let mut next = acked; + let mut pace = tokio::time::interval(PACE); + pace.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); while acked < image.len() { while next < image.len() && next < acked + WINDOW * CHUNK_BYTES { + pace.tick().await; let end = (next + CHUNK_BYTES).min(image.len()); send(transport, chunk_event(next, &image[next..end])).await.map_err(|_| Stop::Resume)?; next = end; diff --git a/docs/protocol.md b/docs/protocol.md index a933221..260cc74 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -143,7 +143,7 @@ The box answers with JSON lines: {"version":1,"event":"xfer.error","code":"sha256"} ``` -`size` and `sha256` are the app image's (`vibebuddy-fw.bin`; the bootloader and partition table never change once a box has the OTA layout). The box writes the slot it isn't running from, `ota_1` after a USB flash, erasing one 4 KB sector ahead of the writes, and checks that the first chunk starts an ESP32-S3 app image. `xfer.ready` and `xfer.ack` carry the offset the box expects next; the computer keeps at most four chunks unacknowledged. A chunk at any other offset changes nothing and is acknowledged with the expected offset again: every way of carrying the link delivers in order, so a chunk only goes missing when the link drops. `crc` is zlib's CRC32 of the chunk's raw bytes. The box keeps the image's size, hash and how far it got in memory, so an `xfer.begin` for the same image, over any way of carrying the link, answers with that offset and the transfer carries on; another image starts over, and a restart forgets. When the link drops or moves, `vibebuddyd` waits up to 3 minutes for it to come back and asks again. +`size` and `sha256` are the app image's (`vibebuddy-fw.bin`; the bootloader and partition table never change once a box has the OTA layout). The box writes the slot it isn't running from, `ota_1` after a USB flash. Before it answers `xfer.begin` it erases what the image needs there, one 64 KB block per main-loop pass (about 15 s for 3 MB), so that while chunks arrive it only writes: an erase stops interrupts long enough to lose bytes on the UART bridge. It checks that the first chunk starts an ESP32-S3 app image. `xfer.ready` and `xfer.ack` carry the offset the box expects next; the computer keeps up to 16 chunks unacknowledged and sends at most 40 a second, under the Relay's 50 messages a second. A chunk at any other offset changes nothing and is acknowledged with the expected offset again: every way of carrying the link delivers in order, so a chunk only goes missing when the link drops. `crc` is zlib's CRC32 of the chunk's raw bytes. The box keeps the image's size, hash and how far it got in memory, so an `xfer.begin` for the same image, over any way of carrying the link, answers with that offset and the transfer carries on; another image starts over, and a restart forgets. When the link drops or moves, `vibebuddyd` waits up to 3 minutes for it to come back and asks again. On `xfer.end` the box reads the slot back, checks the length and SHA-256, writes the next `otadata` entry selecting that slot in the state NEW (in the sector that doesn't hold the running image's entry, so that one is still there to go back to), answers `xfer.done` and restarts. Nothing touches `otadata` before then, so a transfer that fails anywhere leaves the box running what it ran. The new image boots once and confirms itself early in boot, or the bootloader goes back to the old one (ADR-0013); a box that went back reports `FIRMWARE ROLLED BACK` after `TRANSFER 1` until the next update. `vibebuddyd` counts the update done when the box reports its build again without it. diff --git a/firmware-rs/core/src/firmware.rs b/firmware-rs/core/src/firmware.rs index cdda9a5..32129c0 100644 --- a/firmware-rs/core/src/firmware.rs +++ b/firmware-rs/core/src/firmware.rs @@ -592,6 +592,12 @@ impl Firmware { } pub fn poll(&mut self, board: &mut B) { + // A firmware transfer erases its slot one block per pass, so the network and the screen keep running. + match self.transfer.erase_step(board.flash()) { + Ok(Some(offset)) => Self::transfer_reply(board, "xfer.ready", Some(offset)), + Ok(None) => {} + Err(failure) => Self::transfer_error(board, failure), + } let now = board.now_ms(); let outputs = self.lan.poll(now); self.apply_lan(board, outputs); @@ -1550,7 +1556,9 @@ impl Firmware { return Self::transfer_error(board, transfer::Error::BadMessage); }; match self.transfer.begin(board.flash(), kind, size as u32, sha256) { - Ok(offset) => Self::transfer_reply(board, "xfer.ready", Some(offset)), + Ok(Some(offset)) => Self::transfer_reply(board, "xfer.ready", Some(offset)), + // Erasing first; poll answers once it is done. + Ok(None) => {} Err(failure) => Self::transfer_error(board, failure), } } diff --git a/firmware-rs/core/src/transfer.rs b/firmware-rs/core/src/transfer.rs index e88c90f..07d9f9b 100644 --- a/firmware-rs/core/src/transfer.rs +++ b/firmware-rs/core/src/transfer.rs @@ -1,8 +1,10 @@ //! Receiving a firmware image over any link, in chunks, resumably (ADR-0016). The computer sends -//! `xfer.begin` with the image's size and SHA-256, then `xfer.chunk`s of at most [`CHUNK_BYTES`] with up to -//! four unacknowledged, then `xfer.end`. The box writes the OTA slot it isn't running from, erasing one sector -//! ahead of the writes, and acknowledges each chunk with the next offset it expects; a chunk at any other offset -//! is acknowledged with that offset again, since every way of carrying a link delivers in order and a chunk only +//! `xfer.begin` with the image's size and SHA-256, then `xfer.chunk`s of at most [`CHUNK_BYTES`], several +//! unacknowledged, then `xfer.end`. The box erases what the image needs in the OTA slot it isn't running from, +//! one 64 KB block per main-loop pass, before it answers `xfer.ready`: while chunks arrive it only writes, since an +//! erase stops interrupts long enough to lose bytes on the UART bridge, and block erases are far faster than +//! sector ones. It acknowledges each chunk with the next offset it expects; a chunk at any other offset is +//! acknowledged with that offset again, since every way of carrying a link delivers in order and a chunk only //! goes missing when the link drops. //! //! What has been written is kept in memory, so an `xfer.begin` for the same image, over any way of carrying the @@ -13,6 +15,9 @@ use sha2::{Digest, Sha256}; use crate::ota; use crate::storage::{Flash, Region, SECTOR_BYTES, find_partition}; + +/// What one erase step covers: a flash block, erased with one command. +const ERASE_STEP: u32 = 64 * 1024; use crate::voice_pack::crc32; /// Most image bytes in one chunk: as base64 inside its line, about 4.2 KB. @@ -59,7 +64,7 @@ struct Image { target: Region, /// Bytes of the image written so far. next: u32, - /// Flash erased up to here, absolute. + /// Flash erased up to here, absolute; chunks are taken once it reaches [`Image::erase_end`]. erased_until: u32, } @@ -74,6 +79,17 @@ fn is_esp32s3_image(first: &[u8]) -> bool { first.len() >= 14 && first[0] == 0xE9 && first[12..14] == [9, 0] } +impl Image { + /// Where erasing stops: the image's end, rounded up to a whole block within the slot. + fn erase_end(&self) -> u32 { + (self.target.offset + self.size).next_multiple_of(ERASE_STEP).min(self.target.offset + self.target.size) + } + + fn erased(&self) -> bool { + self.erased_until >= self.erase_end() + } +} + impl Transfer { pub fn new() -> Self { Self::default() @@ -88,14 +104,15 @@ impl Transfer { self.image.as_ref().map(|image| (u64::from(image.next) * 100 / u64::from(image.size.max(1))) as u8) } - /// Starts a transfer, or resumes the same one; returns the offset to send from. - pub fn begin(&mut self, flash: &mut dyn Flash, kind: &str, size: u32, sha256: [u8; 32]) -> Result { + /// Starts a transfer, or resumes the same one. Returns the offset to send from once the slot is erased for it; + /// None while erasing, and [`Self::erase_step`] gives the offset when done. + pub fn begin(&mut self, flash: &mut dyn Flash, kind: &str, size: u32, sha256: [u8; 32]) -> Result, Error> { if let Some(image) = &self.image && image.size == size && image.sha256 == sha256 && kind == "firmware" { - return Ok(image.next); + return Ok(image.erased().then_some(image.next)); } self.image = None; if kind != "firmware" { @@ -112,7 +129,23 @@ impl Transfer { return Err(Error::TooBig); } self.image = Some(Image { size, sha256, slot, target, next: 0, erased_until: target.offset }); - Ok(0) + Ok(None) + } + + /// Erases the next block while a transfer is starting; the offset to send from once the last one is done. + /// Called once per main-loop pass, so the network and the screen run between blocks. + pub fn erase_step(&mut self, flash: &mut dyn Flash) -> Result, Error> { + let Some(image) = self.image.as_mut() else { return Ok(None) }; + if image.erased() { + return Ok(None); + } + let end = (image.erased_until + ERASE_STEP).min(image.erase_end()); + if flash.erase(image.erased_until, end).is_err() { + self.image = None; + return Err(Error::Flash); + } + image.erased_until = end; + Ok(image.erased().then_some(image.next)) } /// One chunk at `offset`; returns the offset expected next. An error ends the transfer. @@ -126,6 +159,9 @@ impl Transfer { fn write(&mut self, flash: &mut dyn Flash, offset: u32, data: &[u8], crc: u32) -> Result { let image = self.image.as_mut().ok_or(Error::Sequence)?; + if !image.erased() { + return Err(Error::Sequence); + } if offset != image.next { return Ok(image.next); } @@ -140,11 +176,6 @@ impl Transfer { return Err(Error::TooBig); } let at = image.target.offset + offset; - // One sector at a time, as the writes reach it: the main loop stalls for each erase. - while image.erased_until < at + data.len() as u32 { - flash.erase(image.erased_until, image.erased_until + SECTOR_BYTES).map_err(|_| Error::Flash)?; - image.erased_until += SECTOR_BYTES; - } // Flash writes go in whole words; the last chunk is padded with what erased flash holds anyway. let mut padded = [0xFFu8; CHUNK_BYTES]; padded[..data.len()].copy_from_slice(data); @@ -223,6 +254,18 @@ mod tests { Sha256::digest(data).into() } + /// Begins and erases, as the main loop would; returns the offset to send from. + fn start(transfer: &mut Transfer, flash: &mut MemoryFlash, image: &[u8], sha256: [u8; 32]) -> u32 { + if let Some(offset) = transfer.begin(flash, "firmware", image.len() as u32, sha256).unwrap() { + return offset; + } + loop { + if let Some(offset) = transfer.erase_step(flash).unwrap() { + return offset; + } + } + } + fn send(transfer: &mut Transfer, flash: &mut MemoryFlash, image: &[u8], from: u32) -> u32 { let mut next = from; while (next as usize) < image.len() { @@ -238,7 +281,13 @@ mod tests { let mut flash = flash(); let image = test_image(10_000); let mut transfer = Transfer::new(); - assert_eq!(transfer.begin(&mut flash, "firmware", image.len() as u32, sha(&image)), Ok(0)); + flash.bytes[0x610000..0x620000].fill(0); + assert_eq!(transfer.begin(&mut flash, "firmware", image.len() as u32, sha(&image)), Ok(None), "erasing first"); + let first = &image[..CHUNK_BYTES]; + assert_eq!(transfer.chunk(&mut flash, 0, first, crc(first)), Err(Error::Sequence), "no chunks before ready"); + transfer.begin(&mut flash, "firmware", image.len() as u32, sha(&image)).unwrap(); + assert_eq!(start(&mut transfer, &mut flash, &image, sha(&image)), 0); + assert!(flash.bytes[0x610000..0x620000].iter().all(|&byte| byte == 0xFF), "the block it needs is erased"); assert_eq!(send(&mut transfer, &mut flash, &image, 0), 10_000); assert_eq!(transfer.percent(), Some(100)); transfer.end(&mut flash).unwrap(); @@ -253,11 +302,11 @@ mod tests { let mut flash = flash(); let image = test_image(10_000); let mut transfer = Transfer::new(); - transfer.begin(&mut flash, "firmware", image.len() as u32, sha(&image)).unwrap(); + start(&mut transfer, &mut flash, &image, sha(&image)); let first = &image[..CHUNK_BYTES]; assert_eq!(transfer.chunk(&mut flash, 0, first, crc(first)), Ok(CHUNK_BYTES as u32)); - // The link dropped and came back, perhaps another way: the same image carries on. - assert_eq!(transfer.begin(&mut flash, "firmware", image.len() as u32, sha(&image)), Ok(CHUNK_BYTES as u32)); + // The link dropped and came back, perhaps another way: the same image carries on, with no erasing. + assert_eq!(transfer.begin(&mut flash, "firmware", image.len() as u32, sha(&image)), Ok(Some(CHUNK_BYTES as u32))); // A chunk sent twice, or one ahead of a lost one, gets the expected offset again. assert_eq!(transfer.chunk(&mut flash, 0, first, crc(first)), Ok(CHUNK_BYTES as u32)); let later = &image[2 * CHUNK_BYTES..3 * CHUNK_BYTES]; @@ -267,7 +316,7 @@ mod tests { // Another image starts over. let other = test_image(5_000); - assert_eq!(transfer.begin(&mut flash, "firmware", other.len() as u32, sha(&other)), Ok(0)); + assert_eq!(start(&mut transfer, &mut flash, &other, sha(&other)), 0); } #[test] @@ -278,15 +327,15 @@ mod tests { let mut not_ours = test_image(4_000); not_ours[12] = 2; - transfer.begin(&mut flash, "firmware", 4_000, sha(¬_ours)).unwrap(); + start(&mut transfer, &mut flash, ¬_ours, sha(¬_ours)); assert_eq!(transfer.chunk(&mut flash, 0, ¬_ours[..CHUNK_BYTES], crc(¬_ours[..CHUNK_BYTES])), Err(Error::BadImage)); assert!(!transfer.active()); let image = test_image(4_000); - transfer.begin(&mut flash, "firmware", 4_000, sha(&image)).unwrap(); + start(&mut transfer, &mut flash, &image, sha(&image)); assert_eq!(transfer.chunk(&mut flash, 0, &image[..CHUNK_BYTES], 1), Err(Error::Crc)); - transfer.begin(&mut flash, "firmware", 4_000, [0; 32]).unwrap(); + start(&mut transfer, &mut flash, &image, [0; 32]); send(&mut transfer, &mut flash, &image, 0); assert_eq!(transfer.end(&mut flash), Err(Error::Sha256)); assert_eq!(ota::running_slot(&mut flash, otadata), Ok(0)); diff --git a/firmware-rs/core/tests/firmware.rs b/firmware-rs/core/tests/firmware.rs index 742a76a..14aebfc 100644 --- a/firmware-rs/core/tests/firmware.rs +++ b/firmware-rs/core/tests/firmware.rs @@ -1266,6 +1266,19 @@ mod transfer { serde_json::from_str(lines.last().unwrap()).unwrap() } + /// Sends xfer.begin and runs the main loop until the box answers: it erases the slot first. + fn begin_transfer(firmware: &mut Firmware, board: &mut FakeBoard, line: &str) -> Value { + let mut lines = send(firmware, board, line); + let mut passes = 0; + while !lines.iter().any(|line| line.contains("xfer.")) { + firmware.poll(board); + lines = board.take_lines(); + passes += 1; + assert!(passes < 100, "no answer to xfer.begin"); + } + reply(&lines) + } + #[test] fn a_firmware_image_comes_over_the_link_and_the_box_restarts_into_it() { let (mut firmware, mut board, _) = booted(blank_flash()); @@ -1274,7 +1287,7 @@ mod transfer { image[12..14].copy_from_slice(&[9, 0]); let sha: String = Sha256::digest(&image).iter().map(|byte| format!("{byte:02x}")).collect(); let begin = format!(r#"{{"version":1,"event":"xfer.begin","kind":"firmware","size":{},"sha256":"{sha}"}}"#, image.len()); - assert_eq!(reply(&send(&mut firmware, &mut board, &begin)), serde_json::json!({"version": 1, "event": "xfer.ready", "offset": 0})); + assert_eq!(begin_transfer(&mut firmware, &mut board, &begin), serde_json::json!({"version": 1, "event": "xfer.ready", "offset": 0})); for (index, piece) in image.chunks(CHUNK_BYTES).enumerate() { let line = format!( r#"{{"version":1,"event":"xfer.chunk","offset":{},"crc":{},"data":"{}"}}"#, @@ -1300,7 +1313,7 @@ mod transfer { image[0] = 0xE9; image[12..14].copy_from_slice(&[9, 0]); let sha: String = Sha256::digest(&image).iter().map(|byte| format!("{byte:02x}")).collect(); - send(&mut firmware, &mut board, &format!(r#"{{"version":1,"event":"xfer.begin","kind":"firmware","size":{},"sha256":"{sha}"}}"#, image.len())); + begin_transfer(&mut firmware, &mut board, &format!(r#"{{"version":1,"event":"xfer.begin","kind":"firmware","size":{},"sha256":"{sha}"}}"#, image.len())); let piece = &image[..CHUNK_BYTES]; let line = format!(r#"{{"version":1,"event":"xfer.chunk","offset":0,"crc":{},"data":"{}"}}"#, crc32(0, piece), base64(piece)); assert_eq!(reply(&send(&mut firmware, &mut board, &line))["offset"], CHUNK_BYTES as u64); @@ -1310,7 +1323,7 @@ mod transfer { fn a_bad_chunk_ends_the_transfer_with_its_reason() { let (mut firmware, mut board, _) = booted(blank_flash()); let begin = format!(r#"{{"version":1,"event":"xfer.begin","kind":"firmware","size":100,"sha256":"{}"}}"#, "00".repeat(32)); - send(&mut firmware, &mut board, &begin); + begin_transfer(&mut firmware, &mut board, &begin); let line = format!(r#"{{"version":1,"event":"xfer.chunk","offset":0,"crc":1,"data":"{}"}}"#, base64(&[0xE9; 100])); assert_eq!(reply(&send(&mut firmware, &mut board, &line)), serde_json::json!({"version": 1, "event": "xfer.error", "code": "crc"})); assert_eq!(reply(&send(&mut firmware, &mut board, r#"{"version":1,"event":"xfer.end"}"#))["code"], "sequence"); diff --git a/firmware-rs/device/src/net.rs b/firmware-rs/device/src/net.rs index 30be84c..321db2d 100644 --- a/firmware-rs/device/src/net.rs +++ b/firmware-rs/device/src/net.rs @@ -317,7 +317,8 @@ async fn server(stack: Stack<'static>, slot: usize) { // On the heap, which is PSRAM first: internal RAM is for the Wi-Fi driver and the stack. let mut rx = alloc::vec![0u8; SOCKET_RX_BYTES]; let mut tx = alloc::vec![0u8; SOCKET_TX_BYTES]; - let mut chunk = [0u8; 256]; + // A sealed firmware chunk is about 5.7 KB: read it in a few pieces, not twenty, each one an event for the main loop. + let mut chunk = alloc::vec![0u8; 1024]; loop { let mut socket = TcpSocket::new(stack, &mut rx, &mut tx); socket.set_timeout(Some(Duration::from_secs(60)));