From 03bb0a3e4f6f151edc23ba8fd4958af026533d11 Mon Sep 17 00:00:00 2001 From: Juan Pablo Pino Bravo Date: Fri, 21 Aug 2026 11:15:22 +0200 Subject: [PATCH 1/2] Add vision tracking control modes and enrich SotState with Kalman estimates Add spotlight, active_track, and orbit to ControlMode, with matching SpotlightState/ActiveTrackState/OrbitState messages and Ctrl wrappers. OrbitState carries the commanded orbit angular rate (deg/s). Extend SotState with the tracker confidence and the Kalman velocity estimates (vx, vy, vs, vr) so vision-based control modes can use rate feedforward, and so recorded dives capture the full tracking state. Co-Authored-By: Claude Fable 5 --- protobuf_definitions/control.proto | 24 ++++++++++++++ protobuf_definitions/message_formats.proto | 37 ++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/protobuf_definitions/control.proto b/protobuf_definitions/control.proto index 17f5b976..7a52f25b 100644 --- a/protobuf_definitions/control.proto +++ b/protobuf_definitions/control.proto @@ -105,6 +105,30 @@ message AutoPilotHeaveCtrl { AutoPilotHeaveState state = 1; // State of the auto pilot heave controller- } +// Issue a command to set the vision-based spotlight mode to a desired state. +// +// Requires an actively tracked single-object-tracking (SOT) target; +// the drone refuses to enable the mode otherwise. +message SpotlightCtrl { + SpotlightState state = 1; // State of the spotlight mode. +} + +// Issue a command to set the vision-based active track mode to a desired state. +// +// Requires an actively tracked single-object-tracking (SOT) target; +// the drone refuses to enable the mode otherwise. +message ActiveTrackCtrl { + ActiveTrackState state = 1; // State of the active track mode. +} + +// Issue a command to set the vision-based orbit mode to a desired state. +// +// Requires an actively tracked single-object-tracking (SOT) target; +// the drone refuses to enable the mode otherwise. +message OrbitCtrl { + OrbitState state = 1; // State of the orbit mode, including the orbit angular rate. +} + // Issue a command to start and pause the loaded mission. message RunMissionCtrl { } diff --git a/protobuf_definitions/message_formats.proto b/protobuf_definitions/message_formats.proto index d11bb9e1..938e0ad5 100644 --- a/protobuf_definitions/message_formats.proto +++ b/protobuf_definitions/message_formats.proto @@ -178,6 +178,32 @@ message AutoPilotHeaveState { bool enabled = 1; // If auto pilot heave is enabled. } +// Spotlight state. +// +// Vision-based mode that keeps the tracked object centred in the frame +// (auto heading and vertical centring) while the pilot retains manual +// surge and sway control. +message SpotlightState { + bool enabled = 1; // If spotlight is enabled. +} + +// Active track state. +// +// Vision-based mode that keeps the tracked object centred in the frame and +// automatically controls surge to maintain the desired distance to it. +message ActiveTrackState { + bool enabled = 1; // If active track is enabled. +} + +// Orbit state. +// +// Vision-based mode that keeps the tracked object centred in the frame while +// the drone circles it laterally at the commanded angular rate. +message OrbitState { + bool enabled = 1; // If orbit is enabled. + float rate = 2; // Orbit angular rate (deg/s); positive orbits clockwise seen from above. 0 keeps the current rate. +} + // Control mode from drone supervisor message ControlMode { bool auto_depth = 1; // If auto depth is enabled. @@ -187,6 +213,9 @@ message ControlMode { bool weather_vaning = 5; // If weather vaning is enabled. bool auto_pilot_surge_yaw = 6; // If auto pilot surge yaw is enabled. bool auto_pilot_heave = 7; // If auto pilot heave is enabled. + bool spotlight = 8; // If the vision-based spotlight mode is enabled. + bool active_track = 9; // If the vision-based active track mode is enabled. + bool orbit = 10; // If the vision-based orbit mode is enabled. } // Tilt stabilization state. @@ -1682,6 +1711,9 @@ message OperatorInfo { } // Single-object tracking (SOT) state reported by the computer vision pipeline. +// +// The velocity fields are Kalman filter estimates expressed in the CV frame +// coordinate space (image_width x image_height pixels). message SotState { // Current state of the SOT tracker. enum State { @@ -1694,6 +1726,11 @@ message SotState { BoundingBox bounding_box = 2; // Current tracked bounding box (valid when TRACKING). uint32 image_width = 3; // Width of the source frame in pixels. uint32 image_height = 4; // Height of the source frame in pixels. + float confidence = 5; // Tracker confidence of the last model update (0..1). 0 while coasting and when LOST. + float vx = 6; // Kalman velocity estimate of the bounding box centre, horizontal (px/s). + float vy = 7; // Kalman velocity estimate of the bounding box centre, vertical (px/s). + float vs = 8; // Kalman rate of change of the bounding box area, width * height (px²/s). + float vr = 9; // Kalman rate of change of the bounding box aspect ratio, width / height (1/s). } // Origin of an annotation. Mirrors the AnnotationSource enum in Blueye Cloud. From 2dd2dfa18ac24f87744e769933b9cddb15990ee8 Mon Sep 17 00:00:00 2001 From: Juan Pablo Pino Bravo Date: Sun, 23 Aug 2026 19:58:06 +0200 Subject: [PATCH 2/2] Model the vision tracking modes as a oneof instead of independent bools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three modes are mutually exclusive, but separate ControlMode bools and per-mode Ctrl messages read as independent toggles. Encode the exclusivity structurally instead, following the Instruction.command pattern: - VisionTrackingState carries a oneof over per-mode parameter messages; no field set means no vision tracking mode is active. The per-mode enabled bools are gone (presence is enablement), and Spotlight/ ActiveTrack states are empty extension points for future parameters. - ControlMode.vision_tracking replaces the three draft bools, so the telemetry also echoes the active orbit rate. - A single VisionTrackingCtrl replaces SpotlightCtrl/ActiveTrackCtrl/ OrbitCtrl: mode switches are atomic, disable is an empty state, and the '0 keeps the current rate' convention is dropped — OrbitState.rate is mandatory and non-zero. Co-Authored-By: Claude Fable 5 --- protobuf_definitions/control.proto | 29 ++++++-------------- protobuf_definitions/message_formats.proto | 31 ++++++++++++++-------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/protobuf_definitions/control.proto b/protobuf_definitions/control.proto index 7a52f25b..163f7f85 100644 --- a/protobuf_definitions/control.proto +++ b/protobuf_definitions/control.proto @@ -105,28 +105,15 @@ message AutoPilotHeaveCtrl { AutoPilotHeaveState state = 1; // State of the auto pilot heave controller- } -// Issue a command to set the vision-based spotlight mode to a desired state. +// Issue a command to set the vision-based tracking mode. // -// Requires an actively tracked single-object-tracking (SOT) target; -// the drone refuses to enable the mode otherwise. -message SpotlightCtrl { - SpotlightState state = 1; // State of the spotlight mode. -} - -// Issue a command to set the vision-based active track mode to a desired state. -// -// Requires an actively tracked single-object-tracking (SOT) target; -// the drone refuses to enable the mode otherwise. -message ActiveTrackCtrl { - ActiveTrackState state = 1; // State of the active track mode. -} - -// Issue a command to set the vision-based orbit mode to a desired state. -// -// Requires an actively tracked single-object-tracking (SOT) target; -// the drone refuses to enable the mode otherwise. -message OrbitCtrl { - OrbitState state = 1; // State of the orbit mode, including the orbit angular rate. +// Setting a mode requires an actively tracked single-object-tracking (SOT) +// target; the drone refuses to enable a mode otherwise. Setting a different +// mode while one is active switches atomically. Sending a state with no +// mode set disables vision tracking. An OrbitState with a zero rate is +// rejected. +message VisionTrackingCtrl { + VisionTrackingState state = 1; // The desired vision tracking mode, or none to disable. } // Issue a command to start and pause the loaded mission. diff --git a/protobuf_definitions/message_formats.proto b/protobuf_definitions/message_formats.proto index 938e0ad5..d29405c5 100644 --- a/protobuf_definitions/message_formats.proto +++ b/protobuf_definitions/message_formats.proto @@ -178,30 +178,41 @@ message AutoPilotHeaveState { bool enabled = 1; // If auto pilot heave is enabled. } -// Spotlight state. +// Spotlight mode parameters. // // Vision-based mode that keeps the tracked object centred in the frame // (auto heading and vertical centring) while the pilot retains manual -// surge and sway control. +// surge and sway control. Currently carries no parameters. message SpotlightState { - bool enabled = 1; // If spotlight is enabled. } -// Active track state. +// Active track mode parameters. // // Vision-based mode that keeps the tracked object centred in the frame and // automatically controls surge to maintain the desired distance to it. +// Currently carries no parameters. message ActiveTrackState { - bool enabled = 1; // If active track is enabled. } -// Orbit state. +// Orbit mode parameters. // // Vision-based mode that keeps the tracked object centred in the frame while // the drone circles it laterally at the commanded angular rate. message OrbitState { - bool enabled = 1; // If orbit is enabled. - float rate = 2; // Orbit angular rate (deg/s); positive orbits clockwise seen from above. 0 keeps the current rate. + float rate = 1; // Orbit angular rate (deg/s); positive orbits clockwise seen from above. Must be non-zero. +} + +// The active vision-based tracking mode. +// +// The modes are mutually exclusive, which the oneof encodes: at most one +// mode is set at a time, and no field set means no vision tracking mode +// is active. +message VisionTrackingState { + oneof mode { + SpotlightState spotlight = 1; // Spotlight mode is active. + ActiveTrackState active_track = 2; // Active track mode is active. + OrbitState orbit = 3; // Orbit mode is active, at the reported angular rate. + } } // Control mode from drone supervisor @@ -213,9 +224,7 @@ message ControlMode { bool weather_vaning = 5; // If weather vaning is enabled. bool auto_pilot_surge_yaw = 6; // If auto pilot surge yaw is enabled. bool auto_pilot_heave = 7; // If auto pilot heave is enabled. - bool spotlight = 8; // If the vision-based spotlight mode is enabled. - bool active_track = 9; // If the vision-based active track mode is enabled. - bool orbit = 10; // If the vision-based orbit mode is enabled. + VisionTrackingState vision_tracking = 8; // The active vision-based tracking mode, if any. } // Tilt stabilization state.