From 16a07f6b00a002e9ef27adec02608cfa7939a20a Mon Sep 17 00:00:00 2001 From: adampog <2230698+adampog@users.noreply.github.com> Date: Wed, 30 Sep 2026 23:55:24 -0700 Subject: [PATCH 1/2] Show reefwatch camera frames on the robot's screen Adds a small LAN HTTP server (POST /frame, GET /live) that accepts JPEG frames from the reefwatch detector, decodes them with the existing esp_new_jpeg helper and shows them full screen on LVGL's top layer, so it works over any app and in AI agent mode. - Detections show for 8 s with labels; LEDs go amber and the head can turn toward the camera (CONFIG_STACKCHAN_REEFWATCH_CAMERA_YAW). - Tap a detection to watch that camera live, tap again to stop; the live view can also be started by voice via new self.cameras.* MCP tools, and switches itself off after 5 minutes. - Optional shared token, port, timings in a new Kconfig menu. Co-Authored-By: Claude Opus 5.5 --- firmware/README.md | 22 ++ firmware/main/CMakeLists.txt | 1 + firmware/main/Kconfig.projbuild | 46 ++++ .../common/reefwatch_view/reefwatch_view.cpp | 174 ++++++++++++++++ .../common/reefwatch_view/reefwatch_view.h | 15 ++ firmware/main/hal/hal.h | 14 ++ firmware/main/hal/hal_mcp.cpp | 23 ++ firmware/main/hal/hal_reefwatch.cpp | 196 ++++++++++++++++++ firmware/main/main.cpp | 5 + 9 files changed, 496 insertions(+) create mode 100644 firmware/main/apps/common/reefwatch_view/reefwatch_view.cpp create mode 100644 firmware/main/apps/common/reefwatch_view/reefwatch_view.h create mode 100644 firmware/main/hal/hal_reefwatch.cpp diff --git a/firmware/README.md b/firmware/README.md index 20a73bf65..a1fd9988a 100644 --- a/firmware/README.md +++ b/firmware/README.md @@ -32,3 +32,25 @@ ctest --test-dir build-host-tests --output-on-failure ```bash idf.py flash ``` + +## Reefwatch camera view (adampog fork) + +Shows frames from [reefwatch](https://github.com/adampog/reefwatch) (ESP32-CAM + Coral object detection) +full screen on the robot, over any app and in AI agent mode. + +- **Detections** show for 8 s with boxes and labels; LEDs go amber and, if configured, the head turns toward that camera. +- **Live view**: tap a detection to keep watching that camera, tap again to stop. In AI agent mode you can ask + for it by voice (`self.cameras.show_live` / `self.cameras.stop_live` MCP tools). Switches off after 5 min. + +Set a token (and optionally head angles) in `sdkconfig.defaults.local`: + +``` +CONFIG_STACKCHAN_REEFWATCH_TOKEN="change-me" +CONFIG_STACKCHAN_REEFWATCH_CAMERA_YAW="front=-45,garden=45" +``` + +and point reefwatch at the robot with `REEFWATCH_STACKCHAN_URL=http://` and the same +`REEFWATCH_STACKCHAN_TOKEN`. The robot logs its address on boot (`listening on http://...`). +Protocol: `POST /frame` (JPEG ≤ 320×240, headers `X-Reefwatch-Token`, `X-Camera`, `X-Kind`, `X-Labels`) +and `GET /live` → `{"camera": "front" | null, "fps": 5}`. Code: `main/hal/hal_reefwatch.cpp`, +`main/apps/common/reefwatch_view/`. diff --git a/firmware/main/CMakeLists.txt b/firmware/main/CMakeLists.txt index 22cfebb14..ade1449c5 100644 --- a/firmware/main/CMakeLists.txt +++ b/firmware/main/CMakeLists.txt @@ -297,6 +297,7 @@ idf_component_register(SRCS ${SOURCES} esp_pm esp_psram esp_netif + esp_http_server esp_driver_gpio esp_driver_uart esp_driver_spi diff --git a/firmware/main/Kconfig.projbuild b/firmware/main/Kconfig.projbuild index 89592a0c5..dd97dead4 100644 --- a/firmware/main/Kconfig.projbuild +++ b/firmware/main/Kconfig.projbuild @@ -11,6 +11,52 @@ config STACKCHAN_SERVER_URL endmenu +menu "Reefwatch Camera View" + +config STACKCHAN_REEFWATCH_ENABLE + bool "Show reefwatch camera frames on screen" + default y + help + Runs a small HTTP server on the LAN that accepts JPEG frames from the + reefwatch detector (github.com/adampog/reefwatch) and shows them + full screen. POST /frame takes the JPEG, GET /live tells reefwatch + which camera to stream live. + +config STACKCHAN_REEFWATCH_PORT + int "HTTP port" + default 80 + depends on STACKCHAN_REEFWATCH_ENABLE + +config STACKCHAN_REEFWATCH_TOKEN + string "Shared token (X-Reefwatch-Token header)" + default "" + depends on STACKCHAN_REEFWATCH_ENABLE + help + Must match REEFWATCH_STACKCHAN_TOKEN on the reefwatch side. Empty + accepts any request on the LAN. Set it in sdkconfig.defaults.local. + +config STACKCHAN_REEFWATCH_SHOW_MS + int "How long a detection stays on screen (ms)" + default 8000 + depends on STACKCHAN_REEFWATCH_ENABLE + +config STACKCHAN_REEFWATCH_LIVE_FPS + int "Live view frame rate requested from reefwatch" + default 5 + range 1 15 + depends on STACKCHAN_REEFWATCH_ENABLE + +config STACKCHAN_REEFWATCH_CAMERA_YAW + string "Head yaw per camera, in degrees" + default "" + depends on STACKCHAN_REEFWATCH_ENABLE + help + Turns the head toward a camera when it sees something, e.g. + "front=-45,garden=45" (negative is the robot's left). Empty + keeps the head still. + +endmenu + menu "Xiaozhi Assistant" config OTA_URL diff --git a/firmware/main/apps/common/reefwatch_view/reefwatch_view.cpp b/firmware/main/apps/common/reefwatch_view/reefwatch_view.cpp new file mode 100644 index 000000000..b58edadb4 --- /dev/null +++ b/firmware/main/apps/common/reefwatch_view/reefwatch_view.cpp @@ -0,0 +1,174 @@ +/* + * SPDX-License-Identifier: MIT + */ +#include "reefwatch_view.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if CONFIG_STACKCHAN_REEFWATCH_ENABLE + +static const std::string_view _tag = "reefwatch-view"; + +// Live frames keep the view open; if they stop arriving, close after this long +static constexpr uint32_t _live_idle_ms = 3000; + +static lv_obj_t* _panel = nullptr; +static lv_obj_t* _image = nullptr; +static lv_obj_t* _caption = nullptr; +static lv_timer_t* _timer = nullptr; +static std::shared_ptr _frame; // keeps the pixels alive while LVGL draws them +static std::string _camera; +static bool _detection_on_screen = false; // blocks live frames until it times out or is tapped +static bool _reacted = false; // LEDs changed, put them back on hide +static bool _head_turned = false; +static std::string _reacted_camera; +static std::map _camera_yaw; + +// "front=-45,garden=45" -> {front: -45, garden: 45} +static void parse_camera_yaw() +{ + std::istringstream list(CONFIG_STACKCHAN_REEFWATCH_CAMERA_YAW); + std::string item; + while (std::getline(list, item, ',')) { + auto eq = item.find('='); + if (eq != std::string::npos) { + _camera_yaw[item.substr(0, eq)] = std::atoi(item.c_str() + eq + 1); + } + } +} + +static void hide() +{ + lv_obj_add_flag(_panel, LV_OBJ_FLAG_HIDDEN); + lv_timer_pause(_timer); + lv_image_set_src(_image, nullptr); + _frame.reset(); + + _detection_on_screen = false; + if (_reacted) { + _reacted = false; + GetStackChan().leftNeonLight().setColor(0, 0, 0); + GetStackChan().rightNeonLight().setColor(0, 0, 0); + } + if (_head_turned) { + _head_turned = false; + GetStackChan().motion().yawServo().moveWithSpeed(0, 300); + } +} + +static void on_timer(lv_timer_t* timer) +{ + hide(); +} + +static void on_click(lv_event_t* e) +{ + // Tap a detection to keep watching that camera, tap the live view to stop it + if (GetHAL().getReefwatchLiveCamera().empty() && !_camera.empty()) { + GetHAL().setReefwatchLiveCamera(_camera); + _detection_on_screen = false; + lv_label_set_text_fmt(_caption, "%s - starting live view...", _camera.c_str()); + lv_timer_set_period(_timer, _live_idle_ms + 2000); // reefwatch polls every 2 s + lv_timer_reset(_timer); + } else { + GetHAL().setReefwatchLiveCamera(""); + hide(); + } +} + +static void react_to_detection(const std::string& camera) +{ + // Amber LEDs, within the 0-168 range hal_mcp treats as safe + GetStackChan().leftNeonLight().setColor(168, 90, 0); + GetStackChan().rightNeonLight().setColor(168, 90, 0); + + auto yaw = _camera_yaw.find(camera); + if (yaw != _camera_yaw.end()) { + GetStackChan().motion().yawServo().moveWithSpeed(yaw->second * 10, 300); + _head_turned = true; + } + _reacted = true; + _reacted_camera = camera; +} + +static void show(const ReefwatchFrame_t& frame) +{ + // A live frame never replaces a detection that is still on screen + if (frame.isLive && _detection_on_screen) { + return; + } + + _frame = frame.image; + _camera = frame.camera; + lv_image_set_src(_image, _frame->image_dsc()); + + if (frame.isLive) { + lv_label_set_text_fmt(_caption, "%s - LIVE", frame.camera.c_str()); + lv_timer_set_period(_timer, _live_idle_ms); + } else { + lv_label_set_text_fmt(_caption, "%s - %s", frame.camera.c_str(), frame.labels.c_str()); + lv_timer_set_period(_timer, CONFIG_STACKCHAN_REEFWATCH_SHOW_MS); + if (!_reacted || frame.camera != _reacted_camera) { + react_to_detection(frame.camera); + } + _detection_on_screen = true; + } + + lv_obj_remove_flag(_panel, LV_OBJ_FLAG_HIDDEN); + lv_timer_reset(_timer); + lv_timer_resume(_timer); +} + +void view::reefwatch_view_init() +{ + parse_camera_yaw(); + + LvglLockGuard lock; + + _panel = lv_obj_create(lv_layer_top()); + lv_obj_remove_style_all(_panel); + lv_obj_set_size(_panel, 320, 240); + lv_obj_set_style_bg_color(_panel, lv_color_black(), 0); + lv_obj_set_style_bg_opa(_panel, LV_OPA_COVER, 0); + lv_obj_remove_flag(_panel, LV_OBJ_FLAG_SCROLLABLE); + lv_obj_add_flag(_panel, LV_OBJ_FLAG_CLICKABLE); + lv_obj_add_flag(_panel, LV_OBJ_FLAG_HIDDEN); + lv_obj_add_event_cb(_panel, on_click, LV_EVENT_CLICKED, nullptr); + + _image = lv_image_create(_panel); + lv_obj_center(_image); + + _caption = lv_label_create(_panel); + lv_obj_set_width(_caption, 320); + lv_obj_align(_caption, LV_ALIGN_BOTTOM_MID, 0, 0); + lv_obj_set_style_bg_color(_caption, lv_color_black(), 0); + lv_obj_set_style_bg_opa(_caption, LV_OPA_60, 0); + lv_obj_set_style_text_color(_caption, lv_color_white(), 0); + lv_obj_set_style_pad_all(_caption, 4, 0); + lv_label_set_long_mode(_caption, LV_LABEL_LONG_DOT); + + _timer = lv_timer_create(on_timer, CONFIG_STACKCHAN_REEFWATCH_SHOW_MS, nullptr); + lv_timer_pause(_timer); + + GetHAL().onReefwatchFrame.connect([](const ReefwatchFrame_t& frame) { + LvglLockGuard lock; + show(frame); + }); + + mclog::tagInfo(_tag, "ready"); +} + +#else + +void view::reefwatch_view_init() +{ +} + +#endif diff --git a/firmware/main/apps/common/reefwatch_view/reefwatch_view.h b/firmware/main/apps/common/reefwatch_view/reefwatch_view.h new file mode 100644 index 000000000..072dd88ed --- /dev/null +++ b/firmware/main/apps/common/reefwatch_view/reefwatch_view.h @@ -0,0 +1,15 @@ +/* + * SPDX-License-Identifier: MIT + */ +#pragma once + +namespace view { + +/** + * @brief Full screen overlay for reefwatch camera frames, on LVGL's top layer so it shows over + * any app and in AI agent mode. Tap a detection to watch that camera live, tap again to stop. + * + */ +void reefwatch_view_init(); + +} // namespace view diff --git a/firmware/main/hal/hal.h b/firmware/main/hal/hal.h index b8d3c80f4..1363fb3af 100644 --- a/firmware/main/hal/hal.h +++ b/firmware/main/hal/hal.h @@ -40,6 +40,13 @@ struct WsTextMessage_t { std::string content; }; +struct ReefwatchFrame_t { + std::shared_ptr image; + std::string camera; + std::string labels; + bool isLive = false; +}; + /** * @brief * @@ -253,6 +260,13 @@ class Hal { void startWebSocketAvatarService(std::function onStartLog); + /* -------------------------------- Reefwatch ------------------------------- */ + uitk::Signal onReefwatchFrame; + + void startReefwatchService(); + void setReefwatchLiveCamera(std::string_view camera); // empty to stop the live view + std::string getReefwatchLiveCamera(); + /* ----------------------------------- IMU ---------------------------------- */ uitk::Signal onImuMotionEvent; diff --git a/firmware/main/hal/hal_mcp.cpp b/firmware/main/hal/hal_mcp.cpp index 0d7385171..2bf11242c 100644 --- a/firmware/main/hal/hal_mcp.cpp +++ b/firmware/main/hal/hal_mcp.cpp @@ -8,6 +8,7 @@ #include #include #include +#include using namespace stackchan; @@ -146,4 +147,26 @@ void Hal::xiaozhi_mcp_init() tools::stop_reminder(id); return true; }); + +#if CONFIG_STACKCHAN_REEFWATCH_ENABLE + mclog::tagInfo(_tag, "add cameras.show_live tool"); + mcp_server.AddTool("self.cameras.show_live", + "Show a home security camera live on your screen for up to 5 minutes. " + "Use when the user asks to see or check a camera. Camera names are like 'front' or 'garden'.", + PropertyList({Property("camera", kPropertyTypeString, std::string("front"))}), + [this](const PropertyList& properties) -> ReturnValue { + auto camera = properties["camera"].value(); + mclog::tagInfo(_tag, "cameras.show_live: {}", camera); + setReefwatchLiveCamera(camera); + return true; + }); + + mclog::tagInfo(_tag, "add cameras.stop_live tool"); + mcp_server.AddTool("self.cameras.stop_live", "Stop showing the live camera view.", std::vector{}, + [this](const PropertyList& properties) -> ReturnValue { + mclog::tagInfo(_tag, "cameras.stop_live"); + setReefwatchLiveCamera(""); + return true; + }); +#endif } diff --git a/firmware/main/hal/hal_reefwatch.cpp b/firmware/main/hal/hal_reefwatch.cpp new file mode 100644 index 000000000..c2bbcbc2a --- /dev/null +++ b/firmware/main/hal/hal_reefwatch.cpp @@ -0,0 +1,196 @@ +/* + * SPDX-License-Identifier: MIT + * + * LAN endpoint for the reefwatch camera detector (github.com/adampog/reefwatch): + * + * POST /frame JPEG body (<= 320x240), headers X-Reefwatch-Token, X-Camera, X-Kind (detection|live), X-Labels + * GET /live {"camera": "front" | null, "fps": 5}, polled by reefwatch to know what to stream live + */ +#include "hal.h" +#include "utils/jpeg_to_image/jpeg_decoder.h" +#include +#include +#include +#include +#include +#include +#include +#include + +static const std::string_view _tag = "HAL-Reefwatch"; + +static std::mutex _live_mutex; +static std::string _live_camera; +static uint32_t _live_started_at = 0; + +// Live view switches itself off so a forgotten stream doesn't run all night +static constexpr uint32_t _live_max_ms = 5 * 60 * 1000; + +void Hal::setReefwatchLiveCamera(std::string_view camera) +{ + // Camera names end up in JSON and on screen, so keep them plain + std::string name; + for (char c : camera.substr(0, 32)) { + if (isalnum((unsigned char)c) || c == '-' || c == '_') { + name += c; + } + } + + std::lock_guard lock(_live_mutex); + if (name != _live_camera) { + mclog::tagInfo(_tag, "live view: {}", name.empty() ? "off" : name); + } + _live_camera = name; + _live_started_at = millis(); +} + +std::string Hal::getReefwatchLiveCamera() +{ + std::lock_guard lock(_live_mutex); + if (!_live_camera.empty() && millis() - _live_started_at > _live_max_ms) { + mclog::tagInfo(_tag, "live view timed out"); + _live_camera.clear(); + } + return _live_camera; +} + +#if CONFIG_STACKCHAN_REEFWATCH_ENABLE + +static constexpr size_t _max_jpeg_size = 96 * 1024; + +static std::string get_header(httpd_req_t* req, const char* field, size_t maxLen = 128) +{ + size_t len = httpd_req_get_hdr_value_len(req, field); + if (len == 0) { + return {}; + } + std::string value(std::min(len, maxLen) + 1, '\0'); + httpd_req_get_hdr_value_str(req, field, value.data(), value.size()); // truncates past maxLen + value.resize(strlen(value.c_str())); + return value; +} + +static bool check_token(httpd_req_t* req) +{ + const char* expected = CONFIG_STACKCHAN_REEFWATCH_TOKEN; + if (expected[0] == '\0' || get_header(req, "X-Reefwatch-Token") == expected) { + return true; + } + httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "bad token"); + return false; +} + +static esp_err_t send_no_content(httpd_req_t* req) +{ + httpd_resp_set_status(req, "204 No Content"); + return httpd_resp_send(req, nullptr, 0); +} + +static esp_err_t frame_handler(httpd_req_t* req) +{ + if (!check_token(req)) { + return ESP_OK; + } + + size_t len = req->content_len; + if (len == 0 || len > _max_jpeg_size) { + return httpd_resp_send_err(req, HTTPD_413_CONTENT_TOO_LARGE, "jpeg must be 1 byte to 96 KB"); + } + + std::unique_ptr jpeg( + (uint8_t*)heap_caps_malloc(len, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT), heap_caps_free); + if (!jpeg) { + return httpd_resp_send_err(req, HTTPD_500_INTERNAL_SERVER_ERROR, "out of memory"); + } + + size_t received = 0; + while (received < len) { + int ret = httpd_req_recv(req, (char*)jpeg.get() + received, len - received); + if (ret == HTTPD_SOCK_ERR_TIMEOUT) { + continue; + } + if (ret <= 0) { + return ESP_FAIL; + } + received += ret; + } + + ReefwatchFrame_t frame; + frame.camera = get_header(req, "X-Camera", 32); + frame.labels = get_header(req, "X-Labels"); + frame.isLive = get_header(req, "X-Kind", 16) == "live"; + + // A live frame that was already in flight when the live view was switched off + if (frame.isLive && frame.camera != GetHAL().getReefwatchLiveCamera()) { + return send_no_content(req); + } + + frame.image = jpeg_dec::decode_to_lvgl(jpeg.get(), len); + if (!frame.image) { + return httpd_resp_send_err(req, HTTPD_400_BAD_REQUEST, "could not decode jpeg"); + } + + GetHAL().onReefwatchFrame.emit(frame); + return send_no_content(req); +} + +static esp_err_t live_handler(httpd_req_t* req) +{ + if (!check_token(req)) { + return ESP_OK; + } + + auto camera = GetHAL().getReefwatchLiveCamera(); + auto body = fmt::format(R"({{"camera": {}, "fps": {}}})", + camera.empty() ? "null" : fmt::format("\"{}\"", camera), + CONFIG_STACKCHAN_REEFWATCH_LIVE_FPS); + httpd_resp_set_type(req, "application/json"); + return httpd_resp_send(req, body.data(), body.size()); +} + +static void start_server_task(void* param) +{ + // The server binds to any address, but waiting for Wi-Fi gives a useful log line + while (!WifiManager::GetInstance().IsConnected()) { + vTaskDelay(pdMS_TO_TICKS(1000)); + } + + httpd_config_t config = HTTPD_DEFAULT_CONFIG(); + config.server_port = CONFIG_STACKCHAN_REEFWATCH_PORT; + config.ctrl_port = ESP_HTTPD_DEF_CTRL_PORT + 7; // stay clear of the Wi-Fi config portal's server + config.stack_size = 8192; // JPEG decoding runs on this task + config.task_caps = MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT; // internal SRAM is tight + config.max_open_sockets = 3; + config.lru_purge_enable = true; + + httpd_handle_t server = nullptr; + if (httpd_start(&server, &config) != ESP_OK) { + mclog::tagError(_tag, "failed to start http server on port {}", config.server_port); + vTaskDelete(nullptr); + return; + } + + httpd_uri_t frame_uri = {.uri = "/frame", .method = HTTP_POST, .handler = frame_handler, .user_ctx = nullptr}; + httpd_uri_t live_uri = {.uri = "/live", .method = HTTP_GET, .handler = live_handler, .user_ctx = nullptr}; + httpd_register_uri_handler(server, &frame_uri); + httpd_register_uri_handler(server, &live_uri); + + mclog::tagInfo(_tag, "listening on http://{}:{}", WifiManager::GetInstance().GetIpAddress(), config.server_port); + if (CONFIG_STACKCHAN_REEFWATCH_TOKEN[0] == '\0') { + mclog::tagWarn(_tag, "no token set, any device on the LAN can post frames"); + } + vTaskDelete(nullptr); +} + +void Hal::startReefwatchService() +{ + xTaskCreate(start_server_task, "reefwatch", 4096, nullptr, 3, nullptr); +} + +#else + +void Hal::startReefwatchService() +{ +} + +#endif diff --git a/firmware/main/main.cpp b/firmware/main/main.cpp index 3f11ed65e..f056f9d47 100644 --- a/firmware/main/main.cpp +++ b/firmware/main/main.cpp @@ -9,6 +9,7 @@ #include #include #include +#include using namespace mooncake; using namespace smooth_ui_toolkit; @@ -22,6 +23,10 @@ extern "C" void app_main(void) // HAL init GetHAL().init(); + // Camera frames from reefwatch, shown over any app and in AI agent mode + view::reefwatch_view_init(); + GetHAL().startReefwatchService(); + // Setup ui hal ui_hal::on_delay([](uint32_t ms) { GetHAL().delay(ms); }); ui_hal::on_get_tick([]() { return GetHAL().millis(); }); From f046e58219413f75dd86b1f31f9f4e0365e0f5f3 Mon Sep 17 00:00:00 2001 From: adampog <2230698+adampog@users.noreply.github.com> Date: Thu, 1 Oct 2026 00:20:14 -0700 Subject: [PATCH 2/2] Serve the robot's MCP tools to LAN agents like ESP-Claw The reefwatch HTTP server becomes a shared LAN server (hal_lan_server) that also hosts a JSON-RPC MCP endpoint (POST /mcp_server: initialize, ping, tools/list with cursor paging, tools/call) and advertises it via mDNS as _mcp._tcp, the way ESP-Claw's mcp_client discovers devices. The robot also answers as stackchan.local. StackChan's tools are now McpTool objects shared between xiaozhi and the LAN registry, so the voice agent and LAN agents see the same set. Adds self.robot.set_expression and self.robot.show_message for agents that can't send xiaozhi's emotion messages. dependencies.lock only gains espressif/mdns 1.13.1; every other component stays at its previously locked version. Co-Authored-By: Claude Opus 5.5 --- firmware/README.md | 23 ++- firmware/dependencies.lock | 13 +- firmware/main/Kconfig.projbuild | 41 +++++- firmware/main/hal/hal.h | 4 +- firmware/main/hal/hal_lan_mcp.cpp | 209 +++++++++++++++++++++++++++ firmware/main/hal/hal_lan_server.cpp | 73 ++++++++++ firmware/main/hal/hal_lan_server.h | 21 +++ firmware/main/hal/hal_mcp.cpp | 68 ++++++++- firmware/main/hal/hal_reefwatch.cpp | 42 +----- firmware/main/idf_component.yml | 1 + firmware/main/main.cpp | 2 +- 11 files changed, 438 insertions(+), 59 deletions(-) create mode 100644 firmware/main/hal/hal_lan_mcp.cpp create mode 100644 firmware/main/hal/hal_lan_server.cpp create mode 100644 firmware/main/hal/hal_lan_server.h diff --git a/firmware/README.md b/firmware/README.md index a1fd9988a..89bbef089 100644 --- a/firmware/README.md +++ b/firmware/README.md @@ -49,8 +49,27 @@ CONFIG_STACKCHAN_REEFWATCH_TOKEN="change-me" CONFIG_STACKCHAN_REEFWATCH_CAMERA_YAW="front=-45,garden=45" ``` -and point reefwatch at the robot with `REEFWATCH_STACKCHAN_URL=http://` and the same -`REEFWATCH_STACKCHAN_TOKEN`. The robot logs its address on boot (`listening on http://...`). +and point reefwatch at the robot with `REEFWATCH_STACKCHAN_URL=http://stackchan.local` (or its IP) and the +same `REEFWATCH_STACKCHAN_TOKEN`. The robot logs its address on boot (`listening on http://...`). Protocol: `POST /frame` (JPEG ≤ 320×240, headers `X-Reefwatch-Token`, `X-Camera`, `X-Kind`, `X-Labels`) and `GET /live` → `{"camera": "front" | null, "fps": 5}`. Code: `main/hal/hal_reefwatch.cpp`, `main/apps/common/reefwatch_view/`. + +## LAN MCP server (adampog fork) + +The robot's own MCP tools (`main/hal/hal_mcp.cpp`) are also served to agents on the LAN, such as +[ESP-Claw](https://github.com/adampog/ESP-Claw) running on another ESP32-S3: JSON-RPC over +`POST http://stackchan.local/mcp_server` (`initialize`, `ping`, `tools/list` with cursor paging, +`tools/call`), advertised over mDNS as `_mcp._tcp` with TXT `endpoint=mcp_server`, which is what +ESP-Claw's `mcp_client` discovers. Tools: head angles, LED color, reminders, live camera view, and +`self.robot.set_expression` / `self.robot.show_message` for agents that can't drive the voice +pipeline's emotions. The same tool objects are registered with xiaozhi, so the voice agent sees them too. + +```bash +curl -s http://stackchan.local/mcp_server -d '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' +curl -s http://stackchan.local/mcp_server -d '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"self.robot.set_expression","arguments":{"expression":"happy"}}}' +``` + +There is no authentication: anything on the LAN can move the head. Turn it off with +`CONFIG_STACKCHAN_LAN_MCP_ENABLE=n`; port, hostname and endpoint are in the "LAN Services" Kconfig menu. +Code: `main/hal/hal_lan_server.cpp`, `main/hal/hal_lan_mcp.cpp`. diff --git a/firmware/dependencies.lock b/firmware/dependencies.lock index 325d3982c..afbbdbe7c 100644 --- a/firmware/dependencies.lock +++ b/firmware/dependencies.lock @@ -739,6 +739,16 @@ dependencies: registry_url: https://components.espressif.com/ type: service version: 3.0.3 + espressif/mdns: + component_hash: b679eafd0acae2066e2645bd91d073e33ca5be515e2545cd3c4e55a3e3bce3cb + dependencies: + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com/ + type: service + version: 1.13.1 espressif/usb_host_uvc: component_hash: b0cb9a3a2eaa09dc6e5cc3e47e04d0a73e47c2d3dcfb3afce959224a1fe5fb5e dependencies: @@ -901,6 +911,7 @@ direct_dependencies: - espressif/iot_usbh_rndis - espressif/knob - espressif/led_strip +- espressif/mdns - espressif2022/image_player - idf - lvgl/lvgl @@ -910,6 +921,6 @@ direct_dependencies: - waveshare/esp_lcd_sh8601 - waveshare/esp_lcd_touch_cst9217 - wvirgil123/sscma_client -manifest_hash: 350e17991f1a3226438480b027525b7cde67c34ed4ec6c5847cdfcb22e18dc4a +manifest_hash: 0acfddd452c73d46c5426d1c4a41c483ac5f876c18cbc69ca5b7fc4da0233cf8 target: esp32s3 version: 2.0.0 diff --git a/firmware/main/Kconfig.projbuild b/firmware/main/Kconfig.projbuild index dd97dead4..724736494 100644 --- a/firmware/main/Kconfig.projbuild +++ b/firmware/main/Kconfig.projbuild @@ -11,22 +11,49 @@ config STACKCHAN_SERVER_URL endmenu +menu "LAN Services" + +config STACKCHAN_LAN_SERVER_PORT + int "HTTP port" + default 80 + help + One small HTTP server on the local network hosts the services below. + +config STACKCHAN_LAN_HOSTNAME + string "mDNS hostname" + default "stackchan" + help + The robot answers as .local on the LAN. + +config STACKCHAN_LAN_MCP_ENABLE + bool "MCP server for LAN agents (e.g. ESP-Claw)" + default y + help + Exposes the robot's own MCP tools (head, LEDs, reminders, camera view) + as JSON-RPC over HTTP POST, and advertises them via mDNS as _mcp._tcp + so ESP-Claw's mcp_client can discover them. There is no + authentication: anything on the LAN can move the head. + +config STACKCHAN_LAN_MCP_ENDPOINT + string "MCP endpoint path" + default "mcp_server" + depends on STACKCHAN_LAN_MCP_ENABLE + help + Served at http:///. "mcp_server" is ESP-Claw's default. + +endmenu + menu "Reefwatch Camera View" config STACKCHAN_REEFWATCH_ENABLE bool "Show reefwatch camera frames on screen" default y help - Runs a small HTTP server on the LAN that accepts JPEG frames from the - reefwatch detector (github.com/adampog/reefwatch) and shows them + Accepts JPEG frames from the reefwatch detector + (github.com/adampog/reefwatch) on the LAN server and shows them full screen. POST /frame takes the JPEG, GET /live tells reefwatch which camera to stream live. -config STACKCHAN_REEFWATCH_PORT - int "HTTP port" - default 80 - depends on STACKCHAN_REEFWATCH_ENABLE - config STACKCHAN_REEFWATCH_TOKEN string "Shared token (X-Reefwatch-Token header)" default "" diff --git a/firmware/main/hal/hal.h b/firmware/main/hal/hal.h index 1363fb3af..851d48c50 100644 --- a/firmware/main/hal/hal.h +++ b/firmware/main/hal/hal.h @@ -260,10 +260,12 @@ class Hal { void startWebSocketAvatarService(std::function onStartLog); + /* ------------------------------- LAN server ------------------------------- */ + void startLanServer(); // reefwatch frames + MCP for LAN agents, once Wi-Fi is up + /* -------------------------------- Reefwatch ------------------------------- */ uitk::Signal onReefwatchFrame; - void startReefwatchService(); void setReefwatchLiveCamera(std::string_view camera); // empty to stop the live view std::string getReefwatchLiveCamera(); diff --git a/firmware/main/hal/hal_lan_mcp.cpp b/firmware/main/hal/hal_lan_mcp.cpp new file mode 100644 index 000000000..931334cf6 --- /dev/null +++ b/firmware/main/hal/hal_lan_mcp.cpp @@ -0,0 +1,209 @@ +/* + * SPDX-License-Identifier: MIT + * + * The robot's own MCP tools (hal_mcp.cpp) served to LAN agents such as ESP-Claw as JSON-RPC over + * HTTP POST: initialize, ping, tools/list (paged by cursor) and tools/call. The same McpTool objects + * are registered with xiaozhi, so the voice agent and LAN agents see identical tools. + */ +#include "hal.h" +#include "hal_lan_server.h" +#include +#include +#include +#include +#include +#include +#include + +static std::vector _tools; + +void lan_mcp_add_tool(McpTool* tool) +{ + _tools.push_back(tool); +} + +#if CONFIG_STACKCHAN_LAN_MCP_ENABLE + +static const std::string_view _tag = "HAL-LAN-MCP"; + +static constexpr size_t _max_request_size = 8 * 1024; +static constexpr size_t _max_list_size = 6 * 1024; // ESP-Claw reads responses into an 8 KB buffer + +static cJSON* make_initialize_result(const cJSON* params) +{ + auto requested = cJSON_GetObjectItem(params, "protocolVersion"); + + cJSON* result = cJSON_CreateObject(); + cJSON_AddStringToObject(result, "protocolVersion", + cJSON_IsString(requested) ? requested->valuestring : "2024-11-05"); + cJSON_AddItemToObject(cJSON_AddObjectToObject(result, "capabilities"), "tools", cJSON_CreateObject()); + cJSON* info = cJSON_AddObjectToObject(result, "serverInfo"); + cJSON_AddStringToObject(info, "name", "StackChan"); + cJSON_AddStringToObject(info, "version", FIRMWARE_VERSION); + return result; +} + +// Same paging as xiaozhi: nextCursor is the name of the first tool that didn't fit +static cJSON* make_tools_list_result(const cJSON* params) +{ + auto cursor_json = cJSON_GetObjectItem(params, "cursor"); + std::string cursor = cJSON_IsString(cursor_json) ? cursor_json->valuestring : ""; + + cJSON* result = cJSON_CreateObject(); + cJSON* tools = cJSON_AddArrayToObject(result, "tools"); + size_t size = 0; + bool started = cursor.empty(); + for (auto* tool : _tools) { + if (!started && tool->name() != cursor) { + continue; + } + started = true; + + auto json = tool->to_json(); + if (size + json.size() > _max_list_size && cJSON_GetArraySize(tools) > 0) { + cJSON_AddStringToObject(result, "nextCursor", tool->name().c_str()); + break; + } + size += json.size(); + cJSON_AddItemToArray(tools, cJSON_Parse(json.c_str())); + } + return result; +} + +// Mirrors McpServer::DoToolCall's argument handling, but runs the tool on the calling task +static cJSON* call_tool(const cJSON* params, std::string& error) +{ + auto name = cJSON_GetObjectItem(params, "name"); + if (!cJSON_IsString(name)) { + error = "Missing tool name"; + return nullptr; + } + auto tool = std::find_if(_tools.begin(), _tools.end(), + [name](const McpTool* t) { return t->name() == name->valuestring; }); + if (tool == _tools.end()) { + error = std::string("Unknown tool: ") + name->valuestring; + return nullptr; + } + + auto tool_arguments = cJSON_GetObjectItem(params, "arguments"); + PropertyList arguments = (*tool)->properties(); + try { + for (auto& argument : arguments) { + auto value = cJSON_IsObject(tool_arguments) + ? cJSON_GetObjectItem(tool_arguments, argument.name().c_str()) + : nullptr; + bool found = true; + if (argument.type() == kPropertyTypeBoolean && cJSON_IsBool(value)) { + argument.set_value(cJSON_IsTrue(value)); + } else if (argument.type() == kPropertyTypeInteger && cJSON_IsNumber(value)) { + argument.set_value(value->valueint); + } else if (argument.type() == kPropertyTypeString && cJSON_IsString(value)) { + argument.set_value(value->valuestring); + } else { + found = false; + } + if (!found && !argument.has_default_value()) { + error = "Missing valid argument: " + argument.name(); + return nullptr; + } + } + + mclog::tagInfo(_tag, "tools/call: {}", (*tool)->name()); + return cJSON_Parse((*tool)->Call(arguments).c_str()); + } catch (const std::exception& e) { + error = e.what(); + return nullptr; + } +} + +static esp_err_t send_json(httpd_req_t* req, cJSON* response) +{ + char* body = cJSON_PrintUnformatted(response); + cJSON_Delete(response); + httpd_resp_set_type(req, "application/json"); + esp_err_t err = httpd_resp_sendstr(req, body ? body : "{}"); + cJSON_free(body); + return err; +} + +static esp_err_t send_rpc_error(httpd_req_t* req, const cJSON* id, int code, const std::string& message) +{ + cJSON* response = cJSON_CreateObject(); + cJSON_AddStringToObject(response, "jsonrpc", "2.0"); + cJSON_AddItemToObject(response, "id", id ? cJSON_Duplicate(id, true) : cJSON_CreateNull()); + cJSON* error = cJSON_AddObjectToObject(response, "error"); + cJSON_AddNumberToObject(error, "code", code); + cJSON_AddStringToObject(error, "message", message.c_str()); + return send_json(req, response); +} + +static esp_err_t mcp_handler(httpd_req_t* req) +{ + size_t len = req->content_len; + if (len == 0 || len > _max_request_size) { + return send_rpc_error(req, nullptr, -32600, "Request must be 1 byte to 8 KB"); + } + + std::string body(len, '\0'); + size_t received = 0; + while (received < len) { + int ret = httpd_req_recv(req, body.data() + received, len - received); + if (ret == HTTPD_SOCK_ERR_TIMEOUT) { + continue; + } + if (ret <= 0) { + return ESP_FAIL; + } + received += ret; + } + + std::unique_ptr request(cJSON_Parse(body.c_str()), cJSON_Delete); + auto method = cJSON_GetObjectItem(request.get(), "method"); + if (!request || !cJSON_IsString(method)) { + return send_rpc_error(req, nullptr, -32700, "Parse error"); + } + + // Notifications (no id) get no JSON-RPC response + auto id = cJSON_GetObjectItem(request.get(), "id"); + if (!id) { + httpd_resp_set_status(req, "202 Accepted"); + return httpd_resp_send(req, nullptr, 0); + } + + std::string name = method->valuestring; + auto params = cJSON_GetObjectItem(request.get(), "params"); + cJSON* result = nullptr; + std::string error; + + if (name == "initialize") { + result = make_initialize_result(params); + } else if (name == "ping") { + result = cJSON_CreateObject(); + } else if (name == "tools/list") { + result = make_tools_list_result(params); + } else if (name == "tools/call") { + result = call_tool(params, error); + } else { + return send_rpc_error(req, id, -32601, "Method not found: " + name); + } + if (!result) { + mclog::tagWarn(_tag, "{}: {}", name, error); + return send_rpc_error(req, id, -32602, error); + } + + cJSON* response = cJSON_CreateObject(); + cJSON_AddStringToObject(response, "jsonrpc", "2.0"); + cJSON_AddItemToObject(response, "id", cJSON_Duplicate(id, true)); + cJSON_AddItemToObject(response, "result", result); + return send_json(req, response); +} + +void lan_mcp_register_uris(httpd_handle_t server) +{ + static const std::string uri = "/" CONFIG_STACKCHAN_LAN_MCP_ENDPOINT; + httpd_uri_t mcp_uri = {.uri = uri.c_str(), .method = HTTP_POST, .handler = mcp_handler, .user_ctx = nullptr}; + httpd_register_uri_handler(server, &mcp_uri); + mclog::tagInfo(_tag, "serving {} tools at {}", _tools.size(), uri); +} + +#endif diff --git a/firmware/main/hal/hal_lan_server.cpp b/firmware/main/hal/hal_lan_server.cpp new file mode 100644 index 000000000..d8ada18a8 --- /dev/null +++ b/firmware/main/hal/hal_lan_server.cpp @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: MIT + * + * One HTTP server on the LAN for reefwatch frames and the MCP endpoint, plus an mDNS name + * (stackchan.local) and an _mcp._tcp advertisement so ESP-Claw can discover the robot. + */ +#include "hal.h" +#include "hal_lan_server.h" +#include +#include +#include +#include +#include + +static const std::string_view _tag = "HAL-LAN"; + +static void start_mdns(uint16_t port) +{ + if (mdns_init() != ESP_OK) { + mclog::tagError(_tag, "mdns init failed"); + return; + } + mdns_hostname_set(CONFIG_STACKCHAN_LAN_HOSTNAME); + mdns_instance_name_set("StackChan"); + mdns_service_add(nullptr, "_http", "_tcp", port, nullptr, 0); + +#if CONFIG_STACKCHAN_LAN_MCP_ENABLE + mdns_txt_item_t txt[] = {{"endpoint", CONFIG_STACKCHAN_LAN_MCP_ENDPOINT}}; + mdns_service_add(nullptr, "_mcp", "_tcp", port, txt, sizeof(txt) / sizeof(txt[0])); +#endif +} + +static void start_server_task(void* param) +{ + // The server binds to any address, but waiting for Wi-Fi gives a useful log line and lets mDNS start + while (!WifiManager::GetInstance().IsConnected()) { + vTaskDelay(pdMS_TO_TICKS(1000)); + } + + httpd_config_t config = HTTPD_DEFAULT_CONFIG(); + config.server_port = CONFIG_STACKCHAN_LAN_SERVER_PORT; + config.ctrl_port = ESP_HTTPD_DEF_CTRL_PORT + 7; // stay clear of the Wi-Fi config portal's server + config.stack_size = 8192; // JPEG decoding and MCP tool calls run on this task + config.task_caps = MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT; // internal SRAM is tight; no handler writes flash + config.max_open_sockets = 4; + config.lru_purge_enable = true; + + httpd_handle_t server = nullptr; + if (httpd_start(&server, &config) != ESP_OK) { + mclog::tagError(_tag, "failed to start http server on port {}", config.server_port); + vTaskDelete(nullptr); + return; + } + +#if CONFIG_STACKCHAN_REEFWATCH_ENABLE + reefwatch_register_uris(server); +#endif +#if CONFIG_STACKCHAN_LAN_MCP_ENABLE + lan_mcp_register_uris(server); +#endif + start_mdns(config.server_port); + + mclog::tagInfo(_tag, "listening on http://{}:{} (http://{}.local)", WifiManager::GetInstance().GetIpAddress(), + config.server_port, CONFIG_STACKCHAN_LAN_HOSTNAME); + vTaskDelete(nullptr); +} + +void Hal::startLanServer() +{ +#if CONFIG_STACKCHAN_REEFWATCH_ENABLE || CONFIG_STACKCHAN_LAN_MCP_ENABLE + xTaskCreate(start_server_task, "lan_server", 4096, nullptr, 3, nullptr); +#endif +} diff --git a/firmware/main/hal/hal_lan_server.h b/firmware/main/hal/hal_lan_server.h new file mode 100644 index 000000000..28b108951 --- /dev/null +++ b/firmware/main/hal/hal_lan_server.h @@ -0,0 +1,21 @@ +/* + * SPDX-License-Identifier: MIT + * + * Services hosted on the robot's LAN HTTP server (hal_lan_server.cpp). + */ +#pragma once +#include +#include + +class McpTool; + +#if CONFIG_STACKCHAN_REEFWATCH_ENABLE +void reefwatch_register_uris(httpd_handle_t server); +#endif + +// Tools added here are also served to LAN agents when CONFIG_STACKCHAN_LAN_MCP_ENABLE is set +void lan_mcp_add_tool(McpTool* tool); + +#if CONFIG_STACKCHAN_LAN_MCP_ENABLE +void lan_mcp_register_uris(httpd_handle_t server); +#endif diff --git a/firmware/main/hal/hal_mcp.cpp b/firmware/main/hal/hal_mcp.cpp index 2bf11242c..4e5555d80 100644 --- a/firmware/main/hal/hal_mcp.cpp +++ b/firmware/main/hal/hal_mcp.cpp @@ -4,6 +4,8 @@ * SPDX-License-Identifier: MIT */ #include "hal.h" +#include "hal_lan_server.h" +#include #include #include #include @@ -21,12 +23,21 @@ void Hal::xiaozhi_mcp_init() // https://github.com/78/xiaozhi-esp32/blob/main/docs/mcp-usage.md auto& mcp_server = McpServer::GetInstance(); + // Register each tool with xiaozhi's voice agent and with the LAN MCP server (hal_lan_mcp.cpp) + auto add_tool = [&mcp_server](const std::string& name, const std::string& description, + const PropertyList& properties, + std::function callback) { + auto tool = new McpTool(name, description, properties, callback); + mcp_server.AddTool(tool); + lan_mcp_add_tool(tool); + }; + // System Prompt： // You can control the robot's head. Use get_yaw and get_pitch to sense current position. Use set_yaw for horizontal // movement and set_pitch for vertical movement. All angles are in degrees. mclog::tagInfo(_tag, "add robot.get_head_angles tool"); - mcp_server.AddTool("self.robot.get_head_angles", + add_tool("self.robot.get_head_angles", "Returns current yaw/pitch in degrees. Neutral position is {yaw:0, pitch:0}.", std::vector{}, [this](const PropertyList& properties) -> ReturnValue { LvglLockGuard lock; // StackChan motion update is under the lvgl lock @@ -41,7 +52,7 @@ void Hal::xiaozhi_mcp_init() }); mclog::tagInfo(_tag, "add robot.set_head_angles tool"); - mcp_server.AddTool("self.robot.set_head_angles", + add_tool("self.robot.set_head_angles", "Adjust head position. GUIDELINES: " "1. For natural interaction, stay within +/- 45 degrees. " "2. Only use values > 70 if the user explicitly asks to look far away/behind. " @@ -71,7 +82,7 @@ void Hal::xiaozhi_mcp_init() }); mclog::tagInfo(_tag, "add robot.set_led_color tool"); - mcp_server.AddTool( + add_tool( "self.robot.set_led_color", "Set the color of the robot's INTERNAL onboard LED. This is NOT for room lights. " "Values: 0-168 (safe range). Red=168,0,0; Green=0,168,0; Blue=0,0,168; White=100,100,100; Off=0,0,0.", @@ -94,7 +105,7 @@ void Hal::xiaozhi_mcp_init() }); mclog::tagInfo(_tag, "add robot.create_reminder tool"); - mcp_server.AddTool("self.robot.create_reminder", + add_tool("self.robot.create_reminder", "Create a reminder. Duration is in seconds. Message is what to say when time is up. Set repeat " "to true to repeat the reminder.", PropertyList({Property("duration_seconds", kPropertyTypeInteger, 60, 1, 86400), @@ -119,7 +130,7 @@ void Hal::xiaozhi_mcp_init() }); mclog::tagInfo(_tag, "add robot.get_reminders tool"); - mcp_server.AddTool("self.robot.get_reminders", "Get list of active reminders.", std::vector{}, + add_tool("self.robot.get_reminders", "Get list of active reminders.", std::vector{}, [this](const PropertyList& properties) -> ReturnValue { mclog::tagInfo(_tag, "get_reminders"); auto reminders = tools::get_active_reminders(); @@ -139,7 +150,7 @@ void Hal::xiaozhi_mcp_init() }); mclog::tagInfo(_tag, "add robot.stop_reminder tool"); - mcp_server.AddTool("self.robot.stop_reminder", "Stop a reminder by ID.", + add_tool("self.robot.stop_reminder", "Stop a reminder by ID.", PropertyList({Property("id", kPropertyTypeInteger, -1)}), [this](const PropertyList& properties) -> ReturnValue { int id = properties["id"].value(); @@ -148,9 +159,50 @@ void Hal::xiaozhi_mcp_init() return true; }); + mclog::tagInfo(_tag, "add robot.set_expression tool"); + add_tool("self.robot.set_expression", + "Show a facial expression for a few seconds. " + "Expressions: neutral, happy, angry, sad, doubt, sleepy.", + PropertyList({Property("expression", kPropertyTypeString, std::string("happy")), + Property("seconds", kPropertyTypeInteger, 3, 1, 60)}), + [this](const PropertyList& properties) -> ReturnValue { + static const std::map emotions = { + {"neutral", avatar::Emotion::Neutral}, {"happy", avatar::Emotion::Happy}, + {"angry", avatar::Emotion::Angry}, {"sad", avatar::Emotion::Sad}, + {"doubt", avatar::Emotion::Doubt}, {"sleepy", avatar::Emotion::Sleepy}}; + + auto expression = properties["expression"].value(); + auto emotion = emotions.find(expression); + if (emotion == emotions.end()) { + throw std::invalid_argument("Unknown expression: " + expression); + } + int seconds = properties["seconds"].value(); + mclog::tagInfo(_tag, "set_expression: {} for {}s", expression, seconds); + + LvglLockGuard lock; + GetStackChan().addModifier(std::make_unique(emotion->second, seconds * 1000)); + return true; + }); + + mclog::tagInfo(_tag, "add robot.show_message tool"); + add_tool("self.robot.show_message", + "Show a short message in a speech bubble on your face, e.g. to reply to someone who texted you.", + PropertyList({Property("text", kPropertyTypeString, std::string("")), + Property("seconds", kPropertyTypeInteger, 6, 1, 60)}), + [this](const PropertyList& properties) -> ReturnValue { + auto text = properties["text"].value(); + int seconds = properties["seconds"].value(); + mclog::tagInfo(_tag, "show_message: {}", text); + + LvglLockGuard lock; + GetStackChan().addModifier(std::make_unique(text, seconds * 1000)); + GetStackChan().addModifier(std::make_unique(std::min(seconds, 3) * 1000)); + return true; + }); + #if CONFIG_STACKCHAN_REEFWATCH_ENABLE mclog::tagInfo(_tag, "add cameras.show_live tool"); - mcp_server.AddTool("self.cameras.show_live", + add_tool("self.cameras.show_live", "Show a home security camera live on your screen for up to 5 minutes. " "Use when the user asks to see or check a camera. Camera names are like 'front' or 'garden'.", PropertyList({Property("camera", kPropertyTypeString, std::string("front"))}), @@ -162,7 +214,7 @@ void Hal::xiaozhi_mcp_init() }); mclog::tagInfo(_tag, "add cameras.stop_live tool"); - mcp_server.AddTool("self.cameras.stop_live", "Stop showing the live camera view.", std::vector{}, + add_tool("self.cameras.stop_live", "Stop showing the live camera view.", std::vector{}, [this](const PropertyList& properties) -> ReturnValue { mclog::tagInfo(_tag, "cameras.stop_live"); setReefwatchLiveCamera(""); diff --git a/firmware/main/hal/hal_reefwatch.cpp b/firmware/main/hal/hal_reefwatch.cpp index c2bbcbc2a..30779f825 100644 --- a/firmware/main/hal/hal_reefwatch.cpp +++ b/firmware/main/hal/hal_reefwatch.cpp @@ -7,13 +7,10 @@ * GET /live {"camera": "front" | null, "fps": 5}, polled by reefwatch to know what to stream live */ #include "hal.h" +#include "hal_lan_server.h" #include "utils/jpeg_to_image/jpeg_decoder.h" #include -#include -#include #include -#include -#include #include #include @@ -148,49 +145,16 @@ static esp_err_t live_handler(httpd_req_t* req) return httpd_resp_send(req, body.data(), body.size()); } -static void start_server_task(void* param) +void reefwatch_register_uris(httpd_handle_t server) { - // The server binds to any address, but waiting for Wi-Fi gives a useful log line - while (!WifiManager::GetInstance().IsConnected()) { - vTaskDelay(pdMS_TO_TICKS(1000)); - } - - httpd_config_t config = HTTPD_DEFAULT_CONFIG(); - config.server_port = CONFIG_STACKCHAN_REEFWATCH_PORT; - config.ctrl_port = ESP_HTTPD_DEF_CTRL_PORT + 7; // stay clear of the Wi-Fi config portal's server - config.stack_size = 8192; // JPEG decoding runs on this task - config.task_caps = MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT; // internal SRAM is tight - config.max_open_sockets = 3; - config.lru_purge_enable = true; - - httpd_handle_t server = nullptr; - if (httpd_start(&server, &config) != ESP_OK) { - mclog::tagError(_tag, "failed to start http server on port {}", config.server_port); - vTaskDelete(nullptr); - return; - } - httpd_uri_t frame_uri = {.uri = "/frame", .method = HTTP_POST, .handler = frame_handler, .user_ctx = nullptr}; httpd_uri_t live_uri = {.uri = "/live", .method = HTTP_GET, .handler = live_handler, .user_ctx = nullptr}; httpd_register_uri_handler(server, &frame_uri); httpd_register_uri_handler(server, &live_uri); - mclog::tagInfo(_tag, "listening on http://{}:{}", WifiManager::GetInstance().GetIpAddress(), config.server_port); - if (CONFIG_STACKCHAN_REEFWATCH_TOKEN[0] == '\0') { + if (CONFIG_STACKCHAN_REEFWATCH_TOKEN[0] == 0) { mclog::tagWarn(_tag, "no token set, any device on the LAN can post frames"); } - vTaskDelete(nullptr); -} - -void Hal::startReefwatchService() -{ - xTaskCreate(start_server_task, "reefwatch", 4096, nullptr, 3, nullptr); -} - -#else - -void Hal::startReefwatchService() -{ } #endif diff --git a/firmware/main/idf_component.yml b/firmware/main/idf_component.yml index f124751ca..76a625da3 100644 --- a/firmware/main/idf_component.yml +++ b/firmware/main/idf_component.yml @@ -65,6 +65,7 @@ dependencies: - if: target in [esp32s3] espressif/adc_battery_estimation: ^0.2.0 espressif/esp_new_jpeg: ^0.6.1 + espressif/mdns: ^1.10.1 # SenseCAP Watcher Board wvirgil123/sscma_client: diff --git a/firmware/main/main.cpp b/firmware/main/main.cpp index f056f9d47..afff77429 100644 --- a/firmware/main/main.cpp +++ b/firmware/main/main.cpp @@ -25,7 +25,7 @@ extern "C" void app_main(void) // Camera frames from reefwatch, shown over any app and in AI agent mode view::reefwatch_view_init(); - GetHAL().startReefwatchService(); + GetHAL().startLanServer(); // Setup ui hal ui_hal::on_delay([](uint32_t ms) { GetHAL().delay(ms); });