diff --git a/firmware/README.md b/firmware/README.md index 20a73bf65..89bbef089 100644 --- a/firmware/README.md +++ b/firmware/README.md @@ -32,3 +32,44 @@ ctest --test-dir build-host-tests --output-on-failure ```bash idf.py flash ``` + +## Reefwatch camera view (adampog fork) + +Shows frames from [reefwatch](https://github.com/adampog/reefwatch) (ESP32-CAM + Coral object detection) +full screen on the robot, over any app and in AI agent mode. + +- **Detections** show for 8 s with boxes and labels; LEDs go amber and, if configured, the head turns toward that camera. +- **Live view**: tap a detection to keep watching that camera, tap again to stop. In AI agent mode you can ask + for it by voice (`self.cameras.show_live` / `self.cameras.stop_live` MCP tools). Switches off after 5 min. + +Set a token (and optionally head angles) in `sdkconfig.defaults.local`: + +``` +CONFIG_STACKCHAN_REEFWATCH_TOKEN="change-me" +CONFIG_STACKCHAN_REEFWATCH_CAMERA_YAW="front=-45,garden=45" +``` + +and point reefwatch at the robot with `REEFWATCH_STACKCHAN_URL=http://stackchan.local` (or its IP) and the +same `REEFWATCH_STACKCHAN_TOKEN`. The robot logs its address on boot (`listening on http://...`). +Protocol: `POST /frame` (JPEG ≤ 320×240, headers `X-Reefwatch-Token`, `X-Camera`, `X-Kind`, `X-Labels`) +and `GET /live` → `{"camera": "front" | null, "fps": 5}`. Code: `main/hal/hal_reefwatch.cpp`, +`main/apps/common/reefwatch_view/`. + +## LAN MCP server (adampog fork) + +The robot's own MCP tools (`main/hal/hal_mcp.cpp`) are also served to agents on the LAN, such as +[ESP-Claw](https://github.com/adampog/ESP-Claw) running on another ESP32-S3: JSON-RPC over +`POST http://stackchan.local/mcp_server` (`initialize`, `ping`, `tools/list` with cursor paging, +`tools/call`), advertised over mDNS as `_mcp._tcp` with TXT `endpoint=mcp_server`, which is what +ESP-Claw's `mcp_client` discovers. Tools: head angles, LED color, reminders, live camera view, and +`self.robot.set_expression` / `self.robot.show_message` for agents that can't drive the voice +pipeline's emotions. The same tool objects are registered with xiaozhi, so the voice agent sees them too. + +```bash +curl -s http://stackchan.local/mcp_server -d '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' +curl -s http://stackchan.local/mcp_server -d '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"self.robot.set_expression","arguments":{"expression":"happy"}}}' +``` + +There is no authentication: anything on the LAN can move the head. Turn it off with +`CONFIG_STACKCHAN_LAN_MCP_ENABLE=n`; port, hostname and endpoint are in the "LAN Services" Kconfig menu. +Code: `main/hal/hal_lan_server.cpp`, `main/hal/hal_lan_mcp.cpp`. diff --git a/firmware/dependencies.lock b/firmware/dependencies.lock index 325d3982c..afbbdbe7c 100644 --- a/firmware/dependencies.lock +++ b/firmware/dependencies.lock @@ -739,6 +739,16 @@ dependencies: registry_url: https://components.espressif.com/ type: service version: 3.0.3 + espressif/mdns: + component_hash: b679eafd0acae2066e2645bd91d073e33ca5be515e2545cd3c4e55a3e3bce3cb + dependencies: + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com/ + type: service + version: 1.13.1 espressif/usb_host_uvc: component_hash: b0cb9a3a2eaa09dc6e5cc3e47e04d0a73e47c2d3dcfb3afce959224a1fe5fb5e dependencies: @@ -901,6 +911,7 @@ direct_dependencies: - espressif/iot_usbh_rndis - espressif/knob - espressif/led_strip +- espressif/mdns - espressif2022/image_player - idf - lvgl/lvgl @@ -910,6 +921,6 @@ direct_dependencies: - waveshare/esp_lcd_sh8601 - waveshare/esp_lcd_touch_cst9217 - wvirgil123/sscma_client -manifest_hash: 350e17991f1a3226438480b027525b7cde67c34ed4ec6c5847cdfcb22e18dc4a +manifest_hash: 0acfddd452c73d46c5426d1c4a41c483ac5f876c18cbc69ca5b7fc4da0233cf8 target: esp32s3 version: 2.0.0 diff --git a/firmware/main/CMakeLists.txt b/firmware/main/CMakeLists.txt index 22cfebb14..ade1449c5 100644 --- a/firmware/main/CMakeLists.txt +++ b/firmware/main/CMakeLists.txt @@ -297,6 +297,7 @@ idf_component_register(SRCS ${SOURCES} esp_pm esp_psram esp_netif + esp_http_server esp_driver_gpio esp_driver_uart esp_driver_spi diff --git a/firmware/main/Kconfig.projbuild b/firmware/main/Kconfig.projbuild index 89592a0c5..724736494 100644 --- a/firmware/main/Kconfig.projbuild +++ b/firmware/main/Kconfig.projbuild @@ -11,6 +11,79 @@ config STACKCHAN_SERVER_URL endmenu +menu "LAN Services" + +config STACKCHAN_LAN_SERVER_PORT + int "HTTP port" + default 80 + help + One small HTTP server on the local network hosts the services below. + +config STACKCHAN_LAN_HOSTNAME + string "mDNS hostname" + default "stackchan" + help + The robot answers as .local on the LAN. + +config STACKCHAN_LAN_MCP_ENABLE + bool "MCP server for LAN agents (e.g. ESP-Claw)" + default y + help + Exposes the robot's own MCP tools (head, LEDs, reminders, camera view) + as JSON-RPC over HTTP POST, and advertises them via mDNS as _mcp._tcp + so ESP-Claw's mcp_client can discover them. There is no + authentication: anything on the LAN can move the head. + +config STACKCHAN_LAN_MCP_ENDPOINT + string "MCP endpoint path" + default "mcp_server" + depends on STACKCHAN_LAN_MCP_ENABLE + help + Served at http:///. "mcp_server" is ESP-Claw's default. + +endmenu + +menu "Reefwatch Camera View" + +config STACKCHAN_REEFWATCH_ENABLE + bool "Show reefwatch camera frames on screen" + default y + help + Accepts JPEG frames from the reefwatch detector + (github.com/adampog/reefwatch) on the LAN server and shows them + full screen. POST /frame takes the JPEG, GET /live tells reefwatch + which camera to stream live. + +config STACKCHAN_REEFWATCH_TOKEN + string "Shared token (X-Reefwatch-Token header)" + default "" + depends on STACKCHAN_REEFWATCH_ENABLE + help + Must match REEFWATCH_STACKCHAN_TOKEN on the reefwatch side. Empty + accepts any request on the LAN. Set it in sdkconfig.defaults.local. + +config STACKCHAN_REEFWATCH_SHOW_MS + int "How long a detection stays on screen (ms)" + default 8000 + depends on STACKCHAN_REEFWATCH_ENABLE + +config STACKCHAN_REEFWATCH_LIVE_FPS + int "Live view frame rate requested from reefwatch" + default 5 + range 1 15 + depends on STACKCHAN_REEFWATCH_ENABLE + +config STACKCHAN_REEFWATCH_CAMERA_YAW + string "Head yaw per camera, in degrees" + default "" + depends on STACKCHAN_REEFWATCH_ENABLE + help + Turns the head toward a camera when it sees something, e.g. + "front=-45,garden=45" (negative is the robot's left). Empty + keeps the head still. + +endmenu + menu "Xiaozhi Assistant" config OTA_URL diff --git a/firmware/main/apps/common/reefwatch_view/reefwatch_view.cpp b/firmware/main/apps/common/reefwatch_view/reefwatch_view.cpp new file mode 100644 index 000000000..b58edadb4 --- /dev/null +++ b/firmware/main/apps/common/reefwatch_view/reefwatch_view.cpp @@ -0,0 +1,174 @@ +/* + * SPDX-License-Identifier: MIT + */ +#include "reefwatch_view.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if CONFIG_STACKCHAN_REEFWATCH_ENABLE + +static const std::string_view _tag = "reefwatch-view"; + +// Live frames keep the view open; if they stop arriving, close after this long +static constexpr uint32_t _live_idle_ms = 3000; + +static lv_obj_t* _panel = nullptr; +static lv_obj_t* _image = nullptr; +static lv_obj_t* _caption = nullptr; +static lv_timer_t* _timer = nullptr; +static std::shared_ptr _frame; // keeps the pixels alive while LVGL draws them +static std::string _camera; +static bool _detection_on_screen = false; // blocks live frames until it times out or is tapped +static bool _reacted = false; // LEDs changed, put them back on hide +static bool _head_turned = false; +static std::string _reacted_camera; +static std::map _camera_yaw; + +// "front=-45,garden=45" -> {front: -45, garden: 45} +static void parse_camera_yaw() +{ + std::istringstream list(CONFIG_STACKCHAN_REEFWATCH_CAMERA_YAW); + std::string item; + while (std::getline(list, item, ',')) { + auto eq = item.find('='); + if (eq != std::string::npos) { + _camera_yaw[item.substr(0, eq)] = std::atoi(item.c_str() + eq + 1); + } + } +} + +static void hide() +{ + lv_obj_add_flag(_panel, LV_OBJ_FLAG_HIDDEN); + lv_timer_pause(_timer); + lv_image_set_src(_image, nullptr); + _frame.reset(); + + _detection_on_screen = false; + if (_reacted) { + _reacted = false; + GetStackChan().leftNeonLight().setColor(0, 0, 0); + GetStackChan().rightNeonLight().setColor(0, 0, 0); + } + if (_head_turned) { + _head_turned = false; + GetStackChan().motion().yawServo().moveWithSpeed(0, 300); + } +} + +static void on_timer(lv_timer_t* timer) +{ + hide(); +} + +static void on_click(lv_event_t* e) +{ + // Tap a detection to keep watching that camera, tap the live view to stop it + if (GetHAL().getReefwatchLiveCamera().empty() && !_camera.empty()) { + GetHAL().setReefwatchLiveCamera(_camera); + _detection_on_screen = false; + lv_label_set_text_fmt(_caption, "%s - starting live view...", _camera.c_str()); + lv_timer_set_period(_timer, _live_idle_ms + 2000); // reefwatch polls every 2 s + lv_timer_reset(_timer); + } else { + GetHAL().setReefwatchLiveCamera(""); + hide(); + } +} + +static void react_to_detection(const std::string& camera) +{ + // Amber LEDs, within the 0-168 range hal_mcp treats as safe + GetStackChan().leftNeonLight().setColor(168, 90, 0); + GetStackChan().rightNeonLight().setColor(168, 90, 0); + + auto yaw = _camera_yaw.find(camera); + if (yaw != _camera_yaw.end()) { + GetStackChan().motion().yawServo().moveWithSpeed(yaw->second * 10, 300); + _head_turned = true; + } + _reacted = true; + _reacted_camera = camera; +} + +static void show(const ReefwatchFrame_t& frame) +{ + // A live frame never replaces a detection that is still on screen + if (frame.isLive && _detection_on_screen) { + return; + } + + _frame = frame.image; + _camera = frame.camera; + lv_image_set_src(_image, _frame->image_dsc()); + + if (frame.isLive) { + lv_label_set_text_fmt(_caption, "%s - LIVE", frame.camera.c_str()); + lv_timer_set_period(_timer, _live_idle_ms); + } else { + lv_label_set_text_fmt(_caption, "%s - %s", frame.camera.c_str(), frame.labels.c_str()); + lv_timer_set_period(_timer, CONFIG_STACKCHAN_REEFWATCH_SHOW_MS); + if (!_reacted || frame.camera != _reacted_camera) { + react_to_detection(frame.camera); + } + _detection_on_screen = true; + } + + lv_obj_remove_flag(_panel, LV_OBJ_FLAG_HIDDEN); + lv_timer_reset(_timer); + lv_timer_resume(_timer); +} + +void view::reefwatch_view_init() +{ + parse_camera_yaw(); + + LvglLockGuard lock; + + _panel = lv_obj_create(lv_layer_top()); + lv_obj_remove_style_all(_panel); + lv_obj_set_size(_panel, 320, 240); + lv_obj_set_style_bg_color(_panel, lv_color_black(), 0); + lv_obj_set_style_bg_opa(_panel, LV_OPA_COVER, 0); + lv_obj_remove_flag(_panel, LV_OBJ_FLAG_SCROLLABLE); + lv_obj_add_flag(_panel, LV_OBJ_FLAG_CLICKABLE); + lv_obj_add_flag(_panel, LV_OBJ_FLAG_HIDDEN); + lv_obj_add_event_cb(_panel, on_click, LV_EVENT_CLICKED, nullptr); + + _image = lv_image_create(_panel); + lv_obj_center(_image); + + _caption = lv_label_create(_panel); + lv_obj_set_width(_caption, 320); + lv_obj_align(_caption, LV_ALIGN_BOTTOM_MID, 0, 0); + lv_obj_set_style_bg_color(_caption, lv_color_black(), 0); + lv_obj_set_style_bg_opa(_caption, LV_OPA_60, 0); + lv_obj_set_style_text_color(_caption, lv_color_white(), 0); + lv_obj_set_style_pad_all(_caption, 4, 0); + lv_label_set_long_mode(_caption, LV_LABEL_LONG_DOT); + + _timer = lv_timer_create(on_timer, CONFIG_STACKCHAN_REEFWATCH_SHOW_MS, nullptr); + lv_timer_pause(_timer); + + GetHAL().onReefwatchFrame.connect([](const ReefwatchFrame_t& frame) { + LvglLockGuard lock; + show(frame); + }); + + mclog::tagInfo(_tag, "ready"); +} + +#else + +void view::reefwatch_view_init() +{ +} + +#endif diff --git a/firmware/main/apps/common/reefwatch_view/reefwatch_view.h b/firmware/main/apps/common/reefwatch_view/reefwatch_view.h new file mode 100644 index 000000000..072dd88ed --- /dev/null +++ b/firmware/main/apps/common/reefwatch_view/reefwatch_view.h @@ -0,0 +1,15 @@ +/* + * SPDX-License-Identifier: MIT + */ +#pragma once + +namespace view { + +/** + * @brief Full screen overlay for reefwatch camera frames, on LVGL's top layer so it shows over + * any app and in AI agent mode. Tap a detection to watch that camera live, tap again to stop. + * + */ +void reefwatch_view_init(); + +} // namespace view diff --git a/firmware/main/hal/hal.h b/firmware/main/hal/hal.h index b8d3c80f4..851d48c50 100644 --- a/firmware/main/hal/hal.h +++ b/firmware/main/hal/hal.h @@ -40,6 +40,13 @@ struct WsTextMessage_t { std::string content; }; +struct ReefwatchFrame_t { + std::shared_ptr image; + std::string camera; + std::string labels; + bool isLive = false; +}; + /** * @brief * @@ -253,6 +260,15 @@ class Hal { void startWebSocketAvatarService(std::function onStartLog); + /* ------------------------------- LAN server ------------------------------- */ + void startLanServer(); // reefwatch frames + MCP for LAN agents, once Wi-Fi is up + + /* -------------------------------- Reefwatch ------------------------------- */ + uitk::Signal onReefwatchFrame; + + void setReefwatchLiveCamera(std::string_view camera); // empty to stop the live view + std::string getReefwatchLiveCamera(); + /* ----------------------------------- IMU ---------------------------------- */ uitk::Signal onImuMotionEvent; diff --git a/firmware/main/hal/hal_lan_mcp.cpp b/firmware/main/hal/hal_lan_mcp.cpp new file mode 100644 index 000000000..931334cf6 --- /dev/null +++ b/firmware/main/hal/hal_lan_mcp.cpp @@ -0,0 +1,209 @@ +/* + * SPDX-License-Identifier: MIT + * + * The robot's own MCP tools (hal_mcp.cpp) served to LAN agents such as ESP-Claw as JSON-RPC over + * HTTP POST: initialize, ping, tools/list (paged by cursor) and tools/call. The same McpTool objects + * are registered with xiaozhi, so the voice agent and LAN agents see identical tools. + */ +#include "hal.h" +#include "hal_lan_server.h" +#include +#include +#include +#include +#include +#include +#include + +static std::vector _tools; + +void lan_mcp_add_tool(McpTool* tool) +{ + _tools.push_back(tool); +} + +#if CONFIG_STACKCHAN_LAN_MCP_ENABLE + +static const std::string_view _tag = "HAL-LAN-MCP"; + +static constexpr size_t _max_request_size = 8 * 1024; +static constexpr size_t _max_list_size = 6 * 1024; // ESP-Claw reads responses into an 8 KB buffer + +static cJSON* make_initialize_result(const cJSON* params) +{ + auto requested = cJSON_GetObjectItem(params, "protocolVersion"); + + cJSON* result = cJSON_CreateObject(); + cJSON_AddStringToObject(result, "protocolVersion", + cJSON_IsString(requested) ? requested->valuestring : "2024-11-05"); + cJSON_AddItemToObject(cJSON_AddObjectToObject(result, "capabilities"), "tools", cJSON_CreateObject()); + cJSON* info = cJSON_AddObjectToObject(result, "serverInfo"); + cJSON_AddStringToObject(info, "name", "StackChan"); + cJSON_AddStringToObject(info, "version", FIRMWARE_VERSION); + return result; +} + +// Same paging as xiaozhi: nextCursor is the name of the first tool that didn't fit +static cJSON* make_tools_list_result(const cJSON* params) +{ + auto cursor_json = cJSON_GetObjectItem(params, "cursor"); + std::string cursor = cJSON_IsString(cursor_json) ? cursor_json->valuestring : ""; + + cJSON* result = cJSON_CreateObject(); + cJSON* tools = cJSON_AddArrayToObject(result, "tools"); + size_t size = 0; + bool started = cursor.empty(); + for (auto* tool : _tools) { + if (!started && tool->name() != cursor) { + continue; + } + started = true; + + auto json = tool->to_json(); + if (size + json.size() > _max_list_size && cJSON_GetArraySize(tools) > 0) { + cJSON_AddStringToObject(result, "nextCursor", tool->name().c_str()); + break; + } + size += json.size(); + cJSON_AddItemToArray(tools, cJSON_Parse(json.c_str())); + } + return result; +} + +// Mirrors McpServer::DoToolCall's argument handling, but runs the tool on the calling task +static cJSON* call_tool(const cJSON* params, std::string& error) +{ + auto name = cJSON_GetObjectItem(params, "name"); + if (!cJSON_IsString(name)) { + error = "Missing tool name"; + return nullptr; + } + auto tool = std::find_if(_tools.begin(), _tools.end(), + [name](const McpTool* t) { return t->name() == name->valuestring; }); + if (tool == _tools.end()) { + error = std::string("Unknown tool: ") + name->valuestring; + return nullptr; + } + + auto tool_arguments = cJSON_GetObjectItem(params, "arguments"); + PropertyList arguments = (*tool)->properties(); + try { + for (auto& argument : arguments) { + auto value = cJSON_IsObject(tool_arguments) + ? cJSON_GetObjectItem(tool_arguments, argument.name().c_str()) + : nullptr; + bool found = true; + if (argument.type() == kPropertyTypeBoolean && cJSON_IsBool(value)) { + argument.set_value(cJSON_IsTrue(value)); + } else if (argument.type() == kPropertyTypeInteger && cJSON_IsNumber(value)) { + argument.set_value(value->valueint); + } else if (argument.type() == kPropertyTypeString && cJSON_IsString(value)) { + argument.set_value(value->valuestring); + } else { + found = false; + } + if (!found && !argument.has_default_value()) { + error = "Missing valid argument: " + argument.name(); + return nullptr; + } + } + + mclog::tagInfo(_tag, "tools/call: {}", (*tool)->name()); + return cJSON_Parse((*tool)->Call(arguments).c_str()); + } catch (const std::exception& e) { + error = e.what(); + return nullptr; + } +} + +static esp_err_t send_json(httpd_req_t* req, cJSON* response) +{ + char* body = cJSON_PrintUnformatted(response); + cJSON_Delete(response); + httpd_resp_set_type(req, "application/json"); + esp_err_t err = httpd_resp_sendstr(req, body ? body : "{}"); + cJSON_free(body); + return err; +} + +static esp_err_t send_rpc_error(httpd_req_t* req, const cJSON* id, int code, const std::string& message) +{ + cJSON* response = cJSON_CreateObject(); + cJSON_AddStringToObject(response, "jsonrpc", "2.0"); + cJSON_AddItemToObject(response, "id", id ? cJSON_Duplicate(id, true) : cJSON_CreateNull()); + cJSON* error = cJSON_AddObjectToObject(response, "error"); + cJSON_AddNumberToObject(error, "code", code); + cJSON_AddStringToObject(error, "message", message.c_str()); + return send_json(req, response); +} + +static esp_err_t mcp_handler(httpd_req_t* req) +{ + size_t len = req->content_len; + if (len == 0 || len > _max_request_size) { + return send_rpc_error(req, nullptr, -32600, "Request must be 1 byte to 8 KB"); + } + + std::string body(len, '\0'); + size_t received = 0; + while (received < len) { + int ret = httpd_req_recv(req, body.data() + received, len - received); + if (ret == HTTPD_SOCK_ERR_TIMEOUT) { + continue; + } + if (ret <= 0) { + return ESP_FAIL; + } + received += ret; + } + + std::unique_ptr request(cJSON_Parse(body.c_str()), cJSON_Delete); + auto method = cJSON_GetObjectItem(request.get(), "method"); + if (!request || !cJSON_IsString(method)) { + return send_rpc_error(req, nullptr, -32700, "Parse error"); + } + + // Notifications (no id) get no JSON-RPC response + auto id = cJSON_GetObjectItem(request.get(), "id"); + if (!id) { + httpd_resp_set_status(req, "202 Accepted"); + return httpd_resp_send(req, nullptr, 0); + } + + std::string name = method->valuestring; + auto params = cJSON_GetObjectItem(request.get(), "params"); + cJSON* result = nullptr; + std::string error; + + if (name == "initialize") { + result = make_initialize_result(params); + } else if (name == "ping") { + result = cJSON_CreateObject(); + } else if (name == "tools/list") { + result = make_tools_list_result(params); + } else if (name == "tools/call") { + result = call_tool(params, error); + } else { + return send_rpc_error(req, id, -32601, "Method not found: " + name); + } + if (!result) { + mclog::tagWarn(_tag, "{}: {}", name, error); + return send_rpc_error(req, id, -32602, error); + } + + cJSON* response = cJSON_CreateObject(); + cJSON_AddStringToObject(response, "jsonrpc", "2.0"); + cJSON_AddItemToObject(response, "id", cJSON_Duplicate(id, true)); + cJSON_AddItemToObject(response, "result", result); + return send_json(req, response); +} + +void lan_mcp_register_uris(httpd_handle_t server) +{ + static const std::string uri = "/" CONFIG_STACKCHAN_LAN_MCP_ENDPOINT; + httpd_uri_t mcp_uri = {.uri = uri.c_str(), .method = HTTP_POST, .handler = mcp_handler, .user_ctx = nullptr}; + httpd_register_uri_handler(server, &mcp_uri); + mclog::tagInfo(_tag, "serving {} tools at {}", _tools.size(), uri); +} + +#endif diff --git a/firmware/main/hal/hal_lan_server.cpp b/firmware/main/hal/hal_lan_server.cpp new file mode 100644 index 000000000..d8ada18a8 --- /dev/null +++ b/firmware/main/hal/hal_lan_server.cpp @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: MIT + * + * One HTTP server on the LAN for reefwatch frames and the MCP endpoint, plus an mDNS name + * (stackchan.local) and an _mcp._tcp advertisement so ESP-Claw can discover the robot. + */ +#include "hal.h" +#include "hal_lan_server.h" +#include +#include +#include +#include +#include + +static const std::string_view _tag = "HAL-LAN"; + +static void start_mdns(uint16_t port) +{ + if (mdns_init() != ESP_OK) { + mclog::tagError(_tag, "mdns init failed"); + return; + } + mdns_hostname_set(CONFIG_STACKCHAN_LAN_HOSTNAME); + mdns_instance_name_set("StackChan"); + mdns_service_add(nullptr, "_http", "_tcp", port, nullptr, 0); + +#if CONFIG_STACKCHAN_LAN_MCP_ENABLE + mdns_txt_item_t txt[] = {{"endpoint", CONFIG_STACKCHAN_LAN_MCP_ENDPOINT}}; + mdns_service_add(nullptr, "_mcp", "_tcp", port, txt, sizeof(txt) / sizeof(txt[0])); +#endif +} + +static void start_server_task(void* param) +{ + // The server binds to any address, but waiting for Wi-Fi gives a useful log line and lets mDNS start + while (!WifiManager::GetInstance().IsConnected()) { + vTaskDelay(pdMS_TO_TICKS(1000)); + } + + httpd_config_t config = HTTPD_DEFAULT_CONFIG(); + config.server_port = CONFIG_STACKCHAN_LAN_SERVER_PORT; + config.ctrl_port = ESP_HTTPD_DEF_CTRL_PORT + 7; // stay clear of the Wi-Fi config portal's server + config.stack_size = 8192; // JPEG decoding and MCP tool calls run on this task + config.task_caps = MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT; // internal SRAM is tight; no handler writes flash + config.max_open_sockets = 4; + config.lru_purge_enable = true; + + httpd_handle_t server = nullptr; + if (httpd_start(&server, &config) != ESP_OK) { + mclog::tagError(_tag, "failed to start http server on port {}", config.server_port); + vTaskDelete(nullptr); + return; + } + +#if CONFIG_STACKCHAN_REEFWATCH_ENABLE + reefwatch_register_uris(server); +#endif +#if CONFIG_STACKCHAN_LAN_MCP_ENABLE + lan_mcp_register_uris(server); +#endif + start_mdns(config.server_port); + + mclog::tagInfo(_tag, "listening on http://{}:{} (http://{}.local)", WifiManager::GetInstance().GetIpAddress(), + config.server_port, CONFIG_STACKCHAN_LAN_HOSTNAME); + vTaskDelete(nullptr); +} + +void Hal::startLanServer() +{ +#if CONFIG_STACKCHAN_REEFWATCH_ENABLE || CONFIG_STACKCHAN_LAN_MCP_ENABLE + xTaskCreate(start_server_task, "lan_server", 4096, nullptr, 3, nullptr); +#endif +} diff --git a/firmware/main/hal/hal_lan_server.h b/firmware/main/hal/hal_lan_server.h new file mode 100644 index 000000000..28b108951 --- /dev/null +++ b/firmware/main/hal/hal_lan_server.h @@ -0,0 +1,21 @@ +/* + * SPDX-License-Identifier: MIT + * + * Services hosted on the robot's LAN HTTP server (hal_lan_server.cpp). + */ +#pragma once +#include +#include + +class McpTool; + +#if CONFIG_STACKCHAN_REEFWATCH_ENABLE +void reefwatch_register_uris(httpd_handle_t server); +#endif + +// Tools added here are also served to LAN agents when CONFIG_STACKCHAN_LAN_MCP_ENABLE is set +void lan_mcp_add_tool(McpTool* tool); + +#if CONFIG_STACKCHAN_LAN_MCP_ENABLE +void lan_mcp_register_uris(httpd_handle_t server); +#endif diff --git a/firmware/main/hal/hal_mcp.cpp b/firmware/main/hal/hal_mcp.cpp index 0d7385171..4e5555d80 100644 --- a/firmware/main/hal/hal_mcp.cpp +++ b/firmware/main/hal/hal_mcp.cpp @@ -4,10 +4,13 @@ * SPDX-License-Identifier: MIT */ #include "hal.h" +#include "hal_lan_server.h" +#include #include #include #include #include +#include using namespace stackchan; @@ -20,12 +23,21 @@ void Hal::xiaozhi_mcp_init() // https://github.com/78/xiaozhi-esp32/blob/main/docs/mcp-usage.md auto& mcp_server = McpServer::GetInstance(); + // Register each tool with xiaozhi's voice agent and with the LAN MCP server (hal_lan_mcp.cpp) + auto add_tool = [&mcp_server](const std::string& name, const std::string& description, + const PropertyList& properties, + std::function callback) { + auto tool = new McpTool(name, description, properties, callback); + mcp_server.AddTool(tool); + lan_mcp_add_tool(tool); + }; + // System Prompt： // You can control the robot's head. Use get_yaw and get_pitch to sense current position. Use set_yaw for horizontal // movement and set_pitch for vertical movement. All angles are in degrees. mclog::tagInfo(_tag, "add robot.get_head_angles tool"); - mcp_server.AddTool("self.robot.get_head_angles", + add_tool("self.robot.get_head_angles", "Returns current yaw/pitch in degrees. Neutral position is {yaw:0, pitch:0}.", std::vector{}, [this](const PropertyList& properties) -> ReturnValue { LvglLockGuard lock; // StackChan motion update is under the lvgl lock @@ -40,7 +52,7 @@ void Hal::xiaozhi_mcp_init() }); mclog::tagInfo(_tag, "add robot.set_head_angles tool"); - mcp_server.AddTool("self.robot.set_head_angles", + add_tool("self.robot.set_head_angles", "Adjust head position. GUIDELINES: " "1. For natural interaction, stay within +/- 45 degrees. " "2. Only use values > 70 if the user explicitly asks to look far away/behind. " @@ -70,7 +82,7 @@ void Hal::xiaozhi_mcp_init() }); mclog::tagInfo(_tag, "add robot.set_led_color tool"); - mcp_server.AddTool( + add_tool( "self.robot.set_led_color", "Set the color of the robot's INTERNAL onboard LED. This is NOT for room lights. " "Values: 0-168 (safe range). Red=168,0,0; Green=0,168,0; Blue=0,0,168; White=100,100,100; Off=0,0,0.", @@ -93,7 +105,7 @@ void Hal::xiaozhi_mcp_init() }); mclog::tagInfo(_tag, "add robot.create_reminder tool"); - mcp_server.AddTool("self.robot.create_reminder", + add_tool("self.robot.create_reminder", "Create a reminder. Duration is in seconds. Message is what to say when time is up. Set repeat " "to true to repeat the reminder.", PropertyList({Property("duration_seconds", kPropertyTypeInteger, 60, 1, 86400), @@ -118,7 +130,7 @@ void Hal::xiaozhi_mcp_init() }); mclog::tagInfo(_tag, "add robot.get_reminders tool"); - mcp_server.AddTool("self.robot.get_reminders", "Get list of active reminders.", std::vector{}, + add_tool("self.robot.get_reminders", "Get list of active reminders.", std::vector{}, [this](const PropertyList& properties) -> ReturnValue { mclog::tagInfo(_tag, "get_reminders"); auto reminders = tools::get_active_reminders(); @@ -138,7 +150,7 @@ void Hal::xiaozhi_mcp_init() }); mclog::tagInfo(_tag, "add robot.stop_reminder tool"); - mcp_server.AddTool("self.robot.stop_reminder", "Stop a reminder by ID.", + add_tool("self.robot.stop_reminder", "Stop a reminder by ID.", PropertyList({Property("id", kPropertyTypeInteger, -1)}), [this](const PropertyList& properties) -> ReturnValue { int id = properties["id"].value(); @@ -146,4 +158,67 @@ void Hal::xiaozhi_mcp_init() tools::stop_reminder(id); return true; }); + + mclog::tagInfo(_tag, "add robot.set_expression tool"); + add_tool("self.robot.set_expression", + "Show a facial expression for a few seconds. " + "Expressions: neutral, happy, angry, sad, doubt, sleepy.", + PropertyList({Property("expression", kPropertyTypeString, std::string("happy")), + Property("seconds", kPropertyTypeInteger, 3, 1, 60)}), + [this](const PropertyList& properties) -> ReturnValue { + static const std::map emotions = { + {"neutral", avatar::Emotion::Neutral}, {"happy", avatar::Emotion::Happy}, + {"angry", avatar::Emotion::Angry}, {"sad", avatar::Emotion::Sad}, + {"doubt", avatar::Emotion::Doubt}, {"sleepy", avatar::Emotion::Sleepy}}; + + auto expression = properties["expression"].value(); + auto emotion = emotions.find(expression); + if (emotion == emotions.end()) { + throw std::invalid_argument("Unknown expression: " + expression); + } + int seconds = properties["seconds"].value(); + mclog::tagInfo(_tag, "set_expression: {} for {}s", expression, seconds); + + LvglLockGuard lock; + GetStackChan().addModifier(std::make_unique(emotion->second, seconds * 1000)); + return true; + }); + + mclog::tagInfo(_tag, "add robot.show_message tool"); + add_tool("self.robot.show_message", + "Show a short message in a speech bubble on your face, e.g. to reply to someone who texted you.", + PropertyList({Property("text", kPropertyTypeString, std::string("")), + Property("seconds", kPropertyTypeInteger, 6, 1, 60)}), + [this](const PropertyList& properties) -> ReturnValue { + auto text = properties["text"].value(); + int seconds = properties["seconds"].value(); + mclog::tagInfo(_tag, "show_message: {}", text); + + LvglLockGuard lock; + GetStackChan().addModifier(std::make_unique(text, seconds * 1000)); + GetStackChan().addModifier(std::make_unique(std::min(seconds, 3) * 1000)); + return true; + }); + +#if CONFIG_STACKCHAN_REEFWATCH_ENABLE + mclog::tagInfo(_tag, "add cameras.show_live tool"); + add_tool("self.cameras.show_live", + "Show a home security camera live on your screen for up to 5 minutes. " + "Use when the user asks to see or check a camera. Camera names are like 'front' or 'garden'.", + PropertyList({Property("camera", kPropertyTypeString, std::string("front"))}), + [this](const PropertyList& properties) -> ReturnValue { + auto camera = properties["camera"].value(); + mclog::tagInfo(_tag, "cameras.show_live: {}", camera); + setReefwatchLiveCamera(camera); + return true; + }); + + mclog::tagInfo(_tag, "add cameras.stop_live tool"); + add_tool("self.cameras.stop_live", "Stop showing the live camera view.", std::vector{}, + [this](const PropertyList& properties) -> ReturnValue { + mclog::tagInfo(_tag, "cameras.stop_live"); + setReefwatchLiveCamera(""); + return true; + }); +#endif } diff --git a/firmware/main/hal/hal_reefwatch.cpp b/firmware/main/hal/hal_reefwatch.cpp new file mode 100644 index 000000000..30779f825 --- /dev/null +++ b/firmware/main/hal/hal_reefwatch.cpp @@ -0,0 +1,160 @@ +/* + * SPDX-License-Identifier: MIT + * + * LAN endpoint for the reefwatch camera detector (github.com/adampog/reefwatch): + * + * POST /frame JPEG body (<= 320x240), headers X-Reefwatch-Token, X-Camera, X-Kind (detection|live), X-Labels + * GET /live {"camera": "front" | null, "fps": 5}, polled by reefwatch to know what to stream live + */ +#include "hal.h" +#include "hal_lan_server.h" +#include "utils/jpeg_to_image/jpeg_decoder.h" +#include +#include +#include +#include + +static const std::string_view _tag = "HAL-Reefwatch"; + +static std::mutex _live_mutex; +static std::string _live_camera; +static uint32_t _live_started_at = 0; + +// Live view switches itself off so a forgotten stream doesn't run all night +static constexpr uint32_t _live_max_ms = 5 * 60 * 1000; + +void Hal::setReefwatchLiveCamera(std::string_view camera) +{ + // Camera names end up in JSON and on screen, so keep them plain + std::string name; + for (char c : camera.substr(0, 32)) { + if (isalnum((unsigned char)c) || c == '-' || c == '_') { + name += c; + } + } + + std::lock_guard lock(_live_mutex); + if (name != _live_camera) { + mclog::tagInfo(_tag, "live view: {}", name.empty() ? "off" : name); + } + _live_camera = name; + _live_started_at = millis(); +} + +std::string Hal::getReefwatchLiveCamera() +{ + std::lock_guard lock(_live_mutex); + if (!_live_camera.empty() && millis() - _live_started_at > _live_max_ms) { + mclog::tagInfo(_tag, "live view timed out"); + _live_camera.clear(); + } + return _live_camera; +} + +#if CONFIG_STACKCHAN_REEFWATCH_ENABLE + +static constexpr size_t _max_jpeg_size = 96 * 1024; + +static std::string get_header(httpd_req_t* req, const char* field, size_t maxLen = 128) +{ + size_t len = httpd_req_get_hdr_value_len(req, field); + if (len == 0) { + return {}; + } + std::string value(std::min(len, maxLen) + 1, '\0'); + httpd_req_get_hdr_value_str(req, field, value.data(), value.size()); // truncates past maxLen + value.resize(strlen(value.c_str())); + return value; +} + +static bool check_token(httpd_req_t* req) +{ + const char* expected = CONFIG_STACKCHAN_REEFWATCH_TOKEN; + if (expected[0] == '\0' || get_header(req, "X-Reefwatch-Token") == expected) { + return true; + } + httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "bad token"); + return false; +} + +static esp_err_t send_no_content(httpd_req_t* req) +{ + httpd_resp_set_status(req, "204 No Content"); + return httpd_resp_send(req, nullptr, 0); +} + +static esp_err_t frame_handler(httpd_req_t* req) +{ + if (!check_token(req)) { + return ESP_OK; + } + + size_t len = req->content_len; + if (len == 0 || len > _max_jpeg_size) { + return httpd_resp_send_err(req, HTTPD_413_CONTENT_TOO_LARGE, "jpeg must be 1 byte to 96 KB"); + } + + std::unique_ptr jpeg( + (uint8_t*)heap_caps_malloc(len, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT), heap_caps_free); + if (!jpeg) { + return httpd_resp_send_err(req, HTTPD_500_INTERNAL_SERVER_ERROR, "out of memory"); + } + + size_t received = 0; + while (received < len) { + int ret = httpd_req_recv(req, (char*)jpeg.get() + received, len - received); + if (ret == HTTPD_SOCK_ERR_TIMEOUT) { + continue; + } + if (ret <= 0) { + return ESP_FAIL; + } + received += ret; + } + + ReefwatchFrame_t frame; + frame.camera = get_header(req, "X-Camera", 32); + frame.labels = get_header(req, "X-Labels"); + frame.isLive = get_header(req, "X-Kind", 16) == "live"; + + // A live frame that was already in flight when the live view was switched off + if (frame.isLive && frame.camera != GetHAL().getReefwatchLiveCamera()) { + return send_no_content(req); + } + + frame.image = jpeg_dec::decode_to_lvgl(jpeg.get(), len); + if (!frame.image) { + return httpd_resp_send_err(req, HTTPD_400_BAD_REQUEST, "could not decode jpeg"); + } + + GetHAL().onReefwatchFrame.emit(frame); + return send_no_content(req); +} + +static esp_err_t live_handler(httpd_req_t* req) +{ + if (!check_token(req)) { + return ESP_OK; + } + + auto camera = GetHAL().getReefwatchLiveCamera(); + auto body = fmt::format(R"({{"camera": {}, "fps": {}}})", + camera.empty() ? "null" : fmt::format("\"{}\"", camera), + CONFIG_STACKCHAN_REEFWATCH_LIVE_FPS); + httpd_resp_set_type(req, "application/json"); + return httpd_resp_send(req, body.data(), body.size()); +} + +void reefwatch_register_uris(httpd_handle_t server) +{ + httpd_uri_t frame_uri = {.uri = "/frame", .method = HTTP_POST, .handler = frame_handler, .user_ctx = nullptr}; + httpd_uri_t live_uri = {.uri = "/live", .method = HTTP_GET, .handler = live_handler, .user_ctx = nullptr}; + httpd_register_uri_handler(server, &frame_uri); + httpd_register_uri_handler(server, &live_uri); + + if (CONFIG_STACKCHAN_REEFWATCH_TOKEN[0] == 0) { + mclog::tagWarn(_tag, "no token set, any device on the LAN can post frames"); + } +} + +#endif diff --git a/firmware/main/idf_component.yml b/firmware/main/idf_component.yml index f124751ca..76a625da3 100644 --- a/firmware/main/idf_component.yml +++ b/firmware/main/idf_component.yml @@ -65,6 +65,7 @@ dependencies: - if: target in [esp32s3] espressif/adc_battery_estimation: ^0.2.0 espressif/esp_new_jpeg: ^0.6.1 + espressif/mdns: ^1.10.1 # SenseCAP Watcher Board wvirgil123/sscma_client: diff --git a/firmware/main/main.cpp b/firmware/main/main.cpp index 3f11ed65e..afff77429 100644 --- a/firmware/main/main.cpp +++ b/firmware/main/main.cpp @@ -9,6 +9,7 @@ #include #include #include +#include using namespace mooncake; using namespace smooth_ui_toolkit; @@ -22,6 +23,10 @@ extern "C" void app_main(void) // HAL init GetHAL().init(); + // Camera frames from reefwatch, shown over any app and in AI agent mode + view::reefwatch_view_init(); + GetHAL().startLanServer(); + // Setup ui hal ui_hal::on_delay([](uint32_t ms) { GetHAL().delay(ms); }); ui_hal::on_get_tick([]() { return GetHAL().millis(); });