devttl/xiaozhi-esp32
1
0
Fork 0
forked from xiaozhi/xiaozhi-esp32
Code Pull Requests Activity

Fix sensecap watcher inference (#1501)

Browse Source 
* feat: add keepalive check for Himax client and handle restart

* fix: adjust layout and positioning for top and bottom bars

* fix: fix other param restoring default ​​when modifying one param .
This commit is contained in:
virgil
2025-11-30 10:48:14 +08:00
committed by GitHub
parent 01a12b325f
commit 28db4bd60a
3 changed files with 104 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
main/boards/sensecap-watcher/sensecap_watcher.cc
View File

@@ -52,17 +52,29 @@ class CustomLcdDisplay : public SpiLcdDisplay {
auto text_font = lvgl_theme->text_font()->font();
auto icon_font = lvgl_theme->icon_font()->font();
lv_obj_set_size(status_bar_, LV_HOR_RES, text_font->line_height * 2 + 10);
lv_obj_set_size(top_bar_, LV_HOR_RES, text_font->line_height);
lv_obj_set_style_layout(top_bar_, LV_LAYOUT_NONE, 0);
lv_obj_set_style_pad_top(top_bar_, 10, 0);
lv_obj_set_style_pad_bottom(top_bar_, 1, 0);
lv_obj_set_size(status_bar_, LV_HOR_RES, text_font->line_height);
lv_obj_set_style_layout(status_bar_, LV_LAYOUT_NONE, 0);
lv_obj_set_style_pad_top(status_bar_, 10, 0);
lv_obj_set_style_pad_bottom(status_bar_, 1, 0);
lv_obj_set_y(status_bar_, text_font->line_height);
lv_obj_add_flag(status_bar_, LV_OBJ_FLAG_IGNORE_LAYOUT);
// Reparent mute and battery labels to top_bar_ to allow absolute positioning
lv_obj_set_parent(mute_label_, top_bar_);
lv_obj_set_parent(battery_label_, top_bar_);
lv_obj_set_style_margin_left(battery_label_, 0, 0);
// 针对圆形屏幕调整位置
// network battery mute //
// network mute battery //
// status //
lv_obj_align(battery_label_, LV_ALIGN_TOP_MID, -2.5 * icon_font->line_height, 0);
lv_obj_align(network_label_, LV_ALIGN_TOP_MID, -0.5 * icon_font->line_height, 0);
lv_obj_align(mute_label_, LV_ALIGN_TOP_MID, 1.5 * icon_font->line_height, 0);
lv_obj_align(network_label_, LV_ALIGN_TOP_MID, -1.5 * icon_font->line_height, 0);
lv_obj_align(mute_label_, LV_ALIGN_TOP_MID, 1.0 * icon_font->line_height, 0);
lv_obj_align(battery_label_, LV_ALIGN_TOP_MID, 2.5 * icon_font->line_height, 0);
lv_obj_align(status_label_, LV_ALIGN_BOTTOM_MID, 0, 0);
lv_obj_set_flex_grow(status_label_, 0);
@@ -77,6 +89,10 @@ class CustomLcdDisplay : public SpiLcdDisplay {
lv_obj_set_style_bg_color(low_battery_popup_, lv_color_hex(0xFF0000), 0);
lv_obj_set_width(low_battery_label_, LV_HOR_RES * 0.75);
lv_label_set_long_mode(low_battery_label_, LV_LABEL_LONG_SCROLL_CIRCULAR);
// 针对圆形屏幕调整底部对话框位置，避免被圆角遮挡
lv_obj_set_style_pad_bottom(bottom_bar_, 30, 0);
lv_obj_set_width(chat_message_label_, LV_HOR_RES * 0.75); // 限制宽度，避免文字贴边
}
};

112
main/boards/sensecap-watcher/sscma_camera.cc
View File

@@ -11,11 +11,32 @@
#include <esp_heap_caps.h>
#include <cstring>
#include "application.h"
#include "sscma_client_commands.h"
#define TAG "SscmaCamera"
#define IMG_JPEG_BUF_SIZE 48 * 1024
static bool __himax_keepalive_check(sscma_client_handle_t client)
{
esp_err_t ret = ESP_OK;
sscma_client_reply_t reply = {0};
int retry = 3;
while(retry--) {
ret = sscma_client_request(client, CMD_PREFIX CMD_AT_ID CMD_QUERY CMD_SUFFIX, &reply, true, pdMS_TO_TICKS(2000));
if (reply.payload != NULL) {
sscma_client_reply_clear(&reply);
}
if( ret != ESP_OK ) {
ESP_LOGE(TAG, "Himax keepalive check failed: %d", ret);
vTaskDelay(pdMS_TO_TICKS(100));
} else {
return true;
}
}
return false;
}
SscmaCamera::SscmaCamera(esp_io_expander_handle_t io_exp_handle) {
sscma_client_io_spi_config_t spi_io_config = {0};
spi_io_config.sync_gpio_num = BSP_SSCMA_CLIENT_SPI_SYNC;
@@ -239,6 +260,10 @@ SscmaCamera::SscmaCamera(esp_io_expander_handle_t io_exp_handle) {
};
callback.on_connect = [](sscma_client_handle_t client, const sscma_client_reply_t *reply, void *user_ctx) {
ESP_LOGI(TAG, "SSCMA client connected");
SscmaCamera* self = static_cast<SscmaCamera*>(user_ctx);
if (self) {
self->sscma_restarted_ = true;
}
};
callback.on_log = [](sscma_client_handle_t client, const sscma_client_reply_t *reply, void *user_ctx) {
@@ -345,8 +370,24 @@ SscmaCamera::SscmaCamera(esp_io_expander_handle_t io_exp_handle) {
xTaskCreate([](void* arg) {
auto this_ = (SscmaCamera*)arg;
bool is_inference = false;
int64_t last_keepalive_time = esp_timer_get_time();
while (true)
{
if (this_->sscma_restarted_) {
ESP_LOGI(TAG, "SSCMA restarted detected");
this_->sscma_restarted_ = false;
is_inference = false;
}
if (esp_timer_get_time() - last_keepalive_time > 10 * 1000000) {
last_keepalive_time = esp_timer_get_time();
if (!__himax_keepalive_check(this_->sscma_client_handle_)) {
ESP_LOGE(TAG, "restart himax");
sscma_client_reset(this_->sscma_client_handle_);
vTaskDelay(pdMS_TO_TICKS(100));
}
}
if (this_->inference_en && Application::GetInstance().GetDeviceState() == kDeviceStateIdle ) {
if (!is_inference) {
ESP_LOGI(TAG, "Start inference (enable=1)");
@@ -408,11 +449,12 @@ void SscmaCamera::InitializeMcpTools() {
auto& mcp_server = McpServer::GetInstance();
// 获取模型参数配置
mcp_server.AddTool("self.model.param_get",
"获取模型参数配置\n"
" `threshold`: 检测置信度阈值 (0-100, 默认 75);\n"
" `interval`: 对话结束后的冷却时间，防止频繁打断 (默认 8 秒);\n"
" `duration`: 检测持续时间 (默认 2 秒);\n"
" `target`: 检测目标 (默认 0);",
"获取当前视觉模型检测的参数配置信息。\n"
"返回结果包含：\n"
" `threshold`: 检测置信度阈值 (0-100)，低于此值的检测结果将被忽略；\n"
" `interval`: 触发对话后的冷却时间(秒)，防止频繁打断；\n"
" `duration`: 持续检测确认时间(秒)；\n"
" `target`: 当前关注的检测目标索引。",
PropertyList(),
[this](const PropertyList& properties) -> ReturnValue {
Settings settings("model", false);
@@ -431,25 +473,28 @@ void SscmaCamera::InitializeMcpTools() {
// 设置模型参数配置
mcp_server.AddTool("self.model.param_set",
"模型参数设置\n"
" `threshold`: 检测置信度阈值 (单位百分比, 默认 75);"
" `interval`: 对话结束后的冷却时间，防止频繁打断 (单位秒，默认 8 秒);"
" `duration`: 检测持续时间 (单位秒，默认 2 秒);"
" `target`: 检测目标 (默认 0);",
"配置视觉模型检测参数。当用户希望调整检测灵敏度、频率或特定目标时使用。\n"
"参数(均为可选，未提供的参数将保持当前设置不变)：\n"
" `threshold`: 置信度阈值 (0-100)。提高此值可减少误报，但可能漏检；\n"
" `interval`: 冷却时间(秒)。设置对话结束后多久内不再触发检测；\n"
" `duration`: 持续检测时间(秒)。\n"
" `target`: 设置检测目标的索引 ID。",
PropertyList({
Property("threshold", kPropertyTypeInteger, 75, 0, 100),
Property("interval", kPropertyTypeInteger, 8, 1, 60),
Property("duration", kPropertyTypeInteger, 2, 1, 60),
Property("target", kPropertyTypeInteger, 0, 0, this->model_class_cnt > 0 ? this->model_class_cnt - 1 : 0)
Property("threshold", kPropertyTypeInteger, -1, -1, 100),
Property("interval", kPropertyTypeInteger, -1, -1, 60),
Property("duration", kPropertyTypeInteger, -1, -1, 60),
Property("target", kPropertyTypeInteger, -1, -1, this->model_class_cnt > 0 ? this->model_class_cnt - 1 : 255)
}),
[this](const PropertyList& properties) -> ReturnValue {
Settings settings("model", true);
try {
const Property& threshold_prop = properties["threshold"];
int threshold = threshold_prop.value<int>();
settings.SetInt("threshold", threshold);
this->detect_threshold = threshold;
ESP_LOGI(TAG, "Set detection threshold to %d", threshold);
if (threshold != -1) {
settings.SetInt("threshold", threshold);
this->detect_threshold = threshold;
ESP_LOGI(TAG, "Set detection threshold to %d", threshold);
}
} catch (const std::runtime_error&) {
// threshold parameter not provided, skip
}
@@ -457,9 +502,11 @@ void SscmaCamera::InitializeMcpTools() {
try {
const Property& interval_prop = properties["interval"];
int interval = interval_prop.value<int>();
settings.SetInt("interval", interval);
this->detect_invoke_interval_sec = interval;
ESP_LOGI(TAG, "Set detection interval to %d", interval);
if (interval != -1) {
settings.SetInt("interval", interval);
this->detect_invoke_interval_sec = interval;
ESP_LOGI(TAG, "Set detection interval to %d", interval);
}
} catch (const std::runtime_error&) {
// interval parameter not provided, skip
}
@@ -467,8 +514,10 @@ void SscmaCamera::InitializeMcpTools() {
try {
const Property& duration_prop = properties["duration"];
int duration = duration_prop.value<int>();
settings.SetInt("duration", duration);
this->detect_duration_sec = duration;
if (duration != -1) {
settings.SetInt("duration", duration);
this->detect_duration_sec = duration;
}
} catch (const std::runtime_error&) {
// duration parameter not provided, skip
}
@@ -476,9 +525,11 @@ void SscmaCamera::InitializeMcpTools() {
try {
const Property& target_prop = properties["target"];
int target = target_prop.value<int>();
settings.SetInt("target", target);
this->detect_target = target;
ESP_LOGI(TAG, "Set detection target to %d", target);
if (target != -1) {
settings.SetInt("target", target);
this->detect_target = target;
ESP_LOGI(TAG, "Set detection target to %d", target);
}
} catch (const std::runtime_error&) {
// target_type parameter not provided, skip
}
@@ -488,9 +539,10 @@ void SscmaCamera::InitializeMcpTools() {
// 推理开关获取
mcp_server.AddTool("self.model.enable",
"控制推理开关\n"
" 读取/设置推理是否开启; 0=关闭, 1=开启\n"
"可选字段: `enable`\n",
"控制视觉推理(摄像头检测)功能的开启与关闭，或查询当前状态。\n"
"当用户指令涉及'开启/关闭推理'、'开始/停止检测'时使用。\n"
"参数：\n"
" `enable`: (可选) 整数。1=开启推理，0=关闭推理。若省略则返回当前开关状态。",
PropertyList({
Property("enable", kPropertyTypeInteger, inference_en, 0, 1)
}),
@@ -531,8 +583,8 @@ bool SscmaCamera::Capture() {
return false;
}
ESP_LOGI(TAG, "Capturing image...");
// himax 有缓存数据,需要拍两张照片, 只获取最新的照片即可.
if (sscma_client_sample(sscma_client_handle_, 2) ) {
// himax 可能有缓存数据, 只获取最新的照片即可.
if (sscma_client_sample(sscma_client_handle_, 1) ) {
ESP_LOGE(TAG, "Failed to capture image from SSCMA client");
return false;
}

1
main/boards/sensecap-watcher/sscma_camera.h
View File

@@ -54,6 +54,7 @@ private:
int detect_invoke_interval_sec = 8; // 默认15秒冷却期，避免频繁开始会话
int detect_debounce_sec = 1; // 验证期间人员离开的去抖动时间1秒
int inference_en = 0; // 推理使能开关（0: 关闭, 1: 开启）
bool sscma_restarted_ = false;
sscma_client_model_t *model;
int model_class_cnt = 0;