devttl/xiaozhi-esp32
1
0
Fork 0
forked from xiaozhi/xiaozhi-esp32
Code Pull Requests Activity

feat: sensecap watcher add inference (#1312)

Browse Source 
* feat: Wake up when a person is detected

* fix: Solve the problem of no sound when using WakeWordInvoke

* fix: Solve the problem of triggering dialogue when the person has not left

* feat(vision): 优化视觉检测逻辑并增加配置接口

本次提交旨在优化视觉检测功能,使其行为更自然、更智能,并为用户提供灵活的配置选项。

主要更新包括:

1. 引入了更精细的检测状态机:

   - IDLE: 空闲状态,等待检测目标。

   - VALIDATING: 验证状态,在检测到目标后,持续一段时间(可配置)以确认其存在,防止误触发。

   - COOLDOWN: 冷却状态,在一次成功交互后进入,避免过于频繁的打扰。

2. 新增了用于配置视觉检测的 MCP 工具:

   - self.vision.get_detection_config: 获取当前的检测参数(阈值、冷却间隔、验证时长、目标类型)。

   - self.vision.set_detection_config: 允许用户动态修改这些参数,以适应不同场景。

3. 性能优化:

   - 增加了配置参数的内存缓存,避免了在检测循环中对 NVS 的频繁访问。

* feat: Inference using Model 4

* feat: default inference disable

* feat: version cmd change to output json

* fix: fix image display

* Fix include directives for esp_check and esp_app_desc

---------

Co-authored-by: Spencer <love4yzp@gmail.com>
Co-authored-by: Xiaoxia <terrence@tenclass.com>
This commit is contained in:
virgil
2025-10-20 21:18:44 +08:00
committed by GitHub
parent ac03f8097d
commit a601a5cbc1
4 changed files with 455 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
main/application.cc
View File

@@ -788,13 +788,35 @@ bool Application::UpgradeFirmware(Ota& ota, const std::string& url) {
}
void Application::WakeWordInvoke(const std::string& wake_word) {
if (!protocol_) {
return;
}
if (device_state_ == kDeviceStateIdle) {
ToggleChatState();
Schedule([this, wake_word]() {
if (protocol_) {
protocol_->SendWakeWordDetected(wake_word);
audio_service_.EncodeWakeWord();
if (!protocol_->IsAudioChannelOpened()) {
SetDeviceState(kDeviceStateConnecting);
if (!protocol_->OpenAudioChannel()) {
audio_service_.EnableWakeWordDetection(true);
return;
}
});
}
ESP_LOGI(TAG, "Wake word detected: %s", wake_word.c_str());
#if CONFIG_USE_AFE_WAKE_WORD || CONFIG_USE_CUSTOM_WAKE_WORD
// Encode and send the wake word data to the server
while (auto packet = audio_service_.PopWakeWordPacket()) {
protocol_->SendAudio(std::move(packet));
}
// Set the chat state to wake word detected
protocol_->SendWakeWordDetected(wake_word);
SetListeningMode(aec_mode_ == kAecOff ? kListeningModeAutoStop : kListeningModeRealtime);
#else
SetListeningMode(aec_mode_ == kAecOff ? kListeningModeAutoStop : kListeningModeRealtime);
// Play the pop up sound to indicate the wake word is detected
audio_service_.PlaySound(Lang::Sounds::OGG_POPUP);
#endif
} else if (device_state_ == kDeviceStateSpeaking) {
Schedule([this]() {
AbortSpeaking(kAbortReasonNone);

44
main/boards/sensecap-watcher/sensecap_watcher.cc
View File

@@ -12,7 +12,7 @@
#include "lvgl_theme.h"
#include <esp_log.h>
#include "esp_check.h"
#include <esp_check.h>
#include <esp_lcd_panel_io.h>
#include <esp_lcd_panel_ops.h>
#include <esp_lcd_spd2010.h>
@@ -28,6 +28,7 @@
#include <esp_console.h>
#include <esp_mac.h>
#include <nvs_flash.h>
#include <esp_app_desc.h>
#include "assets/lang_config.h"
@@ -492,6 +493,47 @@ private:
};
ESP_ERROR_CHECK(esp_console_cmd_register(&cmd5));
const esp_console_cmd_t cmd6 = {
.command = "version",
.help = "Read version info",
.hint = NULL,
.func = NULL,
.argtable = NULL,
.func_w_context = [](void *context,int argc, char** argv) -> int {
auto self = static_cast<SensecapWatcher*>(context);
auto app_desc = esp_app_get_description();
const char* region = "UNKNOWN";
#if defined(CONFIG_LANGUAGE_ZH_CN)
region = "CN";
#elif defined(CONFIG_LANGUAGE_EN_US)
region = "US";
#elif defined(CONFIG_LANGUAGE_JA_JP)
region = "JP";
#elif defined(CONFIG_LANGUAGE_ES_ES)
region = "ES";
#elif defined(CONFIG_LANGUAGE_DE_DE)
region = "DE";
#elif defined(CONFIG_LANGUAGE_FR_FR)
region = "FR";
#elif defined(CONFIG_LANGUAGE_IT_IT)
region = "IT";
#elif defined(CONFIG_LANGUAGE_PT_PT)
region = "PT";
#elif defined(CONFIG_LANGUAGE_RU_RU)
region = "RU";
#elif defined(CONFIG_LANGUAGE_KO_KR)
region = "KR";
#endif
printf("{\"type\":0,\"name\":\"VER?\",\"code\":0,\"data\":{\"software\":\"%s\",\"hardware\":\"watcher xiaozhi agent\",\"camera\":%d,\"region\":\"%s\"}}\n",
app_desc->version,
self->GetCamera() == nullptr ? 0 : 1,
region);
return 0;
},
.context =this
};
ESP_ERROR_CHECK(esp_console_cmd_register(&cmd6));
esp_console_dev_uart_config_t hw_config = ESP_CONSOLE_DEV_UART_CONFIG_DEFAULT();
ESP_ERROR_CHECK(esp_console_new_repl_uart(&hw_config, &repl_config, &repl));
ESP_ERROR_CHECK(esp_console_start_repl(repl));

377
main/boards/sensecap-watcher/sscma_camera.cc
View File

@@ -5,10 +5,12 @@
#include "board.h"
#include "system_info.h"
#include "config.h"
#include "settings.h"
#include <esp_log.h>
#include <esp_heap_caps.h>
#include <cstring>
#include "application.h"
#define TAG "SscmaCamera"
@@ -47,28 +49,190 @@ SscmaCamera::SscmaCamera(esp_io_expander_handle_t io_exp_handle) {
sscma_client_callback_t callback = {0};
detection_state = SscmaCamera::IDLE;
state_start_time = 0;
need_start_cooldown = false;
callback.on_event = [](sscma_client_handle_t client, const sscma_client_reply_t *reply, void *user_ctx) {
SscmaCamera* self = static_cast<SscmaCamera*>(user_ctx);
if (!self) return;
char *img = NULL;
int img_size = 0;
if (sscma_utils_fetch_image_from_reply(reply, &img, &img_size) == ESP_OK)
{
ESP_LOGI(TAG, "image_size: %d\n", img_size);
// 将数据通过队列发送出去
SscmaData data;
data.img = (uint8_t*)img;
data.len = img_size;
int box_count = 0;
sscma_client_box_t *boxes = NULL;
int class_count = 0;
sscma_client_class_t *classes = NULL;
int point_count = 0;
sscma_client_point_t *points = NULL;
int model_type = 0;
int obj_cnt = 0;
// 清空队列，保证只保存最新的数据
SscmaData dummy;
while (xQueueReceive(self->sscma_data_queue_, &dummy, 0) == pdPASS) {
if (dummy.img) {
heap_caps_free(dummy.img);
int width = 0, height = 0;
cJSON *data = cJSON_GetObjectItem(reply->payload, "data");
if (data != NULL && cJSON_IsObject(data)) {
cJSON *resolution = cJSON_GetObjectItem(data, "resolution");
if (data != NULL && cJSON_IsArray(resolution) && cJSON_GetArraySize(resolution) == 2) {
width = cJSON_GetArrayItem(resolution, 0)->valueint;
height = cJSON_GetArrayItem(resolution, 1)->valueint;
}
}
switch ((width+height)) {
case (416+416):
{
bool is_object_detected = false;
bool is_need_wake = false;
// 定期更新检测配置参数，避免频繁NVS访问
int64_t cur_tm = esp_timer_get_time();
// 尝试获取检测框数据（目标检测模型）
if (sscma_utils_fetch_boxes_from_reply(reply, &boxes, &box_count) == ESP_OK && box_count > 0) {
for (int i = 0; i < box_count; i++) {
ESP_LOGI(TAG, "[box %d]: x=%d, y=%d, w=%d, h=%d, score=%d, target=%d", i, \
boxes[i].x, boxes[i].y, boxes[i].w, boxes[i].h, boxes[i].score, boxes[i].target);
if (boxes[i].target == self->detect_target && boxes[i].score > self->detect_threshold) {
is_object_detected = true;
model_type = 0;
obj_cnt++;
break;
}
}
} else if (sscma_utils_fetch_classes_from_reply(reply, &classes, &class_count) == ESP_OK && class_count > 0) {
// 尝试获取分类数据（分类模型）
for (int i = 0; i < class_count; i++) {
ESP_LOGI(TAG, "[class %d]: target=%d, score=%d", i,
classes[i].target, classes[i].score);
if (classes[i].target == self->detect_target && classes[i].score > self->detect_threshold) {
is_object_detected = true;
model_type = 1;
obj_cnt++;
}
}
} else if (sscma_utils_fetch_points_from_reply(reply, &points, &point_count) == ESP_OK && point_count > 0) {
// 尝试获取关键点数据（姿态估计模型）
for (int i = 0; i < point_count; i++) {
ESP_LOGI(TAG, "[point %d]: x=%d, y=%d, z=%d, score=%d, target=%d", i,
points[i].x, points[i].y, points[i].z, points[i].score, points[i].target);
if (points[i].target == self->detect_target && points[i].score > self->detect_threshold) {
is_object_detected = true;
model_type = 2;
obj_cnt++;
}
}
}
// 如果需要开始冷却期，现在开始计时
if (self->need_start_cooldown) { // 回调暂停，标志保持，等待回调恢复后开始计时
self->state_start_time = cur_tm;
self->need_start_cooldown = false;
ESP_LOGI(TAG, "Starting cooldown timer");
}
// 状态机驱动的检测逻辑 - 只在人员出现时触发
switch (self->detection_state) {
case SscmaCamera::IDLE:
if (is_object_detected) {
// 人员出现，开始验证（这是从无到有的转换）
self->detection_state = SscmaCamera::VALIDATING;
self->state_start_time = cur_tm; // 记录物体出现时间
self->last_detected_time = cur_tm; // 初始化最后检测时间
ESP_LOGI(TAG, "object appeared, starting validation");
}
break;
case SscmaCamera::VALIDATING:
if (is_object_detected) {
// 更新最后检测到的时间
self->last_detected_time = cur_tm;
// 检查是否验证足够时间
if ((cur_tm - self->state_start_time) >= (self->detect_duration_sec * 1000000)) {
is_need_wake = true;
}
} else {
// 验证期间人员离开，检查去抖动时间
if (self->last_detected_time > 0 &&
(cur_tm - self->last_detected_time) >= self->detect_debounce_sec * 1000000LL) {
// 去抖动时间已过，确认人员已离开，回到空闲
self->detection_state = SscmaCamera::IDLE;
self->last_detected_time = 0;
ESP_LOGI(TAG, "object left during validation (debounced), back to idle");
}
}
break;
case SscmaCamera::COOLDOWN:
// 冷却期，需要满足两个条件：1)object离开 2)过了15秒
if (!is_object_detected &&
(cur_tm - self->state_start_time) >= (self->detect_invoke_interval_sec * 1000000LL)) {
// object离开且冷却时间到，回到空闲状态
self->detection_state = SscmaCamera::IDLE;
ESP_LOGI(TAG, "Cooldown complete and object left, back to idle - ready for next appearance");
}
// 其他情况继续保持冷却状态
break;
}
if( is_need_wake ) {
ESP_LOGI(TAG, "Validation complete, triggering conversation (type=%d, res=%dx%d)",
self->detect_target, width, height);
// 触发对话
std::string wake_word;
if ( model_type == 0 ) {
std::string cached_target_name = "object";
if( self->model != NULL && self->model->classes[self->detect_target] != NULL ) {
cached_target_name = self->model->classes[self->detect_target];
}
wake_word = "<detect>" + std::to_string(obj_cnt) + " " + cached_target_name + " detected </detect>";
} else if ( model_type == 1 ) {
std::string cached_target_name = "object";
if( self->model != NULL && self->model->classes[self->detect_target] != NULL ) {
cached_target_name = self->model->classes[self->detect_target];
}
wake_word = "<detect>" + std::to_string(obj_cnt) + " " + cached_target_name + " detected </detect>";
} else if ( model_type == 2 ) {
std::string cached_target_name = "object";
if( self->model != NULL && self->model->classes[self->detect_target] != NULL ) {
cached_target_name = self->model->classes[self->detect_target];
}
wake_word = "<detect>" + std::to_string(obj_cnt) + " " + cached_target_name + " detected </detect>";
}
printf("wake_word:%s\n", wake_word.c_str());
Application::GetInstance().WakeWordInvoke(wake_word);
// 进入冷却状态，标记需要开始冷却期；如下变量将在会话结束后被使用，等待回调恢复后开始计时
self->detection_state = SscmaCamera::COOLDOWN;
self->need_start_cooldown = true;
}
}
xQueueSend(self->sscma_data_queue_, &data, 0);
// 注意：img 的释放由接收方负责
break;
case (640+480):
if (sscma_utils_fetch_image_from_reply(reply, &img, &img_size) == ESP_OK)
{
ESP_LOGI(TAG, "image_size: %d\n", img_size);
// 将数据通过队列发送出去
SscmaData data;
data.img = (uint8_t*)img;
data.len = img_size;
// 清空队列，保证只保存最新的数据
SscmaData dummy;
while (xQueueReceive(self->sscma_data_queue_, &dummy, 0) == pdPASS) {
if (dummy.img) {
heap_caps_free(dummy.img);
}
}
xQueueSend(self->sscma_data_queue_, &data, 0);
// 注意：img 的释放由接收方负责
}
break;
default:
ESP_LOGI(TAG, "unknown resolution");
break;
}
};
callback.on_connect = [](sscma_client_handle_t client, const sscma_client_reply_t *reply, void *user_ctx) {
@@ -148,6 +312,57 @@ SscmaCamera::SscmaCamera(esp_io_expander_handle_t io_exp_handle) {
ESP_LOGE(TAG, "Failed to allocate memory for preview image");
return;
}
sscma_client_set_model(sscma_client_handle_, 4);
model_class_cnt = 0;
if (sscma_client_get_model(sscma_client_handle_, &model, true) == ESP_OK) {
printf("ID: %d\n", model->id ? model->id : -1);
printf("UUID: %s\n", model->uuid ? model->uuid : "N/A");
printf("Name: %s\n", model->name ? model->name : "N/A");
printf("Version: %s\n", model->ver ? model->ver : "N/A");
printf("URL: %s\n", model->url ? model->url : "N/A");
printf("Checksum: %s\n", model->checksum ? model->checksum : "N/A");
printf("Classes:\n");
if (model->classes[0] != NULL)
{
for (int i = 0; model->classes[i] != NULL; i++)
{
printf(" - %s\n", model->classes[i]);
model_class_cnt++;
}
} else {
printf(" N/A\n");
}
} else {
printf("get model failed\n");
}
ESP_LOGI(TAG, "initialize mcp tools");
InitializeMcpTools();
xTaskCreate([](void* arg) {
auto this_ = (SscmaCamera*)arg;
bool is_inference = false;
while (true)
{
if (this_->inference_en && Application::GetInstance().GetDeviceState() == kDeviceStateIdle ) {
if (!is_inference) {
ESP_LOGI(TAG, "Start inference (enable=1)");
sscma_client_break(this_->sscma_client_handle_);
sscma_client_set_model(this_->sscma_client_handle_, 4);
sscma_client_set_sensor(this_->sscma_client_handle_, 1, 1, true); // 设置分辨率 416X416
sscma_client_invoke(this_->sscma_client_handle_, -1, false, true);
is_inference = true;
}
} else if (is_inference && (!this_->inference_en || Application::GetInstance().GetDeviceState() != kDeviceStateIdle)) {
ESP_LOGI(TAG, "Stop inference (enable=%d state=%d)", this_->inference_en, Application::GetInstance().GetDeviceState());
is_inference = false;
sscma_client_break(this_->sscma_client_handle_);
}
vTaskDelay(pdMS_TO_TICKS(200));
}
}, "sscma_camera", 4096, this, 1, nullptr);
}
SscmaCamera::~SscmaCamera() {
@@ -179,6 +394,121 @@ SscmaCamera::~SscmaCamera() {
}
}
void SscmaCamera::InitializeMcpTools() {
Settings settings("model", false);
detect_threshold = settings.GetInt("threshold", 75);
detect_invoke_interval_sec = settings.GetInt("interval", 8);
detect_duration_sec = settings.GetInt("duration", 2);
detect_target = settings.GetInt("target", 0);
inference_en = settings.GetInt("enable", 0);
auto& mcp_server = McpServer::GetInstance();
// 获取模型参数配置
mcp_server.AddTool("self.model.param_get",
"获取模型参数配置\n"
" `threshold`: 检测置信度阈值 (0-100, 默认 75);\n"
" `interval`: 对话结束后的冷却时间，防止频繁打断 (默认 8 秒);\n"
" `duration`: 检测持续时间 (默认 2 秒);\n"
" `target`: 检测目标 (默认 0);",
PropertyList(),
[this](const PropertyList& properties) -> ReturnValue {
Settings settings("model", false);
int threshold = settings.GetInt("threshold", 75);
int interval = settings.GetInt("interval", 8);
int duration = settings.GetInt("duration", 2);
int target_type = settings.GetInt("target", 0);
std::string result = "{\"threshold\":" + std::to_string(threshold) +
",\"interval\":" + std::to_string(interval) +
",\"duration\":" + std::to_string(duration) +
",\"target_type\":" + std::to_string(target_type) + "}";
return result;
});
// 设置模型参数配置
mcp_server.AddTool("self.model.param_set",
"模型参数设置\n"
" `threshold`: 检测置信度阈值 (单位百分比, 默认 75);"
" `interval`: 对话结束后的冷却时间，防止频繁打断 (单位秒，默认 8 秒);"
" `duration`: 检测持续时间 (单位秒，默认 2 秒);"
" `target`: 检测目标 (默认 0);",
PropertyList({
Property("threshold", kPropertyTypeInteger, 75, 0, 100),
Property("interval", kPropertyTypeInteger, 8, 1, 60),
Property("duration", kPropertyTypeInteger, 2, 1, 60),
Property("target", kPropertyTypeInteger, 0, 0, this->model_class_cnt > 0 ? this->model_class_cnt - 1 : 0)
}),
[this](const PropertyList& properties) -> ReturnValue {
Settings settings("model", true);
try {
const Property& threshold_prop = properties["threshold"];
int threshold = threshold_prop.value<int>();
settings.SetInt("threshold", threshold);
this->detect_threshold = threshold;
ESP_LOGI(TAG, "Set detection threshold to %d", threshold);
} catch (const std::runtime_error&) {
// threshold parameter not provided, skip
}
try {
const Property& interval_prop = properties["interval"];
int interval = interval_prop.value<int>();
settings.SetInt("interval", interval);
this->detect_invoke_interval_sec = interval;
ESP_LOGI(TAG, "Set detection interval to %d", interval);
} catch (const std::runtime_error&) {
// interval parameter not provided, skip
}
try {
const Property& duration_prop = properties["duration"];
int duration = duration_prop.value<int>();
settings.SetInt("duration", duration);
this->detect_duration_sec = duration;
} catch (const std::runtime_error&) {
// duration parameter not provided, skip
}
try {
const Property& target_prop = properties["target"];
int target = target_prop.value<int>();
settings.SetInt("target", target);
this->detect_target = target;
ESP_LOGI(TAG, "Set detection target to %d", target);
} catch (const std::runtime_error&) {
// target_type parameter not provided, skip
}
return "{\"status\": \"success\", \"message\": \"Detection configuration updated\"}";
});
// 推理开关获取
mcp_server.AddTool("self.model.enable",
"控制推理开关\n"
" 读取/设置推理是否开启; 0=关闭, 1=开启\n"
"可选字段: `enable`\n",
PropertyList({
Property("enable", kPropertyTypeInteger, inference_en, 0, 1)
}),
[this](const PropertyList& properties) -> ReturnValue {
Settings settings("model", true);
try {
const Property& enable_prop = properties["enable"];
int en = enable_prop.value<int>();
settings.SetInt("enable", en);
this->inference_en = en;
ESP_LOGI(TAG, "Set inference enable to %d", en);
} catch (const std::runtime_error&) {
// enable not provided -> treat as query
}
// 返回当前配置
int cur_en = settings.GetInt("enable", this->inference_en);
return std::string("{\"enable\":") + std::to_string(cur_en) + "}";
});
}
void SscmaCamera::SetExplainUrl(const std::string& url, const std::string& token) {
explain_url_ = url;
explain_token_ = token;
@@ -194,8 +524,11 @@ bool SscmaCamera::Capture() {
return false;
}
if (sscma_client_set_sensor(sscma_client_handle_, 1, 3, true)) {
ESP_LOGE(TAG, "Failed to set sensor");
return false;
}
ESP_LOGI(TAG, "Capturing image...");
// himax 有缓存数据,需要拍两张照片, 只获取最新的照片即可.
if (sscma_client_sample(sscma_client_handle_, 2) ) {
ESP_LOGE(TAG, "Failed to capture image from SSCMA client");
@@ -245,7 +578,19 @@ bool SscmaCamera::Capture() {
// 显示预览图片
auto display = dynamic_cast<LvglDisplay*>(Board::GetInstance().GetDisplay());
if (display != nullptr) {
auto image = std::make_unique<LvglSourceImage>(&preview_image_);
uint16_t w = preview_image_.header.w;
uint16_t h = preview_image_.header.h;
size_t image_size = w * h * 2;
size_t stride = preview_image_.header.w * 2;
uint8_t* data = (uint8_t*)heap_caps_malloc(image_size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
if (data == nullptr) {
ESP_LOGE(TAG, "Failed to allocate memory for display image");
return true;
}
memcpy(data, preview_image_.data, image_size);
auto image = std::make_unique<LvglAllocatedImage>(data, image_size, w, h, stride, LV_COLOR_FORMAT_RGB565);
display->SetPreviewImage(std::move(image));
}
return true;

24
main/boards/sensecap-watcher/sscma_camera.h
View File

@@ -1,6 +1,7 @@
#ifndef SSCMA_CAMERA_H
#define SSCMA_CAMERA_H
#include <cstdint>
#include <lvgl.h>
#include <thread>
#include <memory>
@@ -35,9 +36,31 @@ private:
jpeg_dec_handle_t jpeg_dec_;
jpeg_dec_io_t *jpeg_io_;
jpeg_dec_header_info_t *jpeg_out_;
// 检测状态机
enum DetectionState {
IDLE, // 空闲状态
VALIDATING, // 验证中（连续检测3秒）
COOLDOWN // 冷却期（等待重新检测）
};
DetectionState detection_state = IDLE;
int64_t state_start_time = 0;
bool need_start_cooldown = false; // 是否需要开始冷却期
int64_t last_detected_time = 0; // 验证期间最后一次检测到物体的时间
int detect_target = 0;
int detect_threshold = 75;
int detect_duration_sec = 2; // 检测持续时间2秒，确认人员持续存在
int detect_invoke_interval_sec = 8; // 默认15秒冷却期，避免频繁开始会话
int detect_debounce_sec = 1; // 验证期间人员离开的去抖动时间1秒
int inference_en = 0; // 推理使能开关（0: 关闭, 1: 开启）
sscma_client_model_t *model;
int model_class_cnt = 0;
public:
SscmaCamera(esp_io_expander_handle_t io_exp_handle);
~SscmaCamera();
void InitializeMcpTools();
virtual void SetExplainUrl(const std::string& url, const std::string& token);
virtual bool Capture();
@@ -45,6 +68,7 @@ public:
virtual bool SetHMirror(bool enabled) override;
virtual bool SetVFlip(bool enabled) override;
virtual std::string Explain(const std::string& question);
};
#endif // ESP32_CAMERA_H