From 3c11cceb43683b676b09a1f2c0f636c4af993e1d Mon Sep 17 00:00:00 2001 From: Xiaoxia Date: Tue, 22 Jul 2025 18:57:25 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=87=AA=E5=AE=9A=E4=B9=89?= =?UTF-8?q?=E5=94=A4=E9=86=92=E8=AF=8D=E5=90=AF=E5=8A=A8=E5=A4=B1=E8=B4=A5?= =?UTF-8?q?=E7=9A=84=E6=8F=90=E5=8D=87=20(#965)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/audio/audio_service.cc | 5 +- main/audio/wake_word.h | 2 +- main/audio/wake_words/afe_wake_word.cc | 6 +- main/audio/wake_words/afe_wake_word.h | 2 +- main/audio/wake_words/custom_wake_word.cc | 71 ++++++++++++++--------- main/audio/wake_words/custom_wake_word.h | 7 ++- main/audio/wake_words/esp_wake_word.cc | 8 ++- main/audio/wake_words/esp_wake_word.h | 2 +- main/boards/xmini-c3-v3/config.json | 1 + 9 files changed, 67 insertions(+), 37 deletions(-) diff --git a/main/audio/audio_service.cc b/main/audio/audio_service.cc index d6b73513..ec6b5931 100644 --- a/main/audio/audio_service.cc +++ b/main/audio/audio_service.cc @@ -468,7 +468,10 @@ void AudioService::EnableWakeWordDetection(bool enable) { ESP_LOGD(TAG, "%s wake word detection", enable ? "Enabling" : "Disabling"); if (enable) { if (!wake_word_initialized_) { - wake_word_->Initialize(codec_); + if (!wake_word_->Initialize(codec_)) { + ESP_LOGE(TAG, "Failed to initialize wake word"); + return; + } wake_word_initialized_ = true; } wake_word_->Start(); diff --git a/main/audio/wake_word.h b/main/audio/wake_word.h index 9725d2fc..71138f71 100644 --- a/main/audio/wake_word.h +++ b/main/audio/wake_word.h @@ -11,7 +11,7 @@ class WakeWord { public: virtual ~WakeWord() = default; - virtual void Initialize(AudioCodec* codec) = 0; + virtual bool Initialize(AudioCodec* codec) = 0; virtual void Feed(const std::vector& data) = 0; virtual void OnWakeWordDetected(std::function callback) = 0; virtual void Start() = 0; diff --git a/main/audio/wake_words/afe_wake_word.cc b/main/audio/wake_words/afe_wake_word.cc index 37145b93..8229b4c2 100644 --- a/main/audio/wake_words/afe_wake_word.cc +++ b/main/audio/wake_words/afe_wake_word.cc @@ -30,14 +30,14 @@ AfeWakeWord::~AfeWakeWord() { vEventGroupDelete(event_group_); } -void AfeWakeWord::Initialize(AudioCodec* codec) { +bool AfeWakeWord::Initialize(AudioCodec* codec) { codec_ = codec; int ref_num = codec_->input_reference() ? 1 : 0; srmodel_list_t *models = esp_srmodel_init("model"); if (models == nullptr || models->num == -1) { ESP_LOGE(TAG, "Failed to initialize wakenet model"); - return; + return false; } for (int i = 0; i < models->num; i++) { ESP_LOGI(TAG, "Model %d: %s", i, models->model_name[i]); @@ -75,6 +75,8 @@ void AfeWakeWord::Initialize(AudioCodec* codec) { this_->AudioDetectionTask(); vTaskDelete(NULL); }, "audio_detection", 4096, this, 3, nullptr); + + return true; } void AfeWakeWord::OnWakeWordDetected(std::function callback) { diff --git a/main/audio/wake_words/afe_wake_word.h b/main/audio/wake_words/afe_wake_word.h index a3d3128c..14b8cf58 100644 --- a/main/audio/wake_words/afe_wake_word.h +++ b/main/audio/wake_words/afe_wake_word.h @@ -23,7 +23,7 @@ public: AfeWakeWord(); ~AfeWakeWord(); - void Initialize(AudioCodec* codec); + bool Initialize(AudioCodec* codec); void Feed(const std::vector& data); void OnWakeWordDetected(std::function callback); void Start(); diff --git a/main/audio/wake_words/custom_wake_word.cc b/main/audio/wake_words/custom_wake_word.cc index c13c94e7..c721a468 100644 --- a/main/audio/wake_words/custom_wake_word.cc +++ b/main/audio/wake_words/custom_wake_word.cc @@ -31,6 +31,12 @@ CustomWakeWord::~CustomWakeWord() { afe_iface_->destroy(afe_data_); } + // 清理 multinet 资源 + if (multinet_model_data_ != nullptr && multinet_ != nullptr) { + multinet_->destroy(multinet_model_data_); + multinet_model_data_ = nullptr; + } + if (wake_word_encode_task_stack_ != nullptr) { heap_caps_free(wake_word_encode_task_stack_); } @@ -38,15 +44,37 @@ CustomWakeWord::~CustomWakeWord() { vEventGroupDelete(event_group_); } -void CustomWakeWord::Initialize(AudioCodec* codec) { +bool CustomWakeWord::Initialize(AudioCodec* codec) { codec_ = codec; - int ref_num = codec_->input_reference() ? 1 : 0; models = esp_srmodel_init("model"); if (models == nullptr || models->num == -1) { ESP_LOGE(TAG, "Failed to initialize wakenet model"); - return; + return false; } + + // 初始化 multinet (命令词识别) + mn_name_ = esp_srmodel_filter(models, ESP_MN_PREFIX, ESP_MN_CHINESE); + if (mn_name_ == nullptr) { + ESP_LOGE(TAG, "Failed to initialize multinet, mn_name is nullptr"); + ESP_LOGI(TAG, "Please refer to https://pcn7cs20v8cr.feishu.cn/wiki/CpQjwQsCJiQSWSkYEvrcxcbVnwh to add custom wake word"); + return false; + } + + ESP_LOGI(TAG, "multinet:%s", mn_name_); + multinet_ = esp_mn_handle_from_name(mn_name_); + multinet_model_data_ = multinet_->create(mn_name_, 2000); // 2秒超时 + multinet_->set_det_threshold(multinet_model_data_, 0.5); + esp_mn_commands_clear(); + esp_mn_commands_add(1, CONFIG_CUSTOM_WAKE_WORD); // 添加自定义唤醒词作为命令词 + esp_mn_commands_update(); + + // 打印所有的命令词 + multinet_->print_active_speech_commands(multinet_model_data_); + ESP_LOGI(TAG, "Custom wake word: %s", CONFIG_CUSTOM_WAKE_WORD); + + // 初始化 afe + int ref_num = codec_->input_reference() ? 1 : 0; std::string input_format; for (int i = 0; i < codec_->input_channels() - ref_num; i++) { input_format.push_back('M'); @@ -70,6 +98,8 @@ void CustomWakeWord::Initialize(AudioCodec* codec) { this_->AudioDetectionTask(); vTaskDelete(NULL); }, "audio_detection", 16384, this, 3, nullptr); + + return true; } void CustomWakeWord::OnWakeWordDetected(std::function callback) { @@ -105,23 +135,16 @@ void CustomWakeWord::AudioDetectionTask() { auto fetch_size = afe_iface_->get_fetch_chunksize(afe_data_); auto feed_size = afe_iface_->get_feed_chunksize(afe_data_); - // 初始化 multinet (命令词识别) - char *mn_name = esp_srmodel_filter(models, ESP_MN_PREFIX, ESP_MN_CHINESE); - ESP_LOGI(TAG, "multinet:%s", mn_name); - esp_mn_iface_t *multinet = esp_mn_handle_from_name(mn_name); - model_iface_data_t *model_data = multinet->create(mn_name, 2000); // 2秒超时 - multinet->set_det_threshold(model_data, 0.5); - esp_mn_commands_clear(); - esp_mn_commands_add(1, CONFIG_CUSTOM_WAKE_WORD); // 添加自定义唤醒词作为命令词 - esp_mn_commands_update(); - int mu_chunksize = multinet->get_samp_chunksize(model_data); + // 检查 multinet 是否已正确初始化 + if (multinet_ == nullptr || multinet_model_data_ == nullptr) { + ESP_LOGE(TAG, "Multinet not initialized properly"); + return; + } + + int mu_chunksize = multinet_->get_samp_chunksize(multinet_model_data_); assert(mu_chunksize == feed_size); - // 打印所有的命令词 - multinet->print_active_speech_commands(model_data); - ESP_LOGI(TAG, "Audio detection task started, feed size: %d fetch size: %d", feed_size, fetch_size); - ESP_LOGI(TAG, "Custom wake word: %s", CONFIG_CUSTOM_WAKE_WORD); // 禁用wakenet，直接使用multinet检测自定义唤醒词 afe_iface_->disable_wakenet(afe_data_); @@ -139,14 +162,14 @@ void CustomWakeWord::AudioDetectionTask() { StoreWakeWordData(res->data, res->data_size / sizeof(int16_t)); // 直接使用multinet检测自定义唤醒词 - esp_mn_state_t mn_state = multinet->detect(model_data, res->data); + esp_mn_state_t mn_state = multinet_->detect(multinet_model_data_, res->data); if (mn_state == ESP_MN_STATE_DETECTING) { // 仍在检测中，继续 continue; } else if (mn_state == ESP_MN_STATE_DETECTED) { // 检测到自定义唤醒词 - esp_mn_results_t *mn_result = multinet->get_results(model_data); + esp_mn_results_t *mn_result = multinet_->get_results(multinet_model_data_); ESP_LOGI(TAG, "Custom wake word detected: command_id=%d, string=%s, prob=%f", mn_result->command_id[0], mn_result->string, mn_result->prob[0]); @@ -163,23 +186,17 @@ void CustomWakeWord::AudioDetectionTask() { } // 清理multinet状态，准备下次检测 - multinet->clean(model_data); + multinet_->clean(multinet_model_data_); ESP_LOGI(TAG, "Ready for next detection"); } } else if (mn_state == ESP_MN_STATE_TIMEOUT) { // 超时，清理状态继续检测 ESP_LOGD(TAG, "Command word detection timeout, cleaning state"); - multinet->clean(model_data); + multinet_->clean(multinet_model_data_); continue; } } - // 清理资源 - if (model_data) { - multinet->destroy(model_data); - model_data = NULL; - } - ESP_LOGI(TAG, "Audio detection task ended"); } diff --git a/main/audio/wake_words/custom_wake_word.h b/main/audio/wake_words/custom_wake_word.h index 66fb0b55..d31bff96 100644 --- a/main/audio/wake_words/custom_wake_word.h +++ b/main/audio/wake_words/custom_wake_word.h @@ -28,7 +28,7 @@ public: CustomWakeWord(); ~CustomWakeWord(); - void Initialize(AudioCodec* codec); + bool Initialize(AudioCodec* codec); void Feed(const std::vector& data); void OnWakeWordDetected(std::function callback); void Start(); @@ -42,6 +42,11 @@ private: esp_afe_sr_iface_t* afe_iface_ = nullptr; esp_afe_sr_data_t* afe_data_ = nullptr; srmodel_list_t *models = nullptr; + + // multinet 相关成员变量 + esp_mn_iface_t* multinet_ = nullptr; + model_iface_data_t* multinet_model_data_ = nullptr; + char* mn_name_ = nullptr; char* wakenet_model_ = NULL; std::vector wake_words_; diff --git a/main/audio/wake_words/esp_wake_word.cc b/main/audio/wake_words/esp_wake_word.cc index dbb94cc2..36fe3deb 100644 --- a/main/audio/wake_words/esp_wake_word.cc +++ b/main/audio/wake_words/esp_wake_word.cc @@ -23,19 +23,19 @@ EspWakeWord::~EspWakeWord() { vEventGroupDelete(event_group_); } -void EspWakeWord::Initialize(AudioCodec* codec) { +bool EspWakeWord::Initialize(AudioCodec* codec) { codec_ = codec; wakenet_model_ = esp_srmodel_init("model"); if (wakenet_model_ == nullptr || wakenet_model_->num == -1) { ESP_LOGE(TAG, "Failed to initialize wakenet model"); - return; + return false; } if(wakenet_model_->num > 1) { ESP_LOGW(TAG, "More than one model found, using the first one"); } else if (wakenet_model_->num == 0) { ESP_LOGE(TAG, "No model found"); - return; + return false; } char *model_name = wakenet_model_->model_name[0]; wakenet_iface_ = (esp_wn_iface_t*)esp_wn_handle_from_name(model_name); @@ -44,6 +44,8 @@ void EspWakeWord::Initialize(AudioCodec* codec) { int frequency = wakenet_iface_->get_samp_rate(wakenet_data_); int audio_chunksize = wakenet_iface_->get_samp_chunksize(wakenet_data_); ESP_LOGI(TAG, "Wake word(%s),freq: %d, chunksize: %d", model_name, frequency, audio_chunksize); + + return true; } void EspWakeWord::OnWakeWordDetected(std::function callback) { diff --git a/main/audio/wake_words/esp_wake_word.h b/main/audio/wake_words/esp_wake_word.h index 8361f541..e0be0e64 100644 --- a/main/audio/wake_words/esp_wake_word.h +++ b/main/audio/wake_words/esp_wake_word.h @@ -24,7 +24,7 @@ public: EspWakeWord(); ~EspWakeWord(); - void Initialize(AudioCodec* codec); + bool Initialize(AudioCodec* codec); void Feed(const std::vector& data); void OnWakeWordDetected(std::function callback); void Start(); diff --git a/main/boards/xmini-c3-v3/config.json b/main/boards/xmini-c3-v3/config.json index 26851771..e830af0d 100644 --- a/main/boards/xmini-c3-v3/config.json +++ b/main/boards/xmini-c3-v3/config.json @@ -6,6 +6,7 @@ "sdkconfig_append": [ "CONFIG_PM_ENABLE=y", "CONFIG_FREERTOS_USE_TICKLESS_IDLE=y", + "CONFIG_USE_ESP_WAKE_WORD=y", "CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y" ] }