diff --git a/main/Kconfig.projbuild b/main/Kconfig.projbuild index 2cb27d67..4184698b 100644 --- a/main/Kconfig.projbuild +++ b/main/Kconfig.projbuild @@ -421,19 +421,26 @@ config USE_CUSTOM_WAKE_WORD config CUSTOM_WAKE_WORD string "Custom Wake Word" - default "ni hao xiao zhi" + default "xiao tu dou" depends on USE_CUSTOM_WAKE_WORD help - 自定义唤醒词,用汉语拼音表示 + 自定义唤醒词,中文用拼音表示,每个字之间用空格隔开 config CUSTOM_WAKE_WORD_DISPLAY string "Custom Wake Word Display" - default "你好小智" + default "小土豆" depends on USE_CUSTOM_WAKE_WORD help - 自定义唤醒词对应问候语 - - + 唤醒后发送给服务器的问候语 + +config CUSTOM_WAKE_WORD_THRESHOLD + int "Custom Wake Word Threshold (%)" + default 20 + range 1 99 + depends on USE_CUSTOM_WAKE_WORD + help + 自定义唤醒词阈值,范围1-99,越小越敏感,默认10 + config USE_AUDIO_PROCESSOR bool "Enable Audio Noise Reduction" default y diff --git a/main/audio/wake_words/afe_wake_word.cc b/main/audio/wake_words/afe_wake_word.cc index 8229b4c2..34fe43a4 100644 --- a/main/audio/wake_words/afe_wake_word.cc +++ b/main/audio/wake_words/afe_wake_word.cc @@ -1,9 +1,7 @@ #include "afe_wake_word.h" -#include "application.h" +#include "audio_service.h" #include -#include -#include #include #define DETECTION_RUNNING_EVENT 1 @@ -27,6 +25,14 @@ AfeWakeWord::~AfeWakeWord() { heap_caps_free(wake_word_encode_task_stack_); } + if (wake_word_encode_task_buffer_ != nullptr) { + heap_caps_free(wake_word_encode_task_buffer_); + } + + if (models_ != nullptr) { + esp_srmodel_deinit(models_); + } + vEventGroupDelete(event_group_); } @@ -34,16 +40,16 @@ bool AfeWakeWord::Initialize(AudioCodec* codec) { codec_ = codec; int ref_num = codec_->input_reference() ? 1 : 0; - srmodel_list_t *models = esp_srmodel_init("model"); - if (models == nullptr || models->num == -1) { + models_ = esp_srmodel_init("model"); + if (models_ == nullptr || models_->num == -1) { ESP_LOGE(TAG, "Failed to initialize wakenet model"); return false; } - for (int i = 0; i < models->num; i++) { - ESP_LOGI(TAG, "Model %d: %s", i, models->model_name[i]); - if (strstr(models->model_name[i], ESP_WN_PREFIX) != NULL) { - wakenet_model_ = models->model_name[i]; - auto words = esp_srmodel_get_wake_words(models, wakenet_model_); + for (int i = 0; i < models_->num; i++) { + ESP_LOGI(TAG, "Model %d: %s", i, models_->model_name[i]); + if (strstr(models_->model_name[i], ESP_WN_PREFIX) != NULL) { + wakenet_model_ = models_->model_name[i]; + auto words = esp_srmodel_get_wake_words(models_, wakenet_model_); // split by ";" to get all wake words std::stringstream ss(words); std::string word; @@ -60,7 +66,7 @@ bool AfeWakeWord::Initialize(AudioCodec* codec) { for (int i = 0; i < ref_num; i++) { input_format.push_back('R'); } - afe_config_t* afe_config = afe_config_init(input_format.c_str(), models, AFE_TYPE_SR, AFE_MODE_HIGH_PERF); + afe_config_t* afe_config = afe_config_init(input_format.c_str(), models_, AFE_TYPE_SR, AFE_MODE_HIGH_PERF); afe_config->aec_init = codec_->input_reference(); afe_config->aec_mode = AEC_MODE_SR_HIGH_PERF; afe_config->afe_perferred_core = 1; @@ -146,10 +152,17 @@ void AfeWakeWord::StoreWakeWordData(const int16_t* data, size_t samples) { } void AfeWakeWord::EncodeWakeWordData() { + const size_t stack_size = 4096 * 7; wake_word_opus_.clear(); if (wake_word_encode_task_stack_ == nullptr) { - wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(4096 * 8, MALLOC_CAP_SPIRAM); + wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM); + assert(wake_word_encode_task_stack_ != nullptr); } + if (wake_word_encode_task_buffer_ == nullptr) { + wake_word_encode_task_buffer_ = (StaticTask_t*)heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL); + assert(wake_word_encode_task_buffer_ != nullptr); + } + wake_word_encode_task_ = xTaskCreateStatic([](void* arg) { auto this_ = (AfeWakeWord*)arg; { @@ -176,7 +189,7 @@ void AfeWakeWord::EncodeWakeWordData() { this_->wake_word_cv_.notify_all(); } vTaskDelete(NULL); - }, "encode_detect_packets", 4096 * 8, this, 2, wake_word_encode_task_stack_, &wake_word_encode_task_buffer_); + }, "encode_wake_word", stack_size, this, 2, wake_word_encode_task_stack_, wake_word_encode_task_buffer_); } bool AfeWakeWord::GetWakeWordOpus(std::vector& opus) { diff --git a/main/audio/wake_words/afe_wake_word.h b/main/audio/wake_words/afe_wake_word.h index 14b8cf58..ca8c6767 100644 --- a/main/audio/wake_words/afe_wake_word.h +++ b/main/audio/wake_words/afe_wake_word.h @@ -7,8 +7,9 @@ #include #include +#include -#include +#include #include #include #include @@ -34,6 +35,7 @@ public: const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; } private: + srmodel_list_t *models_ = nullptr; esp_afe_sr_iface_t* afe_iface_ = nullptr; esp_afe_sr_data_t* afe_data_ = nullptr; char* wakenet_model_ = NULL; @@ -44,10 +46,10 @@ private: std::string last_detected_wake_word_; TaskHandle_t wake_word_encode_task_ = nullptr; - StaticTask_t wake_word_encode_task_buffer_; + StaticTask_t* wake_word_encode_task_buffer_ = nullptr; StackType_t* wake_word_encode_task_stack_ = nullptr; - std::list> wake_word_pcm_; - std::list> wake_word_opus_; + std::deque> wake_word_pcm_; + std::deque> wake_word_opus_; std::mutex wake_word_mutex_; std::condition_variable wake_word_cv_; diff --git a/main/audio/wake_words/custom_wake_word.cc b/main/audio/wake_words/custom_wake_word.cc index c721a468..bf654330 100644 --- a/main/audio/wake_words/custom_wake_word.cc +++ b/main/audio/wake_words/custom_wake_word.cc @@ -1,37 +1,21 @@ #include "custom_wake_word.h" -#include "application.h" +#include "audio_service.h" +#include "system_info.h" #include -#include -#include -#include "esp_wn_iface.h" -#include "esp_wn_models.h" -#include "esp_afe_sr_iface.h" -#include "esp_afe_sr_models.h" #include "esp_mn_iface.h" #include "esp_mn_models.h" #include "esp_mn_speech_commands.h" -#include -#define DETECTION_RUNNING_EVENT 1 #define TAG "CustomWakeWord" CustomWakeWord::CustomWakeWord() - : afe_data_(nullptr), - wake_word_pcm_(), - wake_word_opus_() { - - event_group_ = xEventGroupCreate(); + : wake_word_pcm_(), wake_word_opus_() { } CustomWakeWord::~CustomWakeWord() { - if (afe_data_ != nullptr) { - afe_iface_->destroy(afe_data_); - } - - // 清理 multinet 资源 if (multinet_model_data_ != nullptr && multinet_ != nullptr) { multinet_->destroy(multinet_model_data_); multinet_model_data_ = nullptr; @@ -41,64 +25,41 @@ CustomWakeWord::~CustomWakeWord() { heap_caps_free(wake_word_encode_task_stack_); } - vEventGroupDelete(event_group_); + if (wake_word_encode_task_buffer_ != nullptr) { + heap_caps_free(wake_word_encode_task_buffer_); + } + + if (models_ != nullptr) { + esp_srmodel_deinit(models_); + } } bool CustomWakeWord::Initialize(AudioCodec* codec) { codec_ = codec; - models = esp_srmodel_init("model"); - if (models == nullptr || models->num == -1) { + models_ = esp_srmodel_init("model"); + if (models_ == nullptr || models_->num == -1) { ESP_LOGE(TAG, "Failed to initialize wakenet model"); return false; } // 初始化 multinet (命令词识别) - mn_name_ = esp_srmodel_filter(models, ESP_MN_PREFIX, ESP_MN_CHINESE); + mn_name_ = esp_srmodel_filter(models_, ESP_MN_PREFIX, ESP_MN_CHINESE); if (mn_name_ == nullptr) { ESP_LOGE(TAG, "Failed to initialize multinet, mn_name is nullptr"); ESP_LOGI(TAG, "Please refer to https://pcn7cs20v8cr.feishu.cn/wiki/CpQjwQsCJiQSWSkYEvrcxcbVnwh to add custom wake word"); return false; } - ESP_LOGI(TAG, "multinet:%s", mn_name_); + ESP_LOGI(TAG, "multinet: %s", mn_name_); multinet_ = esp_mn_handle_from_name(mn_name_); - multinet_model_data_ = multinet_->create(mn_name_, 2000); // 2秒超时 - multinet_->set_det_threshold(multinet_model_data_, 0.5); + multinet_model_data_ = multinet_->create(mn_name_, 3000); // 3 秒超时 + multinet_->set_det_threshold(multinet_model_data_, CONFIG_CUSTOM_WAKE_WORD_THRESHOLD / 100.0f); esp_mn_commands_clear(); - esp_mn_commands_add(1, CONFIG_CUSTOM_WAKE_WORD); // 添加自定义唤醒词作为命令词 + esp_mn_commands_add(1, CONFIG_CUSTOM_WAKE_WORD); esp_mn_commands_update(); - // 打印所有的命令词 multinet_->print_active_speech_commands(multinet_model_data_); - ESP_LOGI(TAG, "Custom wake word: %s", CONFIG_CUSTOM_WAKE_WORD); - - // 初始化 afe - int ref_num = codec_->input_reference() ? 1 : 0; - std::string input_format; - for (int i = 0; i < codec_->input_channels() - ref_num; i++) { - input_format.push_back('M'); - } - for (int i = 0; i < ref_num; i++) { - input_format.push_back('R'); - } - - afe_config_t* afe_config = afe_config_init(input_format.c_str(), models, AFE_TYPE_SR, AFE_MODE_HIGH_PERF); - afe_config->aec_init = codec_->input_reference(); - afe_config->aec_mode = AEC_MODE_SR_HIGH_PERF; - afe_config->afe_perferred_core = 1; - afe_config->afe_perferred_priority = 1; - afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM; - - afe_iface_ = esp_afe_handle_from_config(afe_config); - afe_data_ = afe_iface_->create_from_config(afe_config); - - xTaskCreate([](void* arg) { - auto this_ = (CustomWakeWord*)arg; - this_->AudioDetectionTask(); - vTaskDelete(NULL); - }, "audio_detection", 16384, this, 3, nullptr); - return true; } @@ -107,102 +68,54 @@ void CustomWakeWord::OnWakeWordDetected(std::functionreset_buffer(afe_data_); - } + running_ = false; } void CustomWakeWord::Feed(const std::vector& data) { - if (afe_data_ == nullptr) { + if (multinet_model_data_ == nullptr || !running_) { return; } - afe_iface_->feed(afe_data_, data.data()); + + StoreWakeWordData(data); + + esp_mn_state_t mn_state = multinet_->detect(multinet_model_data_, const_cast(data.data())); + + if (mn_state == ESP_MN_STATE_DETECTING) { + return; + } else if (mn_state == ESP_MN_STATE_DETECTED) { + esp_mn_results_t *mn_result = multinet_->get_results(multinet_model_data_); + ESP_LOGI(TAG, "Custom wake word detected: command_id=%d, string=%s, prob=%f", + mn_result->command_id[0], mn_result->string, mn_result->prob[0]); + + if (mn_result->command_id[0] == 1) { + last_detected_wake_word_ = CONFIG_CUSTOM_WAKE_WORD_DISPLAY; + } + running_ = false; + + if (wake_word_detected_callback_) { + wake_word_detected_callback_(last_detected_wake_word_); + } + multinet_->clean(multinet_model_data_); + } else if (mn_state == ESP_MN_STATE_TIMEOUT) { + ESP_LOGD(TAG, "Command word detection timeout, cleaning state"); + multinet_->clean(multinet_model_data_); + } } size_t CustomWakeWord::GetFeedSize() { - if (afe_data_ == nullptr) { + if (multinet_model_data_ == nullptr) { return 0; } - return afe_iface_->get_feed_chunksize(afe_data_) * codec_->input_channels(); + return multinet_->get_samp_chunksize(multinet_model_data_) * codec_->input_channels(); } -void CustomWakeWord::AudioDetectionTask() { - auto fetch_size = afe_iface_->get_fetch_chunksize(afe_data_); - auto feed_size = afe_iface_->get_feed_chunksize(afe_data_); - - // 检查 multinet 是否已正确初始化 - if (multinet_ == nullptr || multinet_model_data_ == nullptr) { - ESP_LOGE(TAG, "Multinet not initialized properly"); - return; - } - - int mu_chunksize = multinet_->get_samp_chunksize(multinet_model_data_); - assert(mu_chunksize == feed_size); - - ESP_LOGI(TAG, "Audio detection task started, feed size: %d fetch size: %d", feed_size, fetch_size); - - // 禁用wakenet,直接使用multinet检测自定义唤醒词 - afe_iface_->disable_wakenet(afe_data_); - - while (true) { - xEventGroupWaitBits(event_group_, DETECTION_RUNNING_EVENT, pdFALSE, pdTRUE, portMAX_DELAY); - - auto res = afe_iface_->fetch_with_delay(afe_data_, portMAX_DELAY); - if (res == nullptr || res->ret_value == ESP_FAIL) { - ESP_LOGW(TAG, "Fetch failed, continue"); - continue; - } - - // 存储音频数据用于语音识别 - StoreWakeWordData(res->data, res->data_size / sizeof(int16_t)); - - // 直接使用multinet检测自定义唤醒词 - esp_mn_state_t mn_state = multinet_->detect(multinet_model_data_, res->data); - - if (mn_state == ESP_MN_STATE_DETECTING) { - // 仍在检测中,继续 - continue; - } else if (mn_state == ESP_MN_STATE_DETECTED) { - // 检测到自定义唤醒词 - esp_mn_results_t *mn_result = multinet_->get_results(multinet_model_data_); - ESP_LOGI(TAG, "Custom wake word detected: command_id=%d, string=%s, prob=%f", - mn_result->command_id[0], mn_result->string, mn_result->prob[0]); - - if (mn_result->command_id[0] == 1) { // 自定义唤醒词 - ESP_LOGI(TAG, "Custom wake word '%s' detected successfully!", CONFIG_CUSTOM_WAKE_WORD); - - // 停止检测 - Stop(); - last_detected_wake_word_ = CONFIG_CUSTOM_WAKE_WORD_DISPLAY; - - // 调用回调 - if (wake_word_detected_callback_) { - wake_word_detected_callback_(last_detected_wake_word_); - } - - // 清理multinet状态,准备下次检测 - multinet_->clean(multinet_model_data_); - ESP_LOGI(TAG, "Ready for next detection"); - } - } else if (mn_state == ESP_MN_STATE_TIMEOUT) { - // 超时,清理状态继续检测 - ESP_LOGD(TAG, "Command word detection timeout, cleaning state"); - multinet_->clean(multinet_model_data_); - continue; - } - } - - ESP_LOGI(TAG, "Audio detection task ended"); -} - -void CustomWakeWord::StoreWakeWordData(const int16_t* data, size_t samples) { +void CustomWakeWord::StoreWakeWordData(const std::vector& data) { // store audio data to wake_word_pcm_ - wake_word_pcm_.emplace_back(std::vector(data, data + samples)); + wake_word_pcm_.push_back(data); // keep about 2 seconds of data, detect duration is 30ms (sample_rate == 16000, chunksize == 512) while (wake_word_pcm_.size() > 2000 / 30) { wake_word_pcm_.pop_front(); @@ -210,10 +123,17 @@ void CustomWakeWord::StoreWakeWordData(const int16_t* data, size_t samples) { } void CustomWakeWord::EncodeWakeWordData() { + const size_t stack_size = 4096 * 7; wake_word_opus_.clear(); if (wake_word_encode_task_stack_ == nullptr) { - wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(4096 * 8, MALLOC_CAP_SPIRAM); + wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM); + assert(wake_word_encode_task_stack_ != nullptr); } + if (wake_word_encode_task_buffer_ == nullptr) { + wake_word_encode_task_buffer_ = (StaticTask_t*)heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL); + assert(wake_word_encode_task_buffer_ != nullptr); + } + wake_word_encode_task_ = xTaskCreateStatic([](void* arg) { auto this_ = (CustomWakeWord*)arg; { @@ -240,7 +160,7 @@ void CustomWakeWord::EncodeWakeWordData() { this_->wake_word_cv_.notify_all(); } vTaskDelete(NULL); - }, "encode_detect_packets", 4096 * 8, this, 2, wake_word_encode_task_stack_, &wake_word_encode_task_buffer_); + }, "encode_wake_word", stack_size, this, 2, wake_word_encode_task_stack_, wake_word_encode_task_buffer_); } bool CustomWakeWord::GetWakeWordOpus(std::vector& opus) { diff --git a/main/audio/wake_words/custom_wake_word.h b/main/audio/wake_words/custom_wake_word.h index d31bff96..824d7fd1 100644 --- a/main/audio/wake_words/custom_wake_word.h +++ b/main/audio/wake_words/custom_wake_word.h @@ -1,24 +1,18 @@ #ifndef CUSTOM_WAKE_WORD_H #define CUSTOM_WAKE_WORD_H -#include -#include -#include - -#include -#include -#include -#include -#include +#include #include #include +#include -#include +#include #include #include #include #include #include +#include #include "audio_codec.h" #include "wake_word.h" @@ -39,32 +33,26 @@ public: const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; } private: - esp_afe_sr_iface_t* afe_iface_ = nullptr; - esp_afe_sr_data_t* afe_data_ = nullptr; - srmodel_list_t *models = nullptr; - // multinet 相关成员变量 esp_mn_iface_t* multinet_ = nullptr; model_iface_data_t* multinet_model_data_ = nullptr; + srmodel_list_t *models_ = nullptr; char* mn_name_ = nullptr; - char* wakenet_model_ = NULL; - std::vector wake_words_; - EventGroupHandle_t event_group_; std::function wake_word_detected_callback_; AudioCodec* codec_ = nullptr; std::string last_detected_wake_word_; + std::atomic running_ = false; TaskHandle_t wake_word_encode_task_ = nullptr; - StaticTask_t wake_word_encode_task_buffer_; + StaticTask_t* wake_word_encode_task_buffer_ = nullptr; StackType_t* wake_word_encode_task_stack_ = nullptr; - std::list> wake_word_pcm_; - std::list> wake_word_opus_; + std::deque> wake_word_pcm_; + std::deque> wake_word_opus_; std::mutex wake_word_mutex_; std::condition_variable wake_word_cv_; - void StoreWakeWordData(const int16_t* data, size_t size); - void AudioDetectionTask(); + void StoreWakeWordData(const std::vector& data); }; #endif diff --git a/main/audio/wake_words/esp_wake_word.cc b/main/audio/wake_words/esp_wake_word.cc index 36fe3deb..28299fd0 100644 --- a/main/audio/wake_words/esp_wake_word.cc +++ b/main/audio/wake_words/esp_wake_word.cc @@ -1,17 +1,10 @@ #include "esp_wake_word.h" -#include "application.h" - #include -#include -#include -#include -#define DETECTION_RUNNING_EVENT 1 #define TAG "EspWakeWord" EspWakeWord::EspWakeWord() { - event_group_ = xEventGroupCreate(); } EspWakeWord::~EspWakeWord() { @@ -19,8 +12,6 @@ EspWakeWord::~EspWakeWord() { wakenet_iface_->destroy(wakenet_data_); esp_srmodel_deinit(wakenet_model_); } - - vEventGroupDelete(event_group_); } bool EspWakeWord::Initialize(AudioCodec* codec) { @@ -53,18 +44,22 @@ void EspWakeWord::OnWakeWordDetected(std::function& data) { + if (wakenet_data_ == nullptr || !running_) { + return; + } + int res = wakenet_iface_->detect(wakenet_data_, (int16_t *)data.data()); if (res > 0) { - Stop(); last_detected_wake_word_ = wakenet_iface_->get_word_name(wakenet_data_, res); + running_ = false; if (wake_word_detected_callback_) { wake_word_detected_callback_(last_detected_wake_word_); diff --git a/main/audio/wake_words/esp_wake_word.h b/main/audio/wake_words/esp_wake_word.h index e0be0e64..475ec2dc 100644 --- a/main/audio/wake_words/esp_wake_word.h +++ b/main/audio/wake_words/esp_wake_word.h @@ -1,20 +1,14 @@ #ifndef ESP_WAKE_WORD_H #define ESP_WAKE_WORD_H -#include -#include -#include - #include #include #include -#include #include #include #include -#include -#include +#include #include "audio_codec.h" #include "wake_word.h" @@ -38,8 +32,8 @@ private: esp_wn_iface_t *wakenet_iface_ = nullptr; model_iface_data_t *wakenet_data_ = nullptr; srmodel_list_t *wakenet_model_ = nullptr; - EventGroupHandle_t event_group_; AudioCodec* codec_ = nullptr; + std::atomic running_ = false; std::function wake_word_detected_callback_; std::string last_detected_wake_word_; diff --git a/main/idf_component.yml b/main/idf_component.yml index 15b2dbee..a933effd 100644 --- a/main/idf_component.yml +++ b/main/idf_component.yml @@ -15,10 +15,10 @@ dependencies: 78/esp_lcd_nv3023: ~1.0.0 78/esp-wifi-connect: ~2.4.3 78/esp-opus-encoder: ~2.4.0 - 78/esp-ml307: ~3.2.0 + 78/esp-ml307: ~3.2.4 78/xiaozhi-fonts: ~1.3.2 espressif/led_strip: ^2.5.5 - espressif/esp_codec_dev: ~1.3.2 + espressif/esp_codec_dev: ~1.3.6 espressif/esp-sr: ~2.1.1 espressif/button: ~4.1.3 espressif/knob: ^1.0.0