diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index 938064e0..f57efdb7 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -230,6 +230,8 @@ if(CONFIG_USE_AFE_WAKE_WORD) list(APPEND SOURCES "audio_processing/afe_wake_word.cc") elseif(CONFIG_USE_ESP_WAKE_WORD) list(APPEND SOURCES "audio_processing/esp_wake_word.cc") +elseif(CONFIG_USE_CUSTOM_WAKE_WORD) + list(APPEND SOURCES "audio_processing/custom_wake_word.cc") else() list(APPEND SOURCES "audio_processing/no_wake_word.cc") endif() diff --git a/main/Kconfig.projbuild b/main/Kconfig.projbuild index 12057109..b9940b68 100644 --- a/main/Kconfig.projbuild +++ b/main/Kconfig.projbuild @@ -404,6 +404,28 @@ config USE_AFE_WAKE_WORD help 需要 ESP32 S3 与 PSRAM 支持 +config USE_CUSTOM_WAKE_WORD + bool "Enable Custom Wake Word Detection" + default n + depends on (IDF_TARGET_ESP32S3 || IDF_TARGET_ESP32P4) && SPIRAM && (!USE_AFE_WAKE_WORD) + help + 需要 ESP32 S3 与 PSRAM 支持 + +config CUSTOM_WAKE_WORD + string "Custom Wake Word" + default "ni hao xiao zhi" + depends on USE_CUSTOM_WAKE_WORD + help + 自定义唤醒词，用汉语拼音表示 + +config CUSTOM_WAKE_WORD_DISPLAY + string "Custom Wake Word Display" + default "你好小智" + depends on USE_CUSTOM_WAKE_WORD + help + 自定义唤醒词对应问候语 + + config USE_AUDIO_PROCESSOR bool "Enable Audio Noise Reduction" default y diff --git a/main/application.cc b/main/application.cc index 09f260ba..2816537b 100644 --- a/main/application.cc +++ b/main/application.cc @@ -21,6 +21,8 @@ #include "afe_wake_word.h" #elif CONFIG_USE_ESP_WAKE_WORD #include "esp_wake_word.h" +#elif CONFIG_USE_CUSTOM_WAKE_WORD +#include "custom_wake_word.h" #else #include "no_wake_word.h" #endif @@ -71,6 +73,8 @@ Application::Application() { wake_word_ = std::make_unique(); #elif CONFIG_USE_ESP_WAKE_WORD wake_word_ = std::make_unique(); +#elif CONFIG_USE_CUSTOM_WAKE_WORD + wake_word_ = std::make_unique(); #else wake_word_ = std::make_unique(); #endif @@ -685,7 +689,7 @@ void Application::Start() { } ESP_LOGI(TAG, "Wake word detected: %s", wake_word.c_str()); -#if CONFIG_USE_AFE_WAKE_WORD +#if CONFIG_USE_AFE_WAKE_WORD || CONFIG_USE_CUSTOM_WAKE_WORD AudioStreamPacket packet; // Encode and send the wake word data to the server while (wake_word_->GetWakeWordOpus(packet.payload)) { diff --git a/main/audio_processing/custom_wake_word.cc b/main/audio_processing/custom_wake_word.cc new file mode 100644 index 00000000..37dc5b7f --- /dev/null +++ b/main/audio_processing/custom_wake_word.cc @@ -0,0 +1,241 @@ +#include "custom_wake_word.h" +#include "application.h" + +#include +#include +#include +#include "esp_wn_iface.h" +#include "esp_wn_models.h" +#include "esp_afe_sr_iface.h" +#include "esp_afe_sr_models.h" +#include "esp_mn_iface.h" +#include "esp_mn_models.h" +#include "esp_mn_speech_commands.h" +#include + +#define DETECTION_RUNNING_EVENT 1 + +#define TAG "CustomWakeWord" + + +CustomWakeWord::CustomWakeWord() + : afe_data_(nullptr), + wake_word_pcm_(), + wake_word_opus_() { + + event_group_ = xEventGroupCreate(); +} + +CustomWakeWord::~CustomWakeWord() { + if (afe_data_ != nullptr) { + afe_iface_->destroy(afe_data_); + } + + if (wake_word_encode_task_stack_ != nullptr) { + heap_caps_free(wake_word_encode_task_stack_); + } + + vEventGroupDelete(event_group_); +} + +void CustomWakeWord::Initialize(AudioCodec* codec) { + codec_ = codec; + int ref_num = codec_->input_reference() ? 1 : 0; + + models = esp_srmodel_init("model"); + if (models == nullptr || models->num == -1) { + ESP_LOGE(TAG, "Failed to initialize wakenet model"); + return; + } + std::string input_format; + for (int i = 0; i < codec_->input_channels() - ref_num; i++) { + input_format.push_back('M'); + } + for (int i = 0; i < ref_num; i++) { + input_format.push_back('R'); + } + + afe_config_t* afe_config = afe_config_init(input_format.c_str(), models, AFE_TYPE_SR, AFE_MODE_HIGH_PERF); + afe_config->aec_init = codec_->input_reference(); + afe_config->aec_mode = AEC_MODE_SR_HIGH_PERF; + afe_config->afe_perferred_core = 1; + afe_config->afe_perferred_priority = 1; + afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM; + + afe_iface_ = esp_afe_handle_from_config(afe_config); + afe_data_ = afe_iface_->create_from_config(afe_config); + + xTaskCreate([](void* arg) { + auto this_ = (CustomWakeWord*)arg; + this_->AudioDetectionTask(); + vTaskDelete(NULL); + }, "audio_detection", 16384, this, 3, nullptr); +} + +void CustomWakeWord::OnWakeWordDetected(std::function callback) { + wake_word_detected_callback_ = callback; +} + +void CustomWakeWord::StartDetection() { + xEventGroupSetBits(event_group_, DETECTION_RUNNING_EVENT); +} + +void CustomWakeWord::StopDetection() { + xEventGroupClearBits(event_group_, DETECTION_RUNNING_EVENT); + if (afe_data_ != nullptr) { + afe_iface_->reset_buffer(afe_data_); + } +} + +bool CustomWakeWord::IsDetectionRunning() { + return xEventGroupGetBits(event_group_) & DETECTION_RUNNING_EVENT; +} + +void CustomWakeWord::Feed(const std::vector& data) { + if (afe_data_ == nullptr) { + return; + } + afe_iface_->feed(afe_data_, data.data()); +} + +size_t CustomWakeWord::GetFeedSize() { + if (afe_data_ == nullptr) { + return 0; + } + return afe_iface_->get_feed_chunksize(afe_data_) * codec_->input_channels(); +} + +void CustomWakeWord::AudioDetectionTask() { + auto fetch_size = afe_iface_->get_fetch_chunksize(afe_data_); + auto feed_size = afe_iface_->get_feed_chunksize(afe_data_); + + // 初始化 multinet (命令词识别) + char *mn_name = esp_srmodel_filter(models, ESP_MN_PREFIX, ESP_MN_CHINESE); + ESP_LOGI(TAG, "multinet:%s", mn_name); + esp_mn_iface_t *multinet = esp_mn_handle_from_name(mn_name); + model_iface_data_t *model_data = multinet->create(mn_name, 2000); // 2秒超时 + multinet->set_det_threshold(model_data, 0.5); + esp_mn_commands_clear(); + esp_mn_commands_add(1, CONFIG_CUSTOM_WAKE_WORD); // 添加自定义唤醒词作为命令词 + esp_mn_commands_update(); + int mu_chunksize = multinet->get_samp_chunksize(model_data); + assert(mu_chunksize == feed_size); + + // 打印所有的命令词 + multinet->print_active_speech_commands(model_data); + + ESP_LOGI(TAG, "Audio detection task started, feed size: %d fetch size: %d", feed_size, fetch_size); + ESP_LOGI(TAG, "Custom wake word: %s", CONFIG_CUSTOM_WAKE_WORD); + + // 禁用wakenet，直接使用multinet检测自定义唤醒词 + afe_iface_->disable_wakenet(afe_data_); + + while (true) { + xEventGroupWaitBits(event_group_, DETECTION_RUNNING_EVENT, pdFALSE, pdTRUE, portMAX_DELAY); + + auto res = afe_iface_->fetch_with_delay(afe_data_, portMAX_DELAY); + if (res == nullptr || res->ret_value == ESP_FAIL) { + ESP_LOGW(TAG, "Fetch failed, continue"); + continue; + } + + // 存储音频数据用于语音识别 + StoreWakeWordData(res->data, res->data_size / sizeof(int16_t)); + + // 直接使用multinet检测自定义唤醒词 + esp_mn_state_t mn_state = multinet->detect(model_data, res->data); + + if (mn_state == ESP_MN_STATE_DETECTING) { + // 仍在检测中，继续 + continue; + } else if (mn_state == ESP_MN_STATE_DETECTED) { + // 检测到自定义唤醒词 + esp_mn_results_t *mn_result = multinet->get_results(model_data); + ESP_LOGI(TAG, "Custom wake word detected: command_id=%d, string=%s, prob=%f", + mn_result->command_id[0], mn_result->string, mn_result->prob[0]); + + if (mn_result->command_id[0] == 1) { // 自定义唤醒词 + ESP_LOGI(TAG, "Custom wake word '%s' detected successfully!", CONFIG_CUSTOM_WAKE_WORD); + + // 停止检测 + StopDetection(); + last_detected_wake_word_ = CONFIG_CUSTOM_WAKE_WORD_DISPLAY; + + // 调用回调 + if (wake_word_detected_callback_) { + wake_word_detected_callback_(last_detected_wake_word_); + } + + // 清理multinet状态，准备下次检测 + multinet->clean(model_data); + ESP_LOGI(TAG, "Ready for next detection"); + } + } else if (mn_state == ESP_MN_STATE_TIMEOUT) { + // 超时，清理状态继续检测 + ESP_LOGD(TAG, "Command word detection timeout, cleaning state"); + multinet->clean(model_data); + continue; + } + } + + // 清理资源 + if (model_data) { + multinet->destroy(model_data); + model_data = NULL; + } + + ESP_LOGI(TAG, "Audio detection task ended"); +} + +void CustomWakeWord::StoreWakeWordData(const int16_t* data, size_t samples) { + // store audio data to wake_word_pcm_ + wake_word_pcm_.emplace_back(std::vector(data, data + samples)); + // keep about 2 seconds of data, detect duration is 30ms (sample_rate == 16000, chunksize == 512) + while (wake_word_pcm_.size() > 2000 / 30) { + wake_word_pcm_.pop_front(); + } +} + +void CustomWakeWord::EncodeWakeWordData() { + wake_word_opus_.clear(); + if (wake_word_encode_task_stack_ == nullptr) { + wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(4096 * 8, MALLOC_CAP_SPIRAM); + } + wake_word_encode_task_ = xTaskCreateStatic([](void* arg) { + auto this_ = (CustomWakeWord*)arg; + { + auto start_time = esp_timer_get_time(); + auto encoder = std::make_unique(16000, 1, OPUS_FRAME_DURATION_MS); + encoder->SetComplexity(0); // 0 is the fastest + + int packets = 0; + for (auto& pcm: this_->wake_word_pcm_) { + encoder->Encode(std::move(pcm), [this_](std::vector&& opus) { + std::lock_guard lock(this_->wake_word_mutex_); + this_->wake_word_opus_.emplace_back(std::move(opus)); + this_->wake_word_cv_.notify_all(); + }); + packets++; + } + this_->wake_word_pcm_.clear(); + + auto end_time = esp_timer_get_time(); + ESP_LOGI(TAG, "Encode wake word opus %d packets in %ld ms", packets, (long)((end_time - start_time) / 1000)); + + std::lock_guard lock(this_->wake_word_mutex_); + this_->wake_word_opus_.push_back(std::vector()); + this_->wake_word_cv_.notify_all(); + } + vTaskDelete(NULL); + }, "encode_detect_packets", 4096 * 8, this, 2, wake_word_encode_task_stack_, &wake_word_encode_task_buffer_); +} + +bool CustomWakeWord::GetWakeWordOpus(std::vector& opus) { + std::unique_lock lock(wake_word_mutex_); + wake_word_cv_.wait(lock, [this]() { + return !wake_word_opus_.empty(); + }); + opus.swap(wake_word_opus_.front()); + wake_word_opus_.pop_front(); + return !opus.empty(); +} diff --git a/main/audio_processing/custom_wake_word.h b/main/audio_processing/custom_wake_word.h new file mode 100644 index 00000000..0c873829 --- /dev/null +++ b/main/audio_processing/custom_wake_word.h @@ -0,0 +1,66 @@ +#ifndef CUSTOM_WAKE_WORD_H +#define CUSTOM_WAKE_WORD_H + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "audio_codec.h" +#include "wake_word.h" + +class CustomWakeWord : public WakeWord { +public: + CustomWakeWord(); + ~CustomWakeWord(); + + void Initialize(AudioCodec* codec); + void Feed(const std::vector& data); + void OnWakeWordDetected(std::function callback); + void StartDetection(); + void StopDetection(); + bool IsDetectionRunning(); + size_t GetFeedSize(); + void EncodeWakeWordData(); + bool GetWakeWordOpus(std::vector& opus); + const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; } + +private: + esp_afe_sr_iface_t* afe_iface_ = nullptr; + esp_afe_sr_data_t* afe_data_ = nullptr; + srmodel_list_t *models = nullptr; + + char* wakenet_model_ = NULL; + std::vector wake_words_; + EventGroupHandle_t event_group_; + std::function wake_word_detected_callback_; + AudioCodec* codec_ = nullptr; + std::string last_detected_wake_word_; + + TaskHandle_t wake_word_encode_task_ = nullptr; + StaticTask_t wake_word_encode_task_buffer_; + StackType_t* wake_word_encode_task_stack_ = nullptr; + std::list> wake_word_pcm_; + std::list> wake_word_opus_; + std::mutex wake_word_mutex_; + std::condition_variable wake_word_cv_; + + void StoreWakeWordData(const int16_t* data, size_t size); + void AudioDetectionTask(); +}; + +#endif diff --git a/partitions/v1/16m_custom_wakeword.csv b/partitions/v1/16m_custom_wakeword.csv new file mode 100644 index 00000000..868294e5 --- /dev/null +++ b/partitions/v1/16m_custom_wakeword.csv @@ -0,0 +1,8 @@ +# ESP-IDF Partition Table +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x4000, +otadata, data, ota, 0xd000, 0x2000, +phy_init, data, phy, 0xf000, 0x1000, +model, data, spiffs, 0x10000, 0x3f0000, +ota_0, app, ota_0, 0x400000, 6M, +ota_1, app, ota_1, 0xa00000, 6M, \ No newline at end of file