#include "custom_wake_word.h" #include "audio_service.h" #include "system_info.h" #include #include "esp_mn_iface.h" #include "esp_mn_models.h" #include "esp_mn_speech_commands.h" #define TAG "CustomWakeWord" CustomWakeWord::CustomWakeWord() : wake_word_pcm_(), wake_word_opus_() { } CustomWakeWord::~CustomWakeWord() { if (multinet_model_data_ != nullptr && multinet_ != nullptr) { multinet_->destroy(multinet_model_data_); multinet_model_data_ = nullptr; } if (wake_word_encode_task_stack_ != nullptr) { heap_caps_free(wake_word_encode_task_stack_); } if (wake_word_encode_task_buffer_ != nullptr) { heap_caps_free(wake_word_encode_task_buffer_); } if (models_ != nullptr) { esp_srmodel_deinit(models_); } } bool CustomWakeWord::Initialize(AudioCodec* codec) { codec_ = codec; models_ = esp_srmodel_init("model"); if (models_ == nullptr || models_->num == -1) { ESP_LOGE(TAG, "Failed to initialize wakenet model"); return false; } // 初始化 multinet (命令词识别) mn_name_ = esp_srmodel_filter(models_, ESP_MN_PREFIX, ESP_MN_CHINESE); if (mn_name_ == nullptr) { ESP_LOGE(TAG, "Failed to initialize multinet, mn_name is nullptr"); ESP_LOGI(TAG, "Please refer to https://pcn7cs20v8cr.feishu.cn/wiki/CpQjwQsCJiQSWSkYEvrcxcbVnwh to add custom wake word"); return false; } ESP_LOGI(TAG, "multinet: %s", mn_name_); multinet_ = esp_mn_handle_from_name(mn_name_); multinet_model_data_ = multinet_->create(mn_name_, 3000); // 3 秒超时 multinet_->set_det_threshold(multinet_model_data_, CONFIG_CUSTOM_WAKE_WORD_THRESHOLD / 100.0f); esp_mn_commands_clear(); esp_mn_commands_add(1, CONFIG_CUSTOM_WAKE_WORD); esp_mn_commands_update(); multinet_->print_active_speech_commands(multinet_model_data_); return true; } void CustomWakeWord::OnWakeWordDetected(std::function callback) { wake_word_detected_callback_ = callback; } void CustomWakeWord::Start() { running_ = true; } void CustomWakeWord::Stop() { running_ = false; } void CustomWakeWord::Feed(const std::vector& data) { if (multinet_model_data_ == nullptr || !running_) { return; } esp_mn_state_t mn_state; // If input channels is 2, we need to fetch the left channel data if (codec_->input_channels() == 2) { auto mono_data = std::vector(data.size() / 2); for (size_t i = 0, j = 0; i < mono_data.size(); ++i, j += 2) { mono_data[i] = data[j]; } StoreWakeWordData(mono_data); mn_state = multinet_->detect(multinet_model_data_, const_cast(mono_data.data())); } else { StoreWakeWordData(data); mn_state = multinet_->detect(multinet_model_data_, const_cast(data.data())); } if (mn_state == ESP_MN_STATE_DETECTING) { return; } else if (mn_state == ESP_MN_STATE_DETECTED) { esp_mn_results_t *mn_result = multinet_->get_results(multinet_model_data_); ESP_LOGI(TAG, "Custom wake word detected: command_id=%d, string=%s, prob=%f", mn_result->command_id[0], mn_result->string, mn_result->prob[0]); if (mn_result->command_id[0] == 1) { last_detected_wake_word_ = CONFIG_CUSTOM_WAKE_WORD_DISPLAY; } running_ = false; if (wake_word_detected_callback_) { wake_word_detected_callback_(last_detected_wake_word_); } multinet_->clean(multinet_model_data_); } else if (mn_state == ESP_MN_STATE_TIMEOUT) { ESP_LOGD(TAG, "Command word detection timeout, cleaning state"); multinet_->clean(multinet_model_data_); } } size_t CustomWakeWord::GetFeedSize() { if (multinet_model_data_ == nullptr) { return 0; } return multinet_->get_samp_chunksize(multinet_model_data_) * codec_->input_channels(); } void CustomWakeWord::StoreWakeWordData(const std::vector& data) { // store audio data to wake_word_pcm_ wake_word_pcm_.push_back(data); // keep about 2 seconds of data, detect duration is 30ms (sample_rate == 16000, chunksize == 512) while (wake_word_pcm_.size() > 2000 / 30) { wake_word_pcm_.pop_front(); } } void CustomWakeWord::EncodeWakeWordData() { const size_t stack_size = 4096 * 7; wake_word_opus_.clear(); if (wake_word_encode_task_stack_ == nullptr) { wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM); assert(wake_word_encode_task_stack_ != nullptr); } if (wake_word_encode_task_buffer_ == nullptr) { wake_word_encode_task_buffer_ = (StaticTask_t*)heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL); assert(wake_word_encode_task_buffer_ != nullptr); } wake_word_encode_task_ = xTaskCreateStatic([](void* arg) { auto this_ = (CustomWakeWord*)arg; { auto start_time = esp_timer_get_time(); auto encoder = std::make_unique(16000, 1, OPUS_FRAME_DURATION_MS); encoder->SetComplexity(0); // 0 is the fastest int packets = 0; for (auto& pcm: this_->wake_word_pcm_) { encoder->Encode(std::move(pcm), [this_](std::vector&& opus) { std::lock_guard lock(this_->wake_word_mutex_); this_->wake_word_opus_.emplace_back(std::move(opus)); this_->wake_word_cv_.notify_all(); }); packets++; } this_->wake_word_pcm_.clear(); auto end_time = esp_timer_get_time(); ESP_LOGI(TAG, "Encode wake word opus %d packets in %ld ms", packets, (long)((end_time - start_time) / 1000)); std::lock_guard lock(this_->wake_word_mutex_); this_->wake_word_opus_.push_back(std::vector()); this_->wake_word_cv_.notify_all(); } vTaskDelete(NULL); }, "encode_wake_word", stack_size, this, 2, wake_word_encode_task_stack_, wake_word_encode_task_buffer_); } bool CustomWakeWord::GetWakeWordOpus(std::vector& opus) { std::unique_lock lock(wake_word_mutex_); wake_word_cv_.wait(lock, [this]() { return !wake_word_opus_.empty(); }); opus.swap(wake_word_opus_.front()); wake_word_opus_.pop_front(); return !opus.empty(); }