devttl/xiaozhi-esp32
1
0
Fork 0
forked from xiaozhi/xiaozhi-esp32
Code Pull Requests Activity

Add custom wakeword threshold option (#1003)

Browse Source
This commit is contained in:
Xiaoxia
2025-07-29 10:56:52 +08:00
committed by GitHub
parent def6301292
commit 345c8be467
8 changed files with 124 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
main/audio/wake_words/afe_wake_word.cc
View File

@@ -1,9 +1,7 @@
#include "afe_wake_word.h"
#include "application.h"
#include "audio_service.h"
#include <esp_log.h>
#include <model_path.h>
#include <arpa/inet.h>
#include <sstream>
#define DETECTION_RUNNING_EVENT 1
@@ -27,6 +25,14 @@ AfeWakeWord::~AfeWakeWord() {
heap_caps_free(wake_word_encode_task_stack_);
}
if (wake_word_encode_task_buffer_ != nullptr) {
heap_caps_free(wake_word_encode_task_buffer_);
}
if (models_ != nullptr) {
esp_srmodel_deinit(models_);
}
vEventGroupDelete(event_group_);
}
@@ -34,16 +40,16 @@ bool AfeWakeWord::Initialize(AudioCodec* codec) {
codec_ = codec;
int ref_num = codec_->input_reference() ? 1 : 0;
srmodel_list_t *models = esp_srmodel_init("model");
if (models == nullptr || models->num == -1) {
models_ = esp_srmodel_init("model");
if (models_ == nullptr || models_->num == -1) {
ESP_LOGE(TAG, "Failed to initialize wakenet model");
return false;
}
for (int i = 0; i < models->num; i++) {
ESP_LOGI(TAG, "Model %d: %s", i, models->model_name[i]);
if (strstr(models->model_name[i], ESP_WN_PREFIX) != NULL) {
wakenet_model_ = models->model_name[i];
auto words = esp_srmodel_get_wake_words(models, wakenet_model_);
for (int i = 0; i < models_->num; i++) {
ESP_LOGI(TAG, "Model %d: %s", i, models_->model_name[i]);
if (strstr(models_->model_name[i], ESP_WN_PREFIX) != NULL) {
wakenet_model_ = models_->model_name[i];
auto words = esp_srmodel_get_wake_words(models_, wakenet_model_);
// split by ";" to get all wake words
std::stringstream ss(words);
std::string word;
@@ -60,7 +66,7 @@ bool AfeWakeWord::Initialize(AudioCodec* codec) {
for (int i = 0; i < ref_num; i++) {
input_format.push_back('R');
}
afe_config_t* afe_config = afe_config_init(input_format.c_str(), models, AFE_TYPE_SR, AFE_MODE_HIGH_PERF);
afe_config_t* afe_config = afe_config_init(input_format.c_str(), models_, AFE_TYPE_SR, AFE_MODE_HIGH_PERF);
afe_config->aec_init = codec_->input_reference();
afe_config->aec_mode = AEC_MODE_SR_HIGH_PERF;
afe_config->afe_perferred_core = 1;
@@ -146,10 +152,17 @@ void AfeWakeWord::StoreWakeWordData(const int16_t* data, size_t samples) {
}
void AfeWakeWord::EncodeWakeWordData() {
const size_t stack_size = 4096 * 7;
wake_word_opus_.clear();
if (wake_word_encode_task_stack_ == nullptr) {
wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(4096 * 8, MALLOC_CAP_SPIRAM);
wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM);
assert(wake_word_encode_task_stack_ != nullptr);
}
if (wake_word_encode_task_buffer_ == nullptr) {
wake_word_encode_task_buffer_ = (StaticTask_t*)heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL);
assert(wake_word_encode_task_buffer_ != nullptr);
}
wake_word_encode_task_ = xTaskCreateStatic([](void* arg) {
auto this_ = (AfeWakeWord*)arg;
{
@@ -176,7 +189,7 @@ void AfeWakeWord::EncodeWakeWordData() {
this_->wake_word_cv_.notify_all();
}
vTaskDelete(NULL);
}, "encode_detect_packets", 4096 * 8, this, 2, wake_word_encode_task_stack_, &wake_word_encode_task_buffer_);
}, "encode_wake_word", stack_size, this, 2, wake_word_encode_task_stack_, wake_word_encode_task_buffer_);
}
bool AfeWakeWord::GetWakeWordOpus(std::vector<uint8_t>& opus) {

10
main/audio/wake_words/afe_wake_word.h
View File

@@ -7,8 +7,9 @@
#include <esp_afe_sr_models.h>
#include <esp_nsn_models.h>
#include <model_path.h>
#include <list>
#include <deque>
#include <string>
#include <vector>
#include <functional>
@@ -34,6 +35,7 @@ public:
const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; }
private:
srmodel_list_t *models_ = nullptr;
esp_afe_sr_iface_t* afe_iface_ = nullptr;
esp_afe_sr_data_t* afe_data_ = nullptr;
char* wakenet_model_ = NULL;
@@ -44,10 +46,10 @@ private:
std::string last_detected_wake_word_;
TaskHandle_t wake_word_encode_task_ = nullptr;
StaticTask_t wake_word_encode_task_buffer_;
StaticTask_t* wake_word_encode_task_buffer_ = nullptr;
StackType_t* wake_word_encode_task_stack_ = nullptr;
std::list<std::vector<int16_t>> wake_word_pcm_;
std::list<std::vector<uint8_t>> wake_word_opus_;
std::deque<std::vector<int16_t>> wake_word_pcm_;
std::deque<std::vector<uint8_t>> wake_word_opus_;
std::mutex wake_word_mutex_;
std::condition_variable wake_word_cv_;

196
main/audio/wake_words/custom_wake_word.cc
View File

@@ -1,37 +1,21 @@
#include "custom_wake_word.h"
#include "application.h"
#include "audio_service.h"
#include "system_info.h"
#include <esp_log.h>
#include <model_path.h>
#include <arpa/inet.h>
#include "esp_wn_iface.h"
#include "esp_wn_models.h"
#include "esp_afe_sr_iface.h"
#include "esp_afe_sr_models.h"
#include "esp_mn_iface.h"
#include "esp_mn_models.h"
#include "esp_mn_speech_commands.h"
#include <sstream>
#define DETECTION_RUNNING_EVENT 1
#define TAG "CustomWakeWord"
CustomWakeWord::CustomWakeWord()
: afe_data_(nullptr),
wake_word_pcm_(),
wake_word_opus_() {
event_group_ = xEventGroupCreate();
: wake_word_pcm_(), wake_word_opus_() {
}
CustomWakeWord::~CustomWakeWord() {
if (afe_data_ != nullptr) {
afe_iface_->destroy(afe_data_);
}
// 清理 multinet 资源
if (multinet_model_data_ != nullptr && multinet_ != nullptr) {
multinet_->destroy(multinet_model_data_);
multinet_model_data_ = nullptr;
@@ -41,64 +25,41 @@ CustomWakeWord::~CustomWakeWord() {
heap_caps_free(wake_word_encode_task_stack_);
}
vEventGroupDelete(event_group_);
if (wake_word_encode_task_buffer_ != nullptr) {
heap_caps_free(wake_word_encode_task_buffer_);
}
if (models_ != nullptr) {
esp_srmodel_deinit(models_);
}
}
bool CustomWakeWord::Initialize(AudioCodec* codec) {
codec_ = codec;
models = esp_srmodel_init("model");
if (models == nullptr || models->num == -1) {
models_ = esp_srmodel_init("model");
if (models_ == nullptr || models_->num == -1) {
ESP_LOGE(TAG, "Failed to initialize wakenet model");
return false;
}
// 初始化 multinet (命令词识别)
mn_name_ = esp_srmodel_filter(models, ESP_MN_PREFIX, ESP_MN_CHINESE);
mn_name_ = esp_srmodel_filter(models_, ESP_MN_PREFIX, ESP_MN_CHINESE);
if (mn_name_ == nullptr) {
ESP_LOGE(TAG, "Failed to initialize multinet, mn_name is nullptr");
ESP_LOGI(TAG, "Please refer to https://pcn7cs20v8cr.feishu.cn/wiki/CpQjwQsCJiQSWSkYEvrcxcbVnwh to add custom wake word");
return false;
}
ESP_LOGI(TAG, "multinet:%s", mn_name_);
ESP_LOGI(TAG, "multinet: %s", mn_name_);
multinet_ = esp_mn_handle_from_name(mn_name_);
multinet_model_data_ = multinet_->create(mn_name_, 2000); // 2秒超时
multinet_->set_det_threshold(multinet_model_data_, 0.5);
multinet_model_data_ = multinet_->create(mn_name_, 3000); // 3 秒超时
multinet_->set_det_threshold(multinet_model_data_, CONFIG_CUSTOM_WAKE_WORD_THRESHOLD / 100.0f);
esp_mn_commands_clear();
esp_mn_commands_add(1, CONFIG_CUSTOM_WAKE_WORD); // 添加自定义唤醒词作为命令词
esp_mn_commands_add(1, CONFIG_CUSTOM_WAKE_WORD);
esp_mn_commands_update();
// 打印所有的命令词
multinet_->print_active_speech_commands(multinet_model_data_);
ESP_LOGI(TAG, "Custom wake word: %s", CONFIG_CUSTOM_WAKE_WORD);
// 初始化 afe
int ref_num = codec_->input_reference() ? 1 : 0;
std::string input_format;
for (int i = 0; i < codec_->input_channels() - ref_num; i++) {
input_format.push_back('M');
}
for (int i = 0; i < ref_num; i++) {
input_format.push_back('R');
}
afe_config_t* afe_config = afe_config_init(input_format.c_str(), models, AFE_TYPE_SR, AFE_MODE_HIGH_PERF);
afe_config->aec_init = codec_->input_reference();
afe_config->aec_mode = AEC_MODE_SR_HIGH_PERF;
afe_config->afe_perferred_core = 1;
afe_config->afe_perferred_priority = 1;
afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM;
afe_iface_ = esp_afe_handle_from_config(afe_config);
afe_data_ = afe_iface_->create_from_config(afe_config);
xTaskCreate([](void* arg) {
auto this_ = (CustomWakeWord*)arg;
this_->AudioDetectionTask();
vTaskDelete(NULL);
}, "audio_detection", 16384, this, 3, nullptr);
return true;
}
@@ -107,102 +68,54 @@ void CustomWakeWord::OnWakeWordDetected(std::function<void(const std::string& wa
}
void CustomWakeWord::Start() {
xEventGroupSetBits(event_group_, DETECTION_RUNNING_EVENT);
running_ = true;
}
void CustomWakeWord::Stop() {
xEventGroupClearBits(event_group_, DETECTION_RUNNING_EVENT);
if (afe_data_ != nullptr) {
afe_iface_->reset_buffer(afe_data_);
}
running_ = false;
}
void CustomWakeWord::Feed(const std::vector<int16_t>& data) {
if (afe_data_ == nullptr) {
if (multinet_model_data_ == nullptr || !running_) {
return;
}
afe_iface_->feed(afe_data_, data.data());
StoreWakeWordData(data);
esp_mn_state_t mn_state = multinet_->detect(multinet_model_data_, const_cast<int16_t*>(data.data()));
if (mn_state == ESP_MN_STATE_DETECTING) {
return;
} else if (mn_state == ESP_MN_STATE_DETECTED) {
esp_mn_results_t *mn_result = multinet_->get_results(multinet_model_data_);
ESP_LOGI(TAG, "Custom wake word detected: command_id=%d, string=%s, prob=%f",
mn_result->command_id[0], mn_result->string, mn_result->prob[0]);
if (mn_result->command_id[0] == 1) {
last_detected_wake_word_ = CONFIG_CUSTOM_WAKE_WORD_DISPLAY;
}
running_ = false;
if (wake_word_detected_callback_) {
wake_word_detected_callback_(last_detected_wake_word_);
}
multinet_->clean(multinet_model_data_);
} else if (mn_state == ESP_MN_STATE_TIMEOUT) {
ESP_LOGD(TAG, "Command word detection timeout, cleaning state");
multinet_->clean(multinet_model_data_);
}
}
size_t CustomWakeWord::GetFeedSize() {
if (afe_data_ == nullptr) {
if (multinet_model_data_ == nullptr) {
return 0;
}
return afe_iface_->get_feed_chunksize(afe_data_) * codec_->input_channels();
return multinet_->get_samp_chunksize(multinet_model_data_) * codec_->input_channels();
}
void CustomWakeWord::AudioDetectionTask() {
auto fetch_size = afe_iface_->get_fetch_chunksize(afe_data_);
auto feed_size = afe_iface_->get_feed_chunksize(afe_data_);
// 检查 multinet 是否已正确初始化
if (multinet_ == nullptr || multinet_model_data_ == nullptr) {
ESP_LOGE(TAG, "Multinet not initialized properly");
return;
}
int mu_chunksize = multinet_->get_samp_chunksize(multinet_model_data_);
assert(mu_chunksize == feed_size);
ESP_LOGI(TAG, "Audio detection task started, feed size: %d fetch size: %d", feed_size, fetch_size);
// 禁用wakenet，直接使用multinet检测自定义唤醒词
afe_iface_->disable_wakenet(afe_data_);
while (true) {
xEventGroupWaitBits(event_group_, DETECTION_RUNNING_EVENT, pdFALSE, pdTRUE, portMAX_DELAY);
auto res = afe_iface_->fetch_with_delay(afe_data_, portMAX_DELAY);
if (res == nullptr || res->ret_value == ESP_FAIL) {
ESP_LOGW(TAG, "Fetch failed, continue");
continue;
}
// 存储音频数据用于语音识别
StoreWakeWordData(res->data, res->data_size / sizeof(int16_t));
// 直接使用multinet检测自定义唤醒词
esp_mn_state_t mn_state = multinet_->detect(multinet_model_data_, res->data);
if (mn_state == ESP_MN_STATE_DETECTING) {
// 仍在检测中，继续
continue;
} else if (mn_state == ESP_MN_STATE_DETECTED) {
// 检测到自定义唤醒词
esp_mn_results_t *mn_result = multinet_->get_results(multinet_model_data_);
ESP_LOGI(TAG, "Custom wake word detected: command_id=%d, string=%s, prob=%f",
mn_result->command_id[0], mn_result->string, mn_result->prob[0]);
if (mn_result->command_id[0] == 1) { // 自定义唤醒词
ESP_LOGI(TAG, "Custom wake word '%s' detected successfully!", CONFIG_CUSTOM_WAKE_WORD);
// 停止检测
Stop();
last_detected_wake_word_ = CONFIG_CUSTOM_WAKE_WORD_DISPLAY;
// 调用回调
if (wake_word_detected_callback_) {
wake_word_detected_callback_(last_detected_wake_word_);
}
// 清理multinet状态，准备下次检测
multinet_->clean(multinet_model_data_);
ESP_LOGI(TAG, "Ready for next detection");
}
} else if (mn_state == ESP_MN_STATE_TIMEOUT) {
// 超时，清理状态继续检测
ESP_LOGD(TAG, "Command word detection timeout, cleaning state");
multinet_->clean(multinet_model_data_);
continue;
}
}
ESP_LOGI(TAG, "Audio detection task ended");
}
void CustomWakeWord::StoreWakeWordData(const int16_t* data, size_t samples) {
void CustomWakeWord::StoreWakeWordData(const std::vector<int16_t>& data) {
// store audio data to wake_word_pcm_
wake_word_pcm_.emplace_back(std::vector<int16_t>(data, data + samples));
wake_word_pcm_.push_back(data);
// keep about 2 seconds of data, detect duration is 30ms (sample_rate == 16000, chunksize == 512)
while (wake_word_pcm_.size() > 2000 / 30) {
wake_word_pcm_.pop_front();
@@ -210,10 +123,17 @@ void CustomWakeWord::StoreWakeWordData(const int16_t* data, size_t samples) {
}
void CustomWakeWord::EncodeWakeWordData() {
const size_t stack_size = 4096 * 7;
wake_word_opus_.clear();
if (wake_word_encode_task_stack_ == nullptr) {
wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(4096 * 8, MALLOC_CAP_SPIRAM);
wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM);
assert(wake_word_encode_task_stack_ != nullptr);
}
if (wake_word_encode_task_buffer_ == nullptr) {
wake_word_encode_task_buffer_ = (StaticTask_t*)heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL);
assert(wake_word_encode_task_buffer_ != nullptr);
}
wake_word_encode_task_ = xTaskCreateStatic([](void* arg) {
auto this_ = (CustomWakeWord*)arg;
{
@@ -240,7 +160,7 @@ void CustomWakeWord::EncodeWakeWordData() {
this_->wake_word_cv_.notify_all();
}
vTaskDelete(NULL);
}, "encode_detect_packets", 4096 * 8, this, 2, wake_word_encode_task_stack_, &wake_word_encode_task_buffer_);
}, "encode_wake_word", stack_size, this, 2, wake_word_encode_task_stack_, wake_word_encode_task_buffer_);
}
bool CustomWakeWord::GetWakeWordOpus(std::vector<uint8_t>& opus) {

32
main/audio/wake_words/custom_wake_word.h
View File

@@ -1,24 +1,18 @@
#ifndef CUSTOM_WAKE_WORD_H
#define CUSTOM_WAKE_WORD_H
#include <freertos/FreeRTOS.h>
#include <freertos/task.h>
#include <freertos/event_groups.h>
#include <esp_afe_sr_models.h>
#include <esp_afe_sr_iface.h>
#include <esp_nsn_models.h>
#include <esp_wn_iface.h>
#include <esp_wn_models.h>
#include <esp_attr.h>
#include <esp_mn_iface.h>
#include <esp_mn_models.h>
#include <model_path.h>
#include <list>
#include <deque>
#include <string>
#include <vector>
#include <functional>
#include <mutex>
#include <condition_variable>
#include <atomic>
#include "audio_codec.h"
#include "wake_word.h"
@@ -39,32 +33,26 @@ public:
const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; }
private:
esp_afe_sr_iface_t* afe_iface_ = nullptr;
esp_afe_sr_data_t* afe_data_ = nullptr;
srmodel_list_t *models = nullptr;
// multinet 相关成员变量
esp_mn_iface_t* multinet_ = nullptr;
model_iface_data_t* multinet_model_data_ = nullptr;
srmodel_list_t *models_ = nullptr;
char* mn_name_ = nullptr;
char* wakenet_model_ = NULL;
std::vector<std::string> wake_words_;
EventGroupHandle_t event_group_;
std::function<void(const std::string& wake_word)> wake_word_detected_callback_;
AudioCodec* codec_ = nullptr;
std::string last_detected_wake_word_;
std::atomic<bool> running_ = false;
TaskHandle_t wake_word_encode_task_ = nullptr;
StaticTask_t wake_word_encode_task_buffer_;
StaticTask_t* wake_word_encode_task_buffer_ = nullptr;
StackType_t* wake_word_encode_task_stack_ = nullptr;
std::list<std::vector<int16_t>> wake_word_pcm_;
std::list<std::vector<uint8_t>> wake_word_opus_;
std::deque<std::vector<int16_t>> wake_word_pcm_;
std::deque<std::vector<uint8_t>> wake_word_opus_;
std::mutex wake_word_mutex_;
std::condition_variable wake_word_cv_;
void StoreWakeWordData(const int16_t* data, size_t size);
void AudioDetectionTask();
void StoreWakeWordData(const std::vector<int16_t>& data);
};
#endif

19
main/audio/wake_words/esp_wake_word.cc
View File

@@ -1,17 +1,10 @@
#include "esp_wake_word.h"
#include "application.h"
#include <esp_log.h>
#include <model_path.h>
#include <arpa/inet.h>
#include <sstream>
#define DETECTION_RUNNING_EVENT 1
#define TAG "EspWakeWord"
EspWakeWord::EspWakeWord() {
event_group_ = xEventGroupCreate();
}
EspWakeWord::~EspWakeWord() {
@@ -19,8 +12,6 @@ EspWakeWord::~EspWakeWord() {
wakenet_iface_->destroy(wakenet_data_);
esp_srmodel_deinit(wakenet_model_);
}
vEventGroupDelete(event_group_);
}
bool EspWakeWord::Initialize(AudioCodec* codec) {
@@ -53,18 +44,22 @@ void EspWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_
}
void EspWakeWord::Start() {
xEventGroupSetBits(event_group_, DETECTION_RUNNING_EVENT);
running_ = true;
}
void EspWakeWord::Stop() {
xEventGroupClearBits(event_group_, DETECTION_RUNNING_EVENT);
running_ = false;
}
void EspWakeWord::Feed(const std::vector<int16_t>& data) {
if (wakenet_data_ == nullptr || !running_) {
return;
}
int res = wakenet_iface_->detect(wakenet_data_, (int16_t *)data.data());
if (res > 0) {
Stop();
last_detected_wake_word_ = wakenet_iface_->get_word_name(wakenet_data_, res);
running_ = false;
if (wake_word_detected_callback_) {
wake_word_detected_callback_(last_detected_wake_word_);

10
main/audio/wake_words/esp_wake_word.h
View File

@@ -1,20 +1,14 @@
#ifndef ESP_WAKE_WORD_H
#define ESP_WAKE_WORD_H
#include <freertos/FreeRTOS.h>
#include <freertos/task.h>
#include <freertos/event_groups.h>
#include <esp_wn_iface.h>
#include <esp_wn_models.h>
#include <model_path.h>
#include <list>
#include <string>
#include <vector>
#include <functional>
#include <mutex>
#include <condition_variable>
#include <atomic>
#include "audio_codec.h"
#include "wake_word.h"
@@ -38,8 +32,8 @@ private:
esp_wn_iface_t *wakenet_iface_ = nullptr;
model_iface_data_t *wakenet_data_ = nullptr;
srmodel_list_t *wakenet_model_ = nullptr;
EventGroupHandle_t event_group_;
AudioCodec* codec_ = nullptr;
std::atomic<bool> running_ = false;
std::function<void(const std::string& wake_word)> wake_word_detected_callback_;
std::string last_detected_wake_word_;