devttl/xiaozhi-esp32
1
0
Fork 0
forked from xiaozhi/xiaozhi-esp32
Code Pull Requests Activity

增加自定义唤醒词启动失败的提升 (#965)

Browse Source
This commit is contained in:
Xiaoxia
2025-07-22 18:57:25 +08:00
committed by GitHub
parent 15f233e773
commit 3c11cceb43
9 changed files with 67 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
main/audio/audio_service.cc
View File

@@ -468,7 +468,10 @@ void AudioService::EnableWakeWordDetection(bool enable) {
ESP_LOGD(TAG, "%s wake word detection", enable ? "Enabling" : "Disabling");
if (enable) {
if (!wake_word_initialized_) {
wake_word_->Initialize(codec_);
if (!wake_word_->Initialize(codec_)) {
ESP_LOGE(TAG, "Failed to initialize wake word");
return;
}
wake_word_initialized_ = true;
}
wake_word_->Start();

2
main/audio/wake_word.h
View File

@@ -11,7 +11,7 @@ class WakeWord {
public:
virtual ~WakeWord() = default;
virtual void Initialize(AudioCodec* codec) = 0;
virtual bool Initialize(AudioCodec* codec) = 0;
virtual void Feed(const std::vector<int16_t>& data) = 0;
virtual void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) = 0;
virtual void Start() = 0;

6
main/audio/wake_words/afe_wake_word.cc
View File

@@ -30,14 +30,14 @@ AfeWakeWord::~AfeWakeWord() {
vEventGroupDelete(event_group_);
}
void AfeWakeWord::Initialize(AudioCodec* codec) {
bool AfeWakeWord::Initialize(AudioCodec* codec) {
codec_ = codec;
int ref_num = codec_->input_reference() ? 1 : 0;
srmodel_list_t *models = esp_srmodel_init("model");
if (models == nullptr || models->num == -1) {
ESP_LOGE(TAG, "Failed to initialize wakenet model");
return;
return false;
}
for (int i = 0; i < models->num; i++) {
ESP_LOGI(TAG, "Model %d: %s", i, models->model_name[i]);
@@ -75,6 +75,8 @@ void AfeWakeWord::Initialize(AudioCodec* codec) {
this_->AudioDetectionTask();
vTaskDelete(NULL);
}, "audio_detection", 4096, this, 3, nullptr);
return true;
}
void AfeWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {

2
main/audio/wake_words/afe_wake_word.h
View File

@@ -23,7 +23,7 @@ public:
AfeWakeWord();
~AfeWakeWord();
void Initialize(AudioCodec* codec);
bool Initialize(AudioCodec* codec);
void Feed(const std::vector<int16_t>& data);
void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback);
void Start();

71
main/audio/wake_words/custom_wake_word.cc
View File

@@ -31,6 +31,12 @@ CustomWakeWord::~CustomWakeWord() {
afe_iface_->destroy(afe_data_);
}
// 清理 multinet 资源
if (multinet_model_data_ != nullptr && multinet_ != nullptr) {
multinet_->destroy(multinet_model_data_);
multinet_model_data_ = nullptr;
}
if (wake_word_encode_task_stack_ != nullptr) {
heap_caps_free(wake_word_encode_task_stack_);
}
@@ -38,15 +44,37 @@ CustomWakeWord::~CustomWakeWord() {
vEventGroupDelete(event_group_);
}
void CustomWakeWord::Initialize(AudioCodec* codec) {
bool CustomWakeWord::Initialize(AudioCodec* codec) {
codec_ = codec;
int ref_num = codec_->input_reference() ? 1 : 0;
models = esp_srmodel_init("model");
if (models == nullptr || models->num == -1) {
ESP_LOGE(TAG, "Failed to initialize wakenet model");
return;
return false;
}
// 初始化 multinet (命令词识别)
mn_name_ = esp_srmodel_filter(models, ESP_MN_PREFIX, ESP_MN_CHINESE);
if (mn_name_ == nullptr) {
ESP_LOGE(TAG, "Failed to initialize multinet, mn_name is nullptr");
ESP_LOGI(TAG, "Please refer to https://pcn7cs20v8cr.feishu.cn/wiki/CpQjwQsCJiQSWSkYEvrcxcbVnwh to add custom wake word");
return false;
}
ESP_LOGI(TAG, "multinet:%s", mn_name_);
multinet_ = esp_mn_handle_from_name(mn_name_);
multinet_model_data_ = multinet_->create(mn_name_, 2000); // 2秒超时
multinet_->set_det_threshold(multinet_model_data_, 0.5);
esp_mn_commands_clear();
esp_mn_commands_add(1, CONFIG_CUSTOM_WAKE_WORD); // 添加自定义唤醒词作为命令词
esp_mn_commands_update();
// 打印所有的命令词
multinet_->print_active_speech_commands(multinet_model_data_);
ESP_LOGI(TAG, "Custom wake word: %s", CONFIG_CUSTOM_WAKE_WORD);
// 初始化 afe
int ref_num = codec_->input_reference() ? 1 : 0;
std::string input_format;
for (int i = 0; i < codec_->input_channels() - ref_num; i++) {
input_format.push_back('M');
@@ -70,6 +98,8 @@ void CustomWakeWord::Initialize(AudioCodec* codec) {
this_->AudioDetectionTask();
vTaskDelete(NULL);
}, "audio_detection", 16384, this, 3, nullptr);
return true;
}
void CustomWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
@@ -105,23 +135,16 @@ void CustomWakeWord::AudioDetectionTask() {
auto fetch_size = afe_iface_->get_fetch_chunksize(afe_data_);
auto feed_size = afe_iface_->get_feed_chunksize(afe_data_);
// 初始化 multinet (命令词识别)
char *mn_name = esp_srmodel_filter(models, ESP_MN_PREFIX, ESP_MN_CHINESE);
ESP_LOGI(TAG, "multinet:%s", mn_name);
esp_mn_iface_t *multinet = esp_mn_handle_from_name(mn_name);
model_iface_data_t *model_data = multinet->create(mn_name, 2000); // 2秒超时
multinet->set_det_threshold(model_data, 0.5);
esp_mn_commands_clear();
esp_mn_commands_add(1, CONFIG_CUSTOM_WAKE_WORD); // 添加自定义唤醒词作为命令词
esp_mn_commands_update();
int mu_chunksize = multinet->get_samp_chunksize(model_data);
// 检查 multinet 是否已正确初始化
if (multinet_ == nullptr || multinet_model_data_ == nullptr) {
ESP_LOGE(TAG, "Multinet not initialized properly");
return;
}
int mu_chunksize = multinet_->get_samp_chunksize(multinet_model_data_);
assert(mu_chunksize == feed_size);
// 打印所有的命令词
multinet->print_active_speech_commands(model_data);
ESP_LOGI(TAG, "Audio detection task started, feed size: %d fetch size: %d", feed_size, fetch_size);
ESP_LOGI(TAG, "Custom wake word: %s", CONFIG_CUSTOM_WAKE_WORD);
// 禁用wakenet，直接使用multinet检测自定义唤醒词
afe_iface_->disable_wakenet(afe_data_);
@@ -139,14 +162,14 @@ void CustomWakeWord::AudioDetectionTask() {
StoreWakeWordData(res->data, res->data_size / sizeof(int16_t));
// 直接使用multinet检测自定义唤醒词
esp_mn_state_t mn_state = multinet->detect(model_data, res->data);
esp_mn_state_t mn_state = multinet_->detect(multinet_model_data_, res->data);
if (mn_state == ESP_MN_STATE_DETECTING) {
// 仍在检测中，继续
continue;
} else if (mn_state == ESP_MN_STATE_DETECTED) {
// 检测到自定义唤醒词
esp_mn_results_t *mn_result = multinet->get_results(model_data);
esp_mn_results_t *mn_result = multinet_->get_results(multinet_model_data_);
ESP_LOGI(TAG, "Custom wake word detected: command_id=%d, string=%s, prob=%f",
mn_result->command_id[0], mn_result->string, mn_result->prob[0]);
@@ -163,23 +186,17 @@ void CustomWakeWord::AudioDetectionTask() {
}
// 清理multinet状态，准备下次检测
multinet->clean(model_data);
multinet_->clean(multinet_model_data_);
ESP_LOGI(TAG, "Ready for next detection");
}
} else if (mn_state == ESP_MN_STATE_TIMEOUT) {
// 超时，清理状态继续检测
ESP_LOGD(TAG, "Command word detection timeout, cleaning state");
multinet->clean(model_data);
multinet_->clean(multinet_model_data_);
continue;
}
}
// 清理资源
if (model_data) {
multinet->destroy(model_data);
model_data = NULL;
}
ESP_LOGI(TAG, "Audio detection task ended");
}

7
main/audio/wake_words/custom_wake_word.h
View File

@@ -28,7 +28,7 @@ public:
CustomWakeWord();
~CustomWakeWord();
void Initialize(AudioCodec* codec);
bool Initialize(AudioCodec* codec);
void Feed(const std::vector<int16_t>& data);
void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback);
void Start();
@@ -43,6 +43,11 @@ private:
esp_afe_sr_data_t* afe_data_ = nullptr;
srmodel_list_t *models = nullptr;
// multinet 相关成员变量
esp_mn_iface_t* multinet_ = nullptr;
model_iface_data_t* multinet_model_data_ = nullptr;
char* mn_name_ = nullptr;
char* wakenet_model_ = NULL;
std::vector<std::string> wake_words_;
EventGroupHandle_t event_group_;

8
main/audio/wake_words/esp_wake_word.cc
View File

@@ -23,19 +23,19 @@ EspWakeWord::~EspWakeWord() {
vEventGroupDelete(event_group_);
}
void EspWakeWord::Initialize(AudioCodec* codec) {
bool EspWakeWord::Initialize(AudioCodec* codec) {
codec_ = codec;
wakenet_model_ = esp_srmodel_init("model");
if (wakenet_model_ == nullptr || wakenet_model_->num == -1) {
ESP_LOGE(TAG, "Failed to initialize wakenet model");
return;
return false;
}
if(wakenet_model_->num > 1) {
ESP_LOGW(TAG, "More than one model found, using the first one");
} else if (wakenet_model_->num == 0) {
ESP_LOGE(TAG, "No model found");
return;
return false;
}
char *model_name = wakenet_model_->model_name[0];
wakenet_iface_ = (esp_wn_iface_t*)esp_wn_handle_from_name(model_name);
@@ -44,6 +44,8 @@ void EspWakeWord::Initialize(AudioCodec* codec) {
int frequency = wakenet_iface_->get_samp_rate(wakenet_data_);
int audio_chunksize = wakenet_iface_->get_samp_chunksize(wakenet_data_);
ESP_LOGI(TAG, "Wake word(%s),freq: %d, chunksize: %d", model_name, frequency, audio_chunksize);
return true;
}
void EspWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {

2
main/audio/wake_words/esp_wake_word.h
View File

@@ -24,7 +24,7 @@ public:
EspWakeWord();
~EspWakeWord();
void Initialize(AudioCodec* codec);
bool Initialize(AudioCodec* codec);
void Feed(const std::vector<int16_t>& data);
void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback);
void Start();

1
main/boards/xmini-c3-v3/config.json
View File

@@ -6,6 +6,7 @@
"sdkconfig_append": [
"CONFIG_PM_ENABLE=y",
"CONFIG_FREERTOS_USE_TICKLESS_IDLE=y",
"CONFIG_USE_ESP_WAKE_WORD=y",
"CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y"
]
}