From 1619217bd9f0457307daa96204378af142ce840e Mon Sep 17 00:00:00 2001 From: Terrence Date: Sat, 22 Mar 2025 06:09:12 +0800 Subject: [PATCH] Upgrade esp-sr to 2.0.2, improve performance --- CMakeLists.txt | 2 +- main/Kconfig.projbuild | 4 +- main/application.cc | 29 ++++---- main/audio_processing/audio_processor.cc | 87 ++++++++++++----------- main/audio_processing/audio_processor.h | 6 +- main/audio_processing/wake_word_detect.cc | 81 +++++++-------------- main/audio_processing/wake_word_detect.h | 6 +- main/boards/esp-sparkbot/config.h | 2 - main/boards/lichuang-c3-dev/config.h | 1 - main/boards/taiji-pi-s3/config.h | 1 - main/idf_component.yml | 2 +- sdkconfig.defaults.esp32s3 | 2 - 12 files changed, 100 insertions(+), 123 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 64c684a4..fedb172a 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ # CMakeLists in this exact order for cmake to work correctly cmake_minimum_required(VERSION 3.16) -set(PROJECT_VER "1.5.0") +set(PROJECT_VER "1.5.1") # Add this line to disable the specific warning add_compile_options(-Wno-missing-field-initializers) diff --git a/main/Kconfig.projbuild b/main/Kconfig.projbuild index d3423fc7..b6a51207 100644 --- a/main/Kconfig.projbuild +++ b/main/Kconfig.projbuild @@ -223,14 +223,14 @@ config USE_WECHAT_MESSAGE_STYLE config USE_AUDIO_PROCESSOR bool "启用音频降噪、增益处理" default y - depends on IDF_TARGET_ESP32S3 && USE_AFE + depends on IDF_TARGET_ESP32S3 && SPIRAM help 需要 ESP32 S3 与 AFE 支持 config USE_WAKE_WORD_DETECT bool "启用唤醒词检测" default y - depends on IDF_TARGET_ESP32S3 && USE_AFE + depends on IDF_TARGET_ESP32S3 && SPIRAM help 需要 ESP32 S3 与 AFE 支持 endmenu diff --git a/main/application.cc b/main/application.cc index 6e4d60ed..56a04756 100644 --- a/main/application.cc +++ b/main/application.cc @@ -484,13 +484,9 @@ void Application::Start() { }); }); }); -#endif - -#if CONFIG_USE_WAKE_WORD_DETECT - wake_word_detect_.Initialize(codec->input_channels(), codec->input_reference()); - wake_word_detect_.OnVadStateChange([this](bool speaking) { - Schedule([this, speaking]() { - if (device_state_ == kDeviceStateListening) { + audio_processor_.OnVadStateChange([this](bool speaking) { + if (device_state_ == kDeviceStateListening) { + Schedule([this, speaking]() { if (speaking) { voice_detected_ = true; } else { @@ -498,10 +494,13 @@ void Application::Start() { } auto led = Board::GetInstance().GetLed(); led->OnStateChanged(); - } - }); + }); + } }); +#endif +#if CONFIG_USE_WAKE_WORD_DETECT + wake_word_detect_.Initialize(codec->input_channels(), codec->input_reference()); wake_word_detect_.OnWakeWordDetected([this](const std::string& wake_word) { Schedule([this, &wake_word]() { if (device_state_ == kDeviceStateIdle) { @@ -528,9 +527,6 @@ void Application::Start() { } else if (device_state_ == kDeviceStateActivating) { SetDeviceState(kDeviceStateIdle); } - - // Resume detection - wake_word_detect_.StartDetection(); }); }); wake_word_detect_.StartDetection(); @@ -738,6 +734,9 @@ void Application::SetDeviceState(DeviceState state) { display->SetEmotion("neutral"); #if CONFIG_USE_AUDIO_PROCESSOR audio_processor_.Stop(); +#endif +#if CONFIG_USE_WAKE_WORD_DETECT + wake_word_detect_.StartDetection(); #endif break; case kDeviceStateConnecting: @@ -752,6 +751,9 @@ void Application::SetDeviceState(DeviceState state) { opus_encoder_->ResetState(); #if CONFIG_USE_AUDIO_PROCESSOR audio_processor_.Start(); +#endif +#if CONFIG_USE_WAKE_WORD_DETECT + wake_word_detect_.StopDetection(); #endif UpdateIotStates(); if (previous_state == kDeviceStateSpeaking) { @@ -765,6 +767,9 @@ void Application::SetDeviceState(DeviceState state) { codec->EnableOutput(true); #if CONFIG_USE_AUDIO_PROCESSOR audio_processor_.Stop(); +#endif +#if CONFIG_USE_WAKE_WORD_DETECT + wake_word_detect_.StartDetection(); #endif break; default: diff --git a/main/audio_processing/audio_processor.cc b/main/audio_processing/audio_processor.cc index 02f38266..bf4e8a30 100644 --- a/main/audio_processing/audio_processor.cc +++ b/main/audio_processing/audio_processor.cc @@ -6,7 +6,7 @@ static const char* TAG = "AudioProcessor"; AudioProcessor::AudioProcessor() - : afe_communication_data_(nullptr) { + : afe_data_(nullptr) { event_group_ = xEventGroupCreate(); } @@ -15,39 +15,30 @@ void AudioProcessor::Initialize(int channels, bool reference) { reference_ = reference; int ref_num = reference_ ? 1 : 0; - afe_config_t afe_config = { - .aec_init = false, - .se_init = true, - .vad_init = false, - .wakenet_init = false, - .voice_communication_init = true, - .voice_communication_agc_init = true, - .voice_communication_agc_gain = 10, - .vad_mode = VAD_MODE_3, - .wakenet_model_name = NULL, - .wakenet_model_name_2 = NULL, - .wakenet_mode = DET_MODE_90, - .afe_mode = SR_MODE_HIGH_PERF, - .afe_perferred_core = 1, - .afe_perferred_priority = 1, - .afe_ringbuf_size = 50, - .memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM, - .afe_linear_gain = 1.0, - .agc_mode = AFE_MN_PEAK_AGC_MODE_2, - .pcm_config = { - .total_ch_num = channels_, - .mic_num = channels_ - ref_num, - .ref_num = ref_num, - .sample_rate = 16000, - }, - .debug_init = false, - .debug_hook = {{ AFE_DEBUG_HOOK_MASE_TASK_IN, NULL }, { AFE_DEBUG_HOOK_FETCH_TASK_IN, NULL }}, - .afe_ns_mode = NS_MODE_SSP, - .afe_ns_model_name = NULL, - .fixed_first_channel = true, - }; + std::string input_format; + for (int i = 0; i < channels_ - ref_num; i++) { + input_format.push_back('M'); + } + for (int i = 0; i < ref_num; i++) { + input_format.push_back('R'); + } - afe_communication_data_ = esp_afe_vc_v1.create_from_config(&afe_config); + afe_config_t* afe_config = afe_config_init(input_format.c_str(), NULL, AFE_TYPE_VC, AFE_MODE_HIGH_PERF); + afe_config->aec_init = false; + afe_config->aec_mode = AEC_MODE_VOIP_HIGH_PERF; + afe_config->ns_init = true; + afe_config->vad_init = true; + afe_config->vad_mode = VAD_MODE_0; + afe_config->vad_min_noise_ms = 100; + afe_config->afe_perferred_core = 1; + afe_config->afe_perferred_priority = 1; + afe_config->agc_init = true; + afe_config->agc_mode = AFE_AGC_MODE_WEBRTC; + afe_config->agc_compression_gain_db = 10; + afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM; + + afe_iface_ = esp_afe_handle_from_config(afe_config); + afe_data_ = afe_iface_->create_from_config(afe_config); xTaskCreate([](void* arg) { auto this_ = (AudioProcessor*)arg; @@ -57,8 +48,8 @@ void AudioProcessor::Initialize(int channels, bool reference) { } AudioProcessor::~AudioProcessor() { - if (afe_communication_data_ != nullptr) { - esp_afe_vc_v1.destroy(afe_communication_data_); + if (afe_data_ != nullptr) { + afe_iface_->destroy(afe_data_); } vEventGroupDelete(event_group_); } @@ -66,10 +57,10 @@ AudioProcessor::~AudioProcessor() { void AudioProcessor::Input(const std::vector& data) { input_buffer_.insert(input_buffer_.end(), data.begin(), data.end()); - auto feed_size = esp_afe_vc_v1.get_feed_chunksize(afe_communication_data_) * channels_; + auto feed_size = afe_iface_->get_feed_chunksize(afe_data_) * channels_; while (input_buffer_.size() >= feed_size) { auto chunk = input_buffer_.data(); - esp_afe_vc_v1.feed(afe_communication_data_, chunk); + afe_iface_->feed(afe_data_, chunk); input_buffer_.erase(input_buffer_.begin(), input_buffer_.begin() + feed_size); } } @@ -80,6 +71,7 @@ void AudioProcessor::Start() { void AudioProcessor::Stop() { xEventGroupClearBits(event_group_, PROCESSOR_RUNNING); + afe_iface_->reset_buffer(afe_data_); } bool AudioProcessor::IsRunning() { @@ -90,16 +82,20 @@ void AudioProcessor::OnOutput(std::function&& data)> c output_callback_ = callback; } +void AudioProcessor::OnVadStateChange(std::function callback) { + vad_state_change_callback_ = callback; +} + void AudioProcessor::AudioProcessorTask() { - auto fetch_size = esp_afe_sr_v1.get_fetch_chunksize(afe_communication_data_); - auto feed_size = esp_afe_sr_v1.get_feed_chunksize(afe_communication_data_); + auto fetch_size = afe_iface_->get_fetch_chunksize(afe_data_); + auto feed_size = afe_iface_->get_feed_chunksize(afe_data_); ESP_LOGI(TAG, "Audio communication task started, feed size: %d fetch size: %d", feed_size, fetch_size); while (true) { xEventGroupWaitBits(event_group_, PROCESSOR_RUNNING, pdFALSE, pdTRUE, portMAX_DELAY); - auto res = esp_afe_vc_v1.fetch(afe_communication_data_); + auto res = afe_iface_->fetch_with_delay(afe_data_, portMAX_DELAY); if ((xEventGroupGetBits(event_group_) & PROCESSOR_RUNNING) == 0) { continue; } @@ -110,6 +106,17 @@ void AudioProcessor::AudioProcessorTask() { continue; } + // VAD state change + if (vad_state_change_callback_) { + if (res->vad_state == VAD_SPEECH && !is_speaking_) { + is_speaking_ = true; + vad_state_change_callback_(true); + } else if (res->vad_state == VAD_SILENCE && is_speaking_) { + is_speaking_ = false; + vad_state_change_callback_(false); + } + } + if (output_callback_) { output_callback_(std::vector(res->data, res->data + res->data_size / sizeof(int16_t))); } diff --git a/main/audio_processing/audio_processor.h b/main/audio_processing/audio_processor.h index 3c8fd90b..7864cec5 100644 --- a/main/audio_processing/audio_processor.h +++ b/main/audio_processing/audio_processor.h @@ -21,14 +21,18 @@ public: void Stop(); bool IsRunning(); void OnOutput(std::function&& data)> callback); + void OnVadStateChange(std::function callback); private: EventGroupHandle_t event_group_ = nullptr; - esp_afe_sr_data_t* afe_communication_data_ = nullptr; + esp_afe_sr_iface_t* afe_iface_ = nullptr; + esp_afe_sr_data_t* afe_data_ = nullptr; std::vector input_buffer_; std::function&& data)> output_callback_; + std::function vad_state_change_callback_; int channels_; bool reference_; + bool is_speaking_ = false; void AudioProcessorTask(); }; diff --git a/main/audio_processing/wake_word_detect.cc b/main/audio_processing/wake_word_detect.cc index 788abfa2..73c900f0 100644 --- a/main/audio_processing/wake_word_detect.cc +++ b/main/audio_processing/wake_word_detect.cc @@ -11,7 +11,7 @@ static const char* TAG = "WakeWordDetect"; WakeWordDetect::WakeWordDetect() - : afe_detection_data_(nullptr), + : afe_data_(nullptr), wake_word_pcm_(), wake_word_opus_() { @@ -19,8 +19,8 @@ WakeWordDetect::WakeWordDetect() } WakeWordDetect::~WakeWordDetect() { - if (afe_detection_data_ != nullptr) { - esp_afe_sr_v1.destroy(afe_detection_data_); + if (afe_data_ != nullptr) { + afe_iface_->destroy(afe_data_); } if (wake_word_encode_task_stack_ != nullptr) { @@ -50,39 +50,22 @@ void WakeWordDetect::Initialize(int channels, bool reference) { } } - afe_config_t afe_config = { - .aec_init = reference_, - .se_init = true, - .vad_init = true, - .wakenet_init = true, - .voice_communication_init = false, - .voice_communication_agc_init = false, - .voice_communication_agc_gain = 10, - .vad_mode = VAD_MODE_3, - .wakenet_model_name = wakenet_model_, - .wakenet_model_name_2 = NULL, - .wakenet_mode = DET_MODE_90, - .afe_mode = SR_MODE_HIGH_PERF, - .afe_perferred_core = 1, - .afe_perferred_priority = 1, - .afe_ringbuf_size = 50, - .memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM, - .afe_linear_gain = 1.0, - .agc_mode = AFE_MN_PEAK_AGC_MODE_2, - .pcm_config = { - .total_ch_num = channels_, - .mic_num = channels_ - ref_num, - .ref_num = ref_num, - .sample_rate = 16000 - }, - .debug_init = false, - .debug_hook = {{ AFE_DEBUG_HOOK_MASE_TASK_IN, NULL }, { AFE_DEBUG_HOOK_FETCH_TASK_IN, NULL }}, - .afe_ns_mode = NS_MODE_SSP, - .afe_ns_model_name = NULL, - .fixed_first_channel = true, - }; - - afe_detection_data_ = esp_afe_sr_v1.create_from_config(&afe_config); + std::string input_format; + for (int i = 0; i < channels_ - ref_num; i++) { + input_format.push_back('M'); + } + for (int i = 0; i < ref_num; i++) { + input_format.push_back('R'); + } + afe_config_t* afe_config = afe_config_init(input_format.c_str(), models, AFE_TYPE_SR, AFE_MODE_HIGH_PERF); + afe_config->aec_init = reference_; + afe_config->aec_mode = AEC_MODE_SR_HIGH_PERF; + afe_config->afe_perferred_core = 1; + afe_config->afe_perferred_priority = 1; + afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM; + + afe_iface_ = esp_afe_handle_from_config(afe_config); + afe_data_ = afe_iface_->create_from_config(afe_config); xTaskCreate([](void* arg) { auto this_ = (WakeWordDetect*)arg; @@ -95,16 +78,13 @@ void WakeWordDetect::OnWakeWordDetected(std::function callback) { - vad_state_change_callback_ = callback; -} - void WakeWordDetect::StartDetection() { xEventGroupSetBits(event_group_, DETECTION_RUNNING_EVENT); } void WakeWordDetect::StopDetection() { xEventGroupClearBits(event_group_, DETECTION_RUNNING_EVENT); + afe_iface_->reset_buffer(afe_data_); } bool WakeWordDetect::IsDetectionRunning() { @@ -114,23 +94,23 @@ bool WakeWordDetect::IsDetectionRunning() { void WakeWordDetect::Feed(const std::vector& data) { input_buffer_.insert(input_buffer_.end(), data.begin(), data.end()); - auto feed_size = esp_afe_sr_v1.get_feed_chunksize(afe_detection_data_) * channels_; + auto feed_size = afe_iface_->get_feed_chunksize(afe_data_) * channels_; while (input_buffer_.size() >= feed_size) { - esp_afe_sr_v1.feed(afe_detection_data_, input_buffer_.data()); + afe_iface_->feed(afe_data_, input_buffer_.data()); input_buffer_.erase(input_buffer_.begin(), input_buffer_.begin() + feed_size); } } void WakeWordDetect::AudioDetectionTask() { - auto fetch_size = esp_afe_sr_v1.get_fetch_chunksize(afe_detection_data_); - auto feed_size = esp_afe_sr_v1.get_feed_chunksize(afe_detection_data_); + auto fetch_size = afe_iface_->get_fetch_chunksize(afe_data_); + auto feed_size = afe_iface_->get_feed_chunksize(afe_data_); ESP_LOGI(TAG, "Audio detection task started, feed size: %d fetch size: %d", feed_size, fetch_size); while (true) { xEventGroupWaitBits(event_group_, DETECTION_RUNNING_EVENT, pdFALSE, pdTRUE, portMAX_DELAY); - auto res = esp_afe_sr_v1.fetch(afe_detection_data_); + auto res = afe_iface_->fetch_with_delay(afe_data_, portMAX_DELAY); if (res == nullptr || res->ret_value == ESP_FAIL) { continue;; } @@ -138,17 +118,6 @@ void WakeWordDetect::AudioDetectionTask() { // Store the wake word data for voice recognition, like who is speaking StoreWakeWordData((uint16_t*)res->data, res->data_size / sizeof(uint16_t)); - // VAD state change - if (vad_state_change_callback_) { - if (res->vad_state == AFE_VAD_SPEECH && !is_speaking_) { - is_speaking_ = true; - vad_state_change_callback_(true); - } else if (res->vad_state == AFE_VAD_SILENCE && is_speaking_) { - is_speaking_ = false; - vad_state_change_callback_(false); - } - } - if (res->wakeup_state == WAKENET_DETECTED) { StopDetection(); last_detected_wake_word_ = wake_words_[res->wake_word_index - 1]; diff --git a/main/audio_processing/wake_word_detect.h b/main/audio_processing/wake_word_detect.h index 0a356b40..54baf5df 100644 --- a/main/audio_processing/wake_word_detect.h +++ b/main/audio_processing/wake_word_detect.h @@ -24,7 +24,6 @@ public: void Initialize(int channels, bool reference); void Feed(const std::vector& data); void OnWakeWordDetected(std::function callback); - void OnVadStateChange(std::function callback); void StartDetection(); void StopDetection(); bool IsDetectionRunning(); @@ -33,14 +32,13 @@ public: const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; } private: - esp_afe_sr_data_t* afe_detection_data_ = nullptr; + esp_afe_sr_iface_t* afe_iface_ = nullptr; + esp_afe_sr_data_t* afe_data_ = nullptr; char* wakenet_model_ = NULL; std::vector wake_words_; std::vector input_buffer_; EventGroupHandle_t event_group_; std::function wake_word_detected_callback_; - std::function vad_state_change_callback_; - bool is_speaking_ = false; int channels_; bool reference_; std::string last_detected_wake_word_; diff --git a/main/boards/esp-sparkbot/config.h b/main/boards/esp-sparkbot/config.h index 2664815b..b26cf162 100644 --- a/main/boards/esp-sparkbot/config.h +++ b/main/boards/esp-sparkbot/config.h @@ -7,8 +7,6 @@ #define AUDIO_INPUT_SAMPLE_RATE 16000 #define AUDIO_OUTPUT_SAMPLE_RATE 16000 -#define AUDIO_INPUT_REFERENCE false - #define AUDIO_I2S_GPIO_MCLK GPIO_NUM_45 #define AUDIO_I2S_GPIO_WS GPIO_NUM_41 #define AUDIO_I2S_GPIO_BCLK GPIO_NUM_39 diff --git a/main/boards/lichuang-c3-dev/config.h b/main/boards/lichuang-c3-dev/config.h index 01304b6a..548ccb44 100644 --- a/main/boards/lichuang-c3-dev/config.h +++ b/main/boards/lichuang-c3-dev/config.h @@ -5,7 +5,6 @@ #define AUDIO_INPUT_SAMPLE_RATE 24000 #define AUDIO_OUTPUT_SAMPLE_RATE 24000 -#define AUDIO_INPUT_REFERENCE true #define AUDIO_I2S_GPIO_MCLK GPIO_NUM_10 #define AUDIO_I2S_GPIO_WS GPIO_NUM_12 diff --git a/main/boards/taiji-pi-s3/config.h b/main/boards/taiji-pi-s3/config.h index d38a9d81..6b9f54aa 100644 --- a/main/boards/taiji-pi-s3/config.h +++ b/main/boards/taiji-pi-s3/config.h @@ -6,7 +6,6 @@ #include #include -#define AUDIO_INPUT_REFERENCE true #define AUDIO_INPUT_SAMPLE_RATE 24000 #define AUDIO_OUTPUT_SAMPLE_RATE 24000 #define AUDIO_DEFAULT_OUTPUT_VOLUME 80 diff --git a/main/idf_component.yml b/main/idf_component.yml index a0cbca64..1a883547 100644 --- a/main/idf_component.yml +++ b/main/idf_component.yml @@ -14,7 +14,7 @@ dependencies: 78/xiaozhi-fonts: "~1.3.2" espressif/led_strip: "^2.4.1" espressif/esp_codec_dev: "~1.3.2" - espressif/esp-sr: "^1.9.0" + espressif/esp-sr: "^2.0.2" espressif/button: "^3.3.1" lvgl/lvgl: "~9.2.2" esp_lvgl_port: "~2.4.4" diff --git a/sdkconfig.defaults.esp32s3 b/sdkconfig.defaults.esp32s3 index 8c9678ba..bdd0b2a2 100644 --- a/sdkconfig.defaults.esp32s3 +++ b/sdkconfig.defaults.esp32s3 @@ -17,6 +17,4 @@ CONFIG_ESP32S3_INSTRUCTION_CACHE_32KB=y CONFIG_ESP32S3_DATA_CACHE_64KB=y CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y -CONFIG_USE_WAKENET=y CONFIG_SR_WN_WN9_NIHAOXIAOZHI_TTS=y -CONFIG_USE_MULTINET=n