diff --git a/CMakeLists.txt b/CMakeLists.txt index 706938dc..6ddc88d2 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ # CMakeLists in this exact order for cmake to work correctly cmake_minimum_required(VERSION 3.16) -set(PROJECT_VER "1.5.2") +set(PROJECT_VER "1.5.4") # Add this line to disable the specific warning add_compile_options(-Wno-missing-field-initializers) diff --git a/main/Kconfig.projbuild b/main/Kconfig.projbuild index 9491e0d2..4c515a64 100644 --- a/main/Kconfig.projbuild +++ b/main/Kconfig.projbuild @@ -228,7 +228,14 @@ config USE_WECHAT_MESSAGE_STYLE bool "使用微信聊天界面风格" default n help - 使用微信聊天界面风格 + 使用微信聊天界面风格 + +config USE_WAKE_WORD_DETECT + bool "启用唤醒词检测" + default y + depends on IDF_TARGET_ESP32S3 && SPIRAM + help + 需要 ESP32 S3 与 AFE 支持 config USE_AUDIO_PROCESSOR bool "启用音频降噪、增益处理" @@ -237,10 +244,11 @@ config USE_AUDIO_PROCESSOR help 需要 ESP32 S3 与 AFE 支持 -config USE_WAKE_WORD_DETECT - bool "启用唤醒词检测" - default y - depends on IDF_TARGET_ESP32S3 && SPIRAM +config USE_REALTIME_CHAT + bool "启用可语音打断的实时对话模式(需要 AEC 支持)" + default n + depends on USE_AUDIO_PROCESSOR && (BOARD_TYPE_ESP_BOX_3 || BOARD_TYPE_ESP_BOX || BOARD_TYPE_LICHUANG_DEV) help - 需要 ESP32 S3 与 AFE 支持 + 需要 ESP32 S3 与 AEC 开启,因为性能不够,不建议和微信聊天界面风格同时开启 + endmenu diff --git a/main/application.cc b/main/application.cc index 56a04756..15d13f40 100644 --- a/main/application.cc +++ b/main/application.cc @@ -203,6 +203,7 @@ void Application::Alert(const char* status, const char* message, const char* emo display->SetEmotion(emotion); display->SetChatMessage("system", message); if (!sound.empty()) { + ResetDecoder(); PlaySound(sound); } } @@ -217,9 +218,8 @@ void Application::DismissAlert() { } void Application::PlaySound(const std::string_view& sound) { - auto codec = Board::GetInstance().GetAudioCodec(); - codec->EnableOutput(true); - SetDecodeSampleRate(16000); + // The assets are encoded at 16000Hz, 60ms frame duration + SetDecodeSampleRate(16000, 60); const char* data = sound.data(); size_t size = sound.size(); for (const char* p = data; p < data + size; ) { @@ -255,9 +255,7 @@ void Application::ToggleChatState() { return; } - keep_listening_ = true; - protocol_->SendStartListening(kListeningModeAutoStop); - SetDeviceState(kDeviceStateListening); + SetListeningMode(realtime_chat_enabled_ ? kListeningModeRealtime : kListeningModeAutoStop); }); } else if (device_state_ == kDeviceStateSpeaking) { Schedule([this]() { @@ -281,7 +279,6 @@ void Application::StartListening() { return; } - keep_listening_ = false; if (device_state_ == kDeviceStateIdle) { Schedule([this]() { if (!protocol_->IsAudioChannelOpened()) { @@ -290,14 +287,13 @@ void Application::StartListening() { return; } } - protocol_->SendStartListening(kListeningModeManualStop); - SetDeviceState(kDeviceStateListening); + + SetListeningMode(kListeningModeManualStop); }); } else if (device_state_ == kDeviceStateSpeaking) { Schedule([this]() { AbortSpeaking(kAbortReasonNone); - protocol_->SendStartListening(kListeningModeManualStop); - SetDeviceState(kDeviceStateListening); + SetListeningMode(kListeningModeManualStop); }); } } @@ -320,12 +316,12 @@ void Application::Start() { /* Setup the audio codec */ auto codec = board.GetAudioCodec(); - opus_decode_sample_rate_ = codec->output_sample_rate(); - opus_decoder_ = std::make_unique(opus_decode_sample_rate_, 1); + opus_decoder_ = std::make_unique(codec->output_sample_rate(), 1, OPUS_FRAME_DURATION_MS); opus_encoder_ = std::make_unique(16000, 1, OPUS_FRAME_DURATION_MS); - // For ML307 boards, we use complexity 5 to save bandwidth - // For other boards, we use complexity 3 to save CPU - if (board.GetBoardType() == "ml307") { + if (realtime_chat_enabled_) { + ESP_LOGI(TAG, "Realtime chat enabled, setting opus encoder complexity to 0"); + opus_encoder_->SetComplexity(0); + } else if (board.GetBoardType() == "ml307") { ESP_LOGI(TAG, "ML307 board detected, setting opus encoder complexity to 5"); opus_encoder_->SetComplexity(5); } else { @@ -337,24 +333,20 @@ void Application::Start() { input_resampler_.Configure(codec->input_sample_rate(), 16000); reference_resampler_.Configure(codec->input_sample_rate(), 16000); } - codec->OnInputReady([this, codec]() { - BaseType_t higher_priority_task_woken = pdFALSE; - xEventGroupSetBitsFromISR(event_group_, AUDIO_INPUT_READY_EVENT, &higher_priority_task_woken); - return higher_priority_task_woken == pdTRUE; - }); - codec->OnOutputReady([this]() { - BaseType_t higher_priority_task_woken = pdFALSE; - xEventGroupSetBitsFromISR(event_group_, AUDIO_OUTPUT_READY_EVENT, &higher_priority_task_woken); - return higher_priority_task_woken == pdTRUE; - }); codec->Start(); + xTaskCreatePinnedToCore([](void* arg) { + Application* app = (Application*)arg; + app->AudioLoop(); + vTaskDelete(NULL); + }, "audio_loop", 4096 * 2, this, 8, &audio_loop_task_handle_, realtime_chat_enabled_ ? 1 : 0); + /* Start the main loop */ - xTaskCreate([](void* arg) { + xTaskCreatePinnedToCore([](void* arg) { Application* app = (Application*)arg; app->MainLoop(); vTaskDelete(NULL); - }, "main_loop", 4096 * 2, this, 4, nullptr); + }, "main_loop", 4096, this, 4, &main_loop_task_handle_, 0); /* Wait for the network to be ready */ board.StartNetwork(); @@ -372,9 +364,7 @@ void Application::Start() { }); protocol_->OnIncomingAudio([this](std::vector&& data) { std::lock_guard lock(mutex_); - if (device_state_ == kDeviceStateSpeaking) { - audio_decode_queue_.emplace_back(std::move(data)); - } + audio_decode_queue_.emplace_back(std::move(data)); }); protocol_->OnAudioChannelOpened([this, codec, &board]() { board.SetPowerSaveMode(false); @@ -382,7 +372,7 @@ void Application::Start() { ESP_LOGW(TAG, "Server sample rate %d does not match device output sample rate %d, resampling may cause distortion", protocol_->server_sample_rate(), codec->output_sample_rate()); } - SetDecodeSampleRate(protocol_->server_sample_rate()); + SetDecodeSampleRate(protocol_->server_sample_rate(), protocol_->server_frame_duration()); auto& thing_manager = iot::ThingManager::GetInstance(); protocol_->SendIotDescriptors(thing_manager.GetDescriptorsJson()); std::string states; @@ -412,13 +402,12 @@ void Application::Start() { }); } else if (strcmp(state->valuestring, "stop") == 0) { Schedule([this]() { + background_task_->WaitForCompletion(); if (device_state_ == kDeviceStateSpeaking) { - background_task_->WaitForCompletion(); - if (keep_listening_) { - protocol_->SendStartListening(kListeningModeAutoStop); - SetDeviceState(kDeviceStateListening); - } else { + if (listening_mode_ == kListeningModeManualStop) { SetDeviceState(kDeviceStateIdle); + } else { + SetDeviceState(kDeviceStateListening); } } }); @@ -474,7 +463,7 @@ void Application::Start() { }, "check_new_version", 4096 * 2, this, 2, nullptr); #if CONFIG_USE_AUDIO_PROCESSOR - audio_processor_.Initialize(codec->input_channels(), codec->input_reference()); + audio_processor_.Initialize(codec, realtime_chat_enabled_); audio_processor_.OnOutput([this](std::vector&& data) { background_task_->Schedule([this, data = std::move(data)]() mutable { opus_encoder_->Encode(std::move(data), [this](std::vector&& opus) { @@ -500,7 +489,7 @@ void Application::Start() { #endif #if CONFIG_USE_WAKE_WORD_DETECT - wake_word_detect_.Initialize(codec->input_channels(), codec->input_reference()); + wake_word_detect_.Initialize(codec); wake_word_detect_.OnWakeWordDetected([this](const std::string& wake_word) { Schedule([this, &wake_word]() { if (device_state_ == kDeviceStateIdle) { @@ -520,8 +509,7 @@ void Application::Start() { // Set the chat state to wake word detected protocol_->SendWakeWordDetected(wake_word); ESP_LOGI(TAG, "Wake word detected: %s", wake_word.c_str()); - keep_listening_ = true; - SetDeviceState(kDeviceStateIdle); + SetListeningMode(realtime_chat_enabled_ ? kListeningModeRealtime : kListeningModeAutoStop); } else if (device_state_ == kDeviceStateSpeaking) { AbortSpeaking(kAbortReasonWakeWordDetected); } else if (device_state_ == kDeviceStateActivating) { @@ -534,6 +522,13 @@ void Application::Start() { SetDeviceState(kDeviceStateIdle); esp_timer_start_periodic(clock_timer_handle_, 1000000); + +#if 0 + while (true) { + SystemInfo::PrintRealTimeStats(pdMS_TO_TICKS(1000)); + vTaskDelay(pdMS_TO_TICKS(10000)); + } +#endif } void Application::OnClockTimer() { @@ -541,7 +536,6 @@ void Application::OnClockTimer() { // Print the debug info every 10 seconds if (clock_ticks_ % 10 == 0) { - // SystemInfo::PrintRealTimeStats(pdMS_TO_TICKS(1000)); int free_sram = heap_caps_get_free_size(MALLOC_CAP_INTERNAL); int min_free_sram = heap_caps_get_minimum_free_size(MALLOC_CAP_INTERNAL); ESP_LOGI(TAG, "Free internal: %u minimal internal: %u", free_sram, min_free_sram); @@ -561,6 +555,7 @@ void Application::OnClockTimer() { } } +// Add a async task to MainLoop void Application::Schedule(std::function callback) { { std::lock_guard lock(mutex_); @@ -574,16 +569,8 @@ void Application::Schedule(std::function callback) { // they should use Schedule to call this function void Application::MainLoop() { while (true) { - auto bits = xEventGroupWaitBits(event_group_, - SCHEDULE_EVENT | AUDIO_INPUT_READY_EVENT | AUDIO_OUTPUT_READY_EVENT, - pdTRUE, pdFALSE, portMAX_DELAY); + auto bits = xEventGroupWaitBits(event_group_, SCHEDULE_EVENT, pdTRUE, pdFALSE, portMAX_DELAY); - if (bits & AUDIO_INPUT_READY_EVENT) { - InputAudio(); - } - if (bits & AUDIO_OUTPUT_READY_EVENT) { - OutputAudio(); - } if (bits & SCHEDULE_EVENT) { std::unique_lock lock(mutex_); std::list> tasks = std::move(main_tasks_); @@ -595,14 +582,18 @@ void Application::MainLoop() { } } -void Application::ResetDecoder() { - std::lock_guard lock(mutex_); - opus_decoder_->ResetState(); - audio_decode_queue_.clear(); - last_output_time_ = std::chrono::steady_clock::now(); +// The Audio Loop is used to input and output audio data +void Application::AudioLoop() { + auto codec = Board::GetInstance().GetAudioCodec(); + while (true) { + OnAudioInput(); + if (codec->output_enabled()) { + OnAudioOutput(); + } + } } -void Application::OutputAudio() { +void Application::OnAudioOutput() { auto now = std::chrono::steady_clock::now(); auto codec = Board::GetInstance().GetAudioCodec(); const int max_silence_seconds = 10; @@ -624,7 +615,6 @@ void Application::OutputAudio() { return; } - last_output_time_ = now; auto opus = std::move(audio_decode_queue_.front()); audio_decode_queue_.pop_front(); lock.unlock(); @@ -638,27 +628,57 @@ void Application::OutputAudio() { if (!opus_decoder_->Decode(std::move(opus), pcm)) { return; } - // Resample if the sample rate is different - if (opus_decode_sample_rate_ != codec->output_sample_rate()) { + if (opus_decoder_->sample_rate() != codec->output_sample_rate()) { int target_size = output_resampler_.GetOutputSamples(pcm.size()); std::vector resampled(target_size); output_resampler_.Process(pcm.data(), pcm.size(), resampled.data()); pcm = std::move(resampled); } - codec->OutputData(pcm); + last_output_time_ = std::chrono::steady_clock::now(); }); } -void Application::InputAudio() { - auto codec = Board::GetInstance().GetAudioCodec(); +void Application::OnAudioInput() { std::vector data; - if (!codec->InputData(data)) { + +#if CONFIG_USE_WAKE_WORD_DETECT + if (wake_word_detect_.IsDetectionRunning()) { + ReadAudio(data, 16000, wake_word_detect_.GetFeedSize()); + wake_word_detect_.Feed(data); return; } +#endif +#if CONFIG_USE_AUDIO_PROCESSOR + if (audio_processor_.IsRunning()) { + ReadAudio(data, 16000, audio_processor_.GetFeedSize()); + audio_processor_.Feed(data); + return; + } +#else + if (device_state_ == kDeviceStateListening) { + ReadAudio(data, 16000, 30 * 16000 / 1000); + background_task_->Schedule([this, data = std::move(data)]() mutable { + opus_encoder_->Encode(std::move(data), [this](std::vector&& opus) { + Schedule([this, opus = std::move(opus)]() { + protocol_->SendAudio(opus); + }); + }); + }); + return; + } +#endif + vTaskDelay(pdMS_TO_TICKS(30)); +} - if (codec->input_sample_rate() != 16000) { +void Application::ReadAudio(std::vector& data, int sample_rate, int samples) { + auto codec = Board::GetInstance().GetAudioCodec(); + if (codec->input_sample_rate() != sample_rate) { + data.resize(samples * codec->input_sample_rate() / sample_rate); + if (!codec->InputData(data)) { + return; + } if (codec->input_channels() == 2) { auto mic_channel = std::vector(data.size() / 2); auto reference_channel = std::vector(data.size() / 2); @@ -680,28 +700,12 @@ void Application::InputAudio() { input_resampler_.Process(data.data(), data.size(), resampled.data()); data = std::move(resampled); } + } else { + data.resize(samples); + if (!codec->InputData(data)) { + return; + } } - -#if CONFIG_USE_WAKE_WORD_DETECT - if (wake_word_detect_.IsDetectionRunning()) { - wake_word_detect_.Feed(data); - } -#endif -#if CONFIG_USE_AUDIO_PROCESSOR - if (audio_processor_.IsRunning()) { - audio_processor_.Input(data); - } -#else - if (device_state_ == kDeviceStateListening) { - background_task_->Schedule([this, data = std::move(data)]() mutable { - opus_encoder_->Encode(std::move(data), [this](std::vector&& opus) { - Schedule([this, opus = std::move(opus)]() { - protocol_->SendAudio(opus); - }); - }); - }); - } -#endif } void Application::AbortSpeaking(AbortReason reason) { @@ -710,6 +714,11 @@ void Application::AbortSpeaking(AbortReason reason) { protocol_->SendAbortSpeaking(reason); } +void Application::SetListeningMode(ListeningMode mode) { + listening_mode_ = mode; + SetDeviceState(kDeviceStateListening); +} + void Application::SetDeviceState(DeviceState state) { if (device_state_ == state) { return; @@ -723,7 +732,6 @@ void Application::SetDeviceState(DeviceState state) { background_task_->WaitForCompletion(); auto& board = Board::GetInstance(); - auto codec = board.GetAudioCodec(); auto display = board.GetDisplay(); auto led = board.GetLed(); led->OnStateChanged(); @@ -747,30 +755,43 @@ void Application::SetDeviceState(DeviceState state) { case kDeviceStateListening: display->SetStatus(Lang::Strings::LISTENING); display->SetEmotion("neutral"); - ResetDecoder(); - opus_encoder_->ResetState(); -#if CONFIG_USE_AUDIO_PROCESSOR - audio_processor_.Start(); -#endif -#if CONFIG_USE_WAKE_WORD_DETECT - wake_word_detect_.StopDetection(); -#endif + + // Update the IoT states before sending the start listening command UpdateIotStates(); - if (previous_state == kDeviceStateSpeaking) { - // FIXME: Wait for the speaker to empty the buffer - vTaskDelay(pdMS_TO_TICKS(120)); + + // Make sure the audio processor is running +#if CONFIG_USE_AUDIO_PROCESSOR + if (!audio_processor_.IsRunning()) { +#else + if (true) { +#endif + // Send the start listening command + protocol_->SendStartListening(listening_mode_); + if (listening_mode_ == kListeningModeAutoStop && previous_state == kDeviceStateSpeaking) { + // FIXME: Wait for the speaker to empty the buffer + vTaskDelay(pdMS_TO_TICKS(120)); + } + opus_encoder_->ResetState(); +#if CONFIG_USE_WAKE_WORD_DETECT + wake_word_detect_.StopDetection(); +#endif +#if CONFIG_USE_AUDIO_PROCESSOR + audio_processor_.Start(); +#endif } break; case kDeviceStateSpeaking: display->SetStatus(Lang::Strings::SPEAKING); - ResetDecoder(); - codec->EnableOutput(true); + + if (listening_mode_ != kListeningModeRealtime) { #if CONFIG_USE_AUDIO_PROCESSOR - audio_processor_.Stop(); + audio_processor_.Stop(); #endif #if CONFIG_USE_WAKE_WORD_DETECT - wake_word_detect_.StartDetection(); + wake_word_detect_.StartDetection(); #endif + } + ResetDecoder(); break; default: // Do nothing @@ -778,19 +799,28 @@ void Application::SetDeviceState(DeviceState state) { } } -void Application::SetDecodeSampleRate(int sample_rate) { - if (opus_decode_sample_rate_ == sample_rate) { +void Application::ResetDecoder() { + std::lock_guard lock(mutex_); + opus_decoder_->ResetState(); + audio_decode_queue_.clear(); + last_output_time_ = std::chrono::steady_clock::now(); + + auto codec = Board::GetInstance().GetAudioCodec(); + codec->EnableOutput(true); +} + +void Application::SetDecodeSampleRate(int sample_rate, int frame_duration) { + if (opus_decoder_->sample_rate() == sample_rate && opus_decoder_->duration_ms() == frame_duration) { return; } - opus_decode_sample_rate_ = sample_rate; opus_decoder_.reset(); - opus_decoder_ = std::make_unique(opus_decode_sample_rate_, 1); + opus_decoder_ = std::make_unique(sample_rate, 1, frame_duration); auto codec = Board::GetInstance().GetAudioCodec(); - if (opus_decode_sample_rate_ != codec->output_sample_rate()) { - ESP_LOGI(TAG, "Resampling audio from %d to %d", opus_decode_sample_rate_, codec->output_sample_rate()); - output_resampler_.Configure(opus_decode_sample_rate_, codec->output_sample_rate()); + if (opus_decoder_->sample_rate() != codec->output_sample_rate()) { + ESP_LOGI(TAG, "Resampling audio from %d to %d", opus_decoder_->sample_rate(), codec->output_sample_rate()); + output_resampler_.Configure(opus_decoder_->sample_rate(), codec->output_sample_rate()); } } diff --git a/main/application.h b/main/application.h index 63a5230c..eb6c90ab 100644 --- a/main/application.h +++ b/main/application.h @@ -88,12 +88,20 @@ private: EventGroupHandle_t event_group_ = nullptr; esp_timer_handle_t clock_timer_handle_ = nullptr; volatile DeviceState device_state_ = kDeviceStateUnknown; - bool keep_listening_ = false; + ListeningMode listening_mode_ = kListeningModeAutoStop; +#if CONFIG_USE_REALTIME_CHAT + bool realtime_chat_enabled_ = true; +#else + bool realtime_chat_enabled_ = false; +#endif bool aborted_ = false; bool voice_detected_ = false; int clock_ticks_ = 0; + TaskHandle_t main_loop_task_handle_ = nullptr; + TaskHandle_t check_new_version_task_handle_ = nullptr; // Audio encode / decode + TaskHandle_t audio_loop_task_handle_ = nullptr; BackgroundTask* background_task_ = nullptr; std::chrono::steady_clock::time_point last_output_time_; std::list> audio_decode_queue_; @@ -101,19 +109,21 @@ private: std::unique_ptr opus_encoder_; std::unique_ptr opus_decoder_; - int opus_decode_sample_rate_ = -1; OpusResampler input_resampler_; OpusResampler reference_resampler_; OpusResampler output_resampler_; void MainLoop(); - void InputAudio(); - void OutputAudio(); + void OnAudioInput(); + void OnAudioOutput(); + void ReadAudio(std::vector& data, int sample_rate, int samples); void ResetDecoder(); - void SetDecodeSampleRate(int sample_rate); + void SetDecodeSampleRate(int sample_rate, int frame_duration); void CheckNewVersion(); void ShowActivationCode(); void OnClockTimer(); + void SetListeningMode(ListeningMode mode); + void AudioLoop(); }; #endif // _APPLICATION_H_ diff --git a/main/audio_codecs/audio_codec.cc b/main/audio_codecs/audio_codec.cc index e7b5a7d2..6cd114d6 100644 --- a/main/audio_codecs/audio_codec.cc +++ b/main/audio_codecs/audio_codec.cc @@ -14,23 +14,11 @@ AudioCodec::AudioCodec() { AudioCodec::~AudioCodec() { } -void AudioCodec::OnInputReady(std::function callback) { - on_input_ready_ = callback; -} - -void AudioCodec::OnOutputReady(std::function callback) { - on_output_ready_ = callback; -} - void AudioCodec::OutputData(std::vector& data) { Write(data.data(), data.size()); } bool AudioCodec::InputData(std::vector& data) { - int duration = 30; - int input_frame_size = input_sample_rate_ / 1000 * duration * input_channels_; - - data.resize(input_frame_size); int samples = Read(data.data(), data.size()); if (samples > 0) { return true; @@ -38,22 +26,6 @@ bool AudioCodec::InputData(std::vector& data) { return false; } -IRAM_ATTR bool AudioCodec::on_sent(i2s_chan_handle_t handle, i2s_event_data_t *event, void *user_ctx) { - auto audio_codec = (AudioCodec*)user_ctx; - if (audio_codec->output_enabled_ && audio_codec->on_output_ready_) { - return audio_codec->on_output_ready_(); - } - return false; -} - -IRAM_ATTR bool AudioCodec::on_recv(i2s_chan_handle_t handle, i2s_event_data_t *event, void *user_ctx) { - auto audio_codec = (AudioCodec*)user_ctx; - if (audio_codec->input_enabled_ && audio_codec->on_input_ready_) { - return audio_codec->on_input_ready_(); - } - return false; -} - void AudioCodec::Start() { Settings settings("audio", false); output_volume_ = settings.GetInt("output_volume", output_volume_); @@ -62,15 +34,6 @@ void AudioCodec::Start() { output_volume_ = 10; } - // 注册音频数据回调 - i2s_event_callbacks_t rx_callbacks = {}; - rx_callbacks.on_recv = on_recv; - i2s_channel_register_event_callback(rx_handle_, &rx_callbacks, this); - - i2s_event_callbacks_t tx_callbacks = {}; - tx_callbacks.on_sent = on_sent; - i2s_channel_register_event_callback(tx_handle_, &tx_callbacks, this); - ESP_ERROR_CHECK(i2s_channel_enable(tx_handle_)); ESP_ERROR_CHECK(i2s_channel_enable(rx_handle_)); diff --git a/main/audio_codecs/audio_codec.h b/main/audio_codecs/audio_codec.h index 8a13ea6d..c97daff2 100644 --- a/main/audio_codecs/audio_codec.h +++ b/main/audio_codecs/audio_codec.h @@ -23,8 +23,6 @@ public: void Start(); void OutputData(std::vector& data); bool InputData(std::vector& data); - void OnOutputReady(std::function callback); - void OnInputReady(std::function callback); inline bool duplex() const { return duplex_; } inline bool input_reference() const { return input_reference_; } @@ -33,13 +31,8 @@ public: inline int input_channels() const { return input_channels_; } inline int output_channels() const { return output_channels_; } inline int output_volume() const { return output_volume_; } - -private: - std::function on_input_ready_; - std::function on_output_ready_; - - IRAM_ATTR static bool on_recv(i2s_chan_handle_t handle, i2s_event_data_t *event, void *user_ctx); - IRAM_ATTR static bool on_sent(i2s_chan_handle_t handle, i2s_event_data_t *event, void *user_ctx); + inline bool input_enabled() const { return input_enabled_; } + inline bool output_enabled() const { return output_enabled_; } protected: i2s_chan_handle_t tx_handle_ = nullptr; diff --git a/main/audio_processing/audio_processor.cc b/main/audio_processing/audio_processor.cc index 5447ab42..ff3efcb1 100644 --- a/main/audio_processing/audio_processor.cc +++ b/main/audio_processing/audio_processor.cc @@ -10,31 +10,41 @@ AudioProcessor::AudioProcessor() event_group_ = xEventGroupCreate(); } -void AudioProcessor::Initialize(int channels, bool reference) { - channels_ = channels; - reference_ = reference; - int ref_num = reference_ ? 1 : 0; +void AudioProcessor::Initialize(AudioCodec* codec, bool realtime_chat) { + codec_ = codec; + int ref_num = codec_->input_reference() ? 1 : 0; std::string input_format; - for (int i = 0; i < channels_ - ref_num; i++) { + for (int i = 0; i < codec_->input_channels() - ref_num; i++) { input_format.push_back('M'); } for (int i = 0; i < ref_num; i++) { input_format.push_back('R'); } + srmodel_list_t *models = esp_srmodel_init("model"); + char* ns_model_name = esp_srmodel_filter(models, ESP_NSNET_PREFIX, NULL); + afe_config_t* afe_config = afe_config_init(input_format.c_str(), NULL, AFE_TYPE_VC, AFE_MODE_HIGH_PERF); - afe_config->aec_init = false; - afe_config->aec_mode = AEC_MODE_VOIP_HIGH_PERF; + if (realtime_chat) { + afe_config->aec_init = true; + afe_config->aec_mode = AEC_MODE_VOIP_LOW_COST; + } else { + afe_config->aec_init = false; + } afe_config->ns_init = true; - afe_config->vad_init = true; - afe_config->vad_mode = VAD_MODE_0; - afe_config->vad_min_noise_ms = 100; + afe_config->ns_model_name = ns_model_name; + afe_config->afe_ns_mode = AFE_NS_MODE_NET; + if (realtime_chat) { + afe_config->vad_init = false; + } else { + afe_config->vad_init = true; + afe_config->vad_mode = VAD_MODE_0; + afe_config->vad_min_noise_ms = 100; + } afe_config->afe_perferred_core = 1; afe_config->afe_perferred_priority = 1; - afe_config->agc_init = true; - afe_config->agc_mode = AFE_AGC_MODE_WEBRTC; - afe_config->agc_compression_gain_db = 10; + afe_config->agc_init = false; afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM; afe_iface_ = esp_afe_handle_from_config(afe_config); @@ -54,15 +64,12 @@ AudioProcessor::~AudioProcessor() { vEventGroupDelete(event_group_); } -void AudioProcessor::Input(const std::vector& data) { - input_buffer_.insert(input_buffer_.end(), data.begin(), data.end()); +size_t AudioProcessor::GetFeedSize() { + return afe_iface_->get_feed_chunksize(afe_data_) * codec_->input_channels(); +} - auto feed_size = afe_iface_->get_feed_chunksize(afe_data_) * channels_; - while (input_buffer_.size() >= feed_size) { - auto chunk = input_buffer_.data(); - afe_iface_->feed(afe_data_, chunk); - input_buffer_.erase(input_buffer_.begin(), input_buffer_.begin() + feed_size); - } +void AudioProcessor::Feed(const std::vector& data) { + afe_iface_->feed(afe_data_, data.data()); } void AudioProcessor::Start() { diff --git a/main/audio_processing/audio_processor.h b/main/audio_processing/audio_processor.h index 7864cec5..3c2c4b75 100644 --- a/main/audio_processing/audio_processor.h +++ b/main/audio_processing/audio_processor.h @@ -10,28 +10,29 @@ #include #include +#include "audio_codec.h" + class AudioProcessor { public: AudioProcessor(); ~AudioProcessor(); - void Initialize(int channels, bool reference); - void Input(const std::vector& data); + void Initialize(AudioCodec* codec, bool realtime_chat); + void Feed(const std::vector& data); void Start(); void Stop(); bool IsRunning(); void OnOutput(std::function&& data)> callback); void OnVadStateChange(std::function callback); + size_t GetFeedSize(); private: EventGroupHandle_t event_group_ = nullptr; esp_afe_sr_iface_t* afe_iface_ = nullptr; esp_afe_sr_data_t* afe_data_ = nullptr; - std::vector input_buffer_; std::function&& data)> output_callback_; std::function vad_state_change_callback_; - int channels_; - bool reference_; + AudioCodec* codec_ = nullptr; bool is_speaking_ = false; void AudioProcessorTask(); diff --git a/main/audio_processing/wake_word_detect.cc b/main/audio_processing/wake_word_detect.cc index 91fd2123..e2b864a2 100644 --- a/main/audio_processing/wake_word_detect.cc +++ b/main/audio_processing/wake_word_detect.cc @@ -30,10 +30,9 @@ WakeWordDetect::~WakeWordDetect() { vEventGroupDelete(event_group_); } -void WakeWordDetect::Initialize(int channels, bool reference) { - channels_ = channels; - reference_ = reference; - int ref_num = reference_ ? 1 : 0; +void WakeWordDetect::Initialize(AudioCodec* codec) { + codec_ = codec; + int ref_num = codec_->input_reference() ? 1 : 0; srmodel_list_t *models = esp_srmodel_init("model"); for (int i = 0; i < models->num; i++) { @@ -51,14 +50,14 @@ void WakeWordDetect::Initialize(int channels, bool reference) { } std::string input_format; - for (int i = 0; i < channels_ - ref_num; i++) { + for (int i = 0; i < codec_->input_channels() - ref_num; i++) { input_format.push_back('M'); } for (int i = 0; i < ref_num; i++) { input_format.push_back('R'); } afe_config_t* afe_config = afe_config_init(input_format.c_str(), models, AFE_TYPE_SR, AFE_MODE_HIGH_PERF); - afe_config->aec_init = reference_; + afe_config->aec_init = codec_->input_reference(); afe_config->aec_mode = AEC_MODE_SR_HIGH_PERF; afe_config->afe_perferred_core = 1; afe_config->afe_perferred_priority = 1; @@ -92,13 +91,11 @@ bool WakeWordDetect::IsDetectionRunning() { } void WakeWordDetect::Feed(const std::vector& data) { - input_buffer_.insert(input_buffer_.end(), data.begin(), data.end()); + afe_iface_->feed(afe_data_, data.data()); +} - auto feed_size = afe_iface_->get_feed_chunksize(afe_data_) * channels_; - while (input_buffer_.size() >= feed_size) { - afe_iface_->feed(afe_data_, input_buffer_.data()); - input_buffer_.erase(input_buffer_.begin(), input_buffer_.begin() + feed_size); - } +size_t WakeWordDetect::GetFeedSize() { + return afe_iface_->get_feed_chunksize(afe_data_) * codec_->input_channels(); } void WakeWordDetect::AudioDetectionTask() { diff --git a/main/audio_processing/wake_word_detect.h b/main/audio_processing/wake_word_detect.h index 54baf5df..583b50c9 100644 --- a/main/audio_processing/wake_word_detect.h +++ b/main/audio_processing/wake_word_detect.h @@ -15,18 +15,20 @@ #include #include +#include "audio_codec.h" class WakeWordDetect { public: WakeWordDetect(); ~WakeWordDetect(); - void Initialize(int channels, bool reference); + void Initialize(AudioCodec* codec); void Feed(const std::vector& data); void OnWakeWordDetected(std::function callback); void StartDetection(); void StopDetection(); bool IsDetectionRunning(); + size_t GetFeedSize(); void EncodeWakeWordData(); bool GetWakeWordOpus(std::vector& opus); const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; } @@ -36,11 +38,9 @@ private: esp_afe_sr_data_t* afe_data_ = nullptr; char* wakenet_model_ = NULL; std::vector wake_words_; - std::vector input_buffer_; EventGroupHandle_t event_group_; std::function wake_word_detected_callback_; - int channels_; - bool reference_; + AudioCodec* codec_ = nullptr; std::string last_detected_wake_word_; TaskHandle_t wake_word_encode_task_ = nullptr; diff --git a/main/boards/esp-box-3/esp_box3_board.cc b/main/boards/esp-box-3/esp_box3_board.cc index 67c58530..b8d9eda1 100644 --- a/main/boards/esp-box-3/esp_box3_board.cc +++ b/main/boards/esp-box-3/esp_box3_board.cc @@ -126,7 +126,11 @@ private: { .text_font = &font_puhui_20_4, .icon_font = &font_awesome_20_4, +#if CONFIG_USE_WECHAT_MESSAGE_STYLE + .emoji_font = font_emoji_32_init(), +#else .emoji_font = font_emoji_64_init(), +#endif }); } diff --git a/main/boards/esp32-s3-touch-amoled-1.8/config.json b/main/boards/esp32-s3-touch-amoled-1.8/config.json index 54df6d48..67194974 100644 --- a/main/boards/esp32-s3-touch-amoled-1.8/config.json +++ b/main/boards/esp32-s3-touch-amoled-1.8/config.json @@ -3,7 +3,9 @@ "builds": [ { "name": "esp32-s3-touch-amoled-1.8", - "sdkconfig_append": [] + "sdkconfig_append": [ + "CONFIG_USE_WECHAT_MESSAGE_STYLE=y" + ] } ] } \ No newline at end of file diff --git a/main/boards/esp32-s3-touch-amoled-1.8/esp32-s3-touch-amoled-1.8.cc b/main/boards/esp32-s3-touch-amoled-1.8/esp32-s3-touch-amoled-1.8.cc index 7e0a595a..0331adaa 100644 --- a/main/boards/esp32-s3-touch-amoled-1.8/esp32-s3-touch-amoled-1.8.cc +++ b/main/boards/esp32-s3-touch-amoled-1.8/esp32-s3-touch-amoled-1.8.cc @@ -87,7 +87,11 @@ public: { .text_font = &font_puhui_30_4, .icon_font = &font_awesome_30_4, +#if CONFIG_USE_WECHAT_MESSAGE_STYLE + .emoji_font = font_emoji_32_init(), +#else .emoji_font = font_emoji_64_init(), +#endif }) { DisplayLockGuard lock(this); lv_obj_set_style_pad_left(status_bar_, LV_HOR_RES * 0.1, 0); diff --git a/main/display/lcd_display.cc b/main/display/lcd_display.cc index 852b6d1a..7488dd1f 100644 --- a/main/display/lcd_display.cc +++ b/main/display/lcd_display.cc @@ -103,6 +103,7 @@ SpiLcdDisplay::SpiLcdDisplay(esp_lcd_panel_io_handle_t panel_io, esp_lcd_panel_h ESP_LOGI(TAG, "Initialize LVGL port"); lvgl_port_cfg_t port_cfg = ESP_LVGL_PORT_INIT_CONFIG(); port_cfg.task_priority = 1; + port_cfg.timer_period_ms = 50; lvgl_port_init(&port_cfg); ESP_LOGI(TAG, "Adding LCD screen"); @@ -278,7 +279,7 @@ void LcdDisplay::SetupUI() { /* Status bar */ status_bar_ = lv_obj_create(container_); - lv_obj_set_size(status_bar_, LV_HOR_RES, fonts_.emoji_font->line_height); + lv_obj_set_size(status_bar_, LV_HOR_RES, LV_SIZE_CONTENT); lv_obj_set_style_radius(status_bar_, 0, 0); lv_obj_set_style_bg_color(status_bar_, current_theme.background, 0); lv_obj_set_style_text_color(status_bar_, current_theme.text, 0); @@ -288,7 +289,7 @@ void LcdDisplay::SetupUI() { lv_obj_set_style_radius(content_, 0, 0); lv_obj_set_width(content_, LV_HOR_RES); lv_obj_set_flex_grow(content_, 1); - lv_obj_set_style_pad_all(content_, 5, 0); + lv_obj_set_style_pad_all(content_, 10, 0); lv_obj_set_style_bg_color(content_, current_theme.chat_background, 0); // Background for chat area lv_obj_set_style_border_color(content_, current_theme.border, 0); // Border color for chat area @@ -309,8 +310,10 @@ void LcdDisplay::SetupUI() { lv_obj_set_style_pad_all(status_bar_, 0, 0); lv_obj_set_style_border_width(status_bar_, 0, 0); lv_obj_set_style_pad_column(status_bar_, 0, 0); - lv_obj_set_style_pad_left(status_bar_, 2, 0); - lv_obj_set_style_pad_right(status_bar_, 2, 0); + lv_obj_set_style_pad_left(status_bar_, 10, 0); + lv_obj_set_style_pad_right(status_bar_, 10, 0); + lv_obj_set_style_pad_top(status_bar_, 2, 0); + lv_obj_set_style_pad_bottom(status_bar_, 2, 0); lv_obj_set_scrollbar_mode(status_bar_, LV_SCROLLBAR_MODE_OFF); // 设置状态栏的内容垂直居中 lv_obj_set_flex_align(status_bar_, LV_FLEX_ALIGN_SPACE_BETWEEN, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER); @@ -366,7 +369,7 @@ void LcdDisplay::SetupUI() { lv_obj_add_flag(low_battery_popup_, LV_OBJ_FLAG_HIDDEN); } -#define MAX_MESSAGES 50 +#define MAX_MESSAGES 20 void LcdDisplay::SetChatMessage(const char* role, const char* content) { DisplayLockGuard lock(this); if (content_ == nullptr) { @@ -376,6 +379,21 @@ void LcdDisplay::SetChatMessage(const char* role, const char* content) { //避免出现空的消息框 if(strlen(content) == 0) return; + // 检查消息数量是否超过限制 + uint32_t child_count = lv_obj_get_child_cnt(content_); + if (child_count >= MAX_MESSAGES) { + // 删除最早的消息(第一个子对象) + lv_obj_t* first_child = lv_obj_get_child(content_, 0); + lv_obj_t* last_child = lv_obj_get_child(content_, child_count - 1); + if (first_child != nullptr) { + lv_obj_del(first_child); + } + // Scroll to the last message immediately + if (last_child != nullptr) { + lv_obj_scroll_to_view_recursive(last_child, LV_ANIM_OFF); + } + } + // Create a message bubble lv_obj_t* msg_bubble = lv_obj_create(content_); lv_obj_set_style_radius(msg_bubble, 8, 0); @@ -431,9 +449,6 @@ void LcdDisplay::SetChatMessage(const char* role, const char* content) { lv_obj_set_width(msg_bubble, LV_SIZE_CONTENT); lv_obj_set_height(msg_bubble, LV_SIZE_CONTENT); - // Add some margin - lv_obj_set_style_margin_right(msg_bubble, 10, 0); - // Don't grow lv_obj_set_style_flex_grow(msg_bubble, 0, 0); } else if (strcmp(role, "assistant") == 0) { @@ -449,9 +464,6 @@ void LcdDisplay::SetChatMessage(const char* role, const char* content) { lv_obj_set_width(msg_bubble, LV_SIZE_CONTENT); lv_obj_set_height(msg_bubble, LV_SIZE_CONTENT); - // Add some margin - lv_obj_set_style_margin_left(msg_bubble, -4, 0); - // Don't grow lv_obj_set_style_flex_grow(msg_bubble, 0, 0); } else if (strcmp(role, "system") == 0) { @@ -487,7 +499,7 @@ void LcdDisplay::SetChatMessage(const char* role, const char* content) { lv_obj_set_parent(msg_bubble, container); // Right align the bubble in the container - lv_obj_align(msg_bubble, LV_ALIGN_RIGHT_MID, -10, 0); + lv_obj_align(msg_bubble, LV_ALIGN_RIGHT_MID, -25, 0); // Auto-scroll to this container lv_obj_scroll_to_view_recursive(container, LV_ANIM_ON); @@ -521,22 +533,6 @@ void LcdDisplay::SetChatMessage(const char* role, const char* content) { // Store reference to the latest message label chat_message_label_ = msg_text; - - // 检查消息数量是否超过限制 - uint32_t msg_count = lv_obj_get_child_cnt(content_); - while (msg_count >= MAX_MESSAGES) { - // 删除最早的消息(第一个子节点) - lv_obj_t* oldest_msg = lv_obj_get_child(content_, 0); - if (oldest_msg != nullptr) { - lv_obj_del(oldest_msg); - msg_count--; - // 删除最早的消息会导致所有气泡整体往上移 - // 所以需要重新滚动到当前消息气泡位置 - lv_obj_scroll_to_view_recursive(msg_bubble, LV_ANIM_ON); - }else{ - break; - } - } } #else void LcdDisplay::SetupUI() { diff --git a/main/display/oled_display.cc b/main/display/oled_display.cc index 248f8786..3b1b905a 100644 --- a/main/display/oled_display.cc +++ b/main/display/oled_display.cc @@ -22,6 +22,7 @@ OledDisplay::OledDisplay(esp_lcd_panel_io_handle_t panel_io, esp_lcd_panel_handl ESP_LOGI(TAG, "Initialize LVGL"); lvgl_port_cfg_t port_cfg = ESP_LVGL_PORT_INIT_CONFIG(); port_cfg.task_priority = 1; + port_cfg.timer_period_ms = 50; lvgl_port_init(&port_cfg); ESP_LOGI(TAG, "Adding LCD screen"); diff --git a/main/idf_component.yml b/main/idf_component.yml index b47c7f5b..5d3cf2fd 100644 --- a/main/idf_component.yml +++ b/main/idf_component.yml @@ -9,8 +9,8 @@ dependencies: espressif/esp_lcd_panel_io_additions: "^1.0.1" 78/esp_lcd_nv3023: "~1.0.0" 78/esp-wifi-connect: "~2.3.1" - 78/esp-opus-encoder: "~2.1.0" - 78/esp-ml307: "~1.7.2" + 78/esp-opus-encoder: "~2.3.0" + 78/esp-ml307: "~1.7.3" 78/xiaozhi-fonts: "~1.3.2" espressif/led_strip: "^2.4.1" espressif/esp_codec_dev: "~1.3.2" diff --git a/main/protocols/mqtt_protocol.cc b/main/protocols/mqtt_protocol.cc index 3a8d07e3..4a118778 100644 --- a/main/protocols/mqtt_protocol.cc +++ b/main/protocols/mqtt_protocol.cc @@ -254,6 +254,10 @@ void MqttProtocol::ParseServerHello(const cJSON* root) { if (sample_rate != NULL) { server_sample_rate_ = sample_rate->valueint; } + auto frame_duration = cJSON_GetObjectItem(audio_params, "frame_duration"); + if (frame_duration != NULL) { + server_frame_duration_ = frame_duration->valueint; + } } auto udp = cJSON_GetObjectItem(root, "udp"); diff --git a/main/protocols/protocol.cc b/main/protocols/protocol.cc index 5b8c9a29..e4e0b56f 100644 --- a/main/protocols/protocol.cc +++ b/main/protocols/protocol.cc @@ -49,7 +49,7 @@ void Protocol::SendWakeWordDetected(const std::string& wake_word) { void Protocol::SendStartListening(ListeningMode mode) { std::string message = "{\"session_id\":\"" + session_id_ + "\""; message += ",\"type\":\"listen\",\"state\":\"start\""; - if (mode == kListeningModeAlwaysOn) { + if (mode == kListeningModeRealtime) { message += ",\"mode\":\"realtime\""; } else if (mode == kListeningModeAutoStop) { message += ",\"mode\":\"auto\""; diff --git a/main/protocols/protocol.h b/main/protocols/protocol.h index 73f7b989..c1ef1d47 100644 --- a/main/protocols/protocol.h +++ b/main/protocols/protocol.h @@ -21,7 +21,7 @@ enum AbortReason { enum ListeningMode { kListeningModeAutoStop, kListeningModeManualStop, - kListeningModeAlwaysOn // 需要 AEC 支持 + kListeningModeRealtime // 需要 AEC 支持 }; class Protocol { @@ -31,6 +31,9 @@ public: inline int server_sample_rate() const { return server_sample_rate_; } + inline int server_frame_duration() const { + return server_frame_duration_; + } inline const std::string& session_id() const { return session_id_; } @@ -60,7 +63,8 @@ protected: std::function on_audio_channel_closed_; std::function on_network_error_; - int server_sample_rate_ = 16000; + int server_sample_rate_ = 24000; + int server_frame_duration_ = 60; bool error_occurred_ = false; std::string session_id_; std::chrono::time_point last_incoming_time_; diff --git a/main/protocols/websocket_protocol.cc b/main/protocols/websocket_protocol.cc index 9a6b997b..e3d25b30 100644 --- a/main/protocols/websocket_protocol.cc +++ b/main/protocols/websocket_protocol.cc @@ -146,6 +146,10 @@ void WebsocketProtocol::ParseServerHello(const cJSON* root) { if (sample_rate != NULL) { server_sample_rate_ = sample_rate->valueint; } + auto frame_duration = cJSON_GetObjectItem(audio_params, "frame_duration"); + if (frame_duration != NULL) { + server_frame_duration_ = frame_duration->valueint; + } } xEventGroupSetBits(event_group_handle_, WEBSOCKET_PROTOCOL_SERVER_HELLO_EVENT);