diff --git a/CMakeLists.txt b/CMakeLists.txt index d5c4564c..631ab4b3 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ # CMakeLists in this exact order for cmake to work correctly cmake_minimum_required(VERSION 3.16) -set(PROJECT_VER "0.9.2") +set(PROJECT_VER "0.9.3") include($ENV{IDF_PATH}/tools/cmake/project.cmake) project(xiaozhi) diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index b63d6193..40fb970e 100755 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -1,6 +1,7 @@ set(SOURCES "audio_codecs/audio_codec.cc" "audio_codecs/no_audio_codec.cc" "audio_codecs/box_audio_codec.cc" + "audio_codecs/es8311_audio_codec.cc" "display/display.cc" "display/no_display.cc" "display/st7789_display.cc" @@ -12,10 +13,11 @@ set(SOURCES "audio_codecs/audio_codec.cc" "application.cc" "ota.cc" "settings.cc" + "background_task.cc" "main.cc" ) -set(INCLUDE_DIRS "." "display" "audio_codecs" "protocols") +set(INCLUDE_DIRS "." "display" "audio_codecs" "protocols" "audio_processing") # 字体 file(GLOB FONT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/fonts/*.c) @@ -38,14 +40,16 @@ elseif(CONFIG_BOARD_TYPE_KEVIN_BOX_1) set(BOARD_TYPE "kevin-box-1") elseif(CONFIG_BOARD_TYPE_KEVIN_BOX_2) set(BOARD_TYPE "kevin-box-2") +elseif(CONFIG_BOARD_TYPE_KEVIN_C3) + set(BOARD_TYPE "kevin-c3") elseif(CONFIG_BOARD_TYPE_LICHUANG_DEV) set(BOARD_TYPE "lichuang-dev") endif() file(GLOB BOARD_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/boards/${BOARD_TYPE}/*.cc) list(APPEND SOURCES ${BOARD_SOURCES}) -if(CONFIG_USE_AFE_SR) - list(APPEND SOURCES "audio_processor.cc" "wake_word_detect.cc") +if(CONFIG_IDF_TARGET_ESP32S3) + list(APPEND SOURCES "audio_processing/audio_processor.cc" "audio_processing/wake_word_detect.cc") endif() idf_component_register(SRCS ${SOURCES} diff --git a/main/Kconfig.projbuild b/main/Kconfig.projbuild index ff548c98..0113fdcc 100644 --- a/main/Kconfig.projbuild +++ b/main/Kconfig.projbuild @@ -46,6 +46,8 @@ choice BOARD_TYPE bool "Kevin Box 1" config BOARD_TYPE_KEVIN_BOX_2 bool "Kevin Box 2" + config BOARD_TYPE_KEVIN_C3 + bool "Kevin C3" config BOARD_TYPE_LICHUANG_DEV bool "立创开发板" endchoice diff --git a/main/application.cc b/main/application.cc index 19b4147c..e1a57777 100644 --- a/main/application.cc +++ b/main/application.cc @@ -22,7 +22,7 @@ extern const char p3_err_wificonfig_start[] asm("_binary_err_wificonfig_p3_start extern const char p3_err_wificonfig_end[] asm("_binary_err_wificonfig_p3_end"); -Application::Application() { +Application::Application() : background_task_(4096 * 8) { event_group_ = xEventGroupCreate(); ota_.SetCheckVersionUrl(CONFIG_OTA_VERSION_URL); @@ -36,9 +36,6 @@ Application::~Application() { if (opus_decoder_ != nullptr) { opus_decoder_destroy(opus_decoder_); } - if (audio_encode_task_stack_ != nullptr) { - heap_caps_free(audio_encode_task_stack_); - } vEventGroupDelete(event_group_); } @@ -116,7 +113,6 @@ void Application::PlayLocalFile(const char* data, size_t size) { std::lock_guard lock(mutex_); audio_decode_queue_.emplace_back(std::move(opus)); } - cv_.notify_all(); } void Application::ToggleChatState() { @@ -157,6 +153,8 @@ void Application::StartListening() { } else if (chat_state_ == kChatStateSpeaking) { AbortSpeaking(kAbortReasonNone); protocol_->SendStartListening(kListeningModeManualStop); + // FIXME: Wait for the speaker to empty the buffer + vTaskDelay(pdMS_TO_TICKS(120)); SetChatState(kChatStateListening); } }); @@ -164,8 +162,10 @@ void Application::StartListening() { void Application::StopListening() { Schedule([this]() { - protocol_->SendStopListening(); - SetChatState(kChatStateIdle); + if (chat_state_ == kChatStateListening) { + protocol_->SendStopListening(); + SetChatState(kChatStateIdle); + } }); } @@ -184,71 +184,32 @@ void Application::Start() { auto codec = board.GetAudioCodec(); opus_decode_sample_rate_ = codec->output_sample_rate(); opus_decoder_ = opus_decoder_create(opus_decode_sample_rate_, 1, NULL); - opus_encoder_.Configure(16000, 1); + opus_encoder_.Configure(16000, 1, OPUS_FRAME_DURATION_MS); if (codec->input_sample_rate() != 16000) { input_resampler_.Configure(codec->input_sample_rate(), 16000); reference_resampler_.Configure(codec->input_sample_rate(), 16000); } - codec->OnInputData([this, codec](std::vector&& data) { - if (codec->input_sample_rate() != 16000) { - if (codec->input_channels() == 2) { - auto mic_channel = std::vector(data.size() / 2); - auto reference_channel = std::vector(data.size() / 2); - for (size_t i = 0, j = 0; i < mic_channel.size(); ++i, j += 2) { - mic_channel[i] = data[j]; - reference_channel[i] = data[j + 1]; - } - auto resampled_mic = std::vector(input_resampler_.GetOutputSamples(mic_channel.size())); - auto resampled_reference = std::vector(reference_resampler_.GetOutputSamples(reference_channel.size())); - input_resampler_.Process(mic_channel.data(), mic_channel.size(), resampled_mic.data()); - reference_resampler_.Process(reference_channel.data(), reference_channel.size(), resampled_reference.data()); - data.resize(resampled_mic.size() + resampled_reference.size()); - for (size_t i = 0, j = 0; i < resampled_mic.size(); ++i, j += 2) { - data[j] = resampled_mic[i]; - data[j + 1] = resampled_reference[i]; - } - } else { - auto resampled = std::vector(input_resampler_.GetOutputSamples(data.size())); - input_resampler_.Process(data.data(), data.size(), resampled.data()); - data = std::move(resampled); - } - } -#ifdef CONFIG_USE_AFE_SR - if (audio_processor_.IsRunning()) { - audio_processor_.Input(data); - } - if (wake_word_detect_.IsDetectionRunning()) { - wake_word_detect_.Feed(data); - } -#else - Schedule([this, data = std::move(data)]() { - if (chat_state_ == kChatStateListening) { - std::lock_guard lock(mutex_); - audio_encode_queue_.emplace_back(std::move(data)); - cv_.notify_all(); - } - }); -#endif + codec->OnInputReady([this, codec]() { + BaseType_t higher_priority_task_woken = pdFALSE; + xEventGroupSetBitsFromISR(event_group_, AUDIO_INPUT_READY_EVENT, &higher_priority_task_woken); + return higher_priority_task_woken == pdTRUE; + }); + codec->OnOutputReady([this]() { + BaseType_t higher_priority_task_woken = pdFALSE; + xEventGroupSetBitsFromISR(event_group_, AUDIO_OUTPUT_READY_EVENT, &higher_priority_task_woken); + return higher_priority_task_woken == pdTRUE; }); - - const size_t opus_stack_size = 4096 * 8; // OPUS encoder / decoder use a lot of stack memory - audio_encode_task_stack_ = (StackType_t*)heap_caps_malloc(opus_stack_size, MALLOC_CAP_SPIRAM); - audio_encode_task_ = xTaskCreateStatic([](void* arg) { - Application* app = (Application*)arg; - app->AudioEncodeTask(); - vTaskDelete(NULL); - }, "opus_encode", opus_stack_size, this, 1, audio_encode_task_stack_, &audio_encode_task_buffer_); - codec->Start(); - /* Wait for the network to be ready */ - board.StartNetwork(); - + /* Start the main loop */ xTaskCreate([](void* arg) { Application* app = (Application*)arg; app->MainLoop(); vTaskDelete(NULL); - }, "main_loop", 4096 * 2, this, 1, nullptr); + }, "main_loop", 4096 * 2, this, 2, nullptr); + + /* Wait for the network to be ready */ + board.StartNetwork(); // Check for new firmware version or get the MQTT broker address xTaskCreate([](void* arg) { @@ -257,12 +218,16 @@ void Application::Start() { vTaskDelete(NULL); }, "check_new_version", 4096 * 2, this, 1, nullptr); -#ifdef CONFIG_USE_AFE_SR +#if CONFIG_IDF_TARGET_ESP32S3 audio_processor_.Initialize(codec->input_channels(), codec->input_reference()); audio_processor_.OnOutput([this](std::vector&& data) { - std::lock_guard lock(mutex_); - audio_encode_queue_.emplace_back(std::move(data)); - cv_.notify_all(); + background_task_.Schedule([this, data = std::move(data)]() { + opus_encoder_.Encode(data, [this](const uint8_t* opus, size_t opus_size) { + Schedule([this, opus = std::string(reinterpret_cast(opus), opus_size)]() { + protocol_->SendAudio(opus); + }); + }); + }); }); wake_word_detect_.Initialize(codec->input_channels(), codec->input_reference()); @@ -326,8 +291,9 @@ void Application::Start() { }); protocol_->OnIncomingAudio([this](const std::string& data) { std::lock_guard lock(mutex_); - audio_decode_queue_.emplace_back(std::move(data)); - cv_.notify_all(); + if (chat_state_ == kChatStateSpeaking) { + audio_decode_queue_.emplace_back(std::move(data)); + } }); protocol_->OnAudioChannelOpened([this, codec, &board]() { if (protocol_->server_sample_rate() != codec->output_sample_rate()) { @@ -350,17 +316,15 @@ void Application::Start() { auto state = cJSON_GetObjectItem(root, "state"); if (strcmp(state->valuestring, "start") == 0) { Schedule([this]() { + aborted_ = false; if (chat_state_ == kChatStateIdle || chat_state_ == kChatStateListening) { - skip_to_end_ = false; - opus_decoder_ctl(opus_decoder_, OPUS_RESET_STATE); SetChatState(kChatStateSpeaking); } }); } else if (strcmp(state->valuestring, "stop") == 0) { Schedule([this]() { - auto codec = Board::GetInstance().GetAudioCodec(); - codec->WaitForOutputDone(); if (chat_state_ == kChatStateSpeaking) { + background_task_.WaitForCompletion(); if (keep_listening_) { protocol_->SendStartListening(kListeningModeAutoStop); SetChatState(kChatStateListening); @@ -399,9 +363,10 @@ void Application::Start() { } void Application::Schedule(std::function callback) { - std::lock_guard lock(mutex_); + mutex_.lock(); main_tasks_.push_back(callback); - cv_.notify_all(); + mutex_.unlock(); + xEventGroupSetBits(event_group_, SCHEDULE_EVENT); } // The Main Loop controls the chat state and websocket connection @@ -409,24 +374,140 @@ void Application::Schedule(std::function callback) { // they should use Schedule to call this function void Application::MainLoop() { while (true) { - std::unique_lock lock(mutex_); - cv_.wait(lock, [this]() { - return !main_tasks_.empty(); - }); - auto task = std::move(main_tasks_.front()); - main_tasks_.pop_front(); - lock.unlock(); - task(); + auto bits = xEventGroupWaitBits(event_group_, + SCHEDULE_EVENT | AUDIO_INPUT_READY_EVENT | AUDIO_OUTPUT_READY_EVENT, + pdTRUE, pdFALSE, portMAX_DELAY); + + if (bits & AUDIO_INPUT_READY_EVENT) { + InputAudio(); + } + if (bits & AUDIO_OUTPUT_READY_EVENT) { + OutputAudio(); + } + if (bits & SCHEDULE_EVENT) { + mutex_.lock(); + std::list> tasks = std::move(main_tasks_); + mutex_.unlock(); + for (auto& task : tasks) { + task(); + } + } } } +void Application::ResetDecoder() { + std::lock_guard lock(mutex_); + opus_decoder_ctl(opus_decoder_, OPUS_RESET_STATE); + audio_decode_queue_.clear(); + last_output_time_ = std::chrono::steady_clock::now(); + Board::GetInstance().GetAudioCodec()->EnableOutput(true); +} + +void Application::OutputAudio() { + auto now = std::chrono::steady_clock::now(); + auto codec = Board::GetInstance().GetAudioCodec(); + const int max_silence_seconds = 10; + + std::unique_lock lock(mutex_); + if (audio_decode_queue_.empty()) { + // Disable the output if there is no audio data for a long time + auto duration = std::chrono::duration_cast(now - last_output_time_).count(); + if (duration > max_silence_seconds) { + codec->EnableOutput(false); + } + return; + } + + if (chat_state_ == kChatStateListening) { + audio_decode_queue_.clear(); + return; + } + + last_output_time_ = now; + auto opus = std::move(audio_decode_queue_.front()); + audio_decode_queue_.pop_front(); + lock.unlock(); + + background_task_.Schedule([this, codec, opus = std::move(opus)]() { + if (aborted_) { + return; + } + int frame_size = opus_decode_sample_rate_ * OPUS_FRAME_DURATION_MS / 1000; + std::vector pcm(frame_size); + + int ret = opus_decode(opus_decoder_, (const unsigned char*)opus.data(), opus.size(), pcm.data(), frame_size, 0); + if (ret < 0) { + ESP_LOGE(TAG, "Failed to decode audio, error code: %d", ret); + return; + } + + // Resample if the sample rate is different + if (opus_decode_sample_rate_ != codec->output_sample_rate()) { + int target_size = output_resampler_.GetOutputSamples(frame_size); + std::vector resampled(target_size); + output_resampler_.Process(pcm.data(), frame_size, resampled.data()); + pcm = std::move(resampled); + } + + codec->OutputData(pcm); + }); +} + +void Application::InputAudio() { + auto codec = Board::GetInstance().GetAudioCodec(); + std::vector data; + if (!codec->InputData(data)) { + return; + } + + if (codec->input_sample_rate() != 16000) { + if (codec->input_channels() == 2) { + auto mic_channel = std::vector(data.size() / 2); + auto reference_channel = std::vector(data.size() / 2); + for (size_t i = 0, j = 0; i < mic_channel.size(); ++i, j += 2) { + mic_channel[i] = data[j]; + reference_channel[i] = data[j + 1]; + } + auto resampled_mic = std::vector(input_resampler_.GetOutputSamples(mic_channel.size())); + auto resampled_reference = std::vector(reference_resampler_.GetOutputSamples(reference_channel.size())); + input_resampler_.Process(mic_channel.data(), mic_channel.size(), resampled_mic.data()); + reference_resampler_.Process(reference_channel.data(), reference_channel.size(), resampled_reference.data()); + data.resize(resampled_mic.size() + resampled_reference.size()); + for (size_t i = 0, j = 0; i < resampled_mic.size(); ++i, j += 2) { + data[j] = resampled_mic[i]; + data[j + 1] = resampled_reference[i]; + } + } else { + auto resampled = std::vector(input_resampler_.GetOutputSamples(data.size())); + input_resampler_.Process(data.data(), data.size(), resampled.data()); + data = std::move(resampled); + } + } + +#if CONFIG_IDF_TARGET_ESP32S3 + if (audio_processor_.IsRunning()) { + audio_processor_.Input(data); + } + if (wake_word_detect_.IsDetectionRunning()) { + wake_word_detect_.Feed(data); + } +#else + if (chat_state_ == kChatStateListening) { + background_task_.Schedule([this, data = std::move(data)]() { + opus_encoder_.Encode(data, [this](const uint8_t* opus, size_t opus_size) { + Schedule([this, opus = std::string(reinterpret_cast(opus), opus_size)]() { + protocol_->SendAudio(opus); + }); + }); + }); + } +#endif +} + void Application::AbortSpeaking(AbortReason reason) { ESP_LOGI(TAG, "Abort speaking"); + aborted_ = true; protocol_->SendAbortSpeaking(reason); - - skip_to_end_ = true; - auto codec = Board::GetInstance().GetAudioCodec(); - codec->ClearOutputQueue(); } void Application::SetChatState(ChatState state) { @@ -444,6 +525,11 @@ void Application::SetChatState(ChatState state) { return; } + chat_state_ = state; + ESP_LOGI(TAG, "STATE: %s", state_str[chat_state_]); + // The state is changed, wait for all background tasks to finish + background_task_.WaitForCompletion(); + auto display = Board::GetInstance().GetDisplay(); auto builtin_led = Board::GetInstance().GetBuiltinLed(); switch (state) { @@ -452,7 +538,7 @@ void Application::SetChatState(ChatState state) { builtin_led->TurnOff(); display->SetStatus("待命"); display->SetEmotion("neutral"); -#ifdef CONFIG_USE_AFE_SR +#ifdef CONFIG_IDF_TARGET_ESP32S3 audio_processor_.Stop(); #endif break; @@ -466,8 +552,9 @@ void Application::SetChatState(ChatState state) { builtin_led->TurnOn(); display->SetStatus("聆听中..."); display->SetEmotion("neutral"); + ResetDecoder(); opus_encoder_.ResetState(); -#ifdef CONFIG_USE_AFE_SR +#if CONFIG_IDF_TARGET_ESP32S3 audio_processor_.Start(); #endif break; @@ -475,7 +562,8 @@ void Application::SetChatState(ChatState state) { builtin_led->SetGreen(); builtin_led->TurnOn(); display->SetStatus("说话中..."); -#ifdef CONFIG_USE_AFE_SR + ResetDecoder(); +#if CONFIG_IDF_TARGET_ESP32S3 audio_processor_.Stop(); #endif break; @@ -487,59 +575,6 @@ void Application::SetChatState(ChatState state) { ESP_LOGE(TAG, "Invalid chat state: %d", chat_state_); return; } - - chat_state_ = state; - ESP_LOGI(TAG, "STATE: %s", state_str[chat_state_]); -} - -void Application::AudioEncodeTask() { - ESP_LOGI(TAG, "Audio encode task started"); - auto codec = Board::GetInstance().GetAudioCodec(); - - while (true) { - std::unique_lock lock(mutex_); - cv_.wait(lock, [this]() { - return !audio_encode_queue_.empty() || !audio_decode_queue_.empty(); - }); - - if (!audio_encode_queue_.empty()) { - auto pcm = std::move(audio_encode_queue_.front()); - audio_encode_queue_.pop_front(); - lock.unlock(); - - opus_encoder_.Encode(pcm, [this](const uint8_t* opus, size_t opus_size) { - Schedule([this, data = std::string(reinterpret_cast(opus), opus_size)]() { - protocol_->SendAudio(data); - }); - }); - } else if (!audio_decode_queue_.empty()) { - auto opus = std::move(audio_decode_queue_.front()); - audio_decode_queue_.pop_front(); - lock.unlock(); - - if (skip_to_end_) { - continue; - } - - int frame_size = opus_decode_sample_rate_ * OPUS_FRAME_DURATION_MS / 1000; - std::vector pcm(frame_size); - - int ret = opus_decode(opus_decoder_, (const unsigned char*)opus.data(), opus.size(), pcm.data(), frame_size, 0); - if (ret < 0) { - ESP_LOGE(TAG, "Failed to decode audio, error code: %d", ret); - continue; - } - - // Resample if the sample rate is different - if (opus_decode_sample_rate_ != codec->output_sample_rate()) { - int target_size = output_resampler_.GetOutputSamples(frame_size); - std::vector resampled(target_size); - output_resampler_.Process(pcm.data(), frame_size, resampled.data()); - pcm = std::move(resampled); - } - codec->OutputData(pcm); - } - } } void Application::SetDecodeSampleRate(int sample_rate) { diff --git a/main/application.h b/main/application.h index fc91f242..ff7923b4 100644 --- a/main/application.h +++ b/main/application.h @@ -16,12 +16,16 @@ #include "display.h" #include "board.h" #include "ota.h" +#include "background_task.h" -#ifdef CONFIG_USE_AFE_SR +#if CONFIG_IDF_TARGET_ESP32S3 #include "wake_word_detect.h" #include "audio_processor.h" #endif +#define SCHEDULE_EVENT (1 << 0) +#define AUDIO_INPUT_READY_EVENT (1 << 1) +#define AUDIO_OUTPUT_READY_EVENT (1 << 2) enum ChatState { kChatStateUnknown, @@ -58,25 +62,22 @@ private: Application(); ~Application(); -#ifdef CONFIG_USE_AFE_SR +#if CONFIG_IDF_TARGET_ESP32S3 WakeWordDetect wake_word_detect_; AudioProcessor audio_processor_; #endif Ota ota_; std::mutex mutex_; - std::condition_variable_any cv_; std::list> main_tasks_; Protocol* protocol_ = nullptr; EventGroupHandle_t event_group_; volatile ChatState chat_state_ = kChatStateUnknown; bool keep_listening_ = false; - bool skip_to_end_ = false; + bool aborted_ = false; // Audio encode / decode - TaskHandle_t audio_encode_task_ = nullptr; - StaticTask_t audio_encode_task_buffer_; - StackType_t* audio_encode_task_stack_ = nullptr; - std::list> audio_encode_queue_; + BackgroundTask background_task_; + std::chrono::steady_clock::time_point last_output_time_; std::list audio_decode_queue_; OpusEncoder opus_encoder_; @@ -88,10 +89,12 @@ private: OpusResampler output_resampler_; void MainLoop(); + void InputAudio(); + void OutputAudio(); + void ResetDecoder(); void SetDecodeSampleRate(int sample_rate); void CheckNewVersion(); - void AudioEncodeTask(); void PlayLocalFile(const char* data, size_t size); }; diff --git a/main/audio_codecs/audio_codec.cc b/main/audio_codecs/audio_codec.cc index cdfb568f..07b0678f 100644 --- a/main/audio_codecs/audio_codec.cc +++ b/main/audio_codecs/audio_codec.cc @@ -9,34 +9,48 @@ #define TAG "AudioCodec" AudioCodec::AudioCodec() { - audio_event_group_ = xEventGroupCreate(); } AudioCodec::~AudioCodec() { - if (audio_input_task_ != nullptr) { - vTaskDelete(audio_input_task_); - } - if (audio_output_task_ != nullptr) { - vTaskDelete(audio_output_task_); - } - if (audio_event_group_ != nullptr) { - vEventGroupDelete(audio_event_group_); - } } -void AudioCodec::OnInputData(std::function&& data)> callback) { - on_input_data_ = callback; +void AudioCodec::OnInputReady(std::function callback) { + on_input_ready_ = callback; +} + +void AudioCodec::OnOutputReady(std::function callback) { + on_output_ready_ = callback; } void AudioCodec::OutputData(std::vector& data) { - std::lock_guard lock(audio_output_queue_mutex_); - audio_output_queue_.emplace_back(std::move(data)); - audio_output_queue_cv_.notify_one(); + Write(data.data(), data.size()); +} + +bool AudioCodec::InputData(std::vector& data) { + int duration = 30; + int input_frame_size = input_sample_rate_ / 1000 * duration * input_channels_; + + data.resize(input_frame_size); + int samples = Read(data.data(), data.size()); + if (samples > 0) { + return true; + } + return false; } IRAM_ATTR bool AudioCodec::on_sent(i2s_chan_handle_t handle, i2s_event_data_t *event, void *user_ctx) { auto audio_codec = (AudioCodec*)user_ctx; - xEventGroupSetBits(audio_codec->audio_event_group_, AUDIO_EVENT_OUTPUT_DONE); + if (audio_codec->output_enabled_ && audio_codec->on_output_ready_) { + return audio_codec->on_output_ready_(); + } + return false; +} + +IRAM_ATTR bool AudioCodec::on_recv(i2s_chan_handle_t handle, i2s_event_data_t *event, void *user_ctx) { + auto audio_codec = (AudioCodec*)user_ctx; + if (audio_codec->input_enabled_ && audio_codec->on_input_ready_) { + return audio_codec->on_input_ready_(); + } return false; } @@ -44,85 +58,20 @@ void AudioCodec::Start() { Settings settings("audio", false); output_volume_ = settings.GetInt("output_volume", output_volume_); - // 注册音频输出回调 - i2s_event_callbacks_t callbacks = {}; - callbacks.on_sent = on_sent; - i2s_channel_register_event_callback(tx_handle_, &callbacks, this); + // 注册音频数据回调 + i2s_event_callbacks_t rx_callbacks = {}; + rx_callbacks.on_recv = on_recv; + i2s_channel_register_event_callback(rx_handle_, &rx_callbacks, this); + + i2s_event_callbacks_t tx_callbacks = {}; + tx_callbacks.on_sent = on_sent; + i2s_channel_register_event_callback(tx_handle_, &tx_callbacks, this); ESP_ERROR_CHECK(i2s_channel_enable(tx_handle_)); ESP_ERROR_CHECK(i2s_channel_enable(rx_handle_)); EnableInput(true); EnableOutput(true); - - // 创建音频输入任务 - if (audio_input_task_ == nullptr) { - xTaskCreate([](void* arg) { - auto audio_device = (AudioCodec*)arg; - audio_device->InputTask(); - }, "audio_input", 4096 * 2, this, 3, &audio_input_task_); - } - // 创建音频输出任务 - if (audio_output_task_ == nullptr) { - xTaskCreate([](void* arg) { - auto audio_device = (AudioCodec*)arg; - audio_device->OutputTask(); - }, "audio_output", 4096 * 2, this, 3, &audio_output_task_); - } -} - -void AudioCodec::InputTask() { - int duration = 30; - int input_frame_size = input_sample_rate_ / 1000 * duration * input_channels_; - - while (true) { - std::vector input_data(input_frame_size); - int samples = Read(input_data.data(), input_data.size()); - if (samples > 0) { - if (on_input_data_) { - on_input_data_(std::move(input_data)); - } - } - } -} - -void AudioCodec::OutputTask() { - while (true) { - std::unique_lock lock(audio_output_queue_mutex_); - if (!audio_output_queue_cv_.wait_for(lock, std::chrono::seconds(30), [this]() { - return !audio_output_queue_.empty(); - })) { - // If timeout, disable output - EnableOutput(false); - continue; - } - auto data = std::move(audio_output_queue_.front()); - audio_output_queue_.pop_front(); - lock.unlock(); - - if (!output_enabled_) { - EnableOutput(true); - } - - xEventGroupClearBits(audio_event_group_, AUDIO_EVENT_OUTPUT_DONE); - Write(data.data(), data.size()); - audio_output_queue_cv_.notify_all(); - } -} - -void AudioCodec::WaitForOutputDone() { - // Wait for the output queue to be empty and the output is done - std::unique_lock lock(audio_output_queue_mutex_); - audio_output_queue_cv_.wait(lock, [this]() { - return audio_output_queue_.empty(); - }); - lock.unlock(); - xEventGroupWaitBits(audio_event_group_, AUDIO_EVENT_OUTPUT_DONE, pdFALSE, pdFALSE, portMAX_DELAY); -} - -void AudioCodec::ClearOutputQueue() { - std::lock_guard lock(audio_output_queue_mutex_); - audio_output_queue_.clear(); } void AudioCodec::SetOutputVolume(int volume) { diff --git a/main/audio_codecs/audio_codec.h b/main/audio_codecs/audio_codec.h index e5336b7a..8a13ea6d 100644 --- a/main/audio_codecs/audio_codec.h +++ b/main/audio_codecs/audio_codec.h @@ -2,21 +2,15 @@ #define _AUDIO_CODEC_H #include -#include #include #include #include #include #include -#include -#include -#include #include "board.h" -#define AUDIO_EVENT_OUTPUT_DONE (1 << 0) - class AudioCodec { public: AudioCodec(); @@ -27,10 +21,10 @@ public: virtual void EnableOutput(bool enable); void Start(); - void OnInputData(std::function&& data)> callback); void OutputData(std::vector& data); - void WaitForOutputDone(); - void ClearOutputQueue(); + bool InputData(std::vector& data); + void OnOutputReady(std::function callback); + void OnInputReady(std::function callback); inline bool duplex() const { return duplex_; } inline bool input_reference() const { return input_reference_; } @@ -41,18 +35,12 @@ public: inline int output_volume() const { return output_volume_; } private: - TaskHandle_t audio_input_task_ = nullptr; - TaskHandle_t audio_output_task_ = nullptr; - std::function&& data)> on_input_data_; - std::list> audio_output_queue_; - std::mutex audio_output_queue_mutex_; - std::condition_variable audio_output_queue_cv_; - EventGroupHandle_t audio_event_group_ = nullptr; + std::function on_input_ready_; + std::function on_output_ready_; + + IRAM_ATTR static bool on_recv(i2s_chan_handle_t handle, i2s_event_data_t *event, void *user_ctx); IRAM_ATTR static bool on_sent(i2s_chan_handle_t handle, i2s_event_data_t *event, void *user_ctx); - void InputTask(); - void OutputTask(); - protected: i2s_chan_handle_t tx_handle_ = nullptr; i2s_chan_handle_t rx_handle_ = nullptr; diff --git a/main/audio_codecs/box_audio_codec.cc b/main/audio_codecs/box_audio_codec.cc index 906b1179..e6399360 100644 --- a/main/audio_codecs/box_audio_codec.cc +++ b/main/audio_codecs/box_audio_codec.cc @@ -28,7 +28,7 @@ BoxAudioCodec::BoxAudioCodec(void* i2c_master_handle, int input_sample_rate, int // Output audio_codec_i2c_cfg_t i2c_cfg = { - .port = I2C_NUM_1, + .port = (i2c_port_t)1, .addr = es8311_addr, .bus_handle = i2c_master_handle, }; @@ -96,8 +96,8 @@ void BoxAudioCodec::CreateDuplexChannels(gpio_num_t mclk, gpio_num_t bclk, gpio_ i2s_chan_config_t chan_cfg = { .id = I2S_NUM_0, .role = I2S_ROLE_MASTER, - .dma_desc_num = 6, - .dma_frame_num = 240, + .dma_desc_num = 2, + .dma_frame_num = 240 * 3, .auto_clear_after_cb = true, .auto_clear_before_cb = false, .intr_priority = 0, diff --git a/main/audio_codecs/es8311_audio_codec.cc b/main/audio_codecs/es8311_audio_codec.cc new file mode 100644 index 00000000..8dfeed31 --- /dev/null +++ b/main/audio_codecs/es8311_audio_codec.cc @@ -0,0 +1,186 @@ +#include "es8311_audio_codec.h" + +#include + +static const char TAG[] = "Es8311AudioCodec"; + +Es8311AudioCodec::Es8311AudioCodec(void* i2c_master_handle, i2c_port_t i2c_port, int input_sample_rate, int output_sample_rate, + gpio_num_t mclk, gpio_num_t bclk, gpio_num_t ws, gpio_num_t dout, gpio_num_t din, + gpio_num_t pa_pin, uint8_t es8311_addr) { + duplex_ = true; // 是否双工 + input_reference_ = false; // 是否使用参考输入，实现回声消除 + input_channels_ = 1; // 输入通道数 + input_sample_rate_ = input_sample_rate; + output_sample_rate_ = output_sample_rate; + + CreateDuplexChannels(mclk, bclk, ws, dout, din); + + // Do initialize of related interface: data_if, ctrl_if and gpio_if + audio_codec_i2s_cfg_t i2s_cfg = { + .port = I2S_NUM_0, + .rx_handle = rx_handle_, + .tx_handle = tx_handle_, + }; + data_if_ = audio_codec_new_i2s_data(&i2s_cfg); + assert(data_if_ != NULL); + + // Output + audio_codec_i2c_cfg_t i2c_cfg = { + .port = i2c_port, + .addr = es8311_addr, + .bus_handle = i2c_master_handle, + }; + ctrl_if_ = audio_codec_new_i2c_ctrl(&i2c_cfg); + assert(ctrl_if_ != NULL); + + gpio_if_ = audio_codec_new_gpio(); + assert(gpio_if_ != NULL); + + es8311_codec_cfg_t es8311_cfg = {}; + es8311_cfg.ctrl_if = ctrl_if_; + es8311_cfg.gpio_if = gpio_if_; + es8311_cfg.codec_mode = ESP_CODEC_DEV_WORK_MODE_BOTH; + es8311_cfg.pa_pin = pa_pin; + es8311_cfg.use_mclk = true; + es8311_cfg.hw_gain.pa_voltage = 5.0; + es8311_cfg.hw_gain.codec_dac_voltage = 3.3; + codec_if_ = es8311_codec_new(&es8311_cfg); + assert(codec_if_ != NULL); + + esp_codec_dev_cfg_t dev_cfg = { + .dev_type = ESP_CODEC_DEV_TYPE_OUT, + .codec_if = codec_if_, + .data_if = data_if_, + }; + output_dev_ = esp_codec_dev_new(&dev_cfg); + assert(output_dev_ != NULL); + dev_cfg.dev_type = ESP_CODEC_DEV_TYPE_IN; + input_dev_ = esp_codec_dev_new(&dev_cfg); + assert(input_dev_ != NULL); + + ESP_LOGI(TAG, "Es8311AudioCodec initialized"); +} + +Es8311AudioCodec::~Es8311AudioCodec() { + ESP_ERROR_CHECK(esp_codec_dev_close(output_dev_)); + esp_codec_dev_delete(output_dev_); + ESP_ERROR_CHECK(esp_codec_dev_close(input_dev_)); + esp_codec_dev_delete(input_dev_); + + audio_codec_delete_codec_if(codec_if_); + audio_codec_delete_ctrl_if(ctrl_if_); + audio_codec_delete_gpio_if(gpio_if_); + audio_codec_delete_data_if(data_if_); +} + +void Es8311AudioCodec::CreateDuplexChannels(gpio_num_t mclk, gpio_num_t bclk, gpio_num_t ws, gpio_num_t dout, gpio_num_t din) { + assert(input_sample_rate_ == output_sample_rate_); + + i2s_chan_config_t chan_cfg = { + .id = I2S_NUM_0, + .role = I2S_ROLE_MASTER, + .dma_desc_num = 2, + .dma_frame_num = 240 * 3, + .auto_clear_after_cb = true, + .auto_clear_before_cb = false, + .intr_priority = 0, + }; + ESP_ERROR_CHECK(i2s_new_channel(&chan_cfg, &tx_handle_, &rx_handle_)); + + i2s_std_config_t std_cfg = { + .clk_cfg = { + .sample_rate_hz = (uint32_t)output_sample_rate_, + .clk_src = I2S_CLK_SRC_DEFAULT, + .ext_clk_freq_hz = 0, + .mclk_multiple = I2S_MCLK_MULTIPLE_256 + }, + .slot_cfg = { + .data_bit_width = I2S_DATA_BIT_WIDTH_16BIT, + .slot_bit_width = I2S_SLOT_BIT_WIDTH_AUTO, + .slot_mode = I2S_SLOT_MODE_STEREO, + .slot_mask = I2S_STD_SLOT_BOTH, + .ws_width = I2S_DATA_BIT_WIDTH_16BIT, + .ws_pol = false, + .bit_shift = true, + .left_align = true, + .big_endian = false, + .bit_order_lsb = false + }, + .gpio_cfg = { + .mclk = mclk, + .bclk = bclk, + .ws = ws, + .dout = dout, + .din = din, + .invert_flags = { + .mclk_inv = false, + .bclk_inv = false, + .ws_inv = false + } + } + }; + + ESP_ERROR_CHECK(i2s_channel_init_std_mode(tx_handle_, &std_cfg)); + ESP_ERROR_CHECK(i2s_channel_init_std_mode(rx_handle_, &std_cfg)); + ESP_LOGI(TAG, "Duplex channels created"); +} + +void Es8311AudioCodec::SetOutputVolume(int volume) { + ESP_ERROR_CHECK(esp_codec_dev_set_out_vol(output_dev_, volume)); + AudioCodec::SetOutputVolume(volume); +} + +void Es8311AudioCodec::EnableInput(bool enable) { + if (enable == input_enabled_) { + return; + } + if (enable) { + esp_codec_dev_sample_info_t fs = { + .bits_per_sample = 16, + .channel = 1, + .channel_mask = 0, + .sample_rate = (uint32_t)output_sample_rate_, + .mclk_multiple = 0, + }; + ESP_ERROR_CHECK(esp_codec_dev_open(input_dev_, &fs)); + ESP_ERROR_CHECK(esp_codec_dev_set_in_gain(input_dev_, 40.0)); + } else { + ESP_ERROR_CHECK(esp_codec_dev_close(input_dev_)); + } + AudioCodec::EnableInput(enable); +} + +void Es8311AudioCodec::EnableOutput(bool enable) { + if (enable == output_enabled_) { + return; + } + if (enable) { + // Play 16bit 1 channel + esp_codec_dev_sample_info_t fs = { + .bits_per_sample = 16, + .channel = 1, + .channel_mask = 0, + .sample_rate = (uint32_t)output_sample_rate_, + .mclk_multiple = 0, + }; + ESP_ERROR_CHECK(esp_codec_dev_open(output_dev_, &fs)); + ESP_ERROR_CHECK(esp_codec_dev_set_out_vol(output_dev_, output_volume_)); + } else { + ESP_ERROR_CHECK(esp_codec_dev_close(output_dev_)); + } + AudioCodec::EnableOutput(enable); +} + +int Es8311AudioCodec::Read(int16_t* dest, int samples) { + if (input_enabled_) { + ESP_ERROR_CHECK_WITHOUT_ABORT(esp_codec_dev_read(input_dev_, (void*)dest, samples * sizeof(int16_t))); + } + return samples; +} + +int Es8311AudioCodec::Write(const int16_t* data, int samples) { + if (output_enabled_) { + ESP_ERROR_CHECK_WITHOUT_ABORT(esp_codec_dev_write(output_dev_, (void*)data, samples * sizeof(int16_t))); + } + return samples; +} \ No newline at end of file diff --git a/main/audio_codecs/es8311_audio_codec.h b/main/audio_codecs/es8311_audio_codec.h new file mode 100644 index 00000000..a10c7ae5 --- /dev/null +++ b/main/audio_codecs/es8311_audio_codec.h @@ -0,0 +1,36 @@ +#ifndef _ES8311_AUDIO_CODEC_H +#define _ES8311_AUDIO_CODEC_H + +#include "audio_codec.h" + +#include +#include +#include + +class Es8311AudioCodec : public AudioCodec { +private: + const audio_codec_data_if_t* data_if_ = nullptr; + const audio_codec_ctrl_if_t* ctrl_if_ = nullptr; + const audio_codec_if_t* codec_if_ = nullptr; + const audio_codec_gpio_if_t* gpio_if_ = nullptr; + + esp_codec_dev_handle_t output_dev_ = nullptr; + esp_codec_dev_handle_t input_dev_ = nullptr; + + void CreateDuplexChannels(gpio_num_t mclk, gpio_num_t bclk, gpio_num_t ws, gpio_num_t dout, gpio_num_t din); + + virtual int Read(int16_t* dest, int samples) override; + virtual int Write(const int16_t* data, int samples) override; + +public: + Es8311AudioCodec(void* i2c_master_handle, i2c_port_t i2c_port, int input_sample_rate, int output_sample_rate, + gpio_num_t mclk, gpio_num_t bclk, gpio_num_t ws, gpio_num_t dout, gpio_num_t din, + gpio_num_t pa_pin, uint8_t es8311_addr); + virtual ~Es8311AudioCodec(); + + virtual void SetOutputVolume(int volume) override; + virtual void EnableInput(bool enable) override; + virtual void EnableOutput(bool enable) override; +}; + +#endif // _ES8311_AUDIO_CODEC_H diff --git a/main/audio_codecs/no_audio_codec.cc b/main/audio_codecs/no_audio_codec.cc index d92d7944..a943faee 100644 --- a/main/audio_codecs/no_audio_codec.cc +++ b/main/audio_codecs/no_audio_codec.cc @@ -23,8 +23,8 @@ NoAudioCodec::NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_n i2s_chan_config_t chan_cfg = { .id = I2S_NUM_0, .role = I2S_ROLE_MASTER, - .dma_desc_num = 6, - .dma_frame_num = 240, + .dma_desc_num = 2, + .dma_frame_num = 240 * 3, .auto_clear_after_cb = false, .auto_clear_before_cb = false, .intr_priority = 0, @@ -75,7 +75,7 @@ NoAudioCodec::NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_n // Create a new channel for speaker i2s_chan_config_t chan_cfg = { - .id = I2S_NUM_0, + .id = (i2s_port_t)0, .role = I2S_ROLE_MASTER, .dma_desc_num = 6, .dma_frame_num = 240, @@ -120,7 +120,7 @@ NoAudioCodec::NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_n ESP_ERROR_CHECK(i2s_channel_init_std_mode(tx_handle_, &std_cfg)); // Create a new channel for MIC - chan_cfg.id = I2S_NUM_1; + chan_cfg.id = (i2s_port_t)1; ESP_ERROR_CHECK(i2s_new_channel(&chan_cfg, nullptr, &rx_handle_)); std_cfg.clk_cfg.sample_rate_hz = (uint32_t)input_sample_rate_; std_cfg.gpio_cfg.bclk = mic_sck; diff --git a/main/audio_processor.cc b/main/audio_processing/audio_processor.cc similarity index 100% rename from main/audio_processor.cc rename to main/audio_processing/audio_processor.cc diff --git a/main/audio_processor.h b/main/audio_processing/audio_processor.h similarity index 100% rename from main/audio_processor.h rename to main/audio_processing/audio_processor.h diff --git a/main/wake_word_detect.cc b/main/audio_processing/wake_word_detect.cc similarity index 100% rename from main/wake_word_detect.cc rename to main/audio_processing/wake_word_detect.cc diff --git a/main/wake_word_detect.h b/main/audio_processing/wake_word_detect.h similarity index 100% rename from main/wake_word_detect.h rename to main/audio_processing/wake_word_detect.h diff --git a/main/background_task.cc b/main/background_task.cc new file mode 100644 index 00000000..5e0146ca --- /dev/null +++ b/main/background_task.cc @@ -0,0 +1,63 @@ +#include "background_task.h" + +#include + +#define TAG "BackgroundTask" + +BackgroundTask::BackgroundTask(uint32_t stack_size) { +#if CONFIG_IDF_TARGET_ESP32S3 + task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM); + background_task_handle_ = xTaskCreateStatic([](void* arg) { + BackgroundTask* task = (BackgroundTask*)arg; + task->BackgroundTaskLoop(); + }, "background_task", stack_size, this, 1, task_stack_, &task_buffer_); +#else + xTaskCreate([](void* arg) { + BackgroundTask* task = (BackgroundTask*)arg; + task->BackgroundTaskLoop(); + }, "background_task", stack_size, this, 1, &background_task_handle_); +#endif +} + +BackgroundTask::~BackgroundTask() { + if (background_task_handle_ != nullptr) { + vTaskDelete(background_task_handle_); + } +} + +void BackgroundTask::Schedule(std::function callback) { + std::lock_guard lock(mutex_); + if (active_tasks_ >= 30) { + ESP_LOGW(TAG, "active_tasks_ == %u", active_tasks_.load()); + } + active_tasks_++; + auto wrapped_callback = [this, callback]() { + callback(); + active_tasks_--; + condition_variable_.notify_all(); + }; + main_tasks_.push_back(wrapped_callback); + condition_variable_.notify_all(); +} + +void BackgroundTask::WaitForCompletion() { + std::unique_lock lock(mutex_); + condition_variable_.wait(lock, [this]() { + return main_tasks_.empty() && active_tasks_ == 0; + }); +} + +void BackgroundTask::BackgroundTaskLoop() { + ESP_LOGI(TAG, "background_task started"); + while (true) { + std::unique_lock lock(mutex_); + condition_variable_.wait(lock, [this]() { return !main_tasks_.empty(); }); + + std::list> tasks = std::move(main_tasks_); + lock.unlock(); + + for (auto& task : tasks) { + task(); + } + } +} diff --git a/main/background_task.h b/main/background_task.h new file mode 100644 index 00000000..859cd72a --- /dev/null +++ b/main/background_task.h @@ -0,0 +1,33 @@ +#ifndef BACKGROUND_TASK_H +#define BACKGROUND_TASK_H + +#include +#include +#include +#include +#include +#include + +class BackgroundTask { +public: + BackgroundTask(uint32_t stack_size = 4096 * 2); + ~BackgroundTask(); + + void Schedule(std::function callback); + void WaitForCompletion(); + +private: + std::mutex mutex_; + std::list> main_tasks_; + std::condition_variable condition_variable_; + TaskHandle_t background_task_handle_ = nullptr; + std::atomic active_tasks_{0}; + + TaskHandle_t task_ = nullptr; + StaticTask_t task_buffer_; + StackType_t* task_stack_ = nullptr; + + void BackgroundTaskLoop(); +}; + +#endif diff --git a/main/boards/bread-compact-ml307/compact_ml307_board.cc b/main/boards/bread-compact-ml307/compact_ml307_board.cc index c614979e..709aacfe 100644 --- a/main/boards/bread-compact-ml307/compact_ml307_board.cc +++ b/main/boards/bread-compact-ml307/compact_ml307_board.cc @@ -22,7 +22,7 @@ private: void InitializeDisplayI2c() { i2c_master_bus_config_t bus_config = { - .i2c_port = I2C_NUM_0, + .i2c_port = (i2c_port_t)0, .sda_io_num = DISPLAY_SDA_PIN, .scl_io_num = DISPLAY_SCL_PIN, .clk_source = I2C_CLK_SRC_DEFAULT, diff --git a/main/boards/bread-compact-wifi/compact_wifi_board.cc b/main/boards/bread-compact-wifi/compact_wifi_board.cc index f5f19a0c..f15f6d93 100644 --- a/main/boards/bread-compact-wifi/compact_wifi_board.cc +++ b/main/boards/bread-compact-wifi/compact_wifi_board.cc @@ -23,7 +23,7 @@ private: void InitializeDisplayI2c() { i2c_master_bus_config_t bus_config = { - .i2c_port = I2C_NUM_0, + .i2c_port = (i2c_port_t)0, .sda_io_num = DISPLAY_SDA_PIN, .scl_io_num = DISPLAY_SCL_PIN, .clk_source = I2C_CLK_SRC_DEFAULT, diff --git a/main/boards/common/board.cc b/main/boards/common/board.cc index af9cdbbc..4398cbf1 100644 --- a/main/boards/common/board.cc +++ b/main/boards/common/board.cc @@ -1,5 +1,6 @@ #include "board.h" #include "system_info.h" +#include "display/no_display.h" #include #include @@ -11,6 +12,12 @@ bool Board::GetBatteryLevel(int &level, bool& charging) { return false; } +Display* Board::GetDisplay() { + static NoDisplay display; + return &display; +} + + std::string Board::GetJson() { /* { @@ -101,4 +108,4 @@ std::string Board::GetJson() { // Close the JSON object json += "}"; return json; -} +} \ No newline at end of file diff --git a/main/boards/common/board.h b/main/boards/common/board.h index 472e6ce3..a0193e15 100644 --- a/main/boards/common/board.h +++ b/main/boards/common/board.h @@ -35,7 +35,7 @@ public: virtual ~Board() = default; virtual Led* GetBuiltinLed() = 0; virtual AudioCodec* GetAudioCodec() = 0; - virtual Display* GetDisplay() = 0; + virtual Display* GetDisplay(); virtual Http* CreateHttp() = 0; virtual WebSocket* CreateWebSocket() = 0; virtual Mqtt* CreateMqtt() = 0; diff --git a/main/boards/common/led.cc b/main/boards/common/led.cc index f0f4d129..e3640af1 100644 --- a/main/boards/common/led.cc +++ b/main/boards/common/led.cc @@ -7,10 +7,6 @@ #define TAG "Led" Led::Led(gpio_num_t gpio) { - mutex_ = xSemaphoreCreateMutex(); - blink_event_group_ = xEventGroupCreate(); - xEventGroupSetBits(blink_event_group_, BLINK_TASK_STOPPED_BIT); - if (gpio == GPIO_NUM_NC) { ESP_LOGI(TAG, "Builtin LED not connected"); return; @@ -29,19 +25,25 @@ Led::Led(gpio_num_t gpio) { led_strip_clear(led_strip_); SetGrey(); + + esp_timer_create_args_t blink_timer_args = { + .callback = [](void *arg) { + auto led = static_cast(arg); + led->OnBlinkTimer(); + }, + .arg = this, + .dispatch_method = ESP_TIMER_TASK, + .name = "Blink Timer", + .skip_unhandled_events = false, + }; + ESP_ERROR_CHECK(esp_timer_create(&blink_timer_args, &blink_timer_)); } Led::~Led() { - StopBlinkInternal(); + esp_timer_stop(blink_timer_); if (led_strip_ != nullptr) { led_strip_del(led_strip_); } - if (mutex_ != nullptr) { - vSemaphoreDelete(mutex_); - } - if (blink_event_group_ != nullptr) { - vEventGroupDelete(blink_event_group_); - } } void Led::SetColor(uint8_t r, uint8_t g, uint8_t b) { @@ -54,21 +56,21 @@ void Led::TurnOn() { if (led_strip_ == nullptr) { return; } - StopBlinkInternal(); - xSemaphoreTake(mutex_, portMAX_DELAY); + + std::lock_guard lock(mutex_); + esp_timer_stop(blink_timer_); led_strip_set_pixel(led_strip_, 0, r_, g_, b_); led_strip_refresh(led_strip_); - xSemaphoreGive(mutex_); } void Led::TurnOff() { if (led_strip_ == nullptr) { return; } - StopBlinkInternal(); - xSemaphoreTake(mutex_, portMAX_DELAY); + + std::lock_guard lock(mutex_); + esp_timer_stop(blink_timer_); led_strip_clear(led_strip_); - xSemaphoreGive(mutex_); } void Led::BlinkOnce() { @@ -87,45 +89,27 @@ void Led::StartBlinkTask(int times, int interval_ms) { if (led_strip_ == nullptr) { return; } - StopBlinkInternal(); - xSemaphoreTake(mutex_, portMAX_DELAY); + + std::lock_guard lock(mutex_); + esp_timer_stop(blink_timer_); - blink_times_ = times; + led_strip_clear(led_strip_); + blink_counter_ = times * 2; blink_interval_ms_ = interval_ms; - should_blink_ = true; + esp_timer_start_periodic(blink_timer_, interval_ms * 1000); +} - xEventGroupClearBits(blink_event_group_, BLINK_TASK_STOPPED_BIT); - xEventGroupSetBits(blink_event_group_, BLINK_TASK_RUNNING_BIT); +void Led::OnBlinkTimer() { + std::lock_guard lock(mutex_); + blink_counter_--; + if (blink_counter_ & 1) { + led_strip_set_pixel(led_strip_, 0, r_, g_, b_); + led_strip_refresh(led_strip_); + } else { + led_strip_clear(led_strip_); - xTaskCreate([](void* obj) { - auto this_ = static_cast(obj); - int count = 0; - while (this_->should_blink_ && (this_->blink_times_ == BLINK_INFINITE || count < this_->blink_times_)) { - xSemaphoreTake(this_->mutex_, portMAX_DELAY); - led_strip_set_pixel(this_->led_strip_, 0, this_->r_, this_->g_, this_->b_); - led_strip_refresh(this_->led_strip_); - xSemaphoreGive(this_->mutex_); - - vTaskDelay(this_->blink_interval_ms_ / portTICK_PERIOD_MS); - if (!this_->should_blink_) break; - - xSemaphoreTake(this_->mutex_, portMAX_DELAY); - led_strip_clear(this_->led_strip_); - xSemaphoreGive(this_->mutex_); - - vTaskDelay(this_->blink_interval_ms_ / portTICK_PERIOD_MS); - if (this_->blink_times_ != BLINK_INFINITE) count++; + if (blink_counter_ == 0) { + esp_timer_stop(blink_timer_); } - this_->blink_task_ = nullptr; - xEventGroupClearBits(this_->blink_event_group_, BLINK_TASK_RUNNING_BIT); - xEventGroupSetBits(this_->blink_event_group_, BLINK_TASK_STOPPED_BIT); - vTaskDelete(NULL); - }, "blink", 2048, this, tskIDLE_PRIORITY, &blink_task_); - - xSemaphoreGive(mutex_); -} - -void Led::StopBlinkInternal() { - should_blink_ = false; - xEventGroupWaitBits(blink_event_group_, BLINK_TASK_STOPPED_BIT, pdFALSE, pdTRUE, portMAX_DELAY); + } } diff --git a/main/boards/common/led.h b/main/boards/common/led.h index efee46e9..e0fe334f 100644 --- a/main/boards/common/led.h +++ b/main/boards/common/led.h @@ -2,10 +2,9 @@ #define _LED_H_ #include -#include -#include -#include +#include #include +#include #define BLINK_INFINITE -1 #define BLINK_TASK_STOPPED_BIT BIT0 @@ -33,17 +32,16 @@ public: void SetBlue(uint8_t brightness = DEFAULT_BRIGHTNESS) { SetColor(0, 0, brightness); } private: - SemaphoreHandle_t mutex_; - EventGroupHandle_t blink_event_group_; + std::mutex mutex_; TaskHandle_t blink_task_ = nullptr; led_strip_handle_t led_strip_ = nullptr; uint8_t r_ = 0, g_ = 0, b_ = 0; - int blink_times_ = 0; + int blink_counter_ = 0; int blink_interval_ms_ = 0; - std::atomic should_blink_{false}; + esp_timer_handle_t blink_timer_ = nullptr; void StartBlinkTask(int times, int interval_ms); - void StopBlinkInternal(); + void OnBlinkTimer(); }; #endif // _LED_H_ diff --git a/main/boards/common/wifi_board.cc b/main/boards/common/wifi_board.cc index 622ea405..0d368045 100644 --- a/main/boards/common/wifi_board.cc +++ b/main/boards/common/wifi_board.cc @@ -62,7 +62,10 @@ void WifiBoard::StartNetwork() { // Wait forever until reset after configuration while (true) { - vTaskDelay(pdMS_TO_TICKS(1000)); + int free_sram = heap_caps_get_free_size(MALLOC_CAP_INTERNAL); + int min_free_sram = heap_caps_get_minimum_free_size(MALLOC_CAP_INTERNAL); + ESP_LOGI(TAG, "Free internal: %u minimal internal: %u", free_sram, min_free_sram); + vTaskDelay(pdMS_TO_TICKS(10000)); } } } diff --git a/main/boards/esp-box-3/esp_box3_board.cc b/main/boards/esp-box-3/esp_box3_board.cc index 17e85f08..ce2f9452 100644 --- a/main/boards/esp-box-3/esp_box3_board.cc +++ b/main/boards/esp-box-3/esp_box3_board.cc @@ -19,7 +19,7 @@ private: void InitializeI2c() { // Initialize I2C peripheral i2c_master_bus_config_t i2c_bus_cfg = { - .i2c_port = I2C_NUM_1, + .i2c_port = (i2c_port_t)1, .sda_io_num = AUDIO_CODEC_I2C_SDA_PIN, .scl_io_num = AUDIO_CODEC_I2C_SCL_PIN, .clk_source = I2C_CLK_SRC_DEFAULT, diff --git a/main/boards/kevin-box-1/kevin_box_board.cc b/main/boards/kevin-box-1/kevin_box_board.cc index ffcc02cd..a1e75208 100644 --- a/main/boards/kevin-box-1/kevin_box_board.cc +++ b/main/boards/kevin-box-1/kevin_box_board.cc @@ -48,7 +48,7 @@ private: void InitializeDisplayI2c() { i2c_master_bus_config_t bus_config = { - .i2c_port = I2C_NUM_0, + .i2c_port = (i2c_port_t)0, .sda_io_num = DISPLAY_SDA_PIN, .scl_io_num = DISPLAY_SCL_PIN, .clk_source = I2C_CLK_SRC_DEFAULT, @@ -65,7 +65,7 @@ private: void InitializeCodecI2c() { // Initialize I2C peripheral i2c_master_bus_config_t i2c_bus_cfg = { - .i2c_port = I2C_NUM_1, + .i2c_port = (i2c_port_t)1, .sda_io_num = AUDIO_CODEC_I2C_SDA_PIN, .scl_io_num = AUDIO_CODEC_I2C_SCL_PIN, .clk_source = I2C_CLK_SRC_DEFAULT, diff --git a/main/boards/kevin-box-2/kevin_box_board.cc b/main/boards/kevin-box-2/kevin_box_board.cc index 26156ff5..a76c3e53 100644 --- a/main/boards/kevin-box-2/kevin_box_board.cc +++ b/main/boards/kevin-box-2/kevin_box_board.cc @@ -86,7 +86,7 @@ private: void InitializeDisplayI2c() { i2c_master_bus_config_t bus_config = { - .i2c_port = I2C_NUM_0, + .i2c_port = (i2c_port_t)0, .sda_io_num = DISPLAY_SDA_PIN, .scl_io_num = DISPLAY_SCL_PIN, .clk_source = I2C_CLK_SRC_DEFAULT, @@ -103,7 +103,7 @@ private: void InitializeCodecI2c() { // Initialize I2C peripheral i2c_master_bus_config_t i2c_bus_cfg = { - .i2c_port = I2C_NUM_1, + .i2c_port = (i2c_port_t)1, .sda_io_num = AUDIO_CODEC_I2C_SDA_PIN, .scl_io_num = AUDIO_CODEC_I2C_SCL_PIN, .clk_source = I2C_CLK_SRC_DEFAULT, @@ -118,10 +118,6 @@ private: } void InitializeButtons() { - // 测试按住说话 - // boot_button_.OnClick([this]() { - // Application::GetInstance().ToggleChatState(); - // }); boot_button_.OnPressDown([this]() { Application::GetInstance().StartListening(); }); diff --git a/main/boards/kevin-c3/config.h b/main/boards/kevin-c3/config.h new file mode 100644 index 00000000..a8034e8e --- /dev/null +++ b/main/boards/kevin-c3/config.h @@ -0,0 +1,24 @@ +#ifndef _BOARD_CONFIG_H_ +#define _BOARD_CONFIG_H_ + +#include + +#define AUDIO_INPUT_SAMPLE_RATE 24000 +#define AUDIO_OUTPUT_SAMPLE_RATE 24000 + +#define AUDIO_I2S_GPIO_MCLK GPIO_NUM_10 +#define AUDIO_I2S_GPIO_WS GPIO_NUM_12 +#define AUDIO_I2S_GPIO_BCLK GPIO_NUM_8 +#define AUDIO_I2S_GPIO_DIN GPIO_NUM_7 +#define AUDIO_I2S_GPIO_DOUT GPIO_NUM_11 + +#define AUDIO_CODEC_PA_PIN GPIO_NUM_13 +#define AUDIO_CODEC_I2C_SDA_PIN GPIO_NUM_0 +#define AUDIO_CODEC_I2C_SCL_PIN GPIO_NUM_1 +#define AUDIO_CODEC_ES8311_ADDR ES8311_CODEC_DEFAULT_ADDR + +#define BUILTIN_LED_GPIO GPIO_NUM_2 +#define BOOT_BUTTON_GPIO GPIO_NUM_9 + + +#endif // _BOARD_CONFIG_H_ diff --git a/main/boards/kevin-c3/kevin_box_board.cc b/main/boards/kevin-c3/kevin_box_board.cc new file mode 100644 index 00000000..47c8c791 --- /dev/null +++ b/main/boards/kevin-c3/kevin_box_board.cc @@ -0,0 +1,78 @@ +#include "wifi_board.h" +#include "audio_codecs/es8311_audio_codec.h" +#include "application.h" +#include "button.h" +#include "led.h" +#include "config.h" + +#include +#include +#include + +#define TAG "KevinBoxBoard" + +class KevinBoxBoard : public WifiBoard { +private: + i2c_master_bus_handle_t codec_i2c_bus_; + Button boot_button_; + + void InitializeCodecI2c() { + // Initialize I2C peripheral + i2c_master_bus_config_t i2c_bus_cfg = { + .i2c_port = I2C_NUM_0, + .sda_io_num = AUDIO_CODEC_I2C_SDA_PIN, + .scl_io_num = AUDIO_CODEC_I2C_SCL_PIN, + .clk_source = I2C_CLK_SRC_DEFAULT, + .glitch_ignore_cnt = 7, + .intr_priority = 0, + .trans_queue_depth = 0, + .flags = { + .enable_internal_pullup = 1, + }, + }; + ESP_ERROR_CHECK(i2c_new_master_bus(&i2c_bus_cfg, &codec_i2c_bus_)); + } + + void InitializeButtons() { + boot_button_.OnClick([this]() { + auto& app = Application::GetInstance(); + if (app.GetChatState() == kChatStateUnknown && !WifiStation::GetInstance().IsConnected()) { + ResetWifiConfiguration(); + } + }); + boot_button_.OnPressDown([this]() { + Application::GetInstance().StartListening(); + }); + boot_button_.OnPressUp([this]() { + Application::GetInstance().StopListening(); + }); + } + +public: + KevinBoxBoard() : + boot_button_(BOOT_BUTTON_GPIO) { + } + + virtual void Initialize() override { + ESP_LOGI(TAG, "Initializing KevinBoxBoard"); + + InitializeCodecI2c(); + InitializeButtons(); + + WifiBoard::Initialize(); + } + + virtual Led* GetBuiltinLed() override { + static Led led(BUILTIN_LED_GPIO); + return &led; + } + + virtual AudioCodec* GetAudioCodec() override { + static Es8311AudioCodec audio_codec(codec_i2c_bus_, I2C_NUM_0, AUDIO_INPUT_SAMPLE_RATE, AUDIO_OUTPUT_SAMPLE_RATE, + AUDIO_I2S_GPIO_MCLK, AUDIO_I2S_GPIO_BCLK, AUDIO_I2S_GPIO_WS, AUDIO_I2S_GPIO_DOUT, AUDIO_I2S_GPIO_DIN, + AUDIO_CODEC_PA_PIN, AUDIO_CODEC_ES8311_ADDR); + return &audio_codec; + } +}; + +DECLARE_BOARD(KevinBoxBoard); diff --git a/main/boards/lichuang-dev/lichuang_dev_board.cc b/main/boards/lichuang-dev/lichuang_dev_board.cc index 3ef44c65..5a0b351d 100644 --- a/main/boards/lichuang-dev/lichuang_dev_board.cc +++ b/main/boards/lichuang-dev/lichuang_dev_board.cc @@ -42,7 +42,7 @@ private: void InitializeI2c() { // Initialize I2C peripheral i2c_master_bus_config_t i2c_bus_cfg = { - .i2c_port = I2C_NUM_1, + .i2c_port = (i2c_port_t)1, .sda_io_num = AUDIO_CODEC_I2C_SDA_PIN, .scl_io_num = AUDIO_CODEC_I2C_SCL_PIN, .clk_source = I2C_CLK_SRC_DEFAULT, @@ -76,7 +76,12 @@ private: if (app.GetChatState() == kChatStateUnknown && !WifiStation::GetInstance().IsConnected()) { ResetWifiConfiguration(); } - app.ToggleChatState(); + }); + boot_button_.OnPressDown([this]() { + Application::GetInstance().StartListening(); + }); + boot_button_.OnPressUp([this]() { + Application::GetInstance().StopListening(); }); } diff --git a/main/idf_component.yml b/main/idf_component.yml index a3bcc214..03f6aff1 100644 --- a/main/idf_component.yml +++ b/main/idf_component.yml @@ -2,7 +2,7 @@ dependencies: 78/esp-wifi-connect: "~1.4.1" 78/esp-opus-encoder: "~1.1.0" - 78/esp-ml307: "~1.6.3" + 78/esp-ml307: "~1.7.0" espressif/led_strip: "^2.4.1" espressif/esp_codec_dev: "^1.3.1" espressif/esp-sr: "^1.9.0" diff --git a/main/ota.cc b/main/ota.cc index 61cd3332..5a82adfe 100644 --- a/main/ota.cc +++ b/main/ota.cc @@ -50,11 +50,11 @@ bool Ota::CheckVersion() { } http->SetHeader("Content-Type", "application/json"); - if (post_data_.length() > 0) { - http->SetContent(post_data_); - http->Open("POST", check_version_url_); - } else { - http->Open("GET", check_version_url_); + std::string method = post_data_.length() > 0 ? "POST" : "GET"; + if (!http->Open(method, check_version_url_, post_data_)) { + ESP_LOGE(TAG, "Failed to open HTTP connection"); + delete http; + return false; } auto response = http->GetBody(); diff --git a/sdkconfig.defaults b/sdkconfig.defaults index 6f3dbccf..1b79fd81 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -10,13 +10,13 @@ CONFIG_PARTITION_TABLE_CUSTOM=y CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" CONFIG_PARTITION_TABLE_OFFSET=0x8000 -CONFIG_USE_WAKENET=y -CONFIG_SR_WN_WN9_NIHAOXIAOZHI_TTS=y -CONFIG_USE_MULTINET=n - ESP_TASK_WDT_TIMEOUT_S=10 CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS=y CONFIG_FREERTOS_USE_STATS_FORMATTING_FUNCTIONS=y CONFIG_LV_COLOR_16_SWAP=y CONFIG_LV_MEM_CUSTOM=y + +CONFIG_MBEDTLS_DYNAMIC_BUFFER=y +CONFIG_ESP_WIFI_IRAM_OPT=n +CONFIG_ESP_WIFI_RX_IRAM_OPT=n diff --git a/sdkconfig.defaults.esp32c3 b/sdkconfig.defaults.esp32c3 index dc2584f3..e3ac6c78 100644 --- a/sdkconfig.defaults.esp32c3 +++ b/sdkconfig.defaults.esp32c3 @@ -1,6 +1,2 @@ CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y - -CONFIG_PARTITION_TABLE_CUSTOM=y -CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_4M.csv" -CONFIG_PARTITION_TABLE_OFFSET=0x8000 \ No newline at end of file diff --git a/sdkconfig.defaults.esp32s3 b/sdkconfig.defaults.esp32s3 index c2d0bfbd..5b99472f 100644 --- a/sdkconfig.defaults.esp32s3 +++ b/sdkconfig.defaults.esp32s3 @@ -16,3 +16,7 @@ CONFIG_MBEDTLS_EXTERNAL_MEM_ALLOC=y CONFIG_ESP32S3_INSTRUCTION_CACHE_32KB=y CONFIG_ESP32S3_DATA_CACHE_64KB=y CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y + +CONFIG_USE_WAKENET=y +CONFIG_SR_WN_WN9_NIHAOXIAOZHI_TTS=y +CONFIG_USE_MULTINET=n