devttl/xiaozhi-esp32
1
0
Fork 0
forked from xiaozhi/xiaozhi-esp32
Code Pull Requests Activity

move all hardware init to boards

Browse Source
This commit is contained in:
Terrence
2024-11-06 06:18:56 +08:00
parent 55ff4e1f74
commit 3575448373
34 changed files with 1150 additions and 669 deletions
Download Patch File Download Diff File Expand all files Collapse all files

219
main/application.cc
View File

@@ -1,7 +1,7 @@
#include "builtin_led.h"
#include "application.h"
#include "system_info.h"
#include "ml307_ssl_transport.h"
#include "application.h"
#include "audio_codec.h"
#include <cstring>
#include <esp_log.h>
@@ -19,24 +19,9 @@ extern const char p3_err_wificonfig_start[] asm("_binary_err_wificonfig_p3_start
extern const char p3_err_wificonfig_end[] asm("_binary_err_wificonfig_p3_end");
Application::Application()
: boot_button_((gpio_num_t)BOOT_BUTTON_GPIO),
volume_up_button_((gpio_num_t)VOLUME_UP_BUTTON_GPIO),
volume_down_button_((gpio_num_t)VOLUME_DOWN_BUTTON_GPIO),
display_(DISPLAY_SDA_PIN, DISPLAY_SCL_PIN)
{
Application::Application() {
event_group_ = xEventGroupCreate();
opus_encoder_.Configure(16000, 1);
opus_decoder_ = opus_decoder_create(opus_decode_sample_rate_, 1, NULL);
if (opus_decode_sample_rate_ != AUDIO_OUTPUT_SAMPLE_RATE) {
output_resampler_.Configure(AUDIO_OUTPUT_SAMPLE_RATE, opus_decode_sample_rate_);
}
if (16000 != AUDIO_INPUT_SAMPLE_RATE) {
input_resampler_.Configure(AUDIO_INPUT_SAMPLE_RATE, 16000);
reference_resampler_.Configure(AUDIO_INPUT_SAMPLE_RATE, 16000);
}
ota_.SetCheckVersionUrl(CONFIG_OTA_VERSION_URL);
ota_.SetHeader("Device-Id", SystemInfo::GetMacAddress().c_str());
}
@@ -58,9 +43,6 @@ Application::~Application() {
if (main_loop_task_stack_ != nullptr) {
heap_caps_free(main_loop_task_stack_);
}
if (audio_device_ != nullptr) {
delete audio_device_;
}
vEventGroupDelete(event_group_);
}
@@ -75,10 +57,11 @@ void Application::CheckNewVersion() {
vTaskDelay(100);
}
SetChatState(kChatStateUpgrading);
ota_.StartUpgrade([this](int progress, size_t speed) {
ota_.StartUpgrade([](int progress, size_t speed) {
char buffer[64];
snprintf(buffer, sizeof(buffer), "Upgrading...\n %d%% %zuKB/s", progress, speed / 1024);
display_.SetText(buffer);
auto display = Board::GetInstance().GetDisplay();
display->SetText(buffer);
});
// If upgrade success, the device will reboot and never reach here
ESP_LOGI(TAG, "Firmware upgrade failed...");
@@ -90,7 +73,8 @@ void Application::CheckNewVersion() {
void Application::Alert(const std::string&& title, const std::string&& message) {
ESP_LOGE(TAG, "Alert: %s, %s", title.c_str(), message.c_str());
display_.ShowNotification(std::string(title + "\n" + message));
auto display = Board::GetInstance().GetDisplay();
display->ShowNotification(std::string(title + "\n" + message));
if (message == "PIN is not ready") {
PlayLocalFile(p3_err_pin_start, p3_err_pin_end - p3_err_pin_start);
@@ -104,7 +88,8 @@ void Application::Alert(const std::string&& title, const std::string&& message)
void Application::PlayLocalFile(const char* data, size_t size) {
ESP_LOGI(TAG, "PlayLocalFile: %zu bytes", size);
SetDecodeSampleRate(16000);
audio_device_->EnableOutput(true);
auto codec = Board::GetInstance().GetAudioCodec();
codec->EnableOutput(true);
{
std::lock_guard<std::mutex> lock(mutex_);
@@ -124,22 +109,58 @@ void Application::PlayLocalFile(const char* data, size_t size) {
}
}
void Application::Start() {
auto& builtin_led = BuiltinLed::GetInstance();
builtin_led.SetBlue();
builtin_led.StartContinuousBlink(100);
void Application::ToggleChatState() {
Schedule([this]() {
if (chat_state_ == kChatStateIdle) {
SetChatState(kChatStateConnecting);
StartWebSocketClient();
if (ws_client_ && ws_client_->IsConnected()) {
opus_encoder_.ResetState();
#ifdef CONFIG_USE_AFE_SR
audio_processor_.Start();
#endif
SetChatState(kChatStateListening);
ESP_LOGI(TAG, "Communication started");
} else {
SetChatState(kChatStateIdle);
}
} else if (chat_state_ == kChatStateSpeaking) {
AbortSpeaking();
} else if (chat_state_ == kChatStateListening) {
if (ws_client_ && ws_client_->IsConnected()) {
ws_client_->Close();
}
}
});
}
void Application::Start() {
auto& board = Board::GetInstance();
board.Initialize();
audio_device_ = board.GetAudioDevice();
audio_device_->Initialize();
audio_device_->EnableInput(true);
audio_device_->EnableOutput(true);
audio_device_->EnableOutput(false);
audio_device_->OnInputData([this](std::vector<int16_t>&& data) {
if (16000 != AUDIO_INPUT_SAMPLE_RATE) {
if (audio_device_->input_channels() == 2) {
auto builtin_led = board.GetBuiltinLed();
builtin_led->SetBlue();
builtin_led->StartContinuousBlink(100);
auto display = board.GetDisplay();
display->SetupUI();
auto codec = board.GetAudioCodec();
opus_decode_sample_rate_ = codec->output_sample_rate();
opus_decoder_ = opus_decoder_create(opus_decode_sample_rate_, 1, NULL);
opus_encoder_.Configure(codec->input_sample_rate(), 1);
if (codec->input_sample_rate() != 16000) {
input_resampler_.Configure(codec->input_sample_rate(), 16000);
reference_resampler_.Configure(codec->input_sample_rate(), 16000);
}
codec->EnableInput(true);
codec->EnableOutput(true);
codec->EnableOutput(false);
codec->OnInputData([this, codec](std::vector<int16_t>&& data) {
if (codec->input_sample_rate() != 16000) {
if (codec->input_channels() == 2) {
auto mic_channel = std::vector<int16_t>(data.size() / 2);
auto reference_channel = std::vector<int16_t>(data.size() / 2);
for (size_t i = 0, j = 0; i < mic_channel.size(); ++i, j += 2) {
@@ -196,70 +217,8 @@ void Application::Start() {
board.StartNetwork();
// Blink the LED to indicate the device is running
builtin_led.SetGreen();
builtin_led.BlinkOnce();
boot_button_.OnClick([this]() {
Schedule([this]() {
if (chat_state_ == kChatStateIdle) {
SetChatState(kChatStateConnecting);
StartWebSocketClient();
if (ws_client_ && ws_client_->IsConnected()) {
opus_encoder_.ResetState();
#ifdef CONFIG_USE_AFE_SR
audio_processor_.Start();
#endif
SetChatState(kChatStateListening);
ESP_LOGI(TAG, "Communication started");
} else {
SetChatState(kChatStateIdle);
}
} else if (chat_state_ == kChatStateSpeaking) {
AbortSpeaking();
} else if (chat_state_ == kChatStateListening) {
if (ws_client_ && ws_client_->IsConnected()) {
ws_client_->Close();
}
}
});
});
volume_up_button_.OnClick([this]() {
Schedule([this]() {
auto volume = audio_device_->output_volume() + 10;
if (volume > 100) {
volume = 100;
}
audio_device_->SetOutputVolume(volume);
display_.ShowNotification("Volume\n" + std::to_string(volume));
});
});
volume_up_button_.OnLongPress([this]() {
Schedule([this]() {
audio_device_->SetOutputVolume(100);
display_.ShowNotification("Volume\n100");
});
});
volume_down_button_.OnClick([this]() {
Schedule([this]() {
auto volume = audio_device_->output_volume() - 10;
if (volume < 0) {
volume = 0;
}
audio_device_->SetOutputVolume(volume);
display_.ShowNotification("Volume\n" + std::to_string(volume));
});
});
volume_down_button_.OnLongPress([this]() {
Schedule([this]() {
audio_device_->SetOutputVolume(0);
display_.ShowNotification("Volume\n0");
});
});
builtin_led->SetGreen();
builtin_led->BlinkOnce();
const size_t main_loop_stack_size = 4096 * 2;
main_loop_task_stack_ = (StackType_t*)heap_caps_malloc(main_loop_stack_size, MALLOC_CAP_SPIRAM);
@@ -277,17 +236,17 @@ void Application::Start() {
}, "check_new_version", 4096 * 2, this, 1, NULL);
#ifdef CONFIG_USE_AFE_SR
wake_word_detect_.Initialize(audio_device_->input_channels(), audio_device_->input_reference());
wake_word_detect_.Initialize(codec->input_channels(), codec->input_reference());
wake_word_detect_.OnVadStateChange([this](bool speaking) {
Schedule([this, speaking]() {
auto& builtin_led = BuiltinLed::GetInstance();
auto builtin_led = Board::GetInstance().GetBuiltinLed();
if (chat_state_ == kChatStateListening) {
if (speaking) {
builtin_led.SetRed(32);
builtin_led->SetRed(32);
} else {
builtin_led.SetRed(8);
builtin_led->SetRed(8);
}
builtin_led.TurnOn();
builtin_led->TurnOn();
}
});
});
@@ -326,7 +285,7 @@ void Application::Start() {
});
wake_word_detect_.StartDetection();
audio_processor_.Initialize(audio_device_->input_channels(), audio_device_->input_reference());
audio_processor_.Initialize(codec->input_channels(), codec->input_reference());
audio_processor_.OnOutput([this](std::vector<int16_t>&& data) {
Schedule([this, data = std::move(data)]() {
if (chat_state_ == kChatStateListening) {
@@ -339,7 +298,7 @@ void Application::Start() {
#endif
chat_state_ = kChatStateIdle;
display_.UpdateDisplay();
display->UpdateDisplay();
}
void Application::Schedule(std::function<void()> callback) {
@@ -397,31 +356,31 @@ void Application::SetChatState(ChatState state) {
chat_state_ = state;
ESP_LOGI(TAG, "STATE: %s", state_str[chat_state_]);
auto& builtin_led = BuiltinLed::GetInstance();
auto builtin_led = Board::GetInstance().GetBuiltinLed();
switch (chat_state_) {
case kChatStateUnknown:
case kChatStateIdle:
builtin_led.TurnOff();
builtin_led->TurnOff();
break;
case kChatStateConnecting:
builtin_led.SetBlue();
builtin_led.TurnOn();
builtin_led->SetBlue();
builtin_led->TurnOn();
break;
case kChatStateListening:
builtin_led.SetRed();
builtin_led.TurnOn();
builtin_led->SetRed();
builtin_led->TurnOn();
break;
case kChatStateSpeaking:
builtin_led.SetGreen();
builtin_led.TurnOn();
builtin_led->SetGreen();
builtin_led->TurnOn();
break;
case kChatStateWakeWordDetected:
builtin_led.SetBlue();
builtin_led.TurnOn();
builtin_led->SetBlue();
builtin_led->TurnOn();
break;
case kChatStateUpgrading:
builtin_led.SetGreen();
builtin_led.StartContinuousBlink(100);
builtin_led->SetGreen();
builtin_led->StartContinuousBlink(100);
break;
}
@@ -450,6 +409,7 @@ BinaryProtocol3* Application::AllocateBinaryProtocol3(const uint8_t* payload, si
void Application::AudioEncodeTask() {
ESP_LOGI(TAG, "Audio encode task started");
const int max_audio_play_queue_size_ = 2; // avoid decoding too fast
auto codec = Board::GetInstance().GetAudioCodec();
while (true) {
std::unique_lock<std::mutex> lock(mutex_);
@@ -490,7 +450,7 @@ void Application::AudioEncodeTask() {
continue;
}
if (opus_decode_sample_rate_ != AUDIO_OUTPUT_SAMPLE_RATE) {
if (opus_decode_sample_rate_ != codec->output_sample_rate()) {
int target_size = output_resampler_.GetOutputSamples(frame_size);
std::vector<int16_t> resampled(target_size);
output_resampler_.Process(packet->pcm.data(), frame_size, resampled.data());
@@ -514,7 +474,8 @@ void Application::HandleAudioPacket(AudioPacket* packet) {
}
// This will block until the audio device has finished playing the audio
audio_device_->OutputData(packet->pcm);
auto codec = Board::GetInstance().GetAudioCodec();
codec->OutputData(packet->pcm);
break;
}
case kAudioPacketTypeStart:
@@ -574,9 +535,11 @@ void Application::SetDecodeSampleRate(int sample_rate) {
opus_decoder_destroy(opus_decoder_);
opus_decode_sample_rate_ = sample_rate;
opus_decoder_ = opus_decoder_create(opus_decode_sample_rate_, 1, NULL);
if (opus_decode_sample_rate_ != AUDIO_OUTPUT_SAMPLE_RATE) {
ESP_LOGI(TAG, "Resampling audio from %d to %d", opus_decode_sample_rate_, AUDIO_OUTPUT_SAMPLE_RATE);
output_resampler_.Configure(opus_decode_sample_rate_, AUDIO_OUTPUT_SAMPLE_RATE);
auto codec = Board::GetInstance().GetAudioCodec();
if (opus_decode_sample_rate_ != codec->output_sample_rate()) {
ESP_LOGI(TAG, "Resampling audio from %d to %d", opus_decode_sample_rate_, codec->output_sample_rate());
output_resampler_.Configure(opus_decode_sample_rate_, codec->output_sample_rate());
}
}
@@ -683,7 +646,8 @@ void Application::StartWebSocketClient() {
ws_client_->OnDisconnected([this]() {
ESP_LOGI(TAG, "Websocket disconnected");
Schedule([this]() {
audio_device_->EnableOutput(false);
auto codec = Board::GetInstance().GetAudioCodec();
codec->EnableOutput(false);
#ifdef CONFIG_USE_AFE_SR
audio_processor_.Stop();
#endif
@@ -699,5 +663,6 @@ void Application::StartWebSocketClient() {
}
// 建立语音通道后打开音频输出，避免待机时喇叭底噪
audio_device_->EnableOutput(true);
auto codec = Board::GetInstance().GetAudioCodec();
codec->EnableOutput(true);
}