devttl/xiaozhi-esp32
1
0
Fork 0
forked from xiaozhi/xiaozhi-esp32
Code Pull Requests Activity

新的opus封装以及优化const会导致的内存拷贝

Browse Source
This commit is contained in:
Terrence
2024-12-04 02:12:20 +08:00
parent 9c1f8a1d06
commit bcfd120b00
15 changed files with 102 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@@ -4,7 +4,7 @@
# CMakeLists in this exact order for cmake to work correctly
cmake_minimum_required(VERSION 3.16)
set(PROJECT_VER "0.9.5")
set(PROJECT_VER "0.9.6")
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
project(xiaozhi)

95
main/application.cc
View File

@@ -21,6 +21,15 @@ extern const char p3_err_pin_end[] asm("_binary_err_pin_p3_end");
extern const char p3_err_wificonfig_start[] asm("_binary_err_wificonfig_p3_start");
extern const char p3_err_wificonfig_end[] asm("_binary_err_wificonfig_p3_end");
static const char* const STATE_STRINGS[] = {
"unknown",
"idle",
"connecting",
"listening",
"speaking",
"upgrading",
"invalid_state"
};
Application::Application() : background_task_(4096 * 8) {
event_group_ = xEventGroupCreate();
@@ -30,13 +39,6 @@ Application::Application() : background_task_(4096 * 8) {
}
Application::~Application() {
if (protocol_ != nullptr) {
delete protocol_;
}
if (opus_decoder_ != nullptr) {
opus_decoder_destroy(opus_decoder_);
}
vEventGroupDelete(event_group_);
}
@@ -83,7 +85,7 @@ void Application::CheckNewVersion() {
}
}
void Application::Alert(const std::string&& title, const std::string&& message) {
void Application::Alert(const std::string& title, const std::string& message) {
ESP_LOGW(TAG, "Alert: %s, %s", title.c_str(), message.c_str());
auto display = Board::GetInstance().GetDisplay();
display->ShowNotification(message);
@@ -105,7 +107,7 @@ void Application::PlayLocalFile(const char* data, size_t size) {
p += sizeof(BinaryProtocol3);
auto payload_size = ntohs(p3->payload_size);
std::string opus;
std::vector<uint8_t> opus;
opus.resize(payload_size);
memcpy(opus.data(), p3->payload, payload_size);
p += payload_size;
@@ -117,10 +119,15 @@ void Application::PlayLocalFile(const char* data, size_t size) {
void Application::ToggleChatState() {
Schedule([this]() {
if (!protocol_) {
ESP_LOGE(TAG, "Protocol not initialized");
return;
}
if (chat_state_ == kChatStateIdle) {
SetChatState(kChatStateConnecting);
if (!protocol_->OpenAudioChannel()) {
ESP_LOGE(TAG, "Failed to open audio channel");
Alert("Error", "Failed to open audio channel");
SetChatState(kChatStateIdle);
return;
}
@@ -138,13 +145,18 @@ void Application::ToggleChatState() {
void Application::StartListening() {
Schedule([this]() {
if (!protocol_) {
ESP_LOGE(TAG, "Protocol not initialized");
return;
}
keep_listening_ = false;
if (chat_state_ == kChatStateIdle) {
if (!protocol_->IsAudioChannelOpened()) {
SetChatState(kChatStateConnecting);
if (!protocol_->OpenAudioChannel()) {
SetChatState(kChatStateIdle);
ESP_LOGE(TAG, "Failed to open audio channel");
Alert("Error", "Failed to open audio channel");
return;
}
}
@@ -183,8 +195,8 @@ void Application::Start() {
/* Setup the audio codec */
auto codec = board.GetAudioCodec();
opus_decode_sample_rate_ = codec->output_sample_rate();
opus_decoder_ = opus_decoder_create(opus_decode_sample_rate_, 1, NULL);
opus_encoder_.Configure(16000, 1, OPUS_FRAME_DURATION_MS);
opus_decoder_ = std::make_unique<OpusDecoderWrapper>(opus_decode_sample_rate_, 1);
opus_encoder_ = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
if (codec->input_sample_rate() != 16000) {
input_resampler_.Configure(codec->input_sample_rate(), 16000);
reference_resampler_.Configure(codec->input_sample_rate(), 16000);
@@ -221,9 +233,9 @@ void Application::Start() {
#if CONFIG_IDF_TARGET_ESP32S3
audio_processor_.Initialize(codec->input_channels(), codec->input_reference());
audio_processor_.OnOutput([this](std::vector<int16_t>&& data) {
background_task_.Schedule([this, data = std::move(data)]() {
opus_encoder_.Encode(data, [this](const uint8_t* opus, size_t opus_size) {
Schedule([this, opus = std::string(reinterpret_cast<const char*>(opus), opus_size)]() {
background_task_.Schedule([this, data = std::move(data)]() mutable {
opus_encoder_->Encode(std::move(data), [this](std::vector<uint8_t>&& opus) {
Schedule([this, opus = std::move(opus)]() {
protocol_->SendAudio(opus);
});
});
@@ -258,7 +270,7 @@ void Application::Start() {
return;
}
std::string opus;
std::vector<uint8_t> opus;
// Encode and send the wake word data to the server
while (wake_word_detect_.GetWakeWordOpus(opus)) {
protocol_->SendAudio(opus);
@@ -282,14 +294,14 @@ void Application::Start() {
// Initialize the protocol
display->SetStatus("初始化协议");
#ifdef CONFIG_CONNECTION_TYPE_WEBSOCKET
protocol_ = new WebsocketProtocol();
protocol_ = std::make_unique<WebsocketProtocol>();
#else
protocol_ = new MqttProtocol();
protocol_ = std::make_unique<MqttProtocol>();
#endif
protocol_->OnNetworkError([this](const std::string& message) {
Alert("Error", std::move(message));
});
protocol_->OnIncomingAudio([this](const std::string& data) {
protocol_->OnIncomingAudio([this](std::vector<uint8_t>&& data) {
std::lock_guard<std::mutex> lock(mutex_);
if (chat_state_ == kChatStateSpeaking) {
audio_decode_queue_.emplace_back(std::move(data));
@@ -363,9 +375,8 @@ void Application::Start() {
}
void Application::Schedule(std::function<void()> callback) {
mutex_.lock();
main_tasks_.push_back(callback);
mutex_.unlock();
std::lock_guard<std::mutex> lock(mutex_);
main_tasks_.push_back(std::move(callback));
xEventGroupSetBits(event_group_, SCHEDULE_EVENT);
}
@@ -397,7 +408,7 @@ void Application::MainLoop() {
void Application::ResetDecoder() {
std::lock_guard<std::mutex> lock(mutex_);
opus_decoder_ctl(opus_decoder_, OPUS_RESET_STATE);
opus_decoder_->ResetState();
audio_decode_queue_.clear();
last_output_time_ = std::chrono::steady_clock::now();
Board::GetInstance().GetAudioCodec()->EnableOutput(true);
@@ -430,24 +441,21 @@ void Application::OutputAudio() {
audio_decode_queue_.pop_front();
lock.unlock();
background_task_.Schedule([this, codec, opus = std::move(opus)]() {
background_task_.Schedule([this, codec, opus = std::move(opus)]() mutable {
if (aborted_) {
return;
}
int frame_size = opus_decode_sample_rate_ * OPUS_FRAME_DURATION_MS / 1000;
std::vector<int16_t> pcm(frame_size);
int ret = opus_decode(opus_decoder_, (const unsigned char*)opus.data(), opus.size(), pcm.data(), frame_size, 0);
if (ret < 0) {
ESP_LOGE(TAG, "Failed to decode audio, error code: %d", ret);
std::vector<int16_t> pcm;
if (!opus_decoder_->Decode(std::move(opus), pcm)) {
return;
}
// Resample if the sample rate is different
if (opus_decode_sample_rate_ != codec->output_sample_rate()) {
int target_size = output_resampler_.GetOutputSamples(frame_size);
int target_size = output_resampler_.GetOutputSamples(pcm.size());
std::vector<int16_t> resampled(target_size);
output_resampler_.Process(pcm.data(), frame_size, resampled.data());
output_resampler_.Process(pcm.data(), pcm.size(), resampled.data());
pcm = std::move(resampled);
}
@@ -495,9 +503,9 @@ void Application::InputAudio() {
}
#else
if (chat_state_ == kChatStateListening) {
background_task_.Schedule([this, data = std::move(data)]() {
opus_encoder_.Encode(data, [this](const uint8_t* opus, size_t opus_size) {
Schedule([this, opus = std::string(reinterpret_cast<const char*>(opus), opus_size)]() {
background_task_.Schedule([this, data = std::move(data)]() mutable {
opus_encoder_->Encode(std::move(data), [this](std::vector<uint8_t>&& opus) {
Schedule([this, opus = std::move(opus)]() {
protocol_->SendAudio(opus);
});
});
@@ -513,22 +521,12 @@ void Application::AbortSpeaking(AbortReason reason) {
}
void Application::SetChatState(ChatState state) {
const char* state_str[] = {
"unknown",
"idle",
"connecting",
"listening",
"speaking",
"upgrading",
"invalid_state"
};
if (chat_state_ == state) {
// No need to update the state
return;
}
chat_state_ = state;
ESP_LOGI(TAG, "STATE: %s", state_str[chat_state_]);
ESP_LOGI(TAG, "STATE: %s", STATE_STRINGS[chat_state_]);
// The state is changed, wait for all background tasks to finish
background_task_.WaitForCompletion();
@@ -555,7 +553,7 @@ void Application::SetChatState(ChatState state) {
display->SetStatus("聆听中...");
display->SetEmotion("neutral");
ResetDecoder();
opus_encoder_.ResetState();
opus_encoder_->ResetState();
#if CONFIG_IDF_TARGET_ESP32S3
audio_processor_.Start();
#endif
@@ -584,9 +582,8 @@ void Application::SetDecodeSampleRate(int sample_rate) {
return;
}
opus_decoder_destroy(opus_decoder_);
opus_decode_sample_rate_ = sample_rate;
opus_decoder_ = opus_decoder_create(opus_decode_sample_rate_, 1, NULL);
opus_decoder_ = std::make_unique<OpusDecoderWrapper>(opus_decode_sample_rate_, 1);
auto codec = Board::GetInstance().GetAudioCodec();
if (opus_decode_sample_rate_ != codec->output_sample_rate()) {

11
main/application.h
View File

@@ -10,6 +10,7 @@
#include <condition_variable>
#include "opus_encoder.h"
#include "opus_decoder.h"
#include "opus_resampler.h"
#include "protocol.h"
@@ -52,7 +53,7 @@ public:
ChatState GetChatState() const { return chat_state_; }
void Schedule(std::function<void()> callback);
void SetChatState(ChatState state);
void Alert(const std::string&& title, const std::string&& message);
void Alert(const std::string& title, const std::string& message);
void AbortSpeaking(AbortReason reason);
void ToggleChatState();
void StartListening();
@@ -69,7 +70,7 @@ private:
Ota ota_;
std::mutex mutex_;
std::list<std::function<void()>> main_tasks_;
Protocol* protocol_ = nullptr;
std::unique_ptr<Protocol> protocol_;
EventGroupHandle_t event_group_;
volatile ChatState chat_state_ = kChatStateUnknown;
bool keep_listening_ = false;
@@ -78,10 +79,10 @@ private:
// Audio encode / decode
BackgroundTask background_task_;
std::chrono::steady_clock::time_point last_output_time_;
std::list<std::string> audio_decode_queue_;
std::list<std::vector<uint8_t>> audio_decode_queue_;
OpusEncoder opus_encoder_;
OpusDecoder* opus_decoder_ = nullptr;
std::unique_ptr<OpusEncoderWrapper> opus_encoder_;
std::unique_ptr<OpusDecoderWrapper> opus_decoder_;
int opus_decode_sample_rate_ = -1;
OpusResampler input_resampler_;

2
main/audio_processing/audio_processor.cc
View File

@@ -63,7 +63,7 @@ AudioProcessor::~AudioProcessor() {
vEventGroupDelete(event_group_);
}
void AudioProcessor::Input(std::vector<int16_t>& data) {
void AudioProcessor::Input(const std::vector<int16_t>& data) {
input_buffer_.insert(input_buffer_.end(), data.begin(), data.end());
auto chunk_size = esp_afe_vc_v1.get_feed_chunksize(afe_communication_data_) * channels_;

2
main/audio_processing/audio_processor.h
View File

@@ -16,7 +16,7 @@ public:
~AudioProcessor();
void Initialize(int channels, bool reference);
void Input(std::vector<int16_t>& data);
void Input(const std::vector<int16_t>& data);
void Start();
void Stop();
bool IsRunning();

44
main/audio_processing/wake_word_detect.cc
View File

@@ -111,7 +111,7 @@ bool WakeWordDetect::IsDetectionRunning() {
return xEventGroupGetBits(event_group_) & DETECTION_RUNNING_EVENT;
}
void WakeWordDetect::Feed(std::vector<int16_t>& data) {
void WakeWordDetect::Feed(const std::vector<int16_t>& data) {
input_buffer_.insert(input_buffer_.end(), data.begin(), data.end());
auto chunk_size = esp_afe_sr_v1.get_feed_chunksize(afe_detection_data_) * channels_;
@@ -163,8 +163,7 @@ void WakeWordDetect::AudioDetectionTask() {
void WakeWordDetect::StoreWakeWordData(uint16_t* data, size_t samples) {
// store audio data to wake_word_pcm_
std::vector<int16_t> pcm(data, data + samples);
wake_word_pcm_.emplace_back(std::move(pcm));
wake_word_pcm_.emplace_back(std::vector<int16_t>(data, data + samples));
// keep about 2 seconds of data, detect duration is 32ms (sample_rate == 16000, chunksize == 512)
while (wake_word_pcm_.size() > 2000 / 32) {
wake_word_pcm_.pop_front();
@@ -178,34 +177,33 @@ void WakeWordDetect::EncodeWakeWordData() {
}
wake_word_encode_task_ = xTaskCreateStatic([](void* arg) {
auto this_ = (WakeWordDetect*)arg;
auto start_time = esp_timer_get_time();
// encode detect packets
OpusEncoder* encoder = new OpusEncoder();
encoder->Configure(16000, 1, 60);
encoder->SetComplexity(0);
for (auto& pcm: this_->wake_word_pcm_) {
encoder->Encode(pcm, [this_](const uint8_t* opus, size_t opus_size) {
std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
this_->wake_word_opus_.emplace_back(std::string(reinterpret_cast<const char*>(opus), opus_size));
this_->wake_word_cv_.notify_all();
});
}
this_->wake_word_pcm_.clear();
auto end_time = esp_timer_get_time();
ESP_LOGI(TAG, "Encode wake word opus %zu packets in %lld ms", this_->wake_word_opus_.size(), (end_time - start_time) / 1000);
{
auto start_time = esp_timer_get_time();
auto encoder = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
encoder->SetComplexity(0); // 0 is the fastest
for (auto& pcm: this_->wake_word_pcm_) {
encoder->Encode(std::move(pcm), [this_](std::vector<uint8_t>&& opus) {
std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
this_->wake_word_opus_.emplace_back(std::move(opus));
this_->wake_word_cv_.notify_all();
});
}
this_->wake_word_pcm_.clear();
auto end_time = esp_timer_get_time();
ESP_LOGI(TAG, "Encode wake word opus %zu packets in %lld ms",
this_->wake_word_opus_.size(), (end_time - start_time) / 1000);
std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
this_->wake_word_opus_.push_back("");
this_->wake_word_opus_.push_back(std::vector<uint8_t>());
this_->wake_word_cv_.notify_all();
}
delete encoder;
vTaskDelete(NULL);
}, "encode_detect_packets", 4096 * 8, this, 1, wake_word_encode_task_stack_, &wake_word_encode_task_buffer_);
}
bool WakeWordDetect::GetWakeWordOpus(std::string& opus) {
bool WakeWordDetect::GetWakeWordOpus(std::vector<uint8_t>& opus) {
std::unique_lock<std::mutex> lock(wake_word_mutex_);
wake_word_cv_.wait(lock, [this]() {
return !wake_word_opus_.empty();

6
main/audio_processing/wake_word_detect.h
View File

@@ -22,14 +22,14 @@ public:
~WakeWordDetect();
void Initialize(int channels, bool reference);
void Feed(std::vector<int16_t>& data);
void Feed(const std::vector<int16_t>& data);
void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback);
void OnVadStateChange(std::function<void(bool speaking)> callback);
void StartDetection();
void StopDetection();
bool IsDetectionRunning();
void EncodeWakeWordData();
bool GetWakeWordOpus(std::string& opus);
bool GetWakeWordOpus(std::vector<uint8_t>& opus);
const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; }
private:
@@ -49,7 +49,7 @@ private:
StaticTask_t wake_word_encode_task_buffer_;
StackType_t* wake_word_encode_task_stack_ = nullptr;
std::list<std::vector<int16_t>> wake_word_pcm_;
std::list<std::string> wake_word_opus_;
std::list<std::vector<uint8_t>> wake_word_opus_;
std::mutex wake_word_mutex_;
std::condition_variable wake_word_cv_;

16
main/background_task.cc
View File

@@ -31,12 +31,16 @@ void BackgroundTask::Schedule(std::function<void()> callback) {
ESP_LOGW(TAG, "active_tasks_ == %u", active_tasks_.load());
}
active_tasks_++;
auto wrapped_callback = [this, callback]() {
callback();
active_tasks_--;
condition_variable_.notify_all();
};
main_tasks_.push_back(wrapped_callback);
main_tasks_.emplace_back([this, cb = std::move(callback)]() {
cb();
{
std::lock_guard<std::mutex> lock(mutex_);
active_tasks_--;
if (main_tasks_.empty() && active_tasks_ == 0) {
condition_variable_.notify_all();
}
}
});
condition_variable_.notify_all();
}

2
main/idf_component.yml
View File

@@ -1,7 +1,7 @@
## IDF Component Manager Manifest File
dependencies:
78/esp-wifi-connect: "~1.4.1"
78/esp-opus-encoder: "~1.1.0"
78/esp-opus-encoder: "~2.0.0"
78/esp-ml307: "~1.7.0"
espressif/led_strip: "^2.4.1"
espressif/esp_codec_dev: "^1.3.1"

6
main/protocols/mqtt_protocol.cc
View File

@@ -105,7 +105,7 @@ void MqttProtocol::SendText(const std::string& text) {
mqtt_->Publish(publish_topic_, text);
}
void MqttProtocol::SendAudio(const std::string& data) {
void MqttProtocol::SendAudio(const std::vector<uint8_t>& data) {
std::lock_guard<std::mutex> lock(channel_mutex_);
if (udp_ == nullptr) {
return;
@@ -202,7 +202,7 @@ bool MqttProtocol::OpenAudioChannel() {
ESP_LOGW(TAG, "Received audio packet with wrong sequence: %lu, expected: %lu", sequence, remote_sequence_ + 1);
}
std::string decrypted;
std::vector<uint8_t> decrypted;
size_t decrypted_size = data.size() - aes_nonce_.size();
size_t nc_off = 0;
uint8_t stream_block[16] = {0};
@@ -215,7 +215,7 @@ bool MqttProtocol::OpenAudioChannel() {
return;
}
if (on_incoming_audio_ != nullptr) {
on_incoming_audio_(decrypted);
on_incoming_audio_(std::move(decrypted));
}
remote_sequence_ = sequence;
});

2
main/protocols/mqtt_protocol.h
View File

@@ -25,7 +25,7 @@ public:
MqttProtocol();
~MqttProtocol();
void SendAudio(const std::string& data) override;
void SendAudio(const std::vector<uint8_t>& data) override;
bool OpenAudioChannel() override;
void CloseAudioChannel() override;
bool IsAudioChannelOpened() const override;

2
main/protocols/protocol.cc
View File

@@ -8,7 +8,7 @@ void Protocol::OnIncomingJson(std::function<void(const cJSON* root)> callback) {
on_incoming_json_ = callback;
}
void Protocol::OnIncomingAudio(std::function<void(const std::string& data)> callback) {
void Protocol::OnIncomingAudio(std::function<void(std::vector<uint8_t>&& data)> callback) {
on_incoming_audio_ = callback;
}

6
main/protocols/protocol.h
View File

@@ -31,7 +31,7 @@ public:
return server_sample_rate_;
}
void OnIncomingAudio(std::function<void(const std::string& data)> callback);
void OnIncomingAudio(std::function<void(std::vector<uint8_t>&& data)> callback);
void OnIncomingJson(std::function<void(const cJSON* root)> callback);
void OnAudioChannelOpened(std::function<void()> callback);
void OnAudioChannelClosed(std::function<void()> callback);
@@ -40,7 +40,7 @@ public:
virtual bool OpenAudioChannel() = 0;
virtual void CloseAudioChannel() = 0;
virtual bool IsAudioChannelOpened() const = 0;
virtual void SendAudio(const std::string& data) = 0;
virtual void SendAudio(const std::vector<uint8_t>& data) = 0;
virtual void SendWakeWordDetected(const std::string& wake_word);
virtual void SendStartListening(ListeningMode mode);
virtual void SendStopListening();
@@ -48,7 +48,7 @@ public:
protected:
std::function<void(const cJSON* root)> on_incoming_json_;
std::function<void(const std::string& data)> on_incoming_audio_;
std::function<void(std::vector<uint8_t>&& data)> on_incoming_audio_;
std::function<void()> on_audio_channel_opened_;
std::function<void()> on_audio_channel_closed_;
std::function<void(const std::string& message)> on_network_error_;

4
main/protocols/websocket_protocol.cc
View File

@@ -23,7 +23,7 @@ WebsocketProtocol::~WebsocketProtocol() {
vEventGroupDelete(event_group_handle_);
}
void WebsocketProtocol::SendAudio(const std::string& data) {
void WebsocketProtocol::SendAudio(const std::vector<uint8_t>& data) {
if (websocket_ == nullptr) {
return;
}
@@ -65,7 +65,7 @@ bool WebsocketProtocol::OpenAudioChannel() {
websocket_->OnData([this](const char* data, size_t len, bool binary) {
if (binary) {
if (on_incoming_audio_ != nullptr) {
on_incoming_audio_(std::string(data, len));
on_incoming_audio_(std::vector<uint8_t>((uint8_t*)data, (uint8_t*)data + len));
}
} else {
// Parse JSON data

2
main/protocols/websocket_protocol.h
View File

@@ -15,7 +15,7 @@ public:
WebsocketProtocol();
~WebsocketProtocol();
void SendAudio(const std::string& data) override;
void SendAudio(const std::vector<uint8_t>& data) override;
bool OpenAudioChannel() override;
void CloseAudioChannel() override;
bool IsAudioChannelOpened() const override;