devttl/xiaozhi-esp32
1
0
Fork 0
forked from xiaozhi/xiaozhi-esp32
Code Pull Requests Activity

新的opus封装以及优化const会导致的内存拷贝

Browse Source
This commit is contained in:
Terrence
2024-12-04 02:12:20 +08:00
parent 9c1f8a1d06
commit bcfd120b00
15 changed files with 102 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
main/audio_processing/audio_processor.cc
View File

@@ -63,7 +63,7 @@ AudioProcessor::~AudioProcessor() {
vEventGroupDelete(event_group_);
}
void AudioProcessor::Input(std::vector<int16_t>& data) {
void AudioProcessor::Input(const std::vector<int16_t>& data) {
input_buffer_.insert(input_buffer_.end(), data.begin(), data.end());
auto chunk_size = esp_afe_vc_v1.get_feed_chunksize(afe_communication_data_) * channels_;

2
main/audio_processing/audio_processor.h
View File

@@ -16,7 +16,7 @@ public:
~AudioProcessor();
void Initialize(int channels, bool reference);
void Input(std::vector<int16_t>& data);
void Input(const std::vector<int16_t>& data);
void Start();
void Stop();
bool IsRunning();

44
main/audio_processing/wake_word_detect.cc
View File

@@ -111,7 +111,7 @@ bool WakeWordDetect::IsDetectionRunning() {
return xEventGroupGetBits(event_group_) & DETECTION_RUNNING_EVENT;
}
void WakeWordDetect::Feed(std::vector<int16_t>& data) {
void WakeWordDetect::Feed(const std::vector<int16_t>& data) {
input_buffer_.insert(input_buffer_.end(), data.begin(), data.end());
auto chunk_size = esp_afe_sr_v1.get_feed_chunksize(afe_detection_data_) * channels_;
@@ -163,8 +163,7 @@ void WakeWordDetect::AudioDetectionTask() {
void WakeWordDetect::StoreWakeWordData(uint16_t* data, size_t samples) {
// store audio data to wake_word_pcm_
std::vector<int16_t> pcm(data, data + samples);
wake_word_pcm_.emplace_back(std::move(pcm));
wake_word_pcm_.emplace_back(std::vector<int16_t>(data, data + samples));
// keep about 2 seconds of data, detect duration is 32ms (sample_rate == 16000, chunksize == 512)
while (wake_word_pcm_.size() > 2000 / 32) {
wake_word_pcm_.pop_front();
@@ -178,34 +177,33 @@ void WakeWordDetect::EncodeWakeWordData() {
}
wake_word_encode_task_ = xTaskCreateStatic([](void* arg) {
auto this_ = (WakeWordDetect*)arg;
auto start_time = esp_timer_get_time();
// encode detect packets
OpusEncoder* encoder = new OpusEncoder();
encoder->Configure(16000, 1, 60);
encoder->SetComplexity(0);
for (auto& pcm: this_->wake_word_pcm_) {
encoder->Encode(pcm, [this_](const uint8_t* opus, size_t opus_size) {
std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
this_->wake_word_opus_.emplace_back(std::string(reinterpret_cast<const char*>(opus), opus_size));
this_->wake_word_cv_.notify_all();
});
}
this_->wake_word_pcm_.clear();
auto end_time = esp_timer_get_time();
ESP_LOGI(TAG, "Encode wake word opus %zu packets in %lld ms", this_->wake_word_opus_.size(), (end_time - start_time) / 1000);
{
auto start_time = esp_timer_get_time();
auto encoder = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
encoder->SetComplexity(0); // 0 is the fastest
for (auto& pcm: this_->wake_word_pcm_) {
encoder->Encode(std::move(pcm), [this_](std::vector<uint8_t>&& opus) {
std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
this_->wake_word_opus_.emplace_back(std::move(opus));
this_->wake_word_cv_.notify_all();
});
}
this_->wake_word_pcm_.clear();
auto end_time = esp_timer_get_time();
ESP_LOGI(TAG, "Encode wake word opus %zu packets in %lld ms",
this_->wake_word_opus_.size(), (end_time - start_time) / 1000);
std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
this_->wake_word_opus_.push_back("");
this_->wake_word_opus_.push_back(std::vector<uint8_t>());
this_->wake_word_cv_.notify_all();
}
delete encoder;
vTaskDelete(NULL);
}, "encode_detect_packets", 4096 * 8, this, 1, wake_word_encode_task_stack_, &wake_word_encode_task_buffer_);
}
bool WakeWordDetect::GetWakeWordOpus(std::string& opus) {
bool WakeWordDetect::GetWakeWordOpus(std::vector<uint8_t>& opus) {
std::unique_lock<std::mutex> lock(wake_word_mutex_);
wake_word_cv_.wait(lock, [this]() {
return !wake_word_opus_.empty();

6
main/audio_processing/wake_word_detect.h
View File

@@ -22,14 +22,14 @@ public:
~WakeWordDetect();
void Initialize(int channels, bool reference);
void Feed(std::vector<int16_t>& data);
void Feed(const std::vector<int16_t>& data);
void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback);
void OnVadStateChange(std::function<void(bool speaking)> callback);
void StartDetection();
void StopDetection();
bool IsDetectionRunning();
void EncodeWakeWordData();
bool GetWakeWordOpus(std::string& opus);
bool GetWakeWordOpus(std::vector<uint8_t>& opus);
const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; }
private:
@@ -49,7 +49,7 @@ private:
StaticTask_t wake_word_encode_task_buffer_;
StackType_t* wake_word_encode_task_stack_ = nullptr;
std::list<std::vector<int16_t>> wake_word_pcm_;
std::list<std::string> wake_word_opus_;
std::list<std::vector<uint8_t>> wake_word_opus_;
std::mutex wake_word_mutex_;
std::condition_variable wake_word_cv_;