devttl/xiaozhi-esp32
1
0
Fork 0
forked from xiaozhi/xiaozhi-esp32
Code Pull Requests Activity

Audio stream packet with timestamp

Browse Source
This commit is contained in:
Xiaoxia
2025-04-28 23:10:24 +08:00
parent 7fdf78408e
commit b804343d30
8 changed files with 70 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
main/application.cc
View File

@@ -245,13 +245,13 @@ void Application::PlaySound(const std::string_view& sound) {
p += sizeof(BinaryProtocol3);
auto payload_size = ntohs(p3->payload_size);
std::vector<uint8_t> opus;
opus.resize(payload_size);
memcpy(opus.data(), p3->payload, payload_size);
AudioStreamPacket packet;
packet.payload.resize(payload_size);
memcpy(packet.payload.data(), p3->payload, payload_size);
p += payload_size;
std::lock_guard<std::mutex> lock(mutex_);
audio_decode_queue_.emplace_back(std::move(opus));
audio_decode_queue_.emplace_back(std::move(packet));
}
}
@@ -391,11 +391,11 @@ void Application::Start() {
SetDeviceState(kDeviceStateIdle);
Alert(Lang::Strings::ERROR, message.c_str(), "sad", Lang::Sounds::P3_EXCLAMATION);
});
protocol_->OnIncomingAudio([this](std::vector<uint8_t>&& data) {
protocol_->OnIncomingAudio([this](AudioStreamPacket&& packet) {
const int max_packets_in_queue = 600 / OPUS_FRAME_DURATION_MS;
std::lock_guard<std::mutex> lock(mutex_);
if (audio_decode_queue_.size() < max_packets_in_queue) {
audio_decode_queue_.emplace_back(std::move(data));
audio_decode_queue_.emplace_back(std::move(packet));
}
});
protocol_->OnAudioChannelOpened([this, codec, &board]() {
@@ -510,8 +510,12 @@ void Application::Start() {
return;
}
opus_encoder_->Encode(std::move(data), [this](std::vector<uint8_t>&& opus) {
Schedule([this, opus = std::move(opus)]() {
protocol_->SendAudio(opus);
AudioStreamPacket packet;
packet.payload = std::move(opus);
packet.timestamp = last_output_timestamp_;
last_output_timestamp_ = 0;
Schedule([this, packet = std::move(packet)]() {
protocol_->SendAudio(packet);
});
});
});
@@ -544,10 +548,10 @@ void Application::Start() {
return;
}
std::vector<uint8_t> opus;
AudioStreamPacket packet;
// Encode and send the wake word data to the server
while (wake_word_detect_.GetWakeWordOpus(opus)) {
protocol_->SendAudio(opus);
while (wake_word_detect_.GetWakeWordOpus(packet.payload)) {
protocol_->SendAudio(packet);
}
// Set the chat state to wake word detected
protocol_->SendWakeWordDetected(wake_word);
@@ -671,20 +675,20 @@ void Application::OnAudioOutput() {
return;
}
auto opus = std::move(audio_decode_queue_.front());
auto packet = std::move(audio_decode_queue_.front());
audio_decode_queue_.pop_front();
lock.unlock();
audio_decode_cv_.notify_all();
busy_decoding_audio_ = true;
background_task_->Schedule([this, codec, opus = std::move(opus)]() mutable {
background_task_->Schedule([this, codec, packet = std::move(packet)]() mutable {
busy_decoding_audio_ = false;
if (aborted_) {
return;
}
std::vector<int16_t> pcm;
if (!opus_decoder_->Decode(std::move(opus), pcm)) {
if (!opus_decoder_->Decode(std::move(packet.payload), pcm)) {
return;
}
// Resample if the sample rate is different
@@ -695,6 +699,7 @@ void Application::OnAudioOutput() {
pcm = std::move(resampled);
}
codec->OutputData(pcm);
last_output_timestamp_ = packet.timestamp;
last_output_time_ = std::chrono::steady_clock::now();
});
}
@@ -730,8 +735,12 @@ void Application::OnAudioInput() {
return;
}
opus_encoder_->Encode(std::move(data), [this](std::vector<uint8_t>&& opus) {
Schedule([this, opus = std::move(opus)]() {
protocol_->SendAudio(opus);
AudioStreamPacket packet;
packet.payload = std::move(opus);
packet.timestamp = last_output_timestamp_;
last_output_timestamp_ = 0;
Schedule([this, packet = std::move(packet)]() {
protocol_->SendAudio(packet);
});
});
});