From 490b8668f65248cbd310271cb97218d73f9349db Mon Sep 17 00:00:00 2001 From: Terrence Date: Sun, 1 Sep 2024 10:37:02 +0800 Subject: [PATCH] esp cert bundle --- main/Application.cc | 33 ++++++++++++++++++--------------- main/Application.h | 4 ---- main/AudioDevice.cc | 2 +- main/BuiltinLed.cc | 2 +- main/OpusEncoder.cc | 25 ++++++++----------------- main/OpusEncoder.h | 12 ++++++++---- main/WebSocketClient.cc | 8 +++++--- main/WebSocketClient.h | 2 +- sdkconfig.defaults | 6 ++---- 9 files changed, 44 insertions(+), 50 deletions(-) diff --git a/main/Application.cc b/main/Application.cc index 5e73d88f..63bf0286 100644 --- a/main/Application.cc +++ b/main/Application.cc @@ -14,7 +14,7 @@ Application::Application() { event_group_ = xEventGroupCreate(); - audio_encode_queue_ = xQueueCreate(100, sizeof(AudioEncoderData)); + audio_encode_queue_ = xQueueCreate(100, sizeof(iovec)); audio_decode_queue_ = xQueueCreate(100, sizeof(AudioPacket*)); srmodel_list_t *models = esp_srmodel_init("model"); @@ -74,16 +74,17 @@ void Application::Start() { }); // OPUS encoder / decoder use a lot of stack memory - audio_encode_task_stack_ = (StackType_t*)malloc(4096 * 8); + const size_t opus_stack_size = 4096 * 8; + audio_encode_task_stack_ = (StackType_t*)malloc(opus_stack_size); xTaskCreateStatic([](void* arg) { Application* app = (Application*)arg; app->AudioEncodeTask(); - }, "opus_encode", 4096 * 8, this, 1, audio_encode_task_stack_, &audio_encode_task_buffer_); - audio_decode_task_stack_ = (StackType_t*)malloc(4096 * 8); + }, "opus_encode", opus_stack_size, this, 1, audio_encode_task_stack_, &audio_encode_task_buffer_); + audio_decode_task_stack_ = (StackType_t*)malloc(opus_stack_size); xTaskCreateStatic([](void* arg) { Application* app = (Application*)arg; app->AudioDecodeTask(); - }, "opus_decode", 4096 * 8, this, 1, audio_decode_task_stack_, &audio_decode_task_buffer_); + }, "opus_decode", opus_stack_size, this, 1, audio_decode_task_stack_, &audio_decode_task_buffer_); wifi_station_.Start(); @@ -305,10 +306,11 @@ void Application::AudioCommunicationTask() { if (chat_state_ == kChatStateListening) { // Send audio data to server - AudioEncoderData data; - data.size = res->data_size; - data.data = malloc(data.size); - memcpy((void*)data.data, res->data, data.size); + iovec data = { + .iov_base = malloc(res->data_size), + .iov_len = (size_t)res->data_size + }; + memcpy(data.iov_base, res->data, res->data_size); xQueueSend(audio_encode_queue_, &data, portMAX_DELAY); } } @@ -317,17 +319,18 @@ void Application::AudioCommunicationTask() { void Application::AudioEncodeTask() { ESP_LOGI(TAG, "Audio encode task started"); while (true) { - AudioEncoderData data; - xQueueReceive(audio_encode_queue_, &data, portMAX_DELAY); + iovec pcm; + xQueueReceive(audio_encode_queue_, &pcm, portMAX_DELAY); // Encode audio data - opus_encoder_.Encode(data.data, data.size, [this](const void* data, size_t size) { + opus_encoder_.Encode(pcm, [this](const iovec opus) { std::lock_guard lock(mutex_); if (ws_client_ && ws_client_->IsConnected()) { - ws_client_->Send((const char*)data, size, true); + ws_client_->Send(opus.iov_base, opus.iov_len, true); } }); - free((void*)data.data); + + free(pcm.iov_base); } } @@ -342,7 +345,7 @@ void Application::AudioDecodeTask() { int ret = opus_decode(opus_decoder_, packet->opus.data(), packet->opus.size(), packet->pcm.data(), frame_size, 0); if (ret < 0) { ESP_LOGE(TAG, "Failed to decode audio, error code: %d", ret); - free(packet); + delete packet; continue; } diff --git a/main/Application.h b/main/Application.h index 9ab59ea2..922e0e91 100644 --- a/main/Application.h +++ b/main/Application.h @@ -19,10 +19,6 @@ #define DETECTION_RUNNING 1 #define COMMUNICATION_RUNNING 2 -struct AudioEncoderData { - const void* data; - size_t size; -}; enum ChatState { kChatStateIdle, diff --git a/main/AudioDevice.cc b/main/AudioDevice.cc index 6d7d9dcf..91a97231 100644 --- a/main/AudioDevice.cc +++ b/main/AudioDevice.cc @@ -3,7 +3,6 @@ #include #define TAG "AudioDevice" -#define SPEAKING BIT0 AudioDevice::AudioDevice() { audio_play_queue_ = xQueueCreate(100, sizeof(AudioPacket*)); @@ -214,6 +213,7 @@ void AudioDevice::AudioPlayTask() { delete p; } breaked_ = false; + playing_ = false; } break; case kAudioPacketTypeData: diff --git a/main/BuiltinLed.cc b/main/BuiltinLed.cc index 4933b63c..dcb164a4 100644 --- a/main/BuiltinLed.cc +++ b/main/BuiltinLed.cc @@ -83,7 +83,7 @@ void BuiltinLed::Blink(int times, int interval_ms) { delete args; this_->blink_task_ = nullptr; vTaskDelete(NULL); - }, "blink", 4096, args, tskIDLE_PRIORITY, &blink_task_); + }, "blink", 1024, args, tskIDLE_PRIORITY, &blink_task_); xSemaphoreGive(mutex_); } diff --git a/main/OpusEncoder.cc b/main/OpusEncoder.cc index 87d4cce0..af65d432 100644 --- a/main/OpusEncoder.cc +++ b/main/OpusEncoder.cc @@ -8,10 +8,6 @@ OpusEncoder::OpusEncoder() { } OpusEncoder::~OpusEncoder() { - if (out_buffer_ != nullptr) { - free(out_buffer_); - } - if (audio_enc_ != nullptr) { opus_encoder_destroy(audio_enc_); } @@ -22,10 +18,6 @@ void OpusEncoder::Configure(int sample_rate, int channels, int duration_ms) { opus_encoder_destroy(audio_enc_); audio_enc_ = nullptr; } - if (out_buffer_ != nullptr) { - free(out_buffer_); - out_buffer_ = nullptr; - } int error; audio_enc_ = opus_encoder_create(sample_rate, channels, OPUS_APPLICATION_VOIP, &error); @@ -40,30 +32,29 @@ void OpusEncoder::Configure(int sample_rate, int channels, int duration_ms) { opus_encoder_ctl(audio_enc_, OPUS_SET_COMPLEXITY(5)); frame_size_ = sample_rate / 1000 * duration_ms; - out_size_ = sample_rate * channels * sizeof(int16_t); - out_buffer_ = (uint8_t*)malloc(out_size_); - assert(out_buffer_ != nullptr); + out_buffer_.resize(sample_rate * channels * sizeof(int16_t)); } -void OpusEncoder::Encode(const void* pcm, size_t pcm_len, std::function handler) { +void OpusEncoder::Encode(const iovec pcm, std::function handler) { if (audio_enc_ == nullptr) { ESP_LOGE(TAG, "Audio encoder is not configured"); return; } - in_buffer_.append((const char*)pcm, pcm_len); + auto pcm_data = (int16_t*)pcm.iov_base; + in_buffer_.insert(in_buffer_.end(), pcm_data, pcm_data + pcm.iov_len / sizeof(int16_t)); - while (in_buffer_.size() >= frame_size_ * sizeof(int16_t)) { - auto ret = opus_encode(audio_enc_, (const opus_int16*)in_buffer_.data(), frame_size_, out_buffer_, out_size_); + while (in_buffer_.size() >= frame_size_) { + auto ret = opus_encode(audio_enc_, in_buffer_.data(), frame_size_, out_buffer_.data(), out_buffer_.size()); if (ret < 0) { ESP_LOGE(TAG, "Failed to encode audio, error code: %ld", ret); return; } if (handler != nullptr) { - handler(out_buffer_, ret); + handler(iovec { out_buffer_.data(), (size_t)ret }); } - in_buffer_.erase(0, frame_size_ * sizeof(int16_t)); + in_buffer_.erase(in_buffer_.begin(), in_buffer_.begin() + frame_size_); } } diff --git a/main/OpusEncoder.h b/main/OpusEncoder.h index 4e27071d..34cdd181 100644 --- a/main/OpusEncoder.h +++ b/main/OpusEncoder.h @@ -3,23 +3,27 @@ #include #include +#include +#include + +#include "lwip/sockets.h" #include "opus.h" + class OpusEncoder { public: OpusEncoder(); ~OpusEncoder(); void Configure(int sample_rate, int channels, int duration_ms = 60); - void Encode(const void* pcm, size_t pcm_len, std::function handler); + void Encode(const iovec pcm, std::function handler); bool IsBufferEmpty() const { return in_buffer_.empty(); } private: struct OpusEncoder* audio_enc_ = nullptr; int frame_size_; - int out_size_; - uint8_t* out_buffer_ = nullptr; - std::string in_buffer_; + std::vector out_buffer_; + std::vector in_buffer_; }; #endif // _OPUS_ENCODER_H_ diff --git a/main/WebSocketClient.cc b/main/WebSocketClient.cc index 5b8eeee2..b553f272 100644 --- a/main/WebSocketClient.cc +++ b/main/WebSocketClient.cc @@ -2,6 +2,7 @@ #include #include "freertos/task.h" #include "esp_log.h" +#include "esp_crt_bundle.h" #define TAG "WebSocket" @@ -13,6 +14,7 @@ WebSocketClient::WebSocketClient(bool auto_reconnect) { esp_websocket_client_config_t config = {}; config.task_prio = 1; config.disable_auto_reconnect = !auto_reconnect; + config.crt_bundle_attach = esp_crt_bundle_attach; client_ = esp_websocket_client_init(&config); assert(client_ != NULL); @@ -93,11 +95,11 @@ bool WebSocketClient::Connect(const char* uri) { return bits & WEBSOCKET_CONNECTED_BIT; } -void WebSocketClient::Send(const char* data, size_t len, bool binary) { +void WebSocketClient::Send(const void* data, size_t len, bool binary) { if (binary) { - esp_websocket_client_send_bin(client_, data, len, portMAX_DELAY); + esp_websocket_client_send_bin(client_, (const char*)data, len, portMAX_DELAY); } else { - esp_websocket_client_send_text(client_, data, len, portMAX_DELAY); + esp_websocket_client_send_text(client_, (const char*)data, len, portMAX_DELAY); } } diff --git a/main/WebSocketClient.h b/main/WebSocketClient.h index 3ac35d34..71372119 100644 --- a/main/WebSocketClient.h +++ b/main/WebSocketClient.h @@ -19,7 +19,7 @@ public: bool IsConnected() const; bool Connect(const char* uri); void Send(const std::string& data); - void Send(const char* data, size_t len, bool binary = false); + void Send(const void* data, size_t len, bool binary = false); void OnConnected(std::function callback); void OnDisconnected(std::function callback); diff --git a/sdkconfig.defaults b/sdkconfig.defaults index 38846b56..e7b9dadf 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -2,14 +2,13 @@ CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_PERF=y CONFIG_BOOTLOADER_LOG_LEVEL_NONE=y CONFIG_BOOTLOADER_SKIP_VALIDATE_ALWAYS=y -CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240=y +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y CONFIG_ESP32S3_INSTRUCTION_CACHE_32KB=y CONFIG_ESP32S3_DATA_CACHE_64KB=y CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y -CONFIG_FLASHMODE_QIO=y -CONFIG_ESP32S3_SPIRAM_SUPPORT=y +CONFIG_SPIRAM=y CONFIG_SPIRAM_MODE_OCT=y CONFIG_SPIRAM_SPEED_80M=y CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=4096 @@ -25,7 +24,6 @@ CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" CONFIG_PARTITION_TABLE_FILENAME="partitions.csv" CONFIG_PARTITION_TABLE_OFFSET=0x8000 -CONFIG_MODEL_IN_SPIFFS=y CONFIG_USE_WAKENET=y CONFIG_SR_WN_WN9_NIHAOXIAOZHI_TTS=y CONFIG_USE_MULTINET=n