From d6b141496738992389adc802291551cf294047d1 Mon Sep 17 00:00:00 2001 From: Dong Ning Date: Thu, 14 Aug 2025 22:11:15 +0800 Subject: [PATCH] =?UTF-8?q?camera=20=E4=BC=98=E5=8C=96=EF=BC=9A=E5=9C=A8?= =?UTF-8?q?=E5=8E=9F=E6=9C=89=E7=9A=84RGB565=E5=A4=84=E7=90=86=E4=B8=8B?= =?UTF-8?q?=EF=BC=8C=E5=AE=B9=E6=98=93=E8=B6=85=E6=97=B6=E6=94=B9=E4=B8=BA?= =?UTF-8?q?JPEG=E6=A0=BC=E5=BC=8F=20(#1029)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * camera 优化 feat(camera): 修改摄像头配置为JPEG格式并优化图像处理逻辑 将摄像头输出格式从RGB565改为JPEG以提高传输效率，同时调整JPEG质量为10 重构预览图像处理逻辑，支持直接处理JPEG格式并自动转换为RGB565 优化Explain方法中的JPEG队列处理，减少内存分配和拷贝操作 * 修复代码缩进 调整代码缩进格式以提升可读性 * fix(esp32_camera): 修复RGB565格式预览图像的字节序问题 添加字节交换处理，将大端序转换为小端序，确保预览图像显示正确 * 使用旧的处理方式 * refactor(esp32_camera): 移除preview_buffer_直接使用preview_image_.data --- .../compact_wifi_board_s3cam.cc | 4 +- main/boards/bread-compact-wifi-s3cam/config.h | 2 +- main/boards/common/esp32_camera.cc | 102 +++++++++++++----- 3 files changed, 78 insertions(+), 30 deletions(-) diff --git a/main/boards/bread-compact-wifi-s3cam/compact_wifi_board_s3cam.cc b/main/boards/bread-compact-wifi-s3cam/compact_wifi_board_s3cam.cc index 45115408..416e1f21 100644 --- a/main/boards/bread-compact-wifi-s3cam/compact_wifi_board_s3cam.cc +++ b/main/boards/bread-compact-wifi-s3cam/compact_wifi_board_s3cam.cc @@ -158,9 +158,9 @@ private: config.pin_pwdn = CAMERA_PIN_PWDN; config.pin_reset = CAMERA_PIN_RESET; config.xclk_freq_hz = XCLK_FREQ_HZ; - config.pixel_format = PIXFORMAT_RGB565; + config.pixel_format = PIXFORMAT_JPEG; config.frame_size = FRAMESIZE_QVGA; - config.jpeg_quality = 12; + config.jpeg_quality = 10; config.fb_count = 1; config.fb_location = CAMERA_FB_IN_PSRAM; config.grab_mode = CAMERA_GRAB_WHEN_EMPTY; diff --git a/main/boards/bread-compact-wifi-s3cam/config.h b/main/boards/bread-compact-wifi-s3cam/config.h index 857d8c08..b0bea05f 100644 --- a/main/boards/bread-compact-wifi-s3cam/config.h +++ b/main/boards/bread-compact-wifi-s3cam/config.h @@ -51,7 +51,7 @@ #define CAMERA_PIN_SIOD GPIO_NUM_4 #define CAMERA_PIN_PWDN GPIO_NUM_NC #define CAMERA_PIN_RESET GPIO_NUM_NC -#define XCLK_FREQ_HZ 20000000 +#define XCLK_FREQ_HZ 50000000 #define DISPLAY_BACKLIGHT_PIN GPIO_NUM_38 diff --git a/main/boards/common/esp32_camera.cc b/main/boards/common/esp32_camera.cc index 40d02edd..f3c37f9d 100644 --- a/main/boards/common/esp32_camera.cc +++ b/main/boards/common/esp32_camera.cc @@ -65,6 +65,7 @@ Esp32Camera::Esp32Camera(const camera_config_t& config) { ESP_LOGE(TAG, "Failed to allocate memory for preview image"); return; } + } Esp32Camera::~Esp32Camera() { @@ -115,14 +116,27 @@ bool Esp32Camera::Capture() { // 显示预览图片 auto display = Board::GetInstance().GetDisplay(); if (display != nullptr) { - auto src = (uint16_t*)fb_->buf; - auto dst = (uint16_t*)preview_image_.data; - size_t pixel_count = fb_->len / 2; - for (size_t i = 0; i < pixel_count; i++) { - // 交换每个16位字内的字节 - dst[i] = __builtin_bswap16(src[i]); + // 检查摄像头输出格式 + if (fb_->format == PIXFORMAT_JPEG) { + // JPEG格式需要先解码为RGB565 + // 分配足够的缓冲区用于RGB565数据 + bool converted = jpg2rgb565(fb_->buf, fb_->len, preview_image_.data, JPG_SCALE_NONE); + if (converted) { + display->SetPreviewImage(&preview_image_); + } else { + ESP_LOGE(TAG, "Failed to convert JPEG to RGB565 for preview"); + } + } else { + // RGB565等格式需要进行字节交换处理 + auto src = (uint16_t*)fb_->buf; + auto dst = (uint16_t*)preview_image_.data; + size_t pixel_count = fb_->len / 2; + for (size_t i = 0; i < pixel_count; i++) { + // 交换每个16位字内的字节 + dst[i] = __builtin_bswap16(src[i]); + } + display->SetPreviewImage(&preview_image_); } - display->SetPreviewImage(&preview_image_); } return true; } @@ -188,26 +202,56 @@ std::string Esp32Camera::Explain(const std::string& question) { return "{\"success\": false, \"message\": \"Image explain URL or token is not set\"}"; } - // 创建局部的 JPEG 队列, 40 entries is about to store 512 * 40 = 20480 bytes of JPEG data - QueueHandle_t jpeg_queue = xQueueCreate(40, sizeof(JpegChunk)); - if (jpeg_queue == nullptr) { - ESP_LOGE(TAG, "Failed to create JPEG queue"); - return "{\"success\": false, \"message\": \"Failed to create JPEG queue\"}"; - } + QueueHandle_t jpeg_queue = nullptr; + // If the format is not JPEG, we need to convert it to JPEG first + if (fb_->format != PIXFORMAT_JPEG) { + // 创建局部的 JPEG 队列, 40 entries is about to store 512 * 40 = 20480 bytes of JPEG data + jpeg_queue = xQueueCreate(40, sizeof(JpegChunk)); + if (jpeg_queue == nullptr) { + ESP_LOGE(TAG, "Failed to create JPEG queue"); + return "{\"success\": false, \"message\": \"Failed to create JPEG queue\"}"; + } - // We spawn a thread to encode the image to JPEG - encoder_thread_ = std::thread([this, jpeg_queue]() { - frame2jpg_cb(fb_, 80, [](void* arg, size_t index, const void* data, size_t len) -> unsigned int { - auto jpeg_queue = (QueueHandle_t)arg; - JpegChunk chunk = { - .data = (uint8_t*)heap_caps_aligned_alloc(16, len, MALLOC_CAP_SPIRAM), - .len = len - }; - memcpy(chunk.data, data, len); + // We spawn a thread to encode the image to JPEG + encoder_thread_ = std::thread([this, jpeg_queue]() { + frame2jpg_cb(fb_, 80, [](void* arg, size_t index, const void* data, size_t len) -> unsigned int { + auto jpeg_queue = (QueueHandle_t)arg; + JpegChunk chunk = { + .data = (uint8_t*)heap_caps_aligned_alloc(16, len, MALLOC_CAP_SPIRAM), + .len = len + }; + if (chunk.data != nullptr) { + memcpy(chunk.data, data, len); + xQueueSend(jpeg_queue, &chunk, portMAX_DELAY); + } + return len; + }, jpeg_queue); + // Add a null chunk to indicate the end of the queue + JpegChunk null_chunk = { .data = nullptr, .len = 0 }; + xQueueSend(jpeg_queue, &null_chunk, portMAX_DELAY); + }); + } else { + // JPEG format, we can send it directly + // To keep the logic consistent, we still use a queue to send the data + jpeg_queue = xQueueCreate(2, sizeof(JpegChunk)); + if (jpeg_queue == nullptr) { + ESP_LOGE(TAG, "Failed to create JPEG queue"); + return "{\"success\": false, \"message\": \"Failed to create JPEG queue\"}"; + } + JpegChunk chunk = { + .data = (uint8_t*)heap_caps_aligned_alloc(16, fb_->len, MALLOC_CAP_SPIRAM), + .len = fb_->len + }; + if (chunk.data != nullptr) { + memcpy(chunk.data, fb_->buf, fb_->len); xQueueSend(jpeg_queue, &chunk, portMAX_DELAY); - return len; - }, jpeg_queue); - }); + } else { + ESP_LOGE(TAG, "Failed to allocate memory for JPEG chunk"); + } + // Add a null chunk to indicate the end of the queue + JpegChunk null_chunk = { .data = nullptr, .len = 0 }; + xQueueSend(jpeg_queue, &null_chunk, portMAX_DELAY); + } auto network = Board::GetInstance().GetNetwork(); auto http = network->CreateHttp(3); @@ -225,7 +269,9 @@ std::string Esp32Camera::Explain(const std::string& question) { if (!http->Open("POST", explain_url_)) { ESP_LOGE(TAG, "Failed to connect to explain URL"); // Clear the queue - encoder_thread_.join(); + if (encoder_thread_.joinable()) { + encoder_thread_.join(); + } JpegChunk chunk; while (xQueueReceive(jpeg_queue, &chunk, portMAX_DELAY) == pdPASS) { if (chunk.data != nullptr) { @@ -273,7 +319,9 @@ std::string Esp32Camera::Explain(const std::string& question) { heap_caps_free(chunk.data); } // Wait for the encoder thread to finish - encoder_thread_.join(); + if (encoder_thread_.joinable()) { + encoder_thread_.join(); + } // 清理队列 vQueueDelete(jpeg_queue);