From ebbb2fa319b66610740cbd148c6c410c1ce571ce Mon Sep 17 00:00:00 2001 From: laride <198868291+laride@users.noreply.github.com> Date: Mon, 17 Nov 2025 15:20:37 +0800 Subject: [PATCH] fix: avoid SRAM overflow in software JPEG encode (#1441) --- main/boards/common/esp32_camera.cc | 15 ++- .../lvgl_display/jpg/image_to_jpeg.cpp | 109 ++++++++++++------ 2 files changed, 78 insertions(+), 46 deletions(-) diff --git a/main/boards/common/esp32_camera.cc b/main/boards/common/esp32_camera.cc index 4288cffc..f8994983 100644 --- a/main/boards/common/esp32_camera.cc +++ b/main/boards/common/esp32_camera.cc @@ -1,19 +1,21 @@ -#include "esp32_camera.h" #include #include #include #include #include -#include "board.h" -#include "display.h" + #include "esp_imgfx_color_convert.h" #include "esp_video_device.h" #include "esp_video_init.h" -#include "jpg/image_to_jpeg.h" #include "linux/videodev2.h" + +#include "esp32_camera.h" +#include "board.h" +#include "display.h" #include "lvgl_display.h" #include "mcp_server.h" #include "system_info.h" +#include "jpg/image_to_jpeg.h" #ifdef CONFIG_XIAOZHI_ENABLE_CAMERA_DEBUG_MODE #undef LOG_LOCAL_LEVEL @@ -193,12 +195,9 @@ Esp32Camera::Esp32Camera(const esp_video_init_config_t& config) { return 0; case V4L2_PIX_FMT_RGB565: return 1; - case V4L2_PIX_FMT_YUV420: #ifdef CONFIG_XIAOZHI_ENABLE_HARDWARE_JPEG_ENCODER +case V4L2_PIX_FMT_YUV420: // 软件 JPEG 编码器不支持 YUV420 格式 return 2; -#else - // 软件 JPEG 编码器不支持 YUV420 格式 - [[fallthrough]]; #endif // CONFIG_XIAOZHI_ENABLE_HARDWARE_JPEG_ENCODER case V4L2_PIX_FMT_GREY: case V4L2_PIX_FMT_YUV422P: diff --git a/main/display/lvgl_display/jpg/image_to_jpeg.cpp b/main/display/lvgl_display/jpg/image_to_jpeg.cpp index 8ea05e1a..bf38eff9 100644 --- a/main/display/lvgl_display/jpg/image_to_jpeg.cpp +++ b/main/display/lvgl_display/jpg/image_to_jpeg.cpp @@ -3,9 +3,12 @@ #include #include #include +#include #include "esp_jpeg_common.h" #include "esp_jpeg_enc.h" +#include "esp_imgfx_color_convert.h" + #if CONFIG_XIAOZHI_ENABLE_HARDWARE_JPEG_ENCODER #include "driver/jpeg_encode.h" #endif @@ -34,7 +37,7 @@ static __always_inline uint8_t expand_6_to_8(uint8_t v) { static uint8_t* convert_input_to_encoder_buf(const uint8_t* src, uint16_t width, uint16_t height, v4l2_pix_fmt_t format, jpeg_pixel_format_t* out_fmt, int* out_size) { - // 直接支持的格式：GRAY、RGB888、YCbYCr(YUYV) + // GRAY 直接作为 JPEG_PIXEL_FORMAT_GRAY 输入 if (format == V4L2_PIX_FMT_GREY) { int sz = (int)width * (int)height; uint8_t* buf = (uint8_t*)jpeg_calloc_align(sz, 16); @@ -63,7 +66,8 @@ static uint8_t* convert_input_to_encoder_buf(const uint8_t* src, uint16_t width, } // V4L2 UYVY (Cb Y Cr Y) -> 重排为 YUYV 再作为 YCbYCr 输入 - if (format == V4L2_PIX_FMT_UYVY) { + // 当前版本暂时不会出现 UYVY 格式 + if (format == V4L2_PIX_FMT_UYVY) [[unlikely]] { int sz = (int)width * (int)height * 2; const uint8_t* s = src; uint8_t* buf = (uint8_t*)jpeg_calloc_align(sz, 16); @@ -87,7 +91,8 @@ static uint8_t* convert_input_to_encoder_buf(const uint8_t* src, uint16_t width, } // V4L2 YUV422P (YUV422 Planar) -> 重排为 YUYV (YCbYCr) - if (format == V4L2_PIX_FMT_YUV422P) { + // 当前版本暂时不会出现 YUV422P 格式 + if (format == V4L2_PIX_FMT_YUV422P) [[unlikely]] { int sz = (int)width * (int)height * 2; const uint8_t* y_plane = src; const uint8_t* u_plane = y_plane + (int)width * (int)height; @@ -119,44 +124,72 @@ static uint8_t* convert_input_to_encoder_buf(const uint8_t* src, uint16_t width, return buf; } - // 其余格式转换为 RGB888 - int rgb_size = (int)width * (int)height * 3; - uint8_t* rgb = (uint8_t*)jpeg_calloc_align(rgb_size, 16); - if (!rgb) - return NULL; - - if (format == V4L2_PIX_FMT_RGB24) { - // V4L2_RGB24 即 RGB888 - memcpy(rgb, src, rgb_size); - } else if (format == V4L2_PIX_FMT_RGB565) { - // RGB565 小端，需要转换为 RGB888 - const uint8_t* p = src; - uint8_t* d = rgb; - int pixels = (int)width * (int)height; - for (int i = 0; i < pixels; i++) { - uint8_t lo = p[0]; // 低字节（LSB） - uint8_t hi = p[1]; // 高字节（MSB） - p += 2; - - uint8_t r5 = (hi >> 3) & 0x1F; - uint8_t g6 = ((hi & 0x07) << 3) | ((lo & 0xE0) >> 5); - uint8_t b5 = lo & 0x1F; - - d[0] = expand_5_to_8(r5); - d[1] = expand_6_to_8(g6); - d[2] = expand_5_to_8(b5); - d += 3; + // RGB 转换为 YUV422 (YCbYCr) 再输入 + // 见 https://github.com/78/xiaozhi-esp32/issues/1380#issuecomment-3497156378 + else if (format == V4L2_PIX_FMT_RGB24 || format == V4L2_PIX_FMT_RGB565 || format == V4L2_PIX_FMT_RGB565X) { + esp_imgfx_pixel_fmt_t in_pixel_fmt = ESP_IMGFX_PIXEL_FMT_RGB888; + uint32_t src_len = 0; + switch (format) { + case V4L2_PIX_FMT_RGB24: + in_pixel_fmt = ESP_IMGFX_PIXEL_FMT_RGB888; + src_len = static_cast(width * height * 3); + break; + case V4L2_PIX_FMT_RGB565: + in_pixel_fmt = ESP_IMGFX_PIXEL_FMT_RGB565_LE; + src_len = static_cast(width * height * 2); + break; + [[unlikely]] case V4L2_PIX_FMT_RGB565X: // 当前版本暂时不会出现 RGB565X + in_pixel_fmt = ESP_IMGFX_PIXEL_FMT_RGB565_BE; + src_len = static_cast(width * height * 2); + break; + [[unlikely]] default: + ESP_LOGE(TAG, "[Unreachable Case] unsupported format: 0x%08x", format); + std::unreachable(); } - } else { - // 其他未覆盖格式，清零 - memset(rgb, 0, rgb_size); + int sz = (int)width * (int)height * 2; + uint8_t* buf = (uint8_t*)jpeg_calloc_align(sz, 16); + if (!buf) + return nullptr; + esp_imgfx_color_convert_cfg_t convert_cfg = { + .in_res = {.width = static_cast(width), + .height = static_cast(height)}, + .in_pixel_fmt = in_pixel_fmt, + .out_pixel_fmt = ESP_IMGFX_PIXEL_FMT_YUYV, + .color_space_std = ESP_IMGFX_COLOR_SPACE_STD_BT601, + }; + esp_imgfx_color_convert_handle_t convert_handle = nullptr; + esp_imgfx_err_t err = esp_imgfx_color_convert_open(&convert_cfg, &convert_handle); + if (err != ESP_IMGFX_ERR_OK || convert_handle == nullptr) { + ESP_LOGE(TAG, "esp_imgfx_color_convert_open failed"); + jpeg_free_align(buf); + return nullptr; + } + esp_imgfx_data_t convert_input_data = { + .data = const_cast(src), + .data_len = static_cast(src_len), + }; + esp_imgfx_data_t convert_output_data = { + .data = buf, + .data_len = static_cast(sz), + }; + err = esp_imgfx_color_convert_process(convert_handle, &convert_input_data, &convert_output_data); + if (err != ESP_IMGFX_ERR_OK) { + ESP_LOGE(TAG, "esp_imgfx_color_convert_process failed"); + jpeg_free_align(buf); + return nullptr; + } + esp_imgfx_color_convert_close(convert_handle); + convert_handle = nullptr; + if (out_fmt) + *out_fmt = JPEG_PIXEL_FORMAT_YCbYCr; + if (out_size) + *out_size = sz; + return buf; } - - if (out_fmt) - *out_fmt = JPEG_PIXEL_FORMAT_RGB888; + ESP_LOGE(TAG, "unsupported format: 0x%08x", format); if (out_size) - *out_size = rgb_size; - return rgb; + *out_size = 0; + return nullptr; } #if CONFIG_XIAOZHI_ENABLE_HARDWARE_JPEG_ENCODER