devttl/xiaozhi-esp32
1
0
Fork 0
forked from xiaozhi/xiaozhi-esp32
Code Pull Requests Activity

feat: support JPEG input (#1455)

Browse Source
This commit is contained in:
laride
2025-11-18 20:34:22 +08:00
committed by GitHub
parent 511349a7bd
commit 860d12a12c
7 changed files with 449 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
main/display/lvgl_display/jpg/image_to_jpeg.cpp
View File

@@ -426,6 +426,19 @@ static bool encode_with_esp_new_jpeg(const uint8_t* src, size_t src_len, uint16_
bool image_to_jpeg(uint8_t* src, size_t src_len, uint16_t width, uint16_t height, v4l2_pix_fmt_t format,
uint8_t quality, uint8_t** out, size_t* out_len) {
#ifdef CONFIG_XIAOZHI_CAMERA_ALLOW_JPEG_INPUT
if (format == V4L2_PIX_FMT_JPEG) {
uint8_t * out_data = (uint8_t*)heap_caps_malloc(src_len, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
if (!out_data) {
ESP_LOGE(TAG, "Failed to allocate memory for JPEG output");
return false;
}
memcpy(out_data, src, src_len);
*out = out_data;
*out_len = src_len;
return true;
}
#endif // CONFIG_XIAOZHI_CAMERA_ALLOW_JPEG_INPUT
#if CONFIG_XIAOZHI_ENABLE_HARDWARE_JPEG_ENCODER
if (encode_with_hw_jpeg(src, src_len, width, height, format, quality, out, out_len, NULL, NULL)) {
return true;
@@ -437,6 +450,13 @@ bool image_to_jpeg(uint8_t* src, size_t src_len, uint16_t width, uint16_t height
bool image_to_jpeg_cb(uint8_t* src, size_t src_len, uint16_t width, uint16_t height, v4l2_pix_fmt_t format,
uint8_t quality, jpg_out_cb cb, void* arg) {
#ifdef CONFIG_XIAOZHI_CAMERA_ALLOW_JPEG_INPUT
if (format == V4L2_PIX_FMT_JPEG) {
cb(arg, 0, src, src_len);
cb(arg, 1, nullptr, 0); // end signal
return true;
}
#endif // CONFIG_XIAOZHI_CAMERA_ALLOW_JPEG_INPUT
#if CONFIG_XIAOZHI_ENABLE_HARDWARE_JPEG_ENCODER
if (encode_with_hw_jpeg(src, src_len, width, height, format, quality, NULL, NULL, cb, arg)) {
return true;

264
main/display/lvgl_display/jpg/jpeg_to_image.c Normal file
View File

@@ -0,0 +1,264 @@
#include <esp_check.h>
#include <esp_err.h>
#include <esp_heap_caps.h>
#include <sys/param.h>
#include "esp_jpeg_common.h"
#include "esp_jpeg_dec.h"
#include "jpeg_to_image.h"
#ifdef CONFIG_XIAOZHI_ENABLE_CAMERA_DEBUG_MODE
#undef LOG_LOCAL_LEVEL
#define LOG_LOCAL_LEVEL MAX(CONFIG_LOG_DEFAULT_LEVEL, ESP_LOG_DEBUG)
#endif // CONFIG_XIAOZHI_ENABLE_CAMERA_DEBUG_MODE
#include <esp_log.h>
#ifdef CONFIG_XIAOZHI_ENABLE_HARDWARE_JPEG_DECODER
#include "driver/jpeg_decode.h"
#endif
#define TAG "jpeg_to_image"
static esp_err_t decode_with_new_jpeg(const uint8_t* src, size_t src_len, uint8_t** out, size_t* out_len, size_t* width,
size_t* height, size_t* stride) {
ESP_LOGD(TAG, "Decoding JPEG with software decoder");
esp_err_t ret = ESP_OK;
jpeg_error_t jpeg_ret = JPEG_ERR_OK;
uint8_t* out_buf = NULL;
jpeg_dec_io_t jpeg_io = {0};
jpeg_dec_header_info_t out_info = {0};
jpeg_dec_config_t config = DEFAULT_JPEG_DEC_CONFIG();
config.output_type = JPEG_PIXEL_FORMAT_RGB565_LE;
config.rotate = JPEG_ROTATE_0D;
jpeg_dec_handle_t jpeg_dec = NULL;
jpeg_ret = jpeg_dec_open(&config, &jpeg_dec);
if (jpeg_ret != JPEG_ERR_OK) {
ESP_LOGE(TAG, "Failed to open JPEG decoder");
ret = ESP_FAIL;
goto jpeg_dec_failed;
}
jpeg_io.inbuf = (uint8_t*)src;
jpeg_io.inbuf_len = (int)src_len;
jpeg_ret = jpeg_dec_parse_header(jpeg_dec, &jpeg_io, &out_info);
if (jpeg_ret != JPEG_ERR_OK) {
ESP_LOGE(TAG, "Failed to parse JPEG header");
ret = ESP_ERR_INVALID_ARG;
goto jpeg_dec_failed;
}
ESP_LOGD(TAG, "JPEG header info: width=%d, height=%d", out_info.width, out_info.height);
out_buf = jpeg_calloc_align(out_info.width * out_info.height * 2, 16);
if (out_buf == NULL) {
ESP_LOGE(TAG, "Failed to allocate memory for JPEG output buffer");
ret = ESP_ERR_NO_MEM;
goto jpeg_dec_failed;
}
jpeg_io.outbuf = out_buf;
jpeg_ret = jpeg_dec_process(jpeg_dec, &jpeg_io);
if (jpeg_ret != JPEG_ERR_OK) {
ESP_LOGE(TAG, "Failed to decode JPEG");
ret = ESP_FAIL;
goto jpeg_dec_failed;
}
ESP_LOG_BUFFER_HEXDUMP(TAG, out_buf, MIN(out_info.width * out_info.height * 2, 256), ESP_LOG_DEBUG);
*out = out_buf;
out_buf = NULL;
*out_len = (size_t)(out_info.width * out_info.height * 2);
*width = (size_t)out_info.width;
*height = (size_t)out_info.height;
*stride = (size_t)out_info.width * 2;
jpeg_dec_close(jpeg_dec);
jpeg_dec = NULL;
return ret;
jpeg_dec_failed:
if (jpeg_dec) {
jpeg_dec_close(jpeg_dec);
jpeg_dec = NULL;
}
if (out_buf) {
jpeg_free_align(out_buf);
out_buf = NULL;
}
*out = NULL;
*out_len = 0;
*width = 0;
*height = 0;
*stride = 0;
return ret;
}
#ifdef CONFIG_XIAOZHI_ENABLE_HARDWARE_JPEG_DECODER
static esp_err_t decode_with_hardware_jpeg(const uint8_t* src, size_t src_len, uint8_t** out, size_t* out_len,
size_t* width, size_t* height, size_t* stride) {
ESP_LOGD(TAG, "Decoding JPEG with hardware decoder");
esp_err_t ret = ESP_OK;
jpeg_decoder_handle_t jpeg_dec = NULL;
uint8_t* bit_stream = NULL;
uint8_t* out_buf = NULL;
size_t out_buf_len = 0;
size_t tx_buffer_size = 0;
size_t rx_buffer_size = 0;
jpeg_decode_engine_cfg_t eng_cfg = {
.intr_priority = 1,
.timeout_ms = 1000,
};
jpeg_decode_cfg_t decode_cfg_rgb = {
.output_format = JPEG_DECODE_OUT_FORMAT_RGB565,
.rgb_order = JPEG_DEC_RGB_ELEMENT_ORDER_BGR,
};
ret = jpeg_new_decoder_engine(&eng_cfg, &jpeg_dec);
if (ret != ESP_OK) {
ESP_LOGE(TAG, "Failed to create JPEG decoder engine");
goto jpeg_hw_dec_failed;
}
jpeg_decode_memory_alloc_cfg_t tx_mem_cfg = {
.buffer_direction = JPEG_DEC_ALLOC_INPUT_BUFFER,
};
jpeg_decode_memory_alloc_cfg_t rx_mem_cfg = {
.buffer_direction = JPEG_DEC_ALLOC_OUTPUT_BUFFER,
};
bit_stream = (uint8_t*)jpeg_alloc_decoder_mem(src_len, &tx_mem_cfg, &tx_buffer_size);
if (bit_stream == NULL || tx_buffer_size < src_len) {
ESP_LOGE(TAG, "Failed to allocate memory for JPEG bit stream");
ret = ESP_ERR_NO_MEM;
goto jpeg_hw_dec_failed;
}
memcpy(bit_stream, src, src_len);
jpeg_decode_picture_info_t header_info;
ESP_GOTO_ON_ERROR(jpeg_decoder_get_info(bit_stream, src_len, &header_info), jpeg_hw_dec_failed, TAG,
"Failed to get JPEG header info");
ESP_LOGD(TAG, "JPEG header info: width=%d, height=%d, sample_method=%d", header_info.width, header_info.height,
(int)header_info.sample_method);
switch (header_info.sample_method) {
case JPEG_DOWN_SAMPLING_GRAY:
case JPEG_DOWN_SAMPLING_YUV444:
out_buf_len = header_info.width * header_info.height * 2;
*stride = header_info.width * 2;
break;
case JPEG_DOWN_SAMPLING_YUV422:
case JPEG_DOWN_SAMPLING_YUV420:
out_buf_len = ((header_info.width + 15) & ~15) * ((header_info.height + 15) & ~15) * 2;
*stride = ((header_info.width + 15) & ~15) * 2;
break;
default:
ESP_LOGE(TAG, "Unsupported JPEG sample method");
ret = ESP_ERR_NOT_SUPPORTED;
goto jpeg_hw_dec_failed;
}
out_buf = (uint8_t*)jpeg_alloc_decoder_mem(out_buf_len, &rx_mem_cfg, &rx_buffer_size);
if (out_buf == NULL || rx_buffer_size < out_buf_len) {
ESP_LOGE(TAG, "Failed to allocate memory for JPEG output buffer");
ret = ESP_ERR_NO_MEM;
goto jpeg_hw_dec_failed;
}
uint32_t out_size = 0;
ESP_GOTO_ON_ERROR(
jpeg_decoder_process(jpeg_dec, &decode_cfg_rgb, bit_stream, src_len, out_buf, out_buf_len, &out_size),
jpeg_hw_dec_failed, TAG, "Failed to decode JPEG");
ESP_LOGD(TAG, "Expected %d bytes, got %" PRIu32 " bytes", out_buf_len, out_size);
if (out_size != out_buf_len) {
ESP_LOGE(TAG, "Decoded image size mismatch: Expected %zu bytes, got %" PRIu32 " bytes", out_buf_len, out_size);
ret = ESP_ERR_INVALID_SIZE;
goto jpeg_hw_dec_failed;
}
if (header_info.sample_method == JPEG_DOWN_SAMPLING_GRAY) {
// convert GRAY8 to RGB565
uint32_t i = header_info.width * header_info.height;
do {
--i;
uint8_t r = (out_buf[i] >> 3) & 0x1F;
uint8_t g = (out_buf[i] >> 2) & 0x3F;
// b is same as r
uint16_t rgb565 = (r << 11) | (g << 5) | r;
out_buf[2 * i + 1] = (rgb565 >> 8) & 0xFF;
out_buf[2 * i] = rgb565 & 0xFF;
} while (i != 0);
out_size = header_info.width * header_info.height * 2;
ESP_LOGD(TAG, "Converted GRAY8 to RGB565, new size: %zu", out_size);
}
ESP_LOG_BUFFER_HEXDUMP(TAG, out_buf, MIN(out_size, 256), ESP_LOG_DEBUG);
*out = out_buf;
out_buf = NULL;
*out_len = (size_t)out_size;
jpeg_del_decoder_engine(jpeg_dec);
jpeg_dec = NULL;
heap_caps_free(bit_stream);
bit_stream = NULL;
*width = header_info.width;
*height = header_info.height;
return ret;
jpeg_hw_dec_failed:
if (out_buf) {
heap_caps_free(out_buf);
out_buf = NULL;
}
if (bit_stream) {
heap_caps_free(bit_stream);
bit_stream = NULL;
}
if (jpeg_dec) {
jpeg_del_decoder_engine(jpeg_dec);
jpeg_dec = NULL;
}
*out = NULL;
*out_len = 0;
*width = 0;
*height = 0;
*stride = 0;
return ret;
}
#endif // CONFIG_XIAOZHI_ENABLE_HARDWARE_JPEG_DECODER
esp_err_t jpeg_to_image(const uint8_t* src, size_t src_len, uint8_t** out, size_t* out_len, size_t* width,
size_t* height, size_t* stride) {
#ifdef CONFIG_XIAOZHI_ENABLE_CAMERA_DEBUG_MODE
esp_log_level_set(TAG, ESP_LOG_DEBUG);
#endif // CONFIG_XIAOZHI_ENABLE_CAMERA_DEBUG_MODE
if (src == NULL || src_len == 0 || out == NULL || out_len == NULL || width == NULL || height == NULL ||
stride == NULL) {
ESP_LOGE(TAG, "Invalid parameters");
return ESP_ERR_INVALID_ARG;
}
#ifdef CONFIG_XIAOZHI_ENABLE_HARDWARE_JPEG_DECODER
esp_err_t ret = decode_with_hardware_jpeg(src, src_len, out, out_len, width, height, stride);
if (ret == ESP_OK) {
return ret;
}
ESP_LOGW(TAG, "Failed to decode with hardware JPEG, fallback to software decoder");
// Fallback to esp_new_jpeg
#endif
return decode_with_new_jpeg(src, src_len, out, out_len, width, height, stride);
}

62
main/display/lvgl_display/jpg/jpeg_to_image.h Normal file
View File

@@ -0,0 +1,62 @@
#include "sdkconfig.h"
#ifndef CONFIG_IDF_TARGET_ESP32
#include <esp_err.h>
#ifdef __cplusplus
extern "C" {
#endif
/**
* @brief Decodes a JPEG image from memory to raw RGB565 pixel data
*
* This function attempts to decode a JPEG image using hardware acceleration first (if enabled),
* falling back to a software decoder if hardware decoding fails or is unavailable.
*
* @param[in] src Pointer to the JPEG bitstream in memory
* @param[in] src_len Length of the JPEG bitstream in bytes
* @param[out] out Pointer to a buffer pointer that will be set to the decoded image data.
* This buffer is allocated internally and MUST be freed by the caller using heap_caps_free().
* @param[out] out_len Pointer to a variable that will receive the size of the decoded image data in bytes
* @param[out] width Pointer to a variable that will receive the image width in pixels
* @param[out] height Pointer to a variable that will receive the image height in pixels
* @param[out] stride Pointer to a variable that will receive the image stride in bytes
*
* @return ESP_OK on successful decoding
* @return ESP_ERR_INVALID_ARG on invalid parameters
* @return ESP_ERR_NO_MEM on memory allocation failure
* @return ESP_FAIL on failure
*
* @attention Memory Management for `*out`:
* - The function allocates memory for the decoded image internally
* - On success, the caller takes ownership of this memory and SHOULD free it using heap_caps_free()
* - On failure, `*out` is guaranteed to be NULL and no freeing is required
* - Example usage:
* @code{.c}
* uint8_t *image = NULL;
* size_t len, width, height;
* if (jpeg_to_image(jpeg_data, jpeg_len, &image, &len, &width, &height)) {
* // Use image data...
* heap_caps_free(image); // Critical: use heap_caps_free
* }
* @endcode
*
* @note Configuration dependency:
* - When CONFIG_XIAOZHI_ENABLE_HARDWARE_JPEG_DECODER is enabled, hardware acceleration is attempted first
* - Both hardware and software paths allocate memory that requires heap_caps_free() for deallocation
* - The decoded image format is always RGB565 (2 bytes per pixel)
*
* @note When using hardware decoder, the decoded image dimensions might be aligned up to 16-byte boundaries.
* For YUV420 or YUV422 compressed images, both width and height will be rounded up to the nearest multiple of 16.
* See details at
* <https://docs.espressif.com/projects/esp-idf/en/stable/esp32p4/api-reference/peripherals/jpeg.html#jpeg-decoder-engine>
*
*/
esp_err_t jpeg_to_image(const uint8_t* src, size_t src_len, uint8_t** out, size_t* out_len, size_t* width,
size_t* height, size_t* stride);
#ifdef __cplusplus
}
#endif
#endif // CONFIG_IDF_TARGET_ESP32