devttl/xiaozhi-esp32
1
0
Fork 0
forked from xiaozhi/xiaozhi-esp32
Code Pull Requests Activity

camera 优化：在原有的RGB565处理下，容易超时改为JPEG格式 (#1029)

Browse Source 
* camera 优化

feat(camera): 修改摄像头配置为JPEG格式并优化图像处理逻辑

将摄像头输出格式从RGB565改为JPEG以提高传输效率，同时调整JPEG质量为10
重构预览图像处理逻辑，支持直接处理JPEG格式并自动转换为RGB565
优化Explain方法中的JPEG队列处理，减少内存分配和拷贝操作

* 修复代码缩进

调整代码缩进格式以提升可读性

* fix(esp32_camera): 修复RGB565格式预览图像的字节序问题

添加字节交换处理，将大端序转换为小端序，确保预览图像显示正确

* 使用旧的处理方式

* refactor(esp32_camera): 移除preview_buffer_直接使用preview_image_.data
This commit is contained in:
Dong Ning
2025-08-14 22:11:15 +08:00
committed by GitHub
parent 00dd89079b
commit d6b1414967
3 changed files with 78 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
main/boards/common/esp32_camera.cc
View File

@@ -65,6 +65,7 @@ Esp32Camera::Esp32Camera(const camera_config_t& config) {
ESP_LOGE(TAG, "Failed to allocate memory for preview image");
return;
}
}
Esp32Camera::~Esp32Camera() {
@@ -115,14 +116,27 @@ bool Esp32Camera::Capture() {
// 显示预览图片
auto display = Board::GetInstance().GetDisplay();
if (display != nullptr) {
auto src = (uint16_t*)fb_->buf;
auto dst = (uint16_t*)preview_image_.data;
size_t pixel_count = fb_->len / 2;
for (size_t i = 0; i < pixel_count; i++) {
// 交换每个16位字内的字节
dst[i] = __builtin_bswap16(src[i]);
// 检查摄像头输出格式
if (fb_->format == PIXFORMAT_JPEG) {
// JPEG格式需要先解码为RGB565
// 分配足够的缓冲区用于RGB565数据
bool converted = jpg2rgb565(fb_->buf, fb_->len, preview_image_.data, JPG_SCALE_NONE);
if (converted) {
display->SetPreviewImage(&preview_image_);
} else {
ESP_LOGE(TAG, "Failed to convert JPEG to RGB565 for preview");
}
} else {
// RGB565等格式需要进行字节交换处理
auto src = (uint16_t*)fb_->buf;
auto dst = (uint16_t*)preview_image_.data;
size_t pixel_count = fb_->len / 2;
for (size_t i = 0; i < pixel_count; i++) {
// 交换每个16位字内的字节
dst[i] = __builtin_bswap16(src[i]);
}
display->SetPreviewImage(&preview_image_);
}
display->SetPreviewImage(&preview_image_);
}
return true;
}
@@ -188,26 +202,56 @@ std::string Esp32Camera::Explain(const std::string& question) {
return "{\"success\": false, \"message\": \"Image explain URL or token is not set\"}";
}
// 创建局部的 JPEG 队列, 40 entries is about to store 512 * 40 = 20480 bytes of JPEG data
QueueHandle_t jpeg_queue = xQueueCreate(40, sizeof(JpegChunk));
if (jpeg_queue == nullptr) {
ESP_LOGE(TAG, "Failed to create JPEG queue");
return "{\"success\": false, \"message\": \"Failed to create JPEG queue\"}";
}
QueueHandle_t jpeg_queue = nullptr;
// If the format is not JPEG, we need to convert it to JPEG first
if (fb_->format != PIXFORMAT_JPEG) {
// 创建局部的 JPEG 队列, 40 entries is about to store 512 * 40 = 20480 bytes of JPEG data
jpeg_queue = xQueueCreate(40, sizeof(JpegChunk));
if (jpeg_queue == nullptr) {
ESP_LOGE(TAG, "Failed to create JPEG queue");
return "{\"success\": false, \"message\": \"Failed to create JPEG queue\"}";
}
// We spawn a thread to encode the image to JPEG
encoder_thread_ = std::thread([this, jpeg_queue]() {
frame2jpg_cb(fb_, 80, [](void* arg, size_t index, const void* data, size_t len) -> unsigned int {
auto jpeg_queue = (QueueHandle_t)arg;
JpegChunk chunk = {
.data = (uint8_t*)heap_caps_aligned_alloc(16, len, MALLOC_CAP_SPIRAM),
.len = len
};
memcpy(chunk.data, data, len);
// We spawn a thread to encode the image to JPEG
encoder_thread_ = std::thread([this, jpeg_queue]() {
frame2jpg_cb(fb_, 80, [](void* arg, size_t index, const void* data, size_t len) -> unsigned int {
auto jpeg_queue = (QueueHandle_t)arg;
JpegChunk chunk = {
.data = (uint8_t*)heap_caps_aligned_alloc(16, len, MALLOC_CAP_SPIRAM),
.len = len
};
if (chunk.data != nullptr) {
memcpy(chunk.data, data, len);
xQueueSend(jpeg_queue, &chunk, portMAX_DELAY);
}
return len;
}, jpeg_queue);
// Add a null chunk to indicate the end of the queue
JpegChunk null_chunk = { .data = nullptr, .len = 0 };
xQueueSend(jpeg_queue, &null_chunk, portMAX_DELAY);
});
} else {
// JPEG format, we can send it directly
// To keep the logic consistent, we still use a queue to send the data
jpeg_queue = xQueueCreate(2, sizeof(JpegChunk));
if (jpeg_queue == nullptr) {
ESP_LOGE(TAG, "Failed to create JPEG queue");
return "{\"success\": false, \"message\": \"Failed to create JPEG queue\"}";
}
JpegChunk chunk = {
.data = (uint8_t*)heap_caps_aligned_alloc(16, fb_->len, MALLOC_CAP_SPIRAM),
.len = fb_->len
};
if (chunk.data != nullptr) {
memcpy(chunk.data, fb_->buf, fb_->len);
xQueueSend(jpeg_queue, &chunk, portMAX_DELAY);
return len;
}, jpeg_queue);
});
} else {
ESP_LOGE(TAG, "Failed to allocate memory for JPEG chunk");
}
// Add a null chunk to indicate the end of the queue
JpegChunk null_chunk = { .data = nullptr, .len = 0 };
xQueueSend(jpeg_queue, &null_chunk, portMAX_DELAY);
}
auto network = Board::GetInstance().GetNetwork();
auto http = network->CreateHttp(3);
@@ -225,7 +269,9 @@ std::string Esp32Camera::Explain(const std::string& question) {
if (!http->Open("POST", explain_url_)) {
ESP_LOGE(TAG, "Failed to connect to explain URL");
// Clear the queue
encoder_thread_.join();
if (encoder_thread_.joinable()) {
encoder_thread_.join();
}
JpegChunk chunk;
while (xQueueReceive(jpeg_queue, &chunk, portMAX_DELAY) == pdPASS) {
if (chunk.data != nullptr) {
@@ -273,7 +319,9 @@ std::string Esp32Camera::Explain(const std::string& question) {
heap_caps_free(chunk.data);
}
// Wait for the encoder thread to finish
encoder_thread_.join();
if (encoder_thread_.joinable()) {
encoder_thread_.join();
}
// 清理队列
vQueueDelete(jpeg_queue);