feat: Add camera functions (lichuang-dev)

main/boards/common/board.cc

@@ -57,6 +57,10 @@ Display* Board::GetDisplay() {
     return &display;
 }
 
+Camera* Board::GetCamera() {
+    return nullptr;
+}
+
 Led* Board::GetLed() {
     static NoLed led;
     return &led;

main/boards/common/board.h

@@ -9,6 +9,7 @@
 
 #include "led/led.h"
 #include "backlight.h"
+#include "camera.h"
 
 void* create_board();
 class AudioCodec;
@@ -39,6 +40,7 @@ public:
     virtual AudioCodec* GetAudioCodec() = 0;
     virtual bool GetTemperature(float& esp32temp);
     virtual Display* GetDisplay();
+    virtual Camera* GetCamera();
     virtual Http* CreateHttp() = 0;
     virtual WebSocket* CreateWebSocket() = 0;
     virtual Mqtt* CreateMqtt() = 0;

main/boards/common/camera.h

@@ -0,0 +1,13 @@
+#ifndef CAMERA_H
+#define CAMERA_H
+
+#include <string>
+
+class Camera {
+public:
+    virtual void SetExplainUrl(const std::string& url, const std::string& token) = 0;
+    virtual bool Capture() = 0;
+    virtual std::string Explain(const std::string& question) = 0;
+};
+
+#endif // CAMERA_H

main/boards/common/esp32_camera.cc

@@ -0,0 +1,212 @@
+#include "esp32_camera.h"
+#include "mcp_server.h"
+#include "display.h"
+#include "board.h"
+
+#include <esp_log.h>
+#include <esp_heap_caps.h>
+#include <img_converters.h>
+#include <cstring>
+
+#define TAG "Esp32Camera"
+
+Esp32Camera::Esp32Camera(const camera_config_t& config) {
+    jpeg_queue_ = xQueueCreate(10, sizeof(JpegChunk));
+    if (jpeg_queue_ == nullptr) {
+        ESP_LOGE(TAG, "Failed to create JPEG queue");
+        return;
+    }
+
+    // camera init
+    esp_err_t err = esp_camera_init(&config); // 配置上面定义的参数
+    if (err != ESP_OK) {
+        ESP_LOGE(TAG, "Camera init failed with error 0x%x", err);
+        return;
+    }
+
+    sensor_t *s = esp_camera_sensor_get(); // 获取摄像头型号
+    if (s->id.PID == GC0308_PID) {
+        s->set_hmirror(s, 0);  // 这里控制摄像头镜像 写1镜像 写0不镜像
+    }
+
+    // 初始化预览图片的内存
+    memset(&preview_image_, 0, sizeof(preview_image_));
+    preview_image_.header.magic = LV_IMAGE_HEADER_MAGIC;
+    preview_image_.header.cf = LV_COLOR_FORMAT_RGB565;
+    preview_image_.header.flags = LV_IMAGE_FLAGS_ALLOCATED | LV_IMAGE_FLAGS_MODIFIABLE;
+
+    if (config.frame_size == FRAMESIZE_VGA) {
+        preview_image_.header.w = 640;
+        preview_image_.header.h = 480;
+    } else if (config.frame_size == FRAMESIZE_QVGA) {
+        preview_image_.header.w = 320;
+        preview_image_.header.h = 240;
+    }
+    preview_image_.header.stride = preview_image_.header.w * 2;
+    preview_image_.data_size = preview_image_.header.w * preview_image_.header.h * 2;
+    preview_image_.data = (uint8_t*)heap_caps_malloc(preview_image_.data_size, MALLOC_CAP_SPIRAM);
+    if (preview_image_.data == nullptr) {
+        ESP_LOGE(TAG, "Failed to allocate memory for preview image");
+        return;
+    }
+}
+
+Esp32Camera::~Esp32Camera() {
+    if (fb_) {
+        esp_camera_fb_return(fb_);
+        fb_ = nullptr;
+    }
+    if (preview_image_.data) {
+        heap_caps_free((void*)preview_image_.data);
+        preview_image_.data = nullptr;
+    }
+    esp_camera_deinit();
+
+    if (jpeg_queue_ != nullptr) {
+        vQueueDelete(jpeg_queue_);
+        jpeg_queue_ = nullptr;
+    }
+}
+
+void Esp32Camera::SetExplainUrl(const std::string& url, const std::string& token) {
+    explain_url_ = url;
+    explain_token_ = token;
+}
+
+bool Esp32Camera::Capture() {
+    if (preview_thread_.joinable()) {
+        preview_thread_.join();
+    }
+    if (encoder_thread_.joinable()) {
+        encoder_thread_.join();
+    }
+
+    int frames_to_get = 2;
+    // Try to get a stable frame
+    for (int i = 0; i < frames_to_get; i++) {
+        if (fb_ != nullptr) {
+            esp_camera_fb_return(fb_);
+        }
+        fb_ = esp_camera_fb_get();
+        if (fb_ == nullptr) {
+            ESP_LOGE(TAG, "Camera capture failed");
+            return false;
+        }
+    }
+
+    preview_thread_ = std::thread([this]() {
+        // 显示预览图片
+        auto display = Board::GetInstance().GetDisplay();
+        if (display != nullptr) {
+            auto src = (uint16_t*)fb_->buf;
+            auto dst = (uint16_t*)preview_image_.data;
+            size_t pixel_count = fb_->len / 2;
+            for (size_t i = 0; i < pixel_count; i++) {
+                // 交换每个16位字内的字节
+                dst[i] = __builtin_bswap16(src[i]);
+            }
+            display->SetPreviewImage(&preview_image_);
+        }
+    });
+    return true;
+}
+
+std::string Esp32Camera::Explain(const std::string& question) {
+    if (explain_url_.empty() || explain_token_.empty()) {
+        return "{\"success\": false, \"message\": \"Image explain URL or token is not set\"}";
+    }
+
+    // We spawn a thread to encode the image to JPEG
+    encoder_thread_ = std::thread([this]() {
+        frame2jpg_cb(fb_, 80, [](void* arg, size_t index, const void* data, size_t len) -> unsigned int {
+            auto jpeg_queue = (QueueHandle_t)arg;
+            JpegChunk chunk = {
+                .data = (uint8_t*)heap_caps_malloc(len, MALLOC_CAP_SPIRAM),
+                .len = len
+            };
+            memcpy(chunk.data, data, len);
+            xQueueSend(jpeg_queue, &chunk, portMAX_DELAY);
+            return len;
+        }, jpeg_queue_);
+    });
+
+    auto http = Board::GetInstance().CreateHttp();
+    // 构造multipart/form-data请求体
+    std::string boundary = "----ESP32_CAMERA_BOUNDARY";
+    
+    // 构造question字段
+    std::string question_field;
+    question_field += "--" + boundary + "\r\n";
+    question_field += "Content-Disposition: form-data; name=\"question\"\r\n";
+    question_field += "\r\n";
+    question_field += question + "\r\n";
+    
+    // 构造文件字段头部
+    std::string file_header;
+    file_header += "--" + boundary + "\r\n";
+    file_header += "Content-Disposition: form-data; name=\"file\"; filename=\"camera.jpg\"\r\n";
+    file_header += "Content-Type: image/jpeg\r\n";
+    file_header += "\r\n";
+    
+    // 构造尾部
+    std::string multipart_footer;
+    multipart_footer += "\r\n--" + boundary + "--\r\n";
+
+    // 配置HTTP客户端，使用分块传输编码
+    http->SetHeader("Authorization", "Bearer " + explain_token_);
+    http->SetHeader("Content-Type", "multipart/form-data; boundary=" + boundary);
+    http->SetHeader("Transfer-Encoding", "chunked");
+    if (!http->Open("POST", explain_url_)) {
+        ESP_LOGE(TAG, "Failed to connect to explain URL");
+        // Clear the queue
+        encoder_thread_.join();
+        JpegChunk chunk;
+        while (xQueueReceive(jpeg_queue_, &chunk, portMAX_DELAY) == pdPASS) {
+            if (chunk.data != nullptr) {
+                heap_caps_free(chunk.data);
+            } else {
+                break;
+            }
+        }
+        return "{\"success\": false, \"message\": \"Failed to connect to explain URL\"}";
+    }
+    
+    // 第一块：question字段
+    http->Write(question_field.c_str(), question_field.size());
+    
+    // 第二块：文件字段头部
+    http->Write(file_header.c_str(), file_header.size());
+    
+    // 第三块：JPEG数据
+    size_t total_sent = 0;
+    while (true) {
+        JpegChunk chunk;
+        if (xQueueReceive(jpeg_queue_, &chunk, portMAX_DELAY) != pdPASS) {
+            ESP_LOGE(TAG, "Failed to receive JPEG chunk");
+            break;
+        }
+        if (chunk.data == nullptr) {
+            break; // The last chunk
+        }
+        http->Write((const char*)chunk.data, chunk.len);
+        total_sent += chunk.len;
+        heap_caps_free(chunk.data);
+    }
+
+    // 第四块：multipart尾部
+    http->Write(multipart_footer.c_str(), multipart_footer.size());
+    
+    // 结束块
+    http->Write("", 0);
+
+    if (http->GetStatusCode() != 200) {
+        ESP_LOGE(TAG, "Failed to upload photo, status code: %d", http->GetStatusCode());
+        return "{\"success\": false, \"message\": \"Failed to upload photo\"}";
+    }
+
+    std::string result = http->ReadAll();
+    http->Close();
+
+    ESP_LOGI(TAG, "Explain image size=%dx%d, compressed size=%d, question=%s\n%s", fb_->width, fb_->height, total_sent, question.c_str(), result.c_str());
+    return result;
+}

main/boards/common/esp32_camera.h

@@ -0,0 +1,38 @@
+#ifndef ESP32_CAMERA_H
+#define ESP32_CAMERA_H
+
+#include <esp_camera.h>
+#include <lvgl.h>
+#include <thread>
+#include <memory>
+
+#include <freertos/FreeRTOS.h>
+#include <freertos/queue.h>
+
+#include "camera.h"
+
+struct JpegChunk {
+    uint8_t* data;
+    size_t len;
+};
+
+class Esp32Camera : public Camera {
+private:
+    camera_fb_t* fb_ = nullptr;
+    lv_img_dsc_t preview_image_;
+    std::string explain_url_;
+    std::string explain_token_;
+    std::thread preview_thread_;
+    std::thread encoder_thread_;
+
+    QueueHandle_t jpeg_queue_ = nullptr;
+public:
+    Esp32Camera(const camera_config_t& config);
+    ~Esp32Camera();
+
+    virtual void SetExplainUrl(const std::string& url, const std::string& token);
+    virtual bool Capture();
+    virtual std::string Explain(const std::string& question);
+};
+
+#endif // ESP32_CAMERA_H