feat: add snapshot mcp tool (#1196)

* use main task to execute tool calls

* feat: add snapshot mcp tool

* fix compiling errors

* 取消 audio input 的 pin core，core1留给显示，可能会对aec性能有影响

* update ml307 version

* remove v1 theme colors

main/display/lvgl_display/gif/lvgl_gif.cc

@@ -34,7 +34,7 @@ LvglGif::LvglGif(const lv_img_dsc_t* img_dsc)
     }
 
     loaded_ = true;
-    ESP_LOGI(TAG, "GIF loaded from image descriptor: %dx%d", gif_->width, gif_->height);
+    ESP_LOGD(TAG, "GIF loaded from image descriptor: %dx%d", gif_->width, gif_->height);
 }
 
 // Destructor
@@ -73,7 +73,7 @@ void LvglGif::Start() {
         // Render first frame
         NextFrame();
         
-        ESP_LOGI(TAG, "GIF animation started");
+        ESP_LOGD(TAG, "GIF animation started");
     }
 }
 
@@ -81,7 +81,7 @@ void LvglGif::Pause() {
     if (timer_) {
         playing_ = false;
         lv_timer_pause(timer_);
-        ESP_LOGI(TAG, "GIF animation paused");
+        ESP_LOGD(TAG, "GIF animation paused");
     }
 }
 
@@ -94,7 +94,7 @@ void LvglGif::Resume() {
     if (timer_) {
         playing_ = true;
         lv_timer_resume(timer_);
-        ESP_LOGI(TAG, "GIF animation resumed");
+        ESP_LOGD(TAG, "GIF animation resumed");
     }
 }
 
@@ -107,7 +107,7 @@ void LvglGif::Stop() {
     if (gif_) {
         gd_rewind(gif_);
         NextFrame();
-        ESP_LOGI(TAG, "GIF animation stopped and rewound");
+        ESP_LOGD(TAG, "GIF animation stopped and rewound");
     }
 }
 
@@ -173,7 +173,7 @@ void LvglGif::NextFrame() {
         if (timer_) {
             lv_timer_pause(timer_);
         }
-        ESP_LOGI(TAG, "GIF animation completed");
+        ESP_LOGD(TAG, "GIF animation completed");
     }
 
     // Render current frame

main/display/lvgl_display/lvgl_display.cc

@@ -4,6 +4,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <font_awesome.h>
+#include <img_converters.h>
 
 #include "lvgl_display.h"
 #include "board.h"
@@ -201,12 +202,7 @@ void LvglDisplay::UpdateStatusBar(bool update_all) {
     esp_pm_lock_release(pm_lock_);
 }
 
-void LvglDisplay::SetPreviewImage(const lv_img_dsc_t* image) {
-    // Do nothing but free the image
-    if (image != nullptr) {
-        heap_caps_free((void*)image->data);
-        heap_caps_free((void*)image);
-    }
+void LvglDisplay::SetPreviewImage(std::unique_ptr<LvglImage> image) {
 }
 
 void LvglDisplay::SetPowerSaveMode(bool on) {
@@ -218,3 +214,29 @@ void LvglDisplay::SetPowerSaveMode(bool on) {
         SetEmotion("neutral");
     }
 }
+
+bool LvglDisplay::SnapshotToJpeg(uint8_t*& jpeg_output_data, size_t& jpeg_output_data_size, int quality) {
+    DisplayLockGuard lock(this);
+
+    lv_obj_t* screen = lv_screen_active();
+    lv_draw_buf_t* draw_buffer = lv_snapshot_take(screen, LV_COLOR_FORMAT_RGB565);
+    if (draw_buffer == nullptr) {
+        return false;
+    }
+
+    // swap bytes
+    uint16_t* data = (uint16_t*)draw_buffer->data;
+    size_t pixel_count = draw_buffer->data_size / 2;
+    for (size_t i = 0; i < pixel_count; i++) {
+        data[i] = __builtin_bswap16(data[i]);
+    }
+
+    if (!fmt2jpg(draw_buffer->data, draw_buffer->data_size, draw_buffer->header.w, draw_buffer->header.h,
+        PIXFORMAT_RGB565, quality, &jpeg_output_data, &jpeg_output_data_size)) {
+        lv_draw_buf_destroy(draw_buffer);
+        return false;
+    }
+
+    lv_draw_buf_destroy(draw_buffer);
+    return true;
+}

main/display/lvgl_display/lvgl_display.h

@@ -2,6 +2,7 @@
 #define LVGL_DISPLAY_H
 
 #include "display.h"
+#include "lvgl_image.h"
 
 #include <lvgl.h>
 #include <esp_timer.h>
@@ -19,9 +20,10 @@ public:
     virtual void SetStatus(const char* status);
     virtual void ShowNotification(const char* notification, int duration_ms = 3000);
     virtual void ShowNotification(const std::string &notification, int duration_ms = 3000);
-    virtual void SetPreviewImage(const lv_img_dsc_t* image);
+    virtual void SetPreviewImage(std::unique_ptr<LvglImage> image);
     virtual void UpdateStatusBar(bool update_all = false);
     virtual void SetPowerSaveMode(bool on);
+    virtual bool SnapshotToJpeg(uint8_t*& jpeg_output_data, size_t& jpeg_output_size, int quality = 80);
 
 protected:
     esp_pm_lock_handle_t pm_lock_ = nullptr;

main/display/lvgl_display/lvgl_image.cc

@@ -2,19 +2,21 @@
 #include <cbin_font.h>
 
 #include <esp_log.h>
+#include <stdexcept>
 #include <cstring>
+#include <esp_heap_caps.h>
 
 #define TAG "LvglImage"
 
 
 LvglRawImage::LvglRawImage(void* data, size_t size) {
     bzero(&image_dsc_, sizeof(image_dsc_));
+    image_dsc_.data_size = size;
+    image_dsc_.data = static_cast<uint8_t*>(data);
     image_dsc_.header.magic = LV_IMAGE_HEADER_MAGIC;
     image_dsc_.header.cf = LV_COLOR_FORMAT_RAW_ALPHA;
     image_dsc_.header.w = 0;
     image_dsc_.header.h = 0;
-    image_dsc_.data_size = size;
-    image_dsc_.data = static_cast<uint8_t*>(data);
 }
 
 bool LvglRawImage::IsGif() const {
@@ -31,3 +33,32 @@ LvglCBinImage::~LvglCBinImage() {
         cbin_img_dsc_delete(image_dsc_);
     }
 }
+
+LvglAllocatedImage::LvglAllocatedImage(void* data, size_t size) {
+    bzero(&image_dsc_, sizeof(image_dsc_));
+    image_dsc_.data_size = size;
+    image_dsc_.data = static_cast<uint8_t*>(data);
+
+    if (lv_image_decoder_get_info(&image_dsc_, &image_dsc_.header) != LV_RESULT_OK) {
+        ESP_LOGE(TAG, "Failed to get image info, data: %p size: %u", data, size);
+        throw std::runtime_error("Failed to get image info");
+    }
+}
+
+LvglAllocatedImage::LvglAllocatedImage(void* data, size_t size, int width, int height, int stride, int color_format) {
+    bzero(&image_dsc_, sizeof(image_dsc_));
+    image_dsc_.data_size = size;
+    image_dsc_.data = static_cast<uint8_t*>(data);
+    image_dsc_.header.magic = LV_IMAGE_HEADER_MAGIC;
+    image_dsc_.header.cf = color_format;
+    image_dsc_.header.w = width;
+    image_dsc_.header.h = height;
+    image_dsc_.header.stride = stride;
+}
+
+LvglAllocatedImage::~LvglAllocatedImage() {
+    if (image_dsc_.data) {
+        heap_caps_free((void*)image_dsc_.data);
+        image_dsc_.data = nullptr;
+    }
+}

main/display/lvgl_display/lvgl_image.h

@@ -39,4 +39,15 @@ public:
 
 private:
     const lv_img_dsc_t* image_dsc_;
+};
+
+class LvglAllocatedImage : public LvglImage {
+public:
+    LvglAllocatedImage(void* data, size_t size);
+    LvglAllocatedImage(void* data, size_t size, int width, int height, int stride, int color_format);
+    virtual ~LvglAllocatedImage();
+    virtual const lv_img_dsc_t* image_dsc() const override { return &image_dsc_; }
+
+private:
+    lv_img_dsc_t image_dsc_;
 };