diff --git a/main/application.cc b/main/application.cc index dce29f91..6373c445 100644 --- a/main/application.cc +++ b/main/application.cc @@ -540,6 +540,12 @@ void Application::Start() { // Play the success sound to indicate the device is ready audio_service_.PlaySound(Lang::Sounds::OGG_SUCCESS); } + + // Start the main event loop task with priority 3 + xTaskCreate([](void* arg) { + ((Application*)arg)->MainEventLoop(); + vTaskDelete(NULL); + }, "main_event_loop", 2048 * 4, this, 3, &main_event_loop_task_handle_); } // Add a async task to MainLoop @@ -555,9 +561,6 @@ void Application::Schedule(std::function callback) { // If other tasks need to access the websocket or chat state, // they should use Schedule to call this function void Application::MainEventLoop() { - // Raise the priority of the main event loop to avoid being interrupted by background tasks (which has priority 2) - vTaskPrioritySet(NULL, 3); - while (true) { auto bits = xEventGroupWaitBits(event_group_, MAIN_EVENT_SCHEDULE | MAIN_EVENT_SEND_AUDIO | @@ -832,11 +835,20 @@ bool Application::CanEnterSleepMode() { } void Application::SendMcpMessage(const std::string& payload) { - Schedule([this, payload]() { - if (protocol_) { + if (protocol_ == nullptr) { + return; + } + + // Make sure you are using main thread to send MCP message + if (xTaskGetCurrentTaskHandle() == main_event_loop_task_handle_) { + ESP_LOGI(TAG, "Send MCP message in main thread"); + protocol_->SendMcpMessage(payload); + } else { + ESP_LOGI(TAG, "Send MCP message in sub thread"); + Schedule([this, payload = std::move(payload)]() { protocol_->SendMcpMessage(payload); - } - }); + }); + } } void Application::SetAecMode(AecMode mode) { diff --git a/main/application.h b/main/application.h index 69ed22e4..bff4e9e9 100644 --- a/main/application.h +++ b/main/application.h @@ -83,6 +83,7 @@ private: bool aborted_ = false; int clock_ticks_ = 0; TaskHandle_t check_new_version_task_handle_ = nullptr; + TaskHandle_t main_event_loop_task_handle_ = nullptr; void OnWakeWordDetected(); void CheckNewVersion(Ota& ota); @@ -91,4 +92,19 @@ private: void SetListeningMode(ListeningMode mode); }; + +class TaskPriorityReset { +public: + TaskPriorityReset(BaseType_t priority) { + original_priority_ = uxTaskPriorityGet(NULL); + vTaskPrioritySet(NULL, priority); + } + ~TaskPriorityReset() { + vTaskPrioritySet(NULL, original_priority_); + } + +private: + BaseType_t original_priority_; +}; + #endif // _APPLICATION_H_ diff --git a/main/audio/audio_service.cc b/main/audio/audio_service.cc index c819b3a9..7cb8dcb7 100644 --- a/main/audio/audio_service.cc +++ b/main/audio/audio_service.cc @@ -100,11 +100,11 @@ void AudioService::Start() { #if CONFIG_USE_AUDIO_PROCESSOR /* Start the audio input task */ - xTaskCreatePinnedToCore([](void* arg) { + xTaskCreate([](void* arg) { AudioService* audio_service = (AudioService*)arg; audio_service->AudioInputTask(); vTaskDelete(NULL); - }, "audio_input", 2048 * 3, this, 8, &audio_input_task_handle_, 1); + }, "audio_input", 2048 * 3, this, 8, &audio_input_task_handle_); /* Start the audio output task */ xTaskCreate([](void* arg) { diff --git a/main/boards/common/esp32_camera.cc b/main/boards/common/esp32_camera.cc index d49c1e0d..8864325e 100644 --- a/main/boards/common/esp32_camera.cc +++ b/main/boards/common/esp32_camera.cc @@ -63,33 +63,26 @@ bool Esp32Camera::Capture() { // 显示预览图片 auto display = dynamic_cast(Board::GetInstance().GetDisplay()); if (display != nullptr) { - // Create a new preview image - auto img_dsc = (lv_img_dsc_t*)heap_caps_calloc(1, sizeof(lv_img_dsc_t), MALLOC_CAP_8BIT); - img_dsc->header.magic = LV_IMAGE_HEADER_MAGIC; - img_dsc->header.cf = LV_COLOR_FORMAT_RGB565; - img_dsc->header.flags = 0; - img_dsc->header.w = fb_->width; - img_dsc->header.h = fb_->height; - img_dsc->header.stride = fb_->width * 2; - img_dsc->data_size = fb_->width * fb_->height * 2; - img_dsc->data = (uint8_t*)heap_caps_malloc(img_dsc->data_size, MALLOC_CAP_SPIRAM); - if (img_dsc->data == nullptr) { + auto data = (uint8_t*)heap_caps_malloc(fb_->len, MALLOC_CAP_SPIRAM); + if (data == nullptr) { ESP_LOGE(TAG, "Failed to allocate memory for preview image"); - heap_caps_free(img_dsc); return false; } auto src = (uint16_t*)fb_->buf; - auto dst = (uint16_t*)img_dsc->data; + auto dst = (uint16_t*)data; size_t pixel_count = fb_->len / 2; for (size_t i = 0; i < pixel_count; i++) { // 交换每个16位字内的字节 dst[i] = __builtin_bswap16(src[i]); } - display->SetPreviewImage(img_dsc); + + auto image = std::make_unique(data, fb_->len, fb_->width, fb_->height, fb_->width * 2, LV_COLOR_FORMAT_RGB565); + display->SetPreviewImage(std::move(image)); } return true; } + bool Esp32Camera::SetHMirror(bool enabled) { sensor_t *s = esp_camera_sensor_get(); if (s == nullptr) { diff --git a/main/boards/sensecap-watcher/sscma_camera.cc b/main/boards/sensecap-watcher/sscma_camera.cc index a6c3e0cf..69342c8f 100644 --- a/main/boards/sensecap-watcher/sscma_camera.cc +++ b/main/boards/sensecap-watcher/sscma_camera.cc @@ -1,6 +1,7 @@ #include "sscma_camera.h" #include "mcp_server.h" #include "lvgl_display.h" +#include "lvgl_image.h" #include "board.h" #include "system_info.h" #include "config.h" @@ -245,7 +246,8 @@ bool SscmaCamera::Capture() { // 显示预览图片 auto display = dynamic_cast(Board::GetInstance().GetDisplay()); if (display != nullptr) { - display->SetPreviewImage(&preview_image_); + auto image = std::make_unique(&preview_image_); + display->SetPreviewImage(std::move(image)); } return true; } diff --git a/main/display/lcd_display.cc b/main/display/lcd_display.cc index f2c7da8c..4035975a 100644 --- a/main/display/lcd_display.cc +++ b/main/display/lcd_display.cc @@ -28,30 +28,30 @@ void LcdDisplay::InitializeLcdThemes() { // light theme auto light_theme = new LvglTheme("light"); - light_theme->set_background_color(lv_color_white()); - light_theme->set_text_color(lv_color_black()); - light_theme->set_chat_background_color(lv_color_hex(0xE0E0E0)); - light_theme->set_user_bubble_color(lv_color_hex(0x95EC69)); - light_theme->set_assistant_bubble_color(lv_color_white()); - light_theme->set_system_bubble_color(lv_color_hex(0xE0E0E0)); - light_theme->set_system_text_color(lv_color_hex(0x666666)); - light_theme->set_border_color(lv_color_hex(0xE0E0E0)); - light_theme->set_low_battery_color(lv_color_black()); + light_theme->set_background_color(lv_color_hex(0xFFFFFF)); //rgb(255, 255, 255) + light_theme->set_text_color(lv_color_hex(0x000000)); //rgb(0, 0, 0) + light_theme->set_chat_background_color(lv_color_hex(0xE0E0E0)); //rgb(224, 224, 224) + light_theme->set_user_bubble_color(lv_color_hex(0x00FF00)); //rgb(0, 128, 0) + light_theme->set_assistant_bubble_color(lv_color_hex(0xDDDDDD)); //rgb(221, 221, 221) + light_theme->set_system_bubble_color(lv_color_hex(0xFFFFFF)); //rgb(255, 255, 255) + light_theme->set_system_text_color(lv_color_hex(0x000000)); //rgb(0, 0, 0) + light_theme->set_border_color(lv_color_hex(0x000000)); //rgb(0, 0, 0) + light_theme->set_low_battery_color(lv_color_hex(0x000000)); //rgb(0, 0, 0) light_theme->set_text_font(text_font); light_theme->set_icon_font(icon_font); light_theme->set_large_icon_font(large_icon_font); // dark theme auto dark_theme = new LvglTheme("dark"); - dark_theme->set_background_color(lv_color_hex(0x121212)); - dark_theme->set_text_color(lv_color_white()); - dark_theme->set_chat_background_color(lv_color_hex(0x1E1E1E)); - dark_theme->set_user_bubble_color(lv_color_hex(0x1A6C37)); - dark_theme->set_assistant_bubble_color(lv_color_hex(0x333333)); - dark_theme->set_system_bubble_color(lv_color_hex(0x2A2A2A)); - dark_theme->set_system_text_color(lv_color_hex(0xAAAAAA)); - dark_theme->set_border_color(lv_color_hex(0x333333)); - dark_theme->set_low_battery_color(lv_color_hex(0xFF0000)); + dark_theme->set_background_color(lv_color_hex(0x000000)); //rgb(0, 0, 0) + dark_theme->set_text_color(lv_color_hex(0xFFFFFF)); //rgb(255, 255, 255) + dark_theme->set_chat_background_color(lv_color_hex(0x1F1F1F)); //rgb(31, 31, 31) + dark_theme->set_user_bubble_color(lv_color_hex(0x00FF00)); //rgb(0, 128, 0) + dark_theme->set_assistant_bubble_color(lv_color_hex(0x222222)); //rgb(34, 34, 34) + dark_theme->set_system_bubble_color(lv_color_hex(0x000000)); //rgb(0, 0, 0) + dark_theme->set_system_text_color(lv_color_hex(0xFFFFFF)); //rgb(255, 255, 255) + dark_theme->set_border_color(lv_color_hex(0xFFFFFF)); //rgb(255, 255, 255) + dark_theme->set_low_battery_color(lv_color_hex(0xFF0000)); //rgb(255, 0, 0) dark_theme->set_text_font(text_font); dark_theme->set_icon_font(icon_font); dark_theme->set_large_icon_font(large_icon_font); @@ -120,6 +120,9 @@ SpiLcdDisplay::SpiLcdDisplay(esp_lcd_panel_io_handle_t panel_io, esp_lcd_panel_h ESP_LOGI(TAG, "Initialize LVGL port"); lvgl_port_cfg_t port_cfg = ESP_LVGL_PORT_INIT_CONFIG(); port_cfg.task_priority = 1; +#if CONFIG_SOC_CPU_CORES_NUM > 1 + port_cfg.task_affinity = 1; +#endif lvgl_port_init(&port_cfg); ESP_LOGI(TAG, "Adding LCD display"); @@ -451,7 +454,7 @@ void LcdDisplay::SetupUI() { lv_obj_add_flag(low_battery_popup_, LV_OBJ_FLAG_HIDDEN); emoji_image_ = lv_img_create(screen); - lv_obj_align(emoji_image_, LV_ALIGN_TOP_MID, 0, text_font->line_height + lvgl_theme->spacing(2)); + lv_obj_align(emoji_image_, LV_ALIGN_TOP_MID, 0, text_font->line_height + lvgl_theme->spacing(8)); // Display AI logo while booting emoji_label_ = lv_label_create(screen); @@ -521,8 +524,7 @@ void LcdDisplay::SetChatMessage(const char* role, const char* content) { lv_obj_t* msg_bubble = lv_obj_create(content_); lv_obj_set_style_radius(msg_bubble, 8, 0); lv_obj_set_scrollbar_mode(msg_bubble, LV_SCROLLBAR_MODE_OFF); - lv_obj_set_style_border_width(msg_bubble, 1, 0); - lv_obj_set_style_border_color(msg_bubble, lvgl_theme->border_color(), 0); + lv_obj_set_style_border_width(msg_bubble, 0, 0); lv_obj_set_style_pad_all(msg_bubble, lvgl_theme->spacing(4), 0); // Create the message text @@ -561,6 +563,7 @@ void LcdDisplay::SetChatMessage(const char* role, const char* content) { if (strcmp(role, "user") == 0) { // User messages are right-aligned with green background lv_obj_set_style_bg_color(msg_bubble, lvgl_theme->user_bubble_color(), 0); + lv_obj_set_style_bg_opa(msg_bubble, LV_OPA_70, 0); // Set text color for contrast lv_obj_set_style_text_color(msg_text, lvgl_theme->text_color(), 0); @@ -576,6 +579,7 @@ void LcdDisplay::SetChatMessage(const char* role, const char* content) { } else if (strcmp(role, "assistant") == 0) { // Assistant messages are left-aligned with white background lv_obj_set_style_bg_color(msg_bubble, lvgl_theme->assistant_bubble_color(), 0); + lv_obj_set_style_bg_opa(msg_bubble, LV_OPA_70, 0); // Set text color for contrast lv_obj_set_style_text_color(msg_text, lvgl_theme->text_color(), 0); @@ -591,6 +595,7 @@ void LcdDisplay::SetChatMessage(const char* role, const char* content) { } else if (strcmp(role, "system") == 0) { // System messages are center-aligned with light gray background lv_obj_set_style_bg_color(msg_bubble, lvgl_theme->system_bubble_color(), 0); + lv_obj_set_style_bg_opa(msg_bubble, LV_OPA_70, 0); // Set text color for contrast lv_obj_set_style_text_color(msg_text, lvgl_theme->system_text_color(), 0); @@ -657,84 +662,88 @@ void LcdDisplay::SetChatMessage(const char* role, const char* content) { chat_message_label_ = msg_text; } -void LcdDisplay::SetPreviewImage(const lv_img_dsc_t* img_dsc) { +void LcdDisplay::SetPreviewImage(std::unique_ptr image) { DisplayLockGuard lock(this); if (content_ == nullptr) { return; } + + if (image == nullptr) { + return; + } auto lvgl_theme = static_cast(current_theme_); - if (img_dsc != nullptr) { - // Create a message bubble for image preview - lv_obj_t* img_bubble = lv_obj_create(content_); - lv_obj_set_style_radius(img_bubble, 8, 0); - lv_obj_set_scrollbar_mode(img_bubble, LV_SCROLLBAR_MODE_OFF); - lv_obj_set_style_border_width(img_bubble, 1, 0); - lv_obj_set_style_border_color(img_bubble, lvgl_theme->border_color(), 0); - lv_obj_set_style_pad_all(img_bubble, lvgl_theme->spacing(4), 0); - - // Set image bubble background color (similar to system message) - lv_obj_set_style_bg_color(img_bubble, lvgl_theme->assistant_bubble_color(), 0); - - // 设置自定义属性标记气泡类型 - lv_obj_set_user_data(img_bubble, (void*)"image"); + // Create a message bubble for image preview + lv_obj_t* img_bubble = lv_obj_create(content_); + lv_obj_set_style_radius(img_bubble, 8, 0); + lv_obj_set_scrollbar_mode(img_bubble, LV_SCROLLBAR_MODE_OFF); + lv_obj_set_style_border_width(img_bubble, 0, 0); + lv_obj_set_style_pad_all(img_bubble, lvgl_theme->spacing(4), 0); + + // Set image bubble background color (similar to system message) + lv_obj_set_style_bg_color(img_bubble, lvgl_theme->assistant_bubble_color(), 0); + lv_obj_set_style_bg_opa(img_bubble, LV_OPA_70, 0); + + // 设置自定义属性标记气泡类型 + lv_obj_set_user_data(img_bubble, (void*)"image"); - // Create the image object inside the bubble - lv_obj_t* preview_image = lv_image_create(img_bubble); - - // Calculate appropriate size for the image - lv_coord_t max_width = LV_HOR_RES * 70 / 100; // 70% of screen width - lv_coord_t max_height = LV_VER_RES * 50 / 100; // 50% of screen height - - // Calculate zoom factor to fit within maximum dimensions - lv_coord_t img_width = img_dsc->header.w; - lv_coord_t img_height = img_dsc->header.h; - if (img_width == 0 || img_height == 0) { - img_width = max_width; - img_height = max_height; - ESP_LOGW(TAG, "Invalid image dimensions: %ld x %ld, using default dimensions: %ld x %ld", img_width, img_height, max_width, max_height); - } - - lv_coord_t zoom_w = (max_width * 256) / img_width; - lv_coord_t zoom_h = (max_height * 256) / img_height; - lv_coord_t zoom = (zoom_w < zoom_h) ? zoom_w : zoom_h; - - // Ensure zoom doesn't exceed 256 (100%) - if (zoom > 256) zoom = 256; - - // Set image properties - lv_image_set_src(preview_image, img_dsc); - lv_image_set_scale(preview_image, zoom); - - // Add event handler to clean up copied data when image is deleted - lv_obj_add_event_cb(preview_image, [](lv_event_t* e) { - lv_img_dsc_t* img_dsc = (lv_img_dsc_t*)lv_event_get_user_data(e); - if (img_dsc != nullptr) { - heap_caps_free((void*)img_dsc->data); - heap_caps_free(img_dsc); - } - }, LV_EVENT_DELETE, (void*)img_dsc); - - // Calculate actual scaled image dimensions - lv_coord_t scaled_width = (img_width * zoom) / 256; - lv_coord_t scaled_height = (img_height * zoom) / 256; - - // Set bubble size to be 16 pixels larger than the image (8 pixels on each side) - lv_obj_set_width(img_bubble, scaled_width + 16); - lv_obj_set_height(img_bubble, scaled_height + 16); - - // Don't grow in flex layout - lv_obj_set_style_flex_grow(img_bubble, 0, 0); - - // Center the image within the bubble - lv_obj_center(preview_image); - - // Left align the image bubble like assistant messages - lv_obj_align(img_bubble, LV_ALIGN_LEFT_MID, 0, 0); - - // Auto-scroll to the image bubble - lv_obj_scroll_to_view_recursive(img_bubble, LV_ANIM_ON); + // Create the image object inside the bubble + lv_obj_t* preview_image = lv_image_create(img_bubble); + + // Calculate appropriate size for the image + lv_coord_t max_width = LV_HOR_RES * 70 / 100; // 70% of screen width + lv_coord_t max_height = LV_VER_RES * 50 / 100; // 50% of screen height + + // Calculate zoom factor to fit within maximum dimensions + auto img_dsc = image->image_dsc(); + lv_coord_t img_width = img_dsc->header.w; + lv_coord_t img_height = img_dsc->header.h; + if (img_width == 0 || img_height == 0) { + img_width = max_width; + img_height = max_height; + ESP_LOGW(TAG, "Invalid image dimensions: %ld x %ld, using default dimensions: %ld x %ld", img_width, img_height, max_width, max_height); } + + lv_coord_t zoom_w = (max_width * 256) / img_width; + lv_coord_t zoom_h = (max_height * 256) / img_height; + lv_coord_t zoom = (zoom_w < zoom_h) ? zoom_w : zoom_h; + + // Ensure zoom doesn't exceed 256 (100%) + if (zoom > 256) zoom = 256; + + // Set image properties + lv_image_set_src(preview_image, img_dsc); + lv_image_set_scale(preview_image, zoom); + + // Add event handler to clean up LvglImage when image is deleted + // We need to transfer ownership of the unique_ptr to the event callback + LvglImage* raw_image = image.release(); // 释放智能指针的所有权 + lv_obj_add_event_cb(preview_image, [](lv_event_t* e) { + LvglImage* img = (LvglImage*)lv_event_get_user_data(e); + if (img != nullptr) { + delete img; // 通过删除 LvglImage 对象来正确释放内存 + } + }, LV_EVENT_DELETE, (void*)raw_image); + + // Calculate actual scaled image dimensions + lv_coord_t scaled_width = (img_width * zoom) / 256; + lv_coord_t scaled_height = (img_height * zoom) / 256; + + // Set bubble size to be 16 pixels larger than the image (8 pixels on each side) + lv_obj_set_width(img_bubble, scaled_width + 16); + lv_obj_set_height(img_bubble, scaled_height + 16); + + // Don't grow in flex layout + lv_obj_set_style_flex_grow(img_bubble, 0, 0); + + // Center the image within the bubble + lv_obj_center(preview_image); + + // Left align the image bubble like assistant messages + lv_obj_align(img_bubble, LV_ALIGN_LEFT_MID, 0, 0); + + // Auto-scroll to the image bubble + lv_obj_scroll_to_view_recursive(img_bubble, LV_ANIM_ON); } #else void LcdDisplay::SetupUI() { @@ -858,37 +867,35 @@ void LcdDisplay::SetupUI() { lv_obj_add_flag(low_battery_popup_, LV_OBJ_FLAG_HIDDEN); } -void LcdDisplay::SetPreviewImage(const lv_img_dsc_t* img_dsc) { +void LcdDisplay::SetPreviewImage(std::unique_ptr image) { DisplayLockGuard lock(this); if (preview_image_ == nullptr) { ESP_LOGE(TAG, "Preview image is not initialized"); return; } - auto old_src = (const lv_img_dsc_t*)lv_image_get_src(preview_image_); - if (old_src != nullptr) { - lv_image_set_src(preview_image_, nullptr); - heap_caps_free((void*)old_src->data); - heap_caps_free((void*)old_src); - } - - if (img_dsc != nullptr) { - // 设置图片源并显示预览图片 - lv_image_set_src(preview_image_, img_dsc); - if (img_dsc->header.w > 0 && img_dsc->header.h > 0) { - // zoom factor 0.5 - lv_image_set_scale(preview_image_, 128 * width_ / img_dsc->header.w); - } - // Hide emoji_box_ - lv_obj_add_flag(emoji_box_, LV_OBJ_FLAG_HIDDEN); - lv_obj_remove_flag(preview_image_, LV_OBJ_FLAG_HIDDEN); - esp_timer_stop(preview_timer_); - ESP_ERROR_CHECK(esp_timer_start_once(preview_timer_, PREVIEW_IMAGE_DURATION_MS * 1000)); - } else { + if (image == nullptr) { esp_timer_stop(preview_timer_); lv_obj_remove_flag(emoji_box_, LV_OBJ_FLAG_HIDDEN); lv_obj_add_flag(preview_image_, LV_OBJ_FLAG_HIDDEN); + preview_image_cached_.reset(); + return; } + + preview_image_cached_ = std::move(image); + auto img_dsc = preview_image_cached_->image_dsc(); + // 设置图片源并显示预览图片 + lv_image_set_src(preview_image_, img_dsc); + if (img_dsc->header.w > 0 && img_dsc->header.h > 0) { + // zoom factor 0.5 + lv_image_set_scale(preview_image_, 128 * width_ / img_dsc->header.w); + } + + // Hide emoji_box_ + lv_obj_add_flag(emoji_box_, LV_OBJ_FLAG_HIDDEN); + lv_obj_remove_flag(preview_image_, LV_OBJ_FLAG_HIDDEN); + esp_timer_stop(preview_timer_); + ESP_ERROR_CHECK(esp_timer_start_once(preview_timer_, PREVIEW_IMAGE_DURATION_MS * 1000)); } void LcdDisplay::SetChatMessage(const char* role, const char* content) { @@ -955,7 +962,8 @@ void LcdDisplay::SetEmotion(const char* emotion) { #if CONFIG_USE_WECHAT_MESSAGE_STYLE // Wechat message style中,如果emotion是neutral,则不显示 - if (strcmp(emotion, "neutral") == 0) { + uint32_t child_count = lv_obj_get_child_cnt(content_); + if (strcmp(emotion, "neutral") == 0 && child_count > 0) { // Stop GIF animation if running if (gif_controller_) { gif_controller_->Stop(); diff --git a/main/display/lcd_display.h b/main/display/lcd_display.h index 54cc4848..42ecc29a 100644 --- a/main/display/lcd_display.h +++ b/main/display/lcd_display.h @@ -31,6 +31,7 @@ protected: lv_obj_t* emoji_box_ = nullptr; lv_obj_t* chat_message_label_ = nullptr; esp_timer_handle_t preview_timer_ = nullptr; + std::unique_ptr preview_image_cached_ = nullptr; void InitializeLcdThemes(); void SetupUI(); @@ -44,8 +45,8 @@ protected: public: ~LcdDisplay(); virtual void SetEmotion(const char* emotion) override; - virtual void SetPreviewImage(const lv_img_dsc_t* img_dsc) override; virtual void SetChatMessage(const char* role, const char* content) override; + virtual void SetPreviewImage(std::unique_ptr image) override; // Add theme switching function virtual void SetTheme(Theme* theme) override; diff --git a/main/display/lvgl_display/gif/lvgl_gif.cc b/main/display/lvgl_display/gif/lvgl_gif.cc index 7d3495f7..172d5ba3 100644 --- a/main/display/lvgl_display/gif/lvgl_gif.cc +++ b/main/display/lvgl_display/gif/lvgl_gif.cc @@ -34,7 +34,7 @@ LvglGif::LvglGif(const lv_img_dsc_t* img_dsc) } loaded_ = true; - ESP_LOGI(TAG, "GIF loaded from image descriptor: %dx%d", gif_->width, gif_->height); + ESP_LOGD(TAG, "GIF loaded from image descriptor: %dx%d", gif_->width, gif_->height); } // Destructor @@ -73,7 +73,7 @@ void LvglGif::Start() { // Render first frame NextFrame(); - ESP_LOGI(TAG, "GIF animation started"); + ESP_LOGD(TAG, "GIF animation started"); } } @@ -81,7 +81,7 @@ void LvglGif::Pause() { if (timer_) { playing_ = false; lv_timer_pause(timer_); - ESP_LOGI(TAG, "GIF animation paused"); + ESP_LOGD(TAG, "GIF animation paused"); } } @@ -94,7 +94,7 @@ void LvglGif::Resume() { if (timer_) { playing_ = true; lv_timer_resume(timer_); - ESP_LOGI(TAG, "GIF animation resumed"); + ESP_LOGD(TAG, "GIF animation resumed"); } } @@ -107,7 +107,7 @@ void LvglGif::Stop() { if (gif_) { gd_rewind(gif_); NextFrame(); - ESP_LOGI(TAG, "GIF animation stopped and rewound"); + ESP_LOGD(TAG, "GIF animation stopped and rewound"); } } @@ -173,7 +173,7 @@ void LvglGif::NextFrame() { if (timer_) { lv_timer_pause(timer_); } - ESP_LOGI(TAG, "GIF animation completed"); + ESP_LOGD(TAG, "GIF animation completed"); } // Render current frame diff --git a/main/display/lvgl_display/lvgl_display.cc b/main/display/lvgl_display/lvgl_display.cc index ffced5b0..2ec00a6d 100644 --- a/main/display/lvgl_display/lvgl_display.cc +++ b/main/display/lvgl_display/lvgl_display.cc @@ -4,6 +4,7 @@ #include #include #include +#include #include "lvgl_display.h" #include "board.h" @@ -201,12 +202,7 @@ void LvglDisplay::UpdateStatusBar(bool update_all) { esp_pm_lock_release(pm_lock_); } -void LvglDisplay::SetPreviewImage(const lv_img_dsc_t* image) { - // Do nothing but free the image - if (image != nullptr) { - heap_caps_free((void*)image->data); - heap_caps_free((void*)image); - } +void LvglDisplay::SetPreviewImage(std::unique_ptr image) { } void LvglDisplay::SetPowerSaveMode(bool on) { @@ -218,3 +214,29 @@ void LvglDisplay::SetPowerSaveMode(bool on) { SetEmotion("neutral"); } } + +bool LvglDisplay::SnapshotToJpeg(uint8_t*& jpeg_output_data, size_t& jpeg_output_data_size, int quality) { + DisplayLockGuard lock(this); + + lv_obj_t* screen = lv_screen_active(); + lv_draw_buf_t* draw_buffer = lv_snapshot_take(screen, LV_COLOR_FORMAT_RGB565); + if (draw_buffer == nullptr) { + return false; + } + + // swap bytes + uint16_t* data = (uint16_t*)draw_buffer->data; + size_t pixel_count = draw_buffer->data_size / 2; + for (size_t i = 0; i < pixel_count; i++) { + data[i] = __builtin_bswap16(data[i]); + } + + if (!fmt2jpg(draw_buffer->data, draw_buffer->data_size, draw_buffer->header.w, draw_buffer->header.h, + PIXFORMAT_RGB565, quality, &jpeg_output_data, &jpeg_output_data_size)) { + lv_draw_buf_destroy(draw_buffer); + return false; + } + + lv_draw_buf_destroy(draw_buffer); + return true; +} diff --git a/main/display/lvgl_display/lvgl_display.h b/main/display/lvgl_display/lvgl_display.h index 47d2e84c..c3a56277 100644 --- a/main/display/lvgl_display/lvgl_display.h +++ b/main/display/lvgl_display/lvgl_display.h @@ -2,6 +2,7 @@ #define LVGL_DISPLAY_H #include "display.h" +#include "lvgl_image.h" #include #include @@ -19,9 +20,10 @@ public: virtual void SetStatus(const char* status); virtual void ShowNotification(const char* notification, int duration_ms = 3000); virtual void ShowNotification(const std::string ¬ification, int duration_ms = 3000); - virtual void SetPreviewImage(const lv_img_dsc_t* image); + virtual void SetPreviewImage(std::unique_ptr image); virtual void UpdateStatusBar(bool update_all = false); virtual void SetPowerSaveMode(bool on); + virtual bool SnapshotToJpeg(uint8_t*& jpeg_output_data, size_t& jpeg_output_size, int quality = 80); protected: esp_pm_lock_handle_t pm_lock_ = nullptr; diff --git a/main/display/lvgl_display/lvgl_image.cc b/main/display/lvgl_display/lvgl_image.cc index eff91a12..eefc0314 100644 --- a/main/display/lvgl_display/lvgl_image.cc +++ b/main/display/lvgl_display/lvgl_image.cc @@ -2,19 +2,21 @@ #include #include +#include #include +#include #define TAG "LvglImage" LvglRawImage::LvglRawImage(void* data, size_t size) { bzero(&image_dsc_, sizeof(image_dsc_)); + image_dsc_.data_size = size; + image_dsc_.data = static_cast(data); image_dsc_.header.magic = LV_IMAGE_HEADER_MAGIC; image_dsc_.header.cf = LV_COLOR_FORMAT_RAW_ALPHA; image_dsc_.header.w = 0; image_dsc_.header.h = 0; - image_dsc_.data_size = size; - image_dsc_.data = static_cast(data); } bool LvglRawImage::IsGif() const { @@ -31,3 +33,32 @@ LvglCBinImage::~LvglCBinImage() { cbin_img_dsc_delete(image_dsc_); } } + +LvglAllocatedImage::LvglAllocatedImage(void* data, size_t size) { + bzero(&image_dsc_, sizeof(image_dsc_)); + image_dsc_.data_size = size; + image_dsc_.data = static_cast(data); + + if (lv_image_decoder_get_info(&image_dsc_, &image_dsc_.header) != LV_RESULT_OK) { + ESP_LOGE(TAG, "Failed to get image info, data: %p size: %u", data, size); + throw std::runtime_error("Failed to get image info"); + } +} + +LvglAllocatedImage::LvglAllocatedImage(void* data, size_t size, int width, int height, int stride, int color_format) { + bzero(&image_dsc_, sizeof(image_dsc_)); + image_dsc_.data_size = size; + image_dsc_.data = static_cast(data); + image_dsc_.header.magic = LV_IMAGE_HEADER_MAGIC; + image_dsc_.header.cf = color_format; + image_dsc_.header.w = width; + image_dsc_.header.h = height; + image_dsc_.header.stride = stride; +} + +LvglAllocatedImage::~LvglAllocatedImage() { + if (image_dsc_.data) { + heap_caps_free((void*)image_dsc_.data); + image_dsc_.data = nullptr; + } +} \ No newline at end of file diff --git a/main/display/lvgl_display/lvgl_image.h b/main/display/lvgl_display/lvgl_image.h index 92f3490c..0bbf39be 100644 --- a/main/display/lvgl_display/lvgl_image.h +++ b/main/display/lvgl_display/lvgl_image.h @@ -39,4 +39,15 @@ public: private: const lv_img_dsc_t* image_dsc_; +}; + +class LvglAllocatedImage : public LvglImage { +public: + LvglAllocatedImage(void* data, size_t size); + LvglAllocatedImage(void* data, size_t size, int width, int height, int stride, int color_format); + virtual ~LvglAllocatedImage(); + virtual const lv_img_dsc_t* image_dsc() const override { return &image_dsc_; } + +private: + lv_img_dsc_t image_dsc_; }; \ No newline at end of file diff --git a/main/display/oled_display.cc b/main/display/oled_display.cc index 918b5e16..66dee400 100644 --- a/main/display/oled_display.cc +++ b/main/display/oled_display.cc @@ -40,6 +40,9 @@ OledDisplay::OledDisplay(esp_lcd_panel_io_handle_t panel_io, esp_lcd_panel_handl lvgl_port_cfg_t port_cfg = ESP_LVGL_PORT_INIT_CONFIG(); port_cfg.task_priority = 1; port_cfg.task_stack = 6144; +#if CONFIG_SOC_CPU_CORES_NUM > 1 + port_cfg.task_affinity = 1; +#endif lvgl_port_init(&port_cfg); ESP_LOGI(TAG, "Adding OLED display"); diff --git a/main/idf_component.yml b/main/idf_component.yml index 4bd068b7..f102d3cb 100644 --- a/main/idf_component.yml +++ b/main/idf_component.yml @@ -15,7 +15,7 @@ dependencies: 78/esp_lcd_nv3023: ~1.0.0 78/esp-wifi-connect: ~2.5.2 78/esp-opus-encoder: ~2.4.1 - 78/esp-ml307: ~3.3.3 + 78/esp-ml307: ~3.3.5 78/xiaozhi-fonts: ~1.5.2 espressif/led_strip: ~3.0.1 espressif/esp_codec_dev: ~1.4.0 diff --git a/main/main.cc b/main/main.cc index c4bce23d..7ceda913 100755 --- a/main/main.cc +++ b/main/main.cc @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include "application.h" #include "system_info.h" @@ -27,5 +29,4 @@ extern "C" void app_main(void) // Launch the application auto& app = Application::GetInstance(); app.Start(); - app.MainEventLoop(); } diff --git a/main/mcp_server.cc b/main/mcp_server.cc index 9c3064e9..5b12e43d 100644 --- a/main/mcp_server.cc +++ b/main/mcp_server.cc @@ -20,8 +20,6 @@ #define TAG "MCP" -#define DEFAULT_TOOLCALL_STACK_SIZE 6144 - McpServer::McpServer() { } @@ -111,6 +109,9 @@ void McpServer::AddCommonTools() { Property("question", kPropertyTypeString) }), [camera](const PropertyList& properties) -> ReturnValue { + // Lower the priority to do the camera capture + TaskPriorityReset priority_reset(1); + if (!camera->Capture()) { throw std::runtime_error("Failed to capture photo"); } @@ -137,11 +138,13 @@ void McpServer::AddUserOnlyTools() { AddUserOnlyTool("self.reboot", "Reboot the system", PropertyList(), [this](const PropertyList& properties) -> ReturnValue { - std::thread([this]() { + auto& app = Application::GetInstance(); + app.Schedule([&app]() { ESP_LOGW(TAG, "User requested reboot"); vTaskDelay(pdMS_TO_TICKS(1000)); - Application::GetInstance().Reboot(); - }).detach(); + + app.Reboot(); + }); return true; }); @@ -155,8 +158,7 @@ void McpServer::AddUserOnlyTools() { ESP_LOGI(TAG, "User requested firmware upgrade from URL: %s", url.c_str()); auto& app = Application::GetInstance(); - app.Schedule([url]() { - auto& app = Application::GetInstance(); + app.Schedule([url, &app]() { auto ota = std::make_unique(); bool success = app.UpgradeFirmware(*ota, url); @@ -185,6 +187,63 @@ void McpServer::AddUserOnlyTools() { } return json; }); + + AddUserOnlyTool("self.screen.snapshot", "Snapshot the screen and upload it to a specific URL", + PropertyList({ + Property("url", kPropertyTypeString), + Property("quality", kPropertyTypeInteger, 80, 1, 100) + }), + [display](const PropertyList& properties) -> ReturnValue { + auto url = properties["url"].value(); + auto quality = properties["quality"].value(); + + uint8_t* jpeg_output_data = nullptr; + size_t jpeg_output_size = 0; + if (!display->SnapshotToJpeg(jpeg_output_data, jpeg_output_size, quality)) { + throw std::runtime_error("Failed to snapshot screen"); + } + + ESP_LOGI(TAG, "Upload snapshot %u bytes to %s", jpeg_output_size, url.c_str()); + + // 构造multipart/form-data请求体 + std::string boundary = "----ESP32_SCREEN_SNAPSHOT_BOUNDARY"; + + auto http = Board::GetInstance().GetNetwork()->CreateHttp(3); + http->SetHeader("Content-Type", "multipart/form-data; boundary=" + boundary); + if (!http->Open("POST", url)) { + free(jpeg_output_data); + throw std::runtime_error("Failed to open URL: " + url); + } + { + // 文件字段头部 + std::string file_header; + file_header += "--" + boundary + "\r\n"; + file_header += "Content-Disposition: form-data; name=\"file\"; filename=\"screenshot.jpg\"\r\n"; + file_header += "Content-Type: image/jpeg\r\n"; + file_header += "\r\n"; + http->Write(file_header.c_str(), file_header.size()); + } + + // JPEG数据 + http->Write((const char*)jpeg_output_data, jpeg_output_size); + free(jpeg_output_data); + + { + // multipart尾部 + std::string multipart_footer; + multipart_footer += "\r\n--" + boundary + "--\r\n"; + http->Write(multipart_footer.c_str(), multipart_footer.size()); + } + http->Write("", 0); + + if (http->GetStatusCode() != 200) { + throw std::runtime_error("Unexpected status code: " + std::to_string(http->GetStatusCode())); + } + std::string result = http->ReadAll(); + http->Close(); + ESP_LOGI(TAG, "Snapshot screen result: %s", result.c_str()); + return true; + }); AddUserOnlyTool("self.screen.preview_image", "Preview an image on the screen", PropertyList({ @@ -197,12 +256,16 @@ void McpServer::AddUserOnlyTools() { if (!http->Open("GET", url)) { throw std::runtime_error("Failed to open URL: " + url); } - if (http->GetStatusCode() != 200) { - throw std::runtime_error("Unexpected status code: " + std::to_string(http->GetStatusCode())); + int status_code = http->GetStatusCode(); + if (status_code != 200) { + throw std::runtime_error("Unexpected status code: " + std::to_string(status_code)); } size_t content_length = http->GetBodyLength(); char* data = (char*)heap_caps_malloc(content_length, MALLOC_CAP_8BIT); + if (data == nullptr) { + throw std::runtime_error("Failed to allocate memory for image: " + url); + } size_t total_read = 0; while (total_read < content_length) { int ret = http->Read(data + total_read, content_length - total_read); @@ -210,24 +273,15 @@ void McpServer::AddUserOnlyTools() { heap_caps_free(data); throw std::runtime_error("Failed to download image: " + url); } + if (ret == 0) { + break; + } total_read += ret; } http->Close(); - auto img_dsc = (lv_img_dsc_t*)heap_caps_calloc(1, sizeof(lv_img_dsc_t), MALLOC_CAP_8BIT); - img_dsc->data_size = content_length; - img_dsc->data = (uint8_t*)data; - if (lv_image_decoder_get_info(img_dsc, &img_dsc->header) != LV_RESULT_OK) { - heap_caps_free(data); - heap_caps_free(img_dsc); - throw std::runtime_error("Failed to get image info"); - } - ESP_LOGI(TAG, "Preview image: %s size: %d resolution: %d x %d", url.c_str(), content_length, img_dsc->header.w, img_dsc->header.h); - - auto& app = Application::GetInstance(); - app.Schedule([display, img_dsc]() { - display->SetPreviewImage(img_dsc); - }); + auto image = std::make_unique(data, content_length); + display->SetPreviewImage(std::move(image)); return true; }); } @@ -379,13 +433,7 @@ void McpServer::ParseMessage(const cJSON* json) { ReplyError(id_int, "Invalid arguments"); return; } - auto stack_size = cJSON_GetObjectItem(params, "stackSize"); - if (stack_size != nullptr && !cJSON_IsNumber(stack_size)) { - ESP_LOGE(TAG, "tools/call: Invalid stackSize"); - ReplyError(id_int, "Invalid stackSize"); - return; - } - DoToolCall(id_int, std::string(tool_name->valuestring), tool_arguments, stack_size ? stack_size->valueint : DEFAULT_TOOLCALL_STACK_SIZE); + DoToolCall(id_int, std::string(tool_name->valuestring), tool_arguments); } else { ESP_LOGE(TAG, "Method not implemented: %s", method_str.c_str()); ReplyError(id_int, "Method not implemented: " + method_str); @@ -465,7 +513,7 @@ void McpServer::GetToolsList(int id, const std::string& cursor, bool list_user_o ReplyResult(id, json); } -void McpServer::DoToolCall(int id, const std::string& tool_name, const cJSON* tool_arguments, int stack_size) { +void McpServer::DoToolCall(int id, const std::string& tool_name, const cJSON* tool_arguments) { auto tool_iter = std::find_if(tools_.begin(), tools_.end(), [&tool_name](const McpTool* tool) { return tool->name() == tool_name; @@ -507,15 +555,9 @@ void McpServer::DoToolCall(int id, const std::string& tool_name, const cJSON* to return; } - // Start a task to receive data with stack size - esp_pthread_cfg_t cfg = esp_pthread_get_default_config(); - cfg.thread_name = "tool_call"; - cfg.stack_size = stack_size; - cfg.prio = 1; - esp_pthread_set_cfg(&cfg); - - // Use a thread to call the tool to avoid blocking the main thread - tool_call_thread_ = std::thread([this, id, tool_iter, arguments = std::move(arguments)]() { + // Use main thread to call the tool + auto& app = Application::GetInstance(); + app.Schedule([this, id, tool_iter, arguments = std::move(arguments)]() { try { ReplyResult(id, (*tool_iter)->Call(arguments)); } catch (const std::exception& e) { @@ -523,5 +565,4 @@ void McpServer::DoToolCall(int id, const std::string& tool_name, const cJSON* to ReplyError(id, e.what()); } }); - tool_call_thread_.detach(); -} \ No newline at end of file +} diff --git a/main/mcp_server.h b/main/mcp_server.h index 547d1384..dacdd551 100644 --- a/main/mcp_server.h +++ b/main/mcp_server.h @@ -336,10 +336,9 @@ private: void ReplyError(int id, const std::string& message); void GetToolsList(int id, const std::string& cursor, bool list_user_only_tools); - void DoToolCall(int id, const std::string& tool_name, const cJSON* tool_arguments, int stack_size); + void DoToolCall(int id, const std::string& tool_name, const cJSON* tool_arguments); std::vector tools_; - std::thread tool_call_thread_; }; #endif // MCP_SERVER_H diff --git a/sdkconfig.defaults b/sdkconfig.defaults index 5c99235b..073f0ad4 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -55,6 +55,7 @@ CONFIG_LV_USE_IMGFONT=y CONFIG_LV_USE_ASSERT_STYLE=y CONFIG_LV_USE_GIF=y CONFIG_LV_USE_LODEPNG=y +CONFIG_LV_USE_SNAPSHOT=y # Use compressed font CONFIG_LV_FONT_FMT_TXT_LARGE=y