forked from xiaozhi/xiaozhi-esp32
feat: Add camera functions (lichuang-dev)
This commit is contained in:
@@ -4,7 +4,7 @@
|
||||
# CMakeLists in this exact order for cmake to work correctly
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
|
||||
set(PROJECT_VER "1.6.3")
|
||||
set(PROJECT_VER "1.6.4")
|
||||
|
||||
# Add this line to disable the specific warning
|
||||
add_compile_options(-Wno-missing-field-initializers)
|
||||
|
||||
@@ -57,6 +57,10 @@ Display* Board::GetDisplay() {
|
||||
return &display;
|
||||
}
|
||||
|
||||
Camera* Board::GetCamera() {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
Led* Board::GetLed() {
|
||||
static NoLed led;
|
||||
return &led;
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
|
||||
#include "led/led.h"
|
||||
#include "backlight.h"
|
||||
#include "camera.h"
|
||||
|
||||
void* create_board();
|
||||
class AudioCodec;
|
||||
@@ -39,6 +40,7 @@ public:
|
||||
virtual AudioCodec* GetAudioCodec() = 0;
|
||||
virtual bool GetTemperature(float& esp32temp);
|
||||
virtual Display* GetDisplay();
|
||||
virtual Camera* GetCamera();
|
||||
virtual Http* CreateHttp() = 0;
|
||||
virtual WebSocket* CreateWebSocket() = 0;
|
||||
virtual Mqtt* CreateMqtt() = 0;
|
||||
|
||||
13
main/boards/common/camera.h
Normal file
13
main/boards/common/camera.h
Normal file
@@ -0,0 +1,13 @@
|
||||
#ifndef CAMERA_H
|
||||
#define CAMERA_H
|
||||
|
||||
#include <string>
|
||||
|
||||
class Camera {
|
||||
public:
|
||||
virtual void SetExplainUrl(const std::string& url, const std::string& token) = 0;
|
||||
virtual bool Capture() = 0;
|
||||
virtual std::string Explain(const std::string& question) = 0;
|
||||
};
|
||||
|
||||
#endif // CAMERA_H
|
||||
212
main/boards/common/esp32_camera.cc
Normal file
212
main/boards/common/esp32_camera.cc
Normal file
@@ -0,0 +1,212 @@
|
||||
#include "esp32_camera.h"
|
||||
#include "mcp_server.h"
|
||||
#include "display.h"
|
||||
#include "board.h"
|
||||
|
||||
#include <esp_log.h>
|
||||
#include <esp_heap_caps.h>
|
||||
#include <img_converters.h>
|
||||
#include <cstring>
|
||||
|
||||
#define TAG "Esp32Camera"
|
||||
|
||||
Esp32Camera::Esp32Camera(const camera_config_t& config) {
|
||||
jpeg_queue_ = xQueueCreate(10, sizeof(JpegChunk));
|
||||
if (jpeg_queue_ == nullptr) {
|
||||
ESP_LOGE(TAG, "Failed to create JPEG queue");
|
||||
return;
|
||||
}
|
||||
|
||||
// camera init
|
||||
esp_err_t err = esp_camera_init(&config); // 配置上面定义的参数
|
||||
if (err != ESP_OK) {
|
||||
ESP_LOGE(TAG, "Camera init failed with error 0x%x", err);
|
||||
return;
|
||||
}
|
||||
|
||||
sensor_t *s = esp_camera_sensor_get(); // 获取摄像头型号
|
||||
if (s->id.PID == GC0308_PID) {
|
||||
s->set_hmirror(s, 0); // 这里控制摄像头镜像 写1镜像 写0不镜像
|
||||
}
|
||||
|
||||
// 初始化预览图片的内存
|
||||
memset(&preview_image_, 0, sizeof(preview_image_));
|
||||
preview_image_.header.magic = LV_IMAGE_HEADER_MAGIC;
|
||||
preview_image_.header.cf = LV_COLOR_FORMAT_RGB565;
|
||||
preview_image_.header.flags = LV_IMAGE_FLAGS_ALLOCATED | LV_IMAGE_FLAGS_MODIFIABLE;
|
||||
|
||||
if (config.frame_size == FRAMESIZE_VGA) {
|
||||
preview_image_.header.w = 640;
|
||||
preview_image_.header.h = 480;
|
||||
} else if (config.frame_size == FRAMESIZE_QVGA) {
|
||||
preview_image_.header.w = 320;
|
||||
preview_image_.header.h = 240;
|
||||
}
|
||||
preview_image_.header.stride = preview_image_.header.w * 2;
|
||||
preview_image_.data_size = preview_image_.header.w * preview_image_.header.h * 2;
|
||||
preview_image_.data = (uint8_t*)heap_caps_malloc(preview_image_.data_size, MALLOC_CAP_SPIRAM);
|
||||
if (preview_image_.data == nullptr) {
|
||||
ESP_LOGE(TAG, "Failed to allocate memory for preview image");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
Esp32Camera::~Esp32Camera() {
|
||||
if (fb_) {
|
||||
esp_camera_fb_return(fb_);
|
||||
fb_ = nullptr;
|
||||
}
|
||||
if (preview_image_.data) {
|
||||
heap_caps_free((void*)preview_image_.data);
|
||||
preview_image_.data = nullptr;
|
||||
}
|
||||
esp_camera_deinit();
|
||||
|
||||
if (jpeg_queue_ != nullptr) {
|
||||
vQueueDelete(jpeg_queue_);
|
||||
jpeg_queue_ = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
void Esp32Camera::SetExplainUrl(const std::string& url, const std::string& token) {
|
||||
explain_url_ = url;
|
||||
explain_token_ = token;
|
||||
}
|
||||
|
||||
bool Esp32Camera::Capture() {
|
||||
if (preview_thread_.joinable()) {
|
||||
preview_thread_.join();
|
||||
}
|
||||
if (encoder_thread_.joinable()) {
|
||||
encoder_thread_.join();
|
||||
}
|
||||
|
||||
int frames_to_get = 2;
|
||||
// Try to get a stable frame
|
||||
for (int i = 0; i < frames_to_get; i++) {
|
||||
if (fb_ != nullptr) {
|
||||
esp_camera_fb_return(fb_);
|
||||
}
|
||||
fb_ = esp_camera_fb_get();
|
||||
if (fb_ == nullptr) {
|
||||
ESP_LOGE(TAG, "Camera capture failed");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
preview_thread_ = std::thread([this]() {
|
||||
// 显示预览图片
|
||||
auto display = Board::GetInstance().GetDisplay();
|
||||
if (display != nullptr) {
|
||||
auto src = (uint16_t*)fb_->buf;
|
||||
auto dst = (uint16_t*)preview_image_.data;
|
||||
size_t pixel_count = fb_->len / 2;
|
||||
for (size_t i = 0; i < pixel_count; i++) {
|
||||
// 交换每个16位字内的字节
|
||||
dst[i] = __builtin_bswap16(src[i]);
|
||||
}
|
||||
display->SetPreviewImage(&preview_image_);
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string Esp32Camera::Explain(const std::string& question) {
|
||||
if (explain_url_.empty() || explain_token_.empty()) {
|
||||
return "{\"success\": false, \"message\": \"Image explain URL or token is not set\"}";
|
||||
}
|
||||
|
||||
// We spawn a thread to encode the image to JPEG
|
||||
encoder_thread_ = std::thread([this]() {
|
||||
frame2jpg_cb(fb_, 80, [](void* arg, size_t index, const void* data, size_t len) -> unsigned int {
|
||||
auto jpeg_queue = (QueueHandle_t)arg;
|
||||
JpegChunk chunk = {
|
||||
.data = (uint8_t*)heap_caps_malloc(len, MALLOC_CAP_SPIRAM),
|
||||
.len = len
|
||||
};
|
||||
memcpy(chunk.data, data, len);
|
||||
xQueueSend(jpeg_queue, &chunk, portMAX_DELAY);
|
||||
return len;
|
||||
}, jpeg_queue_);
|
||||
});
|
||||
|
||||
auto http = Board::GetInstance().CreateHttp();
|
||||
// 构造multipart/form-data请求体
|
||||
std::string boundary = "----ESP32_CAMERA_BOUNDARY";
|
||||
|
||||
// 构造question字段
|
||||
std::string question_field;
|
||||
question_field += "--" + boundary + "\r\n";
|
||||
question_field += "Content-Disposition: form-data; name=\"question\"\r\n";
|
||||
question_field += "\r\n";
|
||||
question_field += question + "\r\n";
|
||||
|
||||
// 构造文件字段头部
|
||||
std::string file_header;
|
||||
file_header += "--" + boundary + "\r\n";
|
||||
file_header += "Content-Disposition: form-data; name=\"file\"; filename=\"camera.jpg\"\r\n";
|
||||
file_header += "Content-Type: image/jpeg\r\n";
|
||||
file_header += "\r\n";
|
||||
|
||||
// 构造尾部
|
||||
std::string multipart_footer;
|
||||
multipart_footer += "\r\n--" + boundary + "--\r\n";
|
||||
|
||||
// 配置HTTP客户端,使用分块传输编码
|
||||
http->SetHeader("Authorization", "Bearer " + explain_token_);
|
||||
http->SetHeader("Content-Type", "multipart/form-data; boundary=" + boundary);
|
||||
http->SetHeader("Transfer-Encoding", "chunked");
|
||||
if (!http->Open("POST", explain_url_)) {
|
||||
ESP_LOGE(TAG, "Failed to connect to explain URL");
|
||||
// Clear the queue
|
||||
encoder_thread_.join();
|
||||
JpegChunk chunk;
|
||||
while (xQueueReceive(jpeg_queue_, &chunk, portMAX_DELAY) == pdPASS) {
|
||||
if (chunk.data != nullptr) {
|
||||
heap_caps_free(chunk.data);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return "{\"success\": false, \"message\": \"Failed to connect to explain URL\"}";
|
||||
}
|
||||
|
||||
// 第一块:question字段
|
||||
http->Write(question_field.c_str(), question_field.size());
|
||||
|
||||
// 第二块:文件字段头部
|
||||
http->Write(file_header.c_str(), file_header.size());
|
||||
|
||||
// 第三块:JPEG数据
|
||||
size_t total_sent = 0;
|
||||
while (true) {
|
||||
JpegChunk chunk;
|
||||
if (xQueueReceive(jpeg_queue_, &chunk, portMAX_DELAY) != pdPASS) {
|
||||
ESP_LOGE(TAG, "Failed to receive JPEG chunk");
|
||||
break;
|
||||
}
|
||||
if (chunk.data == nullptr) {
|
||||
break; // The last chunk
|
||||
}
|
||||
http->Write((const char*)chunk.data, chunk.len);
|
||||
total_sent += chunk.len;
|
||||
heap_caps_free(chunk.data);
|
||||
}
|
||||
|
||||
// 第四块:multipart尾部
|
||||
http->Write(multipart_footer.c_str(), multipart_footer.size());
|
||||
|
||||
// 结束块
|
||||
http->Write("", 0);
|
||||
|
||||
if (http->GetStatusCode() != 200) {
|
||||
ESP_LOGE(TAG, "Failed to upload photo, status code: %d", http->GetStatusCode());
|
||||
return "{\"success\": false, \"message\": \"Failed to upload photo\"}";
|
||||
}
|
||||
|
||||
std::string result = http->ReadAll();
|
||||
http->Close();
|
||||
|
||||
ESP_LOGI(TAG, "Explain image size=%dx%d, compressed size=%d, question=%s\n%s", fb_->width, fb_->height, total_sent, question.c_str(), result.c_str());
|
||||
return result;
|
||||
}
|
||||
38
main/boards/common/esp32_camera.h
Normal file
38
main/boards/common/esp32_camera.h
Normal file
@@ -0,0 +1,38 @@
|
||||
#ifndef ESP32_CAMERA_H
|
||||
#define ESP32_CAMERA_H
|
||||
|
||||
#include <esp_camera.h>
|
||||
#include <lvgl.h>
|
||||
#include <thread>
|
||||
#include <memory>
|
||||
|
||||
#include <freertos/FreeRTOS.h>
|
||||
#include <freertos/queue.h>
|
||||
|
||||
#include "camera.h"
|
||||
|
||||
struct JpegChunk {
|
||||
uint8_t* data;
|
||||
size_t len;
|
||||
};
|
||||
|
||||
class Esp32Camera : public Camera {
|
||||
private:
|
||||
camera_fb_t* fb_ = nullptr;
|
||||
lv_img_dsc_t preview_image_;
|
||||
std::string explain_url_;
|
||||
std::string explain_token_;
|
||||
std::thread preview_thread_;
|
||||
std::thread encoder_thread_;
|
||||
|
||||
QueueHandle_t jpeg_queue_ = nullptr;
|
||||
public:
|
||||
Esp32Camera(const camera_config_t& config);
|
||||
~Esp32Camera();
|
||||
|
||||
virtual void SetExplainUrl(const std::string& url, const std::string& token);
|
||||
virtual bool Capture();
|
||||
virtual std::string Explain(const std::string& question);
|
||||
};
|
||||
|
||||
#endif // ESP32_CAMERA_H
|
||||
@@ -37,5 +37,26 @@
|
||||
#define DISPLAY_BACKLIGHT_PIN GPIO_NUM_42
|
||||
#define DISPLAY_BACKLIGHT_OUTPUT_INVERT true
|
||||
|
||||
/* Camera pins */
|
||||
#define CAMERA_PIN_PWDN -1
|
||||
#define CAMERA_PIN_RESET -1
|
||||
#define CAMERA_PIN_XCLK 5
|
||||
#define CAMERA_PIN_SIOD 1
|
||||
#define CAMERA_PIN_SIOC 2
|
||||
|
||||
#define CAMERA_PIN_D7 9
|
||||
#define CAMERA_PIN_D6 4
|
||||
#define CAMERA_PIN_D5 6
|
||||
#define CAMERA_PIN_D4 15
|
||||
#define CAMERA_PIN_D3 17
|
||||
#define CAMERA_PIN_D2 8
|
||||
#define CAMERA_PIN_D1 18
|
||||
#define CAMERA_PIN_D0 16
|
||||
#define CAMERA_PIN_VSYNC 3
|
||||
#define CAMERA_PIN_HREF 46
|
||||
#define CAMERA_PIN_PCLK 7
|
||||
|
||||
#define XCLK_FREQ_HZ 24000000
|
||||
|
||||
|
||||
#endif // _BOARD_CONFIG_H_
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
#include "config.h"
|
||||
#include "i2c_device.h"
|
||||
#include "iot/thing_manager.h"
|
||||
#include "esp32_camera.h"
|
||||
|
||||
#include <esp_log.h>
|
||||
#include <esp_lcd_panel_vendor.h>
|
||||
@@ -36,6 +37,36 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
class CustomAudioCodec : public BoxAudioCodec {
|
||||
private:
|
||||
Pca9557* pca9557_;
|
||||
|
||||
public:
|
||||
CustomAudioCodec(i2c_master_bus_handle_t i2c_bus, Pca9557* pca9557)
|
||||
: BoxAudioCodec(i2c_bus,
|
||||
AUDIO_INPUT_SAMPLE_RATE,
|
||||
AUDIO_OUTPUT_SAMPLE_RATE,
|
||||
AUDIO_I2S_GPIO_MCLK,
|
||||
AUDIO_I2S_GPIO_BCLK,
|
||||
AUDIO_I2S_GPIO_WS,
|
||||
AUDIO_I2S_GPIO_DOUT,
|
||||
AUDIO_I2S_GPIO_DIN,
|
||||
GPIO_NUM_NC,
|
||||
AUDIO_CODEC_ES8311_ADDR,
|
||||
AUDIO_CODEC_ES7210_ADDR,
|
||||
AUDIO_INPUT_REFERENCE),
|
||||
pca9557_(pca9557) {
|
||||
}
|
||||
|
||||
virtual void EnableOutput(bool enable) override {
|
||||
BoxAudioCodec::EnableOutput(enable);
|
||||
if (enable) {
|
||||
pca9557_->SetOutputState(1, 1);
|
||||
} else {
|
||||
pca9557_->SetOutputState(1, 0);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
class LichuangDevBoard : public WifiBoard {
|
||||
private:
|
||||
@@ -44,6 +75,7 @@ private:
|
||||
Button boot_button_;
|
||||
LcdDisplay* display_;
|
||||
Pca9557* pca9557_;
|
||||
Esp32Camera* camera_;
|
||||
|
||||
void InitializeI2c() {
|
||||
// Initialize I2C peripheral
|
||||
@@ -164,11 +196,39 @@ private:
|
||||
lvgl_port_add_touch(&touch_cfg);
|
||||
}
|
||||
|
||||
// 物联网初始化,添加对 AI 可见设备
|
||||
void InitializeIot() {
|
||||
auto& thing_manager = iot::ThingManager::GetInstance();
|
||||
thing_manager.AddThing(iot::CreateThing("Speaker"));
|
||||
thing_manager.AddThing(iot::CreateThing("Screen"));
|
||||
void InitializeCamera() {
|
||||
// Open camera power
|
||||
pca9557_->SetOutputState(2, 0);
|
||||
|
||||
camera_config_t config = {};
|
||||
config.ledc_channel = LEDC_CHANNEL_2; // LEDC通道选择 用于生成XCLK时钟 但是S3不用
|
||||
config.ledc_timer = LEDC_TIMER_2; // LEDC timer选择 用于生成XCLK时钟 但是S3不用
|
||||
config.pin_d0 = CAMERA_PIN_D0;
|
||||
config.pin_d1 = CAMERA_PIN_D1;
|
||||
config.pin_d2 = CAMERA_PIN_D2;
|
||||
config.pin_d3 = CAMERA_PIN_D3;
|
||||
config.pin_d4 = CAMERA_PIN_D4;
|
||||
config.pin_d5 = CAMERA_PIN_D5;
|
||||
config.pin_d6 = CAMERA_PIN_D6;
|
||||
config.pin_d7 = CAMERA_PIN_D7;
|
||||
config.pin_xclk = CAMERA_PIN_XCLK;
|
||||
config.pin_pclk = CAMERA_PIN_PCLK;
|
||||
config.pin_vsync = CAMERA_PIN_VSYNC;
|
||||
config.pin_href = CAMERA_PIN_HREF;
|
||||
config.pin_sccb_sda = -1; // 这里写-1 表示使用已经初始化的I2C接口
|
||||
config.pin_sccb_scl = CAMERA_PIN_SIOC;
|
||||
config.sccb_i2c_port = 1;
|
||||
config.pin_pwdn = CAMERA_PIN_PWDN;
|
||||
config.pin_reset = CAMERA_PIN_RESET;
|
||||
config.xclk_freq_hz = XCLK_FREQ_HZ;
|
||||
config.pixel_format = PIXFORMAT_RGB565;
|
||||
config.frame_size = FRAMESIZE_VGA;
|
||||
config.jpeg_quality = 12;
|
||||
config.fb_count = 1;
|
||||
config.fb_location = CAMERA_FB_IN_PSRAM;
|
||||
config.grab_mode = CAMERA_GRAB_WHEN_EMPTY;
|
||||
|
||||
camera_ = new Esp32Camera(config);
|
||||
}
|
||||
|
||||
public:
|
||||
@@ -178,24 +238,20 @@ public:
|
||||
InitializeSt7789Display();
|
||||
InitializeTouch();
|
||||
InitializeButtons();
|
||||
InitializeIot();
|
||||
InitializeCamera();
|
||||
|
||||
#if CONFIG_IOT_PROTOCOL_XIAOZHI
|
||||
auto& thing_manager = iot::ThingManager::GetInstance();
|
||||
thing_manager.AddThing(iot::CreateThing("Speaker"));
|
||||
thing_manager.AddThing(iot::CreateThing("Screen"));
|
||||
#endif
|
||||
GetBacklight()->RestoreBrightness();
|
||||
}
|
||||
|
||||
virtual AudioCodec* GetAudioCodec() override {
|
||||
static BoxAudioCodec audio_codec(
|
||||
static CustomAudioCodec audio_codec(
|
||||
i2c_bus_,
|
||||
AUDIO_INPUT_SAMPLE_RATE,
|
||||
AUDIO_OUTPUT_SAMPLE_RATE,
|
||||
AUDIO_I2S_GPIO_MCLK,
|
||||
AUDIO_I2S_GPIO_BCLK,
|
||||
AUDIO_I2S_GPIO_WS,
|
||||
AUDIO_I2S_GPIO_DOUT,
|
||||
AUDIO_I2S_GPIO_DIN,
|
||||
GPIO_NUM_NC,
|
||||
AUDIO_CODEC_ES8311_ADDR,
|
||||
AUDIO_CODEC_ES7210_ADDR,
|
||||
AUDIO_INPUT_REFERENCE);
|
||||
pca9557_);
|
||||
return &audio_codec;
|
||||
}
|
||||
|
||||
@@ -207,6 +263,10 @@ public:
|
||||
static PwmBacklight backlight(DISPLAY_BACKLIGHT_PIN, DISPLAY_BACKLIGHT_OUTPUT_INVERT);
|
||||
return &backlight;
|
||||
}
|
||||
|
||||
virtual Camera* GetCamera() override {
|
||||
return camera_;
|
||||
}
|
||||
};
|
||||
|
||||
DECLARE_BOARD(LichuangDevBoard);
|
||||
|
||||
@@ -11,13 +11,14 @@ dependencies:
|
||||
78/esp_lcd_nv3023: ~1.0.0
|
||||
78/esp-wifi-connect: ~2.4.2
|
||||
78/esp-opus-encoder: ~2.3.2
|
||||
78/esp-ml307: ~2.0.3
|
||||
78/esp-ml307: ~2.1.0
|
||||
78/xiaozhi-fonts: ~1.3.2
|
||||
espressif/led_strip: ^2.5.5
|
||||
espressif/esp_codec_dev: ~1.3.2
|
||||
espressif/esp-sr: ~2.1.1
|
||||
espressif/button: ~4.1.3
|
||||
espressif/knob: ^1.0.0
|
||||
espressif/esp32-camera: ^2.0.15
|
||||
espressif/esp_lcd_touch_ft5x06: ~1.0.7
|
||||
espressif/esp_lcd_touch_gt911: ^1
|
||||
waveshare/esp_lcd_touch_cst9217: ^1.0.3
|
||||
|
||||
@@ -19,15 +19,24 @@ McpServer::McpServer() {
|
||||
AddCommonTools();
|
||||
}
|
||||
|
||||
McpServer::~McpServer() {
|
||||
for (auto tool : tools_) {
|
||||
delete tool;
|
||||
}
|
||||
tools_.clear();
|
||||
}
|
||||
|
||||
void McpServer::AddCommonTools() {
|
||||
auto& board = Board::GetInstance();
|
||||
|
||||
AddTool("self.get_device_status",
|
||||
"Provides the real-time information of the device, including the current status of the audio speaker, screen, battery, network, etc.\n"
|
||||
"Use this tool for: \n"
|
||||
"1. Answering questions about current condition (e.g. what is the current volume of the audio speaker?)\n"
|
||||
"2. As the first step to control the device (e.g. turn up / down the volume of the audio speaker, etc.)",
|
||||
PropertyList(),
|
||||
[](const PropertyList& properties) -> ReturnValue {
|
||||
return Board::GetInstance().GetDeviceStatusJson();
|
||||
[&board](const PropertyList& properties) -> ReturnValue {
|
||||
return board.GetDeviceStatusJson();
|
||||
});
|
||||
|
||||
AddTool("self.audio_speaker.set_volume",
|
||||
@@ -35,47 +44,67 @@ void McpServer::AddCommonTools() {
|
||||
PropertyList({
|
||||
Property("volume", kPropertyTypeInteger, 0, 100)
|
||||
}),
|
||||
[](const PropertyList& properties) -> ReturnValue {
|
||||
auto codec = Board::GetInstance().GetAudioCodec();
|
||||
[&board](const PropertyList& properties) -> ReturnValue {
|
||||
auto codec = board.GetAudioCodec();
|
||||
codec->SetOutputVolume(properties["volume"].value<int>());
|
||||
return true;
|
||||
});
|
||||
|
||||
AddTool("self.screen.set_brightness",
|
||||
"Set the brightness of the screen.",
|
||||
PropertyList({
|
||||
Property("brightness", kPropertyTypeInteger, 0, 100)
|
||||
}),
|
||||
[](const PropertyList& properties) -> ReturnValue {
|
||||
uint8_t brightness = static_cast<uint8_t>(properties["brightness"].value<int>());
|
||||
auto backlight = Board::GetInstance().GetBacklight();
|
||||
if (backlight) {
|
||||
auto backlight = board.GetBacklight();
|
||||
if (backlight) {
|
||||
AddTool("self.screen.set_brightness",
|
||||
"Set the brightness of the screen.",
|
||||
PropertyList({
|
||||
Property("brightness", kPropertyTypeInteger, 0, 100)
|
||||
}),
|
||||
[backlight](const PropertyList& properties) -> ReturnValue {
|
||||
uint8_t brightness = static_cast<uint8_t>(properties["brightness"].value<int>());
|
||||
backlight->SetBrightness(brightness, true);
|
||||
}
|
||||
return true;
|
||||
});
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
AddTool("self.screen.set_theme",
|
||||
"Set the theme of the screen. The theme can be 'light' or 'dark'.",
|
||||
PropertyList({
|
||||
Property("theme", kPropertyTypeString)
|
||||
}),
|
||||
[](const PropertyList& properties) -> ReturnValue {
|
||||
auto display = Board::GetInstance().GetDisplay();
|
||||
if (display) {
|
||||
auto display = board.GetDisplay();
|
||||
if (display && !display->GetTheme().empty()) {
|
||||
AddTool("self.screen.set_theme",
|
||||
"Set the theme of the screen. The theme can be `light` or `dark`.",
|
||||
PropertyList({
|
||||
Property("theme", kPropertyTypeString)
|
||||
}),
|
||||
[display](const PropertyList& properties) -> ReturnValue {
|
||||
display->SetTheme(properties["theme"].value<std::string>().c_str());
|
||||
}
|
||||
return true;
|
||||
});
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
auto camera = board.GetCamera();
|
||||
if (camera) {
|
||||
AddTool("self.camera.take_photo",
|
||||
"Take a photo and explain it. Use this tool after the user asks you to see something.\n"
|
||||
"Args:\n"
|
||||
" `question`: The question that you want to ask about the photo.\n"
|
||||
"Return:\n"
|
||||
" A JSON object that provides the photo information.",
|
||||
PropertyList({
|
||||
Property("question", kPropertyTypeString)
|
||||
}),
|
||||
[camera](const PropertyList& properties) -> ReturnValue {
|
||||
if (!camera->Capture()) {
|
||||
return "{\"success\": false, \"message\": \"Failed to capture photo\"}";
|
||||
}
|
||||
auto question = properties["question"].value<std::string>();
|
||||
return camera->Explain(question);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void McpServer::AddTool(McpTool* tool) {
|
||||
ESP_LOGI(TAG, "Add tool: %s", tool->name().c_str());
|
||||
tools_.push_back(tool);
|
||||
}
|
||||
|
||||
void McpServer::AddTool(const std::string& name, const std::string& description, const PropertyList& properties, std::function<ReturnValue(const PropertyList&)> callback) {
|
||||
tools_.push_back(new McpTool(name, description, properties, callback));
|
||||
AddTool(new McpTool(name, description, properties, callback));
|
||||
}
|
||||
|
||||
void McpServer::ParseMessage(const std::string& message) {
|
||||
@@ -88,6 +117,21 @@ void McpServer::ParseMessage(const std::string& message) {
|
||||
cJSON_Delete(json);
|
||||
}
|
||||
|
||||
void McpServer::ParseCapabilities(const cJSON* capabilities) {
|
||||
auto vision = cJSON_GetObjectItem(capabilities, "vision");
|
||||
if (cJSON_IsObject(vision)) {
|
||||
auto url = cJSON_GetObjectItem(vision, "url");
|
||||
auto token = cJSON_GetObjectItem(vision, "token");
|
||||
if (cJSON_IsString(url) && cJSON_IsString(token)) {
|
||||
ESP_LOGI(TAG, "Setting explain URL: %s, token: %s", url->valuestring, token->valuestring);
|
||||
auto camera = Board::GetInstance().GetCamera();
|
||||
if (camera) {
|
||||
camera->SetExplainUrl(std::string(url->valuestring), std::string(token->valuestring));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void McpServer::ParseMessage(const cJSON* json) {
|
||||
// Check JSONRPC version
|
||||
auto version = cJSON_GetObjectItem(json, "jsonrpc");
|
||||
@@ -123,6 +167,12 @@ void McpServer::ParseMessage(const cJSON* json) {
|
||||
auto id_int = id->valueint;
|
||||
|
||||
if (method_str == "initialize") {
|
||||
if (cJSON_IsObject(params)) {
|
||||
auto capabilities = cJSON_GetObjectItem(params, "capabilities");
|
||||
if (cJSON_IsObject(capabilities)) {
|
||||
ParseCapabilities(capabilities);
|
||||
}
|
||||
}
|
||||
auto app_desc = esp_app_get_description();
|
||||
std::string message = "{\"protocolVersion\":\"2024-11-05\",\"capabilities\":{\"tools\":{}},\"serverInfo\":{\"name\":\"" BOARD_NAME "\",\"version\":\"";
|
||||
message += app_desc->version;
|
||||
|
||||
@@ -262,9 +262,10 @@ public:
|
||||
|
||||
private:
|
||||
McpServer();
|
||||
~McpServer() = default;
|
||||
~McpServer();
|
||||
|
||||
void AddCommonTools();
|
||||
void ParseCapabilities(const cJSON* capabilities);
|
||||
|
||||
void ReplyResult(int id, const std::string& result);
|
||||
void ReplyError(int id, const std::string& message);
|
||||
|
||||
40
main/ota.cc
40
main/ota.cc
@@ -84,18 +84,25 @@ bool Ota::CheckVersion() {
|
||||
return false;
|
||||
}
|
||||
|
||||
auto http = SetupHttp();
|
||||
auto http = std::unique_ptr<Http>(SetupHttp());
|
||||
|
||||
std::string data = board.GetJson();
|
||||
std::string method = data.length() > 0 ? "POST" : "GET";
|
||||
if (!http->Open(method, check_version_url_, data)) {
|
||||
http->SetContent(std::move(data));
|
||||
|
||||
if (!http->Open(method, check_version_url_)) {
|
||||
ESP_LOGE(TAG, "Failed to open HTTP connection");
|
||||
delete http;
|
||||
return false;
|
||||
}
|
||||
|
||||
data = http->GetBody();
|
||||
delete http;
|
||||
auto status_code = http->GetStatusCode();
|
||||
if (status_code != 200) {
|
||||
ESP_LOGE(TAG, "Failed to check version, status code: %d", status_code);
|
||||
return false;
|
||||
}
|
||||
|
||||
data = http->ReadAll();
|
||||
http->Close();
|
||||
|
||||
// Response: { "firmware": { "version": "1.0.0", "url": "http://" } }
|
||||
// Parse the JSON response and check if the version is newer
|
||||
@@ -257,17 +264,15 @@ void Ota::Upgrade(const std::string& firmware_url) {
|
||||
bool image_header_checked = false;
|
||||
std::string image_header;
|
||||
|
||||
auto http = Board::GetInstance().CreateHttp();
|
||||
auto http = std::unique_ptr<Http>(Board::GetInstance().CreateHttp());
|
||||
if (!http->Open("GET", firmware_url)) {
|
||||
ESP_LOGE(TAG, "Failed to open HTTP connection");
|
||||
delete http;
|
||||
return;
|
||||
}
|
||||
|
||||
size_t content_length = http->GetBodyLength();
|
||||
if (content_length == 0) {
|
||||
ESP_LOGE(TAG, "Failed to get content length");
|
||||
delete http;
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -278,7 +283,6 @@ void Ota::Upgrade(const std::string& firmware_url) {
|
||||
int ret = http->Read(buffer, sizeof(buffer));
|
||||
if (ret < 0) {
|
||||
ESP_LOGE(TAG, "Failed to read HTTP data: %s", esp_err_to_name(ret));
|
||||
delete http;
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -309,13 +313,11 @@ void Ota::Upgrade(const std::string& firmware_url) {
|
||||
auto current_version = esp_app_get_description()->version;
|
||||
if (memcmp(new_app_info.version, current_version, sizeof(new_app_info.version)) == 0) {
|
||||
ESP_LOGE(TAG, "Firmware version is the same, skipping upgrade");
|
||||
delete http;
|
||||
return;
|
||||
}
|
||||
|
||||
if (esp_ota_begin(update_partition, OTA_WITH_SEQUENTIAL_WRITES, &update_handle)) {
|
||||
esp_ota_abort(update_handle);
|
||||
delete http;
|
||||
ESP_LOGE(TAG, "Failed to begin OTA");
|
||||
return;
|
||||
}
|
||||
@@ -328,11 +330,10 @@ void Ota::Upgrade(const std::string& firmware_url) {
|
||||
if (err != ESP_OK) {
|
||||
ESP_LOGE(TAG, "Failed to write OTA data: %s", esp_err_to_name(err));
|
||||
esp_ota_abort(update_handle);
|
||||
delete http;
|
||||
return;
|
||||
}
|
||||
}
|
||||
delete http;
|
||||
http->Close();
|
||||
|
||||
esp_err_t err = esp_ota_end(update_handle);
|
||||
if (err != ESP_OK) {
|
||||
@@ -437,25 +438,22 @@ esp_err_t Ota::Activate() {
|
||||
url += "activate";
|
||||
}
|
||||
|
||||
auto http = SetupHttp();
|
||||
auto http = std::unique_ptr<Http>(SetupHttp());
|
||||
|
||||
std::string data = GetActivationPayload();
|
||||
if (!http->Open("POST", url, data)) {
|
||||
http->SetContent(std::move(data));
|
||||
|
||||
if (!http->Open("POST", url)) {
|
||||
ESP_LOGE(TAG, "Failed to open HTTP connection");
|
||||
delete http;
|
||||
return ESP_FAIL;
|
||||
}
|
||||
|
||||
auto status_code = http->GetStatusCode();
|
||||
data = http->GetBody();
|
||||
http->Close();
|
||||
delete http;
|
||||
|
||||
if (status_code == 202) {
|
||||
return ESP_ERR_TIMEOUT;
|
||||
}
|
||||
if (status_code != 200) {
|
||||
ESP_LOGE(TAG, "Failed to activate, code: %d, body: %s", status_code, data.c_str());
|
||||
ESP_LOGE(TAG, "Failed to activate, code: %d, body: %s", status_code, http->ReadAll().c_str());
|
||||
return ESP_FAIL;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user