devttl/xiaozhi-esp32
1
0
Fork 0
forked from xiaozhi/xiaozhi-esp32
Code Pull Requests Activity

Feat sscma camera (#770)

Browse Source 
* feat: add sscma camera

* fix: 修复sscma camera 无法使用等问题

* feat: update README.md

* style: optimize styles

* style: fix styles

* Update sscma_camera.h

---------

Co-authored-by: Xiaoxia <terrence@tenclass.com>
This commit is contained in:
virgil
2025-06-06 14:22:39 +08:00
committed by GitHub
parent 8729fad9a8
commit 5c51fbc2f8
6 changed files with 490 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
main/boards/sensecap-watcher/README.md
View File

@@ -18,6 +18,17 @@ idf.py menuconfig
Xiaozhi Assistant -> Board Type -> SenseCAP Watcher
```
watcher 中一些额外的配置项如下，需要menuconfig 选择， 或者拷贝放入sdkconfig.defaults中.
```
CONFIG_BOARD_TYPE_SENSECAP_WATCHER=y
CONFIG_ESPTOOLPY_FLASHSIZE_32MB=y
CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions/v1/32m.csv"
CONFIG_BOOTLOADER_CACHE_32BIT_ADDR_QUAD_FLASH=y
CONFIG_ESPTOOLPY_FLASH_MODE_AUTO_DETECT=n
CONFIG_IDF_EXPERIMENTAL_FEATURES=y
```
**编译烧入：**
```bash

46
main/boards/sensecap-watcher/config.h
View File

@@ -2,8 +2,29 @@
#define _BOARD_CONFIG_H_
#include <driver/gpio.h>
#include "driver/spi_common.h"
#include "esp_io_expander.h"
// SSCMA Client Configuration
#define CONFIG_SSCMA_EVENT_QUEUE_SIZE 1
#define CONFIG_SSCMA_TX_BUFFER_SIZE 8192
#define CONFIG_SSCMA_RX_BUFFER_SIZE 98304
// SSCMA Client Process Task
#define CONFIG_SSCMA_PROCESS_TASK_STACK_SIZE 10240
#define CONFIG_SSCMA_PROCESS_TASK_PRIORITY 5
#define CONFIG_SSCMA_PROCESS_TASK_AFFINITY_CPU1 1
#define CONFIG_SSCMA_PROCESS_TASK_AFFINITY 1
#define CONFIG_SSCMA_PROCESS_TASK_STACK_ALLOC_EXTERNAL 1
// SSCMA Client Monitor Task
#define CONFIG_SSCMA_MONITOR_TASK_STACK_SIZE 10240
#define CONFIG_SSCMA_MONITOR_TASK_PRIORITY 4
#define CONFIG_SSCMA_MONITOR_TASK_AFFINITY_CPU1 1
#define CONFIG_SSCMA_MONITOR_TASK_AFFINITY 1
#define CONFIG_SSCMA_MONITOR_TASK_STACK_ALLOC_EXTERNAL 1
#define CONFIG_SSCMA_ALLOC_SMALL_SHORTTERM_MEM_EXTERNALLY 1
/* General I2C */
#define BSP_GENERAL_I2C_NUM (I2C_NUM_0)
#define BSP_GENERAL_I2C_SDA (GPIO_NUM_47)
@@ -57,6 +78,16 @@
#define BSP_KNOB_A_PIN GPIO_NUM_41
#define BSP_KNOB_B_PIN GPIO_NUM_42
/* SPI */
#define BSP_SPI2_HOST_SCLK (GPIO_NUM_4)
#define BSP_SPI2_HOST_MOSI (GPIO_NUM_5)
#define BSP_SPI2_HOST_MISO (GPIO_NUM_6)
/* SD Card */
#define BSP_SD_SPI_NUM (SPI2_HOST)
#define BSP_SD_SPI_CS (GPIO_NUM_46)
#define BSP_SD_GPIO_DET (IO_EXPANDER_PIN_NUM_4)
/* QSPI */
#define BSP_SPI3_HOST_PCLK (GPIO_NUM_7)
#define BSP_SPI3_HOST_DATA0 (GPIO_NUM_9)
@@ -103,4 +134,19 @@
#define BSP_BAT_ADC_ATTEN (ADC_ATTEN_DB_2_5) // 0 ~ 1100 mV
#define BSP_BAT_VOL_RATIO ((62 + 20) / 20)
/* Himax */
#define BSP_SSCMA_CLIENT_RST (IO_EXPANDER_PIN_NUM_7)
#define BSP_SSCMA_CLIENT_RST_USE_EXPANDER (true)
#define BSP_SSCMA_CLIENT_SPI_NUM (SPI2_HOST)
#define BSP_SSCMA_CLIENT_SPI_CS (GPIO_NUM_21)
#define BSP_SSCMA_CLIENT_SPI_SYNC (IO_EXPANDER_PIN_NUM_6)
#define BSP_SSCMA_CLIENT_SPI_SYNC_USE_EXPANDER (true)
#define BSP_SSCMA_CLIENT_SPI_CLK (12 * 1000 * 1000)
#define BSP_SSCMA_FLASHER_UART_NUM (UART_NUM_1)
#define BSP_SSCMA_FLASHER_UART_TX (GPIO_NUM_17)
#define BSP_SSCMA_FLASHER_UART_RX (GPIO_NUM_18)
#define BSP_SSCMA_FLASHER_UART_BAUD_RATE (921600)
#endif // _BOARD_CONFIG_H_

41
main/boards/sensecap-watcher/sensecap_watcher.cc
View File

@@ -10,6 +10,7 @@
#include "led/single_led.h"
#include "iot/thing_manager.h"
#include "power_save_timer.h"
#include "sscma_camera.h"
#include <esp_log.h>
#include "esp_check.h"
@@ -97,6 +98,7 @@ private:
uint32_t long_press_cnt_;
button_driver_t* btn_driver_ = nullptr;
static SensecapWatcher* instance_;
SscmaCamera* camera_ = nullptr;
void InitializePowerSaveTimer() {
power_save_timer_ = new PowerSaveTimer(-1, 60, 300);
@@ -288,6 +290,19 @@ private:
}
void InitializeSpi() {
ESP_LOGI(TAG, "Initialize SSCMA SPI bus");
spi_bus_config_t spi_cfg = {0};
spi_cfg.mosi_io_num = BSP_SPI2_HOST_MOSI;
spi_cfg.miso_io_num = BSP_SPI2_HOST_MISO;
spi_cfg.sclk_io_num = BSP_SPI2_HOST_SCLK;
spi_cfg.quadwp_io_num = -1;
spi_cfg.quadhd_io_num = -1;
spi_cfg.isr_cpu_id = ESP_INTR_CPU_AFFINITY_1;
spi_cfg.max_transfer_sz = 4095;
ESP_ERROR_CHECK(spi_bus_initialize(SPI2_HOST, &spi_cfg, SPI_DMA_CH_AUTO));
ESP_LOGI(TAG, "Initialize QSPI bus");
spi_bus_config_t qspi_cfg = {0};
@@ -501,6 +516,27 @@ private:
ESP_ERROR_CHECK(esp_console_start_repl(repl));
}
void InitializeCamera() {
ESP_LOGI(TAG, "Initialize Camera");
// !!!NOTE: SD Card use same SPI bus as sscma client, so we need to disable SD card CS pin first
const gpio_config_t io_config = {
.pin_bit_mask = (1ULL << BSP_SD_SPI_CS),
.mode = GPIO_MODE_OUTPUT,
.pull_up_en = GPIO_PULLUP_ENABLE,
.pull_down_en = GPIO_PULLDOWN_DISABLE,
.intr_type = GPIO_INTR_DISABLE,
};
esp_err_t ret = gpio_config(&io_config);
if (ret != ESP_OK)
return;
gpio_set_level(BSP_SD_SPI_CS, 1);
camera_ = new SscmaCamera(io_exp_handle);
}
public:
SensecapWatcher() {
ESP_LOGI(TAG, "Initialize Sensecap Watcher");
@@ -512,6 +548,7 @@ public:
InitializeButton();
InitializeKnob();
Initializespd2010Display();
InitializeCamera();
InitializeIot();
GetBacklight()->RestoreBrightness();
}
@@ -573,6 +610,10 @@ public:
}
return true;
}
virtual Camera* GetCamera() override {
return camera_;
}
};
DECLARE_BOARD(SensecapWatcher);

334
main/boards/sensecap-watcher/sscma_camera.cc Normal file
View File

@@ -0,0 +1,334 @@
#include "sscma_camera.h"
#include "mcp_server.h"
#include "display.h"
#include "board.h"
#include "system_info.h"
#include "config.h"
#include <esp_log.h>
#include <esp_heap_caps.h>
#include <img_converters.h>
#include <cstring>
#define TAG "SscmaCamera"
#define IMG_JPEG_BUF_SIZE 48 * 1024
SscmaCamera::SscmaCamera(esp_io_expander_handle_t io_exp_handle) {
sscma_client_io_spi_config_t spi_io_config = {0};
spi_io_config.sync_gpio_num = BSP_SSCMA_CLIENT_SPI_SYNC;
spi_io_config.cs_gpio_num = BSP_SSCMA_CLIENT_SPI_CS;
spi_io_config.pclk_hz = BSP_SSCMA_CLIENT_SPI_CLK;
spi_io_config.spi_mode = 0;
spi_io_config.wait_delay = 10; //两个transfer之间至少延时4ms,但当前 FREERTOS_HZ=100, 延时精度只能达到10ms,
spi_io_config.user_ctx = NULL;
spi_io_config.io_expander = io_exp_handle;
spi_io_config.flags.sync_use_expander = BSP_SSCMA_CLIENT_RST_USE_EXPANDER;
sscma_client_new_io_spi_bus((sscma_client_spi_bus_handle_t)BSP_SSCMA_CLIENT_SPI_NUM, &spi_io_config, &sscma_client_io_handle_);
sscma_client_config_t sscma_client_config = SSCMA_CLIENT_CONFIG_DEFAULT();
sscma_client_config.event_queue_size = CONFIG_SSCMA_EVENT_QUEUE_SIZE;
sscma_client_config.tx_buffer_size = CONFIG_SSCMA_TX_BUFFER_SIZE;
sscma_client_config.rx_buffer_size = CONFIG_SSCMA_RX_BUFFER_SIZE;
sscma_client_config.process_task_stack = CONFIG_SSCMA_PROCESS_TASK_STACK_SIZE;
sscma_client_config.process_task_affinity = CONFIG_SSCMA_PROCESS_TASK_AFFINITY;
sscma_client_config.process_task_priority = CONFIG_SSCMA_PROCESS_TASK_PRIORITY;
sscma_client_config.monitor_task_stack = CONFIG_SSCMA_MONITOR_TASK_STACK_SIZE;
sscma_client_config.monitor_task_affinity = CONFIG_SSCMA_MONITOR_TASK_AFFINITY;
sscma_client_config.monitor_task_priority = CONFIG_SSCMA_MONITOR_TASK_PRIORITY;
sscma_client_config.reset_gpio_num = BSP_SSCMA_CLIENT_RST;
sscma_client_config.io_expander = io_exp_handle;
sscma_client_config.flags.reset_use_expander = BSP_SSCMA_CLIENT_RST_USE_EXPANDER;
sscma_client_new(sscma_client_io_handle_, &sscma_client_config, &sscma_client_handle_);
sscma_data_queue_ = xQueueCreate(1, sizeof(SscmaData));
sscma_client_callback_t callback = {0};
callback.on_event = [](sscma_client_handle_t client, const sscma_client_reply_t *reply, void *user_ctx) {
SscmaCamera* self = static_cast<SscmaCamera*>(user_ctx);
if (!self) return;
char *img = NULL;
int img_size = 0;
if (sscma_utils_fetch_image_from_reply(reply, &img, &img_size) == ESP_OK)
{
ESP_LOGI(TAG, "image_size: %d\n", img_size);
// 将数据通过队列发送出去
SscmaData data;
data.img = (uint8_t*)img;
data.len = img_size;
// 清空队列，保证只保存最新的数据
SscmaData dummy;
while (xQueueReceive(self->sscma_data_queue_, &dummy, 0) == pdPASS) {
if (dummy.img) {
heap_caps_free(dummy.img);
}
}
xQueueSend(self->sscma_data_queue_, &data, 0);
// 注意：img 的释放由接收方负责
}
};
callback.on_connect = [](sscma_client_handle_t client, const sscma_client_reply_t *reply, void *user_ctx) {
ESP_LOGI(TAG, "SSCMA client connected");
};
callback.on_log = [](sscma_client_handle_t client, const sscma_client_reply_t *reply, void *user_ctx) {
ESP_LOGI(TAG, "log: %s\n", reply->data);
};
sscma_client_register_callback(sscma_client_handle_, &callback, this);
sscma_client_init(sscma_client_handle_);
// 获取设备信息
sscma_client_info_t *info;
if (sscma_client_get_info(sscma_client_handle_, &info, true) == ESP_OK) {
ESP_LOGI(TAG, "Device Info - ID: %s, Name: %s",
info->id ? info->id : "NULL",
info->name ? info->name : "NULL");
}
sscma_client_set_sensor(sscma_client_handle_, 1, 3, true); // 3 = 640x480
// 初始化JPEG数据的内存
jpeg_data_.len = 0;
jpeg_data_.buf = (uint8_t*)heap_caps_malloc(IMG_JPEG_BUF_SIZE, MALLOC_CAP_SPIRAM);;
if ( jpeg_data_.buf == nullptr ) {
ESP_LOGE(TAG, "Failed to allocate memory for JPEG buffer");
return;
}
//初始化JPEG解码
jpeg_dec_config_t config = { .output_type = JPEG_RAW_TYPE_RGB565_LE, .rotate = JPEG_ROTATE_0D };
jpeg_dec_ = jpeg_dec_open(&config);
if (!jpeg_dec_) {
ESP_LOGE(TAG, "Failed to open JPEG decoder");
return;
}
jpeg_io_ = (jpeg_dec_io_t*)heap_caps_malloc(sizeof(jpeg_dec_io_t), MALLOC_CAP_SPIRAM);
if (!jpeg_io_) {
ESP_LOGE(TAG, "Failed to allocate memory for JPEG IO");
jpeg_dec_close(jpeg_dec_);
return;
}
memset(jpeg_io_, 0, sizeof(jpeg_dec_io_t));
jpeg_out_ = (jpeg_dec_header_info_t*)heap_caps_aligned_alloc(16, sizeof(jpeg_dec_header_info_t), MALLOC_CAP_SPIRAM);
if (!jpeg_out_) {
ESP_LOGE(TAG, "Failed to allocate memory for JPEG output header");
heap_caps_free(jpeg_io_);
jpeg_dec_close(jpeg_dec_);
return;
}
memset(jpeg_out_, 0, sizeof(jpeg_dec_header_info_t));
// 初始化预览图片的内存
memset(&preview_image_, 0, sizeof(preview_image_));
preview_image_.header.magic = LV_IMAGE_HEADER_MAGIC;
preview_image_.header.cf = LV_COLOR_FORMAT_RGB565;
preview_image_.header.flags = LV_IMAGE_FLAGS_ALLOCATED | LV_IMAGE_FLAGS_MODIFIABLE;
preview_image_.header.w = 640;
preview_image_.header.h = 480;
preview_image_.header.stride = preview_image_.header.w * 2;
preview_image_.data_size = preview_image_.header.w * preview_image_.header.h * 2;
preview_image_.data = (uint8_t*)heap_caps_malloc(preview_image_.data_size, MALLOC_CAP_SPIRAM);
if (preview_image_.data == nullptr) {
ESP_LOGE(TAG, "Failed to allocate memory for preview image");
return;
}
}
SscmaCamera::~SscmaCamera() {
if (preview_image_.data) {
heap_caps_free((void*)preview_image_.data);
preview_image_.data = nullptr;
}
if (sscma_client_handle_) {
sscma_client_del(sscma_client_handle_);
}
if (sscma_data_queue_) {
vQueueDelete(sscma_data_queue_);
}
if (jpeg_data_.buf) {
heap_caps_free(jpeg_data_.buf);
jpeg_data_.buf = nullptr;
}
if (jpeg_dec_) {
jpeg_dec_close(jpeg_dec_);
jpeg_dec_ = nullptr;
}
if (jpeg_io_) {
heap_caps_free(jpeg_io_);
jpeg_io_ = nullptr;
}
if (jpeg_out_) {
heap_caps_free(jpeg_out_);
jpeg_out_ = nullptr;
}
}
void SscmaCamera::SetExplainUrl(const std::string& url, const std::string& token) {
explain_url_ = url;
explain_token_ = token;
}
bool SscmaCamera::Capture() {
SscmaData data;
size_t output_len = 0;
int ret = 0;
if (sscma_client_handle_ == nullptr) {
ESP_LOGE(TAG, "SSCMA client handle is not initialized");
return false;
}
ESP_LOGI(TAG, "Capturing image...");
// himax 有缓存数据,需要拍两张照片, 只获取最新的照片即可.
if (sscma_client_sample(sscma_client_handle_, 2) ) {
ESP_LOGE(TAG, "Failed to capture image from SSCMA client");
return false;
}
vTaskDelay(pdMS_TO_TICKS(500)); // 等待SSCMA客户端处理数据
if (xQueueReceive(sscma_data_queue_, &data, pdMS_TO_TICKS(1000)) != pdPASS) {
ESP_LOGE(TAG, "Failed to receive JPEG data from SSCMA client");
return false;
}
if (jpeg_data_.buf == nullptr) {
heap_caps_free(data.img);
return false;
}
ret = mbedtls_base64_decode(jpeg_data_.buf, IMG_JPEG_BUF_SIZE, &jpeg_data_.len, data.img, data.len);
if (ret != 0 || jpeg_data_.len == 0) {
ESP_LOGE(TAG, "Failed to decode base64 image data, ret: %d, output_len: %zu", ret, jpeg_data_.len);
heap_caps_free(data.img);
return false;
}
heap_caps_free(data.img);
//DECODE JPEG
if (!jpeg_dec_ || !jpeg_io_ || !jpeg_out_ || !preview_image_.data) {
return true;
}
jpeg_io_->inbuf = jpeg_data_.buf;
jpeg_io_->inbuf_len = jpeg_data_.len;
ret = jpeg_dec_parse_header(jpeg_dec_, jpeg_io_, jpeg_out_);
if (ret < 0) {
ESP_LOGE(TAG, "Failed to parse JPEG header, ret: %d", ret);
return true;
}
jpeg_io_->outbuf = (unsigned char*)preview_image_.data;
int inbuf_consumed = jpeg_io_->inbuf_len - jpeg_io_->inbuf_remain;
jpeg_io_->inbuf = jpeg_data_.buf + inbuf_consumed;
jpeg_io_->inbuf_len = jpeg_io_->inbuf_remain;
ret = jpeg_dec_process(jpeg_dec_, jpeg_io_);
if (ret != ESP_OK) {
ESP_LOGE(TAG, "Failed to decode JPEG image, ret: %d", ret);
return true;
}
// 显示预览图片
auto display = Board::GetInstance().GetDisplay();
if (display != nullptr) {
display->SetPreviewImage(&preview_image_);
}
return true;
}
bool SscmaCamera::SetHMirror(bool enabled) {
return false;
}
bool SscmaCamera::SetVFlip(bool enabled) {
return false;
}
/**
* @brief 将摄像头捕获的图像发送到远程服务器进行AI分析和解释
*
* 该函数将当前摄像头缓冲区中的图像编码为JPEG格式，并通过HTTP POST请求
* 以multipart/form-data的形式发送到指定的解释服务器。服务器将根据提供的
* 问题对图像进行AI分析并返回结果。
*
* @param question 要向AI提出的关于图像的问题，将作为表单字段发送
* @return std::string 服务器返回的JSON格式响应字符串
* 成功时包含AI分析结果，失败时包含错误信息
* 格式示例：{"success": true, "result": "分析结果"}
* {"success": false, "message": "错误信息"}
*
* @note 调用此函数前必须先调用SetExplainUrl()设置服务器URL
* @note 函数会等待之前的编码线程完成后再开始新的处理
* @warning 如果摄像头缓冲区为空或网络连接失败，将返回错误信息
*/
std::string SscmaCamera::Explain(const std::string& question) {
if (explain_url_.empty()) {
return "{\"success\": false, \"message\": \"Image explain URL or token is not set\"}";
}
auto http = Board::GetInstance().CreateHttp();
// 构造multipart/form-data请求体
std::string boundary = "----ESP32_CAMERA_BOUNDARY";
// 构造question字段
std::string question_field;
question_field += "--" + boundary + "\r\n";
question_field += "Content-Disposition: form-data; name=\"question\"\r\n";
question_field += "\r\n";
question_field += question + "\r\n";
// 构造文件字段头部
std::string file_header;
file_header += "--" + boundary + "\r\n";
file_header += "Content-Disposition: form-data; name=\"file\"; filename=\"camera.jpg\"\r\n";
file_header += "Content-Type: image/jpeg\r\n";
file_header += "\r\n";
// 构造尾部
std::string multipart_footer;
multipart_footer += "\r\n--" + boundary + "--\r\n";
// 配置HTTP客户端，使用分块传输编码
http->SetHeader("Device-Id", SystemInfo::GetMacAddress().c_str());
http->SetHeader("Client-Id", Board::GetInstance().GetUuid().c_str());
if (!explain_token_.empty()) {
http->SetHeader("Authorization", "Bearer " + explain_token_);
}
http->SetHeader("Content-Type", "multipart/form-data; boundary=" + boundary);
http->SetHeader("Transfer-Encoding", "chunked");
if (!http->Open("POST", explain_url_)) {
ESP_LOGE(TAG, "Failed to connect to explain URL");
return "{\"success\": false, \"message\": \"Failed to connect to explain URL\"}";
}
// 第一块：question字段
http->Write(question_field.c_str(), question_field.size());
// 第二块：文件字段头部
http->Write(file_header.c_str(), file_header.size());
// 第三块：JPEG数据
http->Write((const char*)jpeg_data_.buf, jpeg_data_.len);
// 第四块：multipart尾部
http->Write(multipart_footer.c_str(), multipart_footer.size());
// 结束块
http->Write("", 0);
if (http->GetStatusCode() != 200) {
ESP_LOGE(TAG, "Failed to upload photo, status code: %d", http->GetStatusCode());
return "{\"success\": false, \"message\": \"Failed to upload photo\"}";
}
std::string result = http->ReadAll();
http->Close();
ESP_LOGI(TAG, "Explain image size=%d, question=%s\n%s", jpeg_data_.len, question.c_str(), result.c_str());
return result;
}

50
main/boards/sensecap-watcher/sscma_camera.h Normal file
View File

@@ -0,0 +1,50 @@
#ifndef SSCMA_CAMERA_H
#define SSCMA_CAMERA_H
#include <lvgl.h>
#include <thread>
#include <memory>
#include <freertos/FreeRTOS.h>
#include <freertos/queue.h>
#include <esp_io_expander_tca95xx_16bit.h>
#include <esp_jpeg_dec.h>
#include <mbedtls/base64.h>
#include "sscma_client.h"
#include "camera.h"
struct SscmaData {
uint8_t* img;
size_t len;
};
struct JpegData {
uint8_t* buf;
size_t len;
};
class SscmaCamera : public Camera {
private:
lv_img_dsc_t preview_image_;
std::string explain_url_;
std::string explain_token_;
sscma_client_io_handle_t sscma_client_io_handle_;
sscma_client_handle_t sscma_client_handle_;
QueueHandle_t sscma_data_queue_;
JpegData jpeg_data_;
jpeg_dec_handle_t *jpeg_dec_;
jpeg_dec_io_t *jpeg_io_;
jpeg_dec_header_info_t *jpeg_out_;
public:
SscmaCamera(esp_io_expander_handle_t io_exp_handle);
~SscmaCamera();
virtual void SetExplainUrl(const std::string& url, const std::string& token);
virtual bool Capture();
// 翻转控制函数
virtual bool SetHMirror(bool enabled) override;
virtual bool SetVFlip(bool enabled) override;
virtual std::string Explain(const std::string& question);
};
#endif // ESP32_CAMERA_H

8
main/idf_component.yml
View File

@@ -59,6 +59,14 @@ dependencies:
rules:
- if: target in [esp32c3]
sscma_client:
git: https://github.com/Seeed-Studio/SenseCAP-Watcher-Firmware.git
path: components/sscma_client
esp_jpeg_simd:
git: https://github.com/Seeed-Studio/SenseCAP-Watcher-Firmware.git
path: components/esp_jpeg_simd
## Required IDF version
idf:
version: '>=5.4.0'