devttl/xiaozhi-esp32
1
0
Fork 0
forked from xiaozhi/xiaozhi-esp32
Code Pull Requests Activity

use protocol 3

Browse Source
This commit is contained in:
Terrence
2024-10-30 06:58:29 +08:00
parent 3e1e576272
commit 6f5f5a0642
22 changed files with 191 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@@ -4,7 +4,7 @@
# CMakeLists in this exact order for cmake to work correctly
cmake_minimum_required(VERSION 3.16)
set(PROJECT_VER "0.5.0")
set(PROJECT_VER "0.6.0")
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
project(xiaozhi)

94
main/Application.cc
View File

@@ -14,6 +14,13 @@
#define TAG "Application"
extern const char p3_err_reg_start[] asm("_binary_err_reg_p3_start");
extern const char p3_err_reg_end[] asm("_binary_err_reg_p3_end");
extern const char p3_err_pin_start[] asm("_binary_err_pin_p3_start");
extern const char p3_err_pin_end[] asm("_binary_err_pin_p3_end");
extern const char p3_err_wificonfig_start[] asm("_binary_err_wificonfig_p3_start");
extern const char p3_err_wificonfig_end[] asm("_binary_err_wificonfig_p3_end");
Application::Application()
: boot_button_((gpio_num_t)BOOT_BUTTON_GPIO),
@@ -80,6 +87,41 @@ void Application::CheckNewVersion() {
}
}
void Application::Alert(const std::string&& title, const std::string&& message) {
ESP_LOGE(TAG, "Alert: %s, %s", title.c_str(), message.c_str());
display_.ShowNotification(std::string(title + "\n" + message));
if (message == "PIN is not ready") {
PlayLocalFile(p3_err_pin_start, p3_err_pin_end - p3_err_pin_start);
} else if (message == "Configuring WiFi") {
PlayLocalFile(p3_err_wificonfig_start, p3_err_wificonfig_end - p3_err_wificonfig_start);
} else if (message == "Registration denied") {
PlayLocalFile(p3_err_reg_start, p3_err_reg_end - p3_err_reg_start);
}
}
void Application::PlayLocalFile(const char* data, size_t size) {
ESP_LOGI(TAG, "PlayLocalFile: %zu bytes", size);
SetDecodeSampleRate(16000);
{
std::lock_guard<std::mutex> lock(mutex_);
auto packet = new AudioPacket();
packet->type = kAudioPacketTypeStart;
audio_decode_queue_.push_back(packet);
}
ParseBinaryProtocol3(data, size);
{
std::lock_guard<std::mutex> lock(mutex_);
auto packet = new AudioPacket();
packet->type = kAudioPacketTypeStop;
audio_decode_queue_.push_back(packet);
cv_.notify_all();
}
}
void Application::Start() {
auto& builtin_led = BuiltinLed::GetInstance();
builtin_led.SetBlue();
@@ -370,15 +412,12 @@ void Application::SetChatState(ChatState state) {
}
}
BinaryProtocol* Application::AllocateBinaryProtocol(const uint8_t* payload, size_t payload_size) {
auto last_timestamp = 0;
auto protocol = (BinaryProtocol*)heap_caps_malloc(sizeof(BinaryProtocol) + payload_size, MALLOC_CAP_SPIRAM);
protocol->version = htons(PROTOCOL_VERSION);
protocol->type = htons(0);
BinaryProtocol3* Application::AllocateBinaryProtocol3(const uint8_t* payload, size_t payload_size) {
auto protocol = (BinaryProtocol3*)heap_caps_malloc(sizeof(BinaryProtocol3) + payload_size, MALLOC_CAP_SPIRAM);
protocol->type = 0;
protocol->reserved = 0;
protocol->timestamp = htonl(last_timestamp);
protocol->payload_size = htonl(payload_size);
assert(sizeof(BinaryProtocol) == 16);
protocol->payload_size = htons(payload_size);
assert(sizeof(BinaryProtocol3) == 4UL);
memcpy(protocol->payload, payload, payload_size);
return protocol;
}
@@ -400,10 +439,10 @@ void Application::AudioEncodeTask() {
// Encode audio data
opus_encoder_.Encode(pcm, [this](const uint8_t* opus, size_t opus_size) {
auto protocol = AllocateBinaryProtocol(opus, opus_size);
auto protocol = AllocateBinaryProtocol3(opus, opus_size);
Schedule([this, protocol, opus_size]() {
if (ws_client_ && ws_client_->IsConnected()) {
if (!ws_client_->Send(protocol, sizeof(BinaryProtocol) + opus_size, true)) {
if (!ws_client_->Send(protocol, sizeof(BinaryProtocol3) + opus_size, true)) {
ESP_LOGE(TAG, "Failed to send audio data");
}
}
@@ -470,7 +509,11 @@ void Application::HandleAudioPacket(AudioPacket* packet) {
break;
case kAudioPacketTypeStop:
Schedule([this]() {
SetChatState(kChatStateListening);
if (ws_client_ && ws_client_->IsConnected()) {
SetChatState(kChatStateListening);
} else {
SetChatState(kChatStateIdle);
}
});
break;
case kAudioPacketTypeSentenceStart:
@@ -517,6 +560,23 @@ void Application::SetDecodeSampleRate(int sample_rate) {
}
}
void Application::ParseBinaryProtocol3(const char* data, size_t size) {
for (const char* p = data; p < data + size; ) {
auto protocol = (BinaryProtocol3*)p;
p += sizeof(BinaryProtocol3);
auto packet = new AudioPacket();
packet->type = kAudioPacketTypeData;
auto payload_size = ntohs(protocol->payload_size);
packet->opus.resize(payload_size);
memcpy(packet->opus.data(), protocol->payload, payload_size);
p += payload_size;
std::lock_guard<std::mutex> lock(mutex_);
audio_decode_queue_.push_back(packet);
}
}
void Application::StartWebSocketClient() {
if (ws_client_ != nullptr) {
ESP_LOGW(TAG, "WebSocket client already exists");
@@ -545,17 +605,7 @@ void Application::StartWebSocketClient() {
ws_client_->OnData([this](const char* data, size_t len, bool binary) {
if (binary) {
auto protocol = (BinaryProtocol*)data;
auto packet = new AudioPacket();
packet->type = kAudioPacketTypeData;
packet->timestamp = ntohl(protocol->timestamp);
auto payload_size = ntohl(protocol->payload_size);
packet->opus.resize(payload_size);
memcpy(packet->opus.data(), protocol->payload, payload_size);
std::lock_guard<std::mutex> lock(mutex_);
audio_decode_queue_.push_back(packet);
ParseBinaryProtocol3(data, len);
cv_.notify_all();
} else {
// Parse JSON data

17
main/Application.h
View File

@@ -28,13 +28,11 @@
#define DETECTION_RUNNING 1
#define COMMUNICATION_RUNNING 2
#define PROTOCOL_VERSION 2
struct BinaryProtocol {
uint16_t version;
uint16_t type;
uint32_t reserved;
uint32_t timestamp;
uint32_t payload_size;
#define PROTOCOL_VERSION 3
struct BinaryProtocol3 {
uint8_t type;
uint8_t reserved;
uint16_t payload_size;
uint8_t payload[];
} __attribute__((packed));
@@ -78,6 +76,7 @@ public:
Display& GetDisplay() { return display_; }
void Schedule(std::function<void()> callback);
void SetChatState(ChatState state);
void Alert(const std::string&& title, const std::string&& message);
// 删除拷贝构造函数和赋值运算符
Application(const Application&) = delete;
@@ -128,7 +127,8 @@ private:
StackType_t* check_new_version_task_stack_ = nullptr;
void MainLoop();
BinaryProtocol* AllocateBinaryProtocol(const uint8_t* payload, size_t payload_size);
BinaryProtocol3* AllocateBinaryProtocol3(const uint8_t* payload, size_t payload_size);
void ParseBinaryProtocol3(const char* data, size_t size);
void SetDecodeSampleRate(int sample_rate);
void StartWebSocketClient();
void CheckNewVersion();
@@ -136,6 +136,7 @@ private:
void AudioEncodeTask();
void AudioPlayTask();
void HandleAudioPacket(AudioPacket* packet);
void PlayLocalFile(const char* data, size_t size);
};
#endif // _APPLICATION_H_

4
main/Board.cc
View File

@@ -2,3 +2,7 @@
#include <esp_log.h>
// static const char *TAG = "Board";
bool Board::GetBatteryVoltage(int &voltage) {
return false;
}

1
main/Board.h
View File

@@ -25,6 +25,7 @@ public:
virtual Http* CreateHttp() = 0;
virtual WebSocket* CreateWebSocket() = 0;
virtual bool GetNetworkState(std::string& network_name, int& signal_quality, std::string& signal_quality_text) = 0;
virtual bool GetBatteryVoltage(int &voltage);
virtual std::string GetJson() = 0;
protected:

1
main/CMakeLists.txt
View File

@@ -45,6 +45,7 @@ if(CONFIG_USE_AFE_SR)
endif()
idf_component_register(SRCS ${SOURCES}
EMBED_FILES "assets/err_reg.p3" "assets/err_pin.p3" "assets/err_wificonfig.p3"
INCLUDE_DIRS ${INCLUDE_DIRS}
)

11
main/Display.cc
View File

@@ -220,8 +220,15 @@ void Display::UpdateDisplay() {
if (!board.GetNetworkState(network_name, signal_quality, signal_quality_text)) {
text = "No network";
} else {
ESP_LOGI(TAG, "%s CSQ: %d", network_name.c_str(), signal_quality);
text = network_name + "\n" + signal_quality_text + " (" + std::to_string(signal_quality) + ")";
text = network_name + "\n" + signal_quality_text;
if (std::abs(signal_quality) != 99) {
text += " (" + std::to_string(signal_quality) + ")";
}
}
int battery_voltage;
if (board.GetBatteryVoltage(battery_voltage)) {
text += "\n" + std::to_string(battery_voltage) + "mV";
}
SetText(text);
}

33
main/Ml307Board.cc
View File

@@ -30,28 +30,38 @@ static std::string csq_to_string(int csq) {
Ml307Board::Ml307Board() : modem_(ML307_TX_PIN, ML307_RX_PIN, 4096) {
}
void Ml307Board::StartModem() {
void Ml307Board::StartNetwork() {
auto& application = Application::GetInstance();
auto& display = application.GetDisplay();
modem_.SetDebug(false);
modem_.SetBaudRate(921600);
display.SetText(std::string("Wait for network\n"));
int result = modem_.WaitForNetworkReady();
if (result == -1) {
application.Alert("Error", "PIN is not ready");
} else if (result == -2) {
application.Alert("Error", "Registration denied");
}
// Print the ML307 modem information
std::string module_name = modem_.GetModuleName();
ESP_LOGI(TAG, "ML307 Module: %s", module_name.c_str());
display.SetText(std::string("Wait for network\n") + module_name);
modem_.ResetConnections();
modem_.WaitForNetworkReady();
std::string imei = modem_.GetImei();
std::string iccid = modem_.GetIccid();
ESP_LOGI(TAG, "ML307 Module: %s", module_name.c_str());
ESP_LOGI(TAG, "ML307 IMEI: %s", imei.c_str());
ESP_LOGI(TAG, "ML307 ICCID: %s", iccid.c_str());
}
void Ml307Board::StartModem() {
modem_.SetDebug(false);
modem_.SetBaudRate(921600);
StartNetwork();
auto& application = Application::GetInstance();
// If low power, the material ready event will be triggered by the modem because of a reset
modem_.OnMaterialReady([&application]() {
modem_.OnMaterialReady([this, &application]() {
ESP_LOGI(TAG, "ML307 material ready");
application.Schedule([&application]() {
application.Schedule([this, &application]() {
application.SetChatState(kChatStateIdle);
StartNetwork();
});
});
}
@@ -74,6 +84,9 @@ WebSocket* Ml307Board::CreateWebSocket() {
}
bool Ml307Board::GetNetworkState(std::string& network_name, int& signal_quality, std::string& signal_quality_text) {
if (!modem_.network_ready()) {
return false;
}
network_name = modem_.GetCarrierName();
signal_quality = modem_.GetCsq();
signal_quality_text = csq_to_string(signal_quality);

1
main/Ml307Board.h
View File

@@ -9,6 +9,7 @@ protected:
Ml307AtModem modem_;
void StartModem();
void StartNetwork();
public:
Ml307Board();

4
main/SystemReset.cc
View File

@@ -41,6 +41,10 @@ void SystemReset::ResetNvsFlash() {
if (ret != ESP_OK) {
ESP_LOGE(TAG, "Failed to erase NVS flash");
}
ret = nvs_flash_init();
if (ret != ESP_OK) {
ESP_LOGE(TAG, "Failed to initialize NVS flash");
}
}
void SystemReset::ResetToFactory() {

10
main/WakeWordDetect.cc
View File

@@ -180,14 +180,12 @@ void WakeWordDetect::EncodeWakeWordData() {
for (auto& pcm: this_->wake_word_pcm_) {
encoder->Encode(pcm, [this_, &offset](const uint8_t* opus, size_t opus_size) {
size_t protocol_size = sizeof(BinaryProtocol) + opus_size;
size_t protocol_size = sizeof(BinaryProtocol3) + opus_size;
if (offset + protocol_size < this_->wake_word_opus_.size()) {
auto protocol = (BinaryProtocol*)(&this_->wake_word_opus_[offset]);
protocol->version = htons(PROTOCOL_VERSION);
protocol->type = htons(0);
auto protocol = (BinaryProtocol3*)(&this_->wake_word_opus_[offset]);
protocol->type = 0;
protocol->reserved = 0;
protocol->timestamp = 0;
protocol->payload_size = htonl(opus_size);
protocol->payload_size = htons(opus_size);
memcpy(protocol->payload, opus, opus_size);
offset += protocol_size;
}

14
main/WifiBoard.cc
View File

@@ -39,16 +39,13 @@ void WifiBoard::StartWifi() {
display.SetText(std::string("Connect to WiFi\n") + wifi_station.GetSsid());
wifi_station.Start();
if (!wifi_station.IsConnected()) {
application.Alert("Info", "Configuring WiFi");
builtin_led.SetBlue();
builtin_led.Blink(1000, 500);
auto& wifi_ap = WifiConfigurationAp::GetInstance();
wifi_ap.SetSsidPrefix("Xiaozhi");
display.SetText(wifi_ap.GetSsid() + "\n" + wifi_ap.GetWebServerUrl());
wifi_ap.Start();
// Wait until the WiFi configuration is finished
while (true) {
vTaskDelay(pdMS_TO_TICKS(1000));
}
wifi_config_mode_ = true;
}
}
@@ -71,6 +68,13 @@ WebSocket* WifiBoard::CreateWebSocket() {
}
bool WifiBoard::GetNetworkState(std::string& network_name, int& signal_quality, std::string& signal_quality_text) {
if (wifi_config_mode_) {
auto& wifi_ap = WifiConfigurationAp::GetInstance();
network_name = wifi_ap.GetSsid();
signal_quality = -99;
signal_quality_text = wifi_ap.GetWebServerUrl();
return true;
}
auto& wifi_station = WifiStation::GetInstance();
if (!wifi_station.IsConnected()) {
return false;

1
main/WifiBoard.h
View File

@@ -5,6 +5,7 @@
class WifiBoard : public Board {
protected:
bool wifi_config_mode_ = false;
virtual void StartWifi();
public:

BIN
main/assets/err_pin.p3 Normal file
View File

Binary file not shown.

BIN
main/assets/err_reg.p3 Normal file
View File

Binary file not shown.

BIN
main/assets/err_wificonfig.p3 Normal file
View File

Binary file not shown.

4
main/boards/bread-compact-ml307/CompactMl307Board.cc
View File

@@ -1,4 +1,5 @@
#include "Ml307Board.h"
#include "SystemReset.h"
#include <esp_log.h>
#define TAG "CompactMl307Board"
@@ -7,6 +8,9 @@ class CompactMl307Board : public Ml307Board {
public:
virtual void Initialize() override {
ESP_LOGI(TAG, "Initializing CompactMl307Board");
// Check if the reset button is pressed
SystemReset::GetInstance().CheckButtons();
Ml307Board::Initialize();
}

5
main/boards/bread-compact-wifi/CompactWifiBoard.cc
View File

@@ -1,5 +1,5 @@
#include "WifiBoard.h"
#include "SystemReset.h"
#include <esp_log.h>
#define TAG "CompactWifiBoard"
@@ -8,6 +8,9 @@ class CompactWifiBoard : public WifiBoard {
public:
virtual void Initialize() override {
ESP_LOGI(TAG, "Initializing CompactWifiBoard");
// Check if the reset button is pressed
SystemReset::GetInstance().CheckButtons();
WifiBoard::Initialize();
}

35
main/boards/kevin-box-0/KevinBoxBoard.cc
View File

@@ -4,11 +4,17 @@
#include <esp_log.h>
#include <esp_spiffs.h>
#include <driver/gpio.h>
#include <esp_adc/adc_oneshot.h>
#include <esp_adc/adc_cali.h>
#include <esp_adc/adc_cali_scheme.h>
static const char *TAG = "KevinBoxBoard";
class KevinBoxBoard : public Ml307Board {
private:
adc_oneshot_unit_handle_t adc1_handle_;
adc_cali_handle_t adc1_cali_handle_;
void MountStorage() {
// Mount the storage partition
esp_vfs_spiffs_conf_t conf = {
@@ -32,9 +38,31 @@ private:
gpio_config(&ml307_enable_config);
gpio_set_level(GPIO_NUM_15, 1);
}
virtual void InitializeADC() {
adc_oneshot_unit_init_cfg_t init_config1 = {};
init_config1.unit_id = ADC_UNIT_1;
ESP_ERROR_CHECK(adc_oneshot_new_unit(&init_config1, &adc1_handle_));
//-------------ADC1 Config---------------//
adc_oneshot_chan_cfg_t config = {
.atten = ADC_ATTEN_DB_12,
.bitwidth = ADC_BITWIDTH_DEFAULT,
};
ESP_ERROR_CHECK(adc_oneshot_config_channel(adc1_handle_, ADC_CHANNEL_0, &config));
adc_cali_curve_fitting_config_t cali_config = {
.unit_id = ADC_UNIT_1,
.chan = ADC_CHANNEL_0,
.atten = ADC_ATTEN_DB_12,
.bitwidth = ADC_BITWIDTH_DEFAULT,
};
ESP_ERROR_CHECK(adc_cali_create_scheme_curve_fitting(&cali_config, &adc1_cali_handle_));
}
public:
virtual void Initialize() override {
ESP_LOGI(TAG, "Initializing KevinBoxBoard");
InitializeADC();
MountStorage();
Enable4GModule();
Ml307Board::Initialize();
@@ -43,6 +71,13 @@ public:
virtual AudioDevice* CreateAudioDevice() override {
return new BoxAudioDevice();
}
virtual bool GetBatteryVoltage(int &voltage) override {
int adc_reading;
ESP_ERROR_CHECK(adc_oneshot_read(adc1_handle_, ADC_CHANNEL_0, &adc_reading));
ESP_ERROR_CHECK(adc_cali_raw_to_voltage(adc1_cali_handle_, adc_reading, &voltage));
return true;
}
};
DECLARE_BOARD(KevinBoxBoard);

10
main/idf_component.yml
View File

@@ -1,14 +1,14 @@
## IDF Component Manager Manifest File
dependencies:
78/esp-wifi-connect: "^1.2.0"
78/esp-opus-encoder: "^1.0.2"
78/esp-ml307: "^1.2.2"
78/esp-wifi-connect: "~1.2.0"
78/esp-opus-encoder: "~1.0.2"
78/esp-ml307: "~1.3.0"
espressif/led_strip: "^2.4.1"
espressif/esp_codec_dev: "^1.3.1"
espressif/esp-sr: "^1.9.0"
espressif/button: "^3.3.1"
lvgl/lvgl: "^8.4.0"
esp_lvgl_port: "^2.4.1"
lvgl/lvgl: "~8.4.0"
esp_lvgl_port: "~2.4.1"
## Required IDF version
idf:
version: ">=5.3"

4
main/main.cc
View File

@@ -7,15 +7,11 @@
#include "Application.h"
#include "SystemInfo.h"
#include "SystemReset.h"
#define TAG "main"
extern "C" void app_main(void)
{
// Check if the reset button is pressed
SystemReset::GetInstance().CheckButtons();
// Initialize the default event loop
ESP_ERROR_CHECK(esp_event_loop_create_default());

8
versions.py
View File

@@ -56,12 +56,12 @@ def get_app_desc(data):
def get_board_name(folder):
basename = os.path.basename(folder)
if basename.startswith("v0.2"):
return "simple"
if basename.startswith("v0.3") or basename.startswith("v0.4") or basename.startswith("v0.5"):
return "bread-simple"
if basename.startswith("v0.3") or basename.startswith("v0.4") or basename.startswith("v0.5") or basename.startswith("v0.6"):
if "ML307" in basename:
return "compact.4g"
return "bread-compact-ml307"
else:
return "compact.wifi"
return "bread-compact-wifi"
raise Exception(f"Unknown board name: {basename}")
def read_binary(dir_path):