devttl/xiaozhi-esp32
1
0
Fork 0
forked from xiaozhi/xiaozhi-esp32
Code Pull Requests Activity

support ML307, new version 0.3.0

Browse Source
This commit is contained in:
Terrence
2024-10-01 14:16:12 +08:00
parent 8e9be5abc7
commit d26e8d25ff
17 changed files with 1020 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

201
main/Application.cc
View File

@@ -1,5 +1,10 @@
#include "Application.h"
#include "BuiltinLed.h"
#include "TlsTransport.h"
#include "Ml307SslTransport.h"
#include "WifiConfigurationAp.h"
#include "WifiStation.h"
#include <cstring>
#include "esp_log.h"
#include "model_path.h"
@@ -10,7 +15,19 @@
#define TAG "Application"
Application::Application() {
Application::Application()
#ifdef CONFIG_USE_ML307
: ml307_at_modem_(CONFIG_ML307_TX_PIN, CONFIG_ML307_RX_PIN, 4096),
http_(ml307_at_modem_),
firmware_upgrade_(http_)
#else
: http_(),
firmware_upgrade_(http_)
#endif
#ifdef CONFIG_USE_DISPLAY
, display_(CONFIG_DISPLAY_SDA_PIN, CONFIG_DISPLAY_SCL_PIN)
#endif
{
event_group_ = xEventGroupCreate();
audio_encode_queue_ = xQueueCreate(100, sizeof(iovec));
audio_decode_queue_ = xQueueCreate(100, sizeof(AudioPacket*));
@@ -20,8 +37,6 @@ Application::Application() {
ESP_LOGI(TAG, "Model %d: %s", i, models->model_name[i]);
if (strstr(models->model_name[i], ESP_WN_PREFIX) != NULL) {
wakenet_model_ = models->model_name[i];
} else if (strstr(models->model_name[i], ESP_NSNET_PREFIX) != NULL) {
nsnet_model_ = models->model_name[i];
}
}
@@ -32,6 +47,8 @@ Application::Application() {
}
firmware_upgrade_.SetCheckVersionUrl(CONFIG_OTA_VERSION_URL);
firmware_upgrade_.SetHeader("Device-Id", SystemInfo::GetMacAddress().c_str());
firmware_upgrade_.SetPostData(SystemInfo::GetJsonString());
}
Application::~Application() {
@@ -49,9 +66,6 @@ Application::~Application() {
for (auto& pcm : wake_word_pcm_) {
free(pcm.iov_base);
}
for (auto& packet : wake_word_opus_) {
heap_caps_free(packet);
}
if (opus_decoder_ != nullptr) {
opus_decoder_destroy(opus_decoder_);
@@ -77,16 +91,119 @@ void Application::CheckNewVersion() {
vTaskDelay(100);
}
SetChatState(kChatStateUpgrading);
firmware_upgrade_.StartUpgrade();
firmware_upgrade_.StartUpgrade([this](int progress, size_t speed) {
#ifdef CONFIG_USE_DISPLAY
char buffer[64];
snprintf(buffer, sizeof(buffer), "Upgrading...\n %d%% %zuKB/s", progress, speed / 1024);
display_.SetText(buffer);
#endif
});
// If upgrade success, the device will reboot and never reach here
ESP_LOGI(TAG, "Firmware upgrade failed...");
SetChatState(kChatStateIdle);
} else {
firmware_upgrade_.MarkValid();
firmware_upgrade_.MarkCurrentVersionValid();
}
}
#ifdef CONFIG_USE_DISPLAY
#ifdef CONFIG_USE_ML307
static std::string csq_to_string(int csq) {
if (csq == -1) {
return "No network";
} else if (csq >= 0 && csq <= 9) {
return "Very bad";
} else if (csq >= 10 && csq <= 14) {
return "Bad";
} else if (csq >= 15 && csq <= 19) {
return "Fair";
} else if (csq >= 20 && csq <= 24) {
return "Good";
} else if (csq >= 25 && csq <= 31) {
return "Very good";
}
return "Invalid";
}
#else
static std::string rssi_to_string(int rssi) {
if (rssi >= -55) {
return "Very good";
} else if (rssi >= -65) {
return "Good";
} else if (rssi >= -75) {
return "Fair";
} else if (rssi >= -85) {
return "Poor";
} else {
return "No network";
}
}
#endif
void Application::UpdateDisplay() {
while (true) {
if (chat_state_ == kChatStateIdle) {
#ifdef CONFIG_USE_ML307
std::string network_name = ml307_at_modem_.GetCarrierName();
int signal_quality = ml307_at_modem_.GetCsq();
if (signal_quality == -1) {
network_name = "No network";
} else {
ESP_LOGI(TAG, "%s CSQ: %d", network_name.c_str(), signal_quality);
display_.SetText(network_name + "\n" + csq_to_string(signal_quality) + " (" + std::to_string(signal_quality) + ")");
}
#else
auto& wifi_station = WifiStation::GetInstance();
int8_t rssi = wifi_station.GetRssi();
display_.SetText(wifi_station.GetSsid() + "\n" + rssi_to_string(rssi) + " (" + std::to_string(rssi) + ")");
#endif
}
vTaskDelay(pdMS_TO_TICKS(10 * 1000));
}
}
#endif
void Application::Start() {
auto& builtin_led = BuiltinLed::GetInstance();
#ifdef CONFIG_USE_ML307
builtin_led.SetBlue();
builtin_led.StartContinuousBlink(100);
ml307_at_modem_.SetDebug(false);
ml307_at_modem_.SetBaudRate(921600);
// Print the ML307 modem information
std::string module_name = ml307_at_modem_.GetModuleName();
ESP_LOGI(TAG, "ML307 Module: %s", module_name.c_str());
#ifdef CONFIG_USE_DISPLAY
display_.SetText(std::string("Wait for network\n") + module_name);
#endif
ml307_at_modem_.ResetConnections();
ml307_at_modem_.WaitForNetworkReady();
ESP_LOGI(TAG, "ML307 IMEI: %s", ml307_at_modem_.GetImei().c_str());
ESP_LOGI(TAG, "ML307 ICCID: %s", ml307_at_modem_.GetIccid().c_str());
#else
// Try to connect to WiFi, if failed, launch the WiFi configuration AP
auto& wifi_station = WifiStation::GetInstance();
#ifdef CONFIG_USE_DISPLAY
display_.SetText(std::string("Connect to WiFi\n") + wifi_station.GetSsid());
#endif
builtin_led.SetBlue();
builtin_led.StartContinuousBlink(100);
wifi_station.Start();
if (!wifi_station.IsConnected()) {
builtin_led.SetBlue();
builtin_led.Blink(1000, 500);
auto& wifi_ap = WifiConfigurationAp::GetInstance();
wifi_ap.SetSsidPrefix("Xiaozhi");
#ifdef CONFIG_USE_DISPLAY
display_.SetText(wifi_ap.GetSsid() + "\n" + wifi_ap.GetWebServerUrl());
#endif
wifi_ap.Start();
return;
}
#endif
// Initialize the audio device
audio_device_.Start(CONFIG_AUDIO_INPUT_SAMPLE_RATE, CONFIG_AUDIO_OUTPUT_SAMPLE_RATE);
audio_device_.OnStateChanged([this]() {
@@ -108,18 +225,19 @@ void Application::Start() {
xTaskCreateStatic([](void* arg) {
Application* app = (Application*)arg;
app->AudioEncodeTask();
vTaskDelete(NULL);
}, "opus_encode", opus_stack_size, this, 1, audio_encode_task_stack_, &audio_encode_task_buffer_);
audio_decode_task_stack_ = (StackType_t*)malloc(opus_stack_size);
xTaskCreateStatic([](void* arg) {
Application* app = (Application*)arg;
app->AudioDecodeTask();
vTaskDelete(NULL);
}, "opus_decode", opus_stack_size, this, 1, audio_decode_task_stack_, &audio_decode_task_buffer_);
StartCommunication();
StartDetection();
// Blink the LED to indicate the device is running
auto& builtin_led = BuiltinLed::GetInstance();
builtin_led.SetGreen();
builtin_led.BlinkOnce();
xEventGroupSetBits(event_group_, DETECTION_RUNNING);
@@ -130,6 +248,15 @@ void Application::Start() {
app->CheckNewVersion();
vTaskDelete(NULL);
}, "check_new_version", 4096 * 2, this, 1, NULL);
#ifdef CONFIG_USE_DISPLAY
// Launch a task to update the display
xTaskCreate([](void* arg) {
Application* app = (Application*)arg;
app->UpdateDisplay();
vTaskDelete(NULL);
}, "update_display", 4096, this, 1, NULL);
#endif
}
void Application::SetChatState(ChatState state) {
@@ -227,6 +354,7 @@ void Application::StartCommunication() {
xTaskCreate([](void* arg) {
Application* app = (Application*)arg;
app->AudioCommunicationTask();
vTaskDelete(NULL);
}, "audio_communication", 4096 * 2, this, 5, NULL);
}
@@ -267,11 +395,13 @@ void Application::StartDetection() {
xTaskCreate([](void* arg) {
Application* app = (Application*)arg;
app->AudioFeedTask();
vTaskDelete(NULL);
}, "audio_feed", 4096 * 2, this, 5, NULL);
xTaskCreate([](void* arg) {
Application* app = (Application*)arg;
app->AudioDetectionTask();
vTaskDelete(NULL);
}, "audio_detection", 4096 * 2, this, 5, NULL);
}
@@ -303,14 +433,13 @@ void Application::StoreWakeWordData(uint8_t* data, size_t size) {
memcpy(iov.iov_base, data, size);
wake_word_pcm_.push_back(iov);
// keep about 2 seconds of data, detect duration is 32ms (sample_rate == 16000, chunksize == 512)
if (wake_word_pcm_.size() > 2000 / 32) {
while (wake_word_pcm_.size() > 2000 / 32) {
heap_caps_free(wake_word_pcm_.front().iov_base);
wake_word_pcm_.pop_front();
}
}
void Application::EncodeWakeWordData() {
wake_word_opus_.clear();
if (wake_word_encode_task_stack_ == nullptr) {
wake_word_encode_task_stack_ = (StackType_t*)malloc(4096 * 8);
}
@@ -321,18 +450,30 @@ void Application::EncodeWakeWordData() {
OpusEncoder* encoder = new OpusEncoder();
encoder->Configure(CONFIG_AUDIO_INPUT_SAMPLE_RATE, 1, 60);
encoder->SetComplexity(0);
app->wake_word_opus_.resize(4096 * 4);
size_t offset = 0;
for (auto& pcm: app->wake_word_pcm_) {
encoder->Encode(pcm, [app](const iovec opus) {
auto protocol = app->AllocateBinaryProtocol(opus.iov_base, opus.iov_len);
app->wake_word_opus_.push_back(protocol);
encoder->Encode(pcm, [app, &offset](const iovec opus) {
size_t protocol_size = sizeof(BinaryProtocol) + opus.iov_len;
if (offset + protocol_size < app->wake_word_opus_.size()) {
auto protocol = (BinaryProtocol*)(&app->wake_word_opus_[offset]);
protocol->version = htons(PROTOCOL_VERSION);
protocol->type = htons(0);
protocol->reserved = 0;
protocol->timestamp = htonl(app->audio_device_.playing() ? app->audio_device_.last_timestamp() : 0);
protocol->payload_size = htonl(opus.iov_len);
memcpy(protocol->payload, opus.iov_base, opus.iov_len);
offset += protocol_size;
}
});
heap_caps_free(pcm.iov_base);
}
app->wake_word_pcm_.clear();
app->wake_word_opus_.resize(offset);
auto end_time = esp_timer_get_time();
ESP_LOGI(TAG, "Encode wake word data opus packets: %d in %lld ms", app->wake_word_opus_.size(), (end_time - start_time) / 1000);
ESP_LOGI(TAG, "Encode wake word opus: %zu bytes in %lld ms", app->wake_word_opus_.size(), (end_time - start_time) / 1000);
xEventGroupSetBits(app->event_group_, WAKE_WORD_ENCODED);
delete encoder;
vTaskDelete(NULL);
@@ -340,10 +481,7 @@ void Application::EncodeWakeWordData() {
}
void Application::SendWakeWordData() {
for (auto& protocol: wake_word_opus_) {
ws_client_->Send(protocol, sizeof(BinaryProtocol) + ntohl(protocol->payload_size), true);
heap_caps_free(protocol);
}
ws_client_->Send(wake_word_opus_.data(), wake_word_opus_.size(), true);
wake_word_opus_.clear();
}
@@ -412,7 +550,7 @@ void Application::AudioDetectionTask() {
auto res = esp_afe_sr_v1.fetch(afe_detection_data_);
if (res == nullptr || res->ret_value == ESP_FAIL) {
ESP_LOGE(TAG, "Error in fetch");
ESP_LOGE(TAG, "Error in AudioDetectionTask");
if (res != nullptr) {
ESP_LOGI(TAG, "Error code: %d", res->ret_value);
}
@@ -461,6 +599,7 @@ void Application::AudioDetectionTask() {
opus_encoder_.ResetState();
// If connected, the hello message is already sent, so we can start communication
xEventGroupSetBits(event_group_, COMMUNICATION_RUNNING);
ESP_LOGI(TAG, "Communication running");
} else {
SetChatState(kChatStateIdle);
xEventGroupSetBits(event_group_, DETECTION_RUNNING);
@@ -478,7 +617,7 @@ void Application::AudioCommunicationTask() {
auto res = esp_afe_vc_v1.fetch(afe_communication_data_);
if (res == nullptr || res->ret_value == ESP_FAIL) {
ESP_LOGE(TAG, "Error in fetch");
ESP_LOGE(TAG, "Error in AudioCommunicationTask");
if (res != nullptr) {
ESP_LOGI(TAG, "Error code: %d", res->ret_value);
}
@@ -489,16 +628,16 @@ void Application::AudioCommunicationTask() {
{
std::lock_guard<std::recursive_mutex> lock(mutex_);
if (ws_client_ == nullptr || !ws_client_->IsConnected()) {
if (ws_client_ != nullptr) {
delete ws_client_;
ws_client_ = nullptr;
}
xEventGroupClearBits(event_group_, COMMUNICATION_RUNNING);
if (audio_device_.playing()) {
audio_device_.Break();
}
SetChatState(kChatStateIdle);
if (ws_client_ != nullptr) {
delete ws_client_;
ws_client_ = nullptr;
}
xEventGroupSetBits(event_group_, DETECTION_RUNNING);
xEventGroupClearBits(event_group_, COMMUNICATION_RUNNING);
continue;
}
}
@@ -591,7 +730,11 @@ void Application::StartWebSocketClient() {
}
std::string token = "Bearer " + std::string(CONFIG_WEBSOCKET_ACCESS_TOKEN);
ws_client_ = new WebSocketClient();
#ifdef CONFIG_USE_ML307
ws_client_ = new WebSocket(new Ml307SslTransport(ml307_at_modem_, 0));
#else
ws_client_ = new WebSocket(new TlsTransport());
#endif
ws_client_->SetHeader("Authorization", token.c_str());
ws_client_->SetHeader("Device-Id", SystemInfo::GetMacAddress().c_str());
ws_client_->SetHeader("Protocol-Version", std::to_string(PROTOCOL_VERSION).c_str());
@@ -659,6 +802,10 @@ void Application::StartWebSocketClient() {
ESP_LOGE(TAG, "Websocket error: %d", error);
});
ws_client_->OnDisconnected([this]() {
ESP_LOGI(TAG, "Websocket disconnected");
});
if (!ws_client_->Connect(CONFIG_WEBSOCKET_URL)) {
ESP_LOGE(TAG, "Failed to connect to websocket server");
return;