diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..22f77cea --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +tmp/ +components/ +managed_components/ +build/ +.vscode/ +.devcontainer/ +sdkconfig.old +sdkconfig +dependencies.lock \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100755 index 00000000..0bab4a78 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,8 @@ +# For more information about build system see +# https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/build-system.html +# The following five lines of boilerplate have to be in your project's +# CMakeLists in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.16) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(xiaozhi) diff --git a/README.md b/README.md new file mode 100755 index 00000000..15da6b08 --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +# 你好，小智 + +【ESP32+SenseVoice+Qwen72B打造你的AI聊天伴侣！】 + +https://www.bilibili.com/video/BV11msTenEH3/?share_source=copy_web&vd_source=ee1aafe19d6e60cf22e60a93881faeba + diff --git a/main/Application.cc b/main/Application.cc new file mode 100644 index 00000000..5e73d88f --- /dev/null +++ b/main/Application.cc @@ -0,0 +1,430 @@ +#include "Application.h" +#include +#include "esp_log.h" +#include "model_path.h" +#include "SystemInfo.h" +#include "cJSON.h" +#include "silk_resampler.h" + +#define TAG "application" +#define INPUT_SAMPLE_RATE 16000 +#define DECODE_SAMPLE_RATE 24000 +#define OUTPUT_SAMPLE_RATE 24000 + + +Application::Application() { + event_group_ = xEventGroupCreate(); + audio_encode_queue_ = xQueueCreate(100, sizeof(AudioEncoderData)); + audio_decode_queue_ = xQueueCreate(100, sizeof(AudioPacket*)); + + srmodel_list_t *models = esp_srmodel_init("model"); + for (int i = 0; i < models->num; i++) { + ESP_LOGI(TAG, "Model %d: %s", i, models->model_name[i]); + if (strstr(models->model_name[i], ESP_WN_PREFIX) != NULL) { + wakenet_model_ = models->model_name[i]; + } else if (strstr(models->model_name[i], ESP_NSNET_PREFIX) != NULL) { + nsnet_model_ = models->model_name[i]; + } + } + + opus_encoder_.Configure(INPUT_SAMPLE_RATE, 1); + opus_decoder_ = opus_decoder_create(DECODE_SAMPLE_RATE, 1, NULL); + if (DECODE_SAMPLE_RATE != OUTPUT_SAMPLE_RATE) { + assert(0 == silk_resampler_init(&resampler_state_, DECODE_SAMPLE_RATE, OUTPUT_SAMPLE_RATE, 1)); + } +} + +Application::~Application() { + if (afe_detection_data_ != nullptr) { + esp_afe_sr_v1.destroy(afe_detection_data_); + } + + if (afe_communication_data_ != nullptr) { + esp_afe_vc_v1.destroy(afe_communication_data_); + } + + if (opus_decoder_ != nullptr) { + opus_decoder_destroy(opus_decoder_); + } + if (audio_encode_task_stack_ != nullptr) { + free(audio_encode_task_stack_); + } + if (audio_decode_task_stack_ != nullptr) { + free(audio_decode_task_stack_); + } + vQueueDelete(audio_decode_queue_); + vQueueDelete(audio_encode_queue_); + + vEventGroupDelete(event_group_); +} + +void Application::Start() { + audio_device_.Start(INPUT_SAMPLE_RATE, OUTPUT_SAMPLE_RATE); + audio_device_.OnStateChanged([this]() { + if (audio_device_.playing()) { + SetChatState(kChatStateSpeaking); + } else { + // Check if communication is still running + if (xEventGroupGetBits(event_group_) & COMMUNICATION_RUNNING) { + SetChatState(kChatStateListening); + } else { + SetChatState(kChatStateIdle); + } + } + }); + + // OPUS encoder / decoder use a lot of stack memory + audio_encode_task_stack_ = (StackType_t*)malloc(4096 * 8); + xTaskCreateStatic([](void* arg) { + Application* app = (Application*)arg; + app->AudioEncodeTask(); + }, "opus_encode", 4096 * 8, this, 1, audio_encode_task_stack_, &audio_encode_task_buffer_); + audio_decode_task_stack_ = (StackType_t*)malloc(4096 * 8); + xTaskCreateStatic([](void* arg) { + Application* app = (Application*)arg; + app->AudioDecodeTask(); + }, "opus_decode", 4096 * 8, this, 1, audio_decode_task_stack_, &audio_decode_task_buffer_); + + wifi_station_.Start(); + + StartCommunication(); + StartDetection(); + + xEventGroupSetBits(event_group_, DETECTION_RUNNING); +} + +void Application::SetChatState(ChatState state) { + chat_state_ = state; + switch (chat_state_) { + case kChatStateIdle: + ESP_LOGI(TAG, "Chat state: idle"); + builtin_led_.TurnOff(); + break; + case kChatStateConnecting: + ESP_LOGI(TAG, "Chat state: connecting"); + builtin_led_.SetBlue(); + builtin_led_.TurnOn(); + break; + case kChatStateListening: + ESP_LOGI(TAG, "Chat state: listening"); + builtin_led_.SetRed(); + builtin_led_.TurnOn(); + break; + case kChatStateSpeaking: + ESP_LOGI(TAG, "Chat state: speaking"); + builtin_led_.SetGreen(); + builtin_led_.TurnOn(); + break; + } + + std::lock_guard lock(mutex_); + if (ws_client_ && ws_client_->IsConnected()) { + cJSON* root = cJSON_CreateObject(); + cJSON_AddStringToObject(root, "type", "state"); + cJSON_AddStringToObject(root, "state", chat_state_ == kChatStateListening ? "listening" : "speaking"); + char* json = cJSON_PrintUnformatted(root); + ws_client_->Send(json); + cJSON_Delete(root); + free(json); + } +} + +void Application::StartCommunication() { + afe_config_t afe_config = { + .aec_init = false, + .se_init = true, + .vad_init = false, + .wakenet_init = false, + .voice_communication_init = true, + .voice_communication_agc_init = true, + .voice_communication_agc_gain = 10, + .vad_mode = VAD_MODE_3, + .wakenet_model_name = NULL, + .wakenet_model_name_2 = NULL, + .wakenet_mode = DET_MODE_90, + .afe_mode = SR_MODE_HIGH_PERF, + .afe_perferred_core = 0, + .afe_perferred_priority = 5, + .afe_ringbuf_size = 50, + .memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM, + .afe_linear_gain = 1.0, + .agc_mode = AFE_MN_PEAK_AGC_MODE_2, + .pcm_config = { + .total_ch_num = 1, + .mic_num = 1, + .ref_num = 0, + .sample_rate = INPUT_SAMPLE_RATE + }, + .debug_init = false, + .debug_hook = {{ AFE_DEBUG_HOOK_MASE_TASK_IN, NULL }, { AFE_DEBUG_HOOK_FETCH_TASK_IN, NULL }}, + .afe_ns_mode = NS_MODE_SSP, + .afe_ns_model_name = NULL, + .fixed_first_channel = true, + }; + + afe_communication_data_ = esp_afe_vc_v1.create_from_config(&afe_config); + + xTaskCreate([](void* arg) { + Application* app = (Application*)arg; + app->AudioCommunicationTask(); + }, "audio_communication", 4096 * 2, this, 5, NULL); +} + +void Application::StartDetection() { + afe_config_t afe_config = { + .aec_init = false, + .se_init = true, + .vad_init = false, + .wakenet_init = true, + .voice_communication_init = false, + .voice_communication_agc_init = false, + .voice_communication_agc_gain = 10, + .vad_mode = VAD_MODE_3, + .wakenet_model_name = wakenet_model_, + .wakenet_model_name_2 = NULL, + .wakenet_mode = DET_MODE_90, + .afe_mode = SR_MODE_HIGH_PERF, + .afe_perferred_core = 0, + .afe_perferred_priority = 5, + .afe_ringbuf_size = 50, + .memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM, + .afe_linear_gain = 1.0, + .agc_mode = AFE_MN_PEAK_AGC_MODE_2, + .pcm_config = { + .total_ch_num = 1, + .mic_num = 1, + .ref_num = 0, + .sample_rate = INPUT_SAMPLE_RATE + }, + .debug_init = false, + .debug_hook = {{ AFE_DEBUG_HOOK_MASE_TASK_IN, NULL }, { AFE_DEBUG_HOOK_FETCH_TASK_IN, NULL }}, + .afe_ns_mode = NS_MODE_SSP, + .afe_ns_model_name = NULL, + .fixed_first_channel = true, + }; + + afe_detection_data_ = esp_afe_sr_v1.create_from_config(&afe_config); + xTaskCreate([](void* arg) { + Application* app = (Application*)arg; + app->AudioFeedTask(); + }, "audio_feed", 4096 * 2, this, 5, NULL); + + xTaskCreate([](void* arg) { + Application* app = (Application*)arg; + app->AudioDetectionTask(); + }, "audio_detection", 4096 * 2, this, 5, NULL); +} + +void Application::AudioFeedTask() { + int chunk_size = esp_afe_vc_v1.get_feed_chunksize(afe_detection_data_); + int16_t buffer[chunk_size]; + ESP_LOGI(TAG, "Audio feed task started, chunk size: %d", chunk_size); + + while (true) { + audio_device_.Read(buffer, chunk_size); + + auto event_bits = xEventGroupGetBits(event_group_); + if (event_bits & DETECTION_RUNNING) { + esp_afe_sr_v1.feed(afe_detection_data_, buffer); + } else if (event_bits & COMMUNICATION_RUNNING) { + esp_afe_vc_v1.feed(afe_communication_data_, buffer); + } + } + + vTaskDelete(NULL); +} + +void Application::AudioDetectionTask() { + auto chunk_size = esp_afe_sr_v1.get_fetch_chunksize(afe_detection_data_); + ESP_LOGI(TAG, "Audio detection task started, chunk size: %d", chunk_size); + + while (true) { + xEventGroupWaitBits(event_group_, DETECTION_RUNNING, pdFALSE, pdTRUE, portMAX_DELAY); + + auto res = esp_afe_sr_v1.fetch(afe_detection_data_); + if (res == nullptr || res->ret_value == ESP_FAIL) { + ESP_LOGE(TAG, "Error in fetch"); + if (res != nullptr) { + ESP_LOGI(TAG, "Error code: %d", res->ret_value); + } + continue;; + } + + if (res->wakeup_state == WAKENET_DETECTED) { + ESP_LOGI(TAG, "Wakenet detected"); + + xEventGroupClearBits(event_group_, DETECTION_RUNNING); + SetChatState(kChatStateConnecting); + StartWebSocketClient(); + + std::lock_guard lock(mutex_); + if (ws_client_ && ws_client_->IsConnected()) { + // If connected, the hello message is already sent, so we can start communication + xEventGroupSetBits(event_group_, COMMUNICATION_RUNNING); + } else { + SetChatState(kChatStateIdle); + xEventGroupSetBits(event_group_, DETECTION_RUNNING); + } + } + } +} + +void Application::AudioCommunicationTask() { + int chunk_size = esp_afe_vc_v1.get_fetch_chunksize(afe_communication_data_); + ESP_LOGI(TAG, "Audio communication task started, chunk size: %d", chunk_size); + + while (true) { + xEventGroupWaitBits(event_group_, COMMUNICATION_RUNNING, pdFALSE, pdTRUE, portMAX_DELAY); + + auto res = esp_afe_vc_v1.fetch(afe_communication_data_); + if (res == nullptr || res->ret_value == ESP_FAIL) { + ESP_LOGE(TAG, "Error in fetch"); + if (res != nullptr) { + ESP_LOGI(TAG, "Error code: %d", res->ret_value); + } + continue; + } + + // Check if the websocket client is disconnected by the server + { + std::lock_guard lock(mutex_); + if (ws_client_ == nullptr || !ws_client_->IsConnected()) { + if (ws_client_ != nullptr) { + delete ws_client_; + ws_client_ = nullptr; + } + if (audio_device_.playing()) { + audio_device_.Break(); + } + SetChatState(kChatStateIdle); + xEventGroupSetBits(event_group_, DETECTION_RUNNING); + xEventGroupClearBits(event_group_, COMMUNICATION_RUNNING); + continue; + } + } + + if (chat_state_ == kChatStateListening) { + // Send audio data to server + AudioEncoderData data; + data.size = res->data_size; + data.data = malloc(data.size); + memcpy((void*)data.data, res->data, data.size); + xQueueSend(audio_encode_queue_, &data, portMAX_DELAY); + } + } +} + +void Application::AudioEncodeTask() { + ESP_LOGI(TAG, "Audio encode task started"); + while (true) { + AudioEncoderData data; + xQueueReceive(audio_encode_queue_, &data, portMAX_DELAY); + + // Encode audio data + opus_encoder_.Encode(data.data, data.size, [this](const void* data, size_t size) { + std::lock_guard lock(mutex_); + if (ws_client_ && ws_client_->IsConnected()) { + ws_client_->Send((const char*)data, size, true); + } + }); + free((void*)data.data); + } +} + +void Application::AudioDecodeTask() { + int frame_size = DECODE_SAMPLE_RATE / 1000 * opus_duration_ms_; + + while (true) { + AudioPacket* packet; + xQueueReceive(audio_decode_queue_, &packet, portMAX_DELAY); + packet->pcm.resize(frame_size); + + int ret = opus_decode(opus_decoder_, packet->opus.data(), packet->opus.size(), packet->pcm.data(), frame_size, 0); + if (ret < 0) { + ESP_LOGE(TAG, "Failed to decode audio, error code: %d", ret); + free(packet); + continue; + } + + if (DECODE_SAMPLE_RATE != OUTPUT_SAMPLE_RATE) { + int target_size = frame_size * OUTPUT_SAMPLE_RATE / DECODE_SAMPLE_RATE; + std::vector resampled(target_size); + assert(0 == silk_resampler(&resampler_state_, resampled.data(), packet->pcm.data(), frame_size)); + packet->pcm = std::move(resampled); + } + + audio_device_.QueueAudioPacket(packet); + } +} + +void Application::StartWebSocketClient() { + if (ws_client_ != nullptr) { + delete ws_client_; + } + + std::string token = "Bearer " + std::string(CONFIG_WEBSOCKET_ACCESS_TOKEN); + ws_client_ = new WebSocketClient(); + ws_client_->SetHeader("Authorization", token.c_str()); + ws_client_->SetHeader("Device-Id", SystemInfo::GetMacAddress().c_str()); + + ws_client_->OnConnected([this]() { + ESP_LOGI(TAG, "Websocket connected"); + + // Send hello message to describe the client + // keys: message type, version, wakeup_model, audio_params (format, sample_rate, channels) + std::string message = "{"; + message += "\"type\":\"hello\", \"version\":\"1.0\","; + message += "\"wakeup_model\":\"" + std::string(wakenet_model_) + "\","; + message += "\"audio_params\":{"; + message += "\"format\":\"opus\", \"sample_rate\":" + std::to_string(INPUT_SAMPLE_RATE) + ", \"channels\":1"; + message += "}}"; + ws_client_->Send(message); + }); + + ws_client_->OnData([this](const char* data, size_t len, bool binary) { + auto packet = new AudioPacket(); + if (binary) { + auto header = (AudioDataHeader*)data; + packet->type = kAudioPacketTypeData; + packet->timestamp = ntohl(header->timestamp); + + auto payload_size = ntohl(header->payload_size); + packet->opus.resize(payload_size); + memcpy(packet->opus.data(), data + sizeof(AudioDataHeader), payload_size); + } else { + // Parse JSON data + auto root = cJSON_Parse(data); + auto type = cJSON_GetObjectItem(root, "type"); + if (type != NULL) { + if (strcmp(type->valuestring, "tts") == 0) { + auto state = cJSON_GetObjectItem(root, "state"); + if (strcmp(state->valuestring, "start") == 0) { + packet->type = kAudioPacketTypeStart; + } else if (strcmp(state->valuestring, "stop") == 0) { + packet->type = kAudioPacketTypeStop; + } else if (strcmp(state->valuestring, "sentence_end") == 0) { + packet->type = kAudioPacketTypeSentenceEnd; + } else if (strcmp(state->valuestring, "sentence_start") == 0) { + packet->type = kAudioPacketTypeSentenceStart; + packet->text = cJSON_GetObjectItem(root, "text")->valuestring; + } + } + } + cJSON_Delete(root); + } + xQueueSend(audio_decode_queue_, &packet, portMAX_DELAY); + }); + + ws_client_->OnError([this](int error) { + ESP_LOGE(TAG, "Websocket error: %d", error); + }); + + ws_client_->OnClosed([this]() { + ESP_LOGI(TAG, "Websocket closed"); + }); + + if (!ws_client_->Connect(CONFIG_WEBSOCKET_URL)) { + ESP_LOGE(TAG, "Failed to connect to websocket server"); + return; + } +} \ No newline at end of file diff --git a/main/Application.h b/main/Application.h new file mode 100644 index 00000000..9ab59ea2 --- /dev/null +++ b/main/Application.h @@ -0,0 +1,83 @@ +#ifndef _APPLICATION_H_ +#define _APPLICATION_H_ + +#include "WifiStation.h" +#include "AudioDevice.h" +#include "OpusEncoder.h" +#include "WebSocketClient.h" +#include "BuiltinLed.h" + +#include "opus.h" +#include "resampler_structs.h" +#include "freertos/event_groups.h" +#include "freertos/queue.h" +#include "freertos/task.h" +#include "esp_afe_sr_models.h" +#include "esp_nsn_models.h" +#include + +#define DETECTION_RUNNING 1 +#define COMMUNICATION_RUNNING 2 + +struct AudioEncoderData { + const void* data; + size_t size; +}; + +enum ChatState { + kChatStateIdle, + kChatStateConnecting, + kChatStateListening, + kChatStateSpeaking, +}; + +class Application { +public: + Application(); + ~Application(); + void Start(); + +private: + WifiStation wifi_station_; + AudioDevice audio_device_; + BuiltinLed builtin_led_; + + std::recursive_mutex mutex_; + WebSocketClient* ws_client_ = nullptr; + esp_afe_sr_data_t* afe_detection_data_ = nullptr; + esp_afe_sr_data_t* afe_communication_data_ = nullptr; + EventGroupHandle_t event_group_; + char* wakenet_model_ = NULL; + char* nsnet_model_ = NULL; + volatile ChatState chat_state_ = kChatStateIdle; + + // Audio encode / decode + TaskHandle_t audio_feed_task_ = nullptr; + StaticTask_t audio_encode_task_buffer_; + StackType_t* audio_encode_task_stack_ = nullptr; + QueueHandle_t audio_encode_queue_ = nullptr; + + TaskHandle_t audio_decode_task_ = nullptr; + StaticTask_t audio_decode_task_buffer_; + StackType_t* audio_decode_task_stack_ = nullptr; + QueueHandle_t audio_decode_queue_ = nullptr; + + OpusEncoder opus_encoder_; + OpusDecoder* opus_decoder_ = nullptr; + + int opus_duration_ms_ = 60; + silk_resampler_state_struct resampler_state_; + + void SetChatState(ChatState state); + void StartDetection(); + void StartCommunication(); + void StartWebSocketClient(); + + void AudioFeedTask(); + void AudioDetectionTask(); + void AudioCommunicationTask(); + void AudioEncodeTask(); + void AudioDecodeTask(); +}; + +#endif // _APPLICATION_H_ diff --git a/main/AudioDevice.cc b/main/AudioDevice.cc new file mode 100644 index 00000000..6d7d9dcf --- /dev/null +++ b/main/AudioDevice.cc @@ -0,0 +1,235 @@ +#include "AudioDevice.h" +#include "esp_log.h" +#include + +#define TAG "AudioDevice" +#define SPEAKING BIT0 + +AudioDevice::AudioDevice() { + audio_play_queue_ = xQueueCreate(100, sizeof(AudioPacket*)); +} + +AudioDevice::~AudioDevice() { + vQueueDelete(audio_play_queue_); + + if (audio_play_task_ != nullptr) { + vTaskDelete(audio_play_task_); + } + if (rx_handle_ != nullptr) { + ESP_ERROR_CHECK(i2s_channel_disable(rx_handle_)); + } + if (tx_handle_ != nullptr) { + ESP_ERROR_CHECK(i2s_channel_disable(tx_handle_)); + } +} + +void AudioDevice::Start(int input_sample_rate, int output_sample_rate) { + assert(input_sample_rate == 16000); + input_sample_rate_ = input_sample_rate; + output_sample_rate_ = output_sample_rate; + + if (output_sample_rate == 16000) { + CreateDuplexChannels(); + } else { + CreateSimplexChannels(); + } + + ESP_ERROR_CHECK(i2s_channel_enable(tx_handle_)); + ESP_ERROR_CHECK(i2s_channel_enable(rx_handle_)); + + xTaskCreate([](void* arg) { + auto audio_device = (AudioDevice*)arg; + audio_device->AudioPlayTask(); + }, "audio_play", 4096 * 4, this, 5, &audio_play_task_); +} + +void AudioDevice::CreateDuplexChannels() { + duplex_ = true; + + i2s_chan_config_t chan_cfg = { + .id = I2S_NUM_0, + .role = I2S_ROLE_MASTER, + .dma_desc_num = 6, + .dma_frame_num = 240, + .auto_clear_after_cb = false, + .auto_clear_before_cb = false, + .intr_priority = 0, + }; + ESP_ERROR_CHECK(i2s_new_channel(&chan_cfg, &tx_handle_, &rx_handle_)); + + i2s_std_config_t std_cfg = { + .clk_cfg = { + .sample_rate_hz = (uint32_t)output_sample_rate_, + .clk_src = I2S_CLK_SRC_DEFAULT, + .ext_clk_freq_hz = 0, + .mclk_multiple = I2S_MCLK_MULTIPLE_256 + }, + .slot_cfg = { + .data_bit_width = I2S_DATA_BIT_WIDTH_32BIT, + .slot_bit_width = I2S_SLOT_BIT_WIDTH_AUTO, + .slot_mode = I2S_SLOT_MODE_MONO, + .slot_mask = I2S_STD_SLOT_LEFT, + .ws_width = I2S_DATA_BIT_WIDTH_32BIT, + .ws_pol = false, + .bit_shift = true, + .left_align = true, + .big_endian = false, + .bit_order_lsb = false + }, + .gpio_cfg = { + .mclk = I2S_GPIO_UNUSED, + .bclk = GPIO_NUM_5, + .ws = GPIO_NUM_4, + .dout = GPIO_NUM_6, + .din = GPIO_NUM_3, + .invert_flags = { + .mclk_inv = false, + .bclk_inv = false, + .ws_inv = false + } + } + }; + ESP_ERROR_CHECK(i2s_channel_init_std_mode(tx_handle_, &std_cfg)); + ESP_ERROR_CHECK(i2s_channel_init_std_mode(rx_handle_, &std_cfg)); + ESP_LOGI(TAG, "Duplex channels created"); +} + +void AudioDevice::CreateSimplexChannels() { + // Create a new channel for speaker + i2s_chan_config_t chan_cfg = { + .id = I2S_NUM_0, + .role = I2S_ROLE_MASTER, + .dma_desc_num = 6, + .dma_frame_num = 240, + .auto_clear_after_cb = false, + .auto_clear_before_cb = false, + .intr_priority = 0, + }; + ESP_ERROR_CHECK(i2s_new_channel(&chan_cfg, &tx_handle_, nullptr)); + + i2s_std_config_t std_cfg = { + .clk_cfg = { + .sample_rate_hz = (uint32_t)output_sample_rate_, + .clk_src = I2S_CLK_SRC_DEFAULT, + .ext_clk_freq_hz = 0, + .mclk_multiple = I2S_MCLK_MULTIPLE_256 + }, + .slot_cfg = { + .data_bit_width = I2S_DATA_BIT_WIDTH_32BIT, + .slot_bit_width = I2S_SLOT_BIT_WIDTH_AUTO, + .slot_mode = I2S_SLOT_MODE_MONO, + .slot_mask = I2S_STD_SLOT_LEFT, + .ws_width = I2S_DATA_BIT_WIDTH_32BIT, + .ws_pol = false, + .bit_shift = true, + .left_align = true, + .big_endian = false, + .bit_order_lsb = false + }, + .gpio_cfg = { + .mclk = I2S_GPIO_UNUSED, + .bclk = GPIO_NUM_5, + .ws = GPIO_NUM_4, + .dout = GPIO_NUM_6, + .din = I2S_GPIO_UNUSED, + .invert_flags = { + .mclk_inv = false, + .bclk_inv = false, + .ws_inv = false + } + } + }; + ESP_ERROR_CHECK(i2s_channel_init_std_mode(tx_handle_, &std_cfg)); + + // Create a new channel for MIC + chan_cfg.id = I2S_NUM_1; + ESP_ERROR_CHECK(i2s_new_channel(&chan_cfg, nullptr, &rx_handle_)); + std_cfg.clk_cfg.sample_rate_hz = (uint32_t)input_sample_rate_; + std_cfg.gpio_cfg.bclk = GPIO_NUM_11; + std_cfg.gpio_cfg.ws = GPIO_NUM_10; + std_cfg.gpio_cfg.dout = I2S_GPIO_UNUSED; + std_cfg.gpio_cfg.din = GPIO_NUM_3; + ESP_ERROR_CHECK(i2s_channel_init_std_mode(rx_handle_, &std_cfg)); + ESP_LOGI(TAG, "Simplex channels created"); +} + +void AudioDevice::Write(const int16_t* data, int samples) { + int32_t buffer[samples]; + for (int i = 0; i < samples; i++) { + buffer[i] = int32_t(data[i]) << 15; + } + + size_t bytes_written; + ESP_ERROR_CHECK(i2s_channel_write(tx_handle_, buffer, samples * sizeof(int32_t), &bytes_written, portMAX_DELAY)); +} + +int AudioDevice::Read(int16_t* dest, int samples) { + size_t bytes_read; + + int32_t bit32_buffer_[samples]; + if (i2s_channel_read(rx_handle_, bit32_buffer_, samples * sizeof(int32_t), &bytes_read, portMAX_DELAY) != ESP_OK) { + ESP_LOGE(TAG, "Read Failed!"); + return 0; + } + + samples = bytes_read / sizeof(int32_t); + for (int i = 0; i < samples; i++) { + int32_t value = bit32_buffer_[i] >> 12; + dest[i] = (value > INT16_MAX) ? INT16_MAX : (value < -INT16_MAX) ? -INT16_MAX : (int16_t)value; + } + return samples; +} + +void AudioDevice::QueueAudioPacket(AudioPacket* packet) { + xQueueSend(audio_play_queue_, &packet, portMAX_DELAY); +} + +void AudioDevice::AudioPlayTask() { + while (true) { + AudioPacket* packet; + xQueueReceive(audio_play_queue_, &packet, portMAX_DELAY); + + switch (packet->type) + { + case kAudioPacketTypeStart: + playing_ = true; + breaked_ = false; + if (on_state_changed_) { + on_state_changed_(); + } + break; + case kAudioPacketTypeStop: + playing_ = false; + if (on_state_changed_) { + on_state_changed_(); + } + break; + case kAudioPacketTypeSentenceStart: + ESP_LOGI(TAG, "Playing sentence: %s", packet->text.c_str()); + break; + case kAudioPacketTypeSentenceEnd: + if (breaked_) { // Clear the queue + AudioPacket* p; + while (xQueueReceive(audio_play_queue_, &p, 0) == pdTRUE) { + delete p; + } + breaked_ = false; + } + break; + case kAudioPacketTypeData: + Write(packet->pcm.data(), packet->pcm.size()); + break; + default: + ESP_LOGE(TAG, "Unknown audio packet type: %d", packet->type); + } + delete packet; + } +} + +void AudioDevice::OnStateChanged(std::function callback) { + on_state_changed_ = callback; +} + +void AudioDevice::Break() { + breaked_ = true; +} diff --git a/main/AudioDevice.h b/main/AudioDevice.h new file mode 100644 index 00000000..cf5abd82 --- /dev/null +++ b/main/AudioDevice.h @@ -0,0 +1,76 @@ +#ifndef _AUDIO_DEVICE_H +#define _AUDIO_DEVICE_H + +#include "opus.h" +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" +#include "freertos/event_groups.h" +#include "driver/i2s_std.h" + +#include +#include +#include + +enum AudioPacketType { + kAudioPacketTypeUnkonwn = 0, + kAudioPacketTypeStart, + kAudioPacketTypeStop, + kAudioPacketTypeData, + kAudioPacketTypeSentenceStart, + kAudioPacketTypeSentenceEnd +}; + +struct AudioPacket { + AudioPacketType type = kAudioPacketTypeUnkonwn; + std::string text; + std::vector opus; + std::vector pcm; + uint32_t timestamp; +}; + +struct AudioDataHeader { + uint32_t version; + uint32_t reserved; + uint32_t timestamp; + uint32_t payload_size; +} __attribute__((packed)); + +class AudioDevice { +public: + AudioDevice(); + ~AudioDevice(); + + void Start(int input_sample_rate, int output_sample_rate); + int Read(int16_t* dest, int samples); + void Write(const int16_t* data, int samples); + void QueueAudioPacket(AudioPacket* packet); + void OnStateChanged(std::function callback); + void Break(); + + int input_sample_rate() const { return input_sample_rate_; } + int output_sample_rate() const { return output_sample_rate_; } + bool duplex() const { return duplex_; } + bool playing() const { return playing_; } + +private: + bool playing_ = false; + bool breaked_ = false; + bool duplex_ = false; + int input_sample_rate_ = 0; + int output_sample_rate_ = 0; + + i2s_chan_handle_t tx_handle_ = nullptr; + i2s_chan_handle_t rx_handle_ = nullptr; + + QueueHandle_t audio_play_queue_ = nullptr; + TaskHandle_t audio_play_task_ = nullptr; + + EventGroupHandle_t event_group_; + std::function on_state_changed_; + + void CreateDuplexChannels(); + void CreateSimplexChannels(); + void AudioPlayTask(); +}; + +#endif // _AUDIO_DEVICE_H diff --git a/main/BuiltinLed.cc b/main/BuiltinLed.cc new file mode 100644 index 00000000..4933b63c --- /dev/null +++ b/main/BuiltinLed.cc @@ -0,0 +1,89 @@ +#include "BuiltinLed.h" +#include +#include "driver/gpio.h" +#include "esp_log.h" + +#define TAG "builtin_led" + +BuiltinLed::BuiltinLed() { + mutex_ = xSemaphoreCreateMutex(); + + Configure(); + SetGreen(); +} + +BuiltinLed::~BuiltinLed() { + if (blink_task_ != nullptr) { + vTaskDelete(blink_task_); + } + if (led_strip_ != nullptr) { + led_strip_del(led_strip_); + } + vSemaphoreDelete(mutex_); +} + +void BuiltinLed::Configure() { + /* LED strip initialization with the GPIO and pixels number*/ + led_strip_config_t strip_config; + bzero(&strip_config, sizeof(strip_config)); + strip_config.strip_gpio_num = CONFIG_BUILTIN_LED_GPIO; + strip_config.max_leds = 1; + + led_strip_rmt_config_t rmt_config; + bzero(&rmt_config, sizeof(rmt_config)); + rmt_config.resolution_hz = 10 * 1000 * 1000; // 10MHz + + ESP_ERROR_CHECK(led_strip_new_rmt_device(&strip_config, &rmt_config, &led_strip_)); + /* Set all LED off to clear all pixels */ + led_strip_clear(led_strip_); +} + +void BuiltinLed::SetColor(uint8_t r, uint8_t g, uint8_t b) { + r_ = r; + g_ = g; + b_ = b; +} + +void BuiltinLed::TurnOn() { + xSemaphoreTake(mutex_, portMAX_DELAY); + led_strip_set_pixel(led_strip_, 0, r_, g_, b_); + led_strip_refresh(led_strip_); + xSemaphoreGive(mutex_); +} + +void BuiltinLed::TurnOff() { + xSemaphoreTake(mutex_, portMAX_DELAY); + led_strip_clear(led_strip_); + xSemaphoreGive(mutex_); +} + +void BuiltinLed::BlinkOnce() { + Blink(1, 100); +} + +void BuiltinLed::Blink(int times, int interval_ms) { + xSemaphoreTake(mutex_, portMAX_DELAY); + struct BlinkTaskArgs { + BuiltinLed* self; + int times; + int interval_ms; + }; + auto args = new BlinkTaskArgs {this, times, interval_ms}; + + xTaskCreate([](void* obj) { + auto args = (BlinkTaskArgs*) obj; + auto this_ = args->self; + for (int i = 0; i < args->times; i++) { + this_->TurnOn(); + vTaskDelay(args->interval_ms / portTICK_PERIOD_MS); + this_->TurnOff(); + vTaskDelay(args->interval_ms / portTICK_PERIOD_MS); + } + + delete args; + this_->blink_task_ = nullptr; + vTaskDelete(NULL); + }, "blink", 4096, args, tskIDLE_PRIORITY, &blink_task_); + + xSemaphoreGive(mutex_); +} diff --git a/main/BuiltinLed.h b/main/BuiltinLed.h new file mode 100644 index 00000000..1d53306c --- /dev/null +++ b/main/BuiltinLed.h @@ -0,0 +1,33 @@ +#ifndef _BUILTIN_LED_H_ +#define _BUILTIN_LED_H_ + +#include "led_strip.h" +#include "freertos/semphr.h" +#include "freertos/task.h" + +class BuiltinLed { +public: + BuiltinLed(); + ~BuiltinLed(); + + void BlinkOnce(); + void Blink(int times, int interval_ms); + void TurnOn(); + void TurnOff(); + void SetColor(uint8_t r, uint8_t g, uint8_t b); + void SetWhite() { SetColor(128, 128, 128); } + void SetGrey() { SetColor(32, 32, 32); } + void SetRed() { SetColor(128, 0, 0); } + void SetGreen() { SetColor(0, 128, 0); } + void SetBlue() { SetColor(0, 0, 128); } + +private: + SemaphoreHandle_t mutex_; + TaskHandle_t blink_task_ = nullptr; + led_strip_handle_t led_strip_ = nullptr; + uint8_t r_ = 0, g_ = 0, b_ = 0; + + void Configure(); +}; + +#endif // _BUILTIN_LED_H_ diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt new file mode 100755 index 00000000..63225bfe --- /dev/null +++ b/main/CMakeLists.txt @@ -0,0 +1,15 @@ +set(SOURCES "AudioDevice.cc" + "SystemInfo.cc" + "WebSocketClient.cc" + "OpusEncoder.cc" + "BuiltinLed.cc" + "Application.cc" + "WifiConfigurationAp.cc" + "main.cc" + "WifiStation.cc" + ) + +idf_component_register(SRCS ${SOURCES} + INCLUDE_DIRS "." + EMBED_TXTFILES "assets/wifi_configuration_ap.html" + ) diff --git a/main/Kconfig.projbuild b/main/Kconfig.projbuild new file mode 100644 index 00000000..f1345843 --- /dev/null +++ b/main/Kconfig.projbuild @@ -0,0 +1,22 @@ +menu "Xiaozhi Assistant" + +config WEBSOCKET_URL + string "Websocket URL" + default "ws://" + help + Communication with the server through websocket after wake up. + +config WEBSOCKET_ACCESS_TOKEN + string "Websocket Access Token" + default "" + help + Access token for websocket communication. + + +config BUILTIN_LED_GPIO + int "Builtin LED GPIO" + default 48 + help + GPIO number of the builtin LED. + +endmenu diff --git a/main/OpusEncoder.cc b/main/OpusEncoder.cc new file mode 100644 index 00000000..87d4cce0 --- /dev/null +++ b/main/OpusEncoder.cc @@ -0,0 +1,69 @@ +#include "OpusEncoder.h" +#include "esp_err.h" +#include "esp_log.h" + +#define TAG "OpusEncoder" + +OpusEncoder::OpusEncoder() { +} + +OpusEncoder::~OpusEncoder() { + if (out_buffer_ != nullptr) { + free(out_buffer_); + } + + if (audio_enc_ != nullptr) { + opus_encoder_destroy(audio_enc_); + } +} + +void OpusEncoder::Configure(int sample_rate, int channels, int duration_ms) { + if (audio_enc_ != nullptr) { + opus_encoder_destroy(audio_enc_); + audio_enc_ = nullptr; + } + if (out_buffer_ != nullptr) { + free(out_buffer_); + out_buffer_ = nullptr; + } + + int error; + audio_enc_ = opus_encoder_create(sample_rate, channels, OPUS_APPLICATION_VOIP, &error); + if (audio_enc_ == nullptr) { + ESP_LOGE(TAG, "Failed to create audio encoder, error code: %d", error); + return; + } + + // Set DTX + opus_encoder_ctl(audio_enc_, OPUS_SET_DTX(1)); + // Set complexity to 5 + opus_encoder_ctl(audio_enc_, OPUS_SET_COMPLEXITY(5)); + + frame_size_ = sample_rate / 1000 * duration_ms; + out_size_ = sample_rate * channels * sizeof(int16_t); + out_buffer_ = (uint8_t*)malloc(out_size_); + assert(out_buffer_ != nullptr); +} + +void OpusEncoder::Encode(const void* pcm, size_t pcm_len, std::function handler) { + if (audio_enc_ == nullptr) { + ESP_LOGE(TAG, "Audio encoder is not configured"); + return; + } + + in_buffer_.append((const char*)pcm, pcm_len); + + while (in_buffer_.size() >= frame_size_ * sizeof(int16_t)) { + auto ret = opus_encode(audio_enc_, (const opus_int16*)in_buffer_.data(), frame_size_, out_buffer_, out_size_); + if (ret < 0) { + ESP_LOGE(TAG, "Failed to encode audio, error code: %ld", ret); + return; + } + + if (handler != nullptr) { + handler(out_buffer_, ret); + } + + in_buffer_.erase(0, frame_size_ * sizeof(int16_t)); + } +} diff --git a/main/OpusEncoder.h b/main/OpusEncoder.h new file mode 100644 index 00000000..4e27071d --- /dev/null +++ b/main/OpusEncoder.h @@ -0,0 +1,25 @@ +#ifndef _OPUS_ENCODER_H_ +#define _OPUS_ENCODER_H_ + +#include +#include +#include "opus.h" + +class OpusEncoder { +public: + OpusEncoder(); + ~OpusEncoder(); + + void Configure(int sample_rate, int channels, int duration_ms = 60); + void Encode(const void* pcm, size_t pcm_len, std::function handler); + bool IsBufferEmpty() const { return in_buffer_.empty(); } + +private: + struct OpusEncoder* audio_enc_ = nullptr; + int frame_size_; + int out_size_; + uint8_t* out_buffer_ = nullptr; + std::string in_buffer_; +}; + +#endif // _OPUS_ENCODER_H_ diff --git a/main/SystemInfo.cc b/main/SystemInfo.cc new file mode 100644 index 00000000..0888aede --- /dev/null +++ b/main/SystemInfo.cc @@ -0,0 +1,211 @@ +#include "SystemInfo.h" +#include "freertos/task.h" +#include "esp_log.h" +#include "esp_flash.h" +#include "esp_mac.h" +#include "esp_chip_info.h" +#include "esp_system.h" +#include "esp_partition.h" +#include "esp_app_desc.h" +#include "esp_psram.h" + + +#define TAG "SystemInfo" + +size_t SystemInfo::GetFlashSize() { + uint32_t flash_size; + if (esp_flash_get_size(NULL, &flash_size) != ESP_OK) { + ESP_LOGE(TAG, "Failed to get flash size"); + return 0; + } + return (size_t)flash_size; +} + +size_t SystemInfo::GetMinimumFreeHeapSize() { + return esp_get_minimum_free_heap_size(); +} + +size_t SystemInfo::GetFreeHeapSize() { + return esp_get_free_heap_size(); +} + +std::string SystemInfo::GetMacAddress() { + uint8_t mac[6]; + esp_read_mac(mac, ESP_MAC_WIFI_STA); + char mac_str[18]; + snprintf(mac_str, sizeof(mac_str), "%02x:%02x:%02x:%02x:%02x:%02x", mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); + return std::string(mac_str); +} + +std::string SystemInfo::GetChipModelName() { + return std::string(CONFIG_IDF_TARGET); +} + +std::string SystemInfo::GetJsonString() { + /* + { + "flash_size": 4194304, + "psram_size": 0, + "minimum_free_heap_size": 123456, + "mac_address": "00:00:00:00:00:00", + "chip_model_name": "esp32s3", + "chip_info": { + "model": 1, + "cores": 2, + "revision": 0, + "features": 0 + }, + "application": { + "name": "my-app", + "version": "1.0.0", + "compile_time": "2021-01-01T00:00:00Z" + "idf_version": "4.2-dev" + "elf_sha256": "" + }, + "partition_table": { + "app": { + "label": "app", + "type": 1, + "subtype": 2, + "address": 0x10000, + "size": 0x100000 + }, + } + } + */ + std::string json = "{"; + json += "\"flash_size\":" + std::to_string(GetFlashSize()) + ","; + json += "\"psram_size\":" + std::to_string(esp_psram_get_size()) + ","; + json += "\"minimum_free_heap_size\":" + std::to_string(GetMinimumFreeHeapSize()) + ","; + json += "\"mac_address\":\"" + GetMacAddress() + "\","; + json += "\"chip_model_name\":\"" + GetChipModelName() + "\","; + json += "\"chip_info\":{"; + + esp_chip_info_t chip_info; + esp_chip_info(&chip_info); + json += "\"model\":" + std::to_string(chip_info.model) + ","; + json += "\"cores\":" + std::to_string(chip_info.cores) + ","; + json += "\"revision\":" + std::to_string(chip_info.revision) + ","; + json += "\"features\":" + std::to_string(chip_info.features); + json += "},"; + + json += "\"application\":{"; + auto app_desc = esp_app_get_description(); + json += "\"name\":\"" + std::string(app_desc->project_name) + "\","; + json += "\"version\":\"" + std::string(app_desc->version) + "\","; + json += "\"compile_time\":\"" + std::string(app_desc->date) + "T" + std::string(app_desc->time) + "Z\","; + json += "\"idf_version\":\"" + std::string(app_desc->idf_ver) + "\","; + + char sha256_str[65]; + for (int i = 0; i < 32; i++) { + snprintf(sha256_str + i * 2, sizeof(sha256_str) - i * 2, "%02x", app_desc->app_elf_sha256[i]); + } + json += "\"elf_sha256\":\"" + std::string(sha256_str) + "\""; + json += "},"; + + json += "\"partition_table\": ["; + esp_partition_iterator_t it = esp_partition_find(ESP_PARTITION_TYPE_ANY, ESP_PARTITION_SUBTYPE_ANY, NULL); + while (it) { + const esp_partition_t *partition = esp_partition_get(it); + json += "{"; + json += "\"label\":\"" + std::string(partition->label) + "\","; + json += "\"type\":" + std::to_string(partition->type) + ","; + json += "\"subtype\":" + std::to_string(partition->subtype) + ","; + json += "\"address\":" + std::to_string(partition->address) + ","; + json += "\"size\":" + std::to_string(partition->size); + json += "},"; + it = esp_partition_next(it); + } + json.pop_back(); // Remove the last comma + json += "]"; + + // Close the JSON object + json += "}"; + return json; +} + +esp_err_t SystemInfo::PrintRealTimeStats(TickType_t xTicksToWait) { + #define ARRAY_SIZE_OFFSET 5 + TaskStatus_t *start_array = NULL, *end_array = NULL; + UBaseType_t start_array_size, end_array_size; + configRUN_TIME_COUNTER_TYPE start_run_time, end_run_time; + esp_err_t ret; + uint32_t total_elapsed_time; + + //Allocate array to store current task states + start_array_size = uxTaskGetNumberOfTasks() + ARRAY_SIZE_OFFSET; + start_array = (TaskStatus_t*)malloc(sizeof(TaskStatus_t) * start_array_size); + if (start_array == NULL) { + ret = ESP_ERR_NO_MEM; + goto exit; + } + //Get current task states + start_array_size = uxTaskGetSystemState(start_array, start_array_size, &start_run_time); + if (start_array_size == 0) { + ret = ESP_ERR_INVALID_SIZE; + goto exit; + } + + vTaskDelay(xTicksToWait); + + //Allocate array to store tasks states post delay + end_array_size = uxTaskGetNumberOfTasks() + ARRAY_SIZE_OFFSET; + end_array = (TaskStatus_t*)malloc(sizeof(TaskStatus_t) * end_array_size); + if (end_array == NULL) { + ret = ESP_ERR_NO_MEM; + goto exit; + } + //Get post delay task states + end_array_size = uxTaskGetSystemState(end_array, end_array_size, &end_run_time); + if (end_array_size == 0) { + ret = ESP_ERR_INVALID_SIZE; + goto exit; + } + + //Calculate total_elapsed_time in units of run time stats clock period. + total_elapsed_time = (end_run_time - start_run_time); + if (total_elapsed_time == 0) { + ret = ESP_ERR_INVALID_STATE; + goto exit; + } + + printf("| Task | Run Time | Percentage\n"); + //Match each task in start_array to those in the end_array + for (int i = 0; i < start_array_size; i++) { + int k = -1; + for (int j = 0; j < end_array_size; j++) { + if (start_array[i].xHandle == end_array[j].xHandle) { + k = j; + //Mark that task have been matched by overwriting their handles + start_array[i].xHandle = NULL; + end_array[j].xHandle = NULL; + break; + } + } + //Check if matching task found + if (k >= 0) { + uint32_t task_elapsed_time = end_array[k].ulRunTimeCounter - start_array[i].ulRunTimeCounter; + uint32_t percentage_time = (task_elapsed_time * 100UL) / (total_elapsed_time * CONFIG_FREERTOS_NUMBER_OF_CORES); + printf("| %-16s | %8lu | %4lu%%\n", start_array[i].pcTaskName, task_elapsed_time, percentage_time); + } + } + + //Print unmatched tasks + for (int i = 0; i < start_array_size; i++) { + if (start_array[i].xHandle != NULL) { + printf("| %s | Deleted\n", start_array[i].pcTaskName); + } + } + for (int i = 0; i < end_array_size; i++) { + if (end_array[i].xHandle != NULL) { + printf("| %s | Created\n", end_array[i].pcTaskName); + } + } + ret = ESP_OK; + +exit: //Common return path + free(start_array); + free(end_array); + return ret; +} + diff --git a/main/SystemInfo.h b/main/SystemInfo.h new file mode 100644 index 00000000..b0744d2b --- /dev/null +++ b/main/SystemInfo.h @@ -0,0 +1,20 @@ +#ifndef _SYSTEM_INFO_H_ +#define _SYSTEM_INFO_H_ + +#include + +#include "esp_err.h" +#include "freertos/FreeRTOS.h" + +class SystemInfo { +public: + static size_t GetFlashSize(); + static size_t GetMinimumFreeHeapSize(); + static size_t GetFreeHeapSize(); + static std::string GetMacAddress(); + static std::string GetChipModelName(); + static std::string GetJsonString(); + static esp_err_t PrintRealTimeStats(TickType_t xTicksToWait); +}; + +#endif // _SYSTEM_INFO_H_ diff --git a/main/WebSocketClient.cc b/main/WebSocketClient.cc new file mode 100644 index 00000000..5b8eeee2 --- /dev/null +++ b/main/WebSocketClient.cc @@ -0,0 +1,130 @@ +#include "WebSocketClient.h" +#include +#include "freertos/task.h" +#include "esp_log.h" + + +#define TAG "WebSocket" +#define TIMEOUT_TICKS pdMS_TO_TICKS(3000) + +WebSocketClient::WebSocketClient(bool auto_reconnect) { + event_group_ = xEventGroupCreate(); + + esp_websocket_client_config_t config = {}; + config.task_prio = 1; + config.disable_auto_reconnect = !auto_reconnect; + client_ = esp_websocket_client_init(&config); + assert(client_ != NULL); + + esp_websocket_register_events(client_, WEBSOCKET_EVENT_ANY, [](void* arg, esp_event_base_t event_base, int32_t event_id, void* event_data) { + WebSocketClient* ws = (WebSocketClient*)arg; + esp_websocket_event_data_t *data = (esp_websocket_event_data_t *)event_data; + switch (event_id) + { + case WEBSOCKET_EVENT_BEFORE_CONNECT: + break; + case WEBSOCKET_EVENT_CONNECTED: + if (ws->on_connected_) { + ws->on_connected_(); + } + xEventGroupSetBits(ws->event_group_, WEBSOCKET_CONNECTED_BIT); + break; + case WEBSOCKET_EVENT_DISCONNECTED: + xEventGroupSetBits(ws->event_group_, WEBSOCKET_DISCONNECTED_BIT); + break; + case WEBSOCKET_EVENT_DATA: + if (data->data_len != data->payload_len) { + ESP_LOGE(TAG, "Payload segmentating is not supported, data_len: %d, payload_len: %d", data->data_len, data->payload_len); + break; + } + if (data->op_code == 8) { // Websocket close + ESP_LOGI(TAG, "Websocket closed"); + if (ws->on_closed_) { + ws->on_closed_(); + } + } else if (data->op_code == 9) { + // Websocket ping + } else if (data->op_code == 10) { + // Websocket pong + } else if (data->op_code == 1) { + // Websocket text + if (ws->on_data_) { + ws->on_data_(data->data_ptr, data->data_len, false); + } + } else if (data->op_code == 2) { + // Websocket binary + if (ws->on_data_) { + ws->on_data_(data->data_ptr, data->data_len, true); + } + } else { + ESP_LOGI(TAG, "Unknown opcode: %d", data->op_code); + } + break; + case WEBSOCKET_EVENT_ERROR: + if (ws->on_error_) { + ws->on_error_(data->error_handle.error_type); + } + xEventGroupSetBits(ws->event_group_, WEBSOCKET_ERROR_BIT); + break; + case WEBSOCKET_EVENT_CLOSED: + break; + default: + ESP_LOGI(TAG, "Event %ld", event_id); + } + }, this); +} + +WebSocketClient::~WebSocketClient() { + esp_websocket_client_close(client_, TIMEOUT_TICKS); + ESP_LOGI(TAG, "Destroying websocket client"); + esp_websocket_client_destroy(client_); +} + +void WebSocketClient::SetHeader(const char* key, const char* value) { + esp_websocket_client_append_header(client_, key, value); +} + +bool WebSocketClient::Connect(const char* uri) { + esp_websocket_client_set_uri(client_, uri); + esp_websocket_client_start(client_); + + // Wait for the connection to be established or an error + EventBits_t bits = xEventGroupWaitBits(event_group_, WEBSOCKET_CONNECTED_BIT | WEBSOCKET_ERROR_BIT, pdFALSE, pdFALSE, TIMEOUT_TICKS); + return bits & WEBSOCKET_CONNECTED_BIT; +} + +void WebSocketClient::Send(const char* data, size_t len, bool binary) { + if (binary) { + esp_websocket_client_send_bin(client_, data, len, portMAX_DELAY); + } else { + esp_websocket_client_send_text(client_, data, len, portMAX_DELAY); + } +} + +void WebSocketClient::Send(const std::string& data) { + Send(data.c_str(), data.size(), false); +} + +void WebSocketClient::OnClosed(std::function callback) { + on_closed_ = callback; +} + +void WebSocketClient::OnData(std::function callback) { + on_data_ = callback; +} + +void WebSocketClient::OnError(std::function callback) { + on_error_ = callback; +} + +void WebSocketClient::OnConnected(std::function callback) { + on_connected_ = callback; +} + +void WebSocketClient::OnDisconnected(std::function callback) { + on_disconnected_ = callback; +} + +bool WebSocketClient::IsConnected() const { + return esp_websocket_client_is_connected(client_); +} diff --git a/main/WebSocketClient.h b/main/WebSocketClient.h new file mode 100644 index 00000000..3ac35d34 --- /dev/null +++ b/main/WebSocketClient.h @@ -0,0 +1,40 @@ +#ifndef _WEBSOCKET_CLIENT_H_ +#define _WEBSOCKET_CLIENT_H_ + +#include +#include +#include "esp_websocket_client.h" +#include "freertos/event_groups.h" + +#define WEBSOCKET_CONNECTED_BIT BIT0 +#define WEBSOCKET_DISCONNECTED_BIT BIT1 +#define WEBSOCKET_ERROR_BIT BIT2 + +class WebSocketClient { +public: + WebSocketClient(bool auto_reconnect = false); + ~WebSocketClient(); + + void SetHeader(const char* key, const char* value); + bool IsConnected() const; + bool Connect(const char* uri); + void Send(const std::string& data); + void Send(const char* data, size_t len, bool binary = false); + + void OnConnected(std::function callback); + void OnDisconnected(std::function callback); + void OnData(std::function callback); + void OnError(std::function callback); + void OnClosed(std::function callback); + +private: + esp_websocket_client_handle_t client_ = NULL; + EventGroupHandle_t event_group_; + std::function on_data_; + std::function on_error_; + std::function on_closed_; + std::function on_connected_; + std::function on_disconnected_; +}; + +#endif // _WEBSOCKET_CLIENT_H_ diff --git a/main/WifiConfigurationAp.cc b/main/WifiConfigurationAp.cc new file mode 100644 index 00000000..456acdd9 --- /dev/null +++ b/main/WifiConfigurationAp.cc @@ -0,0 +1,259 @@ +#include "WifiConfigurationAp.h" +#include + +#include "esp_err.h" +#include "esp_event.h" +#include "esp_wifi.h" +#include "esp_log.h" +#include "esp_mac.h" +#include "esp_netif.h" +#include "lwip/ip_addr.h" +#include "nvs.h" +#include "nvs_flash.h" +#include "freertos/task.h" + +#define TAG "WifiConfigurationAp" + +extern const char index_html_start[] asm("_binary_wifi_configuration_ap_html_start"); + +#define WIFI_CONNECTED_BIT BIT0 +#define WIFI_FAIL_BIT BIT1 + + +WifiConfigurationAp::WifiConfigurationAp() +{ + event_group_ = xEventGroupCreate(); +} + +std::string WifiConfigurationAp::GetSsid() +{ + // Get MAC and use it to generate a unique SSID + uint8_t mac[6]; + ESP_ERROR_CHECK(esp_read_mac(mac, ESP_MAC_WIFI_SOFTAP)); + char ssid[32]; + snprintf(ssid, sizeof(ssid), "ESP32-%02X%02X%02X", mac[3], mac[4], mac[5]); + return std::string(ssid); +} + +void WifiConfigurationAp::StartAccessPoint() +{ + // Get the SSID + std::string ssid = GetSsid(); + + // Register the WiFi event handler + ESP_ERROR_CHECK(esp_event_handler_register(WIFI_EVENT, ESP_EVENT_ANY_ID, + [](void *ctx, esp_event_base_t event_base, int32_t event_id, void *event_data) { + if (event_id == WIFI_EVENT_AP_STACONNECTED) { + wifi_event_ap_staconnected_t *event = (wifi_event_ap_staconnected_t *)event_data; + ESP_LOGI(TAG, "Station connected: " MACSTR, MAC2STR(event->mac)); + } else if (event_id == WIFI_EVENT_AP_STADISCONNECTED) { + wifi_event_ap_stadisconnected_t *event = (wifi_event_ap_stadisconnected_t *)event_data; + ESP_LOGI(TAG, "Station disconnected: " MACSTR, MAC2STR(event->mac)); + } else if (event_id == WIFI_EVENT_STA_CONNECTED) { + xEventGroupSetBits(static_cast(ctx)->event_group_, WIFI_CONNECTED_BIT); + } else if (event_id == WIFI_EVENT_STA_DISCONNECTED) { + xEventGroupSetBits(static_cast(ctx)->event_group_, WIFI_FAIL_BIT); + } + }, this)); + + // Initialize the TCP/IP stack + ESP_ERROR_CHECK(esp_netif_init()); + + // Create the default event loop + auto netif = esp_netif_create_default_wifi_ap(); + + // Set the router IP address to 192.168.4.1 + esp_netif_ip_info_t ip_info; + IP4_ADDR(&ip_info.ip, 192, 168, 4, 1); + IP4_ADDR(&ip_info.gw, 192, 168, 4, 1); + IP4_ADDR(&ip_info.netmask, 255, 255, 255, 0); + esp_netif_dhcps_stop(netif); + esp_netif_set_ip_info(netif, &ip_info); + esp_netif_dhcps_start(netif); + + // Initialize the WiFi stack in Access Point mode + wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT(); + ESP_ERROR_CHECK(esp_wifi_init(&cfg)); + + // Set the WiFi configuration + wifi_config_t wifi_config = {}; + strcpy((char *)wifi_config.ap.ssid, ssid.c_str()); + wifi_config.ap.ssid_len = ssid.length(); + wifi_config.ap.max_connection = 4; + wifi_config.ap.authmode = WIFI_AUTH_OPEN; + + // Start the WiFi Access Point + ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_APSTA)); + ESP_ERROR_CHECK(esp_wifi_set_config(WIFI_IF_AP, &wifi_config)); + ESP_ERROR_CHECK(esp_wifi_start()); + + ESP_LOGI(TAG, "Access Point started with SSID %s", ssid.c_str()); +} + +void WifiConfigurationAp::StartWebServer() +{ + // Start the web server + httpd_config_t config = HTTPD_DEFAULT_CONFIG(); + config.uri_match_fn = httpd_uri_match_wildcard; + ESP_ERROR_CHECK(httpd_start(&server_, &config)); + + // Register the index.html file + httpd_uri_t index_html = { + .uri = "/", + .method = HTTP_GET, + .handler = [](httpd_req_t *req) -> esp_err_t { + httpd_resp_send(req, index_html_start, strlen(index_html_start)); + return ESP_OK; + }, + .user_ctx = NULL + }; + ESP_ERROR_CHECK(httpd_register_uri_handler(server_, &index_html)); + + // Register the /scan URI + httpd_uri_t scan = { + .uri = "/scan", + .method = HTTP_GET, + .handler = [](httpd_req_t *req) -> esp_err_t { + esp_wifi_scan_start(nullptr, true); + uint16_t ap_num = 0; + esp_wifi_scan_get_ap_num(&ap_num); + wifi_ap_record_t *ap_records = (wifi_ap_record_t *)malloc(ap_num * sizeof(wifi_ap_record_t)); + esp_wifi_scan_get_ap_records(&ap_num, ap_records); + + // Send the scan results as JSON + httpd_resp_set_type(req, "application/json"); + httpd_resp_sendstr_chunk(req, "["); + for (int i = 0; i < ap_num; i++) { + ESP_LOGI(TAG, "SSID: %s, RSSI: %d, Authmode: %d", + (char *)ap_records[i].ssid, ap_records[i].rssi, ap_records[i].authmode); + char buf[128]; + snprintf(buf, sizeof(buf), "{\"ssid\":\"%s\",\"rssi\":%d,\"authmode\":%d}", + (char *)ap_records[i].ssid, ap_records[i].rssi, ap_records[i].authmode); + httpd_resp_sendstr_chunk(req, buf); + if (i < ap_num - 1) { + httpd_resp_sendstr_chunk(req, ","); + } + } + httpd_resp_sendstr_chunk(req, "]"); + httpd_resp_sendstr_chunk(req, NULL); + free(ap_records); + return ESP_OK; + }, + .user_ctx = NULL + }; + ESP_ERROR_CHECK(httpd_register_uri_handler(server_, &scan)); + + // Register the form submission + httpd_uri_t form_submit = { + .uri = "/submit", + .method = HTTP_POST, + .handler = [](httpd_req_t *req) -> esp_err_t { + char buf[128]; + int ret = httpd_req_recv(req, buf, sizeof(buf)); + if (ret <= 0) { + if (ret == HTTPD_SOCK_ERR_TIMEOUT) { + httpd_resp_send_408(req); + } + return ESP_FAIL; + } + buf[ret] = '\0'; + ESP_LOGI(TAG, "Received form data: %s", buf); + + // Parse the form data + char ssid[32], password[64]; + if (sscanf(buf, "ssid=%32[^&]&password=%64s", ssid, password) != 2) { + httpd_resp_send_err(req, HTTPD_400_BAD_REQUEST, "Invalid form data"); + return ESP_FAIL; + } + + // Get this object from the user context + auto *this_ = static_cast(req->user_ctx); + if (!this_->ConnectToWifi(ssid, password)) { + char error[] = "Failed to connect to WiFi"; + char location[128]; + snprintf(location, sizeof(location), "/?error=%s&ssid=%s", error, ssid); + + httpd_resp_set_status(req, "302 Found"); + httpd_resp_set_hdr(req, "Location", location); + httpd_resp_send(req, NULL, 0); + return ESP_OK; + } + + // Set HTML response + httpd_resp_set_status(req, "200 OK"); + httpd_resp_set_type(req, "text/html"); + httpd_resp_send(req, "

Done!

", -1); + + this_->Save(ssid, password); + return ESP_OK; + }, + .user_ctx = this + }; + ESP_ERROR_CHECK(httpd_register_uri_handler(server_, &form_submit)); + + ESP_LOGI(TAG, "Web server started"); +} + +void WifiConfigurationAp::Start() +{ + builtin_led_.SetBlue(); + builtin_led_.Blink(1000, 500); + + StartAccessPoint(); + StartWebServer(); +} + +bool WifiConfigurationAp::ConnectToWifi(const std::string &ssid, const std::string &password) +{ + // auto esp_netif = esp_netif_create_default_wifi_sta(); + + wifi_config_t wifi_config; + bzero(&wifi_config, sizeof(wifi_config)); + strcpy((char *)wifi_config.sta.ssid, ssid.c_str()); + strcpy((char *)wifi_config.sta.password, password.c_str()); + wifi_config.sta.scan_method = WIFI_ALL_CHANNEL_SCAN; + wifi_config.sta.failure_retry_cnt = 1; + + ESP_ERROR_CHECK(esp_wifi_set_config(WIFI_IF_STA, &wifi_config)); + auto ret = esp_wifi_connect(); + if (ret != ESP_OK) { + ESP_LOGE(TAG, "Failed to connect to WiFi: %d", ret); + return false; + } + ESP_LOGI(TAG, "Connecting to WiFi %s", ssid.c_str()); + + // Wait for the connection to complete for 5 seconds + EventBits_t bits = xEventGroupWaitBits(event_group_, WIFI_CONNECTED_BIT | WIFI_FAIL_BIT, pdTRUE, pdFALSE, pdMS_TO_TICKS(10000)); + if (bits & WIFI_CONNECTED_BIT) { + ESP_LOGI(TAG, "Connected to WiFi %s", ssid.c_str()); + return true; + } else { + ESP_LOGE(TAG, "Failed to connect to WiFi %s", ssid.c_str()); + return false; + } +} + +void WifiConfigurationAp::Save(const std::string &ssid, const std::string &password) +{ + // Open the NVS flash + nvs_handle_t nvs_handle; + ESP_ERROR_CHECK(nvs_open("wifi", NVS_READWRITE, &nvs_handle)); + + // Write the SSID and password to the NVS flash + ESP_ERROR_CHECK(nvs_set_str(nvs_handle, "ssid", ssid.c_str())); + ESP_ERROR_CHECK(nvs_set_str(nvs_handle, "password", password.c_str())); + + // Commit the changes + ESP_ERROR_CHECK(nvs_commit(nvs_handle)); + + // Close the NVS flash + nvs_close(nvs_handle); + + ESP_LOGI(TAG, "WiFi configuration saved"); + // Use xTaskCreate to create a new task that restarts the ESP32 + xTaskCreate([](void *ctx) { + ESP_LOGI(TAG, "Restarting the ESP32 in 3 second"); + vTaskDelay(pdMS_TO_TICKS(3000)); + esp_restart(); + }, "restart_task", 4096, NULL, 5, NULL); +} diff --git a/main/WifiConfigurationAp.h b/main/WifiConfigurationAp.h new file mode 100644 index 00000000..8c14df70 --- /dev/null +++ b/main/WifiConfigurationAp.h @@ -0,0 +1,25 @@ +#ifndef _WIFI_CONFIGURATION_AP_H_ +#define _WIFI_CONFIGURATION_AP_H_ + +#include +#include "esp_http_server.h" +#include "BuiltinLed.h" + +class WifiConfigurationAp { +public: + WifiConfigurationAp(); + void Start(); + +private: + BuiltinLed builtin_led_; + httpd_handle_t server_ = NULL; + EventGroupHandle_t event_group_; + + std::string GetSsid(); + void StartAccessPoint(); + void StartWebServer(); + bool ConnectToWifi(const std::string &ssid, const std::string &password); + void Save(const std::string &ssid, const std::string &password); +}; + +#endif // _WIFI_CONFIGURATION_AP_H_ diff --git a/main/WifiStation.cc b/main/WifiStation.cc new file mode 100644 index 00000000..02776d39 --- /dev/null +++ b/main/WifiStation.cc @@ -0,0 +1,100 @@ +#include "WifiStation.h" +#include + +#include "esp_log.h" +#include "esp_wifi.h" +#include "nvs.h" +#include "nvs_flash.h" +#include "esp_netif.h" +#include "esp_system.h" + + +#define TAG "wifi" +#define WIFI_EVENT_CONNECTED BIT0 +#define WIFI_EVENT_FAILED BIT1 +#define MAX_RECONNECT_COUNT 5 + +WifiStation::WifiStation() { + // Get ssid and password from NVS + nvs_handle_t nvs_handle; + ESP_ERROR_CHECK(nvs_open("wifi", NVS_READONLY, &nvs_handle)); + char ssid[32], password[64]; + size_t length = sizeof(ssid); + ESP_ERROR_CHECK(nvs_get_str(nvs_handle, "ssid", ssid, &length)); + length = sizeof(password); + ESP_ERROR_CHECK(nvs_get_str(nvs_handle, "password", password, &length)); + nvs_close(nvs_handle); + + ssid_ = std::string(ssid); + password_ = std::string(password); + + // Create the event group + event_group_ = xEventGroupCreate(); + + // Register event handler + ESP_ERROR_CHECK(esp_event_handler_register(WIFI_EVENT, ESP_EVENT_ANY_ID, + [](void* event_handler_arg, esp_event_base_t event_base, int32_t event_id, void* event_data) { + auto this_ = static_cast(event_handler_arg); + if (event_id == WIFI_EVENT_STA_START) { + esp_wifi_connect(); + } else if (event_id == WIFI_EVENT_STA_DISCONNECTED) { + xEventGroupClearBits(this_->event_group_, WIFI_EVENT_CONNECTED); + if (this_->reconnect_count_ < MAX_RECONNECT_COUNT) { + esp_wifi_connect(); + this_->reconnect_count_++; + ESP_LOGI(TAG, "Reconnecting to WiFi (attempt %d)", this_->reconnect_count_); + } else { + xEventGroupSetBits(this_->event_group_, WIFI_EVENT_FAILED); + ESP_LOGI(TAG, "Failed to connect to WiFi"); + } + } + }, this)); + + ESP_ERROR_CHECK(esp_event_handler_register(IP_EVENT, IP_EVENT_STA_GOT_IP, + [](void* event_handler_arg, esp_event_base_t event_base, int32_t event_id, void* event_data) { + auto this_ = static_cast(event_handler_arg); + auto event = static_cast(event_data); + + char ip_address[16]; + esp_ip4addr_ntoa(&event->ip_info.ip, ip_address, sizeof(ip_address)); + this_->ip_address_ = ip_address; + ESP_LOGI(TAG, "Got IP: %s", this_->ip_address_.c_str()); + xEventGroupSetBits(this_->event_group_, WIFI_EVENT_CONNECTED); + }, this)); +} + + +void WifiStation::Start() { + // Initialize the TCP/IP stack + ESP_ERROR_CHECK(esp_netif_init()); + + // Create the default event loop + esp_netif_create_default_wifi_sta(); + + // Initialize the WiFi stack in station mode + wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT(); + ESP_ERROR_CHECK(esp_wifi_init(&cfg)); + ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_STA)); + + ESP_LOGI(TAG, "Connecting to WiFi ssid=%s password=%s", ssid_.c_str(), password_.c_str()); + wifi_config_t wifi_config; + bzero(&wifi_config, sizeof(wifi_config)); + strcpy((char *)wifi_config.sta.ssid, ssid_.c_str()); + strcpy((char *)wifi_config.sta.password, password_.c_str()); + ESP_ERROR_CHECK(esp_wifi_set_config(WIFI_IF_STA, &wifi_config)); + + // Start the WiFi stack + ESP_ERROR_CHECK(esp_wifi_start()); + + // Wait for the WiFi stack to start + auto bits = xEventGroupWaitBits(event_group_, WIFI_EVENT_CONNECTED | WIFI_EVENT_FAILED, pdFALSE, pdFALSE, portMAX_DELAY); + if (bits & WIFI_EVENT_FAILED) { + ESP_LOGE(TAG, "WifiStation start failed"); + } else { + ESP_LOGI(TAG, "WifiStation started"); + } +} + +bool WifiStation::IsConnected() { + return xEventGroupGetBits(event_group_) & WIFI_EVENT_CONNECTED; +} diff --git a/main/WifiStation.h b/main/WifiStation.h new file mode 100644 index 00000000..1d4912d1 --- /dev/null +++ b/main/WifiStation.h @@ -0,0 +1,23 @@ +#ifndef _WIFI_STATION_H_ +#define _WIFI_STATION_H_ + +#include +#include "esp_event.h" + +class WifiStation { +public: + WifiStation(); + void Start(); + bool IsConnected(); + std::string ssid() { return ssid_; } + std::string ip_address() { return ip_address_; } + +private: + EventGroupHandle_t event_group_; + std::string ssid_; + std::string password_; + std::string ip_address_; + int reconnect_count_ = 0; +}; + +#endif // _WIFI_STATION_H_ diff --git a/main/assets/wifi_configuration_ap.html b/main/assets/wifi_configuration_ap.html new file mode 100644 index 00000000..1b3adf52 --- /dev/null +++ b/main/assets/wifi_configuration_ap.html @@ -0,0 +1,137 @@ + + + + WiFi Configuration + + + + + +

WiFi Configuration

+
+

+

+

+ + +

+

+ + +

+

+ +

+

+

+
+ + + + diff --git a/main/idf_component.yml b/main/idf_component.yml new file mode 100644 index 00000000..efcccb6c --- /dev/null +++ b/main/idf_component.yml @@ -0,0 +1,20 @@ +## IDF Component Manager Manifest File +dependencies: + 78/esp-opus: "*" + espressif/esp_websocket_client: "^1.2.3" + espressif/led_strip: "*" + espressif/esp-sr: "^1.9.0" + ## Required IDF version + idf: + version: ">=4.1.0" + # # Put list of dependencies here + # # For components maintained by Espressif: + # component: "~1.0.0" + # # For 3rd party components: + # username/component: ">=1.0.0,<2.0.0" + # username2/component2: + # version: "~1.0.0" + # # For transient dependencies `public` flag can be set. + # # `public` flag doesn't have an effect dependencies of the `main` component. + # # All dependencies of `main` are public by default. + # public: true diff --git a/main/main.cc b/main/main.cc new file mode 100755 index 00000000..97e83ebb --- /dev/null +++ b/main/main.cc @@ -0,0 +1,66 @@ +#include + +#include "esp_log.h" +#include "esp_err.h" +#include "nvs.h" +#include "nvs_flash.h" +#include "driver/gpio.h" + +#include "WifiConfigurationAp.h" +#include "Application.h" +#include "SystemInfo.h" + +#define TAG "main" +#define STATS_TICKS pdMS_TO_TICKS(1000) + +extern "C" void app_main(void) +{ + // Initialize the default event loop + ESP_ERROR_CHECK(esp_event_loop_create_default()); + + // Configure GPIO1 as INPUT, reset NVS flash if the button is pressed + gpio_config_t io_conf; + io_conf.intr_type = GPIO_INTR_DISABLE; + io_conf.mode = GPIO_MODE_INPUT; + io_conf.pin_bit_mask = 1ULL << 1; + io_conf.pull_down_en = GPIO_PULLDOWN_DISABLE; + io_conf.pull_up_en = GPIO_PULLUP_ENABLE; + gpio_config(&io_conf); + + if (gpio_get_level(GPIO_NUM_1) == 0) { + ESP_LOGI(TAG, "Button is pressed, reset NVS flash"); + nvs_flash_erase(); + } + + // Initialize NVS flash for WiFi configuration + esp_err_t ret = nvs_flash_init(); + if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) { + ESP_ERROR_CHECK(nvs_flash_erase()); + ret = nvs_flash_init(); + } + ESP_ERROR_CHECK(ret); + + // Get the WiFi configuration + nvs_handle_t nvs_handle; + ret = nvs_open("wifi", NVS_READONLY, &nvs_handle); + + // If the WiFi configuration is not found, launch the WiFi configuration AP + if (ret != ESP_OK) { + auto app = new WifiConfigurationAp(); + app->Start(); + return; + } + nvs_close(nvs_handle); + + // Otherwise, launch the application + auto app = new Application(); + app->Start(); + + // Dump CPU usage every 1 second + while (true) { + vTaskDelay(2000 / portTICK_PERIOD_MS); + SystemInfo::PrintRealTimeStats(STATS_TICKS); + int free_sram = heap_caps_get_minimum_free_size(MALLOC_CAP_INTERNAL); + ESP_LOGI(TAG, "Free heap size: %u minimal internal: %u", SystemInfo::GetFreeHeapSize(), free_sram); + } +} diff --git a/main/resampler_structs.h b/main/resampler_structs.h new file mode 100644 index 00000000..9e9457d1 --- /dev/null +++ b/main/resampler_structs.h @@ -0,0 +1,60 @@ +/*********************************************************************** +Copyright (c) 2006-2011, Skype Limited. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef SILK_RESAMPLER_STRUCTS_H +#define SILK_RESAMPLER_STRUCTS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define SILK_RESAMPLER_MAX_FIR_ORDER 36 +#define SILK_RESAMPLER_MAX_IIR_ORDER 6 + +typedef struct _silk_resampler_state_struct{ + opus_int32 sIIR[ SILK_RESAMPLER_MAX_IIR_ORDER ]; /* this must be the first element of this struct */ + union{ + opus_int32 i32[ SILK_RESAMPLER_MAX_FIR_ORDER ]; + opus_int16 i16[ SILK_RESAMPLER_MAX_FIR_ORDER ]; + } sFIR; + opus_int16 delayBuf[ 48 ]; + opus_int resampler_function; + opus_int batchSize; + opus_int32 invRatio_Q16; + opus_int FIR_Order; + opus_int FIR_Fracs; + opus_int Fs_in_kHz; + opus_int Fs_out_kHz; + opus_int inputDelay; + const opus_int16 *Coefs; +} silk_resampler_state_struct; + +#ifdef __cplusplus +} +#endif +#endif /* SILK_RESAMPLER_STRUCTS_H */ + diff --git a/main/silk_resampler.h b/main/silk_resampler.h new file mode 100644 index 00000000..1f4518d1 --- /dev/null +++ b/main/silk_resampler.h @@ -0,0 +1,25 @@ +#ifndef _SILK_RESAMPLER_H_ +#define _SILK_RESAMPLER_H_ + + +/*! + * Initialize/reset the resampler state for a given pair of input/output sampling rates +*/ +extern "C" opus_int silk_resampler_init( + silk_resampler_state_struct *S, /* I/O Resampler state */ + opus_int32 Fs_Hz_in, /* I Input sampling rate (Hz) */ + opus_int32 Fs_Hz_out, /* I Output sampling rate (Hz) */ + opus_int forEnc /* I If 1: encoder; if 0: decoder */ +); + +/*! + * Resampler: convert from one sampling rate to another + */ +extern "C" opus_int silk_resampler( + silk_resampler_state_struct *S, /* I/O Resampler state */ + opus_int16 out[], /* O Output signal */ + const opus_int16 in[], /* I Input signal */ + opus_int32 inLen /* I Number of input samples */ +); + +#endif // _SILK_RESAMPLER_H_ diff --git a/partitions.csv b/partitions.csv new file mode 100644 index 00000000..963fc6e1 --- /dev/null +++ b/partitions.csv @@ -0,0 +1,9 @@ +# ESP-IDF Partition Table +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x4000, +otadata, data, ota, 0xd000, 0x2000, +phy_init, data, phy, 0xf000, 0x1000, +factory, app, factory, 0x10000, 2M, +ota_0, app, ota_0, , 2M, +ota_1, app, ota_1, , 2M, +model, data, spiffs, , 1M, diff --git a/sdkconfig.defaults b/sdkconfig.defaults new file mode 100644 index 00000000..38846b56 --- /dev/null +++ b/sdkconfig.defaults @@ -0,0 +1,34 @@ +CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_PERF=y +CONFIG_BOOTLOADER_LOG_LEVEL_NONE=y +CONFIG_BOOTLOADER_SKIP_VALIDATE_ALWAYS=y + +CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240=y +CONFIG_ESP32S3_INSTRUCTION_CACHE_32KB=y +CONFIG_ESP32S3_DATA_CACHE_64KB=y +CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y + +CONFIG_FLASHMODE_QIO=y + +CONFIG_ESP32S3_SPIRAM_SUPPORT=y +CONFIG_SPIRAM_MODE_OCT=y +CONFIG_SPIRAM_SPEED_80M=y +CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=4096 +CONFIG_SPIRAM_TRY_ALLOCATE_WIFI_LWIP=y +CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL=16384 +CONFIG_SPIRAM_MEMTEST=n + +CONFIG_HTTPD_MAX_REQ_HDR_LEN=2048 +CONFIG_HTTPD_MAX_URI_LEN=2048 + +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" +CONFIG_PARTITION_TABLE_FILENAME="partitions.csv" +CONFIG_PARTITION_TABLE_OFFSET=0x8000 + +CONFIG_MODEL_IN_SPIFFS=y +CONFIG_USE_WAKENET=y +CONFIG_SR_WN_WN9_NIHAOXIAOZHI_TTS=y +CONFIG_USE_MULTINET=n + +CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS=y +CONFIG_FREERTOS_USE_STATS_FORMATTING_FUNCTIONS=y