Add wake word to xmini-c3 (#730)

* esp-hi: MCP protocol is not ready yet

* Add wake word to xmini-c3

main/audio_processing/afe_audio_processor.cc

@@ -3,7 +3,7 @@
 
 #define PROCESSOR_RUNNING 0x01
 
-static const char* TAG = "AfeAudioProcessor";
+#define TAG "AfeAudioProcessor"
 
 AfeAudioProcessor::AfeAudioProcessor()
     : afe_data_(nullptr) {

main/audio_processing/afe_wake_word.cc

@@ -1,4 +1,4 @@
-#include "wake_word_detect.h"
+#include "afe_wake_word.h"
 #include "application.h"
 
 #include <esp_log.h>
@@ -8,9 +8,9 @@
 
 #define DETECTION_RUNNING_EVENT 1
 
-static const char* TAG = "WakeWordDetect";
+#define TAG "AfeWakeWord"
 
-WakeWordDetect::WakeWordDetect()
+AfeWakeWord::AfeWakeWord()
     : afe_data_(nullptr),
       wake_word_pcm_(),
       wake_word_opus_() {
@@ -18,7 +18,7 @@ WakeWordDetect::WakeWordDetect()
     event_group_ = xEventGroupCreate();
 }
 
-WakeWordDetect::~WakeWordDetect() {
+AfeWakeWord::~AfeWakeWord() {
     if (afe_data_ != nullptr) {
         afe_iface_->destroy(afe_data_);
     }
@@ -30,7 +30,7 @@ WakeWordDetect::~WakeWordDetect() {
     vEventGroupDelete(event_group_);
 }
 
-void WakeWordDetect::Initialize(AudioCodec* codec) {
+void AfeWakeWord::Initialize(AudioCodec* codec) {
     codec_ = codec;
     int ref_num = codec_->input_reference() ? 1 : 0;
 
@@ -67,46 +67,46 @@ void WakeWordDetect::Initialize(AudioCodec* codec) {
     afe_data_ = afe_iface_->create_from_config(afe_config);
 
     xTaskCreate([](void* arg) {
-        auto this_ = (WakeWordDetect*)arg;
+        auto this_ = (AfeWakeWord*)arg;
         this_->AudioDetectionTask();
         vTaskDelete(NULL);
     }, "audio_detection", 4096, this, 3, nullptr);
 }
 
-void WakeWordDetect::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
+void AfeWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
     wake_word_detected_callback_ = callback;
 }
 
-void WakeWordDetect::StartDetection() {
+void AfeWakeWord::StartDetection() {
     xEventGroupSetBits(event_group_, DETECTION_RUNNING_EVENT);
 }
 
-void WakeWordDetect::StopDetection() {
+void AfeWakeWord::StopDetection() {
     xEventGroupClearBits(event_group_, DETECTION_RUNNING_EVENT);
     if (afe_data_ != nullptr) {
         afe_iface_->reset_buffer(afe_data_);
     }
 }
 
-bool WakeWordDetect::IsDetectionRunning() {
+bool AfeWakeWord::IsDetectionRunning() {
     return xEventGroupGetBits(event_group_) & DETECTION_RUNNING_EVENT;
 }
 
-void WakeWordDetect::Feed(const std::vector<int16_t>& data) {
+void AfeWakeWord::Feed(const std::vector<int16_t>& data) {
     if (afe_data_ == nullptr) {
         return;
     }
     afe_iface_->feed(afe_data_, data.data());
 }
 
-size_t WakeWordDetect::GetFeedSize() {
+size_t AfeWakeWord::GetFeedSize() {
     if (afe_data_ == nullptr) {
         return 0;
     }
     return afe_iface_->get_feed_chunksize(afe_data_) * codec_->input_channels();
 }
 
-void WakeWordDetect::AudioDetectionTask() {
+void AfeWakeWord::AudioDetectionTask() {
     auto fetch_size = afe_iface_->get_fetch_chunksize(afe_data_);
     auto feed_size = afe_iface_->get_feed_chunksize(afe_data_);
     ESP_LOGI(TAG, "Audio detection task started, feed size: %d fetch size: %d",
@@ -121,7 +121,7 @@ void WakeWordDetect::AudioDetectionTask() {
         }
 
         // Store the wake word data for voice recognition, like who is speaking
-        StoreWakeWordData((uint16_t*)res->data, res->data_size / sizeof(uint16_t));
+        StoreWakeWordData(res->data, res->data_size / sizeof(int16_t));
 
         if (res->wakeup_state == WAKENET_DETECTED) {
             StopDetection();
@@ -134,7 +134,7 @@ void WakeWordDetect::AudioDetectionTask() {
     }
 }
 
-void WakeWordDetect::StoreWakeWordData(uint16_t* data, size_t samples) {
+void AfeWakeWord::StoreWakeWordData(const int16_t* data, size_t samples) {
     // store audio data to wake_word_pcm_
     wake_word_pcm_.emplace_back(std::vector<int16_t>(data, data + samples));
     // keep about 2 seconds of data, detect duration is 30ms (sample_rate == 16000, chunksize == 512)
@@ -143,13 +143,13 @@ void WakeWordDetect::StoreWakeWordData(uint16_t* data, size_t samples) {
     }
 }
 
-void WakeWordDetect::EncodeWakeWordData() {
+void AfeWakeWord::EncodeWakeWordData() {
     wake_word_opus_.clear();
     if (wake_word_encode_task_stack_ == nullptr) {
         wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(4096 * 8, MALLOC_CAP_SPIRAM);
     }
     wake_word_encode_task_ = xTaskCreateStatic([](void* arg) {
-        auto this_ = (WakeWordDetect*)arg;
+        auto this_ = (AfeWakeWord*)arg;
         {
             auto start_time = esp_timer_get_time();
             auto encoder = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
@@ -176,7 +176,7 @@ void WakeWordDetect::EncodeWakeWordData() {
     }, "encode_detect_packets", 4096 * 8, this, 2, wake_word_encode_task_stack_, &wake_word_encode_task_buffer_);
 }
 
-bool WakeWordDetect::GetWakeWordOpus(std::vector<uint8_t>& opus) {
+bool AfeWakeWord::GetWakeWordOpus(std::vector<uint8_t>& opus) {
     std::unique_lock<std::mutex> lock(wake_word_mutex_);
     wake_word_cv_.wait(lock, [this]() {
         return !wake_word_opus_.empty();

main/audio_processing/afe_wake_word.h

@@ -1,5 +1,5 @@
-#ifndef WAKE_WORD_DETECT_H
-#define WAKE_WORD_DETECT_H
+#ifndef AFE_WAKE_WORD_H
+#define AFE_WAKE_WORD_H
 
 #include <freertos/FreeRTOS.h>
 #include <freertos/task.h>
@@ -16,11 +16,12 @@
 #include <condition_variable>
 
 #include "audio_codec.h"
+#include "wake_word.h"
 
-class WakeWordDetect {
+class AfeWakeWord : public WakeWord {
 public:
-    WakeWordDetect();
-    ~WakeWordDetect();
+    AfeWakeWord();
+    ~AfeWakeWord();
 
     void Initialize(AudioCodec* codec);
     void Feed(const std::vector<int16_t>& data);
@@ -51,7 +52,7 @@ private:
     std::mutex wake_word_mutex_;
     std::condition_variable wake_word_cv_;
 
-    void StoreWakeWordData(uint16_t* data, size_t size);
+    void StoreWakeWordData(const int16_t* data, size_t size);
     void AudioDetectionTask();
 };
 

main/audio_processing/esp_wake_word.cc

@@ -1,4 +1,4 @@
-#include "wake_word_no_afe.h"
+#include "esp_wake_word.h"
 #include "application.h"
 
 #include <esp_log.h>
@@ -8,13 +8,13 @@
 
 #define DETECTION_RUNNING_EVENT 1
 
-static const char* TAG = "WakeWordDetect";
+#define TAG "EspWakeWord"
 
-WakeWordDetect::WakeWordDetect() {
+EspWakeWord::EspWakeWord() {
     event_group_ = xEventGroupCreate();
 }
 
-WakeWordDetect::~WakeWordDetect() {
+EspWakeWord::~EspWakeWord() {
     if (wakenet_data_ != nullptr) {
         wakenet_iface_->destroy(wakenet_data_);
         esp_srmodel_deinit(wakenet_model_);
@@ -23,13 +23,16 @@ WakeWordDetect::~WakeWordDetect() {
     vEventGroupDelete(event_group_);
 }
 
-void WakeWordDetect::Initialize(AudioCodec* codec) {
+void EspWakeWord::Initialize(AudioCodec* codec) {
     codec_ = codec;
 
     wakenet_model_ = esp_srmodel_init("model");
 
     if(wakenet_model_->num > 1) {
         ESP_LOGW(TAG, "More than one model found, using the first one");
+    } else if (wakenet_model_->num == 0) {
+        ESP_LOGE(TAG, "No model found");
+        return;
     }
     char *model_name = wakenet_model_->model_name[0];
     wakenet_iface_ = (esp_wn_iface_t*)esp_wn_handle_from_name(model_name);
@@ -40,28 +43,46 @@ void WakeWordDetect::Initialize(AudioCodec* codec) {
     ESP_LOGI(TAG, "Wake word(%s),freq: %d, chunksize: %d", model_name, frequency, audio_chunksize);
 }
 
-void WakeWordDetect::StartDetection() {
+void EspWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
+    wake_word_detected_callback_ = callback;
+}
+
+void EspWakeWord::StartDetection() {
+    ESP_LOGI(TAG, "Start wake word detection");
     xEventGroupSetBits(event_group_, DETECTION_RUNNING_EVENT);
 }
 
-void WakeWordDetect::StopDetection() {
+void EspWakeWord::StopDetection() {
+    ESP_LOGI(TAG, "Stop wake word detection");
     xEventGroupClearBits(event_group_, DETECTION_RUNNING_EVENT);
 }
 
-bool WakeWordDetect::IsDetectionRunning() {
+bool EspWakeWord::IsDetectionRunning() {
     return xEventGroupGetBits(event_group_) & DETECTION_RUNNING_EVENT;
 }
 
-void WakeWordDetect::Feed(const std::vector<int16_t>& data) {
+void EspWakeWord::Feed(const std::vector<int16_t>& data) {
     int res = wakenet_iface_->detect(wakenet_data_, (int16_t *)data.data());
     if (res > 0) {
-        ESP_LOGI(TAG, "Wake word detected");
-        auto& app = Application::GetInstance();
-        app.ToggleChatState();
+        StopDetection();
+        last_detected_wake_word_ = wakenet_iface_->get_word_name(wakenet_data_, res);
+
+        if (wake_word_detected_callback_) {
+            wake_word_detected_callback_(last_detected_wake_word_);
+        }
     }
 }
 
-size_t WakeWordDetect::GetFeedSize() {
-
+size_t EspWakeWord::GetFeedSize() {
+    if (wakenet_data_ == nullptr) {
+        return 0;
+    }
     return wakenet_iface_->get_samp_chunksize(wakenet_data_) * codec_->input_channels();
 }
+
+void EspWakeWord::EncodeWakeWordData() {
+}
+
+bool EspWakeWord::GetWakeWordOpus(std::vector<uint8_t>& opus) {
+    return false;
+}

main/audio_processing/esp_wake_word.h

@@ -1,13 +1,13 @@
-#ifndef WAKE_WORD_DETECT_H
-#define WAKE_WORD_DETECT_H
+#ifndef ESP_WAKE_WORD_H
+#define ESP_WAKE_WORD_H
 
 #include <freertos/FreeRTOS.h>
 #include <freertos/task.h>
 #include <freertos/event_groups.h>
 
-#include "model_path.h"
-#include "esp_wn_iface.h"
-#include "esp_wn_models.h"
+#include <esp_wn_iface.h>
+#include <esp_wn_models.h>
+#include <model_path.h>
 
 #include <list>
 #include <string>
@@ -17,19 +17,23 @@
 #include <condition_variable>
 
 #include "audio_codec.h"
-#include <model_path.h>
+#include "wake_word.h"
 
-class WakeWordDetect {
+class EspWakeWord : public WakeWord {
 public:
-    WakeWordDetect();
-    ~WakeWordDetect();
+    EspWakeWord();
+    ~EspWakeWord();
 
     void Initialize(AudioCodec* codec);
     void Feed(const std::vector<int16_t>& data);
+    void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback);
     void StartDetection();
     void StopDetection();
     bool IsDetectionRunning();
     size_t GetFeedSize();
+    void EncodeWakeWordData();
+    bool GetWakeWordOpus(std::vector<uint8_t>& opus);
+    const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; }
 
 private:
     esp_wn_iface_t *wakenet_iface_ = nullptr;
@@ -37,6 +41,9 @@ private:
     srmodel_list_t *wakenet_model_ = nullptr;
     EventGroupHandle_t event_group_;
     AudioCodec* codec_ = nullptr;
+
+    std::function<void(const std::string& wake_word)> wake_word_detected_callback_;
+    std::string last_detected_wake_word_;
 };
 
 #endif

main/audio_processing/no_audio_processor.cc

@@ -1,13 +1,13 @@
-#include "dummy_audio_processor.h"
+#include "no_audio_processor.h"
 #include <esp_log.h>
 
-#define TAG "DummyAudioProcessor"
+#define TAG "NoAudioProcessor"
 
-void DummyAudioProcessor::Initialize(AudioCodec* codec) {
+void NoAudioProcessor::Initialize(AudioCodec* codec) {
     codec_ = codec;
 }
 
-void DummyAudioProcessor::Feed(const std::vector<int16_t>& data) {
+void NoAudioProcessor::Feed(const std::vector<int16_t>& data) {
     if (!is_running_ || !output_callback_) {
         return;
     }
@@ -15,27 +15,27 @@ void DummyAudioProcessor::Feed(const std::vector<int16_t>& data) {
     output_callback_(std::vector<int16_t>(data));
 }
 
-void DummyAudioProcessor::Start() {
+void NoAudioProcessor::Start() {
     is_running_ = true;
 }
 
-void DummyAudioProcessor::Stop() {
+void NoAudioProcessor::Stop() {
     is_running_ = false;
 }
 
-bool DummyAudioProcessor::IsRunning() {
+bool NoAudioProcessor::IsRunning() {
     return is_running_;
 }
 
-void DummyAudioProcessor::OnOutput(std::function<void(std::vector<int16_t>&& data)> callback) {
+void NoAudioProcessor::OnOutput(std::function<void(std::vector<int16_t>&& data)> callback) {
     output_callback_ = callback;
 }
 
-void DummyAudioProcessor::OnVadStateChange(std::function<void(bool speaking)> callback) {
+void NoAudioProcessor::OnVadStateChange(std::function<void(bool speaking)> callback) {
     vad_state_change_callback_ = callback;
 }
 
-size_t DummyAudioProcessor::GetFeedSize() {
+size_t NoAudioProcessor::GetFeedSize() {
     if (!codec_) {
         return 0;
     }
@@ -43,7 +43,7 @@ size_t DummyAudioProcessor::GetFeedSize() {
     return 30 * codec_->input_sample_rate() / 1000;
 }
 
-void DummyAudioProcessor::EnableDeviceAec(bool enable) {
+void NoAudioProcessor::EnableDeviceAec(bool enable) {
     if (enable) {
         ESP_LOGE(TAG, "Device AEC is not supported");
     }

main/audio_processing/no_audio_processor.h

@@ -7,10 +7,10 @@
 #include "audio_processor.h"
 #include "audio_codec.h"
 
-class DummyAudioProcessor : public AudioProcessor {
+class NoAudioProcessor : public AudioProcessor {
 public:
-    DummyAudioProcessor() = default;
-    ~DummyAudioProcessor() = default;
+    NoAudioProcessor() = default;
+    ~NoAudioProcessor() = default;
 
     void Initialize(AudioCodec* codec) override;
     void Feed(const std::vector<int16_t>& data) override;

main/audio_processing/no_wake_word.cc

@@ -0,0 +1,45 @@
+#include "no_wake_word.h"
+#include <esp_log.h>
+
+#define TAG "NoWakeWord"
+
+void NoWakeWord::Initialize(AudioCodec* codec) {
+    codec_ = codec;
+}
+
+void NoWakeWord::Feed(const std::vector<int16_t>& data) {
+    // Do nothing - no wake word processing
+}
+
+void NoWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
+    // Do nothing - no wake word processing
+}
+
+void NoWakeWord::StartDetection() {
+    // Do nothing - no wake word processing
+}
+
+void NoWakeWord::StopDetection() {
+    // Do nothing - no wake word processing
+}
+
+bool NoWakeWord::IsDetectionRunning() {
+    return false;  // No wake word processing
+}
+
+size_t NoWakeWord::GetFeedSize() {
+    return 0;  // No specific feed size requirement
+}
+
+void NoWakeWord::EncodeWakeWordData() {
+    // Do nothing - no encoding needed
+}
+
+bool NoWakeWord::GetWakeWordOpus(std::vector<uint8_t>& opus) {
+    opus.clear();
+    return false;  // No opus data available
+}
+
+const std::string& NoWakeWord::GetLastDetectedWakeWord() const {
+    return "";  // No wake word detected
+}

main/audio_processing/no_wake_word.h

@@ -0,0 +1,31 @@
+#ifndef NO_WAKE_WORD_H
+#define NO_WAKE_WORD_H
+
+#include <vector>
+#include <functional>
+#include <string>
+
+#include "wake_word.h"
+#include "audio_codec.h"
+
+class NoWakeWord : public WakeWord {
+public:
+    NoWakeWord() = default;
+    ~NoWakeWord() = default;
+
+    void Initialize(AudioCodec* codec) override;
+    void Feed(const std::vector<int16_t>& data) override;
+    void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) override;
+    void StartDetection() override;
+    void StopDetection() override;
+    bool IsDetectionRunning() override;
+    size_t GetFeedSize() override;
+    void EncodeWakeWordData() override;
+    bool GetWakeWordOpus(std::vector<uint8_t>& opus) override;
+    const std::string& GetLastDetectedWakeWord() const override;
+
+private:
+    AudioCodec* codec_ = nullptr;
+};
+
+#endif 

main/audio_processing/wake_word.h

@@ -0,0 +1,26 @@
+#ifndef WAKE_WORD_H
+#define WAKE_WORD_H
+
+#include <string>
+#include <vector>
+#include <functional>
+
+#include "audio_codec.h"
+
+class WakeWord {
+public:
+    virtual ~WakeWord() = default;
+    
+    virtual void Initialize(AudioCodec* codec) = 0;
+    virtual void Feed(const std::vector<int16_t>& data) = 0;
+    virtual void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) = 0;
+    virtual void StartDetection() = 0;
+    virtual void StopDetection() = 0;
+    virtual bool IsDetectionRunning() = 0;
+    virtual size_t GetFeedSize() = 0;
+    virtual void EncodeWakeWordData() = 0;
+    virtual bool GetWakeWordOpus(std::vector<uint8_t>& opus) = 0;
+    virtual const std::string& GetLastDetectedWakeWord() const = 0;
+};
+
+#endif