reco_vosk.cpp

Go to the documentation of this file.

/*
-----------------------------------------------------------------------------
This source file is part of OpenSpace3D
For the latest info, see http://www.openspace3d.com
 
Copyright (c) 2012 I-maginer
 
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU Lesser General Public License as published by the Free Software
Foundation; either version 2 of the License, or (at your option) any later
version.
 
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
 
You should have received a copy of the GNU Lesser General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place - Suite 330, Boston, MA 02111-1307, USA, or go to
http://www.gnu.org/copyleft/lesser.txt
 
-----------------------------------------------------------------------------
*/
 
#include "reco_vosk.h"
 
#define DEF_SAMPLES 16000
#define FRAME_SIZE 320
#define MAX_RECOGNITION_TIME 2500
#define MAX_BUFFER DEF_SAMPLES*10
#define DEF_BUFFER DEF_SAMPLES
 
#include <iostream>
#include <sstream>
#include <algorithm>
 
#include "nlohmann/json.hpp"
 
using json = nlohmann::json;
 
#ifdef _WIN32
#include <windows.h>
 
#define P_SLASH "\\"
 
bool fileExists(const std::string& filePath)
{
  DWORD fileAttributes = GetFileAttributes(filePath.c_str());
  return (fileAttributes != INVALID_FILE_ATTRIBUTES);
}
#else
#include <iostream>
#include <sys/stat.h>
 
bool fileExists(const std::string& filePath)
{
  struct stat buffer;
  return (stat(filePath.c_str(), &buffer) == 0);
}
#endif
 
#define P_SLASH "/"
 
#ifdef ANDROID
#include <android/asset_manager.h>
#include <array>
 
void copyAssetFile(AAssetManager* manager, const std::string& assetFilePath, const std::string& destFilePath)
{
  AAsset* asset = AAssetManager_open(manager, assetFilePath.c_str(), AASSET_MODE_STREAMING);
  if (asset == nullptr)
  {
    // Handle error opening the asset file
    return;
  }
 
  // Get the file size
  off_t assetSize = AAsset_getLength(asset);
 
  // Open the destination file
  std::ofstream destFile(destFilePath, std::ios::out | std::ios::binary);
  if (!destFile.is_open())
  {
    // Handle error opening the destination file
    AAsset_close(asset);
    return;
  }
 
  // Set the buffer size for reading and writing chunks
  const int bufferSize = 8192; // 8 KB
  std::array<char, bufferSize> buffer;
 
  // Copy the asset file in chunks
  while (true)
  {
    int bytesRead = AAsset_read(asset, buffer.data(), bufferSize);
    if (bytesRead <= 0) {
      // End of file or error occurred
      break;
    }
 
    destFile.write(buffer.data(), bytesRead);
  }
 
  // Close the destination file
  destFile.close();
 
  // Close the asset file
  AAsset_close(asset);
}
 
void copyAssetDirectory(AAssetManager* manager, const std::string& assetDir, const std::string& destDir)
{
  // Create the destination directory
  mkdir(destDir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
 
  AAssetDir* assetDirObj = AAssetManager_openDir(manager, assetDir.c_str());
  if (assetDirObj == nullptr)
  {
    // Handle error opening the asset directory
    return;
  }
 
  const char* fileName = nullptr;
  while ((fileName = AAssetDir_getNextFileName(assetDirObj)) != nullptr)
  {
    std::string assetFilePath = assetDir + P_SLASH + std::string(fileName);
    std::string destFilePath = destDir + P_SLASH + std::string(fileName);
 
    copyAssetFile(manager, assetFilePath, destFilePath);
  }
 
  // Close the asset directory
  AAssetDir_close(assetDirObj);
}
 
 
std::string android_extract_dir(std::string dir)
{
  std::string assetDir = dir;
  std::string destDir;
 
  std::transform(assetDir.begin(), assetDir.end(), assetDir.begin(), ::tolower);
  if (assetDir.compare(0, 4, "apk/") != 0)
    return assetDir;
 
  assetDir = assetDir.substr(4).c_str();
 
  struct android_app* mApp = (struct android_app*)SCgetExtra("this_inst");
 
  const char* internalPath = mApp->activity->internalDataPath;
  destDir = std::string(internalPath) + P_SLASH + assetDir;
 
  AAssetManager* manager = mApp->activity->assetManager;
 
  if (fileExists(destDir))
    return destDir;
 
  //Creates destination directories
  std::string subPath;
  for (const char& c : destDir)
  {
    subPath += c;
    if (c == '/')
      mkdir(subPath.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
  }
 
  copyAssetDirectory(manager, assetDir, destDir);
  copyAssetDirectory(manager, assetDir + P_SLASH + "am", destDir + P_SLASH + "am");
  copyAssetDirectory(manager, assetDir + P_SLASH + "conf", destDir + P_SLASH + "conf");
  copyAssetDirectory(manager, assetDir + P_SLASH + "graph", destDir + P_SLASH + "graph");
  copyAssetDirectory(manager, assetDir + P_SLASH + "ivector", destDir + P_SLASH + "ivector");
 
  return destDir;
}
#endif //ANDROID
 
std::string getDirectoryFromFilePath(const std::string& filePath)
{
  size_t found = filePath.find_last_of("/\\");
 
  if (found != std::string::npos)
    return filePath.substr(0, found);
 
  return "";
}
 
std::string getFileNameWithoutExtension(const std::string& filePath)
{
  size_t lastSlash = filePath.find_last_of("/\\");
  size_t lastDot = filePath.find_last_of(".");
  if (lastDot != std::string::npos && (lastSlash == std::string::npos || lastDot > lastSlash))
    return filePath.substr(lastSlash + 1, lastDot - lastSlash - 1);
 
  return filePath.substr(lastSlash + 1);
}
 
std::string getValueFromJson(const std::string& jsonString, const std::string& key)
{
  try
  {
    json jsonData = json::parse(jsonString);
    if (jsonData.contains(key))
    {
      return jsonData[key].get<std::string>();
    }
    else
    {
      // Key not found
      return "";
    }
  }
  catch (const json::parse_error&)
  {
    // JSON parsing error
    return "";
  }
}
 
int countWords(const std::string& text)
{
  std::istringstream iss(text);
  int wordCount = 0;
  std::string word;
 
  while (iss >> word)
    ++wordCount;
 
  return wordCount;
}
 
Recognition::Recognition()
{
  mValid = false;
  mInSpeech = false;
  mBuffer = 0;
  mModel = NULL;
  mRecognizer = NULL;
  mTimeOutSamples = MAX_RECOGNITION_TIME * DEF_SAMPLES / 1000;
}
 
Recognition::Recognition(std::string pathtobin, std::string lang)
{
  mValid = false;
  mInSpeech = false;
  mBuffer = NULL;
  mModel = NULL;
  mRecognizer = NULL;
  mKeySearch = false;
  mTimeOutSamples = MAX_RECOGNITION_TIME * DEF_SAMPLES / 1000;
 
  std::string pathDir = getDirectoryFromFilePath(pathtobin);
 
#ifdef ANDROID
  //Extract the models in a place that kaldi can read on android
  pathDir = android_extract_dir(pathDir);
#endif
 
  mModel = vosk_model_new(pathDir.c_str());
  if (mModel != NULL)
    mRecognizer = vosk_recognizer_new(mModel, DEF_SAMPLES);
 
  if (mRecognizer != NULL)
    mValid = true;
 
  if (!mValid)
  {
    if (mRecognizer != NULL)
      vosk_recognizer_free(mRecognizer);
 
    if (mModel != NULL)
      vosk_model_free(mModel);
 
    mModel = NULL;
    mRecognizer = NULL;
  }
  else
  {
    mBuffer = new Buffer(MAX_BUFFER);
    mThread = std::thread(std::bind(&Recognition::cbThread, this));
  }
}
 
Recognition::~Recognition()
{
  mValid = false;
  if (mThread.joinable())
    mThread.join();
 
  if (mRecognizer != NULL)
    vosk_recognizer_free(mRecognizer);
 
  if (mModel != NULL)
    vosk_model_free(mModel);
 
  SAFE_DELETE(mBuffer);
}
 
 
void Recognition::fillAudioBuffer(const char* data, size_t lenght)
{
  if (mBuffer == NULL)
    return;
 
  mBuffer->fill(data, lenght);
}
 
void Recognition::cbThread()
{
  int final = 0;
  //consume buffer and send callbacks
  char buffer[DEF_BUFFER];
  int dataSize = 0;
  int frameSize = 0;
  int bufferCursor = 0;
 
  int remainingSamples = mTimeOutSamples;
  std::string lastPartial;
 
  while (mValid)
  {
    const std::lock_guard<std::mutex> lock(mMutexConfig);
 
    // Check if recognition has taken too long
    if (remainingSamples <= 0)
    {
      if (mInSpeech)
        OBJpostEvent(RECOGNITION_END_CB, SCOL_PTR this, 0);
 
      // Reset recognition
      vosk_recognizer_reset(mRecognizer);
 
      mInSpeech = false;
      lastPartial = "";
      remainingSamples = mTimeOutSamples;
      dataSize = 0;
    }
 
    //nothing to consume
    if (!mBuffer->getCount())
    {
      std::this_thread::sleep_for(std::chrono::milliseconds(1));
      continue;
    }
    
    //if (mBuffer->getCount() > (DEF_SAMPLES * 5))
    //  mBuffer->consume_safe(DEF_SAMPLES * 4);
    
    if (dataSize == 0)
    {
      dataSize = mBuffer->ProcessBuffer(buffer, (size_t)DEF_BUFFER, 0.01f);
      bufferCursor = 0;
      remainingSamples = mTimeOutSamples;
 
      if (dataSize == 0)
      {
        remainingSamples = 0;
        continue;
      }
    }
 
    frameSize = std::min(dataSize, FRAMESIZE);
    dataSize -= frameSize;
    remainingSamples -= frameSize;
 
    final = vosk_recognizer_accept_waveform(mRecognizer, buffer + bufferCursor, frameSize);
    bufferCursor += frameSize;
 
    if (!final)
    {
      std::string partial = getValueFromJson(vosk_recognizer_partial_result(mRecognizer), "partial").c_str();
      if (!partial.empty() && (partial != "[unk]"))
      {
        if (!mInSpeech)
          OBJpostEvent(RECOGNITION_START_CB, SCOL_PTR this, 0);
        
        if (lastPartial != partial)
        {
          cbData* cbdataText = new cbData(from_utf8(partial));
          OBJpostEvent(RECOGNITION_TEXT_CB, SCOL_PTR this, SCOL_PTR cbdataText);
        }
 
        lastPartial = partial;
        mInSpeech = true;
      }
    }
    else
    {
      std::string result = getValueFromJson(vosk_recognizer_result(mRecognizer), "text").c_str();
 
      if (!result.empty() && (result != "[unk]"))
      {
        if (!mInSpeech)
          OBJpostEvent(RECOGNITION_START_CB, SCOL_PTR this, 0);
 
        if (lastPartial != result)
        {
          cbData* cbdataText = new cbData(from_utf8(result));
          OBJpostEvent(RECOGNITION_TEXT_CB, SCOL_PTR this, SCOL_PTR cbdataText);
        }
 
        OBJpostEvent(RECOGNITION_END_CB, SCOL_PTR this, 0);
        mInSpeech = false;
 
        remainingSamples = mTimeOutSamples;
 
        dataSize = 0;
      }
      else
      {
        if (mInSpeech)
          OBJpostEvent(RECOGNITION_END_CB, SCOL_PTR this, 0);
 
        mInSpeech = false;
        continue;
      }
 
      lastPartial = "";
    }
  }
}
 
 
void Recognition::AddWord(std::string s_Rule, std::string s_Word)
{
  const std::lock_guard<std::mutex> lock(mMutexConfig);
  mKeyWords.push_back(s_Word);
 
  std::string wlist = "[";
  for (unsigned int i = 0; i < mKeyWords.size(); i++)
  {
    std::string w = mKeyWords[i];
    std::transform(w.begin(), w.end(), w.begin(), ::tolower);
    if (i == 0)
      wlist += "\"" + to_utf8(w) + "\"";
    else
      wlist += ",\"" + to_utf8(w) + "\"";
  }
  if (mKeyWords.size() > 0)
    wlist += ", \"[unk]\"";
 
  wlist += "]";
 
  vosk_recognizer_set_grm(mRecognizer, wlist.c_str());
}
 
bool Recognition::initializeObjects()
{
  return mValid;
}
 
int Recognition::getVolume()
{
  return -1;
}
 
 
void Recognition::setVolume(int volume)
{
}