CameraInputAndroid.cpp

Go to the documentation of this file.

/*
-----------------------------------------------------------------------------
This source file is part of OpenSpace3D
For the latest info, see http://www.openspace3d.com
 
Copyright (c) 2012 I-maginer
 
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU Lesser General Public License as published by the Free Software
Foundation; either version 2 of the License, or (at your option) any later
version.
 
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
 
You should have received a copy of the GNU Lesser General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place - Suite 330, Boston, MA 02111-1307, USA, or go to
http://www.gnu.org/copyleft/lesser.txt
 
-----------------------------------------------------------------------------
*/
 
#include "ArManager.h"
#include "ICameraInput.h"
#include "CameraInputAndroid.h"
 
#include <stdexcept>
#include <opencv2/opencv.hpp>
 
#ifdef BT_USE_LIBUVC
 
#include "libusb/libusb.h"
 
 
// Construction
CameraUvcAndroid::CameraUvcAndroid()
{
  mUpdated = false;
  mIsOpen = false;
  mIndex = 0;
  mFrameConv = 0;
  mUvcCtx = 0;
  mUvcDevh = 0;
 
  initJava();
 
  uvc_error_t res;
  res = uvc_init(&mUvcCtx, NULL);
  if (res < 0)
  {
    uvc_perror(res, "uvc_init");
    mUvcCtx = 0;
  }
}
 
void CameraUvcAndroid::initJava()
{
  mApp = (struct android_app*)SCgetExtra("this_inst");
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  libusb_set_option(0, LIBUSB_OPTION_ANDROID_JAVAVM, mApp->activity->vm, 0);
}
 
CameraUvcAndroid::~CameraUvcAndroid()
{
  mApp->activity->vm->DetachCurrentThread();
 
  if (mUvcDevh)
    uvc_close(mUvcDevh);
 
  if (mFrameConv)
    uvc_free_frame(mFrameConv);
 
  if (mUvcCtx)
    uvc_exit(mUvcCtx);
 
  mFrameRGB.release();
  mFrameBuffer.release();
}
 
bool CameraUvcAndroid::isValid()
{
  return (mUvcCtx) ? true : false;
}
 
bool CameraUvcAndroid::checkCameraPermission()
{
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  //wait for default authorization
  jobject oActivity = mApp->activity->clazz;
  jclass cActivity = env->GetObjectClass(oActivity);
 
  if (cActivity)
  {
    bool authPassed = false;
    while (!authPassed)
    {
      jfieldID reqBoolean = env->GetFieldID(cActivity, "mMinAuthPassed", "Z");
      authPassed = (bool) env->GetBooleanField(oActivity, reqBoolean);
      if (!authPassed)
        boost::this_thread::sleep_for(boost::chrono::milliseconds(100));
    }
 
    //check camera permission
    jmethodID camera_permission = env->GetMethodID(cActivity, "RequestCameraPermission", "()V");
    if (camera_permission)
      env->CallVoidMethod(oActivity, camera_permission);
 
    if (env->ExceptionCheck())
    {
      env->ExceptionClear();
      MMechostr(MSKRUNTIME, "Android camera exception : Check android.permission.CAMERA\n");
    }
 
    authPassed = false;
    while (!authPassed)
    {
      jfieldID reqBoolean = env->GetFieldID(cActivity, "mCameraAuthPassed", "Z");
      authPassed = (bool) env->GetBooleanField(oActivity, reqBoolean);
      if (!authPassed)
        boost::this_thread::sleep_for(boost::chrono::milliseconds(100));
    }
    env->DeleteLocalRef(cActivity);
 
    return authPassed;
  }
  else
    return false;
}
 
void CameraUvcAndroid::startPreview(int width, int height, int index)
{
  uvc_error_t res;
  uvc_device_t* uvcDev;
 
  mIndex = index;
  mIsOpen = false;
 
  if (!mUvcCtx)
  {
    res = uvc_init(&mUvcCtx, NULL);
    if (res < 0)
    {
      uvc_perror(res, "uvc_init");
      mUvcCtx = 0;
      return;
    }
  }
 
  if (checkCameraPermission())
  {
    /*
    uvc_device_t** device_list;
    uvc_device_t* dev;
    uvc_device_descriptor_t *device_desc;
    res = uvc_get_device_list(mUvcCtx, &device_list);
 
    if (res < 0)
    {
      uvc_perror(res, "uvc_get_device_list");
      return;
    }
 
    int i = 0;
    // iterate over all devices.
    while (device_list[i] != NULL)
    {
      // get the internal device object so we can use the libusb handle.
      dev = device_list[i];
 
      // get device descriptor.
      res = uvc_get_device_descriptor((uvc_device_t *)dev, &device_desc);
      if (res < 0)
      {
        uvc_free_device_list(device_list, 0);
        return;
      }
 
      uvc_free_device_descriptor(device_desc);
      i++;
    }
    uvc_free_device_list(device_list, 0);
    */
 
    // filter devices: vendor_id, product_id, "serial_num"
    res = uvc_find_device(mUvcCtx, &uvcDev, 0, 0, NULL);
 
    if (res < 0)
    {
      // no devices found
      uvc_perror(res, "uvc_find_device");
      mSize = cv::Size(0, 0);
      return;
    }
    else
    {
      // Try to open the device: requires exclusive access
      res = uvc_open(uvcDev, &mUvcDevh);
 
      if (res < 0)
      {
        // unable to open device
        uvc_perror(res, "uvc_open");
        mSize = cv::Size(0, 0);
        return;
      }
      else
      {
        // Print out a message containing all the information that libuvc knows about the device
        //uvc_print_diag(mUvcDevh, stderr);
 
        const uvc_format_desc_t *format_desc = uvc_get_format_descs(mUvcDevh);
        const uvc_frame_desc_t *frame_desc = format_desc->frame_descs;
        enum uvc_frame_format frame_format;
        mSize = cv::Size(width, height);
        int fps = 30;
 
        switch (format_desc->bDescriptorSubtype)
        {
          case UVC_VS_FORMAT_MJPEG:
            frame_format = UVC_FRAME_FORMAT_MJPEG;
            break;
          case UVC_VS_FORMAT_FRAME_BASED:
            frame_format = UVC_FRAME_FORMAT_H264;
            break;
          default:
            frame_format = UVC_FRAME_FORMAT_YUYV;
            break;
        }
 
        if (frame_desc)
          fps = 10000000 / frame_desc->dwDefaultFrameInterval;
 
        //printf("\nFirst format: (%4s) %dx%d %dfps\n", format_desc->fourccFormat, width, height, fps);
 
        // Try to negotiate first stream profile
        uvc_stream_ctrl_t uvcCtrl;
        res = uvc_get_stream_ctrl_format_size(mUvcDevh, &uvcCtrl, frame_format, width, height, fps);
 
        if (res < 0)
        {
          if (frame_desc)
          {
            // try with frame size
            res = uvc_get_stream_ctrl_format_size(mUvcDevh, &uvcCtrl, frame_format, frame_desc->wWidth, frame_desc->wHeight, fps);
            mSize = cv::Size(frame_desc->wWidth, frame_desc->wHeight);
          }
          else
          {
            res = uvc_get_stream_ctrl_format_size(mUvcDevh, &uvcCtrl, frame_format, 640, 480, fps);
            mSize = cv::Size(640, 480);
          }
        }
 
        // Print out the result
        //uvc_print_stream_ctrl(&uvcCtrl, stderr);
 
        if (res < 0)
        {
          // device doesn't provide a matching stream
          uvc_perror(res, "get_mode");
          mSize = cv::Size(0, 0);
          return;
        }
        else
        {
          if (mFrameConv)
            uvc_free_frame(mFrameConv);
 
          mFrameConv = uvc_allocate_frame(mSize.width * mSize.height * 3);
          mFrameBuffer = cv::Mat(mSize.height,  mSize.width, CV_8UC3);
          mFrameRGB = cv::Mat(mSize.height, mSize.width, CV_8UC3);
 
          // enable auto exposure - see uvc_set_ae_mode documentation
          const uint8_t UVC_AUTO_EXPOSURE_MODE_AUTO = 2;
          res = uvc_set_ae_mode(mUvcDevh, UVC_AUTO_EXPOSURE_MODE_AUTO);
 
          if (res == UVC_ERROR_PIPE)
          {
            // this error indicates that the camera does not support the full AE mode;
            // try again, using aperture priority mode (fixed aperture, variable exposure time)
            const uint8_t UVC_AUTO_EXPOSURE_MODE_APERTURE_PRIORITY = 8;
            res = uvc_set_ae_mode(mUvcDevh, UVC_AUTO_EXPOSURE_MODE_APERTURE_PRIORITY);
 
            if (res < 0)
              uvc_perror(res, " ... uvc_set_ae_mode failed to enable aperture priority mode");
            else
              uvc_perror(res, " ... uvc_set_ae_mode failed to enable auto exposure mode");
          }
 
          // Start the video stream. The library will call user function cb
          res = uvc_start_streaming(mUvcDevh, &uvcCtrl, CameraUvcAndroid::streamCb, (void *) this, 0);
 
          if (res < 0)
          {
            // unable to start stream
            uvc_perror(res, "start_streaming");
            mSize = cv::Size(0, 0);
            return;
          }
 
          mIsOpen = true;
        }
      }
    }
  }
 
  if (!mIsOpen)
    mSize = cv::Size(0, 0);
}
 
void CameraUvcAndroid::stopPreview()
{
  if (mUvcDevh)
  {
    uvc_close(mUvcDevh);
    mUvcDevh = 0;
  }
 
  if (mFrameConv)
  {
    uvc_free_frame(mFrameConv);
    mFrameConv = 0;
  }
 
  // also close context otherwise usb messages are stuck
  if (mUvcCtx)
  {
    uvc_exit(mUvcCtx);
    mUvcCtx = 0;
  }
 
  mIsOpen = false;
}
 
bool CameraUvcAndroid::isOpen()
{
  return mIsOpen;
}
 
void CameraUvcAndroid::streamCb(uvc_frame_t *frame, void *ptr)
{
  CameraUvcAndroid* uvc = static_cast<CameraUvcAndroid*>(ptr);
  uvc_error_t ret;
 
  if (!uvc->mFrameConv)
    return;
 
  ret = uvc_any2bgr(frame, uvc->mFrameConv);
  if (ret)
  {
    ret = uvc_any2rgb(frame, uvc->mFrameConv);
 
    if (ret)
    {
      uvc_perror(ret, "uvc_any2rgb");
      return;
    }
    else
    {
      uvc->mFrameBuffer.data = (uchar*)uvc->mFrameConv->data;
      cv::cvtColor(uvc->mFrameBuffer, uvc->mFrameRGB, cv::COLOR_RGB2BGR);
    }
  }
  else
  {
    uvc->mFrameBuffer.data = (uchar*)uvc->mFrameConv->data;
    uvc->mFrameBuffer.copyTo(uvc->mFrameRGB);
  }
 
  uvc->mUpdated = true;
}
 
cv::Mat CameraUvcAndroid::grabFrame()
{
  mUpdated = false;
  return mFrameRGB;
}
 
cv::Size CameraUvcAndroid::getSize()
{
  return mSize;
}
 
#endif //BT_USE_LIBUVC
 
 
// Constants
const int CameraBackendAndroid::CAMERA_FACING_BACK = 0;
const int CameraBackendAndroid::CAMERA_FACING_FRONT = 1;
const std::string CameraBackendAndroid::android_package = "org/imaginer/scol/CameraBackend";
 
// Construction
CameraBackendAndroid::CameraBackendAndroid()
{
  mUpdated = false;
  mIsOpen = false;
  mIndex = 0;
  initJava();
}
 
void CameraBackendAndroid::initJava()
{
  mApp = (struct android_app*)SCgetExtra("this_inst");
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
}
 
CameraBackendAndroid::jmethodbox CameraBackendAndroid::grabMethod(JNIEnv* env, const char* name, const char* signature)
{
  CameraBackendAndroid::jmethodbox result;
  result.clazz = 0;
  result.instance = 0;
  result.method = 0;
 
  // Get class
  jobject oActivity = mApp->activity->clazz;
  jclass cActivity = env->GetObjectClass(oActivity);
  if (cActivity)
  {
    jfieldID camera_backend_fieldID = env->GetFieldID(cActivity, "mCameraBackend", "Lorg/imaginer/scol/CameraBackend;");
    if (camera_backend_fieldID)
      result.instance = env->GetObjectField(oActivity, camera_backend_fieldID);
    if (result.instance)
      result.clazz = env->GetObjectClass(result.instance);
    if (result.clazz)
      result.method = env->GetMethodID(result.clazz, name, signature);
 
    env->DeleteLocalRef(cActivity);
  }
  return result;
}
 
CameraBackendAndroid::~CameraBackendAndroid()
{
  mApp->activity->vm->DetachCurrentThread();
  mFrameBuffer.release();
  mFrameRGB.release();
}
 
// Java methods
void CameraBackendAndroid::startPreview(int width, int height, int index)
{
  mIndex = index;
 
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  //wait for default authorization
  jobject oActivity = mApp->activity->clazz;
  jclass cActivity = env->GetObjectClass(oActivity);
 
  if (cActivity)
  {
    bool authPassed = false;
    while (!authPassed)
    {
      jfieldID reqBoolean = env->GetFieldID(cActivity, "mMinAuthPassed", "Z");
      authPassed = (bool) env->GetBooleanField(oActivity, reqBoolean);
      if (!authPassed)
        boost::this_thread::sleep_for(boost::chrono::milliseconds(100));
    }
 
    //check camera permission
    jmethodID camera_permission = env->GetMethodID(cActivity, "RequestCameraPermission", "()V");
    if (camera_permission)
      env->CallVoidMethod(oActivity, camera_permission);
 
    if (env->ExceptionCheck())
    {
      env->ExceptionClear();
      MMechostr(MSKRUNTIME, "Android camera exception : Check android.permission.CAMERA\n");
    }
 
    authPassed = false;
    while (!authPassed)
    {
      jfieldID reqBoolean = env->GetFieldID(cActivity, "mCameraAuthPassed", "Z");
      authPassed = (bool) env->GetBooleanField(oActivity, reqBoolean);
      if (!authPassed)
        boost::this_thread::sleep_for(boost::chrono::milliseconds(100));
    }
    env->DeleteLocalRef(cActivity);
  }
 
  LOGI(">>>>>>> JgrabMethod look for 'openCamera'");
  CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "openCamera", "(III)V");
  if (!method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'openCamera'");
    return;
  }
 
  LOGI(">>>>>>> JgrabMethod call method");
  env->CallVoidMethod(method_box.instance, method_box.method,  (jint)width, (jint)height, (jint)index);
  env->DeleteLocalRef(method_box.instance);
  env->DeleteLocalRef(method_box.clazz);
 
  if (env->ExceptionCheck())
  {
    env->ExceptionDescribe();
    env->ExceptionClear();
    LOGI(">>>>>>> 'openCamera' error");
    return;
  }
 
  if (isCameraOpened())
  {
    mSize = getPreviewSize();
    mFrameBuffer = cv::Mat(mSize.height + mSize.height / 2, mSize.width, CV_8UC1);
    mFrameRGB = cv::Mat(mSize.height,  mSize.width, CV_8UC3);
 
    mIsOpen = true;
  }
  else
  {
    mSize = cv::Size(0, 0);
  }
}
 
void CameraBackendAndroid::stopPreview()
{
  if (mIsOpen)
  {
    mIsOpen = false;
 
    JNIEnv* env = 0;
    if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
      mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
    CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "closeCamera", "()V");
    if (!method_box.instance)
    {
      LOGI(">>>>>>> JgrabMethod failed for 'closeCamera'");
      return;
    }
 
    env->CallVoidMethod(method_box.instance, method_box.method);
    env->DeleteLocalRef(method_box.instance);
    env->DeleteLocalRef(method_box.clazz);
 
    if (env->ExceptionCheck())
    {
      env->ExceptionClear();
      LOGI(">>>>>>> JgrabMethod failed for 'closeCamera'");
    }
  }
}
 
void CameraBackendAndroid::torchOn()
{
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "torchOn", "()V");
  if (!method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'torchOn'");
    return;
  }
 
  env->CallVoidMethod(method_box.instance, method_box.method);
  env->DeleteLocalRef(method_box.instance);
  env->DeleteLocalRef(method_box.clazz);
 
  if (env->ExceptionCheck())
  {
    env->ExceptionClear();
    LOGI(">>>>>>> JgrabMethod failed for 'torchOn'");
  }
}
 
void CameraBackendAndroid::torchOff()
{
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "torchOff", "()V");
  if (!method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'torchOn'");
    return;
  }
 
  env->CallVoidMethod(method_box.instance, method_box.method);
  env->DeleteLocalRef(method_box.instance);
  env->DeleteLocalRef(method_box.clazz);
 
  if (env->ExceptionCheck())
  {
    env->ExceptionClear();
    LOGI(">>>>>>> JgrabMethod failed for 'torchOn'");
  }
}
 
int CameraBackendAndroid::getDeviceOrientation()
{
  int ret = 0;
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "getDeviceOrientation", "()I");
  if (!method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'getDeviceOrientation'");
    return false;
  }
 
  ret = (int)env->CallIntMethod(method_box.instance, method_box.method);
  env->DeleteLocalRef(method_box.instance);
  env->DeleteLocalRef(method_box.clazz);
 
  if (env->ExceptionCheck())
    env->ExceptionClear();
 
  //LOGI(">>>>>>> getDeviceOrientation: %d", ret);
  return ret;
}
 
 
bool CameraBackendAndroid::isPortrait()
{
  int degree = getDeviceOrientation();
  if (degree == 90 || degree == 270)
    return true;
  
  return false;
}
 
bool CameraBackendAndroid::isCameraOpened()
{
  bool ret = false;
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "isCameraOpened", "()Z");
  if (!method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'isCameraOpened'");
    return false;
  }
 
  ret = (bool)env->CallBooleanMethod(method_box.instance, method_box.method);
  env->DeleteLocalRef(method_box.instance);
  env->DeleteLocalRef(method_box.clazz);
 
  if (env->ExceptionCheck())
    env->ExceptionClear();
 
  return ret;
}
 
bool CameraBackendAndroid::isOpen()
{
  return mIsOpen;
}
 
cv::Size CameraBackendAndroid::getPreviewSize()
{
  cv::Size nSize(0, 0);
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "getPreviewSize", "()Landroid/graphics/Point;");
  if (!method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'getPreviewSize'");
    return nSize;
  }
 
  jobject previewSize_object = env->CallObjectMethod(method_box.instance, method_box.method);
 
  if (env->ExceptionCheck())
  {
    env->ExceptionClear();
    return nSize;
  }
 
  if (previewSize_object == 0)
    return nSize;
 
  jclass cameraSize_class = env->GetObjectClass(previewSize_object);
  jfieldID size_width_field = env->GetFieldID(cameraSize_class, "x", "I");
  jfieldID size_height_field = env->GetFieldID(cameraSize_class, "y", "I");
 
  int width = env->GetIntField(previewSize_object, size_width_field);
  int height = env->GetIntField(previewSize_object, size_height_field);
  nSize = cv::Size(width, height);
 
  env->DeleteLocalRef(cameraSize_class);
  env->DeleteLocalRef(previewSize_object);
  env->DeleteLocalRef(method_box.instance);
  env->DeleteLocalRef(method_box.clazz);
  return nSize;
}
 
cv::Mat CameraBackendAndroid::grabFrame()
{
  mUpdated = false;
 
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  // get camera facing
  CameraBackendAndroid::jmethodbox facing_method_box = grabMethod(env, "isCameraFacing", "()Z");
  if (!facing_method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'isCameraFacing'");
    return cv::Mat();
  }
 
  bool camIsFacing = (bool)env->CallBooleanMethod(facing_method_box.instance, facing_method_box.method);
 
  if (env->ExceptionCheck())
    env->ExceptionClear();
 
  env->DeleteLocalRef(facing_method_box.instance);
  env->DeleteLocalRef(facing_method_box.clazz);
 
  // get image buffer
  CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "grabFrame", "()[B");
  if (!method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'grabFrame'");
    return cv::Mat();
  }
 
  jbyteArray frame_buffer_object = (jbyteArray)env->CallObjectMethod(method_box.instance, method_box.method);
  if (!frame_buffer_object || env->ExceptionCheck())
  {
    // read exception
    env->ExceptionClear();
    env->DeleteLocalRef(method_box.instance);
    env->DeleteLocalRef(method_box.clazz);
    return cv::Mat();
  }
 
  int length = env->GetArrayLength((jarray)frame_buffer_object);
  env->GetByteArrayRegion(frame_buffer_object, 0, length, (jbyte*)mFrameBuffer.data);
  env->DeleteLocalRef(frame_buffer_object);
  env->DeleteLocalRef(method_box.instance);
  env->DeleteLocalRef(method_box.clazz);
 
  if (mFrameBuffer.data == NULL || env->ExceptionCheck())
  {
    // read exception
    env->ExceptionClear();
    MMechostr(MSKRUNTIME, "Could not get camera frame");
    return cv::Mat();
  }
 
  if (mFrameBuffer.data)
  {
    cv::Mat rgbmat;
    cv::cvtColor(mFrameBuffer, rgbmat, cv::COLOR_YUV2BGR_NV21, 3);
 
    if (!rgbmat.empty())
    {
      //rotate picture
 
      int rotation = getDeviceOrientation();
      //image mirror for front view is managed by user
      if (rotation == 180)
      {
        cv::flip(rgbmat, mFrameRGB, camIsFacing ? 1 : -1);
      }
      else if (rotation == 90)
      {
        cv::transpose(rgbmat, mFrameRGB);
        cv::flip(mFrameRGB, mFrameRGB, camIsFacing ? 0 : 1);
      }
      else if (rotation == 270)
      {
        cv::transpose(rgbmat, mFrameRGB);
        if (mIndex != 1)
          cv::flip(mFrameRGB, mFrameRGB, 0);
      }
        /*else if(camIsFacing)
        {
          cv::flip(rgbmat, mFrameRGB, -1);
        }*/
      else
      {
        rgbmat.copyTo(mFrameRGB);
      }
 
      /*cv::Point center(rgbmat.cols / 2, rgbmat.rows / 2);
      cv::Mat rotationMatrix = cv::getRotationMatrix2D(center, rotation + 90, 1.0);
      cv::warpAffine(rgbmat, mFrameRGB, rotationMatrix, (rotation == 0 || rotation == 180) ? cv::Size(rgbmat.cols, rgbmat.rows) : cv::Size(rgbmat.rows, rgbmat.cols));
      */
 
      mUpdated = true;
    }
 
    if (!mUpdated || mFrameRGB.empty())
      return cv::Mat();
  }
  return mFrameRGB;
}
 
bool CameraBackendAndroid::setFocusPoint(int x, int y)
{
  int w = (int)ANativeWindow_getWidth(mApp->window);
  int h = (int)ANativeWindow_getHeight(mApp->window);
 
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "doTouchFocus", "(IIII)V");
  if (!method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'setFocusPoint'");
    return false;
  }
 
  env->CallVoidMethod(method_box.instance, method_box.method, (jint)x, (jint)y, (jint)w, (jint)h);
 
  if (env->ExceptionCheck())
    env->ExceptionClear();
 
  // Release references
  env->DeleteLocalRef(method_box.instance);
  env->DeleteLocalRef(method_box.clazz);
  return true;
}
 
cv::Size CameraBackendAndroid::getSize()
{
  return mSize;
}
 
 
CameraInputAndroid::CameraInputAndroid(int index)
  : ICameraInput(index)
{
 
#ifdef BT_USE_LIBUVC
  //Try to initialise usb cam first, otherwise fallback on android camera
  mUsbMode = false;
 
  mVI = 0;
 
  mUVC = new CameraUvcAndroid();
  if (mUVC->isValid())
  {
    if (index == 0)
      mUsbMode = true;
    else
      mIndex = index - 1;
  }
 
  if (!mUsbMode)
  {
    SAFE_DELETE(mUVC);
    mVI = new CameraBackendAndroid();
  }
#else
    mVI = new CameraBackendAndroid();
#endif
 
  mBufferSize = cv::Size(640, 480);
  mInitializing = false;
 
  MMechostr(MSKDEBUG, "CameraInputAndroid : Created CameraInputAndroid with size %d:%d", mBufferSize.width, mBufferSize.height);
}
 
bool CameraInputAndroid::Initialize()
{
  if (!mInitializing)
  {
    mInitializing = true;
 
    MMechostr(MSKDEBUG, "CameraInputAndroid::Initialize : Initializing camera backend with size %d:%d", mBufferSize.width, mBufferSize.height);
    cv::Size nSize;
#ifdef BT_USE_LIBUVC
    if (mUsbMode)
    {
      mUVC->startPreview(mBufferSize.width, mBufferSize.height, mIndex);
      if (mUVC->isOpen())
      {
        nSize = mUVC->getSize();
      }
      else
      {
        SAFE_DELETE(mUVC);
        mUsbMode = false;
        mVI = new CameraBackendAndroid();
      }
    }
 
    if (!mUsbMode)
    {
      mVI->startPreview(mBufferSize.width, mBufferSize.height, mIndex);
      nSize = mVI->getSize();
    }
#else
    mVI->startPreview(mBufferSize.width, mBufferSize.height, mIndex);
    nSize = mVI->getSize();
#endif
 
    mInitializing = false;
    if (nSize.width == 0 || nSize.height == 0)
      return false;
 
    mBufferSize = nSize;
  }
  return true;
}
 
void CameraInputAndroid::Close()
{
#ifdef BT_USE_LIBUVC
  if (mUsbMode)
  {
    if (!mInitializing && mUVC->isOpen())
      mUVC->stopPreview();
  }
  else
  {
    if (!mInitializing && mVI->isOpen())
      mVI->stopPreview();
  }
#else
  if (!mInitializing && mVI->isOpen())
    mVI->stopPreview();
#endif
}
 
CameraInputAndroid::~CameraInputAndroid()
{
  Close();
  mFrameCached.release();
 
#ifdef BT_USE_LIBUVC
  SAFE_DELETE(mUVC);
  SAFE_DELETE(mVI);
#else
  SAFE_DELETE(mVI);
#endif
}
 
bool CameraInputAndroid::IsOpened()
{
#ifdef BT_USE_LIBUVC
  if (mUsbMode)
  {
    return mUVC->isOpen();
  }
  else
  {
    return mVI->isOpen();
  }
#else
  return mVI->isOpen();
#endif
}
 
cv::Mat CameraInputAndroid::UpdateImage()
{
  if (!IsOpened())
    throw std::logic_error("CameraInputAndroid::UpdateImage : Device not initialised");
 
#ifdef BT_USE_LIBUVC
  if (mUsbMode)
  {
    mUVC->grabFrame().copyTo(mFrameCached);
  }
  else
  {
    mVI->grabFrame().copyTo(mFrameCached);
  }
#else
  mVI->grabFrame().copyTo(mFrameCached);
#endif
 
  if (mFrameCached.empty())
    throw std::logic_error("CameraInputAndroid::UpdateImage : Frame is empty");
 
  return mFrameCached;
}
 
int CameraInputAndroid::GetWidth()
{
#ifdef BT_USE_LIBUVC
  if (mUsbMode)
  {
    return mBufferSize.width;
  }
  else
  {
    return mVI->isPortrait() ? mBufferSize.height : mBufferSize.width;
  }
#else
  return mVI->isPortrait() ? mBufferSize.height : mBufferSize.width;
#endif
}
 
int CameraInputAndroid::GetHeight()
{
#ifdef BT_USE_LIBUVC
  if (mUsbMode)
  {
    return mBufferSize.height;
  }
  else
  {
    return mVI->isPortrait() ? mBufferSize.width : mBufferSize.height;
  }
#else
  return mVI->isPortrait() ? mBufferSize.width : mBufferSize.height;
#endif
}
 
void CameraInputAndroid::SetSize(int width, int height)
{
  if (IsOpened())
  {
    cv::Size cur;
 
    //MMechostr(MSKDEBUG, "CameraInputAndroid::SetSize : IsOpened : %d x %d\n", width, height);
    if (width < height)
    {
      int pw = height;
      height = width;
      width = pw;
    }
 
    cv::Size nSize;
 
#ifdef BT_USE_LIBUVC
    if (mUsbMode)
    {
      cur = mUVC->getSize();
      if (cur.width == width && cur.height == height)
        return;
 
      mUVC->stopPreview();
      mUVC->startPreview(width, height, mIndex);
      nSize = mUVC->getSize();
 
      MMechostr(MSKDEBUG, "CameraInputAndroid::SetSize : Camera set size : %ix%i\n", mBufferSize.width, mBufferSize.height);
    }
    else
    {
      cur = mVI->getSize();
      if (cur.width == width && cur.height == height)
        return;
 
      mVI->stopPreview();
      mVI->startPreview(width / 2, height / 2, mIndex);
      nSize = mVI->getSize();
 
      MMechostr(MSKDEBUG, "CameraInputAndroid::SetSize : Camera set size : %ix%i orientation : %i\n", mBufferSize.width, mBufferSize.height, mVI->getDeviceOrientation());
    }
#else
    cur = mVI->getSize();
    if (cur.width == width && cur.height == height)
      return;
 
    mVI->stopPreview();
    mVI->startPreview(width / 2, height / 2, mIndex);
    nSize = mVI->getSize();
 
    MMechostr(MSKDEBUG, "CameraInputAndroid::SetSize : Camera set size : %ix%i orientation : %i\n", mBufferSize.width, mBufferSize.height, mVI->getDeviceOrientation());
#endif
 
    mBufferSize = nSize;
  }
  else
  {
    mBufferSize = cv::Size(width, height);
  }
 
}
 
bool CameraInputAndroid::TakeSnapshot(std::string path)
{
  std::cout << "CameraInputAndroid::TakeSnapshot : taking a snapshot of camera" << mIndex << "... ";
  if (!IsOpened())
  {
    std::cout << "CameraInputAndroid::TakeSnapshot : failed. camera not opened yet." << std::endl;
    return false;
  }
 
  cv::Mat frame = UpdateImage();
  if (frame.empty())
  {
    std::cout << "CameraInputAndroid::TakeSnapshot : failed. Frame is empty." << std::endl;
    return false;
  }
 
  if (!cv::imwrite(path, frame))
  {
    std::cout << "CameraInputAndroid::TakeSnapshot : failed. Couldn't write to " << path << "." << std::endl;
    return false;
  }
 
  std::cout << "CameraInputAndroid::TakeSnapshot : success. Frame written to " << path << "." << std::endl;
  return true;
}
 
void CameraInputAndroid::RenderToScreen()
{
  cv::Mat frame;
  try
  {
    MMechostr(MSKDEBUG, "CameraInputAndroid::RenderToScreen : updating frame");
    frame = UpdateImage();
  }
  catch (std::exception e)
  {
    MMechostr(MSKDEBUG, "CameraInputAndroid::RenderToScreen : %s", e.what());
    return;
  }
 
  if (frame.empty())
  {
    MMechostr(MSKDEBUG, "CameraInputAndroid::RenderToScreen : frame empty!");
    return;
  }
 
  ANativeWindow* window = (ANativeWindow*)SCgetExtra("hscol");
  if (!window)
  {
    MMechostr(MSKDEBUG, "CameraInputAndroid::RenderToScreen : error: window empty !");
    return;
  }
  ANativeWindow_Buffer buffer;
 
  // Try lock buffer
  if (ANativeWindow_lock(window, &buffer, 0) < 0)
    return;
 
  //MMechostr(MSKDEBUG, "buffer info :: width = %d height = %d stride = %d format = %d", buffer.width, buffer.height, buffer.stride, buffer.format);
 
  cv::resize(frame, frame, cv::Size(buffer.width, buffer.height), 0, 0, cv::INTER_CUBIC);
 
  for (unsigned int y = 0; y < frame.rows; y++)
  {
    for (unsigned int x = 0; x < frame.cols; x++)
    {
      unsigned long srcByte = (x * 3) + (frame.cols * 3 * y);
      unsigned long destByte = (x * 4) + (buffer.width * 4 * y);
 
      ((unsigned char*)buffer.bits)[destByte + 2] = frame.data[srcByte];
      ((unsigned char*)buffer.bits)[destByte + 1] = frame.data[srcByte + 1];
      ((unsigned char*)buffer.bits)[destByte + 0] = frame.data[srcByte + 2];
      ((unsigned char*)buffer.bits)[destByte + 3] = 255;
    }
  }
 
  // Unlock buffer
  ANativeWindow_unlockAndPost(window);
}
 
bool CameraInputAndroid::SetFocusPoint(int x, int y)
{
#ifdef BT_USE_LIBUVC
  if (mUsbMode)
  {
    return false;
  }
  else
  {
    return mVI->setFocusPoint(x, y);
  }
#else
  return mVI->setFocusPoint(x, y);
#endif
}
 
void CameraInputAndroid::SetTorchState(bool state)
{
#ifdef BT_USE_LIBUVC
  if (mUsbMode)
  {
    return;
  }
  else
  {
    if (state)
      mVI->torchOn();
    else
      mVI->torchOff();
  }
#else
  if (state)
    mVI->torchOn();
  else
    mVI->torchOff();
#endif
}