ArFaceDetector.cpp

Go to the documentation of this file.

/*
-----------------------------------------------------------------------------
This source file is part of OpenSpace3D
For the latest info, see http://www.openspace3d.com
 
Copyright (c) 2012 I-maginer
 
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU Lesser General Public License as published by the Free Software
Foundation; either version 2 of the License, or (at your option) any later
version.
 
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
 
You should have received a copy of the GNU Lesser General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place - Suite 330, Boston, MA 02111-1307, USA, or go to
http://www.gnu.org/copyleft/lesser.txt
 
-----------------------------------------------------------------------------
*/
 
#include "ArManager.h"
#include "ArFaceDetector.h"
#include <vector>
#include <boost/filesystem.hpp>
 
ArFaceDetector::ArFaceDetector()
{
  needUpdate = false;
 
  try
  {
#ifdef ANDROID
    std::string facesxml = CopyFileFromAssetToSD("btdata/lbpcascade_frontalface.xml", "lbpcascade_frontalface.xml", "btdata");
    //std::string leyexml = CopyFileFromAssetToSD("btdata/haarcascade_mcs_lefteye.xml", "haarcascade_mcs_lefteye.xml", "btdata");
    //std::string reyexml = CopyFileFromAssetToSD("btdata/haarcascade_mcs_righteye.xml", "haarcascade_mcs_righteye.xml", "btdata");
    //std::string nosexml = CopyFileFromAssetToSD("btdata/haarcascade_mcs_nose.xml", "haarcascade_mcs_nose.xml", "btdata");
    //std::string mouthxml = CopyFileFromAssetToSD("btdata/haarcascade_mcs_mouth.xml", "haarcascade_mcs_mouth.xml", "btdata");
 
    mFaceClassifier.load(facesxml);
    //mLeftEyeClassifier.load(leyexml);
    //mRightEyeClassifier.load(reyexml);
    //mNoseClassifier.load(nosexml);
    //mMouthClassifier.load(mouthxml);
#else
    mFaceClassifier.load("plugins/btdata/lbpcascade_frontalface.xml");
    //mLeftEyeClassifier.load("plugins/btdata/haarcascade_mcs_lefteye.xml");
    //mRightEyeClassifier.load("plugins/btdata/haarcascade_mcs_righteye.xml");
    //mNoseClassifier.load("plugins/btdata/haarcascade_mcs_nose.xml");
    //mMouthClassifier.load("plugins/btdata/haarcascade_mcs_mouth.xml");
#endif
  }
  catch(std::exception&)
  {
    MMechostr(MSKRUNTIME, "BitmapToolkit Error : Could not load training data for face detector");
  }
 
  // Run thread
  StartThreading(boost::bind(&ArFaceDetector::GoThread, this));
 
  //cv::namedWindow("debug");
}
 
ArFaceDetector::~ArFaceDetector()
{
  StopThreading();
}
 
std::vector<cv::Rect> ArFaceDetector::DetectFaces(const cv::Mat input, std::vector<ArMarker*> faceMarkers, aruco::CameraParameters camParam, bool reversedBitmap)
{
  std::vector<cv::Rect> outVector;
  std::vector<cv::Rect> objList;
 
  if (mFaceClassifier.empty())
    return outVector;
 
  mFaceClassifier.detectMultiScale(input, objList, 1.2, 4, cv::CASCADE_SCALE_IMAGE/*|CV_HAAR_DO_CANNY_PRUNING*/|cv::CASCADE_FIND_BIGGEST_OBJECT, cv::Size(30, 30));
 
  for (unsigned int i = 0; i< objList.size(); i++)
  {
    cv::Mat objROI = input(objList[i]);
    cv::Rect objRect = objList[i];
 
    //search if rect is in the other markers area
    bool tracked = false;
    cv::Rect tRect = objRect;
    tRect.width += 40;
    tRect.height += 40;
    tRect.x -= 20;
    tRect.y -= 20;
    for (unsigned int j = 0; j < faceMarkers.size() && !tracked; j++)
    {
      ArFaceMarker* fmarker = static_cast<ArFaceMarker*> (faceMarkers[j]);
      if (fmarker->IsInitialized())
      {
        std::vector<cv::Point2f> coords = faceMarkers[j]->GetCorners();
        if (coords.size() == 4)
        {
          for (unsigned k = 0; k < coords.size() && !tracked; k++)
          {
            if (tRect.contains(coords[k]))
              tracked = true;
          }
 
          // double check
          if (!tracked)
          {
            cv::Rect nRect;
            nRect.x = (int)coords[0].x;
            nRect.y = (int)coords[0].y;
            nRect.width = (int)coords[2].x - (int)coords[0].x;
            nRect.height = (int)coords[2].y - (int)coords[0].y;
 
            std::vector<cv::Point2f> lrect;
            lrect.push_back(cv::Point2f((float)objRect.x, (float)objRect.y));
            lrect.push_back(cv::Point2f((float)objRect.x + objRect.width, (float)objRect.y));
            lrect.push_back(cv::Point2f((float)objRect.x + objRect.width, (float)objRect.y + objRect.height));
            lrect.push_back(cv::Point2f((float)objRect.x, (float)objRect.y + objRect.height));
 
            for (unsigned k = 0; k < lrect.size() && !tracked; k++)
            {
              if (nRect.contains(lrect[k]))
                tracked = true;
            }
          }
        }
      }
    }
 
    if (!tracked)
    {
      /*
      cv::Mat faceMask = input(objRect);
      std::vector<cv::Rect> LeyeList;
      mLeftEyeClassifier.detectMultiScale(faceMask, LeyeList, 1.3, 6, cv::CASCADE_SCALE_IMAGE, cv::Size(20, 20));
 
      std::vector<cv::Rect> ReyeList;
      mRightEyeClassifier.detectMultiScale(faceMask, ReyeList, 1.3, 6, cv::CASCADE_SCALE_IMAGE, cv::Size(20, 20));
 
      // mask the top of the head
      cv::Mat bMask;
      faceMask.copyTo(bMask);
      cv::rectangle(bMask, cv::Rect(0, 0, bMask.cols, (int)((float)bMask.rows / 2.5f)), cv::Scalar(0), cv::FILLED);
      std::vector<cv::Rect> noseList;
      mNoseClassifier.detectMultiScale(bMask, noseList, 1.3, 4, cv::CASCADE_SCALE_IMAGE, cv::Size(20, 20));
 
      std::vector<cv::Rect> mouthList;
      mMouthClassifier.detectMultiScale(bMask, mouthList, 1.3, 6, cv::CASCADE_SCALE_IMAGE, cv::Size(20, 20));
      
      if (!LeyeList.empty() && !ReyeList.empty() && !noseList.empty() && !mouthList.empty())
      {
        int hdiff = abs(LeyeList[0].y - ReyeList[0].y);
        int sdiff = abs(LeyeList[0].width - ReyeList[0].width);
        int eyemid = LeyeList[0].x + (abs((ReyeList[0].x + ReyeList[0].width) - LeyeList[0].x) / 2);
        int nosepos = noseList[0].x + (noseList[0].width / 2);
        int mdiff = abs(eyemid - nosepos);
        float mindiff = (float)objRect.width / 10;
        if (((float)hdiff < mindiff) && ((float)sdiff < mindiff) && ((float)mdiff < mindiff))
        {
          bool found = false;
          for (unsigned int j = 0; j < faceMarkers.size() && !found; j++)
          {
            ArFaceMarker* fmarker = static_cast<ArFaceMarker*> (faceMarkers[j]);
            if (!fmarker->IsInitialized())
            {
              std::vector<cv::Rect> lRects;
              lRects.push_back(objRect);
              //lRects.push_back(LeyeList[0]);
              //lRects.push_back(ReyeList[0]);
              //lRects.push_back(noseList[0]);
              //lRects.push_back(mouthList[0]);
              fmarker->SetImage(input, lRects);
              found = true;
            }
          }
        }
      }
      */
 
      bool found = false;
      for (unsigned int j = 0; j < faceMarkers.size() && !found; j++)
      {
        ArFaceMarker* fmarker = static_cast<ArFaceMarker*> (faceMarkers[j]);
        if (!fmarker->IsInitialized())
        {
          std::vector<cv::Rect> lRects;
          lRects.push_back(objRect);
          fmarker->SetImage(input, lRects);
          found = true;
        }
      }
    }
 
    outVector.push_back(objRect);
  }
 
  return outVector;
}
 
void ArFaceDetector::GoThread()
{
  while (IsRunning())
  {
    if (needUpdate)
    {
      ArManager* manager = ArManager::GetInstance();
      manager->GetLastData(mLastData);
      needUpdate = false;
 
      //small denoising
      if (mLastData.gray.rows != 480 || mLastData.gray.cols != 640)
      {
        cv::Size nsize;
        if (mLastData.gray.rows >= mLastData.gray.cols)
        {
          nsize.width = (int)(((float)mLastData.gray.cols * 480.0f) / (float)mLastData.gray.rows);
          nsize.height = 480;
        }
        else
        {
          nsize.width = 640;
          nsize.height = (int)(((float)mLastData.gray.rows * 640.0f) / (float)mLastData.gray.cols);
        }
        cv::resize(mLastData.gray, mLastData.gray, nsize, 0, 0, cv::INTER_LINEAR);
      }
 
      cv::equalizeHist(mLastData.gray, mLastData.gray);
      // increase image contrast
      //mLastData.gray.convertTo(mLastData.gray, -1, 1.1, 0);
      //cv::blur(mLastData.gray, mLastData.gray, cv::Size(1, 1));
 
      // Detect all markers seen on image
      //read only
      boost::shared_lock< boost::shared_mutex > lockList(manager->listMutex);
 
      // hide all markers
      bool needFaceDetect = false;
        ArManager::MarkerList::iterator imarker;
      std::vector<ArMarker*> faceMarkers;
      for (imarker = manager->markerList.begin(); imarker != manager->markerList.end(); ++imarker)
      {
        if ((*imarker)->GetType() == ArMarker::AR_FACE_MARKER)
        {
          ArFaceMarker* fmarker = static_cast<ArFaceMarker*> (*imarker);
          if (!fmarker->IsInitialized())
            needFaceDetect = true;
 
          faceMarkers.push_back((*imarker));
        }
      }
 
      //list done
      manager->listMutex.unlock();
 
      if (needFaceDetect)
        DetectFaces(mLastData.gray, faceMarkers, mLastData.arCamParam.camParam, mLastData.reversedBitmap);
 
      std::vector<ArMarker*> foundMarkers;
      for (unsigned int j = 0; j < faceMarkers.size(); j++)
      {
        ArFaceMarker* fmarker = static_cast<ArFaceMarker*> (faceMarkers[j]);
        fmarker->Update(mLastData.gray, mLastData.image, mLastData.arCamParam.camParam, mLastData.reversedBitmap);
        if (faceMarkers[j]->IsVisible())
        {
          foundMarkers.push_back(faceMarkers[j]);
        }
      }
    }
    else
    {
      //prevent cpu burn
      boost::this_thread::sleep_for(boost::chrono::milliseconds(1)); //DO not burn too much CPU
    }
  }
}