saliencySpecializedClasses.hpp

/*M///////////////////////////////////////////////////////////////////////////////////////
 //
 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
 //
 //  By downloading, copying, installing or using the software you agree to this license.
 //  If you do not agree to this license, do not download, install,
 //  copy or use the software.
 //
 //
 //                           License Agreement
 //                For Open Source Computer Vision Library
 //
 // Copyright (C) 2014, OpenCV Foundation, all rights reserved.
 // Third party copyrights are property of their respective owners.
 //
 // Redistribution and use in source and binary forms, with or without modification,
 // are permitted provided that the following conditions are met:
 //
 //   * Redistribution's of source code must retain the above copyright notice,
 //     this list of conditions and the following disclaimer.
 //
 //   * Redistribution's in binary form must reproduce the above copyright notice,
 //     this list of conditions and the following disclaimer in the documentation
 //     and/or other materials provided with the distribution.
 //
 //   * The name of the copyright holders may not be used to endorse or promote products
 //     derived from this software without specific prior written permission.
 //
 // This software is provided by the copyright holders and contributors "as is" and
 // any express or implied warranties, including, but not limited to, the implied
 // warranties of merchantability and fitness for a particular purpose are disclaimed.
 // In no event shall the Intel Corporation or contributors be liable for any direct,
 // indirect, incidental, special, exemplary, or consequential damages
 // (including, but not limited to, procurement of substitute goods or services;
 // loss of use, data, or profits; or business interruption) however caused
 // and on any theory of liability, whether in contract, strict liability,
 // or tort (including negligence or otherwise) arising in any way out of
 // the use of this software, even if advised of the possibility of such damage.
 //
 //M*/
 
#ifndef __OPENCV_SALIENCY_SPECIALIZED_CLASSES_HPP__
#define __OPENCV_SALIENCY_SPECIALIZED_CLASSES_HPP__
 
#include <cstdio>
#include <string>
#include <iostream>
#include <stdint.h>
#include "saliencyBaseClasses.hpp"
#include "opencv2/core.hpp"
 
namespace cv
{
namespace saliency
{
 
 
/************************************ Specific Static Saliency Specialized Classes ************************************/
 
class CV_EXPORTS_W StaticSaliencySpectralResidual : public StaticSaliency
{
public:
 
  StaticSaliencySpectralResidual();
  virtual ~StaticSaliencySpectralResidual();
 
  CV_WRAP static Ptr<StaticSaliencySpectralResidual> create()
  {
    return makePtr<StaticSaliencySpectralResidual>();
  }
 
  CV_WRAP bool computeSaliency( InputArray image, OutputArray saliencyMap )
  {
    if( image.empty() )
      return false;
 
    return computeSaliencyImpl( image, saliencyMap );
  }
 
  CV_WRAP void read( const FileNode& fn ) CV_OVERRIDE;
  void write( FileStorage& fs ) const CV_OVERRIDE;
 
  CV_WRAP int getImageWidth() const
  {
    return resImWidth;
  }
  CV_WRAP inline void setImageWidth(int val)
  {
    resImWidth = val;
  }
  CV_WRAP int getImageHeight() const
  {
    return resImHeight;
  }
  CV_WRAP void setImageHeight(int val)
  {
    resImHeight = val;
  }
 
protected:
  bool computeSaliencyImpl( InputArray image, OutputArray saliencyMap ) CV_OVERRIDE;
  CV_PROP_RW int resImWidth;
  CV_PROP_RW int resImHeight;
 
};
 
 
class CV_EXPORTS_W StaticSaliencyFineGrained : public StaticSaliency
{
public:
 
  StaticSaliencyFineGrained();
 
  CV_WRAP static Ptr<StaticSaliencyFineGrained> create()
  {
    return makePtr<StaticSaliencyFineGrained>();
  }
 
  CV_WRAP bool computeSaliency( InputArray image, OutputArray saliencyMap )
  {
    if( image.empty() )
      return false;
 
    return computeSaliencyImpl( image, saliencyMap );
  }
  virtual ~StaticSaliencyFineGrained();
 
protected:
  bool computeSaliencyImpl( InputArray image, OutputArray saliencyMap ) CV_OVERRIDE;
 
private:
  void calcIntensityChannel(Mat src, Mat dst);
  void copyImage(Mat src, Mat dst);
  void getIntensityScaled(Mat integralImage, Mat gray, Mat saliencyOn, Mat saliencyOff, int neighborhood);
  float getMean(Mat srcArg, Point2i PixArg, int neighbourhood, int centerVal);
  void mixScales(Mat *saliencyOn, Mat intensityOn, Mat *saliencyOff, Mat intensityOff, const int numScales);
  void mixOnOff(Mat intensityOn, Mat intensityOff, Mat intensity);
  void getIntensity(Mat srcArg, Mat dstArg,  Mat dstOnArg,  Mat dstOffArg, bool generateOnOff);
};
 
 
 
 
/************************************ Specific Motion Saliency Specialized Classes ************************************/
 
class CV_EXPORTS_W MotionSaliencyBinWangApr2014 : public MotionSaliency
{
public:
  MotionSaliencyBinWangApr2014();
  virtual ~MotionSaliencyBinWangApr2014();
 
  CV_WRAP static Ptr<MotionSaliencyBinWangApr2014> create()
  {
    return makePtr<MotionSaliencyBinWangApr2014>();
  }
 
  CV_WRAP bool computeSaliency( InputArray image, OutputArray saliencyMap )
  {
    if( image.empty() )
      return false;
 
    return computeSaliencyImpl( image, saliencyMap );
  }
 
  CV_WRAP void setImagesize( int W, int H );
  CV_WRAP bool init();
 
  CV_WRAP int getImageWidth() const
  {
    return imageWidth;
  }
  CV_WRAP inline void setImageWidth(int val)
  {
    imageWidth = val;
  }
  CV_WRAP int getImageHeight() const
  {
    return imageHeight;
  }
  CV_WRAP void setImageHeight(int val)
  {
    imageHeight = val;
  }
 
protected:
  bool computeSaliencyImpl( InputArray image, OutputArray saliencyMap ) CV_OVERRIDE;
 
private:
 
  // classification (and adaptation) functions
  bool fullResolutionDetection( const Mat& image, Mat& highResBFMask );
  bool lowResolutionDetection( const Mat& image, Mat& lowResBFMask );
 
  // Background model maintenance functions
  bool templateOrdering();
  bool templateReplacement( const Mat& finalBFMask, const Mat& image );
 
  // Decision threshold adaptation and Activity control function
  bool activityControl(const Mat& current_noisePixelsMask);
  bool decisionThresholdAdaptation();
 
  // changing structure
  std::vector<Ptr<Mat> > backgroundModel;// The vector represents the background template T0---TK of reference paper.
  // Matrices are two-channel matrix. In the first layer there are the B (background value)
  // for each pixel. In the second layer, there are the C (efficacy) value for each pixel
  Mat potentialBackground;// Two channel Matrix. For each pixel, in the first level there are the Ba value (potential background value)
                          // and in the secon level there are the Ca value, the counter for each potential value.
  Mat epslonPixelsValue;// epslon threshold
 
  Mat activityPixelsValue;// Activity level of each pixel
 
  //vector<Mat> noisePixelMask; // We define a ‘noise-pixel’ as a pixel that has been classified as a foreground pixel during the full resolution
  Mat noisePixelMask;// We define a ‘noise-pixel’ as a pixel that has been classified as a foreground pixel during the full resolution
  //detection process,however, after the low resolution detection, it has become a
  // background pixel. The matrix is  two-channel matrix. In the first layer there is the mask ( the identified noise-pixels are set to 1 while other pixels are 0)
  // for each pixel. In the second layer, there is the value of activity level A for each pixel.
 
  //fixed parameter
  bool activityControlFlag;
  bool neighborhoodCheck;
  int N_DS;// Number of template to be downsampled and used in lowResolutionDetection function
  CV_PROP_RW int imageWidth;// Width of input image
  CV_PROP_RW int imageHeight;//Height of input image
  int K;// Number of background model template
  int N;// NxN is the size of the block for downsampling in the lowlowResolutionDetection
  float alpha;// Learning rate
  int L0, L1;// Upper-bound values for C0 and C1 (efficacy of the first two template (matrices) of backgroundModel
  int thetaL;// T0, T1 swap threshold
  int thetaA;// Potential background value threshold
  int gamma;// Parameter that controls the time that the newly updated long-term background value will remain in the
            // long-term template, regardless of any subsequent background changes. A relatively large (eg gamma=3) will
            //restrain the generation of ghosts.
 
  uchar Ainc;// Activity Incrementation;
  int Bmax;// Upper-bound value for pixel activity
  int Bth;// Max activity threshold
  int Binc, Bdec;// Threshold for pixel-level decision threshold (epslon) adaptation
  float deltaINC, deltaDEC;// Increment-decrement value for epslon adaptation
  int epslonMIN, epslonMAX;// Range values for epslon threshold
 
};
 
/************************************ Specific Objectness Specialized Classes ************************************/
 
class CV_EXPORTS_W ObjectnessBING : public Objectness
{
public:
 
  ObjectnessBING();
  virtual ~ObjectnessBING();
 
  CV_WRAP static Ptr<ObjectnessBING> create()
  {
    return makePtr<ObjectnessBING>();
  }
 
  CV_WRAP bool computeSaliency( InputArray image, OutputArray saliencyMap )
  {
    if( image.empty() )
      return false;
 
    return computeSaliencyImpl( image, saliencyMap );
  }
 
  CV_WRAP void read();
  CV_WRAP void write() const;
 
  CV_WRAP std::vector<float> getobjectnessValues();
 
  CV_WRAP void setTrainingPath( const String& trainingPath );
 
  CV_WRAP void setBBResDir( const String& resultsDir );
 
  CV_WRAP double getBase() const
  {
    return _base;
  }
  CV_WRAP inline void setBase(double val)
  {
    _base = val;
  }
  CV_WRAP int getNSS() const
  {
    return _NSS;
  }
  CV_WRAP void setNSS(int val)
  {
    _NSS = val;
  }
  CV_WRAP int getW() const
  {
    return _W;
  }
  CV_WRAP void setW(int val)
  {
    _W = val;
  }
 
protected:
  bool computeSaliencyImpl( InputArray image, OutputArray objectnessBoundingBox ) CV_OVERRIDE;
 
private:
 
  class FilterTIG
  {
  public:
    void update( Mat &w );
 
    // For a W by H gradient magnitude map, find a W-7 by H-7 CV_32F matching score map
    Mat matchTemplate( const Mat &mag1u );
 
    float dot( int64_t tig1, int64_t tig2, int64_t tig4, int64_t tig8 );
    void reconstruct( Mat &w );// For illustration purpose
 
  private:
    static const int NUM_COMP = 2;// Number of components
    static const int D = 64;// Dimension of TIG
    int64_t _bTIGs[NUM_COMP];// Binary TIG features
    float _coeffs1[NUM_COMP];// Coefficients of binary TIG features
 
    // For efficiently deals with different bits in CV_8U gradient map
    float _coeffs2[NUM_COMP], _coeffs4[NUM_COMP], _coeffs8[NUM_COMP];
  };
 
  template<typename VT, typename ST>
  struct ValStructVec
  {
    ValStructVec();
    int size() const;
    void clear();
    void reserve( int resSz );
    void pushBack( const VT& val, const ST& structVal );
    const VT& operator ()( int i ) const;
    const ST& operator []( int i ) const;
    VT& operator ()( int i );
    ST& operator []( int i );
 
    void sort( bool descendOrder = true );
    const std::vector<ST> &getSortedStructVal();
    std::vector<std::pair<VT, int> > getvalIdxes();
    void append( const ValStructVec<VT, ST> &newVals, int startV = 0 );
 
    std::vector<ST> structVals;  // struct values
    int sz;// size of the value struct vector
    std::vector<std::pair<VT, int> > valIdxes;// Indexes after sort
    bool smaller()
    {
      return true;
    }
    std::vector<ST> sortedStructVals;
  };
 
  enum
  {
    MAXBGR,
    HSV,
    G
  };
 
  double _base, _logBase;  // base for window size quantization
  int _W;// As described in the paper: #Size, Size(_W, _H) of feature window.
  int _NSS;// Size for non-maximal suppress
  int _maxT, _minT, _numT;// The minimal and maximal dimensions of the template
 
  int _Clr;//
  static const char* _clrName[3];
 
// Names and paths to read model and to store results
  std::string _modelName, _bbResDir, _trainingPath, _resultsDir;
 
  std::vector<int> _svmSzIdxs;// Indexes of active size. It's equal to _svmFilters.size() and _svmReW1f.rows
  Mat _svmFilter;// Filters learned at stage I, each is a _H by _W CV_32F matrix
  FilterTIG _tigF;// TIG filter
  Mat _svmReW1f;// Re-weight parameters learned at stage II.
 
// List of the rectangles' objectness value, in the same order as
// the  vector<Vec4i> objectnessBoundingBox returned by the algorithm (in computeSaliencyImpl function)
  std::vector<float> objectnessValues;
 
private:
// functions
 
  inline static float LoG( float x, float y, float delta )
  {
    float d = - ( x * x + y * y ) / ( 2 * delta * delta );
    return -1.0f / ( (float) ( CV_PI ) * pow( delta, 4 ) ) * ( 1 + d ) * exp( d );
  }  // Laplacian of Gaussian
 
// Read matrix from binary file
  static bool matRead( const std::string& filename, Mat& M );
 
  void setColorSpace( int clr = MAXBGR );
 
// Load trained model.
  int loadTrainedModel();// Return -1, 0, or 1 if partial, none, or all loaded
 
// Get potential bounding boxes, each of which is represented by a Vec4i for (minX, minY, maxX, maxY).
// The trained model should be prepared before calling this function: loadTrainedModel() or trainStageI() + trainStageII().
// Use numDet to control the final number of proposed bounding boxes, and number of per size (scale and aspect ratio)
  void getObjBndBoxes( Mat &img3u, ValStructVec<float, Vec4i> &valBoxes, int numDetPerSize = 120 );
  void getObjBndBoxesForSingleImage( Mat img, ValStructVec<float, Vec4i> &boxes, int numDetPerSize );
 
  bool filtersLoaded()
  {
    int n = (int) _svmSzIdxs.size();
    return n > 0 && _svmReW1f.size() == Size( 2, n ) && _svmFilter.size() == Size( _W, _W );
  }
  void predictBBoxSI( Mat &mag3u, ValStructVec<float, Vec4i> &valBoxes, std::vector<int> &sz, int NUM_WIN_PSZ = 100, bool fast = true );
  void predictBBoxSII( ValStructVec<float, Vec4i> &valBoxes, const std::vector<int> &sz );
 
// Calculate the image gradient: center option as in VLFeat
  void gradientMag( Mat &imgBGR3u, Mat &mag1u );
 
  static void gradientRGB( Mat &bgr3u, Mat &mag1u );
  static void gradientGray( Mat &bgr3u, Mat &mag1u );
  static void gradientHSV( Mat &bgr3u, Mat &mag1u );
  static void gradientXY( Mat &x1i, Mat &y1i, Mat &mag1u );
 
  static inline int bgrMaxDist( const Vec3b &u, const Vec3b &v )
  {
    int b = abs( u[0] - v[0] ), g = abs( u[1] - v[1] ), r = abs( u[2] - v[2] );
    b = max( b, g );
    return max( b, r );
  }
  static inline int vecDist3b( const Vec3b &u, const Vec3b &v )
  {
    return abs( u[0] - v[0] ) + abs( u[1] - v[1] ) + abs( u[2] - v[2] );
  }
 
//Non-maximal suppress
  static void nonMaxSup( Mat &matchCost1f, ValStructVec<float, Point> &matchCost, int NSS = 1, int maxPoint = 50, bool fast = true );
 
};
 
 
}
/* namespace saliency */
} /* namespace cv */
 
#endif