saliencySpecializedClasses.hpp

/*M///////////////////////////////////////////////////////////////////////////////////////





//





//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.





//





//  By downloading, copying, installing or using the software you agree to this license.





//  If you do not agree to this license, do not download, install,





//  copy or use the software.





//





//





//                           License Agreement





//                For Open Source Computer Vision Library





//





// Copyright (C) 2014, OpenCV Foundation, all rights reserved.





// Third party copyrights are property of their respective owners.





//





// Redistribution and use in source and binary forms, with or without modification,





// are permitted provided that the following conditions are met:





//





//   * Redistribution's of source code must retain the above copyright notice,





//     this list of conditions and the following disclaimer.





//





//   * Redistribution's in binary form must reproduce the above copyright notice,





//     this list of conditions and the following disclaimer in the documentation





//     and/or other materials provided with the distribution.





//





//   * The name of the copyright holders may not be used to endorse or promote products





//     derived from this software without specific prior written permission.





//





// This software is provided by the copyright holders and contributors "as is" and





// any express or implied warranties, including, but not limited to, the implied





// warranties of merchantability and fitness for a particular purpose are disclaimed.





// In no event shall the Intel Corporation or contributors be liable for any direct,





// indirect, incidental, special, exemplary, or consequential damages





// (including, but not limited to, procurement of substitute goods or services;





// loss of use, data, or profits; or business interruption) however caused





// and on any theory of liability, whether in contract, strict liability,





// or tort (including negligence or otherwise) arising in any way out of





// the use of this software, even if advised of the possibility of such damage.





//





//M*/








#ifndef __OPENCV_SALIENCY_SPECIALIZED_CLASSES_HPP__




#define __OPENCV_SALIENCY_SPECIALIZED_CLASSES_HPP__








#include <cstdio>




#include <string>




#include <iostream>




#include <stdint.h>




#include "saliencyBaseClasses.hpp"




#include "opencv2/core.hpp"








namespace

cv



{



namespace
saliency


{











/************************************ Specific Static Saliency Specialized Classes ************************************/








class
CV_EXPORTS_W
StaticSaliencySpectralResidual
:
public
StaticSaliency



{



public:







StaticSaliencySpectralResidual();



virtual
~StaticSaliencySpectralResidual();







CV_WRAP
static
Ptr<StaticSaliencySpectralResidual>
create()



{



return
makePtr<StaticSaliencySpectralResidual>();



}







CV_WRAP
bool
computeSaliency( InputArray image,
OutputArray
saliencyMap )



{



if( image.empty() )



return
false;







return
computeSaliencyImpl( image, saliencyMap );



}







CV_WRAP
void
read(
const
FileNode& fn ) CV_OVERRIDE;



void
write(
FileStorage& fs )
const
CV_OVERRIDE;







CV_WRAP
int
getImageWidth()
const





{



return
resImWidth;



}



CV_WRAP
inline
void
setImageWidth(int
val)



{



resImWidth = val;



}



CV_WRAP
int
getImageHeight()
const





{



return
resImHeight;



}



CV_WRAP
void
setImageHeight(int
val)



{



resImHeight = val;



}







protected:



bool
computeSaliencyImpl( InputArray image, OutputArray saliencyMap ) CV_OVERRIDE;



CV_PROP_RW
int
resImWidth;



CV_PROP_RW
int
resImHeight;






};











class
CV_EXPORTS_W
StaticSaliencyFineGrained
:
public
StaticSaliency



{



public:







StaticSaliencyFineGrained();







CV_WRAP
static
Ptr<StaticSaliencyFineGrained>
create()



{



return
makePtr<StaticSaliencyFineGrained>();



}







CV_WRAP
bool
computeSaliency( InputArray image,
OutputArray
saliencyMap )



{



if( image.empty() )



return
false;







return
computeSaliencyImpl( image, saliencyMap );



}



virtual
~StaticSaliencyFineGrained();







protected:



bool
computeSaliencyImpl( InputArray image,
OutputArray
saliencyMap ) CV_OVERRIDE;







private:



void
calcIntensityChannel(Mat
src,
Mat
dst);



void
copyImage(Mat
src,
Mat
dst);



void
getIntensityScaled(Mat
integralImage,
Mat
gray,
Mat
saliencyOn,
Mat
saliencyOff,
int
neighborhood);



float
getMean(Mat
srcArg,
Point2i
PixArg,
int
neighbourhood,
int
centerVal);



void
mixScales(Mat
*saliencyOn,
Mat
intensityOn,
Mat
*saliencyOff,
Mat
intensityOff,
const
int
numScales);



void
mixOnOff(Mat
intensityOn,
Mat
intensityOff,
Mat
intensity);



void
getIntensity(Mat
srcArg,
Mat
dstArg,
Mat
dstOnArg,
Mat
dstOffArg,
bool
generateOnOff);


};



















/************************************ Specific Motion Saliency Specialized Classes ************************************/








class
CV_EXPORTS_W
MotionSaliencyBinWangApr2014
:
public
MotionSaliency



{



public:



MotionSaliencyBinWangApr2014();



virtual
~MotionSaliencyBinWangApr2014();







CV_WRAP
static
Ptr<MotionSaliencyBinWangApr2014>
create()



{



return
makePtr<MotionSaliencyBinWangApr2014>();



}







CV_WRAP
bool
computeSaliency( InputArray image,
OutputArray
saliencyMap )



{



if( image.empty() )



return
false;







return
computeSaliencyImpl( image, saliencyMap );



}







CV_WRAP
void
setImagesize(
int
W,
int
H );



CV_WRAP
bool
init();







CV_WRAP
int
getImageWidth()
const





{



return
imageWidth;



}



CV_WRAP
inline
void
setImageWidth(int
val)



{



imageWidth = val;



}



CV_WRAP
int
getImageHeight()
const





{



return
imageHeight;



}



CV_WRAP
void
setImageHeight(int
val)



{



imageHeight = val;



}







protected:



bool
computeSaliencyImpl( InputArray image,
OutputArray
saliencyMap ) CV_OVERRIDE;







private:







// classification (and adaptation) functions




bool
fullResolutionDetection(
const
Mat& image,
Mat& highResBFMask );



bool
lowResolutionDetection(
const
Mat& image,
Mat& lowResBFMask );







// Background model maintenance functions




bool
templateOrdering();



bool
templateReplacement(
const
Mat& finalBFMask,
const
Mat& image );







// Decision threshold adaptation and Activity control function




bool
activityControl(const
Mat& current_noisePixelsMask);



bool
decisionThresholdAdaptation();







// changing structure




std::vector<Ptr<Mat> > backgroundModel;// The vector represents the background template T0---TK of reference paper.




// Matrices are two-channel matrix. In the first layer there are the B (background value)




// for each pixel. In the second layer, there are the C (efficacy) value for each pixel




Mat
potentialBackground;// Two channel Matrix. For each pixel, in the first level there are the Ba value (potential background value)




// and in the secon level there are the Ca value, the counter for each potential value.




Mat
epslonPixelsValue;// epslon threshold








Mat
activityPixelsValue;// Activity level of each pixel








//vector<Mat> noisePixelMask; // We define a ‘noise-pixel’ as a pixel that has been classified as a foreground pixel during the full resolution




Mat
noisePixelMask;// We define a ‘noise-pixel’ as a pixel that has been classified as a foreground pixel during the full resolution




//detection process,however, after the low resolution detection, it has become a




// background pixel. The matrix is  two-channel matrix. In the first layer there is the mask ( the identified noise-pixels are set to 1 while other pixels are 0)




// for each pixel. In the second layer, there is the value of activity level A for each pixel.








//fixed parameter




bool
activityControlFlag;



bool
neighborhoodCheck;



int
N_DS;// Number of template to be downsampled and used in lowResolutionDetection function




CV_PROP_RW
int
imageWidth;// Width of input image




CV_PROP_RW
int
imageHeight;//Height of input image




int
K;// Number of background model template




int
N;// NxN is the size of the block for downsampling in the lowlowResolutionDetection




float
alpha;// Learning rate




int
L0,
L1;// Upper-bound values for C0 and C1 (efficacy of the first two template (matrices) of backgroundModel




int
thetaL;// T0, T1 swap threshold




int
thetaA;// Potential background value threshold




int
gamma;// Parameter that controls the time that the newly updated long-term background value will remain in the




// long-term template, regardless of any subsequent background changes. A relatively large (eg gamma=3) will




//restrain the generation of ghosts.








uchar Ainc;// Activity Incrementation;




int
Bmax;// Upper-bound value for pixel activity




int
Bth;// Max activity threshold




int
Binc, Bdec;// Threshold for pixel-level decision threshold (epslon) adaptation




float
deltaINC, deltaDEC;// Increment-decrement value for epslon adaptation




int
epslonMIN, epslonMAX;// Range values for epslon threshold







};







/************************************ Specific Objectness Specialized Classes ************************************/








class
CV_EXPORTS_W
ObjectnessBING
:
public
Objectness



{



public:







ObjectnessBING();



virtual
~ObjectnessBING();







CV_WRAP
static
Ptr<ObjectnessBING>
create()



{



return
makePtr<ObjectnessBING>();



}







CV_WRAP
bool
computeSaliency( InputArray image,
OutputArray
saliencyMap )



{



if( image.empty() )



return
false;







return
computeSaliencyImpl( image, saliencyMap );



}







CV_WRAP
void
read();



CV_WRAP
void
write()
const;







CV_WRAP std::vector<float> getobjectnessValues();







CV_WRAP
void
setTrainingPath(
const
String& trainingPath );







CV_WRAP
void
setBBResDir(
const
String& resultsDir );







CV_WRAP
double
getBase()
const





{



return
_base;



}



CV_WRAP
inline
void
setBase(double
val)



{



_base = val;



}



CV_WRAP
int
getNSS()
const





{



return
_NSS;



}



CV_WRAP
void
setNSS(int
val)



{



_NSS = val;



}



CV_WRAP
int
getW()
const





{



return
_W;



}



CV_WRAP
void
setW(int
val)



{



_W = val;



}







protected:



bool
computeSaliencyImpl( InputArray image,
OutputArray
objectnessBoundingBox ) CV_OVERRIDE;







private:







class
FilterTIG



{



public:



void
update(
Mat
&w );







// For a W by H gradient magnitude map, find a W-7 by H-7 CV_32F matching score map




Mat
matchTemplate(
const
Mat
&mag1u );







float
dot( int64_t tig1, int64_t tig2, int64_t tig4, int64_t tig8 );



void
reconstruct(
Mat
&w );// For illustration purpose








private:



static
const
int
NUM_COMP = 2;// Number of components




static
const
int
D = 64;// Dimension of TIG




int64_t _bTIGs[NUM_COMP];// Binary TIG features




float
_coeffs1[NUM_COMP];// Coefficients of binary TIG features








// For efficiently deals with different bits in CV_8U gradient map




float
_coeffs2[NUM_COMP], _coeffs4[NUM_COMP], _coeffs8[NUM_COMP];



};







template<typename
VT,
typename
ST>



struct
ValStructVec



{



ValStructVec();



int
size()
const;



void
clear();



void
reserve(
int
resSz );



void
pushBack(
const
VT& val,
const
ST& structVal );



const
VT& operator ()(
int
i )
const;



const
ST& operator [](
int
i )
const;



VT& operator ()(
int
i );



ST& operator [](
int
i );







void
sort(
bool
descendOrder =
true
);



const
std::vector<ST> &getSortedStructVal();



std::vector<std::pair<VT, int> > getvalIdxes();



void
append(
const
ValStructVec<VT, ST> &newVals,
int
startV = 0 );







std::vector<ST> structVals;
// struct values




int
sz;// size of the value struct vector




std::vector<std::pair<VT, int> > valIdxes;// Indexes after sort




bool
smaller()



{



return
true;



}



std::vector<ST> sortedStructVals;



};







enum




{



MAXBGR,



HSV,



G



};







double
_base, _logBase;
// base for window size quantization




int
_W;// As described in the paper: #Size, Size(_W, _H) of feature window.




int
_NSS;// Size for non-maximal suppress




int
_maxT, _minT, _numT;// The minimal and maximal dimensions of the template








int
_Clr;//




static
const
char* _clrName[3];







// Names and paths to read model and to store results




std::string _modelName, _bbResDir, _trainingPath, _resultsDir;







std::vector<int> _svmSzIdxs;// Indexes of active size. It's equal to _svmFilters.size() and _svmReW1f.rows




Mat _svmFilter;// Filters learned at stage I, each is a _H by _W CV_32F matrix




FilterTIG _tigF;// TIG filter




Mat _svmReW1f;// Re-weight parameters learned at stage II.








// List of the rectangles' objectness value, in the same order as




// the  vector<Vec4i> objectnessBoundingBox returned by the algorithm (in computeSaliencyImpl function)




std::vector<float> objectnessValues;







private:



// functions








inline
static
float
LoG(
float
x,
float
y,
float
delta )



{



float
d = - ( x * x + y * y ) / ( 2 * delta * delta );



return
-1.0f / ( (float) ( CV_PI ) *
pow( delta, 4 ) ) * ( 1 + d ) *
exp( d );



}
// Laplacian of Gaussian








// Read matrix from binary file




static
bool
matRead(
const
std::string& filename, Mat& M );







void
setColorSpace(
int
clr = MAXBGR );







// Load trained model.




int
loadTrainedModel();// Return -1, 0, or 1 if partial, none, or all loaded








// Get potential bounding boxes, each of which is represented by a Vec4i for (minX, minY, maxX, maxY).




// The trained model should be prepared before calling this function: loadTrainedModel() or trainStageI() + trainStageII().




// Use numDet to control the final number of proposed bounding boxes, and number of per size (scale and aspect ratio)




void
getObjBndBoxes( Mat &img3u, ValStructVec<float, Vec4i> &valBoxes,
int
numDetPerSize = 120 );



void
getObjBndBoxesForSingleImage( Mat img, ValStructVec<float, Vec4i> &boxes,
int
numDetPerSize );







bool
filtersLoaded()



{



int
n = (int) _svmSzIdxs.size();



return
n > 0 && _svmReW1f.size() == Size( 2, n ) && _svmFilter.size() == Size( _W, _W );



}



void
predictBBoxSI( Mat &mag3u, ValStructVec<float, Vec4i> &valBoxes, std::vector<int> &sz,
int
NUM_WIN_PSZ = 100,
bool
fast =
true
);



void
predictBBoxSII( ValStructVec<float, Vec4i> &valBoxes,
const
std::vector<int> &sz );







// Calculate the image gradient: center option as in VLFeat




void
gradientMag( Mat &imgBGR3u, Mat &mag1u );







static
void
gradientRGB( Mat &bgr3u, Mat &mag1u );



static
void
gradientGray( Mat &bgr3u, Mat &mag1u );



static
void
gradientHSV( Mat &bgr3u, Mat &mag1u );



static
void
gradientXY( Mat &x1i, Mat &y1i, Mat &mag1u );







static
inline
int
bgrMaxDist(
const
Vec3b &u,
const
Vec3b &v )



{



int
b =
abs( u[0] - v[0] ), g =
abs( u[1] - v[1] ), r =
abs( u[2] - v[2] );



b =
max( b, g );



return
max( b, r );



}



static
inline
int
vecDist3b(
const
Vec3b &u,
const
Vec3b &v )



{



return
abs( u[0] - v[0] ) +
abs( u[1] - v[1] ) +
abs( u[2] - v[2] );



}







//Non-maximal suppress




static
void
nonMaxSup( Mat &matchCost1f, ValStructVec<float, Point> &matchCost,
int
NSS = 1,
int
maxPoint = 50,
bool
fast =
true
);






};










}



/* namespace saliency */



}
/* namespace cv */








#endif