Codebook 背景建模

codebook采用量化技术从时间序列中获得背景模型，能够检测像素剧烈变化、或者有移动物体或者更为复杂的背景模型。codebook为每个像素建立一个codebook，每个codebook含有一个或者多个codeword，codeword 记录背景学习的阈值、对应像素的更新时间以及访问频率等，通过这些信息，可以得知每个像素的变化情况，从而获得视频中的背景模型。

1.opencv实现简单Codebook

CodeBook算法为当前图像的每一个像素建立一个CodeBook(CB)结构,每个CodeBook结构又由多个CodeWord(CW)组成。CB和CW的形式如下：

CB={CW1,CW2,…CWn,t}

CW={lHigh,lLow,max,min,t_last,stale}

其中n为一个CB中所包含的CW的数目，当n太小时，退化为简单背景，当n较大时可以对复杂背景进行建模;t为CB更新的次数。CW是一个6元组，其中IHigh和ILow作为更新时的学习上下界，max和min记录当前像素的最大值和最小值。上次更新的时间t_last和陈旧时间stale(记录该CW多久未被访问)用来删除很少使用的CodeWord。

假设当前训练图像I中某一像素为I(x,y)，该像素的CB的更新算法如下，另外记背景阈值的增长判定阈值为Bounds：

(1) CB的访问次数加1；

(2) 遍历CB中的每个CW，如果存在一个CW中的IHigh，ILow满足ILow≤I(x,y)≤IHigh，则转(4)；

(3) 创建一个新的码字CWnew加入到CB中, CWnew的max与min都赋值为I(x,y)，IHigh <- I(x,y) + Bounds，ILow <- I(x,y) – Bounds，并且转(6)；

(4) 更新该码字的t_last，若当前像素值I(x,y)大于该码字的max，则max <- I(x,y)，若I(x,y)小于该码字的min，则min <- I(x,y)；

(5) 更新该码字的学习上下界，以增加背景模型对于复杂背景的适应能力，具体做法是：若IHigh < I(x,y) + Bounds，则IHigh 增长1，若ILow > I(x,y) – Bounds，则ILow减少1；

(6) 更新CB中每个CW的stale。

使用已建立好的CB进行运动目标检测的方法很简单，记判断前景的范围上下界为minMod和maxMod，对于当前待检测图像上的某一像素 I(x,y)，遍历它对应像素背景模型CB中的每一个码字CW，若存在一个CW，使得I(x,y) < max + maxMod并且I(x,y) > min – minMod，则I(x,y)被判断为背景，否则被判断为前景。

在实际使用CodeBook进行运动检测时，除了要隔一定的时间对CB进行更新的同时，需要对CB进行一个时间滤波，目的是去除很少被访问到的CW，其方法是访问每个CW的stale，若stale大于一个阈值（通常设置为总更新次数的一半），移除该CW。

利用opencv实现：代码

#include <cv.h>
#include <highgui.h>

int CVCONTOUR_APPROX_LEVEL = 2;   
int CVCLOSE_ITR = 1;

#define CV_CVX_WHITE	CV_RGB(0xff,0xff,0xff)
#define CV_CVX_BLACK	CV_RGB(0x00,0x00,0x00)
#define CHANNELS 3
typedef struct ce 
{
	uchar learnHigh[CHANNELS]; //High side threshold for learning
	uchar learnLow[CHANNELS];  //Low side threshold for learning
	uchar max[CHANNELS];       //High side of box boundary
	uchar min[CHANNELS];       //Low side of box boundary
	int t_last_update;         //Allow us to kill stale entries
	int stale;                 //max negative run (longest period of inactivity)
} code_element;


//码书结构
typedef struct code_book 
{
	code_element **cb; //指向码字的指针
	int numEntries;    //码书包含的码字数量
	int t;			   //count every access
} codeBook;

codeBook* TcodeBook;//包括所有像素的码书集合

//////////////////////////////////////////////////////////////
// int update_codebook(uchar *p, codeBook &c, unsigned cbBounds)
// Updates the codebook entry with a new data point
// p Pointer to a YUV or HSI pixel
// c Codebook for this pixel
// cbBounds Learning bounds for codebook (cvBounds must be of length equal to numChannels)
// numChannels Number of color channels we’re learning
// codebook index
int update_codebook(uchar* p,codeBook& c,unsigned* cbBounds,int numChannels)
{
	int i = 0 ;
	unsigned int high[3],low[3];
	int n;
	for(n=0; n< numChannels; n++)
	{
		high[n] = *(p+n)+*(cbBounds+n);
		if(high[n] > 255) 
			high[n] = 255;

		low[n] = *(p+n)-*(cbBounds+n);
		if(low[n] < 0) 
			low[n] = 0;
	}
	int matchChannel;

	// SEE IF THIS FITS AN EXISTING CODEWORD
	for(i=0; i<c.numEntries; i++)
	{
		matchChannel = 0;
		for(n=0; n<numChannels; n++)
		{
			if((c.cb[i]->learnLow[n] <= *(p+n)) &&
				//Found an entry for this channel
				(*(p+n) <= c.cb[i]->learnHigh[n]))
			{
				matchChannel++;
			}
		}
		if(matchChannel == numChannels) //If an entry was found
		{
			c.cb[i]->t_last_update = c.t;
			//adjust this codeword for the first channel
			for(n=0; n<numChannels; n++)
			{
				if(c.cb[i]->max[n] < *(p+n))
				{
					c.cb[i]->max[n] = *(p+n);
				}
				else if(c.cb[i]->min[n] > *(p+n))
				{
					c.cb[i]->min[n] = *(p+n);
				}
			}

			break;
		}
	}
	// OVERHEAD TO TRACK POTENTIAL STALE ENTRIES
	//
	for(int s=0; s<c.numEntries; s++)
	{
		// Track which codebook entries are going stale:
		int negRun = c.t - c.cb[s]->t_last_update;
		if(c.cb[s]->stale < negRun) 
			c.cb[s]->stale = negRun;
	}

	// ENTER A NEW CODEWORD IF NEEDED
	if(i == c.numEntries) //if no existing codeword found, make one
	{
		code_element **foo = new code_element* [c.numEntries+1];
		for(int ii=0; ii<c.numEntries; ii++)
		{
			foo[ii] = c.cb[ii];
		}
		foo[c.numEntries] = new code_element;
		if(c.numEntries) delete [] c.cb;
		c.cb = foo;
		for(n=0; n<numChannels; n++) 
		{
			c.cb[c.numEntries]->learnHigh[n] = high[n];
			c.cb[c.numEntries]->learnLow[n] = low[n];
			c.cb[c.numEntries]->max[n] = *(p+n);
			c.cb[c.numEntries]->min[n] = *(p+n);
		}
		c.cb[c.numEntries]->t_last_update = c.t;
		c.cb[c.numEntries]->stale = 0;
		c.numEntries += 1;
	}

	// SLOWLY ADJUST LEARNING BOUNDS
	for(n=0; n<numChannels; n++)
	{
		if(c.cb[i]->learnHigh[n] < high[n]) 
			c.cb[i]->learnHigh[n] += 1;
		if(c.cb[i]->learnLow[n] > low[n]) 
			c.cb[i]->learnLow[n] -= 1;
	}
	return(i);
}

///////////////////////////////////////////////////////////////////
//int clear_stale_entries(codeBook &c)
// During learning, after you’ve learned for some period of time,
// periodically call this to clear out stale codebook entries
//
// c Codebook to clean up
//
// Return
// number of entries cleared
//
int clear_stale_entries(codeBook &c)
{
	int staleThresh = c.t>>1;
	int *keep = new int [c.numEntries];
	int keepCnt = 0;

	// SEE WHICH CODEBOOK ENTRIES ARE TOO STALE
	for(int i=0; i<c.numEntries; i++)
	{
		if(c.cb[i]->stale > staleThresh)
			keep[i] = 0; //Mark for destruction
		else
		{
			keep[i] = 1; //Mark to keep
			keepCnt += 1;
		}
	}
	// KEEP ONLY THE GOOD
	//
	c.t = 0; //Full reset on stale tracking
	code_element **foo = new code_element* [keepCnt];
	int k=0;
	for(int ii=0; ii<c.numEntries; ii++)
	{
		if(keep[ii])
		{
			foo[k] = c.cb[ii];
			//We have to refresh these entries for next clearStale
			foo[k]->t_last_update = 0;
			k++;
		}
	}
	// CLEAN UP
	//
	delete [] keep;
	delete [] c.cb;
	c.cb = foo;
	int numCleared = c.numEntries - keepCnt;
	c.numEntries = keepCnt;
	return(numCleared);
}

////////////////////////////////////////////////////////////
// uchar background_diff( uchar *p, codeBook &c,
// int minMod, int maxMod)
// Given a pixel and a codebook, determine if the pixel is
// covered by the codebook
//
// p Pixel pointer (YUV interleaved)
// c Codebook reference
// numChannels Number of channels we are testing
// maxMod Add this (possibly negative) number onto

// max level when determining if new pixel is foreground
// minMod Subract this (possibly negative) number from
// min level when determining if new pixel is foreground
//
// NOTES:
// minMod and maxMod must have length numChannels,
// e.g. 3 channels => minMod[3], maxMod[3]. There is one min and
// one max threshold per channel.
//
// Return
// 0 => background, 255 => foreground
//
uchar background_diff(
					  uchar* p,
					  codeBook& c,
					  int numChannels,
					  int* minMod,
					  int* maxMod
					  )
{
	int i = 0 ;
	int matchChannel;
	// SEE IF THIS FITS AN EXISTING CODEWORD
	//
	for(i=0; i<c.numEntries; i++)
	{
		matchChannel = 0;
		for(int n=0; n<numChannels; n++) 
		{
			if((c.cb[i]->min[n] - minMod[n] <= *(p+n)) && (*(p+n) <= c.cb[i]->max[n] + maxMod[n])) 
			{
					matchChannel++; //Found an entry for this channel
			} 
			else 
			{
				break;
			}
		}
		if(matchChannel == numChannels) 
		{
			break; //Found an entry that matched all channels
		}
	}
	if(i >= c.numEntries) 
		return(255);
	else
		return(0);
}
void connected_Components(IplImage *mask, int poly1_hull0, float perimScale, int *num, CvRect *bbs, CvPoint *centers)
{
	static CvMemStorage*	mem_storage	= NULL;
	static CvSeq*			contours	= NULL;

	//CLEAN UP RAW MASK
	cvMorphologyEx( mask, mask, NULL, NULL, CV_MOP_OPEN ,1);
	cvMorphologyEx( mask, mask, NULL, NULL, CV_MOP_CLOSE,2);

	//FIND CONTOURS AROUND ONLY BIGGER REGIONS
	if( mem_storage==NULL ) mem_storage = cvCreateMemStorage(0);
	else cvClearMemStorage(mem_storage);

	CvContourScanner scanner = cvStartFindContours(mask,mem_storage,sizeof(CvContour),CV_RETR_EXTERNAL,CV_CHAIN_APPROX_SIMPLE);
	CvSeq* c;
	int numCont = 0;
	while( (c = cvFindNextContour( scanner )) != NULL )
	{
		double len = cvContourPerimeter( c );
		double q = (mask->height + mask->width) /perimScale;   //calculate perimeter len threshold
		if( len < q ) //Get rid of blob if it's perimeter is too small
		{
			cvSubstituteContour( scanner, NULL );
		}
		else //Smooth it's edges if it's large enough
		{
			CvSeq* c_new;
			if(poly1_hull0) //Polygonal approximation of the segmentation
				c_new = cvApproxPoly(c,sizeof(CvContour),mem_storage,CV_POLY_APPROX_DP, CVCONTOUR_APPROX_LEVEL,0);
			else //Convex Hull of the segmentation
				c_new = cvConvexHull2(c,mem_storage,CV_CLOCKWISE,1);
			cvSubstituteContour( scanner, c_new );
			numCont++;
		}
	}
	contours = cvEndFindContours( &scanner );

	// PAINT THE FOUND REGIONS BACK INTO THE IMAGE
	cvZero( mask );
	IplImage *maskTemp;
	//CALC CENTER OF MASS AND OR BOUNDING RECTANGLES
	if(num != NULL)
	{
		int N = *num, numFilled = 0, i=0;
		CvMoments moments;
		double M00, M01, M10;
		maskTemp = cvCloneImage(mask);
		for(i=0, c=contours; c != NULL; c = c->h_next,i++ )
		{
			if(i < N) //Only process up to *num of them
			{
				cvDrawContours(maskTemp,c,CV_CVX_WHITE, CV_CVX_WHITE,-1,CV_FILLED,8);
				//Find the center of each contour
				if(centers != NULL)
				{
					cvMoments(maskTemp,&moments,1);
					M00 = cvGetSpatialMoment(&moments,0,0);
					M10 = cvGetSpatialMoment(&moments,1,0);
					M01 = cvGetSpatialMoment(&moments,0,1);
					centers[i].x = (int)(M10/M00);
					centers[i].y = (int)(M01/M00);
				}
				//Bounding rectangles around blobs
				if(bbs != NULL)
				{
					bbs[i] = cvBoundingRect(c);
				}
				cvZero(maskTemp);
				numFilled++;
			}
			//Draw filled contours into mask
			cvDrawContours(mask,c,CV_CVX_WHITE,CV_CVX_WHITE,-1,CV_FILLED,8); //draw to central mask
		} //end looping over contours
		*num = numFilled;
		cvReleaseImage( &maskTemp);
	}
	else
	{
		for( c=contours; c != NULL; c = c->h_next )
		{
			cvDrawContours(mask,c,CV_CVX_WHITE, CV_CVX_BLACK,-1,CV_FILLED,8);
		}
	}
}
IplImage* pFrame = NULL;
IplImage* pFrameHSV = NULL;
IplImage* pFrImg = NULL;
CvCapture* pCapture = NULL;
int nFrmNum = 0;
//IplImage* pFrImg = NULL;
//IplImage* pBkImg = NULL;

unsigned cbBounds = 5;

int height,width;
int nchannels;
int minMod[3]={30,30,30}, maxMod[3]={30,30,30};

int main(int argc, char* argv[])
{
	//创建窗口
	cvNamedWindow("video", 1);
	cvNamedWindow("HSV空间图像",1);
	cvNamedWindow("foreground",1);
	//使窗口有序排列
	cvMoveWindow("video", 30, 0);
	cvMoveWindow("HSV空间图像", 360, 0);
	cvMoveWindow("foreground", 690, 0);
	//打开视频文件，
	if( !(pCapture = cvCaptureFromFile("tingche.avi")))
	{
		fprintf(stderr, "Can not open video file %s\n");
		return -2;
	}

	int j;
	//逐帧读取视频
	while(pFrame = cvQueryFrame( pCapture ))
	{
		nFrmNum++;
		cvShowImage("video", pFrame);

		if (nFrmNum == 1)
		{
			height =  pFrame->height;
			width = pFrame->width;
			nchannels = pFrame->nChannels;
			pFrameHSV = cvCreateImage(cvSize(pFrame->width, pFrame->height), IPL_DEPTH_8U,3);
			pFrImg = cvCreateImage(cvSize(pFrame->width, pFrame->height), IPL_DEPTH_8U,1);
			//cvCvtColor(pFrame, pFrameHSV, CV_BGR2HSV);//色彩空间转化
			TcodeBook = new codeBook[width*height+1];

			for(j = 0; j < width*height; j++)
			{
				TcodeBook[j].numEntries = 0;
				TcodeBook[j].t = 0;
			}


		}

		if (nFrmNum<=30)
		{
			//cvCvtColor(pFrame, pFrameHSV, CV_BGR2HSV);//色彩空间转化
			cvCopyImage(pFrame,pFrameHSV);
			//学习背景
			for(j = 0; j < width*height; j++)
				update_codebook((uchar*)pFrameHSV->imageData+j*nchannels, TcodeBook[j],&cbBounds,3);


		}
		else
		{
			//cvCvtColor(pFrame, pFrameHSV, CV_BGR2HSV);//色彩空间转化
			cvCopyImage(pFrame,pFrameHSV);

			if(nFrmNum%20 == 0)
			{
				for(j = 0; j < width*height; j++)
					update_codebook((uchar*)pFrameHSV->imageData+j*nchannels, TcodeBook[j],&cbBounds,3);
			}
			if(nFrmNum%40 == 0)
			{
				for(j = 0; j < width*height; j++)
					clear_stale_entries(TcodeBook[j]);
			}
			for(j = 0; j < width*height; j++)
			{
				if(background_diff((uchar*)pFrameHSV->imageData+j*nchannels, TcodeBook[j],3,minMod,maxMod))
				{
					pFrImg->imageData[j] = 255;
				}
				else
				{
					pFrImg->imageData[j] = 0;
				}
			}

			//connected_Components(pFrImg,1,20,NULL,NULL, NULL);

			cvShowImage("foreground", pFrImg);
			cvShowImage("HSV空间图像", pFrameHSV);
		}
		if( cvWaitKey(2) >= 0 )
			break;   

	} // end of while-loop

	for(j = 0; j < width*height; j++)
	{
		if (!TcodeBook[j].cb)
			delete [] TcodeBook[j].cb;
	}
	if (!TcodeBook)
		delete [] TcodeBook;
	//销毁窗口
	cvDestroyWindow("video");
	cvDestroyWindow("HSV空间图像");
	cvDestroyWindow("foreground");
	return 0;
}

该算法测试的时候，对背景的变换有一定的适应性，但是获取的前景目标空洞比较多，需要后面的区域处理上做一些功夫。opencv上面的例子，只是使用图像的亮度作为参数来更新背景模型，对阴影、环境光线的变化等等 处理的并不好。

车进如车库

进入车库后，被更新为背景

又有新的车进来可以看到 前景很多空隙

如果对codebook的参数做一些调整效果会更好，可以参考论文：

http://115.com/file/anpwtz5l#Real-time foreground-background segmentation using a modified codebook model.pdf

http://115.com/file/anpwtz5l#Real-time foreground-background segmentation using a modified codebook model.pdf