微信【跳一跳】 opencv视觉识别 + 物理外挂
视频连接：http://v.youku.com/v_show/id_XMzMyNDQxNTA0OA==.html?spm=a2h3j.8428770.3416059.1
初入门C++ 与 opencv视觉库,写了一个跳一跳的物理挂，现在识别率还比较差，先记录下过程，以后在慢慢修改整理。
一、外挂结构
上位机：USB摄像头连接windows电脑，用作处理识别拍摄到图像数据。
下位机：STM32单片机，用于控制陀机附带电容笔进行物理点击。
单片机部分很简单，所以下文主要记录上位机的内容。
二、上位机程序框架
开发平台：Visual Studio 2012
思路：
- 读取摄像头数据；
- 提取图像中的手机屏幕、进行屏幕矫正；
- 识别人物和方块并计算它们的距离（现方案识别部分采用模板匹配、模板使用photoshop预先裁剪好，加载入程序中）；
- 把计算结果通过串口交给下位机，由下位机带动陀机进行物理点击；
三、程序源码
ScreenExtraction.cpp用于识别屏幕边缘、提取内容和矫正
#include "ScreenExtraction.h" #include "main.h" ScreenExtract::ScreenExtract() { } ScreenExtract::ScreenExtract(Mat srcMat) { setSrc(srcMat); } ScreenExtract::~ScreenExtract() { } Mat ScreenExtract::getDst() { return m_MatDstImageStand; } bool ScreenExtract::setSrc(Mat srcMat) { m_MatSrcImage = srcMat.clone(); m_MatEdgeImage = srcMat.clone(); m_MatHoughImage = srcMat.clone(); m_MatCornerImage = srcMat.clone(); Mat temp(1280,720,srcMat.type()); m_MatDstImageLie = temp.clone(); Mat temp2(720,1280,srcMat.type()); m_MatDstImageStand = temp2.clone(); return true; } Mat ScreenExtract::runExtract() { int cannyThrel=100; int failCNT=0; while(1) { m_MatEdgeImage = EdgeDection(m_MatEdgeImage,cannyThrel); m_MatHoughImage = HoughLine(m_MatEdgeImage); m_MatCornerImage = CornerHarris(m_MatHoughImage,m_PointPerspectiveSrcBuff); if(failCNT>20) { throw("Extract fail"); return m_MatSrcImage; } if(m_PointPerspectiveSrcBuff.size() == 8) { break; } else if(m_PointPerspectiveSrcBuff.size()>8) { if(cannyThrel<500) { cannyThrel+=20; } failCNT++; } else if(m_PointPerspectiveSrcBuff.size()<8) { if(cannyThrel>10) { cannyThrel-=5; } failCNT++; } waitKey(10); } m_MatDstImageLie = perspectiveChange(m_MatSrcImage,m_PointPerspectiveSrcBuff); m_MatDstImageStand = rotation(m_MatDstImageLie,90); return m_MatDstImageStand; } Mat ScreenExtract::runFastExtract(Mat srcMat) { m_MatSrcImage = srcMat.clone(); m_MatDstImageLie = perspectiveChange(m_MatSrcImage,m_PointPerspectiveSrcBuff); m_MatDstImageStand = rotation(m_MatDstImageLie,90); return m_MatDstImageStand; } //private Mat ScreenExtract::EdgeDection(Mat srcImage,int cannyThrel) { Mat cannyEdge; Mat dstImage = m_MatSrcImage.clone(); //降噪 blur(m_MatSrcImage,cannyEdge,Size(3,3)); #ifdef DEBUG_SHOW_ScreenExtract //namedWindow("blur()",CV_WINDOW_AUTOSIZE); //imshow(Windows_Edge,g_dstImage); imshow("blur()",cannyEdge); #endif //运行Canny算子 Canny(cannyEdge,cannyEdge,cannyThrel,cannyThrel*3,3); //先将g_dstImage内的所有元素设置为0 dstImage = Scalar::all(0); //使用Canny算子输出的边缘图g_cannyDetectedEdges作为掩码，来将原图g_srcImage拷贝到目标图g_dstImage中 srcImage.copyTo(dstImage,cannyEdge); #ifdef DEBUG_SHOW_ScreenExtract //namedWindow("EdgeDection()",CV_WINDOW_AUTOSIZE); //imshow(Windows_Edge,g_dstImage); imshow("EdgeDection()",cannyEdge); #endif return cannyEdge; } Mat ScreenExtract::HoughLine(Mat srcImage) { Mat dstImage = srcImage.clone(); //g_houghWithSrc = g_srcImage.clone(); dstImage = Scalar::all(255); //houghlinesP //vector<Vec4i> mylines; //HoughLinesP(g_cannyDetectedEdges,mylines,1,CV_PI/180,valueA+1,valueB,valueC); ////循环遍历绘制每一条线段 // for( size_t i = 0; i < mylines.size(); i++ ) // { // Vec4i lines = mylines[i]; // //line（）划线 // line( dstImage, Point(lines[0], lines[1]), Point(lines[2], lines[3]), Scalar(55,100,195), 1,CV_AA);// CV_AA); // } //houghlines vector<Vec2f> mylines; HoughLines(srcImage, mylines, 1, (CV_PI-0.2)/180,srcImage.cols/8, 0, 0 ); //HoughLines(g_cannyDetectedEdges, mylines, 1, CV_PI/180, valueA+1, valueB, valueC ); //依次在图中绘制出每条线段 for( size_t i = 0; i < mylines.size(); i++ ) { float rho = mylines[i][0], theta = mylines[i][1]; Point pt1, pt2; double a = cos(theta), b = sin(theta); double x0 = a*rho, y0 = b*rho; pt1.x = cvRound(x0 + 3000*(-b)); pt1.y = cvRound(y0 + 3000*(a)); pt2.x = cvRound(x0 - 3000*(-b)); pt2.y = cvRound(y0 - 3000*(a)); line( dstImage, pt1, pt2, Scalar(0,0,0), 1, CV_AA); //line( g_houghWithSrc, pt1, pt2, Scalar(0,255,0), 1, CV_AA); } #ifdef DEBUG_SHOW_ScreenExtract //显示图片 //imshow(Windows_Hough,dstImage); //imshow(Windows_Hough,g_houghLine); imshow("HoughLine()",dstImage); #endif return dstImage; } Mat ScreenExtract::CornerHarris( Mat srcImage ,vector<Point2f> &vecPoint) { //---------------------------【1】定义一些局部变量----------------------------- Mat g_srcImageClone = m_MatSrcImage.clone() ; Mat writeImage(m_MatSrcImage.rows,m_MatSrcImage.cols,m_MatSrcImage.type()); Mat dstImage;//目标图 Mat normImage;//归一化后的图 Mat scaledImage;//线性变换后的八位无符号整型的图 int iCornerThresh = 110; //g_CornerWithHoughWithSrc = g_houghWithSrc; writeImage = Scalar::all(255); //---------------------------【2】初始化--------------------------------------- //置零当前需要显示的两幅图，即清除上一次调用此函数时他们的值 //dstImage = Mat::zeros( srcImage.size(), CV_32FC1 ); //g_srcImageClone=g_srcImage.clone( ); //---------------------------【3】正式检测------------------------------------- //进行角点检测 //cornerHarris( g_srcGrayImage, dstImage, 2, 3, 0.04, BORDER_DEFAULT ); //原图角点检测 cornerHarris( srcImage, dstImage, 2, 3, 0.04, BORDER_DEFAULT ); //霍夫变换后的角点检测 // 归一化与转换 normalize( dstImage, normImage, 0, 255, NORM_MINMAX, CV_32FC1, Mat() ); convertScaleAbs( normImage, scaledImage );//将归一化后的图线性变换成8位无符号整型 //---------------------------【4】进行绘制------------------------------------- // 将检测到的，且符合阈值条件的角点绘制出来 for( int j = normImage.rows/100; j < normImage.rows-normImage.rows/100 ; j++ ) { for( int i = normImage.cols/100; i < normImage.cols-normImage.cols/100; i++ ) { if( (int) normImage.at<float>(j,i) > iCornerThresh ) { circle( g_srcImageClone, Point( i, j ), 5, Scalar(10,10,255), 2, 8, 0 ); //circle( g_CornerWithHoughWithSrc, Point( i, j ), 5, Scalar(10,10,255), 2, 8, 0 ); circle( scaledImage, Point( i, j ), 10, Scalar(0,10,255), -1, 8, 0 ); circle( writeImage, Point( i, j ), 10, Scalar(0,10,255), -1, 8, 0 ); } } } vecPoint = GatherPoint( writeImage,iCornerThresh); #ifdef DEBUG_SHOW_ScreenExtract //---------------------------【4】显示最终效果--------------------------------- //imshow( "CornerHarris", g_CornerWithHoughWithSrc ); //在原图叠加霍夫图上显示 //imshow( "CornerHarris", g_srcImageClone ); //在原图上显示 //imshow( "CornerHarris", scaledImage ); //使用灰度图显示 imshow( "CornerHarris", writeImage ); #endif return scaledImage; } vector<Point2f> ScreenExtract::GatherPoint( Mat srcImage, int CornerThresh ) { //imshow( "GatherPoint", srcImage ); //Mat grayImage=srcImage.clone() ; Mat canny_output; Mat grayImage; vector<vector<Point>>contours; vector<Vec4i>hierarchy; //RNG rng(12345); //转成灰度图 cvtColor(srcImage, grayImage, COLOR_BGR2GRAY); //canny边缘检测 Canny(grayImage, canny_output, 50, 50 * 2, 3); //轮廓提取 findContours(canny_output, contours, hierarchy, RETR_TREE, CHAIN_APPROX_SIMPLE, Point(0, 0)); //计算图像矩 vector<Moments>mu(contours.size()); for (unsigned int i = 0; i < contours.size(); i++) { mu[i] = moments(contours[i], false); } //计算图像的质心 vector<Point2f>mc(contours.size()); for (unsigned int i = 0; i < contours.size(); i++) { mc[i] = Point2f(mu[i].m10 / mu[i].m00, mu[i].m01 / mu[i].m00); } //绘制轮廓 Mat drawing = Mat::zeros(srcImage.size(), CV_8UC3); for (unsigned int i = 0; i < contours.size(); i++) { Scalar color = Scalar(0, 255, 0); drawContours(drawing, contours, i, color, 2, 8, hierarchy, 0, Point()); circle(drawing, mc[i], 4, color, -1, 8, 0); } #ifdef DEBUG_SHOW_ScreenExtract //namedWindow("GatherPoint", WINDOW_AUTOSIZE); imshow("GatherPoint", drawing); #endif return mc; } Mat ScreenExtract::perspectiveChange(Mat srcMat,vector<Point2f> srcBuff) { //Mat变量 Mat dstMat(srcMat.rows,srcMat.cols,srcMat.type()); Mat perspectiveMat( 2,3,CV_32FC1); //透视变换参数 Point2f perspectiveSrcBuff[4]; Point2f perspectiveDesBuff[4]; //透视变换坐标设置 perspectiveSrcBuff[0] = Point2f((srcBuff[0].x+srcBuff[1].x)/2,(srcBuff[0].y+srcBuff[1].y)/2) ; perspectiveSrcBuff[1] = Point2f((srcBuff[2].x+srcBuff[3].x)/2,(srcBuff[2].y+srcBuff[3].y)/2) ; perspectiveSrcBuff[2] = Point2f((srcBuff[4].x+srcBuff[5].x)/2,(srcBuff[4].y+srcBuff[5].y)/2) ; perspectiveSrcBuff[3] = Point2f((srcBuff[6].x+srcBuff[7].x)/2,(srcBuff[6].y+srcBuff[7].y)/2) ; //求变换后坐标 for(int i=0;i<3;i++) { if(0== getPointPlace(srcMat,perspectiveSrcBuff[i]) ) { perspectiveDesBuff[i] = Point2f( 0, 0); } else if(1 == getPointPlace(srcMat,perspectiveSrcBuff[i]) ) { perspectiveDesBuff[i] = Point2f( 0, static_cast<float>(dstMat.rows-1)); } else if(2 == getPointPlace(srcMat,perspectiveSrcBuff[i]) ) { perspectiveDesBuff[i] = Point2f( static_cast<float>(dstMat.cols-1), 0); } else if(3 == getPointPlace(srcMat,perspectiveSrcBuff[i]) ) { perspectiveDesBuff[i] = Point2f( static_cast<float>(dstMat.cols-1), static_cast<float>(dstMat.rows-1)); } } //求透视变换 perspectiveMat = getPerspectiveTransform( perspectiveSrcBuff, perspectiveDesBuff ); //对源图像应用刚刚的透视变换 warpPerspective(srcMat, dstMat, perspectiveMat, dstMat.size()); #ifdef DEBUG_SHOW_ScreenExtract //显示 imshow( "perspectiveChange", dstMat ); #endif return dstMat; } int ScreenExtract::getPointPlace(Mat srcImage,Point2f point) { if(point.x < srcImage.cols/2) { if(point.y < srcImage.rows/2) { return 0; } else { return 1; } } else { if(point.y < srcImage.rows/2) { return 2; } else { return 3; } } } Mat ScreenExtract::rotation(Mat srcMat,float degree) { Mat dstImage(srcMat.rows,srcMat.cols,srcMat.type()); int len = max(srcMat.cols, srcMat.rows); Point2f pt(len/2.f,len/2.f); Mat r = getRotationMatrix2D(pt,degree,1.0); warpAffine(srcMat,dstImage,r,Size(srcMat.rows,srcMat.cols)); return dstImage; }
ScreenExtraction.h为头文件
#ifndef __SCREENEXTRACTION_H #define __SCREENEXTRACTION_H #include <opencv2/highgui/highgui.hpp> #include <opencv2/imgproc/imgproc.hpp> /* #include <opencv2/nonfree/nonfree.hpp> #include<opencv2/legacy/legacy.hpp> */ #include <iostream> using namespace cv; using namespace std; //#define DEBUG_SHOW_ScreenExtract class ScreenExtract { public: ScreenExtract(); ScreenExtract(Mat srcMat); ~ScreenExtract(); Mat getDst(); bool setSrc(Mat srcMat); Mat runExtract(); Mat runFastExtract(Mat srcMat); private: Mat EdgeDection(Mat srcImage,int cannyThrel); Mat HoughLine(Mat srcImage); Mat CornerHarris( Mat srcImage,vector<Point2f> &vecPoint) ; vector<Point2f> GatherPoint( Mat srcImage, int CornerThresh ) ; Mat perspectiveChange(Mat srcMat,vector<Point2f> srcBuff); int ScreenExtract::getPointPlace(Mat srcImage,Point2f point);//获取点在屏幕的位置 Mat rotation(Mat srcImage,float degree); private: Mat m_MatSrcImage; Mat m_MatEdgeImage; Mat m_MatHoughImage; Mat m_MatCornerImage; Mat m_MatDstImageLie; Mat m_MatDstImageStand; vector<Point2f> m_PointPerspectiveSrcBuff; }; #endif
ImageMatch.cpp用于识别内容
#include "ImageMatch.h" ImageMatch::ImageMatch(Mat srcImage) { m_MatSrcImage = srcImage.clone(); m_MatMatchImage = srcImage.clone(); m_MatDstImage = srcImage.clone(); m_iFeatureMaxSize = 50; } ImageMatch::~ImageMatch() { } /******************************************************************************* * Function Name : addObjectFeatureImage * Description : add an objcet feature to feature vector * Input : None * Output : None * Return : succeed/failed *******************************************************************************/ bool ImageMatch::setSrcImage(Mat srcImage) { m_MatSrcImage = srcImage.clone(); return true; } /******************************************************************************* * Function Name : addObjectFeatureImage * Description : add an objcet feature to feature vector * Input : None * Output : None * Return : succeed/failed *******************************************************************************/ bool ImageMatch::addCubeFeatureImage(Mat image) { if(m_vecCubeFeatureImage.size()<m_iFeatureMaxSize) { m_vecCubeFeatureImage.push_back(image); return true; } return false; } bool ImageMatch::addCubeFeatureImage(vector<Mat> vec) { if(m_vecCubeFeatureImage.size()+vec.size()<m_iFeatureMaxSize) { m_vecCubeFeatureImage.insert(m_vecCubeFeatureImage.end(),vec.begin(),vec.end()); return true; } return false; } bool ImageMatch::addConfusingFeatureImage(Mat image) { if(m_vecConfusingFeatureImage.size()<m_iFeatureMaxSize) { m_vecConfusingFeatureImage.push_back(image); return true; } return false; } bool ImageMatch::addConfusingFeatureImage(vector<Mat> vec) { if(m_vecConfusingFeatureImage.size()+vec.size()<m_iFeatureMaxSize) { m_vecConfusingFeatureImage.insert(m_vecConfusingFeatureImage.end(),vec.begin(),vec.end()); return true; } return false; } /******************************************************************************* * Function Name : setPersonFeature * Description : set the person feature * Input : None * Output : None * Return : succeed/failed *******************************************************************************/ bool ImageMatch::setPersonFeature(Mat image) { m_MatPersonFeatureImage = image.clone(); return true; } /******************************************************************************* * Function Name : runMatch * Description : Match main programm * Input : None * Output : None * Return : destination image *******************************************************************************/ Mat ImageMatch::runMatch() { //【0】预处理 Mat dstImage = m_MatSrcImage.clone(); Mat srcImage_Adjust= AdjectDefinition(m_MatSrcImage,100,0); //【1】匹配人物 m_pLocPerson = SearchPerson( srcImage_Adjust, 10, (int)(srcImage_Adjust.rows*0.35), srcImage_Adjust.cols-20, (int)(srcImage_Adjust.rows*0.35) ); #ifdef DEBUG_SHOW_ImageMatch //rectangle( srcImage_Adjust, Rect(loc, Size(m_MatPersonFeatureImage.cols, m_MatPersonFeatureImage.rows) ), Scalar(0, 0, 255), 2, 8, 0 ); circle( srcImage_Adjust, m_pLocPerson, 10, Scalar(0,0,255), 5, 8, 0 ); //imshow("TempleMatch",srcImage_Adjust); #endif //【2】匹配建筑 if(m_pLocPerson.x < srcImage_Adjust.cols*0.5)//人物在左边，搜索右半区域，为了加速 m_pLocBlock = SearchBuilding( srcImage_Adjust, (int)(srcImage_Adjust.cols*0.3)+10, (int)(srcImage_Adjust.rows*0.20), (int)(srcImage_Adjust.cols*0.7)-20, (int)(srcImage_Adjust.rows*0.5)); else//搜索左半区 m_pLocBlock = SearchBuilding( srcImage_Adjust , 10, (int)(srcImage_Adjust.rows*0.20), (int)(srcImage_Adjust.cols*0.7)-20, (int)(srcImage_Adjust.rows*0.5)); #ifdef DEBUG_SHOW_ImageMatch circle( srcImage_Adjust, m_pLocBlock, 20, Scalar(0,255,0), 5, 8, 0 ); imshow("TempleMatch",srcImage_Adjust); #endif m_fDistance = calculateDistance(m_pLocPerson,m_pLocBlock); return dstImage; } /******************************************************************************* * Function Name : changeSrcAndrunMatch * Description : Change src image and run Match * Input : None * Output : None * Return : destination image *******************************************************************************/ Mat ImageMatch::changeSrcAndrunMatch(Mat srcImage) { setSrcImage(srcImage); return runMatch(); } float ImageMatch::getDistance() { float result = m_fDistance*2.1f+0; if(result<0) result=0; return result; } //private /******************************************************************************* * Function Name : vecFeatureImageToVecFeatureImage * Description : update definition * Input : None * Output : None * Return : destination image *******************************************************************************/ bool ImageMatch::FeatureVectorChangeToEdgeVector() { m_vecFeatureEdgeImage.clear(); for(unsigned int i=0;i<m_vecCubeFeatureImage.size();i++) { //m_MatSrcImage = FeatureMatchAndMark(m_vecCubeFeatureImage[i],m_MatSrcImage); m_vecFeatureEdgeImage.push_back ( calEdge(m_vecCubeFeatureImage[i],10)); #ifdef DEBUG_SHOW_ImageMatch imshow("Edge",m_vecFeatureEdgeImage[i]); #endif } return true; } /******************************************************************************* * Function Name : definition * Description : update definition * Input : None * Output : None * Return : destination image *******************************************************************************/ Mat ImageMatch::AdjectDefinition(Mat srcImage,int contrast,int brightness) { Mat dstImage; //平滑 blur(srcImage,dstImage,Size(3,3)); //对比度亮度调节 for(int y = 0; y < dstImage.rows; y++ ) //遍历图片的纵坐标 { for(int x = 0; x < dstImage.cols; x++ )//遍历图片的横坐标 { for(int c = 0; c < 3; c++ ) //分开图像的RGB { //对比度在0-300之间，所以乘以0.01,并用saturate_cast把计算结果转换成uchar类型 dstImage.at<Vec3b>(y,x)[c]= saturate_cast<uchar>( (contrast*0.01)*(dstImage.at<Vec3b>(y,x)[c] ) + brightness ); } } } // imshow("src",srcImage); // imshow("dst",dstImage); //二次平滑 //blur(dstImage,dstImage,Size(3,3)); return dstImage; } /******************************************************************************* * Function Name : addObjectFeatureImage * Description : add an objcet feature to feature vector * Input : None * Output : None * Return : succeed/failed *******************************************************************************/ Mat ImageMatch::calEdge(Mat srcImage,int threl) { Mat dstImage; //canny Canny(srcImage,dstImage,threl,threl*3,3); #ifdef DEBUG_SHOW_ImageMatch imshow("calEdge",dstImage); #endif return dstImage; } /******************************************************************************* * Function Name : SearchPerson * Description : find block point * Input : None * Output : None * Return : destination image *******************************************************************************/ Point ImageMatch::SearchPerson(Mat TargetImage,int ROI_Xstart,int ROI_Ystart,int ROI_length,int ROI_heigh ) { //【0】局部变量用于返回人物坐标 Point dstLoc; //【1】设置ROI区域 Mat imageROI=TargetImage( Rect(ROI_Xstart, ROI_Ystart, ROI_length, ROI_heigh ) ); //【2】模板匹配 float matchaValue = (float)TempleMatch(m_MatPersonFeatureImage, imageROI, dstLoc, CV_TM_CCOEFF_NORMED) ; if( matchaValue < 0.64) { throw("Person match fail"); } else { #ifdef DEBUG_SHOW_ImageMatch //rectangle( imageROI, Rect(dstLoc, Size(m_MatPersonFeatureImage.cols, m_MatPersonFeatureImage.rows) ), Scalar(0, 0, 255), 2, 8, 0 ); //imshow("SearchqPerson",imageROI); #endif //根据输入的ROI设置纠正坐标 dstLoc.x+=ROI_Xstart; dstLoc.y+=ROI_Ystart; //让坐标重新纠正道指向人物与地面的接触中点 dstLoc.x += (int)(m_MatPersonFeatureImage.cols/2); dstLoc.y += (int)(m_MatPersonFeatureImage.rows*0.8); } return dstLoc; } /******************************************************************************* * Function Name : FindBuilding * Description : find block point * Input : None * Output : None * Return : destination image *******************************************************************************/ Point ImageMatch::SearchBuilding(Mat TargetImage, int ROI_Xstart,int ROI_Ystart,int ROI_length,int ROI_heigh) { //【0】定义两个容器用于存放每个矩形特征的最佳匹配点和匹配值 vector<Point> vecCubePoint(m_vecCubeFeatureImage.size()); vector<float> vecCubeValue(m_vecCubeFeatureImage.size()); //【0】定义一些阈值 float completelyMatchThrel = 0.70f;//认为一种图形完全匹配的阈值 float LowestThrel = 40;//认为基本相近的阈值 //【0】定义一些临时变量 Point loc; float matchValue; //【0】最终配置相关变量 bool matchSucceed=false;//是否找到完全匹配特征标记 Point lastLoc(0,0);//最终位置 float lastValue;//最终匹配值 int lastIndex=-1;//最终位置对应的特征序号 //【0】计时 TimeOperation time; int ms=0; //【1】转为灰度图 cvtColor(TargetImage,TargetImage,CV_BGR2GRAY); //【2】设置ROI区域 Mat imageROI=TargetImage( Rect(ROI_Xstart, ROI_Ystart, ROI_length, ROI_heigh ) ); //【3】匹配建筑 for(unsigned int i=0;i<m_vecCubeFeatureImage.size();i++) { matchValue = (float)TempleMatch(m_vecCubeFeatureImage[i],imageROI,loc,CV_TM_CCOEFF_NORMED ); if(matchValue > completelyMatchThrel) { lastLoc = loc ; //纠正坐标 lastLoc.x += ROI_Xstart; lastLoc.y += ROI_Ystart; lastLoc.x += (int)(m_vecCubeFeatureImage[i].cols*0.5); lastLoc.y += (int)(m_vecCubeFeatureImage[i].rows*0.3); //若是任务脚底的建筑，继续找 if(calculateDistance(lastLoc,m_pLocPerson)<LowestThrel+30) { lastLoc.x=0; lastLoc.y=0; continue; } matchSucceed = true; lastValue = matchValue; lastIndex = i; #ifdef DEBUG_SHOW_ImageMatch imshow("matchfeature",m_vecCubeFeatureImage[i]); rectangle( imageROI, Rect(loc, Size(m_vecCubeFeatureImage[i].cols, m_vecCubeFeatureImage[i].rows) ), Scalar(0, 0, 255), 2, 8, 0 ); imshow("SearchPerson",imageROI); #endif break; } else { vecCubePoint[i]=loc; vecCubeValue[i]=matchValue; } } ////【4】没有完全匹配，寻找最佳匹配 if(!matchSucceed) { for(unsigned int i=0;i<m_vecCubeFeatureImage.size();i++) { Mat featureROI; if(m_pLocPerson.x<TargetImage.cols/2)//取特征的右边 { featureROI =m_vecCubeFeatureImage[i]( Rect(m_vecCubeFeatureImage[i].cols*0.4, 0, m_vecCubeFeatureImage[i].cols*0.6-1, m_vecCubeFeatureImage[i].rows ) ); matchValue = (float)TempleMatch(featureROI,imageROI,loc,CV_TM_CCOEFF_NORMED ); if(matchValue > completelyMatchThrel) { lastLoc = loc ; //纠正坐标 lastLoc.x += ROI_Xstart; lastLoc.y += ROI_Ystart; lastLoc.x += (int)(m_vecCubeFeatureImage[i].cols*(0.5-0.4)); lastLoc.y += (int)(m_vecCubeFeatureImage[i].rows*0.3); //若是任务脚底的建筑，继续找 if(calculateDistance(lastLoc,m_pLocPerson)<LowestThrel) { lastLoc.x=0; lastLoc.y=0; continue; } matchSucceed = true; lastValue = matchValue; lastIndex = i; #ifdef DEBUG_SHOW_ImageMatch imshow("matchfeature",m_vecCubeFeatureImage[i]); rectangle( imageROI, Rect(loc, Size(m_vecCubeFeatureImage[i].cols, m_vecCubeFeatureImage[i].rows) ), Scalar(0, 0, 255), 2, 8, 0 ); imshow("SearchPerson",imageROI); #endif break; } } else { featureROI =m_vecCubeFeatureImage[i]( Rect(0, 0, m_vecCubeFeatureImage[i].cols*0.6-1, m_vecCubeFeatureImage[i].rows ) ); matchValue = (float)TempleMatch(featureROI,imageROI,loc,CV_TM_CCOEFF_NORMED ); if(matchValue > completelyMatchThrel) { lastLoc = loc ; //纠正坐标 lastLoc.x += ROI_Xstart; lastLoc.y += ROI_Ystart; lastLoc.x += (int)(m_vecCubeFeatureImage[i].cols*0.5); lastLoc.y += (int)(m_vecCubeFeatureImage[i].rows*0.3); //若是任务脚底的建筑，继续找 if(calculateDistance(lastLoc,m_pLocPerson)<LowestThrel) { lastLoc.x=0; lastLoc.y=0; continue; } matchSucceed = true; lastValue = matchValue; lastIndex = i; #ifdef DEBUG_SHOW_ImageMatch imshow("matchfeature",m_vecCubeFeatureImage[i]); rectangle( imageROI, Rect(loc, Size(m_vecCubeFeatureImage[i].cols, m_vecCubeFeatureImage[i].rows) ), Scalar(0, 0, 255), 2, 8, 0 ); imshow("SearchPerson",imageROI); #endif break; } } } } //易混淆图片匹配 for(unsigned int i=0;i<m_vecConfusingFeatureImage.size();i++) { matchValue = (float)TempleMatch(m_vecConfusingFeatureImage[i],imageROI,loc,CV_TM_CCOEFF_NORMED ); if(matchValue > 0.95) { lastLoc = loc ; //纠正坐标 lastLoc.x += ROI_Xstart; lastLoc.y += ROI_Ystart; lastLoc.x += (int)(m_vecConfusingFeatureImage[i].cols*0.5); lastLoc.y += (int)(m_vecConfusingFeatureImage[i].rows*0.3); //若是任务脚底的建筑，继续找 if(calculateDistance(lastLoc,m_pLocPerson)<LowestThrel+30) { lastLoc.x=0; lastLoc.y=0; continue; } matchSucceed = true; lastValue = matchValue; lastIndex = i; #ifdef DEBUG_SHOW_ImageMatch imshow("matchfeature",m_vecConfusingFeatureImage[i]); rectangle( imageROI, Rect(loc, Size(m_vecConfusingFeatureImage[i].cols, m_vecConfusingFeatureImage[i].rows) ), Scalar(0, 0, 255), 2, 8, 0 ); imshow("SearchPerson",imageROI); #endif break; } else { vecCubePoint[i]=loc; vecCubeValue[i]=matchValue; } } if(!matchSucceed) { float bestValue=0; int bestIndex = 0; unsigned int i=0; bestValue = vecCubeValue[0]; for(i=0;i<vecCubeValue.size();i++) { if(vecCubeValue[i]>bestValue) { bestValue = vecCubeValue[i]; bestIndex = i; } } #ifdef DEBUG_SHOW_ImageMatch imshow("matchfeature",m_vecCubeFeatureImage[bestIndex]); #endif lastLoc = vecCubePoint[bestIndex]; lastIndex=bestIndex; } return lastLoc; } /******************************************************************************* * Function Name : calculateDistance * Description : Temple match ,Get the best matching point and return it * Input : None * Output : None * Return : Point:best matching point. *******************************************************************************/ float ImageMatch::calculateDistance(Point A,Point B) { float distance; distance = powf((float)(B.x - A.x),2) + powf((float)(B.y - A.y),2); distance = sqrtf(distance); return distance; } /******************************************************************************* * Function Name : TempleMatch * Description : Temple match ,Get the best matching point and return it * Input : None * Output : None * Return : Point:best matching point. *******************************************************************************/ double ImageMatch::TempleMatch( Mat tepl ,Mat image, Point &point,int method) { int result_cols = image.cols - tepl.cols + 1; int result_rows = image.rows - tepl.rows + 1; //模板检测 Mat result = Mat( result_cols, result_rows, CV_32FC1 ); matchTemplate( image, tepl, result, method ); //寻找最佳值 double minVal, maxVal; Point minLoc, maxLoc; minMaxLoc( result, &minVal, &maxVal, &minLoc, &maxLoc, Mat() ); //根据方法返回最佳值 if(CV_TM_SQDIFF == method || CV_TM_SQDIFF_NORMED == method) { point = minLoc; return minVal; } else { point = maxLoc; return maxVal; } return 0; }
ImageMatch.h为头文件
#ifndef __IMAGEMATCH_H #define __IMAGEMATCH_H #include <opencv2/highgui/highgui.hpp> #include <opencv2/imgproc/imgproc.hpp> #include <opencv2/core/core.hpp> #include <iostream> #include <vector> #include "TimeOperation.h" #define DEBUG_SHOW_ImageMatch using namespace std; using namespace cv; class ImageMatch { public: ImageMatch(Mat srcImage); ~ImageMatch(); bool setSrcImage(Mat srcImage); bool addCubeFeatureImage(Mat image); bool addCubeFeatureImage(vector<Mat> vec); bool addConfusingFeatureImage(Mat image); bool addConfusingFeatureImage(vector<Mat> vec); bool setPersonFeature(Mat image); Mat runMatch(); Mat changeSrcAndrunMatch(Mat srcImage); float getDistance(); private: bool FeatureVectorChangeToEdgeVector(); Mat AdjectDefinition(Mat srcImage,int contrast=100,int brightness=0); Mat calEdge(Mat srcImage,int threl); Mat FeatureMatchAndMark(Mat FeatureImage,Mat TargetImage); Point SearchPerson(Mat TargetImage,int ROI_Xstart,int ROI_Ystart,int ROI_length,int ROI_heigh ); Point SearchBuilding(Mat TargetImage, int ROI_Xstart,int ROI_Ystart,int ROI_length,int ROI_heigh); float calculateDistance(Point A,Point B); private: double TempleMatch( Mat tepl ,Mat image, Point &point,int method); private: vector<Mat> m_vecCubeFeatureImage; vector<Mat> m_vecConfusingFeatureImage; vector<Mat> m_vecFeatureEdgeImage; unsigned int m_iFeatureMaxSize; Mat m_MatPersonFeatureImage; Mat m_MatSrcImage; Mat m_MatMatchImage; Mat m_MatDstImage; float m_fDistance; Point m_pLocPerson; Point m_pLocBlock; }; #endif
SeralPort.cpp为串口相关内容，用于与下位机通信
#include "SerialPort.h" /******************************************************************************* * Function Name : SerialPort * Description : open serial port. * Input : None * Output : None * Return : succeed/failed *******************************************************************************/ SerialPort::SerialPort(int CommNumber,int BaudRate,int Parity ,int ByteSize , int StopBits ) { //【0】预处理 if(CommNumber<0 || BaudRate<0 || Parity<0 || ByteSize<0 || StopBits<0) { throw("SerialPort parameter wrong"); return; } m_iComNumber = CommNumber; //int 2 str std::stringstream ss; std::string str; ss<<CommNumber; ss>>str; string strComNumber = "COM" + str; const char *pComNumber = strComNumber.c_str(); //【1】打开串口 m_cHCom = CreateFileA( pComNumber, GENERIC_READ | GENERIC_WRITE, 0, NULL, OPEN_EXISTING, 0, 0); //【1.5】超时处理 COMMTIMEOUTS stTimeOuts; GetCommTimeouts(m_cHCom, &stTimeOuts); stTimeOuts.ReadIntervalTimeout = 0; stTimeOuts.ReadTotalTimeoutMultiplier = 0; stTimeOuts.ReadTotalTimeoutConstant = 1; stTimeOuts.WriteTotalTimeoutMultiplier = 0; stTimeOuts.WriteTotalTimeoutConstant = 0; if (!SetCommTimeouts(m_cHCom, &stTimeOuts)) { throw("SerialPort openning time out"); CloseHandle(m_cHCom); return; } //【2】取得并设置端口状态 GetCommState(m_cHCom, &m_cDcb); m_cDcb.DCBlength = sizeof(DCB); m_cDcb.BaudRate = BaudRate; m_cDcb.Parity = Parity; m_cDcb.ByteSize = ByteSize; m_cDcb.StopBits = StopBits; if (!SetCommState(m_cHCom, &m_cDcb)) { throw("SerialPort openning fail"); CloseHandle(m_cHCom); return; } } SerialPort::~SerialPort( ) { CloseHandle(m_cHCom); } /******************************************************************************* * Function Name : SerialPort::send * Description : serialPort send. * Input : None * Output : None * Return : succeed/failed *******************************************************************************/ bool SerialPort::send(vector<unsigned char> vecSend) { DWORD dwWrittenLen; //【0】 把vector换成char* char *cSendBuff = new char[vecSend.size()]; { memcpy(cSendBuff, &vecSend[0], vecSend.size()*sizeof(char)); } if(!WriteFile(m_cHCom,cSendBuff, vecSend.size(),&dwWrittenLen,NULL)) { throw("Sending fail 1"); return false; } if(vecSend.size() != dwWrittenLen) { throw("Sending fail 2"); return false; } return true; } /******************************************************************************* * Function Name : SerialPort::send * Description : serialPort send. * Input : None * Output : None * Return : succeed/failed *******************************************************************************/ bool SerialPort::packAndsend(unsigned short data1) { const int PackageLength=15; const unsigned char PackHead1=0x55; const unsigned char PackHead2=0xaa; static unsigned char number = 0; unsigned short check=0; vector<unsigned char> vecForSend(PackageLength); //报头 vecForSend[0] = PackHead1; vecForSend[1] = PackHead2; //序号 vecForSend[2] = number++; //时间 vecForSend[3] = (char)(data1>>8); vecForSend[4] = (char)data1; //校验 for(int i=0;i<PackageLength-2;i++) { check += vecForSend[i]; } vecForSend[13] = (char)(check>>8); vecForSend[14] = (char)check; //发送 if(send(vecForSend)) { return true; } return false; }
SeralPort.h为头文件
#ifndef __SERIALPORT_H #define __SERIALPORT_H #include <windows.h> #include <iostream> #include <vector> #include <string> #include <sstream> using namespace std; class SerialPort { public: SerialPort(int CommNumber,int BaudRate = 9600,int Parity = 0,int ByteSize = 8, int StopBits = ONESTOPBIT); ~SerialPort(); public: bool send(vector<unsigned char> vecSend); bool packAndsend(unsigned short data1); private: HANDLE m_cHCom; int m_iComNumber; DCB m_cDcb; vector<char> m_vecRecived; }; #endif
TimeOperation.cpp用于计时、定时等操作
#include "TimeOperation.h" using namespace std; TimeOperation::TimeOperation() { m_eSingleStopwatchState = STOPWATCH_STOP; m_eTimeOutDectectionState = TIMEOUTCLOCK_OFF; } TimeOperation::~TimeOperation() { } /******************************************************************************* * Function Name : singleStopwatchRestart * Description : restart stopwatch * Input : None * Output : None * Return : succeed/failed *******************************************************************************/ bool TimeOperation::singularStopwatchRestart() { m_eSingleStopwatchState = STOPWATCH_ON; GetLocalTime(&m_cStopwatchOrigin); //cout<<"start.wSecond:"<<m_cStopwatchOrigin.wMinute<<endl; //cout<<"start.wMilliseconds:"<<m_cStopwatchOrigin.wMilliseconds<<endl; return true; } /******************************************************************************* * Function Name : singleStopwatchPause * Description : pause stopwatch and get time (In Milliseconds). * Input : None * Output : None * Return : succeed/failed *******************************************************************************/ bool TimeOperation::singularStopwatchPause(int &ms) { if(STOPWATCH_ON != m_eSingleStopwatchState ) { return false; } SYSTEMTIME EndTime; GetLocalTime(&EndTime); //cout<<"end.wSecond:"<<EndTime.wMinute<<endl; //cout<<"end.wMilliseconds:"<<EndTime.wMilliseconds<<endl; ms = EndTime.wMilliseconds-m_cStopwatchOrigin.wMilliseconds + ((EndTime.wSecond-m_cStopwatchOrigin.wSecond) + (EndTime.wMinute - m_cStopwatchOrigin.wMinute)*60)*1000; cout<<"ms:"<<ms<<endl; m_eSingleStopwatchState = STOPWATCH_PAUSE; return true; } /******************************************************************************* * Function Name : singleStopwatchPause * Description : pause stopwatch and get time (In Milliseconds). * Input : None * Output : None * Return : succeed/failed *******************************************************************************/ bool TimeOperation::TimeOutDectectionSetClock(SYSTEMTIME &setTime) { m_cTimeOutDectectionThrel = ((setTime.wHour*60+setTime.wMinute)*60+setTime.wSecond)*1000+setTime.wMilliseconds; //set now time SYSTEMTIME nowTime; GetLocalTime(&nowTime); m_cTimeOutDectectionStartTime = ((nowTime.wHour*60+nowTime.wMinute)*60+nowTime.wSecond)*1000+nowTime.wMilliseconds; m_eTimeOutDectectionState = TIMEOUTCLOCK_ON; return true; } /******************************************************************************* * Function Name : TimeOutDectectionSetClock * Description : pause stopwatch and get time (In Milliseconds). * Input : None * Output : None * Return : succeed/failed *******************************************************************************/ bool TimeOperation::TimeOutDectectionSetClock(int ms) { m_cTimeOutDectectionThrel = ms; SYSTEMTIME nowTime; GetLocalTime(&nowTime); m_cTimeOutDectectionStartTime = ((nowTime.wHour*60+nowTime.wMinute)*60+nowTime.wSecond)*1000+nowTime.wMilliseconds; m_eTimeOutDectectionState = TIMEOUTCLOCK_ON; return true; } /******************************************************************************* * Function Name : singleStopwatchPause * Description : pause stopwatch and get time (In Milliseconds). * Input : None * Output : None * Return : succeed/failed *******************************************************************************/ bool TimeOperation::singularTimeOutDectectionCheckClock() { if( TIMEOUTCLOCK_OFF == m_eTimeOutDectectionState ) { return false; } SYSTEMTIME nowTime; GetLocalTime(&nowTime); long int nowTime_ms=((nowTime.wHour*60+nowTime.wMinute)*60+nowTime.wSecond)*1000+nowTime.wMilliseconds; if( nowTime_ms - m_cTimeOutDectectionStartTime> m_cTimeOutDectectionThrel) { m_eTimeOutDectectionState = TIMEOUTCLOCK_OFF; return true; } return false; } /******************************************************************************* * Function Name : singleStopwatchPause * Description : pause stopwatch and get time (In Milliseconds). * Input : None * Output : None * Return : succeed/failed *******************************************************************************/ bool TimeOperation::multipleTimeOutDectectionCheckClock() { if( TIMEOUTCLOCK_OFF == m_eTimeOutDectectionState ) { return false; } SYSTEMTIME nowTime; GetLocalTime(&nowTime); long int nowTime_ms=(nowTime.wHour*60+nowTime.wMinute)*1000+nowTime.wMilliseconds; if( nowTime_ms > m_cTimeOutDectectionThrel) { return true; } return false; }
TimeOperation.h为头文件
#ifndef __TIMEOPERATION_H #define __TIMEOPERATION_H #include <windows.h> #include <iostream> typedef enum { STOPWATCH_ON=0, STOPWATCH_PAUSE=1, STOPWATCH_STOP=2, }StopWatchState; typedef enum { TIMEOUTCLOCK_OFF = 0, TIMEOUTCLOCK_ON = 1, }TimeDectionState; class TimeOperation { public: TimeOperation(); ~TimeOperation(); //stopwatch bool singularStopwatchRestart(); bool singularStopwatchPause(SYSTEMTIME &getTime); bool singularStopwatchPause(int &ms); //timeout detection bool TimeOutDectectionSetClock(SYSTEMTIME &setTime); bool TimeOutDectectionSetClock(int ms); bool singularTimeOutDectectionCheckClock(); bool multipleTimeOutDectectionCheckClock(); private: //stopwatch SYSTEMTIME m_cStopwatchOrigin; StopWatchState m_eSingleStopwatchState; //TimeOut clock long int m_cTimeOutDectectionThrel; long int m_cTimeOutDectectionStartTime; TimeDectionState m_eTimeOutDectectionState; }; #endif
main.cpp主程序
#include "main.h" using namespace cv; using namespace std; typedef enum { Step_ReadCammer = 0, Step_CorrectScreen = 1, Step_MatchFeature = 2, Step_Communication = 3, Step_FastReadCammer =4, Step_FastCorrectScreen =5, }mainFunctionStep; vector<Mat> vecFeature; vector<Mat> vecConfusingFeature; void preconditioning(void) { string Suffix = ".png"; for(int i=1;i<50;i++) { //int 2 str stringstream ss; string index; ss<<i; ss>>index; string name = index+Suffix; Mat Img=imread(name,CV_LOAD_IMAGE_GRAYSCALE); //Mat Img=imread(name); if( !Img.data ) { break; } vecFeature.push_back(Img);//添加灰度特征 } for(int i=101;i<150;i++) { //int 2 str stringstream ss; string index; ss<<i; ss>>index; string name = index+Suffix; Mat Img=imread(name,CV_LOAD_IMAGE_GRAYSCALE); //Mat Img=imread(name); if( !Img.data ) { break; } vecConfusingFeature.push_back(Img);//添加灰度特征 } } int main() { Mat g_srcImage; Mat g_extractImage; float distance; SerialPort *c_COM3; mainFunctionStep step = Step_ReadCammer; //mainFunctionStep step = Step_Communication; //创建校正对象 ScreenExtract Screen; //加载图片 preconditioning(); //打开串口 try { c_COM3 = new SerialPort(3,115200); } catch(char *s) { return -1; } //打开摄像头 VideoCapture cap(0); if(!cap.isOpened()) { return -1; } cap.set(CV_CAP_PROP_FRAME_WIDTH, CammerWidth); cap.set(CV_CAP_PROP_FRAME_HEIGHT, CammerHeigh); while(1) { if( Step_ReadCammer == step ) { //【0】读取测试图片 //g_srcImage = imread("2.jpg"); //if(!g_srcImage.data) //{ // printf("Err"); //} ////namedWindow("【原始图】"); ////imshow("【原始图】",g_srcImage); //【0】读取摄像头 while(1) { cap>>g_srcImage; imshow("vedio",g_srcImage); if((char(waitKey(1))=='q')) { break; } } step = Step_CorrectScreen; } else if( Step_CorrectScreen == step) { //【1】屏幕提取矫正 Screen.setSrc(g_srcImage); Screen.runExtract(); try { g_extractImage = Screen.getDst(); } catch(char *s) { if(s == "Extract fail")//提取失败 { step = Step_ReadCammer; continue; } } imshow("结果",g_extractImage); step = Step_MatchFeature; } else if( Step_MatchFeature == step ) { //【2】特征提取匹配 Mat featureImage1 = imread("小人无背景.png"); if(!featureImage1.data) { printf("featureImage1"); return 0; } //imshow("素材",featureImage1); ImageMatch match(g_extractImage); //加入素材特征 match.setPersonFeature( featureImage1 ); match.addCubeFeatureImage(vecFeature); match.addConfusingFeatureImage(vecConfusingFeature); //运行匹配 try { match.runMatch(); } catch(char *s) { if(s == "Person match fail")//小人匹配失败 { step = Step_ReadCammer; continue; } else if(s == "Block match fail")//方块匹配失败 { step = Step_ReadCammer; continue; } } distance = match.getDistance(); step = Step_Communication; } else if( Step_Communication == step) { //for(;;) //{ // int ms; // TimeOperation time; // time.singularStopwatchRestart(); // c_COM3->packAndsend(5000); // waitKey(1000); // time.singularStopwatchPause(ms); // cout<<"ms"<<ms<<endl; //} c_COM3->packAndsend((unsigned short)distance); step = Step_FastReadCammer; waitKey(100); } else if( Step_FastReadCammer == step) { TimeOperation time; time.TimeOutDectectionSetClock(2*distance+2000); //【0】读取摄像头 while(1) { cap>>g_srcImage; imshow("vedio",g_srcImage); if( time.singularTimeOutDectectionCheckClock()) { break; } else if((char(waitKey(1))=='q')) { break; } } step = Step_FastCorrectScreen; } else if( Step_FastCorrectScreen == step) { //【1】屏幕提取矫正 Screen.runFastExtract(g_srcImage); try { g_extractImage = Screen.getDst(); } catch(char *s) { if(s == "Extract fail")//提取失败 { step = Step_ReadCammer; continue; } } imshow("结果",g_extractImage); step = Step_MatchFeature; } } while((char(waitKey(1))!='q')) {} return 0; }
main.h头文件
#ifndef __MAIN_H #define __MAIN_H #include <opencv2/highgui/highgui.hpp> #include <opencv2/imgproc/imgproc.hpp> #include "ScreenExtraction.h" #include "ImageMatch.h" #include "SerialPort.h" #include "TimeOperation.h" #define DEBUG_SHOW_MAIN #define CammerWidth 1280 #define CammerHeigh 720 #endif
模板匹配的素材下载：
链接：https://pan.baidu.com/s/1c3YXygC 密码：oeal
三、关键程序记录
3.1主程序的步骤
1、预处理（加载图片用于模板匹配）
void preconditioning(void) { string Suffix = ".png"; for(int i=1;i<50;i++) { //int 2 str stringstream ss; string index; ss<<i; ss>>index; string name = index+Suffix; Mat Img=imread(name,CV_LOAD_IMAGE_GRAYSCALE); //Mat Img=imread(name); if( !Img.data ) { break; } vecFeature.push_back(Img);//添加灰度特征 } for(int i=101;i<150;i++) { //int 2 str stringstream ss; string index; ss<<i; ss>>index; string name = index+Suffix; Mat Img=imread(name,CV_LOAD_IMAGE_GRAYSCALE); //Mat Img=imread(name); if( !Img.data ) { break; } vecConfusingFeature.push_back(Img);//添加灰度特征 } }
2、打开摄像头、串口
//打开串口 try { c_COM3 = new SerialPort(3,115200); } catch(char *s) { return -1; } //打开摄像头 VideoCapture cap(0); if(!cap.isOpened()) { return -1; } cap.set(CV_CAP_PROP_FRAME_WIDTH, CammerWidth); cap.set(CV_CAP_PROP_FRAME_HEIGHT, CammerHeigh);
3、主循环部分，在以下5种状态中切换
typedef enum { Step_ReadCammer = 0, Step_CorrectScreen = 1, Step_MatchFeature = 2, Step_Communication = 3, Step_FastReadCammer =4, Step_FastCorrectScreen =5, }mainFunctionStep;
- Step_ReadCammer：读取摄像头，在此状态中读取摄像头数据显示到屏幕上，等待用户开始程序：
- Step_CorrectScreen：矫正屏幕，在此状态中进行屏幕的边缘识别、屏幕内容提取矫正等操作：
- Step_MatchFeature：匹配特征，在此状态中进行识别操作，识别人物、方块
- Step_Communication ：进行通信，告诉下位机需要点击多长时间
- Step_FastReadCammer：快速读取摄像头，此步骤与第一步基本相同，不过不需要在进行人为干预，程序会快速读取图像进行处理
- Step_FastCorrectScreen：快速矫正屏幕，此步骤与第二步相似，但不再做边缘识别角点检测等操作，会跟据第二步的记录数据进行快速矫正
while(1) { if( Step_ReadCammer == step ) { //【0】读取测试图片 //g_srcImage = imread("2.jpg"); //if(!g_srcImage.data) //{ // printf("Err"); //} ////namedWindow("【原始图】"); ////imshow("【原始图】",g_srcImage); //【0】读取摄像头 while(1) { cap>>g_srcImage; imshow("vedio",g_srcImage); if((char(waitKey(1))=='q')) { break; } } step = Step_CorrectScreen; } else if( Step_CorrectScreen == step) { //【1】屏幕提取矫正 Screen.setSrc(g_srcImage); Screen.runExtract(); try { g_extractImage = Screen.getDst(); } catch(char *s) { if(s == "Extract fail")//提取失败 { step = Step_ReadCammer; continue; } } imshow("结果",g_extractImage); step = Step_MatchFeature; } else if( Step_MatchFeature == step ) { //【2】特征提取匹配 Mat featureImage1 = imread("小人无背景.png"); if(!featureImage1.data) { printf("featureImage1"); return 0; } //imshow("素材",featureImage1); ImageMatch match(g_extractImage); //加入素材特征 match.setPersonFeature( featureImage1 ); match.addCubeFeatureImage(vecFeature); match.addConfusingFeatureImage(vecConfusingFeature); //运行匹配 try { match.runMatch(); } catch(char *s) { if(s == "Person match fail")//小人匹配失败 { step = Step_ReadCammer; continue; } else if(s == "Block match fail")//方块匹配失败 { step = Step_ReadCammer; continue; } } distance = match.getDistance(); step = Step_Communication; } else if( Step_Communication == step) { //for(;;) //{ // int ms; // TimeOperation time; // time.singularStopwatchRestart(); // c_COM3->packAndsend(5000); // waitKey(1000); // time.singularStopwatchPause(ms); // cout<<"ms"<<ms<<endl; //} c_COM3->packAndsend((unsigned short)distance); step = Step_FastReadCammer; waitKey(100); } else if( Step_FastReadCammer == step) { TimeOperation time; time.TimeOutDectectionSetClock(2*distance+000); //【0】读取摄像头 while(1) { cap>>g_srcImage; imshow("vedio",g_srcImage); if( time.singularTimeOutDectectionCheckClock()) { break; } else if((char(waitKey(1))=='q')) { break; } } step = Step_FastCorrectScreen; } else if( Step_FastCorrectScreen == step) { //【1】屏幕提取矫正 Screen.runFastExtract(g_srcImage); try { g_extractImage = Screen.getDst(); } catch(char *s) { if(s == "Extract fail")//提取失败 { step = Step_ReadCammer; continue; } } imshow("结果",g_extractImage); step = Step_MatchFeature; } }
3.2关键处理：屏幕识别与提取
在ScreenExtraction类中实现了屏幕识别与提取
其中runExtract()方式为第一次识别时的操作。
runFastExtract()为第二次及以后操作，使用上一次runExtract()中储存的数据，只会保留透视变换操作。
Mat ScreenExtract::runExtract() { int cannyThrel=100; int failCNT=0; while(1) { m_MatEdgeImage = EdgeDection(m_MatEdgeImage,cannyThrel); m_MatHoughImage = HoughLine(m_MatEdgeImage); m_MatCornerImage = CornerHarris(m_MatHoughImage,m_PointPerspectiveSrcBuff); if(failCNT>20) { throw("Extract fail"); return m_MatSrcImage; } if(m_PointPerspectiveSrcBuff.size() == 8) { break; } else if(m_PointPerspectiveSrcBuff.size()>8) { if(cannyThrel<500) { cannyThrel+=20; } failCNT++; } else if(m_PointPerspectiveSrcBuff.size()<8) { if(cannyThrel>10) { cannyThrel-=5; } failCNT++; } waitKey(10); } m_MatDstImageLie = perspectiveChange(m_MatSrcImage,m_PointPerspectiveSrcBuff); m_MatDstImageStand = rotation(m_MatDstImageLie,90); return m_MatDstImageStand; }
runExtract()思路如下：
- 使用Canny边缘检测绘制摄像头的所有边缘。
- 使用霍夫变换勾勒出4条边,所构成的矩形为屏幕部分。
- 使用角点检测找到图片中的角点，并使用circle（）函数把所有角点变为实心圆形，此时在附近的角点会连成一个很大的点，图中会剩下4个大点（如图红点）。
- 使用边缘检测对上一步的图操作，并求出质心，效果如图绿点。
- 使用warpPerspective（）等函数进行透视变换，还原屏幕。
- 在上述操作中，很容易出现检测失败，屏幕中多了很多点等等情况，当检测失败时，自动调整第一步中Canny的阈值，重新进行边缘检测
3.3关键处理：识别小人和方块
识别函数在ImageMatch类中。
预先使用Photoshop提取小人和方块的图片，一张一张进行对比，选出匹配度比较高的一张，事实证明，这样做太麻烦，这个方法实在太蠢，识别率也低，以后再改成别的。
据不完全统计，游戏中的方块有如下种类，提取了一些比较特征，可以在以下素材图片第三节中的连接中下载
runMatch()函数操作思路：
- 匹配小人，计算小人的脚底坐标(如图红圈)。
- 判断小人处于屏幕的左边还是右边，我们只需要搜索一边，缩小搜索范围减少计算量（例如在下图中，小人站在屏幕的左半区，方块匹配只需要搜索屏幕的右半区）。
- 为了减少匹配时间，将裁减好的图转为灰度图，循环使用模板匹配上面的模板、挑出匹配率高的模板、计算方块表面中心点坐标（如图方框为识别最高的模板，和绿色圈为方块表面的中心点）。在实际匹配中发现有些方块靠的很近会被遮挡（如下图的闹钟左边被档），容易造成匹配失败，所以匹配失败的时候用那些模板的左/右半边进行再次匹配。
- 对于容易出错的模板，单独分开来最后识别，并提高阈值。
- 若上述操作都没高匹配的图片出现，则选择匹配度最高的一个。
- 计算小人和方块的距离（红圈和绿圈的距离）。
3.3其他程序
距离和按压时间的转换：
一次函数y=ax+b，实际使用需要根据下位机误差调整
float ImageMatch::getDistance() { float result = m_fDistance*2.1f+0; if(result<0) result=0; return result; }
计时类TimeOperation.cpp：
通过获取Windows的时间函数GetLocalTime（）为基础，实现两个功能
- 秒表(红色)：用来计算某部分程序的运行时间，看看用时多久。
- 等待超时(蓝色)：在串口发送后，需要等待一段时间才进行下一次获取，使用等待超时功能。
串口类 SerialPort.cpp：
class SerialPort { public: SerialPort(int CommNumber,int BaudRate = 9600,int Parity = 0,int ByteSize = 8, int StopBits = ONESTOPBIT); ~SerialPort(); public: bool send(vector<unsigned char> vecSend); bool packAndsend(unsigned short data1); private: HANDLE m_cHCom; int m_iComNumber; DCB m_cDcb; vector<char> m_vecRecived; };
用于打开关闭串口、数据打包发送等等。使用的通信协议如下（十六进制表示）
