yangyangcv的OpenNI驱动玩隔空触摸源代码分析
原文网址在这：用openni的驱动玩隔空触摸
yangyangcv主要做了三个步奏：
1）深度数据做个阈值；
2）blob tracking；
3）TUIO发出去，然后就可以享用各种多点触摸的上层软件了。
当然，我们分析代码最多就到第二步，因为第三步明显上层软件是要另外编的。。。他的源代码可以到这里下载
先上yangyangcv的视频，很炫啊
由于好长时间没用OpenNI1和NiTE1了（OpenNI1和NiTE1被我卸了改OpenNI2和NiTE2了），里面的函数竟然有点忘了，再加上配置CCV什么的，今天就没运行起来，明天我用OpenNI2和NiTE2再来改写一下，一定要把它搞出来。
好，我们先来看程序吧
ofxKinectOpenNI
主要是二值化深度图像的一些函数，利用的原理就是直接设置个深度范围，在这个范围内的东西会被二值化出来，看了一下，好像yangyangcv设置的是1000以内的东西会被二值化为1，也就是，如果你只将手伸进范围内，则手的区域为1即为白色，其他范围内的地方由于没有东西，显示为0，为黑色。
ofxNCoreVision
这是设置CCV的程序了，如果结合注释仔细看的话，以及merryken的CCV入门教程（二），是很容易看明白的，要善用ctrl+f啊
//ofxKinectOpenNI.h
/************************************************************************/ /* simple wrapper for kinect's openni driver * author: yangyang * date: 20110519 * welcome to visit my blog: www.cnblogs.com/yangyangcv */ /************************************************************************/ #ifndef OFXKINEOPENNI_H_INCLUDED #define OFXKINEOPENNI_H_INCLUDED #include "cv.h" #include <XnCppWrapper.h> class ofxKinectOpenNI { private: xn::Context mContext; xn::DepthGenerator mDepthGenerator; xn::ImageGenerator mImageGenerator; xn::DepthMetaData m_DepthMD; xn::ImageMetaData m_ImageMD; XnMapOutputMode mapMode; XnStatus eResult; int camWidth; int camHeight; IplImage *depthImageHeader, *rgbImageHeader; public: ofxKinectOpenNI(); ~ofxKinectOpenNI(); //初始化Kinect bool initKinect(int wid,int hei,double lowbound,double upbound); //设置深度阈值用来做二值化处理 void setUpBound(double bound); void setLowBound(double bound); //彩色图像和二值化图像输入流 bool grabFrame();//color image and bw image int getCamWidth(); int getCamHeight(); //二值化处理上下界 double upBound, lowBound;//up and low threshold for the depth data //二值化图像 IplImage *bwImage;//threshold on the depth date we can get this BW image //彩色图像 IplImage *rgbImage;//image from the rgb camera }; #endif
//ofxKinectOpenNI.cpp
#include "ofxKinectOpenNI.h" #include <stdio.h> ofxKinectOpenNI::ofxKinectOpenNI() { } //初始化图像 bool ofxKinectOpenNI::initKinect(int wid,int hei,double lowbound,double upbound) { //设置上下界 eResult = XN_STATUS_OK; upBound = upbound; lowBound = lowbound; //镜头宽和高 camWidth = wid; camHeight = hei; eResult = mContext.Init(); if (eResult != XN_STATUS_OK) { printf("Kinect Error in initializing Kinect!\n"); return false; } //创建深度图像 eResult = mDepthGenerator.Create( mContext ); if (eResult != XN_STATUS_OK) { printf("Kinect Error in creating depth generator!\n"); return false; } //创建彩色图像 eResult = mImageGenerator.Create( mContext ); if (eResult != XN_STATUS_OK) { printf("Kinect Error in creating image generator!\n"); return false; } //设置帧率等 mapMode.nXRes = camWidth; mapMode.nYRes = camHeight; mapMode.nFPS = 30; eResult = mDepthGenerator.SetMapOutputMode( mapMode ); if (eResult != XN_STATUS_OK) { printf("Kinect Error in setting depth mode!\n"); return false; } eResult = mImageGenerator.SetMapOutputMode( mapMode ); if (eResult != XN_STATUS_OK) { printf("Kinect Error in setting image mode!\n"); return false; } //自动调整角度（这个我还真没用过） eResult = mDepthGenerator.GetAlternativeViewPointCap().SetViewPoint( mImageGenerator ); if (eResult != XN_STATUS_OK) { printf("Kinect Error in setting alternative viewpoint!\n"); return false; } //开始 eResult = mContext.StartGeneratingAll(); if (eResult != XN_STATUS_OK) { printf("Kinect Error in startGeneratingAll!\n"); return false; } //depth image depthImageHeader = cvCreateImageHeader(cvSize(camWidth,camHeight),IPL_DEPTH_16U,1); rgbImageHeader = cvCreateImageHeader(cvSize(camWidth,camHeight),IPL_DEPTH_8U,3); //RGB image rgbImage = cvCreateImage(cvSize(camWidth,camHeight),IPL_DEPTH_8U,3); //BW image bwImage = cvCreateImage(cvSize(camWidth,camHeight),IPL_DEPTH_8U,1); return true; } void ofxKinectOpenNI::setLowBound(double bound) { lowBound = bound; } void ofxKinectOpenNI::setUpBound(double bound) { upBound = bound; } bool ofxKinectOpenNI::grabFrame() { eResult = mContext.WaitNoneUpdateAll( ); if (eResult != XN_STATUS_OK) { printf("Kinect Error in WaitNoneUpdateAll!\n"); return false; } mDepthGenerator.GetMetaData(m_DepthMD); cvSetData(depthImageHeader,(USHORT*)m_DepthMD.Data(),depthImageHeader->widthStep); //we use cvInRangeS() to do the thresholding //cvInRangeS()可用于检查图像中的像素灰度是否属于某一指定范围，下面的代码就是将深度图像转换为二值图像 cvInRangeS( depthImageHeader, cvScalar(lowBound), cvScalar(upBound), bwImage ); mImageGenerator.GetMetaData(m_ImageMD); cvSetData(rgbImageHeader,(uchar*)m_ImageMD.Data(),rgbImageHeader->widthStep); cvCopy(rgbImageHeader,rgbImage); return true; } int ofxKinectOpenNI::getCamWidth() { return camWidth; } int ofxKinectOpenNI::getCamHeight() { return camHeight; } ofxKinectOpenNI::~ofxKinectOpenNI() { mContext.StopGeneratingAll(); mContext.Shutdown(); cvReleaseImageHeader(&rgbImageHeader); cvReleaseImageHeader(&depthImageHeader); cvReleaseImageHeader(&bwImage); cvReleaseImage(&rgbImage); }
//ofxNCoreVision.h
/* * ofxNCoreVision.h * NUI Group Community Core Vision * * Created by NUI Group Dev Team A on 2/1/09. * Copyright 2009 NUI Group/Inc. All rights reserved. * */ #ifndef _ofxNCoreVision_H #define _ofxNCoreVision_H //Main #include "ofMain.h" //Addons #ifdef TARGET_WIN32 #include "ofxffmv.h" #include "ofxKinectOpenNI.h"//my addon for kinect #include "ofxPS3.h" #include "ofxDSVL.h" #endif #include "ofxOpenCv.h" #include "ofxDirList.h" #include "ofxVectorMath.h" #include "ofxNetwork.h" #include "ofxOsc.h" #include "ofxThread.h" #include "ofxXmlSettings.h" //Our Addon #include "ofxNCore.h" //height and width of the source/tracked draw window #define MAIN_WINDOW_HEIGHT 240.0f #define MAIN_WINDOW_WIDTH 320.0f class ofxNCoreVision : public ofxGuiListener//, public BlobManager { //ofxGUI setup stuff enum { propertiesPanel, propertiesPanel_flipV, propertiesPanel_flipH, propertiesPanel_settings, propertiesPanel_pressure, gpuPanel, gpuPanel_use, optionPanel, optionPanel_tuio_osc, optionPanel_tuio_tcp, optionPanel_tuio_height_width, calibrationPanel, calibrationPanel_calibrate, calibrationPanel_warp, sourcePanel, sourcePanel_cam, sourcePanel_nextCam, sourcePanel_previousCam, sourcePanel_video, backgroundPanel, backgroundPanel_remove, backgroundPanel_dynamic, backgroundPanel_learn_rate, smoothPanel, smoothPanel_use, smoothPanel_smooth, amplifyPanel, amplifyPanel_use, amplifyPanel_amp, highpassPanel, highpassPanel_use, highpassPanel_blur, highpassPanel_noise, trackedPanel, trackedPanel_darkblobs, trackedPanel_use, trackedPanel_threshold, trackedPanel_min_movement, trackedPanel_min_blob_size, trackedPanel_max_blob_size, trackedPanel_outlines, trackedPanel_ids, savePanel, kParameter_SaveXml, kParameter_File, }; public: ofxNCoreVision() { ofAddListener(ofEvents.mousePressed, this, &ofxNCoreVision::_mousePressed); ofAddListener(ofEvents.mouseDragged, this, &ofxNCoreVision::_mouseDragged); ofAddListener(ofEvents.mouseReleased, this, &ofxNCoreVision::_mouseReleased); ofAddListener(ofEvents.keyPressed, this, &ofxNCoreVision::_keyPressed); ofAddListener(ofEvents.keyReleased, this, &ofxNCoreVision::_keyReleased); ofAddListener(ofEvents.setup, this, &ofxNCoreVision::_setup); ofAddListener(ofEvents.update, this, &ofxNCoreVision::_update); ofAddListener(ofEvents.draw, this, &ofxNCoreVision::_draw); ofAddListener(ofEvents.exit, this, &ofxNCoreVision::_exit); exited=false; #ifdef TARGET_WIN32 PS3 = NULL; ffmv = NULL; kinect = NULL; dsvl = NULL; #endif vidGrabber = NULL; vidPlayer = NULL; //initialize filter filter = NULL; //fps and dsp calculation frames = 0; fps = 0; lastFPSlog = 0; differenceTime = 0; //bools bCalibration= 0; bFullscreen = 0; bcamera = 0; bKinect = 0; bShowLabels = 1; bMiniMode = 0; bDrawOutlines = 1; bGPUMode = 0; bTUIOMode = 0; showConfiguration = 0; printfToFile = 0; //camera camRate = 30; camWidth = 320; camHeight = 240; //ints/floats backgroundLearnRate = .01; MIN_BLOB_SIZE = 2; MAX_BLOB_SIZE = 100; //if auto tracker is defined then the tracker automagically comes up //on startup.. kinectLowbound = 0; kinectUpbound = 1000; #ifdef STANDALONE bStandaloneMode = true; #else bStandaloneMode = false; #endif } ~ofxNCoreVision() { if( filter != NULL ) { delete filter; } if( vidGrabber != NULL ) { delete vidGrabber; } if( vidPlayer != NULL ) { delete vidPlayer; } #ifdef TARGET_WIN32 if( dsvl != NULL ) { delete dsvl; } if( ffmv != NULL ) { delete ffmv; } if( kinect != NULL ) { delete kinect; } #endif } /**************************************************************** * Public functions ****************************************************************/ //Basic Events called every frame void _setup(ofEventArgs &e); void _update(ofEventArgs &e); void _draw(ofEventArgs &e); void _exit(ofEventArgs &e); //Mouse Events void _mousePressed(ofMouseEventArgs &e); void _mouseDragged(ofMouseEventArgs &e); void _mouseReleased(ofMouseEventArgs &e); //Key Events void _keyPressed(ofKeyEventArgs &e); void _keyReleased(ofKeyEventArgs &e); //GUI void setupControls(); void handleGui(int parameterId, int task, void* data, int length); ofxGui* controls; //image processing stuff void initDevice(); void getPixels(); void grabFrameToCPU(); void grabFrameToGPU(GLuint target); //drawing void drawFingerOutlines(); void drawMiniMode(); void drawFullMode(); //Load/save settings void loadXMLSettings(); void saveSettings(); //Getters std::map<int, Blob> getBlobs(); /*************************************************************** * Video Capture Devices ***************************************************************/ #ifdef TARGET_WIN32 ofxffmv* ffmv; //for firefly mv ofxKinectOpenNI* kinect; ofxPS3* PS3; //for ps3 ofxDSVL* dsvl; #endif ofVideoGrabber* vidGrabber; ofVideoPlayer* vidPlayer; ofxCvColorImage kinectColorImg; /**************************************************************** * Variables in config.xml Settings file *****************************************************************/ int deviceID; int frameseq; int threshold; int wobbleThreshold; int camRate; int camWidth; int camHeight; int winWidth; int winHeight; int MIN_BLOB_SIZE; int MAX_BLOB_SIZE; float backgroundLearnRate; bool showConfiguration; bool printfToFile; bool bcamera; bool bKinect; double kinectLowbound,kinectUpbound; bool bMiniMode; bool bShowInterface; bool bShowPressure; bool bDrawOutlines; bool bTUIOMode; bool bFullscreen; bool bCalibration; bool bShowLabels; bool bNewFrame; //filters bool bAutoBackground; //modes bool bGPUMode; //auto ~ standalone/non-addon bool bStandaloneMode; //exit bool exited; /**************************************************** *End config.xml variables *****************************************************/ //FPS variables int frames; int fps; float lastFPSlog; int differenceTime; //Fonts ofTrueTypeFont verdana; ofTrueTypeFont sidebarTXT; ofTrueTypeFont bigvideo; //Images ofImage background; //Blob Tracker BlobTracker tracker; /**************************************************************** * Private Stuff ****************************************************************/ string videoFileName; int maxBlobs; //Calibration Calibration calib; //Blob Finder ContourFinder contourFinder; //Image filters Filters* filter; CPUImageFilter processedImg; ofxCvColorImage sourceImg; //XML Settings Vars ofxXmlSettings XML; string message; //Communication TUIO myTUIO; string tmpLocalHost; int tmpPort; int tmpFlashPort; //Logging char dateStr [9]; char timeStr [9]; time_t rawtime; struct tm * timeinfo; char fileName [80]; FILE * stream ; }; #endif
//ofxNCoreVision.cpp
/* * ofxNCoreVision.cpp * NUI Group Community Core Vision * * Created by NUI Group Dev Team A on 2/1/09. * Copyright 2009 NUI Group. All rights reserved. * */ #include "ofxNCoreVision.h" #include "../Controls/gui.h" /****************************************************************************** * The setup function is run once to perform initializations in the application *****************************************************************************/ void ofxNCoreVision::_setup(ofEventArgs &e) { //set the title ofSetWindowTitle(" Community Core Vision "); //create filter if ( filter == NULL ){filter = new ProcessFilters();} //Load Settings from config.xml file loadXMLSettings(); //removes the 'x' button on windows which causes a crash due to a GLUT bug #ifdef TARGET_WIN32 //Get rid of 'x' button HWND hwndConsole = FindWindowA(NULL, " Community Core Vision "); HMENU hMnu = ::GetSystemMenu(hwndConsole, FALSE); RemoveMenu(hMnu, SC_CLOSE, MF_BYCOMMAND); #endif if(printfToFile) { /***************************************************************************************************** * LOGGING ******************************************************************************************************/ /* alright first we need to get time and date so our logs can be ordered */ time ( &rawtime ); timeinfo = localtime ( &rawtime ); strftime (fileName,80,"../logs/log_%B_%d_%y_%H_%M_%S.txt",timeinfo); FILE *stream ; sprintf(fileName, ofToDataPath(fileName).c_str()); if((stream = freopen(fileName, "w", stdout)) == NULL){} /******************************************************************************************************/ } printf("printfToFile %i!\n", printfToFile); //Setup Window Properties ofSetWindowShape(winWidth,winHeight); ofSetVerticalSync(false); //Set vertical sync to false for better performance? printf("freedom?"); //load camera/video initDevice(); printf("freedom2?"); //set framerate ofSetFrameRate(camRate * 1.3); //This will be based on camera fps in the future /***************************************************************************************************** * Allocate images (needed for drawing/processing images) ******************************************************************************************************/ processedImg.allocate(camWidth, camHeight); //main Image that'll be processed. processedImg.setUseTexture(false); //We don't need to draw this so don't create a texture sourceImg.allocate(camWidth, camHeight); //Source Image sourceImg.setUseTexture(false); //We don't need to draw this so don't create a texture /******************************************************************************************************/ //Fonts - Is there a way to dynamically change font size? verdana.loadFont("verdana.ttf", 8, true, true); //Font used for small images bigvideo.loadFont("verdana.ttf", 13, true, true); //Font used for big images. //Static Images background.loadImage("images/background.jpg"); //Main (Temp?) Background printf("freedom3?"); //GUI Controls controls = ofxGui::Instance(this); setupControls(); printf("freedom4?"); //Setup Calibration calib.setup(camWidth, camHeight, &tracker); //Allocate Filters filter->allocate( camWidth, camHeight ); kinectColorImg.allocate(camWidth, camHeight); /***************************************************************************************************** * Startup Modes ******************************************************************************************************/ //If Standalone Mode (not an addon) if (bStandaloneMode) { printf("Starting in standalone mode...\n\n"); showConfiguration = true; } if (bMiniMode) { showConfiguration = true; bShowInterface = false; printf("Starting in Mini Mode...\n\n"); ofSetWindowShape(190, 200); //minimized size filter->bMiniMode = bMiniMode; } else{ bShowInterface = true; printf("Starting in full mode...\n\n"); } #ifdef TARGET_WIN32 //get rid of the console window FreeConsole(); #endif printf("Community Core Vision is setup!\n\n"); } /**************************************************************** * Load/Save config.xml file Settings ****************************************************************/ void ofxNCoreVision::loadXMLSettings() { // TODO: a seperate XML to map keyboard commands to action message = "Loading config.xml..."; // Can this load via http? if ( XML.loadFile("config.xml")) message = "Settings Loaded!\n\n"; else message = "No Settings Found...\n\n"; //FAIL //-------------------------------------------------------------- // START BINDING XML TO VARS //-------------------------------------------------------------- winWidth = XML.getValue("CONFIG:WINDOW:WIDTH", 950); winHeight = XML.getValue("CONFIG:WINDOW:HEIGHT", 600); bcamera = XML.getValue("CONFIG:CAMERA_0:USECAMERA", 1); //Kinect settings bKinect = XML.getValue("CONFIG:KINECT_0:USEKINECT", 1); kinectLowbound = XML.getValue("CONFIG:KINECT_0:LOWBOUND",400); kinectUpbound = XML.getValue("CONFIG:KINECT_0:UPBOUND",1000); deviceID = XML.getValue("CONFIG:CAMERA_0:DEVICE", 0); camWidth = XML.getValue("CONFIG:CAMERA_0:WIDTH", 320); camHeight = XML.getValue("CONFIG:CAMERA_0:HEIGHT", 240); camRate = XML.getValue("CONFIG:CAMERA_0:FRAMERATE", 0); videoFileName = XML.getValue("CONFIG:VIDEO:FILENAME", "test_videos/RearDI.m4v"); maxBlobs = XML.getValue("CONFIG:BLOBS:MAXNUMBER", 20); bShowLabels = XML.getValue("CONFIG:BOOLEAN:LABELS",0); bDrawOutlines = XML.getValue("CONFIG:BOOLEAN:OUTLINES",0); filter->bLearnBakground = XML.getValue("CONFIG:BOOLEAN:LEARNBG",0); filter->bVerticalMirror = XML.getValue("CONFIG:BOOLEAN:VMIRROR",0); filter->bHorizontalMirror = XML.getValue("CONFIG:BOOLEAN:HMIRROR",0); //Logging printfToFile = XML.getValue("CONFIG:BOOLEAN:PRINTFTOFILE",0); //Filters filter->bTrackDark = XML.getValue("CONFIG:BOOLEAN:TRACKDARK", 0); filter->bHighpass = XML.getValue("CONFIG:BOOLEAN:HIGHPASS",1); filter->bAmplify = XML.getValue("CONFIG:BOOLEAN:AMPLIFY", 1); filter->bSmooth = XML.getValue("CONFIG:BOOLEAN:SMOOTH", 1); filter->bDynamicBG = XML.getValue("CONFIG:BOOLEAN:DYNAMICBG", 1); //MODES bGPUMode = XML.getValue("CONFIG:BOOLEAN:GPU", 0); bMiniMode = XML.getValue("CONFIG:BOOLEAN:MINIMODE",0); //CONTROLS tracker.MIN_MOVEMENT_THRESHOLD = XML.getValue("CONFIG:INT:MINMOVEMENT",0); MIN_BLOB_SIZE = XML.getValue("CONFIG:INT:MINBLOBSIZE",2); MAX_BLOB_SIZE = XML.getValue("CONFIG:INT:MAXBLOBSIZE",100); backgroundLearnRate = XML.getValue("CONFIG:INT:BGLEARNRATE", 0.01f); //Filter Settings filter->threshold = XML.getValue("CONFIG:INT:THRESHOLD",0); filter->highpassBlur = XML.getValue("CONFIG:INT:HIGHPASSBLUR",0); filter->highpassNoise = XML.getValue("CONFIG:INT:HIGHPASSNOISE",0); filter->highpassAmp = XML.getValue("CONFIG:INT:HIGHPASSAMP",0); filter->smooth = XML.getValue("CONFIG:INT:SMOOTH",0); //NETWORK SETTINGS bTUIOMode = XML.getValue("CONFIG:BOOLEAN:TUIO",0); myTUIO.bOSCMode = XML.getValue("CONFIG:BOOLEAN:OSCMODE",1); myTUIO.bTCPMode = XML.getValue("CONFIG:BOOLEAN:TCPMODE",1); myTUIO.bHeightWidth = XML.getValue("CONFIG:BOOLEAN:HEIGHTWIDTH",0); tmpLocalHost = XML.getValue("CONFIG:NETWORK:LOCALHOST", "localhost"); tmpPort = XML.getValue("CONFIG:NETWORK:TUIOPORT_OUT", 3333); tmpFlashPort = XML.getValue("CONFIG:NETWORK:TUIOFLASHPORT_OUT", 3000); myTUIO.setup(tmpLocalHost.c_str(), tmpPort, tmpFlashPort); //have to convert tmpLocalHost to a const char* //-------------------------------------------------------------- // END XML SETUP } void ofxNCoreVision::saveSettings() { XML.setValue("CONFIG:KINECT_0:USEKINECT", bKinect); XML.setValue("CONFIG:KINECT_0:LOWBOUND",kinectLowbound); XML.setValue("CONFIG:KINECT_0:UPBOUND",kinectUpbound); XML.setValue("CONFIG:CAMERA_0:USECAMERA", bcamera); XML.setValue("CONFIG:CAMERA_0:DEVICE", deviceID); XML.setValue("CONFIG:CAMERA_0:WIDTH", camWidth); XML.setValue("CONFIG:CAMERA_0:HEIGHT", camHeight); XML.setValue("CONFIG:CAMERA_0:FRAMERATE", camRate); XML.setValue("CONFIG:BOOLEAN:PRESSURE",bShowPressure); XML.setValue("CONFIG:BOOLEAN:LABELS",bShowLabels); XML.setValue("CONFIG:BOOLEAN:OUTLINES",bDrawOutlines); XML.setValue("CONFIG:BOOLEAN:LEARNBG", filter->bLearnBakground); XML.setValue("CONFIG:BOOLEAN:VMIRROR", filter->bVerticalMirror); XML.setValue("CONFIG:BOOLEAN:HMIRROR", filter->bHorizontalMirror); XML.setValue("CONFIG:BOOLEAN:PRINTFTOFILE", printfToFile); XML.setValue("CONFIG:BOOLEAN:TRACKDARK", filter->bTrackDark); XML.setValue("CONFIG:BOOLEAN:HIGHPASS", filter->bHighpass); XML.setValue("CONFIG:BOOLEAN:AMPLIFY", filter->bAmplify); XML.setValue("CONFIG:BOOLEAN:SMOOTH", filter->bSmooth); XML.setValue("CONFIG:BOOLEAN:DYNAMICBG", filter->bDynamicBG); XML.setValue("CONFIG:BOOLEAN:GPU", bGPUMode); XML.setValue("CONFIG:INT:MINMOVEMENT", tracker.MIN_MOVEMENT_THRESHOLD); XML.setValue("CONFIG:INT:MINBLOBSIZE", MIN_BLOB_SIZE); XML.setValue("CONFIG:INT:MAXBLOBSIZE", MAX_BLOB_SIZE); XML.setValue("CONFIG:INT:BGLEARNRATE", backgroundLearnRate); XML.setValue("CONFIG:INT:THRESHOLD", filter->threshold); XML.setValue("CONFIG:INT:HIGHPASSBLUR", filter->highpassBlur); XML.setValue("CONFIG:INT:HIGHPASSNOISE", filter->highpassNoise); XML.setValue("CONFIG:INT:HIGHPASSAMP", filter->highpassAmp); XML.setValue("CONFIG:INT:SMOOTH", filter->smooth); XML.setValue("CONFIG:BOOLEAN:MINIMODE", bMiniMode); XML.setValue("CONFIG:BOOLEAN:TUIO",bTUIOMode); XML.setValue("CONFIG:BOOLEAN:HEIGHTWIDTH", myTUIO.bHeightWidth); XML.setValue("CONFIG:BOOLEAN:OSCMODE", myTUIO.bOSCMode); XML.setValue("CONFIG:BOOLEAN:TCPMODE", myTUIO.bTCPMode); // XML.setValue("CONFIG:NETWORK:LOCALHOST", myTUIO.localHost); // XML.setValue("CONFIG:NETWORK:TUIO_PORT_OUT",myTUIO.TUIOPort); XML.saveFile("config.xml"); } /************************************************ * Init Device ************************************************/ //Init Device (camera/video) void ofxNCoreVision::initDevice(){ //save/update log file if(printfToFile) if((stream = freopen(fileName, "a", stdout)) == NULL){} //Pick the Source - camera or video if (bKinect) { kinect = new ofxKinectOpenNI(); if (kinect->initKinect(camWidth,camHeight,kinectLowbound,kinectUpbound)) { printf("Kinect Camera Mode\nAsked for %i by %i - actual size is %i by %i \n\n", camWidth, camHeight, kinect->getCamWidth(), kinect->getCamHeight()); camWidth = kinect->getCamWidth(); camHeight = kinect->getCamHeight(); } else printf("Error when initializing kinect!!!"); } else { if (bcamera) { //check if a firefly, ps3 camera, or other is plugged in #ifdef TARGET_WIN32 /****PS3 - PS3 camera only****/ if(ofxPS3::getDeviceCount() > 0 && PS3 == NULL){ PS3 = new ofxPS3(); PS3->listDevices(); PS3->initPS3(camWidth, camHeight, camRate); camWidth = PS3->getCamWidth(); camHeight = PS3->getCamHeight(); printf("Camera Mode\nAsked for %i by %i - actual size is %i by %i \n\n", camWidth, camHeight, PS3->getCamWidth(), PS3->getCamHeight()); return; } /****ffmv - firefly camera only****/ else if(ofxffmv::getDeviceCount() > 0 && ffmv == NULL){ ffmv = new ofxffmv(); ffmv->listDevices(); ffmv->initFFMV(camWidth,camHeight); printf("Camera Mode\nAsked for %i by %i - actual size is %i by %i \n\n", camWidth, camHeight, ffmv->getCamWidth(), ffmv->getCamHeight()); camWidth = ffmv->getCamWidth(); camHeight = ffmv->getCamHeight(); return; } else if( vidGrabber == NULL ) { vidGrabber = new ofVideoGrabber(); vidGrabber->listDevices(); vidGrabber->setVerbose(true); vidGrabber->initGrabber(camWidth,camHeight); printf("Camera Mode\nAsked for %i by %i - actual size is %i by %i \n\n", camWidth, camHeight, vidGrabber->width, vidGrabber->height); camWidth = vidGrabber->width; camHeight = vidGrabber->height; return; } else if( dsvl == NULL) { dsvl = new ofxDSVL(); dsvl->initDSVL(); printf("Camera Mode\nAsked for %i by %i - actual size is %i by %i \n\n", camWidth, camHeight, dsvl->getCamWidth(), dsvl->getCamHeight()); camWidth = dsvl->getCamWidth(); camHeight = dsvl->getCamHeight(); return; } #else if( vidGrabber == NULL ) { vidGrabber = new ofVideoGrabber(); vidGrabber->listDevices(); vidGrabber->setVerbose(true); vidGrabber->initGrabber(camWidth,camHeight); printf("Camera Mode\nAsked for %i by %i - actual size is %i by %i \n\n", camWidth, camHeight, vidGrabber->width, vidGrabber->height); camWidth = vidGrabber->width; camHeight = vidGrabber->height; return; } #endif }else{ if( vidPlayer == NULL ) { vidPlayer = new ofVideoPlayer(); vidPlayer->loadMovie( videoFileName ); vidPlayer->play(); vidPlayer->setLoopState(OF_LOOP_NORMAL); printf("Video Mode\n\n"); camHeight = vidPlayer->height; camWidth = vidPlayer->width; return; } } } } /****************************************************************************** * The update function runs continuously. Use it to update states and variables *****************************************************************************/ void ofxNCoreVision::_update(ofEventArgs &e) { //save/update log file if(printfToFile) if((stream = freopen(fileName, "a", stdout)) == NULL){} if(exited) return; bNewFrame = false; if (bcamera) //if camera { #ifdef TARGET_WIN32 if(PS3!=NULL)//ps3 camera { bNewFrame = PS3->isFrameNew(); } else if(ffmv!=NULL) { ffmv->grabFrame(); bNewFrame = true; } else if (kinect != NULL) { bNewFrame = kinect->grabFrame(); } else if(vidGrabber !=NULL) { vidGrabber->grabFrame(); bNewFrame = vidGrabber->isFrameNew(); } else if(dsvl !=NULL) { bNewFrame = dsvl->isFrameNew(); } #else vidGrabber->grabFrame(); bNewFrame = vidGrabber->isFrameNew(); #endif } else //if video { vidPlayer->idleMovie(); bNewFrame = vidPlayer->isFrameNew(); } //if no new frame, return if(!bNewFrame){ return; } else//else process camera frame { ofBackground(0, 0, 0); // Calculate FPS of Camera frames++; float time = ofGetElapsedTimeMillis(); if (time > (lastFPSlog + 1000)) { fps = frames; frames = 0; lastFPSlog = time; }//End calculation float beforeTime = ofGetElapsedTimeMillis(); if (bGPUMode) { grabFrameToGPU(filter->gpuSourceTex); filter->applyGPUFilters(); contourFinder.findContours(filter->gpuReadBackImageGS, (MIN_BLOB_SIZE * 2) + 1, ((camWidth * camHeight) * .4) * (MAX_BLOB_SIZE * .001), maxBlobs, false); } else { grabFrameToCPU(); filter->applyCPUFilters( processedImg ); contourFinder.findContours(processedImg, (MIN_BLOB_SIZE * 2) + 1, ((camWidth * camHeight) * .4) * (MAX_BLOB_SIZE * .001), maxBlobs, false); } //Track found contours/blobss tracker.track(&contourFinder); //get DSP time differenceTime = ofGetElapsedTimeMillis() - beforeTime; //Dynamic Background subtraction LearRate if (filter->bDynamicBG) { filter->fLearnRate = backgroundLearnRate * .0001; //If there are no blobs, add the background faster. if (contourFinder.nBlobs > 0) //If there ARE blobs, add the background slower. { filter->fLearnRate = backgroundLearnRate * .0001; } }//End Background Learning rate if (bTUIOMode) { //Start sending OSC myTUIO.sendTUIO(&getBlobs()); } } } /************************************************ * Input Device Stuff ************************************************/ //get pixels from camera void ofxNCoreVision::getPixels(){ #ifdef TARGET_WIN32 if(PS3!=NULL){ sourceImg.setFromPixels(PS3->getPixels(), camWidth, camHeight); //convert to grayscale processedImg = sourceImg; } else if(ffmv != NULL){ processedImg.setFromPixels(ffmv->fcImage[ffmv->getDeviceID()].pData, camWidth, camHeight); } else if(kinect != NULL) { processedImg.setFromPixels((uchar*)(kinect->bwImage->imageData), camWidth, camHeight); if (filter->bVerticalMirror) { cvFlip(kinect->rgbImage,kinect->rgbImage,0); } if (filter->bHorizontalMirror) { cvFlip(kinect->rgbImage,kinect->rgbImage,1); } kinectColorImg = kinect->rgbImage; } else if(vidGrabber != NULL ) { sourceImg.setFromPixels(vidGrabber->getPixels(), camWidth, camHeight); //convert to grayscale processedImg = sourceImg; } else if(dsvl!=NULL) { if(dsvl->getNumByes() != 1){ //if not grayscale sourceImg.setFromPixels(dsvl->getPixels(), camWidth, camHeight); //convert to grayscale processedImg = sourceImg; } else{//if grayscale processedImg.setFromPixels(dsvl->getPixels(), camWidth, camHeight); } } #endif } //Grab frame from CPU void ofxNCoreVision::grabFrameToCPU() { //Set sourceImg as new camera/video frame if (bcamera) { #ifdef TARGET_WIN32 getPixels(); #else sourceImg.setFromPixels(vidGrabber->getPixels(), camWidth, camHeight); //convert to grayscale processedImg = sourceImg; #endif } else { sourceImg.setFromPixels(vidPlayer->getPixels(), camWidth, camHeight); //convert to grayscale processedImg = sourceImg; } } //Grab frame from GPU void ofxNCoreVision::grabFrameToGPU(GLuint target) { //grab the frame to a raw openGL texture if (bcamera) { glEnable(GL_TEXTURE_2D); //glPixelStorei(1); glBindTexture(GL_TEXTURE_2D, target); #ifdef TARGET_WIN32 if(PS3!=NULL) { glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, camWidth, camHeight, GL_RGB, GL_UNSIGNED_BYTE, PS3->getPixels()); } else if(vidGrabber!=NULL) { glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, camWidth, camHeight, GL_RGB, GL_UNSIGNED_BYTE, vidGrabber->getPixels()); } else if(dsvl!=NULL) { glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, camWidth, camHeight, GL_RGB, GL_UNSIGNED_BYTE, dsvl->getPixels()); } #else glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, camWidth, camHeight, GL_RGB, GL_UNSIGNED_BYTE, vidGrabber->getPixels()); #endif glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); glBindTexture(GL_TEXTURE_2D,0); } else { glEnable(GL_TEXTURE_2D); glBindTexture(GL_TEXTURE_2D, target); glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, camWidth, camHeight, GL_RGB, GL_UNSIGNED_BYTE, vidPlayer->getPixels()); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); glBindTexture(GL_TEXTURE_2D,0); } } /****************************************************************************** * The draw function paints the textures onto the screen. It runs after update. *****************************************************************************/ void ofxNCoreVision::_draw(ofEventArgs &e) { if(exited) return; if (showConfiguration) { //if calibration if (bCalibration) { //Don't draw main interface calib.passInContourFinder(contourFinder.nBlobs, contourFinder.blobs); calib.doCalibration(); } //if mini mode else if (bMiniMode) { drawMiniMode(); } //if full mode else if (bShowInterface) { drawFullMode(); if(bDrawOutlines || bShowLabels) drawFingerOutlines(); } //draw gui controls if (!bCalibration && !bMiniMode) {controls->draw();} } } void ofxNCoreVision::drawFullMode(){ ofSetColor(0xFFFFFF); //Draw Background Image background.draw(0, 0); //Draw arrows ofSetColor(187, 200, 203); ofFill(); ofTriangle(680, 420, 680, 460, 700, 440); ofTriangle(70, 420, 70, 460, 50, 440); ofSetColor(255, 255, 0); ofSetColor(0xFFFFFF); //Draw Image Filters To Screen if (bGPUMode) filter->drawGPU(); else filter->draw(); if (kinect != NULL) { kinectColorImg.draw(40, 30, 320, 240); } ofSetColor(0x000000); if (bShowPressure) { bigvideo.drawString("Pressure Map", 140, 20); } else { bigvideo.drawString("Source Image", 140, 20); } bigvideo.drawString("Tracked Image", 475, 20); //draw link to tbeta website ofSetColor(79, 79, 79); ofFill(); ofRect(721, 586, 228, 14); ofSetColor(0xFFFFFF); ofDrawBitmapString("| ~ |tbeta.nuigroup.com", 725, 596); //Display Application information in bottom right string str = "Calc. Time [ms]: "; str+= ofToString(differenceTime, 0)+"\n\n"; if (bcamera) { string str2 = "Camera [Res]: "; str2+= ofToString(camWidth, 0) + " x " + ofToString(camHeight, 0) + "\n"; string str4 = "Camera [fps]: "; str4+= ofToString(fps, 0)+"\n"; ofSetColor(0xFFFFFF); verdana.drawString(str + str2 + str4, 740, 410); } else { string str2 = "Video [Res]: "; str2+= ofToString(vidPlayer->width, 0) + " x " + ofToString(vidPlayer->height, 0) + "\n"; string str4 = "Video [fps]: "; str4+= ofToString(fps, 0)+"\n"; ofSetColor(0xFFFFFF); verdana.drawString(str + str2 + str4, 740, 410); } if (bTUIOMode) { //Draw Port and IP to screen ofSetColor(0xffffff); char buf[256]; if(myTUIO.bOSCMode) sprintf(buf, "Sending OSC messages to:\nHost: %s\nPort: %i", myTUIO.localHost, myTUIO.TUIOPort); else{ if(myTUIO.bIsConnected) sprintf(buf, "Sending TCP messages to:\nPort: %i", myTUIO.TUIOFlashPort); else sprintf(buf, "Could not bind or send TCP to:\nPort: %i", myTUIO.TUIOFlashPort); } verdana.drawString(buf, 740, 480); } ofSetColor(0xFF0000); verdana.drawString("Press spacebar to toggle fast mode", 730, 572); } void ofxNCoreVision::drawMiniMode() { //black background ofSetColor(0,0,0); ofRect(0,0,ofGetWidth(), ofGetHeight()); //draw outlines if (bDrawOutlines){ for (int i=0; i<contourFinder.nBlobs; i++) { contourFinder.blobs[i].drawContours(0,0, camWidth, camHeight+175, ofGetWidth(), ofGetHeight()); } } //draw grey rectagles for text information ofSetColor(128,128,128); ofFill(); ofRect(0,ofGetHeight() - 83, ofGetWidth(), 20); ofRect(0,ofGetHeight() - 62, ofGetWidth(), 20); ofRect(0,ofGetHeight() - 41, ofGetWidth(), 20); ofRect(0,ofGetHeight() - 20, ofGetWidth(), 20); //draw text ofSetColor(250,250,250); verdana.drawString("Calc. Time [ms]: " + ofToString(differenceTime,0),10, ofGetHeight() - 70 ); if (bcamera){ verdana.drawString("Camera [fps]: " + ofToString(fps,0),10, ofGetHeight() - 50 ); } else { verdana.drawString("Video [fps]: " + ofToString(fps,0),10, ofGetHeight() - 50 ); } verdana.drawString("Blob Count: " + ofToString(contourFinder.nBlobs,0),10, ofGetHeight() - 29 ); verdana.drawString("Sending TUIO: " ,10, ofGetHeight() - 9 ); //draw green tuio circle if((myTUIO.bIsConnected || myTUIO.bOSCMode) && bTUIOMode)//green = connected ofSetColor(0x00FF00); else ofSetColor(0xFF0000); //red = not connected ofFill(); ofCircle(ofGetWidth() - 17 , ofGetHeight() - 10, 5); ofNoFill(); } void ofxNCoreVision::drawFingerOutlines() { //Find the blobs for drawing for (int i=0; i<contourFinder.nBlobs; i++) { if (bDrawOutlines) { //Draw contours (outlines) on the source image contourFinder.blobs[i].drawContours(40, 30, camWidth, camHeight, MAIN_WINDOW_WIDTH, MAIN_WINDOW_HEIGHT); } if (bShowLabels) //Show ID label; { float xpos = contourFinder.blobs[i].centroid.x * (MAIN_WINDOW_WIDTH/camWidth); float ypos = contourFinder.blobs[i].centroid.y * (MAIN_WINDOW_HEIGHT/camHeight); ofSetColor(0xCCFFCC); char idStr[1024]; sprintf(idStr, "id: %i", contourFinder.blobs[i].id); verdana.drawString(idStr, xpos + 365, ypos + contourFinder.blobs[i].boundingRect.height/2 + 45); } } ofSetColor(0xFFFFFF); } /***************************************************************************** * KEY EVENTS *****************************************************************************/ void ofxNCoreVision::_keyPressed(ofKeyEventArgs &e) { // detect escape key if(e.key==0x1b) { exited=true; } if (showConfiguration) { switch (e.key) { case 'a': filter->threshold++; controls->update(appPtr->trackedPanel_threshold, kofxGui_Set_Int, &appPtr->filter->threshold, sizeof(int)); break; case 'z': filter->threshold--; controls->update(appPtr->trackedPanel_threshold, kofxGui_Set_Int, &appPtr->filter->threshold, sizeof(int)); break; case 'b': filter->bLearnBakground = true; break; case 'o': bDrawOutlines ? bDrawOutlines = false : bDrawOutlines = true; controls->update(appPtr->trackedPanel_outlines, kofxGui_Set_Bool, &appPtr->bDrawOutlines, sizeof(bool)); break; case 'h': filter->bHorizontalMirror ? filter->bHorizontalMirror = false : filter->bHorizontalMirror = true; controls->update(appPtr->propertiesPanel_flipH, kofxGui_Set_Bool, &appPtr->filter->bHorizontalMirror, sizeof(bool)); break; case 'j': filter->bVerticalMirror ? filter->bVerticalMirror = false : filter->bVerticalMirror = true; controls->update(appPtr->propertiesPanel_flipV, kofxGui_Set_Bool, &appPtr->filter->bVerticalMirror, sizeof(bool)); break; case 't': myTUIO.bOSCMode = !myTUIO.bOSCMode; myTUIO.bTCPMode = false; bTUIOMode = myTUIO.bOSCMode; controls->update(appPtr->optionPanel_tuio_tcp, kofxGui_Set_Bool, &appPtr->myTUIO.bTCPMode, sizeof(bool)); controls->update(appPtr->optionPanel_tuio_osc, kofxGui_Set_Bool, &appPtr->myTUIO.bOSCMode, sizeof(bool)); //clear blobs // myTUIO.blobs.clear(); break; case 'f': myTUIO.bOSCMode = false; myTUIO.bTCPMode = !myTUIO.bTCPMode; bTUIOMode = myTUIO.bTCPMode; controls->update(appPtr->optionPanel_tuio_tcp, kofxGui_Set_Bool, &appPtr->myTUIO.bTCPMode, sizeof(bool)); controls->update(appPtr->optionPanel_tuio_osc, kofxGui_Set_Bool, &appPtr->myTUIO.bOSCMode, sizeof(bool)); //clear blobs // myTUIO.blobs.clear(); break; case 'g': bGPUMode ? bGPUMode = false : bGPUMode = true; controls->update(appPtr->gpuPanel_use, kofxGui_Set_Bool, &appPtr->bGPUMode, sizeof(bool)); filter->bLearnBakground = true; break; case 'v': if (bcamera && vidGrabber != NULL) vidGrabber->videoSettings(); break; case 'l': bShowLabels ? bShowLabels = false : bShowLabels = true; controls->update(appPtr->trackedPanel_ids, kofxGui_Set_Bool, &appPtr->bShowLabels, sizeof(bool)); break; case 'p': bShowPressure ? bShowPressure = false : bShowPressure = true; break; case ' ': if (bMiniMode && !bCalibration) // NEED TO ADD HERE ONLY GO MINI MODE IF NOT CALIBRATING { bMiniMode = false; bShowInterface = true; filter->bMiniMode = bMiniMode; ofSetWindowShape(950,600); //default size } else if(!bCalibration) { bMiniMode = true; bShowInterface = false; filter->bMiniMode = bMiniMode; ofSetWindowShape(190,200); //minimized size } break; case 'x': //Exit Calibrating if (bCalibration) { bShowInterface = true; bCalibration = false; calib.calibrating = false; tracker.isCalibrating = false; if (bFullscreen == true) ofToggleFullscreen(); bFullscreen = false; } break; } } } void ofxNCoreVision::_keyReleased(ofKeyEventArgs &e) { if (showConfiguration) { if ( e.key == 'c' && !bCalibration) { bShowInterface = false; //Enter/Exit Calibration bCalibration = true; calib.calibrating = true; tracker.isCalibrating = true; if (bFullscreen == false) ofToggleFullscreen(); bFullscreen = true; } } if ( e.key == '~' || e.key == '`' && !bMiniMode && !bCalibration) showConfiguration = !showConfiguration; } /***************************************************************************** * MOUSE EVENTS *****************************************************************************/ void ofxNCoreVision::_mouseDragged(ofMouseEventArgs &e) { if (showConfiguration) controls->mouseDragged(e.x, e.y, e.button); //guilistener } void ofxNCoreVision::_mousePressed(ofMouseEventArgs &e) { if (showConfiguration) { controls->mousePressed(e.x, e.y, e.button); //guilistener if (e.x > 722 && e.y > 586){ofLaunchBrowser("http://tbeta.nuigroup.com");} } } void ofxNCoreVision::_mouseReleased(ofMouseEventArgs &e) { if (showConfiguration) controls->mouseReleased(e.x, e.y, 0); //guilistener } /***************************************************************************** * Getters *****************************************************************************/ std::map<int, Blob> ofxNCoreVision::getBlobs(){ return tracker.getTrackedBlobs(); } /***************************************************************************** * ON EXIT *****************************************************************************/ void ofxNCoreVision::_exit(ofEventArgs &e) { //save/update log file if((stream = freopen(fileName, "a", stdout)) == NULL){} saveSettings(); #ifdef TARGET_WIN32 if(PS3!=NULL) delete PS3; if(ffmv!=NULL) delete ffmv; if (kinect != NULL) delete kinect; if(dsvl!=NULL) delete dsvl; #endif if(vidGrabber!=NULL) delete vidGrabber; if(vidPlayer !=NULL) delete vidPlayer; // -------------------------------- SAVE STATE ON EXIT printf("Vision module has exited!\n"); }
