OpenGL应用：实现人脸识别并贴纸的功能
人脸识别贴纸
整个处理过程大致分为3个步骤：
1、使用AVFoundation调用摄像头采集视频流获得图像信息
2、使用CoreImage库判断采集到的图像信息中是否包含有人脸
3、将结果使用OpenGL渲染显示到屏幕上
一、调用摄像头采集视频
self.captureSession = [[AVCaptureSession alloc] init]; [self.captureSession setSessionPreset:AVCaptureSessionPresetHigh]; AVCaptureDevice *captureDevice = nil; NSArray *captureDevices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]; for (AVCaptureDevice *device in captureDevices) { if (device.position == AVCaptureDevicePositionBack) { captureDevice = device; break; } } self.captureDeviceInput = [[AVCaptureDeviceInput alloc] initWithDevice:captureDevice error:nil]; if ([self.captureSession canAddInput:self.captureDeviceInput]) { [self.captureSession addInput:self.captureDeviceInput]; } self.captureDeviceOutput = [[AVCaptureVideoDataOutput alloc] init]; [self.captureDeviceOutput setAlwaysDiscardsLateVideoFrames:YES]; processQueue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0); [self.captureDeviceOutput setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange] forKey:(id)kCVPixelBufferPixelFormatTypeKey]]; [self.captureDeviceOutput setSampleBufferDelegate:delegate queue:processQueue]; if ([self.captureSession canAddOutput:self.captureDeviceOutput]) { [self.captureSession addOutput:self.captureDeviceOutput]; } AVCaptureConnection *captureConnection = [self.captureDeviceOutput connectionWithMediaType:AVMediaTypeVideo]; [captureConnection setVideoOrientation:AVCaptureVideoOrientationPortrait]; [self.captureSession startRunning];
获得视频帧：
- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection { CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); [self.faceDetectionView displayPixelBuffer:pixelBuffer]; }
二、识别图像中的人脸
CIImage *ciImage = [[CIImage alloc] initWithCVPixelBuffer:pixelBuffer]; NSString *accuracy = CIDetectorAccuracyLow; NSDictionary *options = [NSDictionary dictionaryWithObject:accuracy forKey:CIDetectorAccuracy]; CIDetector *detector = [CIDetector detectorOfType:CIDetectorTypeFace context:nil options:options]; NSArray *featuresArray = [detector featuresInImage:ciImage options:nil];
得到的featuresArray便是识别的结果，是一个包含有CIFaceFeature对象的数组，我们可以使用获得的结果判断是否包含有人脸。
三、使用OpenGL渲染原始视频帧和人脸位置贴图
1.讲我们要使用的贴图图片转换成纹理数据，用于识别人脸后的纹理混合
CGImageRef spriteImage = [UIImage imageNamed:fileName].CGImage; if (!spriteImage) { NSLog(@"Failed to load image %@", fileName); exit(1); } size_t width = CGImageGetWidth(spriteImage); size_t height = CGImageGetHeight(spriteImage); GLubyte *spriteData = (GLubyte *)calloc(width * height * 4, sizeof(GLubyte)); CGContextRef context = CGBitmapContextCreate(spriteData, width, height, 8, width * 4, CGImageGetColorSpace(spriteImage), kCGImageAlphaPremultipliedLast); CGContextTranslateCTM(context, 0, height); CGContextScaleCTM (context, 1.0, -1.0); CGContextDrawImage(context, CGRectMake(0, 0, width, height), spriteImage); CGContextRelease(context); GLuint texture; glActiveTexture(GL_TEXTURE2); glGenTextures(1, &texture); glBindTexture(GL_TEXTURE_2D, texture); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR ); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR ); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, (int32_t)width, (int32_t)height, 0, GL_RGBA, GL_UNSIGNED_BYTE, spriteData); free(spriteData); return texture;
2.渲染视频帧，同时检测有没有人脸，如果有，计算出人脸位置，转换坐标，将贴图渲染上去。
- (void)displayPixelBuffer:(CVPixelBufferRef)pixelBuffer { if (pixelBuffer != NULL) { int width = (int)CVPixelBufferGetWidth(pixelBuffer); int height = (int)CVPixelBufferGetHeight(pixelBuffer); if (!_videoTextureCache) { NSLog(@"NO Video Texture Cache"); return; } if ([EAGLContext currentContext] != _context) { [EAGLContext setCurrentContext:_context]; } [self cleanUpTextures]; glActiveTexture(GL_TEXTURE0); CVReturn err; err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, _videoTextureCache, pixelBuffer, NULL, GL_TEXTURE_2D, GL_RED_EXT, width, height, GL_RED_EXT, GL_UNSIGNED_BYTE, 0, &_lumaTexture); if (err) { NSLog(@"Error at CVOpenGLESTextureCacheCreateTextureFromImage %d", err); } glBindTexture(CVOpenGLESTextureGetTarget(_lumaTexture), CVOpenGLESTextureGetName(_lumaTexture)); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); // UV-plane. glActiveTexture(GL_TEXTURE1); err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, _videoTextureCache, pixelBuffer, NULL, GL_TEXTURE_2D, GL_RG_EXT, width / 2, height / 2, GL_RG_EXT, GL_UNSIGNED_BYTE, 1, &_chromaTexture); if (err) { NSLog(@"Error at CVOpenGLESTextureCacheCreateTextureFromImage %d", err); } glBindTexture(CVOpenGLESTextureGetTarget(_chromaTexture), CVOpenGLESTextureGetName(_chromaTexture)); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glBindFramebuffer(GL_FRAMEBUFFER, _frameBuffer); glViewport(0, 0, _backingWidth, _backingHeight); } glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); glEnable(GL_BLEND); glClearColor(0, 0, 0, 1.0); glClear(GL_COLOR_BUFFER_BIT); glBindFramebuffer(GL_FRAMEBUFFER, _frameBuffer); [self.shaderManager useProgram]; glUniform1i(glViewUniforms[UNIFORM_Y], 0); glUniform1i(glViewUniforms[UNIFORM_UV], 1); glUniformMatrix4fv(glViewUniforms[UNIFORM_ROTATE_MATRIX], 1, GL_FALSE, GLKMatrix4MakeXRotation(M_PI).m); GLfloat quadVertexData[] = { -1, -1, 1, -1 , -1, 1, 1, 1, }; // 更新顶点数据 glVertexAttribPointer(glViewAttributes[ATTRIB_VERTEX], 2, GL_FLOAT, 0, 0, quadVertexData); glEnableVertexAttribArray(glViewAttributes[ATTRIB_VERTEX]); GLfloat quadTextureData[] = { // 正常坐标 0, 0, 1, 0, 0, 1, 1, 1 }; glVertexAttribPointer(glViewAttributes[ATTRIB_TEXCOORD], 2, GL_FLOAT, GL_FALSE, 0, quadTextureData); glEnableVertexAttribArray(glViewAttributes[ATTRIB_TEXCOORD]); glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); [LYFaceDetector detectCVPixelBuffer:pixelBuffer completionHandler:^(CIFaceFeature *result, CIImage *ciImage) { if (result) { [self renderTempTexture:result ciImage:ciImage]; } }]; glBindRenderbuffer(GL_RENDERBUFFER, _renderBuffer); if ([EAGLContext currentContext] == _context) { [_context presentRenderbuffer:GL_RENDERBUFFER]; } }
3.转换人脸坐标，计算需要渲染的贴图坐标
- (void)renderTempTexture:(CIFaceFeature *)faceFeature ciImage:(CIImage *)ciImage { dispatch_semaphore_wait(_lock, DISPATCH_TIME_FOREVER); //得到图片的尺寸 CGSize ciImageSize = [ciImage extent].size; //初始化transform CGAffineTransform transform = CGAffineTransformScale(CGAffineTransformIdentity, 1, -1); transform = CGAffineTransformTranslate(transform,0,-ciImageSize.height); // 实现坐标转换 CGSize viewSize =self.layer.bounds.size; CGFloat scale = MIN(viewSize.width / ciImageSize.width,viewSize.height / ciImageSize.height); CGFloat offsetX = (viewSize.width - ciImageSize.width * scale) / 2; CGFloat offsetY = (viewSize.height - ciImageSize.height * scale) / 2; // 缩放 CGAffineTransform scaleTransform = CGAffineTransformMakeScale(scale, scale); //获取人脸的frame CGRect faceViewBounds = CGRectApplyAffineTransform(faceFeature.bounds, transform); // 修正 faceViewBounds = CGRectApplyAffineTransform(faceViewBounds,scaleTransform); faceViewBounds.origin.x += offsetX; faceViewBounds.origin.y += offsetY; NSLog(@"face frame after:%@",NSStringFromCGRect(faceViewBounds)); [self.textureManager useProgram]; glBindTexture(GL_TEXTURE_2D, _myTexture); glUniform1i(glViewUniforms[UNIFORM_TEMP_INPUT_IMG_TEXTURE], 2); CGFloat midX = CGRectGetMidX(self.layer.bounds); CGFloat midY = CGRectGetMidY(self.layer.bounds); CGFloat originX = CGRectGetMinX(faceViewBounds); CGFloat originY = CGRectGetMinY(faceViewBounds); CGFloat maxX = CGRectGetMaxX(faceViewBounds); CGFloat maxY = CGRectGetMaxY(faceViewBounds); //贴图顶点 GLfloat minVertexX = (originX - midX) / midX; GLfloat minVertexY = (midY - maxY) / midY; GLfloat maxVertexX = (maxX - midX) / midX; GLfloat maxVertexY = (midY - originY) / midY; GLfloat quadData[] = { minVertexX, minVertexY, maxVertexX, minVertexY, minVertexX, maxVertexY, maxVertexX, maxVertexY, }; glVertexAttribPointer(glViewAttributes[ATTRIB_TEMP_VERTEX], 2, GL_FLOAT, GL_FALSE, 0, quadData); glEnableVertexAttribArray(glViewAttributes[ATTRIB_TEMP_VERTEX]); GLfloat quadTextureData[] = { // 正常坐标 0, 0, 1, 0, 0, 1, 1, 1 }; glVertexAttribPointer(glViewAttributes[ATTRIB_TEMP_TEXCOORD], 2, GL_FLOAT, GL_FALSE, 0, quadTextureData); glEnableVertexAttribArray(glViewAttributes[ATTRIB_TEMP_TEXCOORD]); glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); dispatch_semaphore_signal(_lock); }
这里只是简单的利用了coreImage返回的CIFaceFeature对象提供的人脸坐标。其实CoreImage返回的CIFaceFeature对象可以提供很多信息，包括人脸坐标、左右眼是否睁开及对应位置、嘴的位置等，所以如果我们需要做更详细的纹理贴图可以分别转换出眼睛、嘴巴的位置，然后使用我们想要的贴图渲染到对应的纹理坐标系中即可。
这个demo有比较详细的应用iOS CoreImage -- 人脸检测/ 换背景/ 抠图 /贴纸/ 实时视频滤镜
最后效果图如下（这里为了简单在得到的视频帧中只处理了一个包含人脸的CIFaceFeature对象）
每天进步一点点。
