由腾讯直播探了探CVPixelBufferRef
https://developer.apple.com/library/archive/documentation/GraphicsImaging/Reference/CoreImageFilterReference/index.html?language=objc#//apple_ref/doc/filter/ci/CISourceInCompositing 各种滤镜的说明
CVPixelBufferRef 也就是 CVImageBufferRef
腾讯直播需要 sendCustomVideoData: ，只要你发格式正确，大小问题不需要考虑（有点无脑操作）
如果通过摄像头数据采集要进行图片的处理，那首选是CIContext和CIImage组合，提供很多API进行各种转换，贴出代码
重要函数：
- (void)createBuffer:(CVPixelBufferRef)originalBuffer size:(CGSize)size
// @property (nonatomic, strong)AVCaptureVideoPreviewLayer *previewLayer; @end -(void)layoutSubviews { [super layoutSubviews]; [_previewLayer setFrame:self.layer.bounds]; } @end @property (nonatomic, strong) AVCaptureSession *session; @property (nonatomic, strong) AVCaptureDeviceInput *input; @property (nonatomic, strong) AVCaptureVideoDataOutput *videoDataOutput; @property (nonatomic, weak) TRTCCloud *trtc; @property (nonatomic, weak) UIImageView *trtcPreviewView; @property (nonatomic, strong) CIContext *ciContext; @end - (void)startCapture:(TRTCCloud*)trtc { self.trtc = trtc; [self.session startRunning]; [self refreshCaptureSessionConfigure]; } -(void)setupDisplayView:(UIView *)view { self.trtcPreviewView = (UIImageView*)view; [self.trtcPreviewView setContentMode:UIViewContentModeScaleAspectFit]; /* [self.previewView removeFromSuperview]; [view addSubview:self.previewView]; [self.previewView mas_remakeConstraints:^(MASConstraintMaker *make) { make.edges.mas_equalTo(view); }];*/ } -(void)setVideoSize:(CGSize)videoSize { _videoSize = videoSize; AVCaptureSessionPreset sessionPreset = [self sessionPresetForVideoSize:self.videoSize]; if (![self.session.sessionPreset isEqualToString:sessionPreset]) { dispatch_async(dispatch_get_main_queue(), ^{ self.session.sessionPreset = sessionPreset; }); } } -(void)setFrameRate:(int)frameRate { BOOL needUpdate = _frameRate != frameRate; _frameRate = frameRate; if (needUpdate) { dispatch_async(dispatch_get_main_queue(), ^{ [self configureFrameRate:frameRate]; }); } } - (void)configureFrameRate:(int)frameRate { // 设置帧率 AVCaptureDevice *captureDevice = self.input.device; AVFrameRateRange *frameRateRange = captureDevice.activeFormat.videoSupportedFrameRateRanges.firstObject; if (frameRate >= frameRateRange.maxFrameRate) { frameRate = frameRateRange.maxFrameRate; } else if (frameRate <= frameRateRange.minFrameRate) { frameRate = frameRateRange.minFrameRate; } self.frameRate = frameRate; if ([captureDevice lockForConfiguration:NULL]){ [captureDevice setActiveVideoMinFrameDuration:CMTimeMake(1, frameRate)]; [captureDevice setActiveVideoMaxFrameDuration:CMTimeMake(1, frameRate)]; [captureDevice unlockForConfiguration]; } } - (void)configureSession { if (!self.session) { self.session = [[AVCaptureSession alloc] init]; } [self setCaptureDevicePosition:AVCaptureDevicePositionFront]; [self setupCaptureVideoDataOutput]; } - (void)refreshCaptureSessionConfigure { dispatch_async(dispatch_get_main_queue(), ^{ [self configureFrameRate:self.frameRate]; self.session.sessionPreset = [self sessionPresetForVideoSize:self.videoSize]; [self.previewView.previewLayer removeFromSuperlayer]; self.previewView.previewLayer = [[AVCaptureVideoPreviewLayer alloc] initWithSession:self.session]; [self.previewView.previewLayer setFrame:self.previewView.layer.bounds]; [self.previewView.layer addSublayer:self.previewView.previewLayer]; [self.previewView.previewLayer setVideoGravity:AVLayerVideoGravityResizeAspectFill]; }); } - (AVCaptureSessionPreset)sessionPresetForVideoSize:(CGSize)size { if (size.width <= 352 && size.height <= 288 && [self.session canSetSessionPreset:AVCaptureSessionPreset352x288]) { return AVCaptureSessionPreset352x288; } else if (size.width <= 640 && size.height <= 480 && [self.session canSetSessionPreset:AVCaptureSessionPreset640x480]) { return AVCaptureSessionPreset640x480; } else if (size.width <= 960 && size.height <= 540 && [self.session canSetSessionPreset:AVCaptureSessionPresetiFrame960x540]) { return AVCaptureSessionPresetiFrame960x540; } else if (size.width <= 1280 && size.height <= 720 && [self.session canSetSessionPreset:AVCaptureSessionPreset1280x720]) { return AVCaptureSessionPreset1280x720; } else if (size.width <= 1920 && size.height <= 1080 && [self.session canSetSessionPreset:AVCaptureSessionPreset1920x1080]) { return AVCaptureSessionPreset1920x1080; } else if (size.width <= 3840 && size.height <= 2160 && [self.session canSetSessionPreset:AVCaptureSessionPreset3840x2160]) { return AVCaptureSessionPreset3840x2160; } return AVCaptureSessionPresetHigh; } - (void)setupCaptureVideoDataOutput { if (!self.videoDataOutput) { // Conigure and add output AVCaptureVideoDataOutput *videoDataOutput = [[AVCaptureVideoDataOutput alloc] init]; [self.session addOutput:videoDataOutput]; videoDataOutput.videoSettings = [NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange] forKey:(id)kCVPixelBufferPixelFormatTypeKey]; videoDataOutput.alwaysDiscardsLateVideoFrames = NO; // Use serial queue to receive audio / video data dispatch_queue_t videoQueue = dispatch_queue_create("videoQueue", NULL); [videoDataOutput setSampleBufferDelegate:self queue:videoQueue]; self.videoDataOutput = videoDataOutput; } } - (void)setCaptureDevicePosition:(AVCaptureDevicePosition)position { NSError *error = nil; AVCaptureDevice *device = [self getCaptureDevicePosition:position]; // Add input AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:device error:&error]; if (error != noErr) { NSLog(@"Configure device input failed:%@",error.localizedDescription); return; } if (self.input) { [self.session removeInput:self.input]; } [self.session addInput:input]; self.input = input; self.session.sessionPreset = [self sessionPresetForVideoSize:self.videoSize]; } #pragma mark - AVCaptureVideoDataOutputSampleBufferDelegate - (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection { CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); [self createBuffer:imageBuffer size:self.videoSize]; } #pragma mark - 裁剪CVPixelBufferRef /* /// aspectRatio (width / height) -(CVPixelBufferRef)createBuffer:(CVPixelBufferRef)buffer aspectRatio:(CGFloat)aspectRatio { CVPixelBufferLockBaseAddress(buffer, 0); size_t num = 2;//CVPixelBufferGetPlaneCount(buffer); void * address[num]; size_t width[num]; size_t height[num]; size_t bytes[num]; size_t rWidth = 0; size_t rHeight = 0; for (size_t i = 0; i != num; i++) { address[i] = CVPixelBufferGetBaseAddressOfPlane(buffer, i); width[i] = CVPixelBufferGetWidthOfPlane(buffer, i); height[i] = CVPixelBufferGetHeightOfPlane(buffer, i); bytes[i] = CVPixelBufferGetBytesPerRowOfPlane(buffer, i); size_t w = 0; size_t h = 0; w = ceil(height[i] * aspectRatio); if (w > width[i]) { w = width[i]; h = ceil(w / aspectRatio); } else { h = height[i]; } if (rWidth == 0 && rHeight == 0) { rWidth = w; rHeight = h; } size_t x = (width[i] - w) / 2; size_t y = (height[i] - h) / 2; size_t startpos = y * bytes[i] + x * (bytes[i] / width[i]); address[i] = address[i] + startpos / (bytes[i] / width[i]); } CVPixelBufferRef re = NULL; CVPixelBufferCreateWithPlanarBytes(kCFAllocatorDefault, rWidth, rHeight, CVPixelBufferGetPixelFormatType(buffer), NULL, CVPixelBufferGetDataSize(buffer), num, address, width, height, bytes, NULL, NULL, NULL, &re); CVPixelBufferUnlockBaseAddress(buffer, 0); return re; }*/ - (void)createBuffer:(CVPixelBufferRef)originalBuffer size:(CGSize)size { if (self.ciContext == nil) { EAGLContext *eaglContext = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES3]; self.ciContext = [CIContext contextWithEAGLContext:eaglContext options:nil]; } CVPixelBufferLockBaseAddress(originalBuffer, 0); CIImage *ciImage = [CIImage imageWithCVImageBuffer:originalBuffer]; /* //此处通过额外的CGImageRef来生成ciImage对象，只是因为直接使用会导致后面的大小缩放会出现异常 CGImageRef originalImageRef = [self.ciContext createCGImage:ciImage fromRect:ciImage.extent]; // CGColorSpaceRef colorSpace = CGImageGetColorSpace(originalImageRef); CGImageRef fixedImageRef = [self image:originalImageRef sizeToFitThat:size]; CGImageRelease(originalImageRef); ciImage = [CIImage imageWithCGImage:fixedImageRef];*/ CGFloat width = ciImage.extent.size.width; CGFloat height = ciImage.extent.size.height; CGFloat ws = width / size.width; CGFloat hs = height / size.height; CGFloat aspectRatio = (ws > hs) ? hs : ws; CGFloat w = floor(size.width * aspectRatio); CGFloat h = floor(size.height * aspectRatio); CGFloat x = ((width - w) / 2); CGFloat y = ((height - h) / 2); CGRect cropRect = CGRectMake(x, y, w, h); ciImage = [ciImage imageByCroppingToRect:cropRect]; ciImage = [ciImage imageByApplyingTransform:CGAffineTransformMakeTranslation(-x, -y)]; CGFloat scale = 1.0 / aspectRatio; ciImage = [ciImage imageByApplyingTransform:CGAffineTransformMakeScale(scale, scale)]; if (self.input.device.position == AVCaptureDevicePositionFront) { ciImage = [ciImage imageByApplyingOrientation:kCGImagePropertyOrientationDownMirrored]; } CVPixelBufferRef re = NULL; NSMutableDictionary *outputPixelBufferAttributes = [NSMutableDictionary dictionary]; [outputPixelBufferAttributes setObject:@(CVPixelBufferGetPixelFormatType(originalBuffer)) forKey:(__bridge NSString *) kCVPixelBufferPixelFormatTypeKey]; [outputPixelBufferAttributes setObject:@(size.width) forKey:(__bridge NSString *) kCVPixelBufferWidthKey]; [outputPixelBufferAttributes setObject:@(size.height) forKey:(__bridge NSString *) kCVPixelBufferHeightKey]; [outputPixelBufferAttributes setObject:@{} forKey:(__bridge NSString *) kCVPixelBufferIOSurfacePropertiesKey]; CVPixelBufferCreate(kCFAllocatorDefault, size.width, size.height, CVPixelBufferGetPixelFormatType(originalBuffer), (__bridge CFDictionaryRef)outputPixelBufferAttributes, &re); // CVOpenGLESTextureCacheCreateTextureFromImage [self.ciContext render:ciImage toCVPixelBuffer:re]; ciImage = [CIImage imageWithCVImageBuffer:re]; UIImage *image = [UIImage imageWithCIImage:ciImage]; // CGImageRelease(fixedImageRef); dispatch_async(dispatch_get_main_queue(), ^{ [self.trtcPreviewView setImage:image]; CVPixelBufferRelease(re); }); // CVOpenGLESTextureCacheCreateTextureFromImage -6683 CVPixelBufferUnlockBaseAddress(originalBuffer, 0); NSLog(@"[PixelBuffer] width:%ld height:%ld",CVPixelBufferGetWidth(re),CVPixelBufferGetHeight(re)); // TRTCVideoPixelFormat_NV12 对应 kCVPixelFormatType_420YpCbCr8BiPlanarFullRange TRTCVideoFrame *frame = [[TRTCVideoFrame alloc] init]; frame.pixelFormat = TRTCVideoPixelFormat_NV12; frame.bufferType = TRTCVideoBufferType_PixelBuffer; frame.pixelBuffer = re; [self.trtc sendCustomVideoData:frame]; /* CVPixelBufferLockBaseAddress(originalBuffer, 0); // if (width >= size.width && height >= size.height) { CGFloat ws = width / size.width; CGFloat hs = height / size.height; CGFloat aspectRatio = (ws > hs) ? hs : ws; CGFloat w = floor(size.width * aspectRatio); CGFloat h = floor(size.height * aspectRatio); CGFloat x = ((width - w) / 2); CGFloat y = ((height - h) / 2); CGRect cropRect = CGRectMake(x, y, w, h); ciImage = [ciImage imageByCroppingToRect:cropRect]; ciImage = [ciImage imageByApplyingTransform:CGAffineTransformMakeTranslation(-x, -y)]; CGFloat scale = 1.0 / aspectRatio; ciImage = [ciImage imageByApplyingTransform:CGAffineTransformMakeScale(scale, scale)]; [ciContext render:ciImage toCVPixelBuffer:originalBuffer]; if (self.input.device.position == AVCaptureDevicePositionFront) { ciImage = [ciImage imageByApplyingOrientation:kCGImagePropertyOrientationDownMirrored]; } UIImage *image = [UIImage imageWithCIImage:ciImage]; dispatch_async(dispatch_get_main_queue(), ^{ [self.trtcPreviewView setImage:image]; }); // CGImageRelease(imageRef); CVPixelBufferUnlockBaseAddress(originalBuffer, 0); // TRTCVideoPixelFormat_NV12 对应 kCVPixelFormatType_420YpCbCr8BiPlanarFullRange TRTCVideoFrame *frame = [[TRTCVideoFrame alloc] init]; frame.pixelFormat = TRTCVideoPixelFormat_NV12; frame.bufferType = TRTCVideoBufferType_PixelBuffer; frame.pixelBuffer = originalBuffer; [self.trtc sendCustomVideoData:frame]; // } else { // CVPixelBufferUnlockBaseAddress(originalBuffer, 0); // }*/ } - (CGImageRef)image:(CGImageRef)image sizeToFitThat:(CGSize)size { size_t oWidth = CGImageGetWidth(image); size_t oHeight = CGImageGetHeight(image); CGFloat ws = 1.0 * size.width / (CGFloat)oWidth; CGFloat hs = 1.0 * size.height / (CGFloat)oHeight; CGFloat aspectRatio = (ws > hs) ? ws : hs; CGFloat w = floor(size.width / aspectRatio); CGFloat h = floor(size.height / aspectRatio); CGFloat x = ((oWidth - w) / 2); CGFloat y = ((oHeight - h) / 2); CGRect cropRect = CGRectMake(x, y, w, h); CGImageRef cropImage = CGImageCreateWithImageInRect(image, cropRect); CGAffineTransform transform = CGAffineTransformIdentity; CGContextRef ctx = CGBitmapContextCreate(NULL, size.width, size.height, CGImageGetBitsPerComponent(cropImage), 0, CGImageGetColorSpace(cropImage), CGImageGetBitmapInfo(cropImage)); CGContextConcatCTM(ctx, transform); CGContextDrawImage(ctx, CGRectMake(0, 0, size.width, size.height), cropImage); // And now we just create a new UIImage from the drawing context CGImageRef scaleImage = CGBitmapContextCreateImage(ctx); CGContextRelease(ctx); CGImageRelease(cropImage); return scaleImage; } /* CVPixelBufferRef buffer = [self createBuffer:originalBuffer aspectRatio:needRotate?(size.height / size.width):(size.width / size.height)]; // CVPixelBufferCreateWithPlanarBytes for YUV input CGSize originalSize = CGSizeMake(CVPixelBufferGetWidth(buffer), CVPixelBufferGetHeight(buffer)); CVPixelBufferLockBaseAddress(buffer, 0); GLubyte *sourceImageBytes = CVPixelBufferGetBaseAddressOfPlane(buffer, 0); CGDataProviderRef dataProvider = CGDataProviderCreateWithData(NULL, sourceImageBytes, CVPixelBufferGetBytesPerRow(buffer) * originalSize.height, NULL); CGColorSpaceRef genericRGBColorspace = CGColorSpaceCreateDeviceRGB(); CGImageRef cgImageFromBytes = CGImageCreate((int)originalSize.width, (int)originalSize.height, 8, 32, CVPixelBufferGetBytesPerRow(buffer), genericRGBColorspace, kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst, dataProvider, NULL, NO, kCGRenderingIntentDefault); GLubyte *imageData = (GLubyte *) calloc(1, (int)size.width * (int)size.height * 4); CGContextRef imageContext = CGBitmapContextCreate(imageData, (int)size.width, (int)size.height, 8, (int)size.width * 4, genericRGBColorspace, kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst); CGAffineTransform transform = CGAffineTransformIdentity; if (needRotate) { transform = CGAffineTransformTranslate(transform, size.height, 0); transform = CGAffineTransformRotate(transform, M_PI_2); } CGContextConcatCTM(imageContext, transform); if (needRotate) { CGContextDrawImage(imageContext, CGRectMake(0.0, 0.0, size.height, size.width), cgImageFromBytes); } else { CGContextDrawImage(imageContext, CGRectMake(0.0, 0.0, size.width, size.height), cgImageFromBytes); } CGImageRef cgimg = CGBitmapContextCreateImage(imageContext); UIImage *img = [UIImage imageWithCGImage:cgimg]; CGImageRelease(cgimg); dispatch_async(dispatch_get_main_queue(), ^{ [self.trtcPreviewView setImage:img]; }); CGImageRelease(cgImageFromBytes); CGContextRelease(imageContext); CGColorSpaceRelease(genericRGBColorspace); CGDataProviderRelease(dataProvider); CVPixelBufferRef pixel_buffer = NULL; CVPixelBufferCreateWithBytes(kCFAllocatorDefault, size.width, size.height, kCVPixelFormatType_32BGRA, imageData, size.width * 4, NULL, NULL, NULL, &pixel_buffer); CVPixelBufferUnlockBaseAddress(buffer, 0); CVPixelBufferRelease(buffer); if (pixel_buffer) { TRTCVideoFrame *frame = [[TRTCVideoFrame alloc] init]; frame.pixelFormat = TRTCVideoPixelFormat_32BGRA; frame.bufferType = TRTCVideoBufferType_PixelBuffer; frame.pixelBuffer = pixel_buffer; [self.trtc sendCustomVideoData:frame]; CVBufferRelease(pixel_buffer); dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.2 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{ free(imageData); }); } */ #pragma mark - priv - (AVCaptureDevice *)getCaptureDevicePosition:(AVCaptureDevicePosition)position { NSArray *devices = nil; if (@available(iOS 10.0, *)) { AVCaptureDeviceDiscoverySession *deviceDiscoverySession = [AVCaptureDeviceDiscoverySession discoverySessionWithDeviceTypes:@[AVCaptureDeviceTypeBuiltInWideAngleCamera] mediaType:AVMediaTypeVideo position:position]; devices = deviceDiscoverySession.devices; } else { #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdeprecated-declarations" devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]; #pragma clang diagnostic pop } for (AVCaptureDevice *device in devices) { if (position == device.position) { return device; } } return NULL; }@end
流程如下：
1.由CVPixelBufferRef生成CIImage对象
2.对CIImage进行各种花式操作，旋转、裁剪、缩放、滤镜等等一顿操作
3.通过CIContext将处理好的CIImage信息写入一个新的CVPixelBufferRef中（由于处理包含了大小缩放等信息，原来的CVPixelBufferRef大小可能存在有变化，所以要新生成一个）
CVPixelBufferRef re = NULL; NSMutableDictionary *outputPixelBufferAttributes = [NSMutableDictionary dictionary]; [outputPixelBufferAttributes setObject:@(CVPixelBufferGetPixelFormatType(originalBuffer)) forKey:(__bridge NSString *) kCVPixelBufferPixelFormatTypeKey]; [outputPixelBufferAttributes setObject:@(size.width) forKey:(__bridge NSString *) kCVPixelBufferWidthKey]; [outputPixelBufferAttributes setObject:@(size.height) forKey:(__bridge NSString *) kCVPixelBufferHeightKey]; [outputPixelBufferAttributes setObject:@{} forKey:(__bridge NSString *) kCVPixelBufferIOSurfacePropertiesKey]; CVPixelBufferCreate(kCFAllocatorDefault, size.width, size.height, CVPixelBufferGetPixelFormatType(originalBuffer), (__bridge CFDictionaryRef)outputPixelBufferAttributes, &re); // CVOpenGLESTextureCacheCreateTextureFromImage [self.ciContext render:ciImage toCVPixelBuffer:re];
注意：
1.UIImage获取CGImage和CIImage是否能取得值，完全要看UIImage创建时，是通过CIImage还是CGImage来创建的
2.如果说通过不熟悉CIImage图片处理的话，也可以通过先转成CGImage来处理，但是...为什么要这么做呢？
3.一定要注意大小的变化，将处理结果写入到新的对象中，否则出来的结果很可能就不是你想要的
