iOS摄像头采集和编码
设计思路
使用AVCaptureSession创建采集会话，获取图像数据后通过VideoToolBox进行编码。
采集参数设置
AVCaptureSession需要AVCaptureDeviceInput作为输入和AVCaptureVideoDataOutput接收输出数据(就是采集图像数据)。
参数设置之间需要分别调用beginConfiguration和commitConfiguration方法。
采集参数设置
//采集参数设置
-(int)doCapturePrepare{
NSError* error;
//获取摄像头设备对象
AVCaptureDevice * device;
NSArray<AVCaptureDevice *> *devices;
AVCaptureDevicePosition position = _facing ? AVCaptureDevicePositionFront : AVCaptureDevicePositionBack;
if (@available(iOS 10.0, *)) {
AVCaptureDeviceDiscoverySession *deviceDiscoverySession = [AVCaptureDeviceDiscoverySession discoverySessionWithDeviceTypes:@[AVCaptureDeviceTypeBuiltInWideAngleCamera] mediaType:AVMediaTypeVideo position:position];
devices = deviceDiscoverySession.devices;
} else {
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wdeprecated-declarations"
devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
#pragma clang diagnostic pop
}
for(AVCaptureDevice * dev in devices)
{
NSLog(@"device : %@", dev);
if([dev position] == position)
{
device = dev;
break;
}
}
//设置摄像头帧率，作用不大
CMTime frameDuration = CMTimeMake(1, 30);
for (AVFrameRateRange *range in [device.activeFormat videoSupportedFrameRateRanges]) {
NSLog(@"support framerate:%@", range);
if (CMTIME_COMPARE_INLINE(frameDuration, >=, range.minFrameDuration) &&
CMTIME_COMPARE_INLINE(frameDuration, <=, range.maxFrameDuration)) {
if ([device lockForConfiguration:&error]) {
[device setActiveVideoMaxFrameDuration:range.minFrameDuration];
[device setActiveVideoMinFrameDuration:range.maxFrameDuration];
[device unlockForConfiguration];
NSLog(@"select framerate:%@", range);
}
}
}
//创建输入
_input = [[AVCaptureDeviceInput alloc] initWithDevice: device error:&error];
if (error) {
NSLog(@"create input failed,%@",error);
return -1;
}else{
NSLog(@"create input succeed.");
}
//创建输出队列
//DISPATCH_QUEUE_SERIAL串行队列
//_dataCallbackQueue = dispatch_queue_create("dataCallbackQueue", DISPATCH_QUEUE_SERIAL);
//创建数据获取线程
_dataCallbackQueue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);
//创建输出
_output = [[AVCaptureVideoDataOutput alloc] init];
//绑定输出队列和代理到输出对象
[_output setSampleBufferDelegate:self queue:_dataCallbackQueue];
//抛弃过期帧，保证实时性
[_output setAlwaysDiscardsLateVideoFrames:YES];
//获取输出对象所支持的像素格式
NSArray *supportedPixelFormats = _output.availableVideoCVPixelFormatTypes;
for (NSNumber *currentPixelFormat in supportedPixelFormats) {
NSLog(@"support format : %@", currentPixelFormat);
}
//设置输出格式
[_output setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange] forKey:(id)kCVPixelBufferPixelFormatTypeKey]];
//创建采集功能会话对象
_captureSession = [[AVCaptureSession alloc] init];
// 改变会话的配置前一定要先开启配置，配置完成后提交配置改变
[_captureSession beginConfiguration];
//设置采集参数
if([_captureSession canSetSessionPreset:AVCaptureSessionPreset640x480])
{
[_captureSession setSessionPreset:AVCaptureSessionPreset640x480];
}
//绑定input和output到session
NSLog(@"input : %@", _input);
if([_captureSession canAddInput:_input])
{
[_captureSession addInput:_input];
}
NSLog(@"output : %@", _output);
if([_captureSession canAddOutput:_output])
{
[_captureSession addOutput: _output];
}
//显示输出画面
AVCaptureVideoPreviewLayer *previewLayer = [AVCaptureVideoPreviewLayer layerWithSession:_captureSession];
previewLayer.frame = CGRectMake(0, 50, self.view.frame.size.width, self.view.frame.size.height - 50);
[self.view.layer addSublayer:previewLayer];
//提交配置变更
[_captureSession commitConfiguration];
return 0;
}
- (void)captureOutput:(AVCaptureOutput *)output
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
fromConnection:(AVCaptureConnection *)connection
{
int64_t cur = CFAbsoluteTimeGetCurrent() * 1000;//ms
if(_lastime == -1)
{
_lastime = cur;
}
int64_t went = cur - _lastime;
int64_t duration = 1000/_fps;
//NSLog(@"duration:%ld", duration);
//NSLog(@"last:%ld,cur:%ld,went:%ld", last, cur, went);
if(went < duration)
{
NSLog(@"drop");
return;
}else{
_lastime = cur - went % duration;
}
//NSLog(@"captureOutput:%@,%@,%@", output, sampleBuffer, connection);
CMVideoFormatDescriptionRef description = CMSampleBufferGetFormatDescription(sampleBuffer);
//NSLog(@"captureOutput:%@", description);
if(CMFormatDescriptionGetMediaType(description) != kCMMediaType_Video)
{
return;
}
//FourCharCode codectype = CMVideoFormatDescriptionGetCodecType(description);
//NSString *scodectype = FOURCC2STR(codectype);
//NSLog(@"codec type:%@", scodectype);
//CVPixelBufferRef是CVImageBufferRef的别名，两者操作几乎一致。
CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
imageBuffer = RotatePixelBuffer(imageBuffer, kCGImagePropertyOrientationRight);
//需先用CVPixelBufferLockBaseAddress()锁定地址才能从主存访问，否则调用CVPixelBufferGetBaseAddressOfPlane等函数则返回NULL或无效值。
CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
//NSLog(@"imageBuffer:%@", imageBuffer);
if(CVPixelBufferIsPlanar(imageBuffer))
{
size_t planars = CVPixelBufferGetPlaneCount(imageBuffer);
if(planars == 2)
{
size_t stride = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer, 0);
size_t width = CVPixelBufferGetWidthOfPlane(imageBuffer, 0);
size_t height = CVPixelBufferGetHeightOfPlane(imageBuffer, 0);
//NSLog(@"buffer stride : %ld, w : %ld, h : %ld", stride, width, height);
void* Y = CVPixelBufferGetBaseAddressOfPlane(imageBuffer, 0);
void* UV = CVPixelBufferGetBaseAddressOfPlane(imageBuffer, 1);
if(Y != nil && UV != nil && _hyuv != NULL)
{
//NSLog(@"frame size %lu",stride * height*3/2);
fwrite(Y, 1, stride * height, _hyuv);
fwrite(UV, 1, stride * height / 2, _hyuv);
}
}
}else{
void* YUV = CVPixelBufferGetBaseAddress(imageBuffer);
size_t size = CVPixelBufferGetDataSize(imageBuffer);
if(YUV != nil && _hyuv != NULL)
{
//NSLog(@"frame size %lu",size);
fwrite(YUV, 1, size, _hyuv);
}
}
fflush(_hyuv);
//编码264
if(_firstime == -1)
{
_firstime = cur;
}
//创建CMTime的pts和duration
CMTime pts = CMTimeMake(cur - _firstime, 1000);//ms
CMTime dur = CMTimeMake(1, _fps);//ms
VTEncodeInfoFlags flags;
//NSLog(@"input pts : %lf", CMTimeGetSeconds(pts));
//开始编码该帧数据
OSStatus statusCode = VTCompressionSessionEncodeFrame(
_compressionSession,
imageBuffer,
pts,
dur,
NULL,
NULL,
&flags
);
if (statusCode != noErr) {
NSLog(@"VTCompressionSessionEncodeFrame failed %d", statusCode);
[self doEncodeDestroy];
}
//unlock
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
CVPixelBufferRelease(imageBuffer);
}
开始/停止采集
isRunning、startRunning、stopRunning简单明了的接口。
开始/停止采集
//开始采集
-(int)doStartCapture{
if(_captureSession != NULL && ![_captureSession isRunning])
{
[_captureSession startRunning];
if([_captureSession isRunning])
{
NSLog(@"start capture succeed.");
return 0;
}
else
{
return -1;
}
}
return 0;
}
//停止采集
-(int)doStopCapture{
if(_captureSession != NULL && [_captureSession isRunning])
{
[_captureSession stopRunning];
if(![_captureSession isRunning])
{
_captureSession = NULL;
NSLog(@"stop capture succeed.");
return 0;
}
else
{
return -1;
}
}
return 0;
}
编码参数设置和销毁
调用VTSessionSetProperty设置需要的编码参数后，调用VTCompressionSessionPrepareToEncodeFrames准备进行编码。
使用VTCompressionSessionEncodeFrame推送数据到编码器。
编码参数设置和销毁
//编码参数设置
-(int)doEncodePrepare{
if([self doEncodeDestroy]!=0)
{
NSLog(@"doEncodeDestroy failed.");
return -1;
}
//创建CompressionSession对象,该对象用于对画面进行编码
OSStatus status = VTCompressionSessionCreate(NULL, // 会话的分配器。传递NULL以使用默认分配器。
_width, // 帧的宽度，以像素为单位。
_height, // 帧的高度，以像素为单位。
kCMVideoCodecType_H264, // 编解码器的类型,表示使用h.264进行编码
NULL, // 指定必须使用的特定视频编码器。传递NULL让视频工具箱选择编码器。
NULL, // 源像素缓冲区所需的属性，用于创建像素缓冲池。如果不希望视频工具箱为您创建一个，请传递NULL
NULL, // 编码数据的分配器。传递NULL以使用默认分配器。
VTCompressionOutputCallbackH264, // 当一次编码结束会在该函数进行回调,可以在该函数中将数据,写入文件中
(__bridge void*)self, // outputCallbackRefCon
&_compressionSession); // 指向一个变量以接收的编码会话。
if (status != noErr){
NSLog(@"VTCompressionSessionCreate failed : %d", status);
return -1;
}
//设置实时编码输出（直播必然是实时输出,否则会有延迟）
status = VTSessionSetProperty(_compressionSession, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue);
if (status != noErr){
NSLog(@"kVTCompressionPropertyKey_RealTime failed : %d", status);
return -1;
}
//设置profile
status = VTSessionSetProperty(_compressionSession, kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_Baseline_AutoLevel);
if (status != noErr){
NSLog(@"kVTCompressionPropertyKey_ProfileLevel failed : %d", status);
return -1;
}
//关闭重排,可以关闭B帧。
status = VTSessionSetProperty(_compressionSession, kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse);
if (status != noErr){
NSLog(@"kVTCompressionPropertyKey_AllowFrameReordering failed : %d", status);
return -1;
}
//设置gop
status = VTSessionSetProperty(_compressionSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, (__bridge CFTypeRef)(@(_fps*10)));
if (status != noErr){
NSLog(@"kVTCompressionPropertyKey_MaxKeyFrameInterval failed : %d", status);
return -1;
}
//设置帧率
status = VTSessionSetProperty(_compressionSession, kVTCompressionPropertyKey_ExpectedFrameRate, (__bridge CFTypeRef)(@(_fps)));
if (status != noErr){
NSLog(@"kVTCompressionPropertyKey_ExpectedFrameRate failed : %d", status);
return -1;
}
//设置码率kVTCompressionPropertyKey_AverageBitRate/kVTCompressionPropertyKey_DataRateLimits
status = VTSessionSetProperty(_compressionSession, kVTCompressionPropertyKey_AverageBitRate, (__bridge CFTypeRef)(@(_bitrate)));
if (status != noErr){
NSLog(@"kVTCompressionPropertyKey_AverageBitRate failed : %d", status);
return -1;
}
status = VTSessionSetProperty(_compressionSession, kVTCompressionPropertyKey_DataRateLimits, (__bridge CFArrayRef)@[@1.0]);
if (status != noErr){
NSLog(@"kVTCompressionPropertyKey_DataRateLimits failed : %d", status);
return -1;
}
//基本设置结束, 准备进行编码
status = VTCompressionSessionPrepareToEncodeFrames(_compressionSession);
if (status != noErr){
NSLog(@"VTCompressionSessionPrepareToEncodeFrames failed : %d", status);
return -1;
}
return 0;
}
//销毁编码设置
-(int)doEncodeDestroy{
if(_compressionSession != NULL)
{
VTCompressionSessionInvalidate(_compressionSession);
CFRelease(_compressionSession);
_compressionSession = NULL;
}
return 0;
}
void VTCompressionOutputCallbackH264(void * CM_NULLABLE outputCallbackRefCon, //自定义回调参数
void * CM_NULLABLE sourceFrameRefCon,
OSStatus status,
VTEncodeInfoFlags infoFlags,
CM_NULLABLE CMSampleBufferRef sampleBuffer)
{
if (status != noErr) {
NSLog(@"encode error : %d", status);
return;
}
if (!CMSampleBufferDataIsReady(sampleBuffer)) {
NSLog(@"sampleBuffer data is not ready ");
return;
}
ViewController* _self = (__bridge ViewController*)outputCallbackRefCon;
//判断是否是关键帧
bool isKeyframe = !CFDictionaryContainsKey((CFArrayGetValueAtIndex(CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true), 0)), kCMSampleAttachmentKey_NotSync);
if (isKeyframe)
{
// 获取编码后的信息（存储于CMFormatDescriptionRef中）
CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(sampleBuffer);
// 获取SPS信息
size_t sparameterSetSize, sparameterSetCount;
const uint8_t *sparameterSet;
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, 0, &sparameterSet, &sparameterSetSize, &sparameterSetCount, 0);
// 获取PPS信息
size_t pparameterSetSize, pparameterSetCount;
const uint8_t *pparameterSet;
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, 1, &pparameterSet, &pparameterSetSize, &pparameterSetCount, 0);
//NSLog(@"sps size : %d", sparameterSetSize);
//NSLog(@"pps size : %d", pparameterSetSize);
if(_self.h264 != NULL)
{
//NSLog(@"write file");
char naluhead[4] = {0x00, 0x00, 0x00, 0x01};
fwrite(naluhead, 1, 4, _self.h264);
fwrite(sparameterSet, 1, sparameterSetSize, _self.h264);
fwrite(naluhead, 1, 4, _self.h264);
fwrite(pparameterSet, 1, pparameterSetSize, _self.h264);
fflush(_self.h264);
}
}
//Float64 pts = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer));
//NSLog(@"output pts : %lf", pts);
// 获取数据块
CMBlockBufferRef dataBuffer = CMSampleBufferGetDataBuffer(sampleBuffer);
size_t length, total;
char *data;
OSStatus ret = CMBlockBufferGetDataPointer(dataBuffer, 0, &length, &total, &data);
if (ret == noErr) {
size_t offset = 0;
static const int AVCCHeaderLength = 4; // 返回的nalu数据前四个字节不是0001的startcode，而是大端模式的帧长度length
// 循环获取nalu数据
while (offset < total - AVCCHeaderLength) {
uint32_t nalulen = 0;
// Read the NAL unit length
memcpy(&nalulen, data + offset, AVCCHeaderLength);
// 从大端转系统端
nalulen = CFSwapInt32BigToHost(nalulen);
//NSLog(@"nalu size : %d", nalulen);
if(_self.h264 != NULL)
{
char naluhead[4] = {0x00, 0x00, 0x00, 0x01};
fwrite(naluhead, 1, 4, _self.h264);
fwrite(data + offset + AVCCHeaderLength, 1, nalulen, _self.h264);
fflush(_self.h264);
}
// 移动到写一个块，转成NALU单元
// Move to the next NAL unit in the block buffer
offset += AVCCHeaderLength + nalulen;
}
}
}
图像处理
有时候可能要对采集数据做旋转镜像等操作，可以参考以下方法。
图像处理
//旋转和镜像操作
static CVPixelBufferRef RotatePixelBuffer(CVPixelBufferRef pixelBuffer, CGImagePropertyOrientation orientation) {
CIImage *image = [CIImage imageWithCVImageBuffer:pixelBuffer];
image = [image imageByApplyingTransform : CGAffineTransformMakeTranslation(-image.extent.origin.x, -image.extent.origin.y)];
image = [image imageByApplyingOrientation : orientation];
CVPixelBufferRef output = NULL;
CVReturn ret = CVPixelBufferCreate(nil,
CGRectGetWidth(image.extent),
CGRectGetHeight(image.extent),
CVPixelBufferGetPixelFormatType(pixelBuffer),
nil,
&output);
if (ret != kCVReturnSuccess) {
NSLog(@"CVPixelBufferCreate failed : %d", ret);
}
else{
// 复用 CIContext
static CIContext *context = nil;
if(context == nil)
{
//方式0
//context = [[CIContext alloc] init];
//方式1
context = [CIContext contextWithOptions: [NSDictionary dictionaryWithObject:[NSNumber numberWithBool:NO] forKey:kCIContextUseSoftwareRenderer]];
//方式2
//EAGLContext* eaglctx = [[EAGLContext alloc] initWithAPI : kEAGLRenderingAPIOpenGLES3];
//context = [CIContext contextWithEAGLContext : eaglctx];
}
[context render : image toCVPixelBuffer : output];//ios9.3
}
return output;
}
完整例子代码
https://github.com/gongluck/AnalysisAVP/tree/master/example/ios/iosCamera
参考
https://www.cnblogs.com/lijinfu-software/articles/11451340.html
https://www.jianshu.com/p/e75d7b573ae5?utm_campaign=maleskine&utm_content=note&utm_medium=seo_notes&utm_source=recommendation
https://www.jianshu.com/p/a0e2d7b3b8a7
https://www.jianshu.com/p/f5f3f94f36c5
https://dikeyking.github.io/2020/01/02/CVPixelBuffer%E8%A3%81%E5%89%AA%E6%97%8B%E8%BD%AC%E7%BC%A9%E6%94%
【免费】FFmpeg/WebRTC/RTMP/NDK/Android音视频流媒体高级
