431 lines
19 KiB
Objective-C
431 lines
19 KiB
Objective-C
//
|
|
// SCCapturerBufferedVideoWriter.m
|
|
// Snapchat
|
|
//
|
|
// Created by Chao Pang on 12/5/17.
|
|
//
|
|
|
|
#import "SCCapturerBufferedVideoWriter.h"
|
|
|
|
#import "SCAudioCaptureSession.h"
|
|
#import "SCCaptureCommon.h"
|
|
#import "SCManagedCapturerUtils.h"
|
|
|
|
#import <SCBase/SCMacros.h>
|
|
#import <SCFoundation/SCAssertWrapper.h>
|
|
#import <SCFoundation/SCDeviceName.h>
|
|
#import <SCFoundation/SCLog.h>
|
|
#import <SCFoundation/SCTrace.h>
|
|
|
|
#import <FBKVOController/FBKVOController.h>
|
|
|
|
@implementation SCCapturerBufferedVideoWriter {
|
|
SCQueuePerformer *_performer;
|
|
__weak id<SCCapturerBufferedVideoWriterDelegate> _delegate;
|
|
FBKVOController *_observeController;
|
|
|
|
AVAssetWriter *_assetWriter;
|
|
AVAssetWriterInput *_audioWriterInput;
|
|
AVAssetWriterInput *_videoWriterInput;
|
|
AVAssetWriterInputPixelBufferAdaptor *_pixelBufferAdaptor;
|
|
CVPixelBufferPoolRef _defaultPixelBufferPool;
|
|
CVPixelBufferPoolRef _nightPixelBufferPool;
|
|
CVPixelBufferPoolRef _lensesPixelBufferPool;
|
|
CMBufferQueueRef _videoBufferQueue;
|
|
CMBufferQueueRef _audioBufferQueue;
|
|
}
|
|
|
|
- (instancetype)initWithPerformer:(id<SCPerforming>)performer
|
|
outputURL:(NSURL *)outputURL
|
|
delegate:(id<SCCapturerBufferedVideoWriterDelegate>)delegate
|
|
error:(NSError **)error
|
|
{
|
|
self = [super init];
|
|
if (self) {
|
|
_performer = performer;
|
|
_delegate = delegate;
|
|
_observeController = [[FBKVOController alloc] initWithObserver:self];
|
|
CMBufferQueueCreate(kCFAllocatorDefault, 0, CMBufferQueueGetCallbacksForUnsortedSampleBuffers(),
|
|
&_videoBufferQueue);
|
|
CMBufferQueueCreate(kCFAllocatorDefault, 0, CMBufferQueueGetCallbacksForUnsortedSampleBuffers(),
|
|
&_audioBufferQueue);
|
|
_assetWriter = [[AVAssetWriter alloc] initWithURL:outputURL fileType:AVFileTypeMPEG4 error:error];
|
|
if (*error) {
|
|
self = nil;
|
|
return self;
|
|
}
|
|
}
|
|
return self;
|
|
}
|
|
|
|
- (BOOL)prepareWritingWithOutputSettings:(SCManagedVideoCapturerOutputSettings *)outputSettings
|
|
{
|
|
SCTraceStart();
|
|
SCAssert([_performer isCurrentPerformer], @"");
|
|
SCAssert(outputSettings, @"empty output setting");
|
|
// Audio
|
|
SCTraceSignal(@"Derive audio output setting");
|
|
NSDictionary *audioOutputSettings = @{
|
|
AVFormatIDKey : @(kAudioFormatMPEG4AAC),
|
|
AVNumberOfChannelsKey : @(1),
|
|
AVSampleRateKey : @(kSCAudioCaptureSessionDefaultSampleRate),
|
|
AVEncoderBitRateKey : @(outputSettings.audioBitRate)
|
|
};
|
|
_audioWriterInput =
|
|
[[AVAssetWriterInput alloc] initWithMediaType:AVMediaTypeAudio outputSettings:audioOutputSettings];
|
|
_audioWriterInput.expectsMediaDataInRealTime = YES;
|
|
|
|
// Video
|
|
SCTraceSignal(@"Derive video output setting");
|
|
size_t outputWidth = outputSettings.width;
|
|
size_t outputHeight = outputSettings.height;
|
|
SCAssert(outputWidth > 0 && outputHeight > 0 && (outputWidth % 2 == 0) && (outputHeight % 2 == 0),
|
|
@"invalid output size");
|
|
NSDictionary *videoCompressionSettings = @{
|
|
AVVideoAverageBitRateKey : @(outputSettings.videoBitRate),
|
|
AVVideoMaxKeyFrameIntervalKey : @(outputSettings.keyFrameInterval)
|
|
};
|
|
NSDictionary *videoOutputSettings = @{
|
|
AVVideoCodecKey : AVVideoCodecH264,
|
|
AVVideoWidthKey : @(outputWidth),
|
|
AVVideoHeightKey : @(outputHeight),
|
|
AVVideoScalingModeKey : AVVideoScalingModeResizeAspectFill,
|
|
AVVideoCompressionPropertiesKey : videoCompressionSettings
|
|
};
|
|
_videoWriterInput =
|
|
[[AVAssetWriterInput alloc] initWithMediaType:AVMediaTypeVideo outputSettings:videoOutputSettings];
|
|
_videoWriterInput.expectsMediaDataInRealTime = YES;
|
|
CGAffineTransform transform = CGAffineTransformMakeTranslation(outputHeight, 0);
|
|
_videoWriterInput.transform = CGAffineTransformRotate(transform, M_PI_2);
|
|
_pixelBufferAdaptor = [[AVAssetWriterInputPixelBufferAdaptor alloc]
|
|
initWithAssetWriterInput:_videoWriterInput
|
|
sourcePixelBufferAttributes:@{
|
|
(NSString *)
|
|
kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange), (NSString *)
|
|
kCVPixelBufferWidthKey : @(outputWidth), (NSString *)
|
|
kCVPixelBufferHeightKey : @(outputHeight)
|
|
}];
|
|
|
|
SCTraceSignal(@"Setup video writer input");
|
|
if ([_assetWriter canAddInput:_videoWriterInput]) {
|
|
[_assetWriter addInput:_videoWriterInput];
|
|
} else {
|
|
return NO;
|
|
}
|
|
|
|
SCTraceSignal(@"Setup audio writer input");
|
|
if ([_assetWriter canAddInput:_audioWriterInput]) {
|
|
[_assetWriter addInput:_audioWriterInput];
|
|
} else {
|
|
return NO;
|
|
}
|
|
|
|
return YES;
|
|
}
|
|
|
|
- (void)appendVideoSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
|
{
|
|
SCAssert([_performer isCurrentPerformer], @"");
|
|
SC_GUARD_ELSE_RETURN(sampleBuffer);
|
|
if (!CMBufferQueueIsEmpty(_videoBufferQueue)) {
|
|
// We need to drain the buffer queue in this case
|
|
while (_videoWriterInput.readyForMoreMediaData) { // TODO: also need to break out in case of errors
|
|
CMSampleBufferRef dequeuedSampleBuffer =
|
|
(CMSampleBufferRef)CMBufferQueueDequeueAndRetain(_videoBufferQueue);
|
|
if (dequeuedSampleBuffer == NULL) {
|
|
break;
|
|
}
|
|
[self _appendVideoSampleBuffer:dequeuedSampleBuffer];
|
|
CFRelease(dequeuedSampleBuffer);
|
|
}
|
|
}
|
|
// Fast path, just append this sample buffer if ready
|
|
if (_videoWriterInput.readyForMoreMediaData) {
|
|
[self _appendVideoSampleBuffer:sampleBuffer];
|
|
} else {
|
|
// It is not ready, queuing the sample buffer
|
|
CMBufferQueueEnqueue(_videoBufferQueue, sampleBuffer);
|
|
}
|
|
}
|
|
|
|
- (void)appendAudioSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
|
{
|
|
SCAssert([_performer isCurrentPerformer], @"");
|
|
SC_GUARD_ELSE_RETURN(sampleBuffer);
|
|
if (!CMBufferQueueIsEmpty(_audioBufferQueue)) {
|
|
// We need to drain the buffer queue in this case
|
|
while (_audioWriterInput.readyForMoreMediaData) {
|
|
CMSampleBufferRef dequeuedSampleBuffer =
|
|
(CMSampleBufferRef)CMBufferQueueDequeueAndRetain(_audioBufferQueue);
|
|
if (dequeuedSampleBuffer == NULL) {
|
|
break;
|
|
}
|
|
[_audioWriterInput appendSampleBuffer:sampleBuffer];
|
|
CFRelease(dequeuedSampleBuffer);
|
|
}
|
|
}
|
|
// fast path, just append this sample buffer if ready
|
|
if ((_audioWriterInput.readyForMoreMediaData)) {
|
|
[_audioWriterInput appendSampleBuffer:sampleBuffer];
|
|
} else {
|
|
// it is not ready, queuing the sample buffer
|
|
CMBufferQueueEnqueue(_audioBufferQueue, sampleBuffer);
|
|
}
|
|
}
|
|
|
|
- (void)startWritingAtSourceTime:(CMTime)sourceTime
|
|
{
|
|
SCTraceStart();
|
|
SCAssert([_performer isCurrentPerformer], @"");
|
|
// To observe the status change on assetWriter because when assetWriter errors out, it only changes the
|
|
// status, no further delegate callbacks etc.
|
|
[_observeController observe:_assetWriter
|
|
keyPath:@keypath(_assetWriter, status)
|
|
options:NSKeyValueObservingOptionNew
|
|
action:@selector(assetWriterStatusChanged:)];
|
|
[_assetWriter startWriting];
|
|
[_assetWriter startSessionAtSourceTime:sourceTime];
|
|
}
|
|
|
|
- (void)cancelWriting
|
|
{
|
|
SCTraceStart();
|
|
SCAssert([_performer isCurrentPerformer], @"");
|
|
CMBufferQueueReset(_videoBufferQueue);
|
|
CMBufferQueueReset(_audioBufferQueue);
|
|
[_assetWriter cancelWriting];
|
|
}
|
|
|
|
- (void)finishWritingAtSourceTime:(CMTime)sourceTime withCompletionHanlder:(dispatch_block_t)completionBlock
|
|
{
|
|
SCTraceStart();
|
|
SCAssert([_performer isCurrentPerformer], @"");
|
|
|
|
while (_audioWriterInput.readyForMoreMediaData && !CMBufferQueueIsEmpty(_audioBufferQueue)) {
|
|
CMSampleBufferRef audioSampleBuffer = (CMSampleBufferRef)CMBufferQueueDequeueAndRetain(_audioBufferQueue);
|
|
if (audioSampleBuffer == NULL) {
|
|
break;
|
|
}
|
|
[_audioWriterInput appendSampleBuffer:audioSampleBuffer];
|
|
CFRelease(audioSampleBuffer);
|
|
}
|
|
while (_videoWriterInput.readyForMoreMediaData && !CMBufferQueueIsEmpty(_videoBufferQueue)) {
|
|
CMSampleBufferRef videoSampleBuffer = (CMSampleBufferRef)CMBufferQueueDequeueAndRetain(_videoBufferQueue);
|
|
if (videoSampleBuffer == NULL) {
|
|
break;
|
|
}
|
|
[_videoWriterInput appendSampleBuffer:videoSampleBuffer];
|
|
CFRelease(videoSampleBuffer);
|
|
}
|
|
|
|
dispatch_block_t finishWritingBlock = ^() {
|
|
[_assetWriter endSessionAtSourceTime:sourceTime];
|
|
[_audioWriterInput markAsFinished];
|
|
[_videoWriterInput markAsFinished];
|
|
[_assetWriter finishWritingWithCompletionHandler:^{
|
|
if (completionBlock) {
|
|
completionBlock();
|
|
}
|
|
}];
|
|
};
|
|
|
|
if (CMBufferQueueIsEmpty(_audioBufferQueue) && CMBufferQueueIsEmpty(_videoBufferQueue)) {
|
|
finishWritingBlock();
|
|
} else {
|
|
// We need to drain the samples from the queues before finish writing
|
|
__block BOOL isAudioDone = NO;
|
|
__block BOOL isVideoDone = NO;
|
|
// Audio
|
|
[_audioWriterInput
|
|
requestMediaDataWhenReadyOnQueue:_performer.queue
|
|
usingBlock:^{
|
|
if (!CMBufferQueueIsEmpty(_audioBufferQueue) &&
|
|
_assetWriter.status == AVAssetWriterStatusWriting) {
|
|
CMSampleBufferRef audioSampleBuffer =
|
|
(CMSampleBufferRef)CMBufferQueueDequeueAndRetain(_audioBufferQueue);
|
|
if (audioSampleBuffer) {
|
|
[_audioWriterInput appendSampleBuffer:audioSampleBuffer];
|
|
CFRelease(audioSampleBuffer);
|
|
}
|
|
} else if (!isAudioDone) {
|
|
isAudioDone = YES;
|
|
}
|
|
if (isAudioDone && isVideoDone) {
|
|
finishWritingBlock();
|
|
}
|
|
}];
|
|
|
|
// Video
|
|
[_videoWriterInput
|
|
requestMediaDataWhenReadyOnQueue:_performer.queue
|
|
usingBlock:^{
|
|
if (!CMBufferQueueIsEmpty(_videoBufferQueue) &&
|
|
_assetWriter.status == AVAssetWriterStatusWriting) {
|
|
CMSampleBufferRef videoSampleBuffer =
|
|
(CMSampleBufferRef)CMBufferQueueDequeueAndRetain(_videoBufferQueue);
|
|
if (videoSampleBuffer) {
|
|
[_videoWriterInput appendSampleBuffer:videoSampleBuffer];
|
|
CFRelease(videoSampleBuffer);
|
|
}
|
|
} else if (!isVideoDone) {
|
|
isVideoDone = YES;
|
|
}
|
|
if (isAudioDone && isVideoDone) {
|
|
finishWritingBlock();
|
|
}
|
|
}];
|
|
}
|
|
}
|
|
|
|
- (void)cleanUp
|
|
{
|
|
_assetWriter = nil;
|
|
_videoWriterInput = nil;
|
|
_audioWriterInput = nil;
|
|
_pixelBufferAdaptor = nil;
|
|
}
|
|
|
|
- (void)dealloc
|
|
{
|
|
CFRelease(_videoBufferQueue);
|
|
CFRelease(_audioBufferQueue);
|
|
CVPixelBufferPoolRelease(_defaultPixelBufferPool);
|
|
CVPixelBufferPoolRelease(_nightPixelBufferPool);
|
|
CVPixelBufferPoolRelease(_lensesPixelBufferPool);
|
|
[_observeController unobserveAll];
|
|
}
|
|
|
|
- (void)assetWriterStatusChanged:(NSDictionary *)change
|
|
{
|
|
SCTraceStart();
|
|
if (_assetWriter.status == AVAssetWriterStatusFailed) {
|
|
SCTraceSignal(@"Asset writer status failed %@, error %@", change, _assetWriter.error);
|
|
[_delegate videoWriterDidFailWritingWithError:[_assetWriter.error copy]];
|
|
}
|
|
}
|
|
|
|
#pragma - Private methods
|
|
|
|
- (CVImageBufferRef)_croppedPixelBufferWithInputPixelBuffer:(CVImageBufferRef)inputPixelBuffer
|
|
{
|
|
SCAssertTrue([SCDeviceName isIphoneX]);
|
|
const size_t inputBufferWidth = CVPixelBufferGetWidth(inputPixelBuffer);
|
|
const size_t inputBufferHeight = CVPixelBufferGetHeight(inputPixelBuffer);
|
|
const size_t croppedBufferWidth = (size_t)(inputBufferWidth * kSCIPhoneXCapturedImageVideoCropRatio) / 2 * 2;
|
|
const size_t croppedBufferHeight =
|
|
(size_t)(croppedBufferWidth * SCManagedCapturedImageAndVideoAspectRatio()) / 2 * 2;
|
|
const size_t offsetPointX = inputBufferWidth - croppedBufferWidth;
|
|
const size_t offsetPointY = (inputBufferHeight - croppedBufferHeight) / 4 * 2;
|
|
|
|
SC_GUARD_ELSE_RUN_AND_RETURN_VALUE((inputBufferWidth >= croppedBufferWidth) &&
|
|
(inputBufferHeight >= croppedBufferHeight) && (offsetPointX % 2 == 0) &&
|
|
(offsetPointY % 2 == 0) &&
|
|
(inputBufferWidth >= croppedBufferWidth + offsetPointX) &&
|
|
(inputBufferHeight >= croppedBufferHeight + offsetPointY),
|
|
SCLogGeneralError(@"Invalid cropping configuration"), NULL);
|
|
|
|
CVPixelBufferRef croppedPixelBuffer = NULL;
|
|
CVPixelBufferPoolRef pixelBufferPool =
|
|
[self _pixelBufferPoolWithInputSize:CGSizeMake(inputBufferWidth, inputBufferHeight)
|
|
croppedSize:CGSizeMake(croppedBufferWidth, croppedBufferHeight)];
|
|
|
|
if (pixelBufferPool) {
|
|
CVReturn result = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pixelBufferPool, &croppedPixelBuffer);
|
|
if ((result != kCVReturnSuccess) || (croppedPixelBuffer == NULL)) {
|
|
SCLogGeneralError(@"[SCCapturerVideoWriterInput] Error creating croppedPixelBuffer");
|
|
return NULL;
|
|
}
|
|
} else {
|
|
SCAssertFail(@"[SCCapturerVideoWriterInput] PixelBufferPool is NULL with inputBufferWidth:%@, "
|
|
@"inputBufferHeight:%@, croppedBufferWidth:%@, croppedBufferHeight:%@",
|
|
@(inputBufferWidth), @(inputBufferHeight), @(croppedBufferWidth), @(croppedBufferHeight));
|
|
return NULL;
|
|
}
|
|
CVPixelBufferLockBaseAddress(inputPixelBuffer, kCVPixelBufferLock_ReadOnly);
|
|
CVPixelBufferLockBaseAddress(croppedPixelBuffer, 0);
|
|
|
|
const size_t planesCount = CVPixelBufferGetPlaneCount(inputPixelBuffer);
|
|
for (int planeIndex = 0; planeIndex < planesCount; planeIndex++) {
|
|
size_t inPlaneHeight = CVPixelBufferGetHeightOfPlane(inputPixelBuffer, planeIndex);
|
|
size_t inPlaneBytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(inputPixelBuffer, planeIndex);
|
|
uint8_t *inPlaneAdress = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(inputPixelBuffer, planeIndex);
|
|
|
|
size_t croppedPlaneHeight = CVPixelBufferGetHeightOfPlane(croppedPixelBuffer, planeIndex);
|
|
size_t croppedPlaneBytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(croppedPixelBuffer, planeIndex);
|
|
uint8_t *croppedPlaneAdress = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(croppedPixelBuffer, planeIndex);
|
|
|
|
// Note that inPlaneBytesPerRow is not strictly 2x of inPlaneWidth for some devices (e.g. iPhone X).
|
|
// However, since UV are packed together in memory, we can use offsetPointX for all planes
|
|
size_t offsetPlaneBytesX = offsetPointX;
|
|
size_t offsetPlaneBytesY = offsetPointY * inPlaneHeight / inputBufferHeight;
|
|
|
|
inPlaneAdress = inPlaneAdress + offsetPlaneBytesY * inPlaneBytesPerRow + offsetPlaneBytesX;
|
|
size_t bytesToCopyPerRow = MIN(inPlaneBytesPerRow - offsetPlaneBytesX, croppedPlaneBytesPerRow);
|
|
for (int i = 0; i < croppedPlaneHeight; i++) {
|
|
memcpy(croppedPlaneAdress, inPlaneAdress, bytesToCopyPerRow);
|
|
inPlaneAdress += inPlaneBytesPerRow;
|
|
croppedPlaneAdress += croppedPlaneBytesPerRow;
|
|
}
|
|
}
|
|
CVPixelBufferUnlockBaseAddress(inputPixelBuffer, kCVPixelBufferLock_ReadOnly);
|
|
CVPixelBufferUnlockBaseAddress(croppedPixelBuffer, 0);
|
|
return croppedPixelBuffer;
|
|
}
|
|
|
|
- (CVPixelBufferPoolRef)_pixelBufferPoolWithInputSize:(CGSize)inputSize croppedSize:(CGSize)croppedSize
|
|
{
|
|
if (CGSizeEqualToSize(inputSize, [SCManagedCaptureDevice defaultActiveFormatResolution])) {
|
|
if (_defaultPixelBufferPool == NULL) {
|
|
_defaultPixelBufferPool = [self _newPixelBufferPoolWithWidth:croppedSize.width height:croppedSize.height];
|
|
}
|
|
return _defaultPixelBufferPool;
|
|
} else if (CGSizeEqualToSize(inputSize, [SCManagedCaptureDevice nightModeActiveFormatResolution])) {
|
|
if (_nightPixelBufferPool == NULL) {
|
|
_nightPixelBufferPool = [self _newPixelBufferPoolWithWidth:croppedSize.width height:croppedSize.height];
|
|
}
|
|
return _nightPixelBufferPool;
|
|
} else {
|
|
if (_lensesPixelBufferPool == NULL) {
|
|
_lensesPixelBufferPool = [self _newPixelBufferPoolWithWidth:croppedSize.width height:croppedSize.height];
|
|
}
|
|
return _lensesPixelBufferPool;
|
|
}
|
|
}
|
|
|
|
- (CVPixelBufferPoolRef)_newPixelBufferPoolWithWidth:(size_t)width height:(size_t)height
|
|
{
|
|
NSDictionary *attributes = @{
|
|
(NSString *) kCVPixelBufferIOSurfacePropertiesKey : @{}, (NSString *)
|
|
kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange), (NSString *)
|
|
kCVPixelBufferWidthKey : @(width), (NSString *)
|
|
kCVPixelBufferHeightKey : @(height)
|
|
};
|
|
CVPixelBufferPoolRef pixelBufferPool = NULL;
|
|
CVReturn result = CVPixelBufferPoolCreate(kCFAllocatorDefault, NULL,
|
|
(__bridge CFDictionaryRef _Nullable)(attributes), &pixelBufferPool);
|
|
if (result != kCVReturnSuccess) {
|
|
SCLogGeneralError(@"[SCCapturerBufferredVideoWriter] Error creating pixel buffer pool %i", result);
|
|
return NULL;
|
|
}
|
|
|
|
return pixelBufferPool;
|
|
}
|
|
|
|
- (void)_appendVideoSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
|
{
|
|
SCAssert([_performer isCurrentPerformer], @"");
|
|
CMTime presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
|
|
CVImageBufferRef inputPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
|
|
if ([SCDeviceName isIphoneX]) {
|
|
CVImageBufferRef croppedPixelBuffer = [self _croppedPixelBufferWithInputPixelBuffer:inputPixelBuffer];
|
|
if (croppedPixelBuffer) {
|
|
[_pixelBufferAdaptor appendPixelBuffer:croppedPixelBuffer withPresentationTime:presentationTime];
|
|
CVPixelBufferRelease(croppedPixelBuffer);
|
|
}
|
|
} else {
|
|
[_pixelBufferAdaptor appendPixelBuffer:inputPixelBuffer withPresentationTime:presentationTime];
|
|
}
|
|
}
|
|
|
|
@end
|