I'm trying to update an old unity app for a client. The app has been crashing on iOS in a plugin they use called NatCorder. They use it to record video only separately and then re-record it with effects and audio gathered separately.
Instead of trying to update the plugin to something else which would be quite the hassle, I noticed the API for the native part of the plugin, where the crash occurs, is very simple, especially if you don't try to support everything the plugin does and the app does not use. So I tried to re-implement that native library using AVFoundation.
I got the video recording right, it captures the camera from the iPhone and writes it to a file properly. However, when the app does the second part, where it sends video and audio frames to the plugin, it fails. The app sends all the video frames and then sends all the audio frames. The video frames are eaten fine by AVFoundation but the audio fails at random points with unknown errors.
I wonder if I'm trying to use incompatible audio-video formats or if I'm using timestamps wrong or something.
Here's my init code. Anything suspicious to you?
void* NCCreateMP4Recorder(int width, int height, float framerate, int bitrate, int keyframeInterval,
int sampleRate, int channelCount, const char* recordingPath, void (*callback)(void*, void*), void* context)
{
Recorder* recorder = calloc(1, sizeof(Recorder));
recorder->context = context;
recorder->callback = callback;
recorder->path = strdup(recordingPath);
recorder->width = width;
recorder->channelCount = channelCount;
recorder->sampleRate = sampleRate;
recorder->height = height;
NSError *error = nil;
NSURL* url = createURLFromArgumentCString(recordingPath);
recorder->writer = [AVAssetWriter assetWriterWithURL:url fileType:AVFileTypeMPEG4 error:&error];
if (recorder->writer == nil)
NSLog(@"Failed creating media writer: %@", error);
NSDictionary *videoSettings = @{
AVVideoCodecKey: AVVideoCodecTypeH264,
AVVideoWidthKey: @(width),
AVVideoHeightKey: @(height),
AVVideoCompressionPropertiesKey: @{
AVVideoAverageBitRateKey: @(bitrate),
AVVideoMaxKeyFrameIntervalKey: @(keyframeInterval),
}
};
recorder->video = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeVideo
outputSettings:videoSettings];
if (recorder->video == nil)
NSLog(@"Failed creating video writer input");
recorder->video.expectsMediaDataInRealTime = true;
NSDictionary* videoSource = [NSDictionary dictionaryWithObjectsAndKeys:
[NSNumber numberWithInt:kCVPixelFormatType_32ARGB], kCVPixelBufferPixelFormatTypeKey,
[NSNumber numberWithInt:width], kCVPixelBufferWidthKey,
[NSNumber numberWithInt:height], kCVPixelBufferHeightKey,
nil];
recorder->videoAdaptor = [AVAssetWriterInputPixelBufferAdaptor
assetWriterInputPixelBufferAdaptorWithAssetWriterInput:recorder->video
sourcePixelBufferAttributes:videoSource];
if (recorder->videoAdaptor == nil)
NSLog(@"Failed creating video adaptor");
if ([recorder->writer canAddInput:recorder->video])
[recorder->writer addInput:recorder->video];
else
NSLog(@"Could not add video input to writer");
if (sampleRate > 0 && channelCount > 0)
{
AudioChannelLayout layout = {
.mChannelLayoutTag = channelCount == 1
? kAudioChannelLayoutTag_Mono
: kAudioChannelLayoutTag_Stereo,
.mChannelBitmap = 0,
.mNumberChannelDescriptions = 0
};
NSDictionary* audioOutputSettings = @{
AVFormatIDKey: @(kAudioFormatMPEG4AAC),
AVNumberOfChannelsKey: @(channelCount),
AVSampleRateKey: @(sampleRate),
AVEncoderBitRateKey: @128000,
AVChannelLayoutKey: [NSData dataWithBytes:&layout length:sizeof(layout)]
};
recorder->audio = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeAudio
outputSettings:audioOutputSettings];
if (!recorder->audio)
NSLog(@"Failed creating audio adaptor");
recorder->audio.expectsMediaDataInRealTime = true;
AudioStreamBasicDescription audioStreamDesc =
{
.mSampleRate = sampleRate,
.mFormatID = kAudioFormatLinearPCM,
.mFormatFlags = kAudioFormatFlagIsPacked | kAudioFormatFlagIsFloat,
.mBytesPerPacket = channelCount * sizeof(float),
.mFramesPerPacket = 1,
.mBytesPerFrame = channelCount * sizeof(float),
.mChannelsPerFrame = channelCount,
.mBitsPerChannel = sizeof(float) * 8,
};
OSStatus status = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &audioStreamDesc,
sizeof(layout), &layout, 0, nil, nil, &recorder->audioDesc);
if (status)
NSLog(@"Failed creating audio format description: %d", (int)status);
if ([recorder->writer canAddInput:recorder->audio])
[recorder->writer addInput:recorder->audio];
else
NSLog(@"Could not add audio input to writer");
}
if (![recorder->writer startWriting])
NSLog(@"Could not start writing: %@", recorder->writer.error);
[recorder->writer startSessionAtSourceTime:kCMTimeZero];
NSLog(@"Recording started to %s", recordingPath);
return recorder;
}