AVFoundation Custom Video Compositor Skipping Frames During AVPlayer Playback Despite 60 FPS Frame Duration

I'm building a Swift video editor with AVFoundation and a custom compositor. Despite setting AVVideoComposition.frameDuration to 60 FPS, I'm seeing significant frame skipping during playback.

Console Output Shows Frame Skipping

Frame #0 at 0.0 ms (fps: 60.0)
Frame #2 at 33.333333333333336 ms (fps: 60.0)
Frame #6 at 100.0 ms (fps: 60.0)
Frame #10 at 166.66666666666666 ms (fps: 60.0)
Frame #32 at 533.3333333333334 ms (fps: 60.0)
Frame #62 at 1033.3333333333335 ms (fps: 60.0)
Frame #96 at 1600.0 ms (fps: 60.0)

Instead of frames every ~16.67ms (60 FPS), I'm getting irregular intervals, sometimes 33ms, 67ms, or hundreds of milliseconds apart.

Renderer.swift (Key Parts)

@MainActor
class Renderer: ObservableObject {
    @Published var playerItem: AVPlayerItem?
    private let assetManager: ProjectAssetManager?
    private let compositorId: String
    
    func buildComposition() async {
        // ... load mouse moves/clicks data ...
        
        let composition = AVMutableComposition()
        let videoTrack = composition.addMutableTrack(
            withMediaType: .video,
            preferredTrackID: kCMPersistentTrackID_Invalid
        )
        
        var currentTime = CMTime.zero
        var layerInstructions: [AVMutableVideoCompositionLayerInstruction] = []
        
        // Insert video segments
        for videoURL in videoURLs {
            let asset = AVAsset(url: videoURL)
            let tracks = try await asset.loadTracks(withMediaType: .video)
            let assetVideoTrack = tracks.first
            let duration = try await asset.load(.duration)
            
            try videoTrack.insertTimeRange(
                CMTimeRange(start: .zero, duration: duration),
                of: assetVideoTrack,
                at: currentTime
            )
            
            let layerInstruction = AVMutableVideoCompositionLayerInstruction(assetTrack: videoTrack)
            let transform = try await assetVideoTrack.load(.preferredTransform)
            layerInstruction.setTransform(transform, at: currentTime)
            layerInstructions.append(layerInstruction)
            
            currentTime = CMTimeAdd(currentTime, duration)
        }
        
        let videoComposition = AVMutableVideoComposition()
        videoComposition.frameDuration = CMTime(value: 1, timescale: 60) // 60 FPS
        
        // Set render size from first video
        if let firstURL = videoURLs.first {
            let firstAsset = AVAsset(url: firstURL)
            let firstTrack = try await firstAsset.loadTracks(withMediaType: .video).first
            let naturalSize = try await firstTrack.load(.naturalSize)
            let transform = try await firstTrack.load(.preferredTransform)
            videoComposition.renderSize = CGSize(
                width: abs(naturalSize.applying(transform).width),
                height: abs(naturalSize.applying(transform).height)
            )
        }
        
        let instruction = CompositorInstruction()
        instruction.timeRange = CMTimeRange(start: .zero, duration: currentTime)
        instruction.layerInstructions = layerInstructions
        instruction.compositorId = compositorId
        videoComposition.instructions = [instruction]
        videoComposition.customVideoCompositorClass = CustomVideoCompositor.self
        
        let playerItem = AVPlayerItem(asset: composition)
        playerItem.videoComposition = videoComposition
        self.playerItem = playerItem
    }
}

class CompositorInstruction: NSObject, AVVideoCompositionInstructionProtocol {
    var timeRange: CMTimeRange = .zero
    var enablePostProcessing: Bool = false
    var containsTweening: Bool = false
    var requiredSourceTrackIDs: [NSValue]?
    var passthroughTrackID: CMPersistentTrackID = kCMPersistentTrackID_Invalid
    var layerInstructions: [AVVideoCompositionLayerInstruction] = []
    var compositorId: String = ""
}

class CustomVideoCompositor: NSObject, AVVideoCompositing {
    var sourcePixelBufferAttributes: [String : Any]? = [
        kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_32BGRA)
    ]
    
    var requiredPixelBufferAttributesForRenderContext: [String : Any] = [
        kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_32BGRA)
    ]
    
    func renderContextChanged(_ newRenderContext: AVVideoCompositionRenderContext) {}
    
    func startRequest(_ asyncVideoCompositionRequest: AVAsynchronousVideoCompositionRequest) {
        guard let sourceTrackID = asyncVideoCompositionRequest.sourceTrackIDs.first?.int32Value,
              let sourcePixelBuffer = asyncVideoCompositionRequest.sourceFrame(byTrackID: sourceTrackID),
              let outputBuffer = asyncVideoCompositionRequest.renderContext.newPixelBuffer() else {
            asyncVideoCompositionRequest.finish(with: NSError(domain: "VideoCompositor", code: -1))
            return
        }
        
        let videoComposition = asyncVideoCompositionRequest.renderContext.videoComposition
        let frameDuration = videoComposition.frameDuration
        let fps = Double(frameDuration.timescale) / Double(frameDuration.value)
        let compositionTime = asyncVideoCompositionRequest.compositionTime
        let seconds = CMTimeGetSeconds(compositionTime)
        let frameInMilliseconds = seconds * 1000
        let frameNumber = Int(round(seconds * fps))
        
        print("Frame #\(frameNumber) at \(frameInMilliseconds) ms (fps: \(fps))")
        
        asyncVideoCompositionRequest.finish(withComposedVideoFrame: outputBuffer)
    }
    
    func cancelAllPendingVideoCompositionRequests() {}
}

VideoPlayerViewModel

@MainActor
class VideoPlayerViewModel: ObservableObject {
    let player = AVPlayer()
    private let renderer: Renderer
    
    func loadVideo() async {
        await renderer.buildComposition()
        if let playerItem = renderer.playerItem {
            player.replaceCurrentItem(with: playerItem)
        }
    }
}

What I've Tried

• Frame skipping is consistent—exact same timestamps on every playback
• Issue persists even with minimal processing (just passing through buffers)
• Occurs regardless of compositor complexity

Please note that I need every frame at exact millisecond intervals for my application. Frame loss or inconsistent frameInMillisecond values are not acceptable.