Capture and process ProRAW images

7:52 - Setting up device and session

// Use the .photo preset

private let session = AVCaptureSession()
private func configureSession() {
  session.beginConfiguration()
	session.sessionPreset = .photo
	//...
}

8:03 - Setting up device and session

// Or optionally find a format that supports highest quality photos

guard let format = device.formats.first(where: { $0.isHighestPhotoQualitySupported }) else {
  // handle failure to find a format that supports highest quality stills
} 
do 
{	
  try device.lockForConfiguration()
  {
    // ...
  }
  device.unlockForConfiguration()
} 
catch 
{
  // handle the exception
}

8:39 - Setting up photo output 1

// Enable ProRAW on the photo output

private let photoOutput = AVCapturePhotoOutput()
private func configurePhotoOutput() {
  photoOutput.isHighResolutionCaptureEnabled = true
	photoOutput.isAppleProRAWEnabled = photoOutput.isAppleProRAWSupported
	//...
}

8:59 - Setting up photo output 2

// Select the desired photo quality prioritization

private let photoOutput = AVCapturePhotoOutput()
private func configurePhotoOutput() {
	photoOutput.isHighResolutionCaptureEnabled = true
	photoOutput.isAppleProRAWEnabled           = photoOutput.isAppleProRAWSupported
	photoOutput.maxPhotoQualityPrioritization  = .quality // or .speed .balanced
  //...
}

9:26 - Prepare for ProRAW capture 1

// Find a supported ProRAW pixel format

guard let proRawPixelFormat = photoOutput.availableRawPhotoPixelFormatTypes.first(
  where: {
    AVCapturePhotoOutput.isAppleProRAWPixelFormat($0) 
  }) 
else {
	// Apple ProRAW is not supported with this device / format
}

// For Bayer RAW pixel format use

AVCapturePhotoOutput.isBayerRAWPixelFormat()

10:09 - Prepare for ProRAW capture 2

// Create photo settings for ProRAW only capture

let photoSettings = AVCapturePhotoSettings(rawPixelFormatType: proRawPixelFormat)

// Create photo settings for processed photo + ProRAW capture

guard let processedPhotoCodecType = photoOutput.availablePhotoCodecTypes.first 
else 
{
	// handle failure to find a processed photo codec type
}
let photoSettings = AVCapturePhotoSettings(rawPixelFormatType: proRawPixelFormat,
	processedFormat: [AVVideoCodecKey: processedPhotoCodecType])

10:53 - Prepare for ProRAW capture 3

// Select a supported thumbnail codec type and thumbnail dimensions

guard let thumbnailPhotoCodecType = photoSettings.availableRawEmbeddedThumbnailPhotoCodecTypes.first 
else 
{
  // handle failure to find an available thumbnail photo codec type
}

let dimensions = device.activeFormat.highResolutionStillImageDimensions

photoSettings.rawEmbeddedThumbnailPhotoFormat = [
  AVVideoCodecKey: thumbnailPhotoCodecType,
  AVVideoWidthKey: dimensions.width,
  AVVideoHeightKey: dimensions.height]

11:08 - Prepare for ProRAW capture 4

// Select the desired quality prioritization for the capture

photoSettings.photoQualityPrioritization = .quality // or .speed .balanced

// Optionally, request a preview image

if let previewPixelFormat = photoSettings.availablePreviewPhotoPixelFormatTypes.first
{	
  photoSettings.previewPhotoFormat = [kCVPixelBufferPixelFormatTypeKey as String: previewPixelFormat]
}

// Capture!

photoOutput.capturePhoto(with: photoSettings, delegate: delegate)

11:44 - Consuming captured ProRAW 1

func photoOutput(_ output: AVCapturePhotoOutput,
                 didFinishProcessingPhoto photo: AVCapturePhoto,
                 error: Error?) 
{
  guard error == nil 
  else 
  {
    // handle failure from the photo capture
  }
	if let preview = photo.previewPixelBuffer 
  { 
    // photo.previewCGImageRepresentation()
    // display the preview
  }
	if photo.isRawPhoto 
  {
		guard let proRAWFileDataRepresentation = photo.fileDataRepresentation() 
    else 
    {
      // handle failure to get ProRAW DNG file data representation
    }
		guard let proRAWPixelBuffer = photo.pixelBuffer 
    else 
    {
      // handle failure to get ProRAW pixel data
    }
		// use the file or pixel data
	}

12:52 - Consuming captured ProRAW 2

// Provide settings for lossless compression with less bits

class AppleProRAWCustomizer: NSObject, AVCapturePhotoFileDataRepresentationCustomizer 
{
	func replacementAppleProRAWCompressionSettings(for photo: AVCapturePhoto,
                                                 defaultSettings: [String : Any],
                                                 maximumBitDepth: Int) -> [String : Any] 
  {
    return [AVVideoAppleProRAWBitDepthKey: min(10, maximumBitDepth),
            AVVideoQualityKey: 1.00]
  }
}

13:35 - Consuming captured ProRAW 3

// Provide settings for lossy compression

class AppleProRAWCustomizer: NSObject, AVCapturePhotoFileDataRepresentationCustomizer 
{
	func replacementAppleProRAWCompressionSettings(
    for photo: AVCapturePhoto,
    defaultSettings: [String : Any],
    maximumBitDepth: Int) -> [String : Any] 
  {
    return [AVVideoAppleProRAWBitDepthKey: min(8, maximumBitDepth),
            AVVideoQualityKey: 0.90]
  }
}

13:51 - Consuming captured ProRAW 4

// Customizing the compression settings for the captured ProRAW photo

func photoOutput(_ output: AVCapturePhotoOutput,
                 didFinishProcessingPhoto photo: AVCapturePhoto,
                 error: Error?) 
{
  guard error == nil 
  else 
  {
    // handle failure from the photo capture
  }
    if photo.isRawPhoto 
  {
		let customizer = AppleProRAWCustomizer()
		guard let customizedFileData = photo.fileDataRepresentation(with: customizer) 
    else 
    {
      // handle failure to get customized ProRAW DNG file data representation
    }
		// use the file data
	}

15:19 - Saving a ProRAW asset with PhotoKit

PHPhotoLibrary.shared().performChanges 
{
    let creationRequest = PHAssetCreationRequest.forAsset()
  
    creationRequest.addResource(with:.photo, 
                                fileURL:proRawFileURL, 
                                options:nil)
} 
completionHandler: 
{ 
  success, error in
  // handle the success and possible error
}

15:45 - Fetching RAW assets from the photo library

// New enum PHAssetCollectionSubtype.smartAlbumRAW

PHAssetCollection.fetchAssetCollections(with: .smartAlbum, 
                                        subtype: .smartAlbumRAW, 
                                        options: nil)

17:16 - Retrieving RAW resources from a PHAsset

let resources = PHAssetResource.assetResources(for: asset)
for resource in resources 
{
  if (resource.type == .photo || resource.type == .alternatePhoto) 
  {
    if let resourceUTType = UTType(resource.uniformTypeIdentifier) 
    {
      if resourceUTType.conforms(to: UTType.rawImage) 
      {
        let resourceManager = PHAssetResourceManager.default()
        resourceManager.requestData(for: resource, options: nil) 
        { 
          data in
          // use the data
        } 
        completionHandler: 
        { 
          error in
          // handle any error 
        }
      }
    }
  }
}

18:28 - Getting CIImages from a ProRAW

// Getting the preview image

let isrc  = CGImageSourceCreateWithURL(url as CFURL, nil)
let cgimg = CGImageSourceCreateThumbnailAtIndex(isrc!, 0, nil)

return CIImage(cgImage: cgimg)

18:36 - Getting CIImages from a ProRAW 2 (New in iOS 15 and macOS 12)

// Getting the preview image

let rawFilter = CIRAWFilter(imageURL: url)

return rawFilter.previewImage

18:44 - Getting CIImages from a ProRAW 3

// Getting the preview image

let rawFilter = CIRAWFilter(imageURL: url)

return rawFilter.previewImage

// Getting segmentation mattes images

return CIImage(contentsOf: url,
               options: [.auxiliarySemanticSegmentationSkinMatte : true])

18:56 - Getting CIImages from a ProRAW 4

// Getting the preview image

let rawFilter = CIRAWFilter(imageURL: url)

return rawFilter.previewImage

// Getting segmentation mattes images

let rawFilter = CIRAWFilter(imageURL: url)

return rawFilter.semanticSegmentationSkinMatte

19:09 - Getting CIImages from a ProRAW 5

// Getting the primary image

return CIImage(contentsOf: url, options:nil)

let rawFilter = CIFilter(imageURL: url, options:nil)

return rawFilter.outputImage

19:31 - Applying common user adjustments

func get_adjusted_raw_image (url: URL) -> CIImage?
{
    // Load the image
    let rawFilter = CIFilter(imageURL: url, options:nil)
    
    // Change one or more filter inputs
    rawFilter.setValue(value, forKey: CIRAWFilterOption.keyName.rawValue)
   
    // Get the adjusted image
    return rawFilter.outputImage
}

19:54 - Applying common user adjustments 2

func get_adjusted_raw_image (url: URL) -> CIImage?
{
    // Load the image
    let rawFilter = CIRAWFilter(imageURL: url)
    
    // Change one or more filter inputs
    rawFilter.property = value
   
    // Get the adjusted image
    return rawFilter.outputImage
}

20:17 - Applying common user adjustments 3

// Exposure

rawFilter.exposure = -1.0

// Temperature and tint

rawFilter.neutralTemperature = 6500  // in °K
rawFilter.neutralTint        = 0.0

// Sharpness

rawFilter.sharpnessAmount = 0.5

// Local tone map strength

rawFilter.localToneMapAmount = 0.5

21:40 - Getting linear scene-referred output 1

// Turn off the filter inputs that apply the default look to the RAW

rawFilter.baselineExposure      = 0.0
rawFilter.shadowBias            = 0.0
rawFilter.boostAmount           = 0.0
rawFilter.localToneMapAmount    = 0.0
rawFilter.isGamutMappingEnabled = false

let linearRawImage = rawFilter.outputImage

22:00 - Getting linear scene-referred output 2

// Use the linear image with other filters

let histogram = CIFilter.areaHistogram()

histogram.inputImage = linearRawImage
histogram.extent     = linearRawImage.extent

// Or render it to a RGBAh buffer

let rd = CIRenderDestination(bitmapData: data.mutableBytes,
                             width: 􀠪imageWidth, 
                             height: 􀠪imageHeight, 
                             bytesPerRow: 􀠪rowBytes,
                             format: .RGBAh)

rd.colorSpace = CGColorSpace(name: CGColorSpace.extendedLinearITUR_2020)

let task = context.startTask(toRender: rawFilter.outputImage, 
                             from: rect, 
                             to: rd, 
                             at: point)

task.waitUntilCompleted()

23:54 - Saving edits to other file formats 1 (8-bit HEIC)

// Saving to 8-bit HEIC

try ciContext.writeHEIFRepresentation(of: rawFilter!.outputImage!,
                                      to: theURL,
                                      format: .RGBA8,
                                      colorSpace: CGColorSpace(name: CGColorSpace.displayP3)!,
                                      options: [:])

24:12 - Saving edits to other file formats 2 (10-bit HEIC)

// Saving to 10-bit HEIC

try ciContext.writeHEIF10Representation(of: rawFilter!.outputImage!,
                                        to: theURL,
                                        format: .RGBA8,
                                        colorSpace: CGColorSpace(name: CGColorSpace.displayP3)!, 
                                        options: [:])

24:51 - Displaying to a Metal Kit View in EDR on Mac

class MyView : MTKView {
  var context: CIContext
  var commandQueue: MTLCommandQueue
  //...
}

25:13 - Displaying to a Metal Kit View in EDR on Mac

// Create a Metal Kit View subclass

class MyView : MTKView {
  var context: CIContext
  var commandQueue: MTLCommandQueue
  //...
}

// Init your Metal Kit View for EDR

colorPixelFormat = MTLPixelFormat.rgba16Float

if let caml = layer as? CAMetalLayer {
  caml.wantsExtendedDynamicRangeContent = true
  //...
}

// Ask the filter for an image designed for EDR and render it

rawFilter.extendedDynamicRangeAmount = 1.0

context.startTask(toRender: rawFilter.outputImage, 
                  from: rect, 
                  to: rd, 
                  at: point)