Explore Packages and Projects with Xcode Playgrounds

8:53 - Playgrounds and resources Demo: Part 1

import UIKit
let image = UIImage(named: "ingredient/orange")

10:18 - Playgrounds and resources Demo: Part 2

import CoreML
let yoloModel = try YOLOv3(configuration: MLModelConfiguration()).model

10:54 - Playgrounds and resources Demo: Part 3

import UIKit
import CoreML
import Vision

let ingredientNames = [
    "banana",
    "orange",
    "almond-milk",
]

let yoloModel = try YOLOv3(configuration: MLModelConfiguration()).model
let model = try VNCoreMLModel(for: yoloModel)

let request = VNCoreMLRequest(model: model) {_,_ in }

11:24 - Recognized Object Visualizer

import Foundation
import SwiftUI
import UIKit

// MARK: Model

/// The result of object detection on an image.
public struct ObjectDetectionResult : Identifiable {
    public var name: String
    public var image: UIImage
    public var id: String
    public var objects: [RecognizedObject]

    public init(name: String, image: UIImage, id: String, objects: [RecognizedObject]) {
        self.id = id
        self.name = name
        self.image = image
        self.objects = objects
    }
}

/// An object recognized by an image classifier.
public struct RecognizedObject : Identifiable {
    public var id: Int
    public var label: String
    public var confidence: Double
    public var boundingBox: CGRect

    public init(id: Int, label: String, confidence: Double, boundingBox: CGRect) {
        self.id = id
        self.label = label
        self.confidence = confidence
        self.boundingBox = boundingBox
    }
}

// MARK: Views

public struct RecognizedObjectVisualizer : View {
    public var results: [ObjectDetectionResult]
    public var imageSize: CGFloat = 400

    public init(withResults results: [ObjectDetectionResult]) {
        self.results = results
    }

    public var body: some View {
        List(results) { result in
            Spacer()

            VStack(alignment: .center) {
                RecognizedObjectsView(
                    image: result.image,
                    objects: result.objects
                )
                .frame(width: imageSize, height: imageSize)

                Text(result.name.capitalized)

                Spacer(minLength: 20)
            }

            Spacer()
        }
    }
}

struct RecognizedObjectsView : View {
    var image: UIImage
    var objects: [RecognizedObject]

    var body: some View {
        GeometryReader { geometry in
            Image(uiImage: image)
                .resizable()
                .overlay(
                    ZStack {
                        ForEach(objects) { object in
                            Rectangle()
                                .stroke(Color.red)
                                .shadow(radius: 2.0)
                                .frame(
                                    width: object.boundingBox.width * geometry.size.width / image.size.width,
                                    height: object.boundingBox.height * geometry.size.height / image.size.height
                                )
                                .position(
                                    x: (object.boundingBox.origin.x + object.boundingBox.size.width / 2.0) * geometry.size.width / image.size.width,
                                    y: geometry.size.height - (object.boundingBox.origin.y + object.boundingBox.size.height / 2.0) * geometry.size.height / image.size.height
                                )
                                .overlay(
                                    Text("\(object.label.capitalized) (\(String(format: "%0.0f", object.confidence * 100.0))%)")
                                        .foregroundColor(Color.red)
                                        .position(
                                            x: (object.boundingBox.origin.x + object.boundingBox.size.width / 2.0) * geometry.size.width / image.size.width,
                                            y: geometry.size.height - (object.boundingBox.origin.y - 20.0) * geometry.size.height / image.size.height
                                        )
                                )
                        }
                    }
                )
        }
    }
}

11:48 - Playgrounds and resources Demo: Part 4

let results = ingredientNames.compactMap { ingredient -> ObjectDetectionResult? in

    guard let image = UIImage(named: "ingredient/\(ingredient)") else { return nil }

    let handler = VNImageRequestHandler(cgImage: image.cgImage!)
    try? handler.perform([request])
    let observations = request.results as! [VNRecognizedObjectObservation]

    let detectedObjects = observations.enumerated().map { (index, observation) -> RecognizedObject in

        // Select only the label with the highest confidence.
        let topLabelObservation = observation.labels[0]
        let objectBounds = VNImageRectForNormalizedRect(observation.boundingBox, Int(image.size.width), Int(image.size.height))

        return RecognizedObject(id: index, label: topLabelObservation.identifier, confidence: Double(topLabelObservation.confidence), boundingBox: objectBounds)
    }

    return ObjectDetectionResult(name: ingredient, image: image, id: ingredient, objects: detectedObjects)
}

results

12:33 - Playgrounds and resources Demo: Part 5

import PlaygroundSupport

PlaygroundPage.current.setLiveView(
    RecognizedObjectVisualizer(withResults: results)
        .frame(width: 500, height: 800)
)