アプリにビジュアルインテリジェンスを統合するためのベストプラクティス

3:21 - Define the content you want to return as an App Entity

// Define the content you want to return as an App Entity
  import AppIntents
  
  struct AlbumEntity: AppEntity {
      var id: String
      @Property var name: String
      @Property var artistName: String
      var coverArtData: Data
      
      var displayRepresentation: DisplayRepresentation {
          DisplayRepresentation( 
              title: "\(name)",
              subtitle: "\(artistName)",
              image: .init(data: coverArtData)
          )   
      }   
      
      static let defaultQuery = AlbumEntityQuery()
      
      static var typeDisplayRepresentation: TypeDisplayRepresentation { "Album" }
  }   
  
  struct AlbumEntityQuery: EntityQuery {
      @Dependency var catalog: AlbumCatalog
      func entities(for identifiers: [String]) async throws -> [AlbumEntity] {
          catalog.albums(for: identifiers)
      }
  }

5:39 - Adopt IntentValueQuery to return results

// Adopt IntentValueQuery to return visual search results
  import AppIntents
  import VisualIntelligence
  
  struct SearchHandler: IntentValueQuery {
      @Dependency var catalog: AlbumCatalog
      @Dependency var concertFinder: ConcertFinder
      
      func values(for input: SemanticContentDescriptor) async throws -> [VisualSearchResult] {
          guard let pixelBuffer = input.pixelBuffer else {
              return []
          }   
          
          let albums = try await catalog.search(matching: pixelBuffer)
          
          return albums.map { VisualSearchResult.album($0) }
      }
  }

  --- snippet.

6:24 - Build a catalog of albums with precomputed feature prints

// Build a catalog of albums with precomputed feature prints
  import Vision
  
  @Observable
  class AlbumCatalog {
      static let shared = AlbumCatalog()
      
      struct CatalogEntry: Sendable {
          let album: AlbumEntity
          let featurePrint: FeaturePrintObservation
      }   
      
      private(set) var entries: [CatalogEntry] = []
      
      private func generateFeaturePrint(
          for image: CGImage
      ) async throws -> FeaturePrintObservation {
          let request = GenerateImageFeaturePrintRequest()
          let result = try await request.perform(on: image)
          return result
      }
  }

6:45 - Search the catalog for albums matching the captured image

// Search the catalog for albums matching the captured image
  func search(matching pixelBuffer: CVReadOnlyPixelBuffer, limit: Int = 10, maxDistance: Double = 1.0) async throws ->
  [AlbumEntity] {
      var cgImage: CGImage?
      _ = pixelBuffer.withUnsafeBuffer { VTCreateCGImageFromCVPixelBuffer($0, options: nil, imageOut: &cgImage) }
      guard let cgImage else { return [] }
      
      let queryPrint = try await generateFeaturePrint(for: cgImage)
      
      return try entries.compactMap { entry -> (album: AlbumEntity, distance: Double)? in
          let distance = try queryPrint.distance(to: entry.featurePrint)
          guard distance <= maxDistance else { return nil }
          return (entry.album, distance)
      }   
      .sorted { $0.distance < $1.distance }
      .prefix(limit)
      .map { $0.album }
  }   
  
  --

8:27 - Create an open intent to land users on the right screen

// Create an open intent to land users on the right screen
  import AppIntents
  
  struct OpenAlbumIntent: OpenIntent {
      static let title: LocalizedStringResource = "Open Album"
      
      @Parameter(title: "Album")
      var target: AlbumEntity
      
      @Dependency var appState: AppState
      
      func perform() async throws -> some IntentResult {
          await appState.openAlbum(id: target.id)
          return .result()
      }
  }

12:05 - Use UnionValue to return multiple visual search result types

// Use UnionValue to return multiple visual search result types
  @UnionValue
  enum VisualSearchResult {
      case album(AlbumEntity)
      case concert(ConcertEntity)
  }   
  
  struct OpenConcertIntent: OpenIntent {
      static let title: LocalizedStringResource = "Open Concert"
      
      @Parameter(title: "Concert")
      var target: ConcertEntity
      
      @Dependency var appState: AppState
      
      func perform() async throws -> some IntentResult {
          await appState.openConcert(id: target.id)
          return .result()
      }
  }

12:18 - Expand the IntentValueQuery to return the UnionValue

// Expand the IntentValueQuery to return the UnionValue
  struct SearchHandler: IntentValueQuery {
      @Dependency var catalog: AlbumCatalog
      @Dependency var concertFinder: ConcertFinder
      
      func values(for input: SemanticContentDescriptor) async throws -> [VisualSearchResult] {
          guard let pixelBuffer = input.pixelBuffer else {
              return []
          }   
          
          let albums = try await catalog.search(matching: pixelBuffer)
          
          let artists = albums.map { $0.artistName }
          
          let concerts = await concertFinder.findNearby(byArtists: artists)

          return albums.map { VisualSearchResult.album($0) }
              + concerts.map { VisualSearchResult.concert($0) }
      }
  }

13:13 - Provide a link to in-app search

// Provide a link to in-app search
  @AppIntent(schema: .visualIntelligence.semanticContentSearch)
  struct SemanticContentSearchIntent: AppIntent {
      static let title: LocalizedStringResource = "Search in app"
      static let openAppWhenRun: Bool = true
      
      var semanticContent: SemanticContentDescriptor
      @Dependency var catalog: AlbumCatalog
      @Dependency var concertFinder: ConcertFinder
      @Dependency var appState: AppState
      
      func perform() async throws -> some IntentResult {
          guard let pixelBuffer = semanticContent.pixelBuffer else { return .result() }
          let albums = try await catalog.search(matching: pixelBuffer)
          let artists = albums.map { $0.artistName }
          let concerts = await concertFinder.findNearby(byArtists: artists)
          await appState.openSearch(albums: albums, concerts: concerts)
          return .result()
      }   
  }

15:24 - Request calendar access and fetch upcoming concerts

// Request calendar access and fetch upcoming concerts
  import EventKit
  
  @Observable
  class UpcomingConcertManager {
      private let eventStore = EKEventStore()
      var upcomingConcerts: [EKEvent] = []
      var authorizationStatus: EKAuthorizationStatus = .notDetermined
      
      func requestAccessAndFetch() async throws {
          let granted = try await eventStore.requestFullAccessToEvents()
          guard granted else {
              authorizationStatus = .denied
              return
          }   
          authorizationStatus = .fullAccess
          await fetchUpcomingConcerts()

          // ...
      }
  }

15:42 - Filter for upcoming events that match known artists in our catalog

// Filter for upcoming events that match known artists in our catalog
  class UpcomingConcertManager {
      func fetchUpcomingConcerts() async {
          let predicate = eventStore.predicateForEvents(
              withStart: .now,
              end: .now.addingTimeInterval(90 * 24 * 60 * 60),
              calendars: nil
          )   
          
          let events = eventStore.events(matching: predicate)
          
          upcomingConcerts = events.filter { event in
              AlbumCatalog.shared.entries.contains { entry in
                  event.title?.localizedCaseInsensitiveContains(entry.album.artistName) == true
              }
          }
      }
  }

15:44 - Observe newly created events

// Observe newly created events
  @Observable
  class UpcomingConcertManager {
      // ...

      func requestAccessAndFetch() async throws {
          // ...

          for await _ in NotificationCenter.default
              .notifications(
                  named: .EKEventStoreChanged
              ) {
              await fetchUpcomingConcerts()
          }
      }
  }