I'm still trying to understand how to correctly convert 3D coordinates to 2D screen coordinates using convert(position:from:) and project(_:)
Below is the example ContentView.swift from the default Augmented Reality App project, with a few important modifications. Two buttons have been added, one that toggles visibility of red circular markers on the screen, and a second button that adds blue spheres to the scene. Additionally a timer has been added to trigger regular screen updates.
When run, the markers should line up with the spheres on screen and follow them on screen, as the camera is moved around. However, the red circles are all very far from their corresponding spheres on screen.
What am I doing wrong in my conversion that is causing the circles to not line up with the spheres?
// ContentView.swift
import SwiftUI
import RealityKit
class Coordinator {
var arView: ARView?
var anchor: AnchorEntity?
var objects: [Entity] = []
}
struct ContentView : View {
let timer = Timer.publish(every: 1.0/30.0, on: .main, in: .common).autoconnect()
var coord = Coordinator()
@State var showMarkers = false
@State var circleColor: Color = .red
var body: some View {
ZStack {
ARViewContainer(coordinator: coord).edgesIgnoringSafeArea(.all)
if showMarkers {
// Add circles to the screen
ForEach(coord.objects) { obj in
Circle()
.offset(projectedPosition(of: obj))
.frame(width: 10.0, height: 10.0)
.foregroundColor(circleColor)
}
}
VStack {
Button(action: { showMarkers = !showMarkers },
label: { Text(showMarkers ? "Hide Markers" : "Show Markers") })
Spacer()
Button(action: { addSphere() },
label: { Text("Add Sphere") })
}
}.onReceive(timer, perform: { _ in
// silly hack to force circles to redraw
if circleColor == .red {
circleColor = Color(#colorLiteral(red: 1, green: 0, blue: 0, alpha: 1))
} else {
circleColor = .red
}
})
}
func addSphere() {
guard let anchor = coord.anchor else { return }
// pick random point for new sphere
let pos = SIMD3<Float>.random(in: 0...0.5)
print("Adding sphere at \(pos)")
// Create a sphere
let mesh = MeshResource.generateSphere(radius: 0.01)
let material = SimpleMaterial(color: .blue, roughness: 0.15, isMetallic: true)
let model = ModelEntity(mesh: mesh, materials: [material])
model.setPosition(pos, relativeTo: anchor)
anchor.addChild(model)
// record sphere for later use
coord.objects.append(model)
}
func projectedPosition(of object: Entity) -> CGPoint {
// convert position of object into "world space"
// (i.e., "the 3D world coordinate system of the scene")
// https://developer.apple.com/documentation/realitykit/entity/convert(position:to:)
let worldCoordinate = object.convert(position: object.position, to: nil)
// project worldCoordinate into "the 2D pixel coordinate system of the view"
// https://developer.apple.com/documentation/realitykit/arview/project(_:)
guard let arView = coord.arView else { return CGPoint(x: -1, y: -1) }
guard let screenPos = arView.project(worldCoordinate) else { return CGPoint(x: -1, y: -1) }
// At this point, screenPos should be the screen coordinate of the object's positions on the screen.
print("3D position \(object.position) mapped to \(screenPos) on screen.")
return screenPos
}
}
struct ARViewContainer: UIViewRepresentable {
var coordinator: Coordinator
func makeUIView(context: Context) -> ARView {
let arView = ARView(frame: .zero)
// Create a sphere model
let mesh = MeshResource.generateSphere(radius: 0.01)
let material = SimpleMaterial(color: .gray, roughness: 0.15, isMetallic: true)
let model = ModelEntity(mesh: mesh, materials: [material])
// Create horizontal plane anchor for the content
let anchor = AnchorEntity(.plane(.horizontal, classification: .any, minimumBounds: SIMD2<Float>(0.2, 0.2)))
anchor.children.append(model)
// Record values needed elsewhere
coordinator.arView = arView
coordinator.anchor = anchor
coordinator.objects.append(model)
// Add the horizontal plane anchor to the scene
arView.scene.anchors.append(anchor)
return arView
}
func updateUIView(_ uiView: ARView, context: Context) {}
}
#Preview {
ContentView()
}