Vision Pro - Throw object by hand

HI @Mathis06

It sounds like you want to use a custom gesture to pick up and throw an entity. Before you do this I want to make sure pinch and gaze won't work for you. The code to pickup and throw using DragGesture is simpler than implementing a custom gesture. Please let me know if that works for you and I'll post the code. With that, I'll assume DragGesture won't work for you and you prefer a custom gesture.

In this example, a person picks something up by putting their open hand inside of an Entity and closing it. The next time they open their hand the object is thrown in the direction their hand is traveling, at the speed their hand is moving.

To determine if a persons hand is close enough to an entity to pick it up, create entities attached to a person's hand. When the they collide with an entity that has a CollisionComponent its time to start monitoring if the hand goes from open to closed.

struct PalmAnchor {
    static func make(_ chirality: AnchoringComponent.Target.Chirality) -> AnchorEntity {
        let palm = AnchorEntity(.hand(chirality, location: .palm))
        
        // This next line is important.
        // Without it the anchor will run it its own physics simulation
        // so collisions between the anchors and the boxes will not register.
        palm.anchoring.physicsSimulation = .none
        let height:Float = 0.02
        
        let shape = ShapeResource.generateBox(width: 0.1, height: height, depth: 0.1)
        palm.components.set(CollisionComponent(shapes: [shape]))
        palm.components.set(ThrowByHandComponent())
        
        return palm
    }
}

Next we create a custom component and system Entity Component System to monitor and respond to changes in a person's hand pose.

struct ThrowByHandComponent: Component {
    var lastPosition:SIMD3<Float>?
    var handPose = HandPoseHelper.HandPose.indeterminate
    var lastHandPose = HandPoseHelper.HandPose.indeterminate
    var entityCollidedWith:ModelEntity?
    var entityCollidedWithPreviousParent:Entity?
}

struct ThrowByHandSystem: System {
    let query = EntityQuery(where: .has(ThrowByHandComponent.self))
    
    public init(scene: RealityKit.Scene) { }
    
    public func update(context: SceneUpdateContext) {
        let entities = context.entities(matching: self.query, updatingSystemWhen: .rendering)
        
        for entity in entities {
            
            if var component = entity.components[ThrowByHandComponent.self] {
                let currentPosition = entity.position(relativeTo: nil)
                
                let handPose = component.handPose
                
                if handPose != .indeterminate {
                    let lastHandPose = component.lastHandPose
                    
                    if lastHandPose != handPose,
                       let entityCollidedWith = component.entityCollidedWith {
                        
                        // Hand went from open to close while under collision.
                        // Time to pickup the object
                        if handPose == .closed {
                            // Change the physics mode to kinematic so we can control its position.
                            entityCollidedWith.components[PhysicsBodyComponent.self]?.mode = .kinematic
                            
                            // Pickup the entity - Reparent to entity a person picked up with its corresponding hand anchor
                            // so the entity moves with the hands. Save the entity's previous parent so
                            // you can restore it later.
                            component.entityCollidedWithPreviousParent = entity.parent
                            entity.addChild(entityCollidedWith, preservingWorldTransform: true)
                        } // Hand went from closed to open. Time to throw
                        else if handPose == .open,
                                let lastPosition = component.lastPosition,
                                let entityCollidedWith = component.entityCollidedWith {
                            let distance = distance(lastPosition, currentPosition)
                            let speed = distance / Float(context.deltaTime)
                            
                            // Change this to make the throws harder of softer.
                            let forceMultiplier:Float = 2.0
                            
                            // Throwing the entity:
                            
                            // If the box is not moving, drop it,
                            // otherwise throw in the direction the person's hand is traveling.
                            let throwDirection = currentPosition == lastPosition ? [0, 0, 0] : normalize(currentPosition - lastPosition)
                            let linearImpulse = throwDirection * speed * forceMultiplier
                            
                            // Change the physics mode to dynamic so its  position is controlled by
                            // the physics simulation.
                            entityCollidedWith.components[PhysicsBodyComponent.self]?.mode = .dynamic
                            
                            // Add the component back to the parent it had before it was picked up
                            // and added to the hand anchor entity.
                            component.entityCollidedWithPreviousParent?.addChild(entityCollidedWith, preservingWorldTransform: true)
                            component.entityCollidedWithPreviousParent = nil
                            entityCollidedWith.applyLinearImpulse(linearImpulse, relativeTo: nil)
                        }
                        
                    }
                    
                    component.lastHandPose = handPose
                }
                
                component.lastPosition = currentPosition
                entity.components[ThrowByHandComponent.self] = component
            }
        }
    }
}

Continued in next post.

Next define the helper used by ThrowByHandSystem to determine if a hand is opened or closed.

struct HandPoseHelper {
    // Define how closed a finger must be before it's considered closed.
    // Closer to 1 means the finger is open.
    // Closer to -1 means the finger is closed.
    let similarityThreshold:Float = 0.2
    let jointsToConsider:[HandSkeleton.JointName] = [
        .indexFingerTip,
        .middleFingerTip,
        .ringFingerTip,
        .littleFingerTip,
    ]
    
    let handAnchor: HandAnchor
    enum HandPose {
        // All the fingers in jointsToConsider are closed
        case closed
        // All the fingers in jointsToConsider are open
        case open
        // Some fingers are open and some are closed
        case indeterminate
    }
    
    func handPose() -> HandPose {
        guard let skeleton = handAnchor.handSkeleton else {return .indeterminate}
        let allJointsTracked = skeleton.allJoints.contains(where: {$0.isTracked == false}) == false
        
        guard allJointsTracked else {return .indeterminate}
        
        let numFingersOpen = jointsToConsider.reduce(0) { sum, jointName in
            let similarity = similarityToKnuckle(skeleton.joint(jointName))
            
            if similarity > similarityThreshold {
                return sum + 1
            }
            
            return sum
        }
        
        if numFingersOpen == 0 {
            return .closed
        }
        
        if numFingersOpen == jointsToConsider.count {
            return .open
        }
        
        return .indeterminate
    }
    
    func similarityToKnuckle(_ fingerTip:HandSkeleton.Joint) -> Float {
        let intermediateTip = fingerTip.parentJoint!
        let intermediateBase = intermediateTip.parentJoint!
        let knuckle = intermediateBase.parentJoint!
        let fingerTipDirection = fingerTip.anchorFromJointTransform.columns.0
        let knuckleDirection = knuckle.anchorFromJointTransform.columns.0
        let similarity = dot(fingerTipDirection, knuckleDirection)

        return similarity
    }
}

Finally put it all together using a reality view in an immersive space.

struct ImmersiveView: View {
    let leftPalm = PalmAnchor.make(.left)
    let rightPalm = PalmAnchor.make(.right)
    let arkitSession = ARKitSession()
    let spatialTrackingSession = SpatialTrackingSession()
    @State var tasks:Set<Task<(), Never>> = []
    
    init() {
        ThrowByHandSystem.registerSystem()
        ThrowByHandComponent.registerComponent()
    }
    
    var body: some View {
        RealityView { content in
            // Build a floor so the cube has something to land on.
            // You'll likely want to use plane detection or scene reconstruction
            content.add(buildFloor())
            
            // Add some cubes to throw around
            for x in -5...5 {
                if let throwable = buildThrowable() {
                    let width = throwable.visualBounds(relativeTo: nil).extents.x * 1.1
                    throwable.position = [Float(x) * width, 0.5, -0.5]
                    content.add(throwable)
                }
            }
            
            content.add(leftPalm)
            content.add(rightPalm)
            
            _ = content.subscribe(to: CollisionEvents.Began.self, on: nil) {
                event in
                let entityA = event.entityA
                let entityB = event.entityB
                entityA.components[ThrowByHandComponent.self]?.entityCollidedWith = entityB as? ModelEntity
            }
            
            _ = content.subscribe(to: CollisionEvents.Ended.self, on: nil) {
                event in
                let entityA = event.entityA
                entityA.components[ThrowByHandComponent.self]?.entityCollidedWith = nil
            }
        }.task {
            // Start a spatial tracking session so our hand anchor entities can receive collisions
            
            let configuration = SpatialTrackingSession.Configuration(
                tracking: [.hand])
            await spatialTrackingSession.run(configuration)
        }.task {
            do {
                // Track the hands so we can determine if the hands are open or closed
                let handTrackingProvider = HandTrackingProvider()
                try await arkitSession.run([handTrackingProvider])
                
                tasks.insert(
                    Task {
                        await processHandTrackingUpdates(handTrackingProvider: handTrackingProvider)
                    }
                )
            } catch {
                assertionFailure("Failed to start hand tracking provider \(error)")
            }
        }
    }
    
    func buildThrowable() -> Entity? {
        let throwable = ModelEntity(mesh: .generateBox(size: 0.2), materials: [SimpleMaterial(color: .red, isMetallic: true)])
        
        throwable.generateCollisionShapes(recursive: true)
        
        guard let shapes = throwable.collision?.shapes else {
            assertionFailure("Failed to read shapes.")
            
            return nil
        }
        
        throwable.components.set(PhysicsBodyComponent(
            shapes: shapes,
            mass: 1.0,
            material: PhysicsMaterialResource.generate(friction: 0.8, restitution: 0.1),
            mode: .dynamic))
        
        return throwable
    }
    
    func buildFloor() -> Entity {
        let floor = Entity()
        let shapes = [ShapeResource.generateBox(width: 30, height: 0.01, depth: 30)]
        
        floor.components.set(CollisionComponent(shapes: shapes))
        floor.components.set(PhysicsBodyComponent(
            shapes: shapes,
            mass: 1.0,
            material: PhysicsMaterialResource.generate(friction: 0.8, restitution: 0),
            mode: .static))
        
        return floor
    }
    
    func processHandTrackingUpdates(handTrackingProvider: HandTrackingProvider) async {
        print("fire")
        for await update in handTrackingProvider.anchorUpdates {
            let anchor = update.anchor

            switch anchor.chirality {
            case .left:
                leftPalm.components[ThrowByHandComponent.self]?.handPose = HandPoseHelper(handAnchor: anchor).handPose()
            case .right:
                rightPalm.components[ThrowByHandComponent.self]?.handPose = HandPoseHelper(handAnchor: anchor).handPose()
            }
        }
    }
}

To help protect people’s privacy, visionOS limits app access to cameras and other sensors in Apple Vision Pro. You need to add an NSWorldSensingUsageDescription and NSHandsTrackingUsageDescription to your app’s information property list to provide a usage description that explains how your app uses the data these sensors provide.

Hello Apple, Ok understand need to have VisionPro OS 2 and last XCode Beta. I've no word, your sample runs very well and enjoy me. I can make many many things from this. "I love you" !!! Thanks and thanks and thanks !!!! Mathis

Hello Me again,

I've tried to change one ligne of your code to have ball in place of the cube. I've replaced

by:		let throwable = ModelEntity(mesh: .generateSphere(radius: 0.1), materials: [SimpleMaterial(color: .red, isMetallic: true)])

and what's funny is the ball not roll when are o the ground, or, to be clear, they roll really strange :(

other test is to use my ball I've in my scene :

				{
					Set_Entity_Params( entity: temp, mass: 0.70, friction: kBoules_Friction, rebond: 0.0005, ralentissement: 1, mode: .static)
					temp.components.set( ImageBasedLightComponent(source: .single( environmentLight!)))
					temp.components.set( ImageBasedLightReceiverComponent( imageBasedLight: temp))
					temp.isEnabled = true
					mLesBoules.append( temp)
					temp.generateCollisionShapes(recursive: true)
				}

	{
		entity.components.set( InputTargetComponent( ))// Permet d'etre saisisable à la main
		
		entity.components.set( GroundingShadowComponent(castsShadow: true))

		var spherePhysics = PhysicsBodyComponent( massProperties: .init(mass: mass), material: .generate(staticFriction: friction, dynamicFriction: friction, restitution: rebond), mode: mode)
		spherePhysics.isRotationLocked = (x:false, y:false, z:false)
		spherePhysics.isContinuousCollisionDetectionEnabled = false
		spherePhysics.angularDamping = 0.9 // rotation de fin de mouvement (+ il roule moins longtemps)
		spherePhysics.linearDamping = ralentissement // amortissement linéaire : Plus le chiffre est haut moins la boule part loin (comme si le terrain etait sablonneux)
		entity.components.set( spherePhysics)
	}

And all running pretty well, I can get my ball in hand, move it, but when I release it from my hand, it disappeared :( Linear impulse it pretty the same than I've with your cubes. But, I don't know why my ball goes very very far of me, also if I make no move, just drop it in place..

I hope you could help me because I've consumed this day and I begun to be crazy 8o(

Cheers and thanks for your help !!! Mathis

Oppsss, it removed my first code line :(

				if let temp = scene.findEntity(named: "Boule_A")
				{
					Set_Entity_Params( entity: temp, mass: 0.70, friction: kBoules_Friction, rebond: 0.0005, ralentissement: 1, mode: .static)
					temp.components.set( ImageBasedLightComponent(source: .single( environmentLight!)))
					temp.components.set( ImageBasedLightReceiverComponent( imageBasedLight: temp))
					temp.isEnabled = true
					mLesBoules.append( temp)
					temp.generateCollisionShapes(recursive: true)
				}

and the function :

	func Set_Entity_Params( entity: Entity, mass: Float, friction: Float, rebond: Float, ralentissement: Float, mode: PhysicsBodyMode = .dynamic)
	{
		entity.components.set( InputTargetComponent( ))// Permet d'etre saisisable à la main
		entity.components.set( GroundingShadowComponent(castsShadow: true))

		var spherePhysics = PhysicsBodyComponent( massProperties: .init(mass: mass), material: .generate(staticFriction: friction, dynamicFriction: friction, restitution: rebond), mode: mode)
		spherePhysics.isRotationLocked = (x:false, y:false, z:false)
		spherePhysics.isContinuousCollisionDetectionEnabled = false
		spherePhysics.angularDamping = 0.9 // rotation de fin de mouvement (+ il roule moins longtemps)
		spherePhysics.linearDamping = ralentissement // amortissement linéaire : Plus le chiffre est haut moins la boule part loin (comme si le terrain etait sablonneux)
		entity.components.set( spherePhysics)
	}

Ok soled, I've changed the velocity index of "let forceMultiplier:Float = 2.0".

From my move, the hand (closed fingers) could be out of the Vision pro cameras (but it is always in scope when finger be opened). When the hand is out of Vision Camera, the code thinks the hand is opened. Existing a way to have the hand considering closed in this case please ?

Cheers and THANKS Mathis

Hi @Mathis06

Glad you like the code! Happy to help.

and what's funny is the ball not roll when are o the ground, or, to be clear, they roll really strange :(

The method generateCollisionShapes always generates a box collision shape. If you change the mesh to a sphere you need to manually create and set the collision shapes to a sphere.

let shape = ShapeResource.generateSphere(radius: radius)
throwable.components.set(CollisionComponent(shapes: [shape]))

When the hand is out of Vision Camera, the code thinks the hand is opened. Existing a way to have the hand considering closed in this case please ?

Change

guard allJointsTracked else {return .indeterminate}

to

guard allJointsTracked else {return .closed}

AMAZING !!!! big big big thanks !!!

Hello @Vision Pro Engineer, fantastic code! I am using your code in order to grab a key and insert it in a door lock. When I put the key inside the hole, I would like to fix the key in that position and rotate it only on z axis.

I tried two different approaches. For the first one, I added in the ImmersiveView this piece of code:

            _ = content.subscribe(to: CollisionEvents.Began.self, on: nil) { event in
                let entityA = event.entityA
                let entityB = event.entityB
                
                let modelEntity = entityB.findEntity(named: "Chiave_TriplaShape") as? ModelEntity
                
                if event.entityB.name == "Chiave_TriplaShape", var physicsBody = entityB.components[PhysicsBodyComponent.self], isInserted, !hasRotated {
                    physicsBody.mode = .dynamic

                    // Lock rotation by setting angular damping to a high value
                    physicsBody.isTranslationLocked = (true, true, true)
                    physicsBody.isRotationLocked = (true, true, false)
                    entityB.components[PhysicsBodyComponent.self] = physicsBody
                }
                
                // insert the key in the unit and disabling dynamic mode
                if event.entityA.name == "ErlenmeyerFlask" && event.entityB.name == "Chiave_TriplaShape", !isInserted {
                    entityB.transform.scale = SIMD3<Float>(x: 0.03, y: 0.03, z: 0.03)
                    entityB.transform.rotation = simd_quatf(real: 0.5963677, imag: SIMD3<Float>(0.59636784, 0.37992808, -0.3799282))
                    entityB.transform.translation = SIMD3<Float>(x: 0.52, y: 1, z: -0.7)
                    
                    if var physicsBody = entityB.components[PhysicsBodyComponent.self] {
                        physicsBody.mode = .kinematic
                        entityB.components.set(physicsBody)
                    }
                    isInserted = true
                }
                
                if modelEntity == nil { return }
                entityA.components[ThrowByHandComponent.self]?.entityCollidedWith = modelEntity
                
            }

For the second approach, I tried instead to add this piece of code in update function of ThrowByHandSystem:

                    let lastHandPose = component.lastHandPose
                    
                    print(component.entityCollidedWithIsInserted)
                    
                    if lastHandPose != handPose,  let entityCollidedWith = component.entityCollidedWith  {
                        if handPose == .closed {
                            entityCollidedWith.components[PhysicsBodyComponent.self]?.mode = .kinematic
                            component.entityCollidedWithPreviousParent = entity.parent
                            entity.addChild(entityCollidedWith, preservingWorldTransform: true)
                        }
                        
                        else if handPose == .closed, component.entityCollidedWithIsInserted, !component.entityCollidedWithIsRotated, let lastOrientation = component.lastOrientation {
                            let rotation = currentOrientation * simd_inverse(lastOrientation)
                            let angularSpeed = rotation.angle / Float(context.deltaTime)
                            let angularForceMultiplier: Float = 2.0
                            let isRotating = rotation.angle > Float.pi / 2
                            let angularImpulse = isRotating ? normalize(rotation.axis) * angularSpeed * angularForceMultiplier : [0, 0, 0]
                       entityCollidedWith.components[PhysicsBodyComponent.self]?.mode = .dynamic
                            component.entityCollidedWithPreviousParent?.addChild(entityCollidedWith, preservingWorldTransform: true)
                            component.entityCollidedWithPreviousParent = nil
                            entityCollidedWith.applyAngularImpulse(angularImpulse, relativeTo: nil)      
                        } else if handPose == .open, let lastPosition = component.lastPosition {
                            let distance = distance(lastPosition, currentPosition)
                            let speed = distance / Float(context.deltaTime)
                            let forceMultiplier: Float =  2.0
                            let isStationary = distance < 0.01
                            let throwDirection = isStationary ? [0, 0, 0] : normalize(currentPosition - lastPosition)
                            let linearImpulse = throwDirection * speed * forceMultiplier

                          entityCollidedWith.components[PhysicsBodyComponent.self]?.mode = .dynamic
                            component.entityCollidedWithPreviousParent?.addChild(entityCollidedWith, preservingWorldTransform: true)
                            component.entityCollidedWithPreviousParent = nil
                            entityCollidedWith.applyLinearImpulse(linearImpulse, relativeTo: nil)
                        }
                    }

                    component.lastHandPose = handPose
                }

Neither of the two approaches seems to work. Do you have any suggestions?

Thank you very much!

Synchron said afferent and efferent nerves teach each other to build neural nets.

I’ve expressed interest in being a beta tester of a teslas haptic suit which can give the sensation of moving that will grow these neural nets to effect motion. My wife’s paralyzed and I’d like to use this loopiness.