The skeletal model moves

Hi @LazyCoder

Here's a snippet to get you started. The snippet enables you to manipulate a humanoid robot by dragging sphere's attached to its hands ands feet. The snippet uses the model available for download here. For more information on the model format, read Rigging a Model for Motion Capture.

To run the snippet, download the model and include it in your project. The snippet looks for a model named "biped_robot".

Please reach out if you have followup questions. If this answers your question please accept the answer.

Here's the source for ImmersiveView. I'll share the code for the SampleModel in my next response.

struct ImmersiveView: View {
    @State fileprivate var model = SampleModel()
    @State var subscription: EventSubscription?
    @State var humanEntity = Entity()
    
    @State var leftHandTarget:ModelEntity
    @State var rightHandTarget:ModelEntity
    @State var leftFootTarget:ModelEntity
    @State var rightFootTarget:ModelEntity
    let root = Entity()
    
    init() {
        let alpha = 0.65
        leftHandTarget = Self.createTargetEntity(color: .blue.withAlphaComponent(alpha))
        rightHandTarget = Self.createTargetEntity(color: .red.withAlphaComponent(alpha))
        leftFootTarget = Self.createTargetEntity(color: .orange.withAlphaComponent(alpha))
        rightFootTarget = Self.createTargetEntity(color: .green.withAlphaComponent(alpha))
    }
    
    var body: some View {
        RealityView { content in
            guard let character = try? await ModelEntity(named: "biped_robot") else {
                fatalError("Failed to load character model.")
            }
            
            character.transform.translation = [0, 0, -1]
            character.transform.scale = SIMD3(repeating: 1.0)
            humanEntity = character
            
            content.add(character)
            
            try? model.setUpIK(character)
            
            subscription = content.subscribe(to: SceneEvents.Update.self) { event in
                model.updateIK()
            }
            
            root.addChild(character)
            
            addTargetEntities()
            
            root.position = [0, 1.5, -1]
            content.add(root)
            
            initializePositions(for: character)
        }.gesture(DragGesture().targetedToAnyEntity()
            .onChanged { event in
                let entity = event.entity
                guard let parent = entity.parent else {return}
                
                entity.position = event.convert(event.location3D, from: .local, to: parent)
                syncPositions()
            }
        )
    }
    
    private func addTargetEntities() {
        setInitialModelPose()
        
        root.addChild(leftHandTarget)
        root.addChild(rightHandTarget)
        root.addChild(leftFootTarget)
        root.addChild(rightFootTarget)
    }
    
    private func setInitialModelPose() {
        leftHandTarget.position = [1.014, 0.543, -1.109]
        rightHandTarget.position = [-1.014, 0.543, -1.109]
        leftFootTarget.position = [0.31, -1.063, -1.07]
        rightFootTarget.position = [-0.31, -1.063, -1.07]
    }
    
    private func initializePositions(for character: ModelEntity) {
        Task {
            // wait for model to load
            var attempts = 0
            let maxAttempts = 10
            let retryDelayInMilliseconds = 100
            
            while model.setUpTargets(character) == false {
                attempts += 1
                
                if attempts >= maxAttempts {
                    fatalError("Model failed to load in time.")
                }
                
                try? await Task.sleep(for: .milliseconds(retryDelayInMilliseconds))
            }
            
            syncPositions()
        }
    }
    
    private func syncPositions() {
        model.setTargetPosition(limb: .leftHand, targetPosition: leftHandTarget.position(relativeTo: humanEntity))
        model.setTargetPosition(limb: .rightHand, targetPosition: rightHandTarget.position(relativeTo: humanEntity))
        model.setTargetPosition(limb: .leftFoot, targetPosition: leftFootTarget.position(relativeTo: humanEntity))
        model.setTargetPosition(limb: .rightFoot, targetPosition: rightFootTarget.position(relativeTo: humanEntity))
    }
    
    private static func createTargetEntity(color: UIColor) -> ModelEntity {
        let radius:Float = 0.05
        let entity = ModelEntity(
            mesh: .generateSphere(radius: radius),
            materials: [SimpleMaterial(color: color, roughness: 0.9, isMetallic: false)]
        )
        
        entity.components.set(CollisionComponent(shapes: [.generateSphere(radius: radius)]))
        entity.components.set(InputTargetComponent())
        entity.components.set(HoverEffectComponent())
        
        return entity
    }
}

Here's the source for SampleModel.

private class SampleModel {
    var humanEntity: ModelEntity?
    
    // Define entities to manipulate the model.
    var leftHandTargetPosition = SIMD3<Float>()
    var rightHandTargetPosition = SIMD3<Float>()
    var leftFootTargetPosition = SIMD3<Float>()
    var rightFootTargetPosition = SIMD3<Float>()
    
    var modelInitialized = false
    var targetsInitialized = false
    
    enum Joints {
        case hips
        case chest
        case leftHand
        case rightHand
        case leftFoot
        case rightFoot
        
        var name: String {
            switch self {
                
            case .hips:
                "root/hips_joint"
            case .chest:
                "root/hips_joint/spine_1_joint/spine_2_joint/spine_3_joint/spine_4_joint/spine_5_joint/spine_6_joint/spine_7_joint"
            case .leftHand:
                "root/hips_joint/spine_1_joint/spine_2_joint/spine_3_joint/spine_4_joint/spine_5_joint/spine_6_joint/spine_7_joint/left_shoulder_1_joint/left_arm_joint/left_forearm_joint/left_hand_joint"
            case .rightHand:
                "root/hips_joint/spine_1_joint/spine_2_joint/spine_3_joint/spine_4_joint/spine_5_joint/spine_6_joint/spine_7_joint/right_shoulder_1_joint/right_arm_joint/right_forearm_joint/right_hand_joint"
            case .leftFoot:
                "root/hips_joint/left_upLeg_joint/left_leg_joint/left_foot_joint"
            case .rightFoot:
                "root/hips_joint/right_upLeg_joint/right_leg_joint/right_foot_joint"
            }
        }
        
        var constraintName: String {
            switch self {
                
            case .hips:
                "hips_constraint"
            case .chest:
                "chest_constraint"
            case .leftHand:
                "left_hand_constraint"
            case .rightHand:
                "right_hand_constraint"
            case .leftFoot:
                "left_foot_constraint"
            case .rightFoot:
                "right_foot_constraint"
            }
        }
    }
    
    enum Limb {
        case leftHand
        case rightHand
        case leftFoot
        case rightFoot
    }
    
    func setTargetPosition(limb: Limb, targetPosition: SIMD3<Float>) {
        switch limb {
        case .leftHand:
            leftHandTargetPosition = targetPosition
        case .rightHand:
            rightHandTargetPosition = targetPosition
        case .leftFoot:
            leftFootTargetPosition = targetPosition
        case .rightFoot:
            rightFootTargetPosition = targetPosition
        }
    }
    
    @MainActor
    func setUpTargets(_ entity: ModelEntity) -> Bool {
        // Get the limb positions in model space.
        guard let leftHandModelSpace = entity.pins.set(named: "leftHand", skeletalJointName: Joints.leftHand.name).position,
              let rightHandModelSpace = entity.pins.set(named: "rightHand", skeletalJointName: Joints.rightHand.name).position,
              let leftFootModelSpace = entity.pins.set(named: "leftFoot", skeletalJointName: Joints.leftFoot.name).position,
              let rightFootModelSpace = entity.pins.set(named: "rightFoot", skeletalJointName: Joints.rightFoot.name).position else {return false}
        
        leftHandTargetPosition  = leftHandModelSpace
        rightHandTargetPosition = rightHandModelSpace
        leftFootTargetPosition  = leftFootModelSpace
        rightFootTargetPosition = rightFootModelSpace
        
        targetsInitialized = true
        
        return true
    }
    
    func setUpIK(_ entity: ModelEntity) throws {
        // Fetch skeleton for rig
        var skeletonIterator = entity.model?.mesh.contents.skeletons.makeIterator()
        guard let modelSkeleton = skeletonIterator?.next() else {
            fatalError("Skeleton not found in model")
        }
        
        // Start with empty rig for the given model skeleton.
        var rig = try IKRig(for: modelSkeleton)
        
        /// Update global rig settings.
        rig.maxIterations = 30
        rig.globalFkWeight = 0.02
        
        // Define constraints for the rig.
        rig.constraints = [
            .parent(named: Joints.hips.constraintName, on: Joints.hips.name, positionWeight: SIMD3(repeating: 90.0), orientationWeight: SIMD3(repeating: 90.0)),
            .parent(named: Joints.chest.constraintName, on: Joints.chest.name, positionWeight: SIMD3(repeating: 120.0), orientationWeight: SIMD3(repeating: 120.0)),
            .point(named: Joints.leftHand.constraintName,  on: Joints.leftHand.name,  positionWeight: SIMD3(repeating: 10.0)),
            .point(named: Joints.rightHand.constraintName, on: Joints.rightHand.name, positionWeight: SIMD3(repeating: 10.0)),
            .point(named: Joints.leftFoot.constraintName,  on: Joints.leftFoot.name,  positionWeight: SIMD3(repeating: 10.0)),
            .point(named: Joints.rightFoot.constraintName, on: Joints.rightFoot.name, positionWeight: SIMD3(repeating: 10.0)),
        ]
        
        let resource = try IKResource(rig: rig)
        
        entity.components.set(IKComponent(resource: resource))
        humanEntity = entity
        
        modelInitialized = true
    }
    
    @MainActor
    func updateIK() {
        guard modelInitialized,
              targetsInitialized,
              let humanEntity,
              let ikComponent = humanEntity.components[IKComponent.self] else { return }
        
        ikComponent.solvers[0].constraints[Joints.leftHand.constraintName]?.target.translation = leftHandTargetPosition
        ikComponent.solvers[0].constraints[Joints.leftHand.constraintName]?.animationOverrideWeight.position = 1.0
        
        ikComponent.solvers[0].constraints[Joints.rightHand.constraintName]?.target.translation = rightHandTargetPosition
        ikComponent.solvers[0].constraints[Joints.rightHand.constraintName]?.animationOverrideWeight.position = 1.0
        
        ikComponent.solvers[0].constraints[Joints.leftFoot.constraintName]?.target.translation = leftFootTargetPosition
        ikComponent.solvers[0].constraints[Joints.leftFoot.constraintName]?.animationOverrideWeight.position = 1.0
        
        ikComponent.solvers[0].constraints[Joints.rightFoot.constraintName]?.target.translation = rightFootTargetPosition
        ikComponent.solvers[0].constraints[Joints.rightFoot.constraintName]?.animationOverrideWeight.position = 1.0
        
        humanEntity.components.set(ikComponent)
    }
}