In the below code I have extracted face mesh vertices from ARKit face anchors and created a custom face mesh using SceneKit SCNGeometry. This enabled me to stretch face mesh vertices as per my requirement.
Now the problem I am facing is as follows. I am trying to apply a lipstick texture material which is of type SCNMaterial. Although ARSCNFaceGeometry lets me apply different textures through SCNMaterial and SCNNode, I am not able to do the same using mu CustomFaceGeometry. When I am applying a lipstick texture which looks like the image attached below, the full face is getting colored or modified, I want only that part of the face which has texture transparency as >0 and I dont want other part of the face to be modified.
Can you give me a detailed solution using code?
// ViewController.swift
import UIKit
import ARKit
import SceneKit
import simd
class ViewController: UIViewController, ARSCNViewDelegate, ARSessionDelegate{
@IBOutlet weak var sceneView: ARSCNView!
let vertexIndicesOfInterest = [250]
var customFaceGeometry: CustomFaceGeometry!
var scnFaceGeometry: SCNGeometry!
private var faceUvGenerator: FaceTextureGenerator!
var faceGeometry: ARSCNFaceGeometry!
override func viewDidLoad() {
super.viewDidLoad()
sceneView.delegate = self
override func viewWillAppear(_ animated: Bool) {
super.viewWillAppear(animated)
let configuration = ARFaceTrackingConfiguration()
sceneView.session.run(configuration)
}
}
extension ViewController {
func renderer(_ renderer: SCNSceneRenderer, didUpdate node: SCNNode, for anchor: ARAnchor) {
guard let faceAnchor = anchor as? ARFaceAnchor else { return }
customFaceGeometry = CustomFaceGeometry(fromFaceAnchor: faceAnchor)
let customGeometryNode = SCNNode(geometry: customFaceGeometry.geometry)
customFaceGeometry.geometry.firstMaterial?.fillMode = .lines
customFaceGeometry.geometry.firstMaterial?.transparency = 0.0
customFaceGeometry.geometry.firstMaterial?.isDoubleSided = true
node.addChildNode(customGeometryNode)
}
func renderer(_ renderer: SCNSceneRenderer, willUpdate node: SCNNode, for anchor: ARAnchor) {
guard let faceAnchor = anchor as? ARFaceAnchor,
let faceMeshNode = node.childNodes.first else { return }
DispatchQueue.main.async {
self.customFaceGeometry.update(withFaceAnchor: faceAnchor, node: faceMeshNode)
}
}
}
class CustomFaceGeometry {
var geometry: SCNGeometry
let lipImage = UIImage(named: "Face.scnassets/lip_arks_y7.png")
init(fromFaceAnchor faceAnchor: ARFaceAnchor) {
self.geometry = CustomFaceGeometry.createCustomSCNGeometry(from: faceAnchor)!
}
static func createCustomFaceGeometry(fromVertices vertices_o: [SCNVector3]) -> SCNGeometry {
var vertices = vertices_o
let vertexData = Data(bytes: vertices, count: vertices.count * MemoryLayout<SCNVector3>.size)
let vertexSource = SCNGeometrySource(data: vertexData,
semantic: .vertex,
vectorCount: vertices.count,
usesFloatComponents: true,
componentsPerVector: 3,
bytesPerComponent: MemoryLayout<Float>.size,
dataOffset: 0,
dataStride: MemoryLayout<SCNVector3>.stride)
let indices: [Int32] = Array(0..<Int32(vertices.count))
let indexData = Data(bytes: indices, count: indices.count * MemoryLayout<Int32>.size)
let element = SCNGeometryElement(data: indexData, primitiveType: .point, primitiveCount: vertices.count, bytesPerIndex: MemoryLayout<Int32>.size)
return SCNGeometry(sources: [vertexSource], elements: [element])
}
static func createGeometry(fromFaceAnchor faceAnchor: ARFaceAnchor) -> SCNGeometry
let vertices = faceAnchor.geometry.vertices.map { SCNVector3($0.x, $0.y, $0.z) }
return CustomFaceGeometry.createCustomFaceGeometry(fromVertices: vertices)
}
func update(withFaceAnchor faceAnchor: ARFaceAnchor, node: SCNNode) {
if let newGeometry = CustomFaceGeometry.createCustomSCNGeometry(from: faceAnchor) {
node.geometry = newGeometry
let lipstickNode = SCNNode(geometry: newGeometry)
let lipstickTextureMaterial = SCNMaterial()
lipstickTextureMaterial.diffuse.contents = lipImage
lipstickTextureMaterial.transparency = 1.0
lipstickNode.geometry?.firstMaterial = lipstickTextureMaterial
node.geometry?.firstMaterial?.fillMode = .lines
node.geometry?.firstMaterial?.transparency = 0.5
}
}
static func createCustomSCNGeometry(from faceAnchor: ARFaceAnchor) -> SCNGeometry? {
let faceGeometry = faceAnchor.geometry
var vertices: [SCNVector3] = faceGeometry.vertices.map { SCNVector3($0.x, $0.y, $0.z) }
print(vertices[250])
let ll_ratio_y = Float(0.969999)
vertices[290] = SCNVector3(x: vertices[290].x, y: vertices[290].y*ll_ratio_y, z: vertices[290].z)
vertices[274] = SCNVector3(x: vertices[274].x, y: vertices[274].y*ll_ratio_y, z: vertices[274].z)
vertices[265] = SCNVector3(x: vertices[265].x, y: vertices[265].y*ll_ratio_y, z: vertices[265].z)
vertices[700] = SCNVector3(x: vertices[700].x, y: vertices[700].y*ll_ratio_y, z: vertices[700].z)
vertices[730] = SCNVector3(x: vertices[730].x, y: vertices[730].y*ll_ratio_y, z: vertices[730].z)
vertices[25] = SCNVector3(x: vertices[25].x, y: vertices[25].y*ll_ratio_y, z: vertices[25].z)
vertices[709] = SCNVector3(x: vertices[709].x, y: vertices[709].y*ll_ratio_y, z: vertices[709].z)
vertices[725] = SCNVector3(x: vertices[725].x, y: vertices[725].y*ll_ratio_y, z: vertices[725].z)
vertices[710] = SCNVector3(x: vertices[710].x, y: vertices[710].y*ll_ratio_y, z: vertices[710].z)
let vertexData = Data(bytes: vertices, count: vertices.count * MemoryLayout<SCNVector3>.size)
let vertexSource = SCNGeometrySource(data: vertexData, semantic: .vertex, vectorCount: vertices.count, usesFloatComponents: true, componentsPerVector: 3, bytesPerComponent: MemoryLayout<Float>.size, dataOffset: 0, dataStride: MemoryLayout<SCNVector3>.stride)
let indices: [UInt16] = faceGeometry.triangleIndices.map(UInt16.init)
let indexData = Data(bytes: indices, count: indices.count * MemoryLayout<UInt16>.size)
let element = SCNGeometryElement(data: indexData, primitiveType: .triangles, primitiveCount: indices.count / 3, bytesPerIndex: MemoryLayout<UInt16>.size)
return SCNGeometry(sources: [vertexSource], elements: [element])
}
}