I am using ARKit with RealityKit to scan objects using LiDAR on iOS. I can generate an OBJ file from ARMeshAnchors, but I am missing the texture export (JPG + MTL).
What I Have So Far:
Successfully capturing mesh using ARMeshAnchor.
Converting mesh into MDLAsset and exporting .obj.
I need help generating the .jpg texture and linking it to the .mtl file.
private func exportScannedObject() {
guard
let camera = arView.session.currentFrame?.camera
else { return }
func convertToAsset(meshAnchors: [ARMeshAnchor]) -> MDLAsset? {
guard let device = MTLCreateSystemDefaultDevice() else {return nil}
let asset = MDLAsset()
for anchor in meshAnchors {
let mdlMesh = anchor.geometry.toMDLMesh(device: device, camera: camera, modelMatrix: anchor.transform)
// Apply a gray material to the mesh
let material = MDLMaterial(name: "GrayMaterial", scatteringFunction: MDLScatteringFunction())
material.setProperty(MDLMaterialProperty(name: "baseColor", semantic: .baseColor, float3: SIMD3(0.5, 0.5, 0.5))) // Gray color
if let submeshes = mdlMesh.submeshes as? [MDLSubmesh] {
for submesh in submeshes {
submesh.material = material
}
}
asset.add(mdlMesh)
}
return asset
}
func export(asset: MDLAsset) throws -> URL {
let directory = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first!
let url = directory.appendingPathComponent("scaned.obj")
if MDLAsset.canExportFileExtension("obj") {
do {
try asset.export(to: url)
return url
} catch let error {
fatalError(error.localizedDescription)
}
} else {
fatalError("Can't export USD")
}
}
if let meshAnchors = arView.session.currentFrame?.anchors.compactMap({ $0 as? ARMeshAnchor }),
let asset = convertToAsset(meshAnchors: meshAnchors) {
do {
let url = try export(asset: asset)
showScanPreview(url)
} catch {
print("export error")
}
}
}
extension ARMeshGeometry {
func vertex(at index: UInt32) -> SIMD3<Float> {
assert(vertices.format == MTLVertexFormat.float3, "Expected three floats (twelve bytes) per vertex.")
let vertexPointer = vertices.buffer.contents().advanced(by: vertices.offset + (vertices.stride * Int(index)))
let vertex = vertexPointer.assumingMemoryBound(to: SIMD3<Float>.self).pointee
return vertex
}
// helps from StackOverflow:
// https://stackoverflow.com/questions/61063571/arkit-3-5-how-to-export-obj-from-new-ipad-pro-with-lidar
func toMDLMesh(device: MTLDevice, camera: ARCamera, modelMatrix: simd_float4x4) -> MDLMesh {
func convertVertexLocalToWorld() {
let verticesPointer = vertices.buffer.contents()
for vertexIndex in 0..<vertices.count {
let vertex = self.vertex(at: UInt32(vertexIndex))
var vertexLocalTransform = matrix_identity_float4x4
vertexLocalTransform.columns.3 = SIMD4<Float>(x: vertex.x, y: vertex.y, z: vertex.z, w: 1)
let vertexWorldPosition = (modelMatrix * vertexLocalTransform).columns.3
let vertexOffset = vertices.offset + vertices.stride * vertexIndex
let componentStride = vertices.stride / 3
verticesPointer.storeBytes(of: vertexWorldPosition.x, toByteOffset: vertexOffset, as: Float.self)
verticesPointer.storeBytes(of: vertexWorldPosition.y, toByteOffset: vertexOffset + componentStride, as: Float.self)
verticesPointer.storeBytes(of: vertexWorldPosition.z, toByteOffset: vertexOffset + (2 * componentStride), as: Float.self)
}
}
convertVertexLocalToWorld()
let allocator = MTKMeshBufferAllocator(device: device);
let data = Data.init(bytes: vertices.buffer.contents(), count: vertices.stride * vertices.count);
let vertexBuffer = allocator.newBuffer(with: data, type: .vertex);
let indexData = Data.init(bytes: faces.buffer.contents(), count: faces.bytesPerIndex * faces.count * faces.indexCountPerPrimitive);
let indexBuffer = allocator.newBuffer(with: indexData, type: .index);
let submesh = MDLSubmesh(indexBuffer: indexBuffer,
indexCount: faces.count * faces.indexCountPerPrimitive,
indexType: .uInt32,
geometryType: .triangles,
material: nil);
let vertexDescriptor = MDLVertexDescriptor();
vertexDescriptor.attributes[0] = MDLVertexAttribute(name: MDLVertexAttributePosition,
format: .float3,
offset: 0,
bufferIndex: 0);
vertexDescriptor.layouts[0] = MDLVertexBufferLayout(stride: vertices.stride);
let mesh = MDLMesh(vertexBuffer: vertexBuffer,
vertexCount: vertices.count,
descriptor: vertexDescriptor,
submeshes: [submesh])
return mesh
}
}
What I Need Help With:
How do I generate the JPG texture from the AR scene?
How do I save an MTL file linking the OBJ model to the texture?
How can I correctly apply the texture when viewing the OBJ in an external 3D viewer?
I appreciate any guidance, including sample code or resources! If you have a complete working solution, I’d love to discuss further via private channels.