Get Reference Image name when tapping ModelEntity

Question

I made the following code to create a Plane with VideoMaterial whenever a Reference Image is detected. It's working great, but I need to get the Name of the corresponding Reference Image when I tap on Plane ModelEntity that's playing a video and I don't know how to achieve it in RealityKit. (SceneKit solution won't help me unfortunately)

class Coordinator: NSObject, ARSessionDelegate {
    var parent: ARViewContainer
    var videoPlayer = AVPlayer()
    
    init(parent: ARViewContainer) {
        self.parent = parent
    }
    
    func session(_ session: ARSession, didAdd anchors: [ARAnchor]) {
        
        guard let validAnchor = anchors[0] as? ARImageAnchor else { return }
        
        let anchor = AnchorEntity(anchor: validAnchor)
        anchor.addChild(createdVideoPlayerNodeFor(validAnchor.referenceImage))
        parent.arView.scene.addAnchor(anchor)
    }
    
    func createdVideoPlayerNodeFor(_ target: ARReferenceImage) -> ModelEntity {

        var videoPlane = ModelEntity()
        if let targetName = target.name,
           let validURL = Bundle.main.url(forResource: targetName, withExtension: "mp4") {
            videoPlayer = AVPlayer(url: validURL)
            videoPlayer.play()
        }
        let videoMaterial = VideoMaterial(avPlayer: videoPlayer)
        videoPlane = ModelEntity(mesh: .generatePlane(width: Float(target.physicalSize.width), 
                                depth: Float(target.physicalSize.height)), 
                            materials: [videoMaterial])
        print (target.name as Any)
        return videoPlane
    }
}

Needs more source code where user making tap on plane model. — Ramis, Nov 21 '22 at 05:52

Andy Jazz · Accepted Answer · 2023-01-06T14:10:26.050

UIKit

Getting reference image name.

If you wanna use pure RealityKit (without any boilerplate code), implement the following approach:

import UIKit
import RealityKit

class ViewController : UIViewController {
    
    @IBOutlet var rkView: ARView!
    
    override func viewDidLoad() {
        super.viewDidLoad()
        
        let model = try! Entity.loadModel(named: "model")
        model.model?.materials[0] = UnlitMaterial(color: .red)
        model.generateCollisionShapes(recursive: true)
        let anchor = AnchorEntity(.image(group: "AR Resources", name: "img01"))
        anchor.addChild(model)
        rkView.scene.anchors.append(anchor)
    }
    
    override func touchesBegan(_ touches: Set<UITouch>,
                              with event: UIEvent?) {
        
        guard let point = touches.first?.location(in: rkView) else { return }
        
        let ray = rkView.ray(through: point)
                
        let castHits = rkView.scene.raycast(origin: ray?.origin ?? [0,0,0], 
                                         direction: ray?.direction ?? [0,0,0])

        guard let castHit: CollisionCastHit = castHits.first else { return }
        
        print((castHit.entity.anchor?.anchoring.target)!)
    }
}

Tap the model.

Result:

//      image(group: "AR Resources", name: "img01")

Substring

To extract a substring containing just a name of a reference image, use this code:

let str = "\((castHit.entity.anchor?.anchoring.target)!)"
            
let start = str.index(str.startIndex, offsetBy: 35)
let end = str.index(str.endIndex, offsetBy: -1)
let name = str[start..<end]
    
print(name)

Result:

//      "img01"

SwiftUI

SwiftUI solution looks different, since you have to update a state, not an event like in UIKit.

import SwiftUI
import RealityKit

struct ARContainer: UIViewRepresentable {
    
    let arView = ARView(frame: .zero)
    @Binding var point: CGPoint
    @Binding var stateSwitcher: Bool
    
    func makeUIView(context: Context) -> ARView {
        
        let model = ModelEntity(mesh: .generateSphere(radius: 0.25))
        model.generateCollisionShapes(recursive: true)
        model.name = "Sphere"
        let anchor = AnchorEntity(.image(group: "...", name: "..."))
        anchor.addChild(model)
        arView.scene.anchors.append(anchor)
        return arView
    }
    func updateUIView(_ view: ARView, context: Context) {
        
        let ray = view.ray(through: point)
        let castHits = view.scene.raycast(origin: ray?.origin ?? [0,0,0],
                                       direction: ray?.direction ?? [0,0,0])
        guard let castHit: CollisionCastHit = castHits.first else { return }
        
        if !stateSwitcher || stateSwitcher {
            print(castHit.entity.name)
        }
    }
}

For switching a state I used toggle() method. This solution takes into account the fact that you can double-click or triple-click on the same point on the screen - but the state will still be updated.

@available(iOS 16.0, macOS 13.0, *)
struct ContentView: View {
    
    @State private var point = CGPoint.zero
    @State private var stateSwitcher = true
    
    var body: some View {
        ARContainer(point: $point, stateSwitcher: $stateSwitcher)
            .onTapGesture {
                point = CGPoint(x: Int($0.x), y: Int($0.y))
                print(point.x)
                stateSwitcher.toggle()
            }
            .ignoresSafeArea()
    }
}

Would it be possible to do this without the use of UIKit, let's say in SwiftUI for example? — user1207524, Dec 31 '22 at 00:11
Yes, it's possible to do this in SwiftUI, however, the implementation will be different. — Andy Jazz, Dec 31 '22 at 05:28
Would you be perhaps able to tell me how to implement it in SwiftUI or at least point me into the right direction, please? — user1207524, Jan 02 '23 at 23:52

Get Reference Image name when tapping ModelEntity

1 Answers1

UIKit

Substring

SwiftUI