Get Size of image in SCNNode / ARKit Swift

Question

I'm trying to scan a Reference-Image an then display the image itself above the printed reference-image. The "virutal" image size should be the same like the printed size.

My idea: get the size of the printed Reference-Image, then scale the image in the SCNNode to this size (or scale the SCNNode to this size?)

But: 1-> How to get the size of the printed image, 2-> for scaling the SCNNode I need the size of this node, too. How to get it?

import UIKit
import SceneKit
import ARKit
import AVKit
import AVFoundation

class ViewController: UIViewController, ARSCNViewDelegate {

@IBOutlet var sceneView: ARSCNView!
private var planeNode: SCNNode?
private var imageNode: SCNNode?
private var animationInfo: AnimationInfo?
private var currentMediaName: String?
private var scrollView: UIScrollView!

override func viewDidLoad() {
    super.viewDidLoad()

    let scene = SCNScene()
    sceneView.scene = scene
    sceneView.delegate = self

}

override func viewWillAppear(_ animated: Bool) {
    super.viewWillAppear(animated)

    // Load reference images to look for from "AR Resources" folder
    guard let referenceImages = ARReferenceImage.referenceImages(inGroupNamed: "AR Resources", bundle: nil) else {
        fatalError("Missing expected asset catalog resources.")
    }

    // Create a session configuration
    let configuration = ARWorldTrackingConfiguration()

    // Add previously loaded images to ARScene configuration as detectionImages
    configuration.detectionImages = referenceImages

    // Run the view's session
    sceneView.session.run(configuration)

    let tap = UITapGestureRecognizer(target: self, action: #selector(handleTap(rec:)))
    //Add recognizer to sceneview
    sceneView.addGestureRecognizer(tap)
}

//Method called when tap
@objc func handleTap(rec: UITapGestureRecognizer){
        let location: CGPoint = rec.location(in: sceneView)
        let hits = self.sceneView.hitTest(location, options: nil)
        if !hits.isEmpty{
            let tappedNode = hits.first?.node
            if tappedNode != nil && tappedNode?.name != nil{

                let stringArr = tappedNode?.name?.components(separatedBy: "-")
                let name = stringArr! [0]
                let size = stringArr! [1].components(separatedBy: ",")
                let width = Float(size [0])
                let height = Float(size [1])
                loadReferenceImage(tappedNode: tappedNode!, name: (name),  width: width!, height: height!)
            }
        }
}

private func playVideo() {
    guard let path = Bundle.main.path(forResource: "video", ofType:"m4v") else {
        debugPrint("video.m4v not found")
        return
    }
    let player = AVPlayer(url: URL(fileURLWithPath: path))
    let playerController = AVPlayerViewController()
    playerController.player = player
    present(playerController, animated: true) {
        player.play()
    }
}





func loadReferenceImage(tappedNode: SCNNode, name: String, width: Float, height: Float){
    print("TAP")
    print(name)

    let currentNode = tappedNode.parent

    if let image = UIImage(named: "col" + name){

        let childNodes = currentNode?.childNodes
        for node in (childNodes)!{
            node.removeFromParentNode()
        }

        let newImage = UIImage(named: "col" + name)
        let newnode = SCNNode(geometry: SCNPlane(width: CGFloat(width), height: CGFloat(height)))
        newnode.geometry?.firstMaterial?.diffuse.contents = newImage
        newnode.scale = SCNVector3(x: 10, y: 10, z: 10)
        currentNode?.removeAnimation(forKey: "spin_around")



        let rotation = SCNVector3((currentNode?.eulerAngles.x)!-0.95,(currentNode?.eulerAngles.y)!,(currentNode?.eulerAngles.z)!)
        currentNode?.eulerAngles = rotation



        //SIZE??????

        let nodex = currentNode?.scale.x
        let nodey = currentNode?.scale.y
        let nodez = currentNode?.scale.z

        let factorx = width / nodex!
        let factory = height / nodey!

        currentNode?.addChildNode(newnode)
    }



}

func renderer(_ renderer: SCNSceneRenderer, didAdd node: SCNNode, for anchor: ARAnchor) {
    guard let imageAnchor = anchor as? ARImageAnchor else {
        return
    }


    // 1. Load scene.
    let planeScene = SCNScene(named: "art.scnassets/plane.scn")!
    let planeNode = planeScene.rootNode.childNode(withName: "planeRootNode", recursively: true)!

    // 2. Calculate size based on planeNode's bounding box.
    let (min, max) = planeNode.boundingBox
    let size = SCNVector3Make(max.x - min.x, max.y - min.y, max.z - min.z)

    // 3. Calculate the ratio of difference between real image and object size.
    // Ignore Y axis because it will be pointed out of the image.
    let widthRatio = Float(imageAnchor.referenceImage.physicalSize.width)/1.2
    let heightRatio = Float(imageAnchor.referenceImage.physicalSize.height)/1.2


    let width = imageAnchor.referenceImage.physicalSize.width
    let height = imageAnchor.referenceImage.physicalSize.height

    let prefix = "-"

    let imageSize = width.description + "," + height.description


    let targetName = imageAnchor.referenceImage.name! + prefix + imageSize


    // Pick smallest value to be sure that object fits into the image.
    let finalRatio = [widthRatio, heightRatio].min()!

    // 4. Set transform from imageAnchor data.
    planeNode.transform = SCNMatrix4(imageAnchor.transform)

    // 5. Animate appearance by scaling model from 0 to previously calculated value.
    let appearanceAction = SCNAction.scale(to: CGFloat(finalRatio), duration: 0.4)
    //test
    appearanceAction.timingMode = .easeOut

    // Set initial scale to 0.
    planeNode.scale = SCNVector3Make(0 , 0, 0)

    //rotate y
    let spin = CABasicAnimation(keyPath: "rotation")
    spin.fromValue = NSValue(scnVector4: SCNVector4(x: 0, y: 1, z: 0, w: 0))
    spin.toValue = NSValue(scnVector4: SCNVector4(x: 0, y: 1, z: 0, w: Float(CGFloat(2 * Double.pi))))
    spin.duration = 4
    spin.repeatCount = .infinity
    planeNode.addAnimation(spin, forKey: "spin_around")

    // Add to root node.
    sceneView.scene.rootNode.addChildNode(planeNode)
    // Run the appearance animation.
    planeNode.runAction(appearanceAction)
    planeNode.name = targetName

    let nodes = planeNode.childNodes
    for node in nodes{
        node.name = targetName
    }

    self.planeNode = planeNode
    self.imageNode = node

}

func renderer(_ renderer: SCNSceneRenderer, didAdd node: SCNNode, for anchor: ARAnchor, updateAtTime time: TimeInterval) {
    guard let imageNode = imageNode, let planeNode = planeNode else {
        return
    }

    // 1. Unwrap animationInfo. Calculate animationInfo if it is nil.
    guard let animationInfo = animationInfo else {
        refreshAnimationVariables(startTime: time,
                                  initialPosition: planeNode.simdWorldPosition,
                                  finalPosition: imageNode.simdWorldPosition,
                                  initialOrientation: planeNode.simdWorldOrientation,
                                  finalOrientation: imageNode.simdWorldOrientation)
        return
    }

    // 2. Calculate new animationInfo if image position or orientation changed.
    if !simd_equal(animationInfo.finalModelPosition, imageNode.simdWorldPosition) || animationInfo.finalModelOrientation != imageNode.simdWorldOrientation {

        refreshAnimationVariables(startTime: time,
                                  initialPosition: planeNode.simdWorldPosition,
                                  finalPosition: imageNode.simdWorldPosition,
                                  initialOrientation: planeNode.simdWorldOrientation,
                                  finalOrientation: imageNode.simdWorldOrientation)
    }

    // 3. Calculate interpolation based on passedTime/totalTime ratio.
    let passedTime = time - animationInfo.startTime
    var t = min(Float(passedTime/animationInfo.duration), 1)
    // Applying curve function to time parameter to achieve "ease out" timing
    t = sin(t * .pi * 0.5)

    // 4. Calculate and set new model position and orientation.
    let f3t = simd_make_float3(t, t, t)
    planeNode.simdWorldPosition = simd_mix(animationInfo.initialModelPosition, animationInfo.finalModelPosition, f3t)
    planeNode.simdWorldOrientation = simd_slerp(animationInfo.initialModelOrientation, animationInfo.finalModelOrientation, t)
    //planeNode.simdWorldOrientation = imageNode.simdWorldOrientation

    guard let currentImageAnchor = anchor as? ARImageAnchor else { return }
}



func refreshAnimationVariables(startTime: TimeInterval, initialPosition: float3, finalPosition: float3, initialOrientation: simd_quatf, finalOrientation: simd_quatf) {
    let distance = simd_distance(initialPosition, finalPosition)
    // Average speed of movement is 0.15 m/s.
    let speed = Float(0.15)
    // Total time is calculated as distance/speed. Min time is set to 0.1s and max is set to 2s.
    let animationDuration = Double(min(max(0.1, distance/speed), 2))
    // Store animation information for later usage.
    animationInfo = AnimationInfo(startTime: startTime,
                                  duration: animationDuration,
                                  initialModelPosition: initialPosition,
                                  finalModelPosition: finalPosition,
                                  initialModelOrientation: initialOrientation,
                                  finalModelOrientation: finalOrientation)
}

}

Answer 1

In order to do this I believe that first you need to get the size in Pixels of the UIImage by

multiplying the size values by the value in the scale property to get the pixel dimensions of the image.

As such an example would be something like so:

guard let image = UIImage(named: "launchScreen") else { return }
let pixelWidth = image.size.width * image.scale
let pixelHeight = image.size.height * image.scale
print(pixelWidth, pixelHeight)

The size of my image when made in Adobe Illustrator was 3072 x 4099, and when I logged the results in the console the dimensions were also the same.

Now the tricky part here is calculating the pixels to a size we can use in ARKit, remembering that different devices have a different PPI (Pixels Per Inch) density.

In my example I am just going to use the PPI of an iPhone7Plus which is 401.

 //1. Get The PPI Of The iPhone7Plus
 let iphone7PlusPixelsPerInch: CGFloat = 401

 //2. To Get The Image Size In Inches We Need To Divide By The PPI
 let inchWidth = pixelWidth/iphone7PlusPixelsPerInch
 let inchHeight = pixelHeight/iphone7PlusPixelsPerInch

 //3. Calculate The Size In Metres (There Are 2.54 Cm's In An Inch)
 let widthInMetres = (inchWidth * 2.54) / 100
 let heightInMeters = (inchHeight * 2.54) / 100

Now we have the size of our Image in Metres it is simple to create an SCNNode of that size e.g:

//1. Generate An SCNPlane With The Same Size As Our Image
let realScaleNode = SCNNode(geometry: SCNPlane(width: widthInMetres, height: heightInMeters))
realScaleNode.geometry?.firstMaterial?.diffuse.contents = image
realScaleNode.position = SCNVector3(0, 0, -1)

//2. Add It To Our Hierachy
self.augmentedRealityView.scene.rootNode.addChildNode(realScaleNode)

Hope it helps...

P.S: This may be useful for helping you get the PPI of the Screen (marchv/UIScreenExtension)