Using Vision to scan images from photo library

Question

Is there a way that I can use the Vision framework to scan an existing image from the user's photo library? As in, not taking a new picture using the camera, but just choosing an image that the user already has?

Answer 1

Yes, you can. Adding on to @Zulqarnayn's answer, here's a working example to detect and draw a bounding box on rectangles.

1. Set up the image view where the image will be displayed

@IBOutlet weak var imageView: UIImageView!

@IBAction func pickImage(_ sender: Any) {
    let picker = UIImagePickerController()
    picker.delegate = self
    self.present(picker, animated: true)
}

override func viewDidLoad() {
    super.viewDidLoad()
    
    imageView.layer.borderWidth = 4
    imageView.layer.borderColor = UIColor.blue.cgColor
    imageView.contentMode = .scaleAspectFill
    
    imageView.backgroundColor = UIColor.green.withAlphaComponent(0.3)
    imageView.layer.masksToBounds = false /// allow image to overflow, for testing purposes
    
}

2. Get the image from the image picker

extension ViewController: UIImagePickerControllerDelegate, UINavigationControllerDelegate {
    func imagePickerController(_ picker: UIImagePickerController, didFinishPickingMediaWithInfo info: [UIImagePickerController.InfoKey : Any]) {
        guard let image = info[.originalImage] as? UIImage else { return }

        /// set the imageView's image
        imageView.image = image

        /// start the request & request handler
        detectCard()

        /// dismiss the picker
        dismiss(animated: true)
    }
}

3. Start the vision request

func detectCard() {
    guard let cgImage = imageView.image?.cgImage else { return }

    /// perform on background thread, so the main screen is not frozen
    DispatchQueue.global(qos: .userInitiated).async {
        
        let request = VNDetectRectanglesRequest { request, error in

            /// this function will be called when the Vision request finishes
            self.handleDetectedRectangle(request: request, error: error)
        }
        
        request.minimumAspectRatio = 0.0
        request.maximumAspectRatio = 1.0
        request.maximumObservations = 1 /// only look for 1 rectangle
        
        let imageRequestHandler = VNImageRequestHandler(cgImage: cgImage, orientation: .up)
        
        do {
            try imageRequestHandler.perform([request])
        } catch let error {
            print("Error: \(error)")
        }
    }
}

4. Get the result from the Vision request

func handleDetectedRectangle(request: VNRequest?, error: Error?) {
    if let results = request?.results {
        if let observation = results.first as? VNRectangleObservation {
            
            /// get back to the main thread
            DispatchQueue.main.async {
                guard let image = self.imageView.image else { return }
                
                let convertedRect = self.getConvertedRect(
                    boundingBox: observation.boundingBox,
                    inImage: image.size,
                    containedIn: self.imageView.bounds.size
                )
                self.drawBoundingBox(rect: convertedRect)
            }
        }
    }
}

5. Convert observation.boundingBox to the UIKit coordinates of the image view, then draw a border around the detected rectangle

I explain this more in detail in this answer.

func getConvertedRect(boundingBox: CGRect, inImage imageSize: CGSize, containedIn containerSize: CGSize) -> CGRect {
    
    let rectOfImage: CGRect
    
    let imageAspect = imageSize.width / imageSize.height
    let containerAspect = containerSize.width / containerSize.height
    
    if imageAspect > containerAspect { /// image extends left and right
        let newImageWidth = containerSize.height * imageAspect /// the width of the overflowing image
        let newX = -(newImageWidth - containerSize.width) / 2
        rectOfImage = CGRect(x: newX, y: 0, width: newImageWidth, height: containerSize.height)
        
    } else { /// image extends top and bottom
        let newImageHeight = containerSize.width * (1 / imageAspect) /// the width of the overflowing image
        let newY = -(newImageHeight - containerSize.height) / 2
        rectOfImage = CGRect(x: 0, y: newY, width: containerSize.width, height: newImageHeight)
    }
    
    let newOriginBoundingBox = CGRect(
    x: boundingBox.origin.x,
    y: 1 - boundingBox.origin.y - boundingBox.height,
    width: boundingBox.width,
    height: boundingBox.height
    )
    
    var convertedRect = VNImageRectForNormalizedRect(newOriginBoundingBox, Int(rectOfImage.width), Int(rectOfImage.height))
    
    /// add the margins
    convertedRect.origin.x += rectOfImage.origin.x
    convertedRect.origin.y += rectOfImage.origin.y
    
    return convertedRect
}

/// draw an orange frame around the detected rectangle, on top of the image view
func drawBoundingBox(rect: CGRect) {
    let uiView = UIView(frame: rect)
    imageView.addSubview(uiView)
    
    uiView.backgroundColor = UIColor.clear
    uiView.layer.borderColor = UIColor.orange.cgColor
    uiView.layer.borderWidth = 3
}

Result | Demo repo

Input image	Result

Answer 2

Yes, you can. First, take an instance of UIImagePickerController & present it.

let picker = UIImagePickerController()
picker.delegate = self
picker.sourceType = .photoLibrary
present(picker, animated: true, completion: nil)

Then implement the delegate method take the desired image

extension YourViewController: UIImagePickerControllerDelegate {
func imagePickerController(_ picker: UIImagePickerController, didFinishPickingMediaWithInfo info: [UIImagePickerController.InfoKey : Any]) {
    if let pickedImage = info[.originalImage] as? UIImage {
        
        ## here start your request & request handler
    }
    
    picker.dismiss(animated: true, completion: nil)
    
    }
}