When app crashes for live capture detect text in get an image from the AVCaptureDataSampleBufferDelegate call back did output

Question

When an app crashes for live capture detect text in getting an image from the AVCaptureDataSampleBufferDelegate call back did output. Get image call back from the send Google vision API I have used that code Did anyone work? My code

import UIKit
import FirebaseMLVision
import FirebaseMLModelInterpreter
import AVKit

protocol ImageDelegate: class {
  func getImage(imageOutFrame: String)
}
var photoOutput: AVCaptureVideoDataOutput!
var videoDataOutputQueue: DispatchQueue!
var previewLayer:AVCaptureVideoPreviewLayer!
var captureDevice : AVCaptureDevice!
var captureSession = AVCaptureSession()
var context = CIContext()
weak var delegate: ImageDelegate?
var vision = Vision.vision()
var textRecognizer = vision.onDeviceTextRecognizer()

extension UIViewController: AVCaptureVideoDataOutputSampleBufferDelegate {      
  // To set the camera and its position to capture
  func setUpAVCapture(capturePreview: UIView) {
    captureSession.sessionPreset = AVCaptureSession.Preset.vga640x480
    guard let device = AVCaptureDevice
      .default(AVCaptureDevice.DeviceType.builtInWideAngleCamera,
               for: .video,
               position: AVCaptureDevice.Position.back) else {
                return
    }
    captureDevice = device
    captureDevice.isFocusModeSupported(.continuousAutoFocus)
    try! captureDevice.lockForConfiguration()
    captureDevice.focusMode = .continuousAutoFocus
    captureDevice.unlockForConfiguration()
    beginSession(capturePreview: capturePreview)
  }
// Function to setup the beginning of the capture session
  func beginSession(capturePreview : UIView){

    var deviceInput: AVCaptureDeviceInput!

    do {
      deviceInput = try AVCaptureDeviceInput(device: captureDevice)
      guard deviceInput != nil else {
        print("error: cant get deviceInput")
        return
      }
      if captureSession.canAddInput(deviceInput){
        captureSession.addInput(deviceInput)
      }         
      photoOutput = AVCaptureVideoDataOutput()
      photoOutput.alwaysDiscardsLateVideoFrames=true
      videoDataOutputQueue = DispatchQueue(label: "VideoDataOutputQueue")
      photoOutput.setSampleBufferDelegate(self, queue: videoDataOutputQueue)            
      if captureSession.canAddOutput(photoOutput){
        captureSession.addOutput(photoOutput)
      }          
      photoOutput.connection(with: .video)?.isEnabled = true          
      previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
      previewLayer.videoGravity = AVLayerVideoGravity.resizeAspectFill          
      let rootLayer :CALayer = capturePreview.layer
      rootLayer.masksToBounds=true          
      rootLayer.addSublayer(previewLayer)          
       captureDevice.configureDesiredFrameRate(0)          
      captureSession.startRunning()
    } catch let error as NSError {
      deviceInput = nil
       exit(0)
      print("error: \(error.localizedDescription)")
    }        
  }     
  public func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
    connection.videoOrientation = AVCaptureVideoOrientation.portrait            
    DispatchQueue.global(qos: .userInteractive).async {
      guard let imag = self.imageFromSampleBuffer(sampleBuffer : sampleBuffer) else { return }
    let image = VisionImage(image: imag)
    textRecognizer.process(image) { result, error in
      guard error == nil, let result = result else {
        return
      }
      let resultText = result.text
      let matched = self.matches(for: RegEx.wholeRegex , in: resultText)          
      let m = matched.joined(separator: "")
      print(m)          
      delegate?.getImage(imageOutFrame: m)
      }
  }
  }      
  func imageFromSampleBuffer(sampleBuffer : CMSampleBuffer) -> UIImage? {
    guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return nil }
    let ciImage = CIImage(cvPixelBuffer: imageBuffer)
    guard let cgImage = context.createCGImage(ciImage, from: ciImage.extent) else { return nil }        
    let img = UIImage(cgImage: cgImage)
    let n = img.fixedOrientation()
    return n
  }
  func stopCamera(){
    captureSession.stopRunning()
    captureDevice = nil
  }      
}    
extension AVCaptureDevice {      
  /// http://stackoverflow.com/questions/21612191/set-a-custom-avframeraterange-for-an-avcapturesession#27566730
  func configureDesiredFrameRate(_ desiredFrameRate: Int) {        
    var isFPSSupported = false        
    do {          
      if let videoSupportedFrameRateRanges = activeFormat.videoSupportedFrameRateRanges as? [AVFrameRateRange] {
        for range in videoSupportedFrameRateRanges {
          if (range.maxFrameRate >= Double(desiredFrameRate) && range.minFrameRate <= Double(desiredFrameRate)) {
            isFPSSupported = true
            break
          }
        }
      }          
      if isFPSSupported {
        try lockForConfiguration()
        activeVideoMaxFrameDuration = CMTimeMake(value: 1, timescale: Int32(desiredFrameRate))
        activeVideoMinFrameDuration = CMTimeMake(value: 1, timescale: Int32(desiredFrameRate))
        unlockForConfiguration()
      }          
    } catch {
      print("lockForConfiguration error: \(error.localizedDescription)")
    }
  }
}

get let resultText = result.text get only doller value for example live capture

xcsrf$16.54245 ssxvcxv

I got an Output for like

$16.54245

• My problem is app getting slow recognition of text.

• 1) I have set frames rate but not working
• 2) I have used Dispatch but not working.

• get frames -> converted to image converted to image -> send google

• vision API detecting text - > converted to get the only exact output

> Please share your code to be appreciated Thanks.

Answer 1

My app also crashes when I initialise a VisionImage object with UIImage in captureOutput method. The reason I don't initialise VisionImage with CMSampleBuffer is that I needed to crop specific part of my camera preview layer. It was difficult for me to crop CMSampleBuffer.

Reason
For my case, the crash happens when a new video frame becomes keep available while the text recogniser is still running because captureOutput was repeatedly called when camera is on.

Solution
I put a flag variable to prevent calling the text recogniser while it is processing image. Check the following code for detail.

func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {

    guard let image = self.imageFromSampleBuffer(sampleBuffer : sampleBuffer) else { return }
    let image = VisionImage(image: image)

    guard isRunningTextRecognizer == false else {
            return
    }

    isRunningTextRecognizer = true
    let textRecognizer = vision.onDeviceTextRecognizer()
    textRecognizer.process(image) { (text, error) in
        //...
        self.isRunningTextRecognizer = false
        //..
    }
}

I hope it help!.