4

I'm trying to get the dimensions of a displayed image to draw bounding boxes over the text I have recognized using apple's Vision framework. So I run the VNRecognizeTextRequest uppon the press of a button with this funcion

func readImage(image:NSImage, completionHandler:@escaping(([VNRecognizedText]?,Error?)->()), comp:@escaping((Double?,Error?)->())) {

var recognizedTexts = [VNRecognizedText]()
var rr = CGRect(x: 0, y: 0, width: image.size.width, height: image.size.height)
let requestHandler = VNImageRequestHandler(cgImage: image.cgImage(forProposedRect: &rr, context: nil, hints: nil)!
, options: [:])
let textRequest = VNRecognizeTextRequest { (request, error) in
    guard let observations = request.results as? [VNRecognizedTextObservation] else { completionHandler(nil,error)
        return
    }
    for currentObservation in observations {
        let topCandidate = currentObservation.topCandidates(1)
        if let recognizedText = topCandidate.first {

            recognizedTexts.append(recognizedText)
        }
    }
    completionHandler(recognizedTexts,nil)
}

textRequest.recognitionLevel = .accurate
textRequest.recognitionLanguages = ["es"]
textRequest.usesLanguageCorrection = true

textRequest.progressHandler = {(request, value, error) in
    comp(value,nil)
}
try? requestHandler.perform([textRequest])

}

and compute the bounding boxes offsets using this struct and function

struct DisplayingRect:Identifiable {

var id = UUID()
var width:CGFloat = 0
var height:CGFloat = 0
var xAxis:CGFloat = 0
var yAxis:CGFloat = 0

init(width:CGFloat, height:CGFloat, xAxis:CGFloat, yAxis:CGFloat) {
    self.width = width
    self.height = height
    self.xAxis = xAxis
    self.yAxis = yAxis
}

}

func createBoundingBoxOffSet(recognizedTexts:[VNRecognizedText], image:NSImage) -> [DisplayingRect] {
var rects = [DisplayingRect]()
let imageSize = image.size
let imageTransform = CGAffineTransform.identity.scaledBy(x: imageSize.width, y: imageSize.height)
for obs in recognizedTexts {
    let observationBounds = try? obs.boundingBox(for: obs.string.startIndex..<obs.string.endIndex)
    let rectangle = observationBounds?.boundingBox.applying(imageTransform)
    print("Rectange: \(rectangle!)")
    let width = rectangle!.width
    let height = rectangle!.height
    let xAxis = rectangle!.origin.x - imageSize.width / 2 + rectangle!.width / 2
    let yAxis = -(rectangle!.origin.y - imageSize.height / 2 + rectangle!.height / 2)
    let rect = DisplayingRect(width: width, height: height, xAxis: xAxis, yAxis: yAxis)
    rects.append(rect)
}

return(rects)

}

I place the rects using this code in the ContentView

        ZStack{
            Image(nsImage: self.img!)
                .scaledToFit()
            ForEach(self.rects) { rect in
                Rectangle()
                    .fill(Color.init(.sRGB, red: 1, green: 0, blue: 0, opacity: 0.2))
                    .frame(width: rect.width, height: rect.height)
                    .offset(x: rect.xAxis, y: rect.yAxis)
            }
        }

If I use the original's image dimensions I get these results

enter image description here

But if I add 

                Image(nsImage: self.img!)
                  .resizable()
                  .scaledToFit()

I get these results enter image description here

Is there a way to get the image dimensions and pass them and get the proper size of the image being displayed? I also need this because I can't show the whole image sometimes and need to scale it.

Thanks a lot

Andy Jazz
  • 49,178
  • 17
  • 136
  • 220
Carlos Maria Caraccia
  • 470
  • 4
  • 22

2 Answers2

12

I would use GeometryReader on background so it reads exactly size of image, as below

@State var imageSize: CGSize = .zero // << or initial from NSImage
...
Image(nsImage: self.img!)
    .resizable()
    .scaledToFit()
    .background(rectReader())

// ... somewhere below 
private func rectReader() -> some View {
    return GeometryReader { (geometry) -> Color in
        let imageSize = geometry.size
        DispatchQueue.main.async {
            print(">> \(imageSize)") // use image actual size in your calculations
            self.imageSize = imageSize
        }
        return .clear
    }
}
Martijn Pieters
  • 1,048,767
  • 296
  • 4,058
  • 3,343
Asperi
  • 228,894
  • 20
  • 464
  • 690
  • I'm getting this warning, the frame actually changes, but I can't capture the value. I get this awkward warning when I want to capture the image size: "Modifying state during view update, this will cause undefined behavior." – Carlos Maria Caraccia Jan 07 '20 at 11:51
  • I see. If you got such warning you have to dispatch modifications asynchronously on the next events cycle, so new values will be applied in the following redraw. [Here](https://stackoverflow.com/a/59518183/12299030) you can find an example of how this can be done. – Asperi Jan 07 '20 at 11:57
  • 1
    Updated with showing how to store fetched imageSize value in `@State`, that can be used in some other part of view code. – Asperi Jan 07 '20 at 12:01
  • Thanks a lot it worked!!! I made the var update through the main queue, I forgot it was an async call. – Carlos Maria Caraccia Jan 07 '20 at 12:02
  • Actually they where @State vars, I thought there was no need of dispatching the modifications. – Carlos Maria Caraccia Jan 07 '20 at 12:08
  • @Asperi Wouldn't it be better to use `PreferenceKey` here, rather than kinda hacking it together with a `DispatchQueue.main.async`? – George Dec 26 '21 at 02:30
1

Rather than pass in the frame to every view, Apple elected to give you a separate GeometryReader view that gets its frame passed in as a parameter to its child closure.

struct Example: View {
  var body: some View {
    GeometryReader { geometry in
      Image(systemName: "check")
        .onAppear {
          print(geometry.frame(in: .local))
        }
    }
  }
}
Josh Homann
  • 15,933
  • 3
  • 30
  • 33