Conundrum; Overlay disables button tapping. Setting .allowsHitTesting(false) disables underlying gestures

Question

I have a nasty problem: I use a custom ViewModifier to be able to zoom (pinch) and pan an Image. On that Image I place pins on positions relative to that image. Those pins are buttons and need to be tap-able. However, the ViewModifier puts an overlay on top, preventing the buttons to be NOT tap-able. Setting the overlay View (PinchZoom) to .allowsHitTesting(false) enables tapping on the buttons, but at the same time disables the gestures (pinch and long press) on the overlay view. Obviously I need both!

Any suggestions?

TIA

Calling View:

import SwiftUI

struct MapImagesView: View {
    var annotationID: Annotation.ID?
    
    @EnvironmentObject var annotationsViewModel: AnnotationsViewModel
    
    @State private var mapImage = AppImages.exampleMapImage
    @State private var tapLocation = CGPoint.zero

    var body: some View {
            ZStack {
                Image(uiImage: mapImage)
                    .resizable()
                    .fixedSize()
                
                mapImagePinSmall()
                    .foregroundColor(.green)
                    .position(tapLocation)
                
                mapImagePinSmall()
                    .foregroundColor(.red)
                    .position(x: 781, y: 1147)

                mapImagePinSmall()
                    .foregroundColor(.blue)
                    .position(x: 998, y: 1415)
            }
            .onAppear {
                dPrint("image size", mapImage.size)
            }
            .frame(width: mapImage.size.width, height: mapImage.size.height)
            .PinchToZoomAndPan(contentSize: mapImage.size, tapLocation: $tapLocation)
    }
}

struct mapImagePinSmall: View {
    var body: some View {
        Button {
            dPrint("Pin tapped!")
        } label: {
            AppSymbols.MapImage.arrowPointUp
                .frame(width: 44, height: 44, alignment: .center)
                .allowsHitTesting(true)
                .background(.gray)
        }
    }
}

ViewModifier:

import SwiftUI
import UIKit

extension View {
    func PinchToZoomAndPan(contentSize: CGSize, tapLocation: Binding<CGPoint>) -> some View {
        modifier(PinchAndZoomModifier(contentSize: contentSize, tapLocation: tapLocation))
    }
}

struct PinchAndZoomModifier: ViewModifier {
    private var contentSize: CGSize
    private var min: CGFloat = 0.75 // 1.0
    private var max: CGFloat = 3.0
    @State var currentScale: CGFloat = 1.0
    
    // The location in the Image frame the user long pressed
    // to send back to the calling View
    @Binding var tapLocation: CGPoint

    init(contentSize: CGSize, tapLocation: Binding<CGPoint>) {
        self.contentSize = contentSize
        self._tapLocation = tapLocation
    }
    
    func body(content: Content) -> some View {
        ScrollView([.horizontal, .vertical]) {
            content
                .frame(width: contentSize.width * currentScale, height: contentSize.height * currentScale, alignment: .center)
                .modifier(PinchToZoom(minScale: min, maxScale: max, scale: $currentScale, longPressLocation: $tapLocation))
        }
        .animation(.easeInOut, value: currentScale)
    }
}

// THREE; Pinch and zoom View to embed in SwiftUI View
class PinchZoomView: UIView {
    let minScale: CGFloat
    let maxScale: CGFloat
    var isPinching: Bool = false
    var scale: CGFloat = 1.0
    let scaleChange: (CGFloat) -> Void
    let location: (CGPoint) -> Void
    
    private var longPressLocation = CGPoint.zero
    
    init(minScale: CGFloat, maxScale: CGFloat, currentScale: CGFloat, scaleChange: @escaping (CGFloat) -> Void, location: @escaping (CGPoint) -> Void) {
        self.minScale = minScale
        self.maxScale = maxScale
        self.scale = currentScale
        self.scaleChange = scaleChange
        self.location = location
        super.init(frame: .zero)
        
        // Gestures
        let pinchGesture = UIPinchGestureRecognizer(target: self, action: #selector(pinch(gesture:)))
        pinchGesture.cancelsTouchesInView = false
        let longPressGesture = UILongPressGestureRecognizer(target: self, action: #selector(longPress(gesture:)))
        addGestureRecognizer(pinchGesture)
        addGestureRecognizer(longPressGesture)
    }
    
    required init?(coder: NSCoder) {
        fatalError()
    }
    
    // location where the user long pressed, to set a pin in the calling View
    // Needs to be corrected for the current zoom scale!
    @objc private func longPress(gesture: UILongPressGestureRecognizer) {
        switch gesture.state {
        case .ended:
            longPressLocation = gesture.location(in: self)
            let correctedLocation = CGPoint(x: longPressLocation.x / scale, y: longPressLocation.y / scale)
            location(correctedLocation)
            dPrint("Long Pressed in UIView on \(longPressLocation) with scale \(scale)")
            dPrint("Correct location for scale: \(correctedLocation)")
        default:
            break
        }
    }
    
    @objc private func pinch(gesture: UIPinchGestureRecognizer) {
        switch gesture.state {
        case .began:
            isPinching = true
            
        case .changed, .ended:
            if gesture.scale <= minScale {
                scale = minScale
            } else if gesture.scale >= maxScale {
                scale = maxScale
            } else {
                scale = gesture.scale
            }
            scaleChange(scale)
        case .cancelled, .failed:
            isPinching = false
            scale = 1.0
        default:
            break
        }
    }
}

// TWO: Bridge UIView to SwiftUI
struct PinchZoom: UIViewRepresentable {
    let minScale: CGFloat
    let maxScale: CGFloat
    @Binding var scale: CGFloat
    @Binding var isPinching: Bool
    
    @Binding var longPressLocation: CGPoint
    
    func makeUIView(context: Context) -> PinchZoomView {
        let pinchZoomView = PinchZoomView(minScale: minScale, maxScale: maxScale, currentScale: scale, scaleChange: { scale = $0 }, location: { longPressLocation = $0 })
        return pinchZoomView
    }
    
    func updateUIView(_ pageControl: PinchZoomView, context: Context) { }
}

// ONE; Modifier to use the UIKit View
struct PinchToZoom: ViewModifier {
    let minScale: CGFloat
    let maxScale: CGFloat
    @Binding var scale: CGFloat
    @State var anchor: UnitPoint = .center
    @State var isPinching: Bool = false
    
    @Binding var longPressLocation: CGPoint
    
    func body(content: Content) -> some View {
        ZStack {
            content
                .scaleEffect(scale, anchor: anchor)
                .animation(.spring(), value: isPinching)
                .overlay(
                    PinchZoom(minScale: minScale, maxScale: maxScale, scale: $scale, isPinching: $isPinching, longPressLocation: $longPressLocation)
                        //Enables the buttons, but disables long press and pinch :(
                        .allowsHitTesting(false)
                )
        }
    }
}