Ok I found a work around to seemingly allow a tap gesture on the background view and still have the calendar working; the idea is to add 4 rectangular views around the calendar view and add your tap gesture to those 4 views.
Add a picker background view (I called mine datePickerBGView) to your main view and then add your UIDatePicker (I called mine picker) to this background view and assign the same center point to both views:
picker.center = datePickerBGView.center
Then call this function:
func addDismissGesture() {
//Work around because with the .inline date picker style tap gestures on the background view interfere with the day selection action on the picker's calendar view
let leftDismissActionTap = UITapGestureRecognizer(target: self, action: #selector(dismissCalendar))
let rightDismissActionTap = UITapGestureRecognizer(target: self, action: #selector(dismissCalendar))
let topDismissActionTap = UITapGestureRecognizer(target: self, action: #selector(dismissCalendar))
let bottomDismissActionTap = UITapGestureRecognizer(target: self, action: #selector(dismissCalendar))
let leftDismissArea = UIView()
let rightDismissArea = UIView()
let topDismissArea = UIView()
let bottomDismissArea = UIView()
leftDismissArea.frame = CGRect(x: 0, y: 0, width: picker.frame.minX - view.frame.minX, height: view.frame.height)
rightDismissArea.frame = CGRect(x: picker.frame.maxX, y: 0, width: view.frame.maxX - picker.frame.maxX, height: view.frame.height)
topDismissArea.frame = CGRect(x: leftDismissArea.frame.maxX, y: 0, width: rightDismissArea.frame.minX - leftDismissArea.frame.maxX, height: picker.frame.minY - view.frame.minY)
bottomDismissArea.frame = CGRect(x: leftDismissArea.frame.maxX, y: picker.frame.maxY, width:rightDismissArea.frame.minX - leftDismissArea.frame.maxX, height: view.frame.maxY - picker.frame.maxY)
datePickerBGView.addSubview(leftDismissArea)
datePickerBGView.addSubview(rightDismissArea)
datePickerBGView.addSubview(topDismissArea)
datePickerBGView.addSubview(bottomDismissArea)
leftDismissArea.addGestureRecognizer(leftDismissActionTap)
rightDismissArea.addGestureRecognizer(rightDismissActionTap)
topDismissArea.addGestureRecognizer(topDismissActionTap)
bottomDismissArea.addGestureRecognizer(bottomDismissActionTap)
leftDismissArea.backgroundColor = UIColor.black.withAlphaComponent(0.3)
rightDismissArea.backgroundColor = UIColor.black.withAlphaComponent(0.3)
topDismissArea.backgroundColor = UIColor.black.withAlphaComponent(0.3)
bottomDismissArea.backgroundColor = UIColor.black.withAlphaComponent(0.3)
datePickerBGView.bringSubviewToFront(picker)
}