Interesting idea - using long dashes on a rounded rect... But I don't believe you'll get your desired results.
You probably want to draw the 4 corners like this:

Your path would start with:
move to: A
line to: B
arc with center: C.x B.y and clockwise from 9 o'clock to 12 o'clock
line to: D
then update your points, and draw the top-right / bottom-right / bottom-left corners.
Here's an example class:
class BracketView: UIView {
var radius: CGFloat = 30
var lineLength: CGFloat = 30
var lineWidth: CGFloat = 10
var lineColor: UIColor = .blue
var shapeLayer: CAShapeLayer!
override class var layerClass: AnyClass {
return CAShapeLayer.self
}
override init(frame: CGRect) {
super.init(frame: frame)
commonInit()
}
required init?(coder: NSCoder) {
super.init(coder: coder)
commonInit()
}
func commonInit() -> Void {
shapeLayer = self.layer as? CAShapeLayer
shapeLayer.fillColor = UIColor.clear.cgColor
shapeLayer.lineCap = .round
}
override func layoutSubviews() {
super.layoutSubviews()
// set these properties here, so we can change them if desired
shapeLayer.strokeColor = lineColor.cgColor
shapeLayer.lineWidth = lineWidth
let pth = UIBezierPath()
var ptA: CGPoint = .zero
var ptB: CGPoint = .zero
var ptC: CGPoint = .zero
var ptD: CGPoint = .zero
var arcCenter: CGPoint = .zero
var startAngle: CGFloat = 0
// top-left corner
startAngle = .pi // 9 o'clock
ptA.x = bounds.minX
ptA.y = bounds.minY + radius + lineLength
ptB.x = bounds.minX
ptB.y = bounds.minY + radius
ptC.x = bounds.minX + radius
ptC.y = bounds.minY
ptD.x = bounds.minX + radius + lineLength
ptD.y = bounds.minY
arcCenter.x = ptC.x
arcCenter.y = ptB.y
pth.move(to: ptA)
pth.addLine(to: ptB)
pth.addArc(withCenter: arcCenter, radius: radius, startAngle: startAngle, endAngle: startAngle + .pi * 0.5, clockwise: true)
pth.addLine(to: ptD)
// top-right corner
startAngle += (.pi * 0.5) // 12 o'clock
ptA.x = bounds.maxX - (radius + lineLength)
ptA.y = bounds.minY
ptB.x = bounds.maxX - radius
ptB.y = bounds.minY
ptC.x = bounds.maxX
ptC.y = bounds.minY + radius
ptD.x = bounds.maxX
ptD.y = bounds.minY + radius + lineLength
arcCenter.x = ptB.x
arcCenter.y = ptC.y
pth.move(to: ptA)
pth.addLine(to: ptB)
pth.addArc(withCenter: arcCenter, radius: radius, startAngle: startAngle, endAngle: startAngle + .pi * 0.5, clockwise: true)
pth.addLine(to: ptD)
// bottom-right corner
startAngle += (.pi * 0.5) // 3 o'clock
ptA.x = bounds.maxX
ptA.y = bounds.maxY - (radius + lineLength)
ptB.x = bounds.maxX
ptB.y = bounds.maxY - radius
ptC.x = bounds.maxX - radius
ptC.y = bounds.maxY
ptD.x = bounds.maxX - (radius + lineLength)
ptD.y = bounds.maxY
arcCenter.x = ptC.x
arcCenter.y = ptB.y
pth.move(to: ptA)
pth.addLine(to: ptB)
pth.addArc(withCenter: arcCenter, radius: radius, startAngle: startAngle, endAngle: startAngle + .pi * 0.5, clockwise: true)
pth.addLine(to: ptD)
// bottom-left corner
startAngle += (.pi * 0.5) // 6 o'clock
ptA.x = bounds.minX + radius + lineLength
ptA.y = bounds.maxY
ptB.x = bounds.minX + radius
ptB.y = bounds.maxY
ptC.x = bounds.minX
ptC.y = bounds.maxY - radius
ptD.x = bounds.minX
ptD.y = bounds.maxY - (radius + lineLength)
arcCenter.x = ptB.x
arcCenter.y = ptC.y
pth.move(to: ptA)
pth.addLine(to: ptB)
pth.addArc(withCenter: arcCenter, radius: radius, startAngle: startAngle, endAngle: startAngle + .pi * 0.5, clockwise: true)
pth.addLine(to: ptD)
shapeLayer.path = pth.cgPath
}
}
Result with a 200 x 200 frame, green background:
