iOS 15 SwiftUI keyboard toolbar random behaviour

Question

I am trying to add a keyboard toolbar for only some of my TextField's in SwiftUI.

I have a view defined like this:

struct Test: View {
    
    @State private var name: String = ""
    @State private var address: String = ""
    @State private var username: String = ""
    @State private var password: String = ""
    
    enum Field: Hashable {
        case nameField
        case addressField
        case usernameField
        case passwordField
    }
    @FocusState private var focusedField: Field?
    
    var body: some View {
        Form {
            Section {
                TextField("Name", text: $name)
                    .focused($focusedField, equals: .nameField)
                    .submitLabel(.next)
                
                TextField("Address", text: $address)
                    .focused($focusedField, equals: .addressField)
                    .submitLabel(.next)
                
                TextField("Username", text: $username)
                    .focused($focusedField, equals: .usernameField)
                    .submitLabel(.next)
                
                SecureField("Password", text: $password)
                    .focused($focusedField, equals: .passwordField)
                    .submitLabel(.done)
            }
        }
        .toolbar {
            ToolbarItemGroup(placement: .keyboard) {
                if focusedField == .addressField {
                    Button("Test", action: {})
                }
                Button("Done", action: {
                    print(focusedField)
                })
            }
        }
    }
}

The problem is, the additional toolbar button appears when the wrong field is selected.

I have also tried using the @FocusedValue like this:

...
@FocusedValue(\.field) var field: Field?
...
   TextField("Name", text: $name)
      .focused($focusedField, equals: .nameField)
      .submitLabel(.next)
      .focusedValue(\.field, .nameField)
...
      .focusedValue(\.field, .addressField)
...
      .focusedValue(\.field, .usernameField)
...
      .focusedValue(\.field, .passwordField)
...
        .toolbar {
            ToolbarItemGroup(placement: .keyboard) {
                if field == .addressField {
                    Button("Test", action: {})
                }
                Button("Done", action: {
                    print(field)
                })
            }
        }

But this gives the same result. It's as if the toolbar is decided, before the focused state is set. I'm not sure if this is a possible bug, or I am doing this wrong?

Although even when I get the above working, it doesn't fix my overall issue of only wanting the toolbar for 1 of the text fields, however when I try setting it like this, I get no toolbar at all.

.toolbar {
    ToolbarItemGroup(placement: .keyboard) {
        if field == .addressField {
            Button("Test", action: {})
        } else {
            EmptyView()
        }
    }
}