SwiftUI ScrollView won't let me scroll up

Question

I'm trying to implement a chat feature, the relevant codes are simple enough

struct ContentView: View {
    @State private var channelId = ""
    @State private var message = ""
    
    @ObservedObject var sig = SignalRService(urlStr: "\(API.HubsEndpoint)/hubs/theHub")
    
    var body: some View {
        VStack {
            HStack {
                TextField("Channel ID", text: $channelId)                   .textFieldStyle(.roundedBorder)
                Button {
                    sig.invoke(method: "JoinRoom", arguments: [channelId]) { error in
                        print("joined")
                        if let err = error {
                            print(err)
                        }
                    }
                } label: {
                    Text("Join")
                }
                .padding(5)
                .background(Color.accentColor)
                .foregroundColor(.white)
                .cornerRadius(2)
            }
            .padding()
            
            Spacer()
            
            ScrollView {
                ScrollViewReader { scrollView in
                    LazyVStack(alignment: .leading, spacing: 5) {
                        ForEach(sig.messages) { msg in
                            HStack(alignment: .top) {
                                Text(msg.user)
                                    .font(.caption)
                                    .bold()
                                Text(msg.message)
                                    .font(.caption)
                            }
                            .id(msg.id)
                            .padding(20)
                            .background(.regularMaterial)
                            .foregroundColor(.white)
                            .cornerRadius(10)
                        }
                    }
                    .onChange(of: sig.messages.last) { newValue in
                        scrollView.scrollTo(newValue?.id)
                    }
                }
                .frame(height: UIScreen.main.bounds.height/4, alignment:.bottom)
            }
            .frame(height: UIScreen.main.bounds.height/4, alignment:.bottom)
            .background(Color.black)
            
            HStack {
                TextField("Message", text: $message)
                    .textFieldStyle(.roundedBorder)
                Button {
                    sig.invoke(method: "SendMessage", arguments: [message, channelId]) { error in
                        print("messaged")
                        if let err = error {
                            print(err)
                        }
                        else {
                            message = ""
                        }
                    }
                } label: {
                    Image(systemName: "paperplane.fill")
                }
                .padding(5)
                .background(Color.accentColor)
                .foregroundColor(.white)
                .cornerRadius(2)
            }
            .padding(.horizontal)
        }

    }
}

And the result: https://i.stack.imgur.com/hvIW4.jpg

It kept snapping to the bottom when I tried to scroll up.

I'm clipping the frame size and increased the padding to exaggerate the views, so it's easier to test.

Any idea what's causing the issue?