Protobuf.net deserialization exception C#

Question

I'm creating a desktop remote control for my LAN computers, I have 2 clients and a server. The connection is based on Ssl stream (over Tcp Stream)

When both of the clients are connected, I can choose which one to watch (using a combo box). However, sometimes, when I switch client to watch, I get ProtoException : Invalid wire-type (6).

NOTE : It usually happens when I switch from one client to the second one and then back to the first one.

Here's my relevant code:

Server:

private void Listen()
{
        _listener.Start();

        while (true) 
        {
            ClientModel client = new ClientModel(_listener.AcceptTcpClient());
            OpenSslStream(client);
            ThreadPool.QueueUserWorkItem(BackgroundClientsPacketReceiver, client);
        }
}

public void SwitchClient(ClientModel value)
{
            ClientModel prevClient = _currentClient;
            _currentClient = value;

            PacketModel<string> pausePacket = new PacketModel<string>
            {
                Value = string.Empty,
                Type = PacketType.PAUSE_ACTIVE_CONNECTION
            };

            if (value != null)
            {
                if (_currentClient.Client.Client.Connected)
                {
                    PacketModel<string> connectPacket = new PacketModel<string>
                    {
                        Value = $"trying to connect from server to {_currentClient.IpAdress}",
                        Type = PacketType.CONNECT_REQ
                    };

                    try
                    {
                        SendPacket(connectPacket);
                    }
                    catch (IOException)
                    {
                        RemoveCurrentClient();
                    }
                }
                else
                {
                    RemoveCurrentClient();
                }
            }
            SendPacket(pausePacket, prevClient);
}

private void BackgroundClientsPacketReceiver(object state)
{
        if (state is null) return;

        ClientModel client = (ClientModel)state;
        while (true)
        {
            ReceivePackets(client, BackgroundClientPacketReceived);
            Thread.Sleep(10);
        }
}

public void ReceivePackets(params object[] args)
{
        ClientModel client = (ClientModel)args[0];
        EventHandler handler = (EventHandler)args[1];

        dynamic packet = client.SslStream.Deserialize();

        BroadcastPacketsEventArgsModel arguments = new BroadcastPacketsEventArgsModel
        {
                    Packet = packet,
                    Client = client,
        };

       PacketReceived?.Invoke(this, arguments);
}

Serializtion/Deserialization code :

public static void Serialize<T>(this SslStream stream, PacketModel<T> value)
{
        Serializer.SerializeWithLengthPrefix(stream,
        value.GetType().AssemblyQualifiedName, PrefixStyle.Base128, 0);

        Serializer.SerializeWithLengthPrefix(stream,
            value, PrefixStyle.Base128, 1);

        stream.Flush();
}

public static dynamic Deserialize(this SslStream stream)
{
        dynamic Deserialize()
        {
            string typeName = Serializer.DeserializeWithLengthPrefix<string>(stream,
                PrefixStyle.Base128, 0);

            Type type = Type.GetType(typeName);

            RuntimeTypeModel model = RuntimeTypeModel.Default;
            model.Add(type, true);

            dynamic obj = model.DeserializeWithLengthPrefix(stream, null,
                type, PrefixStyle.Base128, 1);

            stream.Flush();

            return obj;
        }

        lock (stream)
        return Deserialize();
}

The client code isn't relevant here since it's functioning as expected.

Does someone have a solution for this weird bug?