Sending lots data disconnects the server

Question

I'm building a network library to use in further projects, I know a bit of networking (in general and programming) but I still encounter problems when I want to transfer larger packets on a high frequency. How would this be possible in TCP? One possible solution would be chuncking, that is not a problem at all I'll just put them in a buffer and have the whole packet again. This causes overhead at the server, handling lots of data is consuming lots of resources. And it affects the stability because the server get somewhere stuck and the client throws an ObjectDisposedException at BeginSend (connection is closed). Of course its probably my fault but consider a 'relative' small packet of 2048 bytes (2 kB) which get chunked in 512 bytes (4 chunks):

//main.cs

static void Main(string[] args)
{
    client = new Client("82.72.201.150", 2345);
    client.SendCompleted += OnSend;
    Console.WriteLine(client.Connect());

    var bytes = new byte[2048];
    for (int i = 0; i < bytes.Length; i++)
    {
        bytes[i] = (byte)((i*i + 0x25)%byte.MaxValue);
    }

    while (true)
    {
        SendChuncked(bytes);
        Thread.Sleep(500);
    }
}

static void SendChuncked(byte[] buffer, int chunksize = 512)
{
    var chunks = buffer.Length/chunksize;
    var rest = buffer.Length%chunksize;

    var size = BitConverter.GetBytes(buffer.Length);
    client.Send(size, 0, size.Length);

    for (int i = 0; i < chunks; i++)
    {
       client.Send(buffer, i * chunksize, chunksize);
    }

    if (rest > 0)
    client.Send(buffer, chunks * chunksize, rest);
}

//....
//client.cs

    public struct TransferState
{
    public byte[] buffer;
    public int offset;
    public int count;
    public int handled;
    public ManualResetEvent waitHandle;
    public bool wait;
}

public void Send(byte[] buffer, int offset, int count)
{
    var size = BitConverter.GetBytes(count);

    Send(size, 0, size.Length, true);
    Send(buffer, offset, count, false);
}

private void Send(byte[] buffer, int offset, int count, bool wait, TransferState? state = null)
{
    state = state ?? new TransferState
    {
        buffer = buffer,
        offset = offset,
        count = count,
        handled = 0,
        wait = wait,
        waitHandle = new ManualResetEvent(false)
    };

    socket.BeginSend(buffer, offset, count, SocketFlags.None, SendCallback, state);

    if (wait)
    {
        ((TransferState) state).waitHandle.WaitOne();
    }
}

private void Send(byte[] buffer, int offset, int count, bool wait, TransferState? state = null)
{
    state = state ?? new TransferState
    {
        buffer = buffer,
        offset = offset,
        count = count,
        handled = 0,
        wait = wait,
        waitHandle = new ManualResetEvent(false)
    };

    socket.BeginSend(buffer, offset, count, SocketFlags.None, SendCallback, state);

    if (wait)
    {
        ((TransferState) state).waitHandle.WaitOne();
    }
}

private void SendCallback(IAsyncResult result)
{
    if (result.AsyncState is TransferState == false)
        throw new ArgumentException("Invalid type of state.", "state");

    var state = (TransferState)result.AsyncState;
    var sent = socket.EndSend(result);

    state.handled += sent;
    var tosent = state.count - state.handled;
    var offset = state.offset + state.handled;

    if (tosent > 0)
    {
        Send(state.buffer, offset, tosent, state.wait, state);
    }
    else
    {
        state.waitHandle.Set();
        SendCompleted(this, new TransferCompletedArgs(this, state));
    }
}

//....

public void Receive(TransferState? state = null)
{
    if (state == null)
    {
        var buffer = new byte[sizeof(int)];
        socket.BeginReceive(buffer, 0, sizeof (int), SocketFlags.None, ReceiveCallback, buffer);
    }
    else
    {
        var transferState = state.Value;
        socket.BeginReceive(transferState.buffer, transferState.offset, transferState.count - transferState.handled, SocketFlags.None, ReceiveCallback, transferState);
    }
}

private void ReceiveCallback(IAsyncResult result)
{
    //receiving the amount to receive
    if (result.AsyncState is byte[])
    {
        var buffer = (byte[])result.AsyncState;
        var rec = socket.EndReceive(result);
        if (rec != 4) //TODO: HANDLE MORE PROPERLY
            throw new NotImplementedException("Error while receiving the amoount to receive.");

        var toreceive = BitConverter.ToInt32(buffer, 0);

        var state = new TransferState
            {
                buffer = new byte[toreceive],
                count = toreceive,
                wait = false
            };

        Receive(state);
    }
    //know we know the amount we can receive it till the end
    else if (result.AsyncState is TransferState)
    {
        var state = (TransferState)result.AsyncState;
        var rec = socket.EndReceive(result);

        state.offset += rec;
        state.handled += rec;

        var toreceive = state.count - state.handled;
        if (toreceive > 0)
        {
            Receive(state);
            Debug.WriteLine("[{2}] size mismatch: {0}/{1}", state.handled, state.count, DateTime.Now.ToString("mm:ss.fff"));
        }
        else
        {
            ReceiveCompleted(this, new TransferCompletedArgs(this, state));
            Receive();
        }
    }
    else
    {
        throw new ArgumentException("State is not typeof byte[] or TransferState.");
    }
}

So if its hard to follow:

1. The client tries to connect with the server
2. The server accepts the client
3. The client first sends how long the buffer is
4. The client starts sending the buffer (and loops this)
5. The server reads how long the buffer is and allocates a buffer for it
6. The server reads until the buffer is read and invokes an event (and loops this proces as well)

So I don't know what actually happens, but the server stops receiving. While connected clients still sending data, after couple seconds an ObjectDisposedException is thrown at BeginSend. How would I resolve this from happening? (And have a stable server that can handle lots of traffic.)