12

I have a TCP server windows service developed in .net 4.0 with asynchronous server sockets.

It works, but about 90 % of the time I simply can not stop it: after pressing stop button in Windows Service console, it hangs and stops after about a minute, but its process goes on and TCP communication continues. It hangs at _listener.Close(). Only thing I can do to close the communication is to restart Windows. There can be something with closing sockets. I tried to figure out but I simply can not find the root of the problem.

Client side is not under my control, they are some 100 gadgets sending data via TCP.

Here is my code (updated).

Thanks a lot, any suggestions are highly appreciated!

    public class DeviceTCPServer
{
    public readonly IPAddress IPAddress;
    public readonly int Port;
    public readonly int InputBufferSize;
    public readonly ConcurrentDictionary<Guid, StateObject> Connections;

    public event EventHandler OnStarted;
    public event EventHandler OnStopped;
    public event ServerEventHandler OnConnected;
    public event ServerEventHandler OnDisconnected;
    public event RecievedEventHandler OnRecieved;
    public event DroppedEventHandler OnDropped;
    public event ExceptionEventHandler OnException;
    public event ServerLogEventHandler ServerLog;

    private volatile bool _iAmListening;
    private Socket _listener;
    private Thread _listenerThread;
    private readonly ManualResetEvent _allDone = new ManualResetEvent(false);

    public bool Listening
    {
        get { return _iAmListening; }
    }

    public DeviceTCPServer(IPAddress ipAddress,
                         int port,
                         int inputBufferSize)
    {
        IPAddress = ipAddress;
        Port = port;
        InputBufferSize = inputBufferSize;

        Connections = new ConcurrentDictionary<Guid, StateObject>();
    }

    public void ThreadedStart()
    {
        _listenerThread = new Thread(Start)
        {
            CurrentUICulture = Thread.CurrentThread.CurrentUICulture,
            IsBackground = true
        };

        _listenerThread.Start();
    }

    private void Start()
    {
        try
        {
            var localEP = new IPEndPoint(IPAddress, Port);
            _listener = new Socket(localEP.Address.AddressFamily, SocketType.Stream, ProtocolType.Tcp);

            _listener.Bind(localEP);
            _listener.Listen(10000);

            if (OnStarted != null)
                OnStarted(this, new EventArgs());

            _iAmListening = true;

            var listenerWithCultureInfo = new Tuple<Socket, CultureInfo>(_listener,
                                                                         Thread.CurrentThread.CurrentUICulture);

            while (_iAmListening)
            {
                _allDone.Reset();
                _listener.BeginAccept(AcceptCallback, listenerWithCultureInfo);
                _allDone.WaitOne();
            }
        }
        catch (Exception exc)
        {
            Debug.Assert(false, exc.Message);
            if (OnException != null)
                OnException(this, new ExceptionEventArgs(exc, "Start"));
        }
    }

    public void StopListening()
    {
        try
        {
            _iAmListening = false;
            _allDone.Set();
        }
        catch(Exception exc)
        {
            Debug.Assert(false, exc.Message);
            if (OnException != null)
                OnException(this, new ExceptionEventArgs(exc, "StopListening"));
        }
    }

    public void Stop()
    {
        try
        {
            _listener.Close(0);
            CloseAllConnections();
            _listenerThread.Abort();
        }
        catch (Exception exc)
        {
            Debug.Assert(false, exc.Message);
            if (OnException != null)
                OnException(this, new ExceptionEventArgs(exc, "Stop"));
        }
    }

    private void AcceptCallback(IAsyncResult ar)
    {
        var arTuple = (Tuple<Socket, CultureInfo>)ar.AsyncState;
        var state = new StateObject(arTuple.Item2, InputBufferSize);

        try
        {
            Connections.AddOrUpdate(state.Guid,
                                    state,
                                    (k, v) => v);

            Thread.CurrentThread.CurrentUICulture = state.CurrentUICulture;

            var listener = arTuple.Item1;
            var handler = listener.EndAccept(ar);

            _allDone.Set();
            if (!_iAmListening)
                return;

            state.WorkSocket = handler;
            handler.BeginReceive(state.Buffer, 0, state.InputBufferSize, 0,
                                 RecieveCallBack, state);

            if (OnConnected != null)
                OnConnected(this, new ServerEventArgs(state));
        }
        catch(ObjectDisposedException)
        {
            _allDone.Set();
            return;
        }
        catch (Exception exc)
        {
            Debug.Assert(false, exc.Message);
            if (OnException != null)
                OnException(this, new ExceptionEventArgs(exc, state, "AcceptCallback"));
        }
    }

    public void RecieveCallBack(IAsyncResult ar)
    {
        var state = (StateObject)ar.AsyncState;

        try
        {
            Thread.CurrentThread.CurrentUICulture = state.CurrentUICulture;

            var handler = state.WorkSocket;
            var read = handler.EndReceive(ar);

            var pBinayDataPocketCodecStore = new BinayDataPocketCodecStore();

            if (read > 0)
            {
                state.LastDataReceive = DateTime.Now;

                var data = new byte[read];
                Array.Copy(state.Buffer, 0, data, 0, read);
                state.AddBytesToInputDataCollector(data);

                //check, if pocket is complete
                var allData = state.InputDataCollector.ToArray();
                var codecInitRes = pBinayDataPocketCodecStore.Check(allData);

                if (codecInitRes.Generic.Complete)
                {
                    if (!codecInitRes.Generic.Drop)
                    {
                        if (OnRecieved != null)
                            OnRecieved(this, new RecievedEventArgs(state, allData));
                    }
                    else
                    {
                        if (OnDropped != null)
                            OnDropped(this, new DroppedEventArgs(state, codecInitRes.Generic));

                        //get new data
                        state.ResetInputDataCollector();

                        handler.BeginReceive(state.Buffer, 0, state.InputBufferSize, 0,
                                             RecieveCallBack, state);
                    }
                }
                else
                {
                    //get more data
                    handler.BeginReceive(state.Buffer, 0, state.InputBufferSize, 0,
                                         RecieveCallBack, state);
                }
            }
            else
            {
                if ((handler.Connected == false) || (handler.Available == 0))
                {
                    Close(state);
                }
            }
        }
        catch (Exception exc)
        {
            Debug.Assert(false, exc.Message);
            if (OnException != null)
                OnException(this, new ExceptionEventArgs(exc, state, "RecieveCallBack"));
        }
    }

    public void Send(StateObject state, byte[] data)
    {
        try
        {
            var handler = state.WorkSocket;

            handler.BeginSend(data, 0, data.Length, 0,
                              SendCallback, state);
        }
        catch (Exception exc)
        {
            Debug.Assert(false, exc.Message);
            if (OnException != null)
                OnException(this, new ExceptionEventArgs(exc, state, "Send"));
        }
    }

    private void SendCallback(IAsyncResult ar)
    {
        var state = (StateObject)ar.AsyncState;

        try
        {
            Thread.CurrentThread.CurrentUICulture = state.CurrentUICulture;
            var handler = state.WorkSocket;

            handler.EndSend(ar);
            handler.BeginReceive(state.Buffer, 0, state.InputBufferSize, 0,
                                 RecieveCallBack, state);
        }
        catch (Exception exc)
        {
            Debug.Assert(false, exc.Message);
            if (OnException != null)
                OnException(this, new ExceptionEventArgs(exc, state, "SendCallback"));
        }
    }

    public void Close(StateObject state)
    {
        try
        {
            if (state == null)
                return;

            var handler = state.WorkSocket;

            if (handler == null)
                return;

            if (!handler.Connected)
                return;

            if (handler.Available > 0)
            {
                var data = new byte[handler.Available];
                handler.Receive(data);
            }

            handler.Shutdown(SocketShutdown.Both);
            handler.Close(0);

            if (OnDisconnected != null)
                OnDisconnected(this, new ServerEventArgs(state));

            StateObject removed;
            var removeResult = Connections.TryRemove(state.Guid, out removed);
        }
        catch (Exception exc)
        {
            Debug.Assert(false, exc.Message);
            if (OnException != null)
                OnException(this, new ExceptionEventArgs(exc, "Close"));
        }
    }

    private void CloseAllConnections()
    {
        try
        {
            var connections = Connections.Select(c => c.Value);

            foreach(var connection in connections)
            {
                Close(connection);
            }
        }
        catch(Exception exc)
        {
            Debug.Assert(false, exc.Message);
            if (OnException != null)
                OnException(this, new ExceptionEventArgs(exc, "CloseAllConnections"));
        }
    }

    public override string ToString()
    {
        return string.Format("{0}:{1}", IPAddress, Port);
    }
}
Tom
  • 3,899
  • 22
  • 78
  • 137
  • You should never close a connection from the server side. There is a known issue with TCP if both the client and server closes the connection at same time the connection is left in a half-open/half-close state. When a connection is closed the other end of the connection must ACK. When both end close at the same time one side will close and not ACK. – jdweng Jul 27 '18 at 15:45
  • How should I do that in the above pattern? When and how should I ACK it? Is it possible at all in net or doe sit handle automatically? Anyway, if I don't close, I have hundreds of waiting to close state connections (via netstat). – Tom Jul 27 '18 at 15:58
  • The ack is done in the driver which you have no control. I would set the Keep-alive which will automatically close an idle connection. A server closing an idle connection is unlikely to cause an issue with the client closing at the same time. – jdweng Jul 27 '18 at 16:27

3 Answers3

4

Since this is a service, the usual "wait for the non-background threads to exit" rule doesn't apply, and it is your job to kill the workers (which can including pending async operations). Now; at the moment, you're killing the listener, but that only prevents new sockets from attaching. You should ideally keep track of your clients somewhere, so that you can also kill all the client sockets. When you've done that, you should be fine. Remember to synchronize access to any collection of clients, or use a thread-safe collection type.

Marc Gravell
  • 1,026,079
  • 266
  • 2,566
  • 2,900
  • 1
    As a side note: if possible, note that more modern frameworks have done a *lot* of work to improve async networking; it will be hard to use those features from .NET 4.0, though. Network servers are a great candidate for considering an update to .NET Core 2.1 (and above) - there are *lots* of benefits! – Marc Gravell Jul 27 '18 at 14:28
  • Yep, thx, I tried it but simply does not help. public readonly ConcurrentDictionary Connections; In acceptCallBack: Connections.AddOrUpdate(state.Guid, state, (k, v) => v); And at stop simply iterate and close them by calling Close method, that can be found in my original post. – Tom Jul 27 '18 at 15:04
  • @Tom assuming `handler` is a `Socket`, you don't *dispose* it, and it isn't clear to me if any of the cast/shutdown/close/etc could actually be throwing and the exception swallowed by that `catch` block; I also don't see code that loops and closes all – Marc Gravell Jul 30 '18 at 08:31
  • Dear Marc, I have updated my code in my original question. It still has the same problems. CloseAllConnections method closes the connections. – Tom Jul 31 '18 at 09:51
  • And also note, I don't have any exceptions. Just incoing data even after calling StopListening and Stop. – Tom Jul 31 '18 at 10:37
  • 1
    @Tom it really sounds like there's simply a bug in your `Close` method; have you stepped through it to check that it really is calling shutdown *and* disposing each connection? (I see you calling `handler.Close`, but I don't see any `handler.Dispose()`. Also, I'd be very cautious that if *one* socket throws, you aren't closing the others. Without a full repro, this is very hard to look into, but: as long as you shutdown, close, and dispose all the sockets, it should work - I do high volume socket servers all the time (including one you're using right now) – Marc Gravell Jul 31 '18 at 11:01
  • @Tom additionally, IMO this comes under "sockets are really really hard to get right"; there are tools *explicitly designed* to make it easier, but: they may not work well (or at all) on 4.0 (4.62 should be fine, though); you may find my [series on "pipelines"](https://blog.marcgravell.com/search/label/pipelines) interesting; I'd start with ["part 3"](https://blog.marcgravell.com/2018/07/pipe-dreams-part-3.html) if it was me, as that is the most "hands on" – Marc Gravell Jul 31 '18 at 11:03
  • Thanks. I will check it. The most interesting thing, that it worked for 5 years without any problem, on some 10 servers with some hundres gadgets sending data. And this bug happened first time about 2 weeks ago. :O One thing I could investigate that there was a firmware update in the gadgets and they increased TCP timeouts, but no more info about that side. – Tom Jul 31 '18 at 11:13
  • I could not solve it. Anything I try, it hangs at _listener.Close();. It simply does not closes the listener socket, just hangs forever. Even there are no any connections (by checking netstat command also...). – Tom Aug 03 '18 at 14:18
0

I think you should use the Socket.Shutdown method before closing the socket. As it is stated in the documentation:

When using a connection-oriented Socket, always call the Shutdown method before closing the Socket. This ensures that all data is sent and received on the connected socket before it is closed.

Moreover, maybe I am wrong, but I can not see any calls to the StopListening() method. Since then the _iAmListening field is not set to false, and the callback (AcceptCallback(IAsyncResult ar)) is still responding to the incoming calls.

Hix
  • 29
  • 3
  • No, the listener shocket itself should not be shut down, it is not connected. If you call shutdown, it throws an exception. Other sockets are shut down of course before closing. Stoplistening / stop are called by public method from service stop. – Tom Aug 03 '18 at 15:56
  • What exception is it throwing? – Hix Aug 03 '18 at 17:27
  • "Additional information: A request to send or receive data was disallowed because the socket is not connected and (when sending on a datagram socket using a sendto call) no address was supplied" But it is normal, it is a listener socket, sot it is not the root of my problem. More here: https://stackoverflow.com/questions/23963359/socket-shutdown-throws-socketexception – Tom Aug 03 '18 at 18:40
0

So, finally the problem is solve, there were no any problem with my Windows service!

It was caused by a Microsoft Windows update:

Microsoft KB4338815, which caused closesocket tо hang forever on Intel Xeon processors: https://forum.filezilla-project.org/viewtopic.php?t=49308

Thanks all your efforts to try to help me!

Tom
  • 3,899
  • 22
  • 78
  • 137