ServerSocketChannel stops Accepting after a while on Linux

Question

I noticed that, when I run my application on a Linux OS, after a while, the server just stops accepting clients. This is a screenshot of wireshark when trying to connect to the Server from my host, after it stopped accepting. As you can see the first request is [FIN,ACK] and it seems like that my server can't handle that.

Starting up the server, opens the selector, binds, blocking mode is set to false and the server channel registers for OP_ACCEPT. (the usual standard stuff) And this is the core code of the networking:

private void update(int timeout) throws IOException {

    updateThread = Thread.currentThread();

    synchronized (updateLock) {
    }

    long startTime = System.currentTimeMillis();
    int select = 0;

    if (timeout > 0) {
        select = selector.select(timeout);
    }
    else {
        select = selector.selectNow();
    }

    if (select == 0) {
        ++emptySelects;
        if (emptySelects == 100) {

            emptySelects = 0;
            long elapsedTime = System.currentTimeMillis() - startTime;

            try {
                if (elapsedTime < 25)
                    Thread.sleep(25 - elapsedTime);
            } catch (InterruptedException ie) {
                log.error(ie.getMessage());
            }
        }
    } else {

        emptySelects = 0;
        Set<SelectionKey> keys = selector.selectedKeys();

        synchronized (keys) {
            for (Iterator<SelectionKey> iter = keys.iterator(); iter.hasNext();) {

                // keepAlive();
                SelectionKey selectionKey = iter.next();
                iter.remove();
                Connection fromConnection = (Connection) selectionKey.attachment();

                try {
                    if (!selectionKey.isValid())
                        continue;

                    if (fromConnection != null) {
                        if(selectionKey.isReadable()) {
                            try {
                                while (true) {

                                    Packet packet = fromConnection.getTcpConnection().readPacket();
                                    if(packet == null)
                                        break;
                                     fromConnection.notifyReceived(packet);
                                }
                            } catch (IOException ioe) {
                                fromConnection.notifyException(ioe);
                                fromConnection.close();
                            }
                        } else if (selectionKey.isWritable()) {

                            try {
                                fromConnection.getTcpConnection().writeOperation();
                            } catch (IOException ioe) {
                                fromConnection.notifyException(ioe);
                                fromConnection.close();
                            }
                        }
                    }
                    else {
                        if (selectionKey.isAcceptable()) {
                            ServerSocketChannel serverSocketChannel = this.serverChannel;

                            if (serverSocketChannel != null) {
                                try {
                                    SocketChannel socketChannel = serverSocketChannel.accept();
                                    if (socketChannel != null)
                                        acceptOperation(socketChannel);
                                }
                                catch (IOException ioe) {
                                    log.error(ioe.getMessage());
                                }
                            }
                        }
                        else {
                            selectionKey.channel().close();
                        }
                    }
                }
                catch (CancelledKeyException cke) {
                    if(fromConnection != null) {
                        fromConnection.notifyException(cke);
                        fromConnection.close();
                    }
                    else {
                        selectionKey.channel().close();
                    }
                }
            }
        }
    }

    long time = System.currentTimeMillis();
    List<Connection> connections = this.connections;

    for (Connection connection : connections) {
        if (connection.getTcpConnection().isTimedOut(time)) {
            connection.close();
        }
        /* else {

            if (connection.getTcpConnection().needsKeepAlive(time))
                connection.sendTCP(new Packet((char) 0x0FA3));
        }*/
        if (connection.isIdle())
            connection.notifyIdle();
    }
}

When a write Operation is done, the selection key is accordingly changed to OP_READ and in case of a partial write OP_READ | OP_WRITE.

When Exception is thrown, close() is called from the connection object, which basically does this among other things:

try {
    if(socketChannel != null) {
        socketChannel.close();
        socketChannel = null;

        if(selectionKey != null)
            selectionKey.selector().wakeup();
    }
} catch (IOException ioe) {
    // empty..
}

acceptOperation - method does this:

private void acceptOperation(SocketChannel socketChannel) {
    Connection connection = newConnection();
    connection.initialize(4096, 4096);
    connection.setServer(this);

    try {
        SelectionKey selectionKey = connection.getTcpConnection().accept(selector, socketChannel);
        selectionKey.attach(connection);

        int id = nextConnectionID++;
        if(nextConnectionID == -1)
            nextConnectionID = 1;

        connection.setId(id);
        connection.setConnected(true);
        connection.addConnectionListener(dispatchListener);

        addConnection(connection);
        connection.notifyConnected();
    } catch (IOException ioe) {
        log.error(ioe.getMessage());
        connection.close();
    }
}

accept-Method of the connection object does this:

try {
    this.socketChannel = socketChannel;
    socketChannel.configureBlocking(false);
    Socket socket = socketChannel.socket();
    socket.setTcpNoDelay(true);
    socket.setKeepAlive(true);

    selectionKey = socketChannel.register(selector, SelectionKey.OP_READ);

    lastReadTime = lastWriteTime = System.currentTimeMillis();

    return selectionKey;
} catch (IOException ioe) {
    close();
    throw ioe;
}

Pleae note, I have no impact on the client networking code.
When doing netstat -tulnp after the server suddenly stopped accepting, I can see that Recv-Q column increments by 1 for every connect try.
I hope you can help me out, since I have no idea why it is happening.

Sorry for the maybe long text and thanks in forward!