ZeroMQ dealer--to-dealer high latency compared to winsock

Question

My company is looking into using ZeroMQ as the transport mechanism. First I benchmarked the performance just to get a hunch of what I´m playing with.

So I created an application comparing zmq dealer-to-dealer setup against winsock. I meassured the round-time-trip of sending synchronous messages from a client to a server and then calculating the average.

Here server running winsock:

DWORD RunServerWINSOCKTest(DWORD dwPort)
{
    WSADATA wsaData;
    int iRet = WSAStartup(MAKEWORD(2, 2), &wsaData);
    if (iRet != NO_ERROR)
    {
        printf("WSAStartup failed with error: %d\n", iRet);
        return iRet;
    }

    struct addrinfo hints;
    ZeroMemory(&hints, sizeof(hints));
    hints.ai_family = AF_INET;
    hints.ai_socktype = SOCK_STREAM;
    hints.ai_protocol = IPPROTO_TCP;
    hints.ai_flags = AI_PASSIVE;

    struct addrinfo *result = NULL;
    iRet = getaddrinfo(NULL, std::to_string(dwPort).c_str(), &hints, &result);
    if (iRet != 0)
    {
        WSACleanup();
        return iRet;
    }

    SOCKET ListenSocket = socket(result->ai_family, result->ai_socktype, result->ai_protocol);
    if (ListenSocket == INVALID_SOCKET)
    {
        freeaddrinfo(result);
        WSACleanup();
        return WSAGetLastError();
    }

    iRet = bind(ListenSocket, result->ai_addr, (int)result->ai_addrlen);
    if (iRet == SOCKET_ERROR)
    {
        freeaddrinfo(result);
        closesocket(ListenSocket);
        WSACleanup();
        return WSAGetLastError();
    }

    freeaddrinfo(result);
    iRet = listen(ListenSocket, SOMAXCONN);
    if (iRet == SOCKET_ERROR)
    {
        closesocket(ListenSocket);
        WSACleanup();
        return WSAGetLastError();
    }

    while (true)
    {
        SOCKET ClientSocket = accept(ListenSocket, NULL, NULL);
        if (ClientSocket == INVALID_SOCKET)
        {
            closesocket(ListenSocket);
            WSACleanup();
            return WSAGetLastError();
        }
        char value = 0;
        setsockopt(ClientSocket, IPPROTO_TCP, TCP_NODELAY, &value, sizeof(value));

        char recvbuf[DEFAULT_BUFLEN];
        int recvbuflen = DEFAULT_BUFLEN;
        do {

            iRet = recv(ClientSocket, recvbuf, recvbuflen, 0);
            if (iRet > 0) {
            // Echo the buffer back to the sender
                int iSendResult = send(ClientSocket, recvbuf, iRet, 0);
                if (iSendResult == SOCKET_ERROR)
                {
                    closesocket(ClientSocket);
                    WSACleanup();
                    return WSAGetLastError();
                }
            }
            else if (iRet == 0)
                printf("Connection closing...\n");
            else  {
                closesocket(ClientSocket);
                WSACleanup();
                return 1;
            }

        } while (iRet > 0);

        iRet = shutdown(ClientSocket, SD_SEND);
        if (iRet == SOCKET_ERROR)
        {
            closesocket(ClientSocket);
            WSACleanup();
            return WSAGetLastError();
        }
        closesocket(ClientSocket);
    }
    closesocket(ListenSocket);

    return WSACleanup();
}

Here is the client running winsock:

DWORD RunClientWINSOCKTest(std::string strAddress, DWORD dwPort, DWORD dwMessageSize)
{
    WSADATA wsaData;
    int iRet = WSAStartup(MAKEWORD(2, 2), &wsaData);
    if (iRet != NO_ERROR)
    {
        return iRet;
    }

    SOCKET ConnectSocket = INVALID_SOCKET;
    struct addrinfo *result = NULL,  *ptr = NULL, hints;


    ZeroMemory(&hints, sizeof(hints));
    hints.ai_family = AF_UNSPEC;
    hints.ai_socktype = SOCK_STREAM;
    hints.ai_protocol = IPPROTO_TCP;

    int iResult = getaddrinfo(strAddress.c_str(), std::to_string(dwPort).c_str(), &hints, &result);
    if (iResult != 0) {
        WSACleanup();
        return 1;
    }

    for (ptr = result; ptr != NULL; ptr = ptr->ai_next) {
        ConnectSocket = socket(ptr->ai_family, ptr->ai_socktype, ptr->ai_protocol);
        if (ConnectSocket == INVALID_SOCKET) {
            WSACleanup();
            return 1;
        }

        iResult = connect(ConnectSocket, ptr->ai_addr, (int)ptr->ai_addrlen);
        if (iResult == SOCKET_ERROR) {
            closesocket(ConnectSocket);
            ConnectSocket = INVALID_SOCKET;
            continue;
        }
        break;
    }

    freeaddrinfo(result);

    if (ConnectSocket == INVALID_SOCKET) {
        WSACleanup();
        return 1;
    }


    // Statistics
    UINT64 uint64BytesTransmitted = 0;
    UINT64 uint64StartTime = s_TimeStampGenerator.GetHighResolutionTimeStamp();
    UINT64 uint64WaitForResponse = 0;

    DWORD dwMessageCount = 1000000;

    CHAR cRecvMsg[DEFAULT_BUFLEN];
    SecureZeroMemory(&cRecvMsg, DEFAULT_BUFLEN);

    std::string strSendMsg(dwMessageSize, 'X');

    for (DWORD dwI = 0; dwI < dwMessageCount; dwI++)
    {
        int iRet = send(ConnectSocket, strSendMsg.data(), strSendMsg.size(), 0);
        if (iRet == SOCKET_ERROR) {
            closesocket(ConnectSocket);
            WSACleanup();
            return 1;
        }
        uint64BytesTransmitted += strSendMsg.size();

        UINT64 uint64BeforeRespone = s_TimeStampGenerator.GetHighResolutionTimeStamp();
        iRet = recv(ConnectSocket, cRecvMsg, DEFAULT_BUFLEN, 0);
        if (iRet < 1)
        {
            closesocket(ConnectSocket);
            WSACleanup();
            return 1;
        }
        std::string strMessage(cRecvMsg);

        if (strMessage.compare(strSendMsg) == 0)
        {
            uint64WaitForResponse += (s_TimeStampGenerator.GetHighResolutionTimeStamp() - uint64BeforeRespone);
        }
        else
        {
            return NO_ERROR;
        }
}

    UINT64 uint64ElapsedTime = s_TimeStampGenerator.GetHighResolutionTimeStamp() - uint64StartTime;
    PrintResult(uint64ElapsedTime, uint64WaitForResponse, dwMessageCount, uint64BytesTransmitted, dwMessageSize);

    iResult = shutdown(ConnectSocket, SD_SEND);
    if (iResult == SOCKET_ERROR) {
        closesocket(ConnectSocket);
        WSACleanup();
        return 1;
    }
    closesocket(ConnectSocket);
    return WSACleanup();
}

Here is the server running ZMQ (dealer)

DWORD RunServerZMQTest(DWORD dwPort)
{
    try
    {
        zmq::context_t context(1);
        zmq::socket_t server(context, ZMQ_DEALER);

        // Set options here
        std::string strIdentity = s_set_id(server);
        printf("Created server connection with ID: %s\n", strIdentity.c_str());

        std::string strConnect = "tcp://*:" + std::to_string(dwPort);
        server.bind(strConnect.c_str());

        bool bRunning = true;
        while (bRunning)
        {
            std::string strMessage = s_recv(server);

            if (!s_send(server, strMessage))
            {
                return NO_ERROR;
            }
        }
    }
    catch (zmq::error_t& e)
    {
        return (DWORD)e.num();
    }

return NO_ERROR;

}

Here is the client running ZMQ (dealer)

DWORD RunClientZMQTest(std::string strAddress, DWORD dwPort, DWORD dwMessageSize)
{
    try
    {
        zmq::context_t ctx(1);
        zmq::socket_t client(ctx, ZMQ_DEALER); // ZMQ_REQ

        // Set options here
        std::string strIdentity = s_set_id(client);

        std::string strConnect = "tcp://" + strAddress + ":" + std::to_string(dwPort);
        client.connect(strConnect.c_str());

        if(s_send(client, "INIT"))
        {
            std::string strMessage = s_recv(client);
            if (strMessage.compare("INIT") == 0)
            {
                printf("Client[%s] connected to: %s\n", strIdentity.c_str(), strConnect.c_str());
            }
            else
            {
                return NO_ERROR;
            }
        }
        else
        {
            return NO_ERROR;
        }


        // Statistics
        UINT64 uint64BytesTransmitted   = 0;
        UINT64 uint64StartTime          = s_TimeStampGenerator.GetHighResolutionTimeStamp();
        UINT64 uint64WaitForResponse    = 0;

        DWORD dwMessageCount = 10000000;


        std::string strSendMsg(dwMessageSize, 'X');
        for (DWORD dwI = 0; dwI < dwMessageCount; dwI++)
        {
            if (s_send(client, strSendMsg))
            {
                uint64BytesTransmitted += strSendMsg.size();

                UINT64 uint64BeforeRespone = s_TimeStampGenerator.GetHighResolutionTimeStamp();
                std::string strRecvMsg = s_recv(client);
                if (strRecvMsg.compare(strSendMsg) == 0)
                {
                    uint64WaitForResponse += (s_TimeStampGenerator.GetHighResolutionTimeStamp() - uint64BeforeRespone);
                }
                else
                {
                    return NO_ERROR;
                }
            }
            else
            {
                return NO_ERROR;
            }
        }
        UINT64 uint64ElapsedTime = s_TimeStampGenerator.GetHighResolutionTimeStamp() - uint64StartTime;
        PrintResult(uint64ElapsedTime, uint64WaitForResponse, dwMessageCount, uint64BytesTransmitted, dwMessageSize);
    }
    catch (zmq::error_t& e)
    {
        return (DWORD)e.num();
    }

    return NO_ERROR;
    }

Im running the benchmark locally with message size of 5 bytes and I get the following result:

WINSOCK

Messages sent:                 1 000 000
Time elapsed (us):            48 019 415
Time elapsed (s):                     48.019 415
Message size (bytes):                  5
Msg/s:                            20 825
Bytes/s:                         104 125
Mb/s:                                  0.099
Total   response time (us):   24 537 376
Average repsonse time (us):           24.0

and

ZeroMQ

Messages sent:                 1 000 000
Time elapsed (us):           158 290 708
Time elapsed (s):                    158.290 708    
Message size (bytes):                  5
Msg/s:                             6 317
Bytes/s:                          31 587
Mb/s:                                  0.030
Total   response time (us):  125 524 178    
Average response time (us):          125.0

Can anyone explain why the average response time is so much higher when using ZMQ?

The goal is to find a setup where I can send and receive messages asynchronously without the need to reply. If this can be achieved with a different setup than dealer-dealer, please let me know!

Answer 1

You say you want to send and receive messages asynchronously without the need to reply. Yet the tests done so far are all completely synchronous, essentially request-reply, but on a dealer-dealer socket. Something doesn't compute there. Why not run tests that mimic more closely the design you are aiming for?

ZeroMQ gets a fair amount of it's "faster than TCP" performance by aggregating queued messages into a single message. Obviously, that mechanism cannot be activated in a purely synchronous design with only one message in flight at a time.

As for why this particular test, of very small messages being sent and received purely synchronously, is relatively slow, I cannot say. Have you done profiling? What I will say, again, is that running this test and basing decisions on it doesn't make sense if it doesn't look anything like your final design.

One thing that does look odd is the try/catch block in the ZeroMQ code. That doesn't look fair because the winsock test wasn't written that way. It is known that there is/was a fair amount of overhead in try/catch.

Answer 2

This is only sort of an answer to a little part of your question, but here goes -

Why do you need dealer/dealer? I assume because communication can initiate from either point? You're not tied to dealer/dealer, in particular it limits you to only two endpoints, if you ever add another endpoint on either side of the communication, say, a second client, then each client will only receive half the messages because dealer is strictly round-robin.

What you need for asynchronous communication is some combination of dealer and or router sockets. Neither requires a response, the main differences are in how they choose which connected peer to send a message to:

• Dealer, as said, is strictly round robin, it will send to each connected peer in series
• Router is strictly an addressed message, you have to know the "name" of the peer you want to send to to get the message there.

These two socket types work together because dealer sockets (and request sockets, dealer is a "request-type" socket) send their "name" as part of the message, which the router socket can use to send data back. This is a request/reply paradigm, and you'll see that sort of paradigm enforced in all of the examples in the guide, but you can bend that paradigm to what you're looking for, in particular neither dealer nor router require a reply.

Without knowing your full requirements I can't tell you what sort of ZMQ architecture I would choose, but in general I prefer the expandability of router sockets, it's easier to handle appropriate addressing than it is to shoehorn everything into a single peer... you'll see warnings against doing router/router, and I agree with them to the extent that you should understand what you're doing before you try it, but understanding what you're doing, the implementation isn't that hard.

---

You also have the option, if it fits your requirements, to set up each end with a pub socket, and each with a sub socket, if there are literally no replies ever. If it's strictly a data feed from source to target, and neither peer needs any feedback on what it sends, then this is probably the best choice, even though it means you're dealing with two sockets per end rather than one.

---

None of this addresses performance directly, but the important thing to understand is that zmq sockets are optimized for particular use cases, and as pointed out in John Jefferies' answer, you're breaking that use case for your dealer socket by making the messaging in your test strictly synchronous. The first place to start is to finalize your ZMQ architecture, and then simulate an actual message flow, in particular not adding in arbitrary waits and synchronicity, which will necessarily change the way throughput looks as you're testing it, pretty much by definition.

Answer 3

The OPs issue is a matter of throughput, not latency and is likely a matter of the pattern that is being used in the provided examples. However, you are likely to always find that ZeroMQ has higher latency, which I'll explain, although it might not be useful to the OP in this situation.

ZeroMQ works by buffering messages. Imagine (just as a basic illustration) creating a std::string and appending many small strings to it (many thousands, each including a small header to know the size of these small segments) and then sending this larger string in intervals of 100us, 1000us, 10ms or whatever. On the receiving side, the large string is recived and each smaller message is removed one at a time based on the size header that is sent along with it. This allows you to potentially send millions of messages in batches (although std::string is obviously a bad choice) without the overhead of sending these millions of very small measages one at a time. As a result you take full advantage of your network resources and increase throughput, you also create basic FIFO behavior. However, you also create a delay to allow the buffer to fill, which means an increase in latency.

Imagine (again, just as a basic illustration): if you spend a half second (including string operations, etc) buffering a million messages, this will result in a larger string of a few megabytes. Modern networks can easily send this larger string in the remaining half second. 1000000us (1 second) / 1000000 messages would be 1us per message, right? Wrong - all messages had a half secnd delay to allow the queue to fill, resulting in an increase in latency of upto a half second for all messages. ZeroMQ sends batches much quicker than every 500ms, but the increase in latency that this illustrates is still incurred in ZeroMQ, though its usualy along the lines of a few ms.