25

I have a very simple server/client performance test using boost::asio on Windows and it seems to be performing really poorly. I'm hoping that I'm just using the library incorrectly and would appreciate any advice.

I have a session class that writes a message-length and then writes a message, and then waits to read a message-length and then read a message, and keeps doing this over and over again nonstop. When I run it locally on my own computer I get blazingly fast performance, however; when I run a server on one computer and a client on another computer, even on the same network, the performance slows down, taking as much as 1 second for a read/write operation to occur.

The server source code file is as follows:

#include <cstdlib>
#include <iostream>
#include <boost/asio.hpp>
#include <boost/bind.hpp>

using namespace boost;
using namespace boost::asio;
using namespace boost::asio::ip;
using namespace std;

class Session {
  public:

    Session(io_service& ioService)
      : m_socket(ioService) {}

    tcp::socket& GetSocket() {
      return m_socket;
    }

    void StartRead() {
      m_messageSizeIterator = reinterpret_cast<char*>(&m_messageSize);
      async_read(m_socket, buffer(m_messageSizeIterator, sizeof(m_messageSize)),
        bind(&Session::HandleSizeRead, this, placeholders::error,
        placeholders::bytes_transferred));
    }

    void StartWrite(const char* message, int messageSize) {
      m_messageSize = messageSize;
      m_message = new char[m_messageSize];
      memcpy(m_message, message, m_messageSize);
      async_write(m_socket, buffer(&m_messageSize, sizeof(int)),
        bind(&Session::HandleSizeWritten, this, placeholders::error));
    }

    void HandleSizeRead(const system::error_code& error,
        size_t bytes_transferred) {
      if(!error) {
        m_message = new char[m_messageSize];
        async_read(m_socket, buffer(m_message, m_messageSize),
          bind(&Session::HandleMessageRead, this, placeholders::error,
          placeholders::bytes_transferred));
      } else {
        delete this;
      }
    }

    void HandleMessageRead(const system::error_code& error,
        size_t bytes_transferred) {
      if(!error) {
        cout << string(m_message, m_messageSize) << endl;
        async_write(m_socket, buffer(&m_messageSize, sizeof(int)),
          bind(&Session::HandleSizeWritten, this, placeholders::error));
      } else {
        delete this;
      }
    }

    void HandleSizeWritten(const system::error_code& error) {
      if(!error) {
        async_write(m_socket, buffer(m_message, m_messageSize),
          bind(&Session::HandleMessageWritten, this, placeholders::error));
      } else {
        delete this;
      }
    }

    void HandleMessageWritten(const system::error_code& error) {
      if(!error) {
        delete m_message;
        m_messageSizeIterator = reinterpret_cast<char*>(&m_messageSize);
        async_read(m_socket, buffer(m_messageSizeIterator,
          sizeof(m_messageSize)), bind(&Session::HandleSizeRead, this,
          placeholders::error, placeholders::bytes_transferred));
      } else {
        delete this;
      }
    }

  private:
    tcp::socket m_socket;
    int m_messageSize;
    char* m_messageSizeIterator;
    char* m_message;
};

class Server {
  public:

    Server(io_service& ioService, short port)
        : m_ioService(ioService),
          m_acceptor(ioService, tcp::endpoint(tcp::v4(), port)) {
      Session* new_session = new Session(m_ioService);
      m_acceptor.async_accept(new_session->GetSocket(), bind(&Server::HandleAccept,
        this, new_session,asio::placeholders::error));
    }

    void HandleAccept(Session* new_session, const system::error_code& error) {
      if(!error) {
        new_session->StartRead();
        new_session = new Session(m_ioService);
        m_acceptor.async_accept(new_session->GetSocket(), bind(
          &Server::HandleAccept, this, new_session, placeholders::error));
      } else {
        delete new_session;
      }
    }

  private:
    io_service& m_ioService;
    tcp::acceptor m_acceptor;
};

int main(int argc, char* argv[]) {
  try {
    if(argc != 2) {
      cerr << "Usage: server <port>\n";
      return 1;
    }
    io_service io_service;
    Server s(io_service, atoi(argv[1]));
    io_service.run();
  } catch(std::exception& e) {
    cerr << "Exception: " << e.what() << "\n";
  }
  return 0;
}

And the client code is as follows:

#include <cstdlib>
#include <cstring>
#include <iostream>
#include <boost/bind.hpp>
#include <boost/asio.hpp>

using namespace boost;
using namespace boost::asio;
using namespace boost::asio::ip;
using namespace std;

class Session {
  public:

    Session(io_service& ioService)
      : m_socket(ioService) {}

    tcp::socket& GetSocket() {
      return m_socket;
    }

    void StartRead() {
      m_messageSizeIterator = reinterpret_cast<char*>(&m_messageSize);
      async_read(m_socket, buffer(m_messageSizeIterator, sizeof(m_messageSize)),
        bind(&Session::HandleSizeRead, this, placeholders::error,
        placeholders::bytes_transferred));
    }

    void StartWrite(const char* message, int messageSize) {
      m_messageSize = messageSize;
      m_message = new char[m_messageSize];
      memcpy(m_message, message, m_messageSize);
      async_write(m_socket, buffer(&m_messageSize, sizeof(int)),
        bind(&Session::HandleSizeWritten, this, placeholders::error));
    }

    void HandleSizeRead(const system::error_code& error,
        size_t bytes_transferred) {
      if(!error) {
        m_message = new char[m_messageSize];
        async_read(m_socket, buffer(m_message, m_messageSize),
          bind(&Session::HandleMessageRead, this, placeholders::error,
          placeholders::bytes_transferred));
      } else {
        delete this;
      }
    }

    void HandleMessageRead(const system::error_code& error,
        size_t bytes_transferred) {
      if(!error) {
        cout << string(m_message, m_messageSize) << endl;
        async_write(m_socket, buffer(&m_messageSize, sizeof(int)),
          bind(&Session::HandleSizeWritten, this, placeholders::error));
      } else {
        delete this;
      }
    }

    void HandleSizeWritten(const system::error_code& error) {
      if(!error) {
        async_write(m_socket, buffer(m_message, m_messageSize),
          bind(&Session::HandleMessageWritten, this, placeholders::error));
      } else {
        delete this;
      }
    }

    void HandleMessageWritten(const system::error_code& error) {
      if(!error) {
        delete m_message;
        m_messageSizeIterator = reinterpret_cast<char*>(&m_messageSize);
        async_read(m_socket, buffer(m_messageSizeIterator,
          sizeof(m_messageSize)), bind(&Session::HandleSizeRead, this,
          placeholders::error, placeholders::bytes_transferred));
      } else {
        delete this;
      }
    }

  private:
    tcp::socket m_socket;
    int m_messageSize;
    char* m_messageSizeIterator;
    char* m_message;
};

int main(int argc, char* argv[]) {
  try {
    if(argc != 3) {
      cerr << "Usage: client <host> <port>\n";
      return 1;
    }
    io_service io_service;
    tcp::resolver resolver(io_service);
    tcp::resolver::query query(tcp::v4(), argv[1], argv[2]);
    tcp::resolver::iterator iterator = resolver.resolve(query);
    Session session(io_service);
    tcp::socket& s = session.GetSocket();
    s.connect(*iterator);
    cout << "Enter message: ";
    const int MAX_LENGTH = 1024;
    char request[MAX_LENGTH];
    cin.getline(request, MAX_LENGTH);
    int requestLength = strlen(request);
    session.StartWrite(request, requestLength);
    io_service.run();
  } catch (std::exception& e) {
    cerr << "Exception: " << e.what() << "\n";
  }
  return 0;
}

Any help would be appreciated, thanks.

For my purposes, sending really really small messages and wanting virtual real time replies, disabling Nagle's algorithm turned out to be the cause of the poor performance.

Sam Miller
  • 23,808
  • 4
  • 67
  • 87
Kranar
  • 1,136
  • 2
  • 13
  • 24
  • Have you ruled out the possibilities of something your router could be doing that is causing the problem? What is the CPU usage like on each machine? – bobber205 Jan 11 '10 at 00:14
  • The CPU usage is 0 on both machines. As for the router being the issue, I wrote a similar program without using ASIO and it worked really fast. – Kranar Jan 11 '10 at 00:28
  • 3
    Begin by testing your link using iperf. Then, instrument the whole process - Was the socket created? Did the bind succeed? Did the resolve work? Did the connect to the server work? Did the first send work? Does the first receive work? Does any network API call return any errors? Did anything take longer than expected? Look at the network traffic. Is there is a firewall on the server? Is the Nagle algorithm enabled? Did the server take a long time to respond? Is there some delay you did not expect in the non-network code in the client? – Permaquid Jan 11 '10 at 00:41
  • 2
    Thanks for your help Permaquid. I went one by one through all your suggestions and finally the problem was that Nagle's algorithm was enabled, when for my purposes sending such small messages and desiring virtual real time responses, the best thing to do was to disable it. It now performs very well. – Kranar Jan 11 '10 at 01:03
  • 2
    Kranar, what kind of performance are you getting now? Just out of interest. – Carl Aug 02 '11 at 22:26

2 Answers2

41

You must turn off the Nagle algorithm. Call:

m_socket.set_option(tcp::no_delay(true));

Where appropriate for your code.

janm
  • 17,976
  • 1
  • 43
  • 61
  • +1 for suggestion and for linking in nagle on wiki. However - would have liked to see a warning that turning of Nagle will probably lower overall throughput. – quixver Feb 19 '13 at 21:23
  • 2
    @quixver No nagle will lead to more packets on the network if you are sending bytes with a small gap between them. A timer (ie. Nagle) to coalesce will lead to fewer packets and so improve overall network throughput. This is true for interactive keyboard traffic (eg. telnet, ssh) and was a big part of Ethernet traffic twenty years ago. For program-to-program communication Nagle leads to overall lower throughput (as was the case in the original question) rather than higher throughput. See, for example, that the whole message was passed to async_write(), and so there is no need to wait to send. – janm Feb 20 '13 at 05:04
6

For my purposes, sending really really small messages and wanting virtual real time replies, disabling Nagle's algorithm turned out to be the cause of the poor performance.

Kranar
  • 1,136
  • 2
  • 13
  • 24