In my research group, we recently upgraded the OS on our machines from Red Hat 6.2 to Debian 8.3 and observed that the TCP round trip time through the integrated Intel 1G NICs between our machines had doubled from about 110µs to 220µs.
At first, I thought it was a configuration issue, so I copied all the sysctl configurations (such as tcp_low_latency=1) from the un-upgraded Red Hat machines to the Debian machines and that did not fix the issue. Next, I thought this may have been a Linux distribution issue and installed Red Hat 7.2 on the machines, but the round trip times remained around 220µs.
Finally, I figured that maybe the problem was with Linux kernel versions since Debian 8.3 and Red Hat 7.2 had both used kernel 3.x while Red Hat 6.2 used kernel 2.6. So to test this out, I installed Debian 6.0 with Linux kernel 2.6 and bingo! The times were fast again at 110µs.
Have others also experienced these higher latencies in the latest versions of Linux, and are there known workarounds?
Minimum Working Example
Below is a C++ application that can be used to benchmark the latency. It measures latency by sending a message, waiting for a response, and then sending the next message. It does this 100,000 times with 100-byte messages. Thus, we can divide the execution time of the client by 100,000 to get the round trip latencies. To use this first compile the program:
g++ -o socketpingpong -O3 -std=c++0x Server.cpp
Next run the server-sided version of the application on a host (say on 192.168.0.101). We specify the IP to ensure that we are hosting on a well-known interface.
socketpingpong 192.168.0.101
And then use the Unix utility time to measure the execution time of the client.
time socketpingpong 192.168.0.101 client
Running this experiment between two Debian 8.3 hosts with identical hardware gives the following results.
real 0m22.743s
user 0m0.124s
sys 0m1.992s
Debian 6.0 results are
real 0m11.448s
user 0m0.716s
sys 0m0.312s
Code:
#include <unistd.h>
#include <limits.h>
#include <string.h>
#include <linux/futex.h>
#include <arpa/inet.h>
#include <algorithm>
using namespace std;
static const int PORT = 2444;
static const int COUNT = 100000;
// Message sizes are 100 bytes
static const int SEND_SIZE = 100;
static const int RESP_SIZE = 100;
void serverLoop(const char* srd_addr) {
printf("Creating server via regular sockets\r\n");
int sockfd, newsockfd;
socklen_t clilen;
char buffer[SEND_SIZE];
char bufferOut[RESP_SIZE];
struct sockaddr_in serv_addr, cli_addr;
sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (sockfd < 0)
perror("ERROR opening socket");
bzero((char *) &serv_addr, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = inet_addr(srd_addr);
serv_addr.sin_port = htons(PORT);
fflush(stdout);
if (bind(sockfd, (struct sockaddr *) &serv_addr,
sizeof(serv_addr)) < 0) {
perror("ERROR on binding");
}
listen(sockfd, INT_MAX);
clilen = sizeof(cli_addr);
printf("Started listening on %s port %d\r\n", srd_addr, PORT);
fflush(stdout);
while (true) {
newsockfd = accept(sockfd, (struct sockaddr *) &cli_addr, &clilen);
if (newsockfd < 0)
perror("ERROR on accept");
printf("New connection\r\n");
int status = 1;
while (status > 0) {
// Read
status = read(newsockfd, buffer, SEND_SIZE);
if (status < 0) {
perror("read");
break;
}
if (status == 0) {
printf("connection closed");
break;
}
// Respond
status = write(newsockfd, bufferOut, RESP_SIZE);
if (status < 0) {
perror("write");
break;
}
}
close(newsockfd);
}
close(sockfd);
}
int clientLoop(const char* srd_addr) {
// This example is copied from http://www.binarytides.com/server-client-example-c-sockets-linux/
int sock;
struct sockaddr_in server;
char message[SEND_SIZE] , server_reply[RESP_SIZE];
//Create socket
sock = socket(AF_INET , SOCK_STREAM , 0);
if (sock == -1)
{
printf("Could not create socket");
}
puts("Socket created");
server.sin_addr.s_addr = inet_addr(srd_addr);
server.sin_family = AF_INET;
server.sin_port = htons( PORT );
//Connect to remote server
if (connect(sock , (struct sockaddr *)&server , sizeof(server)) < 0)
{
perror("connect failed. Error");
return 1;
}
printf("Connected to %s on port %d\n", srd_addr, PORT);
// Fill buffer
for (int i = 0; i < SEND_SIZE; ++i) {
message[i] = 'a' + (i % 26);
}
for (int i = 0; i < COUNT; ++i) {
if (send(sock, message, SEND_SIZE, 0) < 0) {
perror("send");
return 1;
}
if ( recv(sock, server_reply, RESP_SIZE, 0) < 0) {
perror("recv");
return 1;
}
}
close(sock);
printf("Sending %d messages of size %d bytes with response sizes of %d bytes\r\n",
COUNT, SEND_SIZE, RESP_SIZE);
return 0;
}
int main(int argc, char** argv) {
if (argc < 2) {
printf("\r\nUsage: socketpingpong <ipaddress> [client]\r\n");
exit(-1);
}
if (argc == 2)
serverLoop(argv[1]);
else
clientLoop(argv[1]);
return 0;
}
