0

I have a server that is using epoll. My question is what is the best way to process all the buffers so I can deal with half messages, or multiple messages coming in a buffer.

For example:

If the message is "Hello From Client Number #".

Buffer's received through epoll:

From Client 1: "Hello From"

From Client 2: "Hello From Client Number 2Hello From Client Number 2"

From Client 1: " Client Number 1HelloFromClient Number 1"

In this scenario, I need to be able to identify that "Hello From" is only half a message and store it somewhere. Then I need to be able to process that two whole messages have gone through from Client 2. And then go back and pick up where I left off from Client 1. I know there are ways of distinguishing messages using delimiters, or sending message length, but I'm not quit sure on how to deal with receiving half a message at all.

Does anyone have any ideas, or any sample code I could look at.

Note I know it's bad that I'm currently processing everything into one buffer. I'm going to change that. Would I need a separate buffer for each client?

Thanks for any help!

void epoll(int listening_port)
{
    char buffer[500];       //buffer for message
    int listen_sock = 0;    //file descriptor (fd) for listening socket
    int conn_sock = 0;      //fd for connecting socket
    int epollfd = 0;         // fd for epoll
    int nfds = 0;           //number of fd's ready for i/o
    int i = 0;              //index to which file descriptor we are lookng at
    int curr_fd = 0;        //fd for socket we are currently looking at
    bool loop = 1;          //boolean value to help identify whether to keep in loop or not
    socklen_t address_len;
    struct sockaddr_in serv_addr;
    struct epoll_event ev, events[EPOLL_MAX_EVENTS];
    ssize_t result = 0;


    bzero(buffer, sizeof(buffer));
    bzero(&serv_addr, sizeof(serv_addr));

    serv_addr.sin_family = AF_INET;
    serv_addr.sin_port = listening_port;
    serv_addr.sin_addr.s_addr = INADDR_ANY;

    listen_sock = create_socket();


    if(bind(listen_sock, SA &serv_addr, sizeof(serv_addr)) != 0)
    {
        perror("Bind failed");
    }
    else
    {
        printf("Bind successful\n");
    }


    set_socket_nonblocking(listen_sock);

    listen_on_socket(listen_sock, SOMAXCONN); //specifying max connections in backlog

    epollfd = initialize_epoll();

    ev.events = EPOLLIN | EPOLLOUT | EPOLLET | EPOLLRDHUP;
    ev.data.fd = listen_sock;

    if(epoll_ctl(epollfd, EPOLL_CTL_ADD, listen_sock, &ev) == ERROR)
    {
        perror("Epoll_ctl: listen sock");
    }
    else
    {
        printf("Successfully added listen socket to epoll\n");
    }

    while (RUN_EPOLL)
    {
        nfds = epoll_wait(epollfd, events, EPOLL_MAX_EVENTS, 0); //waiting for incoming connection;
        if(nfds == ERROR)
        {
            perror("EPOLL_Wait");
        }
        //printf("Finished waiting\i");

        for(i = 0; i < nfds; ++i)
        {
            curr_fd = events[i].data.fd;
            loop = true; //reset looping flag
            //Notification from Listening Socket - Process Incoming Connections

            if (curr_fd == listen_sock) {

                while(loop)
                {
                    conn_sock = accept(listen_sock, SA &serv_addr, &address_len); //accept incoming connection
                    printf("Accepted new incoming connection - socket fd: %d\n", conn_sock);
                    if (conn_sock > 0) //if successful set socket nonblocking and add it to epoll
                    {

                        set_socket_nonblocking(conn_sock);

                        ev.events = EPOLLIN | EPOLLOUT| EPOLLET | EPOLLRDHUP; //setting flags
                        ev.data.fd = conn_sock; //specify fd of new connection in event to follow
                        if (epoll_ctl(epollfd, EPOLL_CTL_ADD, conn_sock, &ev) == ERROR) //add fd to monitored fd's
                        {
                            perror("epoll_ctl: conn_sck");
                        }
                        else
                        {
                            printf("Added %d to monitor list\n", conn_sock);
                        }


                    }
                    else if (conn_sock == ERROR)
                    {
                        if ((errno == EAGAIN) || (errno == EWOULDBLOCK))
                        {
                            printf("All incoming connections processed\n");
                            loop = false;
                        }
                        else
                        {
                            perror("Accept remote socket");
                            loop = false;
                        }
                    }
                }
            }
            else if(events[i].events & EPOLLRDHUP) //detecting if peer shutdown
            {
                printf("Detected socket peer shutdown. Closing now. \n");

                if (epoll_ctl(epollfd, EPOLL_CTL_DEL, curr_fd, NULL) == ERROR) {
                    perror("epoll_ctl: conn_sck");
                }

                close_socket(curr_fd);
            }
            else if(events[i].events & EPOLLIN)
            {
                while(loop)
                {
                    result = recv(curr_fd, buffer, sizeof(buffer), 0);
                    //printf("Length of incoming message is %d\i", result);

                    if(result > 0) //
                    {
                        printf("File Descriptor: %d. Message: %s\n", curr_fd, buffer); //I know this will need to be changd
                        bzero(buffer, sizeof(buffer));
                    }
                    else if(result == ERROR) //Message is completely sent
                    {

                        if(errno == EAGAIN)
                        {

                            loop = false;

                        }
                    }
                    else if(result == 0)
                    {
                        //Removing the fd from the monitored descriptors in epoll
                        if (epoll_ctl(epollfd, EPOLL_CTL_DEL, curr_fd, NULL) == ERROR) {
                            perror("epoll_ctl: conn_sck");
                        }
                        close_socket(curr_fd); //Closing the fd
                        loop = false;
                    }

                }

            }
        }

    }

    close_socket(listen_sock);
    //need to develop way to gracefully close out of epoll

    return;

}
Alex Erling
  • 307
  • 4
  • 19
  • 1
    `recv(curr_fd, buffer, sizeof(buffer), 0);` and `printf("File Descriptor: %d. Message: %s\n", curr_fd, buffer);` will result in undefined behavior if `sizeof(buffer)` bytes are received as there is no space for a `'\0'` terminator. – Andrew Henle Jul 16 '19 at 16:59

1 Answers1

1

Would I need a separate buffer for each client?

Yes. You will need to store a separate client state for each client. As part of that client's state, the partial message buffer should also be stored separately for each client.

Does anyone have any ideas, or any sample code I could look at.

You could look at the code for the facil.io library, and it's raw-HTTP example code.

In the example code you will notice that each HTTP client (protocol / state object) will have it's own target buffer for reading.

The facil.io library uses epoll under the hood (or kqueue on BSD/macOS and poll if you really want to go portable) - so the framework's logic applies to your case.

Sometimes it's possible to use a stack allocated (or per-thread) buffer, but this is only true if you later copy the data you need to keep.

Myst
  • 18,516
  • 2
  • 45
  • 67