coding a simple server - 1 producer, multiple consumer

Question

I'm attempting to create a simple server using the producer/consumer problem, pthreads, and mutexes/condition variables. My server accepts a port number, number of threads to create, how big the buffer should be, and what type of scheduling it should do.

Although I can pass tests in the case where my buffer size is one (my job queue only holds one task), I fail tests where my buffer size is greater than one. I do not know why.

The code is rather long, so I will post what I feel are the more pertinent snippets, but I will post the code in its entirety if asked. Again, this IS NOT EVERYTHING.

Main (includes producer), takes the connector descriptor from client, and passes it to "requestHandle", which in turn takes information from the connfd and stores that into a data structure. It then places the data structure on a "queue" -- in my case, I choose a binary tree, with different sorting techniques (FIFO = First in, first out, SFNF = Shortest file name first, SFF = smallest file first).

There are three pertinent files:

• server.c holds references to the data structure's root, mutex, and lock vars.
• request.c saves pertinent info and enqueues/dequeues in different functions
• request.h holds structure types

What is failing:

When attempting to run the code with a buffer size>1 (i.e., I can place more than one item on the queue), the server appears to be crashing. Here is an example test being run on the server as follows:

class Locks(ServerTest):
   name = "locks"
   description = "many concurrent requests to test locking"
   threads = 8
   buffers = 16
   schedalg = "FIFO"
   num_clients = 20
   loops = 20
   requests = ["/home.html", "/output.cgi?0.3"]
   def many_reqs(self):
      for i in range(self.loops):
         for request in self.requests:
            conn = httplib.HTTPConnection("localhost", self.port, timeout=8)
            if self.quiet_get(conn, request):
              self.client_run(conn)
   def run(self):
       serverProc = self.run_server(threads=self.threads, buffers=self.buffers, schedalg=self.schedalg)
      clients = [threading.Thread(target=self.many_reqs) for i in range(self.num_clients)]
      for client in clients:
         client.start()
      for client in clients:
         client.join()
      serverProc.kill()
      self.done()

Running the test above results in the following (except, you know, printed out a million times):

Client failed with error: timed out
Client failed with error: [Errno 104] Connection reset by peer
unable to send request to server. it may have crashed.

server.c global vars:

 //root of "queue"
 TREE_NODE *root;
 //lock
 pthread_mutex_t *m;
 //Cond. v for queue is full
 pthread_cond_t *full;
 //cond. v for queue is empty
 pthread_cond_t *empty;
 //total buff size
  int buff_sz;
 //num nodes used (<= buff_sz)
 int num_used;
 //schedule_type: 0 = FIFO, 1 = SFNF, 2 = SFF)
 int sched_num;

main in server.c (also producer thread)

int main(int argc, char *argv[])
{
    int listenfd, connfd, port, num_threads, clientlen;
    char *sched_type = malloc(8096);

    struct sockaddr_in clientaddr;

    getargs(&port, &num_threads, &buff_sz, sched_type, argc, argv);

    if (strcmp(sched_type, "FIFO") == 0) sched_num = 0;
    if (strcmp(sched_type, "SFNF") == 0) sched_num = 1;
    if (strcmp(sched_type, "SFF") == 0) sched_num = 2;

//initial set up
root = NULL;

num_used = 0;

m = malloc(sizeof(pthread_mutex_t));
pthread_mutex_init(m, NULL);
full = malloc(sizeof(pthread_cond_t));
pthread_cond_init(full, NULL);
empty = malloc(sizeof(pthread_cond_t));
pthread_cond_init(empty, NULL);


// 
// Create some threads...
//
int i;
void *argum = malloc(sizeof(void *));
for(i = 0; i < num_threads; i++) {

    pthread_t *pthread = malloc(sizeof(pthread_t));
    int err_check = pthread_create (pthread, NULL, thread_handle, argum);

    assert(err_check == 0);
}


listenfd = Open_listenfd(port);
while (1) {
clientlen = sizeof(clientaddr);
connfd = Accept(listenfd, (SA *)&clientaddr, (socklen_t *) &clientlen);

// 
// In general, don't handle the request in the main thread.
// Save the relevant info in a buffer and have one of the worker threads 
// do the work. However, for SFF, you may have to do a little work
// here (e.g., a stat() on the filename) ...
// 

    //PLACE LOCK
    pthread_mutex_lock(m);

    //check condition if queue full, T = wait on full
    while (buff_sz <= num_used) pthread_cond_wait(full, m);

    pthread_mutex_unlock(m);

    requestHandle(connfd, sched_num);

    pthread_mutex_lock(m);
    num_used++;

    //signal empty to wake sleeping threads
    pthread_cond_signal(empty);

   //unlock
    pthread_mutex_unlock(m);

    }

}

consumer thread in server.c:

void *thread_handle(void *ptr){
while(1){

    //place lock
    pthread_mutex_lock(m);

    //check condition if empty, T = wait on empty
    while (num_used < 1) pthread_cond_wait(empty, m);

    //dequeue
     runRequest(sched_num);

    num_used--;

    //signal full
    pthread_cond_signal(full);


    //unlock
    pthread_mutex_unlock(m);

}
return NULL;
}//end thread_handle

enqueue function in request.c

 // handle a request
void requestHandle(int fd, int sched_num){ // enqueue and save stuff

 struct req_info *job = malloc(sizeof(struct req_info));

 job->fd = fd;

 struct stat sbuf;
 char buf[MAXLINE], method[MAXLINE], uri[MAXLINE], version[MAXLINE];
 char filename[MAXLINE], cgiargs[MAXLINE];
 rio_t rio;

 Rio_readinitb(&rio, fd);
 Rio_readlineb(&rio, buf, MAXLINE);
 sscanf(buf, "%s %s %s", method, uri, version);

 printf("%s %s %s\n", method, uri, version);
 job->method = strdup(method);
 job->uri = strdup(uri);
 job->version = strdup(version);

  if (strcasecmp(method, "GET")) {
  requestError(fd, method, "501", "Not Implemented", "CS537 Server does not implement this method");
  return;
 }
  requestReadhdrs(&rio);
  job->stat_rio = rio;

  job->is_static = requestParseURI(uri, filename, cgiargs);

  job->filename = strdup(filename);
  job->arguments = strdup(cgiargs);

  if (stat(filename, &sbuf) < 0) {
     requestError(fd, filename, "404", "Not found", "CS537 Server could not find this file");
     return;
  }

  job->sbuf = sbuf;
  job->filename_len = (int)(strlen(filename));

   struct stat *file_stat = malloc(sizeof(struct stat));
   assert((stat(job->filename, file_stat)) == 0);
  //save file size!
   job->size = (int)(file_stat->st_size);

 //create new node
 TREE_NODE *temp = malloc(sizeof(TREE_NODE));
 temp->job_node = job;
 temp->left = NULL;
 temp->right = NULL;

 //now add node!
 int done = 0;
 TREE_NODE *curr = root;
 TREE_NODE *prev = NULL;

  if (root == NULL){
     root = temp;
     done = 1;
  }

  //FIFO: Add to right
  else if (sched_num == 0){

    while (done != 1){

        if (curr->right == NULL){
            curr->right = temp;
            done = 1;
        }
        curr = curr->right;

    }//end while

 }//end FIFO

 //SFNF: Shortest file name first
 else if (sched_num == 1){

    while (done != 1){

        if (curr == NULL){
            curr = temp;
            if ((prev->job_node)->filename_len <= (temp->job_node)->filename_len) prev->right = temp;
            else prev->left = temp;
            done = 1;
        }

        else if ((curr->job_node)->filename_len <= (temp->job_node)->filename_len){
            prev = curr;
            curr = curr->right;
        }

        else {
            prev = curr;
            curr = curr->left;
        }
    }//end while

 }//end SFNF else if


 //SFF: Smallest file first
 else {

     while (done != 1){

        if (curr == NULL){
            curr = temp;
            if ((prev->job_node)->size <= (temp->job_node)->size) prev->right = temp;
            else prev->left = temp;
            done = 1;
        }

        else if ((curr->job_node)->size <= (temp->job_node)->size){
            prev = curr;
            curr = curr->right;
        }

        else {
            prev = curr;
            curr = curr->left;
        }
    }//end while


  }//end else

}

dequeue function in request.c

//FOR THREADS
void runRequest(int sched_num){ // dequeue and run


   struct req_info *job;

   TREE_NODE *prev = NULL;
   TREE_NODE *child = root;
   TREE_NODE *temp = root;
   int done = 0;

  //FIFO: take root
  if (sched_num == 0){
   if(root == NULL){
       printf("SHIT HAS HIT THE FAN\n");
   }
   root = root->right;
   done = 1;
  }

   //SFNF & SFF: take node all the way left
   else {

    while (done != 1){

        if (child->left == NULL){
            if (child->right == NULL){
                temp = child;
                prev->left = NULL;
                done = 1;
            }
            else {
                temp = child;
                prev->left = child->right;
                done = 1;
            }

        }//end if

    prev = child;
    child = child->left;

    }//end SFNF/SFF while
 }//end else

 job = temp->job_node;


 if (job->is_static) {
    if (!(S_ISREG((job->sbuf).st_mode)) || !(S_IRUSR & (job->sbuf).st_mode)) {
       requestError(job->fd, job->filename, "403", "Forbidden", "CS537 Server could not read this file");
      return;
   }
   requestServeStatic(job->fd, job->filename, (job->sbuf).st_size);
 } else {
    if (!(S_ISREG((job->sbuf).st_mode)) || !(S_IXUSR & (job->sbuf).st_mode)) {
      requestError(job->fd, job->filename, "403", "Forbidden", "CS537 Server could not run this CGI program");
      return;
   }

   requestServeDynamic(job->fd, job->filename, job->arguments);

   Close(job->fd);
 }


}

Last, but not least, the header for request.h:

struct req_info {

  char * method;
  char * uri;
  char * version;
  int fd;
  int size;
  char * filename;
  int filename_len;
  char * arguments;
  int is_static;
  rio_t stat_rio;
  struct stat sbuf;

} req_info;


 typedef struct tree_node TREE_NODE;

 struct tree_node {
 struct req_info *job_node;
 TREE_NODE *left, *right;
 };

 void requestHandle(int fd, int shed_num);
 void runRequest(int sched_num);