Restarting a UDP socket to accept DTLS connection not working

Question

I am starting a UDP Server and then accepting a DTLS connection to read incoming data. The code is working fine. But once I reuse the connection from the client by restarting the client socket Server is unable to read anything. Using Cyassl library for SSL instance creation.

In Client: Socket Binding and DTLS initialisation is fine as it is re-using the DTLS context used earlier.

In Server: Also the socket closing and starting everything is fine but still I am unable to receive in the accepted FD.

But if I put a sleep of exactly 5 seconds between closing and starting the UDP server everything is fine. But I don't want to use blocking I/O like the sleep.

Please do help. Server:

Accepting data received in m_udp_fd:

    if(FD_ISSET(m_udp_fd, &rfds))
    {
        if(m_accepted_flag==0)
        {
            udp_read_connect(m_udp_fd);
            open_dtls_server();
            if(m_ssl_accept(m_udp_fd)>0)
            {
                media_accepted_flag = 1;
            }
        }
        else 
        {
                // Read data
                if(process_dtls_packet(m_udp_fd)<=0)
                {
                    //Clear FD here
                }
        }   

    }

Place where the connection is broken depending whether TCP connection is broken or not. This is getting hit perfectly:

    if (FD_ISSET(m_fd, &rfds))
    {
         if(process_msg(m_fd)<=0)
         {
             closesocket(m_fd);
             FD_CLR(m_fd,recv_fds);
             if(recv_fdmax == m_fd)
             {
                    recv_fdmax = max(x_fd,y_fd);
                    recv_fdmax = max(recv_fdmax,z_fd);
                    recv_fdmax = max(recv_fdmax,p_fd);
                    recv_fdmax = max(recv_fdmax,q_fd);
             }
         }

         if(m_accepted_flag!=0)
         {
                m_accepted_flag = 0;
                x_accepted_flag = 0;
                //This sleep when kept works perfectly
                Sleep(5000);
                close_m_and_x_connection(&recv_fdmax,recv_fds,&m_udp_fd,&x_udp_fd);
         }
    }

UDP Read Connect Function:

    int udp_read_connect(SOCKET sockfd)
    {
         struct sockaddr cliaddr;
         char          b[1500];
         int           n;
         int     len = sizeof(cliaddr);

         n = (int)recvfrom(sockfd, (char*)b, sizeof(b), MSG_PEEK,
                  (struct sockaddr*)&cliaddr, &len);
        if (n > 0) 
        {
              if (connect(sockfd, (const struct sockaddr*)&cliaddr,sizeof(cliaddr)) != 0)
              printf("udp connect failed");
        }
        else
              printf("recvfrom failed");

        return sockfd;
     }

Open DTLS Server Method: Here dtls_ctx and dtls_meth are global variables.

    int open_dtls_server()
    {
        int verify_flag = OFF;       
        int error;

        if(dtls_ctx!= NULL)
            return;

        dtls_ctx  = ssl_create_context(DTLS_V1, &dtls_meth);
        dtls_meth = CyaDTLSv1_server_method();
        if(dtls_meth==NULL)
        {
            return -1;
        }

        dtls_ctx = CyaSSL_CTX_new(dtls_meth);

        if(NULL == dtls_ctx)
        {
           ERR_print_errors_fp(stderr);
           return -1;
        }

       if((status   =   LoadSSLCertificate(dtls_ctx,sslInfo,&error,0)) == 1)
       {
             return 0;
       }
       else
             return 1;

       if(CyaSSL_CTX_use_PrivateKey_file(dtls_ctx, SSL_SERVER_RSA_KEY, SSL_FILETYPE_PEM) <= 0)  
       {
            ERR_print_errors_fp(stderr);
            return -1;      
       }

       if(CyaSSL_CTX_check_private_key(dtls_ctx) != 1)
       {
            return -1;
       }    
       if(verify_flag)
       {    
           if(!CyaSSL_CTX_load_verify_locations(dtls_ctx, SSL_SERVER_RSA_CA_CERT, NULL))
           {
                ERR_print_errors_fp(stderr);
                return -1;      
           }
            CyaSSL_CTX_set_verify(dtls_ctx, SSL_VERIFY_PEER, NULL);
       }

       CyaSSL_CTX_set_options(dtls_ctx, SSL_OP_ALL);
       return 1;
   }

m_ssl_accept Function: ssl_m is also a global variable.

    int m_ssl_accept(int confd)
    {
        int ret;
        if(ssl_m==NULL)
        {
            ssl_m = CyaSSL_new(dtls_ctx);

            if (ssl_m == NULL) 
              printf("SSL_new failed");

            CyaSSL_set_fd(ssl_m, confd);
        }

        do
        {
              if((ret = CyaSSL_accept(ssl_m))!= 1)
              {
                   printf("Handshake Error on M Channel %d with FD: [%d]\n", 
                   return -1;
              }

        }while(ret != 1);

        return 1;
    }