czmq multiples zactor instances crash

Question

I'm trying to write a basic example using zhash and zactor from the czmq library. The main idea of what I'm trying to achieve is:

1. Create 1024 zactor instances and send "START" command after each actor creation.
2. Wait for response from the actor to continue creating zactors.
3. Delete all previously created actors.

I'm a bit lost as to why this code breaks. Every time I reach the actor number 60 the application exits:

...
...
58 actor started!
START command received!
59 actor started!
START command received!
60 actor started!
> Assertion failed: (self), function zsock_set_sndtimeo, file
> src/zsock_option.c, line 1344.
> Abort trap: 6

To compile: gcc -o demo demo.c -g -lczmq

The code is the following:

#include <stdio.h>
#include <czmq.h>

typedef struct {
    zsock_t *pipe;              //  Actor command pipe
    zpoller_t *poller;          //  Socket poller
    int terminated;
} accountactor_t;

typedef struct{
    zactor_t *actor;
    int foo;
} account_t;

accountactor_t *
accountactor_new (zsock_t *pipe, void *args)
{
    accountactor_t *self = (accountactor_t *) zmalloc (sizeof (accountactor_t));
    assert (self);

    self->pipe = pipe;
    self->poller = zpoller_new (self->pipe, NULL);
        self->terminated = 0;
    return self;

}

static void
accountactor_recv_api (accountactor_t *self)
{
//  Get the whole message of the pipe in one go
    zmsg_t *request = zmsg_recv (self->pipe);
    if (!request){
        return;        //  Interrupted
    }

    char *command = zmsg_popstr (request);

    if (streq (command, "START")){
            zsys_debug("START command received!");
        zsock_signal (self->pipe, 0);
    }else
    if (streq (command, "STOP")){
            zsys_debug("STOP command received!");
            zsock_signal (self->pipe, 0);
    }else
    if (streq (command, "$TERM")){
            zsys_debug("$TERM command received!");
        //  The $TERM command is send by zactor_destroy() method
        self->terminated = 1;

    }else {
        zsys_error ("Invalid command '%s'", command);
        zsock_signal (self->pipe, -1);
    }

    zmsg_destroy(&request);
    if(command){
        free(command);
    }
}

void
actor_fcn (zsock_t *pipe, void *args)
{
    accountactor_t * self = accountactor_new (pipe, args);
    if (!self)
        return;          //  Interrupted

        int rc  = 0;
    //  Signal actor successfully initiated
    zsock_signal (self->pipe, 0);

    while (!self->terminated) {
       zsock_t *which = (zsock_t *) zpoller_wait (self->poller, -1);
       if (which == self->pipe){
                        accountactor_recv_api (self);
       }
    }

        if(zpoller_terminated(self->poller)){
            zsys_debug("Poller Interrupted!");
        }else
        if(zpoller_expired(self->poller)){
            zsys_debug("Poller Expired!");
        }

        //  Free object itself
        zpoller_destroy (&self->poller);
        zsock_destroy(&self->pipe);
        free(self);
        self = NULL;

}

void
s_account_free (void *argument)
{
    account_t *account = (account_t *) argument;
        zstr_send (account->actor, "$TERM");
    zactor_destroy(&account->actor);
    free(account);
    zsys_debug("Item removed!");
}

int main(){

    zhash_t *table = zhash_new();
    int i = 0;

    account_t *ptrs[1024];
    char key[10];

    for(i=0; i<1024; i++){

        ptrs[i] = (account_t *) zmalloc (sizeof (account_t));

        ptrs[i]->actor = zactor_new (actor_fcn, NULL);
        sprintf(&key[0],"%d",i);
        zhash_insert(table, key, (void *)ptrs[i]);
        zhash_freefn(table, key, s_account_free);

        zstr_send (ptrs[i]->actor, "START");
        zsock_wait (ptrs[i]->actor);
        zsys_debug("%d actor started!",i);

    }

    i = zhash_size(table);
    // Delete all
    while(i--){
        sprintf(&key[0],"%d",i);
        zhash_delete(table, key);
        free(ptrs[i]);
    }

    return 0;

}

Any ideas? I cannot see why I am reaching this 60 number limit.

Answer 1

What operating system are you on? Is it OS/X?

The current actor implementation still uses PAIR sockets, and these use actual file handles internally, for signalling. Each actor has two PAIR sockets, and each of those uses two file handles, so 60 actors = 240 file handles.

On OS/X the default limit per process is 256. You can raise this, see: http://zeromq.org/docs:tuning-zeromq

On Linux the default is 1024, and you can raise this. On Windows you need to recompile libzmq with FD_SETSIZE set to 16K or somesuch (this is what libzmq master now does, but older versions have lower values).

Quite separately:

• You do not need any hand-shaking on construction, as this is already done by zactor_new (), you'll see all actors send a signal when they've initialized.

• Creating 1024 actors is probably excessive unless you're actually testing system limits. Actors use system threads; for best performance you want one thread per code. For best design, one thread per concurrent thread of work.