Using cron with redlock and cluster in nodejs

Question

I want to run a task every minute and for this I chose cron. Now I am running node using cluster module and it's spawning 4 processes but I want to run cron on only once per process per minute.

Now one solution to this is to run it in master. But that wouldn't help if I launch more than one instance with cluster on each instance.

So I decided to look for locking mechanisms. I found redlock and this is the sample code:

const redlock = new Redlock([redisClient], {
    driftFactor: 0.01, // time in ms
    retryCount: 2,
    retryDelay: 200 // time in ms
});

redlock.on('clientError', function (err) {
    console.error('REDLOCK REDIS ERROR: ', err);
});

const LOCK_KEY = "GOHAN_REMINDER_LOCK";
const LOCK_KEY_TTL = 10000;


const job = new CronJob({
    cronTime: '* * * * *',
    onTick: function () {
        /*
         * Runs every minute
         */
        redlock.lock(LOCK_KEY, LOCK_KEY_TTL).then(function (lock) {

            return Bluebird.try(function () {
                const currentMilitaryTime = moment.utc().format("HH:mm");

                return ChildReminder.getRemindersByDate(currentMilitaryTime).then(function (reminders) {
                    console.log("REMINDERS FOR TIME " + currentMilitaryTime + " :", reminders);
                });
            }).then(function () {
                return lock.unlock()
                    .catch(function (err) {
                        console.error("REDLOCK UNLOCK ERROR: ", err);
                    });
            });
        });

    },
    start: false,
    timeZone: 'UTC'
});

job.start();

The logs I get:

REMINDERS FOR TIME 06:08 : [ anonymous {
    user_id: '764418c2-a874-495c-b8c1-cc555ce9b202',
    child_id: '8c8a273f-3a84-4a66-9fa7-bb9e6c5fcf0a',
    reminder_id: 4,
    label: 'textvoice1',
    mode: 'textvoice',
    text_message: 'How you doin\'',
    voice_message: '1480399630',
    created_at: 2016-11-29T06:07:12.296Z,
    updated_at: 2016-11-29T06:07:12.296Z,
    time: '06:08',
    repeat: '1111111',
    on: true },
  anonymous {
    user_id: '764418c2-a874-495c-b8c1-cc555ce9b202',
    child_id: '8c8a273f-3a84-4a66-9fa7-bb9e6c5fcf0a',
    reminder_id: 5,
    label: 'textvoice1',
    mode: 'textvoice',
    text_message: 'How you doin\'',
    voice_message: '1480399638',
    created_at: 2016-11-29T06:07:19.385Z,
    updated_at: 2016-11-29T06:07:19.385Z,
    time: '06:08',
    repeat: '1111111',
    on: true } ]
REMINDERS FOR TIME 06:08 : [ anonymous {
    user_id: '764418c2-a874-495c-b8c1-cc555ce9b202',
    child_id: '8c8a273f-3a84-4a66-9fa7-bb9e6c5fcf0a',
    reminder_id: 4,
    label: 'textvoice1',
    mode: 'textvoice',
    text_message: 'How you doin\'',
    voice_message: '1480399630',
    created_at: 2016-11-29T06:07:12.296Z,
    updated_at: 2016-11-29T06:07:12.296Z,
    time: '06:08',
    repeat: '1111111',
    on: true },
  anonymous {
    user_id: '764418c2-a874-495c-b8c1-cc555ce9b202',
    child_id: '8c8a273f-3a84-4a66-9fa7-bb9e6c5fcf0a',
    reminder_id: 5,
    label: 'textvoice1',
    mode: 'textvoice',
    text_message: 'How you doin\'',
    voice_message: '1480399638',
    created_at: 2016-11-29T06:07:19.385Z,
    updated_at: 2016-11-29T06:07:19.385Z,
    time: '06:08',
    repeat: '1111111',
    on: true } ]
REMINDERS FOR TIME 06:08 : [ anonymous {
    user_id: '764418c2-a874-495c-b8c1-cc555ce9b202',
    child_id: '8c8a273f-3a84-4a66-9fa7-bb9e6c5fcf0a',
    reminder_id: 4,
    label: 'textvoice1',
    mode: 'textvoice',
    text_message: 'How you doin\'',
    voice_message: '1480399630',
    created_at: 2016-11-29T06:07:12.296Z,
    updated_at: 2016-11-29T06:07:12.296Z,
    time: '06:08',
    repeat: '1111111',
    on: true },
  anonymous {
    user_id: '764418c2-a874-495c-b8c1-cc555ce9b202',
    child_id: '8c8a273f-3a84-4a66-9fa7-bb9e6c5fcf0a',
    reminder_id: 5,
    label: 'textvoice1',
    mode: 'textvoice',
    text_message: 'How you doin\'',
    voice_message: '1480399638',
    created_at: 2016-11-29T06:07:19.385Z,
    updated_at: 2016-11-29T06:07:19.385Z,
    time: '06:08',
    repeat: '1111111',
    on: true } ]
REMINDERS FOR TIME 06:08 : [ anonymous {
    user_id: '764418c2-a874-495c-b8c1-cc555ce9b202',
    child_id: '8c8a273f-3a84-4a66-9fa7-bb9e6c5fcf0a',
    reminder_id: 4,
    label: 'textvoice1',
    mode: 'textvoice',
    text_message: 'How you doin\'',
    voice_message: '1480399630',
    created_at: 2016-11-29T06:07:12.296Z,
    updated_at: 2016-11-29T06:07:12.296Z,
    time: '06:08',
    repeat: '1111111',
    on: true },
  anonymous {
    user_id: '764418c2-a874-495c-b8c1-cc555ce9b202',
    child_id: '8c8a273f-3a84-4a66-9fa7-bb9e6c5fcf0a',
    reminder_id: 5,
    label: 'textvoice1',
    mode: 'textvoice',
    text_message: 'How you doin\'',
    voice_message: '1480399638',
    created_at: 2016-11-29T06:07:19.385Z,
    updated_at: 2016-11-29T06:07:19.385Z,
    time: '06:08',
    repeat: '1111111',
    on: true } ]
REMINDERS FOR TIME 06:08 : [ anonymous {
    user_id: '764418c2-a874-495c-b8c1-cc555ce9b202',
    child_id: '8c8a273f-3a84-4a66-9fa7-bb9e6c5fcf0a',
    reminder_id: 4,
    label: 'textvoice1',
    mode: 'textvoice',
    text_message: 'How you doin\'',
    voice_message: '1480399630',
    created_at: 2016-11-29T06:07:12.296Z,
    updated_at: 2016-11-29T06:07:12.296Z,
    time: '06:08',
    repeat: '1111111',
    on: true },
  anonymous {
    user_id: '764418c2-a874-495c-b8c1-cc555ce9b202',
    child_id: '8c8a273f-3a84-4a66-9fa7-bb9e6c5fcf0a',
    reminder_id: 5,
    label: 'textvoice1',
    mode: 'textvoice',
    text_message: 'How you doin\'',
    voice_message: '1480399638',
    created_at: 2016-11-29T06:07:19.385Z,
    updated_at: 2016-11-29T06:07:19.385Z,
    time: '06:08',
    repeat: '1111111',
    on: true } ]

Basically, the job is executed 4-5 times which I do't want. Any help would be appreciated. Thanks.

EDIT #1: I tried changing the retryCount to 0 and increasing/decreasing the TTL but have not had the results that I want.

Links to packages:

Redlock
Cron

score 1 · Answer 1 · answered Dec 07 '16 at 18:34

1

It seems that if an instance gets the lock and frees it before a second one tries to acquire it, both will run. You will need something a little more robust.

I just wrote a small module for node, cronivo, it uses laterJs and redis, and it should work for you, at least it can give you some ideas.

answered Dec 07 '16 at 18:34

Ivo Udelsmann

142
1
13

I found an alternative way. I am not manually releasing the lock and just depending on the TTL for the lock. This way, I haven't faced any issues so far. Can you tell me if there are any issues possible this way? – Abhyudit Jain Dec 09 '16 at 10:02
If an instance is busy and ends trying to execute after the TTL, it will execute even if it shouldn't. If your TTL is large enough this is very unlikely. I do not think this approach is very good, but it could work. – Ivo Udelsmann Dec 09 '16 at 11:52
I don't mind another process starting but i don't want the work done twice or thrice.... my ttl is 60 seconds... I want to check every minute what reminders need to be sent. So if at 06:11 a process gets reminders for 06:11, then at 06:12 i want reminders for 06:12. What i don't want is 4 process getting reminders for 06:11, because then they will be sent 4 times. – Abhyudit Jain Dec 10 '16 at 05:44

Using cron with redlock and cluster in nodejs

1 Answers1