Semaphore-like queue in javascript?

Question

I have a variable can_run, that can be either 1 or 0, and then I have a queue of functions, that should be run as soon as the variable is switched from 0 to 1 (but only 1 such function at a time).

Right now, what I do is

var can_run=1;
function wait_until_can_run(callback) {
    if (can_run==1) {
        callback();
    } else {
        window.setTimeout(function(){wait_until_can_run(callback)},100);
    }
}

//...somewhere else...

wait_until_can_run( function(){
   can_run=0;
   //start running something
});

//..somewhere else, as a reaction to the task finishing..
can_run=1;

It works, however, it doesn't strike me as very efficient to have about 100 timeouts continuously running. Something like semaphore would be handy in here; but in general, semaphores are not really needed in JavaScript.

So, what to use here?

edit: I have written "queue of functions" but as seen here, I don't really care about the order.

Answer 1

Here is a nice Queue class you can use without the use of timeouts:

var Queue = (function () {

    Queue.prototype.autorun = true;
    Queue.prototype.running = false;
    Queue.prototype.queue = [];

    function Queue(autorun) {
        if (typeof autorun !== "undefined") {
            this.autorun = autorun;
        }
        this.queue = []; //initialize the queue
    };

    Queue.prototype.add = function (callback) {
        var _this = this;
        //add callback to the queue
        this.queue.push(function () {
            var finished = callback();
            if (typeof finished === "undefined" || finished) {
                //  if callback returns `false`, then you have to 
                //  call `next` somewhere in the callback
                _this.dequeue();
            }
        });

        if (this.autorun && !this.running) {
            // if nothing is running, then start the engines!
            this.dequeue();
        }

        return this; // for chaining fun!
    };

    Queue.prototype.dequeue = function () {
        this.running = false;
        //get the first element off the queue
        var shift = this.queue.shift();
        if (shift) {
            this.running = true;
            shift();
        }
        return shift;
    };

    Queue.prototype.next = Queue.prototype.dequeue;

    return Queue;

})();

It can be used like so:

// passing false into the constructor makes it so 
// the queue does not start till we tell it to
var q = new Queue(false).add(function () {
    //start running something
}).add(function () {
    //start running something 2
}).add(function () {
    //start running something 3
});

setTimeout(function () {
    // start the queue
    q.next();
}, 2000);

Fiddle Demo: http://jsfiddle.net/maniator/dUVGX/

---

Updated to use es6 and new es6 Promises:

class Queue {  
  constructor(autorun = true, queue = []) {
    this.running = false;
    this.autorun = autorun;
    this.queue = queue;
  }

  add(cb) {
    this.queue.push((value) => {
        const finished = new Promise((resolve, reject) => {
        const callbackResponse = cb(value);

        if (callbackResponse !== false) {
            resolve(callbackResponse);
        } else {
            reject(callbackResponse);
        }
      });

      finished.then(this.dequeue.bind(this), (() => {}));
    });

    if (this.autorun && !this.running) {
        this.dequeue();
    }

    return this;
  }

  dequeue(value) {
    this.running = this.queue.shift();

    if (this.running) {
        this.running(value);
    }

    return this.running;
  }

  get next() {
    return this.dequeue;
  }
}

It can be used in the same way:

const q = new Queue(false).add(() => {
    console.log('this is a test');

    return {'banana': 42};
}).add((obj) => {
    console.log('test 2', obj);

    return obj.banana;
}).add((number) => {
    console.log('THIS IS A NUMBER', number)
});

// start the sequence
setTimeout(() => q.next(), 2000);

Although now this time if the values passed are a promise etc or a value, it gets passed to the next function automatically.

Fiddle: http://jsfiddle.net/maniator/toefqpsc/

Answer 2

I'm not sure the best way to do this in plain JS but many libraries have Deferred implementations which are very useful for this use case.

With jQuery:

var dfd = $.Deferred();
var callback = function() { 
    // do stuff
};
dfd.done(callback);  // when the deferred is resolved, invoke the callback, you can chain many callbacks here if needed
dfd.resolve(); // this will invoke your callback when you're ready

EDIT One of the nice things about these library supported deferreds are that they are normally compatible with Ajax events, and in turn other Deferred objects so you can create complex chains, trigger events on Ajax complete, or trigger the 'done' callback after multiple conditions are met. This is of course more advanced functionality, but it's nice to have in your back pocket.

Answer 3

In addition to the other useful answers here, if you don't need the extras that those solutions provide, then an asynchronous semaphore is straightforward to implement.

It's a lower-level concept than the other options presented here though, so you may find those to be more convenient for your needs. Still, I think asynchronous semaphores are worth knowing about, even if you use higher-level abstractions in practice.

It looks something like this:

var sem = function(f){
    var busy = 0;
    return function(amount){
        busy += amount;
        if(busy === 0){
            f();
        }
    };
};

And you invoke it like this:

var busy = sem(run_me_asap);

busy is a function that maintains an internal counter of asynchronous actions that it is waiting on. When that internal counter reaches zero, it fires the function run_me_asap, which you supply.

You can increment the internal counter prior to running an asynchronous action with busy(1) and, then the asynchronous actions is responsible for decrementing the counter with busy(-1) once it is complete. This is how we can avoid the need for timers. (If you prefer, you could write sem so that it returns an object with inc and dec methods instead, like in the Wikipedia article; this is just how I do it.)

And that's all you have to do to create an asynchronous semaphore.

Here's an example of it in use. You might define the function run_me_asap as follows.

var funcs = [func1, func2, func3 /*, ...*/];
var run_me_asap = function(){
    funcs.forEach(function(func){
        func();
    });
});

funcs might be the list of functions that you wanted to run in your question. (Maybe this isn't quite what you want, but see my 'N.B.' below.)

Then elsewhere:

var wait_until_ive_finished = function(){
    busy(1);
    do_something_asynchronously_then_run_callback(function(){
        /* ... */
        busy(-1);
    });
    busy(1);
    do_something_else_asynchronously(function(){
        /* ... */
        busy(-1);
    });
};

When both asynchronous operations have completed, busy's counter will be set to zero, and run_me_asap will be invoked.

N.B. How you might use asynchronous semaphores depends on the architecture of your code and your own requirements; what I've set out may not be exactly what you want. I'm just trying to show you how they work; the rest is up to you!

And, one word of advice: if you were to use asynchronous semaphores then I'd recommend that you hide their creation and the calls to busy behind higher-level abstractions so that you're not littering your application code with low-level details.

Answer 4

I'm visiting from nine years in the future to tell anyone who finds this thread to look into AsyncIterable and "for await (...)". I think this promise-based technique would provide an adequate solution.

Here's an example of an AsyncIterable that is essentially a queue of functions as the questioner was looking for.

class AsyncFunctionIterable {

    resolveWaitingPromise = ()=>{};
    functionQueue = [];
    running = false;

    async *[Symbol.asyncIterator]() {
        this.running = true;
        while( this.running ) {
            // If the queue is empty, wait for another item to be added.
            if(this.functionQueue.length===0) {
                await new Promise( (resolve, reject) => { this.resolveWaitingPromise = resolve } );
                if( !this.running ) {
                    break;
                }
            }
            const func = this.functionQueue.shift();
            const value = await func(); // Potentially asynchronous function.
            yield value;
        }
    }

    add( func ) {
        return this.push( func );
    }

    push( func ) {
        this.functionQueue.push( func );
        this.resolveWaitingPromise();
    }

    stop() {
        this.running = false;
        resolveWaitingPromise();
    }

    unshift( func ) {
        this.functionQueue.unshift( func );
        this.resolveWaitingPromise();
    }

}

The following is a functional example of a filesystem walker. Instead of having a function that goes through your disk and builds a large set of data to return with information on all files found on a filesystem, the AsyncFunctionIterator allows you to go file by file with a much smaller memory footprint and almost no wait time before you're able to start processing data.

const { EventEmitter } = require("events");
const FS = require( "fs" );
const Path = require( "path" );


class FileSystemWalker extends EventEmitter {
    basePath = undefined;
    constructor( str ) {
        super();
        this.basePath = str;
    }

    stop() {
        this.emit( "stop" );
    }
    
    /**
     * returns an AsyncIterable which generates {path:string, stats:FS.Stats} objects.
     */
    walk() {
        const queue = new AsyncFunctionIterable();
        this.once( "stop", () => queue.stop() ); 
        queue.push( async () => handlePath( this.basePath ) );
        return queue;

        async function handlePath( curPath ) {
            const stats = await FS.promises.lstat( curPath );
            if( stats.isDirectory() ) {
                let children = await FS.promises.readdir( curPath );
                if( children.length ) {
                    // To keep things in alphabetical order, we'll:
                    // 1: reverse the order 
                    // 2: frontload them on the queue.
                    children = children.reverse(); 
                    for( const child of children ) {
                        queue.unshift( async () => handlePath( Path.resolve( curPath, child ) ) );
                    }
                }
            }
            return { path: curPath, stats };
        }
    }
}

/**
 * Execution by VM begins here.
 */
( async function() {
    const FSWalker = new FileSystemWalker( process.env.HOME );
    setTimeout( () => FSWalker.stop(), 2000 );
    for await ( const file of FSWalker.walk() ) {
        console.log( file.path + ( file.stats.isDirectory() ? Path.sep : "" ) );
    }
} )().then( () => console.log( "DONE" ) ).then( process.exit );

NOTE: In other code I have, I've used this process, but instead of having a queue of functions, I've used a single function and a set of file paths. This lets us work with a much smaller memory footprint, but requires specific code in the AsyncIterator object. The question was looking for a generic function queue. The nice thing about what this way of looking at the problem is that a function queue doesn't just allow for one type of computation, rather you can put whatever function you like in it and execute them one-by-one.

Put both code snippets in one file to see the code in work. I've put a timer in to show an example of stop execution after 2 seconds.