nbqueue

super simple queue module for executing async functions in node.js. nbqueue was created as a module for nodebee, a pure javascript database.

installation

npm install nbqueue

uses

limiting the number of functions being executed

when accessing large amounts of files asynchronously, bad things can happen. for example, take a directory with 20,000 files in it and try accessing them all at once.

with nbqueue you can set the maxProcesses to a sane amount, like 100, and it will execute a maximum of 100 functions at a time. functions added after the limit has been reached will execute as the previous functions finish up.

this is also crucial for maintaining a sane level of forked child processes.

var fs = require( 'fs' )
 
var Queue = require( 'nbqueue' )
 
fs.readdir( directory, function ( err, files )
    {
        // set maxProcesses to 100
        var queue = new Queue( 100 )
 
        files.forEach( function ( file )
            {
                queue.add( function ( next )
                    {
                        fs.readFile( file, function ( err, data )
                            {
                                next( err, data )
                            }
                        )
 
                        // because next takes the same arguments as the readFile callback, you could do
                        // fs.readFile( file, next )
                    }
                )
            }
        )
 
        queue.done( function ( err, results )
            {
                if ( err )
                {
                    // if there were errors, they will be in an array
                    console.log( err )
                }
 
                // results contains an array of the results passed to next()
                console.log( results )
            }
        )
    }
)

executing async functions in order

setting maxProcesses to 1 will ensure only 1 function can be executed at a time.

queues within queues

queue up multiple sets of async functions, and let a the master queue know when they're all finished. here's a not very useful example:

var masterQueue = new Queue()
 
masterQueue.add( function ( nextMaster )
    {
        FS.readdir( '/path/to/directory1', function ( err, files )
            {
                if ( err )
                {
                    return nextMaster( err )
                }
 
                var queue1 = new Queue( 50 )
 
                files.forEach( function ( file )
                    {
                        queue1.add( function ( next )
                            {
                                // do some other stuff here
 
                                next( null, file )
                            }
                        )
                    }
                )
 
                queue1.done( function ( err, results )
                    {
                        nextMaster( err, results )
                    }
                )
            }
        )
    }
)
 
masterQueue.add( function ( nextMaster )
    {
        FS.readdir( '/path/to/directory2', function ( err, files )
            {
                if ( err )
                {
                    return nextMaster( err )
                }
 
                var queue2 = new Queue( 50 )
 
                files.forEach( function ( file )
                    {
                        queue2.add( function ( next )
                            {
                                // do some other stuff here
 
                                next( null, file )
                            }
                        )
                    }
                )
 
                queue2.done( function ( err, results )
                    {
                        nextMaster( err, results )
                    }
                )
            }
        )
    }
)
 
masterQueue.done( function ( err, results )
    {
        if ( err )
        {
            return console.log( err )
        }
 
        // results contains the filenames from both directories
        console.log( results )
    }
)