multi-tasks

multi-tasks

Install:

npm install multi-tasks

How to use:

//see examples/example0
let multiTasks = require('multi-tasks').multiTasks;


//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
let alltasks = [];
for(let i=0;i<50;i++){
    alltasks.push({
        name: `task-${i}`,
        data: `This prop is for a subtask`
    });
};

//Step2, Provide a function to process a certain sub-task and return the result data
let processTask = (task, helper)=>{
    let {taskCount} = task;

    return 'a result which is not a promise'; 
};

//Step3, run!
multiTasks({
    initialTasks: alltasks, 
    processTask, 
    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
    taskId: 'my-task',
    numberOfWorkers: 3, //Assign how many workers are working in parallel
    //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
    shouldTerminate:(info)=>{
        //return true if you need to terminate the whole process
    },
    onFinish: (report)=>{
        console.log('finish callback', report);
    }
});

Have a try:

node examples/example0/run
node examples/example1/run

More about the task processing function

//you can return a promise for async processes
let processTask = (task, helper)=>{
    let {taskCount} = task;

    return new Promise((resolve, reject)=>{
        resolve({
            data:`task${taskCount} complete`
        })
    })
};

//dynamically create a new task while processing
let processTask = (task, helper)=>{
    let {taskCount} = task;

    
    if(taskCount % 2 === 0){
        helper.createNewTasks({//You can dynamically create a new task if find something new while processing
            msg:'a new task'
        });
        return;
    }

    return new Promise((resolve, reject)=>{
        resolve({
            data:`task${taskCount} complete`
        })
    })
};

//this example shows how to generate/handle exceptions
let processTask = (task, helper)=>{
    let {taskCount} = task;

    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
    if(taskCount===3) throw 'exception';
    if(taskCount===4) return Promise.reject({err:'a test error'});
    if(taskCount===5) aaa =  bbb;

    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is OK

    //by default, you should return a promise, all returned data can be found in the results/succ folder
    //but to return a non-promise result is also OK, see above
    return new Promise((resolve, reject)=>{
        setTimeout(()=>{
            if(taskCount % 2 === 0){
                helper.createNewTasks({//You can dynamically create a new task if find something new while processing
                    msg:'a new task'
                });
                resolve()
            }else{
                resolve({
                    data:`task${taskCount} complete`
                })
            }
        }, 10)
    })
}

Changelog:

• 2.0.5 Update readme examples
• 2.0.4 Support resume from a failed task
• 2.0.3 Fix: mkdir bug on windows
• 2.0.2 new feature: support shouldTerminate
• 2.0.1 Avoid possible I/O conflicts.
• 2.0.0 Rewritten with a new architecture to support dynamic tasks.
• 1.2.8 Fix: create task folder failed on MacOS
• 1.2.7 Small updates
• 1.2.6 Support onFinish event
• 1.2.5 Rename numberOfWorks to numberOfWorkers, the old one are still supported ;-)
• 1.2.4 Fix: opt.numberOfWorkers not work
• 1.2.3 Update README
• 1.2.2 Update README and examples
• 1.2.1 Handle exceptions and errors in subtasks
• 1.2.0 Simplified usage by providing the function way and support return Promise
• 1.1.4 Remove make-dir
• 1.1.3 Simplified usage, see example0
• 1.1.2
• 1.1.1 Rename files, updated changelog
• 1.1.0 Simplified the usage of a customized Consumer, see example0
• 1.0.8 Fix examples
• 1.0.7 Remove moment
• 1.0.6 Performance optimization

Github:

https://github.com/zhanglei923/multi-tasks