How to traverse all files, and support pause and continue

Question

I have created a NodeJS (electron) code for read all the files in a specific directory and subdirectories. I don't want to use too much HD resources, that why I use a delay of 5ms between folders.

Now my question. I want the if my NODE process stop? I want to be able to continue from when it is stopped. How should I do that?

In other words: How to keep index of current state while walking in all files and folder, so I can continue the traversing from when it has stopped.

Thank you

My Code:

var walkAll=function(options){
    var x=0
    walk(options.dir,function(){})
    function walk(dir,callback) {
      var files=fs.readdirSync(dir);
      var stat;
      async.eachSeries(files,function(file,next){
        file=dir +'/' + file
        if (dir.match(/Recycle/)) return next() 
        if (dir.match(/.git/)) return next() 
        if (dir.match(/node_modules/)) return next() 
        fs.lstat(file,function(err,stat){
            if(err) return next()
            if(stat.mode==41398) return next()
            if (stat.isDirectory()) {
                setTimeout(function(file){
                    walk(file,next)
                }.bind(null,file),5)
            }
            else{
                x++
                if(false || x % 1000===0) console.log((new Date().valueOf()-start)/1000,x,file)
                next()
            }
        })
      },function(){
        callback()
      })
    }
}

walkAll({
    dir:'c:/',
    delay:1000
});

The first question is how you will get persistance for your index. The index can be deduced thanks to `eachSeries`, which will work like a "loop", so you can increment a variable. — DrakaSAN, Sep 09 '16 at 11:28

DarkKnight · Answer 1 · 2016-09-11T01:27:18.617

Keep a list of sub directories to be visited, and update the list every iteration.

The walk function in the following example takes a previous state, and returns files of next sub directory with next state.

You can save the state before stopping the process, then load the saved state to continue the traversal when restarting.

function walk(state, readdir) {
  let files = [], next = [];
  while (state.length > 0) {
    try {
      const current = state.shift()
      files = readdir(current).map(file => current + '/' + file)
      next = state.concat(files)
      break
    } catch(e) {}
  }
  return [next, files]
}

function main() {
  const {writeFileSync: writeFile, readdirSync: readdir} = require('fs')
  const save = './walk.json'

  let state
  try {
    state = require(save)
  } catch(e) {}
  if (!state || state.length < 1) state = ['.']

  const [nextState, files] = walk(state, readdir)
  console.log(files)
  writeFile(save, JSON.stringify(nextState, null, 2))
}

main()

The whole problem is that I cannot resume travelse if the process restarts — Aminadav Glickshtein, Sep 10 '16 at 18:28

score 2 · Answer 2 · answered Sep 15 '16 at 19:14

an alternate idea,

var miss = require('mississippi')
var fs = require("fs")
var through2 = require("through2")
var path = require("path")

function traverseDir(dirPath) {
  var stack = [path.resolve(dirPath)];
  var filesStack = []
  return miss.from.obj(function(size, next) {

    if (filesStack.length) {
      return next(null, filesStack.shift())
    }

    var self = this;

    try {
      while(stack.length) {
        readADir(stack.pop()).forEach(function (f) {
          if (f.t=="d") {
            stack.push(f.p)
          }
          filesStack.push(f)
        })
        if (filesStack.length) {
          return next(null, filesStack.shift())
        }
      }
      return next(null, null)
    }catch(ex) {
      return next(ex)
    }
  })
}

function readADir (dir) {
  return fs.readdirSync(dir)
    .map(function (f) {return path.join(dir, f)})
    .filter(function (f) { return !f.match(/\.git/) })
    .filter(function (f) { return !f.match(/Recycle/)})
    .filter(function (f) { return !f.match(/node_modules/)})
    .map(function (p) {
      try {
        var stat = fs.lstatSync(p);
        if(stat.mode==41398) return null
        var t = stat.isDirectory() ? "d":"f"
        return { t: t, p: p }
      }catch (ex) {}
      return null
    })
    .filter(function (o) {return o!==null})
}

function loadState(base){
  base = path.resolve(base)
  var state = {base: base, last:null}
  if (fs.existsSync("state.json")) {
    state = JSON.parse(fs.readFileSync("state.json"))
  } else {
    saveState(state)
  }
  return state
}

function saveState(state){
  fs.writeFileSync("state.json", JSON.stringify(state))
}

var state = loadState("..")
var sincePath = state.last;

var filesStream = traverseDir(state.base)
.on('end', function () {
  console.log("end")
})
.pipe(through2.obj(function (chunk, enc, next) {
  if(!sincePath) this.push(chunk)
  if(chunk.p===sincePath) {
    sincePath=null
  }
  next()
}))

var tr = through2.obj(function (chunk, enc, next) {
  state.last = chunk.p
  saveState(state)
  console.log("data %v %j", chunk.t, chunk.p)
  this.push(chunk)
  setTimeout(next, 500)
}).resume()

require('keypress')(process.stdin);
process.stdin.on('keypress', function (ch, key) {
  if(!key) return
  if (key.name == "c") {
    console.log("continue")
    filesStream.pipe(tr)
  } else if (key.name=="p") {
    console.log("pause")
    filesStream.unpipe(tr)
  }
});

console.log("Press 'c' to start")

How to traverse all files, and support pause and continue

2 Answers2