moved to a file appending with grep data store

migrations/1422460849371-doc-lines.coffee

@@ -3,44 +3,33 @@ fs = require("fs")
 mongojs = require("mongojs")
 ObjectId = mongojs.ObjectId
 db = mongojs('sharelatex', ['projects', 'docs'])
-_ = require("underscore")
+_ = require("lodash")
 async = require("async")
+exec = require("child_process").exec
 
-local_db_path = "/tmp/process-db"
+finished_projects_path = "/tmp/finished-projects"
-
-Datastore = require('nedb')
-processDb = new Datastore({ filename: local_db_path})
 
 
 printProgress = ->
-	processDb.count {processed:false}, (err, todo)->
+	exec "wc #{finished_projects_path}", (error, results) ->
-		processDb.count {}, (err, total)->
+		#console.log results
-			console.log "#{todo}/#{total} processed"
+		setTimeout printProgress, 1000 * 30
-			setTimeout printProgress, 1000
 
+checkIfFileHasBeenProccessed = (project_id, callback)->
+	exec "grep #{project_id} #{finished_projects_path}", (error, results) ->
+		hasBeenProcessed = _.include(results, project_id)
+		console.log hasBeenProcessed, project_id
+		callback(null, hasBeenProcessed)
 
-writeProjectIdsToDisk = (callback)->
+getProjectIds = (callback)->
+	console.log "finding all project id's - #{new Date().toString()}"
 	db.projects.find {}, {_id:1}, (err,ids)->
-		console.log "total found projects in mongo #{ids.length}"
+		console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}"
 		ids = _.map ids, (id)-> return id._id.toString()
-		jobs = _.map ids, (id)->
+		callback(err, ids)
-			return (cb)->
-				processDb.findOne {project_id:id}, (err, doc)->
-					if doc?
-						return cb()
-					processDb.insert {project_id:id, processed:false}, cb
-		async.series jobs, (err)->
-			processDb.count {processed:false}, (err, count)->
-				console.log "projects to process: #{count}"
-				callback()
-
-getNextProjectToProccess = (callback)->
-	processDb.findOne {processed:false}, (err, doc)->
-		callback err, doc.project_id
 
 markProjectAsProcessed = (project_id, callback)->
-	processDb.update project_id:project_id, {$set:{processed:true}}, {}, callback
+	fs.appendFile finished_projects_path, "#{project_id}\n", callback
-
 
 getAllDocs = (project_id, callback = (error, docs) ->) ->
 	db.projects.findOne _id:ObjectId(project_id), (error, project) ->
@@ -88,21 +77,33 @@ saveDocsIntoMongo = (project_id, docs, callback)->
 	async.series jobs, callback
 
 
-processNext = ->
+processNext = (project_id, callback)->
-	processDb.count {processed:false}, (err, total)->
+	#console.log("starting to process #{project_id} - #{new Date().toString()}")
-		if total == 0
+	checkIfFileHasBeenProccessed project_id, (err, hasBeenProcessed)->
-			console.log "DONE"
+		if hasBeenProcessed
-			process.exit()
+			return callback()
-		else
-			getNextProjectToProccess (err, project_id)->
 		getAllDocs project_id, (err, docs)->
+			if err?
+				console.error err, project_id, "could not get all docs"
+				return callback()
 			saveDocsIntoMongo project_id, docs, ->
+				if err?
+					console.error err, project_id, "could not save docs into mongo"
+					return callback()
 				markProjectAsProcessed project_id, ->
-							processNext()
+					callback()
 
-processDb.loadDatabase ->
+getProjectIds (err, ids)->
-	writeProjectIdsToDisk ->
+	printProgress()
-		processNext()
+	jobs = _.map ids, (id)->
+		return (cb)->
+			processNext(id, cb)
+	async.series jobs, (err)->
+		if err?
+			console.error err, "at end of jobs"
+		else
+			console.log "finished"
+		process.exit()
 	
 exports.up = (next)->
 

package.json

@@ -4,8 +4,8 @@
   "description": "An online collaborative LaTeX editor",
   "dependencies": {
     "async": "^0.9.0",
+    "lodash": "^3.0.0",
     "mongojs": "^0.18.1",
-    "nedb": "^1.1.1",
     "rimraf": "~2.2.6",
     "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git",
     "underscore": "^1.7.0"