mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-07 20:31:06 -05:00
moved to a file appending with grep data store
This commit is contained in:
parent
93b091d84f
commit
b95f94d90e
2 changed files with 43 additions and 42 deletions
|
@ -3,44 +3,33 @@ fs = require("fs")
|
|||
mongojs = require("mongojs")
|
||||
ObjectId = mongojs.ObjectId
|
||||
db = mongojs('sharelatex', ['projects', 'docs'])
|
||||
_ = require("underscore")
|
||||
_ = require("lodash")
|
||||
async = require("async")
|
||||
exec = require("child_process").exec
|
||||
|
||||
local_db_path = "/tmp/process-db"
|
||||
|
||||
Datastore = require('nedb')
|
||||
processDb = new Datastore({ filename: local_db_path})
|
||||
finished_projects_path = "/tmp/finished-projects"
|
||||
|
||||
|
||||
printProgress = ->
|
||||
processDb.count {processed:false}, (err, todo)->
|
||||
processDb.count {}, (err, total)->
|
||||
console.log "#{todo}/#{total} processed"
|
||||
setTimeout printProgress, 1000
|
||||
exec "wc #{finished_projects_path}", (error, results) ->
|
||||
#console.log results
|
||||
setTimeout printProgress, 1000 * 30
|
||||
|
||||
checkIfFileHasBeenProccessed = (project_id, callback)->
|
||||
exec "grep #{project_id} #{finished_projects_path}", (error, results) ->
|
||||
hasBeenProcessed = _.include(results, project_id)
|
||||
console.log hasBeenProcessed, project_id
|
||||
callback(null, hasBeenProcessed)
|
||||
|
||||
writeProjectIdsToDisk = (callback)->
|
||||
getProjectIds = (callback)->
|
||||
console.log "finding all project id's - #{new Date().toString()}"
|
||||
db.projects.find {}, {_id:1}, (err,ids)->
|
||||
console.log "total found projects in mongo #{ids.length}"
|
||||
console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}"
|
||||
ids = _.map ids, (id)-> return id._id.toString()
|
||||
jobs = _.map ids, (id)->
|
||||
return (cb)->
|
||||
processDb.findOne {project_id:id}, (err, doc)->
|
||||
if doc?
|
||||
return cb()
|
||||
processDb.insert {project_id:id, processed:false}, cb
|
||||
async.series jobs, (err)->
|
||||
processDb.count {processed:false}, (err, count)->
|
||||
console.log "projects to process: #{count}"
|
||||
callback()
|
||||
|
||||
getNextProjectToProccess = (callback)->
|
||||
processDb.findOne {processed:false}, (err, doc)->
|
||||
callback err, doc.project_id
|
||||
callback(err, ids)
|
||||
|
||||
markProjectAsProcessed = (project_id, callback)->
|
||||
processDb.update project_id:project_id, {$set:{processed:true}}, {}, callback
|
||||
|
||||
fs.appendFile finished_projects_path, "#{project_id}\n", callback
|
||||
|
||||
getAllDocs = (project_id, callback = (error, docs) ->) ->
|
||||
db.projects.findOne _id:ObjectId(project_id), (error, project) ->
|
||||
|
@ -88,21 +77,33 @@ saveDocsIntoMongo = (project_id, docs, callback)->
|
|||
async.series jobs, callback
|
||||
|
||||
|
||||
processNext = ->
|
||||
processDb.count {processed:false}, (err, total)->
|
||||
if total == 0
|
||||
console.log "DONE"
|
||||
process.exit()
|
||||
else
|
||||
getNextProjectToProccess (err, project_id)->
|
||||
getAllDocs project_id, (err, docs)->
|
||||
saveDocsIntoMongo project_id, docs, ->
|
||||
markProjectAsProcessed project_id, ->
|
||||
processNext()
|
||||
processNext = (project_id, callback)->
|
||||
#console.log("starting to process #{project_id} - #{new Date().toString()}")
|
||||
checkIfFileHasBeenProccessed project_id, (err, hasBeenProcessed)->
|
||||
if hasBeenProcessed
|
||||
return callback()
|
||||
getAllDocs project_id, (err, docs)->
|
||||
if err?
|
||||
console.error err, project_id, "could not get all docs"
|
||||
return callback()
|
||||
saveDocsIntoMongo project_id, docs, ->
|
||||
if err?
|
||||
console.error err, project_id, "could not save docs into mongo"
|
||||
return callback()
|
||||
markProjectAsProcessed project_id, ->
|
||||
callback()
|
||||
|
||||
processDb.loadDatabase ->
|
||||
writeProjectIdsToDisk ->
|
||||
processNext()
|
||||
getProjectIds (err, ids)->
|
||||
printProgress()
|
||||
jobs = _.map ids, (id)->
|
||||
return (cb)->
|
||||
processNext(id, cb)
|
||||
async.series jobs, (err)->
|
||||
if err?
|
||||
console.error err, "at end of jobs"
|
||||
else
|
||||
console.log "finished"
|
||||
process.exit()
|
||||
|
||||
exports.up = (next)->
|
||||
|
||||
|
|
|
@ -4,8 +4,8 @@
|
|||
"description": "An online collaborative LaTeX editor",
|
||||
"dependencies": {
|
||||
"async": "^0.9.0",
|
||||
"lodash": "^3.0.0",
|
||||
"mongojs": "^0.18.1",
|
||||
"nedb": "^1.1.1",
|
||||
"rimraf": "~2.2.6",
|
||||
"settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git",
|
||||
"underscore": "^1.7.0"
|
||||
|
|
Loading…
Reference in a new issue