mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-07 20:31:06 -05:00
mvp migration script
This commit is contained in:
parent
6fd2e5e6d7
commit
219e9a3305
3 changed files with 136 additions and 1 deletions
|
@ -55,6 +55,8 @@ module.exports = (grunt) ->
|
||||||
grunt.loadNpmTasks 'grunt-execute'
|
grunt.loadNpmTasks 'grunt-execute'
|
||||||
grunt.loadNpmTasks 'grunt-available-tasks'
|
grunt.loadNpmTasks 'grunt-available-tasks'
|
||||||
grunt.loadNpmTasks 'grunt-concurrent'
|
grunt.loadNpmTasks 'grunt-concurrent'
|
||||||
|
grunt.loadNpmTasks "grunt-contrib-coffee"
|
||||||
|
|
||||||
|
|
||||||
execute = {}
|
execute = {}
|
||||||
for service in SERVICES
|
for service in SERVICES
|
||||||
|
@ -70,6 +72,18 @@ module.exports = (grunt) ->
|
||||||
options:
|
options:
|
||||||
limit: SERVICES.length
|
limit: SERVICES.length
|
||||||
logConcurrentOutput: true
|
logConcurrentOutput: true
|
||||||
|
coffee:
|
||||||
|
migrate:
|
||||||
|
expand: true,
|
||||||
|
flatten: false,
|
||||||
|
cwd: './',
|
||||||
|
src: ['./migrations/*.coffee'],
|
||||||
|
dest: './',
|
||||||
|
ext: '.js'
|
||||||
|
options:
|
||||||
|
bare:true
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
availabletasks:
|
availabletasks:
|
||||||
tasks:
|
tasks:
|
||||||
|
@ -147,6 +161,10 @@ module.exports = (grunt) ->
|
||||||
grunt.registerTask "build:upstart_scripts", "Create upstart scripts for each service", () ->
|
grunt.registerTask "build:upstart_scripts", "Create upstart scripts for each service", () ->
|
||||||
Helpers.buildUpstartScripts()
|
Helpers.buildUpstartScripts()
|
||||||
|
|
||||||
|
|
||||||
|
grunt.registerTask 'migrate', 'run migrations', ['coffee:migrate']
|
||||||
|
|
||||||
|
|
||||||
Helpers =
|
Helpers =
|
||||||
installService: (service, callback = (error) ->) ->
|
installService: (service, callback = (error) ->) ->
|
||||||
Helpers.cloneGitRepo service, (error) ->
|
Helpers.cloneGitRepo service, (error) ->
|
||||||
|
|
113
server-ce/migrations/1422460849371-doc-lines.coffee
Normal file
113
server-ce/migrations/1422460849371-doc-lines.coffee
Normal file
|
@ -0,0 +1,113 @@
|
||||||
|
|
||||||
|
fs = require("fs")
|
||||||
|
mongojs = require("mongojs")
|
||||||
|
ObjectId = mongojs.ObjectId
|
||||||
|
db = mongojs('sharelatex', ['projects', 'docs'])
|
||||||
|
_ = require("underscore")
|
||||||
|
async = require("async")
|
||||||
|
|
||||||
|
local_db_path = "/tmp/process-db"
|
||||||
|
|
||||||
|
Datastore = require('nedb')
|
||||||
|
processDb = new Datastore({ filename: local_db_path})
|
||||||
|
|
||||||
|
|
||||||
|
printProgress = ->
|
||||||
|
processDb.count {processed:false}, (err, todo)->
|
||||||
|
processDb.count {}, (err, total)->
|
||||||
|
console.log "#{todo}/#{total} processed"
|
||||||
|
setTimeout printProgress, 1000
|
||||||
|
|
||||||
|
|
||||||
|
writeProjectIdsToDisk = (callback)->
|
||||||
|
db.projects.find {}, {_id:1}, (err,ids)->
|
||||||
|
console.log "total found projects in mongo #{ids.length}"
|
||||||
|
ids = _.map ids, (id)-> return id._id.toString()
|
||||||
|
jobs = _.map ids, (id)->
|
||||||
|
return (cb)->
|
||||||
|
processDb.findOne {project_id:id}, (err, doc)->
|
||||||
|
if doc?
|
||||||
|
return cb()
|
||||||
|
processDb.insert {project_id:id, processed:false}, cb
|
||||||
|
async.series jobs, (err)->
|
||||||
|
processDb.count {processed:false}, (err, count)->
|
||||||
|
console.log "projects to process: #{count}"
|
||||||
|
callback()
|
||||||
|
|
||||||
|
getNextProjectToProccess = (callback)->
|
||||||
|
processDb.findOne {processed:false}, (err, doc)->
|
||||||
|
callback err, doc.project_id
|
||||||
|
|
||||||
|
markProjectAsProcessed = (project_id, callback)->
|
||||||
|
processDb.update project_id:project_id, {$set:{processed:true}}, {}, callback
|
||||||
|
|
||||||
|
|
||||||
|
getAllDocs = (project_id, callback = (error, docs) ->) ->
|
||||||
|
db.projects.findOne _id:ObjectId(project_id), (error, project) ->
|
||||||
|
return callback(error) if error?
|
||||||
|
return callback new Errors.NotFoundError("No such project: #{project_id}") if !project?
|
||||||
|
findAllDocsInProject project, (error, docs) ->
|
||||||
|
return callback(error) if error?
|
||||||
|
return callback null, docs
|
||||||
|
|
||||||
|
findAllDocsInProject = (project, callback = (error, docs) ->) ->
|
||||||
|
callback null, _findAllDocsInFolder project.rootFolder[0]
|
||||||
|
|
||||||
|
_findDocInFolder = (folder = {}, doc_id, currentPath) ->
|
||||||
|
for doc, i in folder.docs or []
|
||||||
|
if doc?._id? and doc._id.toString() == doc_id.toString()
|
||||||
|
return {
|
||||||
|
doc: doc
|
||||||
|
mongoPath: "#{currentPath}.docs.#{i}"
|
||||||
|
}
|
||||||
|
|
||||||
|
for childFolder, i in folder.folders or []
|
||||||
|
result = _findDocInFolder childFolder, doc_id, "#{currentPath}.folders.#{i}"
|
||||||
|
return result if result?
|
||||||
|
|
||||||
|
return null
|
||||||
|
|
||||||
|
_findAllDocsInFolder = (folder = {}) ->
|
||||||
|
docs = folder.docs or []
|
||||||
|
for childFolder in folder.folders or []
|
||||||
|
docs = docs.concat _findAllDocsInFolder childFolder
|
||||||
|
return docs
|
||||||
|
|
||||||
|
insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)->
|
||||||
|
update = {}
|
||||||
|
update["_id"] = ObjectId(doc_id)
|
||||||
|
update["lines"] = lines
|
||||||
|
update["project_id"] = ObjectId(project_id)
|
||||||
|
update["rev"] = oldRev
|
||||||
|
db.docs.insert _id: ObjectId(doc_id), callback
|
||||||
|
|
||||||
|
saveDocsIntoMongo = (project_id, docs, callback)->
|
||||||
|
jobs = _.map docs, (doc)->
|
||||||
|
(cb)->
|
||||||
|
insertDocIntoDocCollection project_id, doc._id, project_id.lines, doc.rev, cb
|
||||||
|
async.series jobs, callback
|
||||||
|
|
||||||
|
|
||||||
|
processNext = ->
|
||||||
|
processDb.count {processed:false}, (err, total)->
|
||||||
|
if total == 0
|
||||||
|
console.log "DONE"
|
||||||
|
process.exit()
|
||||||
|
else
|
||||||
|
getNextProjectToProccess (err, project_id)->
|
||||||
|
getAllDocs project_id, (err, docs)->
|
||||||
|
saveDocsIntoMongo project_id, docs, ->
|
||||||
|
markProjectAsProcessed project_id, ->
|
||||||
|
processNext()
|
||||||
|
|
||||||
|
processDb.loadDatabase ->
|
||||||
|
writeProjectIdsToDisk ->
|
||||||
|
processNext()
|
||||||
|
|
||||||
|
exports.up = (next)->
|
||||||
|
|
||||||
|
next()
|
||||||
|
|
||||||
|
|
||||||
|
exports.down = (next)->
|
||||||
|
next()
|
|
@ -4,8 +4,11 @@
|
||||||
"description": "An online collaborative LaTeX editor",
|
"description": "An online collaborative LaTeX editor",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"async": "^0.9.0",
|
"async": "^0.9.0",
|
||||||
|
"mongojs": "^0.18.1",
|
||||||
|
"nedb": "^1.1.1",
|
||||||
"rimraf": "~2.2.6",
|
"rimraf": "~2.2.6",
|
||||||
"settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git"
|
"settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git",
|
||||||
|
"underscore": "^1.7.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"grunt": "~0.4.2",
|
"grunt": "~0.4.2",
|
||||||
|
@ -14,6 +17,7 @@
|
||||||
"grunt-execute": "~0.1.5",
|
"grunt-execute": "~0.1.5",
|
||||||
"grunt-available-tasks": "~0.4.1",
|
"grunt-available-tasks": "~0.4.1",
|
||||||
"grunt-concurrent": "~0.4.3",
|
"grunt-concurrent": "~0.4.3",
|
||||||
|
"grunt-contrib-coffee": "~0.10.1",
|
||||||
"semver": "~2.2.1",
|
"semver": "~2.2.1",
|
||||||
"knox": "~0.8.9"
|
"knox": "~0.8.9"
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue