From bfe4dd018be4a66dccf3313b2669c50badf06796 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 10 Feb 2015 17:25:13 +0000 Subject: [PATCH] added migrations with east package --- Gruntfile.coffee | 9 +- ...71-doc-lines.coffee => 1_doc_lines.coffee} | 85 ++++++++++++------- migrations/about_migrations.md | 2 + package.json | 3 + 4 files changed, 64 insertions(+), 35 deletions(-) rename migrations/{1422460849371-doc-lines.coffee => 1_doc_lines.coffee} (66%) create mode 100644 migrations/about_migrations.md diff --git a/Gruntfile.coffee b/Gruntfile.coffee index 3b3339e417..d8e5f5ee3c 100644 --- a/Gruntfile.coffee +++ b/Gruntfile.coffee @@ -7,6 +7,8 @@ semver = require "semver" knox = require "knox" crypto = require "crypto" async = require "async" +settings = require("settings-sharelatex") + SERVICES = [{ name: "web" @@ -56,6 +58,7 @@ module.exports = (grunt) -> grunt.loadNpmTasks 'grunt-available-tasks' grunt.loadNpmTasks 'grunt-concurrent' grunt.loadNpmTasks "grunt-contrib-coffee" + grunt.loadNpmTasks "grunt-shell" execute = {} @@ -83,7 +86,9 @@ module.exports = (grunt) -> options: bare:true - + shell: + migrate: + command: "./node_modules/east/bin/east migrate --adapter east-mongo --url #{settings.mongo.url}" availabletasks: tasks: @@ -162,7 +167,7 @@ module.exports = (grunt) -> Helpers.buildUpstartScripts() - grunt.registerTask 'migrate', 'run migrations', ['coffee:migrate'] + grunt.registerTask 'migrate', "compile migrations and run them", ['coffee:migrate', 'shell:migrate'] Helpers = diff --git a/migrations/1422460849371-doc-lines.coffee b/migrations/1_doc_lines.coffee similarity index 66% rename from migrations/1422460849371-doc-lines.coffee rename to migrations/1_doc_lines.coffee index a97d2907ac..9e0f8ae7f6 100644 --- a/migrations/1422460849371-doc-lines.coffee +++ b/migrations/1_doc_lines.coffee @@ -1,8 +1,8 @@ - +Settings = require "settings-sharelatex" fs = require("fs") mongojs = require("mongojs") ObjectId = mongojs.ObjectId -db = mongojs('sharelatex', ['projects', 'docs']) +db = mongojs(Settings.mongo.url, ['projects', 'docs']) _ = require("lodash") async = require("async") exec = require("child_process").exec @@ -19,13 +19,13 @@ checkIfFileHasBeenProccessed = (project_id, callback)-> exec "grep #{project_id} #{finished_projects_path}", (error, results) -> hasBeenProcessed = _.include(results, project_id) #console.log hasBeenProcessed, project_id - callback(null, hasBeenProcessed) + callback(error, hasBeenProcessed) loadProjectIds = (callback)-> + console.log "loading project ids from #{all_projects_path}" fs.readFile all_projects_path, "utf-8", (err, data)-> - console.log data.length ids = data.split("\n") - console.log ids.length + console.log "loaded #{ids.length} project ids from #{all_projects_path}" callback err, ids getAndWriteProjectids = (callback)-> @@ -52,7 +52,6 @@ getAllDocs = (project_id, callback = (error, docs) ->) -> db.projects.findOne _id:ObjectId(project_id), (error, project) -> return callback(error) if error? if !project? - console.error("No such project: #{project_id}") return callback("no such project #{project_id}") findAllDocsInProject project, (error, docs) -> return callback(error) if error? @@ -82,17 +81,34 @@ _findAllDocsInFolder = (folder = {}) -> return docs insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)-> + if !project_id? + return callback("no project id") + if !doc_id? + return callback("no doc id. project=#{project_id}") + if !lines? + return callback("no lines") update = {} - update["_id"] = ObjectId(doc_id) + update["_id"] = ObjectId(doc_id.toString()) update["lines"] = lines update["project_id"] = ObjectId(project_id) - update["rev"] = oldRev - db.docs.insert _id: ObjectId(doc_id), callback + update["rev"] = oldRev || 0 + # console.log update + db.docs.insert update, callback saveDocsIntoMongo = (project_id, docs, callback)-> jobs = _.map docs, (doc)-> (cb)-> - insertDocIntoDocCollection project_id, doc._id, project_id.lines, doc.rev, cb + if !doc? + console.error "null doc in project #{project_id}" + return cb() + insertDocIntoDocCollection project_id, doc._id, doc.lines, doc.rev, (err)-> + if err?.code == 11000 #duplicate key, doc already in there so its not a problem. + err = undefined + if err? + console.log "error inserting doc into doc collection", err + cb(err) + + async.series jobs, callback @@ -101,33 +117,36 @@ processNext = (project_id, callback)-> if hasBeenProcessed console.log "#{project_id} already procssed, skipping" return callback() + console.log "#{project_id} processing" getAllDocs project_id, (err, docs)-> if err? console.error err, project_id, "could not get all docs" - return callback() - saveDocsIntoMongo project_id, docs, -> - if err? - console.error err, project_id, "could not save docs into mongo" - return callback() - markProjectAsProcessed project_id, -> - callback() + return callback(err) + else + saveDocsIntoMongo project_id, docs, (err)-> + if err? + console.error err, project_id, "could not save docs into mongo" + return callback(err) + markProjectAsProcessed project_id, (err)-> + setTimeout( + -> callback(err) + ,100) + -getProjectIds (err, ids)-> - printProgress() - jobs = _.map ids, (id)-> - return (cb)-> - processNext(id, cb) - async.series jobs, (err)-> - if err? - console.error err, "at end of jobs" - else - console.log "finished" - process.exit() -exports.up = (next)-> +exports.migrate = (client, done)-> + getProjectIds (err, ids)-> + printProgress() + jobs = _.map ids, (id)-> + return (cb)-> + processNext(id, cb) + async.series jobs, (err)-> + if err? + console.error err, "at end of jobs" + else + console.log "finished" + done(err) - next() - - -exports.down = (next)-> + +exports.rollback = (next)-> next() diff --git a/migrations/about_migrations.md b/migrations/about_migrations.md new file mode 100644 index 0000000000..146cb9c047 --- /dev/null +++ b/migrations/about_migrations.md @@ -0,0 +1,2 @@ +* if migration is stopped mid way it will start at the beginging next time +* to see the run migrations do db.getCollection('_migrations').find() you can't do db._migrations.find() \ No newline at end of file diff --git a/package.json b/package.json index a50146f3ef..cee7ffc776 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,9 @@ "description": "An online collaborative LaTeX editor", "dependencies": { "async": "^0.9.0", + "east": "^0.2.3", + "east-mongo": "^0.1.2", + "grunt-shell": "^1.1.1", "lodash": "^3.0.0", "mongojs": "^0.18.1", "rimraf": "~2.2.6",