added migrations with east package

server-ce/Gruntfile.coffee

@@ -7,6 +7,8 @@ semver = require "semver"
 knox = require "knox"
 crypto = require "crypto"
 async = require "async"
+settings = require("settings-sharelatex")
+
 
 SERVICES = [{
 	name: "web"
@@ -56,6 +58,7 @@ module.exports = (grunt) ->
 	grunt.loadNpmTasks 'grunt-available-tasks'
 	grunt.loadNpmTasks 'grunt-concurrent'
 	grunt.loadNpmTasks "grunt-contrib-coffee"
+	grunt.loadNpmTasks "grunt-shell"
 
 
 	execute = {}
@@ -83,7 +86,9 @@ module.exports = (grunt) ->
 				options:
 					bare:true
 
-
+		shell:
+			migrate:
+				command: "./node_modules/east/bin/east migrate --adapter east-mongo --url #{settings.mongo.url}"
 
 		availabletasks:
 			tasks:
@@ -162,7 +167,7 @@ module.exports = (grunt) ->
 		Helpers.buildUpstartScripts()
 
 
-	grunt.registerTask 'migrate', 'run migrations', ['coffee:migrate']
+	grunt.registerTask 'migrate', "compile migrations and run them", ['coffee:migrate', 'shell:migrate']
 
 
 	Helpers =

server-ce/migrations/1_doc_lines.coffee

@@ -1,8 +1,8 @@
-
+Settings = require "settings-sharelatex"
 fs = require("fs")
 mongojs = require("mongojs")
 ObjectId = mongojs.ObjectId
-db = mongojs('sharelatex', ['projects', 'docs'])
+db = mongojs(Settings.mongo.url, ['projects', 'docs'])
 _ = require("lodash")
 async = require("async")
 exec = require("child_process").exec
@@ -19,13 +19,13 @@ checkIfFileHasBeenProccessed = (project_id, callback)->
 	exec "grep #{project_id} #{finished_projects_path}", (error, results) ->
 		hasBeenProcessed = _.include(results, project_id)
 		#console.log hasBeenProcessed, project_id
-		callback(null, hasBeenProcessed)
+		callback(error, hasBeenProcessed)
 
 loadProjectIds = (callback)->
+	console.log "loading project ids from #{all_projects_path}"
 	fs.readFile all_projects_path, "utf-8", (err, data)->
-		console.log data.length
 		ids = data.split("\n")
-		console.log ids.length
+		console.log "loaded #{ids.length} project ids from #{all_projects_path}"
 		callback err, ids
 
 getAndWriteProjectids = (callback)->
@@ -52,7 +52,6 @@ getAllDocs = (project_id, callback = (error, docs) ->) ->
 	db.projects.findOne _id:ObjectId(project_id), (error, project) ->
 		return callback(error) if error?
 		if !project?
-			console.error("No such project: #{project_id}")
 			return callback("no such project #{project_id}")
 		findAllDocsInProject project, (error, docs) ->
 			return callback(error) if error?
@@ -82,17 +81,34 @@ _findAllDocsInFolder = (folder = {}) ->
 	return docs
 
 insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)->
+	if !project_id?
+		return callback("no project id")
+	if !doc_id?
+		return callback("no doc id. project=#{project_id}")
+	if !lines?
+		return callback("no lines")
 	update = {}
-	update["_id"] = ObjectId(doc_id)
+	update["_id"] = ObjectId(doc_id.toString())
 	update["lines"] = lines
 	update["project_id"] = ObjectId(project_id)
-	update["rev"] = oldRev
-	db.docs.insert _id: ObjectId(doc_id), callback
+	update["rev"] = oldRev || 0
+	# console.log update
+	db.docs.insert update, callback
 
 saveDocsIntoMongo = (project_id, docs, callback)->
 	jobs = _.map docs, (doc)->
 		(cb)->
-			insertDocIntoDocCollection project_id, doc._id, project_id.lines, doc.rev, cb
+			if !doc?
+				console.error "null doc in project #{project_id}"
+				return cb()
+			insertDocIntoDocCollection project_id, doc._id, doc.lines, doc.rev, (err)->
+				if err?.code == 11000 #duplicate key, doc already in there so its not a problem.
+					err = undefined
+				if err? 
+					console.log "error inserting doc into doc collection", err
+				cb(err)
+
+
 	async.series jobs, callback
 
 
@@ -101,33 +117,36 @@ processNext = (project_id, callback)->
 		if hasBeenProcessed
 			console.log "#{project_id} already procssed, skipping"
 			return callback()
+		console.log "#{project_id} processing"
 		getAllDocs project_id, (err, docs)->
 			if err?
 				console.error err, project_id, "could not get all docs"
-				return callback()
-			saveDocsIntoMongo project_id, docs, ->
-				if err?
-					console.error err, project_id, "could not save docs into mongo"
-					return callback()
-				markProjectAsProcessed project_id, ->
-					callback()
+				return callback(err)
+			else
+				saveDocsIntoMongo project_id, docs, (err)->
+					if err?
+						console.error err, project_id, "could not save docs into mongo"
+						return callback(err)
+					markProjectAsProcessed project_id, (err)->
+						setTimeout(
+							-> callback(err)
+						,100)
+
 
-getProjectIds (err, ids)->
-	printProgress()
-	jobs = _.map ids, (id)->
-		return (cb)->
-			processNext(id, cb)
-	async.series jobs, (err)->
-		if err?
-			console.error err, "at end of jobs"
-		else
-			console.log "finished"
-		process.exit()
 	
-exports.up = (next)->
+exports.migrate = (client, done)->
+	getProjectIds (err, ids)->
+		printProgress()
+		jobs = _.map ids, (id)->
+			return (cb)->
+				processNext(id, cb)
+		async.series jobs, (err)->
+			if err?
+				console.error err, "at end of jobs"
+			else
+				console.log "finished"
+			done(err)
 
-	next()
-
-
-exports.down = (next)->
+
+exports.rollback = (next)->
 	next()

server-ce/migrations/about_migrations.md

@@ -0,0 +1,2 @@
+* if migration is stopped mid way it will start at the beginging next time
+* to see the run migrations do db.getCollection('_migrations').find() you can't do db._migrations.find()

server-ce/package.json

@@ -4,6 +4,9 @@
   "description": "An online collaborative LaTeX editor",
   "dependencies": {
     "async": "^0.9.0",
+    "east": "^0.2.3",
+    "east-mongo": "^0.1.2",
+    "grunt-shell": "^1.1.1",
     "lodash": "^3.0.0",
     "mongojs": "^0.18.1",
     "rimraf": "~2.2.6",