overleaf/server-ce/migrations/1_doc_lines.coffee

Settings = require "settings-sharelatex"
fs = require("fs")
mongojs = require("mongojs")
ObjectId = mongojs.ObjectId
db = mongojs(Settings.mongo.url, ['projects', 'docs'])
_ = require("lodash")
async = require("async")
exec = require("child_process").exec

finished_projects_path = "/tmp/finished-projects"
all_projects_path = "/tmp/all-projects"

printProgress = ->
	exec "wc #{finished_projects_path}", (error, results) ->
		#console.log results
		setTimeout printProgress, 1000 * 30

checkIfFileHasBeenProccessed = (project_id, callback)->
	exec "grep #{project_id} #{finished_projects_path}", (error, results) ->
		hasBeenProcessed = _.include(results, project_id)
		#console.log hasBeenProcessed, project_id
		callback(error, hasBeenProcessed)

loadProjectIds = (callback)->
	console.log "loading project ids from #{all_projects_path}"
	fs.readFile all_projects_path, "utf-8", (err, data)->
		ids = data.split("\n")
		console.log "loaded #{ids.length} project ids from #{all_projects_path}"
		callback err, ids

getAndWriteProjectids = (callback)->
	console.log "finding all project id's - #{new Date().toString()}"
	db.projects.find {}, {_id:1}, (err, ids)->
		console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}"
		ids = _.pluck ids, '_id'
		ids = _.filter ids, (id)-> id?
		fileData = ids.join("\n")
		fs.writeFile all_projects_path, fileData, ->
			callback(err, ids)

getProjectIds = (callback)->
	exists = fs.existsSync all_projects_path
	if exists 
		loadProjectIds callback
	else
		getAndWriteProjectids callback

markProjectAsProcessed = (project_id, callback)->
	fs.appendFile finished_projects_path, "#{project_id}\n", callback

getAllDocs = (project_id, callback = (error, docs) ->) ->
	db.projects.findOne _id:ObjectId(project_id), (error, project) ->
		return callback(error) if error?
		if !project?
			return callback("no such project #{project_id}")
		findAllDocsInProject project, (error, docs) ->
			return callback(error) if error?
			return callback null, docs

findAllDocsInProject = (project, callback = (error, docs) ->) ->
	callback null, _findAllDocsInFolder project.rootFolder[0]

_findDocInFolder = (folder = {}, doc_id, currentPath) ->
	for doc, i in folder.docs or []
		if doc?._id? and doc._id.toString() == doc_id.toString()
			return {
				doc: doc
				mongoPath: "#{currentPath}.docs.#{i}"
			}

	for childFolder, i in folder.folders or []
		result = _findDocInFolder childFolder, doc_id, "#{currentPath}.folders.#{i}"
		return result if result?

	return null

_findAllDocsInFolder = (folder = {}) ->
	docs = folder.docs or []
	for childFolder in folder.folders or []
		docs = docs.concat _findAllDocsInFolder childFolder
	return docs

insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)->
	if !project_id?
		return callback("no project id")
	if !doc_id?
		return callback("no doc id. project=#{project_id}")
	if !lines?
		return callback("no lines")
	update = {}
	update["_id"] = ObjectId(doc_id.toString())
	update["lines"] = lines
	update["project_id"] = ObjectId(project_id)
	update["rev"] = oldRev || 0
	# console.log update
	db.docs.insert update, callback

saveDocsIntoMongo = (project_id, docs, callback)->
	jobs = _.map docs, (doc)->
		(cb)->
			if !doc?
				console.error "null doc in project #{project_id}"
				return cb()
			insertDocIntoDocCollection project_id, doc._id, doc.lines, doc.rev, (err)->
				if err?.code == 11000 #duplicate key, doc already in there so its not a problem.
					err = undefined
				if err? 
					console.log "error inserting doc into doc collection", err
				cb(err)


	async.series jobs, callback


processNext = (project_id, callback)->
	checkIfFileHasBeenProccessed project_id, (err, hasBeenProcessed)->
		if hasBeenProcessed
			console.log "#{project_id} already procssed, skipping"
			return callback()
		console.log "#{project_id} processing"
		getAllDocs project_id, (err, docs)->
			if err?
				console.error err, project_id, "could not get all docs"
				return callback(err)
			else
				saveDocsIntoMongo project_id, docs, (err)->
					if err?
						console.error err, project_id, "could not save docs into mongo"
						return callback(err)
					markProjectAsProcessed project_id, (err)->
						setTimeout(
							-> callback(err)
						,100)


	
exports.migrate = (client, done)->
	getProjectIds (err, ids)->
		printProgress()
		jobs = _.map ids, (id)->
			return (cb)->
				processNext(id, cb)
		async.series jobs, (err)->
			if err?
				console.error err, "at end of jobs"
			else
				console.log "finished"
			done(err)


exports.rollback = (next)->
	next()