From c8dcf853bc3d2e6f6e17d315a92dc6af73e4683c Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 8 Jul 2021 10:08:34 +0100 Subject: [PATCH] Remove migrations These will be replaced by migrations in the web project --- server-ce/Dockerfile | 1 - server-ce/Gruntfile.coffee | 16 - server-ce/init_scripts/99_migrate.sh | 8 - .../migrations/10_update_project_tokens.js | 109 ------ .../migrations/11_set_project_image_name.js | 23 -- .../migrations/12_add_deletedFiles_indexes.js | 24 -- .../migrations/13_add_deleted_docs_index.js | 28 -- .../1_move_doc_lines_to_doc_collection.coffee | 164 --------- .../2_doc_lines_delete_from_project.coffee | 185 ---------- .../3_pack_docHistory_collection.coffee | 342 ------------------ .../migrations/4_update_user_features.coffee | 38 -- .../5_remove_holding_accounts.coffee | 103 ------ .../6_add_track_changes_feature.coffee | 30 -- .../migrations/7_add_token_indexes.coffee | 51 --- .../7_add_track_changes_feature_again.coffee | 31 -- .../migrations/9_create_user_emails_array.js | 49 --- server-ce/migrations/about_migrations.md | 9 - 17 files changed, 1211 deletions(-) delete mode 100755 server-ce/init_scripts/99_migrate.sh delete mode 100644 server-ce/migrations/10_update_project_tokens.js delete mode 100644 server-ce/migrations/11_set_project_image_name.js delete mode 100644 server-ce/migrations/12_add_deletedFiles_indexes.js delete mode 100644 server-ce/migrations/13_add_deleted_docs_index.js delete mode 100644 server-ce/migrations/1_move_doc_lines_to_doc_collection.coffee delete mode 100644 server-ce/migrations/2_doc_lines_delete_from_project.coffee delete mode 100644 server-ce/migrations/3_pack_docHistory_collection.coffee delete mode 100644 server-ce/migrations/4_update_user_features.coffee delete mode 100644 server-ce/migrations/5_remove_holding_accounts.coffee delete mode 100644 server-ce/migrations/6_add_track_changes_feature.coffee delete mode 100644 server-ce/migrations/7_add_token_indexes.coffee delete mode 100644 server-ce/migrations/7_add_track_changes_feature_again.coffee delete mode 100644 server-ce/migrations/9_create_user_emails_array.js delete mode 100644 server-ce/migrations/about_migrations.md diff --git a/server-ce/Dockerfile b/server-ce/Dockerfile index f039da994e..ace431967f 100644 --- a/server-ce/Dockerfile +++ b/server-ce/Dockerfile @@ -12,7 +12,6 @@ ENV SHARELATEX_CONFIG /etc/sharelatex/settings.coffee # ------------------------- ADD ${baseDir}/bin /var/www/sharelatex/bin ADD ${baseDir}/doc /var/www/sharelatex/doc -ADD ${baseDir}/migrations /var/www/sharelatex/migrations ADD ${baseDir}/tasks /var/www/sharelatex/tasks ADD ${baseDir}/Gruntfile.coffee /var/www/sharelatex/Gruntfile.coffee ADD ${baseDir}/package.json /var/www/sharelatex/package.json diff --git a/server-ce/Gruntfile.coffee b/server-ce/Gruntfile.coffee index 3af76ef763..9375096d62 100644 --- a/server-ce/Gruntfile.coffee +++ b/server-ce/Gruntfile.coffee @@ -38,20 +38,6 @@ module.exports = (grunt) -> options: limit: SERVICES.length logConcurrentOutput: true - coffee: - migrate: - expand: true, - flatten: false, - cwd: './', - src: ['./migrations/*.coffee'], - dest: './', - ext: '.js' - options: - bare:true - - shell: - migrate: - command: "./node_modules/east/bin/east migrate --adapter east-mongo --url #{settings?.mongo?.url}" availabletasks: tasks: @@ -115,8 +101,6 @@ module.exports = (grunt) -> grunt.registerTask "check:make", "Check that make is installed", () -> Helpers.checkMake @async() - grunt.registerTask 'migrate', "compile migrations and run them", ["coffee:migrate", 'shell:migrate'] - Helpers = installService: (service, callback = (error) ->) -> diff --git a/server-ce/init_scripts/99_migrate.sh b/server-ce/init_scripts/99_migrate.sh deleted file mode 100755 index f880fa816f..0000000000 --- a/server-ce/init_scripts/99_migrate.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -set -e - -which node -which grunt -ls -al /var/www/sharelatex/migrations -cd /var/www/sharelatex && grunt migrate -v -echo "All migrations finished" diff --git a/server-ce/migrations/10_update_project_tokens.js b/server-ce/migrations/10_update_project_tokens.js deleted file mode 100644 index 57095308f8..0000000000 --- a/server-ce/migrations/10_update_project_tokens.js +++ /dev/null @@ -1,109 +0,0 @@ -const Settings = require('settings-sharelatex') -const Async = require('async') -const mongojs = require('mongojs') -const db = mongojs(Settings.mongo.url, ['users']) - -const indexKeys = { 'tokens.readAndWritePrefix': 1 } -const indexOpts = { - unique: true, - partialFilterExpression: { - 'tokens.readAndWritePrefix': { $exists: true } - }, - background: true -} - -// Index on Prefix -const addReadAndWritePrefixIndex = (db, callback) => { - db.projects.ensureIndex(indexKeys, indexOpts, callback) -} - -const removeReadAndWritePrefixIndex = (db, callback) => { - db.projects.dropIndex(indexKeys, callback) -} - -// Extract prefix data -const extractPrefix = (db, callback) => { - db.projects.find( - { - 'tokens.readAndWrite': { $exists: true }, - 'tokens.readAndWritePrefix': { $exists: false } - }, - { tokens: 1 }, - (err, projects) => { - if (err) { - return callback(err) - } - console.log(`>> Updating ${projects.length} projects`) - Async.eachLimit( - projects, - 5, - (project, cb) => { - const rwToken = project.tokens.readAndWrite - const prefixMatch = rwToken.match(/^(\d+).*$/) - if (!prefixMatch) { - const err = new Error( - `no prefix on token: ${project._id}, ${rwToken}` - ) - console.log(`>> Error, ${err.message}`) - return cb(err) - } - db.projects.update( - { _id: project._id }, - { $set: { 'tokens.readAndWritePrefix': prefixMatch[1] } }, - cb - ) - }, - err => { - if (err) { - return callback(err) - } - console.log('>> done') - callback() - } - ) - } - ) -} - -const erasePrefix = (db, callback) => { - db.projects.update({$unset: 'tokens.readAndWritePrefix'}, callback) -} - - -// Migrations - -exports.migrate = (client, done) => { - console.log(`>> Adding index to projects: ${JSON.stringify(indexKeys)}, with options: ${JSON.stringify(indexOpts)}`) - addReadAndWritePrefixIndex(db, (err) => { - if(err) { - console.log(">> Error while adding index") - return done(err) - } - console.log(">> Extracting tokens.readAndWritePrefix field for existing projects") - extractPrefix(db, (err) => { - if(err) { - console.log(">> Error while extracting prefix data") - return done(err) - } - done() - }) - }) -} - -exports.rollback = (client, done) => { - console.log(`>> Dropping index on projects: ${JSON.stringify(indexKeys)}`) - removeReadAndWritePrefixIndex(db, (err) => { - if(err) { - console.log(">> Error while dropping index") - return done(err) - } - console.log(">> Erasing tokens.readAndWritePrefix field for existing projects") - erasePrefix(db, (err) => { - if(err) { - console.log(">> Error while erasing prefix data") - return done(err) - } - done() - }) - }) -} diff --git a/server-ce/migrations/11_set_project_image_name.js b/server-ce/migrations/11_set_project_image_name.js deleted file mode 100644 index 17c06fd6fc..0000000000 --- a/server-ce/migrations/11_set_project_image_name.js +++ /dev/null @@ -1,23 +0,0 @@ -const Settings = require('settings-sharelatex') -const mongojs = require('mongojs') -const db = mongojs(Settings.mongo.url, ['projects']) - -exports.migrate = (client, done) => { - console.log(`>> Setting 'imageName' in projects`) - - if (!Settings.currentImageName) { - console.log(`>> 'currentImageName' is not defined, no projects updated`) - return done() - } - - console.log(`>> Setting 'imageName' = ${Settings.currentImageName}`) - - db.projects.update( - { imageName: { $exists: false } }, - { $set: { imageName: Settings.currentImageName } }, - { multi: true }, - done - ) -} - -exports.rollback = (client, done) => done() diff --git a/server-ce/migrations/12_add_deletedFiles_indexes.js b/server-ce/migrations/12_add_deletedFiles_indexes.js deleted file mode 100644 index cb6db59305..0000000000 --- a/server-ce/migrations/12_add_deletedFiles_indexes.js +++ /dev/null @@ -1,24 +0,0 @@ -// Internal ticket: https://github.com/overleaf/issues/issues/4094 - -const Settings = require('settings-sharelatex') -const mongojs = require('mongojs') -const db = mongojs(Settings.mongo.url, ['deletedFiles']) - -const INDEX_NAME = 'projectId_1' -const INDEX_KEYS = { projectId: 1 } -const INDEX_OPTIONS = { - name: INDEX_NAME, - background: 1 -} - -exports.migrate = (client, done) => { - db.deletedFiles.ensureIndex( - INDEX_KEYS, - INDEX_OPTIONS, - done - ) -} - -exports.rollback = (client, done) => { - db.deletedFiles.dropIndex(INDEX_NAME, done) -} diff --git a/server-ce/migrations/13_add_deleted_docs_index.js b/server-ce/migrations/13_add_deleted_docs_index.js deleted file mode 100644 index be23e5ee3b..0000000000 --- a/server-ce/migrations/13_add_deleted_docs_index.js +++ /dev/null @@ -1,28 +0,0 @@ -// Internal ticket: https://github.com/overleaf/issues/issues/4211 - -const Settings = require('settings-sharelatex') -const mongojs = require('mongojs') -const db = mongojs(Settings.mongo.url, ['docs']) - -const INDEX_NAME = 'project_id_deleted_deletedAt_1' -const INDEX_KEYS = { - project_id: 1, - deleted: 1, - deletedAt: -1 -} -const INDEX_OPTIONS = { - name: INDEX_NAME, - background: 1 -} - -exports.migrate = (client, done) => { - db.docs.ensureIndex( - INDEX_KEYS, - INDEX_OPTIONS, - done - ) -} - -exports.rollback = (client, done) => { - db.docs.dropIndex(INDEX_NAME, done) -} diff --git a/server-ce/migrations/1_move_doc_lines_to_doc_collection.coffee b/server-ce/migrations/1_move_doc_lines_to_doc_collection.coffee deleted file mode 100644 index ce8c208657..0000000000 --- a/server-ce/migrations/1_move_doc_lines_to_doc_collection.coffee +++ /dev/null @@ -1,164 +0,0 @@ -Settings = require "settings-sharelatex" -bson = require('bson') -BSON = new bson() -fs = require("fs") -mongojs = require("mongojs") -ObjectId = mongojs.ObjectId -console.log Settings.mongo.url -db = mongojs(Settings.mongo.url, ['projects', 'docs']) -_ = require("lodash") -async = require("async") -exec = require("child_process").exec - -finished_projects_path = "/tmp/finished-projects" -all_projects_path = "/tmp/all-projects" -project_too_large_path = "/tmp/large_projects" - - -printProgress = -> - exec "wc #{finished_projects_path}", (error, results) -> - setTimeout printProgress, 1000 * 30 - -checkIfFileHasBeenProcessed = (project_id, callback)-> - exec "grep #{project_id} #{finished_projects_path}", (error, results) -> - hasBeenProcessed = _.include(results, project_id) - callback(error, hasBeenProcessed) - -loadProjectIds = (callback)-> - console.log "loading project ids from #{all_projects_path}" - fs.readFile all_projects_path, "utf-8", (err, data)-> - ids = data.split("\n") - ids = _.filter ids, (id)-> id? and id.length == 24 - console.log "loaded #{ids.length} project ids from #{all_projects_path}" - callback err, ids - -getAndWriteProjectids = (callback)-> - console.log "finding all project id's - #{new Date().toString()}" - db.projects.find {}, {_id:1}, (err, ids)-> - console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}" - ids = _.pluck ids, '_id' - ids = _.filter ids, (id)-> id? - fileData = ids.join("\n") - fs.writeFile all_projects_path, fileData, -> - callback(err, ids) - -markProjectAsToLargeAndFinished = (project_id, callback)-> - console.log "#{project_id} too large" - markProjectAsProcessed project_id, (err)-> - fs.appendFile project_too_large_path, "#{project_id}\n", callback - -getProjectIds = (callback)-> - exists = fs.existsSync all_projects_path - if exists - loadProjectIds callback - else - getAndWriteProjectids callback - -markProjectAsProcessed = (project_id, callback)-> - fs.appendFile finished_projects_path, "#{project_id}\n", callback - -getAllDocs = (project_id, callback = (error, docs) ->) -> - db.projects.findOne _id:ObjectId(project_id), (error, project) -> - return callback(error) if error? - if !project? - console.log "no such project #{project_id}" - return callback() - size = BSON.calculateObjectSize(project) - if size > 12000000 #12mb - return markProjectAsToLargeAndFinished project_id, callback - findAllDocsInProject project, (error, docs) -> - return callback(error) if error? - return callback null, docs - -findAllDocsInProject = (project, callback = (error, docs) ->) -> - callback null, _findAllDocsInFolder project.rootFolder[0] - -_findDocInFolder = (folder = {}, doc_id, currentPath) -> - for doc, i in folder.docs or [] - if doc?._id? and doc._id.toString() == doc_id.toString() - return { - doc: doc - mongoPath: "#{currentPath}.docs.#{i}" - } - - for childFolder, i in folder.folders or [] - result = _findDocInFolder childFolder, doc_id, "#{currentPath}.folders.#{i}" - return result if result? - - return null - -_findAllDocsInFolder = (folder = {}) -> - docs = folder.docs or [] - for childFolder in folder.folders or [] - docs = docs.concat _findAllDocsInFolder childFolder - return docs - -insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)-> - if !project_id? - return callback("no project id") - if !doc_id? - return callback() - if !lines? - lines = [""] - update = {} - update["_id"] = ObjectId(doc_id.toString()) - update["lines"] = lines - update["project_id"] = ObjectId(project_id) - update["rev"] = oldRev || 0 - db.docs.insert update, callback - -saveDocsIntoMongo = (project_id, docs, callback)-> - jobs = _.map docs, (doc)-> - (cb)-> - if !doc? - console.error "null doc in project #{project_id}" #just skip it, not a big deal - return cb() - insertDocIntoDocCollection project_id, doc._id, doc.lines, doc.rev, (err)-> - if err?.code == 11000 #duplicate key, doc already in there so its not a problem. - err = undefined - if err? - console.log "error inserting doc into doc collection", err - cb(err) - - - async.series jobs, callback - - -processNext = (project_id, callback)-> - checkIfFileHasBeenProcessed project_id, (err, hasBeenProcessed)-> - if hasBeenProcessed - console.log "#{project_id} already processed, skipping" - return callback() - console.log "#{project_id} processing" - getAllDocs project_id, (err, docs)-> - if err? - console.error err, project_id, "could not get all docs" - return callback(err) - else - saveDocsIntoMongo project_id, docs, (err)-> - if err? - console.error err, project_id, "could not save docs into mongo" - return callback(err) - markProjectAsProcessed project_id, (err)-> - setTimeout( - -> callback(err) - ,0) - - - -exports.migrate = (client, done = ->)-> - getProjectIds (err, ids)-> - printProgress() - jobs = _.map ids, (id)-> - return (cb)-> - processNext(id, cb) - async.series jobs, (err)-> - if err? - console.error err, "at end of jobs" - else - console.log "finished" - done(err) - - -exports.rollback = (next)-> - next() diff --git a/server-ce/migrations/2_doc_lines_delete_from_project.coffee b/server-ce/migrations/2_doc_lines_delete_from_project.coffee deleted file mode 100644 index 0be28446d8..0000000000 --- a/server-ce/migrations/2_doc_lines_delete_from_project.coffee +++ /dev/null @@ -1,185 +0,0 @@ -Settings = require "settings-sharelatex" -fs = require("fs") -mongojs = require("mongojs") -ObjectId = mongojs.ObjectId -db = mongojs(Settings.mongo.url, ['projects', 'docs']) -_ = require("lodash") -async = require("async") -exec = require("child_process").exec - -finished_projects_path = "/tmp/finished-projects-2" -all_projects_path = "/tmp/all-projects-2" -unmigrated_docs_path = "/tmp/unmigrated-2" - - -printProgress = -> - exec "wc #{finished_projects_path}", (error, results) -> - setTimeout printProgress, 1000 * 30 - -checkIfFileHasBeenProcessed = (project_id, callback)-> - exec "grep #{project_id} #{finished_projects_path}", (error, results) -> - hasBeenProcessed = _.include(results, project_id) - callback(error, hasBeenProcessed) - -loadProjectIds = (callback)-> - console.log "loading project ids from #{all_projects_path}" - fs.readFile all_projects_path, "utf-8", (err, data)-> - ids = data.split("\n") - ids = _.filter ids, (id)-> id? and id.length == 24 - console.log "loaded #{ids.length} project ids from #{all_projects_path}" - callback err, ids - -getAndWriteProjectids = (callback)-> - console.log "finding all project id's - #{new Date().toString()}" - db.projects.find {}, {_id:1}, (err, ids)-> - console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}" - ids = _.pluck ids, '_id' - ids = _.filter ids, (id)-> id? - fileData = ids.join("\n") - fs.writeFile all_projects_path, fileData, -> - callback(err, ids) - -markDocAsUnmigrated = (project_id, doc_id, callback)-> - console.log "#{project_id} #{doc_id} unmigrated" - markProjectAsProcessed project_id, (err)-> - fs.appendFile unmigrated_docs_path, "#{project_id} #{doc_id}\n", callback - -markUnmigratedDocs = (project_id, docs, callback)-> - console.log docs.length, project_id, "unmigrated" - jobs = _.map docs, (doc)-> - (cb)-> - markDocAsUnmigrated project_id, doc._id, cb - async.series jobs, callback - -getProjectIds = (callback)-> - exists = fs.existsSync all_projects_path - if exists - loadProjectIds callback - else - getAndWriteProjectids callback - -markProjectAsProcessed = (project_id, callback)-> - fs.appendFile finished_projects_path, "#{project_id}\n", callback - -getAllDocs = (project_id, callback = (error, docs) ->) -> - excludes = {} - for i in [0..12] - excludes["rootFolder#{Array(i).join(".folders")}.docs.lines"] = 0 - db.projects.findOne _id: ObjectId(project_id.toString()), excludes, (error, project) -> - return callback(error) if error? - if !project? - console.log "no such project #{project_id}" - return callback() - findAllDocsInProject project, (error, docs) -> - return callback(error) if error? - return callback null, docs, project - -findAllDocsInProject = (project, callback = (error, docs) ->) -> - callback null, _findAllDocsInFolder project.rootFolder[0] - -findDocInProject = (project, doc_id, callback = (error, doc, mongoPath) ->) -> - result = _findDocInFolder project.rootFolder[0], doc_id, "rootFolder.0" - if result? - callback null, result.doc, result.mongoPath - else - callback null, null, null - -_findDocInFolder = (folder = {}, doc_id, currentPath) -> - for doc, i in folder.docs or [] - if doc?._id? and doc._id.toString() == doc_id.toString() - return { - doc: doc - mongoPath: "#{currentPath}.docs.#{i}" - } - for childFolder, i in folder.folders or [] - result = _findDocInFolder childFolder, doc_id, "#{currentPath}.folders.#{i}" - return result if result? - - return null - -_findAllDocsInFolder = (folder = {}) -> - docs = folder.docs or [] - for childFolder in folder.folders or [] - docs = docs.concat _findAllDocsInFolder childFolder - return docs - -isDocInDocCollection = (doc, callback)-> - if !doc?._id? or doc._id.length == 0 - return callback(null, true) - db.docs.find({_id: ObjectId(doc._id+"")}, {_id: 1}).limit 1, (err, foundDocs)-> - exists = foundDocs.length > 0 - callback err, exists - -getWhichDocsCanBeDeleted = (docs, callback = (err, docsToBeDeleted, unmigratedDocs)->)-> - docsToBeDeleted = [] - unmigratedDocs = [] - - jobs = _.map docs, (doc)-> - return (cb)-> - isDocInDocCollection doc, (err, exists)-> - if exists - docsToBeDeleted.push doc - else - unmigratedDocs.push doc - cb(err) - async.series jobs, (err)-> - callback err, docsToBeDeleted, unmigratedDocs - -wipeDocLines = (project_id, mongoPath, callback)-> - update = - $unset: {} - update.$unset["#{mongoPath}.lines"] = "" - update.$unset["#{mongoPath}.rev"] = "" - db.projects.update _id: ObjectId(project_id+''), update, callback - - -removeDocLinesFromProject = (docs, project, callback)-> - jobs = _.map docs, (doc)-> - (cb)-> - findDocInProject project, doc._id, (err, doc, mongoPath)-> - wipeDocLines project._id, mongoPath, cb - async.parallelLimit jobs, 5, callback - -processNext = (project_id, callback)-> - if !project_id? or project_id.length == 0 - return callback() - checkIfFileHasBeenProcessed project_id, (err, hasBeenProcessed)-> - if hasBeenProcessed - console.log "#{project_id} already processed, skipping" - return callback() - console.log "#{project_id} processing" - getAllDocs project_id, (err, docs, project)-> - if err? - console.error err, project_id, "could not get all docs" - return callback(err) - else - getWhichDocsCanBeDeleted docs, (err, docsToBeDeleted, unmigratedDocs)-> - if err? - console.error err, project_id, "could not save docs into mongo" - return callback(err) - markUnmigratedDocs project_id, unmigratedDocs, (err)-> - removeDocLinesFromProject docsToBeDeleted, project, (err)-> - if err? - return callback(err) - markProjectAsProcessed project_id, (err)-> - setTimeout( - -> callback(err) - ,0) - -exports.migrate = (client, done = ->)-> - getProjectIds (err, ids)-> - printProgress() - jobs = _.map ids, (id)-> - return (cb)-> - processNext(id, cb) - async.series jobs, (err)-> - if err? - console.error err, "at end of jobs" - else - console.log "finished" - done(err) - - -exports.rollback = (next)-> - next() - diff --git a/server-ce/migrations/3_pack_docHistory_collection.coffee b/server-ce/migrations/3_pack_docHistory_collection.coffee deleted file mode 100644 index affe1fcfd1..0000000000 --- a/server-ce/migrations/3_pack_docHistory_collection.coffee +++ /dev/null @@ -1,342 +0,0 @@ -Settings = require "settings-sharelatex" -fs = require("fs") -mongojs = require("mongojs") -ObjectId = mongojs.ObjectId -db = mongojs(Settings.mongo.url, ['docs','docHistory', 'docHistoryStats']) -_ = require("underscore") -async = require("async") -exec = require("child_process").exec -bson = require('bson') -BSON = new bson() - -logger = { - log: -> - err: -> -} - -needToExit = false -handleExit = () -> - needToExit = true - console.log('Got signal. Shutting down.') - -process.on 'SIGINT', handleExit -process.on 'SIGHUP', handleExit - -finished_docs_path = "/tmp/finished-docs-3" -all_docs_path = "/tmp/all-docs-3" -unmigrated_docs_path = "/tmp/unmigrated-docs-3" - -finished_docs = {} -if fs.existsSync(finished_docs_path) - for id in fs.readFileSync(finished_docs_path,'utf-8').split("\n") - finished_docs[id] = true - -getAndWriteDocids = (callback)-> - console.log "finding all doc id's - #{new Date().toString()}" - db.docs.find {}, {_id:1}, (err, ids)-> - console.log "total found docs in mongo #{ids.length} - #{new Date().toString()}" - ids = _.pluck ids, '_id' - ids = _.filter ids, (id)-> id? - fileData = ids.join("\n") - fs.writeFileSync all_docs_path + ".tmp", fileData - fs.renameSync all_docs_path + ".tmp", all_docs_path - callback(err, ids) - -loadDocIds = (callback)-> - console.log "loading doc ids from #{all_docs_path}" - data = fs.readFileSync all_docs_path, "utf-8" - ids = data.split("\n") - console.log "loaded #{ids.length} doc ids from #{all_docs_path}" - callback null, ids - -getDocIds = (callback)-> - exists = fs.existsSync all_docs_path - if exists - loadDocIds callback - else - getAndWriteDocids callback - -markDocAsProcessed = (doc_id, callback)-> - finished_docs[doc_id] = true - fs.appendFile finished_docs_path, "#{doc_id}\n", callback - -markDocAsUnmigrated = (doc_id, callback)-> - console.log "#{doc_id} unmigrated" - markDocAsProcessed doc_id, (err)-> - fs.appendFile unmigrated_docs_path, "#{doc_id}\n", callback - -checkIfDocHasBeenProcessed = (doc_id, callback)-> - callback(null, finished_docs[doc_id]) - -processNext = (doc_id, callback)-> - if !doc_id? or doc_id.length == 0 - return callback() - if needToExit - return callback(new Error("graceful shutdown")) - checkIfDocHasBeenProcessed doc_id, (err, hasBeenProcessed)-> - if hasBeenProcessed - console.log "#{doc_id} already processed, skipping" - return callback() - PackManager._packDocHistory doc_id, {}, (err) -> - if err? - console.log "error processing #{doc_id}" - markDocAsUnmigrated doc_id, callback - else - markDocAsProcessed doc_id, callback - -updateIndexes = (callback) -> - async.series [ - (cb) -> - console.log "create index" - db.docHistory.ensureIndex { project_id: 1, "meta.end_ts": 1, "meta.start_ts": -1 }, { background: true }, cb - (cb) -> - console.log "drop index" - db.docHistory.dropIndex { project_id: 1, "meta.end_ts": 1 }, cb - (cb) -> - console.log "drop index" - db.docHistory.dropIndex { project_id: 1, "pack.0.meta.end_ts": 1, "meta.end_ts": 1}, cb - ], (err, results) -> - console.log "all done" - callback(err) - -exports.migrate = (client, done = ->)-> - getDocIds (err, ids)-> - totalDocCount = ids.length - alreadyFinishedCount = Object.keys(finished_docs).length - t0 = Date.now() - printProgress = () -> - count = Object.keys(finished_docs).length - processedFraction = (count-alreadyFinishedCount)/totalDocCount - remainingFraction = (totalDocCount-count)/totalDocCount - t = Date.now() - dt = (t-t0)*remainingFraction/processedFraction - estFinishTime = new Date(t + dt) - console.log "completed #{count}/#{totalDocCount} processed=#{processedFraction.toFixed(2)} remaining=#{remainingFraction.toFixed(2)} elapsed=#{(t-t0)/1000} est Finish=#{estFinishTime}" - interval = setInterval printProgress, 3*1000 - - nextId = null - - testFn = () -> - return false if needToExit - id = ids.shift() - while id? and finished_docs[id] # skip finished - id = ids.shift() - nextId = id - return nextId? - - executeFn = (cb) -> - processNext nextId, cb - - async.whilst testFn, executeFn, (err)-> - if err? - console.error err, "at end of jobs" - else - console.log "finished at #{new Date}" - clearInterval interval - done(err) - -exports.rollback = (client, done)-> - done() - -# process.nextTick () -> -# exports.migrate () -> -# console.log "done" - -DAYS = 24 * 3600 * 1000 # one day in milliseconds - -# copied from track-changes/app/coffee/PackManager.coffee - -PackManager = - MAX_SIZE: 1024*1024 # make these configurable parameters - MAX_COUNT: 512 - - convertDocsToPacks: (docs, callback) -> - packs = [] - top = null - docs.forEach (d,i) -> - # skip existing packs - if d.pack? - top = null - return - sz = BSON.calculateObjectSize(d) - # decide if this doc can be added to the current pack - validLength = top? && (top.pack.length < PackManager.MAX_COUNT) - validSize = top? && (top.sz + sz < PackManager.MAX_SIZE) - bothPermanent = top? && (top.expiresAt? is false) && (d.expiresAt? is false) - bothTemporary = top? && (top.expiresAt? is true) && (d.expiresAt? is true) - within1Day = bothTemporary && (d.meta.start_ts - top.meta.start_ts < 24 * 3600 * 1000) - if top? && validLength && validSize && (bothPermanent || (bothTemporary && within1Day)) - top.pack = top.pack.concat {v: d.v, meta: d.meta, op: d.op, _id: d._id} - top.sz += sz - top.n += 1 - top.v_end = d.v - top.meta.end_ts = d.meta.end_ts - top.expiresAt = d.expiresAt if top.expiresAt? - return - else - # create a new pack - top = _.clone(d) - top.pack = [ {v: d.v, meta: d.meta, op: d.op, _id: d._id} ] - top.meta = { start_ts: d.meta.start_ts, end_ts: d.meta.end_ts } - top.sz = sz - top.n = 1 - top.v_end = d.v - delete top.op - delete top._id - packs.push top - - callback(null, packs) - - checkHistory: (docs, callback) -> - errors = [] - prev = null - error = (args...) -> - errors.push args - docs.forEach (d,i) -> - if d.pack? - n = d.pack.length - last = d.pack[n-1] - error('bad pack v_end', d) if d.v_end != last.v - error('bad pack start_ts', d) if d.meta.start_ts != d.pack[0].meta.start_ts - error('bad pack end_ts', d) if d.meta.end_ts != last.meta.end_ts - d.pack.forEach (p, i) -> - prev = v - v = p.v - error('bad version', v, 'in', p) if v <= prev - #error('expired op', p, 'in pack') if p.expiresAt? - else - prev = v - v = d.v - error('bad version', v, 'in', d) if v <= prev - if errors.length - callback(errors) - else - callback() - - insertPack: (packObj, callback) -> - bulk = db.docHistory.initializeOrderedBulkOp() - doc_id = packObj.doc_id - expect_nInserted = 1 - expect_nRemoved = packObj.pack.length - logger.log {doc_id: doc_id}, "adding pack, removing #{expect_nRemoved} ops" - bulk.insert packObj - ids = (op._id for op in packObj.pack) - bulk.find({_id:{$in:ids}}).remove() - bulk.execute (err, result) -> - if err? - logger.error {doc_id: doc_id}, "error adding pack" - callback(err, result) - else if result.nInserted != expect_nInserted or result.nRemoved != expect_nRemoved - logger.error {doc_id: doc_id, result}, "unexpected result adding pack" - callback(new Error( - msg: 'unexpected result' - expected: {expect_nInserted, expect_nRemoved} - ), result) - else - db.docHistoryStats.update {doc_id:doc_id}, { - $inc:{update_count:-expect_nRemoved}, - $currentDate:{last_packed:true} - }, {upsert:true}, () -> - callback(err, result) - - # retrieve document ops/packs and check them - getDocHistory: (doc_id, callback) -> - db.docHistory.find({doc_id:ObjectId(doc_id)}).sort {v:1}, (err, docs) -> - return callback(err) if err? - # for safety, do a consistency check of the history - logger.log {doc_id}, "checking history for document" - PackManager.checkHistory docs, (err) -> - return callback(err) if err? - callback(err, docs) - #PackManager.deleteExpiredPackOps docs, (err) -> - # return callback(err) if err? - # callback err, docs - - packDocHistory: (doc_id, options, callback) -> - if typeof callback == "undefined" and typeof options == 'function' - callback = options - options = {} - LockManager.runWithLock( - "HistoryLock:#{doc_id}", - (releaseLock) -> - PackManager._packDocHistory(doc_id, options, releaseLock) - , callback - ) - - _packDocHistory: (doc_id, options, callback) -> - logger.log {doc_id},"starting pack operation for document history" - - PackManager.getDocHistory doc_id, (err, docs) -> - return callback(err) if err? - origDocs = 0 - origPacks = 0 - for d in docs - if d.pack? then origPacks++ else origDocs++ - PackManager.convertDocsToPacks docs, (err, packs) -> - return callback(err) if err? - total = 0 - for p in packs - total = total + p.pack.length - logger.log {doc_id, origDocs, origPacks, newPacks: packs.length, totalOps: total}, "document stats" - if packs.length - if options['dry-run'] - logger.log {doc_id}, 'dry-run, skipping write packs' - return callback() - PackManager.savePacks packs, (err) -> - return callback(err) if err? - # check the history again - PackManager.getDocHistory doc_id, callback - else - logger.log {doc_id}, "no packs to write" - # keep a record that we checked this one to avoid rechecking it - db.docHistoryStats.update {doc_id:doc_id}, { - $currentDate:{last_checked:true} - }, {upsert:true}, () -> - callback null, null - - DB_WRITE_DELAY: 100 - - savePacks: (packs, callback) -> - async.eachSeries packs, PackManager.safeInsert, (err, result) -> - if err? - logger.log {err, result}, "error writing packs" - callback err, result - else - callback() - - safeInsert: (packObj, callback) -> - PackManager.insertPack packObj, (err, result) -> - setTimeout () -> - callback(err,result) - , PackManager.DB_WRITE_DELAY - - deleteExpiredPackOps: (docs, callback) -> - now = Date.now() - toRemove = [] - toUpdate = [] - docs.forEach (d,i) -> - if d.pack? - newPack = d.pack.filter (op) -> - if op.expiresAt? then op.expiresAt > now else true - if newPack.length == 0 - toRemove.push d - else if newPack.length < d.pack.length - # adjust the pack properties - d.pack = newPack - first = d.pack[0] - last = d.pack[d.pack.length - 1] - d.v_end = last.v - d.meta.start_ts = first.meta.start_ts - d.meta.end_ts = last.meta.end_ts - toUpdate.push d - if toRemove.length or toUpdate.length - bulk = db.docHistory.initializeOrderedBulkOp() - toRemove.forEach (pack) -> - console.log "would remove", pack - #bulk.find({_id:pack._id}).removeOne() - toUpdate.forEach (pack) -> - console.log "would update", pack - #bulk.find({_id:pack._id}).updateOne(pack); - bulk.execute callback - else - callback() diff --git a/server-ce/migrations/4_update_user_features.coffee b/server-ce/migrations/4_update_user_features.coffee deleted file mode 100644 index 680df3c459..0000000000 --- a/server-ce/migrations/4_update_user_features.coffee +++ /dev/null @@ -1,38 +0,0 @@ -Settings = require "settings-sharelatex" -fs = require("fs") -mongojs = require("mongojs") -ObjectId = mongojs.ObjectId -db = mongojs(Settings.mongo.url, ['users']) -_ = require("underscore") - - -handleExit = () -> - console.log('Got signal. Shutting down.') - - -process.on 'SIGINT', handleExit -process.on 'SIGHUP', handleExit - - -exports.migrate = (client, done=()->) -> - patch = { - $set: { - features: { - collaborators: -1 - dropbox: true - versioning: true - references: true - templates: true - compileTimeout: 180 - compileGroup: "standard" - } - } - } - console.log ">> updating all user features: ", patch - db.users.update {}, patch, {multi: true}, (err) -> - console.log "finished updating all user features" - return done(err) - - -exports.rollback = (client, done) -> - done() diff --git a/server-ce/migrations/5_remove_holding_accounts.coffee b/server-ce/migrations/5_remove_holding_accounts.coffee deleted file mode 100644 index ee796b3023..0000000000 --- a/server-ce/migrations/5_remove_holding_accounts.coffee +++ /dev/null @@ -1,103 +0,0 @@ -Settings = require "settings-sharelatex" -mongojs = require("mongojs") -ObjectId = mongojs.ObjectId -db = mongojs(Settings.mongo.url, ['users', 'projects', 'subscriptions']) -async = require "async" - -module.exports = HoldingAccountMigration = - DRY_RUN: true - - findHoldingAccounts: (callback = (error, users) ->) -> - db.users.find({holdingAccount: true, hashedPassword: { $exists: false }}, {holdingAccount: 1, email: 1}, callback) - - deleteUserProjects: (user_id, callback = (error) ->) -> - # Holding accounts can't own projects, so only remove from - # collaberator_refs and readOnly_refs - console.log "[Removing user from projects]", user_id - db.projects.find { - $or: [ - {collaberator_refs: user_id}, - {readOnly_refs: user_id} - ] - }, { collaberator_refs: 1, readOnly_refs: 1 }, (error, projects = []) -> - return callback(error) if error? - jobs = projects.map (project) -> - (cb) -> - console.log "[Removing user from project]", user_id, JSON.stringify(project) - if !project?._id? - throw new Error("no project id") - - if !HoldingAccountMigration.DRY_RUN - db.projects.update { - _id: project._id - }, { - $pull: { - collaberator_refs: user_id, - readOnly_refs: user_id - } - }, (error, result) -> - return cb(error) if error? - console.log "[Removed user from project]", user_id, project._id, result - cb() - else - console.log "[Would have removed user from project]", user_id, project._id - cb() - - async.series jobs, callback - - deleteUser: (user_id, callback = (error) ->) -> - if !user_id? - throw new Error("must have user_id") - if !HoldingAccountMigration.DRY_RUN - db.users.remove {_id: user_id, holdingAccount: true}, (error, result) -> - return callback(error) if error? - console.log "[Removed user]", user_id, result - if result.n != 1 - return callback(new Error("failed to remove user as expected")) - callback() - else - console.log "[Would have removed user]", user_id - callback() - - migrateGroupInvites: (user_id, email, callback = (error) ->) -> - if !user_id? - throw new Error("must have user_id") - if !HoldingAccountMigration.DRY_RUN - db.subscriptions.update {member_ids: user_id}, { - $pull: { member_ids: user_id }, - $addToSet : { invited_emails: email } - }, { multi : true }, (error, result) -> - return callback(error) if error? - console.log "[Migrated user in group accounts]", user_id, email, result - callback() - else - console.log "[Would have migrated user in group accounts]", user_id, email - callback() - - run: (done = () ->) -> - console.log "[Getting list of holding accounts]" - HoldingAccountMigration.findHoldingAccounts (error, users) -> - throw error if error? - console.log "[Got #{users.length} holding accounts]" - i = 0 - jobs = users.map (u) -> - (cb) -> - console.log "[Removing user #{i++}/#{users.length}]" - HoldingAccountMigration.migrateGroupInvites u._id, u.email, (error) -> - return cb(error) if error? - HoldingAccountMigration.deleteUser u._id, (error) -> - return cb(error) if error? - HoldingAccountMigration.deleteUserProjects u._id, (error) -> - return cb(error) if error? - setTimeout cb, 50 # Small delay to not hammer DB - async.series jobs, (error) -> - throw error if error? - console.log "[FINISHED]" - done() - - migrate: (client, done=()->) -> - HoldingAccountMigration.DRY_RUN = false - HoldingAccountMigration.run(done) - - rollback: (client, done) -> - done() diff --git a/server-ce/migrations/6_add_track_changes_feature.coffee b/server-ce/migrations/6_add_track_changes_feature.coffee deleted file mode 100644 index e807102faa..0000000000 --- a/server-ce/migrations/6_add_track_changes_feature.coffee +++ /dev/null @@ -1,30 +0,0 @@ -Settings = require "settings-sharelatex" -fs = require("fs") -mongojs = require("mongojs") -ObjectId = mongojs.ObjectId -db = mongojs(Settings.mongo.url, ['users']) -_ = require("underscore") - - -handleExit = () -> - console.log('Got signal. Shutting down.') - - -process.on 'SIGINT', handleExit -process.on 'SIGHUP', handleExit - - -exports.migrate = (client, done=()->) -> - patch = { - $set: { - 'features.trackChanges': true - } - } - console.log ">> enabling trackChanges feature: ", patch - db.users.update {}, patch, {multi: true}, (err) -> - console.log "finished enabling trackChanges feature" - return done(err) - - -exports.rollback = (client, done) -> - done() diff --git a/server-ce/migrations/7_add_token_indexes.coffee b/server-ce/migrations/7_add_token_indexes.coffee deleted file mode 100644 index 867c209753..0000000000 --- a/server-ce/migrations/7_add_token_indexes.coffee +++ /dev/null @@ -1,51 +0,0 @@ -Settings = require "settings-sharelatex" -fs = require("fs") -mongojs = require("mongojs") -ObjectId = mongojs.ObjectId -db = mongojs(Settings.mongo.url, ['docs','docHistory', 'docHistoryStats']) -_ = require("underscore") -async = require("async") -exec = require("child_process").exec -bson = require('bson') -BSON = new bson() - - -handleExit = () -> - console.log('Got signal. Shutting down.') - - -exports.migrate = (client, done=()->) -> - console.log ">> Adding indexes for token-based project access: " - db.projects.ensureIndex {'tokens.readAndWrite': 1}, { - partialFilterExpression: { 'tokens.readAndWrite': { $exists: true } }, - unique: true, - background: true - }, (err) -> - if err? - return done(err) - db.projects.ensureIndex {'tokens.readOnly': 1}, { - partialFilterExpression: { 'tokens.readOnly': { $exists: true } }, - unique: true, - background: true - }, (err) -> - if err? - return done(err) - db.projects.ensureIndex {tokenAccessReadAndWrite_refs: 1}, { - background: true - }, (err) -> - if err? - return done(err) - db.projects.ensureIndex {tokenAccessOnly_refs: 1}, { - background: true - }, (err) -> - console.log ">> done adding indexes for token-based project access" - done() - - -exports.rollback = (client, done) -> - done() - - -process.on 'SIGINT', handleExit -process.on 'SIGHUP', handleExit - diff --git a/server-ce/migrations/7_add_track_changes_feature_again.coffee b/server-ce/migrations/7_add_track_changes_feature_again.coffee deleted file mode 100644 index b942e61f3f..0000000000 --- a/server-ce/migrations/7_add_track_changes_feature_again.coffee +++ /dev/null @@ -1,31 +0,0 @@ -#This is needed because we forgot to add track changes into the default settings -Settings = require "settings-sharelatex" -fs = require("fs") -mongojs = require("mongojs") -ObjectId = mongojs.ObjectId -db = mongojs(Settings.mongo.url, ['users']) -_ = require("underscore") - - -handleExit = () -> - console.log('Got signal. Shutting down.') - - -process.on 'SIGINT', handleExit -process.on 'SIGHUP', handleExit - - -exports.migrate = (client, done=()->) -> - patch = { - $set: { - 'features.trackChanges': true - } - } - console.log ">> enabling trackChanges feature: ", patch - db.users.update {}, patch, {multi: true}, (err) -> - console.log "finished enabling trackChanges feature" - return done(err) - - -exports.rollback = (client, done) -> - done() diff --git a/server-ce/migrations/9_create_user_emails_array.js b/server-ce/migrations/9_create_user_emails_array.js deleted file mode 100644 index b9216c2a4c..0000000000 --- a/server-ce/migrations/9_create_user_emails_array.js +++ /dev/null @@ -1,49 +0,0 @@ -const Settings = require('settings-sharelatex') -const mongojs = require('mongojs') -const db = mongojs(Settings.mongo.url, ['users']) -const async = require('async') - -const handleExit = () => console.log('Got signal. Shutting down.') -process.on('SIGINT', handleExit) -process.on('SIGHUP', handleExit) - -const initUserEmailsAttribute = (user, callback) => { - const update = { - $set: { - emails: [ - { - email: user.email, - createdAt: new Date() - } - ] - } - } - db.users.update({ _id: user._id }, update, callback) -} - -const updateAllUsersEmailsAttribute = (users, callback) => { - console.log(`updating ${users.length} users`) - async.eachSeries(users, initUserEmailsAttribute, callback) -} - -exports.migrate = (client, done) => - db.users.find( - { emails: { $exists: false } }, - { email: 1 }, - (error, users) => { - if (error) { - callback(error) - } else { - updateAllUsersEmailsAttribute(users, done) - } - } - ) - -exports.rollback = (client, done) => { - const update = { - $unset: { - emails: 1 - } - } - db.users.update({ emails: { $exists: true } }, update, done) -} diff --git a/server-ce/migrations/about_migrations.md b/server-ce/migrations/about_migrations.md deleted file mode 100644 index 8a9d7596ba..0000000000 --- a/server-ce/migrations/about_migrations.md +++ /dev/null @@ -1,9 +0,0 @@ -If migration is stopped mid way it will start at the beginning next time - -To see the run migrations do db.getCollection('_migrations').find() you can't do db._migrations.find() - -When testing, to roll back a migration run: - -``` -./node_modules/east/bin/east rollback 5 --adapter east-mongo --url mongodb://localhost:27017/sharelatex -```