Merge pull request #6247 from overleaf/jpa-cleanup-mongo

[migrations] delete unused collections and properly apply saml indexes

GitOrigin-RevId: eafcde98cbb603c275986a5bca708b9f8221877f

services/web/app/src/infrastructure/mongodb.js

@@ -71,12 +71,7 @@ async function setupDb() {
   db.users = internalDb.collection('users')
   db.userstubs = internalDb.collection('userstubs')
 }
-async function addCollection(name) {
-  await waitForDb()
-  const internalDb = (await clientPromise).db()
 
-  db[name] = internalDb.collection(name)
-}
 async function getCollectionNames() {
   const internalDb = (await clientPromise).db()
 
@@ -84,10 +79,18 @@ async function getCollectionNames() {
   return collections.map(collection => collection.collectionName)
 }
 
+/**
+ * WARNING: Consider using a pre-populated collection from `db` to avoid typos!
+ */
+async function getCollectionInternal(name) {
+  const internalDb = (await clientPromise).db()
+  return internalDb.collection(name)
+}
+
 module.exports = {
   db,
   ObjectId,
-  addCollection,
   getCollectionNames,
+  getCollectionInternal,
   waitForDb,
 }

services/web/migrations/20190912145022_create_projectImportFailures_indexes.js

@@ -1,6 +1,7 @@
 /* eslint-disable no-unused-vars */
 
 const Helpers = require('./lib/helpers')
+const { getCollectionInternal } = require('../app/src/infrastructure/mongodb')
 
 exports.tags = ['saas']
 
@@ -13,17 +14,20 @@ const indexes = [
   },
 ]
 
-exports.migrate = async client => {
+async function getCollection() {
-  const { db } = client
+  // NOTE: This is a stale collection, it will get dropped in a later migration.
+  return await getCollectionInternal('projectImportFailures')
+}
 
-  await Helpers.addIndexesToCollection(db.projectImportFailures, indexes)
+exports.migrate = async client => {
+  const collection = await getCollection()
+  await Helpers.addIndexesToCollection(collection, indexes)
 }
 
 exports.rollback = async client => {
-  const { db } = client
+  const collection = await getCollection()
-
   try {
-    await Helpers.dropIndexesFromCollection(db.projectImportFailures, indexes)
+    await Helpers.dropIndexesFromCollection(collection, indexes)
   } catch (err) {
     console.error('Something went wrong rolling back the migrations', err)
   }

services/web/migrations/20191106102104_saml-log-indexes.js

@@ -1,6 +1,7 @@
 /* eslint-disable no-unused-vars */
 
 const Helpers = require('./lib/helpers')
+const { getCollectionInternal } = require('../app/src/infrastructure/mongodb')
 
 exports.tags = ['saas']
 
@@ -27,17 +28,25 @@ const indexes = [
   },
 ]
 
-exports.migrate = async client => {
+// Export indexes for use in the fix-up migration 20220105130000_fix_saml_indexes.js.
-  const { db } = client
+exports.samlLogsIndexes = indexes
 
-  await Helpers.addIndexesToCollection(db.samllog, indexes)
+async function getCollection() {
+  // This collection was incorrectly named - it should have been `samlLogs`
+  //  instead of `samllog`. The error is corrected by the subsequent migration
+  //  20220105130000_fix_saml_indexes.js.
+  return await getCollectionInternal('samllog')
+}
+
+exports.migrate = async client => {
+  const collection = await getCollection()
+  await Helpers.addIndexesToCollection(collection, indexes)
 }
 
 exports.rollback = async client => {
-  const { db } = client
+  const collection = await getCollection()
-
   try {
-    await Helpers.dropIndexesFromCollection(db.samllog, indexes)
+    await Helpers.dropIndexesFromCollection(collection, indexes)
   } catch (err) {
     console.error('Something went wrong rolling back the migrations', err)
   }

services/web/migrations/20191107191318_saml-indentifiers-index.js

@@ -1,6 +1,7 @@
 /* eslint-disable no-unused-vars */
 
 const Helpers = require('./lib/helpers')
+const { getCollectionInternal } = require('../app/src/infrastructure/mongodb')
 
 exports.tags = ['saas']
 
@@ -15,17 +16,25 @@ const indexes = [
   },
 ]
 
-exports.migrate = async client => {
+// Export indexes for use in the fix-up migration 20220105130000_fix_saml_indexes.js.
-  const { db } = client
+exports.usersIndexes = indexes
 
-  await Helpers.addIndexesToCollection(db.user, indexes)
+async function getCollection() {
+  // This collection was incorrectly named - it should have been `users` instead
+  //  of `user`. The error is corrected by the subsequent migration
+  //  20220105130000_fix_saml_indexes.js.
+  return await getCollectionInternal('user')
+}
+
+exports.migrate = async client => {
+  const collection = await getCollection()
+  await Helpers.addIndexesToCollection(collection, indexes)
 }
 
 exports.rollback = async client => {
-  const { db } = client
+  const collection = await getCollection()
-
   try {
-    await Helpers.dropIndexesFromCollection(db.user, indexes)
+    await Helpers.dropIndexesFromCollection(collection, indexes)
   } catch (err) {
     console.error('Something went wrong rolling back the migrations', err)
   }

services/web/migrations/20200522145727_dropProjectImportFailures.js

@@ -3,8 +3,7 @@ const Helpers = require('./lib/helpers')
 exports.tags = ['saas']
 
 exports.migrate = async client => {
-  const { db } = client
+  await Helpers.dropCollection('projectImportFailures')
-  await Helpers.dropCollection(db, 'projectImportFailures')
 }
 
 exports.rollback = async client => {

services/web/migrations/20200522145741_dropProjectImportBatchRecords.js

@@ -3,8 +3,7 @@ const Helpers = require('./lib/helpers')
 exports.tags = ['saas']
 
 exports.migrate = async client => {
-  const { db } = client
+  await Helpers.dropCollection('projectImportBatchRecords')
-  await Helpers.dropCollection(db, 'projectImportBatchRecords')
 }
 
 exports.rollback = async client => {

services/web/migrations/20210924140139_splittests-name-index.js

@@ -1,4 +1,5 @@
 const Helpers = require('./lib/helpers')
+const { getCollectionInternal } = require('../app/src/infrastructure/mongodb')
 
 exports.tags = ['saas']
 
@@ -12,12 +13,17 @@ const indexes = [
   },
 ]
 
+async function getCollection() {
+  // NOTE: We do not access the splittests collection directly. Fetch it here.
+  return await getCollectionInternal('splittests')
+}
+
 exports.migrate = async client => {
-  const { db } = client
+  const collection = await getCollection()
-  await Helpers.addIndexesToCollection(db.splittests, indexes)
+  await Helpers.addIndexesToCollection(collection, indexes)
 }
 
 exports.rollback = async client => {
-  const { db } = client
+  const collection = await getCollection()
-  Helpers.dropIndexesFromCollection(db.splittests, indexes)
+  await Helpers.dropIndexesFromCollection(collection, indexes)
 }

services/web/migrations/20220105123000_cleanup_unused_collections.js

@@ -0,0 +1,29 @@
+const Helpers = require('./lib/helpers')
+
+exports.tags = ['saas']
+
+const unusedCollections = [
+  'collaberatorcount',
+  'db.subscriptions',
+  'projectsDeletedByMigration',
+  'readonlycount',
+  'samllog',
+  'sharelatex-production.docOps.2013-12-17T02-26-49.0',
+  'sharelatex-production.projects.2013-12-17T02-26-49.1',
+  'sharelatex-production.users.2013-12-17T02-26-49.2',
+  'sharelatex_production.users',
+  'totalwords',
+  'user',
+  'usersDeletedByMigration',
+  'usersEmailDomains',
+]
+
+exports.migrate = async () => {
+  for (const name of unusedCollections) {
+    await Helpers.dropCollection(name)
+  }
+}
+
+exports.rollback = async () => {
+  // We lost the indexes. There is no way back.
+}

services/web/migrations/20220105130000_fix_saml_indexes.js

@@ -0,0 +1,14 @@
+const Helpers = require('./lib/helpers')
+const { samlLogsIndexes } = require('./20191106102104_saml-log-indexes')
+const { usersIndexes } = require('./20191107191318_saml-indentifiers-index')
+
+exports.tags = ['saas']
+
+exports.migrate = async ({ db }) => {
+  // Fix-up the previous SAML migrations that were operating on collections with
+  //  typos in their names.
+  await Helpers.addIndexesToCollection(db.users, usersIndexes)
+  await Helpers.addIndexesToCollection(db.samlLogs, samlLogsIndexes)
+}
+
+exports.rollback = async () => {}

services/web/migrations/lib/adapter.js

@@ -1,9 +1,5 @@
 const Path = require('path')
-const {
+const { waitForDb, db } = require('../../app/src/infrastructure/mongodb')
-  addCollection,
-  waitForDb,
-  db,
-} = require('../../app/src/infrastructure/mongodb')
 
 class Adapter {
   constructor(params) {
@@ -24,12 +20,6 @@ class Adapter {
 
   async connect() {
     await waitForDb()
-
-    await addCollection('projectImportFailures')
-    await addCollection('samllog')
-    await addCollection('user')
-    await addCollection('splittests')
-
     return { db }
   }
 

services/web/migrations/lib/helpers.js

@@ -1,4 +1,9 @@
-const { getCollectionNames } = require('../../app/src/infrastructure/mongodb')
+const {
+  db,
+  getCollectionNames,
+  getCollectionInternal,
+  waitForDb,
+} = require('../../app/src/infrastructure/mongodb')
 
 async function addIndexesToCollection(collection, indexes) {
   return Promise.all(
@@ -13,10 +18,16 @@ async function dropIndexesFromCollection(collection, indexes) {
   return Promise.all(indexes.map(index => collection.dropIndex(index.name)))
 }
 
-async function dropCollection(db, collectionName) {
+async function dropCollection(collectionName) {
+  await waitForDb()
+  if (db[collectionName]) {
+    throw new Error(`blocking drop of an active collection: ${collectionName}`)
+  }
+
   const allCollections = await getCollectionNames()
   if (!allCollections.includes(collectionName)) return
-  return db[collectionName].drop()
+  const collection = await getCollectionInternal(collectionName)
+  await collection.drop()
 }
 
 module.exports = {

services/web/migrations/lib/template.js

@@ -11,5 +11,5 @@ exports.migrate = async client => {
 
 exports.rollback = async client => {
   const { db } = client
-  // Helpers.dropIndexesFromCollection(db.wombats, [{ name: 1 }])
+  // await Helpers.dropIndexesFromCollection(db.wombats, [{ name: 1 }])
 }