From 1f4960165b3016ae03ba5a9a010b14876fc5c7ce Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 5 Oct 2023 13:35:19 +0100 Subject: [PATCH] Merge pull request #15061 from overleaf/bg-server-pro-migrate-emails-script add migrate emails script for server pro GitOrigin-RevId: be5fc166554d08100de175133d413ecb1a29623a --- .../web/app/src/Features/User/UserUpdater.js | 54 +++++ .../scripts/migrate-user-emails.js | 199 ++++++++++++++++++ .../acceptance/src/ServerCEScriptsTests.js | 126 +++++++++++ 3 files changed, 379 insertions(+) create mode 100644 services/web/modules/server-ce-scripts/scripts/migrate-user-emails.js diff --git a/services/web/app/src/Features/User/UserUpdater.js b/services/web/app/src/Features/User/UserUpdater.js index ab6ad1f7cc..b705819e05 100644 --- a/services/web/app/src/Features/User/UserUpdater.js +++ b/services/web/app/src/Features/User/UserUpdater.js @@ -234,6 +234,58 @@ async function setDefaultEmailAddress( } } +/** + * Overwrites the primary email address of a user in the database in-place. + * This function is only intended for use in scripts to migrate email addresses + * where we do not want to trigger all the actions that happen when a user + * changes their own email. It should not be used in any other circumstances. + */ +async function migrateDefaultEmailAddress( + userId, + oldEmail, + newEmail, + auditLog +) { + oldEmail = EmailHelper.parseEmail(oldEmail) + if (oldEmail == null) { + throw new Error('invalid old email') + } + newEmail = EmailHelper.parseEmail(newEmail) + if (newEmail == null) { + throw new Error('invalid new email') + } + const reversedHostname = newEmail.split('@')[1].split('').reverse().join('') + const query = { + _id: userId, + email: oldEmail, + 'emails.email': oldEmail, + } + const update = { + $set: { + email: newEmail, + 'emails.$.email': newEmail, + 'emails.$.reversedHostname': reversedHostname, + }, + } + const result = await updateUser(query, update) + if (result.modifiedCount !== 1) { + throw new Error('email update error') + } + // add a user audit log entry for the email change + await UserAuditLogHandler.promises.addEntry( + userId, + 'migrate-default-email', + auditLog.initiatorId, + auditLog.ipAddress, + { + oldEmail, + newEmail, + // Add optional extra info + ...(auditLog.extraInfo || {}), + } + ) +} + async function confirmEmail(userId, email, affiliationOptions) { // used for initial email confirmation (non-SSO and SSO) // also used for reconfirmation of non-SSO emails @@ -492,6 +544,7 @@ module.exports = { removeEmailAddress: callbackify(removeEmailAddress), removeReconfirmFlag: callbackify(removeReconfirmFlag), setDefaultEmailAddress: callbackify(setDefaultEmailAddress), + migrateDefaultEmailAddress: callbackify(migrateDefaultEmailAddress), updateUser: callbackify(updateUser), promises: { addAffiliationForNewUser, @@ -502,6 +555,7 @@ module.exports = { removeEmailAddress, removeReconfirmFlag, setDefaultEmailAddress, + migrateDefaultEmailAddress, updateUser, }, } diff --git a/services/web/modules/server-ce-scripts/scripts/migrate-user-emails.js b/services/web/modules/server-ce-scripts/scripts/migrate-user-emails.js new file mode 100644 index 0000000000..eb7fb052c4 --- /dev/null +++ b/services/web/modules/server-ce-scripts/scripts/migrate-user-emails.js @@ -0,0 +1,199 @@ +// Script to migrate user emails using a CSV file with the following format: +// +// oldEmail,newEmail +// +// The script will iterate through the CSV file and update the user's email +// address from oldEmail to newEmail, after checking all the email addresses +// for duplicates. +// +// Intended for Server Pro customers migrating user emails from one domain to +// another. + +const minimist = require('minimist') +const { waitForDb } = require('../../../app/src/infrastructure/mongodb') + +const os = require('os') +const fs = require('fs') +const csv = require('csv/sync') +const { parseEmail } = require('../../../app/src/Features/Helpers/EmailHelper') +const UserGetter = require('../../../app/src/Features/User/UserGetter') +const UserUpdater = require('../../../app/src/Features/User/UserUpdater') +const UserSessionsManager = require('../../../app/src/Features/User/UserSessionsManager') + +const hostname = os.hostname() +const scriptTimestamp = new Date().toISOString() + +// support command line option of --commit to actually do the migration +const argv = minimist(process.argv.slice(2), { + boolean: ['commit', 'ignore-missing'], + string: ['admin-id'], + alias: { + 'ignore-missing': 'continue', + }, + default: { + commit: false, + 'ignore-missing': false, + 'admin-id': '000000000000000000000000', // use a dummy admin ID for script audit log entries + }, +}) + +// display usage if no CSV file is provided +if (argv._.length === 0) { + console.log( + 'Usage: node migrate_user_emails.js [--commit] [--continue|--ignore-missing] [--admin-id=ADMIN_USER_ID] ' + ) + console.log(' --commit: actually do the migration (default: false)') + console.log( + ' --continue|--ignore-missing: continue on missing or already-migrated users' + ) + console.log(' --admin-id: admin user ID to use for audit log entries') + console.log(' : CSV file with old and new email addresses') + process.exit(1) +} + +function filterEmails(rows) { + // check that emails have a valid format + const result = [] + const seenOld = new Set() + const seenNew = new Set() + for (const [oldEmail, newEmail] of rows) { + const parsedOld = parseEmail(oldEmail) + const parsedNew = parseEmail(newEmail) + if (!parsedOld) { + throw new Error(`invalid old email "${oldEmail}"`) + } + if (!parsedNew) { + throw new Error(`invalid new email "${newEmail}"`) + } + // Check for duplicates and overlaps + if (seenOld.has(parsedOld)) { + throw new Error(`Duplicate old emails found in CSV file ${oldEmail}.`) + } + if (seenNew.has(parsedNew)) { + throw new Error(`Duplicate new emails found in CSV file ${newEmail}.`) + } + if (seenOld.has(parsedNew) || seenNew.has(parsedOld)) { + throw new Error( + `Old and new emails cannot overlap ${oldEmail} ${newEmail}` + ) + } + seenOld.add(parsedOld) + seenNew.add(parsedNew) + result.push([parsedOld, parsedNew]) + } + return result +} + +async function checkEmailsAgainstDb(emails) { + const result = [] + for (const [oldEmail, newEmail] of emails) { + const userWithEmail = await UserGetter.promises.getUserByMainEmail( + oldEmail, + { + _id: 1, + } + ) + if (!userWithEmail) { + if (argv['ignore-missing']) { + console.log( + `User with email "${oldEmail}" not found, skipping update to "${newEmail}"` + ) + continue + } else { + throw new Error(`no user found with email "${oldEmail}"`) + } + } + const userWithNewEmail = await UserGetter.promises.getUserByAnyEmail( + newEmail, + { + _id: 1, + } + ) + if (userWithNewEmail) { + throw new Error( + `new email "${newEmail}" already exists for user ${userWithNewEmail._id}` + ) + } + result.push([oldEmail, newEmail]) + } + return result +} + +async function doMigration(emails) { + let success = 0 + let failure = 0 + let skipped = 0 + for (const [oldEmail, newEmail] of emails) { + const userWithEmail = await UserGetter.promises.getUserByMainEmail( + oldEmail, + { + _id: 1, + } + ) + if (!userWithEmail) { + if (argv['ignore-missing']) { + continue + } else { + throw new Error(`no user found with email "${oldEmail}"`) + } + } + if (argv.commit) { + console.log( + `Updating user ${userWithEmail._id} email "${oldEmail}" to "${newEmail}"\n` + ) + try { + await UserSessionsManager.promises.revokeAllUserSessions( + userWithEmail, + [] // log out all the user's sessions before changing the email address + ) + + await UserUpdater.promises.migrateDefaultEmailAddress( + userWithEmail._id, + oldEmail, + newEmail, + { + initiatorId: argv['admin-id'], + ipAddress: hostname, + extraInfo: { + script: 'migrate_user_emails.js', + runAt: scriptTimestamp, + }, + } + ) + success++ + } catch (err) { + console.log(err) + failure++ + } + } else { + console.log(`Dry run, skipping update from ${oldEmail} to ${newEmail}`) + skipped++ + } + } + console.log('Success: ', success, 'Failure: ', failure, 'Skipped: ', skipped) + if (failure > 0) { + throw new Error('Some email migrations failed') + } +} + +async function migrateEmails() { + console.log('Starting email migration script') + const csvFilePath = argv._[0] + const csvFile = fs.readFileSync(csvFilePath, 'utf8') + const rows = csv.parse(csvFile) + console.log('Number of users to migrate: ', rows.length) + await waitForDb() + const emails = filterEmails(rows) + const existingUserEmails = await checkEmailsAgainstDb(emails) + await doMigration(existingUserEmails) +} + +migrateEmails() + .then(() => { + console.log('Done.') + process.exit(0) + }) + .catch(error => { + console.error(error) + process.exit(1) + }) diff --git a/services/web/modules/server-ce-scripts/test/acceptance/src/ServerCEScriptsTests.js b/services/web/modules/server-ce-scripts/test/acceptance/src/ServerCEScriptsTests.js index 54d59cfca2..146c212385 100644 --- a/services/web/modules/server-ce-scripts/test/acceptance/src/ServerCEScriptsTests.js +++ b/services/web/modules/server-ce-scripts/test/acceptance/src/ServerCEScriptsTests.js @@ -3,6 +3,7 @@ const { execSync } = require('child_process') const { expect } = require('chai') const { db } = require('../../../../../app/src/infrastructure/mongodb') const User = require('../../../../../test/acceptance/src/helpers/User').promises +const fs = require('fs') /** * @param {string} cmd @@ -131,6 +132,131 @@ describe('ServerCEScripts', function () { }) }) + describe('migrate-user-emails', function () { + let usersToMigrate + let otherUsers + let csv + let csvfail + beforeEach(async function () { + // set up some users to migrate and others to leave alone + usersToMigrate = [] + otherUsers = [] + for (let i = 0; i < 2; i++) { + const user = new User() + await user.login() + usersToMigrate.push(user) + } + for (let i = 0; i < 2; i++) { + const user = new User() + await user.login() + otherUsers.push(user) + } + // write the migration csv to a temporary file + const id = usersToMigrate[0]._id + csv = `/tmp/migration-${id}.csv` + const rows = [] + for (const user of usersToMigrate) { + rows.push(`${user.email},new-${user.email}`) + } + fs.writeFileSync(csv, rows.join('\n')) + // also write a csv with a user that doesn't exist + csvfail = `/tmp/migration-fail-${id}.csv` + fs.writeFileSync( + csvfail, + [ + 'nouser@example.com,nouser@other.example.com', + ...rows, + 'foo@example.com,bar@example.com', + ].join('\n') + ) + }) + + afterEach(function () { + // clean up the temporary files + fs.unlinkSync(csv) + fs.unlinkSync(csvfail) + }) + + it('should do a dry run by default', async function () { + run( + `node modules/server-ce-scripts/scripts/migrate-user-emails.js ${csv}` + ) + for (const user of usersToMigrate) { + const dbEntry = await user.get() + expect(dbEntry.email).to.equal(user.email) + } + for (const user of otherUsers) { + const dbEntry = await user.get() + expect(dbEntry.email).to.equal(user.email) + } + }) + + it('should exit with code 0 when successfully migrating user emails', function () { + run( + `node modules/server-ce-scripts/scripts/migrate-user-emails.js --commit ${csv}` + ) + }) + + it('should migrate the user emails with the --commit option', async function () { + run( + `node modules/server-ce-scripts/scripts/migrate-user-emails.js --commit ${csv}` + ) + for (const user of usersToMigrate) { + const dbEntry = await user.get() + expect(dbEntry.email).to.equal(`new-${user.email}`) + expect(dbEntry.emails).to.have.lengthOf(1) + expect(dbEntry.emails[0].email).to.equal(`new-${user.email}`) + expect(dbEntry.emails[0].reversedHostname).to.equal('moc.elpmaxe') + expect(dbEntry.emails[0].createdAt).to.eql(user.emails[0].createdAt) + } + }) + + it('should leave other user emails unchanged', async function () { + run( + `node modules/server-ce-scripts/scripts/migrate-user-emails.js --commit ${csv}` + ) + for (const user of otherUsers) { + const dbEntry = await user.get() + expect(dbEntry.email).to.equal(user.email) + } + }) + + it('should exit with code 1 when there are failures migrating user emails', function () { + try { + run( + `node modules/server-ce-scripts/scripts/migrate-user-emails.js --commit ${csvfail}` + ) + } catch (e) { + expect(e.status).to.equal(1) + return + } + expect.fail('command should have failed') + }) + + it('should migrate other users when there are failures with the --continue option', async function () { + try { + run( + `node modules/server-ce-scripts/scripts/migrate-user-emails.js --commit ${csvfail}` + ) + } catch (e) { + expect(e.status).to.equal(1) + run( + `node modules/server-ce-scripts/scripts/migrate-user-emails.js --commit --continue ${csvfail}` + ) + for (const user of usersToMigrate) { + const dbEntry = await user.get() + expect(dbEntry.email).to.equal(`new-${user.email}`) + expect(dbEntry.emails).to.have.lengthOf(1) + expect(dbEntry.emails[0].email).to.equal(`new-${user.email}`) + expect(dbEntry.emails[0].reversedHostname).to.equal('moc.elpmaxe') + expect(dbEntry.emails[0].createdAt).to.eql(user.emails[0].createdAt) + } + return + } + expect.fail('command should have failed') + }) + }) + describe('rename-tag', function () { let user beforeEach(async function () {