From 935877222a1bc37ee1adf90461fed308bf21bd27 Mon Sep 17 00:00:00 2001 From: Ersun Warncke Date: Mon, 8 Apr 2019 10:40:45 -0400 Subject: [PATCH] oauth and collabratec migration * make v2 system of record for collabratec id * migrate oauth apps, tokens to v2 * set collabratec id on reg/link from ctec * copy collabratec id from user stub to user on merge GitOrigin-RevId: 4ac63921b030a01ed79bc0da1666d3c9f9545248 --- .../User/ThirdPartyIdentityManager.coffee | 47 +++++--- .../web/app/coffee/models/UserStub.coffee | 2 + services/web/scripts/import-oauth/.gitignore | 2 + .../import_oauth_access_grants.js | 97 ++++++++++++++++ .../import_oauth_access_tokens.js | 109 ++++++++++++++++++ .../import-oauth/import_oauth_applications.js | 48 ++++++++ .../import_user_collabratec_ids.js | 83 +++++++++++++ .../web/scripts/import-oauth/package.json | 5 + .../acceptance/coffee/helpers/User.coffee | 2 +- 9 files changed, 381 insertions(+), 14 deletions(-) create mode 100644 services/web/scripts/import-oauth/.gitignore create mode 100644 services/web/scripts/import-oauth/import_oauth_access_grants.js create mode 100644 services/web/scripts/import-oauth/import_oauth_access_tokens.js create mode 100644 services/web/scripts/import-oauth/import_oauth_applications.js create mode 100644 services/web/scripts/import-oauth/import_user_collabratec_ids.js create mode 100644 services/web/scripts/import-oauth/package.json diff --git a/services/web/app/coffee/Features/User/ThirdPartyIdentityManager.coffee b/services/web/app/coffee/Features/User/ThirdPartyIdentityManager.coffee index 037a94d2ed..30686fd2bf 100644 --- a/services/web/app/coffee/Features/User/ThirdPartyIdentityManager.coffee +++ b/services/web/app/coffee/Features/User/ThirdPartyIdentityManager.coffee @@ -1,29 +1,50 @@ Errors = require "../Errors/Errors" User = require("../../models/User").User +UserStub = require("../../models/UserStub").UserStub UserUpdater = require "./UserUpdater" _ = require "lodash" module.exports = ThirdPartyIdentityManager = login: (providerId, externalUserId, externalData, callback) -> return callback(new Error "invalid arguments") unless providerId? and externalUserId? + query = ThirdPartyIdentityManager._loginQuery providerId, externalUserId + User.findOne query, (err, user) -> + return callback err if err? + return callback(new Errors.ThirdPartyUserNotFoundError()) unless user + return callback(null, user) unless externalData + update = ThirdPartyIdentityManager._loginUpdate user, providerId, externalUserId, externalData + User.findOneAndUpdate query, update, {new: true}, callback + + # attempt to login normally but check for user stub if user not found + loginUserStub: (providerId, externalUserId, externalData, callback) -> + ThirdPartyIdentityManager.login providerId, externalUserId, externalData, (err, user) -> + return callback null, user unless err? + return callback err unless err.name == "ThirdPartyUserNotFoundError" + query = ThirdPartyIdentityManager._loginQuery providerId, externalUserId + UserStub.findOne query, (err, userStub) -> + return callback err if err? + return callback(new Errors.ThirdPartyUserNotFoundError()) unless userStub + return callback(null, userStub) unless externalData + update = ThirdPartyIdentityManager._loginUpdate userStub, providerId, externalUserId, externalData + UserStub.findOneAndUpdate query, update, {new: true}, callback + + _loginQuery: (providerId, externalUserId) -> externalUserId = externalUserId.toString() providerId = providerId.toString() query = "thirdPartyIdentifiers.externalUserId": externalUserId "thirdPartyIdentifiers.providerId": providerId - User.findOne query, (err, user) -> - return callback err if err? - return callback(new Errors.ThirdPartyUserNotFoundError()) unless user - # skip updating data unless passed - return callback(null, user) unless externalData - # get third party identifier object from array - thirdPartyIdentifier = user.thirdPartyIdentifiers.find (tpi) -> - tpi.externalUserId == externalUserId and tpi.providerId == providerId - # do recursive merge of new data over existing data - _.merge(thirdPartyIdentifier.externalData, externalData) - # update user - update = "thirdPartyIdentifiers.$": thirdPartyIdentifier - User.findOneAndUpdate query, update, {new: true}, callback + return query + + _loginUpdate: (user, providerId, externalUserId, externalData) -> + providerId = providerId.toString() + # get third party identifier object from array + thirdPartyIdentifier = user.thirdPartyIdentifiers.find (tpi) -> + tpi.externalUserId == externalUserId and tpi.providerId == providerId + # do recursive merge of new data over existing data + _.merge(thirdPartyIdentifier.externalData, externalData) + update = "thirdPartyIdentifiers.$": thirdPartyIdentifier + return update # register: () -> # this should be implemented once we move to having v2 as the master diff --git a/services/web/app/coffee/models/UserStub.coffee b/services/web/app/coffee/models/UserStub.coffee index 26791972d5..70dd1e60dc 100644 --- a/services/web/app/coffee/models/UserStub.coffee +++ b/services/web/app/coffee/models/UserStub.coffee @@ -8,6 +8,8 @@ UserStubSchema = new Schema first_name : { type : String, default : '' } last_name : { type : String, default : '' } overleaf : { id: { type: Number } } + thirdPartyIdentifiers: { type: Array, default: [] } + confirmed_at: Date conn = mongoose.createConnection(Settings.mongo.url, { server: {poolSize: Settings.mongo.poolSize || 10}, diff --git a/services/web/scripts/import-oauth/.gitignore b/services/web/scripts/import-oauth/.gitignore new file mode 100644 index 0000000000..a56a7ef437 --- /dev/null +++ b/services/web/scripts/import-oauth/.gitignore @@ -0,0 +1,2 @@ +node_modules + diff --git a/services/web/scripts/import-oauth/import_oauth_access_grants.js b/services/web/scripts/import-oauth/import_oauth_access_grants.js new file mode 100644 index 0000000000..80b447e292 --- /dev/null +++ b/services/web/scripts/import-oauth/import_oauth_access_grants.js @@ -0,0 +1,97 @@ +/* eslint-disable max-len */ +/** + * run with: node import_oauth_access_grants /path/oauth_access_grants.csv + * + * where csv is generated from v1 with sql statement like: + * + * \copy ( SELECT oag.token, oag.scopes, oag.redirect_uri, resource_owner_id AS user_id, oa.uid AS client_id, oag.created_at + oag.expires_in * interval '1 second' AS expires_at FROM oauth_access_grants oag JOIN oauth_applications oa ON oag.application_id = oa.id WHERE oa.id = 1 AND oag.revoked_at IS NULL AND oag.created_at + oag.expires_in * interval '1 second' > NOW() ) to 'oauth_access_grants.csv' WITH CSV HEADER; + * + * this query exports the most non-expired oauth authorization codes for collabractec (1) + * + /* eslint-enable */ + +'use strict' + +const OauthApplication = require('../../app/js/models/OauthApplication') + .OauthApplication +const OauthAuthorizationCode = require('../../app/js/models/OauthAuthorizationCode') + .OauthAuthorizationCode +const User = require('../../app/js/models/User').User +const async = require('async') +const csvParser = require('csv-parser') +const fs = require('fs') +const minimist = require('minimist') + +const argv = minimist(process.argv.slice(2)) + +let lineNum = 0 +const records = [] + +fs.createReadStream(argv._[0]) + .pipe(csvParser()) + .on('data', data => { + data.lineNum = ++lineNum + records.push(data) + }) + .on('end', () => { + async.mapSeries(records, loadRecord, function(err) { + if (err) console.error(err) + process.exit() + }) + }) + +function loadRecord(record, cb) { + getOauthApplication(record.client_id, function(err, oauthApplication) { + if (err) return cb(err) + User.findOne( + { 'overleaf.id': parseInt(record.user_id) }, + { _id: 1 }, + function(err, user) { + if (err) return cb(err) + if (!user) { + console.log( + record.lineNum + + ': User not found for ' + + record.user_id + + ' - skipping' + ) + return cb() + } + const newRecord = { + authorizationCode: record.token, + expiresAt: record.expires_at, + oauthApplication_id: oauthApplication._id, + redirectUri: record.redirect_uri, + scope: record.scopes, + user_id: user._id + } + console.log( + record.lineNum + + 'Creating OauthAuthorizationCode for User ' + + user._id + ) + OauthAuthorizationCode.update( + newRecord, + newRecord, + { upsert: true }, + cb + ) + } + ) + }) +} + +const oauthApplications = {} + +function getOauthApplication(clientId, cb) { + if (oauthApplications[clientId]) return cb(null, oauthApplications[clientId]) + OauthApplication.findOne({ id: clientId }, { _id: 1 }, function( + err, + oauthApplication + ) { + if (err) return cb(err) + if (!oauthApplication) return cb(new Error('oauthApplication not found')) + oauthApplications[clientId] = oauthApplication + cb(null, oauthApplication) + }) +} diff --git a/services/web/scripts/import-oauth/import_oauth_access_tokens.js b/services/web/scripts/import-oauth/import_oauth_access_tokens.js new file mode 100644 index 0000000000..e5b1186cb1 --- /dev/null +++ b/services/web/scripts/import-oauth/import_oauth_access_tokens.js @@ -0,0 +1,109 @@ +/* eslint-disable max-len */ +/** + * run with: node import_oauth_access_tokens /path/import_oauth_access_tokens.csv + * + * where csv is generated from v1 with sql statement like: + * + * \copy ( SELECT oat.token, oat.refresh_token, oat.scopes, oat.resource_owner_id AS user_id, u.email, u.confirmed_at, oa.uid AS client_id, oat.created_at + oat.expires_in * interval '1 second' AS expires_at FROM oauth_access_tokens oat LEFT JOIN oauth_access_tokens oat2 ON oat2.previous_refresh_token = oat.refresh_token JOIN oauth_applications oa ON oat.application_id = oa.id JOIN users u ON u.id = oat.resource_owner_id WHERE (oat2.id IS NULL OR oat2.created_at > NOW() - interval '24 hour') AND oat.revoked_at IS NULL AND (oat.application_id = 1 OR (oat.application_id = 2 AND oat.created_at + oat.expires_in * interval '1 second' > NOW())) ) to 'oauth_access_tokens.csv' WITH CSV HEADER; + * + * this query exports the most recent collabractec (1) and gitbridge (2) tokens for + * each user. expired tokens are exported for collabratec but not for gitbridge. + * + * tokens that have been refreshed are not exported if it has been over 24 hours + * since the new token was issued. + */ +/* eslint-enable */ + +'use strict' + +const OauthApplication = require('../../app/js/models/OauthApplication') + .OauthApplication +const OauthAccessToken = require('../../app/js/models/OauthAccessToken') + .OauthAccessToken +const User = require('../../app/js/models/User').User +const UserMapper = require('../../modules/overleaf-integration/app/js/OverleafUsers/UserMapper') +const async = require('async') +const csvParser = require('csv-parser') +const fs = require('fs') +const minimist = require('minimist') + +const argv = minimist(process.argv.slice(2)) + +let lineNum = 0 +const records = [] + +fs.createReadStream(argv._[0]) + .pipe(csvParser()) + .on('data', data => { + data.lineNum = ++lineNum + records.push(data) + }) + .on('end', () => { + async.mapSeries(records, loadRecord, function(err) { + if (err) console.error(err) + process.exit() + }) + }) + +function loadRecord(record, cb) { + getOauthApplication(record.client_id, function(err, oauthApplication) { + if (err) return cb(err) + const overleafId = parseInt(record.user_id) + User.findOne({ 'overleaf.id': overleafId }, { _id: 1 }, function( + err, + user + ) { + if (err) return cb(err) + if (user) { + console.log( + record.lineNum + ': Creating OauthAccessToken for User ' + user._id + ) + createOauthAccessToken(user._id, oauthApplication._id, record, cb) + } else { + // create user stub + const olUser = { + confirmed_at: record.confirmed_at, + email: record.email, + id: overleafId + } + console.log(record.lineNum + ': User not found for ' + record.user_id) + return UserMapper.getSlIdFromOlUser(olUser, function(err, userStubId) { + if (err) return cb(err) + console.log( + record.lineNum + + ': Creating OauthAccessToken for UserStub ' + + userStubId + ) + createOauthAccessToken(userStubId, oauthApplication._id, record, cb) + }) + } + }) + }) +} + +function createOauthAccessToken(userId, oauthApplicationId, record, cb) { + const newRecord = { + accessToken: record.token, + accessTokenExpiresAt: record.expires_at, + oauthApplication_id: oauthApplicationId, + refreshToken: record.refresh_token, + scope: record.scopes, + user_id: userId + } + OauthAccessToken.update(newRecord, newRecord, { upsert: true }, cb) +} + +const oauthApplications = {} + +function getOauthApplication(clientId, cb) { + if (oauthApplications[clientId]) return cb(null, oauthApplications[clientId]) + OauthApplication.findOne({ id: clientId }, { _id: 1 }, function( + err, + oauthApplication + ) { + if (err) return cb(err) + if (!oauthApplication) return cb(new Error('oauthApplication not found')) + oauthApplications[clientId] = oauthApplication + cb(null, oauthApplication) + }) +} diff --git a/services/web/scripts/import-oauth/import_oauth_applications.js b/services/web/scripts/import-oauth/import_oauth_applications.js new file mode 100644 index 0000000000..980d9bba6c --- /dev/null +++ b/services/web/scripts/import-oauth/import_oauth_applications.js @@ -0,0 +1,48 @@ +/* eslint-disable max-len */ +/** + * run with: node import_oauth_applications /path/oauth_applications.csv + * + * where csv is generated from v1 with sql statement like: + * + * \copy (SELECT * FROM oauth_applications) to 'oauth_applications.csv' WITH CSV HEADER; + */ +/* eslint-enable */ + +'use strict' + +const OauthApplication = require('../../app/js/models/OauthApplication') + .OauthApplication +const async = require('async') +const csvParser = require('csv-parser') +const fs = require('fs') +const minimist = require('minimist') + +const argv = minimist(process.argv.slice(2)) + +const records = [] + +fs.createReadStream(argv._[0]) + .pipe(csvParser()) + .on('data', data => records.push(data)) + .on('end', () => { + async.mapSeries(records, loadRecord, function(err) { + if (err) console.error(err) + process.exit() + }) + }) + +function loadRecord(record, cb) { + const newRecord = { + clientSecret: record.secret, + id: record.uid, + // doorkeeper does not define grant types so add all supported + grants: ['authorization_code', 'refresh_token', 'password'], + name: record.name, + // redirect uris are stored new-line separated + redirectUris: record.redirect_uri.split(/\r?\n/), + // scopes are stored space separated + scopes: record.scopes.split(/\s+/) + } + console.log('Creating OauthApplication ' + newRecord.name) + OauthApplication.update(newRecord, newRecord, { upsert: true }, cb) +} diff --git a/services/web/scripts/import-oauth/import_user_collabratec_ids.js b/services/web/scripts/import-oauth/import_user_collabratec_ids.js new file mode 100644 index 0000000000..c9eb86c1f4 --- /dev/null +++ b/services/web/scripts/import-oauth/import_user_collabratec_ids.js @@ -0,0 +1,83 @@ +/* eslint-disable max-len */ +/** + * run with: node import_user_collabratec_ids /path/user_collabratec_ids.csv + * + * where csv is generated from v1 with sql statement like: + * + * \copy ( SELECT ut.user_id, u.email, u.confirmed_at, ut.team_user_id AS collabratec_id FROM user_teams ut JOIN teams t ON ut.team_id = t.id JOIN users u ON ut.user_id = u.id WHERE t.name = 'IEEECollabratec' AND ut.removed_at IS NULL AND ut.team_user_id IS NOT NULL ) to 'user_collabratec_ids.csv' WITH CSV HEADER; + */ +/* eslint-enable */ + +'use strict' + +const UserMapper = require('../../modules/overleaf-integration/app/js/OverleafUsers/UserMapper') +const User = require('../../app/js/models/User').User +const UserStub = require('../../app/js/models/UserStub').UserStub +const async = require('async') +const csvParser = require('csv-parser') +const fs = require('fs') +const minimist = require('minimist') + +const argv = minimist(process.argv.slice(2)) + +let lineNum = 0 +const records = [] + +fs.createReadStream(argv._[0]) + .pipe(csvParser()) + .on('data', data => { + data.lineNum = ++lineNum + records.push(data) + }) + .on('end', () => { + async.mapSeries(records, loadRecord, function(err) { + if (err) console.error(err) + process.exit() + }) + }) + +function loadRecord(record, cb) { + const overleafId = parseInt(record.user_id) + User.findOne( + { 'overleaf.id': overleafId }, + { _id: 1, thirdPartyIdentifiers: 1 }, + function(err, user) { + if (err) return cb(err) + const query = { + 'thirdPartyIdentifiers.providerId': { + $ne: 'collabratec' + } + } + const update = { + $push: { + thirdPartyIdentifiers: { + externalUserId: record.collabratec_id, + externalData: {}, + providerId: 'collabratec' + } + } + } + if (user) { + console.log(record.lineNum + ': setting TPI for User ' + user._id) + query._id = user._id + User.update(query, update, cb) + } else { + // create user stub + const olUser = { + confirmed_at: record.confirmed_at, + email: record.email, + id: overleafId + } + console.log(record.lineNum + ': User not found for ' + record.user_id) + return UserMapper.getSlIdFromOlUser(olUser, function(err, userStubId) { + if (err) return cb(err) + console.log( + record.lineNum + ': setting TPI for UserStub ' + userStubId + ) + query._id = userStubId + UserStub.update(query, update, cb) + }) + } + } + ) +} diff --git a/services/web/scripts/import-oauth/package.json b/services/web/scripts/import-oauth/package.json new file mode 100644 index 0000000000..4abbe35943 --- /dev/null +++ b/services/web/scripts/import-oauth/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "csv-parser": "^2.2.0" + } +} diff --git a/services/web/test/acceptance/coffee/helpers/User.coffee b/services/web/test/acceptance/coffee/helpers/User.coffee index 738d6d206c..314bdc6942 100644 --- a/services/web/test/acceptance/coffee/helpers/User.coffee +++ b/services/web/test/acceptance/coffee/helpers/User.coffee @@ -11,7 +11,7 @@ count = 0 class User constructor: (options = {}) -> @emails = [ - email: "acceptance-test-#{count}@example.com" + email: options.email || "acceptance-test-#{count}@example.com" createdAt: new Date() ] @email = @emails[0].email