oauth and collabratec migration

* make v2 system of record for collabratec id
* migrate oauth apps, tokens to v2
* set collabratec id on reg/link from ctec
* copy collabratec id from user stub to user on merge

GitOrigin-RevId: 4ac63921b030a01ed79bc0da1666d3c9f9545248

services/web/app/coffee/Features/User/ThirdPartyIdentityManager.coffee

@@ -1,29 +1,50 @@
 Errors = require "../Errors/Errors"
 User = require("../../models/User").User
+UserStub = require("../../models/UserStub").UserStub
 UserUpdater = require "./UserUpdater"
 _ = require "lodash"
 
 module.exports = ThirdPartyIdentityManager =
 	login: (providerId, externalUserId, externalData, callback) ->
 		return callback(new Error "invalid arguments") unless providerId? and externalUserId?
+		query = ThirdPartyIdentityManager._loginQuery providerId, externalUserId
+		User.findOne query, (err, user) ->
+			return callback err if err?
+			return callback(new Errors.ThirdPartyUserNotFoundError()) unless user
+			return callback(null, user) unless externalData
+			update = ThirdPartyIdentityManager._loginUpdate user, providerId, externalUserId, externalData
+			User.findOneAndUpdate query, update, {new: true}, callback
+
+	# attempt to login normally but check for user stub if user not found
+	loginUserStub: (providerId, externalUserId, externalData, callback) ->
+		ThirdPartyIdentityManager.login providerId, externalUserId, externalData, (err, user) ->
+			return callback null, user unless err?
+			return callback err unless err.name == "ThirdPartyUserNotFoundError"
+			query = ThirdPartyIdentityManager._loginQuery providerId, externalUserId
+			UserStub.findOne query, (err, userStub) ->
+				return callback err if err?
+				return callback(new Errors.ThirdPartyUserNotFoundError()) unless userStub
+				return callback(null, userStub) unless externalData
+				update = ThirdPartyIdentityManager._loginUpdate userStub, providerId, externalUserId, externalData
+				UserStub.findOneAndUpdate query, update, {new: true}, callback
+
+	_loginQuery: (providerId, externalUserId) ->
 		externalUserId = externalUserId.toString()
 		providerId = providerId.toString()
 		query =
 			"thirdPartyIdentifiers.externalUserId": externalUserId
 			"thirdPartyIdentifiers.providerId": providerId
-		User.findOne query, (err, user) ->
-			return callback err if err?
-			return callback(new Errors.ThirdPartyUserNotFoundError()) unless user
-			# skip updating data unless passed
-			return callback(null, user) unless externalData
-			# get third party identifier object from array
-			thirdPartyIdentifier = user.thirdPartyIdentifiers.find (tpi) ->
-				tpi.externalUserId == externalUserId and tpi.providerId == providerId
-			# do recursive merge of new data over existing data
-			_.merge(thirdPartyIdentifier.externalData, externalData)
-			# update user
-			update = "thirdPartyIdentifiers.$": thirdPartyIdentifier
-			User.findOneAndUpdate query, update, {new: true}, callback
+		return query
+
+	_loginUpdate: (user, providerId, externalUserId, externalData) ->
+		providerId = providerId.toString()
+		# get third party identifier object from array
+		thirdPartyIdentifier = user.thirdPartyIdentifiers.find (tpi) ->
+			tpi.externalUserId == externalUserId and tpi.providerId == providerId
+		# do recursive merge of new data over existing data
+		_.merge(thirdPartyIdentifier.externalData, externalData)
+		update = "thirdPartyIdentifiers.$": thirdPartyIdentifier
+		return update
 
 	# register: () ->
 		# this should be implemented once we move to having v2 as the master

services/web/app/coffee/models/UserStub.coffee

@@ -8,6 +8,8 @@ UserStubSchema = new Schema
 	first_name : { type : String, default : '' }
 	last_name  : { type : String, default : '' }
 	overleaf   : { id: { type: Number } }
+	thirdPartyIdentifiers: { type: Array, default: [] }
+	confirmed_at: Date
 
 conn = mongoose.createConnection(Settings.mongo.url, {
 	server: {poolSize: Settings.mongo.poolSize || 10},

services/web/scripts/import-oauth/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+

services/web/scripts/import-oauth/import_oauth_access_grants.js

@@ -0,0 +1,97 @@
+/* eslint-disable max-len */
+/**
+ * run with: node import_oauth_access_grants /path/oauth_access_grants.csv
+ *
+ * where csv is generated from v1 with sql statement like:
+ *
+ * \copy ( SELECT oag.token, oag.scopes, oag.redirect_uri, resource_owner_id AS user_id, oa.uid AS client_id, oag.created_at + oag.expires_in * interval '1 second' AS expires_at FROM oauth_access_grants oag JOIN oauth_applications oa ON oag.application_id = oa.id WHERE oa.id = 1 AND oag.revoked_at IS NULL AND oag.created_at + oag.expires_in * interval '1 second' > NOW() ) to 'oauth_access_grants.csv' WITH CSV HEADER;
+ *
+ * this query exports the most non-expired oauth authorization codes for collabractec (1)
+ *
+ /* eslint-enable */
+
+'use strict'
+
+const OauthApplication = require('../../app/js/models/OauthApplication')
+  .OauthApplication
+const OauthAuthorizationCode = require('../../app/js/models/OauthAuthorizationCode')
+  .OauthAuthorizationCode
+const User = require('../../app/js/models/User').User
+const async = require('async')
+const csvParser = require('csv-parser')
+const fs = require('fs')
+const minimist = require('minimist')
+
+const argv = minimist(process.argv.slice(2))
+
+let lineNum = 0
+const records = []
+
+fs.createReadStream(argv._[0])
+  .pipe(csvParser())
+  .on('data', data => {
+    data.lineNum = ++lineNum
+    records.push(data)
+  })
+  .on('end', () => {
+    async.mapSeries(records, loadRecord, function(err) {
+      if (err) console.error(err)
+      process.exit()
+    })
+  })
+
+function loadRecord(record, cb) {
+  getOauthApplication(record.client_id, function(err, oauthApplication) {
+    if (err) return cb(err)
+    User.findOne(
+      { 'overleaf.id': parseInt(record.user_id) },
+      { _id: 1 },
+      function(err, user) {
+        if (err) return cb(err)
+        if (!user) {
+          console.log(
+            record.lineNum +
+              ': User not found for ' +
+              record.user_id +
+              ' - skipping'
+          )
+          return cb()
+        }
+        const newRecord = {
+          authorizationCode: record.token,
+          expiresAt: record.expires_at,
+          oauthApplication_id: oauthApplication._id,
+          redirectUri: record.redirect_uri,
+          scope: record.scopes,
+          user_id: user._id
+        }
+        console.log(
+          record.lineNum +
+            'Creating OauthAuthorizationCode for User ' +
+            user._id
+        )
+        OauthAuthorizationCode.update(
+          newRecord,
+          newRecord,
+          { upsert: true },
+          cb
+        )
+      }
+    )
+  })
+}
+
+const oauthApplications = {}
+
+function getOauthApplication(clientId, cb) {
+  if (oauthApplications[clientId]) return cb(null, oauthApplications[clientId])
+  OauthApplication.findOne({ id: clientId }, { _id: 1 }, function(
+    err,
+    oauthApplication
+  ) {
+    if (err) return cb(err)
+    if (!oauthApplication) return cb(new Error('oauthApplication not found'))
+    oauthApplications[clientId] = oauthApplication
+    cb(null, oauthApplication)
+  })
+}

services/web/scripts/import-oauth/import_oauth_access_tokens.js

@@ -0,0 +1,109 @@
+/* eslint-disable max-len */
+/**
+ * run with: node import_oauth_access_tokens /path/import_oauth_access_tokens.csv
+ *
+ * where csv is generated from v1 with sql statement like:
+ *
+ * \copy ( SELECT oat.token, oat.refresh_token, oat.scopes, oat.resource_owner_id AS user_id, u.email, u.confirmed_at, oa.uid AS client_id, oat.created_at + oat.expires_in * interval '1 second' AS expires_at FROM  oauth_access_tokens oat LEFT JOIN oauth_access_tokens oat2 ON oat2.previous_refresh_token = oat.refresh_token JOIN oauth_applications oa ON oat.application_id = oa.id JOIN users u ON u.id = oat.resource_owner_id WHERE (oat2.id IS NULL OR oat2.created_at > NOW() - interval '24 hour') AND oat.revoked_at IS NULL AND (oat.application_id = 1 OR (oat.application_id = 2 AND oat.created_at + oat.expires_in * interval '1 second' > NOW())) ) to 'oauth_access_tokens.csv' WITH CSV HEADER;
+ *
+ * this query exports the most recent collabractec (1) and gitbridge (2) tokens for
+ * each user. expired tokens are exported for collabratec but not for gitbridge.
+ *
+ * tokens that have been refreshed are not exported if it has been over 24 hours
+ * since the new token was issued.
+ */
+/* eslint-enable */
+
+'use strict'
+
+const OauthApplication = require('../../app/js/models/OauthApplication')
+  .OauthApplication
+const OauthAccessToken = require('../../app/js/models/OauthAccessToken')
+  .OauthAccessToken
+const User = require('../../app/js/models/User').User
+const UserMapper = require('../../modules/overleaf-integration/app/js/OverleafUsers/UserMapper')
+const async = require('async')
+const csvParser = require('csv-parser')
+const fs = require('fs')
+const minimist = require('minimist')
+
+const argv = minimist(process.argv.slice(2))
+
+let lineNum = 0
+const records = []
+
+fs.createReadStream(argv._[0])
+  .pipe(csvParser())
+  .on('data', data => {
+    data.lineNum = ++lineNum
+    records.push(data)
+  })
+  .on('end', () => {
+    async.mapSeries(records, loadRecord, function(err) {
+      if (err) console.error(err)
+      process.exit()
+    })
+  })
+
+function loadRecord(record, cb) {
+  getOauthApplication(record.client_id, function(err, oauthApplication) {
+    if (err) return cb(err)
+    const overleafId = parseInt(record.user_id)
+    User.findOne({ 'overleaf.id': overleafId }, { _id: 1 }, function(
+      err,
+      user
+    ) {
+      if (err) return cb(err)
+      if (user) {
+        console.log(
+          record.lineNum + ': Creating OauthAccessToken for User ' + user._id
+        )
+        createOauthAccessToken(user._id, oauthApplication._id, record, cb)
+      } else {
+        // create user stub
+        const olUser = {
+          confirmed_at: record.confirmed_at,
+          email: record.email,
+          id: overleafId
+        }
+        console.log(record.lineNum + ': User not found for ' + record.user_id)
+        return UserMapper.getSlIdFromOlUser(olUser, function(err, userStubId) {
+          if (err) return cb(err)
+          console.log(
+            record.lineNum +
+              ': Creating OauthAccessToken for UserStub ' +
+              userStubId
+          )
+          createOauthAccessToken(userStubId, oauthApplication._id, record, cb)
+        })
+      }
+    })
+  })
+}
+
+function createOauthAccessToken(userId, oauthApplicationId, record, cb) {
+  const newRecord = {
+    accessToken: record.token,
+    accessTokenExpiresAt: record.expires_at,
+    oauthApplication_id: oauthApplicationId,
+    refreshToken: record.refresh_token,
+    scope: record.scopes,
+    user_id: userId
+  }
+  OauthAccessToken.update(newRecord, newRecord, { upsert: true }, cb)
+}
+
+const oauthApplications = {}
+
+function getOauthApplication(clientId, cb) {
+  if (oauthApplications[clientId]) return cb(null, oauthApplications[clientId])
+  OauthApplication.findOne({ id: clientId }, { _id: 1 }, function(
+    err,
+    oauthApplication
+  ) {
+    if (err) return cb(err)
+    if (!oauthApplication) return cb(new Error('oauthApplication not found'))
+    oauthApplications[clientId] = oauthApplication
+    cb(null, oauthApplication)
+  })
+}

services/web/scripts/import-oauth/import_oauth_applications.js

@@ -0,0 +1,48 @@
+/* eslint-disable max-len */
+/**
+ * run with: node import_oauth_applications /path/oauth_applications.csv
+ *
+ * where csv is generated from v1 with sql statement like:
+ *
+ * \copy (SELECT * FROM oauth_applications) to 'oauth_applications.csv' WITH CSV HEADER;
+ */
+/* eslint-enable */
+
+'use strict'
+
+const OauthApplication = require('../../app/js/models/OauthApplication')
+  .OauthApplication
+const async = require('async')
+const csvParser = require('csv-parser')
+const fs = require('fs')
+const minimist = require('minimist')
+
+const argv = minimist(process.argv.slice(2))
+
+const records = []
+
+fs.createReadStream(argv._[0])
+  .pipe(csvParser())
+  .on('data', data => records.push(data))
+  .on('end', () => {
+    async.mapSeries(records, loadRecord, function(err) {
+      if (err) console.error(err)
+      process.exit()
+    })
+  })
+
+function loadRecord(record, cb) {
+  const newRecord = {
+    clientSecret: record.secret,
+    id: record.uid,
+    // doorkeeper does not define grant types so add all supported
+    grants: ['authorization_code', 'refresh_token', 'password'],
+    name: record.name,
+    // redirect uris are stored new-line separated
+    redirectUris: record.redirect_uri.split(/\r?\n/),
+    // scopes are stored space separated
+    scopes: record.scopes.split(/\s+/)
+  }
+  console.log('Creating OauthApplication ' + newRecord.name)
+  OauthApplication.update(newRecord, newRecord, { upsert: true }, cb)
+}

services/web/scripts/import-oauth/import_user_collabratec_ids.js

@@ -0,0 +1,83 @@
+/* eslint-disable max-len */
+/**
+ * run with: node import_user_collabratec_ids /path/user_collabratec_ids.csv
+ *
+ * where csv is generated from v1 with sql statement like:
+ *
+ * \copy ( SELECT ut.user_id, u.email, u.confirmed_at, ut.team_user_id AS collabratec_id FROM user_teams ut JOIN teams t ON ut.team_id = t.id JOIN users u ON ut.user_id = u.id WHERE t.name = 'IEEECollabratec' AND ut.removed_at IS NULL AND ut.team_user_id IS NOT NULL ) to 'user_collabratec_ids.csv' WITH CSV HEADER;
+ */
+/* eslint-enable */
+
+'use strict'
+
+const UserMapper = require('../../modules/overleaf-integration/app/js/OverleafUsers/UserMapper')
+const User = require('../../app/js/models/User').User
+const UserStub = require('../../app/js/models/UserStub').UserStub
+const async = require('async')
+const csvParser = require('csv-parser')
+const fs = require('fs')
+const minimist = require('minimist')
+
+const argv = minimist(process.argv.slice(2))
+
+let lineNum = 0
+const records = []
+
+fs.createReadStream(argv._[0])
+  .pipe(csvParser())
+  .on('data', data => {
+    data.lineNum = ++lineNum
+    records.push(data)
+  })
+  .on('end', () => {
+    async.mapSeries(records, loadRecord, function(err) {
+      if (err) console.error(err)
+      process.exit()
+    })
+  })
+
+function loadRecord(record, cb) {
+  const overleafId = parseInt(record.user_id)
+  User.findOne(
+    { 'overleaf.id': overleafId },
+    { _id: 1, thirdPartyIdentifiers: 1 },
+    function(err, user) {
+      if (err) return cb(err)
+      const query = {
+        'thirdPartyIdentifiers.providerId': {
+          $ne: 'collabratec'
+        }
+      }
+      const update = {
+        $push: {
+          thirdPartyIdentifiers: {
+            externalUserId: record.collabratec_id,
+            externalData: {},
+            providerId: 'collabratec'
+          }
+        }
+      }
+      if (user) {
+        console.log(record.lineNum + ': setting TPI for User ' + user._id)
+        query._id = user._id
+        User.update(query, update, cb)
+      } else {
+        // create user stub
+        const olUser = {
+          confirmed_at: record.confirmed_at,
+          email: record.email,
+          id: overleafId
+        }
+        console.log(record.lineNum + ': User not found for ' + record.user_id)
+        return UserMapper.getSlIdFromOlUser(olUser, function(err, userStubId) {
+          if (err) return cb(err)
+          console.log(
+            record.lineNum + ': setting TPI for UserStub ' + userStubId
+          )
+          query._id = userStubId
+          UserStub.update(query, update, cb)
+        })
+      }
+    }
+  )
+}

services/web/scripts/import-oauth/package.json

@@ -0,0 +1,5 @@
+{
+  "dependencies": {
+    "csv-parser": "^2.2.0"
+  }
+}

services/web/test/acceptance/coffee/helpers/User.coffee

@@ -11,7 +11,7 @@ count = 0
 class User
 	constructor: (options = {}) ->
 		@emails = [
-			email: "acceptance-test-#{count}@example.com"
+			email: options.email || "acceptance-test-#{count}@example.com"
 			createdAt: new Date()
 		]
 		@email = @emails[0].email