github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-09-16 02:52:31 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #3187 from overleaf/jpa-mongodb-native

Browse source 
[misc] migrate the app to the native mongo driver

GitOrigin-RevId: 9030b18c4cf62e3a01d3d8f450bf0e02f9f89c22
This commit is contained in:
Jakob Ackermann 2020-10-01 10:30:26 +02:00 committed by Copybot
parent 022424601d
commit e3c6637339
21 changed files with 169 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
services/web/app.js
View file

@ -24,6 +24,7 @@ if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) {
metrics.memory.monitor(logger)
const Server = require('./app/src/infrastructure/Server')
const mongodb = require('./app/src/infrastructure/mongodb')
if (Settings.catchErrors) {
process.removeAllListeners('uncaughtException')
@ -40,12 +41,20 @@ if (!module.parent) {
if (!process.env['WEB_API_USER'] || !process.env['WEB_API_PASSWORD']) {
throw new Error('No API user and password provided')
}
mongodb
.waitForDb()
.then(() => {
Server.server.listen(port, host, function() {
logger.info(`web starting up, listening on ${host}:${port}`)
logger.info(`${require('http').globalAgent.maxSockets} sockets enabled`)
// wait until the process is ready before monitoring the event loop
metrics.event_loop.monitor(logger)
})
})
.catch(err => {
logger.fatal({ err }, 'Cannot connect to mongo. Exiting.')
process.exit(1)
})
}
// handle SIGTERM for graceful shutdown in kubernetes

8
services/web/app/src/Features/Authentication/AuthenticationManager.js
View file

@ -1,6 +1,6 @@
const Settings = require('settings-sharelatex')
const { User } = require('../../models/User')
const { db, ObjectId } = require('../../infrastructure/mongojs')
const { db, ObjectId } = require('../../infrastructure/mongodb')
const bcrypt = require('bcrypt')
const EmailHelper = require('../Helpers/EmailHelper')
const V1Handler = require('../V1/V1Handler')
@ -15,7 +15,7 @@ const BCRYPT_MINOR_VERSION = Settings.security.bcryptMinorVersion || 'a'
const _checkWriteResult = function(result, callback) {
// for MongoDB
if (result && result.nModified === 1) {
if (result && result.modifiedCount === 1) {
callback(null, true)
} else {
callback(null, false)
@ -26,7 +26,7 @@ const AuthenticationManager = {
authenticate(query, password, callback) {
// Using Mongoose for legacy reasons here. The returned User instance
// gets serialized into the session and there may be subtle differences
// between the user returned by Mongoose vs mongojs (such as default values)
// between the user returned by Mongoose vs mongodb (such as default values)
User.findOne(query, (error, user) => {
if (error) {
return callback(error)
@ -152,7 +152,7 @@ const AuthenticationManager = {
if (error) {
return callback(error)
}
db.users.update(
db.users.updateOne(
{
_id: ObjectId(userId.toString())
},

13
services/web/app/src/Features/Project/ProjectDeleter.js
View file

@ -1,5 +1,5 @@
const _ = require('lodash')
const { db, ObjectId } = require('../../infrastructure/mongojs')
const { db, ObjectId } = require('../../infrastructure/mongodb')
const { callbackify } = require('util')
const { Project } = require('../../models/Project')
const { DeletedProject } = require('../../models/DeletedProject')
@ -290,16 +290,7 @@ async function undeleteProject(projectId, options = {}) {
// create a new document with an _id already specified. We need to
// insert it directly into the collection
// db.projects.insert doesn't work with promisify
await new Promise((resolve, reject) => {
db.projects.insert(restored, err => {
if (err) {
reject(err)
} else {
resolve()
}
})
})
await db.projects.insertOne(restored)
await DeletedProject.deleteOne({ _id: deletedProject._id }).exec()
}

2
services/web/app/src/Features/Project/ProjectGetter.js
View file

@ -13,7 +13,7 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const { db, ObjectId } = require('../../infrastructure/mongojs')
const { db, ObjectId } = require('../../infrastructure/mongodb')
const OError = require('@overleaf/o-error')
const metrics = require('metrics-sharelatex')
const async = require('async')

13
services/web/app/src/Features/Security/OneTimeTokenHandler.js
View file

@ -14,7 +14,7 @@
const Settings = require('settings-sharelatex')
const crypto = require('crypto')
const logger = require('logger-sharelatex')
const { db } = require('../../infrastructure/mongojs')
const { db } = require('../../infrastructure/mongodb')
const Errors = require('../Errors/Errors')
const ONE_HOUR_IN_S = 60 * 60
@ -36,7 +36,7 @@ module.exports = {
const createdAt = new Date()
const expiresAt = new Date(createdAt.getTime() + expiresIn * 1000)
const token = crypto.randomBytes(32).toString('hex')
return db.tokens.insert(
return db.tokens.insertOne(
{
use,
token,
@ -58,24 +58,23 @@ module.exports = {
callback = function(error, data) {}
}
const now = new Date()
return db.tokens.findAndModify(
return db.tokens.findOneAndUpdate(
{
query: {
use,
token,
expiresAt: { $gt: now },
usedAt: { $exists: false }
},
update: {
{
$set: {
usedAt: now
}
}
},
function(error, token) {
function(error, result) {
if (error != null) {
return callback(error)
}
const token = result.value
if (token == null) {
return callback(new Errors.NotFoundError('no token found'))
}

4
services/web/app/src/Features/Subscription/SubscriptionUpdater.js
View file

@ -1,4 +1,4 @@
const { db, ObjectId } = require('../../infrastructure/mongojs')
const { db, ObjectId } = require('../../infrastructure/mongodb')
const OError = require('@overleaf/o-error')
const async = require('async')
const { promisifyAll } = require('../../util/promises')
@ -190,7 +190,7 @@ const SubscriptionUpdater = {
[
cb =>
// 1. upsert subscription
db.subscriptions.update(
db.subscriptions.updateOne(
{ _id: subscription._id },
subscription,
{ upsert: true },

2
services/web/app/src/Features/User/UserGetter.js
View file

@ -1,4 +1,4 @@
const { db, ObjectId } = require('../../infrastructure/mongojs')
const { db, ObjectId } = require('../../infrastructure/mongodb')
const metrics = require('metrics-sharelatex')
const logger = require('logger-sharelatex')
const { promisifyAll } = require('../../util/promises')

2
services/web/app/src/Features/User/UserOnboardingController.js
View file

@ -1,4 +1,4 @@
const { db, ObjectId } = require('../../infrastructure/mongojs')
const { db, ObjectId } = require('../../infrastructure/mongodb')
const UserUpdater = require('./UserUpdater')
const EmailHandler = require('../Email/EmailHandler')
const logger = require('logger-sharelatex')

4
services/web/app/src/Features/User/UserUpdater.js
View file

@ -1,6 +1,6 @@
const logger = require('logger-sharelatex')
const OError = require('@overleaf/o-error')
const { db, ObjectId } = require('../../infrastructure/mongojs')
const { db, ObjectId } = require('../../infrastructure/mongodb')
const metrics = require('metrics-sharelatex')
const async = require('async')
const { callbackify, promisify } = require('util')
@ -200,7 +200,7 @@ const UserUpdater = {
query._id = ObjectId(query._id)
}
db.users.update(query, update, callback)
db.users.updateOne(query, update, callback)
},
//

82
services/web/app/src/infrastructure/mongodb.js Normal file
View file

@ -0,0 +1,82 @@
const Settings = require('settings-sharelatex')
const { MongoClient, ObjectId } = require('mongodb')
const parseMongoUrl = require('parse-mongo-url')
if (
typeof global.beforeEach === 'function' &&
process.argv.join(' ').match(/unit/)
) {
throw new Error(
'It looks like unit tests are running, but you are connecting to Mongo. Missing a stub?'
)
}
const clientPromise = MongoClient.connect(
Settings.mongo.url,
Settings.mongo.options
)
let setupDbPromise
async function waitForDb() {
if (!setupDbPromise) {
setupDbPromise = setupDb()
}
await setupDbPromise
}
const db = {}
async function setupDb() {
const internalDb = (await clientPromise).db(
// TODO(das7pad): remove after upgrading mongodb
parseMongoUrl(Settings.mongo.url).dbName
)
db.contacts = internalDb.collection('contacts')
db.deletedProjects = internalDb.collection('deletedProjects')
db.deletedSubscriptions = internalDb.collection('deletedSubscriptions')
db.deletedUsers = internalDb.collection('deletedUsers')
db.docHistory = internalDb.collection('docHistory')
db.docHistoryIndex = internalDb.collection('docHistoryIndex')
db.docOps = internalDb.collection('docOps')
db.docSnapshots = internalDb.collection('docSnapshots')
db.docs = internalDb.collection('docs')
db.githubSyncEntityVersions = internalDb.collection(
'githubSyncEntityVersions'
)
db.githubSyncProjectStates = internalDb.collection('githubSyncProjectStates')
db.githubSyncUserCredentials = internalDb.collection(
'githubSyncUserCredentials'
)
db.institutions = internalDb.collection('institutions')
db.messages = internalDb.collection('messages')
db.migrations = internalDb.collection('migrations')
db.notifications = internalDb.collection('notifications')
db.oauthAccessTokens = internalDb.collection('oauthAccessTokens')
db.oauthApplications = internalDb.collection('oauthApplications')
db.oauthAuthorizationCodes = internalDb.collection('oauthAuthorizationCodes')
db.projectHistoryFailures = internalDb.collection('projectHistoryFailures')
db.projectHistoryLabels = internalDb.collection('projectHistoryLabels')
db.projectHistoryMetaData = internalDb.collection('projectHistoryMetaData')
db.projectHistorySyncState = internalDb.collection('projectHistorySyncState')
db.projectInvites = internalDb.collection('projectInvites')
db.projects = internalDb.collection('projects')
db.publishers = internalDb.collection('publishers')
db.rooms = internalDb.collection('rooms')
db.samlCache = internalDb.collection('samlCache')
db.samlLogs = internalDb.collection('samlLogs')
db.spellingPreferences = internalDb.collection('spellingPreferences')
db.subscriptions = internalDb.collection('subscriptions')
db.systemmessages = internalDb.collection('systemmessages')
db.tags = internalDb.collection('tags')
db.teamInvites = internalDb.collection('teamInvites')
db.templates = internalDb.collection('templates')
db.tokens = internalDb.collection('tokens')
db.users = internalDb.collection('users')
db.userstubs = internalDb.collection('userstubs')
}
module.exports = {
db,
ObjectId,
waitForDb
}

5
services/web/config/settings.defaults.coffee
View file

@ -41,6 +41,11 @@ module.exports = settings =
# Databases
# ---------
mongo:
options: {
useUnifiedTopology: (process.env['MONGO_USE_UNIFIED_TOPOLOGY'] || 'true') == 'true',
poolSize: parseInt(process.env['MONGO_POOL_SIZE'], 10) || 10,
socketTimeoutMS: parseInt(process.env['MONGO_SOCKET_TIMEOUT'], 10) || 30000,
},
url : process.env['MONGO_CONNECTION_STRING'] || process.env['MONGO_URL'] || "mongodb://#{process.env['MONGO_HOST'] or '127.0.0.1'}/sharelatex"
poolSize: parseInt(process.env['MONGO_POOL_SIZE'], 10) || 10
socketTimeoutMS: parseInt(process.env['MONGO_SOCKET_TIMEOUT'], 10) || 30000

3
services/web/test/acceptance/src/Init.js
View file

@ -1,9 +1,12 @@
const App = require('../../../app.js')
const { exec } = require('child_process')
const { waitForDb } = require('../../../app/src/infrastructure/mongodb')
const { db } = require('../../../app/src/infrastructure/mongojs')
require('logger-sharelatex').logger.level('error')
before(waitForDb)
before(function(done) {
exec('bin/east migrate', (error, stdout, stderr) => {
console.log(stdout)

12
services/web/test/unit/src/Authentication/AuthenticationManagerTests.js
View file

@ -20,7 +20,7 @@ describe('AuthenticationManager', function() {
'../../models/User': {
User: (this.User = {})
},
'../../infrastructure/mongojs': {
'../../infrastructure/mongodb': {
db: (this.db = { users: {} }),
ObjectId
},
@ -99,9 +99,9 @@ describe('AuthenticationManager', function() {
_id: '5c8791477192a80b5e76ca7e',
email: (this.email = 'USER@sharelatex.com')
}
this.db.users.update = sinon
this.db.users.updateOne = sinon
.stub()
.callsArgWith(2, null, { nModified: 1 })
.callsArgWith(2, null, { modifiedCount: 1 })
})
it('should not produce an error', function(done) {
@ -124,7 +124,7 @@ describe('AuthenticationManager', function() {
expect(err).to.not.exist
const {
hashedPassword
} = this.db.users.update.lastCall.args[1].$set
} = this.db.users.updateOne.lastCall.args[1].$set
expect(hashedPassword).to.exist
expect(hashedPassword.length).to.equal(60)
expect(hashedPassword).to.match(/^\$2a\$12\$[a-zA-Z0-9/.]{53}$/)
@ -530,7 +530,7 @@ describe('AuthenticationManager', function() {
this.salt = 'saltaasdfasdfasdf'
this.bcrypt.genSalt = sinon.stub().callsArgWith(2, null, this.salt)
this.bcrypt.hash = sinon.stub().callsArgWith(2, null, this.hashedPassword)
this.db.users.update = sinon.stub().callsArg(2)
this.db.users.updateOne = sinon.stub().callsArg(2)
})
describe('too long', function() {
@ -613,7 +613,7 @@ describe('AuthenticationManager', function() {
})
it("should update the user's password in the database", function() {
const { args } = this.db.users.update.lastCall
const { args } = this.db.users.updateOne.lastCall
expect(args[0]).to.deep.equal({
_id: ObjectId(this.user_id.toString())
})

8
services/web/test/unit/src/Project/ProjectDeleterTests.js
View file

@ -106,7 +106,7 @@ describe('ProjectDeleter', function() {
this.db = {
projects: {
insert: sinon.stub().yields()
insertOne: sinon.stub().resolves()
}
}
@ -143,7 +143,7 @@ describe('ProjectDeleter', function() {
'../Collaborators/CollaboratorsGetter': this.CollaboratorsGetter,
'../Docstore/DocstoreManager': this.DocstoreManager,
'./ProjectDetailsHandler': this.ProjectDetailsHandler,
'../../infrastructure/mongojs': { db: this.db, ObjectId },
'../../infrastructure/mongodb': { db: this.db, ObjectId },
'../History/HistoryManager': this.HistoryManager,
'logger-sharelatex': this.logger,
'../Errors/Errors': Errors
@ -662,7 +662,7 @@ describe('ProjectDeleter', function() {
it('should insert the project into the collection', async function() {
await this.ProjectDeleter.promises.undeleteProject(this.project._id)
sinon.assert.calledWith(
this.db.projects.insert,
this.db.projects.insertOne,
sinon.match({
_id: this.project._id,
name: this.project.name
@ -674,7 +674,7 @@ describe('ProjectDeleter', function() {
this.project.archived = true
await this.ProjectDeleter.promises.undeleteProject(this.project._id)
sinon.assert.calledWith(
this.db.projects.insert,
this.db.projects.insertOne,
sinon.match({ archived: undefined })
)
})

2
services/web/test/unit/src/Project/ProjectGetterTests.js
View file

@ -32,7 +32,7 @@ describe('ProjectGetter', function() {
console: console
},
requires: {
'../../infrastructure/mongojs': {
'../../infrastructure/mongodb': {
db: (this.db = {
projects: {},
users: {}

26
services/web/test/unit/src/Security/OneTimeTokenHandlerTests.js
View file

@ -41,7 +41,7 @@ describe('OneTimeTokenHandler', function() {
randomBytes: () => this.stubbedToken
},
'../Errors/Errors': Errors,
'../../infrastructure/mongojs': {
'../../infrastructure/mongodb': {
db: (this.db = { tokens: {} })
}
}
@ -54,7 +54,7 @@ describe('OneTimeTokenHandler', function() {
describe('getNewToken', function() {
beforeEach(function() {
return (this.db.tokens.insert = sinon.stub().yields())
return (this.db.tokens.insertOne = sinon.stub().yields())
})
describe('normally', function() {
@ -67,7 +67,7 @@ describe('OneTimeTokenHandler', function() {
})
it('should insert a generated token with a 1 hour expiry', function() {
return this.db.tokens.insert
return this.db.tokens.insertOne
.calledWith({
use: 'password',
token: this.stubbedToken,
@ -96,7 +96,7 @@ describe('OneTimeTokenHandler', function() {
})
it('should insert a generated token with a custom expiry', function() {
return this.db.tokens.insert
return this.db.tokens.insertOne
.calledWith({
use: 'password',
token: this.stubbedToken,
@ -118,9 +118,9 @@ describe('OneTimeTokenHandler', function() {
describe('getValueFromTokenAndExpire', function() {
describe('successfully', function() {
beforeEach(function() {
this.db.tokens.findAndModify = sinon
this.db.tokens.findOneAndUpdate = sinon
.stub()
.yields(null, { data: 'mock-data' })
.yields(null, { value: { data: 'mock-data' } })
return this.OneTimeTokenHandler.getValueFromTokenAndExpire(
'password',
'mock-token',
@ -129,18 +129,18 @@ describe('OneTimeTokenHandler', function() {
})
it('should expire the token', function() {
return this.db.tokens.findAndModify
.calledWith({
query: {
return this.db.tokens.findOneAndUpdate
.calledWith(
{
use: 'password',
token: 'mock-token',
expiresAt: { $gt: new Date() },
usedAt: { $exists: false }
},
update: {
{
$set: { usedAt: new Date() }
}
})
)
.should.equal(true)
})
@ -151,7 +151,9 @@ describe('OneTimeTokenHandler', function() {
describe('when a valid token is not found', function() {
beforeEach(function() {
this.db.tokens.findAndModify = sinon.stub().yields(null, null)
this.db.tokens.findOneAndUpdate = sinon
.stub()
.yields(null, { value: null })
return this.OneTimeTokenHandler.getValueFromTokenAndExpire(
'password',
'mock-token',

2
services/web/test/unit/src/Subscription/SubscriptionUpdaterTests.js
View file

@ -110,7 +110,7 @@ describe('SubscriptionUpdater', function() {
warn() {}
},
'settings-sharelatex': this.Settings,
'../../infrastructure/mongojs': { db: {}, ObjectId },
'../../infrastructure/mongodb': { db: {}, ObjectId },
'./FeaturesUpdater': this.FeaturesUpdater,
'../../models/DeletedSubscription': {
DeletedSubscription: this.DeletedSubscription

4
services/web/test/unit/src/Tags/TagsHandlerTests.js
View file

@ -18,12 +18,12 @@ describe('TagsHandler', function() {
this.tagId = ObjectId().toString()
this.projectId = ObjectId().toString()
this.mongojs = { ObjectId: ObjectId }
this.mongodb = { ObjectId: ObjectId }
this.TagMock = sinon.mock(Tag)
this.TagsHandler = SandboxedModule.require(modulePath, {
requires: {
'../../infrastructure/mongojs': this.mongojs,
'../../infrastructure/mongodb': this.mongodb,
'../../models/Tag': { Tag: Tag }
}
})

2
services/web/test/unit/src/User/UserGetterTests.js
View file

@ -48,7 +48,7 @@ describe('UserGetter', function() {
'logger-sharelatex': {
log() {}
},
'../../infrastructure/mongojs': this.Mongo,
'../../infrastructure/mongodb': this.Mongo,
'metrics-sharelatex': {
timeAsyncMethod: sinon.stub()
},

6
services/web/test/unit/src/User/UserOnboardingControllerTests.js
View file

@ -21,7 +21,7 @@ describe('UserOnboardingController', function() {
}
]
this.mongojs = {
this.mongodb = {
db: {
users: {
find: sinon
@ -46,7 +46,7 @@ describe('UserOnboardingController', function() {
this.UserOnboardingController = SandboxedModule.require(modulePath, {
requires: {
'../../infrastructure/mongojs': this.mongojs,
'../../infrastructure/mongodb': this.mongodb,
'./UserUpdater': this.UserUpdater,
'../Email/EmailHandler': this.EmailHandler,
'logger-sharelatex': this.logger
@ -61,7 +61,7 @@ describe('UserOnboardingController', function() {
it('sends onboarding emails', function(done) {
this.res.send = ids => {
ids.length.should.equal(3)
this.mongojs.db.users.find.calledOnce.should.equal(true)
this.mongodb.db.users.find.calledOnce.should.equal(true)
this.EmailHandler.sendEmail.calledThrice.should.equal(true)
this.UserUpdater.updateUser.calledThrice.should.equal(true)
for (var i = 0; i < 3; i++) {

4
services/web/test/unit/src/User/UserUpdaterTests.js
View file

@ -12,7 +12,7 @@ const { expect } = require('chai')
describe('UserUpdater', function() {
beforeEach(function() {
tk.freeze(Date.now())
this.mongojs = {
this.mongodb = {
db: {},
ObjectId(id) {
return id
@ -50,7 +50,7 @@ describe('UserUpdater', function() {
},
requires: {
'logger-sharelatex': this.logger,
'../../infrastructure/mongojs': this.mongojs,
'../../infrastructure/mongodb': this.mongodb,
'metrics-sharelatex': {
timeAsyncMethod: sinon.stub()
},