Merge pull request #10911 from overleaf/em-upgrade-mongo-docstore

Upgrade the Mongo driver in docstore

GitOrigin-RevId: 35e5de558e7900fcb455f1044703912077273d47
This commit is contained in:
Eric Mc Sween 2022-12-15 07:53:03 -05:00 committed by Copybot
parent 1867584bf6
commit 1f566c3556
11 changed files with 129 additions and 78 deletions

119
package-lock.json generated
View file

@ -36209,11 +36209,11 @@
"bunyan": "^1.8.15",
"celebrate": "^13.0.4",
"express": "^4.17.1",
"mongodb": "^3.6.0",
"lodash": "^4.17.21",
"mongodb": "^4.11.0",
"p-map": "^4.0.0",
"request": "^2.88.2",
"streamifier": "^0.1.1",
"underscore": "~1.12.1"
"streamifier": "^0.1.1"
},
"devDependencies": {
"@google-cloud/storage": "^5.1.2",
@ -36258,6 +36258,29 @@
"node": ">=10"
}
},
"services/docstore/node_modules/buffer": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"dependencies": {
"base64-js": "^1.3.1",
"ieee754": "^1.1.13"
}
},
"services/docstore/node_modules/celebrate": {
"version": "13.0.4",
"resolved": "https://registry.npmjs.org/celebrate/-/celebrate-13.0.4.tgz",
@ -36274,6 +36297,14 @@
"integrity": "sha512-X/b2gM7e8zQ7siiE0DhRLeNMpuCkIqec5Jnx4GMk/HWB71a6e5Lz48mH9/GIS/hpLsBRFZfMF1gjXBkY0vq5oA==",
"dev": true
},
"services/docstore/node_modules/denque": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
"engines": {
"node": ">=0.10"
}
},
"services/docstore/node_modules/diff": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
@ -36335,6 +36366,35 @@
"node": ">=10.0.0"
}
},
"services/docstore/node_modules/mongodb": {
"version": "4.11.0",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz",
"integrity": "sha512-9l9n4Nk2BYZzljW3vHah3Z0rfS5npKw6ktnkmFgTcnzaXH1DRm3pDl6VMHu84EVb1lzmSaJC4OzWZqTkB5i2wg==",
"dependencies": {
"bson": "^4.7.0",
"denque": "^2.1.0",
"mongodb-connection-string-url": "^2.5.4",
"socks": "^2.7.1"
},
"engines": {
"node": ">=12.9.0"
},
"optionalDependencies": {
"@aws-sdk/credential-providers": "^3.186.0",
"saslprep": "^1.0.3"
}
},
"services/docstore/node_modules/mongodb/node_modules/bson": {
"version": "4.7.0",
"resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz",
"integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==",
"dependencies": {
"buffer": "^5.6.0"
},
"engines": {
"node": ">=6.9.0"
}
},
"services/docstore/node_modules/sinon": {
"version": "9.0.3",
"resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.3.tgz",
@ -36366,11 +36426,6 @@
"node": ">=8"
}
},
"services/docstore/node_modules/underscore": {
"version": "1.12.1",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz",
"integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw=="
},
"services/document-updater": {
"name": "@overleaf/document-updater",
"dependencies": {
@ -46560,15 +46615,15 @@
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"express": "^4.17.1",
"lodash": "^4.17.21",
"mocha": "^8.4.0",
"mongodb": "^3.6.0",
"mongodb": "^4.11.0",
"p-map": "^4.0.0",
"request": "^2.88.2",
"sandboxed-module": "~2.0.4",
"sinon": "~9.0.2",
"sinon-chai": "^3.7.0",
"streamifier": "^0.1.1",
"underscore": "~1.12.1"
"streamifier": "^0.1.1"
},
"dependencies": {
"@google-cloud/storage": {
@ -46601,6 +46656,15 @@
"xdg-basedir": "^4.0.0"
}
},
"buffer": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
"requires": {
"base64-js": "^1.3.1",
"ieee754": "^1.1.13"
}
},
"celebrate": {
"version": "13.0.4",
"resolved": "https://registry.npmjs.org/celebrate/-/celebrate-13.0.4.tgz",
@ -46617,6 +46681,11 @@
"integrity": "sha512-X/b2gM7e8zQ7siiE0DhRLeNMpuCkIqec5Jnx4GMk/HWB71a6e5Lz48mH9/GIS/hpLsBRFZfMF1gjXBkY0vq5oA==",
"dev": true
},
"denque": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="
},
"diff": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
@ -46660,6 +46729,29 @@
"integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==",
"dev": true
},
"mongodb": {
"version": "4.11.0",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz",
"integrity": "sha512-9l9n4Nk2BYZzljW3vHah3Z0rfS5npKw6ktnkmFgTcnzaXH1DRm3pDl6VMHu84EVb1lzmSaJC4OzWZqTkB5i2wg==",
"requires": {
"@aws-sdk/credential-providers": "^3.186.0",
"bson": "^4.7.0",
"denque": "^2.1.0",
"mongodb-connection-string-url": "^2.5.4",
"saslprep": "^1.0.3",
"socks": "^2.7.1"
},
"dependencies": {
"bson": {
"version": "4.7.0",
"resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz",
"integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==",
"requires": {
"buffer": "^5.6.0"
}
}
}
},
"sinon": {
"version": "9.0.3",
"resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.3.tgz",
@ -46683,11 +46775,6 @@
"requires": {
"has-flag": "^4.0.0"
}
},
"underscore": {
"version": "1.12.1",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz",
"integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw=="
}
}
},

View file

@ -15,7 +15,7 @@ const {
Joi,
errors: handleValidationErrors,
} = require('celebrate')
const mongodb = require('./app/js/mongodb')
const { mongoClient } = require('./app/js/mongodb')
const Errors = require('./app/js/Errors')
const HttpController = require('./app/js/HttpController')
@ -101,8 +101,8 @@ const { host } = Settings.internal.docstore
if (!module.parent) {
// Called directly
mongodb
.waitForDb()
mongoClient
.connect()
.then(() => {
const server = app.listen(port, host, function (err) {
if (err) {

View file

@ -15,7 +15,7 @@ let DocManager
const MongoManager = require('./MongoManager')
const Errors = require('./Errors')
const logger = require('@overleaf/logger')
const _ = require('underscore')
const _ = require('lodash')
const DocArchive = require('./DocArchiveManager')
const RangeManager = require('./RangeManager')
const Settings = require('@overleaf/settings')
@ -157,7 +157,7 @@ module.exports = DocManager = {
)
return callback(err)
}
doc = _.extend(doc, archivedDoc)
Object.assign(doc, archivedDoc)
callback(null, doc)
})
}

View file

@ -12,7 +12,7 @@
const { db, ObjectId } = require('./mongodb')
const request = require('request')
const async = require('async')
const _ = require('underscore')
const _ = require('lodash')
const crypto = require('crypto')
const settings = require('@overleaf/settings')
const { port } = settings.internal.docstore

View file

@ -12,7 +12,7 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let RangeManager
const _ = require('underscore')
const _ = require('lodash')
const { ObjectId } = require('./mongodb')
module.exports = RangeManager = {

View file

@ -1,36 +1,16 @@
const Settings = require('@overleaf/settings')
const { MongoClient, ObjectId } = require('mongodb')
const clientPromise = MongoClient.connect(
Settings.mongo.url,
Settings.mongo.options
)
const mongoClient = new MongoClient(Settings.mongo.url)
const mongoDb = mongoClient.db()
let setupDbPromise
async function waitForDb() {
if (!setupDbPromise) {
setupDbPromise = setupDb()
}
await setupDbPromise
}
const db = {}
async function setupDb() {
const internalDb = (await clientPromise).db()
db.docs = internalDb.collection('docs')
db.docOps = internalDb.collection('docOps')
}
async function addCollection(name) {
await waitForDb()
const internalDb = (await clientPromise).db()
db[name] = internalDb.collection(name)
const db = {
docs: mongoDb.collection('docs'),
docOps: mongoDb.collection('docOps'),
}
module.exports = {
db,
mongoClient,
ObjectId,
addCollection,
waitForDb,
}

View file

@ -9,12 +9,7 @@ const Settings = {
},
},
mongo: {
options: {
useUnifiedTopology:
(process.env.MONGO_USE_UNIFIED_TOPOLOGY || 'true') === 'true',
},
},
mongo: {},
docstore: {
archiveOnSoftDelete: process.env.ARCHIVE_ON_SOFT_DELETE === 'true',

View file

@ -27,11 +27,11 @@
"bunyan": "^1.8.15",
"celebrate": "^13.0.4",
"express": "^4.17.1",
"mongodb": "^3.6.0",
"lodash": "^4.17.21",
"mongodb": "^4.11.0",
"p-map": "^4.0.0",
"request": "^2.88.2",
"streamifier": "^0.1.1",
"underscore": "~1.12.1"
"streamifier": "^0.1.1"
},
"devDependencies": {
"@google-cloud/storage": "^5.1.2",

View file

@ -1,5 +1,4 @@
const app = require('../../../../app')
const { waitForDb } = require('../../../../app/js/mongodb')
require('@overleaf/logger').logger.level('error')
const settings = require('@overleaf/settings')
@ -8,9 +7,6 @@ module.exports = {
initing: false,
callbacks: [],
ensureRunning(callback) {
if (callback == null) {
callback = function () {}
}
if (this.running) {
return callback()
} else if (this.initing) {
@ -18,20 +14,14 @@ module.exports = {
}
this.initing = true
this.callbacks.push(callback)
waitForDb().then(() => {
return app.listen(settings.internal.docstore.port, 'localhost', error => {
if (error != null) {
throw error
}
this.running = true
return (() => {
const result = []
for (callback of Array.from(this.callbacks)) {
result.push(callback())
}
return result
})()
})
app.listen(settings.internal.docstore.port, 'localhost', error => {
if (error != null) {
throw error
}
this.running = true
for (callback of Array.from(this.callbacks)) {
callback()
}
})
},
}

View file

@ -30,7 +30,7 @@ SandboxedModule.configure({
requires: {
'@overleaf/logger': stubs.logger,
},
globals: { Buffer, JSON, console, process },
globals: { Buffer, JSON, Math, console, process },
})
exports.mochaHooks = {

View file

@ -18,7 +18,6 @@ const modulePath = require('path').join(
'../../../app/js/RangeManager'
)
const { ObjectId } = require('mongodb')
const _ = require('underscore')
describe('RangeManager', function () {
beforeEach(function () {