github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #10911 from overleaf/em-upgrade-mongo-docstore

Browse source 
Upgrade the Mongo driver in docstore

GitOrigin-RevId: 35e5de558e7900fcb455f1044703912077273d47
This commit is contained in:
Eric Mc Sween 2022-12-15 07:53:03 -05:00 committed by Copybot
parent 1867584bf6
commit 1f566c3556
11 changed files with 129 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

119
package-lock.json generated
View file

@ -36209,11 +36209,11 @@
"bunyan": "^1.8.15",
"celebrate": "^13.0.4",
"express": "^4.17.1",
"mongodb": "^3.6.0",
"lodash": "^4.17.21",
"mongodb": "^4.11.0",
"p-map": "^4.0.0",
"request": "^2.88.2",
"streamifier": "^0.1.1",
"underscore": "~1.12.1"
"streamifier": "^0.1.1"
},
"devDependencies": {
"@google-cloud/storage": "^5.1.2",
@ -36258,6 +36258,29 @@
"node": ">=10"
}
},
"services/docstore/node_modules/buffer": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"dependencies": {
"base64-js": "^1.3.1",
"ieee754": "^1.1.13"
}
},
"services/docstore/node_modules/celebrate": {
"version": "13.0.4",
"resolved": "https://registry.npmjs.org/celebrate/-/celebrate-13.0.4.tgz",
@ -36274,6 +36297,14 @@
"integrity": "sha512-X/b2gM7e8zQ7siiE0DhRLeNMpuCkIqec5Jnx4GMk/HWB71a6e5Lz48mH9/GIS/hpLsBRFZfMF1gjXBkY0vq5oA==",
"dev": true
},
"services/docstore/node_modules/denque": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
"engines": {
"node": ">=0.10"
}
},
"services/docstore/node_modules/diff": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
@ -36335,6 +36366,35 @@
"node": ">=10.0.0"
}
},
"services/docstore/node_modules/mongodb": {
"version": "4.11.0",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz",
"integrity": "sha512-9l9n4Nk2BYZzljW3vHah3Z0rfS5npKw6ktnkmFgTcnzaXH1DRm3pDl6VMHu84EVb1lzmSaJC4OzWZqTkB5i2wg==",
"dependencies": {
"bson": "^4.7.0",
"denque": "^2.1.0",
"mongodb-connection-string-url": "^2.5.4",
"socks": "^2.7.1"
},
"engines": {
"node": ">=12.9.0"
},
"optionalDependencies": {
"@aws-sdk/credential-providers": "^3.186.0",
"saslprep": "^1.0.3"
}
},
"services/docstore/node_modules/mongodb/node_modules/bson": {
"version": "4.7.0",
"resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz",
"integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==",
"dependencies": {
"buffer": "^5.6.0"
},
"engines": {
"node": ">=6.9.0"
}
},
"services/docstore/node_modules/sinon": {
"version": "9.0.3",
"resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.3.tgz",
@ -36366,11 +36426,6 @@
"node": ">=8"
}
},
"services/docstore/node_modules/underscore": {
"version": "1.12.1",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz",
"integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw=="
},
"services/document-updater": {
"name": "@overleaf/document-updater",
"dependencies": {
@ -46560,15 +46615,15 @@
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"express": "^4.17.1",
"lodash": "^4.17.21",
"mocha": "^8.4.0",
"mongodb": "^3.6.0",
"mongodb": "^4.11.0",
"p-map": "^4.0.0",
"request": "^2.88.2",
"sandboxed-module": "~2.0.4",
"sinon": "~9.0.2",
"sinon-chai": "^3.7.0",
"streamifier": "^0.1.1",
"underscore": "~1.12.1"
"streamifier": "^0.1.1"
},
"dependencies": {
"@google-cloud/storage": {
@ -46601,6 +46656,15 @@
"xdg-basedir": "^4.0.0"
}
},
"buffer": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
"requires": {
"base64-js": "^1.3.1",
"ieee754": "^1.1.13"
}
},
"celebrate": {
"version": "13.0.4",
"resolved": "https://registry.npmjs.org/celebrate/-/celebrate-13.0.4.tgz",
@ -46617,6 +46681,11 @@
"integrity": "sha512-X/b2gM7e8zQ7siiE0DhRLeNMpuCkIqec5Jnx4GMk/HWB71a6e5Lz48mH9/GIS/hpLsBRFZfMF1gjXBkY0vq5oA==",
"dev": true
},
"denque": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="
},
"diff": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
@ -46660,6 +46729,29 @@
"integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==",
"dev": true
},
"mongodb": {
"version": "4.11.0",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz",
"integrity": "sha512-9l9n4Nk2BYZzljW3vHah3Z0rfS5npKw6ktnkmFgTcnzaXH1DRm3pDl6VMHu84EVb1lzmSaJC4OzWZqTkB5i2wg==",
"requires": {
"@aws-sdk/credential-providers": "^3.186.0",
"bson": "^4.7.0",
"denque": "^2.1.0",
"mongodb-connection-string-url": "^2.5.4",
"saslprep": "^1.0.3",
"socks": "^2.7.1"
},
"dependencies": {
"bson": {
"version": "4.7.0",
"resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz",
"integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==",
"requires": {
"buffer": "^5.6.0"
}
}
}
},
"sinon": {
"version": "9.0.3",
"resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.3.tgz",
@ -46683,11 +46775,6 @@
"requires": {
"has-flag": "^4.0.0"
}
},
"underscore": {
"version": "1.12.1",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz",
"integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw=="
}
}
},

6
services/docstore/app.js
View file

@ -15,7 +15,7 @@ const {
Joi,
errors: handleValidationErrors,
} = require('celebrate')
const mongodb = require('./app/js/mongodb')
const { mongoClient } = require('./app/js/mongodb')
const Errors = require('./app/js/Errors')
const HttpController = require('./app/js/HttpController')
@ -101,8 +101,8 @@ const { host } = Settings.internal.docstore
if (!module.parent) {
// Called directly
mongodb
.waitForDb()
mongoClient
.connect()
.then(() => {
const server = app.listen(port, host, function (err) {
if (err) {

4
services/docstore/app/js/DocManager.js
View file

@ -15,7 +15,7 @@ let DocManager
const MongoManager = require('./MongoManager')
const Errors = require('./Errors')
const logger = require('@overleaf/logger')
const _ = require('underscore')
const _ = require('lodash')
const DocArchive = require('./DocArchiveManager')
const RangeManager = require('./RangeManager')
const Settings = require('@overleaf/settings')
@ -157,7 +157,7 @@ module.exports = DocManager = {
)
return callback(err)
}
doc = _.extend(doc, archivedDoc)
Object.assign(doc, archivedDoc)
callback(null, doc)
})
}

2
services/docstore/app/js/HealthChecker.js
View file

@ -12,7 +12,7 @@
const { db, ObjectId } = require('./mongodb')
const request = require('request')
const async = require('async')
const _ = require('underscore')
const _ = require('lodash')
const crypto = require('crypto')
const settings = require('@overleaf/settings')
const { port } = settings.internal.docstore

2
services/docstore/app/js/RangeManager.js
View file

@ -12,7 +12,7 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let RangeManager
const _ = require('underscore')
const _ = require('lodash')
const { ObjectId } = require('./mongodb')
module.exports = RangeManager = {

32
services/docstore/app/js/mongodb.js
View file

@ -1,36 +1,16 @@
const Settings = require('@overleaf/settings')
const { MongoClient, ObjectId } = require('mongodb')
const clientPromise = MongoClient.connect(
Settings.mongo.url,
Settings.mongo.options
)
const mongoClient = new MongoClient(Settings.mongo.url)
const mongoDb = mongoClient.db()
let setupDbPromise
async function waitForDb() {
if (!setupDbPromise) {
setupDbPromise = setupDb()
}
await setupDbPromise
}
const db = {}
async function setupDb() {
const internalDb = (await clientPromise).db()
db.docs = internalDb.collection('docs')
db.docOps = internalDb.collection('docOps')
}
async function addCollection(name) {
await waitForDb()
const internalDb = (await clientPromise).db()
db[name] = internalDb.collection(name)
const db = {
docs: mongoDb.collection('docs'),
docOps: mongoDb.collection('docOps'),
}
module.exports = {
db,
mongoClient,
ObjectId,
addCollection,
waitForDb,
}

7
services/docstore/config/settings.defaults.js
View file

@ -9,12 +9,7 @@ const Settings = {
},
},
mongo: {
options: {
useUnifiedTopology:
(process.env.MONGO_USE_UNIFIED_TOPOLOGY || 'true') === 'true',
},
},
mongo: {},
docstore: {
archiveOnSoftDelete: process.env.ARCHIVE_ON_SOFT_DELETE === 'true',

6
services/docstore/package.json
View file

@ -27,11 +27,11 @@
"bunyan": "^1.8.15",
"celebrate": "^13.0.4",
"express": "^4.17.1",
"mongodb": "^3.6.0",
"lodash": "^4.17.21",
"mongodb": "^4.11.0",
"p-map": "^4.0.0",
"request": "^2.88.2",
"streamifier": "^0.1.1",
"underscore": "~1.12.1"
"streamifier": "^0.1.1"
},
"devDependencies": {
"@google-cloud/storage": "^5.1.2",

26
services/docstore/test/acceptance/js/helpers/DocstoreApp.js
View file

@ -1,5 +1,4 @@
const app = require('../../../../app')
const { waitForDb } = require('../../../../app/js/mongodb')
require('@overleaf/logger').logger.level('error')
const settings = require('@overleaf/settings')
@ -8,9 +7,6 @@ module.exports = {
initing: false,
callbacks: [],
ensureRunning(callback) {
if (callback == null) {
callback = function () {}
}
if (this.running) {
return callback()
} else if (this.initing) {
@ -18,20 +14,14 @@ module.exports = {
}
this.initing = true
this.callbacks.push(callback)
waitForDb().then(() => {
return app.listen(settings.internal.docstore.port, 'localhost', error => {
if (error != null) {
throw error
}
this.running = true
return (() => {
const result = []
for (callback of Array.from(this.callbacks)) {
result.push(callback())
}
return result
})()
})
app.listen(settings.internal.docstore.port, 'localhost', error => {
if (error != null) {
throw error
}
this.running = true
for (callback of Array.from(this.callbacks)) {
callback()
}
})
},
}

2
services/docstore/test/setup.js
View file

@ -30,7 +30,7 @@ SandboxedModule.configure({
requires: {
'@overleaf/logger': stubs.logger,
},
globals: { Buffer, JSON, console, process },
globals: { Buffer, JSON, Math, console, process },
})
exports.mochaHooks = {

1
services/docstore/test/unit/js/RangeManagerTests.js
View file

@ -18,7 +18,6 @@ const modulePath = require('path').join(
'../../../app/js/RangeManager'
)
const { ObjectId } = require('mongodb')
const _ = require('underscore')
describe('RangeManager', function () {
beforeEach(function () {