mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-07 20:31:06 -05:00
Merge pull request #10911 from overleaf/em-upgrade-mongo-docstore
Upgrade the Mongo driver in docstore GitOrigin-RevId: 35e5de558e7900fcb455f1044703912077273d47
This commit is contained in:
parent
1867584bf6
commit
1f566c3556
11 changed files with 129 additions and 78 deletions
119
package-lock.json
generated
119
package-lock.json
generated
|
@ -36209,11 +36209,11 @@
|
|||
"bunyan": "^1.8.15",
|
||||
"celebrate": "^13.0.4",
|
||||
"express": "^4.17.1",
|
||||
"mongodb": "^3.6.0",
|
||||
"lodash": "^4.17.21",
|
||||
"mongodb": "^4.11.0",
|
||||
"p-map": "^4.0.0",
|
||||
"request": "^2.88.2",
|
||||
"streamifier": "^0.1.1",
|
||||
"underscore": "~1.12.1"
|
||||
"streamifier": "^0.1.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@google-cloud/storage": "^5.1.2",
|
||||
|
@ -36258,6 +36258,29 @@
|
|||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"services/docstore/node_modules/buffer": {
|
||||
"version": "5.7.1",
|
||||
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
|
||||
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/feross"
|
||||
},
|
||||
{
|
||||
"type": "patreon",
|
||||
"url": "https://www.patreon.com/feross"
|
||||
},
|
||||
{
|
||||
"type": "consulting",
|
||||
"url": "https://feross.org/support"
|
||||
}
|
||||
],
|
||||
"dependencies": {
|
||||
"base64-js": "^1.3.1",
|
||||
"ieee754": "^1.1.13"
|
||||
}
|
||||
},
|
||||
"services/docstore/node_modules/celebrate": {
|
||||
"version": "13.0.4",
|
||||
"resolved": "https://registry.npmjs.org/celebrate/-/celebrate-13.0.4.tgz",
|
||||
|
@ -36274,6 +36297,14 @@
|
|||
"integrity": "sha512-X/b2gM7e8zQ7siiE0DhRLeNMpuCkIqec5Jnx4GMk/HWB71a6e5Lz48mH9/GIS/hpLsBRFZfMF1gjXBkY0vq5oA==",
|
||||
"dev": true
|
||||
},
|
||||
"services/docstore/node_modules/denque": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
|
||||
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
|
||||
"engines": {
|
||||
"node": ">=0.10"
|
||||
}
|
||||
},
|
||||
"services/docstore/node_modules/diff": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
|
||||
|
@ -36335,6 +36366,35 @@
|
|||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"services/docstore/node_modules/mongodb": {
|
||||
"version": "4.11.0",
|
||||
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz",
|
||||
"integrity": "sha512-9l9n4Nk2BYZzljW3vHah3Z0rfS5npKw6ktnkmFgTcnzaXH1DRm3pDl6VMHu84EVb1lzmSaJC4OzWZqTkB5i2wg==",
|
||||
"dependencies": {
|
||||
"bson": "^4.7.0",
|
||||
"denque": "^2.1.0",
|
||||
"mongodb-connection-string-url": "^2.5.4",
|
||||
"socks": "^2.7.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.9.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@aws-sdk/credential-providers": "^3.186.0",
|
||||
"saslprep": "^1.0.3"
|
||||
}
|
||||
},
|
||||
"services/docstore/node_modules/mongodb/node_modules/bson": {
|
||||
"version": "4.7.0",
|
||||
"resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz",
|
||||
"integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==",
|
||||
"dependencies": {
|
||||
"buffer": "^5.6.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"services/docstore/node_modules/sinon": {
|
||||
"version": "9.0.3",
|
||||
"resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.3.tgz",
|
||||
|
@ -36366,11 +36426,6 @@
|
|||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"services/docstore/node_modules/underscore": {
|
||||
"version": "1.12.1",
|
||||
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz",
|
||||
"integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw=="
|
||||
},
|
||||
"services/document-updater": {
|
||||
"name": "@overleaf/document-updater",
|
||||
"dependencies": {
|
||||
|
@ -46560,15 +46615,15 @@
|
|||
"chai": "^4.3.6",
|
||||
"chai-as-promised": "^7.1.1",
|
||||
"express": "^4.17.1",
|
||||
"lodash": "^4.17.21",
|
||||
"mocha": "^8.4.0",
|
||||
"mongodb": "^3.6.0",
|
||||
"mongodb": "^4.11.0",
|
||||
"p-map": "^4.0.0",
|
||||
"request": "^2.88.2",
|
||||
"sandboxed-module": "~2.0.4",
|
||||
"sinon": "~9.0.2",
|
||||
"sinon-chai": "^3.7.0",
|
||||
"streamifier": "^0.1.1",
|
||||
"underscore": "~1.12.1"
|
||||
"streamifier": "^0.1.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"@google-cloud/storage": {
|
||||
|
@ -46601,6 +46656,15 @@
|
|||
"xdg-basedir": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"buffer": {
|
||||
"version": "5.7.1",
|
||||
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
|
||||
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
|
||||
"requires": {
|
||||
"base64-js": "^1.3.1",
|
||||
"ieee754": "^1.1.13"
|
||||
}
|
||||
},
|
||||
"celebrate": {
|
||||
"version": "13.0.4",
|
||||
"resolved": "https://registry.npmjs.org/celebrate/-/celebrate-13.0.4.tgz",
|
||||
|
@ -46617,6 +46681,11 @@
|
|||
"integrity": "sha512-X/b2gM7e8zQ7siiE0DhRLeNMpuCkIqec5Jnx4GMk/HWB71a6e5Lz48mH9/GIS/hpLsBRFZfMF1gjXBkY0vq5oA==",
|
||||
"dev": true
|
||||
},
|
||||
"denque": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
|
||||
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="
|
||||
},
|
||||
"diff": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
|
||||
|
@ -46660,6 +46729,29 @@
|
|||
"integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==",
|
||||
"dev": true
|
||||
},
|
||||
"mongodb": {
|
||||
"version": "4.11.0",
|
||||
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz",
|
||||
"integrity": "sha512-9l9n4Nk2BYZzljW3vHah3Z0rfS5npKw6ktnkmFgTcnzaXH1DRm3pDl6VMHu84EVb1lzmSaJC4OzWZqTkB5i2wg==",
|
||||
"requires": {
|
||||
"@aws-sdk/credential-providers": "^3.186.0",
|
||||
"bson": "^4.7.0",
|
||||
"denque": "^2.1.0",
|
||||
"mongodb-connection-string-url": "^2.5.4",
|
||||
"saslprep": "^1.0.3",
|
||||
"socks": "^2.7.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"bson": {
|
||||
"version": "4.7.0",
|
||||
"resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz",
|
||||
"integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==",
|
||||
"requires": {
|
||||
"buffer": "^5.6.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"sinon": {
|
||||
"version": "9.0.3",
|
||||
"resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.3.tgz",
|
||||
|
@ -46683,11 +46775,6 @@
|
|||
"requires": {
|
||||
"has-flag": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"underscore": {
|
||||
"version": "1.12.1",
|
||||
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz",
|
||||
"integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw=="
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
@ -15,7 +15,7 @@ const {
|
|||
Joi,
|
||||
errors: handleValidationErrors,
|
||||
} = require('celebrate')
|
||||
const mongodb = require('./app/js/mongodb')
|
||||
const { mongoClient } = require('./app/js/mongodb')
|
||||
const Errors = require('./app/js/Errors')
|
||||
const HttpController = require('./app/js/HttpController')
|
||||
|
||||
|
@ -101,8 +101,8 @@ const { host } = Settings.internal.docstore
|
|||
|
||||
if (!module.parent) {
|
||||
// Called directly
|
||||
mongodb
|
||||
.waitForDb()
|
||||
mongoClient
|
||||
.connect()
|
||||
.then(() => {
|
||||
const server = app.listen(port, host, function (err) {
|
||||
if (err) {
|
||||
|
|
|
@ -15,7 +15,7 @@ let DocManager
|
|||
const MongoManager = require('./MongoManager')
|
||||
const Errors = require('./Errors')
|
||||
const logger = require('@overleaf/logger')
|
||||
const _ = require('underscore')
|
||||
const _ = require('lodash')
|
||||
const DocArchive = require('./DocArchiveManager')
|
||||
const RangeManager = require('./RangeManager')
|
||||
const Settings = require('@overleaf/settings')
|
||||
|
@ -157,7 +157,7 @@ module.exports = DocManager = {
|
|||
)
|
||||
return callback(err)
|
||||
}
|
||||
doc = _.extend(doc, archivedDoc)
|
||||
Object.assign(doc, archivedDoc)
|
||||
callback(null, doc)
|
||||
})
|
||||
}
|
||||
|
|
|
@ -12,7 +12,7 @@
|
|||
const { db, ObjectId } = require('./mongodb')
|
||||
const request = require('request')
|
||||
const async = require('async')
|
||||
const _ = require('underscore')
|
||||
const _ = require('lodash')
|
||||
const crypto = require('crypto')
|
||||
const settings = require('@overleaf/settings')
|
||||
const { port } = settings.internal.docstore
|
||||
|
|
|
@ -12,7 +12,7 @@
|
|||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let RangeManager
|
||||
const _ = require('underscore')
|
||||
const _ = require('lodash')
|
||||
const { ObjectId } = require('./mongodb')
|
||||
|
||||
module.exports = RangeManager = {
|
||||
|
|
|
@ -1,36 +1,16 @@
|
|||
const Settings = require('@overleaf/settings')
|
||||
const { MongoClient, ObjectId } = require('mongodb')
|
||||
|
||||
const clientPromise = MongoClient.connect(
|
||||
Settings.mongo.url,
|
||||
Settings.mongo.options
|
||||
)
|
||||
const mongoClient = new MongoClient(Settings.mongo.url)
|
||||
const mongoDb = mongoClient.db()
|
||||
|
||||
let setupDbPromise
|
||||
async function waitForDb() {
|
||||
if (!setupDbPromise) {
|
||||
setupDbPromise = setupDb()
|
||||
}
|
||||
await setupDbPromise
|
||||
}
|
||||
|
||||
const db = {}
|
||||
async function setupDb() {
|
||||
const internalDb = (await clientPromise).db()
|
||||
|
||||
db.docs = internalDb.collection('docs')
|
||||
db.docOps = internalDb.collection('docOps')
|
||||
}
|
||||
async function addCollection(name) {
|
||||
await waitForDb()
|
||||
const internalDb = (await clientPromise).db()
|
||||
|
||||
db[name] = internalDb.collection(name)
|
||||
const db = {
|
||||
docs: mongoDb.collection('docs'),
|
||||
docOps: mongoDb.collection('docOps'),
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
db,
|
||||
mongoClient,
|
||||
ObjectId,
|
||||
addCollection,
|
||||
waitForDb,
|
||||
}
|
||||
|
|
|
@ -9,12 +9,7 @@ const Settings = {
|
|||
},
|
||||
},
|
||||
|
||||
mongo: {
|
||||
options: {
|
||||
useUnifiedTopology:
|
||||
(process.env.MONGO_USE_UNIFIED_TOPOLOGY || 'true') === 'true',
|
||||
},
|
||||
},
|
||||
mongo: {},
|
||||
|
||||
docstore: {
|
||||
archiveOnSoftDelete: process.env.ARCHIVE_ON_SOFT_DELETE === 'true',
|
||||
|
|
|
@ -27,11 +27,11 @@
|
|||
"bunyan": "^1.8.15",
|
||||
"celebrate": "^13.0.4",
|
||||
"express": "^4.17.1",
|
||||
"mongodb": "^3.6.0",
|
||||
"lodash": "^4.17.21",
|
||||
"mongodb": "^4.11.0",
|
||||
"p-map": "^4.0.0",
|
||||
"request": "^2.88.2",
|
||||
"streamifier": "^0.1.1",
|
||||
"underscore": "~1.12.1"
|
||||
"streamifier": "^0.1.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@google-cloud/storage": "^5.1.2",
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
const app = require('../../../../app')
|
||||
const { waitForDb } = require('../../../../app/js/mongodb')
|
||||
require('@overleaf/logger').logger.level('error')
|
||||
const settings = require('@overleaf/settings')
|
||||
|
||||
|
@ -8,9 +7,6 @@ module.exports = {
|
|||
initing: false,
|
||||
callbacks: [],
|
||||
ensureRunning(callback) {
|
||||
if (callback == null) {
|
||||
callback = function () {}
|
||||
}
|
||||
if (this.running) {
|
||||
return callback()
|
||||
} else if (this.initing) {
|
||||
|
@ -18,20 +14,14 @@ module.exports = {
|
|||
}
|
||||
this.initing = true
|
||||
this.callbacks.push(callback)
|
||||
waitForDb().then(() => {
|
||||
return app.listen(settings.internal.docstore.port, 'localhost', error => {
|
||||
if (error != null) {
|
||||
throw error
|
||||
}
|
||||
this.running = true
|
||||
return (() => {
|
||||
const result = []
|
||||
for (callback of Array.from(this.callbacks)) {
|
||||
result.push(callback())
|
||||
}
|
||||
return result
|
||||
})()
|
||||
})
|
||||
app.listen(settings.internal.docstore.port, 'localhost', error => {
|
||||
if (error != null) {
|
||||
throw error
|
||||
}
|
||||
this.running = true
|
||||
for (callback of Array.from(this.callbacks)) {
|
||||
callback()
|
||||
}
|
||||
})
|
||||
},
|
||||
}
|
||||
|
|
|
@ -30,7 +30,7 @@ SandboxedModule.configure({
|
|||
requires: {
|
||||
'@overleaf/logger': stubs.logger,
|
||||
},
|
||||
globals: { Buffer, JSON, console, process },
|
||||
globals: { Buffer, JSON, Math, console, process },
|
||||
})
|
||||
|
||||
exports.mochaHooks = {
|
||||
|
|
|
@ -18,7 +18,6 @@ const modulePath = require('path').join(
|
|||
'../../../app/js/RangeManager'
|
||||
)
|
||||
const { ObjectId } = require('mongodb')
|
||||
const _ = require('underscore')
|
||||
|
||||
describe('RangeManager', function () {
|
||||
beforeEach(function () {
|
||||
|
|
Loading…
Reference in a new issue