github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #10998 from overleaf/em-upgrade-mongo-document-updater

Browse source 
Upgrade Mongo driver in document updater

GitOrigin-RevId: e3c4d86bbcf86dcf16fdfdbfab2aaf0d8eb1037c
This commit is contained in:
Eric Mc Sween 2023-01-10 07:57:21 -05:00 committed by Copybot
parent 073645d929
commit f76ab26822
8 changed files with 132 additions and 97 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

84
package-lock.json generated
View file

@ -35782,7 +35782,7 @@
"diff-match-patch": "https://github.com/overleaf/diff-match-patch/archive/89805f9c671a77a263fc53461acd62aa7498f688.tar.gz",
"express": "4.17.1",
"lodash": "^4.17.21",
"mongodb": "^3.6.6",
"mongodb": "^4.11.0",
"request": "^2.88.2",
"requestretry": "^4.1.2"
},
@ -35816,6 +35816,40 @@
"node": ">= 0.8"
}
},
"services/document-updater/node_modules/bson": {
"version": "4.7.1",
"resolved": "https://registry.npmjs.org/bson/-/bson-4.7.1.tgz",
"integrity": "sha512-XkuFtlCzi0WSy8D6PMhvrQ/q8VlZHN/2bJ/shJglwuA6TPD2ZP/hHLB7iDxOEWVINHN/UVTxP4pqZqOKMXPIXg==",
"dependencies": {
"buffer": "^5.6.0"
},
"engines": {
"node": ">=6.9.0"
}
},
"services/document-updater/node_modules/buffer": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"dependencies": {
"base64-js": "^1.3.1",
"ieee754": "^1.1.13"
}
},
"services/document-updater/node_modules/bytes": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
@ -35927,6 +35961,23 @@
"node": ">=4"
}
},
"services/document-updater/node_modules/mongodb": {
"version": "4.13.0",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.13.0.tgz",
"integrity": "sha512-+taZ/bV8d1pYuHL4U+gSwkhmDrwkWbH1l4aah4YpmpscMwgFBkufIKxgP/G7m87/NUuQzc2Z75ZTI7ZOyqZLbw==",
"dependencies": {
"bson": "^4.7.0",
"mongodb-connection-string-url": "^2.5.4",
"socks": "^2.7.1"
},
"engines": {
"node": ">=12.9.0"
},
"optionalDependencies": {
"@aws-sdk/credential-providers": "^3.186.0",
"saslprep": "^1.0.3"
}
},
"services/document-updater/node_modules/ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@ -46853,7 +46904,7 @@
"express": "4.17.1",
"lodash": "^4.17.21",
"mocha": "^8.4.0",
"mongodb": "^3.6.6",
"mongodb": "^4.11.0",
"request": "^2.88.2",
"requestretry": "^4.1.2",
"sandboxed-module": "^2.0.4",
@ -46878,6 +46929,23 @@
"type-is": "~1.6.17"
}
},
"bson": {
"version": "4.7.1",
"resolved": "https://registry.npmjs.org/bson/-/bson-4.7.1.tgz",
"integrity": "sha512-XkuFtlCzi0WSy8D6PMhvrQ/q8VlZHN/2bJ/shJglwuA6TPD2ZP/hHLB7iDxOEWVINHN/UVTxP4pqZqOKMXPIXg==",
"requires": {
"buffer": "^5.6.0"
}
},
"buffer": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
"requires": {
"base64-js": "^1.3.1",
"ieee754": "^1.1.13"
}
},
"bytes": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
@ -46968,6 +47036,18 @@
"resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
"integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
},
"mongodb": {
"version": "4.13.0",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.13.0.tgz",
"integrity": "sha512-+taZ/bV8d1pYuHL4U+gSwkhmDrwkWbH1l4aah4YpmpscMwgFBkufIKxgP/G7m87/NUuQzc2Z75ZTI7ZOyqZLbw==",
"requires": {
"@aws-sdk/credential-providers": "^3.186.0",
"bson": "^4.7.0",
"mongodb-connection-string-url": "^2.5.4",
"saslprep": "^1.0.3",
"socks": "^2.7.1"
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",

4
services/document-updater/app.js
View file

@ -225,8 +225,8 @@ const host = Settings.internal.documentupdater.host || 'localhost'
if (!module.parent) {
// Called directly
mongodb
.waitForDb()
mongodb.mongoClient
.connect()
.then(() => {
app.listen(port, host, function (err) {
if (err) {

32
services/document-updater/app/js/mongodb.js
View file

@ -1,37 +1,25 @@
const Settings = require('@overleaf/settings')
const { MongoClient, ObjectId } = require('mongodb')
const clientPromise = MongoClient.connect(
Settings.mongo.url,
Settings.mongo.options
)
const mongoClient = new MongoClient(Settings.mongo.url)
const mongoDb = mongoClient.db()
const db = {
docs: mongoDb.collection('docs'),
docSnapshots: mongoDb.collection('docSnapshots'),
projects: mongoDb.collection('projects'),
}
async function healthCheck() {
const internalDb = (await clientPromise).db()
const res = await internalDb.command({ ping: 1 })
const res = await mongoDb.command({ ping: 1 })
if (!res.ok) {
throw new Error('failed mongo ping')
}
}
let setupDbPromise
async function waitForDb() {
if (!setupDbPromise) {
setupDbPromise = setupDb()
}
await setupDbPromise
}
const db = {}
async function setupDb() {
const internalDb = (await clientPromise).db()
db.docSnapshots = internalDb.collection('docSnapshots')
}
module.exports = {
db,
ObjectId,
mongoClient,
healthCheck: require('util').callbackify(healthCheck),
waitForDb,
}

4
services/document-updater/config/settings.defaults.js
View file

@ -166,10 +166,6 @@ module.exports = {
dispatcherCount: parseInt(process.env.DISPATCHER_COUNT || 10, 10),
mongo: {
options: {
useUnifiedTopology:
(process.env.MONGO_USE_UNIFIED_TOPOLOGY || 'true') === 'true',
},
url:
process.env.MONGO_CONNECTION_STRING ||
`mongodb://${process.env.MONGO_HOST || '127.0.0.1'}/sharelatex`,

2
services/document-updater/package.json
View file

@ -28,7 +28,7 @@
"diff-match-patch": "https://github.com/overleaf/diff-match-patch/archive/89805f9c671a77a263fc53461acd62aa7498f688.tar.gz",
"express": "4.17.1",
"lodash": "^4.17.21",
"mongodb": "^3.6.6",
"mongodb": "^4.11.0",
"request": "^2.88.2",
"requestretry": "^4.1.2"
},

39
services/document-updater/scripts/fix_docs_with_missing_project.js
View file

@ -4,25 +4,14 @@ const rclient = require('@overleaf/redis-wrapper').createClient(
Settings.redis.documentupdater
)
const keys = Settings.redis.documentupdater.key_schema
const ProjectFlusher = require('app/js/ProjectFlusher')
const DocumentManager = require('app/js/DocumentManager')
const ProjectFlusher = require('../app/js/ProjectFlusher')
const DocumentManager = require('../app/js/DocumentManager')
const { mongoClient, db, ObjectId } = require('../app/js/mongodb')
const util = require('util')
const flushAndDeleteDocWithLock = util.promisify(
DocumentManager.flushAndDeleteDocWithLock
)
const { MongoClient, ObjectId } = require('mongodb')
const clientPromise = MongoClient.connect(
Settings.mongo.url,
Settings.mongo.options
)
const db = {}
clientPromise.then(client => {
db.docs = client.db().collection('docs')
})
async function fixDocsWithMissingProjectIds(dockeys, options) {
const docIds = ProjectFlusher._extractIds(dockeys)
for (const docId of docIds) {
@ -86,15 +75,13 @@ async function findAndProcessDocs(options) {
} while (cursor !== '0')
}
clientPromise.then(client => {
findAndProcessDocs({ limit: 1000, dryRun: process.env.DRY_RUN !== 'false' })
.then(result => {
rclient.quit()
client.close()
console.log('DONE')
})
.catch(function (error) {
console.error(error)
process.exit(1)
})
})
findAndProcessDocs({ limit: 1000, dryRun: process.env.DRY_RUN !== 'false' })
.then(result => {
rclient.quit()
mongoClient.close()
console.log('DONE')
})
.catch(function (error) {
console.error(error)
process.exit(1)
})

39
services/document-updater/scripts/remove_deleted_docs.js
View file

@ -4,28 +4,17 @@ const rclient = require('@overleaf/redis-wrapper').createClient(
Settings.redis.documentupdater
)
const keys = Settings.redis.documentupdater.key_schema
const ProjectFlusher = require('app/js/ProjectFlusher')
const RedisManager = require('app/js/RedisManager')
const ProjectFlusher = require('../app/js/ProjectFlusher')
const RedisManager = require('../app/js/RedisManager')
const { mongoClient, db, ObjectId } = require('../app/js/mongodb')
const util = require('util')
const getDoc = util.promisify((projectId, docId, cb) =>
RedisManager.getDoc(projectId, docId, (err, ...args) => cb(err, args))
)
const removeDocFromMemory = util.promisify(RedisManager.removeDocFromMemory)
const { MongoClient, ObjectId } = require('mongodb')
const clientPromise = MongoClient.connect(
Settings.mongo.url,
Settings.mongo.options
)
const summary = { totalDocs: 0, deletedDocs: 0, skippedDocs: 0 }
const db = {}
clientPromise.then(client => {
db.docs = client.db().collection('docs')
db.projects = client.db().collection('projects')
})
async function removeDeletedDocs(dockeys, options) {
const docIds = ProjectFlusher._extractIds(dockeys)
for (const docId of docIds) {
@ -159,15 +148,13 @@ async function findAndProcessDocs(options) {
} while (cursor !== '0')
}
clientPromise.then(client => {
findAndProcessDocs({ limit: 1000, dryRun: process.env.DRY_RUN !== 'false' })
.then(result => {
rclient.quit()
client.close()
console.log('DONE')
})
.catch(function (error) {
console.error(error)
process.exit(1)
})
})
findAndProcessDocs({ limit: 1000, dryRun: process.env.DRY_RUN !== 'false' })
.then(result => {
rclient.quit()
mongoClient.close()
console.log('DONE')
})
.catch(function (error) {
console.error(error)
process.exit(1)
})

25
services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js
View file

@ -9,7 +9,6 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const app = require('../../../../app')
const { waitForDb } = require('../../../../app/js/mongodb')
require('@overleaf/logger').logger.level('fatal')
module.exports = {
@ -27,20 +26,18 @@ module.exports = {
}
this.initing = true
this.callbacks.push(callback)
waitForDb().then(() => {
return app.listen(3003, 'localhost', error => {
if (error != null) {
throw error
app.listen(3003, 'localhost', error => {
if (error != null) {
throw error
}
this.running = true
return (() => {
const result = []
for (callback of Array.from(this.callbacks)) {
result.push(callback())
}
this.running = true
return (() => {
const result = []
for (callback of Array.from(this.callbacks)) {
result.push(callback())
}
return result
})()
})
return result
})()
})
},
}