github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

[misc] migrate the app to the native mongo driver

Browse source 
acceptance tests to follow in a separate commit
This commit is contained in:
Jakob Ackermann 2020-08-21 17:50:01 +01:00
parent a548f0c1a4
commit df0747ec48
12 changed files with 207 additions and 145 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
services/docstore/app.js
View file

@ -10,6 +10,7 @@ const Settings = require('settings-sharelatex')
const logger = require('logger-sharelatex') const logger = require('logger-sharelatex')
const express = require('express') const express = require('express')
const bodyParser = require('body-parser') const bodyParser = require('body-parser')
const mongodb = require('./app/js/mongodb')
const Errors = require('./app/js/Errors') const Errors = require('./app/js/Errors')
const HttpController = require('./app/js/HttpController') const HttpController = require('./app/js/HttpController')
@ -76,12 +77,21 @@ const { host } = Settings.internal.docstore
if (!module.parent) { if (!module.parent) {
// Called directly // Called directly
app.listen(port, host, function (error) { mongodb
if (error != null) { .waitForDb()
throw error .then(() => {
app.listen(port, host, function (err) {
if (err) {
logger.fatal({ err }, `Cannot bind to ${host}:${port}. Exiting.`)
process.exit(1)
} }
return logger.info(`Docstore starting up, listening on ${host}:${port}`) return logger.info(`Docstore starting up, listening on ${host}:${port}`)
}) })
})
.catch((err) => {
logger.fatal({ err }, 'Cannot connect to mongo. Exiting.')
process.exit(1)
})
} }
module.exports = app module.exports = app

15
services/docstore/app/js/HealthChecker.js
View file

@ -10,7 +10,7 @@
* DS207: Consider shorter variations of null checks * DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/ */
const { db, ObjectId } = require('./mongojs') const { getCollection, ObjectId } = require('./mongodb')
const request = require('request') const request = require('request')
const async = require('async') const async = require('async')
const _ = require('underscore') const _ = require('underscore')
@ -19,6 +19,9 @@ const settings = require('settings-sharelatex')
const { port } = settings.internal.docstore const { port } = settings.internal.docstore
const logger = require('logger-sharelatex') const logger = require('logger-sharelatex')
const docsCollectionPromise = getCollection('docs')
const docOpsCollectionPromise = getCollection('docOps')
module.exports = { module.exports = {
check(callback) { check(callback) {
const doc_id = ObjectId() const doc_id = ObjectId()
@ -60,8 +63,14 @@ module.exports = {
} }
}) })
}, },
(cb) => db.docs.remove({ _id: doc_id, project_id }, cb), (cb) =>
(cb) => db.docOps.remove({ doc_id }, cb) docsCollectionPromise.then((docs) =>
docs.deleteOne({ _id: doc_id, project_id }, cb)
),
(cb) =>
docOpsCollectionPromise.then((docOps) =>
docOps.deleteOne({ doc_id }, cb)
)
] ]
return async.series(jobs, callback) return async.series(jobs, callback)
} }

58
services/docstore/app/js/MongoManager.js
View file

@ -11,28 +11,28 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/ */
let MongoManager let MongoManager
const { db, ObjectId } = require('./mongojs') const { getCollection, ObjectId } = require('./mongodb')
const logger = require('logger-sharelatex') const logger = require('logger-sharelatex')
const metrics = require('metrics-sharelatex') const metrics = require('metrics-sharelatex')
const { promisify } = require('util') const { promisify } = require('util')
const docsCollectionPromise = getCollection('docs')
const docOpsCollectionPromise = getCollection('docOps')
module.exports = MongoManager = { module.exports = MongoManager = {
findDoc(project_id, doc_id, filter, callback) { findDoc(project_id, doc_id, filter, callback) {
if (callback == null) { if (callback == null) {
callback = function (error, doc) {} callback = function (error, doc) {}
} }
return db.docs.find( docsCollectionPromise.then((docs) =>
docs.findOne(
{ {
_id: ObjectId(doc_id.toString()), _id: ObjectId(doc_id.toString()),
project_id: ObjectId(project_id.toString()) project_id: ObjectId(project_id.toString())
}, },
filter, filter,
function (error, docs) { callback
if (docs == null) { )
docs = []
}
return callback(error, docs[0])
}
) )
}, },
@ -44,7 +44,9 @@ module.exports = MongoManager = {
if (!options.include_deleted) { if (!options.include_deleted) {
query.deleted = { $ne: true } query.deleted = { $ne: true }
} }
return db.docs.find(query, filter, callback) docsCollectionPromise.then((docs) =>
docs.find(query, filter).toArray(callback)
)
}, },
getArchivedProjectDocs(project_id, callback) { getArchivedProjectDocs(project_id, callback) {
@ -52,7 +54,7 @@ module.exports = MongoManager = {
project_id: ObjectId(project_id.toString()), project_id: ObjectId(project_id.toString()),
inS3: true inS3: true
} }
return db.docs.find(query, {}, callback) docsCollectionPromise.then((docs) => docs.find(query).toArray(callback))
}, },
upsertIntoDocCollection(project_id, doc_id, updates, callback) { upsertIntoDocCollection(project_id, doc_id, updates, callback) {
@ -66,16 +68,19 @@ module.exports = MongoManager = {
} }
} }
update.$set.project_id = ObjectId(project_id) update.$set.project_id = ObjectId(project_id)
return db.docs.update( docsCollectionPromise.then((docs) =>
docs.updateOne(
{ _id: ObjectId(doc_id) }, { _id: ObjectId(doc_id) },
update, update,
{ upsert: true }, { upsert: true },
callback callback
) )
)
}, },
markDocAsDeleted(project_id, doc_id, callback) { markDocAsDeleted(project_id, doc_id, callback) {
return db.docs.update( docsCollectionPromise.then((docs) =>
docs.updateOne(
{ {
_id: ObjectId(doc_id), _id: ObjectId(doc_id),
project_id: ObjectId(project_id) project_id: ObjectId(project_id)
@ -85,6 +90,7 @@ module.exports = MongoManager = {
}, },
callback callback
) )
)
}, },
markDocAsArchived(doc_id, rev, callback) { markDocAsArchived(doc_id, rev, callback) {
@ -99,38 +105,39 @@ module.exports = MongoManager = {
_id: doc_id, _id: doc_id,
rev rev
} }
return db.docs.update(query, update, (err) => callback(err)) docsCollectionPromise.then((docs) =>
docs.updateOne(query, update, callback)
)
}, },
getDocVersion(doc_id, callback) { getDocVersion(doc_id, callback) {
if (callback == null) { if (callback == null) {
callback = function (error, version) {} callback = function (error, version) {}
} }
return db.docOps.find( docOpsCollectionPromise.then((docOps) =>
docOps.findOne(
{ {
doc_id: ObjectId(doc_id) doc_id: ObjectId(doc_id)
}, },
{ {
version: 1 version: 1
}, },
function (error, docs) { function (error, doc) {
if (error != null) { if (error != null) {
return callback(error) return callback(error)
} }
if (docs.length < 1 || docs[0].version == null) { callback(null, (doc && doc.version) || 0)
return callback(null, 0)
} else {
return callback(null, docs[0].version)
}
} }
) )
)
}, },
setDocVersion(doc_id, version, callback) { setDocVersion(doc_id, version, callback) {
if (callback == null) { if (callback == null) {
callback = function (error) {} callback = function (error) {}
} }
return db.docOps.update( docOpsCollectionPromise.then((docOps) =>
docOps.updateOne(
{ {
doc_id: ObjectId(doc_id) doc_id: ObjectId(doc_id)
}, },
@ -142,10 +149,12 @@ module.exports = MongoManager = {
}, },
callback callback
) )
)
}, },
destroyDoc(doc_id, callback) { destroyDoc(doc_id, callback) {
return db.docs.remove( docsCollectionPromise.then((docs) =>
docs.deleteOne(
{ {
_id: ObjectId(doc_id) _id: ObjectId(doc_id)
}, },
@ -153,14 +162,17 @@ module.exports = MongoManager = {
if (err != null) { if (err != null) {
return callback(err) return callback(err)
} }
return db.docOps.remove( docOpsCollectionPromise.then((docOps) =>
docOps.deleteOne(
{ {
doc_id: ObjectId(doc_id) doc_id: ObjectId(doc_id)
}, },
callback callback
) )
)
} }
) )
)
} }
} }

2
services/docstore/app/js/RangeManager.js
View file

@ -13,7 +13,7 @@
*/ */
let RangeManager let RangeManager
const _ = require('underscore') const _ = require('underscore')
const { ObjectId } = require('./mongojs') const { ObjectId } = require('./mongodb')
module.exports = RangeManager = { module.exports = RangeManager = {
shouldUpdateRanges(doc_ranges, incoming_ranges) { shouldUpdateRanges(doc_ranges, incoming_ranges) {

19
services/docstore/app/js/mongodb.js Normal file
View file

@ -0,0 +1,19 @@
const Settings = require('settings-sharelatex')
const { MongoClient, ObjectId } = require('mongodb')
const clientPromise = MongoClient.connect(Settings.mongo.url)
const dbPromise = clientPromise.then((client) => client.db())
async function getCollection(name) {
return (await dbPromise).collection(name)
}
async function waitForDb() {
await clientPromise
}
module.exports = {
ObjectId,
getCollection,
waitForDb
}

64
services/docstore/package-lock.json generated
View file

@ -1438,7 +1438,7 @@
"archy": { "archy": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz", "resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz",
"integrity": "sha512-Xg+9RwCg/0p32teKdGMPTPnVXKD0w3DfHnFTficozsAgsvq2XenPJq/MYpzzQ/v8zrOyJn6Ds39VA4JIDwFfqw==" "integrity": "sha1-+cjBN1fMHde8N5rHeyxipcKGjEA="
}, },
"argparse": { "argparse": {
"version": "1.0.10", "version": "1.0.10",
@ -1727,9 +1727,9 @@
"dev": true "dev": true
}, },
"bson": { "bson": {
"version": "1.1.3", "version": "1.1.5",
"resolved": "https://registry.npmjs.org/bson/-/bson-1.1.3.tgz", "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.5.tgz",
"integrity": "sha512-TdiJxMVnodVS7r0BdL42y/pqC9cL2iKynVwA0Ho3qbsQYr428veL3l7BQyuqiw+Q5SqqoT0m4srSY/BlZ9AxXg==" "integrity": "sha512-kDuEzldR21lHciPQAIulLs1LZlCXdLziXI6Mb/TDkwXhb//UORJNPXgcRs2CuO4H0DcMkpfT3/ySsP3unoZjBg=="
}, },
"buffer": { "buffer": {
"version": "4.9.2", "version": "4.9.2",
@ -1974,7 +1974,7 @@
"commondir": { "commondir": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz",
"integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==" "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs="
}, },
"compressible": { "compressible": {
"version": "2.0.18", "version": "2.0.18",
@ -2225,7 +2225,7 @@
"default-require-extensions": { "default-require-extensions": {
"version": "2.0.0", "version": "2.0.0",
"resolved": "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-2.0.0.tgz", "resolved": "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-2.0.0.tgz",
"integrity": "sha512-B0n2zDIXpzLzKeoEozorDSa1cHc1t0NjmxP0zuAxbizNU2MBqYJJKYXrrFdKuQliojXynrxgd7l4ahfg/+aA5g==", "integrity": "sha1-9fj7sYp9bVCyH2QfZJ67Uiz+JPc=",
"requires": { "requires": {
"strip-bom": "^3.0.0" "strip-bom": "^3.0.0"
} }
@ -2316,7 +2316,7 @@
"each-series": { "each-series": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/each-series/-/each-series-1.0.0.tgz", "resolved": "https://registry.npmjs.org/each-series/-/each-series-1.0.0.tgz",
"integrity": "sha512-4MQloCGGCmT5GJZK5ibgJSvTK1c1QSrNlDvLk6fEyRxjZnXjl+NNFfzhfXpmnWh33Owc9D9klrdzCUi7yc9r4Q==" "integrity": "sha1-+Ibmxm39sl7x/nNWQUbuXLR4r8s="
}, },
"ecc-jsbn": { "ecc-jsbn": {
"version": "0.1.2", "version": "0.1.2",
@ -3184,7 +3184,7 @@
"foreground-child": { "foreground-child": {
"version": "1.5.6", "version": "1.5.6",
"resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-1.5.6.tgz", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-1.5.6.tgz",
"integrity": "sha512-3TOY+4TKV0Ml83PXJQY+JFQaHNV38lzQDIzzXYg1kWdBLenGgoZhAs0CKgzI31vi2pWEpQMq/Yi4bpKwCPkw7g==", "integrity": "sha1-T9ca0t/elnibmApcCilZN8svXOk=",
"requires": { "requires": {
"cross-spawn": "^4", "cross-spawn": "^4",
"signal-exit": "^3.0.0" "signal-exit": "^3.0.0"
@ -3193,7 +3193,7 @@
"cross-spawn": { "cross-spawn": {
"version": "4.0.2", "version": "4.0.2",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz",
"integrity": "sha512-yAXz/pA1tD8Gtg2S98Ekf/sewp3Lcp3YoFKJ4Hkp5h5yLWnKVTDU0kwjKJ8NDCYcfTLfyGkzTikst+jWypT1iA==", "integrity": "sha1-e5JHYhwjrf3ThWAEqCPL45dCTUE=",
"requires": { "requires": {
"lru-cache": "^4.0.1", "lru-cache": "^4.0.1",
"which": "^1.2.9" "which": "^1.2.9"
@ -3211,7 +3211,7 @@
"yallist": { "yallist": {
"version": "2.1.2", "version": "2.1.2",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
"integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==" "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI="
} }
} }
}, },
@ -3584,7 +3584,7 @@
"hasha": { "hasha": {
"version": "3.0.0", "version": "3.0.0",
"resolved": "https://registry.npmjs.org/hasha/-/hasha-3.0.0.tgz", "resolved": "https://registry.npmjs.org/hasha/-/hasha-3.0.0.tgz",
"integrity": "sha512-w0Kz8lJFBoyaurBiNrIvxPqr/gJ6fOfSkpAPOepN3oECqGJag37xPbOv57izi/KP8auHgNYxn5fXtAb+1LsJ6w==", "integrity": "sha1-UqMvq4Vp1BymmmH/GiFPjrfIvTk=",
"requires": { "requires": {
"is-stream": "^1.0.1" "is-stream": "^1.0.1"
}, },
@ -3592,7 +3592,7 @@
"is-stream": { "is-stream": {
"version": "1.1.0", "version": "1.1.0",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
"integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==" "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ="
} }
} }
}, },
@ -3613,9 +3613,9 @@
"integrity": "sha512-kssjab8CvdXfcXMXVcvsXum4Hwdq9XGtRD3TteMEvEbq0LXyiNQr6AprqKqfeaDXze7SxWvRxdpwE6ku7ikLkg==" "integrity": "sha512-kssjab8CvdXfcXMXVcvsXum4Hwdq9XGtRD3TteMEvEbq0LXyiNQr6AprqKqfeaDXze7SxWvRxdpwE6ku7ikLkg=="
}, },
"html-escaper": { "html-escaper": {
"version": "2.0.1", "version": "2.0.2",
"resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.1.tgz", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz",
"integrity": "sha512-hNX23TjWwD3q56HpWjUHOKj1+4KKlnjv9PcmBUYKVpga+2cnb9nDx/B1o0yO4n+RZXZdiNxzx6B24C9aNMTkkQ==" "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg=="
}, },
"http-errors": { "http-errors": {
"version": "1.7.2", "version": "1.7.2",
@ -4301,7 +4301,7 @@
"lodash.flattendeep": { "lodash.flattendeep": {
"version": "4.4.0", "version": "4.4.0",
"resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz", "resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz",
"integrity": "sha512-uHaJFihxmJcEX3kT4I23ABqKKalJ/zDrDg0lsFtc1h+3uw49SIJ5beyhx5ExVRti3AvKoOJngIj7xz3oylPdWQ==" "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI="
}, },
"lodash.get": { "lodash.get": {
"version": "4.4.2", "version": "4.4.2",
@ -4804,12 +4804,12 @@
"optional": true "optional": true
}, },
"mongodb": { "mongodb": {
"version": "3.5.5", "version": "3.6.0",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.5.5.tgz", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.0.tgz",
"integrity": "sha512-GCjDxR3UOltDq00Zcpzql6dQo1sVry60OXJY3TDmFc2SWFY6c8Gn1Ardidc5jDirvJrx2GC3knGOImKphbSL3A==", "integrity": "sha512-/XWWub1mHZVoqEsUppE0GV7u9kanLvHxho6EvBxQbShXTKYF9trhZC2NzbulRGeG7xMJHD8IOWRcdKx5LPjAjQ==",
"requires": { "requires": {
"bl": "^2.2.0", "bl": "^2.2.0",
"bson": "^1.1.1", "bson": "^1.1.4",
"denque": "^1.4.1", "denque": "^1.4.1",
"require_optional": "^1.0.1", "require_optional": "^1.0.1",
"safe-buffer": "^5.1.2", "safe-buffer": "^5.1.2",
@ -5186,7 +5186,7 @@
"os-homedir": { "os-homedir": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
"integrity": "sha512-B5JU3cabzk8c67mRRd3ECmROafjYMXbuzlwtqdM8IbS8ktlTix8aFGb2bAGKrSRIlnfKwovGUUr72JUPyOb6kQ==" "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M="
}, },
"os-tmpdir": { "os-tmpdir": {
"version": "1.0.2", "version": "1.0.2",
@ -5270,7 +5270,7 @@
"parse-mongo-url": { "parse-mongo-url": {
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/parse-mongo-url/-/parse-mongo-url-1.1.1.tgz", "resolved": "https://registry.npmjs.org/parse-mongo-url/-/parse-mongo-url-1.1.1.tgz",
"integrity": "sha512-7bZUusQIrFLwvsLHBnCz2WKYQ5LKO/LwKPnvQxbMIh9gDx8H5ZsknRmLjZdn6GVdrgVOwqDrZKsY0qDLNmRgcw==" "integrity": "sha1-ZiON9fjnwMjKTNlw1KtqE3PrdbU="
}, },
"parse-ms": { "parse-ms": {
"version": "2.1.0", "version": "2.1.0",
@ -6037,7 +6037,7 @@
"pseudomap": { "pseudomap": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
"integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==" "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM="
}, },
"psl": { "psl": {
"version": "1.7.0", "version": "1.7.0",
@ -6249,7 +6249,7 @@
"release-zalgo": { "release-zalgo": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/release-zalgo/-/release-zalgo-1.0.0.tgz", "resolved": "https://registry.npmjs.org/release-zalgo/-/release-zalgo-1.0.0.tgz",
"integrity": "sha512-gUAyHVHPPC5wdqX/LG4LWtRYtgjxyX78oanFNTMMyFEfOqdC54s3eE82imuWKbOeqYht2CrNf64Qb8vgmmtZGA==", "integrity": "sha1-CXALflB0Mpc5Mw5TXFqQ+2eFFzA=",
"requires": { "requires": {
"es6-error": "^4.0.1" "es6-error": "^4.0.1"
} }
@ -6365,7 +6365,7 @@
"resolve-from": { "resolve-from": {
"version": "2.0.0", "version": "2.0.0",
"resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-2.0.0.tgz", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-2.0.0.tgz",
"integrity": "sha512-qpFcKaXsq8+oRoLilkwyc7zHGF5i9Q2/25NIgLQQ/+VVv9rU4qvr6nXVAw1DsnXJyQkZsR4Ytfbtg5ehfcUssQ==" "integrity": "sha1-lICrIOlP+h2egKgEx+oUdhGWa1c="
}, },
"restore-cursor": { "restore-cursor": {
"version": "3.1.0", "version": "3.1.0",
@ -6666,7 +6666,7 @@
"sparse-bitfield": { "sparse-bitfield": {
"version": "3.0.3", "version": "3.0.3",
"resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
"integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=",
"optional": true, "optional": true,
"requires": { "requires": {
"memory-pager": "^1.0.2" "memory-pager": "^1.0.2"
@ -6927,7 +6927,7 @@
"stubs": { "stubs": {
"version": "3.0.0", "version": "3.0.0",
"resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz",
"integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==" "integrity": "sha1-6NK6H6nJBXAwPAMLaQD31fiavls="
}, },
"supports-color": { "supports-color": {
"version": "6.0.0", "version": "6.0.0",
@ -7055,7 +7055,7 @@
"load-json-file": { "load-json-file": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz",
"integrity": "sha512-Kx8hMakjX03tiGTLAIdJ+lL0htKnXjEZN6hk/tozf/WOuYGdZBJrZ+rCJRbVCugsjB3jMLn9746NsQIf5VjBMw==", "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=",
"requires": { "requires": {
"graceful-fs": "^4.1.2", "graceful-fs": "^4.1.2",
"parse-json": "^4.0.0", "parse-json": "^4.0.0",
@ -7083,7 +7083,7 @@
"parse-json": { "parse-json": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
"integrity": "sha512-aOIos8bujGN93/8Ox/jPLh7RwVnPEysynVFE+fQZyg6jKELEHwzgKdLRFHUgXJL6kylijVSBC4BvN9OmsB48Rw==", "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
"requires": { "requires": {
"error-ex": "^1.3.1", "error-ex": "^1.3.1",
"json-parse-better-errors": "^1.0.1" "json-parse-better-errors": "^1.0.1"
@ -7100,12 +7100,12 @@
"pify": { "pify": {
"version": "3.0.0", "version": "3.0.0",
"resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",
"integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==" "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY="
}, },
"read-pkg": { "read-pkg": {
"version": "3.0.0", "version": "3.0.0",
"resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz",
"integrity": "sha512-BLq/cCO9two+lBgiTYNqD6GdtK8s4NpaWrl6/rCO9w0TUS8oJl7cmToOZfRYllKTISY6nt1U7jQ53brmKqY6BA==", "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=",
"requires": { "requires": {
"load-json-file": "^4.0.0", "load-json-file": "^4.0.0",
"normalize-package-data": "^2.3.2", "normalize-package-data": "^2.3.2",
@ -7178,7 +7178,7 @@
"to-mongodb-core": { "to-mongodb-core": {
"version": "2.0.0", "version": "2.0.0",
"resolved": "https://registry.npmjs.org/to-mongodb-core/-/to-mongodb-core-2.0.0.tgz", "resolved": "https://registry.npmjs.org/to-mongodb-core/-/to-mongodb-core-2.0.0.tgz",
"integrity": "sha512-vfXXcGYFP8+0L5IPOtUzzVIvPE/G3GN0TKa/PRBlzPqYyhm+UxhPmvv634EQgO4Ot8dHbBFihOslMJQclY8Z9A==" "integrity": "sha1-NZbsdhOsmtO5ioncua77pWnNJ+s="
}, },
"to-no-case": { "to-no-case": {
"version": "1.0.2", "version": "1.0.2",

1
services/docstore/package.json
View file

@ -26,6 +26,7 @@
"logger-sharelatex": "^2.2.0", "logger-sharelatex": "^2.2.0",
"metrics-sharelatex": "^2.7.0", "metrics-sharelatex": "^2.7.0",
"mongojs": "3.1.0", "mongojs": "3.1.0",
"mongodb": "^3.6.0",
"settings-sharelatex": "^1.1.0", "settings-sharelatex": "^1.1.0",
"streamifier": "^0.1.1", "streamifier": "^0.1.1",
"tiny-async-pool": "^1.1.0", "tiny-async-pool": "^1.1.0",

2
services/docstore/test/unit/js/DocArchiveManagerTests.js
View file

@ -3,7 +3,7 @@ const chai = require('chai')
const { expect } = chai const { expect } = chai
const modulePath = '../../../app/js/DocArchiveManager.js' const modulePath = '../../../app/js/DocArchiveManager.js'
const SandboxedModule = require('sandboxed-module') const SandboxedModule = require('sandboxed-module')
const { ObjectId } = require('mongojs') const { ObjectId } = require('mongodb')
const Errors = require('../../../app/js/Errors') const Errors = require('../../../app/js/Errors')
chai.use(require('chai-as-promised')) chai.use(require('chai-as-promised'))

2
services/docstore/test/unit/js/DocManagerTests.js
View file

@ -19,7 +19,7 @@ const { assert } = require('chai')
chai.should() chai.should()
const { expect } = chai const { expect } = chai
const modulePath = require('path').join(__dirname, '../../../app/js/DocManager') const modulePath = require('path').join(__dirname, '../../../app/js/DocManager')
const { ObjectId } = require('mongojs') const { ObjectId } = require('mongodb')
const Errors = require('../../../app/js/Errors') const Errors = require('../../../app/js/Errors')
describe('DocManager', function () { describe('DocManager', function () {

2
services/docstore/test/unit/js/HttpControllerTests.js
View file

@ -19,7 +19,7 @@ const modulePath = require('path').join(
__dirname, __dirname,
'../../../app/js/HttpController' '../../../app/js/HttpController'
) )
const { ObjectId } = require('mongojs') const { ObjectId } = require('mongodb')
describe('HttpController', function () { describe('HttpController', function () {
beforeEach(function () { beforeEach(function () {

63
services/docstore/test/unit/js/MongoManagerTests.js
View file

@ -16,14 +16,18 @@ const modulePath = require('path').join(
__dirname, __dirname,
'../../../app/js/MongoManager' '../../../app/js/MongoManager'
) )
const { ObjectId } = require('mongojs') const { ObjectId } = require('mongodb')
const { assert } = require('chai') const { assert } = require('chai')
describe('MongoManager', function () { describe('MongoManager', function () {
beforeEach(function () { beforeEach(function () {
this.db = {}
this.MongoManager = SandboxedModule.require(modulePath, { this.MongoManager = SandboxedModule.require(modulePath, {
requires: { requires: {
'./mongojs': { './mongodb': {
getCollection: sinon.stub().callsFake((name) => {
return Promise.resolve((this.db[name] = {}))
}),
db: (this.db = { docs: {}, docOps: {} }), db: (this.db = { docs: {}, docOps: {} }),
ObjectId ObjectId
}, },
@ -36,14 +40,14 @@ describe('MongoManager', function () {
}) })
this.project_id = ObjectId().toString() this.project_id = ObjectId().toString()
this.doc_id = ObjectId().toString() this.doc_id = ObjectId().toString()
this.callback = sinon.stub()
return (this.stubbedErr = new Error('hello world')) return (this.stubbedErr = new Error('hello world'))
}) })
describe('findDoc', function () { describe('findDoc', function () {
beforeEach(function () { beforeEach(function (done) {
this.callback = sinon.stub().callsFake(() => done())
this.doc = { name: 'mock-doc' } this.doc = { name: 'mock-doc' }
this.db.docs.find = sinon.stub().callsArgWith(2, null, [this.doc]) this.db.docs.findOne = sinon.stub().callsArgWith(2, null, this.doc)
this.filter = { lines: true } this.filter = { lines: true }
return this.MongoManager.findDoc( return this.MongoManager.findDoc(
this.project_id, this.project_id,
@ -54,7 +58,7 @@ describe('MongoManager', function () {
}) })
it('should find the doc', function () { it('should find the doc', function () {
return this.db.docs.find this.db.docs.findOne
.calledWith( .calledWith(
{ {
_id: ObjectId(this.doc_id), _id: ObjectId(this.doc_id),
@ -77,13 +81,16 @@ describe('MongoManager', function () {
this.doc2 = { name: 'mock-doc2' } this.doc2 = { name: 'mock-doc2' }
this.doc3 = { name: 'mock-doc3' } this.doc3 = { name: 'mock-doc3' }
this.doc4 = { name: 'mock-doc4' } this.doc4 = { name: 'mock-doc4' }
return (this.db.docs.find = sinon this.db.docs.find = sinon.stub().returns({
toArray: sinon
.stub() .stub()
.callsArgWith(2, null, [this.doc, this.doc3, this.doc4])) .callsArgWith(0, null, [this.doc, this.doc3, this.doc4])
})
}) })
describe('with included_deleted = false', function () { describe('with included_deleted = false', function () {
beforeEach(function () { beforeEach(function (done) {
this.callback = sinon.stub().callsFake(() => done())
return this.MongoManager.getProjectsDocs( return this.MongoManager.getProjectsDocs(
this.project_id, this.project_id,
{ include_deleted: false }, { include_deleted: false },
@ -112,7 +119,8 @@ describe('MongoManager', function () {
}) })
return describe('with included_deleted = true', function () { return describe('with included_deleted = true', function () {
beforeEach(function () { beforeEach(function (done) {
this.callback = sinon.stub().callsFake(() => done())
return this.MongoManager.getProjectsDocs( return this.MongoManager.getProjectsDocs(
this.project_id, this.project_id,
{ include_deleted: true }, { include_deleted: true },
@ -142,7 +150,7 @@ describe('MongoManager', function () {
describe('upsertIntoDocCollection', function () { describe('upsertIntoDocCollection', function () {
beforeEach(function () { beforeEach(function () {
this.db.docs.update = sinon.stub().callsArgWith(3, this.stubbedErr) this.db.docs.updateOne = sinon.stub().callsArgWith(3, this.stubbedErr)
return (this.oldRev = 77) return (this.oldRev = 77)
}) })
@ -152,7 +160,7 @@ describe('MongoManager', function () {
this.doc_id, this.doc_id,
{ lines: this.lines }, { lines: this.lines },
(err) => { (err) => {
const args = this.db.docs.update.args[0] const args = this.db.docs.updateOne.args[0]
assert.deepEqual(args[0], { _id: ObjectId(this.doc_id) }) assert.deepEqual(args[0], { _id: ObjectId(this.doc_id) })
assert.equal(args[1].$set.lines, this.lines) assert.equal(args[1].$set.lines, this.lines)
assert.equal(args[1].$inc.rev, 1) assert.equal(args[1].$inc.rev, 1)
@ -177,7 +185,7 @@ describe('MongoManager', function () {
describe('markDocAsDeleted', function () { describe('markDocAsDeleted', function () {
beforeEach(function () { beforeEach(function () {
this.db.docs.update = sinon.stub().callsArgWith(2, this.stubbedErr) this.db.docs.updateOne = sinon.stub().callsArgWith(2, this.stubbedErr)
return (this.oldRev = 77) return (this.oldRev = 77)
}) })
@ -186,7 +194,7 @@ describe('MongoManager', function () {
this.project_id, this.project_id,
this.doc_id, this.doc_id,
(err) => { (err) => {
const args = this.db.docs.update.args[0] const args = this.db.docs.updateOne.args[0]
assert.deepEqual(args[0], { assert.deepEqual(args[0], {
_id: ObjectId(this.doc_id), _id: ObjectId(this.doc_id),
project_id: ObjectId(this.project_id) project_id: ObjectId(this.project_id)
@ -211,19 +219,19 @@ describe('MongoManager', function () {
describe('destroyDoc', function () { describe('destroyDoc', function () {
beforeEach(function (done) { beforeEach(function (done) {
this.db.docs.remove = sinon.stub().yields() this.db.docs.deleteOne = sinon.stub().yields()
this.db.docOps.remove = sinon.stub().yields() this.db.docOps.deleteOne = sinon.stub().yields()
return this.MongoManager.destroyDoc('123456789012', done) return this.MongoManager.destroyDoc('123456789012', done)
}) })
it('should destroy the doc', function () { it('should destroy the doc', function () {
return sinon.assert.calledWith(this.db.docs.remove, { return sinon.assert.calledWith(this.db.docs.deleteOne, {
_id: ObjectId('123456789012') _id: ObjectId('123456789012')
}) })
}) })
return it('should destroy the docOps', function () { return it('should destroy the docOps', function () {
return sinon.assert.calledWith(this.db.docOps.remove, { return sinon.assert.calledWith(this.db.docOps.deleteOne, {
doc_id: ObjectId('123456789012') doc_id: ObjectId('123456789012')
}) })
}) })
@ -231,14 +239,15 @@ describe('MongoManager', function () {
describe('getDocVersion', function () { describe('getDocVersion', function () {
describe('when the doc exists', function () { describe('when the doc exists', function () {
beforeEach(function () { beforeEach(function (done) {
this.callback = sinon.stub().callsFake(() => done())
this.doc = { version: (this.version = 42) } this.doc = { version: (this.version = 42) }
this.db.docOps.find = sinon.stub().callsArgWith(2, null, [this.doc]) this.db.docOps.findOne = sinon.stub().callsArgWith(2, null, this.doc)
return this.MongoManager.getDocVersion(this.doc_id, this.callback) return this.MongoManager.getDocVersion(this.doc_id, this.callback)
}) })
it('should look for the doc in the database', function () { it('should look for the doc in the database', function () {
return this.db.docOps.find return this.db.docOps.findOne
.calledWith({ doc_id: ObjectId(this.doc_id) }, { version: 1 }) .calledWith({ doc_id: ObjectId(this.doc_id) }, { version: 1 })
.should.equal(true) .should.equal(true)
}) })
@ -249,8 +258,9 @@ describe('MongoManager', function () {
}) })
return describe("when the doc doesn't exist", function () { return describe("when the doc doesn't exist", function () {
beforeEach(function () { beforeEach(function (done) {
this.db.docOps.find = sinon.stub().callsArgWith(2, null, []) this.callback = sinon.stub().callsFake(() => done())
this.db.docOps.findOne = sinon.stub().callsArgWith(2, null, null)
return this.MongoManager.getDocVersion(this.doc_id, this.callback) return this.MongoManager.getDocVersion(this.doc_id, this.callback)
}) })
@ -261,9 +271,10 @@ describe('MongoManager', function () {
}) })
return describe('setDocVersion', function () { return describe('setDocVersion', function () {
beforeEach(function () { beforeEach(function (done) {
this.callback = sinon.stub().callsFake(() => done())
this.version = 42 this.version = 42
this.db.docOps.update = sinon.stub().callsArg(3) this.db.docOps.updateOne = sinon.stub().callsArg(3)
return this.MongoManager.setDocVersion( return this.MongoManager.setDocVersion(
this.doc_id, this.doc_id,
this.version, this.version,
@ -272,7 +283,7 @@ describe('MongoManager', function () {
}) })
it('should update the doc version', function () { it('should update the doc version', function () {
return this.db.docOps.update return this.db.docOps.updateOne
.calledWith( .calledWith(
{ {
doc_id: ObjectId(this.doc_id) doc_id: ObjectId(this.doc_id)

4
services/docstore/test/unit/js/RangeManagerTests.js
View file

@ -18,7 +18,7 @@ const modulePath = require('path').join(
__dirname, __dirname,
'../../../app/js/RangeManager' '../../../app/js/RangeManager'
) )
const { ObjectId } = require('mongojs') const { ObjectId } = require('mongodb')
const { assert } = require('chai') const { assert } = require('chai')
const _ = require('underscore') const _ = require('underscore')
@ -26,7 +26,7 @@ describe('RangeManager', function () {
beforeEach(function () { beforeEach(function () {
return (this.RangeManager = SandboxedModule.require(modulePath, { return (this.RangeManager = SandboxedModule.require(modulePath, {
requires: { requires: {
'./mongojs': { './mongodb': {
ObjectId ObjectId
} }
} }