github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge branches 'jpa-bump-bson', 'jpa-mongo-find-query-cleanup', 'jpa-mongo-js-cleanup' and 'jpa-mongo-update-query-cleanup' into jpa-mongodb-native

Browse source
This commit is contained in:
Jakob Ackermann 2020-09-10 16:41:45 +01:00
commit 3a4aa45e2e
5 changed files with 59 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
services/track-changes/app/js/MongoManager.js
View file

@ -115,16 +115,11 @@ module.exports = MongoManager = {
if (callback == null) { if (callback == null) {
callback = function (error, metadata) {} callback = function (error, metadata) {}
} }
return db.projectHistoryMetaData.find( return db.projectHistoryMetaData.findOne(
{ {
project_id: ObjectId(project_id.toString()) project_id: ObjectId(project_id.toString())
}, },
function (error, results) { callback
if (error != null) {
return callback(error)
}
return callback(null, results[0])
}
) )
}, },

108
services/track-changes/app/js/PackManager.js
View file

@ -275,10 +275,7 @@ module.exports = PackManager = {
'appending updates to existing pack' 'appending updates to existing pack'
) )
Metrics.inc(`append-pack-${temporary ? 'temporary' : 'permanent'}`) Metrics.inc(`append-pack-${temporary ? 'temporary' : 'permanent'}`)
return db.docHistory.findAndModify( return db.docHistory.updateOne(query, update, callback)
{ query, update, new: true, fields: { meta: 1, v_end: 1 } },
callback
)
}, },
// Retrieve all changes for a document // Retrieve all changes for a document
@ -303,7 +300,8 @@ module.exports = PackManager = {
// console.log "query:", query // console.log "query:", query
return db.docHistory return db.docHistory
.find(query) .find(query)
.sort({ v: -1 }, function (err, result) { .sort({ v: -1 })
.toArray(function (err, result) {
if (err != null) { if (err != null) {
return callback(err) return callback(err)
} }
@ -382,20 +380,9 @@ module.exports = PackManager = {
fetchPacksIfNeeded(project_id, doc_id, pack_ids, callback) { fetchPacksIfNeeded(project_id, doc_id, pack_ids, callback) {
let id let id
return db.docHistory.find( return db.docHistory
{ .find({ _id: { $in: pack_ids.map(ObjectId) } }, { _id: 1 })
_id: { .toArray(function (err, loadedPacks) {
$in: (() => {
const result = []
for (id of Array.from(pack_ids)) {
result.push(ObjectId(id))
}
return result
})()
}
},
{ _id: 1 },
function (err, loadedPacks) {
if (err != null) { if (err != null) {
return callback(err) return callback(err)
} }
@ -430,8 +417,7 @@ module.exports = PackManager = {
return callback() return callback()
} }
) )
} })
)
}, },
// Retrieve all changes across a project // Retrieve all changes across a project
@ -440,7 +426,8 @@ module.exports = PackManager = {
// get all the docHistory Entries // get all the docHistory Entries
return db.docHistory return db.docHistory
.find({ project_id: ObjectId(project_id) }, { pack: false }) .find({ project_id: ObjectId(project_id) }, { pack: false })
.sort({ 'meta.end_ts': -1 }, function (err, packs) { .sort({ 'meta.end_ts': -1 })
.toArray(function (err, packs) {
let pack let pack
if (err != null) { if (err != null) {
return callback(err) return callback(err)
@ -451,9 +438,9 @@ module.exports = PackManager = {
allPacks.push(pack) allPacks.push(pack)
seenIds[pack._id] = true seenIds[pack._id] = true
} }
return db.docHistoryIndex.find( return db.docHistoryIndex
{ project_id: ObjectId(project_id) }, .find({ project_id: ObjectId(project_id) })
function (err, indexes) { .toArray(function (err, indexes) {
if (err != null) { if (err != null) {
return callback(err) return callback(err)
} }
@ -472,8 +459,7 @@ module.exports = PackManager = {
null, null,
new ProjectIterator(allPacks, before, PackManager.getPackById) new ProjectIterator(allPacks, before, PackManager.getPackById)
) )
} })
)
}) })
}, },
@ -499,11 +485,9 @@ module.exports = PackManager = {
increaseTTL(pack, callback) { increaseTTL(pack, callback) {
if (pack.expiresAt < new Date(Date.now() + 6 * DAYS)) { if (pack.expiresAt < new Date(Date.now() + 6 * DAYS)) {
// update cache expiry since we are using this pack // update cache expiry since we are using this pack
return db.docHistory.findAndModify( return db.docHistory.updateOne(
{ { _id: pack._id },
query: { _id: pack._id }, { $set: { expiresAt: new Date(Date.now() + 7 * DAYS) } },
update: { $set: { expiresAt: new Date(Date.now() + 7 * DAYS) } }
},
(err) => callback(err, pack) (err) => callback(err, pack)
) )
} else { } else {
@ -619,7 +603,8 @@ module.exports = PackManager = {
} }
return db.docHistory return db.docHistory
.find(query, { pack: false }) .find(query, { pack: false })
.sort({ v: 1 }, function (err, packs) { .sort({ v: 1 })
.toArray(function (err, packs) {
if (err != null) { if (err != null) {
return callback(err) return callback(err)
} }
@ -644,7 +629,8 @@ module.exports = PackManager = {
} }
return db.docHistory return db.docHistory
.find(query, { pack: false }) .find(query, { pack: false })
.sort({ v: 1 }, function (err, packs) { .sort({ v: 1 })
.toArray(function (err, packs) {
if (err != null) { if (err != null) {
return callback(err) return callback(err)
} }
@ -729,15 +715,15 @@ module.exports = PackManager = {
}, },
_insertPacksIntoIndex(project_id, doc_id, newPacks, callback) { _insertPacksIntoIndex(project_id, doc_id, newPacks, callback) {
return db.docHistoryIndex.findAndModify( return db.docHistoryIndex.updateOne(
{ _id: ObjectId(doc_id.toString()) },
{
$setOnInsert: { project_id: ObjectId(project_id.toString()) },
$push: {
packs: { $each: newPacks, $sort: { v: 1 } }
}
},
{ {
query: { _id: ObjectId(doc_id.toString()) },
update: {
$setOnInsert: { project_id: ObjectId(project_id.toString()) },
$push: {
packs: { $each: newPacks, $sort: { v: 1 } }
}
},
upsert: true upsert: true
}, },
callback callback
@ -995,11 +981,9 @@ module.exports = PackManager = {
_markPackAsFinalised(project_id, doc_id, pack_id, callback) { _markPackAsFinalised(project_id, doc_id, pack_id, callback) {
logger.log({ project_id, doc_id, pack_id }, 'marking pack as finalised') logger.log({ project_id, doc_id, pack_id }, 'marking pack as finalised')
return db.docHistory.findAndModify( return db.docHistory.updateOne(
{ { _id: pack_id },
query: { _id: pack_id }, { $set: { finalised: true } },
update: { $set: { finalised: true } }
},
callback callback
) )
}, },
@ -1020,11 +1004,9 @@ module.exports = PackManager = {
markPackAsChecked(project_id, doc_id, pack_id, callback) { markPackAsChecked(project_id, doc_id, pack_id, callback) {
logger.log({ project_id, doc_id, pack_id }, 'marking pack as checked') logger.log({ project_id, doc_id, pack_id }, 'marking pack as checked')
return db.docHistory.findAndModify( return db.docHistory.updateOne(
{ { _id: pack_id },
query: { _id: pack_id }, { $currentDate: { last_checked: true } },
update: { $currentDate: { last_checked: true } }
},
callback callback
) )
}, },
@ -1117,15 +1099,12 @@ module.exports = PackManager = {
{ project_id, doc_id, pack_id }, { project_id, doc_id, pack_id },
'clearing as archive in progress' 'clearing as archive in progress'
) )
return db.docHistoryIndex.findAndModify( return db.docHistoryIndex.updateOne(
{ {
query: { _id: ObjectId(doc_id.toString()),
_id: ObjectId(doc_id.toString()), packs: { $elemMatch: { _id: pack_id, inS3: false } }
packs: { $elemMatch: { _id: pack_id, inS3: false } }
},
fields: { 'packs.$': 1 },
update: { $unset: { 'packs.$.inS3': true } }
}, },
{ $unset: { 'packs.$.inS3': true } },
callback callback
) )
}, },
@ -1155,12 +1134,13 @@ module.exports = PackManager = {
}, },
setTTLOnArchivedPack(project_id, doc_id, pack_id, callback) { setTTLOnArchivedPack(project_id, doc_id, pack_id, callback) {
return db.docHistory.findAndModify( return db.docHistory.updateOne(
{ { _id: pack_id },
query: { _id: pack_id }, { $set: { expiresAt: new Date(Date.now() + 1 * DAYS) } },
update: { $set: { expiresAt: new Date(Date.now() + 1 * DAYS) } }
},
function (err) { function (err) {
if (err) {
return callback(err)
}
logger.log({ project_id, doc_id, pack_id }, 'set expiry on pack') logger.log({ project_id, doc_id, pack_id }, 'set expiry on pack')
return callback() return callback()
} }

15
services/track-changes/app/js/PackWorker.js
View file

@ -78,18 +78,6 @@ logger.log(
`checking for updates, limit=${LIMIT}, delay=${DOCUMENT_PACK_DELAY}, timeout=${TIMEOUT}` `checking for updates, limit=${LIMIT}, delay=${DOCUMENT_PACK_DELAY}, timeout=${TIMEOUT}`
) )
// work around for https://github.com/mafintosh/mongojs/issues/224
db.close = function (callback) {
return this._getServer(function (err, server) {
if (err != null) {
return callback(err)
}
server = server.destroy != null ? server : server.topology
server.destroy(true, true)
return callback()
})
}
const finish = function () { const finish = function () {
if (shutDownTimer != null) { if (shutDownTimer != null) {
logger.log('cancelling timeout') logger.log('cancelling timeout')
@ -191,7 +179,8 @@ if (pending != null) {
.sort({ .sort({
last_checked: 1 last_checked: 1
}) })
.limit(LIMIT, function (err, results) { .limit(LIMIT)
.toArray(function (err, results) {
if (err != null) { if (err != null) {
logger.log({ err }, 'error checking for updates') logger.log({ err }, 'error checking for updates')
finish() finish()

4
services/track-changes/test/unit/js/MongoManager/MongoManagerTests.js
View file

@ -190,7 +190,7 @@ describe('MongoManager', function () {
beforeEach(function () { beforeEach(function () {
this.metadata = { mock: 'metadata' } this.metadata = { mock: 'metadata' }
this.db.projectHistoryMetaData = { this.db.projectHistoryMetaData = {
find: sinon.stub().callsArgWith(1, null, [this.metadata]) findOne: sinon.stub().callsArgWith(1, null, this.metadata)
} }
return this.MongoManager.getProjectMetaData( return this.MongoManager.getProjectMetaData(
this.project_id, this.project_id,
@ -199,7 +199,7 @@ describe('MongoManager', function () {
}) })
it('should look up the meta data in the db', function () { it('should look up the meta data in the db', function () {
return this.db.projectHistoryMetaData.find return this.db.projectHistoryMetaData.findOne
.calledWith({ project_id: ObjectId(this.project_id) }) .calledWith({ project_id: ObjectId(this.project_id) })
.should.equal(true) .should.equal(true)
}) })

19
services/track-changes/test/unit/js/PackManager/PackManagerTests.js
View file

@ -67,6 +67,7 @@ describe('PackManager', function () {
return (this.db.docHistory = { return (this.db.docHistory = {
save: sinon.stub().callsArg(1), save: sinon.stub().callsArg(1),
insert: sinon.stub().callsArg(1), insert: sinon.stub().callsArg(1),
updateOne: sinon.stub().yields(),
findAndModify: sinon.stub().callsArg(1) findAndModify: sinon.stub().callsArg(1)
}) })
}) })
@ -442,23 +443,21 @@ describe('PackManager', function () {
return describe('for a small update that will expire', function () { return describe('for a small update that will expire', function () {
it('should append the update in mongo', function () { it('should append the update in mongo', function () {
return this.db.docHistory.findAndModify return this.db.docHistory.updateOne
.calledWithMatch({ .calledWithMatch(
query: { _id: this.lastUpdate._id }, { _id: this.lastUpdate._id },
update: { {
$push: { pack: { $each: this.newUpdates } }, $push: { pack: { $each: this.newUpdates } },
$set: { v_end: this.newUpdates[this.newUpdates.length - 1].v } $set: { v_end: this.newUpdates[this.newUpdates.length - 1].v }
} }
}) )
.should.equal(true) .should.equal(true)
}) })
it('should set an expiry time in the future', function () { it('should set an expiry time in the future', function () {
return this.db.docHistory.findAndModify return this.db.docHistory.updateOne
.calledWithMatch({ .calledWithMatch(sinon.match.any, {
update: { $set: { expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000) }
$set: { expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000) }
}
}) })
.should.equal(true) .should.equal(true)
}) })