mirror of
https://github.com/overleaf/overleaf.git
synced 2024-09-16 02:52:31 -04:00
00e59a9afe
[scripts] batchedUpdate: try to work around flaky mongo connection setup GitOrigin-RevId: 2bba7117b8f8892a41703569fef3f97d5e8d6835
61 lines
1.8 KiB
JavaScript
61 lines
1.8 KiB
JavaScript
const { promisify } = require('util')
|
|
const { ReadPreference } = require('mongodb')
|
|
const { getNativeDb } = require('../../app/src/infrastructure/Mongoose')
|
|
|
|
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 1000
|
|
|
|
async function getNextBatch(collection, query, maxId) {
|
|
if (maxId) {
|
|
query['_id'] = { $gt: maxId }
|
|
}
|
|
const entries = await collection
|
|
.find(query, { _id: 1 })
|
|
.sort({ _id: 1 })
|
|
.limit(BATCH_SIZE)
|
|
.setReadPreference(ReadPreference.SECONDARY)
|
|
.toArray()
|
|
return entries.map(entry => entry._id)
|
|
}
|
|
|
|
async function performUpdate(collection, nextBatch, update) {
|
|
return collection.updateMany({ _id: { $in: nextBatch } }, update)
|
|
}
|
|
|
|
async function batchedUpdate(collectionName, query, update) {
|
|
// Apparently the mongo driver returns the connection too early.
|
|
// Some secondary connections are not ready as it returns, leading to
|
|
// failing cursor actions with a readPreference set to 'secondary'.
|
|
// TODO(das7pad): revisit/remove this delay after the mongo-driver update.
|
|
await Promise.all([getNativeDb(), promisify(setTimeout)(10 * 1000)])
|
|
|
|
const db = await getNativeDb()
|
|
const collection = db.collection(collectionName)
|
|
|
|
let nextBatch
|
|
let updated = 0
|
|
let maxId
|
|
while ((nextBatch = await getNextBatch(collection, query, maxId)).length) {
|
|
maxId = nextBatch[nextBatch.length - 1]
|
|
updated += nextBatch.length
|
|
console.log(JSON.stringify(nextBatch))
|
|
await performUpdate(collection, nextBatch, update)
|
|
}
|
|
return updated
|
|
}
|
|
|
|
function batchedUpdateWithResultHandling(collection, query, update) {
|
|
batchedUpdate(collection, query, update)
|
|
.then(updated => {
|
|
console.error({ updated })
|
|
process.exit(0)
|
|
})
|
|
.catch(error => {
|
|
console.error({ error })
|
|
process.exit(1)
|
|
})
|
|
}
|
|
|
|
module.exports = {
|
|
batchedUpdate,
|
|
batchedUpdateWithResultHandling
|
|
}
|