github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

[web/scripts] Process users' projects with promiseMapWithLimit() (#9807)

Browse source 
* [web/scripts] Process users' projects with `promiseMapWithLimit()`
* [web/scripts] Added several optimisations to migration script

GitOrigin-RevId: 0dd221a3a8b4bb64f412636faa17e81249ca6c4f
This commit is contained in:
Miguel Serrano 2022-10-04 19:48:54 +02:00 committed by Copybot
parent b03380c0b2
commit 877638f09c
2 changed files with 41 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
services/web/migrations/20220913125500_migrate_auditLog_to_collections.js
View file

@ -4,7 +4,7 @@ exports.tags = ['server-ce', 'server-pro']
exports.migrate = async () => { exports.migrate = async () => {
const options = { const options = {
writeConcurrency: 10, writeConcurrency: 5,
dryRun: false, dryRun: false,
} }
await runScript(options) await runScript(options)

69
services/web/scripts/migrate_audit_logs.js
View file

@ -27,14 +27,22 @@ async function main(options) {
{ _id: ObjectId(options.projectId) }, { _id: ObjectId(options.projectId) },
{ _id: 1, auditLog: 1 } { _id: 1, auditLog: 1 }
) )
await processProject(project, options) if (!project || !project.auditLog) {
console.error('unable to process project', project)
return
}
await processProjectsBatch([project], options)
} else if (options.userId) { } else if (options.userId) {
console.log('migrating userId=' + options.userId) console.log('migrating userId=' + options.userId)
const user = await db.users.findOne( const user = await db.users.findOne(
{ _id: ObjectId(options.userId) }, { _id: ObjectId(options.userId) },
{ _id: 1, auditLog: 1 } { _id: 1, auditLog: 1 }
) )
await processUser(user, options) if (!user || !user.auditLog) {
console.error('unable to process user', user)
return
}
await processUsersBatch([user], options)
} else { } else {
await batchedUpdate( await batchedUpdate(
'users', 'users',
@ -60,51 +68,54 @@ async function main(options) {
} }
async function processUsersBatch(users, options) { async function processUsersBatch(users, options) {
await promiseMapWithLimit(options.writeConcurrency, users, async user => { if (!users || users.length <= 0) {
await processUser(user, options) return
}) }
}
const entries = users
.map(user => user.auditLog.map(log => ({ ...log, userId: user._id })))
.flat()
async function processUser(user, options) {
const entries = user.auditLog.map(log => ({ ...log, userId: user._id }))
if (!options.dryRun && entries?.length > 0) { if (!options.dryRun && entries?.length > 0) {
await db.userAuditLogEntries.insertMany(entries) await db.userAuditLogEntries.insertMany(entries)
} }
if (!options.dryRun) { if (!options.dryRun) {
await db.users.updateOne({ _id: user._id }, { $unset: { auditLog: 1 } }) const userIds = users.map(user => user._id)
await db.users.updateMany(
{ _id: { $in: userIds } },
{ $unset: { auditLog: 1 } }
)
} }
const projects = await db.projects.find( await promiseMapWithLimit(options.writeConcurrency, users, async user => {
{ owner_ref: user._id, auditLog: { $exists: true } }, const projects = await db.projects.find(
{ _id: 1, auditLog: 1 } { owner_ref: user._id, auditLog: { $exists: true } },
) { _id: 1, auditLog: 1 }
projects.forEach(project => processProject(project, options)) )
await processProjectsBatch(projects, options)
})
} }
async function processProjectsBatch(projects, options) { async function processProjectsBatch(projects, options) {
await promiseMapWithLimit( if (!projects || projects.length <= 0) {
options.writeConcurrency, return
projects, }
async project => {
await processProject(project, options)
}
)
}
async function processProject(project, options) { const entries = projects
const entries = project.auditLog.map(log => ({ .map(project =>
...log, project.auditLog.map(log => ({ ...log, projectId: project._id }))
projectId: project._id, )
})) .flat()
if (!options.dryRun && entries?.length > 0) { if (!options.dryRun && entries?.length > 0) {
await db.projectAuditLogEntries.insertMany(entries) await db.projectAuditLogEntries.insertMany(entries)
} }
if (!options.dryRun) { if (!options.dryRun) {
await db.projects.updateOne( const projectIds = projects.map(project => project._id)
{ _id: project._id }, await db.projects.updateMany(
{ _id: { $in: projectIds } },
{ $unset: { auditLog: 1 } } { $unset: { auditLog: 1 } }
) )
} }