mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-21 20:47:08 -05:00
252533b2fd
[mongo-utils] create new home for batchedUpdate helper GitOrigin-RevId: 9f61c5e367a9f4cef63b5cc4c0bbbd3ef57c8ca8
102 lines
2.8 KiB
JavaScript
102 lines
2.8 KiB
JavaScript
import mongodb from 'mongodb-legacy'
|
|
import { promiseMapWithLimit } from '@overleaf/promise-utils'
|
|
import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
|
|
import ChatApiHandler from '../app/src/Features/Chat/ChatApiHandler.js'
|
|
import DeleteOrphanedDataHelper from './delete_orphaned_data_helper.mjs'
|
|
import { ensureMongoTimeout } from './helpers/env_variable_helper.mjs'
|
|
import { db } from '../app/src/infrastructure/mongodb.js'
|
|
|
|
const { ObjectId } = mongodb
|
|
const { getHardDeletedProjectIds } = DeleteOrphanedDataHelper
|
|
|
|
const READ_CONCURRENCY_SECONDARY =
|
|
parseInt(process.env.READ_CONCURRENCY_SECONDARY, 10) || 1000
|
|
const READ_CONCURRENCY_PRIMARY =
|
|
parseInt(process.env.READ_CONCURRENCY_PRIMARY, 10) || 500
|
|
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
|
|
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
|
|
const DRY_RUN = process.env.DRY_RUN !== 'false'
|
|
const MAX_CHATS_TO_DESTROY =
|
|
parseInt(process.env.MAX_CHATS_TO_DESTROY, 10) || false
|
|
// persist fallback in order to keep batchedUpdate in-sync
|
|
process.env.BATCH_SIZE = BATCH_SIZE
|
|
// ensure set mongo timeout to 10mins if otherwise unspecified
|
|
if (!process.env.MONGO_SOCKET_TIMEOUT) {
|
|
ensureMongoTimeout(600000)
|
|
}
|
|
|
|
console.log({
|
|
DRY_RUN,
|
|
WRITE_CONCURRENCY,
|
|
BATCH_SIZE,
|
|
MAX_CHATS_TO_DESTROY,
|
|
})
|
|
|
|
const RESULT = {
|
|
DRY_RUN,
|
|
projectChatsDestroyed: 0,
|
|
continueFrom: null,
|
|
}
|
|
|
|
async function processBatch(rooms) {
|
|
if (rooms.length && rooms[0]._id) {
|
|
RESULT.continueFrom = rooms[0]._id
|
|
}
|
|
const projectIds = Array.from(
|
|
new Set(rooms.map(room => room.project_id.toString()))
|
|
).map(id => new ObjectId(id))
|
|
console.log(
|
|
`Checking projects (${projectIds.length})`,
|
|
JSON.stringify(projectIds)
|
|
)
|
|
|
|
const projectsWithOrphanedChat = await getHardDeletedProjectIds({
|
|
projectIds,
|
|
READ_CONCURRENCY_PRIMARY,
|
|
READ_CONCURRENCY_SECONDARY,
|
|
})
|
|
|
|
console.log(
|
|
`Destroying chat for projects (${projectsWithOrphanedChat.length})`,
|
|
JSON.stringify(projectsWithOrphanedChat)
|
|
)
|
|
if (!DRY_RUN) {
|
|
await promiseMapWithLimit(
|
|
WRITE_CONCURRENCY,
|
|
projectsWithOrphanedChat,
|
|
ChatApiHandler.promises.destroyProject
|
|
)
|
|
}
|
|
RESULT.projectChatsDestroyed += projectsWithOrphanedChat.length
|
|
|
|
console.log(RESULT)
|
|
if (
|
|
MAX_CHATS_TO_DESTROY &&
|
|
RESULT.projectChatsDestroyed >= MAX_CHATS_TO_DESTROY
|
|
) {
|
|
console.log(
|
|
`MAX_CHATS_TO_DELETE limit (${MAX_CHATS_TO_DESTROY}) reached. Stopping.`
|
|
)
|
|
process.exit(0)
|
|
}
|
|
}
|
|
|
|
async function main() {
|
|
const projection = {
|
|
_id: 1,
|
|
project_id: 1,
|
|
}
|
|
await batchedUpdate(db.rooms, {}, processBatch, projection)
|
|
console.log('Final')
|
|
console.log(RESULT)
|
|
}
|
|
|
|
main()
|
|
.then(() => {
|
|
console.log('Done.')
|
|
process.exit(0)
|
|
})
|
|
.catch(error => {
|
|
console.error({ error })
|
|
process.exit(1)
|
|
})
|