const Settings = require('@overleaf/settings') const logger = require('@overleaf/logger') const rclient = require('@overleaf/redis-wrapper').createClient( Settings.redis.documentupdater ) const keys = Settings.redis.documentupdater.key_schema const ProjectFlusher = require('app/js/ProjectFlusher') const DocumentManager = require('app/js/DocumentManager') const util = require('util') const flushAndDeleteDocWithLock = util.promisify( DocumentManager.flushAndDeleteDocWithLock ) const { MongoClient, ObjectId } = require('mongodb') const clientPromise = MongoClient.connect( Settings.mongo.url, Settings.mongo.options ) const db = {} clientPromise.then(client => { db.docs = client.db().collection('docs') }) async function fixDocsWithMissingProjectIds(dockeys, options) { const docIds = ProjectFlusher._extractIds(dockeys) for (const docId of docIds) { const projectId = await rclient.get(keys.projectKey({ doc_id: docId })) logger.debug({ docId, projectId }, 'checking doc') if (!projectId) { try { await insertMissingProjectId(docId, options) } catch (err) { logger.error({ docId, err }, 'error fixing doc without project id') } } } } async function insertMissingProjectId(docId, options) { const doc = await db.docs.findOne({ _id: ObjectId(docId) }) if (!doc) { logger.warn({ docId }, 'doc not found in mongo') return } if (!doc.project_id) { logger.error({ docId }, 'doc does not have project id in mongo') return } logger.debug({ docId, doc }, 'found doc') const projectIdFromMongo = doc.project_id.toString() if (options.dryRun) { logger.info( { projectIdFromMongo, docId }, 'dry run mode - would insert project id in redis' ) return } // set the project id for this doc await rclient.set(keys.projectKey({ doc_id: docId }), projectIdFromMongo) logger.debug({ docId, projectIdFromMongo }, 'inserted project id in redis') if (projectIdFromMongo) { await flushAndDeleteDocWithLock(projectIdFromMongo, docId, {}) logger.info( { docId, projectIdFromMongo }, 'fixed doc with empty project id' ) } return projectIdFromMongo } async function findAndProcessDocs(options) { logger.info({ options }, 'fixing docs with missing projcct id') let cursor = 0 do { const [newCursor, doclinesKeys] = await rclient.scan( cursor, 'MATCH', keys.docLines({ doc_id: '*' }), 'COUNT', options.limit ) await fixDocsWithMissingProjectIds(doclinesKeys, options) cursor = newCursor } while (cursor !== '0') } clientPromise.then(client => { findAndProcessDocs({ limit: 1000, dryRun: process.env.DRY_RUN !== 'false' }) .then(result => { rclient.quit() client.close() console.log('DONE') }) .catch(function (error) { console.error(error) process.exit(1) }) })