Merge pull request #20679 from overleaf/bg-issue19022

Move flush_all_projects endpoint in document-updater into script

GitOrigin-RevId: cb774d860b5928b7fece1a8e21b0b76aecae73ff

package-lock.json

@@ -38056,6 +38056,7 @@
         "diff-match-patch": "overleaf/diff-match-patch#89805f9c671a77a263fc53461acd62aa7498f688",
         "express": "^4.21.0",
         "lodash": "^4.17.21",
+        "minimist": "^1.2.8",
         "mongodb-legacy": "^6.0.1",
         "request": "^2.88.2",
         "requestretry": "^7.1.0"
@@ -48796,6 +48797,7 @@
         "diff-match-patch": "overleaf/diff-match-patch#89805f9c671a77a263fc53461acd62aa7498f688",
         "express": "^4.21.0",
         "lodash": "^4.17.21",
+        "minimist": "^1.2.8",
         "mocha": "^10.2.0",
         "mongodb-legacy": "^6.0.1",
         "request": "^2.88.2",

services/document-updater/app.js

@@ -187,7 +187,6 @@ app.delete(
 app.post('/project/:project_id/block', HttpController.blockProject)
 app.post('/project/:project_id/unblock', HttpController.unblockProject)
 
-app.get('/flush_all_projects', HttpController.flushAllProjects)
 app.get('/flush_queued_projects', HttpController.flushQueuedProjects)
 
 app.get('/total', (req, res, next) => {

services/document-updater/app/js/HttpController.js

@@ -6,7 +6,6 @@ const Errors = require('./Errors')
 const logger = require('@overleaf/logger')
 const Settings = require('@overleaf/settings')
 const Metrics = require('./Metrics')
-const ProjectFlusher = require('./ProjectFlusher')
 const DeleteQueueManager = require('./DeleteQueueManager')
 const { getTotalSizeOfLines } = require('./Limits')
 const async = require('async')
@@ -409,23 +408,6 @@ function resyncProjectHistory(req, res, next) {
   )
 }
 
-function flushAllProjects(req, res, next) {
-  res.setTimeout(5 * 60 * 1000)
-  const options = {
-    limit: req.query.limit || 1000,
-    concurrency: req.query.concurrency || 5,
-    dryRun: req.query.dryRun || false,
-  }
-  ProjectFlusher.flushAllProjects(options, (err, projectIds) => {
-    if (err) {
-      logger.err({ err }, 'error bulk flushing projects')
-      res.sendStatus(500)
-    } else {
-      res.send(projectIds)
-    }
-  })
-}
-
 function flushQueuedProjects(req, res, next) {
   res.setTimeout(10 * 60 * 1000)
   const options = {
@@ -490,7 +472,6 @@ module.exports = {
   deleteComment,
   updateProject,
   resyncProjectHistory,
-  flushAllProjects,
   flushQueuedProjects,
   blockProject,
   unblockProject,

services/document-updater/package.json

@@ -32,6 +32,7 @@
     "diff-match-patch": "overleaf/diff-match-patch#89805f9c671a77a263fc53461acd62aa7498f688",
     "express": "^4.21.0",
     "lodash": "^4.17.21",
+    "minimist": "^1.2.8",
     "mongodb-legacy": "^6.0.1",
     "request": "^2.88.2",
     "requestretry": "^7.1.0"

services/document-updater/scripts/flush_all.js

@@ -1,12 +1,38 @@
 const ProjectFlusher = require('../app/js/ProjectFlusher')
+const minimist = require('minimist')
 
 async function main() {
-  console.log('Flushing all projects')
+  const argv = minimist(process.argv.slice(2), {
-  return await new Promise((resolve, reject) => {
+    default: {
-    const options = {
       limit: 100000,
       concurrency: 5,
-    }
+      'dry-run': false,
+    },
+    boolean: ['dry-run', 'help'],
+    alias: { h: 'help', n: 'dry-run', j: 'concurrency' },
+  })
+
+  if (argv.help) {
+    console.log(`
+Usage: node scripts/flush_all.js [options]
+
+Options:
+  --limit            Number of projects to flush (default: 100000)
+  --concurrency, -j  Number of concurrent flush operations (default: 5)
+  --dryRun, -n       Perform a dry run without making any changes (default: false)
+  --help, -h         Show this help message
+    `)
+    process.exit(0)
+  }
+
+  const options = {
+    limit: argv.limit,
+    concurrency: argv.concurrency,
+    dryRun: argv['dry-run'],
+  }
+  console.log('Flushing all projects with options:', options)
+
+  return await new Promise((resolve, reject) => {
     ProjectFlusher.flushAllProjects(options, err => {
       if (err) {
         reject(err)

services/document-updater/test/unit/js/HttpController/HttpControllerTests.js

@@ -13,7 +13,6 @@ describe('HttpController', function () {
         }),
         './ProjectHistoryRedisManager': (this.ProjectHistoryRedisManager = {}),
         './ProjectManager': (this.ProjectManager = {}),
-        './ProjectFlusher': { flushAllProjects() {} },
         './DeleteQueueManager': (this.DeleteQueueManager = {}),
         './RedisManager': (this.RedisManager = {
           DOC_OPS_TTL: 42,