Merge pull request #16368 from overleaf/em-remove-project-archiver-code

Remove project-archiver related code

GitOrigin-RevId: b3331033658c14a7c4d8745fd7085cd7c65c94bc

services/web/app/src/Features/Project/ProjectDeleter.js

@@ -17,7 +17,6 @@ const DocstoreManager = require('../Docstore/DocstoreManager')
 const EditorRealTimeController = require('../Editor/EditorRealTimeController')
 const HistoryManager = require('../History/HistoryManager')
 const FilestoreHandler = require('../FileStore/FileStoreHandler')
-const TpdsUpdateSender = require('../ThirdPartyDataStore/TpdsUpdateSender')
 const ChatApiHandler = require('../Chat/ChatApiHandler')
 const moment = require('moment')
 const { promiseMapWithLimit } = require('@overleaf/promise-utils')
@@ -382,9 +381,6 @@ async function expireDeletedProject(projectId) {
         historyId
       ),
       FilestoreHandler.promises.deleteProject(deletedProject.project._id),
-      TpdsUpdateSender.promises.deleteProject({
-        projectId: deletedProject.project._id,
-      }),
       ChatApiHandler.promises.destroyProject(deletedProject.project._id),
       hardDeleteDeletedFiles(deletedProject.project._id),
       ProjectAuditLogEntry.deleteMany({ projectId }),

services/web/app/src/Features/ThirdPartyDataStore/TpdsUpdateSender.js

@@ -169,35 +169,6 @@ async function createProject(params) {
   await enqueue(userId, 'standardHttpRequest', job)
 }
 
-async function deleteProject(params) {
-  const { projectId } = params
-  // deletion only applies to project archiver
-  const projectArchiverUrl = _.get(settings, [
-    'apis',
-    'project_archiver',
-    'url',
-  ])
-  // silently do nothing if project archiver url is not in settings
-  if (!projectArchiverUrl) {
-    return
-  }
-  metrics.inc('tpds.delete-project')
-  // send the request directly to project archiver, bypassing third-party-datastore
-  try {
-    await fetchNothing(
-      `${settings.apis.project_archiver.url}/project/${projectId}`,
-      { method: 'DELETE' }
-    )
-    return true
-  } catch (err) {
-    logger.error(
-      { err, projectId },
-      'error deleting project in third party datastore (project_archiver)'
-    )
-    return false
-  }
-}
-
 async function enqueue(group, method, job) {
   const tpdsWorkerUrl = _.get(settings, ['apis', 'tpdsworker', 'url'])
   // silently do nothing if worker url is not in settings
@@ -296,7 +267,6 @@ const TpdsUpdateSender = {
   addFile: callbackify(addFile),
   deleteEntity: callbackify(deleteEntity),
   createProject: callbackify(createProject),
-  deleteProject: callbackify(deleteProject),
   enqueue: callbackify(enqueue),
   moveEntity: callbackify(moveEntity),
   pollDropboxForUser: callbackify(pollDropboxForUser),
@@ -306,7 +276,6 @@ const TpdsUpdateSender = {
     addFile,
     deleteEntity,
     createProject,
-    deleteProject,
     enqueue,
     moveEntity,
     pollDropboxForUser,

services/web/scripts/delete_orphaned_project_archives.js

@@ -1,134 +0,0 @@
-const Settings = require('@overleaf/settings')
-const { fetchJson } = require('@overleaf/fetch-utils')
-const { waitForDb } = require('../app/src/infrastructure/mongodb')
-const { promiseMapWithLimit } = require('@overleaf/promise-utils')
-const { getHardDeletedProjectIds } = require('./delete_orphaned_data_helper')
-const TpdsUpdateSender = require('../app/src/Features/ThirdPartyDataStore/TpdsUpdateSender')
-const { promisify } = require('util')
-const { ObjectId } = require('mongodb')
-const sleep = promisify(setTimeout)
-
-const START_OFFSET = process.env.START_OFFSET
-
-const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 1000
-const DRY_RUN = process.env.DRY_RUN !== 'false'
-const READ_CONCURRENCY_SECONDARY =
-  parseInt(process.env.READ_CONCURRENCY_SECONDARY, 10) || 1000
-const READ_CONCURRENCY_PRIMARY =
-  parseInt(process.env.READ_CONCURRENCY_PRIMARY, 10) || 500
-const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
-
-const LET_USER_DOUBLE_CHECK_INPUTS_FOR =
-  parseInt(process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR, 10) || 10 * 1000
-
-async function main() {
-  await letUserDoubleCheckInputs()
-  await waitForDb()
-
-  let processed = 0
-  let hardDeleted = 0
-  let pageToken = ''
-  let startOffset = START_OFFSET
-  while (pageToken !== undefined) {
-    const url = new URL(`${Settings.apis.project_archiver.url}/project/list`)
-    url.searchParams.append('pageToken', pageToken)
-    url.searchParams.append('startOffset', startOffset)
-    const { nextPageToken, entries } = await fetchJson(url)
-    pageToken = nextPageToken
-    startOffset = undefined
-
-    hardDeleted += await processBatch(entries)
-    processed += entries.length
-    console.log(
-      'processed:',
-      processed.toString().padStart(10, '0'),
-      'hard deleted:',
-      hardDeleted.toString().padStart(10, '0'),
-      'nextPageToken:',
-      nextPageToken,
-      'START_OFFSET:',
-      entries.pop()?.prefix
-    )
-  }
-}
-
-async function processBatch(entries) {
-  const projectIdToPrefix = new Map()
-  for (const { prefix, projectId } of entries) {
-    const prefixes = projectIdToPrefix.get(projectId) || []
-    prefixes.push(prefix)
-    projectIdToPrefix.set(projectId, prefixes)
-  }
-  const projectIds = Array.from(projectIdToPrefix.keys()).map(
-    id => new ObjectId(id)
-  )
-  const projectsWithOrphanedArchive = await getHardDeletedProjectIds({
-    projectIds,
-    READ_CONCURRENCY_PRIMARY,
-    READ_CONCURRENCY_SECONDARY,
-  })
-
-  await promiseMapWithLimit(
-    WRITE_CONCURRENCY,
-    projectsWithOrphanedArchive.flatMap(id =>
-      projectIdToPrefix.get(id.toString())
-    ),
-    hardDeleteProjectArchiverData
-  )
-  return projectsWithOrphanedArchive.length
-}
-
-async function hardDeleteProjectArchiverData(prefix) {
-  console.log(`Destroying hard deleted project archive at '${prefix}/'`)
-  if (DRY_RUN) return
-
-  for (let i = 0; i < 10; i++) {
-    await sleep(1000 * i)
-    try {
-      const ok = await TpdsUpdateSender.promises.deleteProject({
-        projectId: encodeURIComponent(prefix),
-      })
-      if (ok) {
-        return
-      }
-    } catch (e) {
-      console.error(`deletion failed for '${prefix}/'`, e)
-    }
-  }
-  throw new Error(`deletion failed for '${prefix}/', check logs`)
-}
-
-async function letUserDoubleCheckInputs() {
-  console.error(
-    'Options:',
-    JSON.stringify(
-      {
-        BATCH_SIZE,
-        DRY_RUN,
-        LET_USER_DOUBLE_CHECK_INPUTS_FOR,
-        READ_CONCURRENCY_SECONDARY,
-        READ_CONCURRENCY_PRIMARY,
-        START_OFFSET,
-        WRITE_CONCURRENCY,
-      },
-      null,
-      2
-    )
-  )
-  console.error(
-    'Waiting for you to double check inputs for',
-    LET_USER_DOUBLE_CHECK_INPUTS_FOR,
-    'ms'
-  )
-  await sleep(LET_USER_DOUBLE_CHECK_INPUTS_FOR)
-}
-
-main()
-  .then(() => {
-    console.log('Done.')
-    process.exit(0)
-  })
-  .catch(error => {
-    console.error({ error })
-    process.exit(1)
-  })

services/web/test/unit/src/Project/ProjectDeleterTests.js

@@ -127,11 +127,6 @@ describe('ProjectDeleter', function () {
         deleteProject: sinon.stub().resolves(),
       },
     }
-    this.TpdsUpdateSender = {
-      promises: {
-        deleteProject: sinon.stub().resolves(),
-      },
-    }
     this.Features = {
       hasFeature: sinon.stub().returns(true),
     }
@@ -154,7 +149,6 @@ describe('ProjectDeleter', function () {
           this.DocumentUpdaterHandler,
         '../Tags/TagsHandler': this.TagsHandler,
         '../FileStore/FileStoreHandler': this.FileStoreHandler,
-        '../ThirdPartyDataStore/TpdsUpdateSender': this.TpdsUpdateSender,
         '../Chat/ChatApiHandler': this.ChatApiHandler,
         '../Collaborators/CollaboratorsHandler': this.CollaboratorsHandler,
         '../Collaborators/CollaboratorsGetter': this.CollaboratorsGetter,
@@ -496,14 +490,6 @@ describe('ProjectDeleter', function () {
         ).to.have.been.calledWith(this.deletedProjects[0].project._id)
       })
 
-      it('should destroy the files in project-archiver', function () {
-        expect(
-          this.TpdsUpdateSender.promises.deleteProject
-        ).to.have.been.calledWith({
-          projectId: this.deletedProjects[0].project._id,
-        })
-      })
-
       it('should destroy the chat threads and messages', function () {
         expect(
           this.ChatApiHandler.promises.destroyProject
@@ -553,11 +539,6 @@ describe('ProjectDeleter', function () {
           .called
       })
 
-      it('should not destroy the files in project-archiver', function () {
-        expect(this.TpdsUpdateSender.promises.deleteProject).to.not.have.been
-          .called
-      })
-
       it('should not destroy the chat threads and messages', function () {
         expect(this.ChatApiHandler.promises.destroyProject).to.not.have.been
           .called

services/web/test/unit/src/ThirdPartyDataStore/TpdsUpdateSenderTests.js

@@ -18,7 +18,6 @@ const projectName = 'project_name_here'
 const thirdPartyDataStoreApiUrl = 'http://third-party-json-store.herokuapp.com'
 const siteUrl = 'http://www.localhost:3000'
 const filestoreUrl = 'filestore.sharelatex.com'
-const projectArchiverUrl = 'project-archiver.overleaf.com'
 
 describe('TpdsUpdateSender', function () {
   beforeEach(function () {
@@ -440,21 +439,6 @@ describe('TpdsUpdateSender', function () {
     })
   })
 
-  describe('deleteProject', function () {
-    it('should not call request if there is no project archiver url', async function () {
-      await this.TpdsUpdateSender.promises.deleteProject({ projectId })
-      this.FetchUtils.fetchNothing.should.not.have.been.called
-    })
-    it('should make a delete request to project archiver', async function () {
-      this.settings.apis.project_archiver = { url: projectArchiverUrl }
-      await this.TpdsUpdateSender.promises.deleteProject({ projectId })
-      this.FetchUtils.fetchNothing.should.have.been.calledWith(
-        `${projectArchiverUrl}/project/${projectId}`,
-        { method: 'DELETE' }
-      )
-    })
-  })
-
   describe('user not linked to dropbox', function () {
     beforeEach(function () {
       this.UserGetter.promises.getUsers