diff --git a/services/web/app/src/Features/Project/ProjectDeleter.js b/services/web/app/src/Features/Project/ProjectDeleter.js index f159e2bb76..91b742befb 100644 --- a/services/web/app/src/Features/Project/ProjectDeleter.js +++ b/services/web/app/src/Features/Project/ProjectDeleter.js @@ -17,7 +17,6 @@ const DocstoreManager = require('../Docstore/DocstoreManager') const EditorRealTimeController = require('../Editor/EditorRealTimeController') const HistoryManager = require('../History/HistoryManager') const FilestoreHandler = require('../FileStore/FileStoreHandler') -const TpdsUpdateSender = require('../ThirdPartyDataStore/TpdsUpdateSender') const ChatApiHandler = require('../Chat/ChatApiHandler') const moment = require('moment') const { promiseMapWithLimit } = require('@overleaf/promise-utils') @@ -382,9 +381,6 @@ async function expireDeletedProject(projectId) { historyId ), FilestoreHandler.promises.deleteProject(deletedProject.project._id), - TpdsUpdateSender.promises.deleteProject({ - projectId: deletedProject.project._id, - }), ChatApiHandler.promises.destroyProject(deletedProject.project._id), hardDeleteDeletedFiles(deletedProject.project._id), ProjectAuditLogEntry.deleteMany({ projectId }), diff --git a/services/web/app/src/Features/ThirdPartyDataStore/TpdsUpdateSender.js b/services/web/app/src/Features/ThirdPartyDataStore/TpdsUpdateSender.js index 4339260308..7afd5720ce 100644 --- a/services/web/app/src/Features/ThirdPartyDataStore/TpdsUpdateSender.js +++ b/services/web/app/src/Features/ThirdPartyDataStore/TpdsUpdateSender.js @@ -169,35 +169,6 @@ async function createProject(params) { await enqueue(userId, 'standardHttpRequest', job) } -async function deleteProject(params) { - const { projectId } = params - // deletion only applies to project archiver - const projectArchiverUrl = _.get(settings, [ - 'apis', - 'project_archiver', - 'url', - ]) - // silently do nothing if project archiver url is not in settings - if (!projectArchiverUrl) { - return - } - metrics.inc('tpds.delete-project') - // send the request directly to project archiver, bypassing third-party-datastore - try { - await fetchNothing( - `${settings.apis.project_archiver.url}/project/${projectId}`, - { method: 'DELETE' } - ) - return true - } catch (err) { - logger.error( - { err, projectId }, - 'error deleting project in third party datastore (project_archiver)' - ) - return false - } -} - async function enqueue(group, method, job) { const tpdsWorkerUrl = _.get(settings, ['apis', 'tpdsworker', 'url']) // silently do nothing if worker url is not in settings @@ -296,7 +267,6 @@ const TpdsUpdateSender = { addFile: callbackify(addFile), deleteEntity: callbackify(deleteEntity), createProject: callbackify(createProject), - deleteProject: callbackify(deleteProject), enqueue: callbackify(enqueue), moveEntity: callbackify(moveEntity), pollDropboxForUser: callbackify(pollDropboxForUser), @@ -306,7 +276,6 @@ const TpdsUpdateSender = { addFile, deleteEntity, createProject, - deleteProject, enqueue, moveEntity, pollDropboxForUser, diff --git a/services/web/scripts/delete_orphaned_project_archives.js b/services/web/scripts/delete_orphaned_project_archives.js deleted file mode 100644 index 7781574214..0000000000 --- a/services/web/scripts/delete_orphaned_project_archives.js +++ /dev/null @@ -1,134 +0,0 @@ -const Settings = require('@overleaf/settings') -const { fetchJson } = require('@overleaf/fetch-utils') -const { waitForDb } = require('../app/src/infrastructure/mongodb') -const { promiseMapWithLimit } = require('@overleaf/promise-utils') -const { getHardDeletedProjectIds } = require('./delete_orphaned_data_helper') -const TpdsUpdateSender = require('../app/src/Features/ThirdPartyDataStore/TpdsUpdateSender') -const { promisify } = require('util') -const { ObjectId } = require('mongodb') -const sleep = promisify(setTimeout) - -const START_OFFSET = process.env.START_OFFSET - -const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 1000 -const DRY_RUN = process.env.DRY_RUN !== 'false' -const READ_CONCURRENCY_SECONDARY = - parseInt(process.env.READ_CONCURRENCY_SECONDARY, 10) || 1000 -const READ_CONCURRENCY_PRIMARY = - parseInt(process.env.READ_CONCURRENCY_PRIMARY, 10) || 500 -const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10 - -const LET_USER_DOUBLE_CHECK_INPUTS_FOR = - parseInt(process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR, 10) || 10 * 1000 - -async function main() { - await letUserDoubleCheckInputs() - await waitForDb() - - let processed = 0 - let hardDeleted = 0 - let pageToken = '' - let startOffset = START_OFFSET - while (pageToken !== undefined) { - const url = new URL(`${Settings.apis.project_archiver.url}/project/list`) - url.searchParams.append('pageToken', pageToken) - url.searchParams.append('startOffset', startOffset) - const { nextPageToken, entries } = await fetchJson(url) - pageToken = nextPageToken - startOffset = undefined - - hardDeleted += await processBatch(entries) - processed += entries.length - console.log( - 'processed:', - processed.toString().padStart(10, '0'), - 'hard deleted:', - hardDeleted.toString().padStart(10, '0'), - 'nextPageToken:', - nextPageToken, - 'START_OFFSET:', - entries.pop()?.prefix - ) - } -} - -async function processBatch(entries) { - const projectIdToPrefix = new Map() - for (const { prefix, projectId } of entries) { - const prefixes = projectIdToPrefix.get(projectId) || [] - prefixes.push(prefix) - projectIdToPrefix.set(projectId, prefixes) - } - const projectIds = Array.from(projectIdToPrefix.keys()).map( - id => new ObjectId(id) - ) - const projectsWithOrphanedArchive = await getHardDeletedProjectIds({ - projectIds, - READ_CONCURRENCY_PRIMARY, - READ_CONCURRENCY_SECONDARY, - }) - - await promiseMapWithLimit( - WRITE_CONCURRENCY, - projectsWithOrphanedArchive.flatMap(id => - projectIdToPrefix.get(id.toString()) - ), - hardDeleteProjectArchiverData - ) - return projectsWithOrphanedArchive.length -} - -async function hardDeleteProjectArchiverData(prefix) { - console.log(`Destroying hard deleted project archive at '${prefix}/'`) - if (DRY_RUN) return - - for (let i = 0; i < 10; i++) { - await sleep(1000 * i) - try { - const ok = await TpdsUpdateSender.promises.deleteProject({ - projectId: encodeURIComponent(prefix), - }) - if (ok) { - return - } - } catch (e) { - console.error(`deletion failed for '${prefix}/'`, e) - } - } - throw new Error(`deletion failed for '${prefix}/', check logs`) -} - -async function letUserDoubleCheckInputs() { - console.error( - 'Options:', - JSON.stringify( - { - BATCH_SIZE, - DRY_RUN, - LET_USER_DOUBLE_CHECK_INPUTS_FOR, - READ_CONCURRENCY_SECONDARY, - READ_CONCURRENCY_PRIMARY, - START_OFFSET, - WRITE_CONCURRENCY, - }, - null, - 2 - ) - ) - console.error( - 'Waiting for you to double check inputs for', - LET_USER_DOUBLE_CHECK_INPUTS_FOR, - 'ms' - ) - await sleep(LET_USER_DOUBLE_CHECK_INPUTS_FOR) -} - -main() - .then(() => { - console.log('Done.') - process.exit(0) - }) - .catch(error => { - console.error({ error }) - process.exit(1) - }) diff --git a/services/web/test/unit/src/Project/ProjectDeleterTests.js b/services/web/test/unit/src/Project/ProjectDeleterTests.js index f551f8bed4..e0d07d93b9 100644 --- a/services/web/test/unit/src/Project/ProjectDeleterTests.js +++ b/services/web/test/unit/src/Project/ProjectDeleterTests.js @@ -127,11 +127,6 @@ describe('ProjectDeleter', function () { deleteProject: sinon.stub().resolves(), }, } - this.TpdsUpdateSender = { - promises: { - deleteProject: sinon.stub().resolves(), - }, - } this.Features = { hasFeature: sinon.stub().returns(true), } @@ -154,7 +149,6 @@ describe('ProjectDeleter', function () { this.DocumentUpdaterHandler, '../Tags/TagsHandler': this.TagsHandler, '../FileStore/FileStoreHandler': this.FileStoreHandler, - '../ThirdPartyDataStore/TpdsUpdateSender': this.TpdsUpdateSender, '../Chat/ChatApiHandler': this.ChatApiHandler, '../Collaborators/CollaboratorsHandler': this.CollaboratorsHandler, '../Collaborators/CollaboratorsGetter': this.CollaboratorsGetter, @@ -496,14 +490,6 @@ describe('ProjectDeleter', function () { ).to.have.been.calledWith(this.deletedProjects[0].project._id) }) - it('should destroy the files in project-archiver', function () { - expect( - this.TpdsUpdateSender.promises.deleteProject - ).to.have.been.calledWith({ - projectId: this.deletedProjects[0].project._id, - }) - }) - it('should destroy the chat threads and messages', function () { expect( this.ChatApiHandler.promises.destroyProject @@ -553,11 +539,6 @@ describe('ProjectDeleter', function () { .called }) - it('should not destroy the files in project-archiver', function () { - expect(this.TpdsUpdateSender.promises.deleteProject).to.not.have.been - .called - }) - it('should not destroy the chat threads and messages', function () { expect(this.ChatApiHandler.promises.destroyProject).to.not.have.been .called diff --git a/services/web/test/unit/src/ThirdPartyDataStore/TpdsUpdateSenderTests.js b/services/web/test/unit/src/ThirdPartyDataStore/TpdsUpdateSenderTests.js index ed195fd9bf..41c83c7aa6 100644 --- a/services/web/test/unit/src/ThirdPartyDataStore/TpdsUpdateSenderTests.js +++ b/services/web/test/unit/src/ThirdPartyDataStore/TpdsUpdateSenderTests.js @@ -18,7 +18,6 @@ const projectName = 'project_name_here' const thirdPartyDataStoreApiUrl = 'http://third-party-json-store.herokuapp.com' const siteUrl = 'http://www.localhost:3000' const filestoreUrl = 'filestore.sharelatex.com' -const projectArchiverUrl = 'project-archiver.overleaf.com' describe('TpdsUpdateSender', function () { beforeEach(function () { @@ -440,21 +439,6 @@ describe('TpdsUpdateSender', function () { }) }) - describe('deleteProject', function () { - it('should not call request if there is no project archiver url', async function () { - await this.TpdsUpdateSender.promises.deleteProject({ projectId }) - this.FetchUtils.fetchNothing.should.not.have.been.called - }) - it('should make a delete request to project archiver', async function () { - this.settings.apis.project_archiver = { url: projectArchiverUrl } - await this.TpdsUpdateSender.promises.deleteProject({ projectId }) - this.FetchUtils.fetchNothing.should.have.been.calledWith( - `${projectArchiverUrl}/project/${projectId}`, - { method: 'DELETE' } - ) - }) - }) - describe('user not linked to dropbox', function () { beforeEach(function () { this.UserGetter.promises.getUsers