// @ts-check import cleanup from '../storage/support/cleanup.js' import fetch from 'node-fetch' import testServer from './support/test_backup_server.mjs' import { expect } from 'chai' import testProjects from './support/test_projects.js' import { db } from '../../../../storage/lib/mongodb.js' import { ObjectId } from 'mongodb' import { backupPersistor, projectBlobsBucket, chunksBucket, } from '../../../../storage/lib/backupPersistor.mjs' import { makeProjectKey } from '../../../../storage/lib/blob_store/index.js' import config from 'config' import Stream from 'stream' import projectKey from '../../../../storage/lib/project_key.js' /** * @typedef {import("node-fetch").Response} Response */ const { deksBucket } = config.get('backupStore') const deletedProjectsCollection = db.collection('deletedProjects') /** * @param {string} bucket * @param {string} prefix * @return {Promise>} */ async function listS3Bucket(bucket, prefix) { // @ts-ignore access to internal library helper const client = backupPersistor._getClientForBucket(bucket) const response = await client .listObjectsV2({ Bucket: bucket, Prefix: prefix }) .promise() return (response.Contents || []).map(item => item.Key || '') } /** * @param {ObjectId} projectId * @return {Promise} */ async function deleteProject(projectId) { return await fetch(testServer.testUrl(`/project/${projectId}/backup`), { method: 'DELETE', headers: { Authorization: testServer.basicAuthHeader }, }) } /** * @param {string|ObjectId} historyId * @return {Promise} */ async function expectToHaveBackup(historyId) { const prefix = projectKey.format(historyId.toString()) + '/' expect(await listS3Bucket(deksBucket, prefix)).to.have.length(1) expect(await listS3Bucket(chunksBucket, prefix)).to.have.length(2) expect(await listS3Bucket(projectBlobsBucket, prefix)).to.have.length(2) } /** * @param {string|ObjectId} historyId * @return {Promise} */ async function expectToHaveNoBackup(historyId) { const prefix = projectKey.format(historyId.toString()) + '/' expect(await listS3Bucket(deksBucket, prefix)).to.have.length(0) expect(await listS3Bucket(chunksBucket, prefix)).to.have.length(0) expect(await listS3Bucket(projectBlobsBucket, prefix)).to.have.length(0) } describe('backupDeletion', function () { beforeEach(cleanup.everything) beforeEach('create health check projects', async function () { await testProjects.createEmptyProject('42') await testProjects.createEmptyProject('000000000000000000000042') }) beforeEach(testServer.listenOnRandomPort) it('renders 200 on /status', async function () { const response = await fetch(testServer.testUrl('/status')) expect(response.status).to.equal(200) }) it('renders 200 on /health_check', async function () { const response = await fetch(testServer.testUrl('/health_check')) expect(response.status).to.equal(200) }) describe('DELETE /project/:projectId', function () { const postgresHistoryId = '1' const projectIdPostgres = new ObjectId('000000000000000000000001') const projectIdMongoDB = new ObjectId('000000000000000000000002') const projectIdNonDeleted = new ObjectId('000000000000000000000003') const projectIdNonExpired = new ObjectId('000000000000000000000004') const projectIdWithChunks = new ObjectId('000000000000000000000005') const projectIdNoHistoryId = new ObjectId('000000000000000000000006') beforeEach('cleanup s3 buckets', async function () { await backupPersistor.deleteDirectory(deksBucket, '') await backupPersistor.deleteDirectory(chunksBucket, '') await backupPersistor.deleteDirectory(projectBlobsBucket, '') }) beforeEach('populate mongo', async function () { await deletedProjectsCollection.insertMany([ { _id: new ObjectId(), deleterData: { deletedProjectId: projectIdPostgres, deletedAt: new Date('2024-01-01T00:00:00Z'), deletedProjectOverleafHistoryId: postgresHistoryId, }, }, { _id: new ObjectId(), deleterData: { deletedProjectId: projectIdNonExpired, deletedAt: new Date(), deletedProjectOverleafHistoryId: projectIdNonExpired.toString(), }, }, { _id: new ObjectId(), deleterData: { deletedProjectId: projectIdNoHistoryId, deletedAt: new Date('2024-01-01T00:00:00Z'), }, }, ...[projectIdMongoDB, projectIdWithChunks].map(projectId => { return { _id: new ObjectId(), deleterData: { deletedProjectId: projectId, deletedAt: new Date('2024-01-01T00:00:00Z'), deletedProjectOverleafHistoryId: projectId.toString(), }, } }), ]) }) beforeEach('initialize history', async function () { await testProjects.createEmptyProject(projectIdWithChunks.toString()) }) beforeEach('create a file in s3', async function () { const historyIds = [ postgresHistoryId, projectIdMongoDB, projectIdNonDeleted, projectIdNonExpired, projectIdWithChunks, projectIdNoHistoryId, ] const jobs = [] for (const historyId of historyIds) { jobs.push( backupPersistor.sendStream( projectBlobsBucket, makeProjectKey(historyId, 'a'.repeat(40)), Stream.Readable.from(['blob a']), { contentLength: 6 } ) ) jobs.push( backupPersistor.sendStream( projectBlobsBucket, makeProjectKey(historyId, 'b'.repeat(40)), Stream.Readable.from(['blob b']), { contentLength: 6 } ) ) jobs.push( backupPersistor.sendStream( chunksBucket, projectKey.format(historyId) + '/111', Stream.Readable.from(['chunk 1']), { contentLength: 7 } ) ) jobs.push( backupPersistor.sendStream( chunksBucket, projectKey.format(historyId) + '/222', Stream.Readable.from(['chunk 2']), { contentLength: 7 } ) ) } await Promise.all(jobs) }) it('renders 401 without auth', async function () { const response = await fetch( testServer.testUrl('/project/000000000000000000000042/backup'), { method: 'DELETE' } ) expect(response.status).to.equal(401) expect(response.headers.get('www-authenticate')).to.match(/^Basic/) }) it('returns 422 when not deleted', async function () { const response = await deleteProject(projectIdNonDeleted) expect(response.status).to.equal(422) expect(await response.text()).to.equal( 'refusing to delete non-deleted project' ) await expectToHaveBackup(projectIdNonDeleted) }) it('returns 422 when not expired', async function () { const response = await deleteProject(projectIdNonExpired) expect(response.status).to.equal(422) expect(await response.text()).to.equal( 'refusing to delete non-expired project' ) await expectToHaveBackup(projectIdNonExpired) }) it('returns 422 when live-history not deleted', async function () { const response = await deleteProject(projectIdWithChunks) expect(response.status).to.equal(422) expect(await response.text()).to.equal( 'refusing to delete project with remaining chunks' ) await expectToHaveBackup(projectIdWithChunks) }) it('returns 422 when historyId is unknown', async function () { const response = await deleteProject(projectIdNoHistoryId) expect(response.status).to.equal(422) expect(await response.text()).to.equal( 'refusing to delete project with unknown historyId' ) await expectToHaveBackup(projectIdNoHistoryId) }) it('should successfully delete postgres id', async function () { await expectToHaveBackup(postgresHistoryId) const response = await deleteProject(projectIdPostgres) expect(response.status).to.equal(204) await expectToHaveNoBackup(postgresHistoryId) }) it('should successfully delete mongo id', async function () { await expectToHaveBackup(projectIdMongoDB) const response = await deleteProject(projectIdMongoDB) expect(response.status).to.equal(204) await expectToHaveNoBackup(projectIdMongoDB) }) }) })