const chai = require('chai') const { expect } = chai const fs = require('node:fs') const Stream = require('node:stream') const Settings = require('@overleaf/settings') const Path = require('node:path') const FilestoreApp = require('./FilestoreApp') const TestHelper = require('./TestHelper') const fetch = require('node-fetch') const { promisify } = require('node:util') const { Storage } = require('@google-cloud/storage') const streamifier = require('streamifier') chai.use(require('chai-as-promised')) const { ObjectId } = require('mongodb') const ChildProcess = require('node:child_process') const fsWriteFile = promisify(fs.writeFile) const fsStat = promisify(fs.stat) const exec = promisify(ChildProcess.exec) const msleep = promisify(setTimeout) if (!process.env.AWS_ACCESS_KEY_ID) { throw new Error('please provide credentials for the AWS S3 test server') } process.on('unhandledRejection', e => { // eslint-disable-next-line no-console console.log('** Unhandled Promise Rejection **\n', e) throw e }) // store settings for multiple backends, so that we can test each one. // fs will always be available - add others if they are configured const { BackendSettings, s3Config, s3SSECConfig } = require('./TestConfig') const { AlreadyWrittenError, NotFoundError, NotImplementedError, NoKEKMatchedError, } = require('@overleaf/object-persistor/src/Errors') const { PerProjectEncryptedS3Persistor, RootKeyEncryptionKey, } = require('@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor') const { S3Persistor } = require('@overleaf/object-persistor/src/S3Persistor') const crypto = require('node:crypto') const { WritableBuffer } = require('@overleaf/stream-utils') const { gzipSync } = require('node:zlib') describe('Filestore', function () { this.timeout(1000 * 10) const filestoreUrl = `http://127.0.0.1:${Settings.internal.filestore.port}` const seenSockets = [] async function expectNoSockets() { try { await msleep(1000) const { stdout } = await exec('ss -tn') const lines = stdout.split('\n') const header = lines.shift() const badSockets = [] for (const socket of lines) { const fields = socket.split(' ').filter(part => part !== '') if ( fields.length > 2 && parseInt(fields[1]) && !seenSockets.includes(socket) ) { badSockets.push(socket) seenSockets.push(socket) } } if (badSockets.length) { // eslint-disable-next-line no-console console.error( 'ERR: Sockets still have receive buffer after connection closed' ) console.error(header) for (const socket of badSockets) { // eslint-disable-next-line no-console console.error(socket) } throw new Error('Sockets still open after connection closed') } } catch (err) { expect(err).not.to.exist } } // redefine the test suite for every available backend for (const [backendVariantWithShardNumber, backendSettings] of Object.entries( BackendSettings )) { describe(backendVariantWithShardNumber, function () { let app, previousEgress, previousIngress, metricPrefix, projectId, otherProjectId const dataEncryptionKeySize = backendSettings.backend === 's3SSEC' ? 32 : 0 const BUCKET_NAMES = [ process.env.GCS_USER_FILES_BUCKET_NAME, process.env.GCS_TEMPLATE_FILES_BUCKET_NAME, `${process.env.GCS_USER_FILES_BUCKET_NAME}-deleted`, `${process.env.GCS_TEMPLATE_FILES_BUCKET_NAME}-deleted`, ] before('start filestore with new settings', async function () { // create the app with the relevant filestore settings Settings.filestore = backendSettings app = new FilestoreApp() await app.runServer() }) if (backendSettings.gcs) { before('create gcs buckets', async function () { // create test buckets for gcs const storage = new Storage(Settings.filestore.gcs.endpoint) for (const bucketName of BUCKET_NAMES) { await storage.createBucket(bucketName) } }) after('delete gcs buckets', async function () { // tear down all the gcs buckets const storage = new Storage(Settings.filestore.gcs.endpoint) for (const bucketName of BUCKET_NAMES) { const bucket = storage.bucket(bucketName) await bucket.deleteFiles() await bucket.delete() } }) } after('stop filestore app', async function () { await app.stop() }) beforeEach('fetch previous egress metric', async function () { // retrieve previous metrics from the app if (['s3', 's3SSEC', 'gcs'].includes(Settings.filestore.backend)) { metricPrefix = Settings.filestore.backend.replace('SSEC', '') previousEgress = await TestHelper.getMetric( filestoreUrl, `${metricPrefix}_egress` ) } projectId = new ObjectId().toString() otherProjectId = new ObjectId().toString() }) it('should send a 200 for the status endpoint', async function () { const response = await fetch(`${filestoreUrl}/status`) expect(response.status).to.equal(200) const body = await response.text() expect(body).to.contain('filestore') expect(body).to.contain('up') }) describe('with a file on the server', function () { let fileId, fileUrl, constantFileContent const localFileReadPath = '/tmp/filestore_acceptance_tests_file_read.txt' beforeEach('upload file', async function () { fileId = new ObjectId().toString() fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}` constantFileContent = [ 'hello world', `line 2 goes here ${Math.random()}`, 'there are 3 lines in all', ].join('\n') await fsWriteFile(localFileReadPath, constantFileContent) const readStream = fs.createReadStream(localFileReadPath) const res = await fetch(fileUrl, { method: 'POST', body: readStream }) if (!res.ok) throw new Error(res.statusText) }) beforeEach('retrieve previous ingress metric', async function () { // The upload request can bump the ingress metric. // The content hash validation might require a full download // in case the ETag field of the upload response is not a md5 sum. if (['s3', 's3SSEC', 'gcs'].includes(Settings.filestore.backend)) { previousIngress = await TestHelper.getMetric( filestoreUrl, `${metricPrefix}_ingress` ) } }) it('should return 404 for a non-existant id', async function () { const url = fileUrl + '___this_is_clearly_wrong___' const response = await fetch(url) expect(response.status).to.equal(404) }) it('should return the file size on a HEAD request', async function () { const expectedLength = Buffer.byteLength(constantFileContent) const res = await fetch(fileUrl, { method: 'HEAD' }) expect(res.status).to.equal(200) expect(res.headers.get('Content-Length')).to.equal( expectedLength.toString() ) }) it('should be able get the file back', async function () { const res = await fetch(fileUrl) const body = await res.text() expect(body).to.equal(constantFileContent) }) it('should send a 200 for the health-check endpoint using the file', async function () { Settings.health_check = { project_id: projectId, file_id: fileId, } const response = await fetch(`${filestoreUrl}/health_check`) expect(response.status).to.equal(200) const body = await response.text() expect(body).to.equal('OK') }) it('should not leak a socket', async function () { const res = await fetch(fileUrl) if (!res.ok) throw new Error(res.statusText) await res.text() await expectNoSockets() }) it('should be able to get back the first 9 bytes of the file', async function () { const res = await fetch(fileUrl, { headers: { Range: 'bytes=0-8' } }) const body = await res.text() expect(body).to.equal('hello wor') }) it('should be able to get back bytes 4 through 10 of the file', async function () { const res = await fetch(fileUrl, { headers: { Range: 'bytes=4-10' } }) const body = await res.text() expect(body).to.equal('o world') }) it('should be able to delete the file', async function () { const response = await fetch(fileUrl, { method: 'DELETE' }) expect(response.status).to.equal(204) const response2 = await fetch(fileUrl) expect(response2.status).to.equal(404) }) it('should be able to copy files', async function () { const newProjectID = new ObjectId().toString() const newFileId = new ObjectId().toString() const newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${newFileId}` let response = await fetch(newFileUrl, { method: 'PUT', body: JSON.stringify({ source: { project_id: projectId, file_id: fileId, }, }), headers: { 'Content-Type': 'application/json', }, }) expect(response.status).to.equal(200) response = await fetch(fileUrl, { method: 'DELETE' }) expect(response.status).to.equal(204) response = await fetch(newFileUrl) const body = await response.text() expect(body).to.equal(constantFileContent) }) it('should be able to overwrite the file', async function () { const newContent = `here is some different content, ${Math.random()}` const readStream = streamifier.createReadStream(newContent) await fetch(fileUrl, { method: 'POST', body: readStream }) const response = await fetch(fileUrl) const body = await response.text() expect(body).to.equal(newContent) }) describe('IfNoneMatch', function () { if (backendSettings.backend === 'fs') { it('should refuse to handle IfNoneMatch', async function () { await expect( app.persistor.sendStream( Settings.filestore.stores.user_files, `${projectId}/${fileId}`, fs.createReadStream(localFileReadPath), { ifNoneMatch: '*' } ) ).to.be.rejectedWith(NotImplementedError) }) } else { it('should reject sendStream on the same key with IfNoneMatch', async function () { await expect( app.persistor.sendStream( Settings.filestore.stores.user_files, `${projectId}/${fileId}`, fs.createReadStream(localFileReadPath), { ifNoneMatch: '*' } ) ).to.be.rejectedWith(AlreadyWrittenError) }) it('should allow sendStream on a different key with IfNoneMatch', async function () { await app.persistor.sendStream( Settings.filestore.stores.user_files, `${projectId}/${fileId}-other`, fs.createReadStream(localFileReadPath), { ifNoneMatch: '*' } ) }) } }) if (backendSettings.backend !== 'fs') { it('should record an egress metric for the upload', async function () { const metric = await TestHelper.getMetric( filestoreUrl, `${metricPrefix}_egress` ) expect(metric - previousEgress).to.equal( constantFileContent.length + dataEncryptionKeySize ) }) it('should record an ingress metric when downloading the file', async function () { const response = await fetch(fileUrl) expect(response.ok).to.be.true await response.text() const metric = await TestHelper.getMetric( filestoreUrl, `${metricPrefix}_ingress` ) expect(metric - previousIngress).to.equal( constantFileContent.length + dataEncryptionKeySize ) }) it('should record an ingress metric for a partial download', async function () { const response = await fetch(fileUrl, { headers: { Range: 'bytes=0-8' }, }) expect(response.ok).to.be.true await response.text() const metric = await TestHelper.getMetric( filestoreUrl, `${metricPrefix}_ingress` ) expect(metric - previousIngress).to.equal(9 + dataEncryptionKeySize) }) } }) describe('with multiple files', function () { let fileIds, fileUrls, otherFileUrls, projectUrl, otherProjectUrl const localFileReadPaths = [ '/tmp/filestore_acceptance_tests_file_read_1.txt', '/tmp/filestore_acceptance_tests_file_read_2.txt', '/tmp/filestore_acceptance_tests_file_read_3.txt', ] const constantFileContents = [ [ 'hello world', `line 2 goes here ${Math.random()}`, 'there are 3 lines in all', ].join('\n'), [ `for reference: ${Math.random()}`, 'cats are the best animals', 'wombats are a close second', ].join('\n'), [ `another file: ${Math.random()}`, 'with multiple lines', 'the end', ].join('\n'), ] before('create local files', async function () { return await Promise.all([ fsWriteFile(localFileReadPaths[0], constantFileContents[0]), fsWriteFile(localFileReadPaths[1], constantFileContents[1]), fsWriteFile(localFileReadPaths[2], constantFileContents[2]), ]) }) beforeEach('upload two files', async function () { projectUrl = `${filestoreUrl}/project/${projectId}` otherProjectUrl = `${filestoreUrl}/project/${otherProjectId}` fileIds = [ new ObjectId().toString(), new ObjectId().toString(), new ObjectId().toString(), ] fileUrls = [ `${projectUrl}/file/${fileIds[0]}`, `${projectUrl}/file/${fileIds[1]}`, ] otherFileUrls = [`${otherProjectUrl}/file/${fileIds[2]}`] await Promise.all([ fetch(fileUrls[0], { method: 'POST', body: fs.createReadStream(localFileReadPaths[0]), }), fetch(fileUrls[1], { method: 'POST', body: fs.createReadStream(localFileReadPaths[1]), }), fetch(otherFileUrls[0], { method: 'POST', body: fs.createReadStream(localFileReadPaths[2]), }), ]) }) it('should get the directory size', async function () { const response = await fetch( `${filestoreUrl}/project/${projectId}/size` ) const body = await response.text() expect(parseInt(JSON.parse(body)['total bytes'])).to.equal( constantFileContents[0].length + constantFileContents[1].length ) }) it('should store the files', async function () { for (const index in fileUrls) { const response = await fetch(fileUrls[index]) const body = await response.text() expect(body).to.equal(constantFileContents[index]) } }) it('should be able to delete the project', async function () { let response = await fetch(projectUrl, { method: 'DELETE' }) expect(response.status).to.equal(204) for (const index in fileUrls) { response = await fetch(fileUrls[index]) expect(response.status).to.equal(404) } }) it('should not delete files in other projects', async function () { for (const index in otherFileUrls) { const response = await fetch(otherFileUrls[index]) expect(response.status).to.equal(200) } }) it('should not delete a partial project id', async function () { const response = await fetch(`${filestoreUrl}/project/5`, { method: 'DELETE', }) expect(response.status).to.equal(400) }) }) describe('with a large file', function () { this.timeout(1000 * 20) let fileId, fileUrl, largeFileContent, error beforeEach('upload large file', async function () { fileId = new ObjectId().toString() fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}` largeFileContent = '_wombat_'.repeat(1024 * 1024) // 8 megabytes largeFileContent += Math.random() const readStream = streamifier.createReadStream(largeFileContent) const res = await fetch(fileUrl, { method: 'POST', body: readStream }) if (!res.ok) throw new Error(res.statusText) }) it('should be able to get the file back', async function () { const response = await fetch(fileUrl) const body = await response.text() expect(body).to.equal(largeFileContent) }) it('should not throw an error', function () { expect(error).not.to.exist }) it('should not leak a socket', async function () { const response = await fetch(fileUrl) await response.text() await expectNoSockets() }) it('should not leak a socket if the connection is aborted', async function () { const controller = new AbortController() const response = await fetch(fileUrl, { signal: controller.signal }) expect(response.ok).to.be.true controller.abort() await expectNoSockets() }) }) if ( (backendSettings.backend === 's3' && !backendSettings.fallback) || (backendSettings.backend === 'gcs' && backendSettings.fallback?.backend === 's3') ) { describe('with a file in a specific bucket', function () { let constantFileContent, fileId, fileUrl, bucketName beforeEach('upload file into random bucket', async function () { constantFileContent = `This is a file in a different S3 bucket ${Math.random()}` fileId = new ObjectId().toString() bucketName = `random-bucket-${new ObjectId().toString()}` fileUrl = `${filestoreUrl}/bucket/${bucketName}/key/${fileId}` const s3 = new S3Persistor({ ...s3Config(), key: process.env.MINIO_ROOT_USER, secret: process.env.MINIO_ROOT_PASSWORD, })._getClientForBucket(bucketName) await s3 .createBucket({ Bucket: bucketName, }) .promise() await s3 .upload({ Bucket: bucketName, Key: fileId, Body: constantFileContent, }) .promise() }) it('should get the file from the specified bucket', async function () { const response = await fetch(fileUrl) const body = await response.text() expect(body).to.equal(constantFileContent) }) }) } if (backendSettings.backend === 'gcs') { describe('when deleting a file in GCS', function () { let fileId, fileUrl, content, error, dateBefore, dateAfter beforeEach('upload and delete file', async function () { fileId = new ObjectId() fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}` content = '_wombat_' + Math.random() const readStream = streamifier.createReadStream(content) let res = await fetch(fileUrl, { method: 'POST', body: readStream }) if (!res.ok) throw new Error(res.statusText) dateBefore = new Date() res = await fetch(fileUrl, { method: 'DELETE' }) dateAfter = new Date() if (!res.ok) throw new Error(res.statusText) }) it('should not throw an error', function () { expect(error).not.to.exist }) it('should copy the file to the deleted-files bucket', async function () { let date = dateBefore const keys = [] while (date <= dateAfter) { keys.push(`${projectId}/${fileId}-${date.toISOString()}`) date = new Date(date.getTime() + 1) } await TestHelper.expectPersistorToHaveSomeFile( app.persistor, `${Settings.filestore.stores.user_files}-deleted`, keys, content ) }) it('should remove the file from the original bucket', async function () { await TestHelper.expectPersistorNotToHaveFile( app.persistor, Settings.filestore.stores.user_files, `${projectId}/${fileId}` ) }) }) } if (backendSettings.fallback) { describe('with a fallback', function () { let constantFileContent, fileId, fileKey, fileUrl, bucket, fallbackBucket beforeEach('prepare fallback', function () { constantFileContent = `This is yet more file content ${Math.random()}` fileId = new ObjectId().toString() fileKey = `${projectId}/${fileId}` fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}` bucket = Settings.filestore.stores.user_files fallbackBucket = Settings.filestore.fallback.buckets[bucket] }) describe('with a file in the fallback bucket', function () { beforeEach('upload into fallback', async function () { await TestHelper.uploadStringToPersistor( app.persistor.fallbackPersistor, fallbackBucket, fileKey, constantFileContent ) }) it('should not find file in the primary', async function () { await TestHelper.expectPersistorNotToHaveFile( app.persistor.primaryPersistor, bucket, fileKey ) }) it('should find the file in the fallback', async function () { await TestHelper.expectPersistorToHaveFile( app.persistor.fallbackPersistor, fallbackBucket, fileKey, constantFileContent ) }) describe('when copyOnMiss is disabled', function () { beforeEach('swap copyOnMiss=false', function () { app.persistor.settings.copyOnMiss = false }) it('should fetch the file', async function () { const res = await fetch(fileUrl) const body = await res.text() expect(body).to.equal(constantFileContent) }) it('should not copy the file to the primary', async function () { const response = await fetch(fileUrl) expect(response.ok).to.be.true await response.text() await TestHelper.expectPersistorNotToHaveFile( app.persistor.primaryPersistor, bucket, fileKey ) }) }) describe('when copyOnMiss is enabled', function () { beforeEach('swap copyOnMiss=true', function () { app.persistor.settings.copyOnMiss = true }) it('should fetch the file', async function () { const res = await fetch(fileUrl) const body = await res.text() expect(body).to.equal(constantFileContent) }) it('copies the file to the primary', async function () { const response = await fetch(fileUrl) expect(response.ok).to.be.true await response.text() // wait for the file to copy in the background await msleep(1000) await TestHelper.expectPersistorToHaveFile( app.persistor.primaryPersistor, bucket, fileKey, constantFileContent ) }) }) describe('when copying a file', function () { let newFileId, newFileUrl, newFileKey, opts beforeEach('prepare to copy file', function () { const newProjectID = new ObjectId().toString() newFileId = new ObjectId().toString() newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${newFileId}` newFileKey = `${newProjectID}/${newFileId}` opts = { method: 'put', body: JSON.stringify({ source: { project_id: projectId, file_id: fileId, }, }), headers: { 'Content-Type': 'application/json', }, } }) describe('when copyOnMiss is false', function () { beforeEach('copy with copyOnMiss=false', async function () { app.persistor.settings.copyOnMiss = false const response = await fetch(newFileUrl, opts) expect(response.status).to.equal(200) }) it('should leave the old file in the old bucket', async function () { await TestHelper.expectPersistorToHaveFile( app.persistor.fallbackPersistor, fallbackBucket, fileKey, constantFileContent ) }) it('should not create a new file in the old bucket', async function () { await TestHelper.expectPersistorNotToHaveFile( app.persistor.fallbackPersistor, fallbackBucket, newFileKey ) }) it('should create a new file in the new bucket', async function () { await TestHelper.expectPersistorToHaveFile( app.persistor.primaryPersistor, bucket, newFileKey, constantFileContent ) }) it('should not copy the old file to the primary with the old key', async function () { // wait for the file to copy in the background await msleep(1000) await TestHelper.expectPersistorNotToHaveFile( app.persistor.primaryPersistor, bucket, fileKey ) }) }) describe('when copyOnMiss is true', function () { beforeEach('copy with copyOnMiss=false', async function () { app.persistor.settings.copyOnMiss = true const response = await fetch(newFileUrl, opts) expect(response.status).to.equal(200) }) it('should leave the old file in the old bucket', async function () { await TestHelper.expectPersistorToHaveFile( app.persistor.fallbackPersistor, fallbackBucket, fileKey, constantFileContent ) }) it('should not create a new file in the old bucket', async function () { await TestHelper.expectPersistorNotToHaveFile( app.persistor.fallbackPersistor, fallbackBucket, newFileKey ) }) it('should create a new file in the new bucket', async function () { await TestHelper.expectPersistorToHaveFile( app.persistor.primaryPersistor, bucket, newFileKey, constantFileContent ) }) it('should copy the old file to the primary with the old key', async function () { // wait for the file to copy in the background await msleep(1000) await TestHelper.expectPersistorToHaveFile( app.persistor.primaryPersistor, bucket, fileKey, constantFileContent ) }) }) }) }) describe('when sending a file', function () { beforeEach('upload file', async function () { const readStream = streamifier.createReadStream(constantFileContent) const res = await fetch(fileUrl, { method: 'POST', body: readStream, }) if (!res.ok) throw new Error(res.statusText) }) it('should store the file on the primary', async function () { await TestHelper.expectPersistorToHaveFile( app.persistor.primaryPersistor, bucket, fileKey, constantFileContent ) }) it('should not store the file on the fallback', async function () { await TestHelper.expectPersistorNotToHaveFile( app.persistor.fallbackPersistor, fallbackBucket, `${projectId}/${fileId}` ) }) }) describe('when deleting a file', function () { describe('when the file exists on the primary', function () { beforeEach('upload into primary', async function () { await TestHelper.uploadStringToPersistor( app.persistor.primaryPersistor, bucket, fileKey, constantFileContent ) }) it('should delete the file', async function () { const response1 = await fetch(fileUrl, { method: 'DELETE' }) expect(response1.status).to.equal(204) const response2 = await fetch(fileUrl) expect(response2.status).to.equal(404) }) }) describe('when the file exists on the fallback', function () { beforeEach('upload into fallback', async function () { await TestHelper.uploadStringToPersistor( app.persistor.fallbackPersistor, fallbackBucket, fileKey, constantFileContent ) }) it('should delete the file', async function () { const response1 = await fetch(fileUrl, { method: 'DELETE' }) expect(response1.status).to.equal(204) const response2 = await fetch(fileUrl) expect(response2.status).to.equal(404) }) }) describe('when the file exists on both the primary and the fallback', function () { beforeEach( 'upload into both primary and fallback', async function () { await TestHelper.uploadStringToPersistor( app.persistor.primaryPersistor, bucket, fileKey, constantFileContent ) await TestHelper.uploadStringToPersistor( app.persistor.fallbackPersistor, fallbackBucket, fileKey, constantFileContent ) } ) it('should delete the files', async function () { const response1 = await fetch(fileUrl, { method: 'DELETE' }) expect(response1.status).to.equal(204) const response2 = await fetch(fileUrl) expect(response2.status).to.equal(404) }) }) describe('when the file does not exist', function () { it('should return return 204', async function () { // S3 doesn't give us a 404 when the object doesn't exist, so to stay // consistent we merrily return 204 ourselves here as well const response = await fetch(fileUrl, { method: 'DELETE' }) expect(response.status).to.equal(204) }) }) }) }) } describe('with a pdf file', function () { let fileId, fileUrl, localFileSize const localFileReadPath = Path.resolve( __dirname, '../../fixtures/test.pdf' ) beforeEach('upload test.pdf', async function () { fileId = new ObjectId().toString() fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}` const stat = await fsStat(localFileReadPath) localFileSize = stat.size const readStream = fs.createReadStream(localFileReadPath) const res = await fetch(fileUrl, { method: 'POST', body: readStream }) if (!res.ok) throw new Error(res.statusText) }) it('should be able get the file back', async function () { const response = await fetch(fileUrl) const body = await response.text() expect(body.substring(0, 8)).to.equal('%PDF-1.5') }) if (backendSettings.backend !== 'fs') { it('should record an egress metric for the upload', async function () { const metric = await TestHelper.getMetric( filestoreUrl, `${metricPrefix}_egress` ) expect(metric - previousEgress).to.equal( localFileSize + dataEncryptionKeySize ) }) } describe('getting the preview image', function () { this.timeout(1000 * 20) let previewFileUrl beforeEach('prepare previewFileUrl for preview', function () { previewFileUrl = `${fileUrl}?style=preview` }) it('should not time out', async function () { const response = await fetch(previewFileUrl) expect(response.status).to.equal(200) await response.arrayBuffer() }) it('should respond with image data', async function () { // note: this test relies of the imagemagick conversion working const response = await fetch(previewFileUrl) expect(response.status).to.equal(200) const body = await response.text() expect(body.length).to.be.greaterThan(400) expect(body.substr(1, 3)).to.equal('PNG') }) }) describe('warming the cache', function () { this.timeout(1000 * 20) let previewFileUrl beforeEach('prepare previewFileUrl for cacheWarn', function () { previewFileUrl = `${fileUrl}?style=preview&cacheWarm=true` }) it('should not time out', async function () { const response = await fetch(previewFileUrl) expect(response.status).to.equal(200) await response.arrayBuffer() }) it('should not leak sockets', async function () { const response1 = await fetch(previewFileUrl) expect(response1.status).to.equal(200) // do not read the response body, should be destroyed immediately const response2 = await fetch(previewFileUrl) expect(response2.status).to.equal(200) // do not read the response body, should be destroyed immediately await expectNoSockets() }) it("should respond with only an 'OK'", async function () { // note: this test relies of the imagemagick conversion working const response = await fetch(previewFileUrl) const body = await response.text() expect(body).to.equal('OK') }) }) }) describe('with server side encryption', function () { if (backendSettings.backend !== 's3SSEC') return before('sanity check top-level variable', function () { expect(dataEncryptionKeySize).to.equal(32) }) let fileId1, fileId2, fileKey1, fileKey2, fileKeyOtherProject, fileUrl1, fileUrl2 beforeEach('prepare ids', function () { fileId1 = new ObjectId().toString() fileId2 = new ObjectId().toString() fileKey1 = `${projectId}/${fileId1}` fileKey2 = `${projectId}/${fileId2}` fileKeyOtherProject = `${new ObjectId().toString()}/${new ObjectId().toString()}` fileUrl1 = `${filestoreUrl}/project/${projectId}/file/${fileId1}` fileUrl2 = `${filestoreUrl}/project/${projectId}/file/${fileId2}` }) beforeEach('ensure DEK is missing', async function () { // Cannot use test helper expectPersistorNotToHaveFile here, we need to use the KEK. await expect( app.persistor.getDataEncryptionKeySize( backendSettings.stores.user_files, fileKey1 ) ).to.rejectedWith(NotFoundError) }) async function createRandomContent(url, suffix = '') { const content = Math.random().toString() + suffix const res = await fetch(url, { method: 'POST', body: Stream.Readable.from([content]), }) if (!res.ok) throw new Error(res.statusText) return async () => { const res = await fetch(url, { method: 'GET' }) if (!res.ok) throw new Error(res.statusText) expect(await res.text()).to.equal(content) } } it('should create a DEK when asked explicitly', async function () { await app.persistor.generateDataEncryptionKey( backendSettings.stores.user_files, fileKey1 ) expect( await app.persistor.getDataEncryptionKeySize( backendSettings.stores.user_files, fileKey1 ) ).to.equal(32) }) it('should create a DEK from writes', async function () { await createRandomContent(fileUrl1) expect( await app.persistor.getDataEncryptionKeySize( backendSettings.stores.user_files, fileKey1 ) ).to.equal(32) }) it('should not create a DEK from reads', async function () { const res = await fetch(fileUrl1, { method: 'GET', }) if (res.status !== 404) throw new Error(`${res.status} should be 404`) // Cannot use test helper expectPersistorNotToHaveFile here, we need to use the KEK. await expect( app.persistor.getDataEncryptionKeySize( backendSettings.stores.user_files, fileKey1 ) ).to.rejectedWith(NotFoundError) }) it('should never overwrite a data encryption key', async function () { const checkGET = await createRandomContent(fileUrl1) await expect( app.persistor.generateDataEncryptionKey( backendSettings.stores.user_files, fileKey1 ) ).to.rejectedWith(AlreadyWrittenError) await checkGET() }) it('should re-use the data encryption key after a write', async function () { const checkGET1 = await createRandomContent(fileUrl1, '1') const checkGET2 = await createRandomContent(fileUrl2, '2') await checkGET1() await checkGET2() }) describe('kek rotation', function () { const newKEK = new RootKeyEncryptionKey( crypto.generateKeySync('aes', { length: 256 }).export(), Buffer.alloc(32) ) const oldKEK = new RootKeyEncryptionKey( crypto.generateKeySync('aes', { length: 256 }).export(), Buffer.alloc(32) ) const migrationStep0 = new PerProjectEncryptedS3Persistor({ ...s3SSECConfig(), automaticallyRotateDEKEncryption: false, async getRootKeyEncryptionKeys() { return [oldKEK] // only old key }, }) const migrationStep1 = new PerProjectEncryptedS3Persistor({ ...s3SSECConfig(), automaticallyRotateDEKEncryption: false, async getRootKeyEncryptionKeys() { return [oldKEK, newKEK] // new key as fallback }, }) const migrationStep2 = new PerProjectEncryptedS3Persistor({ ...s3SSECConfig(), automaticallyRotateDEKEncryption: true, // <- different compared to partiallyRotated async getRootKeyEncryptionKeys() { return [newKEK, oldKEK] // old keys as fallback }, }) const migrationStep3 = new PerProjectEncryptedS3Persistor({ ...s3SSECConfig(), automaticallyRotateDEKEncryption: true, async getRootKeyEncryptionKeys() { return [newKEK] // only new key }, }) async function checkWrites( fileKey, writer, readersSuccess, readersFailed ) { const content = Math.random().toString() await writer.sendStream( Settings.filestore.stores.user_files, fileKey, Stream.Readable.from([content]) ) for (const persistor of readersSuccess) { await TestHelper.expectPersistorToHaveFile( persistor, backendSettings.stores.user_files, fileKey, content ) } for (const persistor of readersFailed) { await expect( TestHelper.expectPersistorToHaveFile( persistor, backendSettings.stores.user_files, fileKey, content ) ).to.be.rejectedWith(NoKEKMatchedError) } } const stages = [ { name: 'stage 0 - [old]', prev: migrationStep0, cur: migrationStep0, fail: [migrationStep3], }, { name: 'stage 1 - [old,new]', prev: migrationStep0, cur: migrationStep1, fail: [], }, { name: 'stage 2 - [new,old]', prev: migrationStep1, cur: migrationStep2, fail: [], }, { name: 'stage 3 - [new]', prev: migrationStep2, cur: migrationStep3, fail: [migrationStep0], }, ] for (const { name, prev, cur, fail } of stages) { describe(name, function () { this.timeout(1000 * 30) it('can read old writes', async function () { await checkWrites(fileKey1, prev, [prev, cur], fail) await checkWrites(fileKey2, prev, [prev, cur], fail) // check again after access await checkWrites(fileKeyOtherProject, prev, [prev, cur], fail) }) it('can read new writes', async function () { await checkWrites(fileKey1, prev, [prev, cur], fail) await checkWrites(fileKey2, cur, [prev, cur], fail) // check again after access await checkWrites(fileKeyOtherProject, cur, [prev, cur], fail) }) }) } describe('full migration', function () { it('can read old writes if rotated in sequence', async function () { await checkWrites( fileKey1, migrationStep0, [ migrationStep0, migrationStep1, migrationStep2, // migrates migrationStep3, ], [] ) }) it('cannot read/write if not rotated', async function () { await checkWrites( fileKey1, migrationStep0, [migrationStep0], [migrationStep3] ) }) }) }) let s3Client before('create s3 client', function () { s3Client = new S3Persistor(s3Config())._getClientForBucket('') }) async function checkDEKStorage({ dekBucketKeys = [], userFilesBucketKeys = [], }) { await createRandomContent(fileUrl1) const { Contents: dekEntries } = await s3Client .listObjectsV2({ Bucket: process.env.AWS_S3_USER_FILES_DEK_BUCKET_NAME, Prefix: `${projectId}/`, }) .promise() expect(dekEntries).to.have.length(dekBucketKeys.length) // Order is not predictable, use members expect(dekEntries.map(o => o.Key)).to.have.members(dekBucketKeys) const { Contents: userFilesEntries } = await s3Client .listObjectsV2({ Bucket: backendSettings.stores.user_files, Prefix: `${projectId}/`, }) .promise() expect(userFilesEntries).to.have.length(userFilesBucketKeys.length) // Order is not predictable, use members expect(userFilesEntries.map(o => o.Key)).to.have.members( userFilesBucketKeys ) } it('should use a custom bucket for DEKs', async function () { await checkDEKStorage({ dekBucketKeys: [`${projectId}/dek`], userFilesBucketKeys: [fileKey1], }) }) describe('deleteDirectory', function () { let checkGET2 beforeEach('create files', async function () { await createRandomContent(fileUrl1, '1') checkGET2 = await createRandomContent(fileUrl2, '2') }) it('should delete sub-folder and keep DEK', async function () { await app.persistor.deleteDirectory( Settings.filestore.stores.user_files, fileKey1 // not really a sub-folder, but it will do for this test. ) expect( await app.persistor.checkIfObjectExists( Settings.filestore.stores.user_files, fileKey1 ) ).to.equal(false) expect( await app.persistor.checkIfObjectExists( Settings.filestore.stores.user_files, fileKey2 ) ).to.equal(true) expect( await app.persistor.getDataEncryptionKeySize( Settings.filestore.stores.user_files, fileKey2 ) ).to.equal(32) await checkGET2() }) it('should delete project folder and DEK', async function () { await app.persistor.deleteDirectory( Settings.filestore.stores.user_files, `${projectId}/` ) expect( await app.persistor.checkIfObjectExists( Settings.filestore.stores.user_files, fileKey1 ) ).to.equal(false) expect( await app.persistor.checkIfObjectExists( Settings.filestore.stores.user_files, fileKey2 ) ).to.equal(false) await expect( app.persistor.getDataEncryptionKeySize( Settings.filestore.stores.user_files, fileKey2 ) ).to.rejectedWith(NotFoundError) }) }) }) describe('getObjectSize', function () { it('should return a number', async function () { const buf = Buffer.from('hello') const fileId = new ObjectId().toString() const fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}` const res = await fetch(fileUrl, { method: 'POST', body: Stream.Readable.from([buf]), }) if (!res.ok) throw new Error(res.statusText) expect( await app.persistor.getObjectSize( Settings.filestore.stores.user_files, `${projectId}/${fileId}` ) ).to.equal(buf.byteLength) }) }) describe('checkIfObjectExists', function () { it('should return false when the object does not exist', async function () { expect( await app.persistor.checkIfObjectExists( Settings.filestore.stores.user_files, `${projectId}/${new ObjectId().toString()}` ) ).to.equal(false) }) it('should return true when the object exists', async function () { const fileId = new ObjectId().toString() const fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}` const res = await fetch(fileUrl, { method: 'POST', body: Stream.Readable.from(['hello']), }) if (!res.ok) throw new Error(res.statusText) expect( await app.persistor.checkIfObjectExists( Settings.filestore.stores.user_files, `${projectId}/${fileId}` ) ).to.equal(true) }) }) describe('autoGunzip', function () { let key beforeEach('new key', function () { key = `${projectId}/${new ObjectId().toString()}` }) this.timeout(60 * 1000) const body = Buffer.alloc(10 * 1024 * 1024, 'hello') const gzippedBody = gzipSync(body) /** * @param {string} key * @param {Buffer} wantBody * @param {boolean} autoGunzip * @return {Promise} */ async function checkBodyIsTheSame(key, wantBody, autoGunzip) { const s = await app.persistor.getObjectStream( Settings.filestore.stores.user_files, key, { autoGunzip } ) const buf = new WritableBuffer() await Stream.promises.pipeline(s, buf) expect(buf.getContents()).to.deep.equal(wantBody) } if (backendSettings.backend === 'fs') { it('should refuse to handle autoGunzip', async function () { await expect( app.persistor.getObjectStream( Settings.filestore.stores.user_files, key, { autoGunzip: true } ) ).to.be.rejectedWith(NotImplementedError) }) } else { it('should return the raw body with gzip', async function () { await app.persistor.sendStream( Settings.filestore.stores.user_files, key, Stream.Readable.from([gzippedBody]), { contentEncoding: 'gzip' } ) expect( await app.persistor.getObjectSize( Settings.filestore.stores.user_files, key ) ).to.equal(gzippedBody.byteLength) // raw body with autoGunzip=true await checkBodyIsTheSame(key, body, true) // gzip body without autoGunzip=false await checkBodyIsTheSame(key, gzippedBody, false) }) it('should return the raw body without gzip compression', async function () { await app.persistor.sendStream( Settings.filestore.stores.user_files, key, Stream.Readable.from([body]) ) expect( await app.persistor.getObjectSize( Settings.filestore.stores.user_files, key ) ).to.equal(body.byteLength) // raw body with both autoGunzip options await checkBodyIsTheSame(key, body, true) await checkBodyIsTheSame(key, body, false) }) it('should return the gzip body without gzip header', async function () { await app.persistor.sendStream( Settings.filestore.stores.user_files, key, Stream.Readable.from([gzippedBody]) ) expect( await app.persistor.getObjectSize( Settings.filestore.stores.user_files, key ) ).to.equal(gzippedBody.byteLength) // gzip body with both autoGunzip options await checkBodyIsTheSame(key, gzippedBody, true) await checkBodyIsTheSame(key, gzippedBody, false) }) } }) }) } })