diff --git a/services/document-updater/app/js/ProjectHistoryRedisManager.js b/services/document-updater/app/js/ProjectHistoryRedisManager.js index b84edfe74a..888948af09 100644 --- a/services/document-updater/app/js/ProjectHistoryRedisManager.js +++ b/services/document-updater/app/js/ProjectHistoryRedisManager.js @@ -122,6 +122,7 @@ const ProjectHistoryRedisManager = { hash: projectUpdate.hash, metadata: projectUpdate.metadata, projectHistoryId, + createdBlob: projectUpdate.createdBlob ?? false, } if (ranges) { projectUpdate.ranges = ranges diff --git a/services/document-updater/test/unit/js/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.js b/services/document-updater/test/unit/js/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.js index 0f5df2e29f..760385b176 100644 --- a/services/document-updater/test/unit/js/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.js +++ b/services/document-updater/test/unit/js/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.js @@ -162,6 +162,7 @@ describe('ProjectHistoryRedisManager', function () { }, version: this.version, projectHistoryId: this.projectHistoryId, + createdBlob: false, doc: this.doc_id, } @@ -207,6 +208,7 @@ describe('ProjectHistoryRedisManager', function () { hash: '1337', metadata, projectHistoryId: this.projectHistoryId, + createdBlob: false, file: fileId, } @@ -291,6 +293,7 @@ describe('ProjectHistoryRedisManager', function () { }, version: this.version, projectHistoryId: this.projectHistoryId, + createdBlob: false, ranges: historyCompatibleRanges, doc: this.doc_id, } @@ -343,6 +346,7 @@ describe('ProjectHistoryRedisManager', function () { }, version: this.version, projectHistoryId: this.projectHistoryId, + createdBlob: false, doc: this.doc_id, } @@ -394,6 +398,68 @@ describe('ProjectHistoryRedisManager', function () { }, version: this.version, projectHistoryId: this.projectHistoryId, + createdBlob: false, + doc: this.doc_id, + } + + this.ProjectHistoryRedisManager.promises.queueOps + .calledWithExactly(this.project_id, JSON.stringify(update)) + .should.equal(true) + }) + + it('should pass "false" as the createdBlob field if not provided', async function () { + await this.ProjectHistoryRedisManager.promises.queueAddEntity( + this.project_id, + this.projectHistoryId, + 'doc', + this.doc_id, + this.user_id, + this.rawUpdate, + this.source + ) + + const update = { + pathname: this.pathname, + docLines: this.docLines, + meta: { + user_id: this.user_id, + ts: new Date(), + source: this.source, + }, + version: this.version, + projectHistoryId: this.projectHistoryId, + createdBlob: false, + doc: this.doc_id, + } + + this.ProjectHistoryRedisManager.promises.queueOps + .calledWithExactly(this.project_id, JSON.stringify(update)) + .should.equal(true) + }) + + it('should pass through the value of the createdBlob field', async function () { + this.rawUpdate.createdBlob = true + await this.ProjectHistoryRedisManager.promises.queueAddEntity( + this.project_id, + this.projectHistoryId, + 'doc', + this.doc_id, + this.user_id, + this.rawUpdate, + this.source + ) + + const update = { + pathname: this.pathname, + docLines: this.docLines, + meta: { + user_id: this.user_id, + ts: new Date(), + source: this.source, + }, + version: this.version, + projectHistoryId: this.projectHistoryId, + createdBlob: true, doc: this.doc_id, } diff --git a/services/project-history/app/js/HistoryStoreManager.js b/services/project-history/app/js/HistoryStoreManager.js index 1e74912ed3..50bd52d793 100644 --- a/services/project-history/app/js/HistoryStoreManager.js +++ b/services/project-history/app/js/HistoryStoreManager.js @@ -300,8 +300,18 @@ export function createBlobForUpdate(projectId, historyId, update, callback) { if (urlMatch[1] !== projectId) { return callback(new OError('invalid project for blob creation')) } + const fileId = urlMatch[2] const filestoreURL = `${Settings.apis.filestore.url}/project/${projectId}/file/${fileId}` + + if (update.createdBlob) { + logger.debug( + { projectId, fileId }, + 'Skipping blob creation as it has already been created' + ) + return callback(null, { file: update.hash }) + } + fetchStream(filestoreURL, { signal: AbortSignal.timeout(HTTP_REQUEST_TIMEOUT), }) diff --git a/services/project-history/app/js/types.ts b/services/project-history/app/js/types.ts index baf70b59d4..c5f88e66e1 100644 --- a/services/project-history/app/js/types.ts +++ b/services/project-history/app/js/types.ts @@ -76,6 +76,7 @@ export type AddFileUpdate = ProjectUpdateBase & { file: string url: string hash: string + createdBlob?: boolean metadata?: LinkedFileData } diff --git a/services/project-history/test/unit/js/HistoryStoreManager/HistoryStoreManagerTests.js b/services/project-history/test/unit/js/HistoryStoreManager/HistoryStoreManagerTests.js index f8b560a56f..d98c6eb7a5 100644 --- a/services/project-history/test/unit/js/HistoryStoreManager/HistoryStoreManagerTests.js +++ b/services/project-history/test/unit/js/HistoryStoreManager/HistoryStoreManagerTests.js @@ -491,6 +491,44 @@ describe('HistoryStoreManager', function () { expect(this.actualHash).to.equal(this.hash) }) }) + describe('when the createdBlob flag is set on the update', function () { + beforeEach(function (done) { + this.file_id = '012345678901234567890123' + this.update = { + file: true, + createdBlob: true, + url: `http://filestore.other.cloud.provider/project/${this.projectId}/file/${this.file_id}`, + hash: this.hash, + } + this.HistoryStoreManager.createBlobForUpdate( + this.projectId, + this.historyId, + this.update, + (err, { file: hash }) => { + if (err) { + return done(err) + } + this.actualHash = hash + done() + } + ) + }) + + it('should call the callback with the existing hash', function () { + expect(this.actualHash).to.equal(this.hash) + }) + + it('should not request the file from the filestore', function () { + expect(this.FetchUtils.fetchStream).to.not.have.been.called + }) + + it('should log a debug level message', function () { + expect(this.logger.debug).to.have.been.calledWith( + { projectId: this.projectId, fileId: this.file_id }, + 'Skipping blob creation as it has already been created' + ) + }) + }) }) describe('getProjectBlob', function () { diff --git a/services/web/app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js b/services/web/app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js index ea7efcf8ee..7130b7131d 100644 --- a/services/web/app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js +++ b/services/web/app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js @@ -490,6 +490,7 @@ function _getUpdates( url: newEntity.url, hash: newEntity.file != null ? newEntity.file.hash : undefined, metadata: buildFileMetadataForHistory(newEntity.file), + createdBlob: newEntity.createdBlob ?? false, }) } else if (newEntity.path !== oldEntity.path) { // entity renamed diff --git a/services/web/app/src/Features/FileStore/FileStoreHandler.js b/services/web/app/src/Features/FileStore/FileStoreHandler.js index af75f71d84..dc2210af4d 100644 --- a/services/web/app/src/Features/FileStore/FileStoreHandler.js +++ b/services/web/app/src/Features/FileStore/FileStoreHandler.js @@ -100,7 +100,7 @@ const FileStoreHandler = { }) return callback(err) } - callback(err, result.url, result.fileRef) + callback(err, result.url, result.fileRef, true) } ) } @@ -303,6 +303,6 @@ module.exports = FileStoreHandler module.exports.promises = promisifyAll(FileStoreHandler, { multiResult: { uploadFileFromDisk: ['url', 'fileRef'], - uploadFileFromDiskWithHistoryId: ['url', 'fileRef'], + uploadFileFromDiskWithHistoryId: ['url', 'fileRef', 'createdBlob'], }, }) diff --git a/services/web/app/src/Features/Project/ProjectDuplicator.js b/services/web/app/src/Features/Project/ProjectDuplicator.js index 18594d3ff8..25e66c55ba 100644 --- a/services/web/app/src/Features/Project/ProjectDuplicator.js +++ b/services/web/app/src/Features/Project/ProjectDuplicator.js @@ -211,6 +211,7 @@ async function _copyFiles(sourceEntries, sourceProject, targetProject) { if (sourceFile.hash != null) { file.hash = sourceFile.hash } + let createdBlob = false if (file.hash != null) { try { await HistoryManager.promises.copyBlob( @@ -218,6 +219,7 @@ async function _copyFiles(sourceEntries, sourceProject, targetProject) { targetHistoryId, file.hash ) + createdBlob = true } catch (err) { logger.error( { @@ -237,7 +239,7 @@ async function _copyFiles(sourceEntries, sourceProject, targetProject) { targetProject._id, file._id ) - return { file, path, url } + return { createdBlob, file, path, url } } ) return targetEntries diff --git a/services/web/app/src/Features/Uploads/ProjectUploadManager.js b/services/web/app/src/Features/Uploads/ProjectUploadManager.js index 8af189254f..d42bc0eb22 100644 --- a/services/web/app/src/Features/Uploads/ProjectUploadManager.js +++ b/services/web/app/src/Features/Uploads/ProjectUploadManager.js @@ -194,14 +194,14 @@ async function _createFile(project, projectPath, fsPath) { throw new OError('missing history id') } const fileName = Path.basename(projectPath) - const { fileRef, url } = + const { createdBlob, fileRef, url } = await FileStoreHandler.promises.uploadFileFromDiskWithHistoryId( projectId, historyId, { name: fileName }, fsPath ) - return { file: fileRef, path: projectPath, url } + return { createdBlob, file: fileRef, path: projectPath, url } } async function _notifyDocumentUpdater(project, userId, changes) { diff --git a/services/web/test/unit/src/DocumentUpdater/DocumentUpdaterHandlerTests.js b/services/web/test/unit/src/DocumentUpdater/DocumentUpdaterHandlerTests.js index bdfe6ab9db..cec68eac72 100644 --- a/services/web/test/unit/src/DocumentUpdater/DocumentUpdaterHandlerTests.js +++ b/services/web/test/unit/src/DocumentUpdater/DocumentUpdaterHandlerTests.js @@ -1132,6 +1132,7 @@ describe('DocumentUpdaterHandler', function () { hash: undefined, ranges: undefined, metadata: undefined, + createdBlob: false, }, ] @@ -1184,6 +1185,7 @@ describe('DocumentUpdaterHandler', function () { historyRangesSupport: false, hash: '12345', ranges: undefined, + createdBlob: false, metadata: undefined, }, ] @@ -1296,6 +1298,7 @@ describe('DocumentUpdaterHandler', function () { hash: undefined, ranges: undefined, metadata: undefined, + createdBlob: false, }, ] @@ -1398,6 +1401,7 @@ describe('DocumentUpdaterHandler', function () { hash: undefined, ranges: this.ranges, metadata: undefined, + createdBlob: false, }, ] @@ -1444,6 +1448,7 @@ describe('DocumentUpdaterHandler', function () { hash: undefined, ranges: this.ranges, metadata: undefined, + createdBlob: false, }, ] diff --git a/services/web/test/unit/src/Project/ProjectDuplicatorTests.js b/services/web/test/unit/src/Project/ProjectDuplicatorTests.js index df53bbb044..3561d28809 100644 --- a/services/web/test/unit/src/Project/ProjectDuplicatorTests.js +++ b/services/web/test/unit/src/Project/ProjectDuplicatorTests.js @@ -105,16 +105,19 @@ describe('ProjectDuplicator', function () { ] this.fileEntries = [ { + createdBlob: false, path: this.file0Path, file: this.newFile0, url: this.filestoreUrl, }, { + createdBlob: false, path: this.file1Path, file: this.newFile1, url: this.filestoreUrl, }, { + createdBlob: true, path: this.file2Path, file: this.newFile2, url: this.filestoreUrl, diff --git a/services/web/test/unit/src/Uploads/ProjectUploadManagerTests.js b/services/web/test/unit/src/Uploads/ProjectUploadManagerTests.js index abdec03d11..2cfeb5a5cd 100644 --- a/services/web/test/unit/src/Uploads/ProjectUploadManagerTests.js +++ b/services/web/test/unit/src/Uploads/ProjectUploadManagerTests.js @@ -58,7 +58,12 @@ describe('ProjectUploadManager', function () { }, ] this.fileEntries = [ - { file: this.file, path: `/${this.file.name}`, url: this.fileStoreUrl }, + { + file: this.file, + path: `/${this.file.name}`, + url: this.fileStoreUrl, + createdBlob: true, + }, ] this.fs = { @@ -94,9 +99,11 @@ describe('ProjectUploadManager', function () { } this.FileStoreHandler = { promises: { - uploadFileFromDiskWithHistoryId: sinon - .stub() - .resolves({ fileRef: this.file, url: this.fileStoreUrl }), + uploadFileFromDiskWithHistoryId: sinon.stub().resolves({ + fileRef: this.file, + url: this.fileStoreUrl, + createdBlob: true, + }), }, } this.FileSystemImportManager = {