From 8d99ad39645e9d5b1dd5421379c15a879f5e0464 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 24 Jan 2025 12:18:36 +0000 Subject: [PATCH] [web] do not send filestore URLs when feature is disabled (#23095) * [web] do not return createdBlob=true from error path Defense in-depth, currently the only call-site bails out on error. * [web] do not send filestore URLs when feature is disabled GitOrigin-RevId: 7e90cf4c4babffeec337702502234bac73c1f116 --- services/project-history/app/js/types.ts | 2 +- .../DocumentUpdater/DocumentUpdaterHandler.js | 33 +- .../Features/FileStore/FileStoreHandler.js | 5 +- .../acceptance/src/ProjectStructureTests.mjs | 1710 +++++++++-------- .../DocumentUpdaterHandlerTests.js | 239 +++ 5 files changed, 1184 insertions(+), 805 deletions(-) diff --git a/services/project-history/app/js/types.ts b/services/project-history/app/js/types.ts index c5f88e66e1..638fa30ca2 100644 --- a/services/project-history/app/js/types.ts +++ b/services/project-history/app/js/types.ts @@ -210,7 +210,7 @@ export type Doc = { export type File = { file: string - url: string + url?: string path: string _hash: string metadata?: LinkedFileData diff --git a/services/web/app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js b/services/web/app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js index 70e6770053..b20a61d883 100644 --- a/services/web/app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js +++ b/services/web/app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js @@ -9,6 +9,7 @@ const { promisify } = require('util') const { promisifyMultiResult } = require('@overleaf/promise-utils') const ProjectGetter = require('../Project/ProjectGetter') const FileStoreHandler = require('../FileStore/FileStoreHandler') +const Features = require('../../infrastructure/Features') /** * @param {string} projectId @@ -275,11 +276,25 @@ function resyncProjectHistory( doc: doc.doc._id, path: doc.path, })) + const hasFilestore = Features.hasFeature('filestore') + if (!hasFilestore) { + // Files without a hash likely do not have a blob. Abort. + for (const { file } of files) { + if (!file.hash) { + return callback( + new OError('found file with missing hash', { projectId, file }) + ) + } + } + } files = files.map(file => ({ file: file.file._id, path: file.path, - url: FileStoreHandler._buildUrl(projectId, file.file._id), + url: hasFilestore + ? FileStoreHandler._buildUrl(projectId, file.file._id) + : undefined, _hash: file.file.hash, + createdBlob: !hasFilestore, metadata: buildFileMetadataForHistory(file.file), })) @@ -377,6 +392,17 @@ function updateProjectStructure( changes.newDocs, historyRangesSupport ) + const hasFilestore = Features.hasFeature('filestore') + if (!hasFilestore) { + for (const newEntity of changes.newFiles || []) { + if (!newEntity.file.hash) { + // Files without a hash likely do not have a blob. Abort. + return callback( + new OError('found file with missing hash', { newEntity }) + ) + } + } + } const { deletes: fileDeletes, adds: fileAdds, @@ -507,6 +533,7 @@ function _getUpdates( }) } } + const hasFilestore = Features.hasFeature('filestore') for (const id in newEntitiesHash) { const newEntity = newEntitiesHash[id] @@ -521,10 +548,10 @@ function _getUpdates( docLines: newEntity.docLines, ranges: newEntity.ranges, historyRangesSupport, - url: newEntity.url, + url: newEntity.file != null && hasFilestore ? newEntity.url : undefined, hash: newEntity.file != null ? newEntity.file.hash : undefined, metadata: buildFileMetadataForHistory(newEntity.file), - createdBlob: newEntity.createdBlob ?? false, + createdBlob: (newEntity.createdBlob || !hasFilestore) ?? false, }) } else if (newEntity.path !== oldEntity.path) { // entity renamed diff --git a/services/web/app/src/Features/FileStore/FileStoreHandler.js b/services/web/app/src/Features/FileStore/FileStoreHandler.js index 0c1d2cb391..7ca8e496d4 100644 --- a/services/web/app/src/Features/FileStore/FileStoreHandler.js +++ b/services/web/app/src/Features/FileStore/FileStoreHandler.js @@ -45,7 +45,10 @@ const FileStoreHandler = { FileStoreHandler.RETRY_ATTEMPTS, cb => HistoryManager.uploadBlobFromDisk(historyId, hash, size, fsPath, cb), - error => callback(error, true) + error => { + if (error) return callback(error, false) + callback(null, true) + } ) } else { callback(null, false) diff --git a/services/web/modules/history-v1/test/acceptance/src/ProjectStructureTests.mjs b/services/web/modules/history-v1/test/acceptance/src/ProjectStructureTests.mjs index 9e2e91c1de..e4b9a6063f 100644 --- a/services/web/modules/history-v1/test/acceptance/src/ProjectStructureTests.mjs +++ b/services/web/modules/history-v1/test/acceptance/src/ProjectStructureTests.mjs @@ -185,820 +185,715 @@ describe('ProjectStructureChanges', function () { }) } - describe('creating a project from the example template', function () { - let exampleProjectId - - beforeEach(function (done) { - createExampleProject(owner, (err, projectId) => { - exampleProjectId = projectId - done(err) + const cases = [ + { + label: 'with filestore disabled', + disableFilestore: true, + }, + { + label: 'with filestore enabled', + disableFilestore: false, + }, + ] + for (const { label, disableFilestore } of cases) { + describe(label, function () { + const previousSetting = Settings.disableFilestore + beforeEach(function () { + Settings.disableFilestore = disableFilestore }) - }) - - it('should version creating a doc and a file', function () { - const { updates, version } = - MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) - expect(updates.length).to.equal(3) - for (const update of updates.slice(0, 2)) { - expect(update.type).to.equal('add-doc') - expect(update.userId).to.equal(owner._id) - expect(update.docLines).to.be.a('string') - } - expect(_.filter(updates, { pathname: '/main.tex' }).length).to.equal(1) - expect(_.filter(updates, { pathname: '/sample.bib' }).length).to.equal(1) - expect(updates[2].type).to.equal('add-file') - expect(updates[2].userId).to.equal(owner._id) - expect(updates[2].pathname).to.equal('/frog.jpg') - expect(updates[2].url).to.be.a('string') - expect(version).to.equal(3) - }) - }) - - describe('duplicating a project', function () { - let dupProjectId - - beforeEach(function (done) { - createExampleProject(owner, (err, projectId) => { - if (err) { - return done(err) - } - owner.request.post( - { - uri: `/Project/${projectId}/clone`, - json: { - projectName: 'new.tex', - }, - }, - (error, res, body) => { - if (error) { - throw error - } - if (res.statusCode < 200 || res.statusCode >= 300) { - throw new Error(`failed to clone project ${res.statusCode}`) - } - dupProjectId = body.project_id - done() - } - ) + afterEach(function () { + Settings.disableFilestore = previousSetting }) - }) - it('should version the docs and files created', function () { - const { updates, version } = - MockDocUpdaterApi.getProjectStructureUpdates(dupProjectId) - expect(updates.length).to.equal(3) - for (const update of updates.slice(0, 2)) { - expect(update.type).to.equal('add-doc') - expect(update.userId).to.equal(owner._id) - expect(update.docLines).to.be.a('string') - } - expect(_.filter(updates, { pathname: '/main.tex' }).length).to.equal(1) - expect(_.filter(updates, { pathname: '/sample.bib' }).length).to.equal(1) - expect(updates[2].type).to.equal('add-file') - expect(updates[2].userId).to.equal(owner._id) - expect(updates[2].pathname).to.equal('/frog.jpg') - if (Features.hasFeature('project-history-blobs')) { - expect(updates[2].url).to.be.null - } else { - expect(updates[2].url).to.be.a('string') - } - expect(version).to.equal(1) - }) - }) + describe('creating a project from the example template', function () { + let exampleProjectId - describe('adding a doc', function () { - let exampleProjectId, oldVersion - - beforeEach(function (done) { - createExampleProject(owner, (err, projectId) => { - if (err) { - return done(err) - } - exampleProjectId = projectId - MockDocUpdaterApi.reset() - - ProjectGetter.getProject(projectId, (error, project) => { - if (error) { - return done(error) - } - oldVersion = project.version - createExampleDoc(owner, projectId, done) + beforeEach(function (done) { + createExampleProject(owner, (err, projectId) => { + exampleProjectId = projectId + done(err) + }) }) - }) - }) - it('should version the doc added', function (done) { - const { updates, version: newVersion } = - MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) - expect(updates.length).to.equal(1) - const update = updates[0] - expect(update.type).to.equal('add-doc') - expect(update.userId).to.equal(owner._id) - expect(update.pathname).to.equal('/new.tex') - expect(update.docLines).to.be.a('string') - - verifyVersionIncremented( - exampleProjectId, - oldVersion, - newVersion, - 1, - done - ) - }) - }) - - describe('uploading a project', function () { - let exampleProjectId - - beforeEach(function (done) { - uploadExampleProject(owner, 'test_project.zip', (err, projectId) => { - if (err) { - return done(err) - } - exampleProjectId = projectId - done() - }) - }) - - it('should version the docs and files created', function () { - const { updates, version } = - MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) - expect(updates.length).to.equal(2) - expect(updates[0].type).to.equal('add-doc') - expect(updates[0].userId).to.equal(owner._id) - expect(updates[0].pathname).to.equal('/main.tex') - expect(updates[0].docLines).to.equal('Test') - expect(updates[1].type).to.equal('add-file') - expect(updates[1].userId).to.equal(owner._id) - expect(updates[1].pathname).to.equal('/1pixel.png') - expect(updates[1].url).to.be.a('string') - expect(version).to.equal(1) - }) - }) - - describe('uploading a project with files in different encodings', function () { - let updates - beforeEach(function (done) { - uploadExampleProject(owner, 'charsets/charsets.zip', (err, projectId) => { - if (err) { - return done(err) - } - - updates = - MockDocUpdaterApi.getProjectStructureUpdates(projectId).updates - done() - }) - }) - - it('should correctly parse windows-1252', function () { - const update = _.find( - updates, - update => update.pathname === '/test-german-windows-1252.tex' - ) - expect(update.docLines).to.contain( - 'Der schnelle braune Fuchs sprang träge über den Hund.' - ) - }) - - it('should correctly parse German utf8', function () { - const update = _.find( - updates, - update => update.pathname === '/test-german-utf8x.tex' - ) - expect(update.docLines).to.contain( - 'Der schnelle braune Fuchs sprang träge über den Hund.' - ) - }) - - it('should correctly parse little-endian utf16', function () { - const update = _.find( - updates, - update => update.pathname === '/test-greek-utf16-le-bom.tex' - ) - expect(update.docLines).to.contain( - 'Η γρήγορη καστανή αλεπού πήδηξε χαλαρά πάνω από το σκυλί.' - ) - }) - - it('should correctly parse Greek utf8', function () { - const update = _.find( - updates, - update => update.pathname === '/test-greek-utf8x.tex' - ) - expect(update.docLines).to.contain( - 'Η γρήγορη καστανή αλεπού πήδηξε χαλαρά πάνω από το σκυλί.' - ) - }) - }) - - describe('uploading a file', function () { - let exampleProjectId, oldVersion, rootFolderId - - beforeEach(function (done) { - createExampleProject(owner, (err, projectId, folderId) => { - if (err) { - return done(err) - } - exampleProjectId = projectId - rootFolderId = folderId - MockDocUpdaterApi.reset() - ProjectGetter.getProject(projectId, (error, project) => { - if (error) { - throw error - } - - oldVersion = project.version - - uploadExampleFile(owner, projectId, rootFolderId, done) - }) - }) - }) - - it('should version a newly uploaded file', function (done) { - const { updates, version } = - MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) - expect(updates.length).to.equal(1) - const update = updates[0] - expect(update.type).to.equal('add-file') - expect(update.userId).to.equal(owner._id) - expect(update.pathname).to.equal('/1pixel.png') - expect(update.url).to.be.a('string') - - // one file upload - verifyVersionIncremented(exampleProjectId, oldVersion, version, 1, done) - }) - - it('should version a replacement file', function (done) { - MockDocUpdaterApi.reset() - - uploadFile( - owner, - exampleProjectId, - rootFolderId, - '2pixel.png', - '1pixel.png', - 'image/png', - () => { + it('should version creating a doc and a file', function () { const { updates, version } = MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) - expect(updates.length).to.equal(2) - expect(updates[0].type).to.equal('rename-file') - expect(updates[0].userId).to.equal(owner._id) - expect(updates[0].pathname).to.equal('/1pixel.png') - expect(updates[0].newPathname).to.equal('') - expect(updates[1].type).to.equal('add-file') - expect(updates[1].userId).to.equal(owner._id) - expect(updates[1].pathname).to.equal('/1pixel.png') - expect(updates[1].url).to.be.a('string') - - // two file uploads - verifyVersionIncremented( - exampleProjectId, - oldVersion, - version, - 2, - done - ) - } - ) - }) - }) - - describe('moving entities', function () { - let exampleProjectId, - oldVersion, - exampleDocId, - exampleFileId, - exampleFolderId - - beforeEach(function (done) { - createExampleProject(owner, (err, projectId, rootFolderId) => { - if (err) { - return done(err) - } - exampleProjectId = projectId - createExampleDoc(owner, projectId, (err, docId) => { - if (err) { - return done(err) + expect(updates.length).to.equal(3) + for (const update of updates.slice(0, 2)) { + expect(update.type).to.equal('add-doc') + expect(update.userId).to.equal(owner._id) + expect(update.docLines).to.be.a('string') } - exampleDocId = docId - uploadExampleFile(owner, projectId, rootFolderId, (err, fileId) => { + expect(_.filter(updates, { pathname: '/main.tex' }).length).to.equal( + 1 + ) + expect( + _.filter(updates, { pathname: '/sample.bib' }).length + ).to.equal(1) + expect(updates[2].type).to.equal('add-file') + expect(updates[2].userId).to.equal(owner._id) + expect(updates[2].pathname).to.equal('/frog.jpg') + if (disableFilestore) { + expect(updates[2].url).to.not.exist + expect(updates[2].createdBlob).to.be.true + } else { + expect(updates[2].url).to.be.a('string') + } + expect(version).to.equal(3) + }) + }) + + describe('duplicating a project', function () { + let dupProjectId + + beforeEach(function (done) { + createExampleProject(owner, (err, projectId) => { if (err) { return done(err) } - exampleFileId = fileId - createExampleFolder(owner, projectId, (err, folderId) => { - if (err) { - return done(err) - } - exampleFolderId = folderId - - ProjectGetter.getProject(projectId, (error, project) => { + owner.request.post( + { + uri: `/Project/${projectId}/clone`, + json: { + projectName: 'new.tex', + }, + }, + (error, res, body) => { if (error) { throw error } - oldVersion = project.version - MockDocUpdaterApi.reset() + if (res.statusCode < 200 || res.statusCode >= 300) { + throw new Error(`failed to clone project ${res.statusCode}`) + } + dupProjectId = body.project_id done() - }) + } + ) + }) + }) + + it('should version the docs and files created', function () { + const { updates, version } = + MockDocUpdaterApi.getProjectStructureUpdates(dupProjectId) + expect(updates.length).to.equal(3) + for (const update of updates.slice(0, 2)) { + expect(update.type).to.equal('add-doc') + expect(update.userId).to.equal(owner._id) + expect(update.docLines).to.be.a('string') + } + expect(_.filter(updates, { pathname: '/main.tex' }).length).to.equal( + 1 + ) + expect( + _.filter(updates, { pathname: '/sample.bib' }).length + ).to.equal(1) + expect(updates[2].type).to.equal('add-file') + expect(updates[2].userId).to.equal(owner._id) + expect(updates[2].pathname).to.equal('/frog.jpg') + if (disableFilestore) { + expect(updates[2].url).to.not.exist + expect(updates[2].createdBlob).to.be.true + } else if (Features.hasFeature('project-history-blobs')) { + expect(updates[2].url).to.be.null + } else { + expect(updates[2].url).to.be.a('string') + } + expect(version).to.equal(1) + }) + }) + + describe('adding a doc', function () { + let exampleProjectId, oldVersion + + beforeEach(function (done) { + createExampleProject(owner, (err, projectId) => { + if (err) { + return done(err) + } + exampleProjectId = projectId + MockDocUpdaterApi.reset() + + ProjectGetter.getProject(projectId, (error, project) => { + if (error) { + return done(error) + } + oldVersion = project.version + createExampleDoc(owner, projectId, done) }) }) }) - }) - }) - it('should version moving a doc', function (done) { - moveItem( - owner, - exampleProjectId, - 'doc', - exampleDocId, - exampleFolderId, - () => { - const { updates, version } = + it('should version the doc added', function (done) { + const { updates, version: newVersion } = MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) expect(updates.length).to.equal(1) const update = updates[0] - expect(update.type).to.equal('rename-doc') + expect(update.type).to.equal('add-doc') expect(update.userId).to.equal(owner._id) expect(update.pathname).to.equal('/new.tex') - expect(update.newPathname).to.equal('/foo/new.tex') + expect(update.docLines).to.be.a('string') - // 2, because it's a delete and then add verifyVersionIncremented( exampleProjectId, oldVersion, - version, - 2, + newVersion, + 1, done ) - } - ) - }) + }) + }) - it('should version moving a file', function (done) { - moveItem( - owner, - exampleProjectId, - 'file', - exampleFileId, - exampleFolderId, - () => { + describe('uploading a project', function () { + let exampleProjectId + + beforeEach(function (done) { + uploadExampleProject(owner, 'test_project.zip', (err, projectId) => { + if (err) { + return done(err) + } + exampleProjectId = projectId + done() + }) + }) + + it('should version the docs and files created', function () { const { updates, version } = MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) - expect(updates.length).to.equal(1) - const update = updates[0] - expect(update.type).to.equal('rename-file') - expect(update.userId).to.equal(owner._id) - expect(update.pathname).to.equal('/1pixel.png') - expect(update.newPathname).to.equal('/foo/1pixel.png') + expect(updates.length).to.equal(2) + expect(updates[0].type).to.equal('add-doc') + expect(updates[0].userId).to.equal(owner._id) + expect(updates[0].pathname).to.equal('/main.tex') + expect(updates[0].docLines).to.equal('Test') + expect(updates[1].type).to.equal('add-file') + expect(updates[1].userId).to.equal(owner._id) + expect(updates[1].pathname).to.equal('/1pixel.png') + if (disableFilestore) { + expect(updates[1].url).to.not.exist + expect(updates[1].createdBlob).to.be.true + } else { + expect(updates[1].url).to.be.a('string') + } + expect(version).to.equal(1) + }) + }) - // 2, because it's a delete and then add - verifyVersionIncremented( - exampleProjectId, - oldVersion, - version, - 2, - done + describe('uploading a project with files in different encodings', function () { + let updates + beforeEach(function (done) { + uploadExampleProject( + owner, + 'charsets/charsets.zip', + (err, projectId) => { + if (err) { + return done(err) + } + + updates = + MockDocUpdaterApi.getProjectStructureUpdates(projectId).updates + done() + } ) - } - ) - }) + }) - it('should version moving a folder', function (done) { - moveItem( - owner, - exampleProjectId, - 'doc', - exampleDocId, - exampleFolderId, - () => { - MockDocUpdaterApi.reset() + it('should correctly parse windows-1252', function () { + const update = _.find( + updates, + update => update.pathname === '/test-german-windows-1252.tex' + ) + expect(update.docLines).to.contain( + 'Der schnelle braune Fuchs sprang träge über den Hund.' + ) + }) - owner.request.post( - { - uri: `project/${exampleProjectId}/folder`, - json: { - name: 'bar', - }, - }, - (error, res, body) => { + it('should correctly parse German utf8', function () { + const update = _.find( + updates, + update => update.pathname === '/test-german-utf8x.tex' + ) + expect(update.docLines).to.contain( + 'Der schnelle braune Fuchs sprang träge über den Hund.' + ) + }) + + it('should correctly parse little-endian utf16', function () { + const update = _.find( + updates, + update => update.pathname === '/test-greek-utf16-le-bom.tex' + ) + expect(update.docLines).to.contain( + 'Η γρήγορη καστανή αλεπού πήδηξε χαλαρά πάνω από το σκυλί.' + ) + }) + + it('should correctly parse Greek utf8', function () { + const update = _.find( + updates, + update => update.pathname === '/test-greek-utf8x.tex' + ) + expect(update.docLines).to.contain( + 'Η γρήγορη καστανή αλεπού πήδηξε χαλαρά πάνω από το σκυλί.' + ) + }) + }) + + describe('uploading a file', function () { + let exampleProjectId, oldVersion, rootFolderId + + beforeEach(function (done) { + createExampleProject(owner, (err, projectId, folderId) => { + if (err) { + return done(err) + } + exampleProjectId = projectId + rootFolderId = folderId + MockDocUpdaterApi.reset() + ProjectGetter.getProject(projectId, (error, project) => { if (error) { throw error } - const newFolderId = body._id - moveItem( - owner, + oldVersion = project.version + + uploadExampleFile(owner, projectId, rootFolderId, done) + }) + }) + }) + + it('should version a newly uploaded file', function (done) { + const { updates, version } = + MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) + expect(updates.length).to.equal(1) + const update = updates[0] + expect(update.type).to.equal('add-file') + expect(update.userId).to.equal(owner._id) + expect(update.pathname).to.equal('/1pixel.png') + if (disableFilestore) { + expect(update.url).to.not.exist + expect(update.createdBlob).to.be.true + } else { + expect(update.url).to.be.a('string') + } + + // one file upload + verifyVersionIncremented( + exampleProjectId, + oldVersion, + version, + 1, + done + ) + }) + + it('should version a replacement file', function (done) { + MockDocUpdaterApi.reset() + + uploadFile( + owner, + exampleProjectId, + rootFolderId, + '2pixel.png', + '1pixel.png', + 'image/png', + () => { + const { updates, version } = + MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) + expect(updates.length).to.equal(2) + expect(updates[0].type).to.equal('rename-file') + expect(updates[0].userId).to.equal(owner._id) + expect(updates[0].pathname).to.equal('/1pixel.png') + expect(updates[0].newPathname).to.equal('') + expect(updates[1].type).to.equal('add-file') + expect(updates[1].userId).to.equal(owner._id) + expect(updates[1].pathname).to.equal('/1pixel.png') + if (disableFilestore) { + expect(updates[1].url).to.not.exist + expect(updates[1].createdBlob).to.be.true + } else { + expect(updates[1].url).to.be.a('string') + } + + // two file uploads + verifyVersionIncremented( exampleProjectId, - 'folder', - exampleFolderId, - newFolderId, - () => { - const { updates, version } = - MockDocUpdaterApi.getProjectStructureUpdates( - exampleProjectId - ) - expect(updates.length).to.equal(1) - const update = updates[0] - expect(update.type).to.equal('rename-doc') - expect(update.userId).to.equal(owner._id) - expect(update.pathname).to.equal('/foo/new.tex') - expect(update.newPathname).to.equal('/bar/foo/new.tex') + oldVersion, + version, + 2, + done + ) + } + ) + }) + }) - // 5, because it's two file moves plus a folder - verifyVersionIncremented( + describe('moving entities', function () { + let exampleProjectId, + oldVersion, + exampleDocId, + exampleFileId, + exampleFolderId + + beforeEach(function (done) { + createExampleProject(owner, (err, projectId, rootFolderId) => { + if (err) { + return done(err) + } + exampleProjectId = projectId + createExampleDoc(owner, projectId, (err, docId) => { + if (err) { + return done(err) + } + exampleDocId = docId + uploadExampleFile( + owner, + projectId, + rootFolderId, + (err, fileId) => { + if (err) { + return done(err) + } + exampleFileId = fileId + createExampleFolder(owner, projectId, (err, folderId) => { + if (err) { + return done(err) + } + exampleFolderId = folderId + + ProjectGetter.getProject(projectId, (error, project) => { + if (error) { + throw error + } + oldVersion = project.version + MockDocUpdaterApi.reset() + done() + }) + }) + } + ) + }) + }) + }) + + it('should version moving a doc', function (done) { + moveItem( + owner, + exampleProjectId, + 'doc', + exampleDocId, + exampleFolderId, + () => { + const { updates, version } = + MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) + expect(updates.length).to.equal(1) + const update = updates[0] + expect(update.type).to.equal('rename-doc') + expect(update.userId).to.equal(owner._id) + expect(update.pathname).to.equal('/new.tex') + expect(update.newPathname).to.equal('/foo/new.tex') + + // 2, because it's a delete and then add + verifyVersionIncremented( + exampleProjectId, + oldVersion, + version, + 2, + done + ) + } + ) + }) + + it('should version moving a file', function (done) { + moveItem( + owner, + exampleProjectId, + 'file', + exampleFileId, + exampleFolderId, + () => { + const { updates, version } = + MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) + expect(updates.length).to.equal(1) + const update = updates[0] + expect(update.type).to.equal('rename-file') + expect(update.userId).to.equal(owner._id) + expect(update.pathname).to.equal('/1pixel.png') + expect(update.newPathname).to.equal('/foo/1pixel.png') + + // 2, because it's a delete and then add + verifyVersionIncremented( + exampleProjectId, + oldVersion, + version, + 2, + done + ) + } + ) + }) + + it('should version moving a folder', function (done) { + moveItem( + owner, + exampleProjectId, + 'doc', + exampleDocId, + exampleFolderId, + () => { + MockDocUpdaterApi.reset() + + owner.request.post( + { + uri: `project/${exampleProjectId}/folder`, + json: { + name: 'bar', + }, + }, + (error, res, body) => { + if (error) { + throw error + } + const newFolderId = body._id + + moveItem( + owner, exampleProjectId, - oldVersion, - version, - 5, - done + 'folder', + exampleFolderId, + newFolderId, + () => { + const { updates, version } = + MockDocUpdaterApi.getProjectStructureUpdates( + exampleProjectId + ) + expect(updates.length).to.equal(1) + const update = updates[0] + expect(update.type).to.equal('rename-doc') + expect(update.userId).to.equal(owner._id) + expect(update.pathname).to.equal('/foo/new.tex') + expect(update.newPathname).to.equal('/bar/foo/new.tex') + + // 5, because it's two file moves plus a folder + verifyVersionIncremented( + exampleProjectId, + oldVersion, + version, + 5, + done + ) + } ) } ) } ) - } - ) - }) - }) + }) + }) - describe('renaming entities', function () { - let exampleProjectId, - exampleDocId, - exampleFileId, - exampleFolderId, - oldVersion + describe('renaming entities', function () { + let exampleProjectId, + exampleDocId, + exampleFileId, + exampleFolderId, + oldVersion - beforeEach(function (done) { - createExampleProject(owner, (err, projectId, rootFolderId) => { - if (err) { - return done(err) - } - exampleProjectId = projectId - createExampleDoc(owner, projectId, (err, docId) => { - if (err) { - return done(err) - } - exampleDocId = docId - uploadExampleFile(owner, projectId, rootFolderId, (err, fileId) => { + beforeEach(function (done) { + createExampleProject(owner, (err, projectId, rootFolderId) => { if (err) { return done(err) } - exampleFileId = fileId - createExampleFolder(owner, projectId, (err, folderId) => { + exampleProjectId = projectId + createExampleDoc(owner, projectId, (err, docId) => { + if (err) { + return done(err) + } + exampleDocId = docId + uploadExampleFile( + owner, + projectId, + rootFolderId, + (err, fileId) => { + if (err) { + return done(err) + } + exampleFileId = fileId + createExampleFolder(owner, projectId, (err, folderId) => { + if (err) { + return done(err) + } + exampleFolderId = folderId + moveItem(owner, projectId, 'doc', docId, folderId, () => { + moveItem( + owner, + projectId, + 'file', + fileId, + folderId, + () => { + MockDocUpdaterApi.reset() + ProjectGetter.getProject( + exampleProjectId, + (error, project) => { + if (error) { + throw error + } + oldVersion = project.version + done() + } + ) + } + ) + }) + }) + } + ) + }) + }) + }) + + it('should version renaming a doc', function (done) { + renameItem( + owner, + exampleProjectId, + 'Doc', + exampleDocId, + 'wombat.tex', + () => { + const { updates, version } = + MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) + expect(updates.length).to.equal(1) + const update = updates[0] + expect(update.type).to.equal('rename-doc') + expect(update.userId).to.equal(owner._id) + expect(update.pathname).to.equal('/foo/new.tex') + expect(update.newPathname).to.equal('/foo/wombat.tex') + + verifyVersionIncremented( + exampleProjectId, + oldVersion, + version, + 1, + done + ) + } + ) + }) + + it('should version renaming a file', function (done) { + renameItem( + owner, + exampleProjectId, + 'file', + exampleFileId, + 'potato.png', + () => { + const { updates, version } = + MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) + expect(updates.length).to.equal(1) + const update = updates[0] + expect(update.type).to.equal('rename-file') + expect(update.userId).to.equal(owner._id) + expect(update.pathname).to.equal('/foo/1pixel.png') + expect(update.newPathname).to.equal('/foo/potato.png') + + verifyVersionIncremented( + exampleProjectId, + oldVersion, + version, + 1, + done + ) + } + ) + }) + + it('should version renaming a folder', function (done) { + renameItem( + owner, + exampleProjectId, + 'folder', + exampleFolderId, + 'giraffe', + () => { + const { updates, version } = + MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) + expect(updates.length).to.equal(2) + expect(updates[0].type).to.equal('rename-doc') + expect(updates[0].userId).to.equal(owner._id) + expect(updates[0].pathname).to.equal('/foo/new.tex') + expect(updates[0].newPathname).to.equal('/giraffe/new.tex') + expect(updates[1].type).to.equal('rename-file') + expect(updates[1].userId).to.equal(owner._id) + expect(updates[1].pathname).to.equal('/foo/1pixel.png') + expect(updates[1].newPathname).to.equal('/giraffe/1pixel.png') + + verifyVersionIncremented( + exampleProjectId, + oldVersion, + version, + 1, + done + ) + } + ) + }) + }) + + describe('deleting entities', function () { + let exampleProjectId, oldVersion, exampleFolderId + + beforeEach(function (done) { + createExampleProject(owner, (err, projectId) => { + if (err) { + return done(err) + } + exampleProjectId = projectId + createExampleFolder(owner, exampleProjectId, (err, folderId) => { if (err) { return done(err) } exampleFolderId = folderId - moveItem(owner, projectId, 'doc', docId, folderId, () => { - moveItem(owner, projectId, 'file', fileId, folderId, () => { - MockDocUpdaterApi.reset() - ProjectGetter.getProject( - exampleProjectId, - (error, project) => { - if (error) { - throw error - } - oldVersion = project.version - done() - } - ) + createExampleDoc(owner, projectId, (err, docId) => { + if (err) { + return done(err) + } + uploadExampleFile(owner, projectId, folderId, (err, fileId) => { + if (err) { + return done(err) + } + moveItem(owner, projectId, 'doc', docId, folderId, () => { + moveItem(owner, projectId, 'file', fileId, folderId, () => { + MockDocUpdaterApi.reset() + ProjectGetter.getProject( + exampleProjectId, + (error, project) => { + if (error) { + throw error + } + oldVersion = project.version + done() + } + ) + }) + }) }) }) }) }) }) - }) - }) - - it('should version renaming a doc', function (done) { - renameItem( - owner, - exampleProjectId, - 'Doc', - exampleDocId, - 'wombat.tex', - () => { - const { updates, version } = - MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) - expect(updates.length).to.equal(1) - const update = updates[0] - expect(update.type).to.equal('rename-doc') - expect(update.userId).to.equal(owner._id) - expect(update.pathname).to.equal('/foo/new.tex') - expect(update.newPathname).to.equal('/foo/wombat.tex') - - verifyVersionIncremented( - exampleProjectId, - oldVersion, - version, - 1, - done - ) - } - ) - }) - - it('should version renaming a file', function (done) { - renameItem( - owner, - exampleProjectId, - 'file', - exampleFileId, - 'potato.png', - () => { - const { updates, version } = - MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) - expect(updates.length).to.equal(1) - const update = updates[0] - expect(update.type).to.equal('rename-file') - expect(update.userId).to.equal(owner._id) - expect(update.pathname).to.equal('/foo/1pixel.png') - expect(update.newPathname).to.equal('/foo/potato.png') - - verifyVersionIncremented( - exampleProjectId, - oldVersion, - version, - 1, - done - ) - } - ) - }) - - it('should version renaming a folder', function (done) { - renameItem( - owner, - exampleProjectId, - 'folder', - exampleFolderId, - 'giraffe', - () => { - const { updates, version } = - MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) - expect(updates.length).to.equal(2) - expect(updates[0].type).to.equal('rename-doc') - expect(updates[0].userId).to.equal(owner._id) - expect(updates[0].pathname).to.equal('/foo/new.tex') - expect(updates[0].newPathname).to.equal('/giraffe/new.tex') - expect(updates[1].type).to.equal('rename-file') - expect(updates[1].userId).to.equal(owner._id) - expect(updates[1].pathname).to.equal('/foo/1pixel.png') - expect(updates[1].newPathname).to.equal('/giraffe/1pixel.png') - - verifyVersionIncremented( - exampleProjectId, - oldVersion, - version, - 1, - done - ) - } - ) - }) - }) - - describe('deleting entities', function () { - let exampleProjectId, oldVersion, exampleFolderId - - beforeEach(function (done) { - createExampleProject(owner, (err, projectId) => { - if (err) { - return done(err) - } - exampleProjectId = projectId - createExampleFolder(owner, exampleProjectId, (err, folderId) => { - if (err) { - return done(err) - } - exampleFolderId = folderId - createExampleDoc(owner, projectId, (err, docId) => { - if (err) { - return done(err) - } - uploadExampleFile(owner, projectId, folderId, (err, fileId) => { - if (err) { - return done(err) - } - moveItem(owner, projectId, 'doc', docId, folderId, () => { - moveItem(owner, projectId, 'file', fileId, folderId, () => { - MockDocUpdaterApi.reset() - ProjectGetter.getProject( - exampleProjectId, - (error, project) => { - if (error) { - throw error - } - oldVersion = project.version - done() - } - ) - }) - }) - }) - }) - }) - }) - }) - - it('should version deleting a folder', function (done) { - deleteItem(owner, exampleProjectId, 'folder', exampleFolderId, () => { - const { updates, version } = - MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) - expect(updates.length).to.equal(2) - expect(updates[0].type).to.equal('rename-doc') - expect(updates[0].userId).to.equal(owner._id) - expect(updates[0].pathname).to.equal('/foo/new.tex') - expect(updates[0].newPathname).to.equal('') - expect(updates[1].type).to.equal('rename-file') - expect(updates[1].userId).to.equal(owner._id) - expect(updates[1].pathname).to.equal('/foo/1pixel.png') - expect(updates[1].newPathname).to.equal('') - - verifyVersionIncremented(exampleProjectId, oldVersion, version, 1, done) - }) - }) - }) - - describe('tpds', function () { - let projectName, exampleProjectId, oldVersion, rootFolderId - - beforeEach(function (done) { - projectName = `tpds-project-${new ObjectId().toString()}` - owner.createProject(projectName, (error, projectId) => { - if (error) { - throw error - } - exampleProjectId = projectId - fs.mkdir(Settings.path.dumpFolder, { recursive: true }, error => { - if (error) { - throw error - } - ProjectGetter.getProject(exampleProjectId, (error, project) => { - if (error) { - throw error - } - MockDocUpdaterApi.reset() - rootFolderId = project.rootFolder[0]._id.toString() - oldVersion = project.version - done() - }) - }) - }) - }) - - it('should version adding a doc', function (done) { - const req = owner.request.post({ - uri: `/user/${owner._id}/update/${projectName}/test.tex`, - auth: { - user: _.keys(Settings.httpAuthUsers)[0], - pass: _.values(Settings.httpAuthUsers)[0], - sendImmediately: true, - }, - body: fs.createReadStream(Path.join(FILES_PATH, 'test.tex')), - }) - - req.on('error', err => { - throw err - }) - - req.on('response', res => { - if (res.statusCode < 200 || res.statusCode >= 300) { - throw new Error(`failed to upload file ${res.statusCode}`) - } - - const { updates, version } = - MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) - expect(updates.length).to.equal(1) - const update = updates[0] - expect(update.type).to.equal('add-doc') - expect(update.userId).to.equal(owner._id) - expect(update.pathname).to.equal('/test.tex') - expect(update.docLines).to.equal('Test') - - verifyVersionIncremented(exampleProjectId, oldVersion, version, 1, done) - }) - }) - - it('should version adding a new file', function (done) { - const req = owner.request.post({ - uri: `/user/${owner._id}/update/${projectName}/1pixel.png`, - auth: { - user: _.keys(Settings.httpAuthUsers)[0], - pass: _.values(Settings.httpAuthUsers)[0], - sendImmediately: true, - }, - body: fs.createReadStream(Path.join(FILES_PATH, '1pixel.png')), - }) - - req.on('error', err => { - throw err - }) - - req.on('response', res => { - if (res.statusCode < 200 || res.statusCode >= 300) { - throw new Error(`failed to upload file ${res.statusCode}`) - } - - const { updates, version } = - MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) - expect(updates.length).to.equal(1) - const update = updates[0] - expect(update.type).to.equal('add-file') - expect(update.userId).to.equal(owner._id) - expect(update.pathname).to.equal('/1pixel.png') - expect(update.url).to.be.a('string') - - verifyVersionIncremented(exampleProjectId, oldVersion, version, 1, done) - }) - }) - - describe('when there are files in the project', function () { - beforeEach(function (done) { - uploadExampleFile(owner, exampleProjectId, rootFolderId, () => { - createExampleDoc(owner, exampleProjectId, () => { - ProjectGetter.getProject(exampleProjectId, (error, project) => { - if (error) { - throw error - } - MockDocUpdaterApi.reset() - oldVersion = project.version - done() - }) - }) - }) - }) - - it('should version replacing a file', function (done) { - const req = owner.request.post({ - uri: `/user/${owner._id}/update/${projectName}/1pixel.png`, - auth: { - user: _.keys(Settings.httpAuthUsers)[0], - pass: _.values(Settings.httpAuthUsers)[0], - sendImmediately: true, - }, - body: fs.createReadStream(Path.join(FILES_PATH, '2pixel.png')), - }) - - req.on('error', err => { - throw err - }) - - req.on('response', res => { - if (res.statusCode < 200 || res.statusCode >= 300) { - throw new Error(`failed to upload file ${res.statusCode}`) - } - - const { updates, version } = - MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) - expect(updates.length).to.equal(2) - expect(updates[0].type).to.equal('rename-file') - expect(updates[0].userId).to.equal(owner._id) - expect(updates[0].pathname).to.equal('/1pixel.png') - expect(updates[0].newPathname).to.equal('') - expect(updates[1].type).to.equal('add-file') - expect(updates[1].userId).to.equal(owner._id) - expect(updates[1].pathname).to.equal('/1pixel.png') - expect(updates[1].url).to.be.a('string') - - verifyVersionIncremented( - exampleProjectId, - oldVersion, - version, - 1, - done - ) - }) - }) - - it('should version deleting a doc', function (done) { - owner.request.delete( - { - uri: `/user/${owner._id}/update/${projectName}/new.tex`, - auth: { - user: _.keys(Settings.httpAuthUsers)[0], - pass: _.values(Settings.httpAuthUsers)[0], - sendImmediately: true, - }, - }, - (error, res) => { - if (error) { - throw error - } - if (res.statusCode < 200 || res.statusCode >= 300) { - throw new Error(`failed to delete doc ${res.statusCode}`) - } + it('should version deleting a folder', function (done) { + deleteItem(owner, exampleProjectId, 'folder', exampleFolderId, () => { const { updates, version } = MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) - expect(updates.length).to.equal(1) - const update = updates[0] - expect(update.type).to.equal('rename-doc') - expect(update.userId).to.equal(owner._id) - expect(update.pathname).to.equal('/new.tex') - expect(update.newPathname).to.equal('') + expect(updates.length).to.equal(2) + expect(updates[0].type).to.equal('rename-doc') + expect(updates[0].userId).to.equal(owner._id) + expect(updates[0].pathname).to.equal('/foo/new.tex') + expect(updates[0].newPathname).to.equal('') + expect(updates[1].type).to.equal('rename-file') + expect(updates[1].userId).to.equal(owner._id) + expect(updates[1].pathname).to.equal('/foo/1pixel.png') + expect(updates[1].newPathname).to.equal('') verifyVersionIncremented( exampleProjectId, @@ -1007,72 +902,287 @@ describe('ProjectStructureChanges', function () { 1, done ) - } - ) + }) + }) }) - }) - }) - describe('uploading a document', function () { - let exampleProjectId, rootFolderId - beforeEach(function (done) { - createExampleProject(owner, (err, projectId, folderId) => { - if (err) { - return done(err) - } - exampleProjectId = projectId - rootFolderId = folderId - MockDocUpdaterApi.reset() - done() - }) - }) + describe('tpds', function () { + let projectName, exampleProjectId, oldVersion, rootFolderId - describe('with an unusual character set', function () { - it('should correctly handle utf16-le data', function (done) { - uploadFile( - owner, - exampleProjectId, - rootFolderId, - 'charsets/test-greek-utf16-le-bom.tex', - 'test-greek-utf16-le-bom.tex', - 'text/x-tex', - () => { - const { updates } = + beforeEach(function (done) { + projectName = `tpds-project-${new ObjectId().toString()}` + owner.createProject(projectName, (error, projectId) => { + if (error) { + throw error + } + exampleProjectId = projectId + fs.mkdir(Settings.path.dumpFolder, { recursive: true }, error => { + if (error) { + throw error + } + ProjectGetter.getProject(exampleProjectId, (error, project) => { + if (error) { + throw error + } + MockDocUpdaterApi.reset() + rootFolderId = project.rootFolder[0]._id.toString() + oldVersion = project.version + done() + }) + }) + }) + }) + + it('should version adding a doc', function (done) { + const req = owner.request.post({ + uri: `/user/${owner._id}/update/${projectName}/test.tex`, + auth: { + user: _.keys(Settings.httpAuthUsers)[0], + pass: _.values(Settings.httpAuthUsers)[0], + sendImmediately: true, + }, + body: fs.createReadStream(Path.join(FILES_PATH, 'test.tex')), + }) + + req.on('error', err => { + throw err + }) + + req.on('response', res => { + if (res.statusCode < 200 || res.statusCode >= 300) { + throw new Error(`failed to upload file ${res.statusCode}`) + } + + const { updates, version } = MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) expect(updates.length).to.equal(1) const update = updates[0] expect(update.type).to.equal('add-doc') - expect(update.pathname).to.equal('/test-greek-utf16-le-bom.tex') - expect(update.docLines).to.contain( - 'Η γρήγορη καστανή αλεπού πήδηξε χαλαρά πάνω από το σκυλί.' - ) - done() - } - ) - }) + expect(update.userId).to.equal(owner._id) + expect(update.pathname).to.equal('/test.tex') + expect(update.docLines).to.equal('Test') - it('should correctly handle windows1252/iso-8859-1/latin1 data', function (done) { - uploadFile( - owner, - exampleProjectId, - rootFolderId, - 'charsets/test-german-windows-1252.tex', - 'test-german-windows-1252.tex', - 'text/x-tex', - () => { - const { updates } = + verifyVersionIncremented( + exampleProjectId, + oldVersion, + version, + 1, + done + ) + }) + }) + + it('should version adding a new file', function (done) { + const req = owner.request.post({ + uri: `/user/${owner._id}/update/${projectName}/1pixel.png`, + auth: { + user: _.keys(Settings.httpAuthUsers)[0], + pass: _.values(Settings.httpAuthUsers)[0], + sendImmediately: true, + }, + body: fs.createReadStream(Path.join(FILES_PATH, '1pixel.png')), + }) + + req.on('error', err => { + throw err + }) + + req.on('response', res => { + if (res.statusCode < 200 || res.statusCode >= 300) { + throw new Error(`failed to upload file ${res.statusCode}`) + } + + const { updates, version } = MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) expect(updates.length).to.equal(1) const update = updates[0] - expect(update.type).to.equal('add-doc') - expect(update.pathname).to.equal('/test-german-windows-1252.tex') - expect(update.docLines).to.contain( - 'Der schnelle braune Fuchs sprang träge über den Hund.' + expect(update.type).to.equal('add-file') + expect(update.userId).to.equal(owner._id) + expect(update.pathname).to.equal('/1pixel.png') + if (disableFilestore) { + expect(update.url).to.not.exist + expect(update.createdBlob).to.be.true + } else { + expect(update.url).to.be.a('string') + } + + verifyVersionIncremented( + exampleProjectId, + oldVersion, + version, + 1, + done ) + }) + }) + + describe('when there are files in the project', function () { + beforeEach(function (done) { + uploadExampleFile(owner, exampleProjectId, rootFolderId, () => { + createExampleDoc(owner, exampleProjectId, () => { + ProjectGetter.getProject(exampleProjectId, (error, project) => { + if (error) { + throw error + } + MockDocUpdaterApi.reset() + oldVersion = project.version + done() + }) + }) + }) + }) + + it('should version replacing a file', function (done) { + const req = owner.request.post({ + uri: `/user/${owner._id}/update/${projectName}/1pixel.png`, + auth: { + user: _.keys(Settings.httpAuthUsers)[0], + pass: _.values(Settings.httpAuthUsers)[0], + sendImmediately: true, + }, + body: fs.createReadStream(Path.join(FILES_PATH, '2pixel.png')), + }) + + req.on('error', err => { + throw err + }) + + req.on('response', res => { + if (res.statusCode < 200 || res.statusCode >= 300) { + throw new Error(`failed to upload file ${res.statusCode}`) + } + + const { updates, version } = + MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) + expect(updates.length).to.equal(2) + expect(updates[0].type).to.equal('rename-file') + expect(updates[0].userId).to.equal(owner._id) + expect(updates[0].pathname).to.equal('/1pixel.png') + expect(updates[0].newPathname).to.equal('') + expect(updates[1].type).to.equal('add-file') + expect(updates[1].userId).to.equal(owner._id) + expect(updates[1].pathname).to.equal('/1pixel.png') + if (disableFilestore) { + expect(updates[1].url).to.not.exist + expect(updates[1].createdBlob).to.be.true + } else { + expect(updates[1].url).to.be.a('string') + } + + verifyVersionIncremented( + exampleProjectId, + oldVersion, + version, + 1, + done + ) + }) + }) + + it('should version deleting a doc', function (done) { + owner.request.delete( + { + uri: `/user/${owner._id}/update/${projectName}/new.tex`, + auth: { + user: _.keys(Settings.httpAuthUsers)[0], + pass: _.values(Settings.httpAuthUsers)[0], + sendImmediately: true, + }, + }, + (error, res) => { + if (error) { + throw error + } + if (res.statusCode < 200 || res.statusCode >= 300) { + throw new Error(`failed to delete doc ${res.statusCode}`) + } + + const { updates, version } = + MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) + expect(updates.length).to.equal(1) + const update = updates[0] + expect(update.type).to.equal('rename-doc') + expect(update.userId).to.equal(owner._id) + expect(update.pathname).to.equal('/new.tex') + expect(update.newPathname).to.equal('') + + verifyVersionIncremented( + exampleProjectId, + oldVersion, + version, + 1, + done + ) + } + ) + }) + }) + }) + + describe('uploading a document', function () { + let exampleProjectId, rootFolderId + beforeEach(function (done) { + createExampleProject(owner, (err, projectId, folderId) => { + if (err) { + return done(err) + } + exampleProjectId = projectId + rootFolderId = folderId + MockDocUpdaterApi.reset() done() - } - ) + }) + }) + + describe('with an unusual character set', function () { + it('should correctly handle utf16-le data', function (done) { + uploadFile( + owner, + exampleProjectId, + rootFolderId, + 'charsets/test-greek-utf16-le-bom.tex', + 'test-greek-utf16-le-bom.tex', + 'text/x-tex', + () => { + const { updates } = + MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) + expect(updates.length).to.equal(1) + const update = updates[0] + expect(update.type).to.equal('add-doc') + expect(update.pathname).to.equal('/test-greek-utf16-le-bom.tex') + expect(update.docLines).to.contain( + 'Η γρήγορη καστανή αλεπού πήδηξε χαλαρά πάνω από το σκυλί.' + ) + done() + } + ) + }) + + it('should correctly handle windows1252/iso-8859-1/latin1 data', function (done) { + uploadFile( + owner, + exampleProjectId, + rootFolderId, + 'charsets/test-german-windows-1252.tex', + 'test-german-windows-1252.tex', + 'text/x-tex', + () => { + const { updates } = + MockDocUpdaterApi.getProjectStructureUpdates(exampleProjectId) + expect(updates.length).to.equal(1) + const update = updates[0] + expect(update.type).to.equal('add-doc') + expect(update.pathname).to.equal( + '/test-german-windows-1252.tex' + ) + expect(update.docLines).to.contain( + 'Der schnelle braune Fuchs sprang träge über den Hund.' + ) + done() + } + ) + }) + }) }) }) - }) + } }) diff --git a/services/web/test/unit/src/DocumentUpdater/DocumentUpdaterHandlerTests.js b/services/web/test/unit/src/DocumentUpdater/DocumentUpdaterHandlerTests.js index 3e9d2ee9d9..2d5a116337 100644 --- a/services/web/test/unit/src/DocumentUpdater/DocumentUpdaterHandlerTests.js +++ b/services/web/test/unit/src/DocumentUpdater/DocumentUpdaterHandlerTests.js @@ -28,6 +28,7 @@ describe('DocumentUpdaterHandler', function () { url: 'http://project_history.example.com', }, }, + moduleImportSequence: [], } this.source = 'dropbox' @@ -1510,6 +1511,89 @@ describe('DocumentUpdaterHandler', function () { ) }) }) + + describe('with filestore disabled', function () { + beforeEach(function () { + this.settings.disableFilestore = true + }) + it('should add files without URL and with createdBlob', function (done) { + this.fileId = new ObjectId() + this.changes = { + newFiles: [ + { + path: '/bar', + url: 'filestore.example.com/file', + file: { _id: this.fileId, hash: '12345' }, + }, + ], + newProject: { version: this.version }, + } + + const updates = [ + { + type: 'add-file', + id: this.fileId.toString(), + pathname: '/bar', + docLines: undefined, + historyRangesSupport: false, + url: undefined, + hash: '12345', + ranges: undefined, + createdBlob: true, + metadata: undefined, + }, + ] + + this.handler.updateProjectStructure( + this.project_id, + this.projectHistoryId, + this.user_id, + this.changes, + this.source, + () => { + this.request.should.have.been.calledWith({ + url: this.url, + method: 'POST', + json: { + updates, + userId: this.user_id, + version: this.version, + projectHistoryId: this.projectHistoryId, + source: this.source, + }, + timeout: 30 * 1000, + }) + done() + } + ) + }) + it('should flag files without hash', function (done) { + this.fileId = new ObjectId() + this.changes = { + newFiles: [ + { + path: '/bar', + url: 'filestore.example.com/file', + file: { _id: this.fileId }, + }, + ], + newProject: { version: this.version }, + } + + this.handler.updateProjectStructure( + this.project_id, + this.projectHistoryId, + this.user_id, + this.changes, + this.source, + err => { + err.should.match(/found file with missing hash/) + this.request.should.not.have.been.called + done() + } + ) + }) + }) }) }) @@ -1607,6 +1691,7 @@ describe('DocumentUpdaterHandler', function () { _hash: '42', path: '1.png', url: `http://filestore/project/${projectId}/file/${fileId1}`, + createdBlob: false, metadata: undefined, }, { @@ -1614,6 +1699,7 @@ describe('DocumentUpdaterHandler', function () { _hash: '1337', path: '1.bib', url: `http://filestore/project/${projectId}/file/${fileId2}`, + createdBlob: false, metadata: { importedAt: fileCreated2, provider: 'references-provider', @@ -1624,6 +1710,7 @@ describe('DocumentUpdaterHandler', function () { _hash: '21', path: 'bar.txt', url: `http://filestore/project/${projectId}/file/${fileId3}`, + createdBlob: false, metadata: { importedAt: fileCreated3, provider: 'project_output_file', @@ -1641,6 +1728,158 @@ describe('DocumentUpdaterHandler', function () { } ) }) + describe('with filestore disabled', function () { + beforeEach(function () { + this.settings.disableFilestore = true + }) + it('should add files without URL', function (done) { + const fileId1 = new ObjectId() + const fileId2 = new ObjectId() + const fileId3 = new ObjectId() + const fileCreated2 = new Date() + const fileCreated3 = new Date() + const otherProjectId = new ObjectId().toString() + const files = [ + { file: { _id: fileId1, hash: '42' }, path: '1.png' }, + { + file: { + _id: fileId2, + hash: '1337', + created: fileCreated2, + linkedFileData: { + provider: 'references-provider', + }, + }, + path: '1.bib', + }, + { + file: { + _id: fileId3, + hash: '21', + created: fileCreated3, + linkedFileData: { + provider: 'project_output_file', + build_id: '1234-abc', + clsiServerId: 'server-1', + source_project_id: otherProjectId, + source_output_file_path: 'foo/bar.txt', + }, + }, + path: 'bar.txt', + }, + ] + const docs = [] + this.request.yields(null, { statusCode: 200 }) + const projectId = new ObjectId() + const projectHistoryId = 99 + this.handler.resyncProjectHistory( + projectId, + projectHistoryId, + docs, + files, + {}, + () => { + this.request.should.have.been.calledWith({ + url: `${this.settings.apis.documentupdater.url}/project/${projectId}/history/resync`, + method: 'POST', + json: { + docs: [], + files: [ + { + file: fileId1, + _hash: '42', + path: '1.png', + url: undefined, + createdBlob: true, + metadata: undefined, + }, + { + file: fileId2, + _hash: '1337', + path: '1.bib', + url: undefined, + createdBlob: true, + metadata: { + importedAt: fileCreated2, + provider: 'references-provider', + }, + }, + { + file: fileId3, + _hash: '21', + path: 'bar.txt', + url: undefined, + createdBlob: true, + metadata: { + importedAt: fileCreated3, + provider: 'project_output_file', + source_project_id: otherProjectId, + source_output_file_path: 'foo/bar.txt', + // build_id and clsiServerId are omitted + }, + }, + ], + projectHistoryId, + }, + timeout: 6 * 60 * 1000, + }) + done() + } + ) + }) + it('should flag files with missing hashes', function (done) { + const fileId1 = new ObjectId() + const fileId2 = new ObjectId() + const fileId3 = new ObjectId() + const fileCreated2 = new Date() + const fileCreated3 = new Date() + const otherProjectId = new ObjectId().toString() + const files = [ + { file: { _id: fileId1, hash: '42' }, path: '1.png' }, + { + file: { + _id: fileId2, + created: fileCreated2, + linkedFileData: { + provider: 'references-provider', + }, + }, + path: '1.bib', + }, + { + file: { + _id: fileId3, + hash: '21', + created: fileCreated3, + linkedFileData: { + provider: 'project_output_file', + build_id: '1234-abc', + clsiServerId: 'server-1', + source_project_id: otherProjectId, + source_output_file_path: 'foo/bar.txt', + }, + }, + path: 'bar.txt', + }, + ] + const docs = [] + this.request.yields(null, { statusCode: 200 }) + const projectId = new ObjectId() + const projectHistoryId = 99 + this.handler.resyncProjectHistory( + projectId, + projectHistoryId, + docs, + files, + {}, + err => { + err.should.match(/found file with missing hash/) + this.request.should.not.have.been.called + done() + } + ) + }) + }) }) describe('appendToDocument', function () {