From 577497b655f259c2211289d5bb69084ba6917a53 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 21 Aug 2024 13:28:21 +0200 Subject: [PATCH] Merge pull request #19842 from overleaf/jpa-ro-mirror-on-client [misc] add readonly mirror of full project content on the client GitOrigin-RevId: 940bd93bfd587f83ca383d10fc44579b38fc3e88 --- libraries/overleaf-editor-core/lib/types.ts | 19 +++ package-lock.json | 17 +++ .../project-history/app/js/HttpController.js | 39 ++++++ services/project-history/app/js/Router.js | 12 ++ .../project-history/app/js/SnapshotManager.js | 62 ++++++++- .../project-history/app/js/SyncManager.js | 2 +- services/project-history/app/js/types.ts | 6 +- .../test/acceptance/js/GetChangesSince.js | 122 +++++++++++++++++ .../test/acceptance/js/LatestSnapshotTests.js | 78 +++++++++++ .../js/helpers/ProjectHistoryClient.js | 35 +++++ .../js/HttpController/HttpControllerTests.js | 8 ++ .../SnapshotManager/SnapshotManagerTests.js | 6 +- .../unit/js/SyncManager/SyncManagerTests.js | 10 +- .../web/app/src/Features/Errors/Errors.js | 3 + .../src/Features/Project/ProjectController.js | 1 + .../Features/Project/ProjectEditorHandler.js | 1 + .../SplitTests/SplitTestMiddleware.js | 27 ++++ .../web/app/src/infrastructure/RateLimiter.js | 8 ++ services/web/app/src/infrastructure/Server.js | 7 + services/web/app/src/router.js | 14 ++ services/web/config/settings.defaults.js | 2 + .../web/frontend/extracted-translations.json | 5 + .../components/toolbar-header.jsx | 16 +++ .../file-view/components/file-view-header.tsx | 10 +- .../file-view/components/file-view-image.tsx | 17 ++- .../file-view/components/file-view-text.tsx | 11 +- .../file-view/components/file-view.jsx | 8 +- .../features/file-view/types/binary-file.ts | 1 + ...ort-overleaf-editor-core-for-type-check.ts | 1 - .../ide-react/context/react-context-root.tsx | 69 +++++----- .../ide-react/context/snapshot-context.tsx | 127 ++++++++++++++++++ .../js/features/ide-react/util/file-view.ts | 1 + .../frontend/js/infrastructure/fetch-json.ts | 7 + .../shared/context/file-tree-data-context.tsx | 32 ++++- .../web/frontend/stories/decorators/scope.tsx | 72 +++++----- services/web/locales/en.json | 5 + services/web/package.json | 1 + services/web/test/frontend/bootstrap.js | 3 + .../features/file-tree/util/path.test.ts | 4 + .../components/file-view-image.test.jsx | 8 +- .../file-view-refresh-error.test.tsx | 1 + .../components/file-view-text.test.jsx | 2 + .../codemirror-editor-autocomplete.spec.tsx | 5 + .../source-editor/helpers/mock-scope.ts | 2 + .../frontend/helpers/editor-providers.jsx | 70 +++++----- services/web/types/file-ref.ts | 1 + 46 files changed, 816 insertions(+), 142 deletions(-) create mode 100644 services/project-history/test/acceptance/js/GetChangesSince.js create mode 100644 services/project-history/test/acceptance/js/LatestSnapshotTests.js delete mode 100644 services/web/frontend/js/features/full-project-on-client/import-overleaf-editor-core-for-type-check.ts create mode 100644 services/web/frontend/js/features/ide-react/context/snapshot-context.tsx diff --git a/libraries/overleaf-editor-core/lib/types.ts b/libraries/overleaf-editor-core/lib/types.ts index de50f241cf..4ebd56ff8a 100644 --- a/libraries/overleaf-editor-core/lib/types.ts +++ b/libraries/overleaf-editor-core/lib/types.ts @@ -51,6 +51,25 @@ export type StringFileRawData = { trackedChanges?: TrackedChangeRawData[] } +export type RawOrigin = { + kind: string +} + +export type RawChange = { + operations: RawOperation[] + timestamp: string + authors?: (number | null)[] + v2Authors: string[] + origin: RawOrigin + projectVersion: string + v2DocVersions: RawV2DocVersions +} + +export type RawOperation = + | RawEditFileOperation + // TODO(das7pad): add types for all the other operations + | object + export type RawSnapshot = { files: RawFileMap projectVersion?: string diff --git a/package-lock.json b/package-lock.json index 48c4f58f6b..5f8ea582df 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22682,6 +22682,15 @@ "node >=0.6.0" ] }, + "node_modules/fake-indexeddb": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/fake-indexeddb/-/fake-indexeddb-6.0.0.tgz", + "integrity": "sha512-YEboHE5VfopUclOck7LncgIqskAqnv4q0EWbYCaxKKjAvO93c+TJIaBuGy8CBFdbg9nKdpN3AuPRwVBJ4k7NrQ==", + "dev": true, + "engines": { + "node": ">=18" + } + }, "node_modules/fast-copy": { "version": "2.1.7", "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-2.1.7.tgz", @@ -44790,6 +44799,7 @@ "eslint-plugin-react": "^7.32.2", "eslint-plugin-react-hooks": "^4.6.0", "events": "^3.3.0", + "fake-indexeddb": "^6.0.0", "fetch-mock": "^9.10.2", "formik": "^2.2.9", "glob": "^7.1.6", @@ -53365,6 +53375,7 @@ "express-bearer-token": "^2.4.0", "express-http-proxy": "^1.6.0", "express-session": "^1.17.1", + "fake-indexeddb": "^6.0.0", "fetch-mock": "^9.10.2", "formik": "^2.2.9", "fs-extra": "^4.0.2", @@ -65322,6 +65333,12 @@ "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" }, + "fake-indexeddb": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/fake-indexeddb/-/fake-indexeddb-6.0.0.tgz", + "integrity": "sha512-YEboHE5VfopUclOck7LncgIqskAqnv4q0EWbYCaxKKjAvO93c+TJIaBuGy8CBFdbg9nKdpN3AuPRwVBJ4k7NrQ==", + "dev": true + }, "fast-copy": { "version": "2.1.7", "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-2.1.7.tgz", diff --git a/services/project-history/app/js/HttpController.js b/services/project-history/app/js/HttpController.js index 92b032dc76..5746c741c6 100644 --- a/services/project-history/app/js/HttpController.js +++ b/services/project-history/app/js/HttpController.js @@ -17,6 +17,8 @@ import * as RetryManager from './RetryManager.js' import * as FlushManager from './FlushManager.js' import { pipeline } from 'stream' +const ONE_DAY_IN_SECONDS = 24 * 60 * 60 + export function getProjectBlob(req, res, next) { const projectId = req.params.project_id const blobHash = req.params.hash @@ -27,6 +29,7 @@ export function getProjectBlob(req, res, next) { if (err != null) { return next(OError.tag(err)) } + res.setHeader('Cache-Control', `private, max-age=${ONE_DAY_IN_SECONDS}`) pipeline(stream, res, err => { if (err) next(err) // res.end() is already called via 'end' event by pipeline. @@ -216,6 +219,42 @@ export function getRangesSnapshot(req, res, next) { ) } +export function getLatestSnapshot(req, res, next) { + const { project_id: projectId } = req.params + WebApiManager.getHistoryId(projectId, (error, historyId) => { + if (error) return next(OError.tag(error)) + SnapshotManager.getLatestSnapshot( + projectId, + historyId, + (error, { snapshot, version }) => { + if (error != null) { + return next(error) + } + res.json({ snapshot: snapshot.toRaw(), version }) + } + ) + }) +} + +export function getChangesSince(req, res, next) { + const { project_id: projectId } = req.params + const { since } = req.query + WebApiManager.getHistoryId(projectId, (error, historyId) => { + if (error) return next(OError.tag(error)) + SnapshotManager.getChangesSince( + projectId, + historyId, + since, + (error, changes) => { + if (error != null) { + return next(error) + } + res.json(changes.map(c => c.toRaw())) + } + ) + }) +} + export function getProjectSnapshot(req, res, next) { const { project_id: projectId, version } = req.params SnapshotManager.getProjectSnapshot( diff --git a/services/project-history/app/js/Router.js b/services/project-history/app/js/Router.js index 87cb12cc06..851b750735 100644 --- a/services/project-history/app/js/Router.js +++ b/services/project-history/app/js/Router.js @@ -21,6 +21,8 @@ export function initialize(app) { app.delete('/project/:project_id', HttpController.deleteProject) + app.get('/project/:project_id/snapshot', HttpController.getLatestSnapshot) + app.get( '/project/:project_id/diff', validate({ @@ -55,6 +57,16 @@ export function initialize(app) { HttpController.getUpdates ) + app.get( + '/project/:project_id/changes', + validate({ + query: { + since: Joi.number().integer(), + }, + }), + HttpController.getChangesSince + ) + app.get('/project/:project_id/version', HttpController.latestVersion) app.post( diff --git a/services/project-history/app/js/SnapshotManager.js b/services/project-history/app/js/SnapshotManager.js index 6fb85c027f..1f5ee7df20 100644 --- a/services/project-history/app/js/SnapshotManager.js +++ b/services/project-history/app/js/SnapshotManager.js @@ -263,6 +263,15 @@ async function _getSnapshotAtVersion(projectId, version) { return snapshot } +async function getLatestSnapshotFiles(projectId, historyId) { + const { snapshot } = await getLatestSnapshot(projectId, historyId) + const snapshotFiles = await snapshot.loadFiles( + 'lazy', + HistoryStoreManager.getBlobStore(historyId) + ) + return snapshotFiles +} + async function getLatestSnapshot(projectId, historyId) { const data = await HistoryStoreManager.promises.getMostRecentChunk( projectId, @@ -277,11 +286,48 @@ async function getLatestSnapshot(projectId, historyId) { const snapshot = chunk.getSnapshot() const changes = chunk.getChanges() snapshot.applyAll(changes) - const snapshotFiles = await snapshot.loadFiles( - 'lazy', - HistoryStoreManager.getBlobStore(historyId) - ) - return snapshotFiles + return { + snapshot, + version: chunk.getEndVersion(), + } +} + +async function getChangesSince(projectId, historyId, sinceVersion) { + const allChanges = [] + let nextVersion + while (true) { + let data + if (nextVersion) { + data = await HistoryStoreManager.promises.getChunkAtVersion( + projectId, + historyId, + nextVersion + ) + } else { + data = await HistoryStoreManager.promises.getMostRecentChunk( + projectId, + historyId + ) + } + if (data == null || data.chunk == null) { + throw new OError('undefined chunk') + } + const chunk = Core.Chunk.fromRaw(data.chunk) + if (sinceVersion > chunk.getEndVersion()) { + throw new OError('requested version past the end') + } + const changes = chunk.getChanges() + if (chunk.getStartVersion() > sinceVersion) { + allChanges.unshift(...changes) + nextVersion = chunk.getStartVersion() + } else { + allChanges.unshift( + ...changes.slice(sinceVersion - chunk.getStartVersion()) + ) + break + } + } + return allChanges } async function _loadFilesLimit(snapshot, kind, blobStore) { @@ -298,24 +344,30 @@ async function _loadFilesLimit(snapshot, kind, blobStore) { // EXPORTS +const getChangesSinceCb = callbackify(getChangesSince) const getFileSnapshotStreamCb = callbackify(getFileSnapshotStream) const getProjectSnapshotCb = callbackify(getProjectSnapshot) const getLatestSnapshotCb = callbackify(getLatestSnapshot) +const getLatestSnapshotFilesCb = callbackify(getLatestSnapshotFiles) const getRangesSnapshotCb = callbackify(getRangesSnapshot) const getPathsAtVersionCb = callbackify(getPathsAtVersion) export { + getChangesSinceCb as getChangesSince, getFileSnapshotStreamCb as getFileSnapshotStream, getProjectSnapshotCb as getProjectSnapshot, getLatestSnapshotCb as getLatestSnapshot, + getLatestSnapshotFilesCb as getLatestSnapshotFiles, getRangesSnapshotCb as getRangesSnapshot, getPathsAtVersionCb as getPathsAtVersion, } export const promises = { + getChangesSince, getFileSnapshotStream, getProjectSnapshot, getLatestSnapshot, + getLatestSnapshotFiles, getRangesSnapshot, getPathsAtVersion, } diff --git a/services/project-history/app/js/SyncManager.js b/services/project-history/app/js/SyncManager.js index 2cc7d3d674..f0619f2ac3 100644 --- a/services/project-history/app/js/SyncManager.js +++ b/services/project-history/app/js/SyncManager.js @@ -203,7 +203,7 @@ async function expandSyncUpdates( const syncState = await _getResyncState(projectId) // compute the current snapshot from the most recent chunk - const snapshotFiles = await SnapshotManager.promises.getLatestSnapshot( + const snapshotFiles = await SnapshotManager.promises.getLatestSnapshotFiles( projectId, projectHistoryId ) diff --git a/services/project-history/app/js/types.ts b/services/project-history/app/js/types.ts index 95c40d91bb..baf70b59d4 100644 --- a/services/project-history/app/js/types.ts +++ b/services/project-history/app/js/types.ts @@ -1,5 +1,5 @@ import { HistoryRanges } from '../../../document-updater/app/js/types' -import { LinkedFileData } from 'overleaf-editor-core/lib/types' +import { LinkedFileData, RawOrigin } from 'overleaf-editor-core/lib/types' export type Update = | TextUpdate @@ -161,10 +161,6 @@ export type UpdateWithBlob = { : never } -export type RawOrigin = { - kind: string -} - export type TrackingProps = { type: 'insert' | 'delete' userId: string diff --git a/services/project-history/test/acceptance/js/GetChangesSince.js b/services/project-history/test/acceptance/js/GetChangesSince.js new file mode 100644 index 0000000000..559432fc73 --- /dev/null +++ b/services/project-history/test/acceptance/js/GetChangesSince.js @@ -0,0 +1,122 @@ +import { expect } from 'chai' +import mongodb from 'mongodb-legacy' +import nock from 'nock' +import Core from 'overleaf-editor-core' +import * as ProjectHistoryClient from './helpers/ProjectHistoryClient.js' +import * as ProjectHistoryApp from './helpers/ProjectHistoryApp.js' +import latestChunk from '../fixtures/chunks/7-8.json' with { type: 'json' } +import previousChunk from '../fixtures/chunks/4-6.json' with { type: 'json' } +import firstChunk from '../fixtures/chunks/0-3.json' with { type: 'json' } +const { ObjectId } = mongodb + +const MockHistoryStore = () => nock('http://127.0.0.1:3100') +const MockWeb = () => nock('http://127.0.0.1:3000') + +const fixture = path => new URL(`../fixtures/${path}`, import.meta.url) + +describe('GetChangesSince', function () { + let projectId, historyId + beforeEach(function (done) { + projectId = new ObjectId().toString() + historyId = new ObjectId().toString() + ProjectHistoryApp.ensureRunning(error => { + if (error) throw error + + MockHistoryStore().post('/api/projects').reply(200, { + projectId: historyId, + }) + + ProjectHistoryClient.initializeProject(historyId, (error, olProject) => { + if (error) throw error + MockWeb() + .get(`/project/${projectId}/details`) + .reply(200, { + name: 'Test Project', + overleaf: { history: { id: olProject.id } }, + }) + + MockHistoryStore() + .get(`/api/projects/${historyId}/latest/history`) + .replyWithFile(200, fixture('chunks/7-8.json')) + MockHistoryStore() + .get(`/api/projects/${historyId}/versions/6/history`) + .replyWithFile(200, fixture('chunks/4-6.json')) + MockHistoryStore() + .get(`/api/projects/${historyId}/versions/3/history`) + .replyWithFile(200, fixture('chunks/0-3.json')) + + done() + }) + }) + }) + + afterEach(function () { + nock.cleanAll() + }) + + function expectChangesSince(version, changes, done) { + ProjectHistoryClient.getChangesSince( + projectId, + version, + {}, + (error, got) => { + if (error) throw error + expect(got.map(c => Core.Change.fromRaw(c))).to.deep.equal( + changes.map(c => Core.Change.fromRaw(c)) + ) + done() + } + ) + } + + it('should return zero changes since the latest version', function (done) { + expectChangesSince(8, [], done) + }) + + it('should return one change when behind one version', function (done) { + expectChangesSince(7, [latestChunk.chunk.history.changes[1]], done) + }) + + it('should return changes when at the chunk boundary', function (done) { + expect(latestChunk.chunk.startVersion).to.equal(6) + expectChangesSince(6, latestChunk.chunk.history.changes, done) + }) + + it('should return changes spanning multiple chunks', function (done) { + expectChangesSince( + 1, + [ + ...firstChunk.chunk.history.changes.slice(1), + ...previousChunk.chunk.history.changes, + ...latestChunk.chunk.history.changes, + ], + done + ) + }) + + it('should return all changes when going back to the beginning', function (done) { + expectChangesSince( + 0, + [ + ...firstChunk.chunk.history.changes, + ...previousChunk.chunk.history.changes, + ...latestChunk.chunk.history.changes, + ], + done + ) + }) + + it('should return an error when past the end version', function (done) { + ProjectHistoryClient.getChangesSince( + projectId, + 9, + { allowErrors: true }, + (error, body, statusCode) => { + if (error) throw error + expect(statusCode).to.equal(500) + expect(body).to.deep.equal({ message: 'an internal error occurred' }) + done() + } + ) + }) +}) diff --git a/services/project-history/test/acceptance/js/LatestSnapshotTests.js b/services/project-history/test/acceptance/js/LatestSnapshotTests.js new file mode 100644 index 0000000000..6e989ffda3 --- /dev/null +++ b/services/project-history/test/acceptance/js/LatestSnapshotTests.js @@ -0,0 +1,78 @@ +import { expect } from 'chai' +import mongodb from 'mongodb-legacy' +import nock from 'nock' +import * as ProjectHistoryClient from './helpers/ProjectHistoryClient.js' +import * as ProjectHistoryApp from './helpers/ProjectHistoryApp.js' +const { ObjectId } = mongodb + +const MockHistoryStore = () => nock('http://127.0.0.1:3100') +const MockWeb = () => nock('http://127.0.0.1:3000') + +const fixture = path => new URL(`../fixtures/${path}`, import.meta.url) + +describe('LatestSnapshot', function () { + beforeEach(function (done) { + ProjectHistoryApp.ensureRunning(error => { + if (error) { + throw error + } + + this.historyId = new ObjectId().toString() + MockHistoryStore().post('/api/projects').reply(200, { + projectId: this.historyId, + }) + + ProjectHistoryClient.initializeProject( + this.historyId, + (error, v1Project) => { + if (error) { + throw error + } + this.projectId = new ObjectId().toString() + MockWeb() + .get(`/project/${this.projectId}/details`) + .reply(200, { + name: 'Test Project', + overleaf: { history: { id: v1Project.id } }, + }) + done() + } + ) + }) + }) + + afterEach(function () { + nock.cleanAll() + }) + + it('should return the snapshot with applied changes, metadata and without full content', function (done) { + MockHistoryStore() + .get(`/api/projects/${this.historyId}/latest/history`) + .replyWithFile(200, fixture('chunks/0-3.json')) + + ProjectHistoryClient.getLatestSnapshot(this.projectId, (error, body) => { + if (error) { + throw error + } + expect(body).to.deep.equal({ + snapshot: { + files: { + 'main.tex': { + hash: 'f28571f561d198b87c24cc6a98b78e87b665e22d', + stringLength: 20649, + operations: [{ textOperation: [1912, 'Hello world', 18726] }], + metadata: { main: true }, + }, + 'foo.tex': { + hash: '4f785a4c192155b240e3042b3a7388b47603f423', + stringLength: 41, + operations: [{ textOperation: [26, '\n\nFour five six'] }], + }, + }, + }, + version: 3, + }) + done() + }) + }) +}) diff --git a/services/project-history/test/acceptance/js/helpers/ProjectHistoryClient.js b/services/project-history/test/acceptance/js/helpers/ProjectHistoryClient.js index 65f7ceb10b..1cff6dcdf9 100644 --- a/services/project-history/test/acceptance/js/helpers/ProjectHistoryClient.js +++ b/services/project-history/test/acceptance/js/helpers/ProjectHistoryClient.js @@ -108,6 +108,41 @@ export function getFileTreeDiff(projectId, from, to, callback) { ) } +export function getChangesSince(projectId, since, options, callback) { + request.get( + { + url: `http://127.0.0.1:3054/project/${projectId}/changes`, + qs: { + since, + }, + json: true, + }, + (error, res, body) => { + if (error) return callback(error) + if (!options.allowErrors) { + expect(res.statusCode).to.equal(200) + } + callback(null, body, res.statusCode) + } + ) +} + +export function getLatestSnapshot(projectId, callback) { + request.get( + { + url: `http://127.0.0.1:3054/project/${projectId}/snapshot`, + json: true, + }, + (error, res, body) => { + if (error) { + return callback(error) + } + expect(res.statusCode).to.equal(200) + callback(null, body) + } + ) +} + export function getSnapshot(projectId, pathname, version, options, callback) { if (typeof options === 'function') { callback = options diff --git a/services/project-history/test/unit/js/HttpController/HttpControllerTests.js b/services/project-history/test/unit/js/HttpController/HttpControllerTests.js index 1fae6f503c..273bd1d867 100644 --- a/services/project-history/test/unit/js/HttpController/HttpControllerTests.js +++ b/services/project-history/test/unit/js/HttpController/HttpControllerTests.js @@ -84,6 +84,7 @@ describe('HttpController', function () { json: sinon.stub(), send: sinon.stub(), sendStatus: sinon.stub(), + setHeader: sinon.stub(), } }) @@ -105,6 +106,13 @@ describe('HttpController', function () { .should.equal(true) this.pipeline.should.have.been.calledWith(this.stream, this.res) }) + + it('should set caching header', function () { + this.res.setHeader.should.have.been.calledWith( + 'Cache-Control', + 'private, max-age=86400' + ) + }) }) describe('initializeProject', function () { diff --git a/services/project-history/test/unit/js/SnapshotManager/SnapshotManagerTests.js b/services/project-history/test/unit/js/SnapshotManager/SnapshotManagerTests.js index 34350b1548..dbc51464e2 100644 --- a/services/project-history/test/unit/js/SnapshotManager/SnapshotManagerTests.js +++ b/services/project-history/test/unit/js/SnapshotManager/SnapshotManagerTests.js @@ -479,7 +479,7 @@ Four five six\ }) }) - describe('getLatestSnapshot', function () { + describe('getLatestSnapshotFiles', function () { describe('for a project', function () { beforeEach(async function () { this.HistoryStoreManager.promises.getMostRecentChunk.resolves({ @@ -543,7 +543,7 @@ Four five six\ )), getObject: sinon.stub().rejects(), }) - this.data = await this.SnapshotManager.promises.getLatestSnapshot( + this.data = await this.SnapshotManager.promises.getLatestSnapshotFiles( this.projectId, this.historyId ) @@ -571,7 +571,7 @@ Four five six\ beforeEach(async function () { this.HistoryStoreManager.promises.getMostRecentChunk.resolves(null) expect( - this.SnapshotManager.promises.getLatestSnapshot( + this.SnapshotManager.promises.getLatestSnapshotFiles( this.projectId, this.historyId ) diff --git a/services/project-history/test/unit/js/SyncManager/SyncManagerTests.js b/services/project-history/test/unit/js/SyncManager/SyncManagerTests.js index 48a301c165..cbf52ac15e 100644 --- a/services/project-history/test/unit/js/SyncManager/SyncManagerTests.js +++ b/services/project-history/test/unit/js/SyncManager/SyncManagerTests.js @@ -114,7 +114,7 @@ describe('SyncManager', function () { this.SnapshotManager = { promises: { - getLatestSnapshot: sinon.stub(), + getLatestSnapshotFiles: sinon.stub(), }, } @@ -517,7 +517,9 @@ describe('SyncManager', function () { .returns('another.tex') this.UpdateTranslator._convertPathname.withArgs('1.png').returns('1.png') this.UpdateTranslator._convertPathname.withArgs('2.png').returns('2.png') - this.SnapshotManager.promises.getLatestSnapshot.resolves(this.fileMap) + this.SnapshotManager.promises.getLatestSnapshotFiles.resolves( + this.fileMap + ) }) it('returns updates if no sync updates are queued', async function () { @@ -530,8 +532,8 @@ describe('SyncManager', function () { ) expect(expandedUpdates).to.equal(updates) - expect(this.SnapshotManager.promises.getLatestSnapshot).to.not.have.been - .called + expect(this.SnapshotManager.promises.getLatestSnapshotFiles).to.not.have + .been.called expect(this.extendLock).to.not.have.been.called }) diff --git a/services/web/app/src/Features/Errors/Errors.js b/services/web/app/src/Features/Errors/Errors.js index ed504836fa..c00a1097d6 100644 --- a/services/web/app/src/Features/Errors/Errors.js +++ b/services/web/app/src/Features/Errors/Errors.js @@ -5,6 +5,9 @@ const settings = require('@overleaf/settings') // backward-compatible (can be instantiated with string as argument instead // of object) class BackwardCompatibleError extends OError { + /** + * @param {string | { message: string, info?: Object }} messageOrOptions + */ constructor(messageOrOptions) { if (typeof messageOrOptions === 'string') { super(messageOrOptions) diff --git a/services/web/app/src/Features/Project/ProjectController.js b/services/web/app/src/Features/Project/ProjectController.js index 97801a6ddd..89353a502f 100644 --- a/services/web/app/src/Features/Project/ProjectController.js +++ b/services/web/app/src/Features/Project/ProjectController.js @@ -337,6 +337,7 @@ const _ProjectController = { 'revert-file', 'revert-project', 'review-panel-redesign', + !anonymous && 'ro-mirror-on-client', 'track-pdf-download', !anonymous && 'writefull-oauth-promotion', 'ieee-stylesheet', diff --git a/services/web/app/src/Features/Project/ProjectEditorHandler.js b/services/web/app/src/Features/Project/ProjectEditorHandler.js index f157c92b25..7d01708833 100644 --- a/services/web/app/src/Features/Project/ProjectEditorHandler.js +++ b/services/web/app/src/Features/Project/ProjectEditorHandler.js @@ -127,6 +127,7 @@ module.exports = ProjectEditorHandler = { name: file.name, linkedFileData: file.linkedFileData, created: file.created, + hash: file.hash, } }, diff --git a/services/web/app/src/Features/SplitTests/SplitTestMiddleware.js b/services/web/app/src/Features/SplitTests/SplitTestMiddleware.js index f7a5a1dfe6..2e8e1ff97f 100644 --- a/services/web/app/src/Features/SplitTests/SplitTestMiddleware.js +++ b/services/web/app/src/Features/SplitTests/SplitTestMiddleware.js @@ -1,5 +1,7 @@ const SplitTestHandler = require('./SplitTestHandler') const logger = require('@overleaf/logger') +const { expressify } = require('@overleaf/promise-utils') +const Errors = require('../Errors/Errors') function loadAssignmentsInLocals(splitTestNames) { return async function (req, res, next) { @@ -17,6 +19,31 @@ function loadAssignmentsInLocals(splitTestNames) { } } +function ensureSplitTestEnabledForUser( + splitTestName, + enabledVariant = 'enabled' +) { + return expressify(async function (req, res, next) { + const { variant } = await SplitTestHandler.promises.getAssignment( + req, + res, + splitTestName + ) + if (variant !== enabledVariant) { + throw new Errors.ForbiddenError({ + message: 'missing split test access', + info: { + splitTestName, + variant, + enabledVariant, + }, + }) + } + next() + }) +} + module.exports = { loadAssignmentsInLocals, + ensureSplitTestEnabledForUser, } diff --git a/services/web/app/src/infrastructure/RateLimiter.js b/services/web/app/src/infrastructure/RateLimiter.js index c3e863baf7..f0f625a07e 100644 --- a/services/web/app/src/infrastructure/RateLimiter.js +++ b/services/web/app/src/infrastructure/RateLimiter.js @@ -11,6 +11,8 @@ const rclient = RedisWrapper.client('ratelimiter') * Wrapper over the RateLimiterRedis class */ class RateLimiter { + #opts + /** * Create a rate limiter. * @@ -31,6 +33,7 @@ class RateLimiter { */ constructor(name, opts = {}) { this.name = name + this.#opts = Object.assign({}, opts) this._rateLimiter = new RateLimiterFlexible.RateLimiterRedis({ ...opts, keyPrefix: `rate-limit:${name}`, @@ -46,6 +49,11 @@ class RateLimiter { } } + // Readonly access to the options, useful for aligning rate-limits. + getOptions() { + return Object.assign({}, this.#opts) + } + async consume(key, points = 1, options = { method: 'unknown' }) { if (Settings.disableRateLimits) { // Return a fake result in case it's used somewhere diff --git a/services/web/app/src/infrastructure/Server.js b/services/web/app/src/infrastructure/Server.js index f9f7274401..d59f465dbe 100644 --- a/services/web/app/src/infrastructure/Server.js +++ b/services/web/app/src/infrastructure/Server.js @@ -296,6 +296,13 @@ webRouter.use(function addNoCacheHeader(req, res, next) { // don't set no-cache headers on a project file, as it's immutable and can be cached (privately) return next() } + const isProjectBlob = /^\/project\/[a-f0-9]{24}\/blob\/[a-f0-9]{40}$/.test( + req.path + ) + if (isProjectBlob) { + // don't set no-cache headers on a project blobs, as they are immutable and can be cached (privately) + return next() + } const isWikiContent = /^\/learn(-scripts)?(\/|$)/i.test(req.path) if (isWikiContent) { diff --git a/services/web/app/src/router.js b/services/web/app/src/router.js index 972f0b8c2a..b1ba39de6e 100644 --- a/services/web/app/src/router.js +++ b/services/web/app/src/router.js @@ -125,6 +125,20 @@ const rateLimiters = { points: 30, duration: 60 * 60, }), + flushHistory: new RateLimiter('flush-project-history', { + // Allow flushing once every 30s-1s (allow for network jitter). + points: 1, + duration: 30 - 1, + }), + getProjectBlob: new RateLimiter('get-project-blob', { + // Download project in full once per hour + points: Settings.maxEntitiesPerProject, + duration: 60 * 60, + }), + getHistorySnapshot: new RateLimiter( + 'get-history-snapshot', + openProjectRateLimiter.getOptions() + ), endorseEmail: new RateLimiter('endorse-email', { points: 30, duration: 60, diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index d587d683d1..1b07d02409 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -901,6 +901,8 @@ module.exports = { managedGroupEnrollmentInvite: [], ssoCertificateInfo: [], v1ImportDataScreen: [], + snapshotUtils: [], + offlineModeToolbarButtons: [], }, moduleImportSequence: [ diff --git a/services/web/frontend/extracted-translations.json b/services/web/frontend/extracted-translations.json index a1e1381a62..35f78c3d6f 100644 --- a/services/web/frontend/extracted-translations.json +++ b/services/web/frontend/extracted-translations.json @@ -476,6 +476,7 @@ "from_provider": "", "from_url": "", "full_doc_history": "", + "full_project_search": "", "full_width": "", "generate_token": "", "generic_if_problem_continues_contact_us": "", @@ -844,6 +845,7 @@ "my_library": "", "n_items": "", "n_items_plural": "", + "n_matches": "", "n_more_updates_above": "", "n_more_updates_above_plural": "", "n_more_updates_below": "", @@ -1100,6 +1102,7 @@ "refresh_page_after_starting_free_trial": "", "refreshing": "", "regards": "", + "regular_expression": "", "reject": "", "reject_all": "", "reject_change": "", @@ -1183,6 +1186,7 @@ "saving": "", "saving_notification_with_seconds": "", "search": "", + "search_all_project_files": "", "search_bib_files": "", "search_by_citekey_author_year_title": "", "search_command_find": "", @@ -1202,6 +1206,7 @@ "search_replace_all": "", "search_replace_with": "", "search_search_for": "", + "search_terms": "", "search_whole_word": "", "search_within_selection": "", "searched_path_for_lines_containing": "", diff --git a/services/web/frontend/js/features/editor-navigation-toolbar/components/toolbar-header.jsx b/services/web/frontend/js/features/editor-navigation-toolbar/components/toolbar-header.jsx index 64d6d889f6..75247388b9 100644 --- a/services/web/frontend/js/features/editor-navigation-toolbar/components/toolbar-header.jsx +++ b/services/web/frontend/js/features/editor-navigation-toolbar/components/toolbar-header.jsx @@ -15,10 +15,20 @@ import ShareProjectButton from './share-project-button' import importOverleafModules from '../../../../macros/import-overleaf-module.macro' import BackToEditorButton from './back-to-editor-button' import getMeta from '@/utils/meta' +import { isSplitTestEnabled } from '@/utils/splitTestUtils' const [publishModalModules] = importOverleafModules('publishModal') const PublishButton = publishModalModules?.import.default +const offlineModeToolbarButtons = importOverleafModules( + 'offlineModeToolbarButtons' +) +// double opt-in +const enableROMirrorOnClient = + isSplitTestEnabled('ro-mirror-on-client') && + new URLSearchParams(window.location.search).get('ro-mirror-on-client') === + 'enabled' + const ToolbarHeader = React.memo(function ToolbarHeader({ cobranding, onShowLeftMenuClick, @@ -55,6 +65,12 @@ const ToolbarHeader = React.memo(function ToolbarHeader({ )} + {enableROMirrorOnClient && + offlineModeToolbarButtons.map( + ({ path, import: { default: OfflineModeToolbarButton } }) => { + return + } + )} {getMeta('ol-showUpgradePrompt') && } >(null) @@ -88,8 +90,12 @@ export default function FileViewHeader({ file }: FileViewHeaderProps) { )}   diff --git a/services/web/frontend/js/features/file-view/components/file-view-image.tsx b/services/web/frontend/js/features/file-view/components/file-view-image.tsx index 163ce5cc50..8ad6fb7f38 100644 --- a/services/web/frontend/js/features/file-view/components/file-view-image.tsx +++ b/services/web/frontend/js/features/file-view/components/file-view-image.tsx @@ -1,24 +1,29 @@ import { useProjectContext } from '../../../shared/context/project-context' +import { useSnapshotContext } from '@/features/ide-react/context/snapshot-context' +import { BinaryFile } from '@/features/file-view/types/binary-file' export default function FileViewImage({ - fileName, - fileId, + file, onLoad, onError, }: { - fileName: string - fileId: string + file: BinaryFile onLoad: () => void onError: () => void }) { const { _id: projectId } = useProjectContext() + const { fileTreeFromHistory } = useSnapshotContext() return ( {fileName} ) } diff --git a/services/web/frontend/js/features/file-view/components/file-view-text.tsx b/services/web/frontend/js/features/file-view/components/file-view-text.tsx index f066c6028d..cf3f2263dd 100644 --- a/services/web/frontend/js/features/file-view/components/file-view-text.tsx +++ b/services/web/frontend/js/features/file-view/components/file-view-text.tsx @@ -2,6 +2,8 @@ import { useState, useEffect } from 'react' import { useProjectContext } from '../../../shared/context/project-context' import { debugConsole } from '@/utils/debugging' import useAbortController from '../../../shared/hooks/use-abort-controller' +import { BinaryFile } from '@/features/file-view/types/binary-file' +import { useSnapshotContext } from '@/features/ide-react/context/snapshot-context' const MAX_FILE_SIZE = 2 * 1024 * 1024 @@ -10,11 +12,12 @@ export default function FileViewText({ onLoad, onError, }: { - file: { id: string } + file: BinaryFile onLoad: () => void onError: () => void }) { const { _id: projectId } = useProjectContext() + const { fileTreeFromHistory } = useSnapshotContext() const [textPreview, setTextPreview] = useState('') const [shouldShowDots, setShouldShowDots] = useState(false) @@ -27,7 +30,9 @@ export default function FileViewText({ if (inFlight) { return } - let path = `/project/${projectId}/file/${file.id}` + let path = fileTreeFromHistory + ? `/project/${projectId}/blob/${file.hash}` + : `/project/${projectId}/file/${file.id}` const fetchContentLengthTimeout = setTimeout( () => fetchContentLengthController.abort(), 10000 @@ -77,8 +82,10 @@ export default function FileViewText({ clearTimeout(fetchDataTimeout) }) }, [ + fileTreeFromHistory, projectId, file.id, + file.hash, onError, onLoad, inFlight, diff --git a/services/web/frontend/js/features/file-view/components/file-view.jsx b/services/web/frontend/js/features/file-view/components/file-view.jsx index 2c6b6f695c..bb567bb8c5 100644 --- a/services/web/frontend/js/features/file-view/components/file-view.jsx +++ b/services/web/frontend/js/features/file-view/components/file-view.jsx @@ -44,12 +44,7 @@ export default function FileView({ file }) { <> {isImageFile && ( - + )} {isEditableTextFile && ( @@ -91,5 +86,6 @@ FileView.propTypes = { file: PropTypes.shape({ id: PropTypes.string, name: PropTypes.string, + hash: PropTypes.string, }).isRequired, } diff --git a/services/web/frontend/js/features/file-view/types/binary-file.ts b/services/web/frontend/js/features/file-view/types/binary-file.ts index 05c133bf37..a52bc0edb9 100644 --- a/services/web/frontend/js/features/file-view/types/binary-file.ts +++ b/services/web/frontend/js/features/file-view/types/binary-file.ts @@ -26,6 +26,7 @@ export type BinaryFile = type: string selected: boolean linkedFileData?: LinkedFileData[T] + hash: string } export type LinkedFile = Required> diff --git a/services/web/frontend/js/features/full-project-on-client/import-overleaf-editor-core-for-type-check.ts b/services/web/frontend/js/features/full-project-on-client/import-overleaf-editor-core-for-type-check.ts deleted file mode 100644 index 5d9eaa9036..0000000000 --- a/services/web/frontend/js/features/full-project-on-client/import-overleaf-editor-core-for-type-check.ts +++ /dev/null @@ -1 +0,0 @@ -import 'overleaf-editor-core' diff --git a/services/web/frontend/js/features/ide-react/context/react-context-root.tsx b/services/web/frontend/js/features/ide-react/context/react-context-root.tsx index 523072f8ec..0f6b339476 100644 --- a/services/web/frontend/js/features/ide-react/context/react-context-root.tsx +++ b/services/web/frontend/js/features/ide-react/context/react-context-root.tsx @@ -22,6 +22,7 @@ import { UserSettingsProvider } from '@/shared/context/user-settings-context' import { PermissionsProvider } from '@/features/ide-react/context/permissions-context' import { FileTreeOpenProvider } from '@/features/ide-react/context/file-tree-open-context' import { OutlineProvider } from '@/features/ide-react/context/outline-context' +import { SnapshotProvider } from '@/features/ide-react/context/snapshot-context' export const ReactContextRoot: FC = ({ children }) => { return ( @@ -32,39 +33,41 @@ export const ReactContextRoot: FC = ({ children }) => { - - - - - - - - - - - - - - - - - {children} - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + {children} + + + + + + + + + + + + + + + + + diff --git a/services/web/frontend/js/features/ide-react/context/snapshot-context.tsx b/services/web/frontend/js/features/ide-react/context/snapshot-context.tsx new file mode 100644 index 0000000000..686940f612 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/context/snapshot-context.tsx @@ -0,0 +1,127 @@ +import { + createContext, + FC, + useContext, + useEffect, + useMemo, + useState, +} from 'react' +import { Snapshot } from 'overleaf-editor-core' +import { useProjectContext } from '@/shared/context/project-context' +import { debugConsole } from '@/utils/debugging' +import importOverleafModules from '../../../../macros/import-overleaf-module.macro' +import { Folder } from '../../../../../types/folder' + +export const StubSnapshotUtils = { + SnapshotUpdater: class SnapshotUpdater { + // eslint-disable-next-line no-useless-constructor + constructor(readonly projectId: string) {} + refresh(): Promise<{ snapshot: Snapshot; snapshotVersion: number }> { + throw new Error('not implemented') + } + + abort(): void { + throw new Error('not implemented') + } + }, + buildFileTree(snapshot: Snapshot): Folder { + throw new Error('not implemented') + }, + createFolder(_id: string, name: string): Folder { + throw new Error('not implemented') + }, +} + +const { SnapshotUpdater } = + (importOverleafModules('snapshotUtils')[0] + ?.import as typeof StubSnapshotUtils) || StubSnapshotUtils + +export type SnapshotLoadingState = '' | 'loading' | 'error' + +export const SnapshotContext = createContext< + | { + snapshotVersion: number + snapshot?: Snapshot + snapshotLoadingState: SnapshotLoadingState + + fileTreeFromHistory: boolean + setFileTreeFromHistory: (v: boolean) => void + } + | undefined +>(undefined) + +export const SnapshotProvider: FC = ({ children }) => { + const { _id: projectId } = useProjectContext() + const [snapshotLoadingState, setSnapshotLoadingState] = + useState('') + const [snapshotUpdater] = useState(() => new SnapshotUpdater(projectId)) + const [snapshot, setSnapshot] = useState() + const [snapshotVersion, setSnapshotVersion] = useState(-1) + const [fileTreeFromHistory, setFileTreeFromHistory] = useState(false) + + useEffect(() => { + if (!fileTreeFromHistory) return + + let stop = false + let handle: number + const refresh = () => { + setSnapshotLoadingState('loading') + snapshotUpdater + .refresh() + .then(({ snapshot, snapshotVersion }) => { + setSnapshot(snapshot) + setSnapshotVersion(snapshotVersion) + setSnapshotLoadingState('') + }) + .catch(err => { + debugConsole.error(err) + setSnapshotLoadingState('error') + }) + .finally(() => { + if (stop) return + // use a chain of timeouts to avoid concurrent updates + handle = window.setTimeout(refresh, 30_000) + }) + } + + refresh() + return () => { + stop = true + snapshotUpdater.abort() + clearInterval(handle) + } + }, [projectId, fileTreeFromHistory, snapshotUpdater]) + + const value = useMemo( + () => ({ + snapshot, + snapshotVersion, + snapshotLoadingState, + fileTreeFromHistory, + setFileTreeFromHistory, + }), + [ + snapshot, + snapshotVersion, + snapshotLoadingState, + fileTreeFromHistory, + setFileTreeFromHistory, + ] + ) + + return ( + + {children} + + ) +} + +export function useSnapshotContext() { + const context = useContext(SnapshotContext) + if (!context) { + throw new Error( + 'useSnapshotContext is only available within SnapshotProvider' + ) + } + return context +} diff --git a/services/web/frontend/js/features/ide-react/util/file-view.ts b/services/web/frontend/js/features/ide-react/util/file-view.ts index d08a32cd30..af7f7bed3c 100644 --- a/services/web/frontend/js/features/ide-react/util/file-view.ts +++ b/services/web/frontend/js/features/ide-react/util/file-view.ts @@ -10,6 +10,7 @@ export function convertFileRefToBinaryFile(fileRef: FileRef): BinaryFile { selected: true, linkedFileData: fileRef.linkedFileData, created: fileRef.created ? new Date(fileRef.created) : new Date(), + hash: fileRef.hash, } } diff --git a/services/web/frontend/js/infrastructure/fetch-json.ts b/services/web/frontend/js/infrastructure/fetch-json.ts index 57ea417baf..24ab2deca7 100644 --- a/services/web/frontend/js/infrastructure/fetch-json.ts +++ b/services/web/frontend/js/infrastructure/fetch-json.ts @@ -244,3 +244,10 @@ export function getUserFacingMessage(error: Error | null) { return error.message } + +export function isRateLimited(error?: Error | FetchError | any) { + if (error && error instanceof FetchError) { + return error.response?.status === 429 + } + return false +} diff --git a/services/web/frontend/js/shared/context/file-tree-data-context.tsx b/services/web/frontend/js/shared/context/file-tree-data-context.tsx index 22cc1a044d..8186b79e70 100644 --- a/services/web/frontend/js/shared/context/file-tree-data-context.tsx +++ b/services/web/frontend/js/shared/context/file-tree-data-context.tsx @@ -6,6 +6,7 @@ import { useMemo, useState, FC, + useEffect, } from 'react' import useScopeValue from '../hooks/use-scope-value' import { @@ -22,6 +23,14 @@ import { Folder } from '../../../../types/folder' import { Project } from '../../../../types/project' import { MainDocument } from '../../../../types/project-settings' import { FindResult } from '@/features/file-tree/util/path' +import { + StubSnapshotUtils, + useSnapshotContext, +} from '@/features/ide-react/context/snapshot-context' +import importOverleafModules from '../../../macros/import-overleaf-module.macro' +const { buildFileTree, createFolder } = + (importOverleafModules('snapshotUtils')[0] + ?.import as typeof StubSnapshotUtils) || StubSnapshotUtils const FileTreeDataContext = createContext< | { @@ -170,8 +179,29 @@ export const FileTreeDataProvider: FC = ({ children }) => { const [project] = useScopeValue('project') const [openDocId] = useScopeValue('editor.open_doc_id') const [, setOpenDocName] = useScopeValueSetterOnly('editor.open_doc_name') + const { fileTreeFromHistory, snapshot, snapshotVersion } = + useSnapshotContext() - const { rootFolder } = project || {} + const [rootFolder, setRootFolder] = useState(project?.rootFolder) + + useEffect(() => { + if (fileTreeFromHistory) return + setRootFolder(project?.rootFolder) + }, [project, fileTreeFromHistory]) + + useEffect(() => { + if (!fileTreeFromHistory) return + if (!rootFolder || rootFolder?.[0]?._id) { + // Init or replace mongo rootFolder with stub while we load the snapshot. + // In the future, project:joined should only fire once the snapshot is ready. + setRootFolder([createFolder('', '')]) + } + }, [fileTreeFromHistory, rootFolder]) + + useEffect(() => { + if (!fileTreeFromHistory || !snapshot) return + setRootFolder([buildFileTree(snapshot)]) + }, [fileTreeFromHistory, snapshot, snapshotVersion]) const [{ fileTreeData, fileCount }, dispatch] = useReducer( fileTreeMutableReducer, diff --git a/services/web/frontend/stories/decorators/scope.tsx b/services/web/frontend/stories/decorators/scope.tsx index 9f8838125b..5a45cc926b 100644 --- a/services/web/frontend/stories/decorators/scope.tsx +++ b/services/web/frontend/stories/decorators/scope.tsx @@ -42,6 +42,7 @@ import { EditorManagerProvider } from '@/features/ide-react/context/editor-manag import { FileTreeOpenProvider } from '@/features/ide-react/context/file-tree-open-context' import { MetadataProvider } from '@/features/ide-react/context/metadata-context' import { OnlineUsersProvider } from '@/features/ide-react/context/online-users-context' +import { SnapshotProvider } from '@/features/ide-react/context/snapshot-context' const scopeWatchers: [string, (value: any) => void][] = [] @@ -73,7 +74,7 @@ const initialize = () => { { _id: 'test-file-id', name: 'testfile.tex' }, { _id: 'test-bib-file-id', name: 'testsources.bib' }, ], - fileRefs: [{ _id: 'test-image-id', name: 'frog.jpg' }], + fileRefs: [{ _id: 'test-image-id', name: 'frog.jpg', hash: '42' }], folders: [], }, ], @@ -179,6 +180,7 @@ export const ScopeDecorator = ( DetachProvider, EditorProvider, EditorManagerProvider, + SnapshotProvider, FileTreeDataProvider, FileTreeOpenProvider, FileTreePathProvider, @@ -207,39 +209,41 @@ export const ScopeDecorator = ( - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/services/web/locales/en.json b/services/web/locales/en.json index 6ff74fc92e..6733d27171 100644 --- a/services/web/locales/en.json +++ b/services/web/locales/en.json @@ -711,6 +711,7 @@ "full_doc_history": "Full document history", "full_doc_history_info_v2": "You can see all the edits in your project and who made every change. Add labels to quickly access specific versions.", "full_document_history": "Full document <0>history", + "full_project_search": "Full Project Search", "full_width": "Full width", "gallery": "Gallery", "gallery_find_more": "Find More __itemPlural__", @@ -1234,6 +1235,7 @@ "my_library": "My Library", "n_items": "__count__ item", "n_items_plural": "__count__ items", + "n_matches": "__n__ matches", "n_more_updates_above": "__count__ more update above", "n_more_updates_above_plural": "__count__ more updates above", "n_more_updates_below": "__count__ more update below", @@ -1612,6 +1614,7 @@ "registered": "Registered", "registering": "Registering", "registration_error": "Registration error", + "regular_expression": "Regular Expression", "reject": "Reject", "reject_all": "Reject all", "reject_change": "Reject change", @@ -1725,6 +1728,7 @@ "saving_20_percent_no_exclamation": "Saving 20%", "saving_notification_with_seconds": "Saving __docname__... (__seconds__ seconds of unsaved changes)", "search": "Search", + "search_all_project_files": "Search all project files", "search_bib_files": "Search by author, title, year", "search_by_citekey_author_year_title": "Search by citation key, author, title, year", "search_command_find": "Find", @@ -1744,6 +1748,7 @@ "search_replace_all": "Replace All", "search_replace_with": "Replace with", "search_search_for": "Search for", + "search_terms": "Search terms", "search_whole_word": "Whole word", "search_within_selection": "Within selection", "searched_path_for_lines_containing": "Searched __path__ for lines containing \"__query__\"", diff --git a/services/web/package.json b/services/web/package.json index ec9beb854b..1d53f8b05b 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -298,6 +298,7 @@ "eslint-plugin-react": "^7.32.2", "eslint-plugin-react-hooks": "^4.6.0", "events": "^3.3.0", + "fake-indexeddb": "^6.0.0", "fetch-mock": "^9.10.2", "formik": "^2.2.9", "glob": "^7.1.6", diff --git a/services/web/test/frontend/bootstrap.js b/services/web/test/frontend/bootstrap.js index 4d4e995890..306545a3ec 100644 --- a/services/web/test/frontend/bootstrap.js +++ b/services/web/test/frontend/bootstrap.js @@ -92,3 +92,6 @@ addHook(() => '', { exts: ['.css'], ignoreNodeModules: false }) globalThis.HTMLElement.prototype.scrollIntoView = () => {} globalThis.DOMParser = window.DOMParser + +// Polyfill for IndexedDB +require('fake-indexeddb/auto') diff --git a/services/web/test/frontend/features/file-tree/util/path.test.ts b/services/web/test/frontend/features/file-tree/util/path.test.ts index 8622e1d71b..cfdade6f45 100644 --- a/services/web/test/frontend/features/file-tree/util/path.test.ts +++ b/services/web/test/frontend/features/file-tree/util/path.test.ts @@ -34,6 +34,7 @@ describe('Path utils', function () { { _id: 'test-file-in-folder', name: 'example.png', + hash: '42', }, ], folders: [ @@ -50,6 +51,7 @@ describe('Path utils', function () { { _id: 'test-file-in-subfolder', name: 'nested-example.png', + hash: '43', }, ], folders: [], @@ -61,10 +63,12 @@ describe('Path utils', function () { { _id: 'test-image-file', name: 'frog.jpg', + hash: '21', }, { _id: 'uppercase-extension-image-file', name: 'frog.JPG', + hash: '22', }, ], } diff --git a/services/web/test/frontend/features/file-view/components/file-view-image.test.jsx b/services/web/test/frontend/features/file-view/components/file-view-image.test.jsx index b7b4ac5764..7ff204cc2e 100644 --- a/services/web/test/frontend/features/file-view/components/file-view-image.test.jsx +++ b/services/web/test/frontend/features/file-view/components/file-view-image.test.jsx @@ -7,6 +7,7 @@ describe('', function () { const file = { id: '60097ca20454610027c442a8', name: 'file.jpg', + hash: 'hash', linkedFileData: { source_entity_path: '/source-entity-path', provider: 'project_file', @@ -15,12 +16,7 @@ describe('', function () { it('renders an image', function () { renderWithEditorContext( - {}} - onLoad={() => {}} - /> + {}} onLoad={() => {}} /> ) screen.getByRole('img') }) diff --git a/services/web/test/frontend/features/file-view/components/file-view-refresh-error.test.tsx b/services/web/test/frontend/features/file-view/components/file-view-refresh-error.test.tsx index 72b4408357..c4aa796fcc 100644 --- a/services/web/test/frontend/features/file-view/components/file-view-refresh-error.test.tsx +++ b/services/web/test/frontend/features/file-view/components/file-view-refresh-error.test.tsx @@ -16,6 +16,7 @@ describe('', function () { name: 'frog.jpg', type: 'file', selected: true, + hash: '42', } render( diff --git a/services/web/test/frontend/features/file-view/components/file-view-text.test.jsx b/services/web/test/frontend/features/file-view/components/file-view-text.test.jsx index a5c1c10375..36fb9325f6 100644 --- a/services/web/test/frontend/features/file-view/components/file-view-text.test.jsx +++ b/services/web/test/frontend/features/file-view/components/file-view-text.test.jsx @@ -6,6 +6,8 @@ import FileViewText from '../../../../../frontend/js/features/file-view/componen describe('', function () { const file = { + id: '123', + hash: '1234', name: 'example.tex', linkedFileData: { v1_source_doc_id: 'v1-source-id', diff --git a/services/web/test/frontend/features/source-editor/components/codemirror-editor-autocomplete.spec.tsx b/services/web/test/frontend/features/source-editor/components/codemirror-editor-autocomplete.spec.tsx index 22ba46b0f5..eb0f2bc7a4 100644 --- a/services/web/test/frontend/features/source-editor/components/codemirror-editor-autocomplete.spec.tsx +++ b/services/web/test/frontend/features/source-editor/components/codemirror-editor-autocomplete.spec.tsx @@ -44,6 +44,7 @@ describe('autocomplete', { scrollBehavior: false }, function () { { _id: 'test-file-in-folder', name: 'example.png', + hash: '42', }, ], folders: [], @@ -53,10 +54,12 @@ describe('autocomplete', { scrollBehavior: false }, function () { { _id: 'test-image-file', name: 'frog.jpg', + hash: '21', }, { _id: 'uppercase-extension-image-file', name: 'frog.JPG', + hash: '22', }, ], }, @@ -194,6 +197,7 @@ describe('autocomplete', { scrollBehavior: false }, function () { { _id: 'test-file-in-folder', name: 'example.png', + hash: '42', }, ], folders: [], @@ -203,6 +207,7 @@ describe('autocomplete', { scrollBehavior: false }, function () { { _id: 'test-image-file', name: 'frog.jpg', + hash: '43', }, ], }, diff --git a/services/web/test/frontend/features/source-editor/helpers/mock-scope.ts b/services/web/test/frontend/features/source-editor/helpers/mock-scope.ts index 0e2b5462ea..d6dd0eb5b4 100644 --- a/services/web/test/frontend/features/source-editor/helpers/mock-scope.ts +++ b/services/web/test/frontend/features/source-editor/helpers/mock-scope.ts @@ -41,10 +41,12 @@ export const mockScope = (content?: string) => { { _id: figureId, name: 'frog.jpg', + hash: '42', }, { _id: 'fake-figure-id', name: 'unicorn.png', + hash: '43', }, ], }, diff --git a/services/web/test/frontend/helpers/editor-providers.jsx b/services/web/test/frontend/helpers/editor-providers.jsx index 0b86a6746e..e64ef8488e 100644 --- a/services/web/test/frontend/helpers/editor-providers.jsx +++ b/services/web/test/frontend/helpers/editor-providers.jsx @@ -34,6 +34,7 @@ import { ModalsContextProvider } from '@/features/ide-react/context/modals-conte import { OnlineUsersProvider } from '@/features/ide-react/context/online-users-context' import { PermissionsProvider } from '@/features/ide-react/context/permissions-context' import { ReferencesProvider } from '@/features/ide-react/context/references-context' +import { SnapshotProvider } from '@/features/ide-react/context/snapshot-context' // these constants can be imported in tests instead of // using magic strings @@ -159,6 +160,7 @@ export function EditorProviders({ DetachProvider, EditorProvider, EditorManagerProvider, + SnapshotProvider, FileTreeDataProvider, FileTreeOpenProvider, FileTreePathProvider, @@ -187,39 +189,41 @@ export function EditorProviders({ - - - - - - - - - - - - - - - - - {children} - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + {children} + + + + + + + + + + + + + + + + + diff --git a/services/web/types/file-ref.ts b/services/web/types/file-ref.ts index 4b2d86893b..1d220c2f70 100644 --- a/services/web/types/file-ref.ts +++ b/services/web/types/file-ref.ts @@ -3,4 +3,5 @@ export type FileRef = { name: string created?: string linkedFileData?: any + hash: string }