From 3ba3ffe56d9467b6a5b125eb2cabb952a30414c8 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 23 Sep 2024 13:09:01 +0200 Subject: [PATCH] Merge pull request #20277 from overleaf/jpa-ro-mirror-chunk [misc] use Chunk for read-only mirror in frontend GitOrigin-RevId: b97e520c7a63015179792542310d18f346752e92 --- libraries/overleaf-editor-core/lib/chunk.js | 7 +- libraries/overleaf-editor-core/lib/types.ts | 10 ++ .../project-history/app/js/HttpController.js | 42 ++++- services/project-history/app/js/Router.js | 16 +- .../project-history/app/js/SnapshotManager.js | 36 ++++ .../acceptance/js/GetChangesInChunkSince.js | 158 ++++++++++++++++++ .../js/helpers/ProjectHistoryClient.js | 19 +++ 7 files changed, 285 insertions(+), 3 deletions(-) create mode 100644 services/project-history/test/acceptance/js/GetChangesInChunkSince.js diff --git a/libraries/overleaf-editor-core/lib/chunk.js b/libraries/overleaf-editor-core/lib/chunk.js index 897c3fa647..929ecb08b6 100644 --- a/libraries/overleaf-editor-core/lib/chunk.js +++ b/libraries/overleaf-editor-core/lib/chunk.js @@ -6,10 +6,11 @@ const OError = require('@overleaf/o-error') const History = require('./history') /** - * @import { BlobStore } from "./types" + * @import { BlobStore, RawChunk } from "./types" * @import Change from "./change" * @import Snapshot from "./snapshot" */ + class ConflictingEndVersion extends OError { constructor(clientEndVersion, latestEndVersion) { const message = @@ -84,6 +85,10 @@ class Chunk { this.startVersion = startVersion } + /** + * @param {RawChunk} raw + * @return {Chunk} + */ static fromRaw(raw) { return new Chunk(History.fromRaw(raw.history), raw.startVersion) } diff --git a/libraries/overleaf-editor-core/lib/types.ts b/libraries/overleaf-editor-core/lib/types.ts index 4ebd56ff8a..604cc93414 100644 --- a/libraries/overleaf-editor-core/lib/types.ts +++ b/libraries/overleaf-editor-core/lib/types.ts @@ -76,6 +76,16 @@ export type RawSnapshot = { v2DocVersions?: RawV2DocVersions | null } +export type RawHistory = { + snapshot: RawSnapshot + changes: RawChange[] +} + +export type RawChunk = { + history: RawHistory + startVersion: number +} + export type RawFileMap = Record export type RawFile = { metadata?: Object } & RawFileData diff --git a/services/project-history/app/js/HttpController.js b/services/project-history/app/js/HttpController.js index af2f4c36a8..934f1de781 100644 --- a/services/project-history/app/js/HttpController.js +++ b/services/project-history/app/js/HttpController.js @@ -234,6 +234,22 @@ export function getFileMetadataSnapshot(req, res, next) { ) } +export function getMostRecentChunk(req, res, next) { + const { project_id: projectId } = req.params + WebApiManager.getHistoryId(projectId, (error, historyId) => { + if (error) return next(OError.tag(error)) + + HistoryStoreManager.getMostRecentChunk( + projectId, + historyId, + (err, data) => { + if (err) return next(OError.tag(err)) + res.json(data) + } + ) + }) +} + export function getLatestSnapshot(req, res, next) { const { project_id: projectId } = req.params WebApiManager.getHistoryId(projectId, (error, historyId) => { @@ -241,10 +257,11 @@ export function getLatestSnapshot(req, res, next) { SnapshotManager.getLatestSnapshot( projectId, historyId, - (error, { snapshot, version }) => { + (error, details) => { if (error != null) { return next(error) } + const { snapshot, version } = details res.json({ snapshot: snapshot.toRaw(), version }) } ) @@ -270,6 +287,29 @@ export function getChangesSince(req, res, next) { }) } +export function getChangesInChunkSince(req, res, next) { + const { project_id: projectId } = req.params + const { since } = req.query + WebApiManager.getHistoryId(projectId, (error, historyId) => { + if (error) return next(OError.tag(error)) + SnapshotManager.getChangesInChunkSince( + projectId, + historyId, + since, + (error, details) => { + if (error != null) { + return next(error) + } + const { latestStartVersion, changes } = details + res.json({ + latestStartVersion, + changes: changes.map(c => c.toRaw()), + }) + } + ) + }) +} + export function getProjectSnapshot(req, res, next) { const { project_id: projectId, version } = req.params SnapshotManager.getProjectSnapshot( diff --git a/services/project-history/app/js/Router.js b/services/project-history/app/js/Router.js index 08d97cd04d..42fe4c244f 100644 --- a/services/project-history/app/js/Router.js +++ b/services/project-history/app/js/Router.js @@ -22,6 +22,10 @@ export function initialize(app) { app.delete('/project/:project_id', HttpController.deleteProject) app.get('/project/:project_id/snapshot', HttpController.getLatestSnapshot) + app.get( + '/project/:project_id/latest/history', + HttpController.getMostRecentChunk + ) app.get( '/project/:project_id/diff', @@ -61,12 +65,22 @@ export function initialize(app) { '/project/:project_id/changes', validate({ query: { - since: Joi.number().integer(), + since: Joi.number().integer().min(0), }, }), HttpController.getChangesSince ) + app.get( + '/project/:project_id/changes-in-chunk', + validate({ + query: { + since: Joi.number().integer().min(0), + }, + }), + HttpController.getChangesInChunkSince + ) + app.get('/project/:project_id/version', HttpController.latestVersion) app.post( diff --git a/services/project-history/app/js/SnapshotManager.js b/services/project-history/app/js/SnapshotManager.js index abe12ade47..04d3de0360 100644 --- a/services/project-history/app/js/SnapshotManager.js +++ b/services/project-history/app/js/SnapshotManager.js @@ -354,6 +354,39 @@ async function getChangesSince(projectId, historyId, sinceVersion) { return allChanges } +async function getChangesInChunkSince(projectId, historyId, sinceVersion) { + const latestChunk = Core.Chunk.fromRaw( + ( + await HistoryStoreManager.promises.getMostRecentChunk( + projectId, + historyId + ) + ).chunk + ) + if (sinceVersion > latestChunk.getEndVersion()) { + throw new Errors.BadRequestError( + 'requested version past the end of the history' + ) + } + const latestStartVersion = latestChunk.getStartVersion() + let chunk = latestChunk + if (sinceVersion < latestStartVersion) { + chunk = Core.Chunk.fromRaw( + ( + await HistoryStoreManager.promises.getChunkAtVersion( + projectId, + historyId, + sinceVersion + ) + ).chunk + ) + } + const changes = chunk + .getChanges() + .slice(sinceVersion - chunk.getStartVersion()) + return { latestStartVersion, changes } +} + async function _loadFilesLimit(snapshot, kind, blobStore) { await snapshot.fileMap.mapAsync(async file => { // only load changed files or files with tracked changes, others can be @@ -369,6 +402,7 @@ async function _loadFilesLimit(snapshot, kind, blobStore) { // EXPORTS const getChangesSinceCb = callbackify(getChangesSince) +const getChangesInChunkSinceCb = callbackify(getChangesInChunkSince) const getFileSnapshotStreamCb = callbackify(getFileSnapshotStream) const getProjectSnapshotCb = callbackify(getProjectSnapshot) const getLatestSnapshotCb = callbackify(getLatestSnapshot) @@ -379,6 +413,7 @@ const getPathsAtVersionCb = callbackify(getPathsAtVersion) export { getChangesSinceCb as getChangesSince, + getChangesInChunkSinceCb as getChangesInChunkSince, getFileSnapshotStreamCb as getFileSnapshotStream, getProjectSnapshotCb as getProjectSnapshot, getFileMetadataSnapshotCb as getFileMetadataSnapshot, @@ -390,6 +425,7 @@ export { export const promises = { getChangesSince, + getChangesInChunkSince, getFileSnapshotStream, getProjectSnapshot, getLatestSnapshot, diff --git a/services/project-history/test/acceptance/js/GetChangesInChunkSince.js b/services/project-history/test/acceptance/js/GetChangesInChunkSince.js new file mode 100644 index 0000000000..2c8c44babb --- /dev/null +++ b/services/project-history/test/acceptance/js/GetChangesInChunkSince.js @@ -0,0 +1,158 @@ +import { expect } from 'chai' +import mongodb from 'mongodb-legacy' +import nock from 'nock' +import Core from 'overleaf-editor-core' +import * as ProjectHistoryClient from './helpers/ProjectHistoryClient.js' +import * as ProjectHistoryApp from './helpers/ProjectHistoryApp.js' +import latestChunk from '../fixtures/chunks/7-8.json' with { type: 'json' } +import previousChunk from '../fixtures/chunks/4-6.json' with { type: 'json' } +import firstChunk from '../fixtures/chunks/0-3.json' with { type: 'json' } +const { ObjectId } = mongodb + +const MockHistoryStore = () => nock('http://127.0.0.1:3100') +const MockWeb = () => nock('http://127.0.0.1:3000') + +const fixture = path => new URL(`../fixtures/${path}`, import.meta.url) + +describe('GetChangesInChunkSince', function () { + let projectId, historyId + beforeEach(function (done) { + projectId = new ObjectId().toString() + historyId = new ObjectId().toString() + ProjectHistoryApp.ensureRunning(error => { + if (error) throw error + + MockHistoryStore().post('/api/projects').reply(200, { + projectId: historyId, + }) + + ProjectHistoryClient.initializeProject(historyId, (error, olProject) => { + if (error) throw error + MockWeb() + .get(`/project/${projectId}/details`) + .reply(200, { + name: 'Test Project', + overleaf: { history: { id: olProject.id } }, + }) + + MockHistoryStore() + .get(`/api/projects/${historyId}/latest/history`) + .replyWithFile(200, fixture('chunks/7-8.json')) + MockHistoryStore() + .get(`/api/projects/${historyId}/versions/7/history`) + .replyWithFile(200, fixture('chunks/7-8.json')) + MockHistoryStore() + .get(`/api/projects/${historyId}/versions/6/history`) + .replyWithFile(200, fixture('chunks/7-8.json')) + MockHistoryStore() + .get(`/api/projects/${historyId}/versions/5/history`) + .replyWithFile(200, fixture('chunks/4-6.json')) + MockHistoryStore() + .get(`/api/projects/${historyId}/versions/4/history`) + .replyWithFile(200, fixture('chunks/4-6.json')) + MockHistoryStore() + .get(`/api/projects/${historyId}/versions/3/history`) + .replyWithFile(200, fixture('chunks/4-6.json')) + MockHistoryStore() + .get(`/api/projects/${historyId}/versions/2/history`) + .replyWithFile(200, fixture('chunks/0-3.json')) + MockHistoryStore() + .get(`/api/projects/${historyId}/versions/1/history`) + .replyWithFile(200, fixture('chunks/0-3.json')) + MockHistoryStore() + .get(`/api/projects/${historyId}/versions/0/history`) + .replyWithFile(200, fixture('chunks/0-3.json')) + + done() + }) + }) + }) + + afterEach(function () { + nock.cleanAll() + }) + + function expectChangesSince(version, n, changes, done) { + ProjectHistoryClient.getChangesInChunkSince( + projectId, + version, + {}, + (error, got) => { + if (error) throw error + expect(got.latestStartVersion).to.equal(6) + expect(got.changes).to.have.length(n) + expect(got.changes.map(c => Core.Change.fromRaw(c))).to.deep.equal( + changes.map(c => Core.Change.fromRaw(c)) + ) + done() + } + ) + } + + const cases = { + 8: { + name: 'when up-to-date, return zero changes', + n: 0, + changes: [], + }, + 7: { + name: 'when one version behind, return one change', + n: 1, + changes: latestChunk.chunk.history.changes.slice(1), + }, + 6: { + name: 'when at current chunk boundary, return latest chunk in full', + n: 2, + changes: latestChunk.chunk.history.changes, + }, + 5: { + name: 'when one version behind last chunk, return one change', + n: 1, + changes: previousChunk.chunk.history.changes.slice(2), + }, + 4: { + name: 'when in last chunk, return two changes', + n: 2, + changes: previousChunk.chunk.history.changes.slice(1), + }, + 3: { + name: 'when at previous chunk boundary, return just the previous chunk', + n: 3, + changes: previousChunk.chunk.history.changes, + }, + 2: { + name: 'when at end of first chunk, return one change', + n: 1, + changes: firstChunk.chunk.history.changes.slice(2), + }, + 1: { + name: 'when in first chunk, return two changes', + n: 2, + changes: firstChunk.chunk.history.changes.slice(1), + }, + 0: { + name: 'when from zero, return just the first chunk', + n: 3, + changes: firstChunk.chunk.history.changes, + }, + } + + for (const [since, { name, n, changes }] of Object.entries(cases)) { + it(name, function (done) { + expectChangesSince(since, n, changes, done) + }) + } + + it('should return an error when past the end version', function (done) { + ProjectHistoryClient.getChangesInChunkSince( + projectId, + 9, + { allowErrors: true }, + (error, _body, statusCode) => { + if (error) throw error + expect(statusCode).to.equal(400) + done() + } + ) + }) +}) diff --git a/services/project-history/test/acceptance/js/helpers/ProjectHistoryClient.js b/services/project-history/test/acceptance/js/helpers/ProjectHistoryClient.js index 1cff6dcdf9..e0b7eb6acf 100644 --- a/services/project-history/test/acceptance/js/helpers/ProjectHistoryClient.js +++ b/services/project-history/test/acceptance/js/helpers/ProjectHistoryClient.js @@ -127,6 +127,25 @@ export function getChangesSince(projectId, since, options, callback) { ) } +export function getChangesInChunkSince(projectId, since, options, callback) { + request.get( + { + url: `http://127.0.0.1:3054/project/${projectId}/changes-in-chunk`, + qs: { + since, + }, + json: true, + }, + (error, res, body) => { + if (error) return callback(error) + if (!options.allowErrors) { + expect(res.statusCode).to.equal(200) + } + callback(null, body, res.statusCode) + } + ) +} + export function getLatestSnapshot(projectId, callback) { request.get( {