Merge pull request #20277 from overleaf/jpa-ro-mirror-chunk

[misc] use Chunk for read-only mirror in frontend

GitOrigin-RevId: b97e520c7a63015179792542310d18f346752e92
This commit is contained in:
Jakob Ackermann 2024-09-23 13:09:01 +02:00 committed by Copybot
parent 18a052f224
commit 3ba3ffe56d
7 changed files with 285 additions and 3 deletions

View file

@ -6,10 +6,11 @@ const OError = require('@overleaf/o-error')
const History = require('./history')
/**
* @import { BlobStore } from "./types"
* @import { BlobStore, RawChunk } from "./types"
* @import Change from "./change"
* @import Snapshot from "./snapshot"
*/
class ConflictingEndVersion extends OError {
constructor(clientEndVersion, latestEndVersion) {
const message =
@ -84,6 +85,10 @@ class Chunk {
this.startVersion = startVersion
}
/**
* @param {RawChunk} raw
* @return {Chunk}
*/
static fromRaw(raw) {
return new Chunk(History.fromRaw(raw.history), raw.startVersion)
}

View file

@ -76,6 +76,16 @@ export type RawSnapshot = {
v2DocVersions?: RawV2DocVersions | null
}
export type RawHistory = {
snapshot: RawSnapshot
changes: RawChange[]
}
export type RawChunk = {
history: RawHistory
startVersion: number
}
export type RawFileMap = Record<string, RawFile>
export type RawFile = { metadata?: Object } & RawFileData

View file

@ -234,6 +234,22 @@ export function getFileMetadataSnapshot(req, res, next) {
)
}
export function getMostRecentChunk(req, res, next) {
const { project_id: projectId } = req.params
WebApiManager.getHistoryId(projectId, (error, historyId) => {
if (error) return next(OError.tag(error))
HistoryStoreManager.getMostRecentChunk(
projectId,
historyId,
(err, data) => {
if (err) return next(OError.tag(err))
res.json(data)
}
)
})
}
export function getLatestSnapshot(req, res, next) {
const { project_id: projectId } = req.params
WebApiManager.getHistoryId(projectId, (error, historyId) => {
@ -241,10 +257,11 @@ export function getLatestSnapshot(req, res, next) {
SnapshotManager.getLatestSnapshot(
projectId,
historyId,
(error, { snapshot, version }) => {
(error, details) => {
if (error != null) {
return next(error)
}
const { snapshot, version } = details
res.json({ snapshot: snapshot.toRaw(), version })
}
)
@ -270,6 +287,29 @@ export function getChangesSince(req, res, next) {
})
}
export function getChangesInChunkSince(req, res, next) {
const { project_id: projectId } = req.params
const { since } = req.query
WebApiManager.getHistoryId(projectId, (error, historyId) => {
if (error) return next(OError.tag(error))
SnapshotManager.getChangesInChunkSince(
projectId,
historyId,
since,
(error, details) => {
if (error != null) {
return next(error)
}
const { latestStartVersion, changes } = details
res.json({
latestStartVersion,
changes: changes.map(c => c.toRaw()),
})
}
)
})
}
export function getProjectSnapshot(req, res, next) {
const { project_id: projectId, version } = req.params
SnapshotManager.getProjectSnapshot(

View file

@ -22,6 +22,10 @@ export function initialize(app) {
app.delete('/project/:project_id', HttpController.deleteProject)
app.get('/project/:project_id/snapshot', HttpController.getLatestSnapshot)
app.get(
'/project/:project_id/latest/history',
HttpController.getMostRecentChunk
)
app.get(
'/project/:project_id/diff',
@ -61,12 +65,22 @@ export function initialize(app) {
'/project/:project_id/changes',
validate({
query: {
since: Joi.number().integer(),
since: Joi.number().integer().min(0),
},
}),
HttpController.getChangesSince
)
app.get(
'/project/:project_id/changes-in-chunk',
validate({
query: {
since: Joi.number().integer().min(0),
},
}),
HttpController.getChangesInChunkSince
)
app.get('/project/:project_id/version', HttpController.latestVersion)
app.post(

View file

@ -354,6 +354,39 @@ async function getChangesSince(projectId, historyId, sinceVersion) {
return allChanges
}
async function getChangesInChunkSince(projectId, historyId, sinceVersion) {
const latestChunk = Core.Chunk.fromRaw(
(
await HistoryStoreManager.promises.getMostRecentChunk(
projectId,
historyId
)
).chunk
)
if (sinceVersion > latestChunk.getEndVersion()) {
throw new Errors.BadRequestError(
'requested version past the end of the history'
)
}
const latestStartVersion = latestChunk.getStartVersion()
let chunk = latestChunk
if (sinceVersion < latestStartVersion) {
chunk = Core.Chunk.fromRaw(
(
await HistoryStoreManager.promises.getChunkAtVersion(
projectId,
historyId,
sinceVersion
)
).chunk
)
}
const changes = chunk
.getChanges()
.slice(sinceVersion - chunk.getStartVersion())
return { latestStartVersion, changes }
}
async function _loadFilesLimit(snapshot, kind, blobStore) {
await snapshot.fileMap.mapAsync(async file => {
// only load changed files or files with tracked changes, others can be
@ -369,6 +402,7 @@ async function _loadFilesLimit(snapshot, kind, blobStore) {
// EXPORTS
const getChangesSinceCb = callbackify(getChangesSince)
const getChangesInChunkSinceCb = callbackify(getChangesInChunkSince)
const getFileSnapshotStreamCb = callbackify(getFileSnapshotStream)
const getProjectSnapshotCb = callbackify(getProjectSnapshot)
const getLatestSnapshotCb = callbackify(getLatestSnapshot)
@ -379,6 +413,7 @@ const getPathsAtVersionCb = callbackify(getPathsAtVersion)
export {
getChangesSinceCb as getChangesSince,
getChangesInChunkSinceCb as getChangesInChunkSince,
getFileSnapshotStreamCb as getFileSnapshotStream,
getProjectSnapshotCb as getProjectSnapshot,
getFileMetadataSnapshotCb as getFileMetadataSnapshot,
@ -390,6 +425,7 @@ export {
export const promises = {
getChangesSince,
getChangesInChunkSince,
getFileSnapshotStream,
getProjectSnapshot,
getLatestSnapshot,

View file

@ -0,0 +1,158 @@
import { expect } from 'chai'
import mongodb from 'mongodb-legacy'
import nock from 'nock'
import Core from 'overleaf-editor-core'
import * as ProjectHistoryClient from './helpers/ProjectHistoryClient.js'
import * as ProjectHistoryApp from './helpers/ProjectHistoryApp.js'
import latestChunk from '../fixtures/chunks/7-8.json' with { type: 'json' }
import previousChunk from '../fixtures/chunks/4-6.json' with { type: 'json' }
import firstChunk from '../fixtures/chunks/0-3.json' with { type: 'json' }
const { ObjectId } = mongodb
const MockHistoryStore = () => nock('http://127.0.0.1:3100')
const MockWeb = () => nock('http://127.0.0.1:3000')
const fixture = path => new URL(`../fixtures/${path}`, import.meta.url)
describe('GetChangesInChunkSince', function () {
let projectId, historyId
beforeEach(function (done) {
projectId = new ObjectId().toString()
historyId = new ObjectId().toString()
ProjectHistoryApp.ensureRunning(error => {
if (error) throw error
MockHistoryStore().post('/api/projects').reply(200, {
projectId: historyId,
})
ProjectHistoryClient.initializeProject(historyId, (error, olProject) => {
if (error) throw error
MockWeb()
.get(`/project/${projectId}/details`)
.reply(200, {
name: 'Test Project',
overleaf: { history: { id: olProject.id } },
})
MockHistoryStore()
.get(`/api/projects/${historyId}/latest/history`)
.replyWithFile(200, fixture('chunks/7-8.json'))
MockHistoryStore()
.get(`/api/projects/${historyId}/versions/7/history`)
.replyWithFile(200, fixture('chunks/7-8.json'))
MockHistoryStore()
.get(`/api/projects/${historyId}/versions/6/history`)
.replyWithFile(200, fixture('chunks/7-8.json'))
MockHistoryStore()
.get(`/api/projects/${historyId}/versions/5/history`)
.replyWithFile(200, fixture('chunks/4-6.json'))
MockHistoryStore()
.get(`/api/projects/${historyId}/versions/4/history`)
.replyWithFile(200, fixture('chunks/4-6.json'))
MockHistoryStore()
.get(`/api/projects/${historyId}/versions/3/history`)
.replyWithFile(200, fixture('chunks/4-6.json'))
MockHistoryStore()
.get(`/api/projects/${historyId}/versions/2/history`)
.replyWithFile(200, fixture('chunks/0-3.json'))
MockHistoryStore()
.get(`/api/projects/${historyId}/versions/1/history`)
.replyWithFile(200, fixture('chunks/0-3.json'))
MockHistoryStore()
.get(`/api/projects/${historyId}/versions/0/history`)
.replyWithFile(200, fixture('chunks/0-3.json'))
done()
})
})
})
afterEach(function () {
nock.cleanAll()
})
function expectChangesSince(version, n, changes, done) {
ProjectHistoryClient.getChangesInChunkSince(
projectId,
version,
{},
(error, got) => {
if (error) throw error
expect(got.latestStartVersion).to.equal(6)
expect(got.changes).to.have.length(n)
expect(got.changes.map(c => Core.Change.fromRaw(c))).to.deep.equal(
changes.map(c => Core.Change.fromRaw(c))
)
done()
}
)
}
const cases = {
8: {
name: 'when up-to-date, return zero changes',
n: 0,
changes: [],
},
7: {
name: 'when one version behind, return one change',
n: 1,
changes: latestChunk.chunk.history.changes.slice(1),
},
6: {
name: 'when at current chunk boundary, return latest chunk in full',
n: 2,
changes: latestChunk.chunk.history.changes,
},
5: {
name: 'when one version behind last chunk, return one change',
n: 1,
changes: previousChunk.chunk.history.changes.slice(2),
},
4: {
name: 'when in last chunk, return two changes',
n: 2,
changes: previousChunk.chunk.history.changes.slice(1),
},
3: {
name: 'when at previous chunk boundary, return just the previous chunk',
n: 3,
changes: previousChunk.chunk.history.changes,
},
2: {
name: 'when at end of first chunk, return one change',
n: 1,
changes: firstChunk.chunk.history.changes.slice(2),
},
1: {
name: 'when in first chunk, return two changes',
n: 2,
changes: firstChunk.chunk.history.changes.slice(1),
},
0: {
name: 'when from zero, return just the first chunk',
n: 3,
changes: firstChunk.chunk.history.changes,
},
}
for (const [since, { name, n, changes }] of Object.entries(cases)) {
it(name, function (done) {
expectChangesSince(since, n, changes, done)
})
}
it('should return an error when past the end version', function (done) {
ProjectHistoryClient.getChangesInChunkSince(
projectId,
9,
{ allowErrors: true },
(error, _body, statusCode) => {
if (error) throw error
expect(statusCode).to.equal(400)
done()
}
)
})
})

View file

@ -127,6 +127,25 @@ export function getChangesSince(projectId, since, options, callback) {
)
}
export function getChangesInChunkSince(projectId, since, options, callback) {
request.get(
{
url: `http://127.0.0.1:3054/project/${projectId}/changes-in-chunk`,
qs: {
since,
},
json: true,
},
(error, res, body) => {
if (error) return callback(error)
if (!options.allowErrors) {
expect(res.statusCode).to.equal(200)
}
callback(null, body, res.statusCode)
}
)
}
export function getLatestSnapshot(projectId, callback) {
request.get(
{