mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-21 20:47:08 -05:00
Merge pull request #20277 from overleaf/jpa-ro-mirror-chunk
[misc] use Chunk for read-only mirror in frontend GitOrigin-RevId: b97e520c7a63015179792542310d18f346752e92
This commit is contained in:
parent
18a052f224
commit
3ba3ffe56d
7 changed files with 285 additions and 3 deletions
|
@ -6,10 +6,11 @@ const OError = require('@overleaf/o-error')
|
|||
const History = require('./history')
|
||||
|
||||
/**
|
||||
* @import { BlobStore } from "./types"
|
||||
* @import { BlobStore, RawChunk } from "./types"
|
||||
* @import Change from "./change"
|
||||
* @import Snapshot from "./snapshot"
|
||||
*/
|
||||
|
||||
class ConflictingEndVersion extends OError {
|
||||
constructor(clientEndVersion, latestEndVersion) {
|
||||
const message =
|
||||
|
@ -84,6 +85,10 @@ class Chunk {
|
|||
this.startVersion = startVersion
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RawChunk} raw
|
||||
* @return {Chunk}
|
||||
*/
|
||||
static fromRaw(raw) {
|
||||
return new Chunk(History.fromRaw(raw.history), raw.startVersion)
|
||||
}
|
||||
|
|
|
@ -76,6 +76,16 @@ export type RawSnapshot = {
|
|||
v2DocVersions?: RawV2DocVersions | null
|
||||
}
|
||||
|
||||
export type RawHistory = {
|
||||
snapshot: RawSnapshot
|
||||
changes: RawChange[]
|
||||
}
|
||||
|
||||
export type RawChunk = {
|
||||
history: RawHistory
|
||||
startVersion: number
|
||||
}
|
||||
|
||||
export type RawFileMap = Record<string, RawFile>
|
||||
|
||||
export type RawFile = { metadata?: Object } & RawFileData
|
||||
|
|
|
@ -234,6 +234,22 @@ export function getFileMetadataSnapshot(req, res, next) {
|
|||
)
|
||||
}
|
||||
|
||||
export function getMostRecentChunk(req, res, next) {
|
||||
const { project_id: projectId } = req.params
|
||||
WebApiManager.getHistoryId(projectId, (error, historyId) => {
|
||||
if (error) return next(OError.tag(error))
|
||||
|
||||
HistoryStoreManager.getMostRecentChunk(
|
||||
projectId,
|
||||
historyId,
|
||||
(err, data) => {
|
||||
if (err) return next(OError.tag(err))
|
||||
res.json(data)
|
||||
}
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
export function getLatestSnapshot(req, res, next) {
|
||||
const { project_id: projectId } = req.params
|
||||
WebApiManager.getHistoryId(projectId, (error, historyId) => {
|
||||
|
@ -241,10 +257,11 @@ export function getLatestSnapshot(req, res, next) {
|
|||
SnapshotManager.getLatestSnapshot(
|
||||
projectId,
|
||||
historyId,
|
||||
(error, { snapshot, version }) => {
|
||||
(error, details) => {
|
||||
if (error != null) {
|
||||
return next(error)
|
||||
}
|
||||
const { snapshot, version } = details
|
||||
res.json({ snapshot: snapshot.toRaw(), version })
|
||||
}
|
||||
)
|
||||
|
@ -270,6 +287,29 @@ export function getChangesSince(req, res, next) {
|
|||
})
|
||||
}
|
||||
|
||||
export function getChangesInChunkSince(req, res, next) {
|
||||
const { project_id: projectId } = req.params
|
||||
const { since } = req.query
|
||||
WebApiManager.getHistoryId(projectId, (error, historyId) => {
|
||||
if (error) return next(OError.tag(error))
|
||||
SnapshotManager.getChangesInChunkSince(
|
||||
projectId,
|
||||
historyId,
|
||||
since,
|
||||
(error, details) => {
|
||||
if (error != null) {
|
||||
return next(error)
|
||||
}
|
||||
const { latestStartVersion, changes } = details
|
||||
res.json({
|
||||
latestStartVersion,
|
||||
changes: changes.map(c => c.toRaw()),
|
||||
})
|
||||
}
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
export function getProjectSnapshot(req, res, next) {
|
||||
const { project_id: projectId, version } = req.params
|
||||
SnapshotManager.getProjectSnapshot(
|
||||
|
|
|
@ -22,6 +22,10 @@ export function initialize(app) {
|
|||
app.delete('/project/:project_id', HttpController.deleteProject)
|
||||
|
||||
app.get('/project/:project_id/snapshot', HttpController.getLatestSnapshot)
|
||||
app.get(
|
||||
'/project/:project_id/latest/history',
|
||||
HttpController.getMostRecentChunk
|
||||
)
|
||||
|
||||
app.get(
|
||||
'/project/:project_id/diff',
|
||||
|
@ -61,12 +65,22 @@ export function initialize(app) {
|
|||
'/project/:project_id/changes',
|
||||
validate({
|
||||
query: {
|
||||
since: Joi.number().integer(),
|
||||
since: Joi.number().integer().min(0),
|
||||
},
|
||||
}),
|
||||
HttpController.getChangesSince
|
||||
)
|
||||
|
||||
app.get(
|
||||
'/project/:project_id/changes-in-chunk',
|
||||
validate({
|
||||
query: {
|
||||
since: Joi.number().integer().min(0),
|
||||
},
|
||||
}),
|
||||
HttpController.getChangesInChunkSince
|
||||
)
|
||||
|
||||
app.get('/project/:project_id/version', HttpController.latestVersion)
|
||||
|
||||
app.post(
|
||||
|
|
|
@ -354,6 +354,39 @@ async function getChangesSince(projectId, historyId, sinceVersion) {
|
|||
return allChanges
|
||||
}
|
||||
|
||||
async function getChangesInChunkSince(projectId, historyId, sinceVersion) {
|
||||
const latestChunk = Core.Chunk.fromRaw(
|
||||
(
|
||||
await HistoryStoreManager.promises.getMostRecentChunk(
|
||||
projectId,
|
||||
historyId
|
||||
)
|
||||
).chunk
|
||||
)
|
||||
if (sinceVersion > latestChunk.getEndVersion()) {
|
||||
throw new Errors.BadRequestError(
|
||||
'requested version past the end of the history'
|
||||
)
|
||||
}
|
||||
const latestStartVersion = latestChunk.getStartVersion()
|
||||
let chunk = latestChunk
|
||||
if (sinceVersion < latestStartVersion) {
|
||||
chunk = Core.Chunk.fromRaw(
|
||||
(
|
||||
await HistoryStoreManager.promises.getChunkAtVersion(
|
||||
projectId,
|
||||
historyId,
|
||||
sinceVersion
|
||||
)
|
||||
).chunk
|
||||
)
|
||||
}
|
||||
const changes = chunk
|
||||
.getChanges()
|
||||
.slice(sinceVersion - chunk.getStartVersion())
|
||||
return { latestStartVersion, changes }
|
||||
}
|
||||
|
||||
async function _loadFilesLimit(snapshot, kind, blobStore) {
|
||||
await snapshot.fileMap.mapAsync(async file => {
|
||||
// only load changed files or files with tracked changes, others can be
|
||||
|
@ -369,6 +402,7 @@ async function _loadFilesLimit(snapshot, kind, blobStore) {
|
|||
// EXPORTS
|
||||
|
||||
const getChangesSinceCb = callbackify(getChangesSince)
|
||||
const getChangesInChunkSinceCb = callbackify(getChangesInChunkSince)
|
||||
const getFileSnapshotStreamCb = callbackify(getFileSnapshotStream)
|
||||
const getProjectSnapshotCb = callbackify(getProjectSnapshot)
|
||||
const getLatestSnapshotCb = callbackify(getLatestSnapshot)
|
||||
|
@ -379,6 +413,7 @@ const getPathsAtVersionCb = callbackify(getPathsAtVersion)
|
|||
|
||||
export {
|
||||
getChangesSinceCb as getChangesSince,
|
||||
getChangesInChunkSinceCb as getChangesInChunkSince,
|
||||
getFileSnapshotStreamCb as getFileSnapshotStream,
|
||||
getProjectSnapshotCb as getProjectSnapshot,
|
||||
getFileMetadataSnapshotCb as getFileMetadataSnapshot,
|
||||
|
@ -390,6 +425,7 @@ export {
|
|||
|
||||
export const promises = {
|
||||
getChangesSince,
|
||||
getChangesInChunkSince,
|
||||
getFileSnapshotStream,
|
||||
getProjectSnapshot,
|
||||
getLatestSnapshot,
|
||||
|
|
|
@ -0,0 +1,158 @@
|
|||
import { expect } from 'chai'
|
||||
import mongodb from 'mongodb-legacy'
|
||||
import nock from 'nock'
|
||||
import Core from 'overleaf-editor-core'
|
||||
import * as ProjectHistoryClient from './helpers/ProjectHistoryClient.js'
|
||||
import * as ProjectHistoryApp from './helpers/ProjectHistoryApp.js'
|
||||
import latestChunk from '../fixtures/chunks/7-8.json' with { type: 'json' }
|
||||
import previousChunk from '../fixtures/chunks/4-6.json' with { type: 'json' }
|
||||
import firstChunk from '../fixtures/chunks/0-3.json' with { type: 'json' }
|
||||
const { ObjectId } = mongodb
|
||||
|
||||
const MockHistoryStore = () => nock('http://127.0.0.1:3100')
|
||||
const MockWeb = () => nock('http://127.0.0.1:3000')
|
||||
|
||||
const fixture = path => new URL(`../fixtures/${path}`, import.meta.url)
|
||||
|
||||
describe('GetChangesInChunkSince', function () {
|
||||
let projectId, historyId
|
||||
beforeEach(function (done) {
|
||||
projectId = new ObjectId().toString()
|
||||
historyId = new ObjectId().toString()
|
||||
ProjectHistoryApp.ensureRunning(error => {
|
||||
if (error) throw error
|
||||
|
||||
MockHistoryStore().post('/api/projects').reply(200, {
|
||||
projectId: historyId,
|
||||
})
|
||||
|
||||
ProjectHistoryClient.initializeProject(historyId, (error, olProject) => {
|
||||
if (error) throw error
|
||||
MockWeb()
|
||||
.get(`/project/${projectId}/details`)
|
||||
.reply(200, {
|
||||
name: 'Test Project',
|
||||
overleaf: { history: { id: olProject.id } },
|
||||
})
|
||||
|
||||
MockHistoryStore()
|
||||
.get(`/api/projects/${historyId}/latest/history`)
|
||||
.replyWithFile(200, fixture('chunks/7-8.json'))
|
||||
MockHistoryStore()
|
||||
.get(`/api/projects/${historyId}/versions/7/history`)
|
||||
.replyWithFile(200, fixture('chunks/7-8.json'))
|
||||
MockHistoryStore()
|
||||
.get(`/api/projects/${historyId}/versions/6/history`)
|
||||
.replyWithFile(200, fixture('chunks/7-8.json'))
|
||||
MockHistoryStore()
|
||||
.get(`/api/projects/${historyId}/versions/5/history`)
|
||||
.replyWithFile(200, fixture('chunks/4-6.json'))
|
||||
MockHistoryStore()
|
||||
.get(`/api/projects/${historyId}/versions/4/history`)
|
||||
.replyWithFile(200, fixture('chunks/4-6.json'))
|
||||
MockHistoryStore()
|
||||
.get(`/api/projects/${historyId}/versions/3/history`)
|
||||
.replyWithFile(200, fixture('chunks/4-6.json'))
|
||||
MockHistoryStore()
|
||||
.get(`/api/projects/${historyId}/versions/2/history`)
|
||||
.replyWithFile(200, fixture('chunks/0-3.json'))
|
||||
MockHistoryStore()
|
||||
.get(`/api/projects/${historyId}/versions/1/history`)
|
||||
.replyWithFile(200, fixture('chunks/0-3.json'))
|
||||
MockHistoryStore()
|
||||
.get(`/api/projects/${historyId}/versions/0/history`)
|
||||
.replyWithFile(200, fixture('chunks/0-3.json'))
|
||||
|
||||
done()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
afterEach(function () {
|
||||
nock.cleanAll()
|
||||
})
|
||||
|
||||
function expectChangesSince(version, n, changes, done) {
|
||||
ProjectHistoryClient.getChangesInChunkSince(
|
||||
projectId,
|
||||
version,
|
||||
{},
|
||||
(error, got) => {
|
||||
if (error) throw error
|
||||
expect(got.latestStartVersion).to.equal(6)
|
||||
expect(got.changes).to.have.length(n)
|
||||
expect(got.changes.map(c => Core.Change.fromRaw(c))).to.deep.equal(
|
||||
changes.map(c => Core.Change.fromRaw(c))
|
||||
)
|
||||
done()
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
const cases = {
|
||||
8: {
|
||||
name: 'when up-to-date, return zero changes',
|
||||
n: 0,
|
||||
changes: [],
|
||||
},
|
||||
7: {
|
||||
name: 'when one version behind, return one change',
|
||||
n: 1,
|
||||
changes: latestChunk.chunk.history.changes.slice(1),
|
||||
},
|
||||
6: {
|
||||
name: 'when at current chunk boundary, return latest chunk in full',
|
||||
n: 2,
|
||||
changes: latestChunk.chunk.history.changes,
|
||||
},
|
||||
5: {
|
||||
name: 'when one version behind last chunk, return one change',
|
||||
n: 1,
|
||||
changes: previousChunk.chunk.history.changes.slice(2),
|
||||
},
|
||||
4: {
|
||||
name: 'when in last chunk, return two changes',
|
||||
n: 2,
|
||||
changes: previousChunk.chunk.history.changes.slice(1),
|
||||
},
|
||||
3: {
|
||||
name: 'when at previous chunk boundary, return just the previous chunk',
|
||||
n: 3,
|
||||
changes: previousChunk.chunk.history.changes,
|
||||
},
|
||||
2: {
|
||||
name: 'when at end of first chunk, return one change',
|
||||
n: 1,
|
||||
changes: firstChunk.chunk.history.changes.slice(2),
|
||||
},
|
||||
1: {
|
||||
name: 'when in first chunk, return two changes',
|
||||
n: 2,
|
||||
changes: firstChunk.chunk.history.changes.slice(1),
|
||||
},
|
||||
0: {
|
||||
name: 'when from zero, return just the first chunk',
|
||||
n: 3,
|
||||
changes: firstChunk.chunk.history.changes,
|
||||
},
|
||||
}
|
||||
|
||||
for (const [since, { name, n, changes }] of Object.entries(cases)) {
|
||||
it(name, function (done) {
|
||||
expectChangesSince(since, n, changes, done)
|
||||
})
|
||||
}
|
||||
|
||||
it('should return an error when past the end version', function (done) {
|
||||
ProjectHistoryClient.getChangesInChunkSince(
|
||||
projectId,
|
||||
9,
|
||||
{ allowErrors: true },
|
||||
(error, _body, statusCode) => {
|
||||
if (error) throw error
|
||||
expect(statusCode).to.equal(400)
|
||||
done()
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
|
@ -127,6 +127,25 @@ export function getChangesSince(projectId, since, options, callback) {
|
|||
)
|
||||
}
|
||||
|
||||
export function getChangesInChunkSince(projectId, since, options, callback) {
|
||||
request.get(
|
||||
{
|
||||
url: `http://127.0.0.1:3054/project/${projectId}/changes-in-chunk`,
|
||||
qs: {
|
||||
since,
|
||||
},
|
||||
json: true,
|
||||
},
|
||||
(error, res, body) => {
|
||||
if (error) return callback(error)
|
||||
if (!options.allowErrors) {
|
||||
expect(res.statusCode).to.equal(200)
|
||||
}
|
||||
callback(null, body, res.statusCode)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
export function getLatestSnapshot(projectId, callback) {
|
||||
request.get(
|
||||
{
|
||||
|
|
Loading…
Reference in a new issue