overleaf/services/project-history/test/acceptance/js/SendingUpdatesTests.js

2118 lines
56 KiB
JavaScript
Raw Normal View History

import { expect } from 'chai'
import Settings from '@overleaf/settings'
import assert from 'assert'
import async from 'async'
import crypto from 'crypto'
import mongodb from 'mongodb-legacy'
import nock from 'nock'
import * as ProjectHistoryClient from './helpers/ProjectHistoryClient.js'
import * as ProjectHistoryApp from './helpers/ProjectHistoryApp.js'
const { ObjectId } = mongodb
const MockHistoryStore = () => nock('http://127.0.0.1:3100')
const MockFileStore = () => nock('http://127.0.0.1:3009')
const MockWeb = () => nock('http://127.0.0.1:3000')
// Some helper methods to make the tests more compact
function slTextUpdate(historyId, doc, userId, v, ts, op) {
return {
projectHistoryId: historyId,
doc: doc.id,
op,
v,
meta: {
user_id: userId,
ts: ts.getTime(),
pathname: doc.pathname,
doc_length: doc.length,
},
}
}
function slAddDocUpdate(historyId, doc, userId, ts, docLines, ranges = {}) {
return {
projectHistoryId: historyId,
pathname: doc.pathname,
ranges,
docLines,
doc: doc.id,
meta: { user_id: userId, ts: ts.getTime() },
}
}
function slAddDocUpdateWithVersion(
historyId,
doc,
userId,
ts,
docLines,
projectVersion,
ranges = {}
) {
const result = slAddDocUpdate(historyId, doc, userId, ts, docLines, ranges)
result.version = projectVersion
return result
}
function slAddFileUpdate(historyId, file, userId, ts, projectId) {
return {
projectHistoryId: historyId,
pathname: file.pathname,
url: `http://127.0.0.1:3009/project/${projectId}/file/${file.id}`,
file: file.id,
ranges: undefined,
meta: { user_id: userId, ts: ts.getTime() },
}
}
function slRenameUpdate(historyId, doc, userId, ts, pathname, newPathname) {
return {
projectHistoryId: historyId,
pathname,
new_pathname: newPathname,
doc: doc.id,
meta: { user_id: userId, ts: ts.getTime() },
}
}
function olUpdate(doc, userId, ts, operations, v) {
return {
v2Authors: [userId],
timestamp: ts.toJSON(),
authors: [],
operations,
v2DocVersions: {
[doc.id]: {
pathname: doc.pathname.replace(/^\//, ''), // Strip leading /
v: v || 1,
},
},
}
}
function olTextOperation(doc, textOperation) {
return {
pathname: doc.pathname.replace(/^\//, ''), // Strip leading /
textOperation,
}
}
function olAddCommentOperation(doc, commentId, pos, length) {
return {
pathname: doc.pathname.replace(/^\//, ''), // Strip leading /
commentId,
ranges: [{ pos, length }],
}
}
function olTextUpdate(doc, userId, ts, textOperation, v) {
return olUpdate(doc, userId, ts, [olTextOperation(doc, textOperation)], v)
}
function olTextUpdates(doc, userId, ts, textOperations, v) {
return olUpdate(
doc,
userId,
ts,
textOperations.map(textOperation => olTextOperation(doc, textOperation)),
v
)
}
function olRenameUpdate(doc, userId, ts, pathname, newPathname) {
return {
v2Authors: [userId],
timestamp: ts.toJSON(),
authors: [],
operations: [
{
pathname,
newPathname,
},
],
}
}
function olAddDocUpdate(doc, userId, ts, fileHash, rangesHash = undefined) {
const update = {
v2Authors: [userId],
timestamp: ts.toJSON(),
authors: [],
operations: [
{
pathname: doc.pathname.replace(/^\//, ''), // Strip leading /
file: {
hash: fileHash,
},
},
],
}
if (rangesHash) {
update.operations[0].file.rangesHash = rangesHash
}
return update
}
function olAddDocUpdateWithVersion(
doc,
userId,
ts,
fileHash,
version,
rangesHash = undefined
) {
const result = olAddDocUpdate(doc, userId, ts, fileHash, rangesHash)
result.projectVersion = version
return result
}
function olAddFileUpdate(file, userId, ts, fileHash) {
return {
v2Authors: [userId],
timestamp: ts.toJSON(),
authors: [],
operations: [
{
pathname: file.pathname.replace(/^\//, ''), // Strip leading /
file: {
hash: fileHash,
},
},
],
}
}
describe('Sending Updates', function () {
const historyId = new ObjectId().toString()
beforeEach(function (done) {
this.timestamp = new Date()
ProjectHistoryApp.ensureRunning(error => {
if (error) {
return done(error)
}
this.userId = new ObjectId().toString()
this.projectId = new ObjectId().toString()
this.docId = new ObjectId().toString()
this.doc = {
id: this.docId,
pathname: '/main.tex',
length: 5,
}
MockHistoryStore().post('/api/projects').reply(200, {
projectId: historyId,
})
MockWeb()
.get(`/project/${this.projectId}/details`)
.reply(200, {
name: 'Test Project',
overleaf: {
history: {
id: historyId,
},
},
})
ProjectHistoryClient.initializeProject(historyId, done)
})
})
afterEach(function () {
nock.cleanAll()
})
describe('basic update types', function () {
beforeEach(function () {
MockHistoryStore()
.get(`/api/projects/${historyId}/latest/history`)
.reply(200, {
chunk: {
startVersion: 0,
history: {
snapshot: {},
changes: [],
},
},
})
})
it('should send add doc updates to the history store', function (done) {
const fileHash = '0a207c060e61f3b88eaee0a8cd0696f46fb155eb'
const createBlob = MockHistoryStore()
.put(`/api/projects/${historyId}/blobs/${fileHash}`, 'a\nb')
.reply(201)
const addFile = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olAddDocUpdate(this.doc, this.userId, this.timestamp, fileHash),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slAddDocUpdate(
historyId,
this.doc,
this.userId,
this.timestamp,
'a\nb'
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createBlob.isDone(),
'/api/projects/:historyId/blobs/:hash should have been called'
)
assert(
addFile.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
it('should send ranges to the history store', function (done) {
const fileHash = '49e886093b3eacbc12b99a1eb5aeaa44a6b9d90e'
const rangesHash = 'fa9a429ff518bc9e5b2507a96ff0646b566eca65'
const historyRanges = {
trackedChanges: [
{
range: { pos: 4, length: 3 },
tracking: {
type: 'delete',
userId: 'user-id-1',
ts: '2024-01-01T00:00:00.000Z',
},
},
],
comments: [
{
ranges: [{ pos: 0, length: 3 }],
id: 'comment-id-1',
},
],
}
// We need to set up the ranges mock first, as we will call it last..
const createRangesBlob = MockHistoryStore()
.put(`/api/projects/${historyId}/blobs/${rangesHash}`, historyRanges)
.reply(201)
const createBlob = MockHistoryStore()
.put(`/api/projects/${historyId}/blobs/${fileHash}`, 'foo barbaz')
.reply(201)
const addFile = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olAddDocUpdate(
this.doc,
this.userId,
this.timestamp,
fileHash,
rangesHash
),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slAddDocUpdate(
historyId,
this.doc,
this.userId,
this.timestamp,
'foo barbaz',
{
changes: [
{
op: { p: 4, d: 'bar' },
metadata: {
ts: 1704067200000,
user_id: 'user-id-1',
},
},
],
comments: [
{
op: {
p: 0,
c: 'foo',
t: 'comment-id-1',
},
metadata: { resolved: false },
},
],
}
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createBlob.isDone(),
'/api/projects/:historyId/blobs/:hash should have been called to create content blob'
)
assert(
createRangesBlob.isDone(),
'/api/projects/:historyId/blobs/:hash should have been called to create ranges blob'
)
assert(
addFile.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
it('should strip non-BMP characters in add doc updates before sending to the history store', function (done) {
const fileHash = '11509fe05a41f9cdc51ea081342b5a4fc7c8d0fc'
const createBlob = MockHistoryStore()
.put(
`/api/projects/${historyId}/blobs/${fileHash}`,
'a\nb\uFFFD\uFFFDc'
)
.reply(201)
const addFile = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olAddDocUpdate(this.doc, this.userId, this.timestamp, fileHash),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slAddDocUpdate(
historyId,
this.doc,
this.userId,
this.timestamp,
'a\nb\uD800\uDC00c'
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createBlob.isDone(),
'/api/projects/:historyId/blobs/:hash should have been called'
)
assert(
addFile.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
it('should send text updates to the history store', function (done) {
const createChange = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olTextUpdate(this.doc, this.userId, this.timestamp, [3, '\nc', 2]),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(
historyId,
this.doc,
this.userId,
1,
this.timestamp,
[{ p: 3, i: '\nc' }]
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createChange.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
it('should send renames to the history store', function (done) {
const createChange = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olRenameUpdate(
this.doc,
this.userId,
this.timestamp,
'main.tex',
'main2.tex'
),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slRenameUpdate(
historyId,
this.doc,
this.userId,
this.timestamp,
'/main.tex',
'/main2.tex'
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createChange.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
it('should send add file updates to the history store', function (done) {
const file = {
id: new ObjectId().toString(),
pathname: '/test.png',
contents: Buffer.from([1, 2, 3]),
hash: 'aed2973e4b8a7ff1b30ff5c4751e5a2b38989e74',
}
const fileStoreRequest = MockFileStore()
.get(`/project/${this.projectId}/file/${file.id}`)
.reply(200, file.contents)
const createBlob = MockHistoryStore()
.put(`/api/projects/${historyId}/blobs/${file.hash}`, file.contents)
.reply(201)
const addFile = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olAddFileUpdate(file, this.userId, this.timestamp, file.hash),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slAddFileUpdate(
historyId,
file,
this.userId,
this.timestamp,
this.projectId
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
fileStoreRequest.isDone(),
`/project/${this.projectId}/file/${file.id} should have been called`
)
assert(
createBlob.isDone(),
`/api/projects/${historyId}/latest/files should have been called`
)
assert(
addFile.isDone(),
`/api/projects/${historyId}/latest/files should have been called`
)
done()
}
)
})
it('should send a stub to the history store when the file is large', function (done) {
const fileContents = Buffer.alloc(Settings.maxFileSizeInBytes + 1, 'X')
const fileSize = Buffer.byteLength(fileContents)
const fileHash = crypto
.createHash('sha1')
.update('blob ' + fileSize + '\x00')
.update(fileContents, 'utf8')
.digest('hex')
const file = {
id: new ObjectId().toString(),
pathname: '/large.png',
contents: fileContents,
hash: fileHash,
}
const stubContents = [
'FileTooLargeError v1',
'File too large to be stored in history service',
`id project-${this.projectId}-file-${file.id}`,
`size ${fileSize} bytes`,
`hash ${fileHash}`,
'\0', // null byte to make this a binary file
].join('\n')
const stubHash = crypto
.createHash('sha1')
.update('blob ' + Buffer.byteLength(stubContents) + '\x00')
.update(stubContents, 'utf8')
.digest('hex')
const stub = {
id: file.id,
pathname: file.pathname,
contents: stubContents,
hash: stubHash,
}
const fileStoreRequest = MockFileStore()
.get(`/project/${this.projectId}/file/${file.id}`)
.reply(200, file.contents)
const createBlob = MockHistoryStore()
.put(`/api/projects/${historyId}/blobs/${stub.hash}`, stub.contents)
.reply(201)
const addFile = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olAddFileUpdate(stub, this.userId, this.timestamp, stub.hash),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slAddFileUpdate(
historyId,
file,
this.userId,
this.timestamp,
this.projectId
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
addFile.isDone(),
`/api/projects/${historyId}/latest/files should have been called`
)
assert(
createBlob.isDone(),
`/api/projects/${historyId}/latest/files should have been called`
)
assert(
fileStoreRequest.isDone(),
`/project/${this.projectId}/file/${file.id} should have been called`
)
done()
}
)
})
it('should handle comment ops', function (done) {
const createChange = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olUpdate(this.doc, this.userId, this.timestamp, [
olTextOperation(this.doc, [3, '\nc', 2]),
olAddCommentOperation(this.doc, 'comment-id-1', 3, 2),
]),
olUpdate(
this.doc,
this.userId,
this.timestamp,
[olAddCommentOperation(this.doc, 'comment-id-2', 2, 1)],
2
),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(
historyId,
this.doc,
this.userId,
1,
this.timestamp,
[
{ p: 3, i: '\nc' },
{ p: 3, c: '\nc', t: 'comment-id-1' },
]
),
cb
)
},
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(
historyId,
this.doc,
this.userId,
2,
this.timestamp,
[{ p: 2, c: 'b', t: 'comment-id-2' }]
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createChange.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
it('should be able to process lots of updates in batches', function (done) {
const BATCH_SIZE = 500
const createFirstChangeBatch = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olTextUpdate(
this.doc,
this.userId,
this.timestamp,
['a'.repeat(BATCH_SIZE), 6],
BATCH_SIZE - 1
),
])
return true
})
.query({ end_version: 0 })
.reply(204)
const createSecondChangeBatch = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olTextUpdate(
this.doc,
this.userId,
this.timestamp,
['a'.repeat(50), BATCH_SIZE + 6],
BATCH_SIZE - 1 + 50
),
])
return true
})
.query({ end_version: 500 })
.reply(204)
// these need mocking again for the second batch
MockHistoryStore()
.get(`/api/projects/${historyId}/latest/history`)
.reply(200, {
chunk: {
startVersion: BATCH_SIZE,
history: {
snapshot: {},
changes: [],
},
},
})
MockWeb()
.get(`/project/${this.projectId}/details`)
.reply(200, {
name: 'Test Project',
overleaf: {
history: {
id: historyId,
},
},
})
const pushChange = (n, cb) => {
this.doc.length += 1
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(historyId, this.doc, this.userId, n, this.timestamp, [
{ p: 0, i: 'a' },
]),
cb
)
}
async.series(
[
cb => {
async.times(BATCH_SIZE + 50, pushChange, cb)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createFirstChangeBatch.isDone(),
`/api/projects/${historyId}/changes should have been called for the first batch`
)
assert(
createSecondChangeBatch.isDone(),
`/api/projects/${historyId}/changes should have been called for the second batch`
)
done()
}
)
})
})
describe('compressing updates', function () {
beforeEach(function () {
MockHistoryStore()
.get(`/api/projects/${historyId}/latest/history`)
.reply(200, {
chunk: {
startVersion: 0,
history: {
snapshot: {},
changes: [],
},
},
})
})
it('should concat adjacent text updates', function (done) {
const createChange = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olTextUpdate(
this.doc,
this.userId,
this.timestamp,
[3, 'foobaz', 2],
2
),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(
historyId,
this.doc,
this.userId,
1,
this.timestamp,
[
{ p: 3, i: 'foobar' },
{ p: 6, d: 'bar' },
]
),
cb
)
this.doc.length += 3
},
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(
historyId,
this.doc,
this.userId,
2,
this.timestamp,
[{ p: 6, i: 'baz' }]
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createChange.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
it('should take the timestamp of the first update', function (done) {
const timestamp1 = new Date(this.timestamp)
const timestamp2 = new Date(this.timestamp.getTime() + 10000)
const createChange = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olTextUpdate(
this.doc,
this.userId,
timestamp1,
[3, 'foobaz', 2],
2
),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(historyId, this.doc, this.userId, 1, timestamp1, [
{ p: 3, i: 'foo' },
]),
cb
)
this.doc.length += 3
},
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(historyId, this.doc, this.userId, 2, timestamp2, [
{ p: 6, i: 'baz' },
]),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createChange.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
it('should not concat updates more than 60 seconds apart', function (done) {
const timestamp1 = new Date(this.timestamp)
const timestamp2 = new Date(this.timestamp.getTime() + 120000)
const createChange = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olTextUpdate(this.doc, this.userId, timestamp1, [3, 'foo', 2], 1),
olTextUpdate(this.doc, this.userId, timestamp2, [6, 'baz', 2], 2),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(historyId, this.doc, this.userId, 1, timestamp1, [
{ p: 3, i: 'foo' },
]),
cb
)
this.doc.length += 3
},
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(historyId, this.doc, this.userId, 2, timestamp2, [
{ p: 6, i: 'baz' },
]),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createChange.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
it('should not concat updates with different user_ids', function (done) {
const userId1 = new ObjectId().toString()
const userId2 = new ObjectId().toString()
const createChange = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olTextUpdate(this.doc, userId1, this.timestamp, [3, 'foo', 2], 1),
olTextUpdate(this.doc, userId2, this.timestamp, [6, 'baz', 2], 2),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(historyId, this.doc, userId1, 1, this.timestamp, [
{ p: 3, i: 'foo' },
]),
cb
)
this.doc.length += 3
},
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(historyId, this.doc, userId2, 2, this.timestamp, [
{ p: 6, i: 'baz' },
]),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createChange.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
it('should not concat updates with different docs', function (done) {
const doc1 = {
id: new ObjectId().toString(),
pathname: '/doc1.tex',
length: 10,
}
const doc2 = {
id: new ObjectId().toString(),
pathname: '/doc2.tex',
length: 10,
}
const createChange = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olTextUpdate(doc1, this.userId, this.timestamp, [3, 'foo', 7], 1),
olTextUpdate(doc2, this.userId, this.timestamp, [6, 'baz', 4], 2),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(historyId, doc1, this.userId, 1, this.timestamp, [
{ p: 3, i: 'foo' },
]),
cb
)
},
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(historyId, doc2, this.userId, 2, this.timestamp, [
{ p: 6, i: 'baz' },
]),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createChange.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
it('should not send updates without any ops', function (done) {
const createChange = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
// These blank ops can get sent by doc-updater on setDocs from Dropbox that don't change anything
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(
historyId,
this.doc,
this.userId,
1,
this.timestamp,
[]
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
!createChange.isDone(),
`/api/projects/${historyId}/changes should not have been called`
)
done()
}
)
})
it('should not send ops that compress to nothing', function (done) {
const createChange = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(
historyId,
this.doc,
this.userId,
1,
this.timestamp,
[{ i: 'foo', p: 3 }]
),
cb
)
this.doc.length += 3
},
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(
historyId,
this.doc,
this.userId,
2,
this.timestamp,
[{ d: 'foo', p: 3 }]
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
!createChange.isDone(),
`/api/projects/${historyId}/changes should not have been called`
)
done()
}
)
})
it('should not send ops from a diff that are blank', function (done) {
this.doc.length = 300
// Test case taken from a real life document where it was generating blank insert and
// delete ops from a diff, and the blank delete was erroring on the OL history from
// a text operation like [42, 0, 512], where the 0 was invalid.
const createChange = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olTextUpdates(this.doc, this.userId, this.timestamp, [
[
87,
-1,
67,
'|l|ll|}\n\\hline',
-4,
30,
' \\hline',
87,
' \\\\ \\hline',
24,
],
]),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(
historyId,
this.doc,
this.userId,
1,
this.timestamp,
[
{
p: 73,
d: '\\begin{table}[h]\n\\centering\n\\caption{My caption}\n\\label{my-label}\n\\begin{tabular}{lll}\n & A & B \\\\\nLiter t up & 2 & 1 \\\\\nLiter Whiskey & 1 & 2 \\\\\nPris pr. liter & 200 & 250\n\\end{tabular}\n\\end{table}',
},
{
p: 73,
i: '\\begin{table}[]\n\\centering\n\\caption{My caption}\n\\label{my-label}\n\\begin{tabular}{|l|ll|}\n\\hline\n & A & B \\\\ \\hline\nLiter t up & 2 & 1 \\\\\nLiter Whiskey & 1 & 2 \\\\\nPris pr. liter & 200 & 250 \\\\ \\hline\n\\end{tabular}\n\\end{table}',
},
]
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createChange.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
it('should not concat text updates across project structure ops', function (done) {
const newDoc = {
id: new ObjectId().toString(),
pathname: '/main.tex',
hash: '0a207c060e61f3b88eaee0a8cd0696f46fb155eb',
docLines: 'a\nb',
}
MockHistoryStore()
.put(`/api/projects/${historyId}/blobs/${newDoc.hash}`)
.reply(201)
const createChange = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olTextUpdate(
this.doc,
this.userId,
this.timestamp,
[3, 'foo', 2],
1
),
olAddDocUpdate(newDoc, this.userId, this.timestamp, newDoc.hash),
olTextUpdate(
this.doc,
this.userId,
this.timestamp,
[6, 'baz', 2],
2
),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(
historyId,
this.doc,
this.userId,
1,
this.timestamp,
[
{ p: 3, i: 'foobar' },
{ p: 6, d: 'bar' },
]
),
cb
)
this.doc.length += 3
},
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slAddDocUpdate(
historyId,
newDoc,
this.userId,
this.timestamp,
newDoc.docLines
),
cb
)
},
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(
historyId,
this.doc,
this.userId,
2,
this.timestamp,
[{ p: 6, i: 'baz' }]
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createChange.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
it('should track the doc length when splitting ops', function (done) {
this.doc.length = 10
const createChange = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olTextUpdate(this.doc, this.userId, this.timestamp, [3, -3, 4], 1),
olTextUpdate(
this.doc,
this.userId,
this.timestamp,
[3, 'barbaz', 4],
2
), // This has a base length of 10
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(
historyId,
this.doc,
this.userId,
1,
this.timestamp,
[
{ p: 3, d: 'foo' },
{ p: 3, i: 'bar' }, // Make sure the length of the op generated from this is 7, not 10
]
),
cb
)
},
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(
historyId,
this.doc,
this.userId,
2,
this.timestamp,
[{ p: 6, i: 'baz' }]
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createChange.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
})
describe('with bad pathnames', function () {
beforeEach(function () {
MockHistoryStore()
.get(`/api/projects/${historyId}/latest/history`)
.reply(200, {
chunk: {
startVersion: 0,
history: {
snapshot: {},
changes: [],
},
},
})
})
it('should replace \\ with _ and workaround * in pathnames', function (done) {
const doc = {
id: this.doc.id,
pathname: '\\main.tex',
hash: 'b07b6b7a27667965f733943737124395c7577bea',
docLines: 'aaabbbccc',
length: 9,
}
MockHistoryStore()
.put(`/api/projects/${historyId}/blobs/${doc.hash}`)
.reply(201)
const createChange = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olAddDocUpdate(
{ id: doc.id, pathname: '_main.tex' },
this.userId,
this.timestamp,
doc.hash
),
olRenameUpdate(
{ id: doc.id, pathname: '_main.tex' },
this.userId,
this.timestamp,
'_main.tex',
'_main2.tex'
),
olTextUpdate(
{ id: doc.id, pathname: '_main2.tex' },
this.userId,
this.timestamp,
[3, 'foo', 6],
2
),
olRenameUpdate(
{ id: doc.id, pathname: '_main2.tex' },
this.userId,
this.timestamp,
'_main2.tex',
'_main__ASTERISK__.tex'
),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slAddDocUpdate(
historyId,
doc,
this.userId,
this.timestamp,
doc.docLines
),
cb
)
},
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slRenameUpdate(
historyId,
doc,
this.userId,
this.timestamp,
'/\\main.tex',
'/\\main2.tex'
),
cb
)
doc.pathname = '\\main2.tex'
},
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(historyId, doc, this.userId, 2, this.timestamp, [
{ p: 3, i: 'foo' },
]),
cb
)
},
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slRenameUpdate(
historyId,
doc,
this.userId,
this.timestamp,
'/\\main2.tex',
'/\\main*.tex'
),
cb
)
doc.pathname = '\\main*.tex'
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createChange.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
it('should workaround pathnames beginning with spaces', function (done) {
const doc = {
id: this.doc.id,
pathname: 'main.tex',
hash: 'b07b6b7a27667965f733943737124395c7577bea',
docLines: 'aaabbbccc',
length: 9,
}
MockHistoryStore()
.put(`/api/projects/${historyId}/blobs/${doc.hash}`)
.reply(201)
const createChange = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olAddDocUpdate(
{ id: doc.id, pathname: 'main.tex' },
this.userId,
this.timestamp,
doc.hash
),
olRenameUpdate(
{ id: doc.id },
this.userId,
this.timestamp,
'main.tex',
'foo/__SPACE__main.tex'
),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slAddDocUpdate(
historyId,
doc,
this.userId,
this.timestamp,
doc.docLines
),
cb
)
},
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slRenameUpdate(
historyId,
doc,
this.userId,
this.timestamp,
'/main.tex',
'/foo/ main.tex'
),
cb
)
doc.pathname = '/foo/ main.tex'
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createChange.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
})
describe('with bad response from filestore', function () {
beforeEach(function () {
MockHistoryStore()
.get(`/api/projects/${historyId}/latest/history`)
.reply(200, {
chunk: {
startVersion: 0,
history: {
snapshot: {},
changes: [],
},
},
})
})
it('should return a 500 if the filestore returns a 500', function (done) {
const file = {
id: new ObjectId().toString(),
pathname: '/test.png',
contents: Buffer.from([1, 2, 3]),
hash: 'aed2973e4b8a7ff1b30ff5c4751e5a2b38989e74',
}
const fileStoreRequest = MockFileStore()
.get(`/project/${this.projectId}/file/${file.id}`)
.reply(500)
const createBlob = MockHistoryStore()
.put(`/api/projects/${historyId}/blobs/${file.hash}`, file.contents)
.reply(201)
const addFile = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olAddFileUpdate(file, this.userId, this.timestamp, file.hash),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slAddFileUpdate(
historyId,
file,
this.userId,
this.timestamp,
this.projectId
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(
this.projectId,
{ allowErrors: true },
(error, res) => {
if (error) {
return cb(error)
}
expect(res.statusCode).to.equal(500)
cb()
}
)
},
],
error => {
if (error) {
return done(error)
}
assert(
fileStoreRequest.isDone(),
`/project/${this.projectId}/file/${file.id} should have been called`
)
assert(
!createBlob.isDone(),
`/api/projects/${historyId}/latest/files should not have been called`
)
assert(
!addFile.isDone(),
`/api/projects/${historyId}/latest/files should not have been called`
)
done()
}
)
})
it('should return a 500 if the filestore request errors', function (done) {
const file = {
id: new ObjectId().toString(),
pathname: '/test.png',
contents: Buffer.from([1, 2, 3]),
hash: 'aed2973e4b8a7ff1b30ff5c4751e5a2b38989e74',
}
const fileStoreRequest = MockFileStore()
.get(`/project/${this.projectId}/file/${file.id}`)
.replyWithError('oh no!')
const createBlob = MockHistoryStore()
.put(`/api/projects/${historyId}/blobs/${file.hash}`, file.contents)
.reply(201)
const addFile = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olAddFileUpdate(file, this.userId, this.timestamp, file.hash),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slAddFileUpdate(
historyId,
file,
this.userId,
this.timestamp,
this.projectId
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(
this.projectId,
{ allowErrors: true },
(error, res) => {
if (error) {
return cb(error)
}
expect(res.statusCode).to.equal(500)
cb()
}
)
},
],
error => {
if (error) {
return done(error)
}
assert(
fileStoreRequest.isDone(),
`/project/${this.projectId}/file/${file.id} should have been called`
)
assert(
!createBlob.isDone(),
`/api/projects/${historyId}/latest/files should not have been called`
)
assert(
!addFile.isDone(),
`/api/projects/${historyId}/latest/files should not have been called`
)
done()
}
)
})
})
describe('with an existing projectVersion field', function () {
beforeEach(function () {
MockHistoryStore()
.get(`/api/projects/${historyId}/latest/history`)
.reply(200, {
chunk: {
startVersion: 0,
history: {
snapshot: { projectVersion: '100.0' },
changes: [],
},
},
})
})
it('should discard project structure updates which have already been applied', function (done) {
const newDoc = []
for (let i = 0; i <= 2; i++) {
newDoc[i] = {
id: new ObjectId().toString(),
pathname: `/main${i}.tex`,
hash: '0a207c060e61f3b88eaee0a8cd0696f46fb155eb',
docLines: 'a\nb',
}
}
MockHistoryStore()
.put(`/api/projects/${historyId}/blobs/${newDoc[0].hash}`)
.times(3)
.reply(201)
const createChange = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olAddDocUpdateWithVersion(
newDoc[1],
this.userId,
this.timestamp,
newDoc[1].hash,
'101.0'
),
olAddDocUpdateWithVersion(
newDoc[2],
this.userId,
this.timestamp,
newDoc[2].hash,
'102.0'
),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slAddDocUpdateWithVersion(
historyId,
newDoc[0],
this.userId,
this.timestamp,
newDoc[0].docLines,
'100.0'
),
cb
)
},
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slAddDocUpdateWithVersion(
historyId,
newDoc[1],
this.userId,
this.timestamp,
newDoc[1].docLines,
'101.0'
),
cb
)
},
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slAddDocUpdateWithVersion(
historyId,
newDoc[2],
this.userId,
this.timestamp,
newDoc[2].docLines,
'102.0'
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createChange.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
})
describe('with an existing docVersions field', function () {
beforeEach(function () {
MockHistoryStore()
.get(`/api/projects/${historyId}/latest/history`)
.reply(200, {
chunk: {
startVersion: 0,
history: {
snapshot: { v2DocVersions: { [this.doc.id]: { v: 100 } } }, // version 100 below already applied
changes: [],
},
},
})
})
it('should discard doc updates which have already been applied', function (done) {
const createChange = MockHistoryStore()
.post(`/api/projects/${historyId}/legacy_changes`, body => {
expect(body).to.deep.equal([
olTextUpdate(
this.doc,
this.userId,
this.timestamp,
[6, 'baz', 2],
101
),
])
return true
})
.query({ end_version: 0 })
.reply(204)
async.series(
[
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(
historyId,
this.doc,
this.userId,
100,
this.timestamp,
[
{ p: 3, i: 'foobar' }, // these ops should be skipped
{ p: 6, d: 'bar' },
]
),
cb
)
this.doc.length += 3
},
cb => {
ProjectHistoryClient.pushRawUpdate(
this.projectId,
slTextUpdate(
historyId,
this.doc,
this.userId,
101,
this.timestamp,
[
{ p: 6, i: 'baz' }, // this op should be applied
]
),
cb
)
},
cb => {
ProjectHistoryClient.flushProject(this.projectId, cb)
},
],
error => {
if (error) {
return done(error)
}
assert(
createChange.isDone(),
`/api/projects/${historyId}/changes should have been called`
)
done()
}
)
})
})
})