overleaf/services/document-updater/test/acceptance/js/ApplyingUpdatesToADocTests.js
2021-07-13 12:04:42 +01:00

847 lines
24 KiB
JavaScript

/* eslint-disable
camelcase,
handle-callback-err,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const sinon = require('sinon')
const { expect } = require('chai')
const async = require('async')
const Settings = require('@overleaf/settings')
const rclient_history = require('@overleaf/redis-wrapper').createClient(
Settings.redis.history
) // note: this is track changes, not project-history
const rclient_project_history = require('@overleaf/redis-wrapper').createClient(
Settings.redis.project_history
)
const rclient_du = require('@overleaf/redis-wrapper').createClient(
Settings.redis.documentupdater
)
const Keys = Settings.redis.documentupdater.key_schema
const HistoryKeys = Settings.redis.history.key_schema
const ProjectHistoryKeys = Settings.redis.project_history.key_schema
const MockTrackChangesApi = require('./helpers/MockTrackChangesApi')
const MockWebApi = require('./helpers/MockWebApi')
const DocUpdaterClient = require('./helpers/DocUpdaterClient')
const DocUpdaterApp = require('./helpers/DocUpdaterApp')
describe('Applying updates to a doc', function () {
before(function (done) {
this.lines = ['one', 'two', 'three']
this.version = 42
this.update = {
doc: this.doc_id,
op: [
{
i: 'one and a half\n',
p: 4,
},
],
v: this.version,
}
this.result = ['one', 'one and a half', 'two', 'three']
return DocUpdaterApp.ensureRunning(done)
})
describe('when the document is not loaded', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
sinon.spy(MockWebApi, 'getDocument')
this.startTime = Date.now()
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: this.version,
})
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.update,
error => {
if (error != null) {
throw error
}
return setTimeout(done, 200)
}
)
return null
})
after(function () {
return MockWebApi.getDocument.restore()
})
it('should load the document from the web API', function () {
return MockWebApi.getDocument
.calledWith(this.project_id, this.doc_id)
.should.equal(true)
})
it('should update the doc', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
doc.lines.should.deep.equal(this.result)
return done()
}
)
return null
})
it('should push the applied updates to the track changes api', function (done) {
rclient_history.lrange(
HistoryKeys.uncompressedHistoryOps({ doc_id: this.doc_id }),
0,
-1,
(error, updates) => {
if (error != null) {
throw error
}
JSON.parse(updates[0]).op.should.deep.equal(this.update.op)
return rclient_history.sismember(
HistoryKeys.docsWithHistoryOps({ project_id: this.project_id }),
this.doc_id,
(error, result) => {
if (error != null) {
throw error
}
result.should.equal(1)
return done()
}
)
}
)
return null
})
it('should push the applied updates to the project history changes api', function (done) {
rclient_project_history.lrange(
ProjectHistoryKeys.projectHistoryOps({ project_id: this.project_id }),
0,
-1,
(error, updates) => {
if (error != null) {
throw error
}
JSON.parse(updates[0]).op.should.deep.equal(this.update.op)
return done()
}
)
return null
})
it('should set the first op timestamp', function (done) {
rclient_project_history.get(
ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
project_id: this.project_id,
}),
(error, result) => {
if (error != null) {
throw error
}
result.should.be.within(this.startTime, Date.now())
this.firstOpTimestamp = result
return done()
}
)
return null
})
return describe('when sending another update', function () {
before(function (done) {
this.timeout = 10000
this.second_update = Object.create(this.update)
this.second_update.v = this.version + 1
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.second_update,
error => {
if (error != null) {
throw error
}
return setTimeout(done, 200)
}
)
return null
})
return it('should not change the first op timestamp', function (done) {
rclient_project_history.get(
ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
project_id: this.project_id,
}),
(error, result) => {
if (error != null) {
throw error
}
result.should.equal(this.firstOpTimestamp)
return done()
}
)
return null
})
})
})
describe('when the document is loaded', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: this.version,
})
DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, error => {
if (error != null) {
throw error
}
sinon.spy(MockWebApi, 'getDocument')
return DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.update,
error => {
if (error != null) {
throw error
}
return setTimeout(done, 200)
}
)
})
return null
})
after(function () {
return MockWebApi.getDocument.restore()
})
it('should not need to call the web api', function () {
return MockWebApi.getDocument.called.should.equal(false)
})
it('should update the doc', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
doc.lines.should.deep.equal(this.result)
return done()
}
)
return null
})
it('should push the applied updates to the track changes api', function (done) {
rclient_history.lrange(
HistoryKeys.uncompressedHistoryOps({ doc_id: this.doc_id }),
0,
-1,
(error, updates) => {
JSON.parse(updates[0]).op.should.deep.equal(this.update.op)
return rclient_history.sismember(
HistoryKeys.docsWithHistoryOps({ project_id: this.project_id }),
this.doc_id,
(error, result) => {
result.should.equal(1)
return done()
}
)
}
)
return null
})
return it('should push the applied updates to the project history changes api', function (done) {
rclient_project_history.lrange(
ProjectHistoryKeys.projectHistoryOps({ project_id: this.project_id }),
0,
-1,
(error, updates) => {
JSON.parse(updates[0]).op.should.deep.equal(this.update.op)
return done()
}
)
return null
})
})
describe('when the document is loaded and is using project-history only', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: this.version,
projectHistoryType: 'project-history',
})
DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, error => {
if (error != null) {
throw error
}
sinon.spy(MockWebApi, 'getDocument')
return DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.update,
error => {
if (error != null) {
throw error
}
return setTimeout(done, 200)
}
)
})
return null
})
after(function () {
return MockWebApi.getDocument.restore()
})
it('should update the doc', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
doc.lines.should.deep.equal(this.result)
return done()
}
)
return null
})
it('should not push any applied updates to the track changes api', function (done) {
rclient_history.lrange(
HistoryKeys.uncompressedHistoryOps({ doc_id: this.doc_id }),
0,
-1,
(error, updates) => {
updates.length.should.equal(0)
return done()
}
)
return null
})
return it('should push the applied updates to the project history changes api', function (done) {
rclient_project_history.lrange(
ProjectHistoryKeys.projectHistoryOps({ project_id: this.project_id }),
0,
-1,
(error, updates) => {
JSON.parse(updates[0]).op.should.deep.equal(this.update.op)
return done()
}
)
return null
})
})
describe('when the document has been deleted', function () {
describe('when the ops come in a single linear order', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
const lines = ['', '', '']
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines,
version: 0,
})
this.updates = [
{ doc_id: this.doc_id, v: 0, op: [{ i: 'h', p: 0 }] },
{ doc_id: this.doc_id, v: 1, op: [{ i: 'e', p: 1 }] },
{ doc_id: this.doc_id, v: 2, op: [{ i: 'l', p: 2 }] },
{ doc_id: this.doc_id, v: 3, op: [{ i: 'l', p: 3 }] },
{ doc_id: this.doc_id, v: 4, op: [{ i: 'o', p: 4 }] },
{ doc_id: this.doc_id, v: 5, op: [{ i: ' ', p: 5 }] },
{ doc_id: this.doc_id, v: 6, op: [{ i: 'w', p: 6 }] },
{ doc_id: this.doc_id, v: 7, op: [{ i: 'o', p: 7 }] },
{ doc_id: this.doc_id, v: 8, op: [{ i: 'r', p: 8 }] },
{ doc_id: this.doc_id, v: 9, op: [{ i: 'l', p: 9 }] },
{ doc_id: this.doc_id, v: 10, op: [{ i: 'd', p: 10 }] },
]
this.my_result = ['hello world', '', '']
return done()
})
it('should be able to continue applying updates when the project has been deleted', function (done) {
let update
const actions = []
for (update of Array.from(this.updates.slice(0, 6))) {
;(update => {
return actions.push(callback =>
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
update,
callback
)
)
})(update)
}
actions.push(callback =>
DocUpdaterClient.deleteDoc(this.project_id, this.doc_id, callback)
)
for (update of Array.from(this.updates.slice(6))) {
;(update => {
return actions.push(callback =>
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
update,
callback
)
)
})(update)
}
async.series(actions, error => {
if (error != null) {
throw error
}
return DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
doc.lines.should.deep.equal(this.my_result)
return done()
}
)
})
return null
})
it('should push the applied updates to the track changes api', function (done) {
rclient_history.lrange(
HistoryKeys.uncompressedHistoryOps({ doc_id: this.doc_id }),
0,
-1,
(error, updates) => {
updates = Array.from(updates).map(u => JSON.parse(u))
for (let i = 0; i < this.updates.length; i++) {
const appliedUpdate = this.updates[i]
appliedUpdate.op.should.deep.equal(updates[i].op)
}
return rclient_history.sismember(
HistoryKeys.docsWithHistoryOps({ project_id: this.project_id }),
this.doc_id,
(error, result) => {
result.should.equal(1)
return done()
}
)
}
)
return null
})
return it('should store the doc ops in the correct order', function (done) {
rclient_du.lrange(
Keys.docOps({ doc_id: this.doc_id }),
0,
-1,
(error, updates) => {
updates = Array.from(updates).map(u => JSON.parse(u))
for (let i = 0; i < this.updates.length; i++) {
const appliedUpdate = this.updates[i]
appliedUpdate.op.should.deep.equal(updates[i].op)
}
return done()
}
)
return null
})
})
return describe('when older ops come in after the delete', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
const lines = ['', '', '']
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines,
version: 0,
})
this.updates = [
{ doc_id: this.doc_id, v: 0, op: [{ i: 'h', p: 0 }] },
{ doc_id: this.doc_id, v: 1, op: [{ i: 'e', p: 1 }] },
{ doc_id: this.doc_id, v: 2, op: [{ i: 'l', p: 2 }] },
{ doc_id: this.doc_id, v: 3, op: [{ i: 'l', p: 3 }] },
{ doc_id: this.doc_id, v: 4, op: [{ i: 'o', p: 4 }] },
{ doc_id: this.doc_id, v: 0, op: [{ i: 'world', p: 1 }] },
]
this.my_result = ['hello', 'world', '']
return done()
})
return it('should be able to continue applying updates when the project has been deleted', function (done) {
let update
const actions = []
for (update of Array.from(this.updates.slice(0, 5))) {
;(update => {
return actions.push(callback =>
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
update,
callback
)
)
})(update)
}
actions.push(callback =>
DocUpdaterClient.deleteDoc(this.project_id, this.doc_id, callback)
)
for (update of Array.from(this.updates.slice(5))) {
;(update => {
return actions.push(callback =>
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
update,
callback
)
)
})(update)
}
async.series(actions, error => {
if (error != null) {
throw error
}
return DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
doc.lines.should.deep.equal(this.my_result)
return done()
}
)
})
return null
})
})
})
describe('with a broken update', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
this.broken_update = {
doc_id: this.doc_id,
v: this.version,
op: [{ d: 'not the correct content', p: 0 }],
}
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: this.version,
})
DocUpdaterClient.subscribeToAppliedOps(
(this.messageCallback = sinon.stub())
)
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.broken_update,
error => {
if (error != null) {
throw error
}
return setTimeout(done, 200)
}
)
return null
})
it('should not update the doc', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
doc.lines.should.deep.equal(this.lines)
return done()
}
)
return null
})
return it('should send a message with an error', function () {
this.messageCallback.called.should.equal(true)
const [channel, message] = Array.from(this.messageCallback.args[0])
channel.should.equal('applied-ops')
return JSON.parse(message).should.deep.include({
project_id: this.project_id,
doc_id: this.doc_id,
error: 'Delete component does not match',
})
})
})
describe('with enough updates to flush to the track changes api', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
const updates = []
for (let v = 0; v <= 199; v++) {
// Should flush after 100 ops
updates.push({
doc_id: this.doc_id,
op: [{ i: v.toString(), p: 0 }],
v,
})
}
sinon.spy(MockTrackChangesApi, 'flushDoc')
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: 0,
})
// Send updates in chunks to causes multiple flushes
const actions = []
for (let i = 0; i <= 19; i++) {
;(i => {
return actions.push(cb => {
return DocUpdaterClient.sendUpdates(
this.project_id,
this.doc_id,
updates.slice(i * 10, (i + 1) * 10),
cb
)
})
})(i)
}
async.series(actions, error => {
if (error != null) {
throw error
}
return setTimeout(done, 2000)
})
return null
})
after(function () {
return MockTrackChangesApi.flushDoc.restore()
})
return it('should flush the doc twice', function () {
return MockTrackChangesApi.flushDoc.calledTwice.should.equal(true)
})
})
describe('when there is no version in Mongo', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
})
const update = {
doc: this.doc_id,
op: this.update.op,
v: 0,
}
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
update,
error => {
if (error != null) {
throw error
}
return setTimeout(done, 200)
}
)
return null
})
return it('should update the doc (using version = 0)', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
doc.lines.should.deep.equal(this.result)
return done()
}
)
return null
})
})
describe('when the sending duplicate ops', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: this.version,
})
DocUpdaterClient.subscribeToAppliedOps(
(this.messageCallback = sinon.stub())
)
// One user delete 'one', the next turns it into 'once'. The second becomes a NOP.
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
{
doc: this.doc_id,
op: [
{
i: 'one and a half\n',
p: 4,
},
],
v: this.version,
meta: {
source: 'ikHceq3yfAdQYzBo4-xZ',
},
},
error => {
if (error != null) {
throw error
}
return setTimeout(() => {
return DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
{
doc: this.doc_id,
op: [
{
i: 'one and a half\n',
p: 4,
},
],
v: this.version,
dupIfSource: ['ikHceq3yfAdQYzBo4-xZ'],
meta: {
source: 'ikHceq3yfAdQYzBo4-xZ',
},
},
error => {
if (error != null) {
throw error
}
return setTimeout(done, 200)
}
)
}, 200)
}
)
return null
})
it('should update the doc', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
doc.lines.should.deep.equal(this.result)
return done()
}
)
return null
})
return it('should return a message about duplicate ops', function () {
this.messageCallback.calledTwice.should.equal(true)
this.messageCallback.args[0][0].should.equal('applied-ops')
expect(JSON.parse(this.messageCallback.args[0][1]).op.dup).to.be.undefined
this.messageCallback.args[1][0].should.equal('applied-ops')
return expect(
JSON.parse(this.messageCallback.args[1][1]).op.dup
).to.equal(true)
})
})
return describe('when sending updates for a non-existing doc id', function () {
before(function (done) {
;[this.project_id, this.doc_id] = Array.from([
DocUpdaterClient.randomId(),
DocUpdaterClient.randomId(),
])
this.non_existing = {
doc_id: this.doc_id,
v: this.version,
op: [{ d: 'content', p: 0 }],
}
DocUpdaterClient.subscribeToAppliedOps(
(this.messageCallback = sinon.stub())
)
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
this.non_existing,
error => {
if (error != null) {
throw error
}
return setTimeout(done, 200)
}
)
return null
})
it('should not update or create a doc', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
this.doc_id,
(error, res, doc) => {
res.statusCode.should.equal(404)
return done()
}
)
return null
})
return it('should send a message with an error', function () {
this.messageCallback.called.should.equal(true)
const [channel, message] = Array.from(this.messageCallback.args[0])
channel.should.equal('applied-ops')
return JSON.parse(message).should.deep.include({
project_id: this.project_id,
doc_id: this.doc_id,
error: `doc not not found: /project/${this.project_id}/doc/${this.doc_id}`,
})
})
})
})