/* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from * DS102: Remove unnecessary code created because of implicit returns * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ const sinon = require("sinon"); const chai = require("chai"); chai.should(); const { expect } = chai; const async = require("async"); const Settings = require('settings-sharelatex'); const rclient_history = require("redis-sharelatex").createClient(Settings.redis.history); // note: this is track changes, not project-history const rclient_project_history = require("redis-sharelatex").createClient(Settings.redis.project_history); const rclient_du = require("redis-sharelatex").createClient(Settings.redis.documentupdater); const Keys = Settings.redis.documentupdater.key_schema; const HistoryKeys = Settings.redis.history.key_schema; const ProjectHistoryKeys = Settings.redis.project_history.key_schema; const MockTrackChangesApi = require("./helpers/MockTrackChangesApi"); const MockWebApi = require("./helpers/MockWebApi"); const DocUpdaterClient = require("./helpers/DocUpdaterClient"); const DocUpdaterApp = require("./helpers/DocUpdaterApp"); describe("Applying updates to a doc", function() { before(function(done) { this.lines = ["one", "two", "three"]; this.version = 42; this.update = { doc: this.doc_id, op: [{ i: "one and a half\n", p: 4 }], v: this.version }; this.result = ["one", "one and a half", "two", "three"]; return DocUpdaterApp.ensureRunning(done); }); describe("when the document is not loaded", function() { before(function(done) { [this.project_id, this.doc_id] = Array.from([DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]); sinon.spy(MockWebApi, "getDocument"); this.startTime = Date.now(); MockWebApi.insertDoc(this.project_id, this.doc_id, {lines: this.lines, version: this.version}); DocUpdaterClient.sendUpdate(this.project_id, this.doc_id, this.update, function(error) { if (error != null) { throw error; } return setTimeout(done, 200); }); return null; }); after(() => MockWebApi.getDocument.restore()); it("should load the document from the web API", function() { return MockWebApi.getDocument .calledWith(this.project_id, this.doc_id) .should.equal(true); }); it("should update the doc", function(done) { DocUpdaterClient.getDoc(this.project_id, this.doc_id, (error, res, doc) => { doc.lines.should.deep.equal(this.result); return done(); }); return null; }); it("should push the applied updates to the track changes api", function(done) { rclient_history.lrange(HistoryKeys.uncompressedHistoryOps({doc_id: this.doc_id}), 0, -1, (error, updates) => { if (error != null) { throw error; } JSON.parse(updates[0]).op.should.deep.equal(this.update.op); return rclient_history.sismember(HistoryKeys.docsWithHistoryOps({project_id: this.project_id}), this.doc_id, (error, result) => { if (error != null) { throw error; } result.should.equal(1); return done(); }); }); return null; }); it("should push the applied updates to the project history changes api", function(done) { rclient_project_history.lrange(ProjectHistoryKeys.projectHistoryOps({project_id: this.project_id}), 0, -1, (error, updates) => { if (error != null) { throw error; } JSON.parse(updates[0]).op.should.deep.equal(this.update.op); return done(); }); return null; }); it("should set the first op timestamp", function(done) { rclient_project_history.get(ProjectHistoryKeys.projectHistoryFirstOpTimestamp({project_id: this.project_id}), (error, result) => { if (error != null) { throw error; } result.should.be.within(this.startTime, Date.now()); this.firstOpTimestamp = result; return done(); }); return null; }); return describe("when sending another update", function() { before(function(done) { this.timeout = 10000; this.second_update = Object.create(this.update); this.second_update.v = this.version + 1; DocUpdaterClient.sendUpdate(this.project_id, this.doc_id, this.second_update, function(error) { if (error != null) { throw error; } return setTimeout(done, 200); }); return null; }); return it("should not change the first op timestamp", function(done) { rclient_project_history.get(ProjectHistoryKeys.projectHistoryFirstOpTimestamp({project_id: this.project_id}), (error, result) => { if (error != null) { throw error; } result.should.equal(this.firstOpTimestamp); return done(); }); return null; }); }); }); describe("when the document is loaded", function() { before(function(done) { [this.project_id, this.doc_id] = Array.from([DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]); MockWebApi.insertDoc(this.project_id, this.doc_id, {lines: this.lines, version: this.version}); DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, error => { if (error != null) { throw error; } sinon.spy(MockWebApi, "getDocument"); return DocUpdaterClient.sendUpdate(this.project_id, this.doc_id, this.update, function(error) { if (error != null) { throw error; } return setTimeout(done, 200); }); }); return null; }); after(() => MockWebApi.getDocument.restore()); it("should not need to call the web api", () => MockWebApi.getDocument.called.should.equal(false)); it("should update the doc", function(done) { DocUpdaterClient.getDoc(this.project_id, this.doc_id, (error, res, doc) => { doc.lines.should.deep.equal(this.result); return done(); }); return null; }); it("should push the applied updates to the track changes api", function(done) { rclient_history.lrange(HistoryKeys.uncompressedHistoryOps({doc_id: this.doc_id}), 0, -1, (error, updates) => { JSON.parse(updates[0]).op.should.deep.equal(this.update.op); return rclient_history.sismember(HistoryKeys.docsWithHistoryOps({project_id: this.project_id}), this.doc_id, (error, result) => { result.should.equal(1); return done(); }); }); return null; }); return it("should push the applied updates to the project history changes api", function(done) { rclient_project_history.lrange(ProjectHistoryKeys.projectHistoryOps({project_id: this.project_id}), 0, -1, (error, updates) => { JSON.parse(updates[0]).op.should.deep.equal(this.update.op); return done(); }); return null; }); }); describe("when the document is loaded and is using project-history only", function() { before(function(done) { [this.project_id, this.doc_id] = Array.from([DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]); MockWebApi.insertDoc(this.project_id, this.doc_id, {lines: this.lines, version: this.version, projectHistoryType: 'project-history'}); DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, error => { if (error != null) { throw error; } sinon.spy(MockWebApi, "getDocument"); return DocUpdaterClient.sendUpdate(this.project_id, this.doc_id, this.update, function(error) { if (error != null) { throw error; } return setTimeout(done, 200); }); }); return null; }); after(() => MockWebApi.getDocument.restore()); it("should update the doc", function(done) { DocUpdaterClient.getDoc(this.project_id, this.doc_id, (error, res, doc) => { doc.lines.should.deep.equal(this.result); return done(); }); return null; }); it("should not push any applied updates to the track changes api", function(done) { rclient_history.lrange(HistoryKeys.uncompressedHistoryOps({doc_id: this.doc_id}), 0, -1, (error, updates) => { updates.length.should.equal(0); return done(); }); return null; }); return it("should push the applied updates to the project history changes api", function(done) { rclient_project_history.lrange(ProjectHistoryKeys.projectHistoryOps({project_id: this.project_id}), 0, -1, (error, updates) => { JSON.parse(updates[0]).op.should.deep.equal(this.update.op); return done(); }); return null; }); }); describe("when the document has been deleted", function() { describe("when the ops come in a single linear order", function() { before(function(done) { [this.project_id, this.doc_id] = Array.from([DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]); const lines = ["", "", ""]; MockWebApi.insertDoc(this.project_id, this.doc_id, {lines, version: 0}); this.updates = [ { doc_id: this.doc_id, v: 0, op: [{i: "h", p: 0} ] }, { doc_id: this.doc_id, v: 1, op: [{i: "e", p: 1} ] }, { doc_id: this.doc_id, v: 2, op: [{i: "l", p: 2} ] }, { doc_id: this.doc_id, v: 3, op: [{i: "l", p: 3} ] }, { doc_id: this.doc_id, v: 4, op: [{i: "o", p: 4} ] }, { doc_id: this.doc_id, v: 5, op: [{i: " ", p: 5} ] }, { doc_id: this.doc_id, v: 6, op: [{i: "w", p: 6} ] }, { doc_id: this.doc_id, v: 7, op: [{i: "o", p: 7} ] }, { doc_id: this.doc_id, v: 8, op: [{i: "r", p: 8} ] }, { doc_id: this.doc_id, v: 9, op: [{i: "l", p: 9} ] }, { doc_id: this.doc_id, v: 10, op: [{i: "d", p: 10}] } ]; this.my_result = ["hello world", "", ""]; return done(); }); it("should be able to continue applying updates when the project has been deleted", function(done) { let update; const actions = []; for (update of Array.from(this.updates.slice(0,6))) { (update => { return actions.push(callback => DocUpdaterClient.sendUpdate(this.project_id, this.doc_id, update, callback)); })(update); } actions.push(callback => DocUpdaterClient.deleteDoc(this.project_id, this.doc_id, callback)); for (update of Array.from(this.updates.slice(6))) { (update => { return actions.push(callback => DocUpdaterClient.sendUpdate(this.project_id, this.doc_id, update, callback)); })(update); } async.series(actions, error => { if (error != null) { throw error; } return DocUpdaterClient.getDoc(this.project_id, this.doc_id, (error, res, doc) => { doc.lines.should.deep.equal(this.my_result); return done(); }); }); return null; }); it("should push the applied updates to the track changes api", function(done) { rclient_history.lrange(HistoryKeys.uncompressedHistoryOps({doc_id: this.doc_id}), 0, -1, (error, updates) => { updates = (Array.from(updates).map((u) => JSON.parse(u))); for (let i = 0; i < this.updates.length; i++) { const appliedUpdate = this.updates[i]; appliedUpdate.op.should.deep.equal(updates[i].op); } return rclient_history.sismember(HistoryKeys.docsWithHistoryOps({project_id: this.project_id}), this.doc_id, (error, result) => { result.should.equal(1); return done(); }); }); return null; }); return it("should store the doc ops in the correct order", function(done) { rclient_du.lrange(Keys.docOps({doc_id: this.doc_id}), 0, -1, (error, updates) => { updates = (Array.from(updates).map((u) => JSON.parse(u))); for (let i = 0; i < this.updates.length; i++) { const appliedUpdate = this.updates[i]; appliedUpdate.op.should.deep.equal(updates[i].op); } return done(); }); return null; }); }); return describe("when older ops come in after the delete", function() { before(function(done) { [this.project_id, this.doc_id] = Array.from([DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]); const lines = ["", "", ""]; MockWebApi.insertDoc(this.project_id, this.doc_id, {lines, version: 0}); this.updates = [ { doc_id: this.doc_id, v: 0, op: [{i: "h", p: 0} ] }, { doc_id: this.doc_id, v: 1, op: [{i: "e", p: 1} ] }, { doc_id: this.doc_id, v: 2, op: [{i: "l", p: 2} ] }, { doc_id: this.doc_id, v: 3, op: [{i: "l", p: 3} ] }, { doc_id: this.doc_id, v: 4, op: [{i: "o", p: 4} ] }, { doc_id: this.doc_id, v: 0, op: [{i: "world", p: 1} ] } ]; this.my_result = ["hello", "world", ""]; return done(); }); return it("should be able to continue applying updates when the project has been deleted", function(done) { let update; const actions = []; for (update of Array.from(this.updates.slice(0,5))) { (update => { return actions.push(callback => DocUpdaterClient.sendUpdate(this.project_id, this.doc_id, update, callback)); })(update); } actions.push(callback => DocUpdaterClient.deleteDoc(this.project_id, this.doc_id, callback)); for (update of Array.from(this.updates.slice(5))) { (update => { return actions.push(callback => DocUpdaterClient.sendUpdate(this.project_id, this.doc_id, update, callback)); })(update); } async.series(actions, error => { if (error != null) { throw error; } return DocUpdaterClient.getDoc(this.project_id, this.doc_id, (error, res, doc) => { doc.lines.should.deep.equal(this.my_result); return done(); }); }); return null; }); }); }); describe("with a broken update", function() { before(function(done) { [this.project_id, this.doc_id] = Array.from([DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]); this.broken_update = { doc_id: this.doc_id, v: this.version, op: [{d: "not the correct content", p: 0} ] }; MockWebApi.insertDoc(this.project_id, this.doc_id, {lines: this.lines, version: this.version}); DocUpdaterClient.subscribeToAppliedOps(this.messageCallback = sinon.stub()); DocUpdaterClient.sendUpdate(this.project_id, this.doc_id, this.broken_update, function(error) { if (error != null) { throw error; } return setTimeout(done, 200); }); return null; }); it("should not update the doc", function(done) { DocUpdaterClient.getDoc(this.project_id, this.doc_id, (error, res, doc) => { doc.lines.should.deep.equal(this.lines); return done(); }); return null; }); return it("should send a message with an error", function() { this.messageCallback.called.should.equal(true); const [channel, message] = Array.from(this.messageCallback.args[0]); channel.should.equal("applied-ops"); return JSON.parse(message).should.deep.include({ project_id: this.project_id, doc_id: this.doc_id, error:'Delete component does not match' }); }); }); describe("with enough updates to flush to the track changes api", function() { before(function(done) { [this.project_id, this.doc_id] = Array.from([DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]); const updates = []; for (let v = 0; v <= 199; v++) { // Should flush after 100 ops updates.push({ doc_id: this.doc_id, op: [{i: v.toString(), p: 0}], v }); } sinon.spy(MockTrackChangesApi, "flushDoc"); MockWebApi.insertDoc(this.project_id, this.doc_id, {lines: this.lines, version: 0}); // Send updates in chunks to causes multiple flushes const actions = []; for (let i = 0; i <= 19; i++) { (i => { return actions.push(cb => { return DocUpdaterClient.sendUpdates(this.project_id, this.doc_id, updates.slice(i*10, (i+1)*10), cb); }); })(i); } async.series(actions, error => { if (error != null) { throw error; } return setTimeout(done, 2000); }); return null; }); after(() => MockTrackChangesApi.flushDoc.restore()); return it("should flush the doc twice", () => MockTrackChangesApi.flushDoc.calledTwice.should.equal(true)); }); describe("when there is no version in Mongo", function() { before(function(done) { [this.project_id, this.doc_id] = Array.from([DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]); MockWebApi.insertDoc(this.project_id, this.doc_id, { lines: this.lines }); const update = { doc: this.doc_id, op: this.update.op, v: 0 }; DocUpdaterClient.sendUpdate(this.project_id, this.doc_id, update, function(error) { if (error != null) { throw error; } return setTimeout(done, 200); }); return null; }); return it("should update the doc (using version = 0)", function(done) { DocUpdaterClient.getDoc(this.project_id, this.doc_id, (error, res, doc) => { doc.lines.should.deep.equal(this.result); return done(); }); return null; }); }); return describe("when the sending duplicate ops", function() { before(function(done) { [this.project_id, this.doc_id] = Array.from([DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]); MockWebApi.insertDoc(this.project_id, this.doc_id, {lines: this.lines, version: this.version}); DocUpdaterClient.subscribeToAppliedOps(this.messageCallback = sinon.stub()); // One user delete 'one', the next turns it into 'once'. The second becomes a NOP. DocUpdaterClient.sendUpdate(this.project_id, this.doc_id, { doc: this.doc_id, op: [{ i: "one and a half\n", p: 4 }], v: this.version, meta: { source: "ikHceq3yfAdQYzBo4-xZ" } }, error => { if (error != null) { throw error; } return setTimeout(() => { return DocUpdaterClient.sendUpdate(this.project_id, this.doc_id, { doc: this.doc_id, op: [{ i: "one and a half\n", p: 4 }], v: this.version, dupIfSource: ["ikHceq3yfAdQYzBo4-xZ"], meta: { source: "ikHceq3yfAdQYzBo4-xZ" } }, error => { if (error != null) { throw error; } return setTimeout(done, 200); }); } , 200); }); return null; }); it("should update the doc", function(done) { DocUpdaterClient.getDoc(this.project_id, this.doc_id, (error, res, doc) => { doc.lines.should.deep.equal(this.result); return done(); }); return null; }); return it("should return a message about duplicate ops", function() { this.messageCallback.calledTwice.should.equal(true); this.messageCallback.args[0][0].should.equal("applied-ops"); expect(JSON.parse(this.messageCallback.args[0][1]).op.dup).to.be.undefined; this.messageCallback.args[1][0].should.equal("applied-ops"); return expect(JSON.parse(this.messageCallback.args[1][1]).op.dup).to.equal(true); }); }); });