diff --git a/services/document-updater/app/coffee/UpdateManager.coffee b/services/document-updater/app/coffee/UpdateManager.coffee index e821926015..5064725aa0 100644 --- a/services/document-updater/app/coffee/UpdateManager.coffee +++ b/services/document-updater/app/coffee/UpdateManager.coffee @@ -4,6 +4,7 @@ RealTimeRedisManager = require "./RealTimeRedisManager" ShareJsUpdateManager = require "./ShareJsUpdateManager" HistoryManager = require "./HistoryManager" Settings = require('settings-sharelatex') +_ = require("underscore") async = require("async") logger = require('logger-sharelatex') Metrics = require "./Metrics" @@ -69,7 +70,7 @@ module.exports = UpdateManager = profile = new Profiler("applyUpdate", {project_id, doc_id}) UpdateManager._sanitizeUpdate update profile.log("sanitizeUpdate") - DocumentManager.getDoc project_id, doc_id, (error, lines, version, ranges) -> + DocumentManager.getDoc project_id, doc_id, (error, lines, version, ranges, pathname) -> profile.log("getDoc") return callback(error) if error? if !lines? or !version? @@ -78,6 +79,7 @@ module.exports = UpdateManager = profile.log("sharejs.applyUpdate") return callback(error) if error? RangesManager.applyUpdate project_id, doc_id, ranges, appliedOps, updatedDocLines, (error, new_ranges) -> + UpdateManager._addProjectHistoryMetadataToOps(appliedOps, pathname, updatedDocLines) profile.log("RangesManager.applyUpdate") return callback(error) if error? RedisManager.updateDocument project_id, doc_id, updatedDocLines, version, appliedOps, new_ranges, (error, doc_ops_length, project_ops_length) -> @@ -108,15 +110,15 @@ module.exports = UpdateManager = _handleErrorInsideLock: (doc_id, lockValue, original_error, callback = (error) ->) -> LockManager.releaseLock doc_id, lockValue, (lock_error) -> callback(original_error) - + _sanitizeUpdate: (update) -> # In Javascript, characters are 16-bits wide. It does not understand surrogates as characters. - # + # # From Wikipedia (http://en.wikipedia.org/wiki/Plane_(Unicode)#Basic_Multilingual_Plane): # "The High Surrogates (U+D800–U+DBFF) and Low Surrogate (U+DC00–U+DFFF) codes are reserved # for encoding non-BMP characters in UTF-16 by using a pair of 16-bit codes: one High Surrogate # and one Low Surrogate. A single surrogate code point will never be assigned a character."" - # + # # The main offender seems to be \uD835 as a stand alone character, which would be the first # 16-bit character of a blackboard bold character (http://www.fileformat.info/info/unicode/char/1d400/index.htm). # Something must be going on client side that is screwing up the encoding and splitting the @@ -127,3 +129,12 @@ module.exports = UpdateManager = op.i = op.i.replace(/[\uD800-\uDFFF]/g, "\uFFFD") return update + _addProjectHistoryMetadataToOps: (ops, pathname, lines) -> + doc_length = _.reduce lines, + (chars, line) -> chars + line.length, + 0 + doc_length += lines.length - 1 # count newline characters + ops.forEach (op) -> + op.meta ||= {} + op.meta.pathname = pathname + op.meta.doc_length = doc_length diff --git a/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee b/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee index b68698bc49..946f5d9fbe 100644 --- a/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee @@ -163,14 +163,16 @@ describe "UpdateManager", -> @lines = ["original", "lines"] @ranges = { entries: "mock", comments: "mock" } @updated_ranges = { entries: "updated", comments: "updated" } - @appliedOps = ["mock-applied-ops"] + @appliedOps = [ {v: 42, op: "mock-op-42"}, { v: 45, op: "mock-op-45" }] @doc_ops_length = sinon.stub() @project_ops_length = sinon.stub() - @DocumentManager.getDoc = sinon.stub().yields(null, @lines, @version, @ranges) + @pathname = '/a/b/c.tex' + @DocumentManager.getDoc = sinon.stub().yields(null, @lines, @version, @ranges, @pathname) @RangesManager.applyUpdate = sinon.stub().yields(null, @updated_ranges) @ShareJsUpdateManager.applyUpdate = sinon.stub().yields(null, @updatedDocLines, @version, @appliedOps) @RedisManager.updateDocument = sinon.stub().yields(null, @doc_ops_length, @project_ops_length) @RealTimeRedisManager.sendData = sinon.stub() + @UpdateManager._addProjectHistoryMetadataToOps = sinon.stub() @HistoryManager.recordAndFlushHistoryOps = sinon.stub().callsArg(5) describe "normally", -> @@ -192,6 +194,11 @@ describe "UpdateManager", -> .calledWith(@project_id, @doc_id, @updatedDocLines, @version, @appliedOps, @updated_ranges) .should.equal true + it "shoould add metadata to the ops" , -> + @UpdateManager._addProjectHistoryMetadataToOps + .calledWith(@appliedOps, @pathname, @updatedDocLines) + .should.equal true + it "should push the applied ops into the history queue", -> @HistoryManager.recordAndFlushHistoryOps .calledWith(@project_id, @doc_id, @appliedOps, @doc_ops_length, @project_ops_length) @@ -231,6 +238,28 @@ describe "UpdateManager", -> it "should call the callback with the error", -> @callback.calledWith(@error).should.equal true + describe "_addProjectHistoryMetadataToOps", -> + it "should add pathname and doc_length metadata to the ops", -> + lines = [ + 'some' + 'test' + 'data' + ] + appliedOps = [ {v: 42, op: "mock-op-42"}, { v: 45, op: "mock-op-45" }] + @UpdateManager._addProjectHistoryMetadataToOps(appliedOps, @pathname, lines) + appliedOps.should.deep.equal [{ + v: 42 + op: "mock-op-42" + meta: + pathname: @pathname + doc_length: 14 + }, { + v: 45 + op: "mock-op-45" + meta: + pathname: @pathname + doc_length: 14 + }] describe "lockUpdatesAndDo", -> beforeEach ->