commit e1a7d4f24a969ec4e47b97fbb6fe55c1ef9984d6 Author: James Allen Date: Wed Feb 12 10:40:42 2014 +0000 Initial open sourcing diff --git a/services/document-updater/.gitignore b/services/document-updater/.gitignore new file mode 100644 index 0000000000..5755e37b12 --- /dev/null +++ b/services/document-updater/.gitignore @@ -0,0 +1,46 @@ +compileFolder + +Compiled source # +################### +*.com +*.class +*.dll +*.exe +*.o +*.so + +# Packages # +############ +# it's better to unpack these files and commit the raw source +# git has its own built in compression methods +*.7z +*.dmg +*.gz +*.iso +*.jar +*.rar +*.tar +*.zip + +# Logs and databases # +###################### +*.log +*.sql +*.sqlite + +# OS generated files # +###################### +.DS_Store? +ehthumbs.db +Icon? +Thumbs.db + +/node_modules/* + +app.js +app/js/* + +test/unit/js/* +test/acceptance/js/* + +**.swp diff --git a/services/document-updater/Gruntfile.coffee b/services/document-updater/Gruntfile.coffee new file mode 100644 index 0000000000..30dd63e708 --- /dev/null +++ b/services/document-updater/Gruntfile.coffee @@ -0,0 +1,111 @@ +module.exports = (grunt) -> + grunt.loadNpmTasks 'grunt-contrib-coffee' + grunt.loadNpmTasks 'grunt-contrib-clean' + grunt.loadNpmTasks 'grunt-mocha-test' + grunt.loadNpmTasks 'grunt-available-tasks' + grunt.loadNpmTasks 'grunt-execute' + grunt.loadNpmTasks 'grunt-bunyan' + + grunt.initConfig + execute: + app: + src: "app.js" + + bunyan: + strict: false + + coffee: + app_dir: + expand: true, + flatten: false, + cwd: 'app/coffee', + src: ['**/*.coffee'], + dest: 'app/js/', + ext: '.js' + + app: + src: 'app.coffee' + dest: 'app.js' + + acceptance_tests: + expand: true, + flatten: false, + cwd: 'test/acceptance/coffee', + src: ['**/*.coffee'], + dest: 'test/acceptance/js/', + ext: '.js' + + unit_tests: + expand: true, + flatten: false, + cwd: 'test/unit/coffee', + src: ['**/*.coffee'], + dest: 'test/unit/js/', + ext: '.js' + + clean: + app: ["app/js"] + acceptance_tests: ["test/unit/js"] + + mochaTest: + unit: + src: ['test/unit/js/**/*.js'] + options: + reporter: grunt.option('reporter') or 'spec' + grep: grunt.option("grep") + acceptance: + src: ['test/acceptance/js/**/*.js'] + options: + reporter: grunt.option('reporter') or 'spec' + grep: grunt.option("grep") + timeout: 10000 + + availabletasks: + tasks: + options: + filter: 'exclude', + tasks: [ + 'coffee' + 'clean' + 'mochaTest' + 'availabletasks' + 'execute' + 'bunyan' + ] + groups: + "Compile tasks": [ + "compile:server" + "compile:tests" + "compile" + "compile:unit_tests" + "compile:acceptance_tests" + "install" + ] + "Test tasks": [ + "test:unit" + "test:acceptance" + ] + "Run tasks": [ + "run" + "default" + ] + "Misc": [ + "help" + ] + + grunt.registerTask 'help', 'Display this help list', 'availabletasks' + + grunt.registerTask 'compile:server', 'Compile the server side coffee script', ['clean:app', 'coffee:app', 'coffee:app_dir'] + grunt.registerTask 'compile:unit_tests', 'Compile the unit tests', ['coffee:unit_tests'] + grunt.registerTask 'compile:acceptance_tests', 'Compile the acceptance tests', ['clean:acceptance_tests', 'coffee:acceptance_tests'] + grunt.registerTask 'compile:tests', 'Compile all the tests', ['compile:acceptance_tests', 'compile:unit_tests'] + grunt.registerTask 'compile', 'Compiles everything need to run document-updater-sharelatex', ['compile:server'] + + grunt.registerTask 'install', "Compile everything when installing as an npm module", ['compile'] + + grunt.registerTask 'test:unit', 'Run the unit tests (use --grep= for individual tests)', ['compile:unit_tests', 'mochaTest:unit'] + grunt.registerTask 'test:acceptance', 'Run the acceptance tests (use --grep= for individual tests)', ['compile:acceptance_tests', 'mochaTest:acceptance'] + + grunt.registerTask 'run', "Compile and run the document-updater-sharelatex server", ['compile', 'bunyan', 'execute'] + grunt.registerTask 'default', 'run' + diff --git a/services/document-updater/app.coffee b/services/document-updater/app.coffee new file mode 100644 index 0000000000..f35b50fe30 --- /dev/null +++ b/services/document-updater/app.coffee @@ -0,0 +1,68 @@ +express = require('express') +http = require("http") +Settings = require('settings-sharelatex') +logger = require('logger-sharelatex') +logger.initialize("documentupdater") +RedisManager = require('./app/js/RedisManager.js') +UpdateManager = require('./app/js/UpdateManager.js') +Keys = require('./app/js/RedisKeyBuilder') +redis = require('redis') +rclient = redis.createClient(Settings.redis.port, Settings.redis.host) +rclient.auth(Settings.redis.password) +metrics = require('./app/js/Metrics') +Errors = require "./app/js/Errors" +HttpController = require "./app/js/HttpController" + +app = express() +app.configure -> + app.use(express.logger(':remote-addr - [:date] - :user-agent ":method :url" :status - :response-time ms')); + app.use express.bodyParser() + app.use app.router + +app.configure 'development', ()-> + console.log "Development Enviroment" + app.use express.errorHandler({ dumpExceptions: true, showStack: true }) + +app.configure 'production', ()-> + console.log "Production Enviroment" + app.use express.logger() + app.use express.errorHandler() + +rclient.subscribe("pending-updates") +rclient.on "message", (channel, doc_key)-> + [project_id, doc_id] = Keys.splitProjectIdAndDocId(doc_key) + UpdateManager.processOutstandingUpdatesWithLock project_id, doc_id, (error) -> + logger.error err: error, project_id: project_id, doc_id: doc_id, "error processing update" if error? + +UpdateManager.resumeProcessing() + +app.use (req, res, next)-> + metrics.inc "http-request" + next() + +app.get '/project/:project_id/doc/:doc_id', HttpController.getDoc +app.post '/project/:project_id/doc/:doc_id', HttpController.setDoc +app.post '/project/:project_id/doc/:doc_id/flush', HttpController.flushDocIfLoaded +app.delete '/project/:project_id/doc/:doc_id', HttpController.flushAndDeleteDoc +app.delete '/project/:project_id', HttpController.deleteProject +app.post '/project/:project_id/flush', HttpController.flushProject + +app.get '/total', (req, res)-> + timer = new metrics.Timer("http.allDocList") + RedisManager.getCountOfDocsInMemory (err, count)-> + timer.done() + res.send {total:count} + +app.get '/status', (req, res)-> + res.send('document updater is alive') + +app.use (error, req, res, next) -> + logger.error err: error, "request errored" + if error instanceof Errors.NotFoundError + res.send 404 + else + res.send(500, "Oops, something went wrong") + +port = Settings.internal?.documentupdater?.port or Settings.apis?.documentupdater?.port or 3003 +app.listen port, "localhost", -> + logger.log("documentupdater-sharelatex server listening on port #{port}") diff --git a/services/document-updater/app/DocumentUpdater.js b/services/document-updater/app/DocumentUpdater.js new file mode 100644 index 0000000000..11e69368dd --- /dev/null +++ b/services/document-updater/app/DocumentUpdater.js @@ -0,0 +1,181 @@ +(function(exports){ + Ace = require('aceserverside-sharelatex') + Range = Ace.Range + + //look at applyDeltas method + exports.applyChange = function(aceDoc, change, callback) { + var r = change.range; + var range = new Range(r.start.row, r.start.column, r.end.row, r.end.column); + if('insertText'==change.action){ + aceDoc.insert(change.range.start, change.text); + }else if('insertLines'==change.action){ + aceDoc.insertLines(change.range.start.row, change.lines); + }else if('removeText'==change.action){ + aceDoc.remove(range); + }else if('removeLines'==change.action){ + aceDoc.removeLines(range.start.row, range.end.row-1); + } + + if(typeof callback === 'function'){ + callback(null, aceDoc); + }; + } + +})(typeof exports === 'undefined'? this['documentUpdater']={}: exports); + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/services/document-updater/app/coffee/DiffCodec.coffee b/services/document-updater/app/coffee/DiffCodec.coffee new file mode 100644 index 0000000000..ba5966648e --- /dev/null +++ b/services/document-updater/app/coffee/DiffCodec.coffee @@ -0,0 +1,31 @@ +diff_match_patch = require("../lib/diff_match_patch").diff_match_patch +dmp = new diff_match_patch() + +module.exports = DiffCodec = + ADDED: 1 + REMOVED: -1 + UNCHANGED: 0 + + diffAsShareJsOp: (before, after, callback = (error, ops) ->) -> + diffs = dmp.diff_main(before.join("\n"), after.join("\n")) + dmp.diff_cleanupSemantic(diffs) + + ops = [] + position = 0 + for diff in diffs + type = diff[0] + content = diff[1] + if type == @ADDED + ops.push + i: content + p: position + position += content.length + else if type == @REMOVED + ops.push + d: content + p: position + else if type == @UNCHANGED + position += content.length + else + throw "Unknown type" + callback null, ops diff --git a/services/document-updater/app/coffee/DocOpsManager.coffee b/services/document-updater/app/coffee/DocOpsManager.coffee new file mode 100644 index 0000000000..0e90f5b462 --- /dev/null +++ b/services/document-updater/app/coffee/DocOpsManager.coffee @@ -0,0 +1,127 @@ +RedisManager = require "./RedisManager" +mongojs = require("./mongojs") +db = mongojs.db +ObjectId = mongojs.ObjectId +logger = require "logger-sharelatex" +async = require "async" +Metrics = require("./Metrics") + +module.exports = DocOpsManager = + flushDocOpsToMongo: (project_id, doc_id, _callback = (error) ->) -> + timer = new Metrics.Timer("docOpsManager.flushDocOpsToMongo") + callback = (args...) -> + timer.done() + _callback(args...) + + DocOpsManager.getDocVersionInMongo doc_id, (error, mongoVersion) -> + return callback(error) if error? + RedisManager.getDocVersion doc_id, (error, redisVersion) -> + return callback(error) if error? + if !mongoVersion? or !redisVersion? or mongoVersion > redisVersion + logger.error doc_id: doc_id, redisVersion: redisVersion, mongoVersion: mongoVersion, "mongo version is ahead of redis" + return callback(new Error("inconsistent versions")) + + RedisManager.getPreviousDocOps doc_id, mongoVersion, -1, (error, ops) -> + return callback(error) if error? + if ops.length != redisVersion - mongoVersion + logger.error doc_id: doc_id, redisVersion: redisVersion, mongoVersion: mongoVersion, opsLength: ops.length, "version difference does not match ops length" + return callback(new Error("inconsistent versions")) + logger.log doc_id: doc_id, redisVersion: redisVersion, mongoVersion: mongoVersion, "flushing doc ops to mongo" + DocOpsManager._appendDocOpsInMongo doc_id, ops, redisVersion, (error) -> + return callback(error) if error? + callback null + + getPreviousDocOps: (project_id, doc_id, start, end, _callback = (error, ops) ->) -> + timer = new Metrics.Timer("docOpsManager.getPreviousDocOps") + callback = (args...) -> + timer.done() + _callback(args...) + + DocOpsManager._ensureOpsAreLoaded project_id, doc_id, start, (error) -> + return callback(error) if error? + RedisManager.getPreviousDocOps doc_id, start, end, (error, ops) -> + return callback(error) if error? + callback null, ops + + pushDocOp: (project_id, doc_id, op, callback = (error) ->) -> + RedisManager.pushDocOp doc_id, op, callback + + _ensureOpsAreLoaded: (project_id, doc_id, backToVersion, callback = (error) ->) -> + RedisManager.getDocVersion doc_id, (error, redisVersion) -> + return callback(error) if error? + RedisManager.getDocOpsLength doc_id, (error, opsLength) -> + return callback(error) if error? + oldestVersionInRedis = redisVersion - opsLength + if oldestVersionInRedis > backToVersion + # _getDocOpsFromMongo(, 4, 6, ...) will return the ops in positions 4 and 5, but not 6. + logger.log doc_id: doc_id, backToVersion: backToVersion, oldestVersionInRedis: oldestVersionInRedis, "loading old ops from mongo" + DocOpsManager._getDocOpsFromMongo doc_id, backToVersion, oldestVersionInRedis, (error, ops) -> + logger.log doc_id: doc_id, backToVersion: backToVersion, oldestVersionInRedis: oldestVersionInRedis, ops: ops, "loaded old ops from mongo" + return callback(error) if error? + RedisManager.prependDocOps doc_id, ops, (error) -> + return callback(error) if error? + callback null + else + logger.log doc_id: doc_id, backToVersion: backToVersion, oldestVersionInRedis: oldestVersionInRedis, "ops already in redis" + callback() + + getDocVersionInMongo: (doc_id, callback = (error, version) ->) -> + t = new Metrics.Timer("mongo-time") + db.docOps.find { + doc_id: ObjectId(doc_id) + }, { + version: 1 + }, (error, docs) -> + t.done() + return callback(error) if error? + if docs.length < 1 or !docs[0].version? + return callback null, 0 + else + return callback null, docs[0].version + + APPEND_OPS_BATCH_SIZE: 100 + + _appendDocOpsInMongo: (doc_id, docOps, newVersion, callback = (error) ->) -> + currentVersion = newVersion - docOps.length + batchSize = DocOpsManager.APPEND_OPS_BATCH_SIZE + noOfBatches = Math.ceil(docOps.length / batchSize) + if noOfBatches <= 0 + return callback() + jobs = [] + for batchNo in [0..(noOfBatches-1)] + do (batchNo) -> + jobs.push (callback) -> + batch = docOps.slice(batchNo * batchSize, (batchNo + 1) * batchSize) + currentVersion += batch.length + logger.log doc_id: doc_id, batchNo: batchNo, "appending doc op batch to Mongo" + t = new Metrics.Timer("mongo-time") + db.docOps.update { + doc_id: ObjectId(doc_id) + }, { + $push: docOps: { $each: batch, $slice: -100 } + $set: version: currentVersion + }, { + upsert: true + }, (err)-> + t.done() + callback(err) + + async.series jobs, (error) -> callback(error) + + _getDocOpsFromMongo: (doc_id, start, end, callback = (error, ops) ->) -> + DocOpsManager.getDocVersionInMongo doc_id, (error, version) -> + return callback(error) if error? + offset = - (version - start) # Negative tells mongo to count from the end backwards + limit = end - start + t = new Metrics.Timer("mongo-time") + db.docOps.find { + doc_id: ObjectId(doc_id) + }, { + docOps: $slice: [offset, limit] + }, (error, docs) -> + t.done() + if docs.length < 1 or !docs[0].docOps? + return callback null, [] + else + return callback null, docs[0].docOps + diff --git a/services/document-updater/app/coffee/DocumentManager.coffee b/services/document-updater/app/coffee/DocumentManager.coffee new file mode 100644 index 0000000000..aa64ac3d7f --- /dev/null +++ b/services/document-updater/app/coffee/DocumentManager.coffee @@ -0,0 +1,127 @@ +RedisManager = require "./RedisManager" +PersistenceManager = require "./PersistenceManager" +DocOpsManager = require "./DocOpsManager" +DiffCodec = require "./DiffCodec" +logger = require "logger-sharelatex" +Metrics = require "./Metrics" + +module.exports = DocumentManager = + getDoc: (project_id, doc_id, _callback = (error, lines, version) ->) -> + timer = new Metrics.Timer("docManager.getDoc") + callback = (args...) -> + timer.done() + _callback(args...) + + RedisManager.getDoc doc_id, (error, lines, version) -> + return callback(error) if error? + if !lines? or !version? + logger.log project_id: project_id, doc_id: doc_id, "doc not in redis so getting from persistence API" + PersistenceManager.getDoc project_id, doc_id, (error, lines) -> + return callback(error) if error? + DocOpsManager.getDocVersionInMongo doc_id, (error, version) -> + return callback(error) if error? + logger.log project_id: project_id, doc_id: doc_id, lines: lines, version: version, "got doc from persistence API" + RedisManager.putDocInMemory project_id, doc_id, lines, version, (error) -> + return callback(error) if error? + callback null, lines, version + else + callback null, lines, version + + getDocAndRecentOps: (project_id, doc_id, fromVersion, _callback = (error, lines, version, recentOps) ->) -> + timer = new Metrics.Timer("docManager.getDocAndRecentOps") + callback = (args...) -> + timer.done() + _callback(args...) + + DocumentManager.getDoc project_id, doc_id, (error, lines, version) -> + return callback(error) if error? + if fromVersion == -1 + callback null, lines, version, [] + else + DocOpsManager.getPreviousDocOps project_id, doc_id, fromVersion, version, (error, ops) -> + return callback(error) if error? + callback null, lines, version, ops + + setDoc: (project_id, doc_id, newLines, _callback = (error) ->) -> + timer = new Metrics.Timer("docManager.setDoc") + callback = (args...) -> + timer.done() + _callback(args...) + + if !newLines? + return callback(new Error("No lines were provided to setDoc")) + + UpdateManager = require "./UpdateManager" + DocumentManager.getDoc project_id, doc_id, (error, oldLines, version) -> + return callback(error) if error? + + if oldLines? and oldLines.length > 0 and oldLines[0].text? + logger.log doc_id: doc_id, project_id: project_id, oldLines: oldLines, newLines: newLines, "document is JSON so not updating" + return callback(null) + + logger.log doc_id: doc_id, project_id: project_id, oldLines: oldLines, newLines: newLines, "setting a document via http" + DiffCodec.diffAsShareJsOp oldLines, newLines, (error, op) -> + return callback(error) if error? + update = + doc: doc_id + op: op + v: version + meta: + type: "external" + UpdateManager.applyUpdates project_id, doc_id, [update], (error) -> + return callback(error) if error? + DocumentManager.flushDocIfLoaded project_id, doc_id, (error) -> + return callback(error) if error? + callback null + + + flushDocIfLoaded: (project_id, doc_id, _callback = (error) ->) -> + timer = new Metrics.Timer("docManager.flushDocIfLoaded") + callback = (args...) -> + timer.done() + _callback(args...) + + RedisManager.getDoc doc_id, (error, lines, version) -> + return callback(error) if error? + if !lines? or !version? + logger.log project_id: project_id, doc_id: doc_id, "doc is not loaded so not flushing" + callback null + else + logger.log project_id: project_id, doc_id: doc_id, "flushing doc" + PersistenceManager.setDoc project_id, doc_id, lines, (error) -> + return callback(error) if error? + DocOpsManager.flushDocOpsToMongo project_id, doc_id, (error) -> + return callback(error) if error? + callback null + + flushAndDeleteDoc: (project_id, doc_id, _callback = (error) ->) -> + timer = new Metrics.Timer("docManager.flushAndDeleteDoc") + callback = (args...) -> + timer.done() + _callback(args...) + + DocumentManager.flushDocIfLoaded project_id, doc_id, (error) -> + return callback(error) if error? + RedisManager.removeDocFromMemory project_id, doc_id, (error) -> + return callback(error) if error? + callback null + + getDocWithLock: (project_id, doc_id, callback = (error, lines, version) ->) -> + UpdateManager = require "./UpdateManager" + UpdateManager.lockUpdatesAndDo DocumentManager.getDoc, project_id, doc_id, callback + + getDocAndRecentOpsWithLock: (project_id, doc_id, fromVersion, callback = (error, lines, version) ->) -> + UpdateManager = require "./UpdateManager" + UpdateManager.lockUpdatesAndDo DocumentManager.getDocAndRecentOps, project_id, doc_id, fromVersion, callback + + setDocWithLock: (project_id, doc_id, lines, callback = (error) ->) -> + UpdateManager = require "./UpdateManager" + UpdateManager.lockUpdatesAndDo DocumentManager.setDoc, project_id, doc_id, lines, callback + + flushDocIfLoadedWithLock: (project_id, doc_id, callback = (error) ->) -> + UpdateManager = require "./UpdateManager" + UpdateManager.lockUpdatesAndDo DocumentManager.flushDocIfLoaded, project_id, doc_id, callback + + flushAndDeleteDocWithLock: (project_id, doc_id, callback = (error) ->) -> + UpdateManager = require "./UpdateManager" + UpdateManager.lockUpdatesAndDo DocumentManager.flushAndDeleteDoc, project_id, doc_id, callback diff --git a/services/document-updater/app/coffee/Errors.coffee b/services/document-updater/app/coffee/Errors.coffee new file mode 100644 index 0000000000..4a29822efc --- /dev/null +++ b/services/document-updater/app/coffee/Errors.coffee @@ -0,0 +1,10 @@ +NotFoundError = (message) -> + error = new Error(message) + error.name = "NotFoundError" + error.__proto__ = NotFoundError.prototype + return error +NotFoundError.prototype.__proto__ = Error.prototype + +module.exports = Errors = + NotFoundError: NotFoundError + diff --git a/services/document-updater/app/coffee/HttpController.coffee b/services/document-updater/app/coffee/HttpController.coffee new file mode 100644 index 0000000000..391d02ee37 --- /dev/null +++ b/services/document-updater/app/coffee/HttpController.coffee @@ -0,0 +1,85 @@ +DocumentManager = require "./DocumentManager" +ProjectManager = require "./ProjectManager" +Errors = require "./Errors" +logger = require "logger-sharelatex" +Metrics = require "./Metrics" + +module.exports = HttpController = + getDoc: (req, res, next = (error) ->) -> + doc_id = req.params.doc_id + project_id = req.params.project_id + logger.log project_id: project_id, doc_id: doc_id, "getting doc via http" + timer = new Metrics.Timer("http.getDoc") + + if req.query?.fromVersion? + fromVersion = parseInt(req.query.fromVersion, 10) + else + fromVersion = -1 + + DocumentManager.getDocAndRecentOpsWithLock project_id, doc_id, fromVersion, (error, lines, version, ops) -> + timer.done() + return next(error) if error? + logger.log project_id: project_id, doc_id: doc_id, "got doc via http" + if !lines? or !version? + return next(new Errors.NotFoundError("document not found")) + res.send JSON.stringify + id: doc_id + lines: lines + version: version + ops: ops + + setDoc: (req, res, next = (error) ->) -> + doc_id = req.params.doc_id + project_id = req.params.project_id + lines = req.body.lines + logger.log project_id: project_id, doc_id: doc_id, lines: lines, "setting doc via http" + timer = new Metrics.Timer("http.setDoc") + DocumentManager.setDocWithLock project_id, doc_id, lines, (error) -> + timer.done() + return next(error) if error? + logger.log project_id: project_id, doc_id: doc_id, "set doc via http" + res.send 204 # No Content + + + flushDocIfLoaded: (req, res, next = (error) ->) -> + doc_id = req.params.doc_id + project_id = req.params.project_id + logger.log project_id: project_id, doc_id: doc_id, "flushing doc via http" + timer = new Metrics.Timer("http.flushDoc") + DocumentManager.flushDocIfLoadedWithLock project_id, doc_id, (error) -> + timer.done() + return next(error) if error? + logger.log project_id: project_id, doc_id: doc_id, "flushed doc via http" + res.send 204 # No Content + + flushAndDeleteDoc: (req, res, next = (error) ->) -> + doc_id = req.params.doc_id + project_id = req.params.project_id + logger.log project_id: project_id, doc_id: doc_id, "deleting doc via http" + timer = new Metrics.Timer("http.deleteDoc") + DocumentManager.flushAndDeleteDocWithLock project_id, doc_id, (error) -> + timer.done() + return next(error) if error? + logger.log project_id: project_id, doc_id: doc_id, "deleted doc via http" + res.send 204 # No Content + + flushProject: (req, res, next = (error) ->) -> + project_id = req.params.project_id + logger.log project_id: project_id, "flushing project via http" + timer = new Metrics.Timer("http.flushProject") + ProjectManager.flushProjectWithLocks project_id, (error) -> + timer.done() + return next(error) if error? + logger.log project_id: project_id, "flushed project via http" + res.send 204 # No Content + + deleteProject: (req, res, next = (error) ->) -> + project_id = req.params.project_id + logger.log project_id: project_id, "deleting project via http" + timer = new Metrics.Timer("http.deleteProject") + ProjectManager.flushAndDeleteProjectWithLocks project_id, (error) -> + timer.done() + return next(error) if error? + logger.log project_id: project_id, "deleted project via http" + res.send 204 # No Content + diff --git a/services/document-updater/app/coffee/LockManager.coffee b/services/document-updater/app/coffee/LockManager.coffee new file mode 100644 index 0000000000..9a3d6cf761 --- /dev/null +++ b/services/document-updater/app/coffee/LockManager.coffee @@ -0,0 +1,55 @@ +metrics = require('./Metrics') +Settings = require('settings-sharelatex') +redis = require('redis') +redisConf = Settings.redis?.web or Settings.redis or {host: "localhost", port: 6379} +rclient = redis.createClient(redisConf.port, redisConf.host) +rclient.auth(redisConf.password) +keys = require('./RedisKeyBuilder') +logger = require "logger-sharelatex" + +module.exports = LockManager = + LOCK_TEST_INTERVAL: 50 # 50ms between each test of the lock + MAX_LOCK_WAIT_TIME: 10000 # 10s maximum time to spend trying to get the lock + + tryLock : (doc_id, callback = (err, isFree)->)-> + tenSeconds = 10 + rclient.set keys.blockingKey(doc_id: doc_id), "locked", "EX", 10, "NX", (err, gotLock)-> + return callback(err) if err? + if gotLock == "OK" + metrics.inc "doc-not-blocking" + callback err, true + else + metrics.inc "doc-blocking" + logger.log doc_id: doc_id, redis_response: gotLock, "doc is locked" + callback err, false + + getLock: (doc_id, callback = (error) ->) -> + startTime = Date.now() + do attempt = () -> + if Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME + return callback(new Error("Timeout")) + + LockManager.tryLock doc_id, (error, gotLock) -> + return callback(error) if error? + if gotLock + callback(null) + else + setTimeout attempt, LockManager.LOCK_TEST_INTERVAL + + checkLock: (doc_id, callback = (err, isFree)->)-> + multi = rclient.multi() + multi.exists keys.blockingKey(doc_id:doc_id) + multi.exec (err, replys)-> + return callback(err) if err? + exists = parseInt replys[0] + if exists == 1 + metrics.inc "doc-blocking" + callback err, false + else + metrics.inc "doc-not-blocking" + callback err, true + + releaseLock: (doc_id, callback)-> + rclient.del keys.blockingKey(doc_id:doc_id), callback + + diff --git a/services/document-updater/app/coffee/Metrics.coffee b/services/document-updater/app/coffee/Metrics.coffee new file mode 100644 index 0000000000..0b98550c0e --- /dev/null +++ b/services/document-updater/app/coffee/Metrics.coffee @@ -0,0 +1,23 @@ +StatsD = require('lynx') +statsd = new StatsD('localhost', 8125, {on_error:->}) + +buildKey = (key)-> "doc-updater.#{process.env.NODE_ENV}.#{key}" + +module.exports = + set : (key, value, sampleRate = 1)-> + statsd.set buildKey(key), value, sampleRate + + inc : (key, sampleRate = 1)-> + statsd.increment buildKey(key), sampleRate + + Timer : class + constructor :(key, sampleRate = 1)-> + this.start = new Date() + this.key = buildKey(key) + done:-> + timeSpan = new Date - this.start + statsd.timing(this.key, timeSpan, this.sampleRate) + + gauge : (key, value, sampleRate = 1)-> + statsd.gauge key, value, sampleRate + diff --git a/services/document-updater/app/coffee/PersistenceManager.coffee b/services/document-updater/app/coffee/PersistenceManager.coffee new file mode 100644 index 0000000000..eb1a7366c2 --- /dev/null +++ b/services/document-updater/app/coffee/PersistenceManager.coffee @@ -0,0 +1,66 @@ +request = require "request" +Settings = require "settings-sharelatex" +Errors = require "./Errors" +Metrics = require "./Metrics" + +module.exports = PersistenceManager = + getDoc: (project_id, doc_id, _callback = (error, lines) ->) -> + timer = new Metrics.Timer("persistenceManager.getDoc") + callback = (args...) -> + timer.done() + _callback(args...) + + url = "#{Settings.apis.web.url}/project/#{project_id}/doc/#{doc_id}" + request { + url: url + method: "GET" + headers: + "accept": "application/json" + auth: + user: Settings.apis.web.user + pass: Settings.apis.web.pass + sendImmediately: true + jar: false + }, (error, res, body) -> + return callback(error) if error? + if res.statusCode >= 200 and res.statusCode < 300 + try + body = JSON.parse body + catch e + return callback(e) + return callback null, body.lines + else if res.statusCode == 404 + return callback(new Errors.NotFoundError("doc not not found: #{url}")) + else + return callback(new Error("error accessing web API: #{url} #{res.statusCode}")) + + setDoc: (project_id, doc_id, lines, _callback = (error) ->) -> + timer = new Metrics.Timer("persistenceManager.setDoc") + callback = (args...) -> + timer.done() + _callback(args...) + + url = "#{Settings.apis.web.url}/project/#{project_id}/doc/#{doc_id}" + request { + url: url + method: "POST" + body: JSON.stringify + lines: lines + headers: + "content-type": "application/json" + auth: + user: Settings.apis.web.user + pass: Settings.apis.web.pass + sendImmediately: true + jar: false + }, (error, res, body) -> + return callback(error) if error? + if res.statusCode >= 200 and res.statusCode < 300 + return callback null + else if res.statusCode == 404 + return callback(new Errors.NotFoundError("doc not not found: #{url}")) + else + return callback(new Error("error accessing web API: #{url} #{res.statusCode}")) + + + diff --git a/services/document-updater/app/coffee/ProjectManager.coffee b/services/document-updater/app/coffee/ProjectManager.coffee new file mode 100644 index 0000000000..f0f62b6d1b --- /dev/null +++ b/services/document-updater/app/coffee/ProjectManager.coffee @@ -0,0 +1,60 @@ +RedisManager = require "./RedisManager" +DocumentManager = require "./DocumentManager" +async = require "async" +logger = require "logger-sharelatex" +Metrics = require "./Metrics" + +module.exports = ProjectManager = + flushProjectWithLocks: (project_id, _callback = (error) ->) -> + timer = new Metrics.Timer("projectManager.flushProjectWithLocks") + callback = (args...) -> + timer.done() + _callback(args...) + + RedisManager.getDocIdsInProject project_id, (error, doc_ids) -> + return callback(error) if error? + jobs = [] + errors = [] + for doc_id in (doc_ids or []) + do (doc_id) -> + jobs.push (callback) -> + DocumentManager.flushDocIfLoadedWithLock project_id, doc_id, (error) -> + if error? + logger.error err: error, project_id: project_id, doc_id: doc_id, "error flushing doc" + errors.push(error) + callback() + + logger.log project_id: project_id, doc_ids: doc_ids, "flushing docs" + async.series jobs, () -> + if errors.length > 0 + callback new Error("Errors flushing docs. See log for details") + else + callback(null) + + flushAndDeleteProjectWithLocks: (project_id, _callback = (error) ->) -> + timer = new Metrics.Timer("projectManager.flushAndDeleteProjectWithLocks") + callback = (args...) -> + timer.done() + _callback(args...) + + RedisManager.getDocIdsInProject project_id, (error, doc_ids) -> + return callback(error) if error? + jobs = [] + errors = [] + for doc_id in (doc_ids or []) + do (doc_id) -> + jobs.push (callback) -> + DocumentManager.flushAndDeleteDocWithLock project_id, doc_id, (error) -> + if error? + logger.error err: error, project_id: project_id, doc_id: doc_id, "error deleting doc" + errors.push(error) + callback() + + logger.log project_id: project_id, doc_ids: doc_ids, "deleting docs" + async.series jobs, () -> + if errors.length > 0 + callback new Error("Errors deleting docs. See log for details") + else + callback(null) + + diff --git a/services/document-updater/app/coffee/RedisKeyBuilder.coffee b/services/document-updater/app/coffee/RedisKeyBuilder.coffee new file mode 100644 index 0000000000..a444341ea1 --- /dev/null +++ b/services/document-updater/app/coffee/RedisKeyBuilder.coffee @@ -0,0 +1,28 @@ +ALLDOCSKEY = "AllDocIds" +PROJECTKEY = "ProjectId" +BLOCKINGKEY = "Blocking" +CHANGEQUE = "ChangeQue" +DOCSINPROJECT = "DocsIn" +PENDINGUPDATESKEY = "PendingUpdates" +DOCLINES = "doclines" +DOCOPS = "DocOps" +DOCVERSION = "DocVersion" +DOCIDSWITHPENDINGUPDATES = "DocsWithPendingUpdates" + +module.exports = + + allDocs : ALLDOCSKEY + docLines : (op)-> DOCLINES+":"+op.doc_id + docOps : (op)-> DOCOPS+":"+op.doc_id + docVersion : (op)-> DOCVERSION+":"+op.doc_id + projectKey : (op)-> PROJECTKEY+":"+op.doc_id + blockingKey : (op)-> BLOCKINGKEY+":"+op.doc_id + changeQue : (op)-> CHANGEQUE+":"+op.project_id + docsInProject : (op)-> DOCSINPROJECT+":"+op.project_id + pendingUpdates : (op)-> PENDINGUPDATESKEY+":"+op.doc_id + docsWithPendingUpdates : DOCIDSWITHPENDINGUPDATES + combineProjectIdAndDocId: (project_id, doc_id) -> "#{project_id}:#{doc_id}" + splitProjectIdAndDocId: (project_and_doc_id) -> project_and_doc_id.split(":") + now : (key)-> + d = new Date() + d.getDate()+":"+(d.getMonth()+1)+":"+d.getFullYear()+":"+key diff --git a/services/document-updater/app/coffee/RedisManager.coffee b/services/document-updater/app/coffee/RedisManager.coffee new file mode 100644 index 0000000000..79bb06036d --- /dev/null +++ b/services/document-updater/app/coffee/RedisManager.coffee @@ -0,0 +1,184 @@ +Settings = require('settings-sharelatex') +redis = require('redis') +redisConf = Settings.redis?.web or Settings.redis or {host: "localhost", port: 6379} +rclient = redis.createClient(redisConf.port, redisConf.host) +rclient.auth(redisConf.password) +async = require('async') +_ = require('underscore') +keys = require('./RedisKeyBuilder') +logger = require('logger-sharelatex') +metrics = require('./Metrics') + +module.exports = + putDocInMemory : (project_id, doc_id, docLines, version, callback)-> + timer = new metrics.Timer("redis.put-doc") + logger.log project_id:project_id, doc_id:doc_id, docLines:docLines, version: version, "putting doc in redis" + multi = rclient.multi() + multi.set keys.docLines(doc_id:doc_id), JSON.stringify(docLines) + multi.set keys.projectKey({doc_id:doc_id}), project_id + multi.set keys.docVersion(doc_id:doc_id), version + multi.del keys.docOps(doc_id:doc_id) + multi.sadd keys.allDocs, doc_id + multi.sadd keys.docsInProject(project_id:project_id), doc_id + multi.exec (err, replys)-> + timer.done() + callback(err) + + removeDocFromMemory : (project_id, doc_id, callback)-> + logger.log project_id:project_id, doc_id:doc_id, "removing doc from redis" + multi = rclient.multi() + multi.get keys.docLines(doc_id:doc_id) + multi.del keys.docLines(doc_id:doc_id) + multi.del keys.projectKey(doc_id:doc_id) + multi.del keys.docVersion(doc_id:doc_id) + multi.del keys.docOps(doc_id:doc_id) + multi.srem keys.docsInProject(project_id:project_id), doc_id + multi.srem keys.allDocs, doc_id + multi.exec (err, replys)-> + if err? + logger.err project_id:project_id, doc_id:doc_id, err:err, "error removing doc from redis" + callback(err, null) + else + docLines = replys[0] + logger.log project_id:project_id, doc_id:doc_id, docLines:docLines, "removed doc from redis" + callback() + + getDoc : (doc_id, callback = (error, lines, version) ->)-> + timer = new metrics.Timer("redis.get-doc") + multi = rclient.multi() + linesKey = keys.docLines(doc_id:doc_id) + multi.get linesKey + multi.get keys.docVersion(doc_id:doc_id) + multi.exec (error, result)-> + timer.done() + return callback(error) if error? + try + docLines = JSON.parse result[0] + catch e + return callback(e) + version = parseInt(result[1] or 0, 10) + callback null, docLines, version + + getDocVersion: (doc_id, callback = (error, version) ->) -> + rclient.get keys.docVersion(doc_id: doc_id), (error, version) -> + return callback(error) if error? + version = parseInt(version, 10) + callback null, version + + getCountOfDocsInMemory : (callback)-> + rclient.smembers keys.allDocs, (err, members)-> + len = members.length + callback null, len + + setDocument : (doc_id, docLines, version, callback = (error) ->)-> + multi = rclient.multi() + multi.set keys.docLines(doc_id:doc_id), JSON.stringify(docLines) + multi.set keys.docVersion(doc_id:doc_id), version + multi.incr keys.now("docsets") + multi.exec (error, replys) -> callback(error) + + getPendingUpdatesForDoc : (doc_id, callback)-> + multi = rclient.multi() + multi.lrange keys.pendingUpdates(doc_id:doc_id), 0 , -1 + multi.del keys.pendingUpdates(doc_id:doc_id) + multi.exec (error, replys) -> + jsonUpdates = replys[0] + updates = [] + for jsonUpdate in jsonUpdates + try + update = JSON.parse jsonUpdate + catch e + return callback e + updates.push update + callback error, updates + + getUpdatesLength: (doc_id, callback)-> + rclient.llen keys.pendingUpdates(doc_id:doc_id), callback + + getDocsWithPendingUpdates: (callback = (error, docs) ->) -> + rclient.smembers keys.docsWithPendingUpdates, (error, doc_keys) -> + return callback(error) if error? + docs = doc_keys.map (doc_key) -> + [project_id, doc_id] = keys.splitProjectIdAndDocId(doc_key) + return { + doc_id: doc_id + project_id: project_id + } + callback null, docs + + clearDocFromPendingUpdatesSet: (project_id, doc_id, callback = (error) ->) -> + doc_key = keys.combineProjectIdAndDocId(project_id, doc_id) + rclient.srem keys.docsWithPendingUpdates, doc_key, callback + + getPreviousDocOps: (doc_id, start, end, callback = (error, jsonOps) ->) -> + # TODO: parse the ops and return them as objects, not JSON + rclient.llen keys.docOps(doc_id: doc_id), (error, length) -> + return callback(error) if error? + rclient.get keys.docVersion(doc_id: doc_id), (error, version) -> + return callback(error) if error? + version = parseInt(version, 10) + first_version_in_redis = version - length + + if start < first_version_in_redis or end > version + error = new Error("doc ops range is not loaded in redis") + logger.error err: error, length: length, version: version, start: start, end: end, "inconsistent version or length" + return callback(error) + + start = start - first_version_in_redis + if end > -1 + end = end - first_version_in_redis + + if isNaN(start) or isNaN(end) + error = new Error("inconsistent version or lengths") + logger.error err: error, length: length, version: version, start: start, end: end, "inconsistent version or length" + return callback(error) + + rclient.lrange keys.docOps(doc_id: doc_id), start, end, (error, jsonOps) -> + return callback(error) if error? + try + ops = jsonOps.map (jsonOp) -> JSON.parse jsonOp + catch e + return callback(e) + callback null, ops + + pushDocOp: (doc_id, op, callback = (error, new_version) ->) -> + # TODO: take a raw op object and JSONify it here + jsonOp = JSON.stringify op + rclient.rpush keys.docOps(doc_id: doc_id), jsonOp, (error) -> + return callback(error) if error? + rclient.incr keys.docVersion(doc_id: doc_id), (error, version) -> + return callback(error) if error? + version = parseInt(version, 10) + callback null, version + + prependDocOps: (doc_id, ops, callback = (error) ->) -> + jsonOps = ops.map (op) -> JSON.stringify op + rclient.lpush keys.docOps(doc_id: doc_id), jsonOps.reverse(), callback + + getDocOpsLength: (doc_id, callback = (error, length) ->) -> + rclient.llen keys.docOps(doc_id: doc_id), callback + + getDocIdsInProject: (project_id, callback = (error, doc_ids) ->) -> + rclient.smembers keys.docsInProject(project_id: project_id), callback + + +getDocumentsProjectId = (doc_id, callback)-> + rclient.get keys.projectKey({doc_id:doc_id}), (err, project_id)-> + callback err, {doc_id:doc_id, project_id:project_id} + +getAllProjectDocsIds = (project_id, callback)-> + rclient.SMEMBERS keys.docsInProject(project_id:project_id), (err, doc_ids)-> + if callback? + callback(err, doc_ids) + +getDocumentsAndExpire = (doc_ids, callback)-> + multi = rclient.multi() + oneDay = 86400 + doc_ids.forEach (doc_id)-> + # rclient.expire keys.docLines(doc_id:doc_id), oneDay, -> + doc_ids.forEach (doc_id)-> + multi.get keys.docLines(doc_id:doc_id) + multi.exec (err, docsLines)-> + callback err, docsLines + + diff --git a/services/document-updater/app/coffee/ShareJsDB.coffee b/services/document-updater/app/coffee/ShareJsDB.coffee new file mode 100644 index 0000000000..3704121b6d --- /dev/null +++ b/services/document-updater/app/coffee/ShareJsDB.coffee @@ -0,0 +1,58 @@ +Keys = require('./RedisKeyBuilder') +Settings = require('settings-sharelatex') +DocumentManager = require "./DocumentManager" +RedisManager = require "./RedisManager" +DocOpsManager = require "./DocOpsManager" +Errors = require "./Errors" + +module.exports = ShareJsDB = + getOps: (doc_key, start, end, callback) -> + if start == end + return callback null, [] + + # In redis, lrange values are inclusive. + if end? + end-- + else + end = -1 + + [project_id, doc_id] = Keys.splitProjectIdAndDocId(doc_key) + DocOpsManager.getPreviousDocOps project_id, doc_id, start, end, (error, ops) -> + return callback error if error? + callback null, ops + + writeOp: (doc_key, opData, callback) -> + [project_id, doc_id] = Keys.splitProjectIdAndDocId(doc_key) + DocOpsManager.pushDocOp project_id, doc_id, {op:opData.op, meta:opData.meta}, (error, version) -> + return callback error if error? + + if version == opData.v + 1 + callback() + else + # The document has been corrupted by the change. For now, throw an exception. + # Later, rebuild the snapshot. + callback "Version mismatch in db.append. '#{doc_id}' is corrupted." + + getSnapshot: (doc_key, callback) -> + [project_id, doc_id] = Keys.splitProjectIdAndDocId(doc_key) + DocumentManager.getDoc project_id, doc_id, (error, lines, version) -> + return callback(error) if error? + if !lines? or !version? + return callback(new Errors.NotFoundError("document not found: #{doc_id}")) + + if lines.length > 0 and lines[0].text? + type = "json" + snapshot = lines: lines + else + type = "text" + snapshot = lines.join("\n") + callback null, + snapshot: snapshot + v: parseInt(version, 10) + type: type + + # To be able to remove a doc from the ShareJS memory + # we need to called Model::delete, which calls this + # method on the database. However, we will handle removing + # it from Redis ourselves + delete: (docName, dbMeta, callback) -> callback() diff --git a/services/document-updater/app/coffee/ShareJsUpdateManager.coffee b/services/document-updater/app/coffee/ShareJsUpdateManager.coffee new file mode 100644 index 0000000000..a5b2e88e4f --- /dev/null +++ b/services/document-updater/app/coffee/ShareJsUpdateManager.coffee @@ -0,0 +1,68 @@ +ShareJsModel = require "./sharejs/server/model" +ShareJsDB = require "./ShareJsDB" +async = require "async" +logger = require "logger-sharelatex" +Settings = require('settings-sharelatex') +Keys = require "./RedisKeyBuilder" +{EventEmitter} = require "events" +util = require "util" + +redis = require('redis') +redisConf = Settings.redis?.web or Settings.redis or {host: "localhost", port: 6379} +rclient = redis.createClient(redisConf.port, redisConf.host) +rclient.auth(redisConf.password) + +ShareJsModel:: = {} +util.inherits ShareJsModel, EventEmitter + +module.exports = ShareJsUpdateManager = + getNewShareJsModel: () -> new ShareJsModel(ShareJsDB) + + applyUpdates: (project_id, doc_id, updates, callback = (error, updatedDocLines) ->) -> + logger.log project_id: project_id, doc_id: doc_id, updates: updates, "applying sharejs updates" + jobs = [] + + # We could use a global model for all docs, but we're hitting issues with the + # internal state of ShareJS not being accessible for clearing caches, and + # getting stuck due to queued callbacks (line 260 of sharejs/server/model.coffee) + # This adds a small but hopefully acceptable overhead (~12ms per 1000 updates on + # my 2009 MBP). + model = @getNewShareJsModel() + @_listenForOps(model) + doc_key = Keys.combineProjectIdAndDocId(project_id, doc_id) + for update in updates + do (update) => + jobs.push (callback) => + model.applyOp doc_key, update, callback + + async.series jobs, (error) => + logger.log project_id: project_id, doc_id: doc_id, error: error, "applied updates" + if error? + @_sendError(project_id, doc_id, error) + return callback(error) + model.getSnapshot doc_key, (error, data) => + if error? + @_sendError(project_id, doc_id, error) + return callback(error) + if typeof data.snapshot == "string" + docLines = data.snapshot.split("\n") + else + docLines = data.snapshot.lines + callback(null, docLines, data.v) + + _listenForOps: (model) -> + model.on "applyOp", (doc_key, opData) -> + [project_id, doc_id] = Keys.splitProjectIdAndDocId(doc_key) + data = JSON.stringify + project_id: project_id + doc_id: doc_id + op: opData + rclient.publish "applied-ops", data + + _sendError: (project_id, doc_id, error) -> + data = JSON.stringify + project_id: project_id + doc_id: doc_id + error: error.message || error + rclient.publish "applied-ops", data + diff --git a/services/document-updater/app/coffee/UpdateManager.coffee b/services/document-updater/app/coffee/UpdateManager.coffee new file mode 100644 index 0000000000..a1db456457 --- /dev/null +++ b/services/document-updater/app/coffee/UpdateManager.coffee @@ -0,0 +1,79 @@ +LockManager = require "./LockManager" +RedisManager = require "./RedisManager" +ShareJsUpdateManager = require "./ShareJsUpdateManager" +Settings = require('settings-sharelatex') +async = require("async") +logger = require('logger-sharelatex') +Metrics = require "./Metrics" + +module.exports = UpdateManager = + resumeProcessing: (callback = (error) ->) -> + RedisManager.getDocsWithPendingUpdates (error, docs) => + return callback(error) if error? + jobs = for doc in (docs or []) + do (doc) => + (callback) => @processOutstandingUpdatesWithLock doc.project_id, doc.doc_id, callback + + async.parallelLimit jobs, 5, callback + + processOutstandingUpdates: (project_id, doc_id, _callback = (error) ->) -> + timer = new Metrics.Timer("updateManager.processOutstandingUpdates") + callback = (args...) -> + timer.done() + _callback(args...) + + UpdateManager.fetchAndApplyUpdates project_id, doc_id, (error) => + return callback(error) if error? + RedisManager.clearDocFromPendingUpdatesSet project_id, doc_id, (error) => + return callback(error) if error? + callback() + + processOutstandingUpdatesWithLock: (project_id, doc_id, callback = (error) ->) -> + LockManager.tryLock doc_id, (error, gotLock) => + return callback(error) if error? + return callback() if !gotLock + UpdateManager.processOutstandingUpdates project_id, doc_id, (error) -> + return UpdateManager._handleErrorInsideLock(doc_id, error, callback) if error? + LockManager.releaseLock doc_id, (error) => + return callback(error) if error? + UpdateManager.continueProcessingUpdatesWithLock project_id, doc_id, callback + + continueProcessingUpdatesWithLock: (project_id, doc_id, callback = (error) ->) -> + RedisManager.getUpdatesLength doc_id, (error, length) => + return callback(error) if error? + if length > 0 + UpdateManager.processOutstandingUpdatesWithLock project_id, doc_id, callback + else + callback() + + fetchAndApplyUpdates: (project_id, doc_id, callback = (error) ->) -> + RedisManager.getPendingUpdatesForDoc doc_id, (error, updates) => + return callback(error) if error? + if updates.length == 0 + return callback() + UpdateManager.applyUpdates project_id, doc_id, updates, callback + + applyUpdates: (project_id, doc_id, updates, callback = (error) ->) -> + ShareJsUpdateManager.applyUpdates project_id, doc_id, updates, (error, updatedDocLines, version) -> + return callback(error) if error? + logger.log doc_id: doc_id, version: version, "updating doc via sharejs" + RedisManager.setDocument doc_id, updatedDocLines, version, callback + + lockUpdatesAndDo: (method, project_id, doc_id, args..., callback) -> + LockManager.getLock doc_id, (error) -> + return callback(error) if error? + UpdateManager.processOutstandingUpdates project_id, doc_id, (error) -> + return UpdateManager._handleErrorInsideLock(doc_id, error, callback) if error? + method project_id, doc_id, args..., (error, response_args...) -> + return UpdateManager._handleErrorInsideLock(doc_id, error, callback) if error? + LockManager.releaseLock doc_id, (error) -> + return callback(error) if error? + callback null, response_args... + # We held the lock for a while so updates might have queued up + UpdateManager.continueProcessingUpdatesWithLock project_id, doc_id + + _handleErrorInsideLock: (doc_id, original_error, callback = (error) ->) -> + LockManager.releaseLock doc_id, (lock_error) -> + callback(original_error) + + diff --git a/services/document-updater/app/coffee/mongojs.coffee b/services/document-updater/app/coffee/mongojs.coffee new file mode 100644 index 0000000000..9a1ae72bc0 --- /dev/null +++ b/services/document-updater/app/coffee/mongojs.coffee @@ -0,0 +1,7 @@ +Settings = require "settings-sharelatex" +mongojs = require "mongojs" +db = mongojs.connect(Settings.mongo.url, ["docOps"]) +module.exports = + db: db + ObjectId: mongojs.ObjectId + diff --git a/services/document-updater/app/coffee/sharejs/README.md b/services/document-updater/app/coffee/sharejs/README.md new file mode 100644 index 0000000000..22e68842dd --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/README.md @@ -0,0 +1,48 @@ +This directory contains all the operational transform code. Each file defines a type. + +Most of the types in here are for testing or demonstration. The only types which are sent to the webclient +are `text` and `json`. + + +# An OT type + +All OT types have the following fields: + +`name`: _(string)_ Name of the type. Should match the filename. +`create() -> snapshot`: Function which creates and returns a new document snapshot + +`apply(snapshot, op) -> snapshot`: A function which creates a new document snapshot with the op applied +`transform(op1, op2, side) -> op1'`: OT transform function. + +Given op1, op2, `apply(s, op2, transform(op1, op2, 'left')) == apply(s, op1, transform(op2, op1, 'right'))`. + +Transform and apply must never modify their arguments. + + +Optional properties: + +`tp2`: _(bool)_ True if the transform function supports TP2. This allows p2p architectures to work. +`compose(op1, op2) -> op`: Create and return a new op which has the same effect as op1 + op2. +`serialize(snapshot) -> JSON object`: Serialize a document to something we can JSON.stringify() +`deserialize(object) -> snapshot`: Deserialize a JSON object into the document's internal snapshot format +`prune(op1', op2, side) -> op1`: Inserse transform function. Only required for TP2 types. +`normalize(op) -> op`: Fix up an op to make it valid. Eg, remove skips of size zero. +`api`: _(object)_ Set of helper methods which will be mixed in to the client document object for manipulating documents. See below. + + +# Examples + +`count` and `simple` are two trivial OT type definitions if you want to take a look. JSON defines +the ot-for-JSON type (see the wiki for documentation) and all the text types define different text +implementations. (I still have no idea which one I like the most, and they're fun to write!) + + +# API + +Types can also define API functions. These methods are mixed into the client's Doc object when a document is created. +You can use them to help construct ops programatically (so users don't need to understand how ops are structured). + +For example, the three text types defined here (text, text-composable and text-tp2) all provide the text API, supplying +`.insert()`, `.del()`, `.getLength` and `.getText` methods. + +See text-api.coffee for an example. diff --git a/services/document-updater/app/coffee/sharejs/count.coffee b/services/document-updater/app/coffee/sharejs/count.coffee new file mode 100644 index 0000000000..da28355efb --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/count.coffee @@ -0,0 +1,22 @@ +# This is a simple type used for testing other OT code. Each op is [expectedSnapshot, increment] + +exports.name = 'count' +exports.create = -> 1 + +exports.apply = (snapshot, op) -> + [v, inc] = op + throw new Error "Op #{v} != snapshot #{snapshot}" unless snapshot == v + snapshot + inc + +# transform op1 by op2. Return transformed version of op1. +exports.transform = (op1, op2) -> + throw new Error "Op1 #{op1[0]} != op2 #{op2[0]}" unless op1[0] == op2[0] + [op1[0] + op2[1], op1[1]] + +exports.compose = (op1, op2) -> + throw new Error "Op1 #{op1} + 1 != op2 #{op2}" unless op1[0] + op1[1] == op2[0] + [op1[0], op1[1] + op2[1]] + +exports.generateRandomOp = (doc) -> + [[doc, 1], doc + 1] + diff --git a/services/document-updater/app/coffee/sharejs/helpers.coffee b/services/document-updater/app/coffee/sharejs/helpers.coffee new file mode 100644 index 0000000000..093b32e1bb --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/helpers.coffee @@ -0,0 +1,65 @@ +# These methods let you build a transform function from a transformComponent function +# for OT types like text and JSON in which operations are lists of components +# and transforming them requires N^2 work. + +# Add transform and transformX functions for an OT type which has transformComponent defined. +# transformComponent(destination array, component, other component, side) +exports['_bt'] = bootstrapTransform = (type, transformComponent, checkValidOp, append) -> + transformComponentX = (left, right, destLeft, destRight) -> + transformComponent destLeft, left, right, 'left' + transformComponent destRight, right, left, 'right' + + # Transforms rightOp by leftOp. Returns ['rightOp', clientOp'] + type.transformX = type['transformX'] = transformX = (leftOp, rightOp) -> + checkValidOp leftOp + checkValidOp rightOp + + newRightOp = [] + + for rightComponent in rightOp + # Generate newLeftOp by composing leftOp by rightComponent + newLeftOp = [] + + k = 0 + while k < leftOp.length + nextC = [] + transformComponentX leftOp[k], rightComponent, newLeftOp, nextC + k++ + + if nextC.length == 1 + rightComponent = nextC[0] + else if nextC.length == 0 + append newLeftOp, l for l in leftOp[k..] + rightComponent = null + break + else + # Recurse. + [l_, r_] = transformX leftOp[k..], nextC + append newLeftOp, l for l in l_ + append newRightOp, r for r in r_ + rightComponent = null + break + + append newRightOp, rightComponent if rightComponent? + leftOp = newLeftOp + + [leftOp, newRightOp] + + # Transforms op with specified type ('left' or 'right') by otherOp. + type.transform = type['transform'] = (op, otherOp, type) -> + throw new Error "type must be 'left' or 'right'" unless type == 'left' or type == 'right' + + return op if otherOp.length == 0 + + # TODO: Benchmark with and without this line. I _think_ it'll make a big difference...? + return transformComponent [], op[0], otherOp[0], type if op.length == 1 and otherOp.length == 1 + + if type == 'left' + [left, _] = transformX op, otherOp + left + else + [_, right] = transformX otherOp, op + right + +if typeof WEB is 'undefined' + exports.bootstrapTransform = bootstrapTransform diff --git a/services/document-updater/app/coffee/sharejs/index.coffee b/services/document-updater/app/coffee/sharejs/index.coffee new file mode 100644 index 0000000000..6f3bb8ec20 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/index.coffee @@ -0,0 +1,15 @@ + +register = (file) -> + type = require file + exports[type.name] = type + try require "#{file}-api" + +# Import all the built-in types. +register './simple' +register './count' + +register './text' +register './text-composable' +register './text-tp2' + +register './json' diff --git a/services/document-updater/app/coffee/sharejs/json-api.coffee b/services/document-updater/app/coffee/sharejs/json-api.coffee new file mode 100644 index 0000000000..8819dee798 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/json-api.coffee @@ -0,0 +1,180 @@ +# API for JSON OT + +json = require './json' if typeof WEB is 'undefined' + +if WEB? + extendDoc = exports.extendDoc + exports.extendDoc = (name, fn) -> + SubDoc::[name] = fn + extendDoc name, fn + +depath = (path) -> + if path.length == 1 and path[0].constructor == Array + path[0] + else path + +class SubDoc + constructor: (@doc, @path) -> + at: (path...) -> @doc.at @path.concat depath path + get: -> @doc.getAt @path + # for objects and lists + set: (value, cb) -> @doc.setAt @path, value, cb + # for strings and lists. + insert: (pos, value, cb) -> @doc.insertAt @path, pos, value, cb + # for strings + del: (pos, length, cb) -> @doc.deleteTextAt @path, length, pos, cb + # for objects and lists + remove: (cb) -> @doc.removeAt @path, cb + push: (value, cb) -> @insert @get().length, value, cb + move: (from, to, cb) -> @doc.moveAt @path, from, to, cb + add: (amount, cb) -> @doc.addAt @path, amount, cb + on: (event, cb) -> @doc.addListener @path, event, cb + removeListener: (l) -> @doc.removeListener l + + # text API compatibility + getLength: -> @get().length + getText: -> @get() + +traverse = (snapshot, path) -> + container = data:snapshot + key = 'data' + elem = container + for p in path + elem = elem[key] + key = p + throw new Error 'bad path' if typeof elem == 'undefined' + {elem, key} + +pathEquals = (p1, p2) -> + return false if p1.length != p2.length + for e,i in p1 + return false if e != p2[i] + true + +json.api = + provides: {json:true} + + at: (path...) -> new SubDoc this, depath path + + get: -> @snapshot + set: (value, cb) -> @setAt [], value, cb + + getAt: (path) -> + {elem, key} = traverse @snapshot, path + return elem[key] + + setAt: (path, value, cb) -> + {elem, key} = traverse @snapshot, path + op = {p:path} + if elem.constructor == Array + op.li = value + op.ld = elem[key] if typeof elem[key] != 'undefined' + else if typeof elem == 'object' + op.oi = value + op.od = elem[key] if typeof elem[key] != 'undefined' + else throw new Error 'bad path' + @submitOp [op], cb + + removeAt: (path, cb) -> + {elem, key} = traverse @snapshot, path + throw new Error 'no element at that path' unless typeof elem[key] != 'undefined' + op = {p:path} + if elem.constructor == Array + op.ld = elem[key] + else if typeof elem == 'object' + op.od = elem[key] + else throw new Error 'bad path' + @submitOp [op], cb + + insertAt: (path, pos, value, cb) -> + {elem, key} = traverse @snapshot, path + op = {p:path.concat pos} + if elem[key].constructor == Array + op.li = value + else if typeof elem[key] == 'string' + op.si = value + @submitOp [op], cb + + moveAt: (path, from, to, cb) -> + op = [{p:path.concat(from), lm:to}] + @submitOp op, cb + + addAt: (path, amount, cb) -> + op = [{p:path, na:amount}] + @submitOp op, cb + + deleteTextAt: (path, length, pos, cb) -> + {elem, key} = traverse @snapshot, path + op = [{p:path.concat(pos), sd:elem[key][pos...(pos + length)]}] + @submitOp op, cb + + addListener: (path, event, cb) -> + l = {path, event, cb} + @_listeners.push l + l + removeListener: (l) -> + i = @_listeners.indexOf l + return false if i < 0 + @_listeners.splice i, 1 + return true + _register: -> + @_listeners = [] + @on 'change', (op) -> + for c in op + if c.na != undefined or c.si != undefined or c.sd != undefined + # no change to structure + continue + to_remove = [] + for l, i in @_listeners + # Transform a dummy op by the incoming op to work out what + # should happen to the listener. + dummy = {p:l.path, na:0} + xformed = @type.transformComponent [], dummy, c, 'left' + if xformed.length == 0 + # The op was transformed to noop, so we should delete the listener. + to_remove.push i + else if xformed.length == 1 + # The op remained, so grab its new path into the listener. + l.path = xformed[0].p + else + throw new Error "Bad assumption in json-api: xforming an 'si' op will always result in 0 or 1 components." + to_remove.sort (a, b) -> b - a + for i in to_remove + @_listeners.splice i, 1 + @on 'remoteop', (op) -> + for c in op + match_path = if c.na == undefined then c.p[...c.p.length-1] else c.p + for {path, event, cb} in @_listeners + if pathEquals path, match_path + switch event + when 'insert' + if c.li != undefined and c.ld == undefined + cb(c.p[c.p.length-1], c.li) + else if c.oi != undefined and c.od == undefined + cb(c.p[c.p.length-1], c.oi) + else if c.si != undefined + cb(c.p[c.p.length-1], c.si) + when 'delete' + if c.li == undefined and c.ld != undefined + cb(c.p[c.p.length-1], c.ld) + else if c.oi == undefined and c.od != undefined + cb(c.p[c.p.length-1], c.od) + else if c.sd != undefined + cb(c.p[c.p.length-1], c.sd) + when 'replace' + if c.li != undefined and c.ld != undefined + cb(c.p[c.p.length-1], c.ld, c.li) + else if c.oi != undefined and c.od != undefined + cb(c.p[c.p.length-1], c.od, c.oi) + when 'move' + if c.lm != undefined + cb(c.p[c.p.length-1], c.lm) + when 'add' + if c.na != undefined + cb(c.na) + else if (common = @type.commonPath match_path, path)? + if event == 'child op' + if match_path.length == path.length == common + throw new Error "paths match length and have commonality, but aren't equal?" + child_path = c.p[common+1..] + cb(child_path, c) diff --git a/services/document-updater/app/coffee/sharejs/json.coffee b/services/document-updater/app/coffee/sharejs/json.coffee new file mode 100644 index 0000000000..b03b0947ef --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/json.coffee @@ -0,0 +1,441 @@ +# This is the implementation of the JSON OT type. +# +# Spec is here: https://github.com/josephg/ShareJS/wiki/JSON-Operations + +if WEB? + text = exports.types.text +else + text = require './text' + +json = {} + +json.name = 'json' + +json.create = -> null + +json.invertComponent = (c) -> + c_ = {p: c.p} + c_.sd = c.si if c.si != undefined + c_.si = c.sd if c.sd != undefined + c_.od = c.oi if c.oi != undefined + c_.oi = c.od if c.od != undefined + c_.ld = c.li if c.li != undefined + c_.li = c.ld if c.ld != undefined + c_.na = -c.na if c.na != undefined + if c.lm != undefined + c_.lm = c.p[c.p.length-1] + c_.p = c.p[0...c.p.length - 1].concat([c.lm]) + c_ + +json.invert = (op) -> json.invertComponent c for c in op.slice().reverse() + +json.checkValidOp = (op) -> + +isArray = (o) -> Object.prototype.toString.call(o) == '[object Array]' +json.checkList = (elem) -> + throw new Error 'Referenced element not a list' unless isArray(elem) + +json.checkObj = (elem) -> + throw new Error "Referenced element not an object (it was #{JSON.stringify elem})" unless elem.constructor is Object + +json.apply = (snapshot, op) -> + json.checkValidOp op + op = clone op + + container = {data: clone snapshot} + + try + for c, i in op + parent = null + parentkey = null + elem = container + key = 'data' + + for p in c.p + parent = elem + parentkey = key + elem = elem[key] + key = p + + throw new Error 'Path invalid' unless parent? + + if c.na != undefined + # Number add + throw new Error 'Referenced element not a number' unless typeof elem[key] is 'number' + elem[key] += c.na + + else if c.si != undefined + # String insert + throw new Error "Referenced element not a string (it was #{JSON.stringify elem})" unless typeof elem is 'string' + parent[parentkey] = elem[...key] + c.si + elem[key..] + else if c.sd != undefined + # String delete + throw new Error 'Referenced element not a string' unless typeof elem is 'string' + throw new Error 'Deleted string does not match' unless elem[key...key + c.sd.length] == c.sd + parent[parentkey] = elem[...key] + elem[key + c.sd.length..] + + else if c.li != undefined && c.ld != undefined + # List replace + json.checkList elem + + # Should check the list element matches c.ld + elem[key] = c.li + else if c.li != undefined + # List insert + json.checkList elem + + elem.splice key, 0, c.li + else if c.ld != undefined + # List delete + json.checkList elem + + # Should check the list element matches c.ld here too. + elem.splice key, 1 + else if c.lm != undefined + # List move + json.checkList elem + if c.lm != key + e = elem[key] + # Remove it... + elem.splice key, 1 + # And insert it back. + elem.splice c.lm, 0, e + + else if c.oi != undefined + # Object insert / replace + json.checkObj elem + + # Should check that elem[key] == c.od + elem[key] = c.oi + else if c.od != undefined + # Object delete + json.checkObj elem + + # Should check that elem[key] == c.od + delete elem[key] + else + throw new Error 'invalid / missing instruction in op' + catch error + # TODO: Roll back all already applied changes. Write tests before implementing this code. + throw error + + container.data + +# Checks if two paths, p1 and p2 match. +json.pathMatches = (p1, p2, ignoreLast) -> + return false unless p1.length == p2.length + + for p, i in p1 + return false if p != p2[i] and (!ignoreLast or i != p1.length - 1) + + true + +json.append = (dest, c) -> + c = clone c + if dest.length != 0 and json.pathMatches c.p, (last = dest[dest.length - 1]).p + if last.na != undefined and c.na != undefined + dest[dest.length - 1] = { p: last.p, na: last.na + c.na } + else if last.li != undefined and c.li == undefined and c.ld == last.li + # insert immediately followed by delete becomes a noop. + if last.ld != undefined + # leave the delete part of the replace + delete last.li + else + dest.pop() + else if last.od != undefined and last.oi == undefined and + c.oi != undefined and c.od == undefined + last.oi = c.oi + else if c.lm != undefined and c.p[c.p.length-1] == c.lm + null # don't do anything + else + dest.push c + else + dest.push c + +json.compose = (op1, op2) -> + json.checkValidOp op1 + json.checkValidOp op2 + + newOp = clone op1 + json.append newOp, c for c in op2 + + newOp + +json.normalize = (op) -> + newOp = [] + + op = [op] unless isArray op + + for c in op + c.p ?= [] + json.append newOp, c + + newOp + +# hax, copied from test/types/json. Apparently this is still the fastest way to deep clone an object, assuming +# we have browser support for JSON. +# http://jsperf.com/cloning-an-object/12 +clone = (o) -> JSON.parse(JSON.stringify o) + +json.commonPath = (p1, p2) -> + p1 = p1.slice() + p2 = p2.slice() + p1.unshift('data') + p2.unshift('data') + p1 = p1[...p1.length-1] + p2 = p2[...p2.length-1] + return -1 if p2.length == 0 + i = 0 + while p1[i] == p2[i] && i < p1.length + i++ + if i == p2.length + return i-1 + return + +# transform c so it applies to a document with otherC applied. +json.transformComponent = (dest, c, otherC, type) -> + c = clone c + c.p.push(0) if c.na != undefined + otherC.p.push(0) if otherC.na != undefined + + common = json.commonPath c.p, otherC.p + common2 = json.commonPath otherC.p, c.p + + cplength = c.p.length + otherCplength = otherC.p.length + + c.p.pop() if c.na != undefined # hax + otherC.p.pop() if otherC.na != undefined + + if otherC.na + if common2? && otherCplength >= cplength && otherC.p[common2] == c.p[common2] + if c.ld != undefined + oc = clone otherC + oc.p = oc.p[cplength..] + c.ld = json.apply clone(c.ld), [oc] + else if c.od != undefined + oc = clone otherC + oc.p = oc.p[cplength..] + c.od = json.apply clone(c.od), [oc] + json.append dest, c + return dest + + if common2? && otherCplength > cplength && c.p[common2] == otherC.p[common2] + # transform based on c + if c.ld != undefined + oc = clone otherC + oc.p = oc.p[cplength..] + c.ld = json.apply clone(c.ld), [oc] + else if c.od != undefined + oc = clone otherC + oc.p = oc.p[cplength..] + c.od = json.apply clone(c.od), [oc] + + + if common? + commonOperand = cplength == otherCplength + # transform based on otherC + if otherC.na != undefined + # this case is handled above due to icky path hax + else if otherC.si != undefined || otherC.sd != undefined + # String op vs string op - pass through to text type + if c.si != undefined || c.sd != undefined + throw new Error("must be a string?") unless commonOperand + + # Convert an op component to a text op component + convert = (component) -> + newC = p:component.p[component.p.length - 1] + if component.si + newC.i = component.si + else + newC.d = component.sd + newC + + tc1 = convert c + tc2 = convert otherC + + res = [] + text._tc res, tc1, tc2, type + for tc in res + jc = { p: c.p[...common] } + jc.p.push(tc.p) + jc.si = tc.i if tc.i? + jc.sd = tc.d if tc.d? + json.append dest, jc + return dest + else if otherC.li != undefined && otherC.ld != undefined + if otherC.p[common] == c.p[common] + # noop + if !commonOperand + # we're below the deleted element, so -> noop + return dest + else if c.ld != undefined + # we're trying to delete the same element, -> noop + if c.li != undefined and type == 'left' + # we're both replacing one element with another. only one can + # survive! + c.ld = clone otherC.li + else + return dest + else if otherC.li != undefined + if c.li != undefined and c.ld == undefined and commonOperand and c.p[common] == otherC.p[common] + # in li vs. li, left wins. + if type == 'right' + c.p[common]++ + else if otherC.p[common] <= c.p[common] + c.p[common]++ + + if c.lm != undefined + if commonOperand + # otherC edits the same list we edit + if otherC.p[common] <= c.lm + c.lm++ + # changing c.from is handled above. + else if otherC.ld != undefined + if c.lm != undefined + if commonOperand + if otherC.p[common] == c.p[common] + # they deleted the thing we're trying to move + return dest + # otherC edits the same list we edit + p = otherC.p[common] + from = c.p[common] + to = c.lm + if p < to || (p == to && from < to) + c.lm-- + + if otherC.p[common] < c.p[common] + c.p[common]-- + else if otherC.p[common] == c.p[common] + if otherCplength < cplength + # we're below the deleted element, so -> noop + return dest + else if c.ld != undefined + if c.li != undefined + # we're replacing, they're deleting. we become an insert. + delete c.ld + else + # we're trying to delete the same element, -> noop + return dest + else if otherC.lm != undefined + if c.lm != undefined and cplength == otherCplength + # lm vs lm, here we go! + from = c.p[common] + to = c.lm + otherFrom = otherC.p[common] + otherTo = otherC.lm + if otherFrom != otherTo + # if otherFrom == otherTo, we don't need to change our op. + + # where did my thing go? + if from == otherFrom + # they moved it! tie break. + if type == 'left' + c.p[common] = otherTo + if from == to # ugh + c.lm = otherTo + else + return dest + else + # they moved around it + if from > otherFrom + c.p[common]-- + if from > otherTo + c.p[common]++ + else if from == otherTo + if otherFrom > otherTo + c.p[common]++ + if from == to # ugh, again + c.lm++ + + # step 2: where am i going to put it? + if to > otherFrom + c.lm-- + else if to == otherFrom + if to > from + c.lm-- + if to > otherTo + c.lm++ + else if to == otherTo + # if we're both moving in the same direction, tie break + if (otherTo > otherFrom and to > from) or + (otherTo < otherFrom and to < from) + if type == 'right' + c.lm++ + else + if to > from + c.lm++ + else if to == otherFrom + c.lm-- + else if c.li != undefined and c.ld == undefined and commonOperand + # li + from = otherC.p[common] + to = otherC.lm + p = c.p[common] + if p > from + c.p[common]-- + if p > to + c.p[common]++ + else + # ld, ld+li, si, sd, na, oi, od, oi+od, any li on an element beneath + # the lm + # + # i.e. things care about where their item is after the move. + from = otherC.p[common] + to = otherC.lm + p = c.p[common] + if p == from + c.p[common] = to + else + if p > from + c.p[common]-- + if p > to + c.p[common]++ + else if p == to + if from > to + c.p[common]++ + else if otherC.oi != undefined && otherC.od != undefined + if c.p[common] == otherC.p[common] + if c.oi != undefined and commonOperand + # we inserted where someone else replaced + if type == 'right' + # left wins + return dest + else + # we win, make our op replace what they inserted + c.od = otherC.oi + else + # -> noop if the other component is deleting the same object (or any + # parent) + return dest + else if otherC.oi != undefined + if c.oi != undefined and c.p[common] == otherC.p[common] + # left wins if we try to insert at the same place + if type == 'left' + json.append dest, {p:c.p, od:otherC.oi} + else + return dest + else if otherC.od != undefined + if c.p[common] == otherC.p[common] + return dest if !commonOperand + if c.oi != undefined + delete c.od + else + return dest + + json.append dest, c + return dest + +if WEB? + exports.types ||= {} + + # This is kind of awful - come up with a better way to hook this helper code up. + exports._bt(json, json.transformComponent, json.checkValidOp, json.append) + + # [] is used to prevent closure from renaming types.text + exports.types.json = json +else + module.exports = json + + require('./helpers').bootstrapTransform(json, json.transformComponent, json.checkValidOp, json.append) + diff --git a/services/document-updater/app/coffee/sharejs/model.coffee b/services/document-updater/app/coffee/sharejs/model.coffee new file mode 100644 index 0000000000..284d6fd770 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/model.coffee @@ -0,0 +1,603 @@ +# The model of all the ops. Responsible for applying & transforming remote deltas +# and managing the storage layer. +# +# Actual storage is handled by the database wrappers in db/*, wrapped by DocCache + +{EventEmitter} = require 'events' + +queue = require './syncqueue' +types = require '../types' + +isArray = (o) -> Object.prototype.toString.call(o) == '[object Array]' + +# This constructor creates a new Model object. There will be one model object +# per server context. +# +# The model object is responsible for a lot of things: +# +# - It manages the interactions with the database +# - It maintains (in memory) a set of all active documents +# - It calls out to the OT functions when necessary +# +# The model is an event emitter. It emits the following events: +# +# create(docName, data): A document has been created with the specified name & data +module.exports = Model = (db, options) -> + # db can be null if the user doesn't want persistance. + + return new Model(db, options) if !(this instanceof Model) + + model = this + + options ?= {} + + # This is a cache of 'live' documents. + # + # The cache is a map from docName -> { + # ops:[{op, meta}] + # snapshot + # type + # v + # meta + # eventEmitter + # reapTimer + # committedVersion: v + # snapshotWriteLock: bool to make sure writeSnapshot isn't re-entrant + # dbMeta: database specific data + # opQueue: syncQueue for processing ops + # } + # + # The ops list contains the document's last options.numCachedOps ops. (Or all + # of them if we're using a memory store). + # + # Documents are stored in this set so long as the document has been accessed in + # the last few seconds (options.reapTime) OR at least one client has the document + # open. I don't know if I should keep open (but not being edited) documents live - + # maybe if a client has a document open but the document isn't being edited, I should + # flush it from the cache. + # + # In any case, the API to model is designed such that if we want to change that later + # it should be pretty easy to do so without any external-to-the-model code changes. + docs = {} + + # This is a map from docName -> [callback]. It is used when a document hasn't been + # cached and multiple getSnapshot() / getVersion() requests come in. All requests + # are added to the callback list and called when db.getSnapshot() returns. + # + # callback(error, snapshot data) + awaitingGetSnapshot = {} + + # The time that documents which no clients have open will stay in the cache. + # Should be > 0. + options.reapTime ?= 3000 + + # The number of operations the cache holds before reusing the space + options.numCachedOps ?= 10 + + # This option forces documents to be reaped, even when there's no database backend. + # This is useful when you don't care about persistance and don't want to gradually + # fill memory. + # + # You might want to set reapTime to a day or something. + options.forceReaping ?= false + + # Until I come up with a better strategy, we'll save a copy of the document snapshot + # to the database every ~20 submitted ops. + options.opsBeforeCommit ?= 20 + + # It takes some processing time to transform client ops. The server will punt ops back to the + # client to transform if they're too old. + options.maximumAge ?= 40 + + # **** Cache API methods + + # Its important that all ops are applied in order. This helper method creates the op submission queue + # for a single document. This contains the logic for transforming & applying ops. + makeOpQueue = (docName, doc) -> queue (opData, callback) -> + return callback 'Version missing' unless opData.v >= 0 + return callback 'Op at future version' if opData.v > doc.v + + # Punt the transforming work back to the client if the op is too old. + return callback 'Op too old' if opData.v + options.maximumAge < doc.v + + opData.meta ||= {} + opData.meta.ts = Date.now() + + # We'll need to transform the op to the current version of the document. This + # calls the callback immediately if opVersion == doc.v. + getOps docName, opData.v, doc.v, (error, ops) -> + return callback error if error + + unless doc.v - opData.v == ops.length + # This should never happen. It indicates that we didn't get all the ops we + # asked for. Its important that the submitted op is correctly transformed. + console.error "Could not get old ops in model for document #{docName}" + console.error "Expected ops #{opData.v} to #{doc.v} and got #{ops.length} ops" + return callback 'Internal error' + + if ops.length > 0 + try + # If there's enough ops, it might be worth spinning this out into a webworker thread. + for oldOp in ops + # Dup detection works by sending the id(s) the op has been submitted with previously. + # If the id matches, we reject it. The client can also detect the op has been submitted + # already if it sees its own previous id in the ops it sees when it does catchup. + if oldOp.meta.source and opData.dupIfSource and oldOp.meta.source in opData.dupIfSource + return callback 'Op already submitted' + + opData.op = doc.type.transform opData.op, oldOp.op, 'left' + opData.v++ + catch error + console.error error.stack + return callback error.message + + try + snapshot = doc.type.apply doc.snapshot, opData.op + catch error + console.error error.stack + return callback error.message + + # The op data should be at the current version, and the new document data should be at + # the next version. + # + # This should never happen in practice, but its a nice little check to make sure everything + # is hunky-dory. + unless opData.v == doc.v + # This should never happen. + console.error "Version mismatch detected in model. File a ticket - this is a bug." + console.error "Expecting #{opData.v} == #{doc.v}" + return callback 'Internal error' + + #newDocData = {snapshot, type:type.name, v:opVersion + 1, meta:docData.meta} + writeOp = db?.writeOp or (docName, newOpData, callback) -> callback() + + writeOp docName, opData, (error) -> + if error + # The user should probably know about this. + console.warn "Error writing ops to database: #{error}" + return callback error + + options.stats?.writeOp?() + + # This is needed when we emit the 'change' event, below. + oldSnapshot = doc.snapshot + + # All the heavy lifting is now done. Finally, we'll update the cache with the new data + # and (maybe!) save a new document snapshot to the database. + + doc.v = opData.v + 1 + doc.snapshot = snapshot + + doc.ops.push opData + doc.ops.shift() if db and doc.ops.length > options.numCachedOps + + model.emit 'applyOp', docName, opData, snapshot, oldSnapshot + doc.eventEmitter.emit 'op', opData, snapshot, oldSnapshot + + # The callback is called with the version of the document at which the op was applied. + # This is the op.v after transformation, and its doc.v - 1. + callback null, opData.v + + # I need a decent strategy here for deciding whether or not to save the snapshot. + # + # The 'right' strategy looks something like "Store the snapshot whenever the snapshot + # is smaller than the accumulated op data". For now, I'll just store it every 20 + # ops or something. (Configurable with doc.committedVersion) + if !doc.snapshotWriteLock and doc.committedVersion + options.opsBeforeCommit <= doc.v + tryWriteSnapshot docName, (error) -> + console.warn "Error writing snapshot #{error}. This is nonfatal" if error + + # Add the data for the given docName to the cache. The named document shouldn't already + # exist in the doc set. + # + # Returns the new doc. + add = (docName, error, data, committedVersion, ops, dbMeta) -> + callbacks = awaitingGetSnapshot[docName] + delete awaitingGetSnapshot[docName] + + if error + callback error for callback in callbacks if callbacks + else + doc = docs[docName] = + snapshot: data.snapshot + v: data.v + type: data.type + meta: data.meta + + # Cache of ops + ops: ops or [] + + eventEmitter: new EventEmitter + + # Timer before the document will be invalidated from the cache (if the document has no + # listeners) + reapTimer: null + + # Version of the snapshot thats in the database + committedVersion: committedVersion ? data.v + snapshotWriteLock: false + dbMeta: dbMeta + + doc.opQueue = makeOpQueue docName, doc + + refreshReapingTimeout docName + model.emit 'add', docName, data + callback null, doc for callback in callbacks if callbacks + + doc + + # This is a little helper wrapper around db.getOps. It does two things: + # + # - If there's no database set, it returns an error to the callback + # - It adds version numbers to each op returned from the database + # (These can be inferred from context so the DB doesn't store them, but its useful to have them). + getOpsInternal = (docName, start, end, callback) -> + return callback? 'Document does not exist' unless db + + db.getOps docName, start, end, (error, ops) -> + return callback? error if error + + v = start + op.v = v++ for op in ops + + callback? null, ops + + # Load the named document into the cache. This function is re-entrant. + # + # The callback is called with (error, doc) + load = (docName, callback) -> + if docs[docName] + # The document is already loaded. Return immediately. + options.stats?.cacheHit? 'getSnapshot' + return callback null, docs[docName] + + # We're a memory store. If we don't have it, nobody does. + return callback 'Document does not exist' unless db + + callbacks = awaitingGetSnapshot[docName] + + # The document is being loaded already. Add ourselves as a callback. + return callbacks.push callback if callbacks + + options.stats?.cacheMiss? 'getSnapshot' + + # The document isn't loaded and isn't being loaded. Load it. + awaitingGetSnapshot[docName] = [callback] + db.getSnapshot docName, (error, data, dbMeta) -> + return add docName, error if error + + type = types[data.type] + unless type + console.warn "Type '#{data.type}' missing" + return callback "Type not found" + data.type = type + + committedVersion = data.v + + # The server can close without saving the most recent document snapshot. + # In this case, there are extra ops which need to be applied before + # returning the snapshot. + getOpsInternal docName, data.v, null, (error, ops) -> + return callback error if error + + if ops.length > 0 + console.log "Catchup #{docName} #{data.v} -> #{data.v + ops.length}" + + try + for op in ops + data.snapshot = type.apply data.snapshot, op.op + data.v++ + catch e + # This should never happen - it indicates that whats in the + # database is invalid. + console.error "Op data invalid for #{docName}: #{e.stack}" + return callback 'Op data invalid' + + model.emit 'load', docName, data + add docName, error, data, committedVersion, ops, dbMeta + + # This makes sure the cache contains a document. If the doc cache doesn't contain + # a document, it is loaded from the database and stored. + # + # Documents are stored so long as either: + # - They have been accessed within the past #{PERIOD} + # - At least one client has the document open + refreshReapingTimeout = (docName) -> + doc = docs[docName] + return unless doc + + # I want to let the clients list be updated before this is called. + process.nextTick -> + # This is an awkward way to find out the number of clients on a document. If this + # causes performance issues, add a numClients field to the document. + # + # The first check is because its possible that between refreshReapingTimeout being called and this + # event being fired, someone called delete() on the document and hence the doc is something else now. + if doc == docs[docName] and + doc.eventEmitter.listeners('op').length == 0 and + (db or options.forceReaping) and + doc.opQueue.busy is false + + clearTimeout doc.reapTimer + doc.reapTimer = reapTimer = setTimeout -> + tryWriteSnapshot docName, -> + # If the reaping timeout has been refreshed while we're writing the snapshot, or if we're + # in the middle of applying an operation, don't reap. + delete docs[docName] if docs[docName].reapTimer is reapTimer and doc.opQueue.busy is false + , options.reapTime + + tryWriteSnapshot = (docName, callback) -> + return callback?() unless db + + doc = docs[docName] + + # The doc is closed + return callback?() unless doc + + # The document is already saved. + return callback?() if doc.committedVersion is doc.v + + return callback? 'Another snapshot write is in progress' if doc.snapshotWriteLock + + doc.snapshotWriteLock = true + + options.stats?.writeSnapshot?() + + writeSnapshot = db?.writeSnapshot or (docName, docData, dbMeta, callback) -> callback() + + data = + v: doc.v + meta: doc.meta + snapshot: doc.snapshot + # The database doesn't know about object types. + type: doc.type.name + + # Commit snapshot. + writeSnapshot docName, data, doc.dbMeta, (error, dbMeta) -> + doc.snapshotWriteLock = false + + # We have to use data.v here because the version in the doc could + # have been updated between the call to writeSnapshot() and now. + doc.committedVersion = data.v + doc.dbMeta = dbMeta + + callback? error + + # *** Model interface methods + + # Create a new document. + # + # data should be {snapshot, type, [meta]}. The version of a new document is 0. + @create = (docName, type, meta, callback) -> + [meta, callback] = [{}, meta] if typeof meta is 'function' + + return callback? 'Invalid document name' if docName.match /\// + return callback? 'Document already exists' if docs[docName] + + type = types[type] if typeof type == 'string' + return callback? 'Type not found' unless type + + data = + snapshot:type.create() + type:type.name + meta:meta or {} + v:0 + + done = (error, dbMeta) -> + # dbMeta can be used to cache extra state needed by the database to access the document, like an ID or something. + return callback? error if error + + # From here on we'll store the object version of the type name. + data.type = type + add docName, null, data, 0, [], dbMeta + model.emit 'create', docName, data + callback?() + + if db + db.create docName, data, done + else + done() + + # Perminantly deletes the specified document. + # If listeners are attached, they are removed. + # + # The callback is called with (error) if there was an error. If error is null / undefined, the + # document was deleted. + # + # WARNING: This isn't well supported throughout the code. (Eg, streaming clients aren't told about the + # deletion. Subsequent op submissions will fail). + @delete = (docName, callback) -> + doc = docs[docName] + + if doc + clearTimeout doc.reapTimer + delete docs[docName] + + done = (error) -> + model.emit 'delete', docName unless error + callback? error + + if db + db.delete docName, doc?.dbMeta, done + else + done (if !doc then 'Document does not exist') + + # This gets all operations from [start...end]. (That is, its not inclusive.) + # + # end can be null. This means 'get me all ops from start'. + # + # Each op returned is in the form {op:o, meta:m, v:version}. + # + # Callback is called with (error, [ops]) + # + # If the document does not exist, getOps doesn't necessarily return an error. This is because + # its awkward to figure out whether or not the document exists for things + # like the redis database backend. I guess its a bit gross having this inconsistant + # with the other DB calls, but its certainly convenient. + # + # Use getVersion() to determine if a document actually exists, if thats what you're + # after. + @getOps = getOps = (docName, start, end, callback) -> + # getOps will only use the op cache if its there. It won't fill the op cache in. + throw new Error 'start must be 0+' unless start >= 0 + + [end, callback] = [null, end] if typeof end is 'function' + + ops = docs[docName]?.ops + + if ops + version = docs[docName].v + + # Ops contains an array of ops. The last op in the list is the last op applied + end ?= version + start = Math.min start, end + + return callback null, [] if start == end + + # Base is the version number of the oldest op we have cached + base = version - ops.length + + # If the database is null, we'll trim to the ops we do have and hope thats enough. + if start >= base or db is null + refreshReapingTimeout docName + options.stats?.cacheHit 'getOps' + + return callback null, ops[(start - base)...(end - base)] + + options.stats?.cacheMiss 'getOps' + + getOpsInternal docName, start, end, callback + + # Gets the snapshot data for the specified document. + # getSnapshot(docName, callback) + # Callback is called with (error, {v: , type: , snapshot: , meta: }) + @getSnapshot = (docName, callback) -> + load docName, (error, doc) -> + callback error, if doc then {v:doc.v, type:doc.type, snapshot:doc.snapshot, meta:doc.meta} + + # Gets the latest version # of the document. + # getVersion(docName, callback) + # callback is called with (error, version). + @getVersion = (docName, callback) -> + load docName, (error, doc) -> callback error, doc?.v + + # Apply an op to the specified document. + # The callback is passed (error, applied version #) + # opData = {op:op, v:v, meta:metadata} + # + # Ops are queued before being applied so that the following code applies op C before op B: + # model.applyOp 'doc', OPA, -> model.applyOp 'doc', OPB + # model.applyOp 'doc', OPC + @applyOp = (docName, opData, callback) -> + # All the logic for this is in makeOpQueue, above. + load docName, (error, doc) -> + return callback error if error + + process.nextTick -> doc.opQueue opData, (error, newVersion) -> + refreshReapingTimeout docName + callback? error, newVersion + + # TODO: store (some) metadata in DB + # TODO: op and meta should be combineable in the op that gets sent + @applyMetaOp = (docName, metaOpData, callback) -> + {path, value} = metaOpData.meta + + return callback? "path should be an array" unless isArray path + + load docName, (error, doc) -> + if error? + callback? error + else + applied = false + switch path[0] + when 'shout' + doc.eventEmitter.emit 'op', metaOpData + applied = true + + model.emit 'applyMetaOp', docName, path, value if applied + callback? null, doc.v + + # Listen to all ops from the specified version. If version is in the past, all + # ops since that version are sent immediately to the listener. + # + # The callback is called once the listener is attached, but before any ops have been passed + # to the listener. + # + # This will _not_ edit the document metadata. + # + # If there are any listeners, we don't purge the document from the cache. But be aware, this behaviour + # might change in a future version. + # + # version is the document version at which the document is opened. It can be left out if you want to open + # the document at the most recent version. + # + # listener is called with (opData) each time an op is applied. + # + # callback(error, openedVersion) + @listen = (docName, version, listener, callback) -> + [version, listener, callback] = [null, version, listener] if typeof version is 'function' + + load docName, (error, doc) -> + return callback? error if error + + clearTimeout doc.reapTimer + + if version? + getOps docName, version, null, (error, data) -> + return callback? error if error + + doc.eventEmitter.on 'op', listener + callback? null, version + for op in data + listener op + + # The listener may well remove itself during the catchup phase. If this happens, break early. + # This is done in a quite inefficient way. (O(n) where n = #listeners on doc) + break unless listener in doc.eventEmitter.listeners 'op' + + else # Version is null / undefined. Just add the listener. + doc.eventEmitter.on 'op', listener + callback? null, doc.v + + # Remove a listener for a particular document. + # + # removeListener(docName, listener) + # + # This is synchronous. + @removeListener = (docName, listener) -> + # The document should already be loaded. + doc = docs[docName] + throw new Error 'removeListener called but document not loaded' unless doc + + doc.eventEmitter.removeListener 'op', listener + refreshReapingTimeout docName + + # Flush saves all snapshot data to the database. I'm not sure whether or not this is actually needed - + # sharejs will happily replay uncommitted ops when documents are re-opened anyway. + @flush = (callback) -> + return callback?() unless db + + pendingWrites = 0 + + for docName, doc of docs + if doc.committedVersion < doc.v + pendingWrites++ + # I'm hoping writeSnapshot will always happen in another thread. + tryWriteSnapshot docName, -> + process.nextTick -> + pendingWrites-- + callback?() if pendingWrites is 0 + + # If nothing was queued, terminate immediately. + callback?() if pendingWrites is 0 + + # Close the database connection. This is needed so nodejs can shut down cleanly. + @closeDb = -> + db?.close?() + db = null + + return + +# Model inherits from EventEmitter. +Model:: = new EventEmitter + diff --git a/services/document-updater/app/coffee/sharejs/server/model.coffee b/services/document-updater/app/coffee/sharejs/server/model.coffee new file mode 100644 index 0000000000..284d6fd770 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/server/model.coffee @@ -0,0 +1,603 @@ +# The model of all the ops. Responsible for applying & transforming remote deltas +# and managing the storage layer. +# +# Actual storage is handled by the database wrappers in db/*, wrapped by DocCache + +{EventEmitter} = require 'events' + +queue = require './syncqueue' +types = require '../types' + +isArray = (o) -> Object.prototype.toString.call(o) == '[object Array]' + +# This constructor creates a new Model object. There will be one model object +# per server context. +# +# The model object is responsible for a lot of things: +# +# - It manages the interactions with the database +# - It maintains (in memory) a set of all active documents +# - It calls out to the OT functions when necessary +# +# The model is an event emitter. It emits the following events: +# +# create(docName, data): A document has been created with the specified name & data +module.exports = Model = (db, options) -> + # db can be null if the user doesn't want persistance. + + return new Model(db, options) if !(this instanceof Model) + + model = this + + options ?= {} + + # This is a cache of 'live' documents. + # + # The cache is a map from docName -> { + # ops:[{op, meta}] + # snapshot + # type + # v + # meta + # eventEmitter + # reapTimer + # committedVersion: v + # snapshotWriteLock: bool to make sure writeSnapshot isn't re-entrant + # dbMeta: database specific data + # opQueue: syncQueue for processing ops + # } + # + # The ops list contains the document's last options.numCachedOps ops. (Or all + # of them if we're using a memory store). + # + # Documents are stored in this set so long as the document has been accessed in + # the last few seconds (options.reapTime) OR at least one client has the document + # open. I don't know if I should keep open (but not being edited) documents live - + # maybe if a client has a document open but the document isn't being edited, I should + # flush it from the cache. + # + # In any case, the API to model is designed such that if we want to change that later + # it should be pretty easy to do so without any external-to-the-model code changes. + docs = {} + + # This is a map from docName -> [callback]. It is used when a document hasn't been + # cached and multiple getSnapshot() / getVersion() requests come in. All requests + # are added to the callback list and called when db.getSnapshot() returns. + # + # callback(error, snapshot data) + awaitingGetSnapshot = {} + + # The time that documents which no clients have open will stay in the cache. + # Should be > 0. + options.reapTime ?= 3000 + + # The number of operations the cache holds before reusing the space + options.numCachedOps ?= 10 + + # This option forces documents to be reaped, even when there's no database backend. + # This is useful when you don't care about persistance and don't want to gradually + # fill memory. + # + # You might want to set reapTime to a day or something. + options.forceReaping ?= false + + # Until I come up with a better strategy, we'll save a copy of the document snapshot + # to the database every ~20 submitted ops. + options.opsBeforeCommit ?= 20 + + # It takes some processing time to transform client ops. The server will punt ops back to the + # client to transform if they're too old. + options.maximumAge ?= 40 + + # **** Cache API methods + + # Its important that all ops are applied in order. This helper method creates the op submission queue + # for a single document. This contains the logic for transforming & applying ops. + makeOpQueue = (docName, doc) -> queue (opData, callback) -> + return callback 'Version missing' unless opData.v >= 0 + return callback 'Op at future version' if opData.v > doc.v + + # Punt the transforming work back to the client if the op is too old. + return callback 'Op too old' if opData.v + options.maximumAge < doc.v + + opData.meta ||= {} + opData.meta.ts = Date.now() + + # We'll need to transform the op to the current version of the document. This + # calls the callback immediately if opVersion == doc.v. + getOps docName, opData.v, doc.v, (error, ops) -> + return callback error if error + + unless doc.v - opData.v == ops.length + # This should never happen. It indicates that we didn't get all the ops we + # asked for. Its important that the submitted op is correctly transformed. + console.error "Could not get old ops in model for document #{docName}" + console.error "Expected ops #{opData.v} to #{doc.v} and got #{ops.length} ops" + return callback 'Internal error' + + if ops.length > 0 + try + # If there's enough ops, it might be worth spinning this out into a webworker thread. + for oldOp in ops + # Dup detection works by sending the id(s) the op has been submitted with previously. + # If the id matches, we reject it. The client can also detect the op has been submitted + # already if it sees its own previous id in the ops it sees when it does catchup. + if oldOp.meta.source and opData.dupIfSource and oldOp.meta.source in opData.dupIfSource + return callback 'Op already submitted' + + opData.op = doc.type.transform opData.op, oldOp.op, 'left' + opData.v++ + catch error + console.error error.stack + return callback error.message + + try + snapshot = doc.type.apply doc.snapshot, opData.op + catch error + console.error error.stack + return callback error.message + + # The op data should be at the current version, and the new document data should be at + # the next version. + # + # This should never happen in practice, but its a nice little check to make sure everything + # is hunky-dory. + unless opData.v == doc.v + # This should never happen. + console.error "Version mismatch detected in model. File a ticket - this is a bug." + console.error "Expecting #{opData.v} == #{doc.v}" + return callback 'Internal error' + + #newDocData = {snapshot, type:type.name, v:opVersion + 1, meta:docData.meta} + writeOp = db?.writeOp or (docName, newOpData, callback) -> callback() + + writeOp docName, opData, (error) -> + if error + # The user should probably know about this. + console.warn "Error writing ops to database: #{error}" + return callback error + + options.stats?.writeOp?() + + # This is needed when we emit the 'change' event, below. + oldSnapshot = doc.snapshot + + # All the heavy lifting is now done. Finally, we'll update the cache with the new data + # and (maybe!) save a new document snapshot to the database. + + doc.v = opData.v + 1 + doc.snapshot = snapshot + + doc.ops.push opData + doc.ops.shift() if db and doc.ops.length > options.numCachedOps + + model.emit 'applyOp', docName, opData, snapshot, oldSnapshot + doc.eventEmitter.emit 'op', opData, snapshot, oldSnapshot + + # The callback is called with the version of the document at which the op was applied. + # This is the op.v after transformation, and its doc.v - 1. + callback null, opData.v + + # I need a decent strategy here for deciding whether or not to save the snapshot. + # + # The 'right' strategy looks something like "Store the snapshot whenever the snapshot + # is smaller than the accumulated op data". For now, I'll just store it every 20 + # ops or something. (Configurable with doc.committedVersion) + if !doc.snapshotWriteLock and doc.committedVersion + options.opsBeforeCommit <= doc.v + tryWriteSnapshot docName, (error) -> + console.warn "Error writing snapshot #{error}. This is nonfatal" if error + + # Add the data for the given docName to the cache. The named document shouldn't already + # exist in the doc set. + # + # Returns the new doc. + add = (docName, error, data, committedVersion, ops, dbMeta) -> + callbacks = awaitingGetSnapshot[docName] + delete awaitingGetSnapshot[docName] + + if error + callback error for callback in callbacks if callbacks + else + doc = docs[docName] = + snapshot: data.snapshot + v: data.v + type: data.type + meta: data.meta + + # Cache of ops + ops: ops or [] + + eventEmitter: new EventEmitter + + # Timer before the document will be invalidated from the cache (if the document has no + # listeners) + reapTimer: null + + # Version of the snapshot thats in the database + committedVersion: committedVersion ? data.v + snapshotWriteLock: false + dbMeta: dbMeta + + doc.opQueue = makeOpQueue docName, doc + + refreshReapingTimeout docName + model.emit 'add', docName, data + callback null, doc for callback in callbacks if callbacks + + doc + + # This is a little helper wrapper around db.getOps. It does two things: + # + # - If there's no database set, it returns an error to the callback + # - It adds version numbers to each op returned from the database + # (These can be inferred from context so the DB doesn't store them, but its useful to have them). + getOpsInternal = (docName, start, end, callback) -> + return callback? 'Document does not exist' unless db + + db.getOps docName, start, end, (error, ops) -> + return callback? error if error + + v = start + op.v = v++ for op in ops + + callback? null, ops + + # Load the named document into the cache. This function is re-entrant. + # + # The callback is called with (error, doc) + load = (docName, callback) -> + if docs[docName] + # The document is already loaded. Return immediately. + options.stats?.cacheHit? 'getSnapshot' + return callback null, docs[docName] + + # We're a memory store. If we don't have it, nobody does. + return callback 'Document does not exist' unless db + + callbacks = awaitingGetSnapshot[docName] + + # The document is being loaded already. Add ourselves as a callback. + return callbacks.push callback if callbacks + + options.stats?.cacheMiss? 'getSnapshot' + + # The document isn't loaded and isn't being loaded. Load it. + awaitingGetSnapshot[docName] = [callback] + db.getSnapshot docName, (error, data, dbMeta) -> + return add docName, error if error + + type = types[data.type] + unless type + console.warn "Type '#{data.type}' missing" + return callback "Type not found" + data.type = type + + committedVersion = data.v + + # The server can close without saving the most recent document snapshot. + # In this case, there are extra ops which need to be applied before + # returning the snapshot. + getOpsInternal docName, data.v, null, (error, ops) -> + return callback error if error + + if ops.length > 0 + console.log "Catchup #{docName} #{data.v} -> #{data.v + ops.length}" + + try + for op in ops + data.snapshot = type.apply data.snapshot, op.op + data.v++ + catch e + # This should never happen - it indicates that whats in the + # database is invalid. + console.error "Op data invalid for #{docName}: #{e.stack}" + return callback 'Op data invalid' + + model.emit 'load', docName, data + add docName, error, data, committedVersion, ops, dbMeta + + # This makes sure the cache contains a document. If the doc cache doesn't contain + # a document, it is loaded from the database and stored. + # + # Documents are stored so long as either: + # - They have been accessed within the past #{PERIOD} + # - At least one client has the document open + refreshReapingTimeout = (docName) -> + doc = docs[docName] + return unless doc + + # I want to let the clients list be updated before this is called. + process.nextTick -> + # This is an awkward way to find out the number of clients on a document. If this + # causes performance issues, add a numClients field to the document. + # + # The first check is because its possible that between refreshReapingTimeout being called and this + # event being fired, someone called delete() on the document and hence the doc is something else now. + if doc == docs[docName] and + doc.eventEmitter.listeners('op').length == 0 and + (db or options.forceReaping) and + doc.opQueue.busy is false + + clearTimeout doc.reapTimer + doc.reapTimer = reapTimer = setTimeout -> + tryWriteSnapshot docName, -> + # If the reaping timeout has been refreshed while we're writing the snapshot, or if we're + # in the middle of applying an operation, don't reap. + delete docs[docName] if docs[docName].reapTimer is reapTimer and doc.opQueue.busy is false + , options.reapTime + + tryWriteSnapshot = (docName, callback) -> + return callback?() unless db + + doc = docs[docName] + + # The doc is closed + return callback?() unless doc + + # The document is already saved. + return callback?() if doc.committedVersion is doc.v + + return callback? 'Another snapshot write is in progress' if doc.snapshotWriteLock + + doc.snapshotWriteLock = true + + options.stats?.writeSnapshot?() + + writeSnapshot = db?.writeSnapshot or (docName, docData, dbMeta, callback) -> callback() + + data = + v: doc.v + meta: doc.meta + snapshot: doc.snapshot + # The database doesn't know about object types. + type: doc.type.name + + # Commit snapshot. + writeSnapshot docName, data, doc.dbMeta, (error, dbMeta) -> + doc.snapshotWriteLock = false + + # We have to use data.v here because the version in the doc could + # have been updated between the call to writeSnapshot() and now. + doc.committedVersion = data.v + doc.dbMeta = dbMeta + + callback? error + + # *** Model interface methods + + # Create a new document. + # + # data should be {snapshot, type, [meta]}. The version of a new document is 0. + @create = (docName, type, meta, callback) -> + [meta, callback] = [{}, meta] if typeof meta is 'function' + + return callback? 'Invalid document name' if docName.match /\// + return callback? 'Document already exists' if docs[docName] + + type = types[type] if typeof type == 'string' + return callback? 'Type not found' unless type + + data = + snapshot:type.create() + type:type.name + meta:meta or {} + v:0 + + done = (error, dbMeta) -> + # dbMeta can be used to cache extra state needed by the database to access the document, like an ID or something. + return callback? error if error + + # From here on we'll store the object version of the type name. + data.type = type + add docName, null, data, 0, [], dbMeta + model.emit 'create', docName, data + callback?() + + if db + db.create docName, data, done + else + done() + + # Perminantly deletes the specified document. + # If listeners are attached, they are removed. + # + # The callback is called with (error) if there was an error. If error is null / undefined, the + # document was deleted. + # + # WARNING: This isn't well supported throughout the code. (Eg, streaming clients aren't told about the + # deletion. Subsequent op submissions will fail). + @delete = (docName, callback) -> + doc = docs[docName] + + if doc + clearTimeout doc.reapTimer + delete docs[docName] + + done = (error) -> + model.emit 'delete', docName unless error + callback? error + + if db + db.delete docName, doc?.dbMeta, done + else + done (if !doc then 'Document does not exist') + + # This gets all operations from [start...end]. (That is, its not inclusive.) + # + # end can be null. This means 'get me all ops from start'. + # + # Each op returned is in the form {op:o, meta:m, v:version}. + # + # Callback is called with (error, [ops]) + # + # If the document does not exist, getOps doesn't necessarily return an error. This is because + # its awkward to figure out whether or not the document exists for things + # like the redis database backend. I guess its a bit gross having this inconsistant + # with the other DB calls, but its certainly convenient. + # + # Use getVersion() to determine if a document actually exists, if thats what you're + # after. + @getOps = getOps = (docName, start, end, callback) -> + # getOps will only use the op cache if its there. It won't fill the op cache in. + throw new Error 'start must be 0+' unless start >= 0 + + [end, callback] = [null, end] if typeof end is 'function' + + ops = docs[docName]?.ops + + if ops + version = docs[docName].v + + # Ops contains an array of ops. The last op in the list is the last op applied + end ?= version + start = Math.min start, end + + return callback null, [] if start == end + + # Base is the version number of the oldest op we have cached + base = version - ops.length + + # If the database is null, we'll trim to the ops we do have and hope thats enough. + if start >= base or db is null + refreshReapingTimeout docName + options.stats?.cacheHit 'getOps' + + return callback null, ops[(start - base)...(end - base)] + + options.stats?.cacheMiss 'getOps' + + getOpsInternal docName, start, end, callback + + # Gets the snapshot data for the specified document. + # getSnapshot(docName, callback) + # Callback is called with (error, {v: , type: , snapshot: , meta: }) + @getSnapshot = (docName, callback) -> + load docName, (error, doc) -> + callback error, if doc then {v:doc.v, type:doc.type, snapshot:doc.snapshot, meta:doc.meta} + + # Gets the latest version # of the document. + # getVersion(docName, callback) + # callback is called with (error, version). + @getVersion = (docName, callback) -> + load docName, (error, doc) -> callback error, doc?.v + + # Apply an op to the specified document. + # The callback is passed (error, applied version #) + # opData = {op:op, v:v, meta:metadata} + # + # Ops are queued before being applied so that the following code applies op C before op B: + # model.applyOp 'doc', OPA, -> model.applyOp 'doc', OPB + # model.applyOp 'doc', OPC + @applyOp = (docName, opData, callback) -> + # All the logic for this is in makeOpQueue, above. + load docName, (error, doc) -> + return callback error if error + + process.nextTick -> doc.opQueue opData, (error, newVersion) -> + refreshReapingTimeout docName + callback? error, newVersion + + # TODO: store (some) metadata in DB + # TODO: op and meta should be combineable in the op that gets sent + @applyMetaOp = (docName, metaOpData, callback) -> + {path, value} = metaOpData.meta + + return callback? "path should be an array" unless isArray path + + load docName, (error, doc) -> + if error? + callback? error + else + applied = false + switch path[0] + when 'shout' + doc.eventEmitter.emit 'op', metaOpData + applied = true + + model.emit 'applyMetaOp', docName, path, value if applied + callback? null, doc.v + + # Listen to all ops from the specified version. If version is in the past, all + # ops since that version are sent immediately to the listener. + # + # The callback is called once the listener is attached, but before any ops have been passed + # to the listener. + # + # This will _not_ edit the document metadata. + # + # If there are any listeners, we don't purge the document from the cache. But be aware, this behaviour + # might change in a future version. + # + # version is the document version at which the document is opened. It can be left out if you want to open + # the document at the most recent version. + # + # listener is called with (opData) each time an op is applied. + # + # callback(error, openedVersion) + @listen = (docName, version, listener, callback) -> + [version, listener, callback] = [null, version, listener] if typeof version is 'function' + + load docName, (error, doc) -> + return callback? error if error + + clearTimeout doc.reapTimer + + if version? + getOps docName, version, null, (error, data) -> + return callback? error if error + + doc.eventEmitter.on 'op', listener + callback? null, version + for op in data + listener op + + # The listener may well remove itself during the catchup phase. If this happens, break early. + # This is done in a quite inefficient way. (O(n) where n = #listeners on doc) + break unless listener in doc.eventEmitter.listeners 'op' + + else # Version is null / undefined. Just add the listener. + doc.eventEmitter.on 'op', listener + callback? null, doc.v + + # Remove a listener for a particular document. + # + # removeListener(docName, listener) + # + # This is synchronous. + @removeListener = (docName, listener) -> + # The document should already be loaded. + doc = docs[docName] + throw new Error 'removeListener called but document not loaded' unless doc + + doc.eventEmitter.removeListener 'op', listener + refreshReapingTimeout docName + + # Flush saves all snapshot data to the database. I'm not sure whether or not this is actually needed - + # sharejs will happily replay uncommitted ops when documents are re-opened anyway. + @flush = (callback) -> + return callback?() unless db + + pendingWrites = 0 + + for docName, doc of docs + if doc.committedVersion < doc.v + pendingWrites++ + # I'm hoping writeSnapshot will always happen in another thread. + tryWriteSnapshot docName, -> + process.nextTick -> + pendingWrites-- + callback?() if pendingWrites is 0 + + # If nothing was queued, terminate immediately. + callback?() if pendingWrites is 0 + + # Close the database connection. This is needed so nodejs can shut down cleanly. + @closeDb = -> + db?.close?() + db = null + + return + +# Model inherits from EventEmitter. +Model:: = new EventEmitter + diff --git a/services/document-updater/app/coffee/sharejs/server/syncqueue.coffee b/services/document-updater/app/coffee/sharejs/server/syncqueue.coffee new file mode 100644 index 0000000000..746450b010 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/server/syncqueue.coffee @@ -0,0 +1,42 @@ +# A synchronous processing queue. The queue calls process on the arguments, +# ensuring that process() is only executing once at a time. +# +# process(data, callback) _MUST_ eventually call its callback. +# +# Example: +# +# queue = require 'syncqueue' +# +# fn = queue (data, callback) -> +# asyncthing data, -> +# callback(321) +# +# fn(1) +# fn(2) +# fn(3, (result) -> console.log(result)) +# +# ^--- async thing will only be running once at any time. + +module.exports = (process) -> + throw new Error('process is not a function') unless typeof process == 'function' + queue = [] + + enqueue = (data, callback) -> + queue.push [data, callback] + flush() + + enqueue.busy = false + + flush = -> + return if enqueue.busy or queue.length == 0 + + enqueue.busy = true + [data, callback] = queue.shift() + process data, (result...) -> # TODO: Make this not use varargs - varargs are really slow. + enqueue.busy = false + # This is called after busy = false so a user can check if enqueue.busy is set in the callback. + callback.apply null, result if callback + flush() + + enqueue + diff --git a/services/document-updater/app/coffee/sharejs/simple.coffee b/services/document-updater/app/coffee/sharejs/simple.coffee new file mode 100644 index 0000000000..996b1a5ddc --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/simple.coffee @@ -0,0 +1,38 @@ +# This is a really simple OT type. Its not compiled with the web client, but it could be. +# +# Its mostly included for demonstration purposes and its used in a lot of unit tests. +# +# This defines a really simple text OT type which only allows inserts. (No deletes). +# +# Ops look like: +# {position:#, text:"asdf"} +# +# Document snapshots look like: +# {str:string} + +module.exports = + # The name of the OT type. The type is stored in types[type.name]. The name can be + # used in place of the actual type in all the API methods. + name: 'simple' + + # Create a new document snapshot + create: -> {str:""} + + # Apply the given op to the document snapshot. Returns the new snapshot. + # + # The original snapshot should not be modified. + apply: (snapshot, op) -> + throw new Error 'Invalid position' unless 0 <= op.position <= snapshot.str.length + + str = snapshot.str + str = str.slice(0, op.position) + op.text + str.slice(op.position) + {str} + + # transform op1 by op2. Return transformed version of op1. + # sym describes the symmetry of the op. Its 'left' or 'right' depending on whether the + # op being transformed comes from the client or the server. + transform: (op1, op2, sym) -> + pos = op1.position + pos += op2.text.length if op2.position < pos or (op2.position == pos and sym is 'left') + + return {position:pos, text:op1.text} diff --git a/services/document-updater/app/coffee/sharejs/syncqueue.coffee b/services/document-updater/app/coffee/sharejs/syncqueue.coffee new file mode 100644 index 0000000000..746450b010 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/syncqueue.coffee @@ -0,0 +1,42 @@ +# A synchronous processing queue. The queue calls process on the arguments, +# ensuring that process() is only executing once at a time. +# +# process(data, callback) _MUST_ eventually call its callback. +# +# Example: +# +# queue = require 'syncqueue' +# +# fn = queue (data, callback) -> +# asyncthing data, -> +# callback(321) +# +# fn(1) +# fn(2) +# fn(3, (result) -> console.log(result)) +# +# ^--- async thing will only be running once at any time. + +module.exports = (process) -> + throw new Error('process is not a function') unless typeof process == 'function' + queue = [] + + enqueue = (data, callback) -> + queue.push [data, callback] + flush() + + enqueue.busy = false + + flush = -> + return if enqueue.busy or queue.length == 0 + + enqueue.busy = true + [data, callback] = queue.shift() + process data, (result...) -> # TODO: Make this not use varargs - varargs are really slow. + enqueue.busy = false + # This is called after busy = false so a user can check if enqueue.busy is set in the callback. + callback.apply null, result if callback + flush() + + enqueue + diff --git a/services/document-updater/app/coffee/sharejs/text-api.coffee b/services/document-updater/app/coffee/sharejs/text-api.coffee new file mode 100644 index 0000000000..96243ceffb --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/text-api.coffee @@ -0,0 +1,32 @@ +# Text document API for text + +text = require './text' if typeof WEB is 'undefined' + +text.api = + provides: {text:true} + + # The number of characters in the string + getLength: -> @snapshot.length + + # Get the text contents of a document + getText: -> @snapshot + + insert: (pos, text, callback) -> + op = [{p:pos, i:text}] + + @submitOp op, callback + op + + del: (pos, length, callback) -> + op = [{p:pos, d:@snapshot[pos...(pos + length)]}] + + @submitOp op, callback + op + + _register: -> + @on 'remoteop', (op) -> + for component in op + if component.i != undefined + @emit 'insert', component.p, component.i + else + @emit 'delete', component.p, component.d diff --git a/services/document-updater/app/coffee/sharejs/text-composable-api.coffee b/services/document-updater/app/coffee/sharejs/text-composable-api.coffee new file mode 100644 index 0000000000..7b27ac163a --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/text-composable-api.coffee @@ -0,0 +1,43 @@ +# Text document API for text + +if WEB? + type = exports.types['text-composable'] +else + type = require './text-composable' + +type.api = + provides: {'text':true} + + # The number of characters in the string + 'getLength': -> @snapshot.length + + # Get the text contents of a document + 'getText': -> @snapshot + + 'insert': (pos, text, callback) -> + op = type.normalize [pos, 'i':text, (@snapshot.length - pos)] + + @submitOp op, callback + op + + 'del': (pos, length, callback) -> + op = type.normalize [pos, 'd':@snapshot[pos...(pos + length)], (@snapshot.length - pos - length)] + + @submitOp op, callback + op + + _register: -> + @on 'remoteop', (op) -> + pos = 0 + for component in op + if typeof component is 'number' + pos += component + else if component.i != undefined + @emit 'insert', pos, component.i + pos += component.i.length + else + # delete + @emit 'delete', pos, component.d + # We don't increment pos, because the position + # specified is after the delete has happened. + diff --git a/services/document-updater/app/coffee/sharejs/text-composable.coffee b/services/document-updater/app/coffee/sharejs/text-composable.coffee new file mode 100644 index 0000000000..992b567bf0 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/text-composable.coffee @@ -0,0 +1,261 @@ +# An alternate composable implementation for text. This is much closer +# to the implementation used by google wave. +# +# Ops are lists of components which iterate over the whole document. +# Components are either: +# A number N: Skip N characters in the original document +# {i:'str'}: Insert 'str' at the current position in the document +# {d:'str'}: Delete 'str', which appears at the current position in the document +# +# Eg: [3, {i:'hi'}, 5, {d:'internet'}] +# +# Snapshots are strings. + +p = -> #require('util').debug +i = -> #require('util').inspect + +exports = if WEB? then {} else module.exports + +exports.name = 'text-composable' + +exports.create = -> '' + +# -------- Utility methods + +checkOp = (op) -> + throw new Error('Op must be an array of components') unless Array.isArray(op) + last = null + for c in op + if typeof(c) == 'object' + throw new Error("Invalid op component: #{i c}") unless (c.i? && c.i.length > 0) or (c.d? && c.d.length > 0) + else + throw new Error('Op components must be objects or numbers') unless typeof(c) == 'number' + throw new Error('Skip components must be a positive number') unless c > 0 + throw new Error('Adjacent skip components should be added') if typeof(last) == 'number' + + last = c + +# Makes a function for appending components to a given op. +# Exported for the randomOpGenerator. +exports._makeAppend = makeAppend = (op) -> (component) -> + if component == 0 || component.i == '' || component.d == '' + return + else if op.length == 0 + op.push component + else if typeof(component) == 'number' && typeof(op[op.length - 1]) == 'number' + op[op.length - 1] += component + else if component.i? && op[op.length - 1].i? + op[op.length - 1].i += component.i + else if component.d? && op[op.length - 1].d? + op[op.length - 1].d += component.d + else + op.push component + +# checkOp op + +# Makes 2 functions for taking components from the start of an op, and for peeking +# at the next op that could be taken. +makeTake = (op) -> + # The index of the next component to take + idx = 0 + # The offset into the component + offset = 0 + + # Take up to length n from the front of op. If n is null, take the next + # op component. If indivisableField == 'd', delete components won't be separated. + # If indivisableField == 'i', insert components won't be separated. + take = (n, indivisableField) -> + return null if idx == op.length + #assert.notStrictEqual op.length, i, 'The op is too short to traverse the document' + + if typeof(op[idx]) == 'number' + if !n? or op[idx] - offset <= n + c = op[idx] - offset + ++idx; offset = 0 + c + else + offset += n + n + else + # Take from the string + field = if op[idx].i then 'i' else 'd' + c = {} + if !n? or op[idx][field].length - offset <= n or field == indivisableField + c[field] = op[idx][field][offset..] + ++idx; offset = 0 + else + c[field] = op[idx][field][offset...(offset + n)] + offset += n + c + + peekType = () -> + op[idx] + + [take, peekType] + +# Find and return the length of an op component +componentLength = (component) -> + if typeof(component) == 'number' + component + else if component.i? + component.i.length + else + component.d.length + +# Normalize an op, removing all empty skips and empty inserts / deletes. Concatenate +# adjacent inserts and deletes. +exports.normalize = (op) -> + newOp = [] + append = makeAppend newOp + append component for component in op + newOp + +# Apply the op to the string. Returns the new string. +exports.apply = (str, op) -> + p "Applying #{i op} to '#{str}'" + throw new Error('Snapshot should be a string') unless typeof(str) == 'string' + checkOp op + + pos = 0 + newDoc = [] + + for component in op + if typeof(component) == 'number' + throw new Error('The op is too long for this document') if component > str.length + newDoc.push str[...component] + str = str[component..] + else if component.i? + newDoc.push component.i + else + throw new Error("The deleted text '#{component.d}' doesn't match the next characters in the document '#{str[...component.d.length]}'") unless component.d == str[...component.d.length] + str = str[component.d.length..] + + throw new Error("The applied op doesn't traverse the entire document") unless '' == str + + newDoc.join '' + +# transform op1 by op2. Return transformed version of op1. +# op1 and op2 are unchanged by transform. +exports.transform = (op, otherOp, side) -> + throw new Error "side (#{side} must be 'left' or 'right'" unless side == 'left' or side == 'right' + + checkOp op + checkOp otherOp + newOp = [] + + append = makeAppend newOp + [take, peek] = makeTake op + + for component in otherOp + if typeof(component) == 'number' # Skip + length = component + while length > 0 + chunk = take(length, 'i') + throw new Error('The op traverses more elements than the document has') unless chunk != null + + append chunk + length -= componentLength chunk unless typeof(chunk) == 'object' && chunk.i? + else if component.i? # Insert + if side == 'left' + # The left insert should go first. + o = peek() + append take() if o?.i + + # Otherwise, skip the inserted text. + append(component.i.length) + else # Delete. + #assert.ok component.d + length = component.d.length + while length > 0 + chunk = take(length, 'i') + throw new Error('The op traverses more elements than the document has') unless chunk != null + + if typeof(chunk) == 'number' + length -= chunk + else if chunk.i? + append(chunk) + else + #assert.ok chunk.d + # The delete is unnecessary now. + length -= chunk.d.length + + # Append extras from op1 + while (component = take()) + throw new Error "Remaining fragments in the op: #{i component}" unless component?.i? + append component + + newOp + + +# Compose 2 ops into 1 op. +exports.compose = (op1, op2) -> + p "COMPOSE #{i op1} + #{i op2}" + checkOp op1 + checkOp op2 + + result = [] + + append = makeAppend result + [take, _] = makeTake op1 + + for component in op2 + if typeof(component) == 'number' # Skip + length = component + while length > 0 + chunk = take(length, 'd') + throw new Error('The op traverses more elements than the document has') unless chunk != null + + append chunk + length -= componentLength chunk unless typeof(chunk) == 'object' && chunk.d? + + else if component.i? # Insert + append {i:component.i} + + else # Delete + offset = 0 + while offset < component.d.length + chunk = take(component.d.length - offset, 'd') + throw new Error('The op traverses more elements than the document has') unless chunk != null + + # If its delete, append it. If its skip, drop it and decrease length. If its insert, check the strings match, drop it and decrease length. + if typeof(chunk) == 'number' + append {d:component.d[offset...(offset + chunk)]} + offset += chunk + else if chunk.i? + throw new Error("The deleted text doesn't match the inserted text") unless component.d[offset...(offset + chunk.i.length)] == chunk.i + offset += chunk.i.length + # The ops cancel each other out. + else + # Delete + append chunk + + # Append extras from op1 + while (component = take()) + throw new Error "Trailing stuff in op1 #{i component}" unless component?.d? + append component + + result + + +invertComponent = (c) -> + if typeof(c) == 'number' + c + else if c.i? + {d:c.i} + else + {i:c.d} + +# Invert an op +exports.invert = (op) -> + result = [] + append = makeAppend result + + append(invertComponent component) for component in op + + result + +if window? + window.ot ||= {} + window.ot.types ||= {} + window.ot.types.text = exports + diff --git a/services/document-updater/app/coffee/sharejs/text-tp2-api.coffee b/services/document-updater/app/coffee/sharejs/text-tp2-api.coffee new file mode 100644 index 0000000000..d661b5ae37 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/text-tp2-api.coffee @@ -0,0 +1,89 @@ +# Text document API for text-tp2 + +if WEB? + type = exports.types['text-tp2'] +else + type = require './text-tp2' + +{_takeDoc:takeDoc, _append:append} = type + +appendSkipChars = (op, doc, pos, maxlength) -> + while (maxlength == undefined || maxlength > 0) and pos.index < doc.data.length + part = takeDoc doc, pos, maxlength, true + maxlength -= part.length if maxlength != undefined and typeof part is 'string' + append op, (part.length || part) + +type['api'] = + 'provides': {'text':true} + + # The number of characters in the string + 'getLength': -> @snapshot.charLength + + # Flatten a document into a string + 'getText': -> + strings = (elem for elem in @snapshot.data when typeof elem is 'string') + strings.join '' + + 'insert': (pos, text, callback) -> + pos = 0 if pos == undefined + + op = [] + docPos = {index:0, offset:0} + + appendSkipChars op, @snapshot, docPos, pos + append op, {'i':text} + appendSkipChars op, @snapshot, docPos + + @submitOp op, callback + op + + 'del': (pos, length, callback) -> + op = [] + docPos = {index:0, offset:0} + + appendSkipChars op, @snapshot, docPos, pos + + while length > 0 + part = takeDoc @snapshot, docPos, length, true + if typeof part is 'string' + append op, {'d':part.length} + length -= part.length + else + append op, part + + appendSkipChars op, @snapshot, docPos + + @submitOp op, callback + op + + '_register': -> + # Interpret recieved ops + generate more detailed events for them + @on 'remoteop', (op, snapshot) -> + textPos = 0 + docPos = {index:0, offset:0} + + for component in op + if typeof component is 'number' + # Skip + remainder = component + while remainder > 0 + part = takeDoc snapshot, docPos, remainder + if typeof part is 'string' + textPos += part.length + remainder -= part.length || part + else if component.i != undefined + # Insert + if typeof component.i is 'string' + @emit 'insert', textPos, component.i + textPos += component.i.length + else + # Delete + remainder = component.d + while remainder > 0 + part = takeDoc snapshot, docPos, remainder + if typeof part is 'string' + @emit 'delete', textPos, part + remainder -= part.length || part + + return + diff --git a/services/document-updater/app/coffee/sharejs/text-tp2.coffee b/services/document-updater/app/coffee/sharejs/text-tp2.coffee new file mode 100644 index 0000000000..d19cbdcef4 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/text-tp2.coffee @@ -0,0 +1,322 @@ +# A TP2 implementation of text, following this spec: +# http://code.google.com/p/lightwave/source/browse/trunk/experimental/ot/README +# +# A document is made up of a string and a set of tombstones inserted throughout +# the string. For example, 'some ', (2 tombstones), 'string'. +# +# This is encoded in a document as: {s:'some string', t:[5, -2, 6]} +# +# Ops are lists of components which iterate over the whole document. +# Components are either: +# N: Skip N characters in the original document +# {i:'str'}: Insert 'str' at the current position in the document +# {i:N}: Insert N tombstones at the current position in the document +# {d:N}: Delete (tombstone) N characters at the current position in the document +# +# Eg: [3, {i:'hi'}, 5, {d:8}] +# +# Snapshots are lists with characters and tombstones. Characters are stored in strings +# and adjacent tombstones are flattened into numbers. +# +# Eg, the document: 'Hello .....world' ('.' denotes tombstoned (deleted) characters) +# would be represented by a document snapshot of ['Hello ', 5, 'world'] + +type = + name: 'text-tp2' + tp2: true + create: -> {charLength:0, totalLength:0, positionCache:[], data:[]} + serialize: (doc) -> + throw new Error 'invalid doc snapshot' unless doc.data + doc.data + deserialize: (data) -> + doc = type.create() + doc.data = data + + for component in data + if typeof component is 'string' + doc.charLength += component.length + doc.totalLength += component.length + else + doc.totalLength += component + + doc + + +checkOp = (op) -> + throw new Error('Op must be an array of components') unless Array.isArray(op) + last = null + for c in op + if typeof(c) == 'object' + if c.i != undefined + throw new Error('Inserts must insert a string or a +ive number') unless (typeof(c.i) == 'string' and c.i.length > 0) or (typeof(c.i) == 'number' and c.i > 0) + else if c.d != undefined + throw new Error('Deletes must be a +ive number') unless typeof(c.d) == 'number' and c.d > 0 + else + throw new Error('Operation component must define .i or .d') + else + throw new Error('Op components must be objects or numbers') unless typeof(c) == 'number' + throw new Error('Skip components must be a positive number') unless c > 0 + throw new Error('Adjacent skip components should be combined') if typeof(last) == 'number' + + last = c + +# Take the next part from the specified position in a document snapshot. +# position = {index, offset}. It will be updated. +type._takeDoc = takeDoc = (doc, position, maxlength, tombsIndivisible) -> + throw new Error 'Operation goes past the end of the document' if position.index >= doc.data.length + + part = doc.data[position.index] + # peel off data[0] + result = if typeof(part) == 'string' + if maxlength != undefined + part[position.offset...(position.offset + maxlength)] + else + part[position.offset...] + else + if maxlength == undefined or tombsIndivisible + part - position.offset + else + Math.min(maxlength, part - position.offset) + + resultLen = result.length || result + + if (part.length || part) - position.offset > resultLen + position.offset += resultLen + else + position.index++ + position.offset = 0 + + result + +# Append a part to the end of a document +type._appendDoc = appendDoc = (doc, p) -> + return if p == 0 or p == '' + + if typeof p is 'string' + doc.charLength += p.length + doc.totalLength += p.length + else + doc.totalLength += p + + data = doc.data + if data.length == 0 + data.push p + else if typeof(data[data.length - 1]) == typeof(p) + data[data.length - 1] += p + else + data.push p + return + +# Apply the op to the document. The document is not modified in the process. +type.apply = (doc, op) -> + unless doc.totalLength != undefined and doc.charLength != undefined and doc.data.length != undefined + throw new Error('Snapshot is invalid') + + checkOp op + + newDoc = type.create() + position = {index:0, offset:0} + + for component in op + if typeof(component) is 'number' + remainder = component + while remainder > 0 + part = takeDoc doc, position, remainder + + appendDoc newDoc, part + remainder -= part.length || part + + else if component.i != undefined + appendDoc newDoc, component.i + else if component.d != undefined + remainder = component.d + while remainder > 0 + part = takeDoc doc, position, remainder + remainder -= part.length || part + appendDoc newDoc, component.d + + newDoc + +# Append an op component to the end of the specified op. +# Exported for the randomOpGenerator. +type._append = append = (op, component) -> + if component == 0 || component.i == '' || component.i == 0 || component.d == 0 + return + else if op.length == 0 + op.push component + else + last = op[op.length - 1] + if typeof(component) == 'number' && typeof(last) == 'number' + op[op.length - 1] += component + else if component.i != undefined && last.i? && typeof(last.i) == typeof(component.i) + last.i += component.i + else if component.d != undefined && last.d? + last.d += component.d + else + op.push component + +# Makes 2 functions for taking components from the start of an op, and for peeking +# at the next op that could be taken. +makeTake = (op) -> + # The index of the next component to take + index = 0 + # The offset into the component + offset = 0 + + # Take up to length maxlength from the op. If maxlength is not defined, there is no max. + # If insertsIndivisible is true, inserts (& insert tombstones) won't be separated. + # + # Returns null when op is fully consumed. + take = (maxlength, insertsIndivisible) -> + return null if index == op.length + + e = op[index] + if typeof((current = e)) == 'number' or typeof((current = e.i)) == 'number' or (current = e.d) != undefined + if !maxlength? or current - offset <= maxlength or (insertsIndivisible and e.i != undefined) + # Return the rest of the current element. + c = current - offset + ++index; offset = 0 + else + offset += maxlength + c = maxlength + if e.i != undefined then {i:c} else if e.d != undefined then {d:c} else c + else + # Take from the inserted string + if !maxlength? or e.i.length - offset <= maxlength or insertsIndivisible + result = {i:e.i[offset..]} + ++index; offset = 0 + else + result = {i:e.i[offset...offset + maxlength]} + offset += maxlength + result + + peekType = -> op[index] + + [take, peekType] + +# Find and return the length of an op component +componentLength = (component) -> + if typeof(component) == 'number' + component + else if typeof(component.i) == 'string' + component.i.length + else + # This should work because c.d and c.i must be +ive. + component.d or component.i + +# Normalize an op, removing all empty skips and empty inserts / deletes. Concatenate +# adjacent inserts and deletes. +type.normalize = (op) -> + newOp = [] + append newOp, component for component in op + newOp + +# This is a helper method to transform and prune. goForwards is true for transform, false for prune. +transformer = (op, otherOp, goForwards, side) -> + checkOp op + checkOp otherOp + newOp = [] + + [take, peek] = makeTake op + + for component in otherOp + length = componentLength component + + if component.i != undefined # Insert text or tombs + if goForwards # transform - insert skips over inserted parts + if side == 'left' + # The left insert should go first. + append newOp, take() while peek()?.i != undefined + + # In any case, skip the inserted text. + append newOp, length + + else # Prune. Remove skips for inserts. + while length > 0 + chunk = take length, true + + throw new Error 'The transformed op is invalid' unless chunk != null + throw new Error 'The transformed op deletes locally inserted characters - it cannot be purged of the insert.' if chunk.d != undefined + + if typeof chunk is 'number' + length -= chunk + else + append newOp, chunk + + else # Skip or delete + while length > 0 + chunk = take length, true + throw new Error('The op traverses more elements than the document has') unless chunk != null + + append newOp, chunk + length -= componentLength chunk unless chunk.i + + # Append extras from op1 + while (component = take()) + throw new Error "Remaining fragments in the op: #{component}" unless component.i != undefined + append newOp, component + + newOp + +# transform op1 by op2. Return transformed version of op1. +# op1 and op2 are unchanged by transform. +# side should be 'left' or 'right', depending on if op1.id <> op2.id. 'left' == client op. +type.transform = (op, otherOp, side) -> + throw new Error "side (#{side}) should be 'left' or 'right'" unless side == 'left' or side == 'right' + transformer op, otherOp, true, side + +# Prune is the inverse of transform. +type.prune = (op, otherOp) -> transformer op, otherOp, false + +# Compose 2 ops into 1 op. +type.compose = (op1, op2) -> + return op2 if op1 == null or op1 == undefined + + checkOp op1 + checkOp op2 + + result = [] + + [take, _] = makeTake op1 + + for component in op2 + + if typeof(component) == 'number' # Skip + # Just copy from op1. + length = component + while length > 0 + chunk = take length + throw new Error('The op traverses more elements than the document has') unless chunk != null + + append result, chunk + length -= componentLength chunk + + else if component.i != undefined # Insert + append result, {i:component.i} + + else # Delete + length = component.d + while length > 0 + chunk = take length + throw new Error('The op traverses more elements than the document has') unless chunk != null + + chunkLength = componentLength chunk + if chunk.i != undefined + append result, {i:chunkLength} + else + append result, {d:chunkLength} + + length -= chunkLength + + # Append extras from op1 + while (component = take()) + throw new Error "Remaining fragments in op1: #{component}" unless component.i != undefined + append result, component + + result + +if WEB? + exports.types['text-tp2'] = type +else + module.exports = type + diff --git a/services/document-updater/app/coffee/sharejs/text.coffee b/services/document-updater/app/coffee/sharejs/text.coffee new file mode 100644 index 0000000000..c64b4dfa68 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/text.coffee @@ -0,0 +1,209 @@ +# A simple text implementation +# +# Operations are lists of components. +# Each component either inserts or deletes at a specified position in the document. +# +# Components are either: +# {i:'str', p:100}: Insert 'str' at position 100 in the document +# {d:'str', p:100}: Delete 'str' at position 100 in the document +# +# Components in an operation are executed sequentially, so the position of components +# assumes previous components have already executed. +# +# Eg: This op: +# [{i:'abc', p:0}] +# is equivalent to this op: +# [{i:'a', p:0}, {i:'b', p:1}, {i:'c', p:2}] + +# NOTE: The global scope here is shared with other sharejs files when built with closure. +# Be careful what ends up in your namespace. + +text = {} + +text.name = 'text' + +text.create = -> '' + +strInject = (s1, pos, s2) -> s1[...pos] + s2 + s1[pos..] + +checkValidComponent = (c) -> + throw new Error 'component missing position field' if typeof c.p != 'number' + + i_type = typeof c.i + d_type = typeof c.d + throw new Error 'component needs an i or d field' unless (i_type == 'string') ^ (d_type == 'string') + + throw new Error 'position cannot be negative' unless c.p >= 0 + +checkValidOp = (op) -> + checkValidComponent(c) for c in op + true + +text.apply = (snapshot, op) -> + checkValidOp op + for component in op + if component.i? + snapshot = strInject snapshot, component.p, component.i + else + deleted = snapshot[component.p...(component.p + component.d.length)] + throw new Error "Delete component '#{component.d}' does not match deleted text '#{deleted}'" unless component.d == deleted + snapshot = snapshot[...component.p] + snapshot[(component.p + component.d.length)..] + + snapshot + + +# Exported for use by the random op generator. +# +# For simplicity, this version of append does not compress adjacent inserts and deletes of +# the same text. It would be nice to change that at some stage. +text._append = append = (newOp, c) -> + return if c.i == '' or c.d == '' + if newOp.length == 0 + newOp.push c + else + last = newOp[newOp.length - 1] + + # Compose the insert into the previous insert if possible + if last.i? && c.i? and last.p <= c.p <= (last.p + last.i.length) + newOp[newOp.length - 1] = {i:strInject(last.i, c.p - last.p, c.i), p:last.p} + else if last.d? && c.d? and c.p <= last.p <= (c.p + c.d.length) + newOp[newOp.length - 1] = {d:strInject(c.d, last.p - c.p, last.d), p:c.p} + else + newOp.push c + +text.compose = (op1, op2) -> + checkValidOp op1 + checkValidOp op2 + + newOp = op1.slice() + append newOp, c for c in op2 + + newOp + +# Attempt to compress the op components together 'as much as possible'. +# This implementation preserves order and preserves create/delete pairs. +text.compress = (op) -> text.compose [], op + +text.normalize = (op) -> + newOp = [] + + # Normalize should allow ops which are a single (unwrapped) component: + # {i:'asdf', p:23}. + # There's no good way to test if something is an array: + # http://perfectionkills.com/instanceof-considered-harmful-or-how-to-write-a-robust-isarray/ + # so this is probably the least bad solution. + op = [op] if op.i? or op.p? + + for c in op + c.p ?= 0 + append newOp, c + + newOp + +# This helper method transforms a position by an op component. +# +# If c is an insert, insertAfter specifies whether the transform +# is pushed after the insert (true) or before it (false). +# +# insertAfter is optional for deletes. +transformPosition = (pos, c, insertAfter) -> + if c.i? + if c.p < pos || (c.p == pos && insertAfter) + pos + c.i.length + else + pos + else + # I think this could also be written as: Math.min(c.p, Math.min(c.p - otherC.p, otherC.d.length)) + # but I think its harder to read that way, and it compiles using ternary operators anyway + # so its no slower written like this. + if pos <= c.p + pos + else if pos <= c.p + c.d.length + c.p + else + pos - c.d.length + +# Helper method to transform a cursor position as a result of an op. +# +# Like transformPosition above, if c is an insert, insertAfter specifies whether the cursor position +# is pushed after an insert (true) or before it (false). +text.transformCursor = (position, op, side) -> + insertAfter = side == 'right' + position = transformPosition position, c, insertAfter for c in op + position + +# Transform an op component by another op component. Asymmetric. +# The result will be appended to destination. +# +# exported for use in JSON type +text._tc = transformComponent = (dest, c, otherC, side) -> + checkValidOp [c] + checkValidOp [otherC] + + if c.i? + append dest, {i:c.i, p:transformPosition(c.p, otherC, side == 'right')} + + else # Delete + if otherC.i? # delete vs insert + s = c.d + if c.p < otherC.p + append dest, {d:s[...otherC.p - c.p], p:c.p} + s = s[(otherC.p - c.p)..] + if s != '' + append dest, {d:s, p:c.p + otherC.i.length} + + else # Delete vs delete + if c.p >= otherC.p + otherC.d.length + append dest, {d:c.d, p:c.p - otherC.d.length} + else if c.p + c.d.length <= otherC.p + append dest, c + else + # They overlap somewhere. + newC = {d:'', p:c.p} + if c.p < otherC.p + newC.d = c.d[...(otherC.p - c.p)] + if c.p + c.d.length > otherC.p + otherC.d.length + newC.d += c.d[(otherC.p + otherC.d.length - c.p)..] + + # This is entirely optional - just for a check that the deleted + # text in the two ops matches + intersectStart = Math.max c.p, otherC.p + intersectEnd = Math.min c.p + c.d.length, otherC.p + otherC.d.length + cIntersect = c.d[intersectStart - c.p...intersectEnd - c.p] + otherIntersect = otherC.d[intersectStart - otherC.p...intersectEnd - otherC.p] + throw new Error 'Delete ops delete different text in the same region of the document' unless cIntersect == otherIntersect + + if newC.d != '' + # This could be rewritten similarly to insert v delete, above. + newC.p = transformPosition newC.p, otherC + append dest, newC + + dest + +invertComponent = (c) -> + if c.i? + {d:c.i, p:c.p} + else + {i:c.d, p:c.p} + +# No need to use append for invert, because the components won't be able to +# cancel with one another. +text.invert = (op) -> (invertComponent c for c in op.slice().reverse()) + + +if WEB? + exports.types ||= {} + + # This is kind of awful - come up with a better way to hook this helper code up. + bootstrapTransform(text, transformComponent, checkValidOp, append) + + # [] is used to prevent closure from renaming types.text + exports.types.text = text +else + module.exports = text + + # The text type really shouldn't need this - it should be possible to define + # an efficient transform function by making a sort of transform map and passing each + # op component through it. + require('./helpers').bootstrapTransform(text, transformComponent, checkValidOp, append) + diff --git a/services/document-updater/app/coffee/sharejs/types/count.coffee b/services/document-updater/app/coffee/sharejs/types/count.coffee new file mode 100644 index 0000000000..da28355efb --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/types/count.coffee @@ -0,0 +1,22 @@ +# This is a simple type used for testing other OT code. Each op is [expectedSnapshot, increment] + +exports.name = 'count' +exports.create = -> 1 + +exports.apply = (snapshot, op) -> + [v, inc] = op + throw new Error "Op #{v} != snapshot #{snapshot}" unless snapshot == v + snapshot + inc + +# transform op1 by op2. Return transformed version of op1. +exports.transform = (op1, op2) -> + throw new Error "Op1 #{op1[0]} != op2 #{op2[0]}" unless op1[0] == op2[0] + [op1[0] + op2[1], op1[1]] + +exports.compose = (op1, op2) -> + throw new Error "Op1 #{op1} + 1 != op2 #{op2}" unless op1[0] + op1[1] == op2[0] + [op1[0], op1[1] + op2[1]] + +exports.generateRandomOp = (doc) -> + [[doc, 1], doc + 1] + diff --git a/services/document-updater/app/coffee/sharejs/types/helpers.coffee b/services/document-updater/app/coffee/sharejs/types/helpers.coffee new file mode 100644 index 0000000000..093b32e1bb --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/types/helpers.coffee @@ -0,0 +1,65 @@ +# These methods let you build a transform function from a transformComponent function +# for OT types like text and JSON in which operations are lists of components +# and transforming them requires N^2 work. + +# Add transform and transformX functions for an OT type which has transformComponent defined. +# transformComponent(destination array, component, other component, side) +exports['_bt'] = bootstrapTransform = (type, transformComponent, checkValidOp, append) -> + transformComponentX = (left, right, destLeft, destRight) -> + transformComponent destLeft, left, right, 'left' + transformComponent destRight, right, left, 'right' + + # Transforms rightOp by leftOp. Returns ['rightOp', clientOp'] + type.transformX = type['transformX'] = transformX = (leftOp, rightOp) -> + checkValidOp leftOp + checkValidOp rightOp + + newRightOp = [] + + for rightComponent in rightOp + # Generate newLeftOp by composing leftOp by rightComponent + newLeftOp = [] + + k = 0 + while k < leftOp.length + nextC = [] + transformComponentX leftOp[k], rightComponent, newLeftOp, nextC + k++ + + if nextC.length == 1 + rightComponent = nextC[0] + else if nextC.length == 0 + append newLeftOp, l for l in leftOp[k..] + rightComponent = null + break + else + # Recurse. + [l_, r_] = transformX leftOp[k..], nextC + append newLeftOp, l for l in l_ + append newRightOp, r for r in r_ + rightComponent = null + break + + append newRightOp, rightComponent if rightComponent? + leftOp = newLeftOp + + [leftOp, newRightOp] + + # Transforms op with specified type ('left' or 'right') by otherOp. + type.transform = type['transform'] = (op, otherOp, type) -> + throw new Error "type must be 'left' or 'right'" unless type == 'left' or type == 'right' + + return op if otherOp.length == 0 + + # TODO: Benchmark with and without this line. I _think_ it'll make a big difference...? + return transformComponent [], op[0], otherOp[0], type if op.length == 1 and otherOp.length == 1 + + if type == 'left' + [left, _] = transformX op, otherOp + left + else + [_, right] = transformX otherOp, op + right + +if typeof WEB is 'undefined' + exports.bootstrapTransform = bootstrapTransform diff --git a/services/document-updater/app/coffee/sharejs/types/index.coffee b/services/document-updater/app/coffee/sharejs/types/index.coffee new file mode 100644 index 0000000000..6f3bb8ec20 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/types/index.coffee @@ -0,0 +1,15 @@ + +register = (file) -> + type = require file + exports[type.name] = type + try require "#{file}-api" + +# Import all the built-in types. +register './simple' +register './count' + +register './text' +register './text-composable' +register './text-tp2' + +register './json' diff --git a/services/document-updater/app/coffee/sharejs/types/json-api.coffee b/services/document-updater/app/coffee/sharejs/types/json-api.coffee new file mode 100644 index 0000000000..8819dee798 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/types/json-api.coffee @@ -0,0 +1,180 @@ +# API for JSON OT + +json = require './json' if typeof WEB is 'undefined' + +if WEB? + extendDoc = exports.extendDoc + exports.extendDoc = (name, fn) -> + SubDoc::[name] = fn + extendDoc name, fn + +depath = (path) -> + if path.length == 1 and path[0].constructor == Array + path[0] + else path + +class SubDoc + constructor: (@doc, @path) -> + at: (path...) -> @doc.at @path.concat depath path + get: -> @doc.getAt @path + # for objects and lists + set: (value, cb) -> @doc.setAt @path, value, cb + # for strings and lists. + insert: (pos, value, cb) -> @doc.insertAt @path, pos, value, cb + # for strings + del: (pos, length, cb) -> @doc.deleteTextAt @path, length, pos, cb + # for objects and lists + remove: (cb) -> @doc.removeAt @path, cb + push: (value, cb) -> @insert @get().length, value, cb + move: (from, to, cb) -> @doc.moveAt @path, from, to, cb + add: (amount, cb) -> @doc.addAt @path, amount, cb + on: (event, cb) -> @doc.addListener @path, event, cb + removeListener: (l) -> @doc.removeListener l + + # text API compatibility + getLength: -> @get().length + getText: -> @get() + +traverse = (snapshot, path) -> + container = data:snapshot + key = 'data' + elem = container + for p in path + elem = elem[key] + key = p + throw new Error 'bad path' if typeof elem == 'undefined' + {elem, key} + +pathEquals = (p1, p2) -> + return false if p1.length != p2.length + for e,i in p1 + return false if e != p2[i] + true + +json.api = + provides: {json:true} + + at: (path...) -> new SubDoc this, depath path + + get: -> @snapshot + set: (value, cb) -> @setAt [], value, cb + + getAt: (path) -> + {elem, key} = traverse @snapshot, path + return elem[key] + + setAt: (path, value, cb) -> + {elem, key} = traverse @snapshot, path + op = {p:path} + if elem.constructor == Array + op.li = value + op.ld = elem[key] if typeof elem[key] != 'undefined' + else if typeof elem == 'object' + op.oi = value + op.od = elem[key] if typeof elem[key] != 'undefined' + else throw new Error 'bad path' + @submitOp [op], cb + + removeAt: (path, cb) -> + {elem, key} = traverse @snapshot, path + throw new Error 'no element at that path' unless typeof elem[key] != 'undefined' + op = {p:path} + if elem.constructor == Array + op.ld = elem[key] + else if typeof elem == 'object' + op.od = elem[key] + else throw new Error 'bad path' + @submitOp [op], cb + + insertAt: (path, pos, value, cb) -> + {elem, key} = traverse @snapshot, path + op = {p:path.concat pos} + if elem[key].constructor == Array + op.li = value + else if typeof elem[key] == 'string' + op.si = value + @submitOp [op], cb + + moveAt: (path, from, to, cb) -> + op = [{p:path.concat(from), lm:to}] + @submitOp op, cb + + addAt: (path, amount, cb) -> + op = [{p:path, na:amount}] + @submitOp op, cb + + deleteTextAt: (path, length, pos, cb) -> + {elem, key} = traverse @snapshot, path + op = [{p:path.concat(pos), sd:elem[key][pos...(pos + length)]}] + @submitOp op, cb + + addListener: (path, event, cb) -> + l = {path, event, cb} + @_listeners.push l + l + removeListener: (l) -> + i = @_listeners.indexOf l + return false if i < 0 + @_listeners.splice i, 1 + return true + _register: -> + @_listeners = [] + @on 'change', (op) -> + for c in op + if c.na != undefined or c.si != undefined or c.sd != undefined + # no change to structure + continue + to_remove = [] + for l, i in @_listeners + # Transform a dummy op by the incoming op to work out what + # should happen to the listener. + dummy = {p:l.path, na:0} + xformed = @type.transformComponent [], dummy, c, 'left' + if xformed.length == 0 + # The op was transformed to noop, so we should delete the listener. + to_remove.push i + else if xformed.length == 1 + # The op remained, so grab its new path into the listener. + l.path = xformed[0].p + else + throw new Error "Bad assumption in json-api: xforming an 'si' op will always result in 0 or 1 components." + to_remove.sort (a, b) -> b - a + for i in to_remove + @_listeners.splice i, 1 + @on 'remoteop', (op) -> + for c in op + match_path = if c.na == undefined then c.p[...c.p.length-1] else c.p + for {path, event, cb} in @_listeners + if pathEquals path, match_path + switch event + when 'insert' + if c.li != undefined and c.ld == undefined + cb(c.p[c.p.length-1], c.li) + else if c.oi != undefined and c.od == undefined + cb(c.p[c.p.length-1], c.oi) + else if c.si != undefined + cb(c.p[c.p.length-1], c.si) + when 'delete' + if c.li == undefined and c.ld != undefined + cb(c.p[c.p.length-1], c.ld) + else if c.oi == undefined and c.od != undefined + cb(c.p[c.p.length-1], c.od) + else if c.sd != undefined + cb(c.p[c.p.length-1], c.sd) + when 'replace' + if c.li != undefined and c.ld != undefined + cb(c.p[c.p.length-1], c.ld, c.li) + else if c.oi != undefined and c.od != undefined + cb(c.p[c.p.length-1], c.od, c.oi) + when 'move' + if c.lm != undefined + cb(c.p[c.p.length-1], c.lm) + when 'add' + if c.na != undefined + cb(c.na) + else if (common = @type.commonPath match_path, path)? + if event == 'child op' + if match_path.length == path.length == common + throw new Error "paths match length and have commonality, but aren't equal?" + child_path = c.p[common+1..] + cb(child_path, c) diff --git a/services/document-updater/app/coffee/sharejs/types/json.coffee b/services/document-updater/app/coffee/sharejs/types/json.coffee new file mode 100644 index 0000000000..b03b0947ef --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/types/json.coffee @@ -0,0 +1,441 @@ +# This is the implementation of the JSON OT type. +# +# Spec is here: https://github.com/josephg/ShareJS/wiki/JSON-Operations + +if WEB? + text = exports.types.text +else + text = require './text' + +json = {} + +json.name = 'json' + +json.create = -> null + +json.invertComponent = (c) -> + c_ = {p: c.p} + c_.sd = c.si if c.si != undefined + c_.si = c.sd if c.sd != undefined + c_.od = c.oi if c.oi != undefined + c_.oi = c.od if c.od != undefined + c_.ld = c.li if c.li != undefined + c_.li = c.ld if c.ld != undefined + c_.na = -c.na if c.na != undefined + if c.lm != undefined + c_.lm = c.p[c.p.length-1] + c_.p = c.p[0...c.p.length - 1].concat([c.lm]) + c_ + +json.invert = (op) -> json.invertComponent c for c in op.slice().reverse() + +json.checkValidOp = (op) -> + +isArray = (o) -> Object.prototype.toString.call(o) == '[object Array]' +json.checkList = (elem) -> + throw new Error 'Referenced element not a list' unless isArray(elem) + +json.checkObj = (elem) -> + throw new Error "Referenced element not an object (it was #{JSON.stringify elem})" unless elem.constructor is Object + +json.apply = (snapshot, op) -> + json.checkValidOp op + op = clone op + + container = {data: clone snapshot} + + try + for c, i in op + parent = null + parentkey = null + elem = container + key = 'data' + + for p in c.p + parent = elem + parentkey = key + elem = elem[key] + key = p + + throw new Error 'Path invalid' unless parent? + + if c.na != undefined + # Number add + throw new Error 'Referenced element not a number' unless typeof elem[key] is 'number' + elem[key] += c.na + + else if c.si != undefined + # String insert + throw new Error "Referenced element not a string (it was #{JSON.stringify elem})" unless typeof elem is 'string' + parent[parentkey] = elem[...key] + c.si + elem[key..] + else if c.sd != undefined + # String delete + throw new Error 'Referenced element not a string' unless typeof elem is 'string' + throw new Error 'Deleted string does not match' unless elem[key...key + c.sd.length] == c.sd + parent[parentkey] = elem[...key] + elem[key + c.sd.length..] + + else if c.li != undefined && c.ld != undefined + # List replace + json.checkList elem + + # Should check the list element matches c.ld + elem[key] = c.li + else if c.li != undefined + # List insert + json.checkList elem + + elem.splice key, 0, c.li + else if c.ld != undefined + # List delete + json.checkList elem + + # Should check the list element matches c.ld here too. + elem.splice key, 1 + else if c.lm != undefined + # List move + json.checkList elem + if c.lm != key + e = elem[key] + # Remove it... + elem.splice key, 1 + # And insert it back. + elem.splice c.lm, 0, e + + else if c.oi != undefined + # Object insert / replace + json.checkObj elem + + # Should check that elem[key] == c.od + elem[key] = c.oi + else if c.od != undefined + # Object delete + json.checkObj elem + + # Should check that elem[key] == c.od + delete elem[key] + else + throw new Error 'invalid / missing instruction in op' + catch error + # TODO: Roll back all already applied changes. Write tests before implementing this code. + throw error + + container.data + +# Checks if two paths, p1 and p2 match. +json.pathMatches = (p1, p2, ignoreLast) -> + return false unless p1.length == p2.length + + for p, i in p1 + return false if p != p2[i] and (!ignoreLast or i != p1.length - 1) + + true + +json.append = (dest, c) -> + c = clone c + if dest.length != 0 and json.pathMatches c.p, (last = dest[dest.length - 1]).p + if last.na != undefined and c.na != undefined + dest[dest.length - 1] = { p: last.p, na: last.na + c.na } + else if last.li != undefined and c.li == undefined and c.ld == last.li + # insert immediately followed by delete becomes a noop. + if last.ld != undefined + # leave the delete part of the replace + delete last.li + else + dest.pop() + else if last.od != undefined and last.oi == undefined and + c.oi != undefined and c.od == undefined + last.oi = c.oi + else if c.lm != undefined and c.p[c.p.length-1] == c.lm + null # don't do anything + else + dest.push c + else + dest.push c + +json.compose = (op1, op2) -> + json.checkValidOp op1 + json.checkValidOp op2 + + newOp = clone op1 + json.append newOp, c for c in op2 + + newOp + +json.normalize = (op) -> + newOp = [] + + op = [op] unless isArray op + + for c in op + c.p ?= [] + json.append newOp, c + + newOp + +# hax, copied from test/types/json. Apparently this is still the fastest way to deep clone an object, assuming +# we have browser support for JSON. +# http://jsperf.com/cloning-an-object/12 +clone = (o) -> JSON.parse(JSON.stringify o) + +json.commonPath = (p1, p2) -> + p1 = p1.slice() + p2 = p2.slice() + p1.unshift('data') + p2.unshift('data') + p1 = p1[...p1.length-1] + p2 = p2[...p2.length-1] + return -1 if p2.length == 0 + i = 0 + while p1[i] == p2[i] && i < p1.length + i++ + if i == p2.length + return i-1 + return + +# transform c so it applies to a document with otherC applied. +json.transformComponent = (dest, c, otherC, type) -> + c = clone c + c.p.push(0) if c.na != undefined + otherC.p.push(0) if otherC.na != undefined + + common = json.commonPath c.p, otherC.p + common2 = json.commonPath otherC.p, c.p + + cplength = c.p.length + otherCplength = otherC.p.length + + c.p.pop() if c.na != undefined # hax + otherC.p.pop() if otherC.na != undefined + + if otherC.na + if common2? && otherCplength >= cplength && otherC.p[common2] == c.p[common2] + if c.ld != undefined + oc = clone otherC + oc.p = oc.p[cplength..] + c.ld = json.apply clone(c.ld), [oc] + else if c.od != undefined + oc = clone otherC + oc.p = oc.p[cplength..] + c.od = json.apply clone(c.od), [oc] + json.append dest, c + return dest + + if common2? && otherCplength > cplength && c.p[common2] == otherC.p[common2] + # transform based on c + if c.ld != undefined + oc = clone otherC + oc.p = oc.p[cplength..] + c.ld = json.apply clone(c.ld), [oc] + else if c.od != undefined + oc = clone otherC + oc.p = oc.p[cplength..] + c.od = json.apply clone(c.od), [oc] + + + if common? + commonOperand = cplength == otherCplength + # transform based on otherC + if otherC.na != undefined + # this case is handled above due to icky path hax + else if otherC.si != undefined || otherC.sd != undefined + # String op vs string op - pass through to text type + if c.si != undefined || c.sd != undefined + throw new Error("must be a string?") unless commonOperand + + # Convert an op component to a text op component + convert = (component) -> + newC = p:component.p[component.p.length - 1] + if component.si + newC.i = component.si + else + newC.d = component.sd + newC + + tc1 = convert c + tc2 = convert otherC + + res = [] + text._tc res, tc1, tc2, type + for tc in res + jc = { p: c.p[...common] } + jc.p.push(tc.p) + jc.si = tc.i if tc.i? + jc.sd = tc.d if tc.d? + json.append dest, jc + return dest + else if otherC.li != undefined && otherC.ld != undefined + if otherC.p[common] == c.p[common] + # noop + if !commonOperand + # we're below the deleted element, so -> noop + return dest + else if c.ld != undefined + # we're trying to delete the same element, -> noop + if c.li != undefined and type == 'left' + # we're both replacing one element with another. only one can + # survive! + c.ld = clone otherC.li + else + return dest + else if otherC.li != undefined + if c.li != undefined and c.ld == undefined and commonOperand and c.p[common] == otherC.p[common] + # in li vs. li, left wins. + if type == 'right' + c.p[common]++ + else if otherC.p[common] <= c.p[common] + c.p[common]++ + + if c.lm != undefined + if commonOperand + # otherC edits the same list we edit + if otherC.p[common] <= c.lm + c.lm++ + # changing c.from is handled above. + else if otherC.ld != undefined + if c.lm != undefined + if commonOperand + if otherC.p[common] == c.p[common] + # they deleted the thing we're trying to move + return dest + # otherC edits the same list we edit + p = otherC.p[common] + from = c.p[common] + to = c.lm + if p < to || (p == to && from < to) + c.lm-- + + if otherC.p[common] < c.p[common] + c.p[common]-- + else if otherC.p[common] == c.p[common] + if otherCplength < cplength + # we're below the deleted element, so -> noop + return dest + else if c.ld != undefined + if c.li != undefined + # we're replacing, they're deleting. we become an insert. + delete c.ld + else + # we're trying to delete the same element, -> noop + return dest + else if otherC.lm != undefined + if c.lm != undefined and cplength == otherCplength + # lm vs lm, here we go! + from = c.p[common] + to = c.lm + otherFrom = otherC.p[common] + otherTo = otherC.lm + if otherFrom != otherTo + # if otherFrom == otherTo, we don't need to change our op. + + # where did my thing go? + if from == otherFrom + # they moved it! tie break. + if type == 'left' + c.p[common] = otherTo + if from == to # ugh + c.lm = otherTo + else + return dest + else + # they moved around it + if from > otherFrom + c.p[common]-- + if from > otherTo + c.p[common]++ + else if from == otherTo + if otherFrom > otherTo + c.p[common]++ + if from == to # ugh, again + c.lm++ + + # step 2: where am i going to put it? + if to > otherFrom + c.lm-- + else if to == otherFrom + if to > from + c.lm-- + if to > otherTo + c.lm++ + else if to == otherTo + # if we're both moving in the same direction, tie break + if (otherTo > otherFrom and to > from) or + (otherTo < otherFrom and to < from) + if type == 'right' + c.lm++ + else + if to > from + c.lm++ + else if to == otherFrom + c.lm-- + else if c.li != undefined and c.ld == undefined and commonOperand + # li + from = otherC.p[common] + to = otherC.lm + p = c.p[common] + if p > from + c.p[common]-- + if p > to + c.p[common]++ + else + # ld, ld+li, si, sd, na, oi, od, oi+od, any li on an element beneath + # the lm + # + # i.e. things care about where their item is after the move. + from = otherC.p[common] + to = otherC.lm + p = c.p[common] + if p == from + c.p[common] = to + else + if p > from + c.p[common]-- + if p > to + c.p[common]++ + else if p == to + if from > to + c.p[common]++ + else if otherC.oi != undefined && otherC.od != undefined + if c.p[common] == otherC.p[common] + if c.oi != undefined and commonOperand + # we inserted where someone else replaced + if type == 'right' + # left wins + return dest + else + # we win, make our op replace what they inserted + c.od = otherC.oi + else + # -> noop if the other component is deleting the same object (or any + # parent) + return dest + else if otherC.oi != undefined + if c.oi != undefined and c.p[common] == otherC.p[common] + # left wins if we try to insert at the same place + if type == 'left' + json.append dest, {p:c.p, od:otherC.oi} + else + return dest + else if otherC.od != undefined + if c.p[common] == otherC.p[common] + return dest if !commonOperand + if c.oi != undefined + delete c.od + else + return dest + + json.append dest, c + return dest + +if WEB? + exports.types ||= {} + + # This is kind of awful - come up with a better way to hook this helper code up. + exports._bt(json, json.transformComponent, json.checkValidOp, json.append) + + # [] is used to prevent closure from renaming types.text + exports.types.json = json +else + module.exports = json + + require('./helpers').bootstrapTransform(json, json.transformComponent, json.checkValidOp, json.append) + diff --git a/services/document-updater/app/coffee/sharejs/types/model.coffee b/services/document-updater/app/coffee/sharejs/types/model.coffee new file mode 100644 index 0000000000..284d6fd770 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/types/model.coffee @@ -0,0 +1,603 @@ +# The model of all the ops. Responsible for applying & transforming remote deltas +# and managing the storage layer. +# +# Actual storage is handled by the database wrappers in db/*, wrapped by DocCache + +{EventEmitter} = require 'events' + +queue = require './syncqueue' +types = require '../types' + +isArray = (o) -> Object.prototype.toString.call(o) == '[object Array]' + +# This constructor creates a new Model object. There will be one model object +# per server context. +# +# The model object is responsible for a lot of things: +# +# - It manages the interactions with the database +# - It maintains (in memory) a set of all active documents +# - It calls out to the OT functions when necessary +# +# The model is an event emitter. It emits the following events: +# +# create(docName, data): A document has been created with the specified name & data +module.exports = Model = (db, options) -> + # db can be null if the user doesn't want persistance. + + return new Model(db, options) if !(this instanceof Model) + + model = this + + options ?= {} + + # This is a cache of 'live' documents. + # + # The cache is a map from docName -> { + # ops:[{op, meta}] + # snapshot + # type + # v + # meta + # eventEmitter + # reapTimer + # committedVersion: v + # snapshotWriteLock: bool to make sure writeSnapshot isn't re-entrant + # dbMeta: database specific data + # opQueue: syncQueue for processing ops + # } + # + # The ops list contains the document's last options.numCachedOps ops. (Or all + # of them if we're using a memory store). + # + # Documents are stored in this set so long as the document has been accessed in + # the last few seconds (options.reapTime) OR at least one client has the document + # open. I don't know if I should keep open (but not being edited) documents live - + # maybe if a client has a document open but the document isn't being edited, I should + # flush it from the cache. + # + # In any case, the API to model is designed such that if we want to change that later + # it should be pretty easy to do so without any external-to-the-model code changes. + docs = {} + + # This is a map from docName -> [callback]. It is used when a document hasn't been + # cached and multiple getSnapshot() / getVersion() requests come in. All requests + # are added to the callback list and called when db.getSnapshot() returns. + # + # callback(error, snapshot data) + awaitingGetSnapshot = {} + + # The time that documents which no clients have open will stay in the cache. + # Should be > 0. + options.reapTime ?= 3000 + + # The number of operations the cache holds before reusing the space + options.numCachedOps ?= 10 + + # This option forces documents to be reaped, even when there's no database backend. + # This is useful when you don't care about persistance and don't want to gradually + # fill memory. + # + # You might want to set reapTime to a day or something. + options.forceReaping ?= false + + # Until I come up with a better strategy, we'll save a copy of the document snapshot + # to the database every ~20 submitted ops. + options.opsBeforeCommit ?= 20 + + # It takes some processing time to transform client ops. The server will punt ops back to the + # client to transform if they're too old. + options.maximumAge ?= 40 + + # **** Cache API methods + + # Its important that all ops are applied in order. This helper method creates the op submission queue + # for a single document. This contains the logic for transforming & applying ops. + makeOpQueue = (docName, doc) -> queue (opData, callback) -> + return callback 'Version missing' unless opData.v >= 0 + return callback 'Op at future version' if opData.v > doc.v + + # Punt the transforming work back to the client if the op is too old. + return callback 'Op too old' if opData.v + options.maximumAge < doc.v + + opData.meta ||= {} + opData.meta.ts = Date.now() + + # We'll need to transform the op to the current version of the document. This + # calls the callback immediately if opVersion == doc.v. + getOps docName, opData.v, doc.v, (error, ops) -> + return callback error if error + + unless doc.v - opData.v == ops.length + # This should never happen. It indicates that we didn't get all the ops we + # asked for. Its important that the submitted op is correctly transformed. + console.error "Could not get old ops in model for document #{docName}" + console.error "Expected ops #{opData.v} to #{doc.v} and got #{ops.length} ops" + return callback 'Internal error' + + if ops.length > 0 + try + # If there's enough ops, it might be worth spinning this out into a webworker thread. + for oldOp in ops + # Dup detection works by sending the id(s) the op has been submitted with previously. + # If the id matches, we reject it. The client can also detect the op has been submitted + # already if it sees its own previous id in the ops it sees when it does catchup. + if oldOp.meta.source and opData.dupIfSource and oldOp.meta.source in opData.dupIfSource + return callback 'Op already submitted' + + opData.op = doc.type.transform opData.op, oldOp.op, 'left' + opData.v++ + catch error + console.error error.stack + return callback error.message + + try + snapshot = doc.type.apply doc.snapshot, opData.op + catch error + console.error error.stack + return callback error.message + + # The op data should be at the current version, and the new document data should be at + # the next version. + # + # This should never happen in practice, but its a nice little check to make sure everything + # is hunky-dory. + unless opData.v == doc.v + # This should never happen. + console.error "Version mismatch detected in model. File a ticket - this is a bug." + console.error "Expecting #{opData.v} == #{doc.v}" + return callback 'Internal error' + + #newDocData = {snapshot, type:type.name, v:opVersion + 1, meta:docData.meta} + writeOp = db?.writeOp or (docName, newOpData, callback) -> callback() + + writeOp docName, opData, (error) -> + if error + # The user should probably know about this. + console.warn "Error writing ops to database: #{error}" + return callback error + + options.stats?.writeOp?() + + # This is needed when we emit the 'change' event, below. + oldSnapshot = doc.snapshot + + # All the heavy lifting is now done. Finally, we'll update the cache with the new data + # and (maybe!) save a new document snapshot to the database. + + doc.v = opData.v + 1 + doc.snapshot = snapshot + + doc.ops.push opData + doc.ops.shift() if db and doc.ops.length > options.numCachedOps + + model.emit 'applyOp', docName, opData, snapshot, oldSnapshot + doc.eventEmitter.emit 'op', opData, snapshot, oldSnapshot + + # The callback is called with the version of the document at which the op was applied. + # This is the op.v after transformation, and its doc.v - 1. + callback null, opData.v + + # I need a decent strategy here for deciding whether or not to save the snapshot. + # + # The 'right' strategy looks something like "Store the snapshot whenever the snapshot + # is smaller than the accumulated op data". For now, I'll just store it every 20 + # ops or something. (Configurable with doc.committedVersion) + if !doc.snapshotWriteLock and doc.committedVersion + options.opsBeforeCommit <= doc.v + tryWriteSnapshot docName, (error) -> + console.warn "Error writing snapshot #{error}. This is nonfatal" if error + + # Add the data for the given docName to the cache. The named document shouldn't already + # exist in the doc set. + # + # Returns the new doc. + add = (docName, error, data, committedVersion, ops, dbMeta) -> + callbacks = awaitingGetSnapshot[docName] + delete awaitingGetSnapshot[docName] + + if error + callback error for callback in callbacks if callbacks + else + doc = docs[docName] = + snapshot: data.snapshot + v: data.v + type: data.type + meta: data.meta + + # Cache of ops + ops: ops or [] + + eventEmitter: new EventEmitter + + # Timer before the document will be invalidated from the cache (if the document has no + # listeners) + reapTimer: null + + # Version of the snapshot thats in the database + committedVersion: committedVersion ? data.v + snapshotWriteLock: false + dbMeta: dbMeta + + doc.opQueue = makeOpQueue docName, doc + + refreshReapingTimeout docName + model.emit 'add', docName, data + callback null, doc for callback in callbacks if callbacks + + doc + + # This is a little helper wrapper around db.getOps. It does two things: + # + # - If there's no database set, it returns an error to the callback + # - It adds version numbers to each op returned from the database + # (These can be inferred from context so the DB doesn't store them, but its useful to have them). + getOpsInternal = (docName, start, end, callback) -> + return callback? 'Document does not exist' unless db + + db.getOps docName, start, end, (error, ops) -> + return callback? error if error + + v = start + op.v = v++ for op in ops + + callback? null, ops + + # Load the named document into the cache. This function is re-entrant. + # + # The callback is called with (error, doc) + load = (docName, callback) -> + if docs[docName] + # The document is already loaded. Return immediately. + options.stats?.cacheHit? 'getSnapshot' + return callback null, docs[docName] + + # We're a memory store. If we don't have it, nobody does. + return callback 'Document does not exist' unless db + + callbacks = awaitingGetSnapshot[docName] + + # The document is being loaded already. Add ourselves as a callback. + return callbacks.push callback if callbacks + + options.stats?.cacheMiss? 'getSnapshot' + + # The document isn't loaded and isn't being loaded. Load it. + awaitingGetSnapshot[docName] = [callback] + db.getSnapshot docName, (error, data, dbMeta) -> + return add docName, error if error + + type = types[data.type] + unless type + console.warn "Type '#{data.type}' missing" + return callback "Type not found" + data.type = type + + committedVersion = data.v + + # The server can close without saving the most recent document snapshot. + # In this case, there are extra ops which need to be applied before + # returning the snapshot. + getOpsInternal docName, data.v, null, (error, ops) -> + return callback error if error + + if ops.length > 0 + console.log "Catchup #{docName} #{data.v} -> #{data.v + ops.length}" + + try + for op in ops + data.snapshot = type.apply data.snapshot, op.op + data.v++ + catch e + # This should never happen - it indicates that whats in the + # database is invalid. + console.error "Op data invalid for #{docName}: #{e.stack}" + return callback 'Op data invalid' + + model.emit 'load', docName, data + add docName, error, data, committedVersion, ops, dbMeta + + # This makes sure the cache contains a document. If the doc cache doesn't contain + # a document, it is loaded from the database and stored. + # + # Documents are stored so long as either: + # - They have been accessed within the past #{PERIOD} + # - At least one client has the document open + refreshReapingTimeout = (docName) -> + doc = docs[docName] + return unless doc + + # I want to let the clients list be updated before this is called. + process.nextTick -> + # This is an awkward way to find out the number of clients on a document. If this + # causes performance issues, add a numClients field to the document. + # + # The first check is because its possible that between refreshReapingTimeout being called and this + # event being fired, someone called delete() on the document and hence the doc is something else now. + if doc == docs[docName] and + doc.eventEmitter.listeners('op').length == 0 and + (db or options.forceReaping) and + doc.opQueue.busy is false + + clearTimeout doc.reapTimer + doc.reapTimer = reapTimer = setTimeout -> + tryWriteSnapshot docName, -> + # If the reaping timeout has been refreshed while we're writing the snapshot, or if we're + # in the middle of applying an operation, don't reap. + delete docs[docName] if docs[docName].reapTimer is reapTimer and doc.opQueue.busy is false + , options.reapTime + + tryWriteSnapshot = (docName, callback) -> + return callback?() unless db + + doc = docs[docName] + + # The doc is closed + return callback?() unless doc + + # The document is already saved. + return callback?() if doc.committedVersion is doc.v + + return callback? 'Another snapshot write is in progress' if doc.snapshotWriteLock + + doc.snapshotWriteLock = true + + options.stats?.writeSnapshot?() + + writeSnapshot = db?.writeSnapshot or (docName, docData, dbMeta, callback) -> callback() + + data = + v: doc.v + meta: doc.meta + snapshot: doc.snapshot + # The database doesn't know about object types. + type: doc.type.name + + # Commit snapshot. + writeSnapshot docName, data, doc.dbMeta, (error, dbMeta) -> + doc.snapshotWriteLock = false + + # We have to use data.v here because the version in the doc could + # have been updated between the call to writeSnapshot() and now. + doc.committedVersion = data.v + doc.dbMeta = dbMeta + + callback? error + + # *** Model interface methods + + # Create a new document. + # + # data should be {snapshot, type, [meta]}. The version of a new document is 0. + @create = (docName, type, meta, callback) -> + [meta, callback] = [{}, meta] if typeof meta is 'function' + + return callback? 'Invalid document name' if docName.match /\// + return callback? 'Document already exists' if docs[docName] + + type = types[type] if typeof type == 'string' + return callback? 'Type not found' unless type + + data = + snapshot:type.create() + type:type.name + meta:meta or {} + v:0 + + done = (error, dbMeta) -> + # dbMeta can be used to cache extra state needed by the database to access the document, like an ID or something. + return callback? error if error + + # From here on we'll store the object version of the type name. + data.type = type + add docName, null, data, 0, [], dbMeta + model.emit 'create', docName, data + callback?() + + if db + db.create docName, data, done + else + done() + + # Perminantly deletes the specified document. + # If listeners are attached, they are removed. + # + # The callback is called with (error) if there was an error. If error is null / undefined, the + # document was deleted. + # + # WARNING: This isn't well supported throughout the code. (Eg, streaming clients aren't told about the + # deletion. Subsequent op submissions will fail). + @delete = (docName, callback) -> + doc = docs[docName] + + if doc + clearTimeout doc.reapTimer + delete docs[docName] + + done = (error) -> + model.emit 'delete', docName unless error + callback? error + + if db + db.delete docName, doc?.dbMeta, done + else + done (if !doc then 'Document does not exist') + + # This gets all operations from [start...end]. (That is, its not inclusive.) + # + # end can be null. This means 'get me all ops from start'. + # + # Each op returned is in the form {op:o, meta:m, v:version}. + # + # Callback is called with (error, [ops]) + # + # If the document does not exist, getOps doesn't necessarily return an error. This is because + # its awkward to figure out whether or not the document exists for things + # like the redis database backend. I guess its a bit gross having this inconsistant + # with the other DB calls, but its certainly convenient. + # + # Use getVersion() to determine if a document actually exists, if thats what you're + # after. + @getOps = getOps = (docName, start, end, callback) -> + # getOps will only use the op cache if its there. It won't fill the op cache in. + throw new Error 'start must be 0+' unless start >= 0 + + [end, callback] = [null, end] if typeof end is 'function' + + ops = docs[docName]?.ops + + if ops + version = docs[docName].v + + # Ops contains an array of ops. The last op in the list is the last op applied + end ?= version + start = Math.min start, end + + return callback null, [] if start == end + + # Base is the version number of the oldest op we have cached + base = version - ops.length + + # If the database is null, we'll trim to the ops we do have and hope thats enough. + if start >= base or db is null + refreshReapingTimeout docName + options.stats?.cacheHit 'getOps' + + return callback null, ops[(start - base)...(end - base)] + + options.stats?.cacheMiss 'getOps' + + getOpsInternal docName, start, end, callback + + # Gets the snapshot data for the specified document. + # getSnapshot(docName, callback) + # Callback is called with (error, {v: , type: , snapshot: , meta: }) + @getSnapshot = (docName, callback) -> + load docName, (error, doc) -> + callback error, if doc then {v:doc.v, type:doc.type, snapshot:doc.snapshot, meta:doc.meta} + + # Gets the latest version # of the document. + # getVersion(docName, callback) + # callback is called with (error, version). + @getVersion = (docName, callback) -> + load docName, (error, doc) -> callback error, doc?.v + + # Apply an op to the specified document. + # The callback is passed (error, applied version #) + # opData = {op:op, v:v, meta:metadata} + # + # Ops are queued before being applied so that the following code applies op C before op B: + # model.applyOp 'doc', OPA, -> model.applyOp 'doc', OPB + # model.applyOp 'doc', OPC + @applyOp = (docName, opData, callback) -> + # All the logic for this is in makeOpQueue, above. + load docName, (error, doc) -> + return callback error if error + + process.nextTick -> doc.opQueue opData, (error, newVersion) -> + refreshReapingTimeout docName + callback? error, newVersion + + # TODO: store (some) metadata in DB + # TODO: op and meta should be combineable in the op that gets sent + @applyMetaOp = (docName, metaOpData, callback) -> + {path, value} = metaOpData.meta + + return callback? "path should be an array" unless isArray path + + load docName, (error, doc) -> + if error? + callback? error + else + applied = false + switch path[0] + when 'shout' + doc.eventEmitter.emit 'op', metaOpData + applied = true + + model.emit 'applyMetaOp', docName, path, value if applied + callback? null, doc.v + + # Listen to all ops from the specified version. If version is in the past, all + # ops since that version are sent immediately to the listener. + # + # The callback is called once the listener is attached, but before any ops have been passed + # to the listener. + # + # This will _not_ edit the document metadata. + # + # If there are any listeners, we don't purge the document from the cache. But be aware, this behaviour + # might change in a future version. + # + # version is the document version at which the document is opened. It can be left out if you want to open + # the document at the most recent version. + # + # listener is called with (opData) each time an op is applied. + # + # callback(error, openedVersion) + @listen = (docName, version, listener, callback) -> + [version, listener, callback] = [null, version, listener] if typeof version is 'function' + + load docName, (error, doc) -> + return callback? error if error + + clearTimeout doc.reapTimer + + if version? + getOps docName, version, null, (error, data) -> + return callback? error if error + + doc.eventEmitter.on 'op', listener + callback? null, version + for op in data + listener op + + # The listener may well remove itself during the catchup phase. If this happens, break early. + # This is done in a quite inefficient way. (O(n) where n = #listeners on doc) + break unless listener in doc.eventEmitter.listeners 'op' + + else # Version is null / undefined. Just add the listener. + doc.eventEmitter.on 'op', listener + callback? null, doc.v + + # Remove a listener for a particular document. + # + # removeListener(docName, listener) + # + # This is synchronous. + @removeListener = (docName, listener) -> + # The document should already be loaded. + doc = docs[docName] + throw new Error 'removeListener called but document not loaded' unless doc + + doc.eventEmitter.removeListener 'op', listener + refreshReapingTimeout docName + + # Flush saves all snapshot data to the database. I'm not sure whether or not this is actually needed - + # sharejs will happily replay uncommitted ops when documents are re-opened anyway. + @flush = (callback) -> + return callback?() unless db + + pendingWrites = 0 + + for docName, doc of docs + if doc.committedVersion < doc.v + pendingWrites++ + # I'm hoping writeSnapshot will always happen in another thread. + tryWriteSnapshot docName, -> + process.nextTick -> + pendingWrites-- + callback?() if pendingWrites is 0 + + # If nothing was queued, terminate immediately. + callback?() if pendingWrites is 0 + + # Close the database connection. This is needed so nodejs can shut down cleanly. + @closeDb = -> + db?.close?() + db = null + + return + +# Model inherits from EventEmitter. +Model:: = new EventEmitter + diff --git a/services/document-updater/app/coffee/sharejs/types/simple.coffee b/services/document-updater/app/coffee/sharejs/types/simple.coffee new file mode 100644 index 0000000000..996b1a5ddc --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/types/simple.coffee @@ -0,0 +1,38 @@ +# This is a really simple OT type. Its not compiled with the web client, but it could be. +# +# Its mostly included for demonstration purposes and its used in a lot of unit tests. +# +# This defines a really simple text OT type which only allows inserts. (No deletes). +# +# Ops look like: +# {position:#, text:"asdf"} +# +# Document snapshots look like: +# {str:string} + +module.exports = + # The name of the OT type. The type is stored in types[type.name]. The name can be + # used in place of the actual type in all the API methods. + name: 'simple' + + # Create a new document snapshot + create: -> {str:""} + + # Apply the given op to the document snapshot. Returns the new snapshot. + # + # The original snapshot should not be modified. + apply: (snapshot, op) -> + throw new Error 'Invalid position' unless 0 <= op.position <= snapshot.str.length + + str = snapshot.str + str = str.slice(0, op.position) + op.text + str.slice(op.position) + {str} + + # transform op1 by op2. Return transformed version of op1. + # sym describes the symmetry of the op. Its 'left' or 'right' depending on whether the + # op being transformed comes from the client or the server. + transform: (op1, op2, sym) -> + pos = op1.position + pos += op2.text.length if op2.position < pos or (op2.position == pos and sym is 'left') + + return {position:pos, text:op1.text} diff --git a/services/document-updater/app/coffee/sharejs/types/syncqueue.coffee b/services/document-updater/app/coffee/sharejs/types/syncqueue.coffee new file mode 100644 index 0000000000..746450b010 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/types/syncqueue.coffee @@ -0,0 +1,42 @@ +# A synchronous processing queue. The queue calls process on the arguments, +# ensuring that process() is only executing once at a time. +# +# process(data, callback) _MUST_ eventually call its callback. +# +# Example: +# +# queue = require 'syncqueue' +# +# fn = queue (data, callback) -> +# asyncthing data, -> +# callback(321) +# +# fn(1) +# fn(2) +# fn(3, (result) -> console.log(result)) +# +# ^--- async thing will only be running once at any time. + +module.exports = (process) -> + throw new Error('process is not a function') unless typeof process == 'function' + queue = [] + + enqueue = (data, callback) -> + queue.push [data, callback] + flush() + + enqueue.busy = false + + flush = -> + return if enqueue.busy or queue.length == 0 + + enqueue.busy = true + [data, callback] = queue.shift() + process data, (result...) -> # TODO: Make this not use varargs - varargs are really slow. + enqueue.busy = false + # This is called after busy = false so a user can check if enqueue.busy is set in the callback. + callback.apply null, result if callback + flush() + + enqueue + diff --git a/services/document-updater/app/coffee/sharejs/types/text-api.coffee b/services/document-updater/app/coffee/sharejs/types/text-api.coffee new file mode 100644 index 0000000000..96243ceffb --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/types/text-api.coffee @@ -0,0 +1,32 @@ +# Text document API for text + +text = require './text' if typeof WEB is 'undefined' + +text.api = + provides: {text:true} + + # The number of characters in the string + getLength: -> @snapshot.length + + # Get the text contents of a document + getText: -> @snapshot + + insert: (pos, text, callback) -> + op = [{p:pos, i:text}] + + @submitOp op, callback + op + + del: (pos, length, callback) -> + op = [{p:pos, d:@snapshot[pos...(pos + length)]}] + + @submitOp op, callback + op + + _register: -> + @on 'remoteop', (op) -> + for component in op + if component.i != undefined + @emit 'insert', component.p, component.i + else + @emit 'delete', component.p, component.d diff --git a/services/document-updater/app/coffee/sharejs/types/text-composable-api.coffee b/services/document-updater/app/coffee/sharejs/types/text-composable-api.coffee new file mode 100644 index 0000000000..7b27ac163a --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/types/text-composable-api.coffee @@ -0,0 +1,43 @@ +# Text document API for text + +if WEB? + type = exports.types['text-composable'] +else + type = require './text-composable' + +type.api = + provides: {'text':true} + + # The number of characters in the string + 'getLength': -> @snapshot.length + + # Get the text contents of a document + 'getText': -> @snapshot + + 'insert': (pos, text, callback) -> + op = type.normalize [pos, 'i':text, (@snapshot.length - pos)] + + @submitOp op, callback + op + + 'del': (pos, length, callback) -> + op = type.normalize [pos, 'd':@snapshot[pos...(pos + length)], (@snapshot.length - pos - length)] + + @submitOp op, callback + op + + _register: -> + @on 'remoteop', (op) -> + pos = 0 + for component in op + if typeof component is 'number' + pos += component + else if component.i != undefined + @emit 'insert', pos, component.i + pos += component.i.length + else + # delete + @emit 'delete', pos, component.d + # We don't increment pos, because the position + # specified is after the delete has happened. + diff --git a/services/document-updater/app/coffee/sharejs/types/text-composable.coffee b/services/document-updater/app/coffee/sharejs/types/text-composable.coffee new file mode 100644 index 0000000000..992b567bf0 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/types/text-composable.coffee @@ -0,0 +1,261 @@ +# An alternate composable implementation for text. This is much closer +# to the implementation used by google wave. +# +# Ops are lists of components which iterate over the whole document. +# Components are either: +# A number N: Skip N characters in the original document +# {i:'str'}: Insert 'str' at the current position in the document +# {d:'str'}: Delete 'str', which appears at the current position in the document +# +# Eg: [3, {i:'hi'}, 5, {d:'internet'}] +# +# Snapshots are strings. + +p = -> #require('util').debug +i = -> #require('util').inspect + +exports = if WEB? then {} else module.exports + +exports.name = 'text-composable' + +exports.create = -> '' + +# -------- Utility methods + +checkOp = (op) -> + throw new Error('Op must be an array of components') unless Array.isArray(op) + last = null + for c in op + if typeof(c) == 'object' + throw new Error("Invalid op component: #{i c}") unless (c.i? && c.i.length > 0) or (c.d? && c.d.length > 0) + else + throw new Error('Op components must be objects or numbers') unless typeof(c) == 'number' + throw new Error('Skip components must be a positive number') unless c > 0 + throw new Error('Adjacent skip components should be added') if typeof(last) == 'number' + + last = c + +# Makes a function for appending components to a given op. +# Exported for the randomOpGenerator. +exports._makeAppend = makeAppend = (op) -> (component) -> + if component == 0 || component.i == '' || component.d == '' + return + else if op.length == 0 + op.push component + else if typeof(component) == 'number' && typeof(op[op.length - 1]) == 'number' + op[op.length - 1] += component + else if component.i? && op[op.length - 1].i? + op[op.length - 1].i += component.i + else if component.d? && op[op.length - 1].d? + op[op.length - 1].d += component.d + else + op.push component + +# checkOp op + +# Makes 2 functions for taking components from the start of an op, and for peeking +# at the next op that could be taken. +makeTake = (op) -> + # The index of the next component to take + idx = 0 + # The offset into the component + offset = 0 + + # Take up to length n from the front of op. If n is null, take the next + # op component. If indivisableField == 'd', delete components won't be separated. + # If indivisableField == 'i', insert components won't be separated. + take = (n, indivisableField) -> + return null if idx == op.length + #assert.notStrictEqual op.length, i, 'The op is too short to traverse the document' + + if typeof(op[idx]) == 'number' + if !n? or op[idx] - offset <= n + c = op[idx] - offset + ++idx; offset = 0 + c + else + offset += n + n + else + # Take from the string + field = if op[idx].i then 'i' else 'd' + c = {} + if !n? or op[idx][field].length - offset <= n or field == indivisableField + c[field] = op[idx][field][offset..] + ++idx; offset = 0 + else + c[field] = op[idx][field][offset...(offset + n)] + offset += n + c + + peekType = () -> + op[idx] + + [take, peekType] + +# Find and return the length of an op component +componentLength = (component) -> + if typeof(component) == 'number' + component + else if component.i? + component.i.length + else + component.d.length + +# Normalize an op, removing all empty skips and empty inserts / deletes. Concatenate +# adjacent inserts and deletes. +exports.normalize = (op) -> + newOp = [] + append = makeAppend newOp + append component for component in op + newOp + +# Apply the op to the string. Returns the new string. +exports.apply = (str, op) -> + p "Applying #{i op} to '#{str}'" + throw new Error('Snapshot should be a string') unless typeof(str) == 'string' + checkOp op + + pos = 0 + newDoc = [] + + for component in op + if typeof(component) == 'number' + throw new Error('The op is too long for this document') if component > str.length + newDoc.push str[...component] + str = str[component..] + else if component.i? + newDoc.push component.i + else + throw new Error("The deleted text '#{component.d}' doesn't match the next characters in the document '#{str[...component.d.length]}'") unless component.d == str[...component.d.length] + str = str[component.d.length..] + + throw new Error("The applied op doesn't traverse the entire document") unless '' == str + + newDoc.join '' + +# transform op1 by op2. Return transformed version of op1. +# op1 and op2 are unchanged by transform. +exports.transform = (op, otherOp, side) -> + throw new Error "side (#{side} must be 'left' or 'right'" unless side == 'left' or side == 'right' + + checkOp op + checkOp otherOp + newOp = [] + + append = makeAppend newOp + [take, peek] = makeTake op + + for component in otherOp + if typeof(component) == 'number' # Skip + length = component + while length > 0 + chunk = take(length, 'i') + throw new Error('The op traverses more elements than the document has') unless chunk != null + + append chunk + length -= componentLength chunk unless typeof(chunk) == 'object' && chunk.i? + else if component.i? # Insert + if side == 'left' + # The left insert should go first. + o = peek() + append take() if o?.i + + # Otherwise, skip the inserted text. + append(component.i.length) + else # Delete. + #assert.ok component.d + length = component.d.length + while length > 0 + chunk = take(length, 'i') + throw new Error('The op traverses more elements than the document has') unless chunk != null + + if typeof(chunk) == 'number' + length -= chunk + else if chunk.i? + append(chunk) + else + #assert.ok chunk.d + # The delete is unnecessary now. + length -= chunk.d.length + + # Append extras from op1 + while (component = take()) + throw new Error "Remaining fragments in the op: #{i component}" unless component?.i? + append component + + newOp + + +# Compose 2 ops into 1 op. +exports.compose = (op1, op2) -> + p "COMPOSE #{i op1} + #{i op2}" + checkOp op1 + checkOp op2 + + result = [] + + append = makeAppend result + [take, _] = makeTake op1 + + for component in op2 + if typeof(component) == 'number' # Skip + length = component + while length > 0 + chunk = take(length, 'd') + throw new Error('The op traverses more elements than the document has') unless chunk != null + + append chunk + length -= componentLength chunk unless typeof(chunk) == 'object' && chunk.d? + + else if component.i? # Insert + append {i:component.i} + + else # Delete + offset = 0 + while offset < component.d.length + chunk = take(component.d.length - offset, 'd') + throw new Error('The op traverses more elements than the document has') unless chunk != null + + # If its delete, append it. If its skip, drop it and decrease length. If its insert, check the strings match, drop it and decrease length. + if typeof(chunk) == 'number' + append {d:component.d[offset...(offset + chunk)]} + offset += chunk + else if chunk.i? + throw new Error("The deleted text doesn't match the inserted text") unless component.d[offset...(offset + chunk.i.length)] == chunk.i + offset += chunk.i.length + # The ops cancel each other out. + else + # Delete + append chunk + + # Append extras from op1 + while (component = take()) + throw new Error "Trailing stuff in op1 #{i component}" unless component?.d? + append component + + result + + +invertComponent = (c) -> + if typeof(c) == 'number' + c + else if c.i? + {d:c.i} + else + {i:c.d} + +# Invert an op +exports.invert = (op) -> + result = [] + append = makeAppend result + + append(invertComponent component) for component in op + + result + +if window? + window.ot ||= {} + window.ot.types ||= {} + window.ot.types.text = exports + diff --git a/services/document-updater/app/coffee/sharejs/types/text-tp2-api.coffee b/services/document-updater/app/coffee/sharejs/types/text-tp2-api.coffee new file mode 100644 index 0000000000..d661b5ae37 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/types/text-tp2-api.coffee @@ -0,0 +1,89 @@ +# Text document API for text-tp2 + +if WEB? + type = exports.types['text-tp2'] +else + type = require './text-tp2' + +{_takeDoc:takeDoc, _append:append} = type + +appendSkipChars = (op, doc, pos, maxlength) -> + while (maxlength == undefined || maxlength > 0) and pos.index < doc.data.length + part = takeDoc doc, pos, maxlength, true + maxlength -= part.length if maxlength != undefined and typeof part is 'string' + append op, (part.length || part) + +type['api'] = + 'provides': {'text':true} + + # The number of characters in the string + 'getLength': -> @snapshot.charLength + + # Flatten a document into a string + 'getText': -> + strings = (elem for elem in @snapshot.data when typeof elem is 'string') + strings.join '' + + 'insert': (pos, text, callback) -> + pos = 0 if pos == undefined + + op = [] + docPos = {index:0, offset:0} + + appendSkipChars op, @snapshot, docPos, pos + append op, {'i':text} + appendSkipChars op, @snapshot, docPos + + @submitOp op, callback + op + + 'del': (pos, length, callback) -> + op = [] + docPos = {index:0, offset:0} + + appendSkipChars op, @snapshot, docPos, pos + + while length > 0 + part = takeDoc @snapshot, docPos, length, true + if typeof part is 'string' + append op, {'d':part.length} + length -= part.length + else + append op, part + + appendSkipChars op, @snapshot, docPos + + @submitOp op, callback + op + + '_register': -> + # Interpret recieved ops + generate more detailed events for them + @on 'remoteop', (op, snapshot) -> + textPos = 0 + docPos = {index:0, offset:0} + + for component in op + if typeof component is 'number' + # Skip + remainder = component + while remainder > 0 + part = takeDoc snapshot, docPos, remainder + if typeof part is 'string' + textPos += part.length + remainder -= part.length || part + else if component.i != undefined + # Insert + if typeof component.i is 'string' + @emit 'insert', textPos, component.i + textPos += component.i.length + else + # Delete + remainder = component.d + while remainder > 0 + part = takeDoc snapshot, docPos, remainder + if typeof part is 'string' + @emit 'delete', textPos, part + remainder -= part.length || part + + return + diff --git a/services/document-updater/app/coffee/sharejs/types/text-tp2.coffee b/services/document-updater/app/coffee/sharejs/types/text-tp2.coffee new file mode 100644 index 0000000000..d19cbdcef4 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/types/text-tp2.coffee @@ -0,0 +1,322 @@ +# A TP2 implementation of text, following this spec: +# http://code.google.com/p/lightwave/source/browse/trunk/experimental/ot/README +# +# A document is made up of a string and a set of tombstones inserted throughout +# the string. For example, 'some ', (2 tombstones), 'string'. +# +# This is encoded in a document as: {s:'some string', t:[5, -2, 6]} +# +# Ops are lists of components which iterate over the whole document. +# Components are either: +# N: Skip N characters in the original document +# {i:'str'}: Insert 'str' at the current position in the document +# {i:N}: Insert N tombstones at the current position in the document +# {d:N}: Delete (tombstone) N characters at the current position in the document +# +# Eg: [3, {i:'hi'}, 5, {d:8}] +# +# Snapshots are lists with characters and tombstones. Characters are stored in strings +# and adjacent tombstones are flattened into numbers. +# +# Eg, the document: 'Hello .....world' ('.' denotes tombstoned (deleted) characters) +# would be represented by a document snapshot of ['Hello ', 5, 'world'] + +type = + name: 'text-tp2' + tp2: true + create: -> {charLength:0, totalLength:0, positionCache:[], data:[]} + serialize: (doc) -> + throw new Error 'invalid doc snapshot' unless doc.data + doc.data + deserialize: (data) -> + doc = type.create() + doc.data = data + + for component in data + if typeof component is 'string' + doc.charLength += component.length + doc.totalLength += component.length + else + doc.totalLength += component + + doc + + +checkOp = (op) -> + throw new Error('Op must be an array of components') unless Array.isArray(op) + last = null + for c in op + if typeof(c) == 'object' + if c.i != undefined + throw new Error('Inserts must insert a string or a +ive number') unless (typeof(c.i) == 'string' and c.i.length > 0) or (typeof(c.i) == 'number' and c.i > 0) + else if c.d != undefined + throw new Error('Deletes must be a +ive number') unless typeof(c.d) == 'number' and c.d > 0 + else + throw new Error('Operation component must define .i or .d') + else + throw new Error('Op components must be objects or numbers') unless typeof(c) == 'number' + throw new Error('Skip components must be a positive number') unless c > 0 + throw new Error('Adjacent skip components should be combined') if typeof(last) == 'number' + + last = c + +# Take the next part from the specified position in a document snapshot. +# position = {index, offset}. It will be updated. +type._takeDoc = takeDoc = (doc, position, maxlength, tombsIndivisible) -> + throw new Error 'Operation goes past the end of the document' if position.index >= doc.data.length + + part = doc.data[position.index] + # peel off data[0] + result = if typeof(part) == 'string' + if maxlength != undefined + part[position.offset...(position.offset + maxlength)] + else + part[position.offset...] + else + if maxlength == undefined or tombsIndivisible + part - position.offset + else + Math.min(maxlength, part - position.offset) + + resultLen = result.length || result + + if (part.length || part) - position.offset > resultLen + position.offset += resultLen + else + position.index++ + position.offset = 0 + + result + +# Append a part to the end of a document +type._appendDoc = appendDoc = (doc, p) -> + return if p == 0 or p == '' + + if typeof p is 'string' + doc.charLength += p.length + doc.totalLength += p.length + else + doc.totalLength += p + + data = doc.data + if data.length == 0 + data.push p + else if typeof(data[data.length - 1]) == typeof(p) + data[data.length - 1] += p + else + data.push p + return + +# Apply the op to the document. The document is not modified in the process. +type.apply = (doc, op) -> + unless doc.totalLength != undefined and doc.charLength != undefined and doc.data.length != undefined + throw new Error('Snapshot is invalid') + + checkOp op + + newDoc = type.create() + position = {index:0, offset:0} + + for component in op + if typeof(component) is 'number' + remainder = component + while remainder > 0 + part = takeDoc doc, position, remainder + + appendDoc newDoc, part + remainder -= part.length || part + + else if component.i != undefined + appendDoc newDoc, component.i + else if component.d != undefined + remainder = component.d + while remainder > 0 + part = takeDoc doc, position, remainder + remainder -= part.length || part + appendDoc newDoc, component.d + + newDoc + +# Append an op component to the end of the specified op. +# Exported for the randomOpGenerator. +type._append = append = (op, component) -> + if component == 0 || component.i == '' || component.i == 0 || component.d == 0 + return + else if op.length == 0 + op.push component + else + last = op[op.length - 1] + if typeof(component) == 'number' && typeof(last) == 'number' + op[op.length - 1] += component + else if component.i != undefined && last.i? && typeof(last.i) == typeof(component.i) + last.i += component.i + else if component.d != undefined && last.d? + last.d += component.d + else + op.push component + +# Makes 2 functions for taking components from the start of an op, and for peeking +# at the next op that could be taken. +makeTake = (op) -> + # The index of the next component to take + index = 0 + # The offset into the component + offset = 0 + + # Take up to length maxlength from the op. If maxlength is not defined, there is no max. + # If insertsIndivisible is true, inserts (& insert tombstones) won't be separated. + # + # Returns null when op is fully consumed. + take = (maxlength, insertsIndivisible) -> + return null if index == op.length + + e = op[index] + if typeof((current = e)) == 'number' or typeof((current = e.i)) == 'number' or (current = e.d) != undefined + if !maxlength? or current - offset <= maxlength or (insertsIndivisible and e.i != undefined) + # Return the rest of the current element. + c = current - offset + ++index; offset = 0 + else + offset += maxlength + c = maxlength + if e.i != undefined then {i:c} else if e.d != undefined then {d:c} else c + else + # Take from the inserted string + if !maxlength? or e.i.length - offset <= maxlength or insertsIndivisible + result = {i:e.i[offset..]} + ++index; offset = 0 + else + result = {i:e.i[offset...offset + maxlength]} + offset += maxlength + result + + peekType = -> op[index] + + [take, peekType] + +# Find and return the length of an op component +componentLength = (component) -> + if typeof(component) == 'number' + component + else if typeof(component.i) == 'string' + component.i.length + else + # This should work because c.d and c.i must be +ive. + component.d or component.i + +# Normalize an op, removing all empty skips and empty inserts / deletes. Concatenate +# adjacent inserts and deletes. +type.normalize = (op) -> + newOp = [] + append newOp, component for component in op + newOp + +# This is a helper method to transform and prune. goForwards is true for transform, false for prune. +transformer = (op, otherOp, goForwards, side) -> + checkOp op + checkOp otherOp + newOp = [] + + [take, peek] = makeTake op + + for component in otherOp + length = componentLength component + + if component.i != undefined # Insert text or tombs + if goForwards # transform - insert skips over inserted parts + if side == 'left' + # The left insert should go first. + append newOp, take() while peek()?.i != undefined + + # In any case, skip the inserted text. + append newOp, length + + else # Prune. Remove skips for inserts. + while length > 0 + chunk = take length, true + + throw new Error 'The transformed op is invalid' unless chunk != null + throw new Error 'The transformed op deletes locally inserted characters - it cannot be purged of the insert.' if chunk.d != undefined + + if typeof chunk is 'number' + length -= chunk + else + append newOp, chunk + + else # Skip or delete + while length > 0 + chunk = take length, true + throw new Error('The op traverses more elements than the document has') unless chunk != null + + append newOp, chunk + length -= componentLength chunk unless chunk.i + + # Append extras from op1 + while (component = take()) + throw new Error "Remaining fragments in the op: #{component}" unless component.i != undefined + append newOp, component + + newOp + +# transform op1 by op2. Return transformed version of op1. +# op1 and op2 are unchanged by transform. +# side should be 'left' or 'right', depending on if op1.id <> op2.id. 'left' == client op. +type.transform = (op, otherOp, side) -> + throw new Error "side (#{side}) should be 'left' or 'right'" unless side == 'left' or side == 'right' + transformer op, otherOp, true, side + +# Prune is the inverse of transform. +type.prune = (op, otherOp) -> transformer op, otherOp, false + +# Compose 2 ops into 1 op. +type.compose = (op1, op2) -> + return op2 if op1 == null or op1 == undefined + + checkOp op1 + checkOp op2 + + result = [] + + [take, _] = makeTake op1 + + for component in op2 + + if typeof(component) == 'number' # Skip + # Just copy from op1. + length = component + while length > 0 + chunk = take length + throw new Error('The op traverses more elements than the document has') unless chunk != null + + append result, chunk + length -= componentLength chunk + + else if component.i != undefined # Insert + append result, {i:component.i} + + else # Delete + length = component.d + while length > 0 + chunk = take length + throw new Error('The op traverses more elements than the document has') unless chunk != null + + chunkLength = componentLength chunk + if chunk.i != undefined + append result, {i:chunkLength} + else + append result, {d:chunkLength} + + length -= chunkLength + + # Append extras from op1 + while (component = take()) + throw new Error "Remaining fragments in op1: #{component}" unless component.i != undefined + append result, component + + result + +if WEB? + exports.types['text-tp2'] = type +else + module.exports = type + diff --git a/services/document-updater/app/coffee/sharejs/types/text.coffee b/services/document-updater/app/coffee/sharejs/types/text.coffee new file mode 100644 index 0000000000..c64b4dfa68 --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/types/text.coffee @@ -0,0 +1,209 @@ +# A simple text implementation +# +# Operations are lists of components. +# Each component either inserts or deletes at a specified position in the document. +# +# Components are either: +# {i:'str', p:100}: Insert 'str' at position 100 in the document +# {d:'str', p:100}: Delete 'str' at position 100 in the document +# +# Components in an operation are executed sequentially, so the position of components +# assumes previous components have already executed. +# +# Eg: This op: +# [{i:'abc', p:0}] +# is equivalent to this op: +# [{i:'a', p:0}, {i:'b', p:1}, {i:'c', p:2}] + +# NOTE: The global scope here is shared with other sharejs files when built with closure. +# Be careful what ends up in your namespace. + +text = {} + +text.name = 'text' + +text.create = -> '' + +strInject = (s1, pos, s2) -> s1[...pos] + s2 + s1[pos..] + +checkValidComponent = (c) -> + throw new Error 'component missing position field' if typeof c.p != 'number' + + i_type = typeof c.i + d_type = typeof c.d + throw new Error 'component needs an i or d field' unless (i_type == 'string') ^ (d_type == 'string') + + throw new Error 'position cannot be negative' unless c.p >= 0 + +checkValidOp = (op) -> + checkValidComponent(c) for c in op + true + +text.apply = (snapshot, op) -> + checkValidOp op + for component in op + if component.i? + snapshot = strInject snapshot, component.p, component.i + else + deleted = snapshot[component.p...(component.p + component.d.length)] + throw new Error "Delete component '#{component.d}' does not match deleted text '#{deleted}'" unless component.d == deleted + snapshot = snapshot[...component.p] + snapshot[(component.p + component.d.length)..] + + snapshot + + +# Exported for use by the random op generator. +# +# For simplicity, this version of append does not compress adjacent inserts and deletes of +# the same text. It would be nice to change that at some stage. +text._append = append = (newOp, c) -> + return if c.i == '' or c.d == '' + if newOp.length == 0 + newOp.push c + else + last = newOp[newOp.length - 1] + + # Compose the insert into the previous insert if possible + if last.i? && c.i? and last.p <= c.p <= (last.p + last.i.length) + newOp[newOp.length - 1] = {i:strInject(last.i, c.p - last.p, c.i), p:last.p} + else if last.d? && c.d? and c.p <= last.p <= (c.p + c.d.length) + newOp[newOp.length - 1] = {d:strInject(c.d, last.p - c.p, last.d), p:c.p} + else + newOp.push c + +text.compose = (op1, op2) -> + checkValidOp op1 + checkValidOp op2 + + newOp = op1.slice() + append newOp, c for c in op2 + + newOp + +# Attempt to compress the op components together 'as much as possible'. +# This implementation preserves order and preserves create/delete pairs. +text.compress = (op) -> text.compose [], op + +text.normalize = (op) -> + newOp = [] + + # Normalize should allow ops which are a single (unwrapped) component: + # {i:'asdf', p:23}. + # There's no good way to test if something is an array: + # http://perfectionkills.com/instanceof-considered-harmful-or-how-to-write-a-robust-isarray/ + # so this is probably the least bad solution. + op = [op] if op.i? or op.p? + + for c in op + c.p ?= 0 + append newOp, c + + newOp + +# This helper method transforms a position by an op component. +# +# If c is an insert, insertAfter specifies whether the transform +# is pushed after the insert (true) or before it (false). +# +# insertAfter is optional for deletes. +transformPosition = (pos, c, insertAfter) -> + if c.i? + if c.p < pos || (c.p == pos && insertAfter) + pos + c.i.length + else + pos + else + # I think this could also be written as: Math.min(c.p, Math.min(c.p - otherC.p, otherC.d.length)) + # but I think its harder to read that way, and it compiles using ternary operators anyway + # so its no slower written like this. + if pos <= c.p + pos + else if pos <= c.p + c.d.length + c.p + else + pos - c.d.length + +# Helper method to transform a cursor position as a result of an op. +# +# Like transformPosition above, if c is an insert, insertAfter specifies whether the cursor position +# is pushed after an insert (true) or before it (false). +text.transformCursor = (position, op, side) -> + insertAfter = side == 'right' + position = transformPosition position, c, insertAfter for c in op + position + +# Transform an op component by another op component. Asymmetric. +# The result will be appended to destination. +# +# exported for use in JSON type +text._tc = transformComponent = (dest, c, otherC, side) -> + checkValidOp [c] + checkValidOp [otherC] + + if c.i? + append dest, {i:c.i, p:transformPosition(c.p, otherC, side == 'right')} + + else # Delete + if otherC.i? # delete vs insert + s = c.d + if c.p < otherC.p + append dest, {d:s[...otherC.p - c.p], p:c.p} + s = s[(otherC.p - c.p)..] + if s != '' + append dest, {d:s, p:c.p + otherC.i.length} + + else # Delete vs delete + if c.p >= otherC.p + otherC.d.length + append dest, {d:c.d, p:c.p - otherC.d.length} + else if c.p + c.d.length <= otherC.p + append dest, c + else + # They overlap somewhere. + newC = {d:'', p:c.p} + if c.p < otherC.p + newC.d = c.d[...(otherC.p - c.p)] + if c.p + c.d.length > otherC.p + otherC.d.length + newC.d += c.d[(otherC.p + otherC.d.length - c.p)..] + + # This is entirely optional - just for a check that the deleted + # text in the two ops matches + intersectStart = Math.max c.p, otherC.p + intersectEnd = Math.min c.p + c.d.length, otherC.p + otherC.d.length + cIntersect = c.d[intersectStart - c.p...intersectEnd - c.p] + otherIntersect = otherC.d[intersectStart - otherC.p...intersectEnd - otherC.p] + throw new Error 'Delete ops delete different text in the same region of the document' unless cIntersect == otherIntersect + + if newC.d != '' + # This could be rewritten similarly to insert v delete, above. + newC.p = transformPosition newC.p, otherC + append dest, newC + + dest + +invertComponent = (c) -> + if c.i? + {d:c.i, p:c.p} + else + {i:c.d, p:c.p} + +# No need to use append for invert, because the components won't be able to +# cancel with one another. +text.invert = (op) -> (invertComponent c for c in op.slice().reverse()) + + +if WEB? + exports.types ||= {} + + # This is kind of awful - come up with a better way to hook this helper code up. + bootstrapTransform(text, transformComponent, checkValidOp, append) + + # [] is used to prevent closure from renaming types.text + exports.types.text = text +else + module.exports = text + + # The text type really shouldn't need this - it should be possible to define + # an efficient transform function by making a sort of transform map and passing each + # op component through it. + require('./helpers').bootstrapTransform(text, transformComponent, checkValidOp, append) + diff --git a/services/document-updater/app/coffee/sharejs/types/web-prelude.coffee b/services/document-updater/app/coffee/sharejs/types/web-prelude.coffee new file mode 100644 index 0000000000..3c045532dc --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/types/web-prelude.coffee @@ -0,0 +1,11 @@ +# This is included at the top of each compiled type file for the web. + +`/** + @const + @type {boolean} +*/ +var WEB = true; +` + +exports = window['sharejs'] + diff --git a/services/document-updater/app/coffee/sharejs/web-prelude.coffee b/services/document-updater/app/coffee/sharejs/web-prelude.coffee new file mode 100644 index 0000000000..3c045532dc --- /dev/null +++ b/services/document-updater/app/coffee/sharejs/web-prelude.coffee @@ -0,0 +1,11 @@ +# This is included at the top of each compiled type file for the web. + +`/** + @const + @type {boolean} +*/ +var WEB = true; +` + +exports = window['sharejs'] + diff --git a/services/document-updater/app/lib/diff_match_patch.js b/services/document-updater/app/lib/diff_match_patch.js new file mode 100644 index 0000000000..112130e097 --- /dev/null +++ b/services/document-updater/app/lib/diff_match_patch.js @@ -0,0 +1,2193 @@ +/** + * Diff Match and Patch + * + * Copyright 2006 Google Inc. + * http://code.google.com/p/google-diff-match-patch/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Computes the difference between two texts to create a patch. + * Applies the patch onto another text, allowing for errors. + * @author fraser@google.com (Neil Fraser) + */ + +/** + * Class containing the diff, match and patch methods. + * @constructor + */ +function diff_match_patch() { + + // Defaults. + // Redefine these in your program to override the defaults. + + // Number of seconds to map a diff before giving up (0 for infinity). + this.Diff_Timeout = 1.0; + // Cost of an empty edit operation in terms of edit characters. + this.Diff_EditCost = 4; + // At what point is no match declared (0.0 = perfection, 1.0 = very loose). + this.Match_Threshold = 0.5; + // How far to search for a match (0 = exact location, 1000+ = broad match). + // A match this many characters away from the expected location will add + // 1.0 to the score (0.0 is a perfect match). + this.Match_Distance = 1000; + // When deleting a large block of text (over ~64 characters), how close do + // the contents have to be to match the expected contents. (0.0 = perfection, + // 1.0 = very loose). Note that Match_Threshold controls how closely the + // end points of a delete need to match. + this.Patch_DeleteThreshold = 0.5; + // Chunk size for context length. + this.Patch_Margin = 4; + + // The number of bits in an int. + this.Match_MaxBits = 32; +} + + +// DIFF FUNCTIONS + + +/** + * The data structure representing a diff is an array of tuples: + * [[DIFF_DELETE, 'Hello'], [DIFF_INSERT, 'Goodbye'], [DIFF_EQUAL, ' world.']] + * which means: delete 'Hello', add 'Goodbye' and keep ' world.' + */ +var DIFF_DELETE = -1; +var DIFF_INSERT = 1; +var DIFF_EQUAL = 0; + +/** @typedef {{0: number, 1: string}} */ +diff_match_patch.Diff; + + +/** + * Find the differences between two texts. Simplifies the problem by stripping + * any common prefix or suffix off the texts before diffing. + * @param {string} text1 Old string to be diffed. + * @param {string} text2 New string to be diffed. + * @param {boolean=} opt_checklines Optional speedup flag. If present and false, + * then don't run a line-level diff first to identify the changed areas. + * Defaults to true, which does a faster, slightly less optimal diff. + * @param {number} opt_deadline Optional time when the diff should be complete + * by. Used internally for recursive calls. Users should set DiffTimeout + * instead. + * @return {!Array.} Array of diff tuples. + */ +diff_match_patch.prototype.diff_main = function(text1, text2, opt_checklines, + opt_deadline) { + // Set a deadline by which time the diff must be complete. + if (typeof opt_deadline == 'undefined') { + if (this.Diff_Timeout <= 0) { + opt_deadline = Number.MAX_VALUE; + } else { + opt_deadline = (new Date).getTime() + this.Diff_Timeout * 1000; + } + } + var deadline = opt_deadline; + + // Check for null inputs. + if (text1 == null || text2 == null) { + throw new Error('Null input. (diff_main)'); + } + + // Check for equality (speedup). + if (text1 == text2) { + if (text1) { + return [[DIFF_EQUAL, text1]]; + } + return []; + } + + if (typeof opt_checklines == 'undefined') { + opt_checklines = true; + } + var checklines = opt_checklines; + + // Trim off common prefix (speedup). + var commonlength = this.diff_commonPrefix(text1, text2); + var commonprefix = text1.substring(0, commonlength); + text1 = text1.substring(commonlength); + text2 = text2.substring(commonlength); + + // Trim off common suffix (speedup). + commonlength = this.diff_commonSuffix(text1, text2); + var commonsuffix = text1.substring(text1.length - commonlength); + text1 = text1.substring(0, text1.length - commonlength); + text2 = text2.substring(0, text2.length - commonlength); + + // Compute the diff on the middle block. + var diffs = this.diff_compute_(text1, text2, checklines, deadline); + + // Restore the prefix and suffix. + if (commonprefix) { + diffs.unshift([DIFF_EQUAL, commonprefix]); + } + if (commonsuffix) { + diffs.push([DIFF_EQUAL, commonsuffix]); + } + this.diff_cleanupMerge(diffs); + return diffs; +}; + + +/** + * Find the differences between two texts. Assumes that the texts do not + * have any common prefix or suffix. + * @param {string} text1 Old string to be diffed. + * @param {string} text2 New string to be diffed. + * @param {boolean} checklines Speedup flag. If false, then don't run a + * line-level diff first to identify the changed areas. + * If true, then run a faster, slightly less optimal diff. + * @param {number} deadline Time when the diff should be complete by. + * @return {!Array.} Array of diff tuples. + * @private + */ +diff_match_patch.prototype.diff_compute_ = function(text1, text2, checklines, + deadline) { + var diffs; + + if (!text1) { + // Just add some text (speedup). + return [[DIFF_INSERT, text2]]; + } + + if (!text2) { + // Just delete some text (speedup). + return [[DIFF_DELETE, text1]]; + } + + var longtext = text1.length > text2.length ? text1 : text2; + var shorttext = text1.length > text2.length ? text2 : text1; + var i = longtext.indexOf(shorttext); + if (i != -1) { + // Shorter text is inside the longer text (speedup). + diffs = [[DIFF_INSERT, longtext.substring(0, i)], + [DIFF_EQUAL, shorttext], + [DIFF_INSERT, longtext.substring(i + shorttext.length)]]; + // Swap insertions for deletions if diff is reversed. + if (text1.length > text2.length) { + diffs[0][0] = diffs[2][0] = DIFF_DELETE; + } + return diffs; + } + + if (shorttext.length == 1) { + // Single character string. + // After the previous speedup, the character can't be an equality. + return [[DIFF_DELETE, text1], [DIFF_INSERT, text2]]; + } + + // Check to see if the problem can be split in two. + var hm = this.diff_halfMatch_(text1, text2); + if (hm) { + // A half-match was found, sort out the return data. + var text1_a = hm[0]; + var text1_b = hm[1]; + var text2_a = hm[2]; + var text2_b = hm[3]; + var mid_common = hm[4]; + // Send both pairs off for separate processing. + var diffs_a = this.diff_main(text1_a, text2_a, checklines, deadline); + var diffs_b = this.diff_main(text1_b, text2_b, checklines, deadline); + // Merge the results. + return diffs_a.concat([[DIFF_EQUAL, mid_common]], diffs_b); + } + + if (checklines && text1.length > 100 && text2.length > 100) { + return this.diff_lineMode_(text1, text2, deadline); + } + + return this.diff_bisect_(text1, text2, deadline); +}; + + +/** + * Do a quick line-level diff on both strings, then rediff the parts for + * greater accuracy. + * This speedup can produce non-minimal diffs. + * @param {string} text1 Old string to be diffed. + * @param {string} text2 New string to be diffed. + * @param {number} deadline Time when the diff should be complete by. + * @return {!Array.} Array of diff tuples. + * @private + */ +diff_match_patch.prototype.diff_lineMode_ = function(text1, text2, deadline) { + // Scan the text on a line-by-line basis first. + var a = this.diff_linesToChars_(text1, text2); + text1 = a.chars1; + text2 = a.chars2; + var linearray = a.lineArray; + + var diffs = this.diff_main(text1, text2, false, deadline); + + // Convert the diff back to original text. + this.diff_charsToLines_(diffs, linearray); + // Eliminate freak matches (e.g. blank lines) + this.diff_cleanupSemantic(diffs); + + // Rediff any replacement blocks, this time character-by-character. + // Add a dummy entry at the end. + diffs.push([DIFF_EQUAL, '']); + var pointer = 0; + var count_delete = 0; + var count_insert = 0; + var text_delete = ''; + var text_insert = ''; + while (pointer < diffs.length) { + switch (diffs[pointer][0]) { + case DIFF_INSERT: + count_insert++; + text_insert += diffs[pointer][1]; + break; + case DIFF_DELETE: + count_delete++; + text_delete += diffs[pointer][1]; + break; + case DIFF_EQUAL: + // Upon reaching an equality, check for prior redundancies. + if (count_delete >= 1 && count_insert >= 1) { + // Delete the offending records and add the merged ones. + diffs.splice(pointer - count_delete - count_insert, + count_delete + count_insert); + pointer = pointer - count_delete - count_insert; + var a = this.diff_main(text_delete, text_insert, false, deadline); + for (var j = a.length - 1; j >= 0; j--) { + diffs.splice(pointer, 0, a[j]); + } + pointer = pointer + a.length; + } + count_insert = 0; + count_delete = 0; + text_delete = ''; + text_insert = ''; + break; + } + pointer++; + } + diffs.pop(); // Remove the dummy entry at the end. + + return diffs; +}; + + +/** + * Find the 'middle snake' of a diff, split the problem in two + * and return the recursively constructed diff. + * See Myers 1986 paper: An O(ND) Difference Algorithm and Its Variations. + * @param {string} text1 Old string to be diffed. + * @param {string} text2 New string to be diffed. + * @param {number} deadline Time at which to bail if not yet complete. + * @return {!Array.} Array of diff tuples. + * @private + */ +diff_match_patch.prototype.diff_bisect_ = function(text1, text2, deadline) { + // Cache the text lengths to prevent multiple calls. + var text1_length = text1.length; + var text2_length = text2.length; + var max_d = Math.ceil((text1_length + text2_length) / 2); + var v_offset = max_d; + var v_length = 2 * max_d; + var v1 = new Array(v_length); + var v2 = new Array(v_length); + // Setting all elements to -1 is faster in Chrome & Firefox than mixing + // integers and undefined. + for (var x = 0; x < v_length; x++) { + v1[x] = -1; + v2[x] = -1; + } + v1[v_offset + 1] = 0; + v2[v_offset + 1] = 0; + var delta = text1_length - text2_length; + // If the total number of characters is odd, then the front path will collide + // with the reverse path. + var front = (delta % 2 != 0); + // Offsets for start and end of k loop. + // Prevents mapping of space beyond the grid. + var k1start = 0; + var k1end = 0; + var k2start = 0; + var k2end = 0; + for (var d = 0; d < max_d; d++) { + // Bail out if deadline is reached. + if ((new Date()).getTime() > deadline) { + break; + } + + // Walk the front path one step. + for (var k1 = -d + k1start; k1 <= d - k1end; k1 += 2) { + var k1_offset = v_offset + k1; + var x1; + if (k1 == -d || (k1 != d && v1[k1_offset - 1] < v1[k1_offset + 1])) { + x1 = v1[k1_offset + 1]; + } else { + x1 = v1[k1_offset - 1] + 1; + } + var y1 = x1 - k1; + while (x1 < text1_length && y1 < text2_length && + text1.charAt(x1) == text2.charAt(y1)) { + x1++; + y1++; + } + v1[k1_offset] = x1; + if (x1 > text1_length) { + // Ran off the right of the graph. + k1end += 2; + } else if (y1 > text2_length) { + // Ran off the bottom of the graph. + k1start += 2; + } else if (front) { + var k2_offset = v_offset + delta - k1; + if (k2_offset >= 0 && k2_offset < v_length && v2[k2_offset] != -1) { + // Mirror x2 onto top-left coordinate system. + var x2 = text1_length - v2[k2_offset]; + if (x1 >= x2) { + // Overlap detected. + return this.diff_bisectSplit_(text1, text2, x1, y1, deadline); + } + } + } + } + + // Walk the reverse path one step. + for (var k2 = -d + k2start; k2 <= d - k2end; k2 += 2) { + var k2_offset = v_offset + k2; + var x2; + if (k2 == -d || (k2 != d && v2[k2_offset - 1] < v2[k2_offset + 1])) { + x2 = v2[k2_offset + 1]; + } else { + x2 = v2[k2_offset - 1] + 1; + } + var y2 = x2 - k2; + while (x2 < text1_length && y2 < text2_length && + text1.charAt(text1_length - x2 - 1) == + text2.charAt(text2_length - y2 - 1)) { + x2++; + y2++; + } + v2[k2_offset] = x2; + if (x2 > text1_length) { + // Ran off the left of the graph. + k2end += 2; + } else if (y2 > text2_length) { + // Ran off the top of the graph. + k2start += 2; + } else if (!front) { + var k1_offset = v_offset + delta - k2; + if (k1_offset >= 0 && k1_offset < v_length && v1[k1_offset] != -1) { + var x1 = v1[k1_offset]; + var y1 = v_offset + x1 - k1_offset; + // Mirror x2 onto top-left coordinate system. + x2 = text1_length - x2; + if (x1 >= x2) { + // Overlap detected. + return this.diff_bisectSplit_(text1, text2, x1, y1, deadline); + } + } + } + } + } + // Diff took too long and hit the deadline or + // number of diffs equals number of characters, no commonality at all. + return [[DIFF_DELETE, text1], [DIFF_INSERT, text2]]; +}; + + +/** + * Given the location of the 'middle snake', split the diff in two parts + * and recurse. + * @param {string} text1 Old string to be diffed. + * @param {string} text2 New string to be diffed. + * @param {number} x Index of split point in text1. + * @param {number} y Index of split point in text2. + * @param {number} deadline Time at which to bail if not yet complete. + * @return {!Array.} Array of diff tuples. + * @private + */ +diff_match_patch.prototype.diff_bisectSplit_ = function(text1, text2, x, y, + deadline) { + var text1a = text1.substring(0, x); + var text2a = text2.substring(0, y); + var text1b = text1.substring(x); + var text2b = text2.substring(y); + + // Compute both diffs serially. + var diffs = this.diff_main(text1a, text2a, false, deadline); + var diffsb = this.diff_main(text1b, text2b, false, deadline); + + return diffs.concat(diffsb); +}; + + +/** + * Split two texts into an array of strings. Reduce the texts to a string of + * hashes where each Unicode character represents one line. + * @param {string} text1 First string. + * @param {string} text2 Second string. + * @return {{chars1: string, chars2: string, lineArray: !Array.}} + * An object containing the encoded text1, the encoded text2 and + * the array of unique strings. + * The zeroth element of the array of unique strings is intentionally blank. + * @private + */ +diff_match_patch.prototype.diff_linesToChars_ = function(text1, text2) { + var lineArray = []; // e.g. lineArray[4] == 'Hello\n' + var lineHash = {}; // e.g. lineHash['Hello\n'] == 4 + + // '\x00' is a valid character, but various debuggers don't like it. + // So we'll insert a junk entry to avoid generating a null character. + lineArray[0] = ''; + + /** + * Split a text into an array of strings. Reduce the texts to a string of + * hashes where each Unicode character represents one line. + * Modifies linearray and linehash through being a closure. + * @param {string} text String to encode. + * @return {string} Encoded string. + * @private + */ + function diff_linesToCharsMunge_(text) { + var chars = ''; + // Walk the text, pulling out a substring for each line. + // text.split('\n') would would temporarily double our memory footprint. + // Modifying text would create many large strings to garbage collect. + var lineStart = 0; + var lineEnd = -1; + // Keeping our own length variable is faster than looking it up. + var lineArrayLength = lineArray.length; + while (lineEnd < text.length - 1) { + lineEnd = text.indexOf('\n', lineStart); + if (lineEnd == -1) { + lineEnd = text.length - 1; + } + var line = text.substring(lineStart, lineEnd + 1); + lineStart = lineEnd + 1; + + if (lineHash.hasOwnProperty ? lineHash.hasOwnProperty(line) : + (lineHash[line] !== undefined)) { + chars += String.fromCharCode(lineHash[line]); + } else { + chars += String.fromCharCode(lineArrayLength); + lineHash[line] = lineArrayLength; + lineArray[lineArrayLength++] = line; + } + } + return chars; + } + + var chars1 = diff_linesToCharsMunge_(text1); + var chars2 = diff_linesToCharsMunge_(text2); + return {chars1: chars1, chars2: chars2, lineArray: lineArray}; +}; + + +/** + * Rehydrate the text in a diff from a string of line hashes to real lines of + * text. + * @param {!Array.} diffs Array of diff tuples. + * @param {!Array.} lineArray Array of unique strings. + * @private + */ +diff_match_patch.prototype.diff_charsToLines_ = function(diffs, lineArray) { + for (var x = 0; x < diffs.length; x++) { + var chars = diffs[x][1]; + var text = []; + for (var y = 0; y < chars.length; y++) { + text[y] = lineArray[chars.charCodeAt(y)]; + } + diffs[x][1] = text.join(''); + } +}; + + +/** + * Determine the common prefix of two strings. + * @param {string} text1 First string. + * @param {string} text2 Second string. + * @return {number} The number of characters common to the start of each + * string. + */ +diff_match_patch.prototype.diff_commonPrefix = function(text1, text2) { + // Quick check for common null cases. + if (!text1 || !text2 || text1.charAt(0) != text2.charAt(0)) { + return 0; + } + // Binary search. + // Performance analysis: http://neil.fraser.name/news/2007/10/09/ + var pointermin = 0; + var pointermax = Math.min(text1.length, text2.length); + var pointermid = pointermax; + var pointerstart = 0; + while (pointermin < pointermid) { + if (text1.substring(pointerstart, pointermid) == + text2.substring(pointerstart, pointermid)) { + pointermin = pointermid; + pointerstart = pointermin; + } else { + pointermax = pointermid; + } + pointermid = Math.floor((pointermax - pointermin) / 2 + pointermin); + } + return pointermid; +}; + + +/** + * Determine the common suffix of two strings. + * @param {string} text1 First string. + * @param {string} text2 Second string. + * @return {number} The number of characters common to the end of each string. + */ +diff_match_patch.prototype.diff_commonSuffix = function(text1, text2) { + // Quick check for common null cases. + if (!text1 || !text2 || + text1.charAt(text1.length - 1) != text2.charAt(text2.length - 1)) { + return 0; + } + // Binary search. + // Performance analysis: http://neil.fraser.name/news/2007/10/09/ + var pointermin = 0; + var pointermax = Math.min(text1.length, text2.length); + var pointermid = pointermax; + var pointerend = 0; + while (pointermin < pointermid) { + if (text1.substring(text1.length - pointermid, text1.length - pointerend) == + text2.substring(text2.length - pointermid, text2.length - pointerend)) { + pointermin = pointermid; + pointerend = pointermin; + } else { + pointermax = pointermid; + } + pointermid = Math.floor((pointermax - pointermin) / 2 + pointermin); + } + return pointermid; +}; + + +/** + * Determine if the suffix of one string is the prefix of another. + * @param {string} text1 First string. + * @param {string} text2 Second string. + * @return {number} The number of characters common to the end of the first + * string and the start of the second string. + * @private + */ +diff_match_patch.prototype.diff_commonOverlap_ = function(text1, text2) { + // Cache the text lengths to prevent multiple calls. + var text1_length = text1.length; + var text2_length = text2.length; + // Eliminate the null case. + if (text1_length == 0 || text2_length == 0) { + return 0; + } + // Truncate the longer string. + if (text1_length > text2_length) { + text1 = text1.substring(text1_length - text2_length); + } else if (text1_length < text2_length) { + text2 = text2.substring(0, text1_length); + } + var text_length = Math.min(text1_length, text2_length); + // Quick check for the worst case. + if (text1 == text2) { + return text_length; + } + + // Start by looking for a single character match + // and increase length until no match is found. + // Performance analysis: http://neil.fraser.name/news/2010/11/04/ + var best = 0; + var length = 1; + while (true) { + var pattern = text1.substring(text_length - length); + var found = text2.indexOf(pattern); + if (found == -1) { + return best; + } + length += found; + if (found == 0 || text1.substring(text_length - length) == + text2.substring(0, length)) { + best = length; + length++; + } + } +}; + + +/** + * Do the two texts share a substring which is at least half the length of the + * longer text? + * This speedup can produce non-minimal diffs. + * @param {string} text1 First string. + * @param {string} text2 Second string. + * @return {Array.} Five element Array, containing the prefix of + * text1, the suffix of text1, the prefix of text2, the suffix of + * text2 and the common middle. Or null if there was no match. + * @private + */ +diff_match_patch.prototype.diff_halfMatch_ = function(text1, text2) { + if (this.Diff_Timeout <= 0) { + // Don't risk returning a non-optimal diff if we have unlimited time. + return null; + } + var longtext = text1.length > text2.length ? text1 : text2; + var shorttext = text1.length > text2.length ? text2 : text1; + if (longtext.length < 4 || shorttext.length * 2 < longtext.length) { + return null; // Pointless. + } + var dmp = this; // 'this' becomes 'window' in a closure. + + /** + * Does a substring of shorttext exist within longtext such that the substring + * is at least half the length of longtext? + * Closure, but does not reference any external variables. + * @param {string} longtext Longer string. + * @param {string} shorttext Shorter string. + * @param {number} i Start index of quarter length substring within longtext. + * @return {Array.} Five element Array, containing the prefix of + * longtext, the suffix of longtext, the prefix of shorttext, the suffix + * of shorttext and the common middle. Or null if there was no match. + * @private + */ + function diff_halfMatchI_(longtext, shorttext, i) { + // Start with a 1/4 length substring at position i as a seed. + var seed = longtext.substring(i, i + Math.floor(longtext.length / 4)); + var j = -1; + var best_common = ''; + var best_longtext_a, best_longtext_b, best_shorttext_a, best_shorttext_b; + while ((j = shorttext.indexOf(seed, j + 1)) != -1) { + var prefixLength = dmp.diff_commonPrefix(longtext.substring(i), + shorttext.substring(j)); + var suffixLength = dmp.diff_commonSuffix(longtext.substring(0, i), + shorttext.substring(0, j)); + if (best_common.length < suffixLength + prefixLength) { + best_common = shorttext.substring(j - suffixLength, j) + + shorttext.substring(j, j + prefixLength); + best_longtext_a = longtext.substring(0, i - suffixLength); + best_longtext_b = longtext.substring(i + prefixLength); + best_shorttext_a = shorttext.substring(0, j - suffixLength); + best_shorttext_b = shorttext.substring(j + prefixLength); + } + } + if (best_common.length * 2 >= longtext.length) { + return [best_longtext_a, best_longtext_b, + best_shorttext_a, best_shorttext_b, best_common]; + } else { + return null; + } + } + + // First check if the second quarter is the seed for a half-match. + var hm1 = diff_halfMatchI_(longtext, shorttext, + Math.ceil(longtext.length / 4)); + // Check again based on the third quarter. + var hm2 = diff_halfMatchI_(longtext, shorttext, + Math.ceil(longtext.length / 2)); + var hm; + if (!hm1 && !hm2) { + return null; + } else if (!hm2) { + hm = hm1; + } else if (!hm1) { + hm = hm2; + } else { + // Both matched. Select the longest. + hm = hm1[4].length > hm2[4].length ? hm1 : hm2; + } + + // A half-match was found, sort out the return data. + var text1_a, text1_b, text2_a, text2_b; + if (text1.length > text2.length) { + text1_a = hm[0]; + text1_b = hm[1]; + text2_a = hm[2]; + text2_b = hm[3]; + } else { + text2_a = hm[0]; + text2_b = hm[1]; + text1_a = hm[2]; + text1_b = hm[3]; + } + var mid_common = hm[4]; + return [text1_a, text1_b, text2_a, text2_b, mid_common]; +}; + + +/** + * Reduce the number of edits by eliminating semantically trivial equalities. + * @param {!Array.} diffs Array of diff tuples. + */ +diff_match_patch.prototype.diff_cleanupSemantic = function(diffs) { + var changes = false; + var equalities = []; // Stack of indices where equalities are found. + var equalitiesLength = 0; // Keeping our own length var is faster in JS. + /** @type {?string} */ + var lastequality = null; + // Always equal to diffs[equalities[equalitiesLength - 1]][1] + var pointer = 0; // Index of current position. + // Number of characters that changed prior to the equality. + var length_insertions1 = 0; + var length_deletions1 = 0; + // Number of characters that changed after the equality. + var length_insertions2 = 0; + var length_deletions2 = 0; + while (pointer < diffs.length) { + if (diffs[pointer][0] == DIFF_EQUAL) { // Equality found. + equalities[equalitiesLength++] = pointer; + length_insertions1 = length_insertions2; + length_deletions1 = length_deletions2; + length_insertions2 = 0; + length_deletions2 = 0; + lastequality = diffs[pointer][1]; + } else { // An insertion or deletion. + if (diffs[pointer][0] == DIFF_INSERT) { + length_insertions2 += diffs[pointer][1].length; + } else { + length_deletions2 += diffs[pointer][1].length; + } + // Eliminate an equality that is smaller or equal to the edits on both + // sides of it. + if (lastequality && (lastequality.length <= + Math.max(length_insertions1, length_deletions1)) && + (lastequality.length <= Math.max(length_insertions2, + length_deletions2))) { + // Duplicate record. + diffs.splice(equalities[equalitiesLength - 1], 0, + [DIFF_DELETE, lastequality]); + // Change second copy to insert. + diffs[equalities[equalitiesLength - 1] + 1][0] = DIFF_INSERT; + // Throw away the equality we just deleted. + equalitiesLength--; + // Throw away the previous equality (it needs to be reevaluated). + equalitiesLength--; + pointer = equalitiesLength > 0 ? equalities[equalitiesLength - 1] : -1; + length_insertions1 = 0; // Reset the counters. + length_deletions1 = 0; + length_insertions2 = 0; + length_deletions2 = 0; + lastequality = null; + changes = true; + } + } + pointer++; + } + + // Normalize the diff. + if (changes) { + this.diff_cleanupMerge(diffs); + } + this.diff_cleanupSemanticLossless(diffs); + + // Find any overlaps between deletions and insertions. + // e.g: abcxxxxxxdef + // -> abcxxxdef + // e.g: xxxabcdefxxx + // -> defxxxabc + // Only extract an overlap if it is as big as the edit ahead or behind it. + pointer = 1; + while (pointer < diffs.length) { + if (diffs[pointer - 1][0] == DIFF_DELETE && + diffs[pointer][0] == DIFF_INSERT) { + var deletion = diffs[pointer - 1][1]; + var insertion = diffs[pointer][1]; + var overlap_length1 = this.diff_commonOverlap_(deletion, insertion); + var overlap_length2 = this.diff_commonOverlap_(insertion, deletion); + if (overlap_length1 >= overlap_length2) { + if (overlap_length1 >= deletion.length / 2 || + overlap_length1 >= insertion.length / 2) { + // Overlap found. Insert an equality and trim the surrounding edits. + diffs.splice(pointer, 0, + [DIFF_EQUAL, insertion.substring(0, overlap_length1)]); + diffs[pointer - 1][1] = + deletion.substring(0, deletion.length - overlap_length1); + diffs[pointer + 1][1] = insertion.substring(overlap_length1); + pointer++; + } + } else { + if (overlap_length2 >= deletion.length / 2 || + overlap_length2 >= insertion.length / 2) { + // Reverse overlap found. + // Insert an equality and swap and trim the surrounding edits. + diffs.splice(pointer, 0, + [DIFF_EQUAL, deletion.substring(0, overlap_length2)]); + diffs[pointer - 1][0] = DIFF_INSERT; + diffs[pointer - 1][1] = + insertion.substring(0, insertion.length - overlap_length2); + diffs[pointer + 1][0] = DIFF_DELETE; + diffs[pointer + 1][1] = + deletion.substring(overlap_length2); + pointer++; + } + } + pointer++; + } + pointer++; + } +}; + + +/** + * Look for single edits surrounded on both sides by equalities + * which can be shifted sideways to align the edit to a word boundary. + * e.g: The cat came. -> The cat came. + * @param {!Array.} diffs Array of diff tuples. + */ +diff_match_patch.prototype.diff_cleanupSemanticLossless = function(diffs) { + /** + * Given two strings, compute a score representing whether the internal + * boundary falls on logical boundaries. + * Scores range from 6 (best) to 0 (worst). + * Closure, but does not reference any external variables. + * @param {string} one First string. + * @param {string} two Second string. + * @return {number} The score. + * @private + */ + function diff_cleanupSemanticScore_(one, two) { + if (!one || !two) { + // Edges are the best. + return 6; + } + + // Each port of this function behaves slightly differently due to + // subtle differences in each language's definition of things like + // 'whitespace'. Since this function's purpose is largely cosmetic, + // the choice has been made to use each language's native features + // rather than force total conformity. + var char1 = one.charAt(one.length - 1); + var char2 = two.charAt(0); + var nonAlphaNumeric1 = char1.match(diff_match_patch.nonAlphaNumericRegex_); + var nonAlphaNumeric2 = char2.match(diff_match_patch.nonAlphaNumericRegex_); + var whitespace1 = nonAlphaNumeric1 && + char1.match(diff_match_patch.whitespaceRegex_); + var whitespace2 = nonAlphaNumeric2 && + char2.match(diff_match_patch.whitespaceRegex_); + var lineBreak1 = whitespace1 && + char1.match(diff_match_patch.linebreakRegex_); + var lineBreak2 = whitespace2 && + char2.match(diff_match_patch.linebreakRegex_); + var blankLine1 = lineBreak1 && + one.match(diff_match_patch.blanklineEndRegex_); + var blankLine2 = lineBreak2 && + two.match(diff_match_patch.blanklineStartRegex_); + + if (blankLine1 || blankLine2) { + // Five points for blank lines. + return 5; + } else if (lineBreak1 || lineBreak2) { + // Four points for line breaks. + return 4; + } else if (nonAlphaNumeric1 && !whitespace1 && whitespace2) { + // Three points for end of sentences. + return 3; + } else if (whitespace1 || whitespace2) { + // Two points for whitespace. + return 2; + } else if (nonAlphaNumeric1 || nonAlphaNumeric2) { + // One point for non-alphanumeric. + return 1; + } + return 0; + } + + var pointer = 1; + // Intentionally ignore the first and last element (don't need checking). + while (pointer < diffs.length - 1) { + if (diffs[pointer - 1][0] == DIFF_EQUAL && + diffs[pointer + 1][0] == DIFF_EQUAL) { + // This is a single edit surrounded by equalities. + var equality1 = diffs[pointer - 1][1]; + var edit = diffs[pointer][1]; + var equality2 = diffs[pointer + 1][1]; + + // First, shift the edit as far left as possible. + var commonOffset = this.diff_commonSuffix(equality1, edit); + if (commonOffset) { + var commonString = edit.substring(edit.length - commonOffset); + equality1 = equality1.substring(0, equality1.length - commonOffset); + edit = commonString + edit.substring(0, edit.length - commonOffset); + equality2 = commonString + equality2; + } + + // Second, step character by character right, looking for the best fit. + var bestEquality1 = equality1; + var bestEdit = edit; + var bestEquality2 = equality2; + var bestScore = diff_cleanupSemanticScore_(equality1, edit) + + diff_cleanupSemanticScore_(edit, equality2); + while (edit.charAt(0) === equality2.charAt(0)) { + equality1 += edit.charAt(0); + edit = edit.substring(1) + equality2.charAt(0); + equality2 = equality2.substring(1); + var score = diff_cleanupSemanticScore_(equality1, edit) + + diff_cleanupSemanticScore_(edit, equality2); + // The >= encourages trailing rather than leading whitespace on edits. + if (score >= bestScore) { + bestScore = score; + bestEquality1 = equality1; + bestEdit = edit; + bestEquality2 = equality2; + } + } + + if (diffs[pointer - 1][1] != bestEquality1) { + // We have an improvement, save it back to the diff. + if (bestEquality1) { + diffs[pointer - 1][1] = bestEquality1; + } else { + diffs.splice(pointer - 1, 1); + pointer--; + } + diffs[pointer][1] = bestEdit; + if (bestEquality2) { + diffs[pointer + 1][1] = bestEquality2; + } else { + diffs.splice(pointer + 1, 1); + pointer--; + } + } + } + pointer++; + } +}; + +// Define some regex patterns for matching boundaries. +diff_match_patch.nonAlphaNumericRegex_ = /[^a-zA-Z0-9]/; +diff_match_patch.whitespaceRegex_ = /\s/; +diff_match_patch.linebreakRegex_ = /[\r\n]/; +diff_match_patch.blanklineEndRegex_ = /\n\r?\n$/; +diff_match_patch.blanklineStartRegex_ = /^\r?\n\r?\n/; + +/** + * Reduce the number of edits by eliminating operationally trivial equalities. + * @param {!Array.} diffs Array of diff tuples. + */ +diff_match_patch.prototype.diff_cleanupEfficiency = function(diffs) { + var changes = false; + var equalities = []; // Stack of indices where equalities are found. + var equalitiesLength = 0; // Keeping our own length var is faster in JS. + /** @type {?string} */ + var lastequality = null; + // Always equal to diffs[equalities[equalitiesLength - 1]][1] + var pointer = 0; // Index of current position. + // Is there an insertion operation before the last equality. + var pre_ins = false; + // Is there a deletion operation before the last equality. + var pre_del = false; + // Is there an insertion operation after the last equality. + var post_ins = false; + // Is there a deletion operation after the last equality. + var post_del = false; + while (pointer < diffs.length) { + if (diffs[pointer][0] == DIFF_EQUAL) { // Equality found. + if (diffs[pointer][1].length < this.Diff_EditCost && + (post_ins || post_del)) { + // Candidate found. + equalities[equalitiesLength++] = pointer; + pre_ins = post_ins; + pre_del = post_del; + lastequality = diffs[pointer][1]; + } else { + // Not a candidate, and can never become one. + equalitiesLength = 0; + lastequality = null; + } + post_ins = post_del = false; + } else { // An insertion or deletion. + if (diffs[pointer][0] == DIFF_DELETE) { + post_del = true; + } else { + post_ins = true; + } + /* + * Five types to be split: + * ABXYCD + * AXCD + * ABXC + * AXCD + * ABXC + */ + if (lastequality && ((pre_ins && pre_del && post_ins && post_del) || + ((lastequality.length < this.Diff_EditCost / 2) && + (pre_ins + pre_del + post_ins + post_del) == 3))) { + // Duplicate record. + diffs.splice(equalities[equalitiesLength - 1], 0, + [DIFF_DELETE, lastequality]); + // Change second copy to insert. + diffs[equalities[equalitiesLength - 1] + 1][0] = DIFF_INSERT; + equalitiesLength--; // Throw away the equality we just deleted; + lastequality = null; + if (pre_ins && pre_del) { + // No changes made which could affect previous entry, keep going. + post_ins = post_del = true; + equalitiesLength = 0; + } else { + equalitiesLength--; // Throw away the previous equality. + pointer = equalitiesLength > 0 ? + equalities[equalitiesLength - 1] : -1; + post_ins = post_del = false; + } + changes = true; + } + } + pointer++; + } + + if (changes) { + this.diff_cleanupMerge(diffs); + } +}; + + +/** + * Reorder and merge like edit sections. Merge equalities. + * Any edit section can move as long as it doesn't cross an equality. + * @param {!Array.} diffs Array of diff tuples. + */ +diff_match_patch.prototype.diff_cleanupMerge = function(diffs) { + diffs.push([DIFF_EQUAL, '']); // Add a dummy entry at the end. + var pointer = 0; + var count_delete = 0; + var count_insert = 0; + var text_delete = ''; + var text_insert = ''; + var commonlength; + while (pointer < diffs.length) { + switch (diffs[pointer][0]) { + case DIFF_INSERT: + count_insert++; + text_insert += diffs[pointer][1]; + pointer++; + break; + case DIFF_DELETE: + count_delete++; + text_delete += diffs[pointer][1]; + pointer++; + break; + case DIFF_EQUAL: + // Upon reaching an equality, check for prior redundancies. + if (count_delete + count_insert > 1) { + if (count_delete !== 0 && count_insert !== 0) { + // Factor out any common prefixies. + commonlength = this.diff_commonPrefix(text_insert, text_delete); + if (commonlength !== 0) { + if ((pointer - count_delete - count_insert) > 0 && + diffs[pointer - count_delete - count_insert - 1][0] == + DIFF_EQUAL) { + diffs[pointer - count_delete - count_insert - 1][1] += + text_insert.substring(0, commonlength); + } else { + diffs.splice(0, 0, [DIFF_EQUAL, + text_insert.substring(0, commonlength)]); + pointer++; + } + text_insert = text_insert.substring(commonlength); + text_delete = text_delete.substring(commonlength); + } + // Factor out any common suffixies. + commonlength = this.diff_commonSuffix(text_insert, text_delete); + if (commonlength !== 0) { + diffs[pointer][1] = text_insert.substring(text_insert.length - + commonlength) + diffs[pointer][1]; + text_insert = text_insert.substring(0, text_insert.length - + commonlength); + text_delete = text_delete.substring(0, text_delete.length - + commonlength); + } + } + // Delete the offending records and add the merged ones. + if (count_delete === 0) { + diffs.splice(pointer - count_insert, + count_delete + count_insert, [DIFF_INSERT, text_insert]); + } else if (count_insert === 0) { + diffs.splice(pointer - count_delete, + count_delete + count_insert, [DIFF_DELETE, text_delete]); + } else { + diffs.splice(pointer - count_delete - count_insert, + count_delete + count_insert, [DIFF_DELETE, text_delete], + [DIFF_INSERT, text_insert]); + } + pointer = pointer - count_delete - count_insert + + (count_delete ? 1 : 0) + (count_insert ? 1 : 0) + 1; + } else if (pointer !== 0 && diffs[pointer - 1][0] == DIFF_EQUAL) { + // Merge this equality with the previous one. + diffs[pointer - 1][1] += diffs[pointer][1]; + diffs.splice(pointer, 1); + } else { + pointer++; + } + count_insert = 0; + count_delete = 0; + text_delete = ''; + text_insert = ''; + break; + } + } + if (diffs[diffs.length - 1][1] === '') { + diffs.pop(); // Remove the dummy entry at the end. + } + + // Second pass: look for single edits surrounded on both sides by equalities + // which can be shifted sideways to eliminate an equality. + // e.g: ABAC -> ABAC + var changes = false; + pointer = 1; + // Intentionally ignore the first and last element (don't need checking). + while (pointer < diffs.length - 1) { + if (diffs[pointer - 1][0] == DIFF_EQUAL && + diffs[pointer + 1][0] == DIFF_EQUAL) { + // This is a single edit surrounded by equalities. + if (diffs[pointer][1].substring(diffs[pointer][1].length - + diffs[pointer - 1][1].length) == diffs[pointer - 1][1]) { + // Shift the edit over the previous equality. + diffs[pointer][1] = diffs[pointer - 1][1] + + diffs[pointer][1].substring(0, diffs[pointer][1].length - + diffs[pointer - 1][1].length); + diffs[pointer + 1][1] = diffs[pointer - 1][1] + diffs[pointer + 1][1]; + diffs.splice(pointer - 1, 1); + changes = true; + } else if (diffs[pointer][1].substring(0, diffs[pointer + 1][1].length) == + diffs[pointer + 1][1]) { + // Shift the edit over the next equality. + diffs[pointer - 1][1] += diffs[pointer + 1][1]; + diffs[pointer][1] = + diffs[pointer][1].substring(diffs[pointer + 1][1].length) + + diffs[pointer + 1][1]; + diffs.splice(pointer + 1, 1); + changes = true; + } + } + pointer++; + } + // If shifts were made, the diff needs reordering and another shift sweep. + if (changes) { + this.diff_cleanupMerge(diffs); + } +}; + + +/** + * loc is a location in text1, compute and return the equivalent location in + * text2. + * e.g. 'The cat' vs 'The big cat', 1->1, 5->8 + * @param {!Array.} diffs Array of diff tuples. + * @param {number} loc Location within text1. + * @return {number} Location within text2. + */ +diff_match_patch.prototype.diff_xIndex = function(diffs, loc) { + var chars1 = 0; + var chars2 = 0; + var last_chars1 = 0; + var last_chars2 = 0; + var x; + for (x = 0; x < diffs.length; x++) { + if (diffs[x][0] !== DIFF_INSERT) { // Equality or deletion. + chars1 += diffs[x][1].length; + } + if (diffs[x][0] !== DIFF_DELETE) { // Equality or insertion. + chars2 += diffs[x][1].length; + } + if (chars1 > loc) { // Overshot the location. + break; + } + last_chars1 = chars1; + last_chars2 = chars2; + } + // Was the location was deleted? + if (diffs.length != x && diffs[x][0] === DIFF_DELETE) { + return last_chars2; + } + // Add the remaining character length. + return last_chars2 + (loc - last_chars1); +}; + + +/** + * Convert a diff array into a pretty HTML report. + * @param {!Array.} diffs Array of diff tuples. + * @return {string} HTML representation. + */ +diff_match_patch.prototype.diff_prettyHtml = function(diffs) { + var html = []; + var pattern_amp = /&/g; + var pattern_lt = //g; + var pattern_para = /\n/g; + for (var x = 0; x < diffs.length; x++) { + var op = diffs[x][0]; // Operation (insert, delete, equal) + var data = diffs[x][1]; // Text of change. + var text = data.replace(pattern_amp, '&').replace(pattern_lt, '<') + .replace(pattern_gt, '>').replace(pattern_para, '¶
'); + switch (op) { + case DIFF_INSERT: + html[x] = '' + text + ''; + break; + case DIFF_DELETE: + html[x] = '' + text + ''; + break; + case DIFF_EQUAL: + html[x] = '' + text + ''; + break; + } + } + return html.join(''); +}; + + +/** + * Compute and return the source text (all equalities and deletions). + * @param {!Array.} diffs Array of diff tuples. + * @return {string} Source text. + */ +diff_match_patch.prototype.diff_text1 = function(diffs) { + var text = []; + for (var x = 0; x < diffs.length; x++) { + if (diffs[x][0] !== DIFF_INSERT) { + text[x] = diffs[x][1]; + } + } + return text.join(''); +}; + + +/** + * Compute and return the destination text (all equalities and insertions). + * @param {!Array.} diffs Array of diff tuples. + * @return {string} Destination text. + */ +diff_match_patch.prototype.diff_text2 = function(diffs) { + var text = []; + for (var x = 0; x < diffs.length; x++) { + if (diffs[x][0] !== DIFF_DELETE) { + text[x] = diffs[x][1]; + } + } + return text.join(''); +}; + + +/** + * Compute the Levenshtein distance; the number of inserted, deleted or + * substituted characters. + * @param {!Array.} diffs Array of diff tuples. + * @return {number} Number of changes. + */ +diff_match_patch.prototype.diff_levenshtein = function(diffs) { + var levenshtein = 0; + var insertions = 0; + var deletions = 0; + for (var x = 0; x < diffs.length; x++) { + var op = diffs[x][0]; + var data = diffs[x][1]; + switch (op) { + case DIFF_INSERT: + insertions += data.length; + break; + case DIFF_DELETE: + deletions += data.length; + break; + case DIFF_EQUAL: + // A deletion and an insertion is one substitution. + levenshtein += Math.max(insertions, deletions); + insertions = 0; + deletions = 0; + break; + } + } + levenshtein += Math.max(insertions, deletions); + return levenshtein; +}; + + +/** + * Crush the diff into an encoded string which describes the operations + * required to transform text1 into text2. + * E.g. =3\t-2\t+ing -> Keep 3 chars, delete 2 chars, insert 'ing'. + * Operations are tab-separated. Inserted text is escaped using %xx notation. + * @param {!Array.} diffs Array of diff tuples. + * @return {string} Delta text. + */ +diff_match_patch.prototype.diff_toDelta = function(diffs) { + var text = []; + for (var x = 0; x < diffs.length; x++) { + switch (diffs[x][0]) { + case DIFF_INSERT: + text[x] = '+' + encodeURI(diffs[x][1]); + break; + case DIFF_DELETE: + text[x] = '-' + diffs[x][1].length; + break; + case DIFF_EQUAL: + text[x] = '=' + diffs[x][1].length; + break; + } + } + return text.join('\t').replace(/%20/g, ' '); +}; + + +/** + * Given the original text1, and an encoded string which describes the + * operations required to transform text1 into text2, compute the full diff. + * @param {string} text1 Source string for the diff. + * @param {string} delta Delta text. + * @return {!Array.} Array of diff tuples. + * @throws {!Error} If invalid input. + */ +diff_match_patch.prototype.diff_fromDelta = function(text1, delta) { + var diffs = []; + var diffsLength = 0; // Keeping our own length var is faster in JS. + var pointer = 0; // Cursor in text1 + var tokens = delta.split(/\t/g); + for (var x = 0; x < tokens.length; x++) { + // Each token begins with a one character parameter which specifies the + // operation of this token (delete, insert, equality). + var param = tokens[x].substring(1); + switch (tokens[x].charAt(0)) { + case '+': + try { + diffs[diffsLength++] = [DIFF_INSERT, decodeURI(param)]; + } catch (ex) { + // Malformed URI sequence. + throw new Error('Illegal escape in diff_fromDelta: ' + param); + } + break; + case '-': + // Fall through. + case '=': + var n = parseInt(param, 10); + if (isNaN(n) || n < 0) { + throw new Error('Invalid number in diff_fromDelta: ' + param); + } + var text = text1.substring(pointer, pointer += n); + if (tokens[x].charAt(0) == '=') { + diffs[diffsLength++] = [DIFF_EQUAL, text]; + } else { + diffs[diffsLength++] = [DIFF_DELETE, text]; + } + break; + default: + // Blank tokens are ok (from a trailing \t). + // Anything else is an error. + if (tokens[x]) { + throw new Error('Invalid diff operation in diff_fromDelta: ' + + tokens[x]); + } + } + } + if (pointer != text1.length) { + throw new Error('Delta length (' + pointer + + ') does not equal source text length (' + text1.length + ').'); + } + return diffs; +}; + + +// MATCH FUNCTIONS + + +/** + * Locate the best instance of 'pattern' in 'text' near 'loc'. + * @param {string} text The text to search. + * @param {string} pattern The pattern to search for. + * @param {number} loc The location to search around. + * @return {number} Best match index or -1. + */ +diff_match_patch.prototype.match_main = function(text, pattern, loc) { + // Check for null inputs. + if (text == null || pattern == null || loc == null) { + throw new Error('Null input. (match_main)'); + } + + loc = Math.max(0, Math.min(loc, text.length)); + if (text == pattern) { + // Shortcut (potentially not guaranteed by the algorithm) + return 0; + } else if (!text.length) { + // Nothing to match. + return -1; + } else if (text.substring(loc, loc + pattern.length) == pattern) { + // Perfect match at the perfect spot! (Includes case of null pattern) + return loc; + } else { + // Do a fuzzy compare. + return this.match_bitap_(text, pattern, loc); + } +}; + + +/** + * Locate the best instance of 'pattern' in 'text' near 'loc' using the + * Bitap algorithm. + * @param {string} text The text to search. + * @param {string} pattern The pattern to search for. + * @param {number} loc The location to search around. + * @return {number} Best match index or -1. + * @private + */ +diff_match_patch.prototype.match_bitap_ = function(text, pattern, loc) { + if (pattern.length > this.Match_MaxBits) { + throw new Error('Pattern too long for this browser.'); + } + + // Initialise the alphabet. + var s = this.match_alphabet_(pattern); + + var dmp = this; // 'this' becomes 'window' in a closure. + + /** + * Compute and return the score for a match with e errors and x location. + * Accesses loc and pattern through being a closure. + * @param {number} e Number of errors in match. + * @param {number} x Location of match. + * @return {number} Overall score for match (0.0 = good, 1.0 = bad). + * @private + */ + function match_bitapScore_(e, x) { + var accuracy = e / pattern.length; + var proximity = Math.abs(loc - x); + if (!dmp.Match_Distance) { + // Dodge divide by zero error. + return proximity ? 1.0 : accuracy; + } + return accuracy + (proximity / dmp.Match_Distance); + } + + // Highest score beyond which we give up. + var score_threshold = this.Match_Threshold; + // Is there a nearby exact match? (speedup) + var best_loc = text.indexOf(pattern, loc); + if (best_loc != -1) { + score_threshold = Math.min(match_bitapScore_(0, best_loc), score_threshold); + // What about in the other direction? (speedup) + best_loc = text.lastIndexOf(pattern, loc + pattern.length); + if (best_loc != -1) { + score_threshold = + Math.min(match_bitapScore_(0, best_loc), score_threshold); + } + } + + // Initialise the bit arrays. + var matchmask = 1 << (pattern.length - 1); + best_loc = -1; + + var bin_min, bin_mid; + var bin_max = pattern.length + text.length; + var last_rd; + for (var d = 0; d < pattern.length; d++) { + // Scan for the best match; each iteration allows for one more error. + // Run a binary search to determine how far from 'loc' we can stray at this + // error level. + bin_min = 0; + bin_mid = bin_max; + while (bin_min < bin_mid) { + if (match_bitapScore_(d, loc + bin_mid) <= score_threshold) { + bin_min = bin_mid; + } else { + bin_max = bin_mid; + } + bin_mid = Math.floor((bin_max - bin_min) / 2 + bin_min); + } + // Use the result from this iteration as the maximum for the next. + bin_max = bin_mid; + var start = Math.max(1, loc - bin_mid + 1); + var finish = Math.min(loc + bin_mid, text.length) + pattern.length; + + var rd = Array(finish + 2); + rd[finish + 1] = (1 << d) - 1; + for (var j = finish; j >= start; j--) { + // The alphabet (s) is a sparse hash, so the following line generates + // warnings. + var charMatch = s[text.charAt(j - 1)]; + if (d === 0) { // First pass: exact match. + rd[j] = ((rd[j + 1] << 1) | 1) & charMatch; + } else { // Subsequent passes: fuzzy match. + rd[j] = (((rd[j + 1] << 1) | 1) & charMatch) | + (((last_rd[j + 1] | last_rd[j]) << 1) | 1) | + last_rd[j + 1]; + } + if (rd[j] & matchmask) { + var score = match_bitapScore_(d, j - 1); + // This match will almost certainly be better than any existing match. + // But check anyway. + if (score <= score_threshold) { + // Told you so. + score_threshold = score; + best_loc = j - 1; + if (best_loc > loc) { + // When passing loc, don't exceed our current distance from loc. + start = Math.max(1, 2 * loc - best_loc); + } else { + // Already passed loc, downhill from here on in. + break; + } + } + } + } + // No hope for a (better) match at greater error levels. + if (match_bitapScore_(d + 1, loc) > score_threshold) { + break; + } + last_rd = rd; + } + return best_loc; +}; + + +/** + * Initialise the alphabet for the Bitap algorithm. + * @param {string} pattern The text to encode. + * @return {!Object} Hash of character locations. + * @private + */ +diff_match_patch.prototype.match_alphabet_ = function(pattern) { + var s = {}; + for (var i = 0; i < pattern.length; i++) { + s[pattern.charAt(i)] = 0; + } + for (var i = 0; i < pattern.length; i++) { + s[pattern.charAt(i)] |= 1 << (pattern.length - i - 1); + } + return s; +}; + + +// PATCH FUNCTIONS + + +/** + * Increase the context until it is unique, + * but don't let the pattern expand beyond Match_MaxBits. + * @param {!diff_match_patch.patch_obj} patch The patch to grow. + * @param {string} text Source text. + * @private + */ +diff_match_patch.prototype.patch_addContext_ = function(patch, text) { + if (text.length == 0) { + return; + } + var pattern = text.substring(patch.start2, patch.start2 + patch.length1); + var padding = 0; + + // Look for the first and last matches of pattern in text. If two different + // matches are found, increase the pattern length. + while (text.indexOf(pattern) != text.lastIndexOf(pattern) && + pattern.length < this.Match_MaxBits - this.Patch_Margin - + this.Patch_Margin) { + padding += this.Patch_Margin; + pattern = text.substring(patch.start2 - padding, + patch.start2 + patch.length1 + padding); + } + // Add one chunk for good luck. + padding += this.Patch_Margin; + + // Add the prefix. + var prefix = text.substring(patch.start2 - padding, patch.start2); + if (prefix) { + patch.diffs.unshift([DIFF_EQUAL, prefix]); + } + // Add the suffix. + var suffix = text.substring(patch.start2 + patch.length1, + patch.start2 + patch.length1 + padding); + if (suffix) { + patch.diffs.push([DIFF_EQUAL, suffix]); + } + + // Roll back the start points. + patch.start1 -= prefix.length; + patch.start2 -= prefix.length; + // Extend the lengths. + patch.length1 += prefix.length + suffix.length; + patch.length2 += prefix.length + suffix.length; +}; + + +/** + * Compute a list of patches to turn text1 into text2. + * Use diffs if provided, otherwise compute it ourselves. + * There are four ways to call this function, depending on what data is + * available to the caller: + * Method 1: + * a = text1, b = text2 + * Method 2: + * a = diffs + * Method 3 (optimal): + * a = text1, b = diffs + * Method 4 (deprecated, use method 3): + * a = text1, b = text2, c = diffs + * + * @param {string|!Array.} a text1 (methods 1,3,4) or + * Array of diff tuples for text1 to text2 (method 2). + * @param {string|!Array.} opt_b text2 (methods 1,4) or + * Array of diff tuples for text1 to text2 (method 3) or undefined (method 2). + * @param {string|!Array.} opt_c Array of diff tuples + * for text1 to text2 (method 4) or undefined (methods 1,2,3). + * @return {!Array.} Array of Patch objects. + */ +diff_match_patch.prototype.patch_make = function(a, opt_b, opt_c) { + var text1, diffs; + if (typeof a == 'string' && typeof opt_b == 'string' && + typeof opt_c == 'undefined') { + // Method 1: text1, text2 + // Compute diffs from text1 and text2. + text1 = /** @type {string} */(a); + diffs = this.diff_main(text1, /** @type {string} */(opt_b), true); + if (diffs.length > 2) { + this.diff_cleanupSemantic(diffs); + this.diff_cleanupEfficiency(diffs); + } + } else if (a && typeof a == 'object' && typeof opt_b == 'undefined' && + typeof opt_c == 'undefined') { + // Method 2: diffs + // Compute text1 from diffs. + diffs = /** @type {!Array.} */(a); + text1 = this.diff_text1(diffs); + } else if (typeof a == 'string' && opt_b && typeof opt_b == 'object' && + typeof opt_c == 'undefined') { + // Method 3: text1, diffs + text1 = /** @type {string} */(a); + diffs = /** @type {!Array.} */(opt_b); + } else if (typeof a == 'string' && typeof opt_b == 'string' && + opt_c && typeof opt_c == 'object') { + // Method 4: text1, text2, diffs + // text2 is not used. + text1 = /** @type {string} */(a); + diffs = /** @type {!Array.} */(opt_c); + } else { + throw new Error('Unknown call format to patch_make.'); + } + + if (diffs.length === 0) { + return []; // Get rid of the null case. + } + var patches = []; + var patch = new diff_match_patch.patch_obj(); + var patchDiffLength = 0; // Keeping our own length var is faster in JS. + var char_count1 = 0; // Number of characters into the text1 string. + var char_count2 = 0; // Number of characters into the text2 string. + // Start with text1 (prepatch_text) and apply the diffs until we arrive at + // text2 (postpatch_text). We recreate the patches one by one to determine + // context info. + var prepatch_text = text1; + var postpatch_text = text1; + for (var x = 0; x < diffs.length; x++) { + var diff_type = diffs[x][0]; + var diff_text = diffs[x][1]; + + if (!patchDiffLength && diff_type !== DIFF_EQUAL) { + // A new patch starts here. + patch.start1 = char_count1; + patch.start2 = char_count2; + } + + switch (diff_type) { + case DIFF_INSERT: + patch.diffs[patchDiffLength++] = diffs[x]; + patch.length2 += diff_text.length; + postpatch_text = postpatch_text.substring(0, char_count2) + diff_text + + postpatch_text.substring(char_count2); + break; + case DIFF_DELETE: + patch.length1 += diff_text.length; + patch.diffs[patchDiffLength++] = diffs[x]; + postpatch_text = postpatch_text.substring(0, char_count2) + + postpatch_text.substring(char_count2 + + diff_text.length); + break; + case DIFF_EQUAL: + if (diff_text.length <= 2 * this.Patch_Margin && + patchDiffLength && diffs.length != x + 1) { + // Small equality inside a patch. + patch.diffs[patchDiffLength++] = diffs[x]; + patch.length1 += diff_text.length; + patch.length2 += diff_text.length; + } else if (diff_text.length >= 2 * this.Patch_Margin) { + // Time for a new patch. + if (patchDiffLength) { + this.patch_addContext_(patch, prepatch_text); + patches.push(patch); + patch = new diff_match_patch.patch_obj(); + patchDiffLength = 0; + // Unlike Unidiff, our patch lists have a rolling context. + // http://code.google.com/p/google-diff-match-patch/wiki/Unidiff + // Update prepatch text & pos to reflect the application of the + // just completed patch. + prepatch_text = postpatch_text; + char_count1 = char_count2; + } + } + break; + } + + // Update the current character count. + if (diff_type !== DIFF_INSERT) { + char_count1 += diff_text.length; + } + if (diff_type !== DIFF_DELETE) { + char_count2 += diff_text.length; + } + } + // Pick up the leftover patch if not empty. + if (patchDiffLength) { + this.patch_addContext_(patch, prepatch_text); + patches.push(patch); + } + + return patches; +}; + + +/** + * Given an array of patches, return another array that is identical. + * @param {!Array.} patches Array of Patch objects. + * @return {!Array.} Array of Patch objects. + */ +diff_match_patch.prototype.patch_deepCopy = function(patches) { + // Making deep copies is hard in JavaScript. + var patchesCopy = []; + for (var x = 0; x < patches.length; x++) { + var patch = patches[x]; + var patchCopy = new diff_match_patch.patch_obj(); + patchCopy.diffs = []; + for (var y = 0; y < patch.diffs.length; y++) { + patchCopy.diffs[y] = patch.diffs[y].slice(); + } + patchCopy.start1 = patch.start1; + patchCopy.start2 = patch.start2; + patchCopy.length1 = patch.length1; + patchCopy.length2 = patch.length2; + patchesCopy[x] = patchCopy; + } + return patchesCopy; +}; + + +/** + * Merge a set of patches onto the text. Return a patched text, as well + * as a list of true/false values indicating which patches were applied. + * @param {!Array.} patches Array of Patch objects. + * @param {string} text Old text. + * @return {!Array.>} Two element Array, containing the + * new text and an array of boolean values. + */ +diff_match_patch.prototype.patch_apply = function(patches, text) { + if (patches.length == 0) { + return [text, []]; + } + + // Deep copy the patches so that no changes are made to originals. + patches = this.patch_deepCopy(patches); + + var nullPadding = this.patch_addPadding(patches); + text = nullPadding + text + nullPadding; + + this.patch_splitMax(patches); + // delta keeps track of the offset between the expected and actual location + // of the previous patch. If there are patches expected at positions 10 and + // 20, but the first patch was found at 12, delta is 2 and the second patch + // has an effective expected position of 22. + var delta = 0; + var results = []; + for (var x = 0; x < patches.length; x++) { + var expected_loc = patches[x].start2 + delta; + var text1 = this.diff_text1(patches[x].diffs); + var start_loc; + var end_loc = -1; + if (text1.length > this.Match_MaxBits) { + // patch_splitMax will only provide an oversized pattern in the case of + // a monster delete. + start_loc = this.match_main(text, text1.substring(0, this.Match_MaxBits), + expected_loc); + if (start_loc != -1) { + end_loc = this.match_main(text, + text1.substring(text1.length - this.Match_MaxBits), + expected_loc + text1.length - this.Match_MaxBits); + if (end_loc == -1 || start_loc >= end_loc) { + // Can't find valid trailing context. Drop this patch. + start_loc = -1; + } + } + } else { + start_loc = this.match_main(text, text1, expected_loc); + } + if (start_loc == -1) { + // No match found. :( + results[x] = false; + // Subtract the delta for this failed patch from subsequent patches. + delta -= patches[x].length2 - patches[x].length1; + } else { + // Found a match. :) + results[x] = true; + delta = start_loc - expected_loc; + var text2; + if (end_loc == -1) { + text2 = text.substring(start_loc, start_loc + text1.length); + } else { + text2 = text.substring(start_loc, end_loc + this.Match_MaxBits); + } + if (text1 == text2) { + // Perfect match, just shove the replacement text in. + text = text.substring(0, start_loc) + + this.diff_text2(patches[x].diffs) + + text.substring(start_loc + text1.length); + } else { + // Imperfect match. Run a diff to get a framework of equivalent + // indices. + var diffs = this.diff_main(text1, text2, false); + if (text1.length > this.Match_MaxBits && + this.diff_levenshtein(diffs) / text1.length > + this.Patch_DeleteThreshold) { + // The end points match, but the content is unacceptably bad. + results[x] = false; + } else { + this.diff_cleanupSemanticLossless(diffs); + var index1 = 0; + var index2; + for (var y = 0; y < patches[x].diffs.length; y++) { + var mod = patches[x].diffs[y]; + if (mod[0] !== DIFF_EQUAL) { + index2 = this.diff_xIndex(diffs, index1); + } + if (mod[0] === DIFF_INSERT) { // Insertion + text = text.substring(0, start_loc + index2) + mod[1] + + text.substring(start_loc + index2); + } else if (mod[0] === DIFF_DELETE) { // Deletion + text = text.substring(0, start_loc + index2) + + text.substring(start_loc + this.diff_xIndex(diffs, + index1 + mod[1].length)); + } + if (mod[0] !== DIFF_DELETE) { + index1 += mod[1].length; + } + } + } + } + } + } + // Strip the padding off. + text = text.substring(nullPadding.length, text.length - nullPadding.length); + return [text, results]; +}; + + +/** + * Add some padding on text start and end so that edges can match something. + * Intended to be called only from within patch_apply. + * @param {!Array.} patches Array of Patch objects. + * @return {string} The padding string added to each side. + */ +diff_match_patch.prototype.patch_addPadding = function(patches) { + var paddingLength = this.Patch_Margin; + var nullPadding = ''; + for (var x = 1; x <= paddingLength; x++) { + nullPadding += String.fromCharCode(x); + } + + // Bump all the patches forward. + for (var x = 0; x < patches.length; x++) { + patches[x].start1 += paddingLength; + patches[x].start2 += paddingLength; + } + + // Add some padding on start of first diff. + var patch = patches[0]; + var diffs = patch.diffs; + if (diffs.length == 0 || diffs[0][0] != DIFF_EQUAL) { + // Add nullPadding equality. + diffs.unshift([DIFF_EQUAL, nullPadding]); + patch.start1 -= paddingLength; // Should be 0. + patch.start2 -= paddingLength; // Should be 0. + patch.length1 += paddingLength; + patch.length2 += paddingLength; + } else if (paddingLength > diffs[0][1].length) { + // Grow first equality. + var extraLength = paddingLength - diffs[0][1].length; + diffs[0][1] = nullPadding.substring(diffs[0][1].length) + diffs[0][1]; + patch.start1 -= extraLength; + patch.start2 -= extraLength; + patch.length1 += extraLength; + patch.length2 += extraLength; + } + + // Add some padding on end of last diff. + patch = patches[patches.length - 1]; + diffs = patch.diffs; + if (diffs.length == 0 || diffs[diffs.length - 1][0] != DIFF_EQUAL) { + // Add nullPadding equality. + diffs.push([DIFF_EQUAL, nullPadding]); + patch.length1 += paddingLength; + patch.length2 += paddingLength; + } else if (paddingLength > diffs[diffs.length - 1][1].length) { + // Grow last equality. + var extraLength = paddingLength - diffs[diffs.length - 1][1].length; + diffs[diffs.length - 1][1] += nullPadding.substring(0, extraLength); + patch.length1 += extraLength; + patch.length2 += extraLength; + } + + return nullPadding; +}; + + +/** + * Look through the patches and break up any which are longer than the maximum + * limit of the match algorithm. + * Intended to be called only from within patch_apply. + * @param {!Array.} patches Array of Patch objects. + */ +diff_match_patch.prototype.patch_splitMax = function(patches) { + var patch_size = this.Match_MaxBits; + for (var x = 0; x < patches.length; x++) { + if (patches[x].length1 <= patch_size) { + continue; + } + var bigpatch = patches[x]; + // Remove the big old patch. + patches.splice(x--, 1); + var start1 = bigpatch.start1; + var start2 = bigpatch.start2; + var precontext = ''; + while (bigpatch.diffs.length !== 0) { + // Create one of several smaller patches. + var patch = new diff_match_patch.patch_obj(); + var empty = true; + patch.start1 = start1 - precontext.length; + patch.start2 = start2 - precontext.length; + if (precontext !== '') { + patch.length1 = patch.length2 = precontext.length; + patch.diffs.push([DIFF_EQUAL, precontext]); + } + while (bigpatch.diffs.length !== 0 && + patch.length1 < patch_size - this.Patch_Margin) { + var diff_type = bigpatch.diffs[0][0]; + var diff_text = bigpatch.diffs[0][1]; + if (diff_type === DIFF_INSERT) { + // Insertions are harmless. + patch.length2 += diff_text.length; + start2 += diff_text.length; + patch.diffs.push(bigpatch.diffs.shift()); + empty = false; + } else if (diff_type === DIFF_DELETE && patch.diffs.length == 1 && + patch.diffs[0][0] == DIFF_EQUAL && + diff_text.length > 2 * patch_size) { + // This is a large deletion. Let it pass in one chunk. + patch.length1 += diff_text.length; + start1 += diff_text.length; + empty = false; + patch.diffs.push([diff_type, diff_text]); + bigpatch.diffs.shift(); + } else { + // Deletion or equality. Only take as much as we can stomach. + diff_text = diff_text.substring(0, + patch_size - patch.length1 - this.Patch_Margin); + patch.length1 += diff_text.length; + start1 += diff_text.length; + if (diff_type === DIFF_EQUAL) { + patch.length2 += diff_text.length; + start2 += diff_text.length; + } else { + empty = false; + } + patch.diffs.push([diff_type, diff_text]); + if (diff_text == bigpatch.diffs[0][1]) { + bigpatch.diffs.shift(); + } else { + bigpatch.diffs[0][1] = + bigpatch.diffs[0][1].substring(diff_text.length); + } + } + } + // Compute the head context for the next patch. + precontext = this.diff_text2(patch.diffs); + precontext = + precontext.substring(precontext.length - this.Patch_Margin); + // Append the end context for this patch. + var postcontext = this.diff_text1(bigpatch.diffs) + .substring(0, this.Patch_Margin); + if (postcontext !== '') { + patch.length1 += postcontext.length; + patch.length2 += postcontext.length; + if (patch.diffs.length !== 0 && + patch.diffs[patch.diffs.length - 1][0] === DIFF_EQUAL) { + patch.diffs[patch.diffs.length - 1][1] += postcontext; + } else { + patch.diffs.push([DIFF_EQUAL, postcontext]); + } + } + if (!empty) { + patches.splice(++x, 0, patch); + } + } + } +}; + + +/** + * Take a list of patches and return a textual representation. + * @param {!Array.} patches Array of Patch objects. + * @return {string} Text representation of patches. + */ +diff_match_patch.prototype.patch_toText = function(patches) { + var text = []; + for (var x = 0; x < patches.length; x++) { + text[x] = patches[x]; + } + return text.join(''); +}; + + +/** + * Parse a textual representation of patches and return a list of Patch objects. + * @param {string} textline Text representation of patches. + * @return {!Array.} Array of Patch objects. + * @throws {!Error} If invalid input. + */ +diff_match_patch.prototype.patch_fromText = function(textline) { + var patches = []; + if (!textline) { + return patches; + } + var text = textline.split('\n'); + var textPointer = 0; + var patchHeader = /^@@ -(\d+),?(\d*) \+(\d+),?(\d*) @@$/; + while (textPointer < text.length) { + var m = text[textPointer].match(patchHeader); + if (!m) { + throw new Error('Invalid patch string: ' + text[textPointer]); + } + var patch = new diff_match_patch.patch_obj(); + patches.push(patch); + patch.start1 = parseInt(m[1], 10); + if (m[2] === '') { + patch.start1--; + patch.length1 = 1; + } else if (m[2] == '0') { + patch.length1 = 0; + } else { + patch.start1--; + patch.length1 = parseInt(m[2], 10); + } + + patch.start2 = parseInt(m[3], 10); + if (m[4] === '') { + patch.start2--; + patch.length2 = 1; + } else if (m[4] == '0') { + patch.length2 = 0; + } else { + patch.start2--; + patch.length2 = parseInt(m[4], 10); + } + textPointer++; + + while (textPointer < text.length) { + var sign = text[textPointer].charAt(0); + try { + var line = decodeURI(text[textPointer].substring(1)); + } catch (ex) { + // Malformed URI sequence. + throw new Error('Illegal escape in patch_fromText: ' + line); + } + if (sign == '-') { + // Deletion. + patch.diffs.push([DIFF_DELETE, line]); + } else if (sign == '+') { + // Insertion. + patch.diffs.push([DIFF_INSERT, line]); + } else if (sign == ' ') { + // Minor equality. + patch.diffs.push([DIFF_EQUAL, line]); + } else if (sign == '@') { + // Start of next patch. + break; + } else if (sign === '') { + // Blank line? Whatever. + } else { + // WTF? + throw new Error('Invalid patch mode "' + sign + '" in: ' + line); + } + textPointer++; + } + } + return patches; +}; + + +/** + * Class representing one patch operation. + * @constructor + */ +diff_match_patch.patch_obj = function() { + /** @type {!Array.} */ + this.diffs = []; + /** @type {?number} */ + this.start1 = null; + /** @type {?number} */ + this.start2 = null; + /** @type {number} */ + this.length1 = 0; + /** @type {number} */ + this.length2 = 0; +}; + + +/** + * Emmulate GNU diff's format. + * Header: @@ -382,8 +481,9 @@ + * Indicies are printed as 1-based, not 0-based. + * @return {string} The GNU diff string. + */ +diff_match_patch.patch_obj.prototype.toString = function() { + var coords1, coords2; + if (this.length1 === 0) { + coords1 = this.start1 + ',0'; + } else if (this.length1 == 1) { + coords1 = this.start1 + 1; + } else { + coords1 = (this.start1 + 1) + ',' + this.length1; + } + if (this.length2 === 0) { + coords2 = this.start2 + ',0'; + } else if (this.length2 == 1) { + coords2 = this.start2 + 1; + } else { + coords2 = (this.start2 + 1) + ',' + this.length2; + } + var text = ['@@ -' + coords1 + ' +' + coords2 + ' @@\n']; + var op; + // Escape the body of the patch with %xx notation. + for (var x = 0; x < this.diffs.length; x++) { + switch (this.diffs[x][0]) { + case DIFF_INSERT: + op = '+'; + break; + case DIFF_DELETE: + op = '-'; + break; + case DIFF_EQUAL: + op = ' '; + break; + } + text[x + 1] = op + encodeURI(this.diffs[x][1]) + '\n'; + } + return text.join('').replace(/%20/g, ' '); +}; + + +// Export these global variables so that they survive Google's JS compiler. +// In a browser, 'this' will be 'window'. +// Users of node.js should 'require' the uncompressed version since Google's +// JS compiler may break the following exports for non-browser environments. +this['diff_match_patch'] = diff_match_patch; +this['DIFF_DELETE'] = DIFF_DELETE; +this['DIFF_INSERT'] = DIFF_INSERT; +this['DIFF_EQUAL'] = DIFF_EQUAL; diff --git a/services/document-updater/config/settings.development.coffee b/services/document-updater/config/settings.development.coffee new file mode 100755 index 0000000000..d730bb0f2d --- /dev/null +++ b/services/document-updater/config/settings.development.coffee @@ -0,0 +1,23 @@ +Path = require('path') +http = require('http') +http.globalAgent.maxSockets = 300 + +module.exports = + internal: + documentupdater: + port: 3003 + + apis: + web: + url: "http://localhost:3000" + user: "sharelatex" + pass: "password" + + redis: + web: + port:"6379" + host:"localhost" + password:"" + + mongo: + url: 'mongodb://127.0.0.1/sharelatex' diff --git a/services/document-updater/package.json b/services/document-updater/package.json new file mode 100644 index 0000000000..ff65e225fe --- /dev/null +++ b/services/document-updater/package.json @@ -0,0 +1,30 @@ +{ + "name": "document-updater-sharelatex", + "version": "0.0.1", + "dependencies": { + "express": "3.3.4", + "underscore": "1.2.2", + "redis": "0.7.2", + "chai": "", + "request": "2.25.0", + "sandboxed-module": "~0.2.0", + "chai-spies": "", + "async": "", + "lynx": "0.0.11", + "coffee-script": "1.4.0", + "settings-sharelatex": "git+ssh://git@bitbucket.org:sharelatex/settings-sharelatex.git#master", + "logger-sharelatex": "git+ssh://git@bitbucket.org:sharelatex/logger-sharelatex.git#bunyan", + "sinon": "~1.5.2", + "mongojs": "0.9.11" + }, + "devDependencies": { + "grunt-execute": "~0.1.5", + "grunt-contrib-clean": "~0.5.0", + "grunt-mocha-test": "~0.9.0", + "grunt": "~0.4.2", + "grunt-available-tasks": "~0.4.1", + "grunt-contrib-coffee": "~0.10.0", + "bunyan": "~0.22.1", + "grunt-bunyan": "~0.5.0" + } +} diff --git a/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee b/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee new file mode 100644 index 0000000000..5108a4c2cc --- /dev/null +++ b/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee @@ -0,0 +1,215 @@ +sinon = require "sinon" +chai = require("chai") +chai.should() +async = require "async" +mongojs = require "../../../app/js/mongojs" +db = mongojs.db +ObjectId = mongojs.ObjectId + +MockWebApi = require "./helpers/MockWebApi" +DocUpdaterClient = require "./helpers/DocUpdaterClient" + +describe "Applying updates to a doc", -> + before -> + @lines = ["one", "two", "three"] + @update = + doc: @doc_id + op: [{ + i: "one and a half\n" + p: 4 + }] + v: 0 + @result = ["one", "one and a half", "two", "three"] + + describe "when the document is not loaded", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines + } + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + + after -> + MockWebApi.getDocument.restore() + + it "should load the document from the web API", -> + MockWebApi.getDocument + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should update the doc", (done) -> + DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) => + doc.lines.should.deep.equal @result + done() + + describe "when the document is loaded", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines + } + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> + throw error if error? + setTimeout done, 200 + + after -> + MockWebApi.getDocument.restore() + + it "should not need to call the web api", -> + MockWebApi.getDocument.called.should.equal false + + it "should update the doc", (done) -> + DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) => + doc.lines.should.deep.equal @result + done() + + describe "when the document has been deleted", -> + describe "when the ops come in a single linear order", -> + before -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + @lines = ["", "", ""] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines + } + + @updates = [ + { doc_id: @doc_id, v: 0, op: [i: "h", p: 0 ] } + { doc_id: @doc_id, v: 1, op: [i: "e", p: 1 ] } + { doc_id: @doc_id, v: 2, op: [i: "l", p: 2 ] } + { doc_id: @doc_id, v: 3, op: [i: "l", p: 3 ] } + { doc_id: @doc_id, v: 4, op: [i: "o", p: 4 ] } + { doc_id: @doc_id, v: 5, op: [i: " ", p: 5 ] } + { doc_id: @doc_id, v: 6, op: [i: "w", p: 6 ] } + { doc_id: @doc_id, v: 7, op: [i: "o", p: 7 ] } + { doc_id: @doc_id, v: 8, op: [i: "r", p: 8 ] } + { doc_id: @doc_id, v: 9, op: [i: "l", p: 9 ] } + { doc_id: @doc_id, v: 10, op: [i: "d", p: 10] } + ] + @result = ["hello world", "", ""] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines + } + + it "should be able to continue applying updates when the project has been deleted", (done) -> + actions = [] + for update in @updates.slice(0,6) + do (update) => + actions.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc_id, update, callback + actions.push (callback) => DocUpdaterClient.deleteDoc @project_id, @doc_id, callback + for update in @updates.slice(6) + do (update) => + actions.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc_id, update, callback + + async.series actions, (error) => + throw error if error? + DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) => + doc.lines.should.deep.equal @result + done() + + describe "when older ops come in after the delete", -> + before -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + @lines = ["", "", ""] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines + } + + @updates = [ + { doc_id: @doc_id, v: 0, op: [i: "h", p: 0 ] } + { doc_id: @doc_id, v: 1, op: [i: "e", p: 1 ] } + { doc_id: @doc_id, v: 2, op: [i: "l", p: 2 ] } + { doc_id: @doc_id, v: 3, op: [i: "l", p: 3 ] } + { doc_id: @doc_id, v: 4, op: [i: "o", p: 4 ] } + { doc_id: @doc_id, v: 0, op: [i: "world", p: 1 ] } + ] + @result = ["hello", "world", ""] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines + } + + it "should be able to continue applying updates when the project has been deleted", (done) -> + actions = [] + for update in @updates.slice(0,5) + do (update) => + actions.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc_id, update, callback + actions.push (callback) => DocUpdaterClient.deleteDoc @project_id, @doc_id, callback + for update in @updates.slice(5) + do (update) => + actions.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc_id, update, callback + + async.series actions, (error) => + throw error if error? + DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) => + doc.lines.should.deep.equal @result + done() + + describe "when the mongo array has been trimmed", -> + before -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + @lines = ["", "", ""] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines + } + + @updates = [ + { doc_id: @doc_id, v: 0, op: [i: "h", p: 0 ] } + { doc_id: @doc_id, v: 1, op: [i: "e", p: 1 ] } + { doc_id: @doc_id, v: 2, op: [i: "l", p: 2 ] } + { doc_id: @doc_id, v: 3, op: [i: "l", p: 3 ] } + { doc_id: @doc_id, v: 4, op: [i: "o", p: 4 ] } + { doc_id: @doc_id, v: 3, op: [i: "world", p: 4 ] } + ] + @result = ["hello", "world", ""] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines + } + + it "should be able to reload the required ops from the trimmed mongo array", (done) -> + actions = [] + # Apply first set of ops + for update in @updates.slice(0,5) + do (update) => + actions.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc_id, update, callback + # Delete doc from redis and trim ops back to version 3 + actions.push (callback) => DocUpdaterClient.deleteDoc @project_id, @doc_id, callback + actions.push (callback) => + db.docOps.update({doc_id: ObjectId(@doc_id)}, {$push: docOps: { $each: [], $slice: -2 }}, callback) + # Apply older update back from version 3 + for update in @updates.slice(5) + do (update) => + actions.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc_id, update, callback + # Flush ops to mongo + actions.push (callback) => DocUpdaterClient.flushDoc @project_id, @doc_id, callback + + async.series actions, (error) => + throw error if error? + DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) => + db.docOps.find {doc_id: ObjectId(@doc_id)}, (error, docOps) => + # Check mongo array has been trimmed + docOps = docOps[0] + docOps.docOps.length.should.equal 3 + # Check ops have all be applied properly + doc.lines.should.deep.equal @result + done() + + describe "with a broken update", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines + } + DocUpdaterClient.sendUpdate @project_id, @doc_id, @undefined, (error) -> + throw error if error? + setTimeout done, 200 + + it "should not update the doc", (done) -> + DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) => + doc.lines.should.deep.equal @lines + done() + diff --git a/services/document-updater/test/acceptance/coffee/DeletingADocumentTests.coffee b/services/document-updater/test/acceptance/coffee/DeletingADocumentTests.coffee new file mode 100644 index 0000000000..d28f37cd6d --- /dev/null +++ b/services/document-updater/test/acceptance/coffee/DeletingADocumentTests.coffee @@ -0,0 +1,89 @@ +sinon = require "sinon" +chai = require("chai") +chai.should() + +MockWebApi = require "./helpers/MockWebApi" +DocUpdaterClient = require "./helpers/DocUpdaterClient" + +describe "Deleting a document", -> + before -> + @lines = ["one", "two", "three"] + @update = + doc: @doc_id + op: [{ + i: "one and a half\n" + p: 4 + }] + v: 0 + @result = ["one", "one and a half", "two", "three"] + + describe "when the updated doc exists in the doc updater", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines + } + sinon.spy MockWebApi, "setDocumentLines" + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) => + throw error if error? + setTimeout () => + DocUpdaterClient.deleteDoc @project_id, @doc_id, (error, res, body) => + @statusCode = res.statusCode + done() + , 200 + + after -> + MockWebApi.setDocumentLines.restore() + MockWebApi.getDocument.restore() + + it "should return a 204 status code", -> + @statusCode.should.equal 204 + + it "should send the updated document to the web api", -> + MockWebApi.setDocumentLines + .calledWith(@project_id, @doc_id, @result) + .should.equal true + + it "should need to reload the doc if read again", (done) -> + MockWebApi.getDocument.called.should.equal.false + DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) => + MockWebApi.getDocument + .calledWith(@project_id, @doc_id) + .should.equal true + done() + + describe "when the doc is not in the doc updater", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines + } + sinon.spy MockWebApi, "setDocumentLines" + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.deleteDoc @project_id, @doc_id, (error, res, body) => + @statusCode = res.statusCode + done() + + after -> + MockWebApi.setDocumentLines.restore() + MockWebApi.getDocument.restore() + + it "should return a 204 status code", -> + @statusCode.should.equal 204 + + it "should not need to send the updated document to the web api", -> + MockWebApi.setDocumentLines.called.should.equal false + + it "should need to reload the doc if read again", (done) -> + MockWebApi.getDocument.called.should.equal.false + DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) => + MockWebApi.getDocument + .calledWith(@project_id, @doc_id) + .should.equal true + done() + + + diff --git a/services/document-updater/test/acceptance/coffee/DeletingAProjectTests.coffee b/services/document-updater/test/acceptance/coffee/DeletingAProjectTests.coffee new file mode 100644 index 0000000000..7b07ed6a25 --- /dev/null +++ b/services/document-updater/test/acceptance/coffee/DeletingAProjectTests.coffee @@ -0,0 +1,81 @@ +sinon = require "sinon" +chai = require("chai") +chai.should() +async = require "async" + +MockWebApi = require "./helpers/MockWebApi" +DocUpdaterClient = require "./helpers/DocUpdaterClient" + +describe "Deleting a project", -> + before -> + @project_id = DocUpdaterClient.randomId() + @docs = [{ + id: doc_id0 = DocUpdaterClient.randomId() + lines: ["one", "two", "three"] + update: + doc: doc_id0 + op: [{ + i: "one and a half\n" + p: 4 + }] + v: 0 + updatedLines: ["one", "one and a half", "two", "three"] + }, { + id: doc_id1 = DocUpdaterClient.randomId() + lines: ["four", "five", "six"] + update: + doc: doc_id1 + op: [{ + i: "four and a half\n" + p: 5 + }] + v: 0 + updatedLines: ["four", "four and a half", "five", "six"] + }] + for doc in @docs + MockWebApi.insertDoc @project_id, doc.id, { + lines: doc.lines + } + + describe "with documents which have been updated", -> + before (done) -> + sinon.spy MockWebApi, "setDocumentLines" + async.series @docs.map((doc) => + (callback) => + DocUpdaterClient.preloadDoc @project_id, doc.id, (error) => + return callback(error) if error? + DocUpdaterClient.sendUpdate @project_id, doc.id, doc.update, (error) => + callback(error) + ), (error) => + throw error if error? + setTimeout () => + DocUpdaterClient.deleteProject @project_id, (error, res, body) => + @statusCode = res.statusCode + done() + , 200 + + after -> + MockWebApi.setDocumentLines.restore() + + it "should return a 204 status code", -> + @statusCode.should.equal 204 + + it "should send each document to the web api", -> + for doc in @docs + MockWebApi.setDocumentLines + .calledWith(@project_id, doc.id, doc.updatedLines) + .should.equal true + + it "should need to reload the docs if read again", (done) -> + sinon.spy MockWebApi, "getDocument" + async.series @docs.map((doc) => + (callback) => + MockWebApi.getDocument.calledWith(@project_id, doc.id).should.equal false + DocUpdaterClient.getDoc @project_id, doc.id, (error, res, returnedDoc) => + MockWebApi.getDocument.calledWith(@project_id, doc.id).should.equal true + callback() + ), () -> + MockWebApi.getDocument.restore() + done() + + diff --git a/services/document-updater/test/acceptance/coffee/FlushingAProjectTests.coffee b/services/document-updater/test/acceptance/coffee/FlushingAProjectTests.coffee new file mode 100644 index 0000000000..02b44e3fd6 --- /dev/null +++ b/services/document-updater/test/acceptance/coffee/FlushingAProjectTests.coffee @@ -0,0 +1,76 @@ +sinon = require "sinon" +chai = require("chai") +chai.should() +async = require "async" + +MockWebApi = require "./helpers/MockWebApi" +DocUpdaterClient = require "./helpers/DocUpdaterClient" + +describe "Flushing a project", -> + before -> + @project_id = DocUpdaterClient.randomId() + @docs = [{ + id: doc_id0 = DocUpdaterClient.randomId() + lines: ["one", "two", "three"] + update: + doc: doc_id0 + op: [{ + i: "one and a half\n" + p: 4 + }] + v: 0 + updatedLines: ["one", "one and a half", "two", "three"] + }, { + id: doc_id1 = DocUpdaterClient.randomId() + lines: ["four", "five", "six"] + update: + doc: doc_id1 + op: [{ + i: "four and a half\n" + p: 5 + }] + v: 0 + updatedLines: ["four", "four and a half", "five", "six"] + }] + for doc in @docs + MockWebApi.insertDoc @project_id, doc.id, { + lines: doc.lines + } + + describe "with documents which have been updated", -> + before (done) -> + sinon.spy MockWebApi, "setDocumentLines" + async.series @docs.map((doc) => + (callback) => + DocUpdaterClient.preloadDoc @project_id, doc.id, (error) => + return callback(error) if error? + DocUpdaterClient.sendUpdate @project_id, doc.id, doc.update, (error) => + callback(error) + ), (error) => + throw error if error? + setTimeout () => + DocUpdaterClient.flushProject @project_id, (error, res, body) => + @statusCode = res.statusCode + done() + , 200 + + after -> + MockWebApi.setDocumentLines.restore() + + it "should return a 204 status code", -> + @statusCode.should.equal 204 + + it "should send each document to the web api", -> + for doc in @docs + MockWebApi.setDocumentLines + .calledWith(@project_id, doc.id, doc.updatedLines) + .should.equal true + + it "should update the lines in the doc updater", (done) -> + async.series @docs.map((doc) => + (callback) => + DocUpdaterClient.getDoc @project_id, doc.id, (error, res, returnedDoc) => + returnedDoc.lines.should.deep.equal doc.updatedLines + callback() + ), done + diff --git a/services/document-updater/test/acceptance/coffee/FlushingDocsTests.coffee b/services/document-updater/test/acceptance/coffee/FlushingDocsTests.coffee new file mode 100644 index 0000000000..aaaef99936 --- /dev/null +++ b/services/document-updater/test/acceptance/coffee/FlushingDocsTests.coffee @@ -0,0 +1,97 @@ +sinon = require "sinon" +chai = require("chai") +chai.should() +async = require "async" + +MockWebApi = require "./helpers/MockWebApi" +DocUpdaterClient = require "./helpers/DocUpdaterClient" +mongojs = require "../../../app/js/mongojs" +db = mongojs.db +ObjectId = mongojs.ObjectId + +describe "Flushing a doc to Mongo", -> + before -> + @lines = ["one", "two", "three"] + @update = + doc: @doc_id + op: [{ + i: "one and a half\n" + p: 4 + }] + v: 0 + @result = ["one", "one and a half", "two", "three"] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines + } + + describe "when the updated doc exists in the doc updater", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines + } + sinon.spy MockWebApi, "setDocumentLines" + + DocUpdaterClient.sendUpdates @project_id, @doc_id, [@update], (error) => + throw error if error? + setTimeout () => + DocUpdaterClient.flushDoc @project_id, @doc_id, done + , 200 + + after -> + MockWebApi.setDocumentLines.restore() + + it "should flush the updated document to the web api", -> + MockWebApi.setDocumentLines + .calledWith(@project_id, @doc_id, @result) + .should.equal true + + it "should flush the doc ops to Mongo", (done) -> + db.docOps.find doc_id: ObjectId(@doc_id), (error, docs) => + doc = docs[0] + doc.docOps[0].op.should.deep.equal @update.op + done() + + describe "when the doc has a large number of ops to be flushed", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines + } + @updates = [] + for v in [0..999] + @updates.push + doc_id: @doc_id, + op: [i: v.toString(), p: 0] + v: v + + DocUpdaterClient.sendUpdates @project_id, @doc_id, @updates, (error) => + throw error if error? + setTimeout () => + DocUpdaterClient.flushDoc @project_id, @doc_id, done + , 200 + + it "should flush the doc ops to Mongo in order", (done) -> + db.docOps.find doc_id: ObjectId(@doc_id), (error, docs) => + doc = docs[0] + updates = @updates.slice(-100) + for update, i in doc.docOps + update.op.should.deep.equal updates[i].op + done() + + describe "when the doc does not exist in the doc updater", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines + } + sinon.spy MockWebApi, "setDocumentLines" + DocUpdaterClient.flushDoc @project_id, @doc_id, done + + after -> + MockWebApi.setDocumentLines.restore() + + it "should not flush the doc to the web api", -> + MockWebApi.setDocumentLines.called.should.equal false + + diff --git a/services/document-updater/test/acceptance/coffee/GettingADocumentTests.coffee b/services/document-updater/test/acceptance/coffee/GettingADocumentTests.coffee new file mode 100644 index 0000000000..0e8456e45f --- /dev/null +++ b/services/document-updater/test/acceptance/coffee/GettingADocumentTests.coffee @@ -0,0 +1,107 @@ +sinon = require "sinon" +chai = require("chai") +chai.should() + +MockWebApi = require "./helpers/MockWebApi" +DocUpdaterClient = require "./helpers/DocUpdaterClient" + +describe "Getting a document", -> + describe "when the document is not loaded", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines = ["one", "two", "three"] + } + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, @returnedDoc) => done() + + after -> + MockWebApi.getDocument.restore() + + it "should load the document from the web API", -> + MockWebApi.getDocument + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should return the document lines", -> + @returnedDoc.lines.should.deep.equal @lines + + it "should return the document at version 0", -> + @returnedDoc.version.should.equal 0 + + describe "when the document is already loaded", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines = ["one", "two", "three"] + } + + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, @returnedDoc) => done() + + after -> + MockWebApi.getDocument.restore() + + it "should not load the document from the web API", -> + MockWebApi.getDocument.called.should.equal false + + it "should return the document lines", -> + @returnedDoc.lines.should.deep.equal @lines + + describe "when the request asks for some recent ops", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines = ["one", "two", "three"] + } + + @updates = for v in [0..99] + doc_id: @doc_id, + op: [i: v.toString(), p: 0] + v: v + + DocUpdaterClient.sendUpdates @project_id, @doc_id, @updates, (error) => + throw error if error? + sinon.spy MockWebApi, "getDocument" + DocUpdaterClient.getDocAndRecentOps @project_id, @doc_id, 90, (error, res, @returnedDoc) => done() + + after -> + MockWebApi.getDocument.restore() + + it "should return the recent ops", -> + @returnedDoc.ops.length.should.equal 10 + for update, i in @updates.slice(90, -1) + @returnedDoc.ops[i].op.should.deep.equal update.op + + + describe "when the document does not exist", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) => + @statusCode = res.statusCode + done() + + it "should return 404", -> + @statusCode.should.equal 404 + + describe "when the web api returns an error", -> + before (done) -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + sinon.stub MockWebApi, "getDocument", (project_id, doc_id, callback = (error, doc) ->) -> + callback new Error("oops") + DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) => + @statusCode = res.statusCode + done() + + after -> + MockWebApi.getDocument.restore() + + it "should return 500", -> + @statusCode.should.equal 500 + + + + + diff --git a/services/document-updater/test/acceptance/coffee/SettingADocumentTests.coffee b/services/document-updater/test/acceptance/coffee/SettingADocumentTests.coffee new file mode 100644 index 0000000000..cc0f30834a --- /dev/null +++ b/services/document-updater/test/acceptance/coffee/SettingADocumentTests.coffee @@ -0,0 +1,58 @@ +sinon = require "sinon" +chai = require("chai") +chai.should() + +MockWebApi = require "./helpers/MockWebApi" +DocUpdaterClient = require "./helpers/DocUpdaterClient" + +describe "Setting a document", -> + before -> + [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + @lines = ["one", "two", "three"] + @update = + doc: @doc_id + op: [{ + i: "one and a half\n" + p: 4 + }] + v: 0 + @result = ["one", "one and a half", "two", "three"] + @newLines = ["these", "are", "the", "new", "lines"] + MockWebApi.insertDoc @project_id, @doc_id, { + lines: @lines + } + + describe "when the updated doc exists in the doc updater", -> + before (done) -> + sinon.spy MockWebApi, "setDocumentLines" + DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => + throw error if error? + DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) => + throw error if error? + setTimeout () => + DocUpdaterClient.setDocLines @project_id, @doc_id, @newLines, (error, res, body) => + @statusCode = res.statusCode + done() + , 200 + + after -> + MockWebApi.setDocumentLines.restore() + + it "should return a 204 status code", -> + @statusCode.should.equal 204 + + it "should send the updated document to the web api", -> + MockWebApi.setDocumentLines + .calledWith(@project_id, @doc_id, @newLines) + .should.equal true + + it "should update the lines in the doc updater", (done) -> + DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) => + doc.lines.should.deep.equal @newLines + done() + + it "should bump the version in the doc updater", (done) -> + DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) => + doc.version.should.equal 2 + done() + diff --git a/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee b/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee new file mode 100644 index 0000000000..4ddef90d26 --- /dev/null +++ b/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee @@ -0,0 +1,66 @@ +rclient = require("redis").createClient() +request = require("request").defaults(jar: false) +async = require "async" + +module.exports = DocUpdaterClient = + randomId: () -> + return require("../../../../app/js/mongojs").ObjectId().toString() + + sendUpdate: (project_id, doc_id, update, callback = (error) ->) -> + rclient.rpush "PendingUpdates:#{doc_id}", JSON.stringify(update), (error)-> + return callback(error) if error? + doc_key = "#{project_id}:#{doc_id}" + rclient.sadd "DocsWithPendingUpdates", doc_key, (error) -> + return callback(error) if error? + rclient.publish "pending-updates", doc_key, callback + + sendUpdates: (project_id, doc_id, updates, callback = (error) ->) -> + DocUpdaterClient.preloadDoc project_id, doc_id, (error) -> + return callback(error) if error? + jobs = [] + for update in updates + do (update) -> + jobs.push (callback) -> + DocUpdaterClient.sendUpdate project_id, doc_id, update, callback + async.series jobs, callback + + getDoc: (project_id, doc_id, callback = (error, res, body) ->) -> + request.get "http://localhost:3003/project/#{project_id}/doc/#{doc_id}", (error, res, body) -> + if body? and res.statusCode >= 200 and res.statusCode < 300 + body = JSON.parse(body) + callback error, res, body + + getDocAndRecentOps: (project_id, doc_id, fromVersion, callback = (error, res, body) ->) -> + request.get "http://localhost:3003/project/#{project_id}/doc/#{doc_id}?fromVersion=#{fromVersion}", (error, res, body) -> + if body? and res.statusCode >= 200 and res.statusCode < 300 + body = JSON.parse(body) + callback error, res, body + + preloadDoc: (project_id, doc_id, callback = (error) ->) -> + DocUpdaterClient.getDoc project_id, doc_id, callback + + flushDoc: (project_id, doc_id, callback = (error) ->) -> + request.post "http://localhost:3003/project/#{project_id}/doc/#{doc_id}/flush", (error, res, body) -> + callback error, res, body + + setDocLines: (project_id, doc_id, lines, callback = (error) ->) -> + request.post { + url: "http://localhost:3003/project/#{project_id}/doc/#{doc_id}" + json: + lines: lines + }, (error, res, body) -> + callback error, res, body + + deleteDoc: (project_id, doc_id, callback = (error) ->) -> + request.del "http://localhost:3003/project/#{project_id}/doc/#{doc_id}", (error, res, body) -> + callback error, res, body + + flushProject: (project_id, callback = () ->) -> + request.post "http://localhost:3003/project/#{project_id}/flush", callback + + deleteProject: (project_id, callback = () ->) -> + request.del "http://localhost:3003/project/#{project_id}", callback + + + + diff --git a/services/document-updater/test/acceptance/coffee/helpers/MockWebApi.coffee b/services/document-updater/test/acceptance/coffee/helpers/MockWebApi.coffee new file mode 100644 index 0000000000..7d50eb8377 --- /dev/null +++ b/services/document-updater/test/acceptance/coffee/helpers/MockWebApi.coffee @@ -0,0 +1,40 @@ +express = require("express") +app = express() + +module.exports = MockWebApi = + docs: {} + + clearDocs: () -> @docs = {} + + insertDoc: (project_id, doc_id, doc) -> + @docs["#{project_id}:#{doc_id}"] = doc + + setDocumentLines: (project_id, doc_id, lines, callback = (error) ->) -> + @docs["#{project_id}:#{doc_id}"] ||= {} + @docs["#{project_id}:#{doc_id}"].lines = lines + callback null + + getDocument: (project_id, doc_id, callback = (error, doc) ->) -> + callback null, @docs["#{project_id}:#{doc_id}"] + + run: () -> + app.get "/project/:project_id/doc/:doc_id", (req, res, next) => + @getDocument req.params.project_id, req.params.doc_id, (error, doc) -> + if error? + res.send 500 + else if doc? + res.send JSON.stringify doc + else + res.send 404 + + app.post "/project/:project_id/doc/:doc_id", express.bodyParser(), (req, res, next) => + @setDocumentLines req.params.project_id, req.params.doc_id, req.body.lines, (error) -> + if error? + res.send 500 + else + res.send 204 + + app.listen(3000) + +MockWebApi.run() + diff --git a/services/document-updater/test/unit/coffee/AddingDocsToMemory.coffee b/services/document-updater/test/unit/coffee/AddingDocsToMemory.coffee new file mode 100644 index 0000000000..1ca00bb305 --- /dev/null +++ b/services/document-updater/test/unit/coffee/AddingDocsToMemory.coffee @@ -0,0 +1,58 @@ +require('coffee-script') +assert = require('assert') +path = require('path') +modulePath = path.join __dirname, '../../../app/js/RedisManager.js' +keys = require(path.join __dirname, '../../../app/js/RedisKeyBuilder.js') +project_id = 1234 +doc_id = 5678 +loadModule = require('./module-loader').loadModule + +describe 'putting a doc into memory', ()-> + lines = ["this is one line", "and another line"] + version = 42 + + potentialSets = {} + potentialSets[keys.docLines(doc_id:doc_id)] = lines + potentialSets[keys.projectKey(doc_id:doc_id)] = project_id + potentialSets[keys.docVersion(doc_id:doc_id)] = version + + potentialSAdds = {} + potentialSAdds[keys.allDocs] = doc_id + potentialSAdds[keys.docsInProject(project_id:project_id)] = doc_id + + potentialDels = {} + potentialDels[keys.docOps(doc_id:doc_id)] = true + + mocks = + "logger-sharelatex": log:-> + redis: + createClient : ()-> + auth:-> + multi: ()-> + set:(key, value)-> + result = potentialSets[key] + delete potentialSets[key] + if key == keys.docLines(doc_id:doc_id) + value = JSON.parse(value) + assert.deepEqual result, value + incr:()-> + sadd:(key, value)-> + result = potentialSAdds[key] + delete potentialSAdds[key] + assert.equal result, value + del: (key) -> + result = potentialDels[key] + delete potentialDels[key] + assert.equal result, true + exec:(callback)-> + callback() + + redisManager = loadModule(modulePath, mocks).module.exports + + it 'should put a all data into memory', (done)-> + redisManager.putDocInMemory project_id, doc_id, lines, version, ()-> + assert.deepEqual potentialSets, {} + assert.deepEqual potentialSAdds, {} + assert.deepEqual potentialDels, {} + done() + diff --git a/services/document-updater/test/unit/coffee/CheckingUpdatesLength.coffee b/services/document-updater/test/unit/coffee/CheckingUpdatesLength.coffee new file mode 100644 index 0000000000..4f76c48ae7 --- /dev/null +++ b/services/document-updater/test/unit/coffee/CheckingUpdatesLength.coffee @@ -0,0 +1,27 @@ +assert = require('chai').assert +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../app/js/RedisManager.js" +SandboxedModule = require('sandboxed-module') + +doc_id = "1234" + +describe 'Document Manager - getUpdatesLength ', -> + + beforeEach -> + + @llenStub = sinon.stub() + @redisManager = SandboxedModule.require modulePath, requires: + redis: + createClient:=> + auth:-> + llen:@llenStub + + it "should the number of things to process in the que", (done)-> + + @llenStub.callsArgWith(1, null, 3) + @redisManager.getUpdatesLength doc_id, (err, len)=> + @llenStub.calledWith("PendingUpdates:#{doc_id}").should.equal true + len.should.equal 3 + done() diff --git a/services/document-updater/test/unit/coffee/DiffCodec/DiffCodecTests.coffee b/services/document-updater/test/unit/coffee/DiffCodec/DiffCodecTests.coffee new file mode 100644 index 0000000000..bcd07c0479 --- /dev/null +++ b/services/document-updater/test/unit/coffee/DiffCodec/DiffCodecTests.coffee @@ -0,0 +1,56 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +expect = chai.expect +modulePath = "../../../../app/js/DiffCodec.js" +SandboxedModule = require('sandboxed-module') + +describe "DiffCodec", -> + beforeEach -> + @callback = sinon.stub() + @DiffCodec = SandboxedModule.require modulePath + + describe "diffAsShareJsOps", -> + it "should insert new text correctly", (done) -> + @before = ["hello world"] + @after = ["hello beautiful world"] + @DiffCodec.diffAsShareJsOp @before, @after, (error, ops) -> + expect(ops).to.deep.equal [ + i: "beautiful " + p: 6 + ] + done() + + it "should shift later inserts by previous inserts", (done) -> + @before = ["the boy played with the ball"] + @after = ["the tall boy played with the red ball"] + @DiffCodec.diffAsShareJsOp @before, @after, (error, ops) -> + expect(ops).to.deep.equal [ + { i: "tall ", p: 4 } + { i: "red ", p: 29 } + ] + done() + + it "should delete text correctly", (done) -> + @before = ["hello beautiful world"] + @after = ["hello world"] + @DiffCodec.diffAsShareJsOp @before, @after, (error, ops) -> + expect(ops).to.deep.equal [ + d: "beautiful " + p: 6 + ] + done() + + + it "should shift later deletes by the first deletes", (done) -> + @before = ["the tall boy played with the red ball"] + @after = ["the boy played with the ball"] + @DiffCodec.diffAsShareJsOp @before, @after, (error, ops) -> + expect(ops).to.deep.equal [ + { d: "tall ", p: 4 } + { d: "red ", p: 24 } + ] + done() + + + diff --git a/services/document-updater/test/unit/coffee/DocOpsManager/DocOpsManagerTests.coffee b/services/document-updater/test/unit/coffee/DocOpsManager/DocOpsManagerTests.coffee new file mode 100644 index 0000000000..83e0ff48cf --- /dev/null +++ b/services/document-updater/test/unit/coffee/DocOpsManager/DocOpsManagerTests.coffee @@ -0,0 +1,309 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/DocOpsManager.js" +SandboxedModule = require('sandboxed-module') +ObjectId = require("../../../../app/js/mongojs").ObjectId + +describe "DocOpsManager", -> + beforeEach -> + @doc_id = ObjectId().toString() + @project_id = "project-id" + @callback = sinon.stub() + @DocOpsManager = SandboxedModule.require modulePath, requires: + "./RedisManager": @RedisManager = {} + "./mongojs": + db: @db = { docOps: {} } + ObjectId: ObjectId + "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } + "./Metrics": @Metrics = + Timer: class Timer + done: sinon.stub() + + describe "flushDocOpsToMongo", -> + describe "when versions are consistent", -> + beforeEach -> + @mongo_version = 40 + @redis_version = 42 + @ops = [ "mock-op-1", "mock-op-2" ] + @DocOpsManager.getDocVersionInMongo = sinon.stub().callsArgWith(1, null, @mongo_version) + @RedisManager.getDocVersion = sinon.stub().callsArgWith(1, null, @redis_version) + @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) + @DocOpsManager._appendDocOpsInMongo = sinon.stub().callsArg(3) + @DocOpsManager.flushDocOpsToMongo @project_id, @doc_id, @callback + + it "should get the version from Mongo", -> + @DocOpsManager.getDocVersionInMongo + .calledWith(@doc_id) + .should.equal true + + it "should get the version from REdis", -> + @RedisManager.getDocVersion + .calledWith(@doc_id) + .should.equal true + + it "should get all doc ops since the version in Mongo", -> + @RedisManager.getPreviousDocOps + .calledWith(@doc_id, @mongo_version, -1) + .should.equal true + + it "should update Mongo with the new ops", -> + @DocOpsManager._appendDocOpsInMongo + .calledWith(@doc_id, @ops, @redis_version) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when the number of ops does not match the difference in versions", -> + beforeEach -> + @mongo_version = 40 + @redis_version = 45 + @ops = [ "mock-op-1", "mock-op-2" ] + @DocOpsManager.getDocVersionInMongo = sinon.stub().callsArgWith(1, null, @mongo_version) + @RedisManager.getDocVersion = sinon.stub().callsArgWith(1, null, @redis_version) + @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) + @DocOpsManager._appendDocOpsInMongo = sinon.stub().callsArg(3) + @DocOpsManager.flushDocOpsToMongo @project_id, @doc_id, @callback + + it "should call the callback with an error", -> + @callback.calledWith(new Error("inconsistet versions")).should.equal true + + it "should log an error", -> + @logger.error + .calledWith(doc_id: @doc_id, mongoVersion: @mongo_version, redisVersion: @redis_version, opsLength: @ops.length, "version difference does not match ops length") + .should.equal true + + it "should not modify mongo", -> + @DocOpsManager._appendDocOpsInMongo.called.should.equal false + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when redis version is behind mongo version", -> + beforeEach -> + @mongo_version = 40 + @redis_version = 30 + @DocOpsManager.getDocVersionInMongo = sinon.stub().callsArgWith(1, null, @mongo_version) + @RedisManager.getDocVersion = sinon.stub().callsArgWith(1, null, @redis_version) + @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) + @DocOpsManager._appendDocOpsInMongo = sinon.stub().callsArg(3) + @DocOpsManager.flushDocOpsToMongo @project_id, @doc_id, @callback + + it "should call the callback with an error", -> + @callback.calledWith(new Error("inconsistet versions")).should.equal true + + it "should log an error", -> + @logger.error + .calledWith(doc_id: @doc_id, mongoVersion: @mongo_version, redisVersion: @redis_version, "mongo version is ahead of redis") + .should.equal true + + it "should not modify mongo", -> + @DocOpsManager._appendDocOpsInMongo.called.should.equal false + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "getPreviousDocOps", -> + beforeEach -> + @ops = [ "mock-op-1", "mock-op-2" ] + @start = 30 + @end = 32 + @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) + @DocOpsManager._ensureOpsAreLoaded = sinon.stub().callsArg(3) + @DocOpsManager.getPreviousDocOps @project_id, @doc_id, @start, @end, @callback + + it "should ensure the ops are loaded back far enough", -> + @DocOpsManager._ensureOpsAreLoaded + .calledWith(@project_id, @doc_id, @start) + .should.equal true + + it "should get the previous doc ops", -> + @RedisManager.getPreviousDocOps + .calledWith(@doc_id, @start, @end) + .should.equal true + + it "should call the callback with the ops", -> + @callback.calledWith(null, @ops).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "_ensureOpsAreLoaded", -> + describe "when the ops are not loaded", -> + beforeEach -> + @redisVersion = 42 + @redisOpsLength = 10 + @backToVersion = 30 + @ops = [ "mock-op-1", "mock-op-2" ] + @RedisManager.getDocVersion = sinon.stub().callsArgWith(1, null, @redisVersion) + @RedisManager.getDocOpsLength = sinon.stub().callsArgWith(1, null, @redisOpsLength) + @DocOpsManager._getDocOpsFromMongo = sinon.stub().callsArgWith(3, null, @ops) + @RedisManager.prependDocOps = sinon.stub().callsArgWith(2, null) + @DocOpsManager._ensureOpsAreLoaded @project_id, @doc_id, @backToVersion, @callback + + it "should get the doc version from redis", -> + @RedisManager.getDocVersion + .calledWith(@doc_id) + .should.equal true + + it "should get the doc ops length in redis", -> + @RedisManager.getDocOpsLength + .calledWith(@doc_id) + .should.equal true + + it "should get the doc ops that need loading from Mongo", -> + @DocOpsManager._getDocOpsFromMongo + .calledWith(@doc_id, @backToVersion, @redisVersion - @redisOpsLength) + .should.equal true + + it "should prepend the retrieved ops to redis", -> + @RedisManager.prependDocOps + .calledWith(@doc_id, @ops) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + describe "when the ops are loaded", -> + beforeEach -> + @redisVersion = 42 + @redisOpsLength = 10 + @backToVersion = 35 + @RedisManager.getDocVersion = sinon.stub().callsArgWith(1, null, @redisVersion) + @RedisManager.getDocOpsLength = sinon.stub().callsArgWith(1, null, @redisOpsLength) + @DocOpsManager._getDocOpsFromMongo = sinon.stub().callsArgWith(3, null, @ops) + @RedisManager.prependDocOps = sinon.stub().callsArgWith(2, null) + @DocOpsManager._ensureOpsAreLoaded @project_id, @doc_id, @backToVersion, @callback + + it "should not need to get the docs from Mongo or put any into redis", -> + @DocOpsManager._getDocOpsFromMongo.called.should.equal false + @RedisManager.prependDocOps.called.should.equal false + + it "should call the callback", -> + @callback.called.should.equal true + + describe "getDocVersionInMongo", -> + describe "when the doc exists", -> + beforeEach -> + @doc = + version: @version = 42 + @db.docOps.find = sinon.stub().callsArgWith(2, null, [@doc]) + @DocOpsManager.getDocVersionInMongo @doc_id, @callback + + it "should look for the doc in the database", -> + @db.docOps.find + .calledWith({ doc_id: ObjectId(@doc_id) }, {version: 1}) + .should.equal true + + it "should call the callback with the version", -> + @callback.calledWith(null, @version).should.equal true + + describe "when the doc doesn't exist", -> + beforeEach -> + @db.docOps.find = sinon.stub().callsArgWith(2, null, []) + @DocOpsManager.getDocVersionInMongo @doc_id, @callback + + it "should call the callback with 0", -> + @callback.calledWith(null, 0).should.equal true + + describe "_appendDocOpsInMongo", -> + describe "with a small set of updates", -> + beforeEach (done) -> + @ops = [ "mock-op-1", "mock-op-2" ] + @version = 42 + @db.docOps.update = sinon.stub().callsArg(3) + @DocOpsManager._appendDocOpsInMongo @doc_id, @ops, @version, (error) => + @callback(error) + done() + + it "should update the database", -> + @db.docOps.update + .calledWith({ + doc_id: ObjectId(@doc_id) + }, { + $push: docOps: { $each: @ops, $slice: -100 } + $set: version: @version + }, { + upsert: true + }) + .should.equal true + + it "should call the callbak", -> + @callback.called.should.equal true + + describe "with a large set of updates", -> + beforeEach (done) -> + @ops = [ "mock-op-1", "mock-op-2", "mock-op-3", "mock-op-4", "mock-op-5" ] + @version = 42 + @DocOpsManager.APPEND_OPS_BATCH_SIZE = 2 + @db.docOps.update = sinon.stub().callsArg(3) + @DocOpsManager._appendDocOpsInMongo @doc_id, @ops, @version, (error) => + @callback(error) + done() + + it "should update the database in batches", -> + @db.docOps.update + .calledWith({ doc_id: ObjectId(@doc_id) }, { + $push: docOps: { $each: @ops.slice(0,2), $slice: -100 } + $set: version: @version - 3 + }, { upsert: true }) + .should.equal true + @db.docOps.update + .calledWith({ doc_id: ObjectId(@doc_id) }, { + $push: docOps: { $each: @ops.slice(2,4), $slice: -100 } + $set: version: @version - 1 + }, { upsert: true }) + .should.equal true + @db.docOps.update + .calledWith({ doc_id: ObjectId(@doc_id) }, { + $push: docOps: { $each: @ops.slice(4,5), $slice: -100 } + $set: version: @version + }, { upsert: true }) + .should.equal true + + it "should call the callbak", -> + @callback.called.should.equal true + + describe "with no updates", -> + beforeEach (done) -> + @ops = [] + @version = 42 + @db.docOps.update = sinon.stub().callsArg(3) + @DocOpsManager._appendDocOpsInMongo @doc_id, @ops, @version, (error) => + @callback(error) + done() + + it "should not try to update the database", -> + @db.docOps.update.called.should.equal false + + describe "_getDocsOpsFromMongo", -> + beforeEach -> + @version = 42 + @start = 32 + @limit = 5 + @doc = + docOps: ["mock-ops"] + @DocOpsManager.getDocVersionInMongo = sinon.stub().callsArgWith(1, null, @version) + @db.docOps.find = sinon.stub().callsArgWith(2, null, [@doc]) + @DocOpsManager._getDocOpsFromMongo @doc_id, @start, @start + @limit, @callback + + it "should get the current version", -> + @DocOpsManager.getDocVersionInMongo + .calledWith(@doc_id) + .should.equal true + + it "should get the doc ops", -> + @db.docOps.find + .calledWith({ doc_id: ObjectId(@doc_id) }, { + docOps: $slice: [-(@version - @start), @limit] + }) + .should.equal true + + it "should return the ops", -> + @callback.calledWith(null, @doc.docOps).should.equal true + + diff --git a/services/document-updater/test/unit/coffee/DocumentManager/flushAndDeleteDocTests.coffee b/services/document-updater/test/unit/coffee/DocumentManager/flushAndDeleteDocTests.coffee new file mode 100644 index 0000000000..85a25ee5a7 --- /dev/null +++ b/services/document-updater/test/unit/coffee/DocumentManager/flushAndDeleteDocTests.coffee @@ -0,0 +1,41 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/DocumentManager.js" +SandboxedModule = require('sandboxed-module') + +describe "DocumentUpdater - flushAndDeleteDoc", -> + beforeEach -> + @DocumentManager = SandboxedModule.require modulePath, requires: + "./RedisManager": @RedisManager = {} + "./PersistenceManager": @PersistenceManager = {} + "logger-sharelatex": @logger = {log: sinon.stub()} + "./DocOpsManager" :{} + "./Metrics": @Metrics = + Timer: class Timer + done: sinon.stub() + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @callback = sinon.stub() + + describe "successfully", -> + beforeEach -> + @RedisManager.removeDocFromMemory = sinon.stub().callsArg(2) + @DocumentManager.flushDocIfLoaded = sinon.stub().callsArgWith(2) + @DocumentManager.flushAndDeleteDoc @project_id, @doc_id, @callback + + it "should flush the doc", -> + @DocumentManager.flushDocIfLoaded + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should remove the doc from redis", -> + @RedisManager.removeDocFromMemory + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should call the callback without error", -> + @callback.calledWith(null).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true diff --git a/services/document-updater/test/unit/coffee/DocumentManager/flushDocTests.coffee b/services/document-updater/test/unit/coffee/DocumentManager/flushDocTests.coffee new file mode 100644 index 0000000000..079341a536 --- /dev/null +++ b/services/document-updater/test/unit/coffee/DocumentManager/flushDocTests.coffee @@ -0,0 +1,73 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/DocumentManager.js" +SandboxedModule = require('sandboxed-module') + +describe "DocumentUpdater - flushDocIfLoaded", -> + beforeEach -> + @DocumentManager = SandboxedModule.require modulePath, requires: + "./RedisManager": @RedisManager = {} + "./PersistenceManager": @PersistenceManager = {} + "./DocOpsManager": @DocOpsManager = {} + "logger-sharelatex": @logger = {log: sinon.stub()} + "./Metrics": @Metrics = + Timer: class Timer + done: sinon.stub() + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @lines = ["one", "two", "three"] + @version = 42 + @callback = sinon.stub() + + describe "when the doc is in Redis", -> + beforeEach -> + @RedisManager.getDoc = sinon.stub().callsArgWith(1, null, @lines, @version) + @PersistenceManager.setDoc = sinon.stub().callsArgWith(3) + @DocOpsManager.flushDocOpsToMongo = sinon.stub().callsArgWith(2) + @DocumentManager.flushDocIfLoaded @project_id, @doc_id, @callback + + it "should get the doc from redis", -> + @RedisManager.getDoc + .calledWith(@doc_id) + .should.equal true + + it "should write the doc lines to the persistence layer", -> + @PersistenceManager.setDoc + .calledWith(@project_id, @doc_id, @lines) + .should.equal true + + it "should write the doc ops to mongo", -> + @DocOpsManager.flushDocOpsToMongo + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should call the callback without error", -> + @callback.calledWith(null).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when the document is not in Redis", -> + beforeEach -> + @RedisManager.getDoc = sinon.stub().callsArgWith(1, null, null, null) + @PersistenceManager.setDoc = sinon.stub().callsArgWith(3) + @DocOpsManager.flushDocOpsToMongo = sinon.stub().callsArgWith(2) + @DocumentManager.flushDocIfLoaded @project_id, @doc_id, @callback + + it "should get the doc from redis", -> + @RedisManager.getDoc + .calledWith(@doc_id) + .should.equal true + + it "should not write anything to the persistence layer", -> + @PersistenceManager.setDoc.called.should.equal false + @DocOpsManager.flushDocOpsToMongo.called.should.equal false + + it "should call the callback without error", -> + @callback.calledWith(null).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + diff --git a/services/document-updater/test/unit/coffee/DocumentManager/getDocAndRecentOpsTests.coffee b/services/document-updater/test/unit/coffee/DocumentManager/getDocAndRecentOpsTests.coffee new file mode 100644 index 0000000000..7a296cc47d --- /dev/null +++ b/services/document-updater/test/unit/coffee/DocumentManager/getDocAndRecentOpsTests.coffee @@ -0,0 +1,67 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/DocumentManager.js" +SandboxedModule = require('sandboxed-module') + +describe "DocumentUpdater - getDocAndRecentOps", -> + beforeEach -> + @DocumentManager = SandboxedModule.require modulePath, requires: + "./RedisManager": @RedisManager = {} + "./PersistenceManager": @PersistenceManager = {} + "./DocOpsManager": @DocOpsManager = {} + "logger-sharelatex": @logger = {log: sinon.stub()} + "./Metrics": @Metrics = + Timer: class Timer + done: sinon.stub() + + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @lines = ["one", "two", "three"] + @version = 42 + @fromVersion = 40 + @ops = ["mock-op-1", "mock-op-2"] + @callback = sinon.stub() + + describe "with a previous version specified", -> + beforeEach -> + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version) + @DocOpsManager.getPreviousDocOps = sinon.stub().callsArgWith(4, null, @ops) + @DocumentManager.getDocAndRecentOps @project_id, @doc_id, @fromVersion, @callback + + it "should get the doc", -> + @DocumentManager.getDoc + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should get the doc ops", -> + @DocOpsManager.getPreviousDocOps + .calledWith(@project_id, @doc_id, @fromVersion, @version) + .should.equal true + + it "should call the callback with the doc info", -> + @callback.calledWith(null, @lines, @version, @ops).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "with no previous version specified", -> + beforeEach -> + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version) + @DocOpsManager.getPreviousDocOps = sinon.stub().callsArgWith(4, null, @ops) + @DocumentManager.getDocAndRecentOps @project_id, @doc_id, -1, @callback + + it "should get the doc", -> + @DocumentManager.getDoc + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should not need to get the doc ops", -> + @DocOpsManager.getPreviousDocOps.called.should.equal false + + it "should call the callback with the doc info", -> + @callback.calledWith(null, @lines, @version, []).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + diff --git a/services/document-updater/test/unit/coffee/DocumentManager/getDocTests.coffee b/services/document-updater/test/unit/coffee/DocumentManager/getDocTests.coffee new file mode 100644 index 0000000000..93de1725fa --- /dev/null +++ b/services/document-updater/test/unit/coffee/DocumentManager/getDocTests.coffee @@ -0,0 +1,75 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/DocumentManager.js" +SandboxedModule = require('sandboxed-module') + +describe "DocumentUpdater - getDoc", -> + beforeEach -> + @DocumentManager = SandboxedModule.require modulePath, requires: + "./RedisManager": @RedisManager = {} + "./PersistenceManager": @PersistenceManager = {} + "./DocOpsManager": @DocOpsManager = {} + "logger-sharelatex": @logger = {log: sinon.stub()} + "./Metrics": @Metrics = + Timer: class Timer + done: sinon.stub() + + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @lines = ["one", "two", "three"] + @version = 42 + @callback = sinon.stub() + + describe "when the doc exists in Redis", -> + beforeEach -> + @RedisManager.getDoc = sinon.stub().callsArgWith(1, null, @lines, @version) + @DocumentManager.getDoc @project_id, @doc_id, @callback + + it "should get the doc from Redis", -> + @RedisManager.getDoc + .calledWith(@doc_id) + .should.equal true + + it "should call the callback with the doc info", -> + @callback.calledWith(null, @lines, @version).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when the doc does not exist in Redis", -> + beforeEach -> + @RedisManager.getDoc = sinon.stub().callsArgWith(1, null, null, null) + @PersistenceManager.getDoc = sinon.stub().callsArgWith(2, null, @lines) + @DocOpsManager.getDocVersionInMongo = sinon.stub().callsArgWith(1, null, @version) + @RedisManager.putDocInMemory = sinon.stub().callsArg(4) + @DocumentManager.getDoc @project_id, @doc_id, @callback + + it "should try to get the doc from Redis", -> + @RedisManager.getDoc + .calledWith(@doc_id) + .should.equal true + + it "should get the doc version from Mongo", -> + @DocOpsManager.getDocVersionInMongo + .calledWith(@doc_id) + .should.equal true + + it "should get the doc from the PersistenceManager", -> + @PersistenceManager.getDoc + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should set the doc in Redis", -> + @RedisManager.putDocInMemory + .calledWith(@project_id, @doc_id, @lines, @version) + .should.equal true + + it "should call the callback with the doc info", -> + @callback.calledWith(null, @lines, @version).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + + diff --git a/services/document-updater/test/unit/coffee/DocumentManager/setDocTests.coffee b/services/document-updater/test/unit/coffee/DocumentManager/setDocTests.coffee new file mode 100644 index 0000000000..d4b5e931b8 --- /dev/null +++ b/services/document-updater/test/unit/coffee/DocumentManager/setDocTests.coffee @@ -0,0 +1,105 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/DocumentManager.js" +SandboxedModule = require('sandboxed-module') + +describe "DocumentManager - setDoc", -> + beforeEach -> + @DocumentManager = SandboxedModule.require modulePath, requires: + "./RedisManager": @RedisManager = {} + "./PersistenceManager": @PersistenceManager = {} + "./DiffCodec": @DiffCodec = {} + "./DocOpsManager":{} + "./UpdateManager": @UpdateManager = {} + "logger-sharelatex": @logger = {log: sinon.stub()} + "./Metrics": @Metrics = + Timer: class Timer + done: sinon.stub() + + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @version = 42 + @ops = ["mock-ops"] + @callback = sinon.stub() + + describe "with plain tex lines", -> + beforeEach -> + @beforeLines = ["before", "lines"] + @afterLines = ["after", "lines"] + + describe "successfully", -> + beforeEach -> + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @beforeLines, @version) + @DiffCodec.diffAsShareJsOp = sinon.stub().callsArgWith(2, null, @ops) + @UpdateManager.applyUpdates = sinon.stub().callsArgWith(3, null) + @DocumentManager.flushDocIfLoaded = sinon.stub().callsArg(2) + @DocumentManager.setDoc @project_id, @doc_id, @afterLines, @callback + + it "should get the current doc lines", -> + @DocumentManager.getDoc + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should return a diff of the old and new lines", -> + @DiffCodec.diffAsShareJsOp + .calledWith(@beforeLines, @afterLines) + .should.equal true + + it "should apply the diff as a ShareJS op", -> + @UpdateManager.applyUpdates + .calledWith(@project_id, @doc_id, [doc: @doc_id, v: @version, op: @ops, meta: { type: "external" }]) + .should.equal true + + it "should flush the doc to Mongo", -> + @DocumentManager.flushDocIfLoaded + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should call the callback", -> + @callback.calledWith(null).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "with json lines", -> + beforeEach -> + @beforeLines = [text: "before", text: "lines"] + @afterLines = ["after", "lines"] + + describe "successfully", -> + beforeEach -> + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @beforeLines, @version) + @DiffCodec.diffAsShareJsOp = sinon.stub().callsArgWith(2, null, @ops) + @UpdateManager.applyUpdates = sinon.stub().callsArgWith(3, null) + @DocumentManager.flushDocIfLoaded = sinon.stub().callsArg(2) + @DocumentManager.setDoc @project_id, @doc_id, @afterLines, @callback + + it "should get the current doc lines", -> + @DocumentManager.getDoc + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should return not try to get a diff", -> + @DiffCodec.diffAsShareJsOp.called.should.equal false + + it "should call the callback", -> + @callback.calledWith(null).should.equal true + + describe "without new lines", -> + beforeEach -> + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @beforeLines, @version) + @DocumentManager.setDoc @project_id, @doc_id, null, @callback + + it "should return teh callback with an error", -> + @callback.calledWith(new Error("No lines were passed to setDoc")) + + it "should not try to get the doc lines", -> + @DocumentManager.getDoc.called.should.equal false + + + + + + + diff --git a/services/document-updater/test/unit/coffee/GettingDoc.coffee b/services/document-updater/test/unit/coffee/GettingDoc.coffee new file mode 100644 index 0000000000..824fe14a3f --- /dev/null +++ b/services/document-updater/test/unit/coffee/GettingDoc.coffee @@ -0,0 +1,41 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../app/js/RedisManager.js" +SandboxedModule = require('sandboxed-module') + +describe 'RedisManager - getDoc', -> + beforeEach -> + @rclient = {} + @rclient.auth = () -> + @rclient.multi = () => @rclient + + @RedisManager = SandboxedModule.require modulePath, requires: + "redis": @redis = + createClient: () => @rclient + + @doc_id = "doc-id-123" + @lines = ["one", "two", "three"] + @jsonlines = JSON.stringify @lines + @version = 42 + @callback = sinon.stub() + + @rclient.get = sinon.stub() + @rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version]) + + @RedisManager.getDoc @doc_id, @callback + + it "should get the lines from redis", -> + @rclient.get + .calledWith("doclines:#{@doc_id}") + .should.equal true + + it "should get the version from", -> + @rclient.get + .calledWith("DocVersion:#{@doc_id}") + .should.equal true + + it 'should return the document', -> + @callback + .calledWith(null, @lines, @version) + .should.equal true diff --git a/services/document-updater/test/unit/coffee/GettingListOfPendingUpdates.coffee b/services/document-updater/test/unit/coffee/GettingListOfPendingUpdates.coffee new file mode 100644 index 0000000000..cb98e8f601 --- /dev/null +++ b/services/document-updater/test/unit/coffee/GettingListOfPendingUpdates.coffee @@ -0,0 +1,42 @@ +assert = require('assert') +should = require('chai').should() +path = require('path') +modulePath = path.join __dirname, '../../../app/js/RedisManager.js' +_ = require('underscore') +loadModule = require('./module-loader').loadModule +keys = require(path.join __dirname, '../../../app/js/RedisKeyBuilder.js') + +describe 'getting entire list of pending updates', ()-> + + doc_id = 123 + redisMemory = {} + correctUpdates = [{"update1"}, {"update2"}, {"update3"}] + jsonCorrectUpdates = _.map correctUpdates, (d)-> JSON.stringify d + redisMemory[keys.pendingUpdates(doc_id:doc_id)] = jsonCorrectUpdates + redisMemory[keys.pendingUpdates(doc_id:"notThis")] = JSON.stringify([{"updatex"}, {"updatez"}]) + + redisReturn = [] + + mocks = + redis: + createClient: ()-> + auth:-> + multi: ()-> + lrange:(key, start, end)-> + key.should.equal(keys.pendingUpdates(doc_id:doc_id)) + start.should.equal(0) + end.should.equal(-1) + redisReturn.push(redisMemory[key]) + del : (key)-> + key.should.equal(keys.pendingUpdates(doc_id:doc_id)) + redisReturn.push(1) + exec: (callback)-> + callback(null, redisReturn) + + redisManager = loadModule(modulePath, mocks).module.exports + + it 'should have 3 elements in array', (done)-> + redisManager.getPendingUpdatesForDoc doc_id, (err, listOfUpdates)-> + listOfUpdates.length.should.equal(3) + done() + diff --git a/services/document-updater/test/unit/coffee/GettingTotalNumberOfDocs.coffee b/services/document-updater/test/unit/coffee/GettingTotalNumberOfDocs.coffee new file mode 100644 index 0000000000..ae4af4825d --- /dev/null +++ b/services/document-updater/test/unit/coffee/GettingTotalNumberOfDocs.coffee @@ -0,0 +1,47 @@ +require('coffee-script') +assert = require('assert') +should = require('chai').should() +path = require('path') +modulePath = path.join __dirname, '../../../app/js/RedisManager.js' +keys = require(path.join __dirname, '../../../app/js/RedisKeyBuilder.js') +loadModule = require('./module-loader').loadModule + +describe 'getting cound of docs from memory', ()-> + + project_id = "12345" + doc_id1 = "docid1" + doc_id2 = "docid2" + doc_id3 = "docid3" + + redisMemory = {} + redisManager = undefined + + beforeEach (done)-> + mocks = + "logger-sharelatex": log:-> + redis: + createClient : ()-> + auth:-> + smembers:(key, callback)-> + callback(null, redisMemory[key]) + multi: ()-> + set:(key, value)-> + redisMemory[key] = value + sadd:(key, value)-> + if !redisMemory[key]? + redisMemory[key] = [] + redisMemory[key].push value + del:()-> + exec:(callback)-> + callback() + + redisManager = loadModule(modulePath, mocks).module.exports + redisManager.putDocInMemory project_id, doc_id1, 0, ["line"], -> + redisManager.putDocInMemory project_id, doc_id2, 0, ["ledf"], -> + redisManager.putDocInMemory project_id, doc_id3, 0, ["ledf"], -> + done() + + it 'should return total', (done)-> + redisManager.getCountOfDocsInMemory (err, count)-> + assert.equal count, 3 + done() diff --git a/services/document-updater/test/unit/coffee/HttpController/deleteProjectTests.coffee b/services/document-updater/test/unit/coffee/HttpController/deleteProjectTests.coffee new file mode 100644 index 0000000000..e3c6eda35c --- /dev/null +++ b/services/document-updater/test/unit/coffee/HttpController/deleteProjectTests.coffee @@ -0,0 +1,63 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/HttpController.js" +SandboxedModule = require('sandboxed-module') +Errors = require "../../../../app/js/Errors.js" + +describe "HttpController - deleteProject", -> + beforeEach -> + @HttpController = SandboxedModule.require modulePath, requires: + "./DocumentManager": @DocumentManager = {} + "./ProjectManager": @ProjectManager = {} + "logger-sharelatex" : @logger = { log: sinon.stub() } + "./Metrics": @Metrics = {} + + @Metrics.Timer = class Timer + done: sinon.stub() + + @project_id = "project-id-123" + @res = + send: sinon.stub() + @req = + params: + project_id: @project_id + @next = sinon.stub() + + describe "successfully", -> + beforeEach -> + @ProjectManager.flushAndDeleteProjectWithLocks = sinon.stub().callsArgWith(1) + @HttpController.deleteProject(@req, @res, @next) + + it "should delete the project", -> + @ProjectManager.flushAndDeleteProjectWithLocks + .calledWith(@project_id) + .should.equal true + + it "should return a successful No Content response", -> + @res.send + .calledWith(204) + .should.equal true + + it "should log the request", -> + @logger.log + .calledWith(project_id: @project_id, "deleting project via http") + .should.equal true + + it "should time the request", -> + @Metrics.Timer::done.called.should.equal true + + describe "when an errors occurs", -> + beforeEach -> + @ProjectManager.flushAndDeleteProjectWithLocks = sinon.stub().callsArgWith(1, new Error("oops")) + @HttpController.deleteProject(@req, @res, @next) + + it "should call next with the error", -> + @next + .calledWith(new Error("oops")) + .should.equal true + + + + + diff --git a/services/document-updater/test/unit/coffee/HttpController/flushAndDeleteDocTests.coffee b/services/document-updater/test/unit/coffee/HttpController/flushAndDeleteDocTests.coffee new file mode 100644 index 0000000000..f586b6c4f8 --- /dev/null +++ b/services/document-updater/test/unit/coffee/HttpController/flushAndDeleteDocTests.coffee @@ -0,0 +1,64 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/HttpController.js" +SandboxedModule = require('sandboxed-module') +Errors = require "../../../../app/js/Errors.js" + +describe "HttpController - flushAndDeleteDoc", -> + beforeEach -> + @HttpController = SandboxedModule.require modulePath, requires: + "./DocumentManager": @DocumentManager = {} + "./ProjectManager":{} + "logger-sharelatex" : @logger = { log: sinon.stub() } + "./Metrics": @Metrics = {} + + @Metrics.Timer = class Timer + done: sinon.stub() + + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @res = + send: sinon.stub() + @req = + params: + project_id: @project_id + doc_id: @doc_id + @next = sinon.stub() + + describe "successfully", -> + beforeEach -> + @DocumentManager.flushAndDeleteDocWithLock = sinon.stub().callsArgWith(2) + @HttpController.flushAndDeleteDoc(@req, @res, @next) + + it "should flush and delete the doc", -> + @DocumentManager.flushAndDeleteDocWithLock + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should return a successful No Content response", -> + @res.send + .calledWith(204) + .should.equal true + + it "should log the request", -> + @logger.log + .calledWith(doc_id: @doc_id, project_id: @project_id, "deleting doc via http") + .should.equal true + + it "should time the request", -> + @Metrics.Timer::done.called.should.equal true + + describe "when an errors occurs", -> + beforeEach -> + @DocumentManager.flushAndDeleteDocWithLock = sinon.stub().callsArgWith(2, new Error("oops")) + @HttpController.flushAndDeleteDoc(@req, @res, @next) + + it "should call next with the error", -> + @next + .calledWith(new Error("oops")) + .should.equal true + + + + diff --git a/services/document-updater/test/unit/coffee/HttpController/flushDocIfLoadedTests.coffee b/services/document-updater/test/unit/coffee/HttpController/flushDocIfLoadedTests.coffee new file mode 100644 index 0000000000..69c0137676 --- /dev/null +++ b/services/document-updater/test/unit/coffee/HttpController/flushDocIfLoadedTests.coffee @@ -0,0 +1,65 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/HttpController.js" +SandboxedModule = require('sandboxed-module') +Errors = require "../../../../app/js/Errors.js" + +describe "HttpController - flushDocIfLoaded", -> + beforeEach -> + @HttpController = SandboxedModule.require modulePath, requires: + "./DocumentManager": @DocumentManager = {} + "./ProjectManager": {} + "logger-sharelatex" : @logger = { log: sinon.stub() } + "./Metrics": @Metrics = {} + + @Metrics.Timer = class Timer + done: sinon.stub() + + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @lines = ["one", "two", "three"] + @version = 42 + @res = + send: sinon.stub() + @req = + params: + project_id: @project_id + doc_id: @doc_id + @next = sinon.stub() + + describe "successfully", -> + beforeEach -> + @DocumentManager.flushDocIfLoadedWithLock = sinon.stub().callsArgWith(2) + @HttpController.flushDocIfLoaded(@req, @res, @next) + + it "should flush the doc", -> + @DocumentManager.flushDocIfLoadedWithLock + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should return a successful No Content response", -> + @res.send + .calledWith(204) + .should.equal true + + it "should log the request", -> + @logger.log + .calledWith(doc_id: @doc_id, project_id: @project_id, "flushing doc via http") + .should.equal true + + it "should time the request", -> + @Metrics.Timer::done.called.should.equal true + + describe "when an errors occurs", -> + beforeEach -> + @DocumentManager.flushDocIfLoadedWithLock = sinon.stub().callsArgWith(2, new Error("oops")) + @HttpController.flushDocIfLoaded(@req, @res, @next) + + it "should call next with the error", -> + @next + .calledWith(new Error("oops")) + .should.equal true + + + diff --git a/services/document-updater/test/unit/coffee/HttpController/flushProjectTests.coffee b/services/document-updater/test/unit/coffee/HttpController/flushProjectTests.coffee new file mode 100644 index 0000000000..5175cd4280 --- /dev/null +++ b/services/document-updater/test/unit/coffee/HttpController/flushProjectTests.coffee @@ -0,0 +1,62 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/HttpController.js" +SandboxedModule = require('sandboxed-module') +Errors = require "../../../../app/js/Errors.js" + +describe "HttpController - flushProject", -> + beforeEach -> + @HttpController = SandboxedModule.require modulePath, requires: + "./DocumentManager": @DocumentManager = {} + "./ProjectManager": @ProjectManager = {} + "logger-sharelatex" : @logger = { log: sinon.stub() } + "./Metrics": @Metrics = {} + + @Metrics.Timer = class Timer + done: sinon.stub() + + @project_id = "project-id-123" + @res = + send: sinon.stub() + @req = + params: + project_id: @project_id + @next = sinon.stub() + + describe "successfully", -> + beforeEach -> + @ProjectManager.flushProjectWithLocks = sinon.stub().callsArgWith(1) + @HttpController.flushProject(@req, @res, @next) + + it "should flush the project", -> + @ProjectManager.flushProjectWithLocks + .calledWith(@project_id) + .should.equal true + + it "should return a successful No Content response", -> + @res.send + .calledWith(204) + .should.equal true + + it "should log the request", -> + @logger.log + .calledWith(project_id: @project_id, "flushing project via http") + .should.equal true + + it "should time the request", -> + @Metrics.Timer::done.called.should.equal true + + describe "when an errors occurs", -> + beforeEach -> + @ProjectManager.flushProjectWithLocks = sinon.stub().callsArgWith(1, new Error("oops")) + @HttpController.flushProject(@req, @res, @next) + + it "should call next with the error", -> + @next + .calledWith(new Error("oops")) + .should.equal true + + + + diff --git a/services/document-updater/test/unit/coffee/HttpController/getDocTests.coffee b/services/document-updater/test/unit/coffee/HttpController/getDocTests.coffee new file mode 100644 index 0000000000..4ec493bc4b --- /dev/null +++ b/services/document-updater/test/unit/coffee/HttpController/getDocTests.coffee @@ -0,0 +1,110 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/HttpController.js" +SandboxedModule = require('sandboxed-module') +Errors = require "../../../../app/js/Errors.js" + +describe "HttpController - getDoc", -> + beforeEach -> + @HttpController = SandboxedModule.require modulePath, requires: + "./DocumentManager": @DocumentManager = {} + "./ProjectManager": {} + "logger-sharelatex" : @logger = { log: sinon.stub() } + "./Metrics": @Metrics = {} + + @Metrics.Timer = class Timer + done: sinon.stub() + + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @lines = ["one", "two", "three"] + @ops = ["mock-op-1", "mock-op-2"] + @version = 42 + @fromVersion = 42 + @res = + send: sinon.stub() + @req = + params: + project_id: @project_id + doc_id: @doc_id + @next = sinon.stub() + + describe "when the document exists and no recent ops are requested", -> + beforeEach -> + @DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, @lines, @version, []) + @HttpController.getDoc(@req, @res, @next) + + it "should get the doc", -> + @DocumentManager.getDocAndRecentOpsWithLock + .calledWith(@project_id, @doc_id, -1) + .should.equal true + + it "should return the doc as JSON", -> + @res.send + .calledWith(JSON.stringify({ + id: @doc_id + lines: @lines + version: @version + ops: [] + })) + .should.equal true + + it "should log the request", -> + @logger.log + .calledWith(doc_id: @doc_id, project_id: @project_id, "getting doc via http") + .should.equal true + + it "should time the request", -> + @Metrics.Timer::done.called.should.equal true + + describe "when recent ops are requested", -> + beforeEach -> + @DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, @lines, @version, @ops) + @req.query = fromVersion: "#{@fromVersion}" + @HttpController.getDoc(@req, @res, @next) + + it "should get the doc", -> + @DocumentManager.getDocAndRecentOpsWithLock + .calledWith(@project_id, @doc_id, @fromVersion) + .should.equal true + + it "should return the doc as JSON", -> + @res.send + .calledWith(JSON.stringify({ + id: @doc_id + lines: @lines + version: @version + ops: @ops + })) + .should.equal true + + it "should log the request", -> + @logger.log + .calledWith(doc_id: @doc_id, project_id: @project_id, "getting doc via http") + .should.equal true + + it "should time the request", -> + @Metrics.Timer::done.called.should.equal true + + describe "when the document does not exist", -> + beforeEach -> + @DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, null, null) + @HttpController.getDoc(@req, @res, @next) + + it "should call next with NotFoundError", -> + @next + .calledWith(new Errors.NotFoundError("not found")) + .should.equal true + + describe "when an errors occurs", -> + beforeEach -> + @DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, new Error("oops"), null, null) + @HttpController.getDoc(@req, @res, @next) + + it "should call next with the error", -> + @next + .calledWith(new Error("oops")) + .should.equal true + + diff --git a/services/document-updater/test/unit/coffee/HttpController/setDocTests.coffee b/services/document-updater/test/unit/coffee/HttpController/setDocTests.coffee new file mode 100644 index 0000000000..2c3924c030 --- /dev/null +++ b/services/document-updater/test/unit/coffee/HttpController/setDocTests.coffee @@ -0,0 +1,67 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/HttpController.js" +SandboxedModule = require('sandboxed-module') +Errors = require "../../../../app/js/Errors.js" + +describe "HttpController - setDoc", -> + beforeEach -> + @HttpController = SandboxedModule.require modulePath, requires: + "./DocumentManager": @DocumentManager = {} + "./ProjectManager": {} + "logger-sharelatex" : @logger = { log: sinon.stub() } + "./Metrics": @Metrics = {} + + @Metrics.Timer = class Timer + done: sinon.stub() + + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @lines = ["one", "two", "three"] + @res = + send: sinon.stub() + @req = + params: + project_id: @project_id + doc_id: @doc_id + body: + lines: @lines + @next = sinon.stub() + + describe "successfully", -> + beforeEach -> + @DocumentManager.setDocWithLock = sinon.stub().callsArgWith(3) + @HttpController.setDoc(@req, @res, @next) + + it "should set the doc", -> + @DocumentManager.setDocWithLock + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should return a successful No Content response", -> + @res.send + .calledWith(204) + .should.equal true + + it "should log the request", -> + @logger.log + .calledWith(doc_id: @doc_id, project_id: @project_id, lines: @lines, "setting doc via http") + .should.equal true + + it "should time the request", -> + @Metrics.Timer::done.called.should.equal true + + describe "when an errors occurs", -> + beforeEach -> + @DocumentManager.setDocWithLock = sinon.stub().callsArgWith(3, new Error("oops")) + @HttpController.setDoc(@req, @res, @next) + + it "should call next with the error", -> + @next + .calledWith(new Error("oops")) + .should.equal true + + + + diff --git a/services/document-updater/test/unit/coffee/LockManager/CheckingTheLock.coffee b/services/document-updater/test/unit/coffee/LockManager/CheckingTheLock.coffee new file mode 100644 index 0000000000..ac72cbae93 --- /dev/null +++ b/services/document-updater/test/unit/coffee/LockManager/CheckingTheLock.coffee @@ -0,0 +1,50 @@ +require('coffee-script') +sinon = require('sinon') +assert = require('assert') +path = require('path') +modulePath = path.join __dirname, '../../../../app/js/LockManager.js' +keys = require(path.join __dirname, '../../../../app/js/RedisKeyBuilder.js') +project_id = 1234 +doc_id = 5678 +blockingKey = "Blocking:#{doc_id}" +loadModule = require('../module-loader').loadModule + +describe 'Lock Manager - checking the lock', ()-> + + existsStub = sinon.stub() + setStub = sinon.stub() + exireStub = sinon.stub() + execStub = sinon.stub() + + mocks = + "logger-sharelatex": log:-> + + redis: + createClient : ()-> + auth:-> + multi: -> + exists: existsStub + expire: exireStub + set: setStub + exec: execStub + LockManager = loadModule(modulePath, mocks).module.exports + + it 'should check if lock exists but not set or expire', (done)-> + execStub.callsArgWith(0, null, ["1"]) + LockManager.checkLock doc_id, (err, docIsLocked)-> + existsStub.calledWith(blockingKey).should.equal true + setStub.called.should.equal false + exireStub.called.should.equal false + done() + + it 'should return true if the key does not exists', (done)-> + execStub.callsArgWith(0, null, "0") + LockManager.checkLock doc_id, (err, free)-> + free.should.equal true + done() + + it 'should return false if the key does exists', (done)-> + execStub.callsArgWith(0, null, "1") + LockManager.checkLock doc_id, (err, free)-> + free.should.equal false + done() diff --git a/services/document-updater/test/unit/coffee/LockManager/ReleasingTheLock.coffee b/services/document-updater/test/unit/coffee/LockManager/ReleasingTheLock.coffee new file mode 100644 index 0000000000..81f42a3f59 --- /dev/null +++ b/services/document-updater/test/unit/coffee/LockManager/ReleasingTheLock.coffee @@ -0,0 +1,28 @@ +require('coffee-script') +sinon = require('sinon') +assert = require('assert') +path = require('path') +modulePath = path.join __dirname, '../../../../app/js/LockManager.js' +keys = require(path.join __dirname, '../../../../app/js/RedisKeyBuilder.js') +project_id = 1234 +doc_id = 5678 +loadModule = require('../module-loader').loadModule + +describe 'LockManager - releasing the lock', ()-> + + deleteStub = sinon.stub().callsArgWith(1) + mocks = + "logger-sharelatex": log:-> + + redis: + createClient : ()-> + auth:-> + del:deleteStub + + LockManager = loadModule(modulePath, mocks).module.exports + + it 'should put a all data into memory', (done)-> + LockManager.releaseLock doc_id, -> + deleteStub.calledWith("Blocking:#{doc_id}").should.equal true + done() + diff --git a/services/document-updater/test/unit/coffee/LockManager/getLockTests.coffee b/services/document-updater/test/unit/coffee/LockManager/getLockTests.coffee new file mode 100644 index 0000000000..f378650d95 --- /dev/null +++ b/services/document-updater/test/unit/coffee/LockManager/getLockTests.coffee @@ -0,0 +1,69 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/LockManager.js" +SandboxedModule = require('sandboxed-module') + +describe 'LockManager - getting the lock', -> + beforeEach -> + @LockManager = SandboxedModule.require modulePath, requires: + "logger-sharelatex": log:-> + redis: + createClient : () => + auth:-> + @callback = sinon.stub() + @doc_id = "doc-id-123" + + describe "when the lock is not set", -> + beforeEach (done) -> + @LockManager.tryLock = sinon.stub().callsArgWith(1, null, true) + @LockManager.getLock @doc_id, (args...) => + @callback(args...) + done() + + it "should try to get the lock", -> + @LockManager.tryLock + .calledWith(@doc_id) + .should.equal true + + it "should only need to try once", -> + @LockManager.tryLock.callCount.should.equal 1 + + it "should return the callback", -> + @callback.calledWith(null).should.equal true + + describe "when the lock is initially set", -> + beforeEach (done) -> + startTime = Date.now() + @LockManager.LOCK_TEST_INTERVAL = 5 + @LockManager.tryLock = (doc_id, callback = (error, isFree) ->) -> + if Date.now() - startTime < 20 + callback null, false + else + callback null, true + sinon.spy @LockManager, "tryLock" + + @LockManager.getLock @doc_id, (args...) => + @callback(args...) + done() + + it "should call tryLock multiple times until free", -> + (@LockManager.tryLock.callCount > 1).should.equal true + + it "should return the callback", -> + @callback.calledWith(null).should.equal true + + describe "when the lock times out", -> + beforeEach (done) -> + time = Date.now() + @LockManager.MAX_LOCK_WAIT_TIME = 5 + @LockManager.tryLock = sinon.stub().callsArgWith(1, null, false) + @LockManager.getLock @doc_id, (args...) => + @callback(args...) + done() + + it "should return the callback with an error", -> + @callback.calledWith(new Error("timeout")).should.equal true + + + diff --git a/services/document-updater/test/unit/coffee/LockManager/tryLockTests.coffee b/services/document-updater/test/unit/coffee/LockManager/tryLockTests.coffee new file mode 100644 index 0000000000..cff2b9538b --- /dev/null +++ b/services/document-updater/test/unit/coffee/LockManager/tryLockTests.coffee @@ -0,0 +1,37 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/LockManager.js" +SandboxedModule = require('sandboxed-module') + +describe 'LockManager - trying the lock', -> + beforeEach -> + @LockManager = SandboxedModule.require modulePath, requires: + "logger-sharelatex": log:-> + redis: + createClient : () => + auth:-> + set: @set = sinon.stub() + @callback = sinon.stub() + @doc_id = "doc-id-123" + + describe "when the lock is not set", -> + beforeEach -> + @set.callsArgWith(5, null, "OK") + @LockManager.tryLock @doc_id, @callback + + it "should set the lock key with an expiry if it is not set", -> + @set.calledWith("Blocking:#{@doc_id}", "locked", "EX", 10, "NX") + .should.equal true + + it "should return the callback with true", -> + @callback.calledWith(null, true).should.equal true + + describe "when the lock is already set", -> + beforeEach -> + @set.callsArgWith(5, null, null) + @LockManager.tryLock @doc_id, @callback + + it "should return the callback with false", -> + @callback.calledWith(null, false).should.equal true + diff --git a/services/document-updater/test/unit/coffee/PersistenceManager/getDocTests.coffee b/services/document-updater/test/unit/coffee/PersistenceManager/getDocTests.coffee new file mode 100644 index 0000000000..c5cfc35ac8 --- /dev/null +++ b/services/document-updater/test/unit/coffee/PersistenceManager/getDocTests.coffee @@ -0,0 +1,85 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/PersistenceManager.js" +SandboxedModule = require('sandboxed-module') +Errors = require "../../../../app/js/Errors" + +describe "PersistenceManager.getDoc", -> + beforeEach -> + @PersistenceManager = SandboxedModule.require modulePath, requires: + "request": @request = sinon.stub() + "settings-sharelatex": @Settings = {} + "./Metrics": @Metrics = + Timer: class Timer + done: sinon.stub() + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @lines = ["one", "two", "three"] + @callback = sinon.stub() + @Settings.apis = + web: + url: @url = "www.example.com" + user: @user = "sharelatex" + pass: @pass = "password" + + describe "with a successful response from the web api", -> + beforeEach -> + @request.callsArgWith(1, null, {statusCode: 200}, JSON.stringify(lines: @lines)) + @PersistenceManager.getDoc(@project_id, @doc_id, @callback) + + it "should call the web api", -> + @request + .calledWith({ + url: "#{@url}/project/#{@project_id}/doc/#{@doc_id}" + method: "GET" + headers: + "accept": "application/json" + auth: + user: @user + pass: @pass + sendImmediately: true + jar: false + }) + .should.equal true + + it "should call the callback with the doc lines", -> + @callback.calledWith(null, @lines).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when request returns an error", -> + beforeEach -> + @request.callsArgWith(1, @error = new Error("oops"), null, null) + @PersistenceManager.getDoc(@project_id, @doc_id, @callback) + + it "should return the error", -> + @callback.calledWith(@error).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when the request returns 404", -> + beforeEach -> + @request.callsArgWith(1, null, {statusCode: 404}, "") + @PersistenceManager.getDoc(@project_id, @doc_id, @callback) + + it "should return a NotFoundError", -> + @callback.calledWith(new Errors.NotFoundError("not found")).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when the request returns an error status code", -> + beforeEach -> + @request.callsArgWith(1, null, {statusCode: 500}, "") + @PersistenceManager.getDoc(@project_id, @doc_id, @callback) + + it "should return an error", -> + @callback.calledWith(new Error("web api error")).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + diff --git a/services/document-updater/test/unit/coffee/PersistenceManager/setDocTests.coffee b/services/document-updater/test/unit/coffee/PersistenceManager/setDocTests.coffee new file mode 100644 index 0000000000..cd9d962d3b --- /dev/null +++ b/services/document-updater/test/unit/coffee/PersistenceManager/setDocTests.coffee @@ -0,0 +1,86 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/PersistenceManager.js" +SandboxedModule = require('sandboxed-module') +Errors = require "../../../../app/js/Errors" + +describe "PersistenceManager.setDoc", -> + beforeEach -> + @PersistenceManager = SandboxedModule.require modulePath, requires: + "request": @request = sinon.stub() + "settings-sharelatex": @Settings = {} + "./Metrics": @Metrics = + Timer: class Timer + done: sinon.stub() + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @lines = ["one", "two", "three"] + @callback = sinon.stub() + @Settings.apis = + web: + url: @url = "www.example.com" + user: @user = "sharelatex" + pass: @pass = "password" + + describe "with a successful response from the web api", -> + beforeEach -> + @request.callsArgWith(1, null, {statusCode: 200}, JSON.stringify(lines: @lines)) + @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @callback) + + it "should call the web api", -> + @request + .calledWith({ + url: "#{@url}/project/#{@project_id}/doc/#{@doc_id}" + body: JSON.stringify + lines: @lines + method: "POST" + headers: + "content-type": "application/json" + auth: + user: @user + pass: @pass + sendImmediately: true + jar: false + }) + .should.equal true + + it "should call the callback without error", -> + @callback.calledWith(null).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when request returns an error", -> + beforeEach -> + @request.callsArgWith(1, @error = new Error("oops"), null, null) + @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @callback) + + it "should return the error", -> + @callback.calledWith(@error).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when the request returns 404", -> + beforeEach -> + @request.callsArgWith(1, null, {statusCode: 404}, "") + @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @callback) + + it "should return a NotFoundError", -> + @callback.calledWith(new Errors.NotFoundError("not found")).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when the request returns an error status code", -> + beforeEach -> + @request.callsArgWith(1, null, {statusCode: 500}, "") + @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @callback) + + it "should return an error", -> + @callback.calledWith(new Error("web api error")).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + diff --git a/services/document-updater/test/unit/coffee/ProjectManager/flushAndDeleteProjectTests.coffee b/services/document-updater/test/unit/coffee/ProjectManager/flushAndDeleteProjectTests.coffee new file mode 100644 index 0000000000..fc2ea998f6 --- /dev/null +++ b/services/document-updater/test/unit/coffee/ProjectManager/flushAndDeleteProjectTests.coffee @@ -0,0 +1,75 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/ProjectManager.js" +SandboxedModule = require('sandboxed-module') + +describe "ProjectManager - flushAndDeleteProject", -> + beforeEach -> + @ProjectManager = SandboxedModule.require modulePath, requires: + "./RedisManager": @RedisManager = {} + "./DocumentManager": @DocumentManager = {} + "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } + "./Metrics": @Metrics = + Timer: class Timer + done: sinon.stub() + @project_id = "project-id-123" + @callback = sinon.stub() + + describe "successfully", -> + beforeEach (done) -> + @doc_ids = ["doc-id-1", "doc-id-2", "doc-id-3"] + @RedisManager.getDocIdsInProject = sinon.stub().callsArgWith(1, null, @doc_ids) + @DocumentManager.flushAndDeleteDocWithLock = sinon.stub().callsArg(2) + @ProjectManager.flushAndDeleteProjectWithLocks @project_id, (error) => + @callback(error) + done() + + it "should get the doc ids in the project", -> + @RedisManager.getDocIdsInProject + .calledWith(@project_id) + .should.equal true + + it "should delete each doc in the project", -> + for doc_id in @doc_ids + @DocumentManager.flushAndDeleteDocWithLock + .calledWith(@project_id, doc_id) + .should.equal true + + it "should call the callback without error", -> + @callback.calledWith(null).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when a doc errors", -> + beforeEach (done) -> + @doc_ids = ["doc-id-1", "doc-id-2", "doc-id-3"] + @RedisManager.getDocIdsInProject = sinon.stub().callsArgWith(1, null, @doc_ids) + @DocumentManager.flushAndDeleteDocWithLock = sinon.spy (project_id, doc_id, callback = (error) ->) => + if doc_id == "doc-id-1" + callback(@error = new Error("oops, something went wrong")) + else + callback() + @ProjectManager.flushAndDeleteProjectWithLocks @project_id, (error) => + @callback(error) + done() + + it "should still flush each doc in the project", -> + for doc_id in @doc_ids + @DocumentManager.flushAndDeleteDocWithLock + .calledWith(@project_id, doc_id) + .should.equal true + + it "should record the error", -> + @logger.error + .calledWith(err: @error, project_id: @project_id, doc_id: "doc-id-1", "error deleting doc") + .should.equal true + + it "should call the callback with an error", -> + @callback.calledWith(new Error()).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + diff --git a/services/document-updater/test/unit/coffee/ProjectManager/flushProjectTests.coffee b/services/document-updater/test/unit/coffee/ProjectManager/flushProjectTests.coffee new file mode 100644 index 0000000000..301740c015 --- /dev/null +++ b/services/document-updater/test/unit/coffee/ProjectManager/flushProjectTests.coffee @@ -0,0 +1,75 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/ProjectManager.js" +SandboxedModule = require('sandboxed-module') + +describe "ProjectManager - flushProject", -> + beforeEach -> + @ProjectManager = SandboxedModule.require modulePath, requires: + "./RedisManager": @RedisManager = {} + "./DocumentManager": @DocumentManager = {} + "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } + "./Metrics": @Metrics = + Timer: class Timer + done: sinon.stub() + @project_id = "project-id-123" + @callback = sinon.stub() + + describe "successfully", -> + beforeEach (done) -> + @doc_ids = ["doc-id-1", "doc-id-2", "doc-id-3"] + @RedisManager.getDocIdsInProject = sinon.stub().callsArgWith(1, null, @doc_ids) + @DocumentManager.flushDocIfLoadedWithLock = sinon.stub().callsArg(2) + @ProjectManager.flushProjectWithLocks @project_id, (error) => + @callback(error) + done() + + it "should get the doc ids in the project", -> + @RedisManager.getDocIdsInProject + .calledWith(@project_id) + .should.equal true + + it "should flush each doc in the project", -> + for doc_id in @doc_ids + @DocumentManager.flushDocIfLoadedWithLock + .calledWith(@project_id, doc_id) + .should.equal true + + it "should call the callback without error", -> + @callback.calledWith(null).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when a doc errors", -> + beforeEach (done) -> + @doc_ids = ["doc-id-1", "doc-id-2", "doc-id-3"] + @RedisManager.getDocIdsInProject = sinon.stub().callsArgWith(1, null, @doc_ids) + @DocumentManager.flushDocIfLoadedWithLock = sinon.spy (project_id, doc_id, callback = (error) ->) => + if doc_id == "doc-id-1" + callback(@error = new Error("oops, something went wrong")) + else + callback() + @ProjectManager.flushProjectWithLocks @project_id, (error) => + @callback(error) + done() + + it "should still flush each doc in the project", -> + for doc_id in @doc_ids + @DocumentManager.flushDocIfLoadedWithLock + .calledWith(@project_id, doc_id) + .should.equal true + + it "should record the error", -> + @logger.error + .calledWith(err: @error, project_id: @project_id, doc_id: "doc-id-1", "error flushing doc") + .should.equal true + + it "should call the callback with an error", -> + @callback.calledWith(new Error()).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + diff --git a/services/document-updater/test/unit/coffee/RedisManager/clearDocFromPendingUpdatesSetTests.coffee b/services/document-updater/test/unit/coffee/RedisManager/clearDocFromPendingUpdatesSetTests.coffee new file mode 100644 index 0000000000..676d454167 --- /dev/null +++ b/services/document-updater/test/unit/coffee/RedisManager/clearDocFromPendingUpdatesSetTests.coffee @@ -0,0 +1,27 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/RedisManager" +SandboxedModule = require('sandboxed-module') + +describe "RedisManager.clearDocFromPendingUpdatesSet", -> + beforeEach -> + @project_id = "project-id" + @doc_id = "document-id" + @callback = sinon.stub() + @RedisManager = SandboxedModule.require modulePath, requires: + "redis" : createClient: () => + @rclient = auth:-> + + @rclient.srem = sinon.stub().callsArg(2) + @RedisManager.clearDocFromPendingUpdatesSet(@project_id, @doc_id, @callback) + + it "should get the docs with pending updates", -> + @rclient.srem + .calledWith("DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}") + .should.equal true + + it "should return the callback", -> + @callback.called.should.equal true + + diff --git a/services/document-updater/test/unit/coffee/RedisManager/getDocsWithPendingUpdatesTests.coffee b/services/document-updater/test/unit/coffee/RedisManager/getDocsWithPendingUpdatesTests.coffee new file mode 100644 index 0000000000..602197ad57 --- /dev/null +++ b/services/document-updater/test/unit/coffee/RedisManager/getDocsWithPendingUpdatesTests.coffee @@ -0,0 +1,33 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/RedisManager" +SandboxedModule = require('sandboxed-module') + +describe "RedisManager.getDocsWithPendingUpdates", -> + beforeEach -> + @callback = sinon.stub() + @RedisManager = SandboxedModule.require modulePath, requires: + "redis" : createClient: () => + @rclient = auth:-> + + @docs = [{ + doc_id: "doc-id-1" + project_id: "project-id-1" + }, { + doc_id: "doc-id-2" + project_id: "project-id-2" + }] + @doc_keys = @docs.map (doc) -> "#{doc.project_id}:#{doc.doc_id}" + + @rclient.smembers = sinon.stub().callsArgWith(1, null, @doc_keys) + @RedisManager.getDocsWithPendingUpdates(@callback) + + it "should get the docs with pending updates", -> + @rclient.smembers + .calledWith("DocsWithPendingUpdates") + .should.equal true + + it "should return the docs with pending updates", -> + @callback.calledWith(null, @docs).should.equal true + diff --git a/services/document-updater/test/unit/coffee/RedisManager/getPendingUpdatesForDocTests.coffee b/services/document-updater/test/unit/coffee/RedisManager/getPendingUpdatesForDocTests.coffee new file mode 100644 index 0000000000..4910f1498f --- /dev/null +++ b/services/document-updater/test/unit/coffee/RedisManager/getPendingUpdatesForDocTests.coffee @@ -0,0 +1,56 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/RedisManager.js" +SandboxedModule = require('sandboxed-module') + +describe "RedisManager.getPendingUpdatesForDoc", -> + beforeEach -> + @RedisManager = SandboxedModule.require modulePath, requires: + "redis": createClient: () => + @rclient = + auth: () -> + multi: () => @rclient + "logger-sharelatex": @logger = {log: sinon.stub()} + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @callback = sinon.stub() + @rclient.lrange = sinon.stub() + @rclient.del = sinon.stub() + + describe "successfully", -> + beforeEach -> + @updates = [ + { op: [{ i: "foo", p: 4 }] } + { op: [{ i: "foo", p: 4 }] } + ] + @jsonUpdates = @updates.map (update) -> JSON.stringify update + @rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonUpdates]) + @RedisManager.getPendingUpdatesForDoc @doc_id, @callback + + it "should get the pending updates", -> + @rclient.lrange + .calledWith("PendingUpdates:#{@doc_id}", 0, -1) + .should.equal true + + it "should delete the pending updates", -> + @rclient.del + .calledWith("PendingUpdates:#{@doc_id}") + .should.equal true + + it "should call the callback with the updates", -> + @callback.calledWith(null, @updates).should.equal true + + describe "when the JSON doesn't parse", -> + beforeEach -> + @jsonUpdates = [ + JSON.stringify { op: [{ i: "foo", p: 4 }] } + "broken json" + ] + @rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonUpdates]) + @RedisManager.getPendingUpdatesForDoc @doc_id, @callback + + it "should return an error to the callback", -> + @callback.calledWith(new Error("JSON parse error")).should.equal true + + diff --git a/services/document-updater/test/unit/coffee/RedisManager/getPreviousDocOpsTests.coffee b/services/document-updater/test/unit/coffee/RedisManager/getPreviousDocOpsTests.coffee new file mode 100644 index 0000000000..775418313e --- /dev/null +++ b/services/document-updater/test/unit/coffee/RedisManager/getPreviousDocOpsTests.coffee @@ -0,0 +1,99 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/RedisManager" +SandboxedModule = require('sandboxed-module') + +describe "RedisManager.getPreviousDocOpsTests", -> + beforeEach -> + @callback = sinon.stub() + @RedisManager = SandboxedModule.require modulePath, requires: + "redis" : createClient: () => + @rclient = + auth: -> + multi: => @rclient + "logger-sharelatex": @logger = { error: sinon.stub(), log: sinon.stub() } + @doc_id = "doc-id-123" + + describe "with a start and an end value", -> + beforeEach -> + @first_version_in_redis = 30 + @version = 70 + @length = @version - @first_version_in_redis + @start = 50 + @end = 60 + @ops = [ + { "mock": "op-1" }, + { "mock": "op-2" } + ] + @jsonOps = @ops.map (op) -> JSON.stringify op + @rclient.llen = sinon.stub().callsArgWith(1, null, @length) + @rclient.get = sinon.stub().callsArgWith(1, null, @version.toString()) + @rclient.lrange = sinon.stub().callsArgWith(3, null, @jsonOps) + @RedisManager.getPreviousDocOps(@doc_id, @start, @end, @callback) + + it "should get the length of the existing doc ops", -> + @rclient.llen + .calledWith("DocOps:#{@doc_id}") + .should.equal true + + it "should get the current version of the doc", -> + @rclient.get + .calledWith("DocVersion:#{@doc_id}") + .should.equal true + + it "should get the appropriate docs ops", -> + @rclient.lrange + .calledWith("DocOps:#{@doc_id}", @start - @first_version_in_redis, @end - @first_version_in_redis) + .should.equal true + + it "should return the docs with the doc ops deserialized", -> + @callback.calledWith(null, @ops).should.equal true + + describe "with an end value of -1", -> + beforeEach -> + @first_version_in_redis = 30 + @version = 70 + @length = @version - @first_version_in_redis + @start = 50 + @end = -1 + @ops = [ + { "mock": "op-1" }, + { "mock": "op-2" } + ] + @jsonOps = @ops.map (op) -> JSON.stringify op + @rclient.llen = sinon.stub().callsArgWith(1, null, @length) + @rclient.get = sinon.stub().callsArgWith(1, null, @version.toString()) + @rclient.lrange = sinon.stub().callsArgWith(3, null, @jsonOps) + @RedisManager.getPreviousDocOps(@doc_id, @start, @end, @callback) + + it "should get the appropriate docs ops to the end of list", -> + @rclient.lrange + .calledWith("DocOps:#{@doc_id}", @start - @first_version_in_redis, -1) + .should.equal true + + it "should return the docs with the doc ops deserialized", -> + @callback.calledWith(null, @ops).should.equal true + + describe "when the requested range is not in Redis", -> + beforeEach -> + @first_version_in_redis = 30 + @version = 70 + @length = @version - @first_version_in_redis + @start = 20 + @end = -1 + @ops = [ + { "mock": "op-1" }, + { "mock": "op-2" } + ] + @jsonOps = @ops.map (op) -> JSON.stringify op + @rclient.llen = sinon.stub().callsArgWith(1, null, @length) + @rclient.get = sinon.stub().callsArgWith(1, null, @version.toString()) + @rclient.lrange = sinon.stub().callsArgWith(3, null, @jsonOps) + @RedisManager.getPreviousDocOps(@doc_id, @start, @end, @callback) + + it "should return an error", -> + @callback.calledWith(new Error("range is not loaded in redis")).should.equal true + + it "should log out the problem", -> + @logger.error.called.should.equal true diff --git a/services/document-updater/test/unit/coffee/RedisManager/prependDocOpsTests.coffee b/services/document-updater/test/unit/coffee/RedisManager/prependDocOpsTests.coffee new file mode 100644 index 0000000000..b4a8192d12 --- /dev/null +++ b/services/document-updater/test/unit/coffee/RedisManager/prependDocOpsTests.coffee @@ -0,0 +1,32 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/RedisManager" +SandboxedModule = require('sandboxed-module') + +describe "RedisManager.clearDocFromPendingUpdatesSet", -> + beforeEach -> + @doc_id = "document-id" + @callback = sinon.stub() + @RedisManager = SandboxedModule.require modulePath, requires: + "redis" : createClient: () => + @rclient = auth:-> + + @rclient.lpush = sinon.stub().callsArg(2) + @ops = [ + { "mock" : "op-1" }, + { "mock" : "op-2" } + ] + @reversedJsonOps = @ops.map((op) -> JSON.stringify op).reverse() + @RedisManager.prependDocOps(@doc_id, @ops, @callback) + + it "should push the reversed JSONed ops", -> + @rclient.lpush + .calledWith("DocOps:#{@doc_id}", @reversedJsonOps) + .should.equal true + + it "should return the callback", -> + @callback.called.should.equal true + + + diff --git a/services/document-updater/test/unit/coffee/RedisManager/pushDocOpTests.coffee b/services/document-updater/test/unit/coffee/RedisManager/pushDocOpTests.coffee new file mode 100644 index 0000000000..0c76730437 --- /dev/null +++ b/services/document-updater/test/unit/coffee/RedisManager/pushDocOpTests.coffee @@ -0,0 +1,37 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/RedisManager" +SandboxedModule = require('sandboxed-module') + +describe "RedisManager.getPreviousDocOpsTests", -> + beforeEach -> + @callback = sinon.stub() + @RedisManager = SandboxedModule.require modulePath, requires: + "redis" : createClient: () => + @rclient = + auth: -> + multi: => @rclient + @doc_id = "doc-id-123" + + beforeEach -> + @version = 70 + @op = + { "mock": "op-1" } + @jsonOp = JSON.stringify @op + @rclient.rpush = sinon.stub().callsArgWith(2, null) + @rclient.incr = sinon.stub().callsArgWith(1, null, @version.toString()) + @RedisManager.pushDocOp(@doc_id, @op, @callback) + + it "should push the op into redis", -> + @rclient.rpush + .calledWith("DocOps:#{@doc_id}", @jsonOp) + .should.equal true + + it "should increment the version number", -> + @rclient.incr + .calledWith("DocVersion:#{@doc_id}") + .should.equal true + + it "should call the callback with the new version", -> + @callback.calledWith(null, @version).should.equal true diff --git a/services/document-updater/test/unit/coffee/RemovingSingleDocFromMemory.coffee b/services/document-updater/test/unit/coffee/RemovingSingleDocFromMemory.coffee new file mode 100644 index 0000000000..9fd0136aad --- /dev/null +++ b/services/document-updater/test/unit/coffee/RemovingSingleDocFromMemory.coffee @@ -0,0 +1,73 @@ +require('coffee-script') +_ = require("underscore") +assert = require('assert') +sinon = require('sinon') +path = require('path') +modulePath = path.join __dirname, '../../../app/js/RedisManager.js' +keys = require(path.join __dirname, '../../../app/js/RedisKeyBuilder.js') +loadModule = require('./module-loader').loadModule + +describe 'removing single doc from memory', ()-> + + project_id = "12345" + doc_id1 = "docid1" + doc_id2 = "docid2" + doc_id3 = "docid3" + + redisMemory = undefined + redisManager = undefined + self = @ + beforeEach (done)-> + redisMemory = {} + + mocks = + "logger-sharelatex": + error:-> + log:-> + redis: + createClient : -> + auth:-> + multi: -> + get:-> + set:(key, value)-> + redisMemory[key] = value + sadd:(key, value)-> + if !redisMemory[key]? + redisMemory[key] = [] + redisMemory[key].push value + del : (key)-> + delete redisMemory[key] + srem : (key, member)-> + index = redisMemory[key].indexOf(member) + redisMemory[key].splice(index, 1) + exec:(callback)-> + callback(null, []) + + redisManager = loadModule(modulePath, mocks).module.exports + redisManager.putDocInMemory project_id, doc_id1, 0, ["line"], -> + redisManager.putDocInMemory project_id, doc_id2, 0, ["ledf"], -> + redisManager.putDocInMemory project_id, doc_id3, 0, ["ledf"], -> + done() + + it 'should remove doc lines from memory', (done)-> + keyExists = false + redisManager.removeDocFromMemory project_id, doc_id1, ()-> + assert.equal redisMemory[keys.docLines(doc_id:doc_id1)], undefined + keys = _.keys(redisMemory) + containsKey(keys, doc_id1) + keys.forEach (sets)-> + containsKey sets, doc_id1 + _.each redisMemory, (value)-> + if value.indexOf(doc_id1) != -1 + assert.equal false, "#{doc_id1} found in value #{value}" + done() + + +containsKey = (haystack, key)-> + if haystack.forEach? + haystack.forEach (area)-> + if area.indexOf(key) != -1 + assert.equal false, "#{key} found in haystack in #{area}" + + + diff --git a/services/document-updater/test/unit/coffee/ShareJsDB/GetOpsTests.coffee b/services/document-updater/test/unit/coffee/ShareJsDB/GetOpsTests.coffee new file mode 100644 index 0000000000..4812619574 --- /dev/null +++ b/services/document-updater/test/unit/coffee/ShareJsDB/GetOpsTests.coffee @@ -0,0 +1,54 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/ShareJsDB.js" +SandboxedModule = require('sandboxed-module') + +describe "ShareJsDB.getOps", -> + beforeEach -> + @doc_id = "document-id" + @project_id = "project-id" + @doc_key = "#{@project_id}:#{@doc_id}" + @callback = sinon.stub() + @ops = [{p: 20, t: "foo"}] + @redis_ops = (JSON.stringify(op) for op in @ops) + @ShareJsDB = SandboxedModule.require modulePath, requires: + "./RedisManager": @RedisManager = {} + "./DocOpsManager": @DocOpsManager = {} + "./DocumentManager":{} + + describe "with start == end", -> + beforeEach -> + @start = @end = 42 + @ShareJsDB.getOps @doc_key, @start, @end, @callback + + it "should return an empty array", -> + @callback.calledWith(null, []).should.equal true + + describe "with a non empty range", -> + beforeEach -> + @start = 35 + @end = 42 + @DocOpsManager.getPreviousDocOps = sinon.stub().callsArgWith(4, null, @ops) + @ShareJsDB.getOps @doc_key, @start, @end, @callback + + it "should get the range from redis", -> + @DocOpsManager.getPreviousDocOps + .calledWith(@project_id, @doc_id, @start, @end-1) + .should.equal true + + it "should return the ops", -> + @callback.calledWith(null, @ops).should.equal true + + describe "with no specified end", -> + beforeEach -> + @start = 35 + @end = null + @DocOpsManager.getPreviousDocOps = sinon.stub().callsArgWith(4, null, @ops) + @ShareJsDB.getOps @doc_key, @start, @end, @callback + + it "should get until the end of the list", -> + @DocOpsManager.getPreviousDocOps + .calledWith(@project_id, @doc_id, @start, -1) + .should.equal true + diff --git a/services/document-updater/test/unit/coffee/ShareJsDB/GetSnapshotTests.coffee b/services/document-updater/test/unit/coffee/ShareJsDB/GetSnapshotTests.coffee new file mode 100644 index 0000000000..ef433c1f90 --- /dev/null +++ b/services/document-updater/test/unit/coffee/ShareJsDB/GetSnapshotTests.coffee @@ -0,0 +1,85 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +expect = chai.expect +modulePath = "../../../../app/js/ShareJsDB.js" +SandboxedModule = require('sandboxed-module') +Errors = require "../../../../app/js/Errors" + +describe "ShareJsDB.getSnapshot", -> + beforeEach -> + @doc_id = "document-id" + @project_id = "project-id" + @doc_key = "#{@project_id}:#{@doc_id}" + @callback = sinon.stub() + @ShareJsDB = SandboxedModule.require modulePath, requires: + "./DocumentManager": @DocumentManager = {} + "./RedisManager": {} + "./DocOpsManager": {} + + @version = 42 + + describe "with a text document", -> + beforeEach -> + @lines = ["one", "two", "three"] + + describe "successfully", -> + beforeEach -> + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version) + @ShareJsDB.getSnapshot @doc_key, @callback + + it "should get the doc", -> + @DocumentManager.getDoc + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should return the doc lines", -> + @callback.args[0][1].snapshot.should.equal @lines.join("\n") + + it "should return the doc version", -> + @callback.args[0][1].v.should.equal @version + + it "should return the type as text", -> + @callback.args[0][1].type.should.equal "text" + + describe "when the doclines do not exist", -> + beforeEach -> + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, null, null) + @ShareJsDB.getSnapshot @doc_key, @callback + + it "should return the callback with a NotFoundError", -> + @callback.calledWith(new Errors.NotFoundError("not found")).should.equal true + + describe "when getDoc returns an error", -> + beforeEach -> + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, @error = new Error("oops"), null, null) + @ShareJsDB.getSnapshot @doc_key, @callback + + it "should return the callback with an error", -> + @callback.calledWith(@error).should.equal true + + describe "with a JSON document", -> + beforeEach -> + @lines = [{text: "one"}, {text:"two"}, {text:"three"}] + + describe "successfully", -> + beforeEach -> + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version) + @ShareJsDB.getSnapshot @doc_key, @callback + + it "should get the doc", -> + @DocumentManager.getDoc + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should return the doc lines", -> + expect(@callback.args[0][1].snapshot).to.deep.equal lines: @lines + + it "should return the doc version", -> + @callback.args[0][1].v.should.equal @version + + it "should return the type as text", -> + @callback.args[0][1].type.should.equal "json" + + + diff --git a/services/document-updater/test/unit/coffee/ShareJsDB/WriteOpsTests.coffee b/services/document-updater/test/unit/coffee/ShareJsDB/WriteOpsTests.coffee new file mode 100644 index 0000000000..b28f23d2f4 --- /dev/null +++ b/services/document-updater/test/unit/coffee/ShareJsDB/WriteOpsTests.coffee @@ -0,0 +1,53 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/ShareJsDB.js" +SandboxedModule = require('sandboxed-module') + +describe "ShareJsDB.writeOps", -> + beforeEach -> + @project_id = "project-id" + @doc_id = "document-id" + @doc_key = "#{@project_id}:#{@doc_id}" + @callback = sinon.stub() + @opData = + op: {p: 20, t: "foo"} + meta: {source: "bar"} + @ShareJsDB = SandboxedModule.require modulePath, requires: + "./RedisManager": @RedisManager = {} + "./DocOpsManager": @DocOpsManager = {} + "./DocumentManager": {} + + describe "writing an op", -> + beforeEach -> + @version = 42 + @opData.v = @version + @DocOpsManager.pushDocOp = sinon.stub().callsArgWith(3, null, @version+1) + @ShareJsDB.writeOp @doc_key, @opData, @callback + + it "should write the op to redis", -> + op = + op: @opData.op + meta: @opData.meta + @DocOpsManager.pushDocOp + .calledWith(@project_id, @doc_id, op) + .should.equal true + + it "should call the callback without an error", -> + @callback.called.should.equal true + (@callback.args[0][0]?).should.equal false + + describe "writing an op at the wrong version", -> + beforeEach -> + @version = 42 + @mismatch = 5 + @opData.v = @version + @DocOpsManager.pushDocOp = sinon.stub().callsArgWith(3, null, @version + @mismatch) + @ShareJsDB.writeOp @doc_key, @opData, @callback + + it "should call the callback with an error", -> + @callback.calledWith(sinon.match.string).should.equal true + + + + diff --git a/services/document-updater/test/unit/coffee/ShareJsUpdateManagerTests.coffee b/services/document-updater/test/unit/coffee/ShareJsUpdateManagerTests.coffee new file mode 100644 index 0000000000..af5a475836 --- /dev/null +++ b/services/document-updater/test/unit/coffee/ShareJsUpdateManagerTests.coffee @@ -0,0 +1,174 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../app/js/ShareJsUpdateManager.js" +SandboxedModule = require('sandboxed-module') + +describe "ShareJsUpdateManager", -> + beforeEach -> + @project_id = "project-id-123" + @doc_id = "document-id-123" + @callback = sinon.stub() + @ShareJsUpdateManager = SandboxedModule.require modulePath, + requires: + "./sharejs/server/model": + class Model + constructor: (@db) -> + "./ShareJsDB" : @ShareJsDB = { mockDB: true } + "redis" : createClient: () => @rclient = auth:-> + "logger-sharelatex": @logger = { log: sinon.stub() } + globals: + clearTimeout: @clearTimeout = sinon.stub() + + describe "applyUpdates", -> + beforeEach -> + @version = 34 + @model = + applyOp: sinon.stub().callsArg(2) + getSnapshot: sinon.stub() + @ShareJsUpdateManager.getNewShareJsModel = sinon.stub().returns(@model) + @ShareJsUpdateManager._listenForOps = sinon.stub() + @ShareJsUpdateManager.removeDocFromCache = sinon.stub().callsArg(1) + + describe "with a text document", -> + beforeEach -> + @updates = [ + {p: 4, t: "foo"} + {p: 6, t: "bar"} + ] + @updatedDocLines = ["one", "two"] + + describe "successfully", -> + beforeEach (done) -> + @model.getSnapshot.callsArgWith(1, null, {snapshot: @updatedDocLines.join("\n"), v: @version}) + @ShareJsUpdateManager.applyUpdates @project_id, @doc_id, @updates, (err, docLines, version) => + @callback(err, docLines, version) + done() + + it "should create a new ShareJs model", -> + @ShareJsUpdateManager.getNewShareJsModel + .called.should.equal true + + it "should listen for ops on the model", -> + @ShareJsUpdateManager._listenForOps + .calledWith(@model) + .should.equal true + + it "should send each update to ShareJs", -> + for update in @updates + @model.applyOp + .calledWith("#{@project_id}:#{@doc_id}", update).should.equal true + + it "should get the updated doc lines", -> + @model.getSnapshot + .calledWith("#{@project_id}:#{@doc_id}") + .should.equal true + + it "should return the updated doc lines", -> + @callback.calledWith(null, @updatedDocLines, @version).should.equal true + + describe "when applyOp fails", -> + beforeEach (done) -> + @error = new Error("Something went wrong") + @ShareJsUpdateManager._sendError = sinon.stub() + @model.applyOp = sinon.stub().callsArgWith(2, @error) + @ShareJsUpdateManager.applyUpdates @project_id, @doc_id, @updates, (err, docLines, version) => + @callback(err, docLines, version) + done() + + it "should call sendError with the error", -> + @ShareJsUpdateManager._sendError + .calledWith(@project_id, @doc_id, @error) + .should.equal true + + it "should call the callback with the error", -> + @callback.calledWith(@error).should.equal true + + describe "when getSnapshot fails", -> + beforeEach (done) -> + @error = new Error("Something went wrong") + @ShareJsUpdateManager._sendError = sinon.stub() + @model.getSnapshot.callsArgWith(1, @error) + @ShareJsUpdateManager.applyUpdates @project_id, @doc_id, @updates, (err, docLines, version) => + @callback(err, docLines, version) + done() + + it "should call sendError with the error", -> + @ShareJsUpdateManager._sendError + .calledWith(@project_id, @doc_id, @error) + .should.equal true + + it "should call the callback with the error", -> + @callback.calledWith(@error).should.equal true + + describe "with a JSON document", -> + beforeEach -> + @updates = [ + {p: ["lines", 0], dl: { foo: "bar "}} + ] + @docLines = [text: "one", text: "two"] + + describe "successfully", -> + beforeEach (done) -> + @model.getSnapshot.callsArgWith(1, null, {snapshot: {lines: @docLines}, v: @version}) + @ShareJsUpdateManager.applyUpdates @project_id, @doc_id, @updates, (err, docLines, version) => + @callback(err, docLines, version) + done() + + it "should create a new ShareJs model", -> + @ShareJsUpdateManager.getNewShareJsModel + .called.should.equal true + + it "should listen for ops on the model", -> + @ShareJsUpdateManager._listenForOps + .calledWith(@model) + .should.equal true + + it "should send each update to ShareJs", -> + for update in @updates + @model.applyOp + .calledWith("#{@project_id}:#{@doc_id}", update).should.equal true + + it "should get the updated doc lines", -> + @model.getSnapshot + .calledWith("#{@project_id}:#{@doc_id}") + .should.equal true + + it "should return the updated doc lines", -> + @callback.calledWith(null, @docLines, @version).should.equal true + + describe "_listenForOps", -> + beforeEach -> + @model = on: (event, callback) => + @callback = callback + sinon.spy @model, "on" + @ShareJsUpdateManager._listenForOps(@model) + + it "should listen to the model for updates", -> + @model.on.calledWith("applyOp") + .should.equal true + + describe "the callback", -> + beforeEach -> + @opData = + op: {t: "foo", p: 1} + meta: source: "bar" + @rclient.publish = sinon.stub() + @callback("#{@project_id}:#{@doc_id}", @opData) + + it "should publish the op to redis", -> + @rclient.publish + .calledWith("applied-ops", JSON.stringify(project_id: @project_id, doc_id: @doc_id, op: @opData)) + .should.equal true + + describe "_sendError", -> + beforeEach -> + @error_text = "Something went wrong" + @rclient.publish = sinon.stub() + @ShareJsUpdateManager._sendError(@project_id, @doc_id, new Error(@error_text)) + + it "should publish the error to the redis stream", -> + @rclient.publish + .calledWith("applied-ops", JSON.stringify(project_id: @project_id, doc_id: @doc_id, error: @error_text)) + .should.equal true + diff --git a/services/document-updater/test/unit/coffee/UpdateManager/ApplyingUpdates.coffee b/services/document-updater/test/unit/coffee/UpdateManager/ApplyingUpdates.coffee new file mode 100644 index 0000000000..f421d545a7 --- /dev/null +++ b/services/document-updater/test/unit/coffee/UpdateManager/ApplyingUpdates.coffee @@ -0,0 +1,198 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/UpdateManager.js" +SandboxedModule = require('sandboxed-module') + +describe "UpdateManager", -> + beforeEach -> + @project_id = "project-id-123" + @doc_id = "document-id-123" + @callback = sinon.stub() + @UpdateManager = SandboxedModule.require modulePath, requires: + "./LockManager" : @LockManager = {} + "./RedisManager" : @RedisManager = {} + "./ShareJsUpdateManager" : @ShareJsUpdateManager = {} + "logger-sharelatex": @logger = { log: sinon.stub() } + "./Metrics": @Metrics = + Timer: class Timer + done: sinon.stub() + + describe "resumeProcessing", -> + beforeEach (done) -> + @docs = [{ + doc_id: "doc-1" + project_id: "project-1" + }, { + doc_id: "doc-2" + project_id: "project-2" + }, { + doc_id: "doc-3" + project_id: "project-3" + }] + @RedisManager.getDocsWithPendingUpdates = sinon.stub().callsArgWith(0, null, @docs) + @UpdateManager.processOutstandingUpdatesWithLock = sinon.stub().callsArg(2) + @UpdateManager.resumeProcessing(done) + + it "should the docs that haven't been processed yet", -> + @RedisManager.getDocsWithPendingUpdates + .called.should.equal true + + it "should call processOutstandingUpdatesWithLock for each doc", -> + for doc in @docs + @UpdateManager.processOutstandingUpdatesWithLock + .calledWith(doc.project_id, doc.doc_id) + .should.equal true + + describe "processOutstandingUpdates", -> + beforeEach -> + @UpdateManager.fetchAndApplyUpdates = sinon.stub().callsArg(2) + @RedisManager.clearDocFromPendingUpdatesSet = sinon.stub().callsArg(2) + @UpdateManager.processOutstandingUpdates @project_id, @doc_id, @callback + + it "should apply the updates", -> + @UpdateManager.fetchAndApplyUpdates.calledWith(@project_id, @doc_id).should.equal true + + it "should clear the doc from the process pending set", -> + @RedisManager.clearDocFromPendingUpdatesSet + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "processOutstandingUpdatesWithLock", -> + describe "when the lock is free", -> + beforeEach -> + @LockManager.tryLock = sinon.stub().callsArgWith(1, null, true) + @LockManager.releaseLock = sinon.stub().callsArg(1) + @UpdateManager.continueProcessingUpdatesWithLock = sinon.stub().callsArg(2) + @UpdateManager.processOutstandingUpdates = sinon.stub().callsArg(2) + + describe "successfully", -> + beforeEach -> + @UpdateManager.processOutstandingUpdatesWithLock @project_id, @doc_id, @callback + + it "should acquire the lock", -> + @LockManager.tryLock.calledWith(@doc_id).should.equal true + + it "should free the lock", -> + @LockManager.releaseLock.calledWith(@doc_id).should.equal true + + it "should process the outstanding updates", -> + @UpdateManager.processOutstandingUpdates.calledWith(@project_id, @doc_id).should.equal true + + it "should do everything with the lock acquired", -> + @UpdateManager.processOutstandingUpdates.calledAfter(@LockManager.tryLock).should.equal true + @UpdateManager.processOutstandingUpdates.calledBefore(@LockManager.releaseLock).should.equal true + + it "should continue processing new updates that may have come in", -> + @UpdateManager.continueProcessingUpdatesWithLock.calledWith(@project_id, @doc_id).should.equal true + + it "should return the callback", -> + @callback.called.should.equal true + + describe "when processOutstandingUpdates returns an error", -> + beforeEach -> + @UpdateManager.processOutstandingUpdates = sinon.stub().callsArgWith(2, @error = new Error("Something went wrong")) + @UpdateManager.processOutstandingUpdatesWithLock @project_id, @doc_id, @callback + + it "should free the lock", -> + @LockManager.releaseLock.calledWith(@doc_id).should.equal true + + it "should return the error in the callback", -> + @callback.calledWith(@error).should.equal true + + describe "when the lock is taken", -> + beforeEach -> + @LockManager.tryLock = sinon.stub().callsArgWith(1, null, false) + @UpdateManager.processOutstandingUpdates = sinon.stub().callsArg(2) + @UpdateManager.processOutstandingUpdatesWithLock @project_id, @doc_id, @callback + + it "should return the callback", -> + @callback.called.should.equal true + + it "should not process the updates", -> + @UpdateManager.processOutstandingUpdates.called.should.equal false + + describe "continueProcessingUpdatesWithLock", -> + describe "when there are outstanding updates", -> + beforeEach -> + @RedisManager.getUpdatesLength = sinon.stub().callsArgWith(1, null, 3) + @UpdateManager.processOutstandingUpdatesWithLock = sinon.stub().callsArg(2) + @UpdateManager.continueProcessingUpdatesWithLock @project_id, @doc_id, @callback + + it "should process the outstanding updates", -> + @UpdateManager.processOutstandingUpdatesWithLock.calledWith(@project_id, @doc_id).should.equal true + + it "should return the callback", -> + @callback.called.should.equal true + + describe "when there are no outstanding updates", -> + beforeEach -> + @RedisManager.getUpdatesLength = sinon.stub().callsArgWith(1, null, 0) + @UpdateManager.processOutstandingUpdatesWithLock = sinon.stub().callsArg(2) + @UpdateManager.continueProcessingUpdatesWithLock @project_id, @doc_id, @callback + + it "should not try to process the outstanding updates", -> + @UpdateManager.processOutstandingUpdatesWithLock.called.should.equal false + + it "should return the callback", -> + @callback.called.should.equal true + + describe "fetchAndApplyUpdates", -> + describe "with updates", -> + beforeEach -> + @updates = [{p: 1, t: "foo"}] + @updatedDocLines = ["updated", "lines"] + @version = 34 + @RedisManager.getPendingUpdatesForDoc = sinon.stub().callsArgWith(1, null, @updates) + @UpdateManager.applyUpdates = sinon.stub().callsArgWith(3, null, @updatedDocLines, @version) + @UpdateManager.fetchAndApplyUpdates @project_id, @doc_id, @callback + + it "should get the pending updates", -> + @RedisManager.getPendingUpdatesForDoc.calledWith(@doc_id).should.equal true + + it "should apply the updates", -> + @UpdateManager.applyUpdates + .calledWith(@project_id, @doc_id, @updates) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + describe "when there are no updates", -> + beforeEach -> + @updates = [] + @RedisManager.getPendingUpdatesForDoc = sinon.stub().callsArgWith(1, null, @updates) + @UpdateManager.applyUpdates = sinon.stub() + @RedisManager.setDocument = sinon.stub() + @UpdateManager.fetchAndApplyUpdates @project_id, @doc_id, @callback + + it "should not call applyUpdates", -> + @UpdateManager.applyUpdates.called.should.equal false + + it "should call the callback", -> + @callback.called.should.equal true + + describe "applyUpdates", -> + beforeEach -> + @updates = [{p: 1, t: "foo"}] + @updatedDocLines = ["updated", "lines"] + @version = 34 + @ShareJsUpdateManager.applyUpdates = sinon.stub().callsArgWith(3, null, @updatedDocLines, @version) + @RedisManager.setDocument = sinon.stub().callsArg(3) + @UpdateManager.applyUpdates @project_id, @doc_id, @updates, @callback + + it "should save the document", -> + @RedisManager.setDocument + .calledWith(@doc_id, @updatedDocLines, @version) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + diff --git a/services/document-updater/test/unit/coffee/UpdateManager/lockUpdatesAndDoTests.coffee b/services/document-updater/test/unit/coffee/UpdateManager/lockUpdatesAndDoTests.coffee new file mode 100644 index 0000000000..74e5a689fa --- /dev/null +++ b/services/document-updater/test/unit/coffee/UpdateManager/lockUpdatesAndDoTests.coffee @@ -0,0 +1,83 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/UpdateManager.js" +SandboxedModule = require('sandboxed-module') + +describe 'UpdateManager - lockUpdatesAndDo', -> + beforeEach -> + @UpdateManager = SandboxedModule.require modulePath, requires: + "./LockManager" : @LockManager = {} + "./RedisManager" : @RedisManager = {} + "./ShareJsUpdateManager" : @ShareJsUpdateManager = {} + "logger-sharelatex": @logger = { log: sinon.stub() } + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @method = sinon.stub().callsArgWith(3, null, @response_arg1) + @callback = sinon.stub() + @arg1 = "argument 1" + @response_arg1 = "response argument 1" + @LockManager.getLock = sinon.stub().callsArgWith(1, null, true) + @LockManager.releaseLock = sinon.stub().callsArg(1) + + describe "successfully", -> + beforeEach -> + @UpdateManager.continueProcessingUpdatesWithLock = sinon.stub() + @UpdateManager.processOutstandingUpdates = sinon.stub().callsArg(2) + @UpdateManager.lockUpdatesAndDo @method, @project_id, @doc_id, @arg1, @callback + + it "should lock the doc", -> + @LockManager.getLock + .calledWith(@doc_id) + .should.equal true + + it "should process any outstanding updates", -> + @UpdateManager.processOutstandingUpdates + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should call the method", -> + @method + .calledWith(@project_id, @doc_id, @arg1) + .should.equal true + + it "should return the method response to the callback", -> + @callback + .calledWith(null, @response_arg1) + .should.equal true + + it "should release the lock", -> + @LockManager.releaseLock + .calledWith(@doc_id) + .should.equal true + + it "should continue processing updates", -> + @UpdateManager.continueProcessingUpdatesWithLock + .calledWith(@project_id, @doc_id) + .should.equal true + + describe "when processOutstandingUpdates returns an error", -> + beforeEach -> + @UpdateManager.processOutstandingUpdates = sinon.stub().callsArgWith(2, @error = new Error("Something went wrong")) + @UpdateManager.lockUpdatesAndDo @method, @project_id, @doc_id, @arg1, @callback + + it "should free the lock", -> + @LockManager.releaseLock.calledWith(@doc_id).should.equal true + + it "should return the error in the callback", -> + @callback.calledWith(@error).should.equal true + + describe "when the method returns an error", -> + beforeEach -> + @UpdateManager.processOutstandingUpdates = sinon.stub().callsArg(2) + @method = sinon.stub().callsArgWith(3, @error = new Error("something went wrong"), @response_arg1) + @UpdateManager.lockUpdatesAndDo @method, @project_id, @doc_id, @arg1, @callback + + it "should free the lock", -> + @LockManager.releaseLock.calledWith(@doc_id).should.equal true + + it "should return the error in the callback", -> + @callback.calledWith(@error).should.equal true + + + diff --git a/services/document-updater/test/unit/js/module-loader.js b/services/document-updater/test/unit/js/module-loader.js new file mode 100644 index 0000000000..ac4cae7601 --- /dev/null +++ b/services/document-updater/test/unit/js/module-loader.js @@ -0,0 +1,29 @@ +var vm = require('vm'); +var fs = require('fs'); +var path = require('path'); + +module.exports.loadModule = function(filePath, mocks) { + mocks = mocks || {}; + + // this is necessary to allow relative path modules within loaded file + // i.e. requiring ./some inside file /a/b.js needs to be resolved to /a/some + var resolveModule = function(module) { + if (module.charAt(0) !== '.') return module; + return path.resolve(path.dirname(filePath), module); + }; + + var exports = {}; + var context = { + require: function(name) { + return mocks[name] || require(resolveModule(name)); + }, + console: console, + exports: exports, + module: { + exports: exports + } + }; + file = fs.readFileSync(filePath); + vm.runInNewContext(file, context); + return context; +};