diff --git a/services/document-updater/app/js/sharejs/LICENSE b/services/document-updater/app/js/sharejs/LICENSE new file mode 100644 index 0000000000..3e6b73ed85 --- /dev/null +++ b/services/document-updater/app/js/sharejs/LICENSE @@ -0,0 +1,22 @@ +Licensed under the standard MIT license: + +Copyright 2011 Joseph Gentle. +Copyright 2012-2024 Overleaf. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/services/document-updater/app/js/sharejs/README.md b/services/document-updater/app/js/sharejs/README.md index 22e68842dd..f5919dd9c1 100644 --- a/services/document-updater/app/js/sharejs/README.md +++ b/services/document-updater/app/js/sharejs/README.md @@ -1,48 +1,6 @@ -This directory contains all the operational transform code. Each file defines a type. +This folder contains a modified version of the ShareJS source code, forked from [v0.5.0](https://github.com/josephg/ShareJS/tree/v0.5.0/). -Most of the types in here are for testing or demonstration. The only types which are sent to the webclient -are `text` and `json`. +The original CoffeeScript code has been decaffeinated to JavaScript, and further modified. Some folders have been removed. See https://github.com/josephg/ShareJS/blob/v0.5.0/src/types/README.md for the original README. +The original code, and the current modified code in this directory, are published under the MIT license. -# An OT type - -All OT types have the following fields: - -`name`: _(string)_ Name of the type. Should match the filename. -`create() -> snapshot`: Function which creates and returns a new document snapshot - -`apply(snapshot, op) -> snapshot`: A function which creates a new document snapshot with the op applied -`transform(op1, op2, side) -> op1'`: OT transform function. - -Given op1, op2, `apply(s, op2, transform(op1, op2, 'left')) == apply(s, op1, transform(op2, op1, 'right'))`. - -Transform and apply must never modify their arguments. - - -Optional properties: - -`tp2`: _(bool)_ True if the transform function supports TP2. This allows p2p architectures to work. -`compose(op1, op2) -> op`: Create and return a new op which has the same effect as op1 + op2. -`serialize(snapshot) -> JSON object`: Serialize a document to something we can JSON.stringify() -`deserialize(object) -> snapshot`: Deserialize a JSON object into the document's internal snapshot format -`prune(op1', op2, side) -> op1`: Inserse transform function. Only required for TP2 types. -`normalize(op) -> op`: Fix up an op to make it valid. Eg, remove skips of size zero. -`api`: _(object)_ Set of helper methods which will be mixed in to the client document object for manipulating documents. See below. - - -# Examples - -`count` and `simple` are two trivial OT type definitions if you want to take a look. JSON defines -the ot-for-JSON type (see the wiki for documentation) and all the text types define different text -implementations. (I still have no idea which one I like the most, and they're fun to write!) - - -# API - -Types can also define API functions. These methods are mixed into the client's Doc object when a document is created. -You can use them to help construct ops programatically (so users don't need to understand how ops are structured). - -For example, the three text types defined here (text, text-composable and text-tp2) all provide the text API, supplying -`.insert()`, `.del()`, `.getLength` and `.getText` methods. - -See text-api.coffee for an example. diff --git a/services/document-updater/app/js/sharejs/count.js b/services/document-updater/app/js/sharejs/count.js deleted file mode 100644 index 246f6b7985..0000000000 --- a/services/document-updater/app/js/sharejs/count.js +++ /dev/null @@ -1,37 +0,0 @@ -// TODO: This file was created by bulk-decaffeinate. -// Sanity-check the conversion and remove this comment. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// This is a simple type used for testing other OT code. Each op is [expectedSnapshot, increment] - -exports.name = 'count' -exports.create = () => 1 - -exports.apply = function (snapshot, op) { - const [v, inc] = Array.from(op) - if (snapshot !== v) { - throw new Error(`Op ${v} != snapshot ${snapshot}`) - } - return snapshot + inc -} - -// transform op1 by op2. Return transformed version of op1. -exports.transform = function (op1, op2) { - if (op1[0] !== op2[0]) { - throw new Error(`Op1 ${op1[0]} != op2 ${op2[0]}`) - } - return [op1[0] + op2[1], op1[1]] -} - -exports.compose = function (op1, op2) { - if (op1[0] + op1[1] !== op2[0]) { - throw new Error(`Op1 ${op1} + 1 != op2 ${op2}`) - } - return [op1[0], op1[1] + op2[1]] -} - -exports.generateRandomOp = doc => [[doc, 1], doc + 1] diff --git a/services/document-updater/app/js/sharejs/helpers.js b/services/document-updater/app/js/sharejs/helpers.js deleted file mode 100644 index 0342655bc4..0000000000 --- a/services/document-updater/app/js/sharejs/helpers.js +++ /dev/null @@ -1,116 +0,0 @@ -/* eslint-disable - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// These methods let you build a transform function from a transformComponent function -// for OT types like text and JSON in which operations are lists of components -// and transforming them requires N^2 work. - -// Add transform and transformX functions for an OT type which has transformComponent defined. -// transformComponent(destination array, component, other component, side) -let bootstrapTransform -exports._bt = bootstrapTransform = function ( - type, - transformComponent, - checkValidOp, - append -) { - let transformX - const transformComponentX = function (left, right, destLeft, destRight) { - transformComponent(destLeft, left, right, 'left') - return transformComponent(destRight, right, left, 'right') - } - - // Transforms rightOp by leftOp. Returns ['rightOp', clientOp'] - type.transformX = - type.transformX = - transformX = - function (leftOp, rightOp) { - checkValidOp(leftOp) - checkValidOp(rightOp) - - const newRightOp = [] - - for (let rightComponent of Array.from(rightOp)) { - // Generate newLeftOp by composing leftOp by rightComponent - const newLeftOp = [] - - let k = 0 - while (k < leftOp.length) { - let l - const nextC = [] - transformComponentX(leftOp[k], rightComponent, newLeftOp, nextC) - k++ - - if (nextC.length === 1) { - rightComponent = nextC[0] - } else if (nextC.length === 0) { - for (l of Array.from(leftOp.slice(k))) { - append(newLeftOp, l) - } - rightComponent = null - break - } else { - // Recurse. - const [l_, r_] = Array.from(transformX(leftOp.slice(k), nextC)) - for (l of Array.from(l_)) { - append(newLeftOp, l) - } - for (const r of Array.from(r_)) { - append(newRightOp, r) - } - rightComponent = null - break - } - } - - if (rightComponent != null) { - append(newRightOp, rightComponent) - } - leftOp = newLeftOp - } - - return [leftOp, newRightOp] - } - - // Transforms op with specified type ('left' or 'right') by otherOp. - return (type.transform = type.transform = - function (op, otherOp, type) { - let _ - if (type !== 'left' && type !== 'right') { - throw new Error("type must be 'left' or 'right'") - } - - if (otherOp.length === 0) { - return op - } - - // TODO: Benchmark with and without this line. I _think_ it'll make a big difference...? - if (op.length === 1 && otherOp.length === 1) { - return transformComponent([], op[0], otherOp[0], type) - } - - if (type === 'left') { - let left - ;[left, _] = Array.from(transformX(op, otherOp)) - return left - } else { - let right - ;[_, right] = Array.from(transformX(otherOp, op)) - return right - } - }) -} - -if (typeof WEB === 'undefined') { - exports.bootstrapTransform = bootstrapTransform -} diff --git a/services/document-updater/app/js/sharejs/index.js b/services/document-updater/app/js/sharejs/index.js deleted file mode 100644 index 7e3d6bbf26..0000000000 --- a/services/document-updater/app/js/sharejs/index.js +++ /dev/null @@ -1,25 +0,0 @@ -// TODO: This file was created by bulk-decaffeinate. -// Sanity-check the conversion and remove this comment. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ - -const register = function (file) { - const type = require(file) - exports[type.name] = type - try { - return require(`${file}-api`) - } catch (error) {} -} - -// Import all the built-in types. -register('./simple') -register('./count') - -register('./text') -register('./text-composable') -register('./text-tp2') - -register('./json') diff --git a/services/document-updater/app/js/sharejs/json-api.js b/services/document-updater/app/js/sharejs/json-api.js deleted file mode 100644 index e591f66743..0000000000 --- a/services/document-updater/app/js/sharejs/json-api.js +++ /dev/null @@ -1,356 +0,0 @@ -/* eslint-disable - no-undef, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS205: Consider reworking code to avoid use of IIFEs - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// API for JSON OT - -let json -if (typeof WEB === 'undefined') { - json = require('./json') -} - -if (typeof WEB !== 'undefined' && WEB !== null) { - const { extendDoc } = exports - exports.extendDoc = function (name, fn) { - SubDoc.prototype[name] = fn - return extendDoc(name, fn) - } -} - -const depath = function (path) { - if (path.length === 1 && path[0].constructor === Array) { - return path[0] - } else { - return path - } -} - -class SubDoc { - constructor(doc, path) { - this.doc = doc - this.path = path - } - - at(...path) { - return this.doc.at(this.path.concat(depath(path))) - } - - get() { - return this.doc.getAt(this.path) - } - - // for objects and lists - set(value, cb) { - return this.doc.setAt(this.path, value, cb) - } - - // for strings and lists. - insert(pos, value, cb) { - return this.doc.insertAt(this.path, pos, value, cb) - } - - // for strings - del(pos, length, cb) { - return this.doc.deleteTextAt(this.path, length, pos, cb) - } - - // for objects and lists - remove(cb) { - return this.doc.removeAt(this.path, cb) - } - - push(value, cb) { - return this.insert(this.get().length, value, cb) - } - - move(from, to, cb) { - return this.doc.moveAt(this.path, from, to, cb) - } - - add(amount, cb) { - return this.doc.addAt(this.path, amount, cb) - } - - on(event, cb) { - return this.doc.addListener(this.path, event, cb) - } - - removeListener(l) { - return this.doc.removeListener(l) - } - - // text API compatibility - getLength() { - return this.get().length - } - - getText() { - return this.get() - } -} - -const traverse = function (snapshot, path) { - const container = { data: snapshot } - let key = 'data' - let elem = container - for (const p of Array.from(path)) { - elem = elem[key] - key = p - if (typeof elem === 'undefined') { - throw new Error('bad path') - } - } - return { elem, key } -} - -const pathEquals = function (p1, p2) { - if (p1.length !== p2.length) { - return false - } - for (let i = 0; i < p1.length; i++) { - const e = p1[i] - if (e !== p2[i]) { - return false - } - } - return true -} - -json.api = { - provides: { json: true }, - - at(...path) { - return new SubDoc(this, depath(path)) - }, - - get() { - return this.snapshot - }, - set(value, cb) { - return this.setAt([], value, cb) - }, - - getAt(path) { - const { elem, key } = traverse(this.snapshot, path) - return elem[key] - }, - - setAt(path, value, cb) { - const { elem, key } = traverse(this.snapshot, path) - const op = { p: path } - if (elem.constructor === Array) { - op.li = value - if (typeof elem[key] !== 'undefined') { - op.ld = elem[key] - } - } else if (typeof elem === 'object') { - op.oi = value - if (typeof elem[key] !== 'undefined') { - op.od = elem[key] - } - } else { - throw new Error('bad path') - } - return this.submitOp([op], cb) - }, - - removeAt(path, cb) { - const { elem, key } = traverse(this.snapshot, path) - if (typeof elem[key] === 'undefined') { - throw new Error('no element at that path') - } - const op = { p: path } - if (elem.constructor === Array) { - op.ld = elem[key] - } else if (typeof elem === 'object') { - op.od = elem[key] - } else { - throw new Error('bad path') - } - return this.submitOp([op], cb) - }, - - insertAt(path, pos, value, cb) { - const { elem, key } = traverse(this.snapshot, path) - const op = { p: path.concat(pos) } - if (elem[key].constructor === Array) { - op.li = value - } else if (typeof elem[key] === 'string') { - op.si = value - } - return this.submitOp([op], cb) - }, - - moveAt(path, from, to, cb) { - const op = [{ p: path.concat(from), lm: to }] - return this.submitOp(op, cb) - }, - - addAt(path, amount, cb) { - const op = [{ p: path, na: amount }] - return this.submitOp(op, cb) - }, - - deleteTextAt(path, length, pos, cb) { - const { elem, key } = traverse(this.snapshot, path) - const op = [{ p: path.concat(pos), sd: elem[key].slice(pos, pos + length) }] - return this.submitOp(op, cb) - }, - - addListener(path, event, cb) { - const l = { path, event, cb } - this._listeners.push(l) - return l - }, - removeListener(l) { - const i = this._listeners.indexOf(l) - if (i < 0) { - return false - } - this._listeners.splice(i, 1) - return true - }, - _register() { - this._listeners = [] - this.on('change', function (op) { - return (() => { - const result = [] - for (const c of Array.from(op)) { - let i - if (c.na !== undefined || c.si !== undefined || c.sd !== undefined) { - // no change to structure - continue - } - const toRemove = [] - for (i = 0; i < this._listeners.length; i++) { - // Transform a dummy op by the incoming op to work out what - // should happen to the listener. - const l = this._listeners[i] - const dummy = { p: l.path, na: 0 } - const xformed = this.type.transformComponent([], dummy, c, 'left') - if (xformed.length === 0) { - // The op was transformed to noop, so we should delete the listener. - toRemove.push(i) - } else if (xformed.length === 1) { - // The op remained, so grab its new path into the listener. - l.path = xformed[0].p - } else { - throw new Error( - "Bad assumption in json-api: xforming an 'si' op will always result in 0 or 1 components." - ) - } - } - toRemove.sort((a, b) => b - a) - result.push( - (() => { - const result1 = [] - for (i of Array.from(toRemove)) { - result1.push(this._listeners.splice(i, 1)) - } - return result1 - })() - ) - } - return result - })() - }) - return this.on('remoteop', function (op) { - return (() => { - const result = [] - for (const c of Array.from(op)) { - const matchPath = - c.na === undefined ? c.p.slice(0, c.p.length - 1) : c.p - result.push( - (() => { - const result1 = [] - for (const { path, event, cb } of Array.from(this._listeners)) { - let common - if (pathEquals(path, matchPath)) { - switch (event) { - case 'insert': - if (c.li !== undefined && c.ld === undefined) { - result1.push(cb(c.p[c.p.length - 1], c.li)) - } else if (c.oi !== undefined && c.od === undefined) { - result1.push(cb(c.p[c.p.length - 1], c.oi)) - } else if (c.si !== undefined) { - result1.push(cb(c.p[c.p.length - 1], c.si)) - } else { - result1.push(undefined) - } - break - case 'delete': - if (c.li === undefined && c.ld !== undefined) { - result1.push(cb(c.p[c.p.length - 1], c.ld)) - } else if (c.oi === undefined && c.od !== undefined) { - result1.push(cb(c.p[c.p.length - 1], c.od)) - } else if (c.sd !== undefined) { - result1.push(cb(c.p[c.p.length - 1], c.sd)) - } else { - result1.push(undefined) - } - break - case 'replace': - if (c.li !== undefined && c.ld !== undefined) { - result1.push(cb(c.p[c.p.length - 1], c.ld, c.li)) - } else if (c.oi !== undefined && c.od !== undefined) { - result1.push(cb(c.p[c.p.length - 1], c.od, c.oi)) - } else { - result1.push(undefined) - } - break - case 'move': - if (c.lm !== undefined) { - result1.push(cb(c.p[c.p.length - 1], c.lm)) - } else { - result1.push(undefined) - } - break - case 'add': - if (c.na !== undefined) { - result1.push(cb(c.na)) - } else { - result1.push(undefined) - } - break - default: - result1.push(undefined) - } - } else if ( - (common = this.type.commonPath(matchPath, path)) != null - ) { - if (event === 'child op') { - if ( - matchPath.length === path.length && - path.length === common - ) { - throw new Error( - "paths match length and have commonality, but aren't equal?" - ) - } - const childPath = c.p.slice(common + 1) - result1.push(cb(childPath, c)) - } else { - result1.push(undefined) - } - } else { - result1.push(undefined) - } - } - return result1 - })() - ) - } - return result - })() - }) - }, -} diff --git a/services/document-updater/app/js/sharejs/json.js b/services/document-updater/app/js/sharejs/json.js deleted file mode 100644 index 3422d6158e..0000000000 --- a/services/document-updater/app/js/sharejs/json.js +++ /dev/null @@ -1,630 +0,0 @@ -/* eslint-disable - no-return-assign, - no-undef, - no-useless-catch, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// This is the implementation of the JSON OT type. -// -// Spec is here: https://github.com/josephg/ShareJS/wiki/JSON-Operations - -let text -if (typeof WEB !== 'undefined' && WEB !== null) { - ;({ text } = exports.types) -} else { - text = require('./text') -} - -const json = {} - -json.name = 'json' - -json.create = () => null - -json.invertComponent = function (c) { - const c_ = { p: c.p } - if (c.si !== undefined) { - c_.sd = c.si - } - if (c.sd !== undefined) { - c_.si = c.sd - } - if (c.oi !== undefined) { - c_.od = c.oi - } - if (c.od !== undefined) { - c_.oi = c.od - } - if (c.li !== undefined) { - c_.ld = c.li - } - if (c.ld !== undefined) { - c_.li = c.ld - } - if (c.na !== undefined) { - c_.na = -c.na - } - if (c.lm !== undefined) { - c_.lm = c.p[c.p.length - 1] - c_.p = c.p.slice(0, c.p.length - 1).concat([c.lm]) - } - return c_ -} - -json.invert = op => - Array.from(op.slice().reverse()).map(c => json.invertComponent(c)) - -json.checkValidOp = function (op) {} - -const isArray = o => Object.prototype.toString.call(o) === '[object Array]' -json.checkList = function (elem) { - if (!isArray(elem)) { - throw new Error('Referenced element not a list') - } -} - -json.checkObj = function (elem) { - if (elem.constructor !== Object) { - throw new Error( - `Referenced element not an object (it was ${JSON.stringify(elem)})` - ) - } -} - -json.apply = function (snapshot, op) { - json.checkValidOp(op) - op = clone(op) - - const container = { data: clone(snapshot) } - - try { - for (let i = 0; i < op.length; i++) { - const c = op[i] - let parent = null - let parentkey = null - let elem = container - let key = 'data' - - for (const p of Array.from(c.p)) { - parent = elem - parentkey = key - elem = elem[key] - key = p - - if (parent == null) { - throw new Error('Path invalid') - } - } - - if (c.na !== undefined) { - // Number add - if (typeof elem[key] !== 'number') { - throw new Error('Referenced element not a number') - } - elem[key] += c.na - } else if (c.si !== undefined) { - // String insert - if (typeof elem !== 'string') { - throw new Error( - `Referenced element not a string (it was ${JSON.stringify(elem)})` - ) - } - parent[parentkey] = elem.slice(0, key) + c.si + elem.slice(key) - } else if (c.sd !== undefined) { - // String delete - if (typeof elem !== 'string') { - throw new Error('Referenced element not a string') - } - if (elem.slice(key, key + c.sd.length) !== c.sd) { - throw new Error('Deleted string does not match') - } - parent[parentkey] = elem.slice(0, key) + elem.slice(key + c.sd.length) - } else if (c.li !== undefined && c.ld !== undefined) { - // List replace - json.checkList(elem) - - // Should check the list element matches c.ld - elem[key] = c.li - } else if (c.li !== undefined) { - // List insert - json.checkList(elem) - - elem.splice(key, 0, c.li) - } else if (c.ld !== undefined) { - // List delete - json.checkList(elem) - - // Should check the list element matches c.ld here too. - elem.splice(key, 1) - } else if (c.lm !== undefined) { - // List move - json.checkList(elem) - if (c.lm !== key) { - const e = elem[key] - // Remove it... - elem.splice(key, 1) - // And insert it back. - elem.splice(c.lm, 0, e) - } - } else if (c.oi !== undefined) { - // Object insert / replace - json.checkObj(elem) - - // Should check that elem[key] == c.od - elem[key] = c.oi - } else if (c.od !== undefined) { - // Object delete - json.checkObj(elem) - - // Should check that elem[key] == c.od - delete elem[key] - } else { - throw new Error('invalid / missing instruction in op') - } - } - } catch (error) { - // TODO: Roll back all already applied changes. Write tests before implementing this code. - throw error - } - - return container.data -} - -// Checks if two paths, p1 and p2 match. -json.pathMatches = function (p1, p2, ignoreLast) { - if (p1.length !== p2.length) { - return false - } - - for (let i = 0; i < p1.length; i++) { - const p = p1[i] - if (p !== p2[i] && (!ignoreLast || i !== p1.length - 1)) { - return false - } - } - - return true -} - -json.append = function (dest, c) { - let last - c = clone(c) - if ( - dest.length !== 0 && - json.pathMatches(c.p, (last = dest[dest.length - 1]).p) - ) { - if (last.na !== undefined && c.na !== undefined) { - return (dest[dest.length - 1] = { p: last.p, na: last.na + c.na }) - } else if ( - last.li !== undefined && - c.li === undefined && - c.ld === last.li - ) { - // insert immediately followed by delete becomes a noop. - if (last.ld !== undefined) { - // leave the delete part of the replace - return delete last.li - } else { - return dest.pop() - } - } else if ( - last.od !== undefined && - last.oi === undefined && - c.oi !== undefined && - c.od === undefined - ) { - return (last.oi = c.oi) - } else if (c.lm !== undefined && c.p[c.p.length - 1] === c.lm) { - return null // don't do anything - } else { - return dest.push(c) - } - } else { - return dest.push(c) - } -} - -json.compose = function (op1, op2) { - json.checkValidOp(op1) - json.checkValidOp(op2) - - const newOp = clone(op1) - for (const c of Array.from(op2)) { - json.append(newOp, c) - } - - return newOp -} - -json.normalize = function (op) { - const newOp = [] - - if (!isArray(op)) { - op = [op] - } - - for (const c of Array.from(op)) { - if (c.p == null) { - c.p = [] - } - json.append(newOp, c) - } - - return newOp -} - -// hax, copied from test/types/json. Apparently this is still the fastest way to deep clone an object, assuming -// we have browser support for JSON. -// http://jsperf.com/cloning-an-object/12 -const clone = o => JSON.parse(JSON.stringify(o)) - -json.commonPath = function (p1, p2) { - p1 = p1.slice() - p2 = p2.slice() - p1.unshift('data') - p2.unshift('data') - p1 = p1.slice(0, p1.length - 1) - p2 = p2.slice(0, p2.length - 1) - if (p2.length === 0) { - return -1 - } - let i = 0 - while (p1[i] === p2[i] && i < p1.length) { - i++ - if (i === p2.length) { - return i - 1 - } - } -} - -// transform c so it applies to a document with otherC applied. -json.transformComponent = function (dest, c, otherC, type) { - let oc - c = clone(c) - if (c.na !== undefined) { - c.p.push(0) - } - if (otherC.na !== undefined) { - otherC.p.push(0) - } - - const common = json.commonPath(c.p, otherC.p) - const common2 = json.commonPath(otherC.p, c.p) - - const cplength = c.p.length - const otherCplength = otherC.p.length - - if (c.na !== undefined) { - c.p.pop() - } // hax - if (otherC.na !== undefined) { - otherC.p.pop() - } - - if (otherC.na) { - if ( - common2 != null && - otherCplength >= cplength && - otherC.p[common2] === c.p[common2] - ) { - if (c.ld !== undefined) { - oc = clone(otherC) - oc.p = oc.p.slice(cplength) - c.ld = json.apply(clone(c.ld), [oc]) - } else if (c.od !== undefined) { - oc = clone(otherC) - oc.p = oc.p.slice(cplength) - c.od = json.apply(clone(c.od), [oc]) - } - } - json.append(dest, c) - return dest - } - - if ( - common2 != null && - otherCplength > cplength && - c.p[common2] === otherC.p[common2] - ) { - // transform based on c - if (c.ld !== undefined) { - oc = clone(otherC) - oc.p = oc.p.slice(cplength) - c.ld = json.apply(clone(c.ld), [oc]) - } else if (c.od !== undefined) { - oc = clone(otherC) - oc.p = oc.p.slice(cplength) - c.od = json.apply(clone(c.od), [oc]) - } - } - - if (common != null) { - let from, p, to - const commonOperand = cplength === otherCplength - // transform based on otherC - if (otherC.na !== undefined) { - // this case is handled above due to icky path hax - } else if (otherC.si !== undefined || otherC.sd !== undefined) { - // String op vs string op - pass through to text type - if (c.si !== undefined || c.sd !== undefined) { - if (!commonOperand) { - throw new Error('must be a string?') - } - - // Convert an op component to a text op component - const convert = function (component) { - const newC = { p: component.p[component.p.length - 1] } - if (component.si) { - newC.i = component.si - } else { - newC.d = component.sd - } - return newC - } - - const tc1 = convert(c) - const tc2 = convert(otherC) - - const res = [] - text._tc(res, tc1, tc2, type) - for (const tc of Array.from(res)) { - const jc = { p: c.p.slice(0, common) } - jc.p.push(tc.p) - if (tc.i != null) { - jc.si = tc.i - } - if (tc.d != null) { - jc.sd = tc.d - } - json.append(dest, jc) - } - return dest - } - } else if (otherC.li !== undefined && otherC.ld !== undefined) { - if (otherC.p[common] === c.p[common]) { - // noop - if (!commonOperand) { - // we're below the deleted element, so -> noop - return dest - } else if (c.ld !== undefined) { - // we're trying to delete the same element, -> noop - if (c.li !== undefined && type === 'left') { - // we're both replacing one element with another. only one can - // survive! - c.ld = clone(otherC.li) - } else { - return dest - } - } - } - } else if (otherC.li !== undefined) { - if ( - c.li !== undefined && - c.ld === undefined && - commonOperand && - c.p[common] === otherC.p[common] - ) { - // in li vs. li, left wins. - if (type === 'right') { - c.p[common]++ - } - } else if (otherC.p[common] <= c.p[common]) { - c.p[common]++ - } - - if (c.lm !== undefined) { - if (commonOperand) { - // otherC edits the same list we edit - if (otherC.p[common] <= c.lm) { - c.lm++ - } - } - } - // changing c.from is handled above. - } else if (otherC.ld !== undefined) { - if (c.lm !== undefined) { - if (commonOperand) { - if (otherC.p[common] === c.p[common]) { - // they deleted the thing we're trying to move - return dest - } - // otherC edits the same list we edit - p = otherC.p[common] - from = c.p[common] - to = c.lm - if (p < to || (p === to && from < to)) { - c.lm-- - } - } - } - - if (otherC.p[common] < c.p[common]) { - c.p[common]-- - } else if (otherC.p[common] === c.p[common]) { - if (otherCplength < cplength) { - // we're below the deleted element, so -> noop - return dest - } else if (c.ld !== undefined) { - if (c.li !== undefined) { - // we're replacing, they're deleting. we become an insert. - delete c.ld - } else { - // we're trying to delete the same element, -> noop - return dest - } - } - } - } else if (otherC.lm !== undefined) { - if (c.lm !== undefined && cplength === otherCplength) { - // lm vs lm, here we go! - from = c.p[common] - to = c.lm - const otherFrom = otherC.p[common] - const otherTo = otherC.lm - if (otherFrom !== otherTo) { - // if otherFrom == otherTo, we don't need to change our op. - - // where did my thing go? - if (from === otherFrom) { - // they moved it! tie break. - if (type === 'left') { - c.p[common] = otherTo - if (from === to) { - // ugh - c.lm = otherTo - } - } else { - return dest - } - } else { - // they moved around it - if (from > otherFrom) { - c.p[common]-- - } - if (from > otherTo) { - c.p[common]++ - } else if (from === otherTo) { - if (otherFrom > otherTo) { - c.p[common]++ - if (from === to) { - // ugh, again - c.lm++ - } - } - } - - // step 2: where am i going to put it? - if (to > otherFrom) { - c.lm-- - } else if (to === otherFrom) { - if (to > from) { - c.lm-- - } - } - if (to > otherTo) { - c.lm++ - } else if (to === otherTo) { - // if we're both moving in the same direction, tie break - if ( - (otherTo > otherFrom && to > from) || - (otherTo < otherFrom && to < from) - ) { - if (type === 'right') { - c.lm++ - } - } else { - if (to > from) { - c.lm++ - } else if (to === otherFrom) { - c.lm-- - } - } - } - } - } - } else if (c.li !== undefined && c.ld === undefined && commonOperand) { - // li - from = otherC.p[common] - to = otherC.lm - p = c.p[common] - if (p > from) { - c.p[common]-- - } - if (p > to) { - c.p[common]++ - } - } else { - // ld, ld+li, si, sd, na, oi, od, oi+od, any li on an element beneath - // the lm - // - // i.e. things care about where their item is after the move. - from = otherC.p[common] - to = otherC.lm - p = c.p[common] - if (p === from) { - c.p[common] = to - } else { - if (p > from) { - c.p[common]-- - } - if (p > to) { - c.p[common]++ - } else if (p === to) { - if (from > to) { - c.p[common]++ - } - } - } - } - } else if (otherC.oi !== undefined && otherC.od !== undefined) { - if (c.p[common] === otherC.p[common]) { - if (c.oi !== undefined && commonOperand) { - // we inserted where someone else replaced - if (type === 'right') { - // left wins - return dest - } else { - // we win, make our op replace what they inserted - c.od = otherC.oi - } - } else { - // -> noop if the other component is deleting the same object (or any - // parent) - return dest - } - } - } else if (otherC.oi !== undefined) { - if (c.oi !== undefined && c.p[common] === otherC.p[common]) { - // left wins if we try to insert at the same place - if (type === 'left') { - json.append(dest, { p: c.p, od: otherC.oi }) - } else { - return dest - } - } - } else if (otherC.od !== undefined) { - if (c.p[common] === otherC.p[common]) { - if (!commonOperand) { - return dest - } - if (c.oi !== undefined) { - delete c.od - } else { - return dest - } - } - } - } - - json.append(dest, c) - return dest -} - -if (typeof WEB !== 'undefined' && WEB !== null) { - if (!exports.types) { - exports.types = {} - } - - // This is kind of awful - come up with a better way to hook this helper code up. - exports._bt(json, json.transformComponent, json.checkValidOp, json.append) - - // [] is used to prevent closure from renaming types.text - exports.types.json = json -} else { - module.exports = json - - require('./helpers').bootstrapTransform( - json, - json.transformComponent, - json.checkValidOp, - json.append - ) -} diff --git a/services/document-updater/app/js/sharejs/model.js b/services/document-updater/app/js/sharejs/model.js deleted file mode 100644 index d927811895..0000000000 --- a/services/document-updater/app/js/sharejs/model.js +++ /dev/null @@ -1,882 +0,0 @@ -/* eslint-disable - no-console, - no-return-assign, - n/no-callback-literal, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS103: Rewrite code to no longer use __guard__ - * DS104: Avoid inline assignments - * DS204: Change includes calls to have a more natural evaluation order - * DS205: Consider reworking code to avoid use of IIFEs - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// The model of all the ops. Responsible for applying & transforming remote deltas -// and managing the storage layer. -// -// Actual storage is handled by the database wrappers in db/*, wrapped by DocCache - -let Model -const { EventEmitter } = require('events') - -const queue = require('./syncqueue') -const types = require('../types') - -const isArray = o => Object.prototype.toString.call(o) === '[object Array]' - -// This constructor creates a new Model object. There will be one model object -// per server context. -// -// The model object is responsible for a lot of things: -// -// - It manages the interactions with the database -// - It maintains (in memory) a set of all active documents -// - It calls out to the OT functions when necessary -// -// The model is an event emitter. It emits the following events: -// -// create(docName, data): A document has been created with the specified name & data -module.exports = Model = function (db, options) { - // db can be null if the user doesn't want persistance. - - let getOps - if (!(this instanceof Model)) { - return new Model(db, options) - } - - const model = this - - if (options == null) { - options = {} - } - - // This is a cache of 'live' documents. - // - // The cache is a map from docName -> { - // ops:[{op, meta}] - // snapshot - // type - // v - // meta - // eventEmitter - // reapTimer - // committedVersion: v - // snapshotWriteLock: bool to make sure writeSnapshot isn't re-entrant - // dbMeta: database specific data - // opQueue: syncQueue for processing ops - // } - // - // The ops list contains the document's last options.numCachedOps ops. (Or all - // of them if we're using a memory store). - // - // Documents are stored in this set so long as the document has been accessed in - // the last few seconds (options.reapTime) OR at least one client has the document - // open. I don't know if I should keep open (but not being edited) documents live - - // maybe if a client has a document open but the document isn't being edited, I should - // flush it from the cache. - // - // In any case, the API to model is designed such that if we want to change that later - // it should be pretty easy to do so without any external-to-the-model code changes. - const docs = {} - - // This is a map from docName -> [callback]. It is used when a document hasn't been - // cached and multiple getSnapshot() / getVersion() requests come in. All requests - // are added to the callback list and called when db.getSnapshot() returns. - // - // callback(error, snapshot data) - const awaitingGetSnapshot = {} - - // The time that documents which no clients have open will stay in the cache. - // Should be > 0. - if (options.reapTime == null) { - options.reapTime = 3000 - } - - // The number of operations the cache holds before reusing the space - if (options.numCachedOps == null) { - options.numCachedOps = 10 - } - - // This option forces documents to be reaped, even when there's no database backend. - // This is useful when you don't care about persistance and don't want to gradually - // fill memory. - // - // You might want to set reapTime to a day or something. - if (options.forceReaping == null) { - options.forceReaping = false - } - - // Until I come up with a better strategy, we'll save a copy of the document snapshot - // to the database every ~20 submitted ops. - if (options.opsBeforeCommit == null) { - options.opsBeforeCommit = 20 - } - - // It takes some processing time to transform client ops. The server will punt ops back to the - // client to transform if they're too old. - if (options.maximumAge == null) { - options.maximumAge = 40 - } - - // **** Cache API methods - - // Its important that all ops are applied in order. This helper method creates the op submission queue - // for a single document. This contains the logic for transforming & applying ops. - const makeOpQueue = (docName, doc) => - queue(function (opData, callback) { - if (!(opData.v >= 0)) { - return callback('Version missing') - } - if (opData.v > doc.v) { - return callback('Op at future version') - } - - // Punt the transforming work back to the client if the op is too old. - if (opData.v + options.maximumAge < doc.v) { - return callback('Op too old') - } - - if (!opData.meta) { - opData.meta = {} - } - opData.meta.ts = Date.now() - - // We'll need to transform the op to the current version of the document. This - // calls the callback immediately if opVersion == doc.v. - return getOps(docName, opData.v, doc.v, function (error, ops) { - let snapshot - if (error) { - return callback(error) - } - - if (doc.v - opData.v !== ops.length) { - // This should never happen. It indicates that we didn't get all the ops we - // asked for. Its important that the submitted op is correctly transformed. - console.error( - `Could not get old ops in model for document ${docName}` - ) - console.error( - `Expected ops ${opData.v} to ${doc.v} and got ${ops.length} ops` - ) - return callback('Internal error') - } - - if (ops.length > 0) { - try { - // If there's enough ops, it might be worth spinning this out into a webworker thread. - for (const oldOp of Array.from(ops)) { - // Dup detection works by sending the id(s) the op has been submitted with previously. - // If the id matches, we reject it. The client can also detect the op has been submitted - // already if it sees its own previous id in the ops it sees when it does catchup. - if ( - oldOp.meta.source && - opData.dupIfSource && - Array.from(opData.dupIfSource).includes(oldOp.meta.source) - ) { - return callback('Op already submitted') - } - - opData.op = doc.type.transform(opData.op, oldOp.op, 'left') - opData.v++ - } - } catch (error1) { - error = error1 - return callback(error.message) - } - } - - try { - snapshot = doc.type.apply(doc.snapshot, opData.op) - } catch (error2) { - error = error2 - return callback(error.message) - } - - // The op data should be at the current version, and the new document data should be at - // the next version. - // - // This should never happen in practice, but its a nice little check to make sure everything - // is hunky-dory. - if (opData.v !== doc.v) { - // This should never happen. - console.error( - 'Version mismatch detected in model. File a ticket - this is a bug.' - ) - console.error(`Expecting ${opData.v} == ${doc.v}`) - return callback('Internal error') - } - - // newDocData = {snapshot, type:type.name, v:opVersion + 1, meta:docData.meta} - const writeOp = - (db != null ? db.writeOp : undefined) || - ((docName, newOpData, callback) => callback()) - - return writeOp(docName, opData, function (error) { - if (error) { - // The user should probably know about this. - console.warn(`Error writing ops to database: ${error}`) - return callback(error) - } - - __guardMethod__(options.stats, 'writeOp', o => o.writeOp()) - - // This is needed when we emit the 'change' event, below. - const oldSnapshot = doc.snapshot - - // All the heavy lifting is now done. Finally, we'll update the cache with the new data - // and (maybe!) save a new document snapshot to the database. - - doc.v = opData.v + 1 - doc.snapshot = snapshot - - doc.ops.push(opData) - if (db && doc.ops.length > options.numCachedOps) { - doc.ops.shift() - } - - model.emit('applyOp', docName, opData, snapshot, oldSnapshot) - doc.eventEmitter.emit('op', opData, snapshot, oldSnapshot) - - // The callback is called with the version of the document at which the op was applied. - // This is the op.v after transformation, and its doc.v - 1. - callback(null, opData.v) - - // I need a decent strategy here for deciding whether or not to save the snapshot. - // - // The 'right' strategy looks something like "Store the snapshot whenever the snapshot - // is smaller than the accumulated op data". For now, I'll just store it every 20 - // ops or something. (Configurable with doc.committedVersion) - if ( - !doc.snapshotWriteLock && - doc.committedVersion + options.opsBeforeCommit <= doc.v - ) { - return tryWriteSnapshot(docName, function (error) { - if (error) { - return console.warn( - `Error writing snapshot ${error}. This is nonfatal` - ) - } - }) - } - }) - }) - }) - - // Add the data for the given docName to the cache. The named document shouldn't already - // exist in the doc set. - // - // Returns the new doc. - const add = function (docName, error, data, committedVersion, ops, dbMeta) { - let callback, doc - const callbacks = awaitingGetSnapshot[docName] - delete awaitingGetSnapshot[docName] - - if (error) { - if (callbacks) { - for (callback of Array.from(callbacks)) { - callback(error) - } - } - } else { - doc = docs[docName] = { - snapshot: data.snapshot, - v: data.v, - type: data.type, - meta: data.meta, - - // Cache of ops - ops: ops || [], - - eventEmitter: new EventEmitter(), - - // Timer before the document will be invalidated from the cache (if the document has no - // listeners) - reapTimer: null, - - // Version of the snapshot thats in the database - committedVersion: committedVersion != null ? committedVersion : data.v, - snapshotWriteLock: false, - dbMeta, - } - - doc.opQueue = makeOpQueue(docName, doc) - - refreshReapingTimeout(docName) - model.emit('add', docName, data) - if (callbacks) { - for (callback of Array.from(callbacks)) { - callback(null, doc) - } - } - } - - return doc - } - - // This is a little helper wrapper around db.getOps. It does two things: - // - // - If there's no database set, it returns an error to the callback - // - It adds version numbers to each op returned from the database - // (These can be inferred from context so the DB doesn't store them, but its useful to have them). - const getOpsInternal = function (docName, start, end, callback) { - if (!db) { - return typeof callback === 'function' - ? callback('Document does not exist') - : undefined - } - - return db.getOps(docName, start, end, function (error, ops) { - if (error) { - return typeof callback === 'function' ? callback(error) : undefined - } - - let v = start - for (const op of Array.from(ops)) { - op.v = v++ - } - - return typeof callback === 'function' ? callback(null, ops) : undefined - }) - } - - // Load the named document into the cache. This function is re-entrant. - // - // The callback is called with (error, doc) - const load = function (docName, callback) { - if (docs[docName]) { - // The document is already loaded. Return immediately. - __guardMethod__(options.stats, 'cacheHit', o => o.cacheHit('getSnapshot')) - return callback(null, docs[docName]) - } - - // We're a memory store. If we don't have it, nobody does. - if (!db) { - return callback('Document does not exist') - } - - const callbacks = awaitingGetSnapshot[docName] - - // The document is being loaded already. Add ourselves as a callback. - if (callbacks) { - return callbacks.push(callback) - } - - __guardMethod__(options.stats, 'cacheMiss', o1 => - o1.cacheMiss('getSnapshot') - ) - - // The document isn't loaded and isn't being loaded. Load it. - awaitingGetSnapshot[docName] = [callback] - return db.getSnapshot(docName, function (error, data, dbMeta) { - if (error) { - return add(docName, error) - } - - const type = types[data.type] - if (!type) { - console.warn(`Type '${data.type}' missing`) - return callback('Type not found') - } - data.type = type - - const committedVersion = data.v - - // The server can close without saving the most recent document snapshot. - // In this case, there are extra ops which need to be applied before - // returning the snapshot. - return getOpsInternal(docName, data.v, null, function (error, ops) { - if (error) { - return callback(error) - } - - if (ops.length > 0) { - console.log(`Catchup ${docName} ${data.v} -> ${data.v + ops.length}`) - - try { - for (const op of Array.from(ops)) { - data.snapshot = type.apply(data.snapshot, op.op) - data.v++ - } - } catch (e) { - // This should never happen - it indicates that whats in the - // database is invalid. - console.error(`Op data invalid for ${docName}: ${e.stack}`) - return callback('Op data invalid') - } - } - - model.emit('load', docName, data) - return add(docName, error, data, committedVersion, ops, dbMeta) - }) - }) - } - - // This makes sure the cache contains a document. If the doc cache doesn't contain - // a document, it is loaded from the database and stored. - // - // Documents are stored so long as either: - // - They have been accessed within the past #{PERIOD} - // - At least one client has the document open - function refreshReapingTimeout(docName) { - const doc = docs[docName] - if (!doc) { - return - } - - // I want to let the clients list be updated before this is called. - return process.nextTick(function () { - // This is an awkward way to find out the number of clients on a document. If this - // causes performance issues, add a numClients field to the document. - // - // The first check is because its possible that between refreshReapingTimeout being called and this - // event being fired, someone called delete() on the document and hence the doc is something else now. - if ( - doc === docs[docName] && - doc.eventEmitter.listeners('op').length === 0 && - (db || options.forceReaping) && - doc.opQueue.busy === false - ) { - let reapTimer - clearTimeout(doc.reapTimer) - return (doc.reapTimer = reapTimer = - setTimeout( - () => - tryWriteSnapshot(docName, function () { - // If the reaping timeout has been refreshed while we're writing the snapshot, or if we're - // in the middle of applying an operation, don't reap. - if ( - docs[docName].reapTimer === reapTimer && - doc.opQueue.busy === false - ) { - return delete docs[docName] - } - }), - options.reapTime - )) - } - }) - } - - function tryWriteSnapshot(docName, callback) { - if (!db) { - return typeof callback === 'function' ? callback() : undefined - } - - const doc = docs[docName] - - // The doc is closed - if (!doc) { - return typeof callback === 'function' ? callback() : undefined - } - - // The document is already saved. - if (doc.committedVersion === doc.v) { - return typeof callback === 'function' ? callback() : undefined - } - - if (doc.snapshotWriteLock) { - return typeof callback === 'function' - ? callback('Another snapshot write is in progress') - : undefined - } - - doc.snapshotWriteLock = true - - __guardMethod__(options.stats, 'writeSnapshot', o => o.writeSnapshot()) - - const writeSnapshot = - (db != null ? db.writeSnapshot : undefined) || - ((docName, docData, dbMeta, callback) => callback()) - - const data = { - v: doc.v, - meta: doc.meta, - snapshot: doc.snapshot, - // The database doesn't know about object types. - type: doc.type.name, - } - - // Commit snapshot. - return writeSnapshot(docName, data, doc.dbMeta, function (error, dbMeta) { - doc.snapshotWriteLock = false - - // We have to use data.v here because the version in the doc could - // have been updated between the call to writeSnapshot() and now. - doc.committedVersion = data.v - doc.dbMeta = dbMeta - - return typeof callback === 'function' ? callback(error) : undefined - }) - } - - // *** Model interface methods - - // Create a new document. - // - // data should be {snapshot, type, [meta]}. The version of a new document is 0. - this.create = function (docName, type, meta, callback) { - if (typeof meta === 'function') { - ;[meta, callback] = Array.from([{}, meta]) - } - - if (docName.match(/\//)) { - return typeof callback === 'function' - ? callback('Invalid document name') - : undefined - } - if (docs[docName]) { - return typeof callback === 'function' - ? callback('Document already exists') - : undefined - } - - if (typeof type === 'string') { - type = types[type] - } - if (!type) { - return typeof callback === 'function' - ? callback('Type not found') - : undefined - } - - const data = { - snapshot: type.create(), - type: type.name, - meta: meta || {}, - v: 0, - } - - const done = function (error, dbMeta) { - // dbMeta can be used to cache extra state needed by the database to access the document, like an ID or something. - if (error) { - return typeof callback === 'function' ? callback(error) : undefined - } - - // From here on we'll store the object version of the type name. - data.type = type - add(docName, null, data, 0, [], dbMeta) - model.emit('create', docName, data) - return typeof callback === 'function' ? callback() : undefined - } - - if (db) { - return db.create(docName, data, done) - } else { - return done() - } - } - - // Perminantly deletes the specified document. - // If listeners are attached, they are removed. - // - // The callback is called with (error) if there was an error. If error is null / undefined, the - // document was deleted. - // - // WARNING: This isn't well supported throughout the code. (Eg, streaming clients aren't told about the - // deletion. Subsequent op submissions will fail). - this.delete = function (docName, callback) { - const doc = docs[docName] - - if (doc) { - clearTimeout(doc.reapTimer) - delete docs[docName] - } - - const done = function (error) { - if (!error) { - model.emit('delete', docName) - } - return typeof callback === 'function' ? callback(error) : undefined - } - - if (db) { - return db.delete(docName, doc != null ? doc.dbMeta : undefined, done) - } else { - return done(!doc ? 'Document does not exist' : undefined) - } - } - - // This gets all operations from [start...end]. (That is, its not inclusive.) - // - // end can be null. This means 'get me all ops from start'. - // - // Each op returned is in the form {op:o, meta:m, v:version}. - // - // Callback is called with (error, [ops]) - // - // If the document does not exist, getOps doesn't necessarily return an error. This is because - // its awkward to figure out whether or not the document exists for things - // like the redis database backend. I guess its a bit gross having this inconsistant - // with the other DB calls, but its certainly convenient. - // - // Use getVersion() to determine if a document actually exists, if thats what you're - // after. - this.getOps = getOps = function (docName, start, end, callback) { - // getOps will only use the op cache if its there. It won't fill the op cache in. - if (!(start >= 0)) { - throw new Error('start must be 0+') - } - - if (typeof end === 'function') { - ;[end, callback] = Array.from([null, end]) - } - - const ops = docs[docName] != null ? docs[docName].ops : undefined - - if (ops) { - const version = docs[docName].v - - // Ops contains an array of ops. The last op in the list is the last op applied - if (end == null) { - end = version - } - start = Math.min(start, end) - - if (start === end) { - return callback(null, []) - } - - // Base is the version number of the oldest op we have cached - const base = version - ops.length - - // If the database is null, we'll trim to the ops we do have and hope thats enough. - if (start >= base || db === null) { - refreshReapingTimeout(docName) - if (options.stats != null) { - options.stats.cacheHit('getOps') - } - - return callback(null, ops.slice(start - base, end - base)) - } - } - - if (options.stats != null) { - options.stats.cacheMiss('getOps') - } - - return getOpsInternal(docName, start, end, callback) - } - - // Gets the snapshot data for the specified document. - // getSnapshot(docName, callback) - // Callback is called with (error, {v: , type: , snapshot: , meta: }) - this.getSnapshot = (docName, callback) => - load(docName, (error, doc) => - callback( - error, - doc - ? { v: doc.v, type: doc.type, snapshot: doc.snapshot, meta: doc.meta } - : undefined - ) - ) - - // Gets the latest version # of the document. - // getVersion(docName, callback) - // callback is called with (error, version). - this.getVersion = (docName, callback) => - load(docName, (error, doc) => - callback(error, doc != null ? doc.v : undefined) - ) - - // Apply an op to the specified document. - // The callback is passed (error, applied version #) - // opData = {op:op, v:v, meta:metadata} - // - // Ops are queued before being applied so that the following code applies op C before op B: - // model.applyOp 'doc', OPA, -> model.applyOp 'doc', OPB - // model.applyOp 'doc', OPC - this.applyOp = ( - docName, - opData, - callback // All the logic for this is in makeOpQueue, above. - ) => - load(docName, function (error, doc) { - if (error) { - return callback(error) - } - - return process.nextTick(() => - doc.opQueue(opData, function (error, newVersion) { - refreshReapingTimeout(docName) - return typeof callback === 'function' - ? callback(error, newVersion) - : undefined - }) - ) - }) - - // TODO: store (some) metadata in DB - // TODO: op and meta should be combineable in the op that gets sent - this.applyMetaOp = function (docName, metaOpData, callback) { - const { path, value } = metaOpData.meta - - if (!isArray(path)) { - return typeof callback === 'function' - ? callback('path should be an array') - : undefined - } - - return load(docName, function (error, doc) { - if (error != null) { - return typeof callback === 'function' ? callback(error) : undefined - } else { - let applied = false - switch (path[0]) { - case 'shout': - doc.eventEmitter.emit('op', metaOpData) - applied = true - break - } - - if (applied) { - model.emit('applyMetaOp', docName, path, value) - } - return typeof callback === 'function' - ? callback(null, doc.v) - : undefined - } - }) - } - - // Listen to all ops from the specified version. If version is in the past, all - // ops since that version are sent immediately to the listener. - // - // The callback is called once the listener is attached, but before any ops have been passed - // to the listener. - // - // This will _not_ edit the document metadata. - // - // If there are any listeners, we don't purge the document from the cache. But be aware, this behaviour - // might change in a future version. - // - // version is the document version at which the document is opened. It can be left out if you want to open - // the document at the most recent version. - // - // listener is called with (opData) each time an op is applied. - // - // callback(error, openedVersion) - this.listen = function (docName, version, listener, callback) { - if (typeof version === 'function') { - ;[version, listener, callback] = Array.from([null, version, listener]) - } - - return load(docName, function (error, doc) { - if (error) { - return typeof callback === 'function' ? callback(error) : undefined - } - - clearTimeout(doc.reapTimer) - - if (version != null) { - return getOps(docName, version, null, function (error, data) { - if (error) { - return typeof callback === 'function' ? callback(error) : undefined - } - - doc.eventEmitter.on('op', listener) - if (typeof callback === 'function') { - callback(null, version) - } - return (() => { - const result = [] - for (const op of Array.from(data)) { - let needle - listener(op) - - // The listener may well remove itself during the catchup phase. If this happens, break early. - // This is done in a quite inefficient way. (O(n) where n = #listeners on doc) - if ( - ((needle = listener), - !Array.from(doc.eventEmitter.listeners('op')).includes(needle)) - ) { - break - } else { - result.push(undefined) - } - } - return result - })() - }) - } else { - // Version is null / undefined. Just add the listener. - doc.eventEmitter.on('op', listener) - return typeof callback === 'function' - ? callback(null, doc.v) - : undefined - } - }) - } - - // Remove a listener for a particular document. - // - // removeListener(docName, listener) - // - // This is synchronous. - this.removeListener = function (docName, listener) { - // The document should already be loaded. - const doc = docs[docName] - if (!doc) { - throw new Error('removeListener called but document not loaded') - } - - doc.eventEmitter.removeListener('op', listener) - return refreshReapingTimeout(docName) - } - - // Flush saves all snapshot data to the database. I'm not sure whether or not this is actually needed - - // sharejs will happily replay uncommitted ops when documents are re-opened anyway. - this.flush = function (callback) { - if (!db) { - return typeof callback === 'function' ? callback() : undefined - } - - let pendingWrites = 0 - - for (const docName in docs) { - const doc = docs[docName] - if (doc.committedVersion < doc.v) { - pendingWrites++ - // I'm hoping writeSnapshot will always happen in another thread. - tryWriteSnapshot(docName, () => - process.nextTick(function () { - pendingWrites-- - if (pendingWrites === 0) { - return typeof callback === 'function' ? callback() : undefined - } - }) - ) - } - } - - // If nothing was queued, terminate immediately. - if (pendingWrites === 0) { - return typeof callback === 'function' ? callback() : undefined - } - } - - // Close the database connection. This is needed so nodejs can shut down cleanly. - this.closeDb = function () { - __guardMethod__(db, 'close', o => o.close()) - return (db = null) - } -} - -// Model inherits from EventEmitter. -Model.prototype = new EventEmitter() - -function __guardMethod__(obj, methodName, transform) { - if ( - typeof obj !== 'undefined' && - obj !== null && - typeof obj[methodName] === 'function' - ) { - return transform(obj, methodName) - } else { - return undefined - } -} diff --git a/services/document-updater/app/js/sharejs/simple.js b/services/document-updater/app/js/sharejs/simple.js deleted file mode 100644 index 41f7eed285..0000000000 --- a/services/document-updater/app/js/sharejs/simple.js +++ /dev/null @@ -1,54 +0,0 @@ -// TODO: This file was created by bulk-decaffeinate. -// Sanity-check the conversion and remove this comment. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// This is a really simple OT type. Its not compiled with the web client, but it could be. -// -// Its mostly included for demonstration purposes and its used in a lot of unit tests. -// -// This defines a really simple text OT type which only allows inserts. (No deletes). -// -// Ops look like: -// {position:#, text:"asdf"} -// -// Document snapshots look like: -// {str:string} - -module.exports = { - // The name of the OT type. The type is stored in types[type.name]. The name can be - // used in place of the actual type in all the API methods. - name: 'simple', - - // Create a new document snapshot - create() { - return { str: '' } - }, - - // Apply the given op to the document snapshot. Returns the new snapshot. - // - // The original snapshot should not be modified. - apply(snapshot, op) { - if (!(op.position >= 0 && op.position <= snapshot.str.length)) { - throw new Error('Invalid position') - } - - let { str } = snapshot - str = str.slice(0, op.position) + op.text + str.slice(op.position) - return { str } - }, - - // transform op1 by op2. Return transformed version of op1. - // sym describes the symmetry of the op. Its 'left' or 'right' depending on whether the - // op being transformed comes from the client or the server. - transform(op1, op2, sym) { - let pos = op1.position - if (op2.position < pos || (op2.position === pos && sym === 'left')) { - pos += op2.text.length - } - - return { position: pos, text: op1.text } - }, -} diff --git a/services/document-updater/app/js/sharejs/syncqueue.js b/services/document-updater/app/js/sharejs/syncqueue.js deleted file mode 100644 index 77959230b6..0000000000 --- a/services/document-updater/app/js/sharejs/syncqueue.js +++ /dev/null @@ -1,60 +0,0 @@ -// TODO: This file was created by bulk-decaffeinate. -// Sanity-check the conversion and remove this comment. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// A synchronous processing queue. The queue calls process on the arguments, -// ensuring that process() is only executing once at a time. -// -// process(data, callback) _MUST_ eventually call its callback. -// -// Example: -// -// queue = require 'syncqueue' -// -// fn = queue (data, callback) -> -// asyncthing data, -> -// callback(321) -// -// fn(1) -// fn(2) -// fn(3, (result) -> console.log(result)) -// -// ^--- async thing will only be running once at any time. - -module.exports = function (process) { - if (typeof process !== 'function') { - throw new Error('process is not a function') - } - const queue = [] - - const enqueue = function (data, callback) { - queue.push([data, callback]) - return flush() - } - - enqueue.busy = false - - function flush() { - if (enqueue.busy || queue.length === 0) { - return - } - - enqueue.busy = true - const [data, callback] = Array.from(queue.shift()) - return process(data, function (...result) { - // TODO: Make this not use varargs - varargs are really slow. - enqueue.busy = false - // This is called after busy = false so a user can check if enqueue.busy is set in the callback. - if (callback) { - callback.apply(null, result) - } - return flush() - }) - } - - return enqueue -} diff --git a/services/document-updater/app/js/sharejs/text-api.js b/services/document-updater/app/js/sharejs/text-api.js deleted file mode 100644 index aa2beef446..0000000000 --- a/services/document-updater/app/js/sharejs/text-api.js +++ /dev/null @@ -1,52 +0,0 @@ -// TODO: This file was created by bulk-decaffeinate. -// Sanity-check the conversion and remove this comment. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// Text document API for text - -let text -if (typeof WEB === 'undefined') { - text = require('./text') -} - -text.api = { - provides: { text: true }, - - // The number of characters in the string - getLength() { - return this.snapshot.length - }, - - // Get the text contents of a document - getText() { - return this.snapshot - }, - - insert(pos, text, callback) { - const op = [{ p: pos, i: text }] - - this.submitOp(op, callback) - return op - }, - - del(pos, length, callback) { - const op = [{ p: pos, d: this.snapshot.slice(pos, pos + length) }] - - this.submitOp(op, callback) - return op - }, - - _register() { - return this.on('remoteop', function (op) { - return Array.from(op).map(component => - component.i !== undefined - ? this.emit('insert', component.p, component.i) - : this.emit('delete', component.p, component.d) - ) - }) - }, -} diff --git a/services/document-updater/app/js/sharejs/text-composable-api.js b/services/document-updater/app/js/sharejs/text-composable-api.js deleted file mode 100644 index 122e119ae4..0000000000 --- a/services/document-updater/app/js/sharejs/text-composable-api.js +++ /dev/null @@ -1,76 +0,0 @@ -/* eslint-disable - no-undef, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS205: Consider reworking code to avoid use of IIFEs - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// Text document API for text - -let type -if (typeof WEB !== 'undefined' && WEB !== null) { - type = exports.types['text-composable'] -} else { - type = require('./text-composable') -} - -type.api = { - provides: { text: true }, - - // The number of characters in the string - getLength() { - return this.snapshot.length - }, - - // Get the text contents of a document - getText() { - return this.snapshot - }, - - insert(pos, text, callback) { - const op = type.normalize([pos, { i: text }, this.snapshot.length - pos]) - - this.submitOp(op, callback) - return op - }, - - del(pos, length, callback) { - const op = type.normalize([ - pos, - { d: this.snapshot.slice(pos, pos + length) }, - this.snapshot.length - pos - length, - ]) - - this.submitOp(op, callback) - return op - }, - - _register() { - return this.on('remoteop', function (op) { - let pos = 0 - return (() => { - const result = [] - for (const component of Array.from(op)) { - if (typeof component === 'number') { - result.push((pos += component)) - } else if (component.i !== undefined) { - this.emit('insert', pos, component.i) - result.push((pos += component.i.length)) - } else { - // delete - result.push(this.emit('delete', pos, component.d)) - } - } - return result - })() - }) - }, -} -// We don't increment pos, because the position -// specified is after the delete has happened. diff --git a/services/document-updater/app/js/sharejs/text-composable.js b/services/document-updater/app/js/sharejs/text-composable.js deleted file mode 100644 index f5e9d91181..0000000000 --- a/services/document-updater/app/js/sharejs/text-composable.js +++ /dev/null @@ -1,401 +0,0 @@ -/* eslint-disable - no-cond-assign, - no-return-assign, - no-undef, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS205: Consider reworking code to avoid use of IIFEs - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// An alternate composable implementation for text. This is much closer -// to the implementation used by google wave. -// -// Ops are lists of components which iterate over the whole document. -// Components are either: -// A number N: Skip N characters in the original document -// {i:'str'}: Insert 'str' at the current position in the document -// {d:'str'}: Delete 'str', which appears at the current position in the document -// -// Eg: [3, {i:'hi'}, 5, {d:'internet'}] -// -// Snapshots are strings. - -let makeAppend -const p = function () {} // require('util').debug -const i = function () {} // require('util').inspect - -const exports = typeof WEB !== 'undefined' && WEB !== null ? {} : module.exports - -exports.name = 'text-composable' - -exports.create = () => '' - -// -------- Utility methods - -const checkOp = function (op) { - if (!Array.isArray(op)) { - throw new Error('Op must be an array of components') - } - let last = null - return (() => { - const result = [] - for (const c of Array.from(op)) { - if (typeof c === 'object') { - if ( - (c.i == null || !(c.i.length > 0)) && - (c.d == null || !(c.d.length > 0)) - ) { - throw new Error(`Invalid op component: ${i(c)}`) - } - } else { - if (typeof c !== 'number') { - throw new Error('Op components must be objects or numbers') - } - if (!(c > 0)) { - throw new Error('Skip components must be a positive number') - } - if (typeof last === 'number') { - throw new Error('Adjacent skip components should be added') - } - } - - result.push((last = c)) - } - return result - })() -} - -// Makes a function for appending components to a given op. -// Exported for the randomOpGenerator. -exports._makeAppend = makeAppend = op => - function (component) { - if (component === 0 || component.i === '' || component.d === '') { - return - } - if (op.length === 0) { - return op.push(component) - } else if ( - typeof component === 'number' && - typeof op[op.length - 1] === 'number' - ) { - return (op[op.length - 1] += component) - } else if (component.i != null && op[op.length - 1].i != null) { - return (op[op.length - 1].i += component.i) - } else if (component.d != null && op[op.length - 1].d != null) { - return (op[op.length - 1].d += component.d) - } else { - return op.push(component) - } - } - -// checkOp op - -// Makes 2 functions for taking components from the start of an op, and for peeking -// at the next op that could be taken. -const makeTake = function (op) { - // The index of the next component to take - let idx = 0 - // The offset into the component - let offset = 0 - - // Take up to length n from the front of op. If n is null, take the next - // op component. If indivisableField == 'd', delete components won't be separated. - // If indivisableField == 'i', insert components won't be separated. - const take = function (n, indivisableField) { - let c - if (idx === op.length) { - return null - } - // assert.notStrictEqual op.length, i, 'The op is too short to traverse the document' - - if (typeof op[idx] === 'number') { - if (n == null || op[idx] - offset <= n) { - c = op[idx] - offset - ++idx - offset = 0 - return c - } else { - offset += n - return n - } - } else { - // Take from the string - const field = op[idx].i ? 'i' : 'd' - c = {} - if ( - n == null || - op[idx][field].length - offset <= n || - field === indivisableField - ) { - c[field] = op[idx][field].slice(offset) - ++idx - offset = 0 - } else { - c[field] = op[idx][field].slice(offset, offset + n) - offset += n - } - return c - } - } - - const peekType = () => op[idx] - - return [take, peekType] -} - -// Find and return the length of an op component -const componentLength = function (component) { - if (typeof component === 'number') { - return component - } else if (component.i != null) { - return component.i.length - } else { - return component.d.length - } -} - -// Normalize an op, removing all empty skips and empty inserts / deletes. Concatenate -// adjacent inserts and deletes. -exports.normalize = function (op) { - const newOp = [] - const append = makeAppend(newOp) - for (const component of Array.from(op)) { - append(component) - } - return newOp -} - -// Apply the op to the string. Returns the new string. -exports.apply = function (str, op) { - p(`Applying ${i(op)} to '${str}'`) - if (typeof str !== 'string') { - throw new Error('Snapshot should be a string') - } - checkOp(op) - - const pos = 0 - const newDoc = [] - - for (const component of Array.from(op)) { - if (typeof component === 'number') { - if (component > str.length) { - throw new Error('The op is too long for this document') - } - newDoc.push(str.slice(0, component)) - str = str.slice(component) - } else if (component.i != null) { - newDoc.push(component.i) - } else { - if (component.d !== str.slice(0, component.d.length)) { - throw new Error( - `The deleted text '${ - component.d - }' doesn't match the next characters in the document '${str.slice( - 0, - component.d.length - )}'` - ) - } - str = str.slice(component.d.length) - } - } - - if (str !== '') { - throw new Error("The applied op doesn't traverse the entire document") - } - - return newDoc.join('') -} - -// transform op1 by op2. Return transformed version of op1. -// op1 and op2 are unchanged by transform. -exports.transform = function (op, otherOp, side) { - let component - if (side !== 'left' && side !== 'right') { - throw new Error(`side (${side} must be 'left' or 'right'`) - } - - checkOp(op) - checkOp(otherOp) - const newOp = [] - - const append = makeAppend(newOp) - const [take, peek] = Array.from(makeTake(op)) - - for (component of Array.from(otherOp)) { - let chunk, length - if (typeof component === 'number') { - // Skip - length = component - while (length > 0) { - chunk = take(length, 'i') - if (chunk === null) { - throw new Error( - 'The op traverses more elements than the document has' - ) - } - - append(chunk) - if (typeof chunk !== 'object' || chunk.i == null) { - length -= componentLength(chunk) - } - } - } else if (component.i != null) { - // Insert - if (side === 'left') { - // The left insert should go first. - const o = peek() - if (o != null ? o.i : undefined) { - append(take()) - } - } - - // Otherwise, skip the inserted text. - append(component.i.length) - } else { - // Delete. - // assert.ok component.d - ;({ length } = component.d) - while (length > 0) { - chunk = take(length, 'i') - if (chunk === null) { - throw new Error( - 'The op traverses more elements than the document has' - ) - } - - if (typeof chunk === 'number') { - length -= chunk - } else if (chunk.i != null) { - append(chunk) - } else { - // assert.ok chunk.d - // The delete is unnecessary now. - length -= chunk.d.length - } - } - } - } - - // Append extras from op1 - while ((component = take())) { - if ((component != null ? component.i : undefined) == null) { - throw new Error(`Remaining fragments in the op: ${i(component)}`) - } - append(component) - } - - return newOp -} - -// Compose 2 ops into 1 op. -exports.compose = function (op1, op2) { - let component - p(`COMPOSE ${i(op1)} + ${i(op2)}`) - checkOp(op1) - checkOp(op2) - - const result = [] - - const append = makeAppend(result) - const [take, _] = Array.from(makeTake(op1)) - - for (component of Array.from(op2)) { - let chunk, length - if (typeof component === 'number') { - // Skip - length = component - while (length > 0) { - chunk = take(length, 'd') - if (chunk === null) { - throw new Error( - 'The op traverses more elements than the document has' - ) - } - - append(chunk) - if (typeof chunk !== 'object' || chunk.d == null) { - length -= componentLength(chunk) - } - } - } else if (component.i != null) { - // Insert - append({ i: component.i }) - } else { - // Delete - let offset = 0 - while (offset < component.d.length) { - chunk = take(component.d.length - offset, 'd') - if (chunk === null) { - throw new Error( - 'The op traverses more elements than the document has' - ) - } - - // If its delete, append it. If its skip, drop it and decrease length. If its insert, check the strings match, drop it and decrease length. - if (typeof chunk === 'number') { - append({ d: component.d.slice(offset, offset + chunk) }) - offset += chunk - } else if (chunk.i != null) { - if (component.d.slice(offset, offset + chunk.i.length) !== chunk.i) { - throw new Error("The deleted text doesn't match the inserted text") - } - offset += chunk.i.length - // The ops cancel each other out. - } else { - // Delete - append(chunk) - } - } - } - } - - // Append extras from op1 - while ((component = take())) { - if ((component != null ? component.d : undefined) == null) { - throw new Error(`Trailing stuff in op1 ${i(component)}`) - } - append(component) - } - - return result -} - -const invertComponent = function (c) { - if (typeof c === 'number') { - return c - } else if (c.i != null) { - return { d: c.i } - } else { - return { i: c.d } - } -} - -// Invert an op -exports.invert = function (op) { - const result = [] - const append = makeAppend(result) - - for (const component of Array.from(op)) { - append(invertComponent(component)) - } - - return result -} - -if (typeof window !== 'undefined' && window !== null) { - if (!window.ot) { - window.ot = {} - } - if (!window.ot.types) { - window.ot.types = {} - } - window.ot.types.text = exports -} diff --git a/services/document-updater/app/js/sharejs/text-tp2-api.js b/services/document-updater/app/js/sharejs/text-tp2-api.js deleted file mode 100644 index 1e1b40d176..0000000000 --- a/services/document-updater/app/js/sharejs/text-tp2-api.js +++ /dev/null @@ -1,133 +0,0 @@ -/* eslint-disable - no-undef, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS205: Consider reworking code to avoid use of IIFEs - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// Text document API for text-tp2 - -let type -if (typeof WEB !== 'undefined' && WEB !== null) { - type = exports.types['text-tp2'] -} else { - type = require('./text-tp2') -} - -const { _takeDoc: takeDoc, _append: append } = type - -const appendSkipChars = (op, doc, pos, maxlength) => - (() => { - const result = [] - while ( - (maxlength === undefined || maxlength > 0) && - pos.index < doc.data.length - ) { - const part = takeDoc(doc, pos, maxlength, true) - if (maxlength !== undefined && typeof part === 'string') { - maxlength -= part.length - } - result.push(append(op, part.length || part)) - } - return result - })() - -type.api = { - provides: { text: true }, - - // The number of characters in the string - getLength() { - return this.snapshot.charLength - }, - - // Flatten a document into a string - getText() { - const strings = Array.from(this.snapshot.data).filter( - elem => typeof elem === 'string' - ) - return strings.join('') - }, - - insert(pos, text, callback) { - if (pos === undefined) { - pos = 0 - } - - const op = [] - const docPos = { index: 0, offset: 0 } - - appendSkipChars(op, this.snapshot, docPos, pos) - append(op, { i: text }) - appendSkipChars(op, this.snapshot, docPos) - - this.submitOp(op, callback) - return op - }, - - del(pos, length, callback) { - const op = [] - const docPos = { index: 0, offset: 0 } - - appendSkipChars(op, this.snapshot, docPos, pos) - - while (length > 0) { - const part = takeDoc(this.snapshot, docPos, length, true) - if (typeof part === 'string') { - append(op, { d: part.length }) - length -= part.length - } else { - append(op, part) - } - } - - appendSkipChars(op, this.snapshot, docPos) - - this.submitOp(op, callback) - return op - }, - - _register() { - // Interpret recieved ops + generate more detailed events for them - return this.on('remoteop', function (op, snapshot) { - let textPos = 0 - const docPos = { index: 0, offset: 0 } - - for (const component of Array.from(op)) { - let part, remainder - if (typeof component === 'number') { - // Skip - remainder = component - while (remainder > 0) { - part = takeDoc(snapshot, docPos, remainder) - if (typeof part === 'string') { - textPos += part.length - } - remainder -= part.length || part - } - } else if (component.i !== undefined) { - // Insert - if (typeof component.i === 'string') { - this.emit('insert', textPos, component.i) - textPos += component.i.length - } - } else { - // Delete - remainder = component.d - while (remainder > 0) { - part = takeDoc(snapshot, docPos, remainder) - if (typeof part === 'string') { - this.emit('delete', textPos, part) - } - remainder -= part.length || part - } - } - } - }) - }, -} diff --git a/services/document-updater/app/js/sharejs/text-tp2.js b/services/document-updater/app/js/sharejs/text-tp2.js deleted file mode 100644 index e2a40cf6bc..0000000000 --- a/services/document-updater/app/js/sharejs/text-tp2.js +++ /dev/null @@ -1,499 +0,0 @@ -/* eslint-disable - no-cond-assign, - no-return-assign, - no-undef, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS103: Rewrite code to no longer use __guard__ - * DS205: Consider reworking code to avoid use of IIFEs - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// A TP2 implementation of text, following this spec: -// http://code.google.com/p/lightwave/source/browse/trunk/experimental/ot/README -// -// A document is made up of a string and a set of tombstones inserted throughout -// the string. For example, 'some ', (2 tombstones), 'string'. -// -// This is encoded in a document as: {s:'some string', t:[5, -2, 6]} -// -// Ops are lists of components which iterate over the whole document. -// Components are either: -// N: Skip N characters in the original document -// {i:'str'}: Insert 'str' at the current position in the document -// {i:N}: Insert N tombstones at the current position in the document -// {d:N}: Delete (tombstone) N characters at the current position in the document -// -// Eg: [3, {i:'hi'}, 5, {d:8}] -// -// Snapshots are lists with characters and tombstones. Characters are stored in strings -// and adjacent tombstones are flattened into numbers. -// -// Eg, the document: 'Hello .....world' ('.' denotes tombstoned (deleted) characters) -// would be represented by a document snapshot of ['Hello ', 5, 'world'] - -let append, appendDoc, takeDoc -const type = { - name: 'text-tp2', - tp2: true, - create() { - return { charLength: 0, totalLength: 0, positionCache: [], data: [] } - }, - serialize(doc) { - if (!doc.data) { - throw new Error('invalid doc snapshot') - } - return doc.data - }, - deserialize(data) { - const doc = type.create() - doc.data = data - - for (const component of Array.from(data)) { - if (typeof component === 'string') { - doc.charLength += component.length - doc.totalLength += component.length - } else { - doc.totalLength += component - } - } - - return doc - }, -} - -const checkOp = function (op) { - if (!Array.isArray(op)) { - throw new Error('Op must be an array of components') - } - let last = null - return (() => { - const result = [] - for (const c of Array.from(op)) { - if (typeof c === 'object') { - if (c.i !== undefined) { - if ( - (typeof c.i !== 'string' || !(c.i.length > 0)) && - (typeof c.i !== 'number' || !(c.i > 0)) - ) { - throw new Error('Inserts must insert a string or a +ive number') - } - } else if (c.d !== undefined) { - if (typeof c.d !== 'number' || !(c.d > 0)) { - throw new Error('Deletes must be a +ive number') - } - } else { - throw new Error('Operation component must define .i or .d') - } - } else { - if (typeof c !== 'number') { - throw new Error('Op components must be objects or numbers') - } - if (!(c > 0)) { - throw new Error('Skip components must be a positive number') - } - if (typeof last === 'number') { - throw new Error('Adjacent skip components should be combined') - } - } - - result.push((last = c)) - } - return result - })() -} - -// Take the next part from the specified position in a document snapshot. -// position = {index, offset}. It will be updated. -type._takeDoc = takeDoc = function ( - doc, - position, - maxlength, - tombsIndivisible -) { - if (position.index >= doc.data.length) { - throw new Error('Operation goes past the end of the document') - } - - const part = doc.data[position.index] - // peel off data[0] - const result = - typeof part === 'string' - ? maxlength !== undefined - ? part.slice(position.offset, position.offset + maxlength) - : part.slice(position.offset) - : maxlength === undefined || tombsIndivisible - ? part - position.offset - : Math.min(maxlength, part - position.offset) - - const resultLen = result.length || result - - if ((part.length || part) - position.offset > resultLen) { - position.offset += resultLen - } else { - position.index++ - position.offset = 0 - } - - return result -} - -// Append a part to the end of a document -type._appendDoc = appendDoc = function (doc, p) { - if (p === 0 || p === '') { - return - } - - if (typeof p === 'string') { - doc.charLength += p.length - doc.totalLength += p.length - } else { - doc.totalLength += p - } - - const { data } = doc - if (data.length === 0) { - data.push(p) - } else if (typeof data[data.length - 1] === typeof p) { - data[data.length - 1] += p - } else { - data.push(p) - } -} - -// Apply the op to the document. The document is not modified in the process. -type.apply = function (doc, op) { - if ( - doc.totalLength === undefined || - doc.charLength === undefined || - doc.data.length === undefined - ) { - throw new Error('Snapshot is invalid') - } - - checkOp(op) - - const newDoc = type.create() - const position = { index: 0, offset: 0 } - - for (const component of Array.from(op)) { - let part, remainder - if (typeof component === 'number') { - remainder = component - while (remainder > 0) { - part = takeDoc(doc, position, remainder) - - appendDoc(newDoc, part) - remainder -= part.length || part - } - } else if (component.i !== undefined) { - appendDoc(newDoc, component.i) - } else if (component.d !== undefined) { - remainder = component.d - while (remainder > 0) { - part = takeDoc(doc, position, remainder) - remainder -= part.length || part - } - appendDoc(newDoc, component.d) - } - } - - return newDoc -} - -// Append an op component to the end of the specified op. -// Exported for the randomOpGenerator. -type._append = append = function (op, component) { - if ( - component === 0 || - component.i === '' || - component.i === 0 || - component.d === 0 - ) { - return - } - if (op.length === 0) { - return op.push(component) - } else { - const last = op[op.length - 1] - if (typeof component === 'number' && typeof last === 'number') { - return (op[op.length - 1] += component) - } else if ( - component.i !== undefined && - last.i != null && - typeof last.i === typeof component.i - ) { - return (last.i += component.i) - } else if (component.d !== undefined && last.d != null) { - return (last.d += component.d) - } else { - return op.push(component) - } - } -} - -// Makes 2 functions for taking components from the start of an op, and for peeking -// at the next op that could be taken. -const makeTake = function (op) { - // The index of the next component to take - let index = 0 - // The offset into the component - let offset = 0 - - // Take up to length maxlength from the op. If maxlength is not defined, there is no max. - // If insertsIndivisible is true, inserts (& insert tombstones) won't be separated. - // - // Returns null when op is fully consumed. - const take = function (maxlength, insertsIndivisible) { - let current - if (index === op.length) { - return null - } - - const e = op[index] - if ( - typeof (current = e) === 'number' || - typeof (current = e.i) === 'number' || - (current = e.d) !== undefined - ) { - let c - if ( - maxlength == null || - current - offset <= maxlength || - (insertsIndivisible && e.i !== undefined) - ) { - // Return the rest of the current element. - c = current - offset - ++index - offset = 0 - } else { - offset += maxlength - c = maxlength - } - if (e.i !== undefined) { - return { i: c } - } else if (e.d !== undefined) { - return { d: c } - } else { - return c - } - } else { - // Take from the inserted string - let result - if ( - maxlength == null || - e.i.length - offset <= maxlength || - insertsIndivisible - ) { - result = { i: e.i.slice(offset) } - ++index - offset = 0 - } else { - result = { i: e.i.slice(offset, offset + maxlength) } - offset += maxlength - } - return result - } - } - - const peekType = () => op[index] - - return [take, peekType] -} - -// Find and return the length of an op component -const componentLength = function (component) { - if (typeof component === 'number') { - return component - } else if (typeof component.i === 'string') { - return component.i.length - } else { - // This should work because c.d and c.i must be +ive. - return component.d || component.i - } -} - -// Normalize an op, removing all empty skips and empty inserts / deletes. Concatenate -// adjacent inserts and deletes. -type.normalize = function (op) { - const newOp = [] - for (const component of Array.from(op)) { - append(newOp, component) - } - return newOp -} - -// This is a helper method to transform and prune. goForwards is true for transform, false for prune. -const transformer = function (op, otherOp, goForwards, side) { - let component - checkOp(op) - checkOp(otherOp) - const newOp = [] - - const [take, peek] = Array.from(makeTake(op)) - - for (component of Array.from(otherOp)) { - let chunk - let length = componentLength(component) - - if (component.i !== undefined) { - // Insert text or tombs - if (goForwards) { - // transform - insert skips over inserted parts - if (side === 'left') { - // The left insert should go first. - while (__guard__(peek(), x => x.i) !== undefined) { - append(newOp, take()) - } - } - - // In any case, skip the inserted text. - append(newOp, length) - } else { - // Prune. Remove skips for inserts. - while (length > 0) { - chunk = take(length, true) - - if (chunk === null) { - throw new Error('The transformed op is invalid') - } - if (chunk.d !== undefined) { - throw new Error( - 'The transformed op deletes locally inserted characters - it cannot be purged of the insert.' - ) - } - - if (typeof chunk === 'number') { - length -= chunk - } else { - append(newOp, chunk) - } - } - } - } else { - // Skip or delete - while (length > 0) { - chunk = take(length, true) - if (chunk === null) { - throw new Error( - 'The op traverses more elements than the document has' - ) - } - - append(newOp, chunk) - if (!chunk.i) { - length -= componentLength(chunk) - } - } - } - } - - // Append extras from op1 - while ((component = take())) { - if (component.i === undefined) { - throw new Error(`Remaining fragments in the op: ${component}`) - } - append(newOp, component) - } - - return newOp -} - -// transform op1 by op2. Return transformed version of op1. -// op1 and op2 are unchanged by transform. -// side should be 'left' or 'right', depending on if op1.id <> op2.id. 'left' == client op. -type.transform = function (op, otherOp, side) { - if (side !== 'left' && side !== 'right') { - throw new Error(`side (${side}) should be 'left' or 'right'`) - } - return transformer(op, otherOp, true, side) -} - -// Prune is the inverse of transform. -type.prune = (op, otherOp) => transformer(op, otherOp, false) - -// Compose 2 ops into 1 op. -type.compose = function (op1, op2) { - let component - if (op1 === null || op1 === undefined) { - return op2 - } - - checkOp(op1) - checkOp(op2) - - const result = [] - - const [take, _] = Array.from(makeTake(op1)) - - for (component of Array.from(op2)) { - let chunk, length - if (typeof component === 'number') { - // Skip - // Just copy from op1. - length = component - while (length > 0) { - chunk = take(length) - if (chunk === null) { - throw new Error( - 'The op traverses more elements than the document has' - ) - } - - append(result, chunk) - length -= componentLength(chunk) - } - } else if (component.i !== undefined) { - // Insert - append(result, { i: component.i }) - } else { - // Delete - length = component.d - while (length > 0) { - chunk = take(length) - if (chunk === null) { - throw new Error( - 'The op traverses more elements than the document has' - ) - } - - const chunkLength = componentLength(chunk) - if (chunk.i !== undefined) { - append(result, { i: chunkLength }) - } else { - append(result, { d: chunkLength }) - } - - length -= chunkLength - } - } - } - - // Append extras from op1 - while ((component = take())) { - if (component.i === undefined) { - throw new Error(`Remaining fragments in op1: ${component}`) - } - append(result, component) - } - - return result -} - -if (typeof WEB !== 'undefined' && WEB !== null) { - exports.types['text-tp2'] = type -} else { - module.exports = type -} - -function __guard__(value, transform) { - return typeof value !== 'undefined' && value !== null - ? transform(value) - : undefined -} diff --git a/services/document-updater/app/js/sharejs/text.js b/services/document-updater/app/js/sharejs/text.js deleted file mode 100644 index 254cab572b..0000000000 --- a/services/document-updater/app/js/sharejs/text.js +++ /dev/null @@ -1,313 +0,0 @@ -/* eslint-disable - no-return-assign, - no-undef, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// A simple text implementation -// -// Operations are lists of components. -// Each component either inserts or deletes at a specified position in the document. -// -// Components are either: -// {i:'str', p:100}: Insert 'str' at position 100 in the document -// {d:'str', p:100}: Delete 'str' at position 100 in the document -// -// Components in an operation are executed sequentially, so the position of components -// assumes previous components have already executed. -// -// Eg: This op: -// [{i:'abc', p:0}] -// is equivalent to this op: -// [{i:'a', p:0}, {i:'b', p:1}, {i:'c', p:2}] - -// NOTE: The global scope here is shared with other sharejs files when built with closure. -// Be careful what ends up in your namespace. - -let append, transformComponent -const text = {} - -text.name = 'text' - -text.create = () => '' - -const strInject = (s1, pos, s2) => s1.slice(0, pos) + s2 + s1.slice(pos) - -const checkValidComponent = function (c) { - if (typeof c.p !== 'number') { - throw new Error('component missing position field') - } - - const iType = typeof c.i - const dType = typeof c.d - if (!((iType === 'string') ^ (dType === 'string'))) { - throw new Error('component needs an i or d field') - } - - if (!(c.p >= 0)) { - throw new Error('position cannot be negative') - } -} - -const checkValidOp = function (op) { - for (const c of Array.from(op)) { - checkValidComponent(c) - } - return true -} - -text.apply = function (snapshot, op) { - checkValidOp(op) - for (const component of Array.from(op)) { - if (component.i != null) { - snapshot = strInject(snapshot, component.p, component.i) - } else { - const deleted = snapshot.slice( - component.p, - component.p + component.d.length - ) - if (component.d !== deleted) { - throw new Error( - `Delete component '${component.d}' does not match deleted text '${deleted}'` - ) - } - snapshot = - snapshot.slice(0, component.p) + - snapshot.slice(component.p + component.d.length) - } - } - - return snapshot -} - -// Exported for use by the random op generator. -// -// For simplicity, this version of append does not compress adjacent inserts and deletes of -// the same text. It would be nice to change that at some stage. -text._append = append = function (newOp, c) { - if (c.i === '' || c.d === '') { - return - } - if (newOp.length === 0) { - return newOp.push(c) - } else { - const last = newOp[newOp.length - 1] - - // Compose the insert into the previous insert if possible - if ( - last.i != null && - c.i != null && - last.p <= c.p && - c.p <= last.p + last.i.length - ) { - return (newOp[newOp.length - 1] = { - i: strInject(last.i, c.p - last.p, c.i), - p: last.p, - }) - } else if ( - last.d != null && - c.d != null && - c.p <= last.p && - last.p <= c.p + c.d.length - ) { - return (newOp[newOp.length - 1] = { - d: strInject(c.d, last.p - c.p, last.d), - p: c.p, - }) - } else { - return newOp.push(c) - } - } -} - -text.compose = function (op1, op2) { - checkValidOp(op1) - checkValidOp(op2) - - const newOp = op1.slice() - for (const c of Array.from(op2)) { - append(newOp, c) - } - - return newOp -} - -// Attempt to compress the op components together 'as much as possible'. -// This implementation preserves order and preserves create/delete pairs. -text.compress = op => text.compose([], op) - -text.normalize = function (op) { - const newOp = [] - - // Normalize should allow ops which are a single (unwrapped) component: - // {i:'asdf', p:23}. - // There's no good way to test if something is an array: - // http://perfectionkills.com/instanceof-considered-harmful-or-how-to-write-a-robust-isarray/ - // so this is probably the least bad solution. - if (op.i != null || op.p != null) { - op = [op] - } - - for (const c of Array.from(op)) { - if (c.p == null) { - c.p = 0 - } - append(newOp, c) - } - - return newOp -} - -// This helper method transforms a position by an op component. -// -// If c is an insert, insertAfter specifies whether the transform -// is pushed after the insert (true) or before it (false). -// -// insertAfter is optional for deletes. -const transformPosition = function (pos, c, insertAfter) { - if (c.i != null) { - if (c.p < pos || (c.p === pos && insertAfter)) { - return pos + c.i.length - } else { - return pos - } - } else { - // I think this could also be written as: Math.min(c.p, Math.min(c.p - otherC.p, otherC.d.length)) - // but I think its harder to read that way, and it compiles using ternary operators anyway - // so its no slower written like this. - if (pos <= c.p) { - return pos - } else if (pos <= c.p + c.d.length) { - return c.p - } else { - return pos - c.d.length - } - } -} - -// Helper method to transform a cursor position as a result of an op. -// -// Like transformPosition above, if c is an insert, insertAfter specifies whether the cursor position -// is pushed after an insert (true) or before it (false). -text.transformCursor = function (position, op, side) { - const insertAfter = side === 'right' - for (const c of Array.from(op)) { - position = transformPosition(position, c, insertAfter) - } - return position -} - -// Transform an op component by another op component. Asymmetric. -// The result will be appended to destination. -// -// exported for use in JSON type -text._tc = transformComponent = function (dest, c, otherC, side) { - checkValidOp([c]) - checkValidOp([otherC]) - - if (c.i != null) { - append(dest, { - i: c.i, - p: transformPosition(c.p, otherC, side === 'right'), - }) - } else { - // Delete - if (otherC.i != null) { - // delete vs insert - let s = c.d - if (c.p < otherC.p) { - append(dest, { d: s.slice(0, otherC.p - c.p), p: c.p }) - s = s.slice(otherC.p - c.p) - } - if (s !== '') { - append(dest, { d: s, p: c.p + otherC.i.length }) - } - } else { - // Delete vs delete - if (c.p >= otherC.p + otherC.d.length) { - append(dest, { d: c.d, p: c.p - otherC.d.length }) - } else if (c.p + c.d.length <= otherC.p) { - append(dest, c) - } else { - // They overlap somewhere. - const newC = { d: '', p: c.p } - if (c.p < otherC.p) { - newC.d = c.d.slice(0, otherC.p - c.p) - } - if (c.p + c.d.length > otherC.p + otherC.d.length) { - newC.d += c.d.slice(otherC.p + otherC.d.length - c.p) - } - - // This is entirely optional - just for a check that the deleted - // text in the two ops matches - const intersectStart = Math.max(c.p, otherC.p) - const intersectEnd = Math.min( - c.p + c.d.length, - otherC.p + otherC.d.length - ) - const cIntersect = c.d.slice(intersectStart - c.p, intersectEnd - c.p) - const otherIntersect = otherC.d.slice( - intersectStart - otherC.p, - intersectEnd - otherC.p - ) - if (cIntersect !== otherIntersect) { - throw new Error( - 'Delete ops delete different text in the same region of the document' - ) - } - - if (newC.d !== '') { - // This could be rewritten similarly to insert v delete, above. - newC.p = transformPosition(newC.p, otherC) - append(dest, newC) - } - } - } - } - - return dest -} - -const invertComponent = function (c) { - if (c.i != null) { - return { d: c.i, p: c.p } - } else { - return { i: c.d, p: c.p } - } -} - -// No need to use append for invert, because the components won't be able to -// cancel with one another. -text.invert = op => - Array.from(op.slice().reverse()).map(c => invertComponent(c)) - -if (typeof WEB !== 'undefined' && WEB !== null) { - if (!exports.types) { - exports.types = {} - } - - // This is kind of awful - come up with a better way to hook this helper code up. - bootstrapTransform(text, transformComponent, checkValidOp, append) - - // [] is used to prevent closure from renaming types.text - exports.types.text = text -} else { - module.exports = text - - // The text type really shouldn't need this - it should be possible to define - // an efficient transform function by making a sort of transform map and passing each - // op component through it. - require('./helpers').bootstrapTransform( - text, - transformComponent, - checkValidOp, - append - ) -} diff --git a/services/document-updater/app/js/sharejs/types/README.md b/services/document-updater/app/js/sharejs/types/README.md new file mode 100644 index 0000000000..22e68842dd --- /dev/null +++ b/services/document-updater/app/js/sharejs/types/README.md @@ -0,0 +1,48 @@ +This directory contains all the operational transform code. Each file defines a type. + +Most of the types in here are for testing or demonstration. The only types which are sent to the webclient +are `text` and `json`. + + +# An OT type + +All OT types have the following fields: + +`name`: _(string)_ Name of the type. Should match the filename. +`create() -> snapshot`: Function which creates and returns a new document snapshot + +`apply(snapshot, op) -> snapshot`: A function which creates a new document snapshot with the op applied +`transform(op1, op2, side) -> op1'`: OT transform function. + +Given op1, op2, `apply(s, op2, transform(op1, op2, 'left')) == apply(s, op1, transform(op2, op1, 'right'))`. + +Transform and apply must never modify their arguments. + + +Optional properties: + +`tp2`: _(bool)_ True if the transform function supports TP2. This allows p2p architectures to work. +`compose(op1, op2) -> op`: Create and return a new op which has the same effect as op1 + op2. +`serialize(snapshot) -> JSON object`: Serialize a document to something we can JSON.stringify() +`deserialize(object) -> snapshot`: Deserialize a JSON object into the document's internal snapshot format +`prune(op1', op2, side) -> op1`: Inserse transform function. Only required for TP2 types. +`normalize(op) -> op`: Fix up an op to make it valid. Eg, remove skips of size zero. +`api`: _(object)_ Set of helper methods which will be mixed in to the client document object for manipulating documents. See below. + + +# Examples + +`count` and `simple` are two trivial OT type definitions if you want to take a look. JSON defines +the ot-for-JSON type (see the wiki for documentation) and all the text types define different text +implementations. (I still have no idea which one I like the most, and they're fun to write!) + + +# API + +Types can also define API functions. These methods are mixed into the client's Doc object when a document is created. +You can use them to help construct ops programatically (so users don't need to understand how ops are structured). + +For example, the three text types defined here (text, text-composable and text-tp2) all provide the text API, supplying +`.insert()`, `.del()`, `.getLength` and `.getText` methods. + +See text-api.coffee for an example. diff --git a/services/document-updater/app/js/sharejs/web-prelude.js b/services/document-updater/app/js/sharejs/web-prelude.js deleted file mode 100644 index a4c3a0f22e..0000000000 --- a/services/document-updater/app/js/sharejs/web-prelude.js +++ /dev/null @@ -1,14 +0,0 @@ -/* eslint-disable - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -// This is included at the top of each compiled type file for the web. - -/** - @const - @type {boolean} -*/ -const WEB = true - -const exports = window.sharejs