/* eslint-disable camelcase, n/handle-callback-err, max-len, */ // Migrated from services/web/frontend/js/ide/editor/Document.js // @ts-ignore import RangesTracker from '@overleaf/ranges-tracker' import { ShareJsDoc } from './share-js-doc' import { debugConsole } from '@/utils/debugging' import { Socket } from '@/features/ide-react/connection/types/socket' import { IdeEventEmitter } from '@/features/ide-react/create-ide-event-emitter' import { EditorFacade } from '@/features/source-editor/extensions/realtime' import { EventLog } from '@/features/ide-react/editor/event-log' import EditorWatchdogManager from '@/features/ide-react/connection/editor-watchdog-manager' import EventEmitter from '@/utils/EventEmitter' import { AnyOperation, Change, CommentOperation, } from '../../../../../types/change' import { isCommentOperation, isDeleteOperation, isInsertOperation, } from '@/utils/operations' import { decodeUtf8 } from '@/utils/decode-utf8' import { ShareJsOperation, TrackChangesIdSeeds, Version, } from '@/features/ide-react/editor/types/document' const MAX_PENDING_OP_SIZE = 64 type JoinCallback = (error?: Error) => void type LeaveCallback = JoinCallback type Update = Record type Message = { meta: { tc: string user_id: string } } type ErrorMetadata = Record function getOpSize(op: AnyOperation) { if (isInsertOperation(op)) { return op.i.length } if (isDeleteOperation(op)) { return op.d.length } return 0 } function getShareJsOpSize(shareJsOp: ShareJsOperation) { return shareJsOp.reduce((total, op) => total + getOpSize(op), 0) } export class Document extends EventEmitter { private connected: boolean private wantToBeJoined = false private chaosMonkeyTimer: number | null = null private track_changes_as: string | null = null private joinCallbacks: JoinCallback[] = [] private leaveCallbacks: LeaveCallback[] = [] doc?: ShareJsDoc cm6?: EditorFacade oldInflightOp?: ShareJsOperation ranges: RangesTracker joined = false // This is set and read in useCodeMirrorScope docName = '' constructor( readonly doc_id: string, readonly socket: Socket, private readonly globalEditorWatchdogManager: EditorWatchdogManager, private readonly ideEventEmitter: IdeEventEmitter, private readonly eventLog: EventLog ) { super() this.connected = this.socket.socket.connected this.bindToEditorEvents() this.bindToSocketEvents() } attachToCM6(cm6: EditorFacade) { this.cm6 = cm6 if (this.doc) { this.doc.attachToCM6(this.cm6) } if (this.cm6) { this.cm6.on('change', this.checkConsistency) } if (this.doc) { this.ideEventEmitter.emit('document:opened', this.doc) } } detachFromCM6() { if (this.doc) { this.doc.detachFromCM6() } if (this.cm6) { this.cm6.off('change', this.checkConsistency) } delete this.cm6 this.clearChaosMonkey() if (this.doc) { this.ideEventEmitter.emit('document:closed', this.doc) } } submitOp(...ops: AnyOperation[]) { this.doc?.submitOp(ops) } private checkConsistency = (editor: EditorFacade) => { // We've been seeing a lot of errors when I think there shouldn't be // any, which may be related to this check happening before the change is // applied. If we use a timeout, hopefully we can reduce this. window.setTimeout(() => { const editorValue = editor?.getValue() const sharejsValue = this.doc?.getSnapshot() if (editorValue !== sharejsValue) { return this.onError( new Error('Editor text does not match server text'), {}, editorValue ) } }, 0) } getSnapshot() { return this.doc?.getSnapshot() } getType() { return this.doc?.getType() } getInflightOp(): ShareJsOperation | undefined { return this.doc?.getInflightOp() } getPendingOp(): ShareJsOperation | undefined { return this.doc?.getPendingOp() } getRecentAck() { return this.doc?.getRecentAck() } hasBufferedOps() { return this.doc?.hasBufferedOps() } setTrackingChanges(track_changes: boolean) { if (this.doc) { this.doc.track_changes = track_changes } } getTrackingChanges() { return !!this.doc?.track_changes } setTrackChangesIdSeeds(id_seeds: TrackChangesIdSeeds) { if (this.doc) { this.doc.track_changes_id_seeds = id_seeds } } private onUpdateAppliedHandler = (update: any) => this.onUpdateApplied(update) // TODO: MIGRATION: Create proper types for error and message private onErrorHandler = (error: Error, message: { doc_id: string }) => { // 'otUpdateError' are emitted per doc socket.io room, hence we can be // sure that message.doc_id exists. if (message.doc_id !== this.doc_id) { // This error is for another doc. Do not action it. We could open // a modal that has the wrong context on it. return } this.onError(error, message) } private onDisconnectHandler = () => this.onDisconnect() private bindToSocketEvents() { this.socket.on('otUpdateApplied', this.onUpdateAppliedHandler) this.socket.on('otUpdateError', this.onErrorHandler) return this.socket.on('disconnect', this.onDisconnectHandler) } private unBindFromSocketEvents() { this.socket.removeListener('otUpdateApplied', this.onUpdateAppliedHandler) this.socket.removeListener('otUpdateError', this.onErrorHandler) return this.socket.removeListener('disconnect', this.onDisconnectHandler) } private bindToEditorEvents() { this.ideEventEmitter.on('project:joined', this.onReconnect) } private unBindFromEditorEvents() { this.ideEventEmitter.off('project:joined', this.onReconnect) } leaveAndCleanUp(cb?: (error?: Error) => void) { return this.leave((error?: Error) => { this.cleanUp() if (cb) cb(error) }) } leaveAndCleanUpPromise() { return new Promise((resolve, reject) => { this.leaveAndCleanUp((error?: Error) => { if (error) { reject(error) } else { resolve() } }) }) } join(callback?: JoinCallback) { this.wantToBeJoined = true this.cancelLeave() if (this.connected) { this.joinDoc(callback) } else if (callback) { this.joinCallbacks.push(callback) } } leave(callback?: LeaveCallback) { this.flush() // force an immediate flush when leaving document this.wantToBeJoined = false this.cancelJoin() if (this.doc?.hasBufferedOps()) { debugConsole.log( '[leave] Doc has buffered ops, pushing callback for later' ) if (callback) { this.leaveCallbacks.push(callback) } } else if (!this.connected) { debugConsole.log('[leave] Not connected, returning now') callback?.() } else { debugConsole.log('[leave] Leaving now') this.leaveDoc(callback) } } flush() { return this.doc?.flushPendingOps() } chaosMonkey(line = 0, char = 'a') { const orig = char let copy: string | null = null let pos = 0 const timer = () => { if (copy == null || !copy.length) { copy = orig.slice() + ' ' + new Date() + '\n' line += Math.random() > 0.1 ? 1 : -2 if (line < 0) { line = 0 } pos = 0 } char = copy[0] copy = copy.slice(1) if (this.cm6) { this.cm6.view.dispatch({ changes: { from: Math.min(pos, this.cm6.view.state.doc.length), insert: char, }, }) } pos += 1 this.chaosMonkeyTimer = window.setTimeout( timer, 100 + (Math.random() < 0.1 ? 1000 : 0) ) } timer() } clearChaosMonkey() { const timer = this.chaosMonkeyTimer if (timer) { this.chaosMonkeyTimer = null window.clearTimeout(timer) } } pollSavedStatus() { // returns false if doc has ops waiting to be acknowledged or // sent that haven't changed since the last time we checked. // Otherwise returns true. let saved const inflightOp = this.getInflightOp() const pendingOp = this.getPendingOp() const recentAck = this.getRecentAck() const pendingOpSize = pendingOp ? getShareJsOpSize(pendingOp) : 0 if (inflightOp == null && pendingOp == null) { // There's nothing going on, this is OK. saved = true debugConsole.log('[pollSavedStatus] no inflight or pending ops') } else if (inflightOp && inflightOp === this.oldInflightOp) { // The same inflight op has been sitting unacked since we // last checked, this is bad. saved = false debugConsole.log('[pollSavedStatus] inflight op is same as before') } else if ( pendingOp != null && recentAck && pendingOpSize < MAX_PENDING_OP_SIZE ) { // There is an op waiting to go to server but it is small and // within the flushDelay, this is OK for now. saved = true debugConsole.log( '[pollSavedStatus] pending op (small with recent ack) assume ok', pendingOp, pendingOpSize ) } else { // In any other situation, assume the document is unsaved. saved = false debugConsole.log( `[pollSavedStatus] assuming not saved (inflightOp?: ${ inflightOp != null }, pendingOp?: ${pendingOp != null})` ) } this.oldInflightOp = inflightOp return saved } private cancelLeave() { this.leaveCallbacks = [] } private cancelJoin() { this.joinCallbacks = [] } private onUpdateApplied(update: Update) { this.eventLog.pushEvent('received-update', { doc_id: this.doc_id, remote_doc_id: update?.doc, wantToBeJoined: this.wantToBeJoined, update, hasDoc: !!this.doc, }) if (update?.doc === this.doc_id && this.doc != null) { this.eventLog.pushEvent('received-update:processing', { update, }) // FIXME: change this back to processUpdateFromServer when redis fixed this.doc.processUpdateFromServerInOrder(update) if (!this.wantToBeJoined) { return this.leave() } } } private onDisconnect() { debugConsole.log('[onDisconnect] disconnecting') this.connected = false this.joined = false return this.doc != null ? this.doc.updateConnectionState('disconnected') : undefined } private onReconnect = () => { debugConsole.log('[onReconnect] reconnected (joined project)') this.eventLog.pushEvent('reconnected:afterJoinProject') this.connected = true if (this.wantToBeJoined || this.doc?.hasBufferedOps()) { debugConsole.log( `[onReconnect] Rejoining (wantToBeJoined: ${ this.wantToBeJoined } OR hasBufferedOps: ${this.doc?.hasBufferedOps()})` ) this.joinDoc((error?: Error) => { if (error) { this.onError(error) return } this.doc?.updateConnectionState('ok') this.doc?.flushPendingOps() this.callJoinCallbacks() }) } } private callJoinCallbacks() { for (const callback of this.joinCallbacks) { callback() } this.joinCallbacks = [] } private joinDoc(callback?: JoinCallback) { if (this.doc) { this.eventLog.pushEvent('joinDoc:existing', { doc_id: this.doc_id, version: this.doc.getVersion(), }) return this.socket.emit( 'joinDoc', this.doc_id, this.doc.getVersion(), { encodeRanges: true }, (error, docLines, version, updates, ranges) => { if (error) { callback?.(error) return } this.joined = true this.doc?.catchUp(updates) this.decodeRanges(ranges) this.catchUpRanges(ranges?.changes, ranges?.comments) callback?.() } ) } else { this.eventLog.pushEvent('joinDoc:new', { doc_id: this.doc_id, }) this.socket.emit( 'joinDoc', this.doc_id, { encodeRanges: true }, (error, docLines, version, updates, ranges) => { if (error) { callback?.(error) return } this.joined = true this.eventLog.pushEvent('joinDoc:inited', { doc_id: this.doc_id, version, }) this.doc = new ShareJsDoc( this.doc_id, docLines, version, this.socket, this.globalEditorWatchdogManager, this.ideEventEmitter, this.eventLog ) this.decodeRanges(ranges) this.ranges = new RangesTracker(ranges?.changes, ranges?.comments) this.bindToShareJsDocEvents() callback?.() } ) } } private decodeRanges(ranges: RangesTracker) { try { if (ranges.changes) { for (const change of ranges.changes) { if (isInsertOperation(change.op)) { change.op.i = decodeUtf8(change.op.i) } if (isDeleteOperation(change.op)) { change.op.d = decodeUtf8(change.op.d) } } } return (() => { if (!ranges.comments) { return [] } return ranges.comments.map((comment: Change) => comment.op.c != null ? (comment.op.c = decodeUtf8(comment.op.c)) : undefined ) })() } catch (err) { debugConsole.error(err) } } private leaveDoc(callback?: LeaveCallback) { this.eventLog.pushEvent('leaveDoc', { doc_id: this.doc_id, }) debugConsole.log('[leaveDoc] Sending leaveDoc request') this.socket.emit('leaveDoc', this.doc_id, error => { if (error) { callback?.(error) return } this.joined = false for (const leaveCallback of this.leaveCallbacks) { debugConsole.log('[_leaveDoc] Calling buffered callback', leaveCallback) leaveCallback(error) } this.leaveCallbacks = [] callback?.() }) } cleanUp() { // if we arrive here from _onError the pending and inflight ops will have been cleared if (this.hasBufferedOps()) { debugConsole.log( `[cleanUp] Document (${this.doc_id}) has buffered ops, refusing to remove from openDocs` ) return // return immediately, do not unbind from events } else { this.emit('detach', this.doc_id) } this.unBindFromEditorEvents() this.unBindFromSocketEvents() } private bindToShareJsDocEvents() { if (!this.doc) { return } this.doc.on('error', (error: Error, meta: ErrorMetadata) => this.onError(error, meta) ) this.doc.on('externalUpdate', (update: Update) => { this.eventLog.pushEvent('externalUpdate', { doc_id: this.doc_id }) return this.trigger('externalUpdate', update) }) this.doc.on('remoteop', (...ops: AnyOperation[]) => { this.eventLog.pushEvent('remoteop', { doc_id: this.doc_id }) return this.trigger('remoteop', ...ops) }) this.doc.on('op:sent', (op: AnyOperation) => { this.eventLog.pushEvent('op:sent', { doc_id: this.doc_id, op, }) return this.trigger('op:sent') }) this.doc.on('op:acknowledged', (op: AnyOperation) => { this.eventLog.pushEvent('op:acknowledged', { doc_id: this.doc_id, op, }) this.ideEventEmitter.emit('ide:opAcknowledged', { doc_id: this.doc_id, op, }) return this.trigger('op:acknowledged') }) this.doc.on('op:timeout', (op: AnyOperation) => { this.eventLog.pushEvent('op:timeout', { doc_id: this.doc_id, op, }) this.trigger('op:timeout') return this.onError(new Error('op timed out')) }) this.doc.on( 'flush', (inflightOp: AnyOperation, pendingOp: AnyOperation, version: Version) => { return this.eventLog.pushEvent('flush', { doc_id: this.doc_id, inflightOp, pendingOp, v: version, }) } ) let docChangedTimeout: number | null = null this.doc.on( 'change', (ops: AnyOperation[], oldSnapshot: any, msg: Message) => { this.applyOpsToRanges(ops, msg) if (docChangedTimeout) { window.clearTimeout(docChangedTimeout) } docChangedTimeout = window.setTimeout(() => { window.dispatchEvent( new CustomEvent('doc:changed', { detail: { id: this.doc_id } }) ) this.ideEventEmitter.emit('doc:changed', { doc_id: this.doc_id }) }, 50) } ) this.doc.on('flipped_pending_to_inflight', () => { return this.trigger('flipped_pending_to_inflight') }) let docSavedTimeout: number | null this.doc.on('saved', () => { if (docSavedTimeout) { window.clearTimeout(docSavedTimeout) } docSavedTimeout = window.setTimeout(() => { window.dispatchEvent( new CustomEvent('doc:saved', { detail: { id: this.doc_id } }) ) this.ideEventEmitter.emit('doc:saved', { doc_id: this.doc_id }) }, 50) }) } private onError( error: Error, meta: ErrorMetadata = {}, editorContent?: string ) { meta.doc_id = this.doc_id debugConsole.log('ShareJS error', error, meta) if (error.message === 'no project_id found on client') { debugConsole.log('ignoring error, will wait to join project') return } if (this.doc) { this.doc.clearInflightAndPendingOps() } this.trigger('error', error, meta, editorContent) // The clean-up should run after the error is triggered because the error triggers a // disconnect. If we run the clean-up first, we remove our event handlers and miss // the disconnect event, which means we try to leaveDoc when the connection comes back. // This could interfere with the new connection of a new instance of this document. this.cleanUp() } private applyOpsToRanges(ops: AnyOperation[], msg?: Message) { let old_id_seed let track_changes_as = null const remote_op = msg != null if (remote_op && msg?.meta.tc) { old_id_seed = this.ranges.getIdSeed() this.ranges.setIdSeed(msg.meta.tc) track_changes_as = msg.meta.user_id } else if (!remote_op && this.track_changes_as != null) { track_changes_as = this.track_changes_as } this.ranges.track_changes = track_changes_as != null for (const op of this.filterOps(ops)) { this.ranges.applyOp(op, { user_id: track_changes_as }) } if (old_id_seed != null) { this.ranges.setIdSeed(old_id_seed) } if (remote_op) { // With remote ops, the editor hasn't been updated when we receive this // op, so defer updating track changes until it has return window.setTimeout(() => this.emit('ranges:dirty')) } else { return this.emit('ranges:dirty') } } private catchUpRanges(changes: Change[], comments: CommentOperation[]) { // We've just been given the current server's ranges, but need to apply any local ops we have. // Reset to the server state then apply our local ops again. if (changes == null) { changes = [] } if (comments == null) { comments = [] } this.emit('ranges:clear') this.ranges.changes = changes this.ranges.comments = comments this.ranges.track_changes = this.doc?.track_changes for (const op of this.filterOps(this.doc?.getInflightOp() || [])) { this.ranges.setIdSeed(this.doc?.track_changes_id_seeds?.inflight) this.ranges.applyOp(op, { user_id: this.track_changes_as }) } for (const op of this.filterOps(this.doc?.getPendingOp() || [])) { this.ranges.setIdSeed(this.doc?.track_changes_id_seeds?.pending) this.ranges.applyOp(op, { user_id: this.track_changes_as }) } return this.emit('ranges:redraw') } private filterOps(ops: AnyOperation[]) { // Read-only token users can't see/edit comment, so we filter out comment // ops to avoid highlighting comment ranges. if (window.isRestrictedTokenMember) { return ops.filter(op => !isCommentOperation(op)) } else { return ops } } }