overleaf/services/web/frontend/js/features/ide-react/editor/document-container.ts
Eric Mc Sween 8b4b454b97 Merge pull request #20052 from overleaf/em-tracked-deletes-limit
Include tracked deletes when limiting document size

GitOrigin-RevId: 9d7b2870001fb942eaa92db78993fee1ddb3937c
2024-08-27 08:04:31 +00:00

745 lines
21 KiB
TypeScript

/* eslint-disable camelcase */
// Migrated from services/web/frontend/js/ide/editor/Document.js
import RangesTracker from '@overleaf/ranges-tracker'
import { ShareJsDoc } from './share-js-doc'
import { debugConsole } from '@/utils/debugging'
import { Socket } from '@/features/ide-react/connection/types/socket'
import { IdeEventEmitter } from '@/features/ide-react/create-ide-event-emitter'
import { EditorFacade } from '@/features/source-editor/extensions/realtime'
import { EventLog } from '@/features/ide-react/editor/event-log'
import EditorWatchdogManager from '@/features/ide-react/connection/editor-watchdog-manager'
import EventEmitter from '@/utils/EventEmitter'
import {
AnyOperation,
Change,
CommentOperation,
EditOperation,
} from '../../../../../types/change'
import {
isCommentOperation,
isDeleteOperation,
isInsertOperation,
} from '@/utils/operations'
import { decodeUtf8 } from '@/utils/decode-utf8'
import {
ShareJsOperation,
TrackChangesIdSeeds,
Version,
} from '@/features/ide-react/editor/types/document'
import { ThreadId } from '../../../../../types/review-panel/review-panel'
import getMeta from '@/utils/meta'
const MAX_PENDING_OP_SIZE = 64
type JoinCallback = (error?: Error) => void
type LeaveCallback = JoinCallback
type Update =
| {
v: number
doc: string
}
| {
v: number
doc: string
op: AnyOperation[]
meta: {
type?: string
source: string
user_id: string
ts: number
}
hash?: string
lastV?: number
}
type Message = {
meta: {
tc: string
user_id: string
}
}
type ErrorMetadata = Record<string, any>
function getOpSize(op: AnyOperation) {
if (isInsertOperation(op)) {
return op.i.length
}
if (isDeleteOperation(op)) {
return op.d.length
}
return 0
}
function getShareJsOpSize(shareJsOp: ShareJsOperation) {
return shareJsOp.reduce((total, op) => total + getOpSize(op), 0)
}
// TODO: define these in RangesTracker
type _RangesTracker = Omit<RangesTracker, 'changes' | 'comments'> & {
changes: Change<EditOperation>[]
comments: Change<CommentOperation>[]
track_changes?: boolean
}
export type RangesTrackerWithResolvedThreadIds = _RangesTracker & {
resolvedThreadIds: Record<ThreadId, boolean>
}
export class DocumentContainer extends EventEmitter {
private connected: boolean
private wantToBeJoined = false
private chaosMonkeyTimer: number | null = null
public track_changes_as: string | null = null
private joinCallbacks: JoinCallback[] = []
private leaveCallbacks: LeaveCallback[] = []
doc?: ShareJsDoc
cm6?: EditorFacade
oldInflightOp?: ShareJsOperation
ranges?: _RangesTracker | RangesTrackerWithResolvedThreadIds
joined = false
// This is set and read in useCodeMirrorScope
docName = ''
constructor(
readonly doc_id: string,
readonly socket: Socket,
private readonly globalEditorWatchdogManager: EditorWatchdogManager,
private readonly ideEventEmitter: IdeEventEmitter,
private readonly eventLog: EventLog,
private readonly detachDoc: (docId: string, doc: DocumentContainer) => void
) {
super()
this.connected = this.socket.socket.connected
this.bindToEditorEvents()
this.bindToSocketEvents()
}
attachToCM6(cm6: EditorFacade) {
this.cm6 = cm6
if (this.doc) {
this.doc.attachToCM6(this.cm6)
}
this.cm6.on('change', this.checkConsistency)
}
detachFromCM6() {
if (this.doc) {
this.doc.detachFromCM6()
}
if (this.cm6) {
this.cm6.off('change', this.checkConsistency)
}
delete this.cm6
this.clearChaosMonkey()
if (this.doc) {
this.ideEventEmitter.emit('document:closed', this.doc)
}
}
submitOp(...ops: AnyOperation[]) {
this.doc?.submitOp(ops)
}
private checkConsistency = (editor: EditorFacade) => {
// We've been seeing a lot of errors when I think there shouldn't be
// any, which may be related to this check happening before the change is
// applied. If we use a timeout, hopefully we can reduce this.
window.setTimeout(() => {
const editorValue = editor?.getValue()
const sharejsValue = this.doc?.getSnapshot()
if (editorValue !== sharejsValue) {
return this.onError(
new Error('Editor text does not match server text'),
{},
editorValue
)
}
}, 0)
}
getSnapshot() {
return this.doc?.getSnapshot()
}
getType() {
return this.doc?.getType()
}
getInflightOp(): ShareJsOperation | undefined {
return this.doc?.getInflightOp()
}
getPendingOp(): ShareJsOperation | undefined {
return this.doc?.getPendingOp()
}
getRecentAck() {
return this.doc?.getRecentAck()
}
hasBufferedOps() {
return this.doc?.hasBufferedOps()
}
setTrackingChanges(track_changes: boolean) {
if (this.doc) {
this.doc.track_changes = track_changes
}
}
getTrackingChanges() {
return !!this.doc?.track_changes
}
setTrackChangesIdSeeds(id_seeds: TrackChangesIdSeeds) {
if (this.doc) {
this.doc.track_changes_id_seeds = id_seeds
}
}
private onUpdateAppliedHandler = (update: any) => this.onUpdateApplied(update)
private onErrorHandler = (error: Error, message: ErrorMetadata) => {
// 'otUpdateError' are emitted per doc socket.io room, hence we can be
// sure that message.doc_id exists.
if (message.doc_id !== this.doc_id) {
// This error is for another doc. Do not action it. We could open
// a modal that has the wrong context on it.
return
}
this.onError(error, message)
}
private onDisconnectHandler = () => this.onDisconnect()
private bindToSocketEvents() {
this.socket.on('otUpdateApplied', this.onUpdateAppliedHandler)
this.socket.on('otUpdateError', this.onErrorHandler)
return this.socket.on('disconnect', this.onDisconnectHandler)
}
private unBindFromSocketEvents() {
this.socket.removeListener('otUpdateApplied', this.onUpdateAppliedHandler)
this.socket.removeListener('otUpdateError', this.onErrorHandler)
return this.socket.removeListener('disconnect', this.onDisconnectHandler)
}
private bindToEditorEvents() {
this.ideEventEmitter.on('project:joined', this.onReconnect)
}
private unBindFromEditorEvents() {
this.ideEventEmitter.off('project:joined', this.onReconnect)
}
leaveAndCleanUp(cb?: (error?: Error) => void) {
return this.leave((error?: Error) => {
this.cleanUp()
if (cb) cb(error)
})
}
leaveAndCleanUpPromise() {
return new Promise<void>((resolve, reject) => {
this.leaveAndCleanUp((error?: Error) => {
if (error) {
reject(error)
} else {
resolve()
}
})
})
}
join(callback?: JoinCallback) {
this.wantToBeJoined = true
this.cancelLeave()
if (this.connected) {
this.joinDoc(callback)
} else if (callback) {
this.joinCallbacks.push(callback)
}
}
leave(callback?: LeaveCallback) {
this.flush() // force an immediate flush when leaving document
this.wantToBeJoined = false
this.cancelJoin()
if (this.doc?.hasBufferedOps()) {
debugConsole.log(
'[leave] Doc has buffered ops, pushing callback for later'
)
if (callback) {
this.leaveCallbacks.push(callback)
}
} else if (!this.connected) {
debugConsole.log('[leave] Not connected, returning now')
callback?.()
} else {
debugConsole.log('[leave] Leaving now')
this.leaveDoc(callback)
}
}
flush() {
return this.doc?.flushPendingOps()
}
chaosMonkey(line = 0, char = 'a') {
const orig = char
let copy: string | null = null
let pos = 0
const timer = () => {
if (copy == null || !copy.length) {
copy = orig.slice() + ' ' + new Date() + '\n'
line += Math.random() > 0.1 ? 1 : -2
if (line < 0) {
line = 0
}
pos = 0
}
char = copy[0]
copy = copy.slice(1)
if (this.cm6) {
this.cm6.view.dispatch({
changes: {
from: Math.min(pos, this.cm6.view.state.doc.length),
insert: char,
},
})
}
pos += 1
this.chaosMonkeyTimer = window.setTimeout(
timer,
100 + (Math.random() < 0.1 ? 1000 : 0)
)
}
timer()
}
clearChaosMonkey() {
const timer = this.chaosMonkeyTimer
if (timer) {
this.chaosMonkeyTimer = null
window.clearTimeout(timer)
}
}
pollSavedStatus() {
// returns false if doc has ops waiting to be acknowledged or
// sent that haven't changed since the last time we checked.
// Otherwise returns true.
let saved
const inflightOp = this.getInflightOp()
const pendingOp = this.getPendingOp()
const recentAck = this.getRecentAck()
const pendingOpSize = pendingOp ? getShareJsOpSize(pendingOp) : 0
if (inflightOp == null && pendingOp == null) {
// There's nothing going on, this is OK.
saved = true
debugConsole.log('[pollSavedStatus] no inflight or pending ops')
} else if (inflightOp && inflightOp === this.oldInflightOp) {
// The same inflight op has been sitting unacked since we
// last checked, this is bad.
saved = false
debugConsole.log('[pollSavedStatus] inflight op is same as before')
} else if (
pendingOp != null &&
recentAck &&
pendingOpSize < MAX_PENDING_OP_SIZE
) {
// There is an op waiting to go to server but it is small and
// within the flushDelay, this is OK for now.
saved = true
debugConsole.log(
'[pollSavedStatus] pending op (small with recent ack) assume ok',
pendingOp,
pendingOpSize
)
} else {
// In any other situation, assume the document is unsaved.
saved = false
debugConsole.log(
`[pollSavedStatus] assuming not saved (inflightOp?: ${
inflightOp != null
}, pendingOp?: ${pendingOp != null})`
)
}
this.oldInflightOp = inflightOp
return saved
}
private cancelLeave() {
this.leaveCallbacks = []
}
private cancelJoin() {
this.joinCallbacks = []
}
private onUpdateApplied(update: Update) {
this.eventLog.pushEvent('received-update', {
doc_id: this.doc_id,
remote_doc_id: update?.doc,
wantToBeJoined: this.wantToBeJoined,
update,
hasDoc: !!this.doc,
})
if (update?.doc === this.doc_id && this.doc != null) {
this.eventLog.pushEvent('received-update:processing', {
update,
})
// FIXME: change this back to processUpdateFromServer when redis fixed
this.doc.processUpdateFromServerInOrder(update)
if (!this.wantToBeJoined) {
return this.leave()
}
}
}
private onDisconnect() {
debugConsole.log('[onDisconnect] disconnecting')
this.connected = false
this.joined = false
return this.doc != null
? this.doc.updateConnectionState('disconnected')
: undefined
}
private onReconnect = () => {
debugConsole.log('[onReconnect] reconnected (joined project)')
this.eventLog.pushEvent('reconnected:afterJoinProject')
this.connected = true
if (this.wantToBeJoined || this.doc?.hasBufferedOps()) {
debugConsole.log(
`[onReconnect] Rejoining (wantToBeJoined: ${
this.wantToBeJoined
} OR hasBufferedOps: ${this.doc?.hasBufferedOps()})`
)
this.joinDoc((error?: Error) => {
if (error) {
this.onError(error)
return
}
this.doc?.updateConnectionState('ok')
this.doc?.flushPendingOps()
this.callJoinCallbacks()
})
}
}
private callJoinCallbacks() {
for (const callback of this.joinCallbacks) {
callback()
}
this.joinCallbacks = []
}
private joinDoc(callback?: JoinCallback) {
if (this.doc) {
this.eventLog.pushEvent('joinDoc:existing', {
doc_id: this.doc_id,
version: this.doc.getVersion(),
})
return this.socket.emit(
'joinDoc',
this.doc_id,
this.doc.getVersion(),
{ encodeRanges: true, age: this.doc.getTimeSinceLastServerActivity() },
(error, docLines, version, updates, ranges) => {
if (error) {
callback?.(error)
return
}
this.joined = true
this.doc?.catchUp(updates)
this.decodeRanges(ranges)
this.catchUpRanges(ranges?.changes, ranges?.comments)
callback?.()
}
)
} else {
this.eventLog.pushEvent('joinDoc:new', {
doc_id: this.doc_id,
})
this.socket.emit(
'joinDoc',
this.doc_id,
{ encodeRanges: true },
(error, docLines, version, updates, ranges) => {
if (error) {
callback?.(error)
return
}
this.joined = true
this.eventLog.pushEvent('joinDoc:inited', {
doc_id: this.doc_id,
version,
})
this.doc = new ShareJsDoc(
this.doc_id,
docLines,
version,
this.socket,
this.globalEditorWatchdogManager,
this.ideEventEmitter,
this.eventLog
)
this.decodeRanges(ranges)
this.ranges = new RangesTracker(ranges?.changes, ranges?.comments)
this.bindToShareJsDocEvents()
callback?.()
}
)
}
}
private decodeRanges(ranges: RangesTracker) {
try {
if (ranges.changes) {
for (const change of ranges.changes) {
if (isInsertOperation(change.op)) {
change.op.i = decodeUtf8(change.op.i)
}
if (isDeleteOperation(change.op)) {
change.op.d = decodeUtf8(change.op.d)
}
}
}
return (() => {
if (!ranges.comments) {
return []
}
return ranges.comments.map((comment: Change<CommentOperation>) =>
comment.op.c != null
? (comment.op.c = decodeUtf8(comment.op.c))
: undefined
)
})()
} catch (err) {
debugConsole.error(err)
}
}
private leaveDoc(callback?: LeaveCallback) {
this.eventLog.pushEvent('leaveDoc', {
doc_id: this.doc_id,
})
debugConsole.log('[leaveDoc] Sending leaveDoc request')
this.socket.emit('leaveDoc', this.doc_id, error => {
if (error) {
callback?.(error)
return
}
this.joined = false
for (const leaveCallback of this.leaveCallbacks) {
debugConsole.log('[_leaveDoc] Calling buffered callback', leaveCallback)
leaveCallback(error)
}
this.leaveCallbacks = []
callback?.()
})
}
cleanUp() {
// if we arrive here from _onError the pending and inflight ops will have been cleared
if (this.hasBufferedOps()) {
debugConsole.log(
`[cleanUp] Document (${this.doc_id}) has buffered ops, refusing to remove from openDocs`
)
return // return immediately, do not unbind from events
}
this.detachDoc(this.doc_id, this)
this.unBindFromEditorEvents()
this.unBindFromSocketEvents()
}
private bindToShareJsDocEvents() {
if (!this.doc) {
return
}
this.doc.on('error', (error: Error, meta: ErrorMetadata) =>
this.onError(error, meta)
)
this.doc.on('externalUpdate', (update: Update) => {
this.eventLog.pushEvent('externalUpdate', { doc_id: this.doc_id })
return this.trigger('externalUpdate', update)
})
this.doc.on('remoteop', (...ops: AnyOperation[]) => {
this.eventLog.pushEvent('remoteop', { doc_id: this.doc_id })
return this.trigger('remoteop', ...ops)
})
this.doc.on('op:sent', (op: AnyOperation) => {
this.eventLog.pushEvent('op:sent', {
doc_id: this.doc_id,
op,
})
return this.trigger('op:sent')
})
this.doc.on('op:acknowledged', (op: AnyOperation) => {
this.eventLog.pushEvent('op:acknowledged', {
doc_id: this.doc_id,
op,
})
this.ideEventEmitter.emit('ide:opAcknowledged', {
doc_id: this.doc_id,
op,
})
return this.trigger('op:acknowledged')
})
this.doc.on('op:timeout', (op: AnyOperation) => {
this.eventLog.pushEvent('op:timeout', {
doc_id: this.doc_id,
op,
})
this.trigger('op:timeout')
return this.onError(new Error('op timed out'))
})
this.doc.on(
'flush',
(inflightOp: AnyOperation, pendingOp: AnyOperation, version: Version) => {
return this.eventLog.pushEvent('flush', {
doc_id: this.doc_id,
inflightOp,
pendingOp,
v: version,
})
}
)
let docChangedTimeout: number | null = null
this.doc.on(
'change',
(ops: AnyOperation[], oldSnapshot: any, msg: Message) => {
this.applyOpsToRanges(ops, msg)
if (docChangedTimeout) {
window.clearTimeout(docChangedTimeout)
}
docChangedTimeout = window.setTimeout(() => {
window.dispatchEvent(
new CustomEvent('doc:changed', { detail: { id: this.doc_id } })
)
this.ideEventEmitter.emit('doc:changed', { doc_id: this.doc_id })
}, 50)
}
)
this.doc.on('flipped_pending_to_inflight', () => {
return this.trigger('flipped_pending_to_inflight')
})
let docSavedTimeout: number | null
this.doc.on('saved', () => {
if (docSavedTimeout) {
window.clearTimeout(docSavedTimeout)
}
docSavedTimeout = window.setTimeout(() => {
window.dispatchEvent(
new CustomEvent('doc:saved', { detail: { id: this.doc_id } })
)
this.ideEventEmitter.emit('doc:saved', { doc_id: this.doc_id })
}, 50)
})
}
private onError(
error: Error,
meta: ErrorMetadata = {},
editorContent?: string
) {
meta.doc_id = this.doc_id
debugConsole.log('ShareJS error', error, meta)
if (error.message === 'no project_id found on client') {
debugConsole.log('ignoring error, will wait to join project')
return
}
if (this.doc) {
this.doc.clearInflightAndPendingOps()
}
this.trigger('error', error, meta, editorContent)
// The clean-up should run after the error is triggered because the error triggers a
// disconnect. If we run the clean-up first, we remove our event handlers and miss
// the disconnect event, which means we try to leaveDoc when the connection comes back.
// This could interfere with the new connection of a new instance of this document.
this.cleanUp()
}
private applyOpsToRanges(ops: AnyOperation[], msg?: Message) {
let old_id_seed
let track_changes_as = null
const remote_op = msg != null
if (remote_op && msg?.meta.tc) {
old_id_seed = this.ranges!.getIdSeed()
this.ranges!.setIdSeed(msg.meta.tc)
track_changes_as = msg.meta.user_id
} else if (!remote_op && this.track_changes_as != null) {
track_changes_as = this.track_changes_as
}
this.ranges!.track_changes = track_changes_as != null
for (const op of this.filterOps(ops)) {
this.ranges!.applyOp(op, { user_id: track_changes_as })
}
if (old_id_seed != null) {
this.ranges!.setIdSeed(old_id_seed)
}
if (remote_op) {
// With remote ops, the editor hasn't been updated when we receive this
// op, so defer updating track changes until it has
return window.setTimeout(() => this.emit('ranges:dirty'))
} else {
return this.emit('ranges:dirty')
}
}
private catchUpRanges(
changes: Change<EditOperation>[],
comments: Change<CommentOperation>[]
) {
// We've just been given the current server's ranges, but need to apply any local ops we have.
// Reset to the server state then apply our local ops again.
if (changes == null) {
changes = []
}
if (comments == null) {
comments = []
}
this.emit('ranges:clear')
this.ranges!.changes = changes
this.ranges!.comments = comments
this.ranges!.track_changes = this.doc?.track_changes ?? false
for (const op of this.filterOps(this.doc?.getInflightOp() || [])) {
this.ranges!.setIdSeed(this.doc?.track_changes_id_seeds?.inflight)
this.ranges!.applyOp(op, { user_id: this.track_changes_as })
}
for (const op of this.filterOps(this.doc?.getPendingOp() || [])) {
this.ranges!.setIdSeed(this.doc?.track_changes_id_seeds?.pending)
this.ranges!.applyOp(op, { user_id: this.track_changes_as })
}
return this.emit('ranges:redraw')
}
private filterOps(ops: AnyOperation[]) {
// Read-only token users can't see/edit comment, so we filter out comment
// ops to avoid highlighting comment ranges.
if (getMeta('ol-isRestrictedTokenMember')) {
return ops.filter(op => !isCommentOperation(op))
} else {
return ops
}
}
}